From 3628e6d21ae6a967a40770c5ad972861faa9eeb0 Mon Sep 17 00:00:00 2001 From: static Date: Sun, 11 Jan 2026 13:19:54 +0900 Subject: [PATCH] =?UTF-8?q?=EC=97=85=EB=A1=9C=EB=93=9C=ED=95=A0=20?= =?UTF-8?q?=EB=95=8C=EC=97=90=EB=8F=84=20=EC=8A=A4=ED=8A=B8=EB=A6=AC?= =?UTF-8?q?=EB=B0=8D=20=EB=B0=A9=EC=8B=9D=EC=9C=BC=EB=A1=9C=20=EC=B2=98?= =?UTF-8?q?=EB=A6=AC=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + pnpm-lock.yaml | 9 + src/lib/indexedDB/keyStore.ts | 6 +- src/lib/modules/crypto/aes.ts | 2 +- src/lib/modules/crypto/sha.ts | 13 + src/lib/modules/crypto/util.ts | 4 +- src/lib/modules/file/upload.svelte.ts | 314 +++++++++++++++--- src/lib/modules/thumbnail.ts | 17 + .../(main)/directory/[[id]]/service.svelte.ts | 4 +- 9 files changed, 308 insertions(+), 62 deletions(-) diff --git a/package.json b/package.json index c16b700..17dad8d 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@eslint/compat": "^2.0.0", "@eslint/js": "^9.39.2", "@iconify-json/material-symbols": "^1.2.50", + "@noble/hashes": "^2.0.1", "@sveltejs/adapter-node": "^5.4.0", "@sveltejs/kit": "^2.49.2", "@sveltejs/vite-plugin-svelte": "^6.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4e336f..025aacd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,6 +48,9 @@ importers: '@iconify-json/material-symbols': specifier: ^1.2.50 version: 1.2.50 + '@noble/hashes': + specifier: ^2.0.1 + version: 2.0.1 '@sveltejs/adapter-node': specifier: ^5.4.0 version: 5.4.0(@sveltejs/kit@2.49.2(@sveltejs/vite-plugin-svelte@6.2.1(svelte@5.46.1)(vite@7.3.0(@types/node@25.0.3)(jiti@1.21.7)(yaml@2.8.0)))(svelte@5.46.1)(vite@7.3.0(@types/node@25.0.3)(jiti@1.21.7)(yaml@2.8.0))) @@ -414,6 +417,10 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@noble/hashes@2.0.1': + resolution: {integrity: sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==} + engines: {node: '>= 20.19.0'} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2217,6 +2224,8 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@noble/hashes@2.0.1': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 diff --git a/src/lib/indexedDB/keyStore.ts b/src/lib/indexedDB/keyStore.ts index 7a4c89e..86b8b79 100644 --- a/src/lib/indexedDB/keyStore.ts +++ b/src/lib/indexedDB/keyStore.ts @@ -70,12 +70,12 @@ export const storeMasterKeys = async (keys: MasterKey[]) => { }; export const getHmacSecrets = async () => { - return await keyStore.hmacSecret.toArray(); + return (await keyStore.hmacSecret.toArray()).filter(({ secret }) => secret.extractable); }; export const storeHmacSecrets = async (secrets: HmacSecret[]) => { - if (secrets.some(({ secret }) => secret.extractable)) { - throw new Error("Hmac secrets must be nonextractable"); + if (secrets.some(({ secret }) => !secret.extractable)) { + throw new Error("Hmac secrets must be extractable"); } await keyStore.hmacSecret.bulkPut(secrets); }; diff --git a/src/lib/modules/crypto/aes.ts b/src/lib/modules/crypto/aes.ts index fe11afb..4035343 100644 --- a/src/lib/modules/crypto/aes.ts +++ b/src/lib/modules/crypto/aes.ts @@ -77,7 +77,7 @@ export const unwrapHmacSecret = async (hmacSecretWrapped: string, masterKey: Cry name: "HMAC", hash: "SHA-256", } satisfies HmacImportParams, - false, // Nonextractable + true, // Extractable ["sign", "verify"], ), }; diff --git a/src/lib/modules/crypto/sha.ts b/src/lib/modules/crypto/sha.ts index 9bf2dea..883ac10 100644 --- a/src/lib/modules/crypto/sha.ts +++ b/src/lib/modules/crypto/sha.ts @@ -1,7 +1,20 @@ +import { hmac } from "@noble/hashes/hmac.js"; +import { sha256 } from "@noble/hashes/sha2.js"; + export const digestMessage = async (message: BufferSource) => { return await crypto.subtle.digest("SHA-256", message); }; +export const createStreamingHmac = async (hmacSecret: CryptoKey) => { + const keyBytes = new Uint8Array(await crypto.subtle.exportKey("raw", hmacSecret)); + const h = hmac.create(sha256, keyBytes); + + return { + update: (data: Uint8Array) => h.update(data), + digest: () => h.digest(), + }; +}; + export const generateHmacSecret = async () => { return { hmacSecret: await crypto.subtle.generateKey( diff --git a/src/lib/modules/crypto/util.ts b/src/lib/modules/crypto/util.ts index a3e3bc0..215eaf2 100644 --- a/src/lib/modules/crypto/util.ts +++ b/src/lib/modules/crypto/util.ts @@ -9,8 +9,8 @@ export const decodeString = (data: ArrayBuffer) => { return textDecoder.decode(data); }; -export const encodeToBase64 = (data: ArrayBuffer) => { - return btoa(String.fromCharCode(...new Uint8Array(data))); +export const encodeToBase64 = (data: ArrayBuffer | Uint8Array) => { + return btoa(String.fromCharCode(...(data instanceof ArrayBuffer ? new Uint8Array(data) : data))); }; export const decodeFromBase64 = (data: string) => { diff --git a/src/lib/modules/file/upload.svelte.ts b/src/lib/modules/file/upload.svelte.ts index 2bb6c7c..ac3010e 100644 --- a/src/lib/modules/file/upload.svelte.ts +++ b/src/lib/modules/file/upload.svelte.ts @@ -1,6 +1,6 @@ import axios from "axios"; import ExifReader from "exifreader"; -import { limitFunction } from "p-limit"; +import pLimit, { limitFunction } from "p-limit"; import { CHUNK_SIZE } from "$lib/constants"; import { encodeToBase64, @@ -11,9 +11,10 @@ import { encryptChunk, digestMessage, signMessageHmac, + createStreamingHmac, } from "$lib/modules/crypto"; import { Scheduler } from "$lib/modules/scheduler"; -import { generateThumbnail } from "$lib/modules/thumbnail"; +import { generateThumbnail, generateThumbnailFromFile } from "$lib/modules/thumbnail"; import type { FileThumbnailUploadRequest } from "$lib/server/schemas"; import type { MasterKey, HmacSecret } from "$lib/stores"; import { trpc } from "$trpc/client"; @@ -41,7 +42,7 @@ export type LiveFileUploadState = FileUploadState & { }; const scheduler = new Scheduler< - { fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined + { fileId: number; fileBuffer?: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined >(); let uploadingFiles: FileUploadState[] = $state([]); @@ -77,6 +78,33 @@ const requestDuplicateFileScan = limitFunction( { concurrency: 1 }, ); +const isImageFile = (fileType: string) => fileType.startsWith("image/"); + +const requestDuplicateFileScanStreaming = limitFunction( + async (file: File, hmacSecret: HmacSecret, onDuplicate: () => Promise) => { + const hmacStream = await createStreamingHmac(hmacSecret.secret); + const reader = file.stream().getReader(); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + hmacStream.update(value); + } + + const fileSigned = encodeToBase64(hmacStream.digest()); + const files = await trpc().file.listByHash.query({ + hskVersion: hmacSecret.version, + contentHmac: fileSigned, + }); + if (files.length === 0 || (await onDuplicate())) { + return { fileSigned }; + } else { + return {}; + } + }, + { concurrency: 1 }, +); + const getFileType = (file: File) => { if (file.type) return file.type; if (file.name.endsWith(".heic")) return "image/heic"; @@ -235,6 +263,148 @@ const requestFileUpload = limitFunction( { concurrency: 1 }, ); +const uploadFileStreaming = async ( + state: FileUploadState, + file: File, + masterKey: MasterKey, + hmacSecret: HmacSecret, + fileSigned: string, + parentId: DirectoryId, +) => { + state.status = "uploading"; + + const fileType = getFileType(file); + const { dataKey, dataKeyVersion } = await generateDataKey(); + const dataKeyWrapped = await wrapDataKey(dataKey, masterKey.key); + + const nameEncrypted = await encryptString(file.name, dataKey); + const lastModifiedAtEncrypted = await encryptString(file.lastModified.toString(), dataKey); + + // Calculate total chunks for metadata + const totalChunks = Math.ceil(file.size / CHUNK_SIZE); + const metadata = { + chunks: totalChunks, + parent: parentId, + mekVersion: masterKey.version, + dek: dataKeyWrapped, + dekVersion: dataKeyVersion, + hskVersion: hmacSecret.version, + contentType: fileType, + name: nameEncrypted.ciphertext, + nameIv: nameEncrypted.iv, + lastModifiedAt: lastModifiedAtEncrypted.ciphertext, + lastModifiedAtIv: lastModifiedAtEncrypted.iv, + }; + + const { uploadId } = await trpc().file.startUpload.mutate(metadata); + + // Stream file, encrypt, and upload with concurrency limit + const reader = file.stream().getReader(); + const limit = pLimit(4); + let buffer = new Uint8Array(0); + let chunkIndex = 0; + const uploadPromises: Promise[] = []; + + const totalBytes = file.size; + let uploadedBytes = 0; + const startTime = Date.now(); + + const uploadChunk = async ( + index: number, + encryptedChunk: ArrayBuffer, + chunkHash: string, + originalChunkSize: number, + ) => { + const response = await fetch(`/api/file/upload/${uploadId}/chunks/${index}`, { + method: "POST", + headers: { + "Content-Type": "application/octet-stream", + "Content-Digest": `sha-256=:${chunkHash}:`, + }, + body: encryptedChunk, + }); + + if (!response.ok) { + throw new Error(`Chunk upload failed: ${response.status} ${response.statusText}`); + } + + // Update progress after upload completes + uploadedBytes += originalChunkSize; + const elapsed = (Date.now() - startTime) / 1000; + const rate = uploadedBytes / elapsed; + const remaining = totalBytes - uploadedBytes; + const estimated = rate > 0 ? remaining / rate : undefined; + + state.progress = uploadedBytes / totalBytes; + state.rate = rate; + state.estimated = estimated; + }; + + while (true) { + const { done, value } = await reader.read(); + if (done && buffer.length === 0) break; + + if (value) { + const newBuffer = new Uint8Array(buffer.length + value.length); + newBuffer.set(buffer); + newBuffer.set(value, buffer.length); + buffer = newBuffer; + } + + while (buffer.length >= CHUNK_SIZE || (done && buffer.length > 0)) { + const chunkSize = Math.min(CHUNK_SIZE, buffer.length); + const chunk = buffer.slice(0, chunkSize); + buffer = buffer.slice(chunkSize); + + const encryptedChunk = await encryptChunk(chunk.buffer.slice(0, chunk.byteLength), dataKey); + const chunkHash = encodeToBase64(await digestMessage(encryptedChunk)); + const currentIndex = chunkIndex++; + + uploadPromises.push( + limit(() => uploadChunk(currentIndex, encryptedChunk, chunkHash, chunkSize)), + ); + } + + if (done) break; + } + + await Promise.all(uploadPromises); + + const { file: fileId } = await trpc().file.completeUpload.mutate({ + uploadId, + contentHmac: fileSigned, + }); + + // Generate and upload thumbnail for video files + if (fileType.startsWith("video/")) { + try { + const thumbnail = await generateThumbnailFromFile(file); + if (thumbnail) { + const thumbnailBuffer = await thumbnail.arrayBuffer(); + const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey); + + const thumbnailForm = new FormData(); + thumbnailForm.set( + "metadata", + JSON.stringify({ + dekVersion: dataKeyVersion.toISOString(), + contentIv: encodeToBase64(thumbnailEncrypted.iv), + } satisfies FileThumbnailUploadRequest), + ); + thumbnailForm.set("content", new Blob([thumbnailEncrypted.ciphertext])); + await axios.post(`/api/file/${fileId}/thumbnail/upload`, thumbnailForm); + } + } catch (e) { + // Thumbnail upload failure is not critical + console.error(e); + } + } + + state.status = "uploaded"; + + return { fileId }; +}; + export const uploadFile = async ( file: File, parentId: "root" | number, @@ -249,69 +419,103 @@ export const uploadFile = async ( }); const state = uploadingFiles.at(-1)!; + const fileType = getFileType(file); + + // Image files: use buffer-based approach (need EXIF + thumbnail) + if (isImageFile(fileType)) { + return await scheduler.schedule(file.size, async () => { + state.status = "encryption-pending"; + + try { + const { fileBuffer, fileSigned } = await requestDuplicateFileScan( + file, + hmacSecret, + onDuplicate, + ); + if (!fileBuffer || !fileSigned) { + state.status = "canceled"; + uploadingFiles = uploadingFiles.filter((file) => file !== state); + return undefined; + } + + const { + dataKeyWrapped, + dataKeyVersion, + fileType, + chunksEncrypted, + nameEncrypted, + createdAtEncrypted, + lastModifiedAtEncrypted, + thumbnail, + } = await encryptFile(state, file, fileBuffer, masterKey); + + const metadata = { + chunks: chunksEncrypted.length, + parent: parentId, + mekVersion: masterKey.version, + dek: dataKeyWrapped, + dekVersion: dataKeyVersion, + hskVersion: hmacSecret.version, + contentType: fileType, + name: nameEncrypted.ciphertext, + nameIv: nameEncrypted.iv, + createdAt: createdAtEncrypted?.ciphertext, + createdAtIv: createdAtEncrypted?.iv, + lastModifiedAt: lastModifiedAtEncrypted.ciphertext, + lastModifiedAtIv: lastModifiedAtEncrypted.iv, + }; + + let thumbnailForm = null; + if (thumbnail) { + thumbnailForm = new FormData(); + thumbnailForm.set( + "metadata", + JSON.stringify({ + dekVersion: dataKeyVersion.toISOString(), + contentIv: encodeToBase64(thumbnail.iv), + } satisfies FileThumbnailUploadRequest), + ); + thumbnailForm.set("content", new Blob([thumbnail.ciphertext])); + } + + const { fileId } = await requestFileUpload( + state, + metadata, + chunksEncrypted, + fileSigned, + thumbnailForm, + ); + return { fileId, fileBuffer, thumbnailBuffer: thumbnail?.plaintext }; + } catch (e) { + state.status = "error"; + throw e; + } + }); + } + + // Video and other files: use streaming approach return await scheduler.schedule(file.size, async () => { state.status = "encryption-pending"; try { - const { fileBuffer, fileSigned } = await requestDuplicateFileScan( - file, - hmacSecret, - onDuplicate, - ); - if (!fileBuffer || !fileSigned) { + // 1st pass: streaming HMAC for duplicate check + const { fileSigned } = await requestDuplicateFileScanStreaming(file, hmacSecret, onDuplicate); + if (!fileSigned) { state.status = "canceled"; - uploadingFiles = uploadingFiles.filter((file) => file !== state); + uploadingFiles = uploadingFiles.filter((f) => f !== state); return undefined; } - const { - dataKeyWrapped, - dataKeyVersion, - fileType, - chunksEncrypted, - nameEncrypted, - createdAtEncrypted, - lastModifiedAtEncrypted, - thumbnail, - } = await encryptFile(state, file, fileBuffer, masterKey); - - const metadata = { - chunks: chunksEncrypted.length, - parent: parentId, - mekVersion: masterKey.version, - dek: dataKeyWrapped, - dekVersion: dataKeyVersion, - hskVersion: hmacSecret.version, - contentType: fileType, - name: nameEncrypted.ciphertext, - nameIv: nameEncrypted.iv, - createdAt: createdAtEncrypted?.ciphertext, - createdAtIv: createdAtEncrypted?.iv, - lastModifiedAt: lastModifiedAtEncrypted.ciphertext, - lastModifiedAtIv: lastModifiedAtEncrypted.iv, - }; - - let thumbnailForm = null; - if (thumbnail) { - thumbnailForm = new FormData(); - thumbnailForm.set( - "metadata", - JSON.stringify({ - dekVersion: dataKeyVersion.toISOString(), - contentIv: encodeToBase64(thumbnail.iv), - } satisfies FileThumbnailUploadRequest), - ); - thumbnailForm.set("content", new Blob([thumbnail.ciphertext])); - } - - const { fileId } = await requestFileUpload( + // 2nd pass: streaming encrypt + upload + const { fileId } = await uploadFileStreaming( state, - metadata, - chunksEncrypted, + file, + masterKey, + hmacSecret, fileSigned, - thumbnailForm, + parentId, ); - return { fileId, fileBuffer, thumbnailBuffer: thumbnail?.plaintext }; + return { fileId, fileBuffer: undefined, thumbnailBuffer: undefined }; } catch (e) { state.status = "error"; throw e; diff --git a/src/lib/modules/thumbnail.ts b/src/lib/modules/thumbnail.ts index d9a995b..739c7af 100644 --- a/src/lib/modules/thumbnail.ts +++ b/src/lib/modules/thumbnail.ts @@ -125,3 +125,20 @@ export const generateThumbnail = async (fileBuffer: ArrayBuffer, fileType: strin export const getThumbnailUrl = (thumbnailBuffer: ArrayBuffer) => { return `data:image/webp;base64,${encodeToBase64(thumbnailBuffer)}`; }; + +export const generateThumbnailFromFile = async (file: File) => { + const fileType = file.type || (file.name.endsWith(".heic") ? "image/heic" : ""); + if (!fileType.startsWith("video/")) return null; + + let url; + try { + url = URL.createObjectURL(file); + return await generateVideoThumbnail(url); + } catch { + return null; + } finally { + if (url) { + URL.revokeObjectURL(url); + } + } +}; diff --git a/src/routes/(main)/directory/[[id]]/service.svelte.ts b/src/routes/(main)/directory/[[id]]/service.svelte.ts index f83bbaf..ccd5b14 100644 --- a/src/routes/(main)/directory/[[id]]/service.svelte.ts +++ b/src/routes/(main)/directory/[[id]]/service.svelte.ts @@ -88,7 +88,9 @@ export const requestFileUpload = async ( const res = await uploadFile(file, parentId, hmacSecret, masterKey, onDuplicate); if (!res) return false; - storeFileCache(res.fileId, res.fileBuffer); // Intended + if (res.fileBuffer) { + storeFileCache(res.fileId, res.fileBuffer); // Intended + } if (res.thumbnailBuffer) { storeFileThumbnailCache(res.fileId, res.thumbnailBuffer); // Intended }