diff --git a/package.json b/package.json index 3479c2c..60b381c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "arkvault", "private": true, - "version": "0.6.0", + "version": "0.7.0", "type": "module", "scripts": { "dev": "vite dev", diff --git a/src/lib/modules/file/upload.svelte.ts b/src/lib/modules/file/upload.svelte.ts index 679de5b..5a23cc1 100644 --- a/src/lib/modules/file/upload.svelte.ts +++ b/src/lib/modules/file/upload.svelte.ts @@ -10,6 +10,7 @@ import { digestMessage, signMessageHmac, } from "$lib/modules/crypto"; +import { Scheduler } from "$lib/modules/scheduler"; import { generateThumbnail } from "$lib/modules/thumbnail"; import type { FileThumbnailUploadRequest, @@ -23,6 +24,7 @@ export interface FileUploadState { name: string; parentId: DirectoryId; status: + | "queued" | "encryption-pending" | "encrypting" | "upload-pending" @@ -36,13 +38,16 @@ export interface FileUploadState { } export type LiveFileUploadState = FileUploadState & { - status: "encryption-pending" | "encrypting" | "upload-pending" | "uploading"; + status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading"; }; +const scheduler = new Scheduler< + { fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined +>(); let uploadingFiles: FileUploadState[] = $state([]); const isFileUploading = (status: FileUploadState["status"]) => - ["encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status); + ["queued", "encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status); export const getUploadingFiles = (parentId?: DirectoryId) => { return uploadingFiles.filter( @@ -183,80 +188,85 @@ export const uploadFile = async ( hmacSecret: HmacSecret, masterKey: MasterKey, onDuplicate: () => Promise, -): Promise< - { fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined -> => { +) => { uploadingFiles.push({ name: file.name, parentId, - status: "encryption-pending", + status: "queued", }); const state = uploadingFiles.at(-1)!; - try { - const { fileBuffer, fileSigned } = await requestDuplicateFileScan( - file, - hmacSecret, - onDuplicate, - ); - if (!fileBuffer || !fileSigned) { - state.status = "canceled"; - uploadingFiles = uploadingFiles.filter((file) => file !== state); - return undefined; - } + return await scheduler.schedule( + async () => file.size, + async () => { + state.status = "encryption-pending"; - const { - dataKeyWrapped, - dataKeyVersion, - fileType, - fileEncrypted, - fileEncryptedHash, - nameEncrypted, - createdAtEncrypted, - lastModifiedAtEncrypted, - thumbnail, - } = await encryptFile(state, file, fileBuffer, masterKey); + try { + const { fileBuffer, fileSigned } = await requestDuplicateFileScan( + file, + hmacSecret, + onDuplicate, + ); + if (!fileBuffer || !fileSigned) { + state.status = "canceled"; + uploadingFiles = uploadingFiles.filter((file) => file !== state); + return undefined; + } - const form = new FormData(); - form.set( - "metadata", - JSON.stringify({ - parent: parentId, - mekVersion: masterKey.version, - dek: dataKeyWrapped, - dekVersion: dataKeyVersion.toISOString(), - hskVersion: hmacSecret.version, - contentHmac: fileSigned, - contentType: fileType, - contentIv: fileEncrypted.iv, - name: nameEncrypted.ciphertext, - nameIv: nameEncrypted.iv, - createdAt: createdAtEncrypted?.ciphertext, - createdAtIv: createdAtEncrypted?.iv, - lastModifiedAt: lastModifiedAtEncrypted.ciphertext, - lastModifiedAtIv: lastModifiedAtEncrypted.iv, - } satisfies FileUploadRequest), - ); - form.set("content", new Blob([fileEncrypted.ciphertext])); - form.set("checksum", fileEncryptedHash); + const { + dataKeyWrapped, + dataKeyVersion, + fileType, + fileEncrypted, + fileEncryptedHash, + nameEncrypted, + createdAtEncrypted, + lastModifiedAtEncrypted, + thumbnail, + } = await encryptFile(state, file, fileBuffer, masterKey); - let thumbnailForm = null; - if (thumbnail) { - thumbnailForm = new FormData(); - thumbnailForm.set( - "metadata", - JSON.stringify({ - dekVersion: dataKeyVersion.toISOString(), - contentIv: thumbnail.iv, - } satisfies FileThumbnailUploadRequest), - ); - thumbnailForm.set("content", new Blob([thumbnail.ciphertext])); - } + const form = new FormData(); + form.set( + "metadata", + JSON.stringify({ + parent: parentId, + mekVersion: masterKey.version, + dek: dataKeyWrapped, + dekVersion: dataKeyVersion.toISOString(), + hskVersion: hmacSecret.version, + contentHmac: fileSigned, + contentType: fileType, + contentIv: fileEncrypted.iv, + name: nameEncrypted.ciphertext, + nameIv: nameEncrypted.iv, + createdAt: createdAtEncrypted?.ciphertext, + createdAtIv: createdAtEncrypted?.iv, + lastModifiedAt: lastModifiedAtEncrypted.ciphertext, + lastModifiedAtIv: lastModifiedAtEncrypted.iv, + } satisfies FileUploadRequest), + ); + form.set("content", new Blob([fileEncrypted.ciphertext])); + form.set("checksum", fileEncryptedHash); - const { fileId } = await requestFileUpload(state, form, thumbnailForm); - return { fileId, fileBuffer, thumbnailBuffer: thumbnail?.plaintext }; - } catch (e) { - state.status = "error"; - throw e; - } + let thumbnailForm = null; + if (thumbnail) { + thumbnailForm = new FormData(); + thumbnailForm.set( + "metadata", + JSON.stringify({ + dekVersion: dataKeyVersion.toISOString(), + contentIv: thumbnail.iv, + } satisfies FileThumbnailUploadRequest), + ); + thumbnailForm.set("content", new Blob([thumbnail.ciphertext])); + } + + const { fileId } = await requestFileUpload(state, form, thumbnailForm); + return { fileId, fileBuffer, thumbnailBuffer: thumbnail?.plaintext }; + } catch (e) { + state.status = "error"; + throw e; + } + }, + ); }; diff --git a/src/lib/modules/scheduler.ts b/src/lib/modules/scheduler.ts new file mode 100644 index 0000000..31ce449 --- /dev/null +++ b/src/lib/modules/scheduler.ts @@ -0,0 +1,41 @@ +export class Scheduler { + private tasks = 0; + private memoryUsage = 0; + private queue: (() => void)[] = []; + + constructor(public memoryLimit = 100 * 1024 * 1024 /* 100 MiB */) {} + + private next() { + if (this.memoryUsage < this.memoryLimit) { + this.queue.shift()?.(); + } + } + + async schedule(estimateMemoryUsage: () => Promise, task: () => Promise) { + if (this.tasks++ > 0) { + await new Promise((resolve) => { + this.queue.push(resolve); + }); + } + + while (this.memoryUsage >= this.memoryLimit) { + await new Promise((resolve) => { + this.queue.unshift(resolve); + }); + } + + let taskMemoryUsage = 0; + + try { + taskMemoryUsage = await estimateMemoryUsage(); + this.memoryUsage += taskMemoryUsage; + this.next(); + + return await task(); + } finally { + this.tasks--; + this.memoryUsage -= taskMemoryUsage; + this.next(); + } + } +} diff --git a/src/routes/(fullscreen)/file/uploads/File.svelte b/src/routes/(fullscreen)/file/uploads/File.svelte index 7b40ac5..4c620ee 100644 --- a/src/routes/(fullscreen)/file/uploads/File.svelte +++ b/src/routes/(fullscreen)/file/uploads/File.svelte @@ -18,7 +18,7 @@
- {#if state.status === "encryption-pending"} + {#if state.status === "queued" || state.status === "encryption-pending"} {:else if state.status === "encrypting"} @@ -37,7 +37,9 @@ {state.name}

- {#if state.status === "encryption-pending"} + {#if state.status === "queued"} + 대기 중 + {:else if state.status === "encryption-pending"} 준비 중 {:else if state.status === "encrypting"} 암호화하는 중 diff --git a/src/routes/(fullscreen)/settings/thumbnail/+page.svelte b/src/routes/(fullscreen)/settings/thumbnail/+page.svelte index 2c06964..50a11de 100644 --- a/src/routes/(fullscreen)/settings/thumbnail/+page.svelte +++ b/src/routes/(fullscreen)/settings/thumbnail/+page.svelte @@ -4,17 +4,36 @@ import { BottomDiv, Button, FullscreenDiv } from "$lib/components/atoms"; import { IconEntryButton, TopBar } from "$lib/components/molecules"; import { deleteAllFileThumbnailCaches } from "$lib/modules/file"; - import { bulkGetFileInfo } from "$lib/modules/filesystem"; + import { bulkGetFileInfo, type MaybeFileInfo } from "$lib/modules/filesystem"; import { masterKeyStore } from "$lib/stores"; + import { sortEntries } from "$lib/utils"; import File from "./File.svelte"; - import { persistentStates, requestThumbnailGeneration } from "./service.svelte"; + import { + getThumbnailGenerationStatus, + clearThumbnailGenerationStatuses, + requestThumbnailGeneration, + type GenerationStatus, + } from "./service.svelte"; import IconDelete from "~icons/material-symbols/delete"; let { data } = $props(); + let fileInfos: MaybeFileInfo[] = $state([]); + let files = $derived( + fileInfos + .map((info) => ({ + info, + status: getThumbnailGenerationStatus(info.id), + })) + .filter( + (file): file is { info: MaybeFileInfo; status: Exclude } => + file.status !== "uploaded", + ), + ); + const generateAllThumbnails = () => { - persistentStates.files.forEach(({ info }) => { + files.forEach(({ info }) => { if (info.exists) { requestThumbnailGeneration(info); } @@ -22,13 +41,12 @@ }; onMount(async () => { - const fileInfos = await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!); - persistentStates.files = persistentStates.files.map(({ id, status }) => ({ - id, - info: fileInfos.get(id)!, - status, - })); + fileInfos = sortEntries( + Array.from((await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!)).values()), + ); }); + + $effect(() => clearThumbnailGenerationStatuses); @@ -43,19 +61,19 @@ 저장된 썸네일 모두 삭제하기

- {#if persistentStates.files.length > 0} + {#if files.length > 0}

썸네일이 누락된 파일

- {persistentStates.files.length}개 파일의 썸네일이 존재하지 않아요. + {files.length}개 파일의 썸네일이 존재하지 않아요.

- {#each persistentStates.files as { info, status } (info.id)} + {#each files as { info, status } (info.id)} {#if info.exists} goto(`/file/${id}`)} onGenerateThumbnailClick={requestThumbnailGeneration} /> @@ -66,7 +84,7 @@
{/if}
- {#if persistentStates.files.length > 0} + {#if files.length > 0} diff --git a/src/routes/(fullscreen)/settings/thumbnail/File.svelte b/src/routes/(fullscreen)/settings/thumbnail/File.svelte index 6b3e0d9..3cc5f4c 100644 --- a/src/routes/(fullscreen)/settings/thumbnail/File.svelte +++ b/src/routes/(fullscreen)/settings/thumbnail/File.svelte @@ -10,7 +10,6 @@ onclick(info)} - actionButtonIcon={!$generationStatus || $generationStatus === "error" ? IconCamera : undefined} + actionButtonIcon={!status || status === "error" ? IconCamera : undefined} onActionButtonClick={() => onGenerateThumbnailClick(info)} actionButtonClass="text-gray-800" > - {@const subtext = - $generationStatus && $generationStatus !== "uploaded" - ? subtexts[$generationStatus] - : formatDateTime(info.createdAt ?? info.lastModifiedAt)} + {@const subtext = status + ? subtexts[status] + : formatDateTime(info.createdAt ?? info.lastModifiedAt)} diff --git a/src/routes/(fullscreen)/settings/thumbnail/service.svelte.ts b/src/routes/(fullscreen)/settings/thumbnail/service.svelte.ts index 23e863f..85226b0 100644 --- a/src/routes/(fullscreen)/settings/thumbnail/service.svelte.ts +++ b/src/routes/(fullscreen)/settings/thumbnail/service.svelte.ts @@ -1,8 +1,9 @@ import { limitFunction } from "p-limit"; -import { get, writable, type Writable } from "svelte/store"; +import { SvelteMap } from "svelte/reactivity"; import { encryptData } from "$lib/modules/crypto"; import { storeFileThumbnailCache } from "$lib/modules/file"; -import type { FileInfo, MaybeFileInfo } from "$lib/modules/filesystem"; +import type { FileInfo } from "$lib/modules/filesystem"; +import { Scheduler } from "$lib/modules/scheduler"; import { generateThumbnail as doGenerateThumbnail } from "$lib/modules/thumbnail"; import { requestFileDownload, requestFileThumbnailUpload } from "$lib/services/file"; @@ -15,41 +16,31 @@ export type GenerationStatus = | "uploaded" | "error"; -interface File { - id: number; - info: MaybeFileInfo; - status?: Writable; -} +const scheduler = new Scheduler(); +const statuses = new SvelteMap(); -const workingFiles = new Map>(); +export const getThumbnailGenerationStatus = (fileId: number) => { + return statuses.get(fileId); +}; -let queue: (() => void)[] = []; -let memoryUsage = 0; -const memoryLimit = 100 * 1024 * 1024; // 100 MiB - -export const persistentStates = $state({ - files: [] as File[], -}); - -export const getGenerationStatus = (fileId: number) => { - return workingFiles.get(fileId); +export const clearThumbnailGenerationStatuses = () => { + for (const [id, status] of statuses) { + if (status === "uploaded" || status === "error") { + statuses.delete(id); + } + } }; const generateThumbnail = limitFunction( - async ( - status: Writable, - fileBuffer: ArrayBuffer, - fileType: string, - dataKey: CryptoKey, - ) => { - status.set("generating"); + async (fileId: number, fileBuffer: ArrayBuffer, fileType: string, dataKey: CryptoKey) => { + statuses.set(fileId, "generating"); const thumbnail = await doGenerateThumbnail(fileBuffer, fileType); if (!thumbnail) return null; const thumbnailBuffer = await thumbnail.arrayBuffer(); const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey); - status.set("upload-pending"); + statuses.set(fileId, "upload-pending"); return { plaintext: thumbnailBuffer, ...thumbnailEncrypted }; }, { concurrency: 4 }, @@ -57,106 +48,55 @@ const generateThumbnail = limitFunction( const requestThumbnailUpload = limitFunction( async ( - status: Writable, fileId: number, dataKeyVersion: Date, thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: string }, ) => { - status.set("uploading"); + statuses.set(fileId, "uploading"); const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnail); if (!res.ok) return false; - - status.set("uploaded"); - workingFiles.delete(fileId); - persistentStates.files = persistentStates.files.filter(({ id }) => id != fileId); - + statuses.set(fileId, "uploaded"); storeFileThumbnailCache(fileId, thumbnail.plaintext); // Intended return true; }, { concurrency: 4 }, ); -const enqueue = async ( - status: Writable | undefined, - fileInfo: FileInfo, - priority = false, -) => { - if (status) { - status.set("queued"); - } else { - status = writable("queued"); - workingFiles.set(fileInfo.id, status); - persistentStates.files = persistentStates.files.map((file) => - file.id === fileInfo.id ? { ...file, status } : file, - ); - } - - let resolver; - const promise = new Promise((resolve) => { - resolver = resolve; - }); - - if (priority) { - queue = [resolver!, ...queue]; - } else { - queue.push(resolver!); - } - - await promise; -}; - export const requestThumbnailGeneration = async (fileInfo: FileInfo) => { - let status = workingFiles.get(fileInfo.id); - if (status && get(status) !== "error") return; - - if (workingFiles.values().some((status) => get(status) !== "error")) { - await enqueue(status, fileInfo); - } - while (memoryUsage >= memoryLimit) { - await enqueue(status, fileInfo, true); - } - + const status = statuses.get(fileInfo.id); if (status) { - status.set("generation-pending"); + if (status !== "error") return; } else { - status = writable("generation-pending"); - workingFiles.set(fileInfo.id, status); - persistentStates.files = persistentStates.files.map((file) => - file.id === fileInfo.id ? { ...file, status } : file, - ); + statuses.set(fileInfo.id, "queued"); } - let fileSize = 0; try { - const file = await requestFileDownload( - fileInfo.id, - fileInfo.contentIv!, - fileInfo.dataKey?.key!, - ); - fileSize = file.byteLength; + let file: ArrayBuffer | undefined; - memoryUsage += fileSize; - if (memoryUsage < memoryLimit) { - queue.shift()?.(); - } - - const thumbnail = await generateThumbnail( - status, - file, - fileInfo.contentType, - fileInfo.dataKey?.key!, + await scheduler.schedule( + async () => { + statuses.set(fileInfo.id, "generation-pending"); + file = await requestFileDownload(fileInfo.id, fileInfo.contentIv!, fileInfo.dataKey?.key!); + return file.byteLength; + }, + async () => { + const thumbnail = await generateThumbnail( + fileInfo.id, + file!, + fileInfo.contentType, + fileInfo.dataKey?.key!, + ); + if ( + !thumbnail || + !(await requestThumbnailUpload(fileInfo.id, fileInfo.dataKey?.version!, thumbnail)) + ) { + statuses.set(fileInfo.id, "error"); + } + }, ); - if ( - !thumbnail || - !(await requestThumbnailUpload(status, fileInfo.id, fileInfo.dataKey?.version!, thumbnail)) - ) { - status.set("error"); - } - } catch { - status.set("error"); - } finally { - memoryUsage -= fileSize; - queue.shift()?.(); + } catch (e) { + statuses.set(fileInfo.id, "error"); + throw e; } }; diff --git a/src/routes/(main)/directory/[[id]]/DirectoryEntries/UploadingFile.svelte b/src/routes/(main)/directory/[[id]]/DirectoryEntries/UploadingFile.svelte index 0ec7263..30e6e20 100644 --- a/src/routes/(main)/directory/[[id]]/DirectoryEntries/UploadingFile.svelte +++ b/src/routes/(main)/directory/[[id]]/DirectoryEntries/UploadingFile.svelte @@ -20,7 +20,9 @@ {state.name}

- {#if state.status === "encryption-pending"} + {#if state.status === "queued"} + 대기 중 + {:else if state.status === "encryption-pending"} 준비 중 {:else if state.status === "encrypting"} 암호화하는 중 diff --git a/src/routes/api/trpc/[trpc]/+server.ts b/src/routes/api/trpc/[trpc]/+server.ts index ec1320a..9c8d8a9 100644 --- a/src/routes/api/trpc/[trpc]/+server.ts +++ b/src/routes/api/trpc/[trpc]/+server.ts @@ -6,6 +6,7 @@ import type { RequestHandler } from "./$types"; const trpcHandler: RequestHandler = (event) => fetchRequestHandler({ endpoint: "/api/trpc", + allowMethodOverride: true, req: event.request, router: appRouter, createContext: () => createContext(event), diff --git a/src/trpc/client.ts b/src/trpc/client.ts index 9c4c5db..f62916a 100644 --- a/src/trpc/client.ts +++ b/src/trpc/client.ts @@ -9,6 +9,7 @@ const createClient = (fetch: typeof globalThis.fetch) => httpBatchLink({ url: "/api/trpc", maxURLLength: 4096, + methodOverride: "POST", transformer: superjson, fetch, }),