동시에 업로드할 수 있는 파일의 메모리 용량을 제한하여 메모리 부족으로 인해 발생하던 크래시 해결

package.json

@@ -1,7 +1,7 @@
 {
   "name": "arkvault",
   "private": true,
-  "version": "0.6.0",
+  "version": "0.7.0",
   "type": "module",
   "scripts": {
     "dev": "vite dev",

src/lib/modules/file/upload.svelte.ts

@@ -10,6 +10,7 @@ import {
   digestMessage,
   signMessageHmac,
 } from "$lib/modules/crypto";
+import { Scheduler } from "$lib/modules/scheduler";
 import { generateThumbnail } from "$lib/modules/thumbnail";
 import type {
   FileThumbnailUploadRequest,
@@ -23,6 +24,7 @@ export interface FileUploadState {
   name: string;
   parentId: DirectoryId;
   status:
+    | "queued"
     | "encryption-pending"
     | "encrypting"
     | "upload-pending"
@@ -36,13 +38,16 @@ export interface FileUploadState {
 }
 
 export type LiveFileUploadState = FileUploadState & {
-  status: "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
+  status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
 };
 
+const scheduler = new Scheduler<
+  { fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
+>();
 let uploadingFiles: FileUploadState[] = $state([]);
 
 const isFileUploading = (status: FileUploadState["status"]) =>
-  ["encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status);
+  ["queued", "encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status);
 
 export const getUploadingFiles = (parentId?: DirectoryId) => {
   return uploadingFiles.filter(
@@ -183,16 +188,19 @@ export const uploadFile = async (
   hmacSecret: HmacSecret,
   masterKey: MasterKey,
   onDuplicate: () => Promise<boolean>,
-): Promise<
-  { fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
-> => {
+) => {
   uploadingFiles.push({
     name: file.name,
     parentId,
-    status: "encryption-pending",
+    status: "queued",
   });
   const state = uploadingFiles.at(-1)!;
 
+  return await scheduler.schedule(
+    async () => file.size,
+    async () => {
+      state.status = "encryption-pending";
+
       try {
         const { fileBuffer, fileSigned } = await requestDuplicateFileScan(
           file,
@@ -259,4 +267,6 @@ export const uploadFile = async (
         state.status = "error";
         throw e;
       }
+    },
+  );
 };

src/lib/modules/scheduler.ts

@@ -0,0 +1,41 @@
+export class Scheduler<T = void> {
+  private tasks = 0;
+  private memoryUsage = 0;
+  private queue: (() => void)[] = [];
+
+  constructor(public memoryLimit = 100 * 1024 * 1024 /* 100 MiB */) {}
+
+  private next() {
+    if (this.memoryUsage < this.memoryLimit) {
+      this.queue.shift()?.();
+    }
+  }
+
+  async schedule(estimateMemoryUsage: () => Promise<number>, task: () => Promise<T>) {
+    if (this.tasks++ > 0) {
+      await new Promise<void>((resolve) => {
+        this.queue.push(resolve);
+      });
+    }
+
+    while (this.memoryUsage >= this.memoryLimit) {
+      await new Promise<void>((resolve) => {
+        this.queue.unshift(resolve);
+      });
+    }
+
+    let taskMemoryUsage = 0;
+
+    try {
+      taskMemoryUsage = await estimateMemoryUsage();
+      this.memoryUsage += taskMemoryUsage;
+      this.next();
+
+      return await task();
+    } finally {
+      this.tasks--;
+      this.memoryUsage -= taskMemoryUsage;
+      this.next();
+    }
+  }
+}

src/routes/(fullscreen)/file/uploads/File.svelte

@@ -18,7 +18,7 @@
 
 <div class="flex h-14 items-center gap-x-4 p-2">
   <div class="flex-shrink-0 text-lg text-gray-600">
-    {#if state.status === "encryption-pending"}
+    {#if state.status === "queued" || state.status === "encryption-pending"}
       <IconPending />
     {:else if state.status === "encrypting"}
       <IconLockClock />
@@ -37,7 +37,9 @@
       {state.name}
     </p>
     <p class="text-xs text-gray-800">
-      {#if state.status === "encryption-pending"}
+      {#if state.status === "queued"}
+        대기 중
+      {:else if state.status === "encryption-pending"}
         준비 중
       {:else if state.status === "encrypting"}
         암호화하는 중

src/routes/(fullscreen)/settings/thumbnail/+page.svelte

@@ -4,17 +4,36 @@
   import { BottomDiv, Button, FullscreenDiv } from "$lib/components/atoms";
   import { IconEntryButton, TopBar } from "$lib/components/molecules";
   import { deleteAllFileThumbnailCaches } from "$lib/modules/file";
-  import { bulkGetFileInfo } from "$lib/modules/filesystem";
+  import { bulkGetFileInfo, type MaybeFileInfo } from "$lib/modules/filesystem";
   import { masterKeyStore } from "$lib/stores";
+  import { sortEntries } from "$lib/utils";
   import File from "./File.svelte";
-  import { persistentStates, requestThumbnailGeneration } from "./service.svelte";
+  import {
+    getThumbnailGenerationStatus,
+    clearThumbnailGenerationStatuses,
+    requestThumbnailGeneration,
+    type GenerationStatus,
+  } from "./service.svelte";
 
   import IconDelete from "~icons/material-symbols/delete";
 
   let { data } = $props();
 
+  let fileInfos: MaybeFileInfo[] = $state([]);
+  let files = $derived(
+    fileInfos
+      .map((info) => ({
+        info,
+        status: getThumbnailGenerationStatus(info.id),
+      }))
+      .filter(
+        (file): file is { info: MaybeFileInfo; status: Exclude<GenerationStatus, "uploaded"> } =>
+          file.status !== "uploaded",
+      ),
+  );
+
   const generateAllThumbnails = () => {
-    persistentStates.files.forEach(({ info }) => {
+    files.forEach(({ info }) => {
       if (info.exists) {
         requestThumbnailGeneration(info);
       }
@@ -22,13 +41,12 @@
   };
 
   onMount(async () => {
-    const fileInfos = await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!);
-    persistentStates.files = persistentStates.files.map(({ id, status }) => ({
-      id,
-      info: fileInfos.get(id)!,
-      status,
-    }));
+    fileInfos = sortEntries(
+      Array.from((await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!)).values()),
+    );
   });
+
+  $effect(() => clearThumbnailGenerationStatuses);
 </script>
 
 <svelte:head>
@@ -43,19 +61,19 @@
         저장된 썸네일 모두 삭제하기
       </IconEntryButton>
     </div>
-    {#if persistentStates.files.length > 0}
+    {#if files.length > 0}
       <div class="flex-grow space-y-2 bg-white p-4">
         <p class="text-lg font-bold text-gray-800">썸네일이 누락된 파일</p>
         <div class="space-y-4">
           <p class="break-keep text-gray-800">
-            {persistentStates.files.length}개 파일의 썸네일이 존재하지 않아요.
+            {files.length}개 파일의 썸네일이 존재하지 않아요.
           </p>
           <div class="space-y-2">
-            {#each persistentStates.files as { info, status } (info.id)}
+            {#each files as { info, status } (info.id)}
               {#if info.exists}
                 <File
                   {info}
-                  generationStatus={status}
+                  {status}
                   onclick={({ id }) => goto(`/file/${id}`)}
                   onGenerateThumbnailClick={requestThumbnailGeneration}
                 />
@@ -66,7 +84,7 @@
       </div>
     {/if}
   </div>
-  {#if persistentStates.files.length > 0}
+  {#if files.length > 0}
     <BottomDiv class="px-4">
       <Button onclick={generateAllThumbnails} class="w-full">모두 썸네일 생성하기</Button>
     </BottomDiv>

src/routes/(fullscreen)/settings/thumbnail/File.svelte

@@ -10,7 +10,6 @@
 </script>
 
 <script lang="ts">
-  import type { Writable } from "svelte/store";
   import { ActionEntryButton } from "$lib/components/atoms";
   import { DirectoryEntryLabel } from "$lib/components/molecules";
   import type { FileInfo } from "$lib/modules/filesystem";
@@ -23,22 +22,21 @@
     info: FileInfo;
     onclick: (file: FileInfo) => void;
     onGenerateThumbnailClick: (file: FileInfo) => void;
-    generationStatus?: Writable<GenerationStatus>;
+    status: Exclude<GenerationStatus, "uploaded"> | undefined;
   }
 
-  let { info, onclick, onGenerateThumbnailClick, generationStatus }: Props = $props();
+  let { info, onclick, onGenerateThumbnailClick, status }: Props = $props();
 </script>
 
 <ActionEntryButton
   class="h-14"
   onclick={() => onclick(info)}
-  actionButtonIcon={!$generationStatus || $generationStatus === "error" ? IconCamera : undefined}
+  actionButtonIcon={!status || status === "error" ? IconCamera : undefined}
   onActionButtonClick={() => onGenerateThumbnailClick(info)}
   actionButtonClass="text-gray-800"
 >
-  {@const subtext =
-    $generationStatus && $generationStatus !== "uploaded"
-      ? subtexts[$generationStatus]
+  {@const subtext = status
+    ? subtexts[status]
     : formatDateTime(info.createdAt ?? info.lastModifiedAt)}
   <DirectoryEntryLabel type="file" name={info.name} {subtext} />
 </ActionEntryButton>

src/routes/(fullscreen)/settings/thumbnail/service.svelte.ts

@@ -1,8 +1,9 @@
 import { limitFunction } from "p-limit";
-import { get, writable, type Writable } from "svelte/store";
+import { SvelteMap } from "svelte/reactivity";
 import { encryptData } from "$lib/modules/crypto";
 import { storeFileThumbnailCache } from "$lib/modules/file";
-import type { FileInfo, MaybeFileInfo } from "$lib/modules/filesystem";
+import type { FileInfo } from "$lib/modules/filesystem";
+import { Scheduler } from "$lib/modules/scheduler";
 import { generateThumbnail as doGenerateThumbnail } from "$lib/modules/thumbnail";
 import { requestFileDownload, requestFileThumbnailUpload } from "$lib/services/file";
 
@@ -15,41 +16,31 @@ export type GenerationStatus =
   | "uploaded"
   | "error";
 
-interface File {
-  id: number;
-  info: MaybeFileInfo;
-  status?: Writable<GenerationStatus>;
-}
+const scheduler = new Scheduler();
+const statuses = new SvelteMap<number, GenerationStatus>();
 
-const workingFiles = new Map<number, Writable<GenerationStatus>>();
+export const getThumbnailGenerationStatus = (fileId: number) => {
+  return statuses.get(fileId);
+};
 
-let queue: (() => void)[] = [];
-let memoryUsage = 0;
-const memoryLimit = 100 * 1024 * 1024; // 100 MiB
-
-export const persistentStates = $state({
-  files: [] as File[],
-});
-
-export const getGenerationStatus = (fileId: number) => {
-  return workingFiles.get(fileId);
+export const clearThumbnailGenerationStatuses = () => {
+  for (const [id, status] of statuses) {
+    if (status === "uploaded" || status === "error") {
+      statuses.delete(id);
+    }
+  }
 };
 
 const generateThumbnail = limitFunction(
-  async (
-    status: Writable<GenerationStatus>,
-    fileBuffer: ArrayBuffer,
-    fileType: string,
-    dataKey: CryptoKey,
-  ) => {
-    status.set("generating");
+  async (fileId: number, fileBuffer: ArrayBuffer, fileType: string, dataKey: CryptoKey) => {
+    statuses.set(fileId, "generating");
 
     const thumbnail = await doGenerateThumbnail(fileBuffer, fileType);
     if (!thumbnail) return null;
 
     const thumbnailBuffer = await thumbnail.arrayBuffer();
     const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey);
-    status.set("upload-pending");
+    statuses.set(fileId, "upload-pending");
     return { plaintext: thumbnailBuffer, ...thumbnailEncrypted };
   },
   { concurrency: 4 },
@@ -57,106 +48,55 @@ const generateThumbnail = limitFunction(
 
 const requestThumbnailUpload = limitFunction(
   async (
-    status: Writable<GenerationStatus>,
     fileId: number,
     dataKeyVersion: Date,
     thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: string },
   ) => {
-    status.set("uploading");
+    statuses.set(fileId, "uploading");
 
     const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnail);
     if (!res.ok) return false;
-
-    status.set("uploaded");
-    workingFiles.delete(fileId);
-    persistentStates.files = persistentStates.files.filter(({ id }) => id != fileId);
-
+    statuses.set(fileId, "uploaded");
     storeFileThumbnailCache(fileId, thumbnail.plaintext); // Intended
     return true;
   },
   { concurrency: 4 },
 );
 
-const enqueue = async (
-  status: Writable<GenerationStatus> | undefined,
-  fileInfo: FileInfo,
-  priority = false,
-) => {
-  if (status) {
-    status.set("queued");
-  } else {
-    status = writable("queued");
-    workingFiles.set(fileInfo.id, status);
-    persistentStates.files = persistentStates.files.map((file) =>
-      file.id === fileInfo.id ? { ...file, status } : file,
-    );
-  }
-
-  let resolver;
-  const promise = new Promise((resolve) => {
-    resolver = resolve;
-  });
-
-  if (priority) {
-    queue = [resolver!, ...queue];
-  } else {
-    queue.push(resolver!);
-  }
-
-  await promise;
-};
-
 export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
-  let status = workingFiles.get(fileInfo.id);
-  if (status && get(status) !== "error") return;
-
-  if (workingFiles.values().some((status) => get(status) !== "error")) {
-    await enqueue(status, fileInfo);
-  }
-  while (memoryUsage >= memoryLimit) {
-    await enqueue(status, fileInfo, true);
-  }
-
+  const status = statuses.get(fileInfo.id);
   if (status) {
-    status.set("generation-pending");
+    if (status !== "error") return;
   } else {
-    status = writable("generation-pending");
-    workingFiles.set(fileInfo.id, status);
-    persistentStates.files = persistentStates.files.map((file) =>
-      file.id === fileInfo.id ? { ...file, status } : file,
-    );
+    statuses.set(fileInfo.id, "queued");
   }
 
-  let fileSize = 0;
   try {
-    const file = await requestFileDownload(
-      fileInfo.id,
-      fileInfo.contentIv!,
-      fileInfo.dataKey?.key!,
-    );
-    fileSize = file.byteLength;
-
-    memoryUsage += fileSize;
-    if (memoryUsage < memoryLimit) {
-      queue.shift()?.();
-    }
+    let file: ArrayBuffer | undefined;
 
+    await scheduler.schedule(
+      async () => {
+        statuses.set(fileInfo.id, "generation-pending");
+        file = await requestFileDownload(fileInfo.id, fileInfo.contentIv!, fileInfo.dataKey?.key!);
+        return file.byteLength;
+      },
+      async () => {
         const thumbnail = await generateThumbnail(
-      status,
-      file,
+          fileInfo.id,
+          file!,
           fileInfo.contentType,
           fileInfo.dataKey?.key!,
         );
         if (
           !thumbnail ||
-      !(await requestThumbnailUpload(status, fileInfo.id, fileInfo.dataKey?.version!, thumbnail))
+          !(await requestThumbnailUpload(fileInfo.id, fileInfo.dataKey?.version!, thumbnail))
         ) {
-      status.set("error");
+          statuses.set(fileInfo.id, "error");
         }
-  } catch {
-    status.set("error");
-  } finally {
-    memoryUsage -= fileSize;
-    queue.shift()?.();
+      },
+    );
+  } catch (e) {
+    statuses.set(fileInfo.id, "error");
+    throw e;
   }
 };

src/routes/(main)/directory/[[id]]/DirectoryEntries/UploadingFile.svelte

@@ -20,7 +20,9 @@
       {state.name}
     </p>
     <p class="text-xs">
-      {#if state.status === "encryption-pending"}
+      {#if state.status === "queued"}
+        대기 중
+      {:else if state.status === "encryption-pending"}
         준비 중
       {:else if state.status === "encrypting"}
         암호화하는 중

src/routes/api/trpc/[trpc]/+server.ts

@@ -6,6 +6,7 @@ import type { RequestHandler } from "./$types";
 const trpcHandler: RequestHandler = (event) =>
   fetchRequestHandler({
     endpoint: "/api/trpc",
+    allowMethodOverride: true,
     req: event.request,
     router: appRouter,
     createContext: () => createContext(event),

src/trpc/client.ts

@@ -9,6 +9,7 @@ const createClient = (fetch: typeof globalThis.fetch) =>
       httpBatchLink({
         url: "/api/trpc",
         maxURLLength: 4096,
+        methodOverride: "POST",
         transformer: superjson,
         fetch,
       }),