파일 업로드 스케쥴링 구현

암호화는 동시에 최대 4개까지, 업로드는 1개까지 가능하도록 설정했습니다.

src/hooks.client.ts

@@ -1,6 +1,6 @@
 import type { ClientInit } from "@sveltejs/kit";
 import { getClientKey, getMasterKeys, getHmacSecrets } from "$lib/indexedDB";
-import { prepareFileCache } from "$lib/modules/cache";
+import { prepareFileCache } from "$lib/modules/file";
 import { prepareOpfs } from "$lib/modules/opfs";
 import { clientKeyStore, masterKeyStore, hmacSecretStore } from "$lib/stores";
 

src/lib/modules/file/index.ts

@@ -0,0 +1,3 @@
+export * from "./cache";
+export * from "./info";
+export * from "./upload";

src/lib/modules/file/upload.ts

@@ -0,0 +1,221 @@
+import axios from "axios";
+import ExifReader from "exifreader";
+import { limitFunction } from "p-limit";
+import { writable, type Writable } from "svelte/store";
+import {
+  encodeToBase64,
+  generateDataKey,
+  wrapDataKey,
+  encryptData,
+  encryptString,
+  signMessageHmac,
+} from "$lib/modules/crypto";
+import type {
+  DuplicateFileScanRequest,
+  DuplicateFileScanResponse,
+  FileUploadRequest,
+} from "$lib/server/schemas";
+import {
+  fileUploadStatusStore,
+  type MasterKey,
+  type HmacSecret,
+  type FileUploadStatus,
+} from "$lib/stores";
+
+const requestDuplicateFileScan = limitFunction(
+  async (file: File, hmacSecret: HmacSecret, onDuplicate: () => Promise<boolean>) => {
+    const fileBuffer = await file.arrayBuffer();
+    const fileSigned = encodeToBase64(await signMessageHmac(fileBuffer, hmacSecret.secret));
+
+    const res = await axios.post("/api/file/scanDuplicates", {
+      hskVersion: hmacSecret.version,
+      contentHmac: fileSigned,
+    } satisfies DuplicateFileScanRequest);
+    const { files }: DuplicateFileScanResponse = res.data;
+
+    if (files.length === 0 || (await onDuplicate())) {
+      return { fileBuffer, fileSigned };
+    } else {
+      return {};
+    }
+  },
+  { concurrency: 1 },
+);
+
+const getFileType = (file: File) => {
+  if (file.type) return file.type;
+  if (file.name.endsWith(".heic")) return "image/heic";
+  throw new Error("Unknown file type");
+};
+
+const extractExifDateTime = (fileBuffer: ArrayBuffer) => {
+  const exif = ExifReader.load(fileBuffer);
+  const dateTimeOriginal = exif["DateTimeOriginal"]?.description;
+  const offsetTimeOriginal = exif["OffsetTimeOriginal"]?.description;
+  if (!dateTimeOriginal) return undefined;
+
+  const [date, time] = dateTimeOriginal.split(" ");
+  if (!date || !time) return undefined;
+
+  const [year, month, day] = date.split(":").map(Number);
+  const [hour, minute, second] = time.split(":").map(Number);
+  if (!year || !month || !day || !hour || !minute || !second) return undefined;
+
+  if (!offsetTimeOriginal) {
+    // No timezone information.. Assume local timezone
+    return new Date(year, month - 1, day, hour, minute, second);
+  }
+
+  const offsetSign = offsetTimeOriginal[0] === "+" ? 1 : -1;
+  const [offsetHour, offsetMinute] = offsetTimeOriginal.slice(1).split(":").map(Number);
+
+  const utcDate = Date.UTC(year, month - 1, day, hour, minute, second);
+  const offsetMs = offsetSign * ((offsetHour ?? 0) * 60 + (offsetMinute ?? 0)) * 60 * 1000;
+  return new Date(utcDate - offsetMs);
+};
+
+const encryptFile = limitFunction(
+  async (
+    status: Writable<FileUploadStatus>,
+    file: File,
+    fileBuffer: ArrayBuffer,
+    masterKey: MasterKey,
+  ) => {
+    status.update((value) => {
+      value.status = "encrypting";
+      return value;
+    });
+
+    const fileType = getFileType(file);
+
+    let createdAt;
+    if (fileType.startsWith("image/")) {
+      createdAt = extractExifDateTime(fileBuffer);
+    }
+
+    const { dataKey, dataKeyVersion } = await generateDataKey();
+    const dataKeyWrapped = await wrapDataKey(dataKey, masterKey.key);
+
+    const fileEncrypted = await encryptData(fileBuffer, dataKey);
+    const nameEncrypted = await encryptString(file.name, dataKey);
+    const createdAtEncrypted =
+      createdAt && (await encryptString(createdAt.getTime().toString(), dataKey));
+    const lastModifiedAtEncrypted = await encryptString(file.lastModified.toString(), dataKey);
+
+    status.update((value) => {
+      value.status = "upload-pending";
+      return value;
+    });
+
+    return {
+      dataKeyWrapped,
+      dataKeyVersion,
+      fileEncrypted,
+      fileType,
+      nameEncrypted,
+      createdAtEncrypted,
+      lastModifiedAtEncrypted,
+    };
+  },
+  { concurrency: 4 },
+);
+
+const uploadFileInternal = limitFunction(
+  async (status: Writable<FileUploadStatus>, form: FormData) => {
+    status.update((value) => {
+      value.status = "uploading";
+      return value;
+    });
+
+    await axios.post("/api/file/upload", form, {
+      onUploadProgress: ({ progress, rate, estimated }) => {
+        status.update((value) => {
+          value.progress = progress;
+          value.rate = rate;
+          value.estimated = estimated;
+          return value;
+        });
+      },
+    });
+
+    status.update((value) => {
+      value.status = "uploaded";
+      return value;
+    });
+  },
+  { concurrency: 1 },
+);
+
+export const uploadFile = async (
+  file: File,
+  parentId: "root" | number,
+  hmacSecret: HmacSecret,
+  masterKey: MasterKey,
+  onDuplicate: () => Promise<boolean>,
+) => {
+  const status = writable<FileUploadStatus>({
+    name: file.name,
+    parentId,
+    status: "encryption-pending",
+  });
+  fileUploadStatusStore.update((value) => {
+    value.push(status);
+    return value;
+  });
+
+  try {
+    const { fileBuffer, fileSigned } = await requestDuplicateFileScan(
+      file,
+      hmacSecret,
+      onDuplicate,
+    );
+    if (!fileBuffer || !fileSigned) {
+      status.update((value) => {
+        value.status = "canceled";
+        return value;
+      });
+      return false;
+    }
+
+    const {
+      dataKeyWrapped,
+      dataKeyVersion,
+      fileEncrypted,
+      fileType,
+      nameEncrypted,
+      createdAtEncrypted,
+      lastModifiedAtEncrypted,
+    } = await encryptFile(status, file, fileBuffer, masterKey);
+
+    const form = new FormData();
+    form.set(
+      "metadata",
+      JSON.stringify({
+        parentId,
+        mekVersion: masterKey.version,
+        dek: dataKeyWrapped,
+        dekVersion: dataKeyVersion.toISOString(),
+        hskVersion: hmacSecret.version,
+        contentHmac: fileSigned,
+        contentType: fileType,
+        contentIv: fileEncrypted.iv,
+        name: nameEncrypted.ciphertext,
+        nameIv: nameEncrypted.iv,
+        createdAt: createdAtEncrypted?.ciphertext,
+        createdAtIv: createdAtEncrypted?.iv,
+        lastModifiedAt: lastModifiedAtEncrypted.ciphertext,
+        lastModifiedAtIv: lastModifiedAtEncrypted.iv,
+      } as FileUploadRequest),
+    );
+    form.set("content", new Blob([fileEncrypted.ciphertext]));
+
+    await uploadFileInternal(status, form);
+    return true;
+  } catch (e) {
+    status.update((value) => {
+      value.status = "error";
+      return value;
+    });
+    throw e;
+  }
+};

src/lib/stores/file.ts

@@ -1,4 +1,4 @@
-import type { Writable } from "svelte/store";
+import { writable, type Writable } from "svelte/store";
 
 export type DirectoryInfo =
   | {
@@ -29,6 +29,24 @@ export interface FileInfo {
   lastModifiedAt: Date;
 }
 
+export interface FileUploadStatus {
+  name: string;
+  parentId: "root" | number;
+  status:
+    | "encryption-pending"
+    | "encrypting"
+    | "upload-pending"
+    | "uploading"
+    | "uploaded"
+    | "canceled"
+    | "error";
+  progress?: number;
+  rate?: number;
+  estimated?: number;
+}
+
 export const directoryInfoStore = new Map<"root" | number, Writable<DirectoryInfo | null>>();
 
 export const fileInfoStore = new Map<number, Writable<FileInfo | null>>();
+
+export const fileUploadStatusStore = writable<Writable<FileUploadStatus>[]>([]);

src/routes/(fullscreen)/file/[id]/service.ts

@@ -1,4 +1,4 @@
-import { getFileCache, storeFileCache } from "$lib/modules/cache";
+import { getFileCache, storeFileCache } from "$lib/modules/file";
 import { decryptData } from "$lib/modules/crypto";
 
 export const requestFileDownload = async (

src/routes/(fullscreen)/setting/cache/+page.svelte

@@ -3,8 +3,7 @@
   import type { Writable } from "svelte/store";
   import { TopBar } from "$lib/components";
   import type { FileCacheIndex } from "$lib/indexedDB";
-  import { getFileCacheIndex } from "$lib/modules/cache";
-  import { getFileInfo } from "$lib/modules/file";
+  import { getFileCacheIndex, getFileInfo } from "$lib/modules/file";
   import { masterKeyStore, type FileInfo } from "$lib/stores";
   import File from "./File.svelte";
   import { formatFileSize, deleteFileCache as doDeleteFileCache } from "./service";

src/routes/(fullscreen)/setting/cache/service.ts

@@ -1,4 +1,4 @@
-import { deleteFileCache as doDeleteFileCache } from "$lib/modules/cache";
+import { deleteFileCache as doDeleteFileCache } from "$lib/modules/file";
 
 export { formatDate, formatFileSize } from "$lib/modules/util";
 

src/routes/(main)/directory/[[id]]/+page.svelte

@@ -16,7 +16,6 @@
   import {
     requestHmacSecretDownload,
     requestDirectoryCreation,
-    requestDuplicateFileScan,
     requestFileUpload,
     requestDirectoryEntryRename,
     requestDirectoryEntryDeletion,
@@ -25,17 +24,11 @@
 
   import IconAdd from "~icons/material-symbols/add";
 
-  interface LoadedFile {
-    file: File;
-    fileBuffer: ArrayBuffer;
-    fileSigned: string;
-  }
-
   let { data } = $props();
 
   let info: Writable<DirectoryInfo | null> | undefined = $state();
   let fileInput: HTMLInputElement | undefined = $state();
-  let loadedFile: LoadedFile | undefined = $state();
+  let resolveForDuplicateFileModal: ((res: boolean) => void) | undefined = $state();
   let selectedEntry: SelectedDirectoryEntry | undefined = $state();
 
   let isCreateBottomSheetOpen = $state(false);
@@ -52,43 +45,32 @@
     info = getDirectoryInfo(data.id, $masterKeyStore?.get(1)?.key!); // TODO: FIXME
   };
 
-  const uploadFile = (loadedFile: LoadedFile) => {
-    requestFileUpload(
-      loadedFile.file,
-      loadedFile.fileBuffer,
-      loadedFile.fileSigned,
-      data.id,
-      $masterKeyStore?.get(1)!,
-      $hmacSecretStore?.get(1)!,
-    )
-      .then(() => {
+  const uploadFile = () => {
+    const file = fileInput?.files?.[0];
+    if (!file) return;
+
+    fileInput!.value = "";
+
+    requestFileUpload(file, data.id, $hmacSecretStore?.get(1)!, $masterKeyStore?.get(1)!, () => {
+      return new Promise((resolve) => {
+        resolveForDuplicateFileModal = resolve;
+        isDuplicateFileModalOpen = true;
+      });
+    })
+      .then((res) => {
+        if (!res) return;
+
         // TODO: FIXME
         info = getDirectoryInfo(data.id, $masterKeyStore?.get(1)?.key!);
         window.alert("파일이 업로드되었어요.");
       })
       .catch((e: Error) => {
         // TODO: FIXME
+        console.error(e);
         window.alert(`파일 업로드에 실패했어요.\n${e.message}`);
       });
   };
 
-  const loadAndUploadFile = async () => {
-    const file = fileInput?.files?.[0];
-    if (!file) return;
-
-    fileInput!.value = "";
-
-    const scanRes = await requestDuplicateFileScan(file, $hmacSecretStore?.get(1)!);
-    if (scanRes === null) {
-      throw new Error("Failed to scan duplicate files");
-    } else if (scanRes.isDuplicate) {
-      loadedFile = { ...scanRes, file };
-      isDuplicateFileModalOpen = true;
-    } else {
-      uploadFile({ ...scanRes, file });
-    }
-  };
-
   onMount(async () => {
     if (!$hmacSecretStore && !(await requestHmacSecretDownload($masterKeyStore?.get(1)?.key!))) {
       throw new Error("Failed to download hmac secrets");
@@ -104,7 +86,7 @@
   <title>파일</title>
 </svelte:head>
 
-<input bind:this={fileInput} onchange={loadAndUploadFile} type="file" class="hidden" />
+<input bind:this={fileInput} onchange={uploadFile} type="file" class="hidden" />
 
 <div class="flex min-h-full flex-col px-4">
   {#if data.id !== "root"}
@@ -148,13 +130,14 @@
 <DuplicateFileModal
   bind:isOpen={isDuplicateFileModalOpen}
   onclose={() => {
+    resolveForDuplicateFileModal?.(false);
+    resolveForDuplicateFileModal = undefined;
     isDuplicateFileModalOpen = false;
-    loadedFile = undefined;
   }}
   onDuplicateClick={() => {
-    uploadFile(loadedFile!);
+    resolveForDuplicateFileModal?.(true);
+    resolveForDuplicateFileModal = undefined;
     isDuplicateFileModalOpen = false;
-    loadedFile = undefined;
   }}
 />
 

src/routes/(main)/directory/[[id]]/DuplicateFileModal.svelte

@@ -18,14 +18,7 @@
       <p>그래도 업로드할까요?</p>
     </div>
     <div class="flex gap-2">
-      <Button
-        color="gray"
-        onclick={() => {
-          isOpen = false;
-        }}
-      >
-        아니요
-      </Button>
+      <Button color="gray" onclick={onclose}>아니요</Button>
       <Button onclick={onDuplicateClick}>업로드할게요</Button>
     </div>
   </div>

src/routes/(main)/directory/[[id]]/service.ts

@@ -1,24 +1,12 @@
-import ExifReader from "exifreader";
 import { callGetApi, callPostApi } from "$lib/hooks";
 import { storeHmacSecrets } from "$lib/indexedDB";
-import { deleteFileCache } from "$lib/modules/cache";
-import {
-  encodeToBase64,
-  generateDataKey,
-  wrapDataKey,
-  unwrapHmacSecret,
-  encryptData,
-  encryptString,
-  signMessageHmac,
-} from "$lib/modules/crypto";
+import { deleteFileCache, uploadFile } from "$lib/modules/file";
+import { generateDataKey, wrapDataKey, unwrapHmacSecret, encryptString } from "$lib/modules/crypto";
 import type {
   DirectoryRenameRequest,
   DirectoryCreateRequest,
   FileRenameRequest,
-  FileUploadRequest,
   HmacSecretListResponse,
-  DuplicateFileScanRequest,
-  DuplicateFileScanResponse,
   DirectoryDeleteResponse,
 } from "$lib/server/schemas";
 import { hmacSecretStore, type MasterKey, type HmacSecret } from "$lib/stores";
@@ -68,106 +56,14 @@ export const requestDirectoryCreation = async (
   });
 };
 
-export const requestDuplicateFileScan = async (file: File, hmacSecret: HmacSecret) => {
-  const fileBuffer = await file.arrayBuffer();
-  const fileSigned = encodeToBase64(await signMessageHmac(fileBuffer, hmacSecret.secret));
-  const res = await callPostApi<DuplicateFileScanRequest>("/api/file/scanDuplicates", {
-    hskVersion: hmacSecret.version,
-    contentHmac: fileSigned,
-  });
-  if (!res.ok) return null;
-
-  const { files }: DuplicateFileScanResponse = await res.json();
-  return {
-    fileBuffer,
-    fileSigned,
-    isDuplicate: files.length > 0,
-  };
-};
-
-const extractExifDateTime = (fileBuffer: ArrayBuffer) => {
-  const exif = ExifReader.load(fileBuffer);
-  const dateTimeOriginal = exif["DateTimeOriginal"]?.description;
-  const offsetTimeOriginal = exif["OffsetTimeOriginal"]?.description;
-  if (!dateTimeOriginal) return undefined;
-
-  const [date, time] = dateTimeOriginal.split(" ");
-  if (!date || !time) return undefined;
-
-  const [year, month, day] = date.split(":").map(Number);
-  const [hour, minute, second] = time.split(":").map(Number);
-  if (!year || !month || !day || !hour || !minute || !second) return undefined;
-
-  if (!offsetTimeOriginal) {
-    // No timezone information -> Local timezone
-    return new Date(year, month - 1, day, hour, minute, second);
-  }
-
-  const offsetSign = offsetTimeOriginal[0] === "+" ? 1 : -1;
-  const [offsetHour, offsetMinute] = offsetTimeOriginal.slice(1).split(":").map(Number);
-
-  const utcDate = Date.UTC(year, month - 1, day, hour, minute, second);
-  const offsetMs = offsetSign * ((offsetHour ?? 0) * 60 + (offsetMinute ?? 0)) * 60 * 1000;
-  return new Date(utcDate - offsetMs);
-};
-
 export const requestFileUpload = async (
   file: File,
-  fileBuffer: ArrayBuffer,
-  fileSigned: string,
   parentId: "root" | number,
-  masterKey: MasterKey,
   hmacSecret: HmacSecret,
+  masterKey: MasterKey,
+  onDuplicate: () => Promise<boolean>,
 ) => {
-  let createdAt = undefined;
-  if (file.type.startsWith("image/")) {
-    createdAt = extractExifDateTime(fileBuffer);
-  }
-
-  const { dataKey, dataKeyVersion } = await generateDataKey();
-  const fileEncrypted = await encryptData(fileBuffer, dataKey);
-  const nameEncrypted = await encryptString(file.name, dataKey);
-  const createdAtEncrypted =
-    createdAt && (await encryptString(createdAt.getTime().toString(), dataKey));
-  const lastModifiedAtEncrypted = await encryptString(file.lastModified.toString(), dataKey);
-
-  const form = new FormData();
-  form.set(
-    "metadata",
-    JSON.stringify({
-      parentId,
-      mekVersion: masterKey.version,
-      dek: await wrapDataKey(dataKey, masterKey.key),
-      dekVersion: dataKeyVersion.toISOString(),
-      hskVersion: hmacSecret.version,
-      contentHmac: fileSigned,
-      contentType: file.type,
-      contentIv: fileEncrypted.iv,
-      name: nameEncrypted.ciphertext,
-      nameIv: nameEncrypted.iv,
-      createdAt: createdAtEncrypted?.ciphertext,
-      createdAtIv: createdAtEncrypted?.iv,
-      lastModifiedAt: lastModifiedAtEncrypted.ciphertext,
-      lastModifiedAtIv: lastModifiedAtEncrypted.iv,
-    } satisfies FileUploadRequest),
-  );
-  form.set("content", new Blob([fileEncrypted.ciphertext]));
-
-  return new Promise<void>((resolve, reject) => {
-    // TODO: Progress, Scheduling, ...
-
-    const xhr = new XMLHttpRequest();
-    xhr.addEventListener("load", () => {
-      if (xhr.status === 200) {
-        resolve();
-      } else {
-        reject(new Error(xhr.responseText));
-      }
-    });
-
-    xhr.open("POST", "/api/file/upload");
-    xhr.send(form);
-  });
+  return await uploadFile(file, parentId, hmacSecret, masterKey, onDuplicate);
 };
 
 export const requestDirectoryEntryRename = async (