이전 버전에서 업로드된 파일을 청크 업로드 방식으로 마이그레이션할 수 있는 기능 추가

src/lib/server/db/error.ts

@@ -9,6 +9,7 @@ type IntegrityErrorMessages =
   // File
   | "Directory not found"
   | "File not found"
+  | "File is not legacy"
   | "File not found in category"
   | "File already added to category"
   | "Invalid DEK version"

src/lib/server/db/file.ts

@@ -334,6 +334,16 @@ export const getAllFileIds = async (userId: number) => {
   return files.map(({ id }) => id);
 };
 
+export const getLegacyFileIds = async (userId: number) => {
+  const files = await db
+    .selectFrom("file")
+    .select("id")
+    .where("user_id", "=", userId)
+    .where("encrypted_content_iv", "is not", null)
+    .execute();
+  return files.map(({ id }) => id);
+};
+
 export const getAllFileIdsByContentHmac = async (
   userId: number,
   hskVersion: number,
@@ -482,6 +492,52 @@ export const unregisterFile = async (userId: number, fileId: number) => {
   });
 };
 
+export const migrateFileContent = async (
+  trx: typeof db,
+  userId: number,
+  fileId: number,
+  newPath: string,
+  encContentHash: string,
+) => {
+  const file = await trx
+    .selectFrom("file")
+    .select(["path", "encrypted_content_iv"])
+    .where("id", "=", fileId)
+    .where("user_id", "=", userId)
+    .limit(1)
+    .forUpdate()
+    .executeTakeFirst();
+
+  if (!file) {
+    throw new IntegrityError("File not found");
+  }
+  if (!file.encrypted_content_iv) {
+    throw new IntegrityError("File is not legacy");
+  }
+
+  await trx
+    .updateTable("file")
+    .set({
+      path: newPath,
+      encrypted_content_iv: null,
+      encrypted_content_hash: encContentHash,
+    })
+    .where("id", "=", fileId)
+    .where("user_id", "=", userId)
+    .execute();
+
+  await trx
+    .insertInto("file_log")
+    .values({
+      file_id: fileId,
+      timestamp: new Date(),
+      action: "migrate",
+    })
+    .execute();
+
+  return file.path;
+};
+
 export const addFileToCategory = async (fileId: number, categoryId: number) => {
   await db.transaction().execute(async (trx) => {
     try {

src/lib/server/db/schema/file.ts

@@ -41,7 +41,7 @@ interface FileLogTable {
   id: Generated<number>;
   file_id: number;
   timestamp: ColumnType<Date, Date, never>;
-  action: "create" | "rename" | "add-to-category" | "remove-from-category";
+  action: "create" | "rename" | "migrate" | "add-to-category" | "remove-from-category";
   new_name: Ciphertext | null;
   category_id: number | null;
 }

src/lib/server/db/schema/upload.ts

@@ -3,7 +3,7 @@ import type { Ciphertext } from "./utils";
 
 interface UploadSessionTable {
   id: string;
-  type: "file" | "thumbnail";
+  type: "file" | "thumbnail" | "migration";
   user_id: number;
   path: string;
   total_chunks: number;

src/lib/server/db/upload.ts

@@ -31,6 +31,11 @@ interface ThumbnailUploadSession extends BaseUploadSession {
   dekVersion: Date;
 }
 
+interface MigrationUploadSession extends BaseUploadSession {
+  type: "migration";
+  fileId: number;
+}
+
 export const createFileUploadSession = async (
   params: Omit<FileUploadSession, "type" | "uploadedChunks">,
 ) => {
@@ -118,6 +123,39 @@ export const createThumbnailUploadSession = async (
   });
 };
 
+export const createMigrationUploadSession = async (
+  params: Omit<MigrationUploadSession, "type" | "uploadedChunks">,
+) => {
+  await db.transaction().execute(async (trx) => {
+    const file = await trx
+      .selectFrom("file")
+      .select("encrypted_content_iv")
+      .where("id", "=", params.fileId)
+      .where("user_id", "=", params.userId)
+      .limit(1)
+      .forUpdate()
+      .executeTakeFirst();
+    if (!file) {
+      throw new IntegrityError("File not found");
+    } else if (!file.encrypted_content_iv) {
+      throw new IntegrityError("File is not legacy");
+    }
+
+    await trx
+      .insertInto("upload_session")
+      .values({
+        id: params.id,
+        type: "migration",
+        user_id: params.userId,
+        path: params.path,
+        total_chunks: params.totalChunks,
+        expires_at: params.expiresAt,
+        file_id: params.fileId,
+      })
+      .execute();
+  });
+};
+
 export const getUploadSession = async (sessionId: string, userId: number) => {
   const session = await db
     .selectFrom("upload_session")
@@ -148,7 +186,7 @@ export const getUploadSession = async (sessionId: string, userId: number) => {
       encCreatedAt: session.encrypted_created_at,
       encLastModifiedAt: session.encrypted_last_modified_at!,
     } satisfies FileUploadSession;
-  } else {
+  } else if (session.type === "thumbnail") {
     return {
       type: "thumbnail",
       id: session.id,
@@ -160,6 +198,17 @@ export const getUploadSession = async (sessionId: string, userId: number) => {
       fileId: session.file_id!,
       dekVersion: session.data_encryption_key_version!,
     } satisfies ThumbnailUploadSession;
+  } else {
+    return {
+      type: "migration",
+      id: session.id,
+      userId: session.user_id,
+      path: session.path,
+      totalChunks: session.total_chunks,
+      uploadedChunks: session.uploaded_chunks,
+      expiresAt: session.expires_at,
+      fileId: session.file_id!,
+    } satisfies MigrationUploadSession;
   }
 };
 

src/routes/(fullscreen)/settings/migration/+page.server.ts

@@ -0,0 +1,7 @@
+import { createCaller } from "$trpc/router.server";
+import type { PageServerLoad } from "./$types";
+
+export const load: PageServerLoad = async (event) => {
+  const files = await createCaller(event).file.listLegacy();
+  return { files };
+};

src/routes/(fullscreen)/settings/migration/+page.svelte

@@ -0,0 +1,79 @@
+<script lang="ts">
+  import { onMount } from "svelte";
+  import { goto } from "$app/navigation";
+  import { BottomDiv, Button, FullscreenDiv } from "$lib/components/atoms";
+  import { TopBar } from "$lib/components/molecules";
+  import { bulkGetFileInfo, type MaybeFileInfo } from "$lib/modules/filesystem";
+  import { masterKeyStore } from "$lib/stores";
+  import { sortEntries } from "$lib/utils";
+  import File from "./File.svelte";
+  import { getMigrationState, clearMigrationStates, requestFileMigration } from "./service.svelte";
+
+  let { data } = $props();
+
+  let fileInfos: MaybeFileInfo[] = $state([]);
+  let files = $derived(
+    fileInfos
+      .map((info) => ({
+        info,
+        state: getMigrationState(info.id),
+      }))
+      .filter((file) => file.state?.status !== "completed"),
+  );
+
+  const migrateAllFiles = () => {
+    files.forEach(({ info }) => {
+      if (info.exists) {
+        requestFileMigration(info);
+      }
+    });
+  };
+
+  onMount(async () => {
+    fileInfos = sortEntries(
+      Array.from((await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!)).values()),
+    );
+  });
+
+  $effect(() => clearMigrationStates);
+</script>
+
+<svelte:head>
+  <title>암호화 마이그레이션</title>
+</svelte:head>
+
+<TopBar title="암호화 마이그레이션" />
+<FullscreenDiv>
+  {#if files.length > 0}
+    <div class="space-y-4 pb-4">
+      <p class="break-keep text-gray-800">
+        이전 버전의 ArkVault에서 업로드된 {files.length}개 파일을 다시 암호화할 수 있어요.
+      </p>
+      <div class="space-y-2">
+        {#each files as { info, state } (info.id)}
+          {#if info.exists}
+            <File
+              {info}
+              {state}
+              onclick={({ id }) => goto(`/file/${id}`)}
+              onMigrateClick={requestFileMigration}
+            />
+          {/if}
+        {/each}
+      </div>
+    </div>
+    <BottomDiv>
+      <Button onclick={migrateAllFiles} class="w-full">모두 다시 암호화하기</Button>
+    </BottomDiv>
+  {:else}
+    <div class="flex flex-grow items-center justify-center">
+      <p class="text-gray-500">
+        {#if data.files.length === 0}
+          마이그레이션할 파일이 없어요.
+        {:else}
+          파일 목록을 불러오고 있어요.
+        {/if}
+      </p>
+    </div>
+  {/if}
+</FullscreenDiv>

src/routes/(fullscreen)/settings/migration/File.svelte

@@ -0,0 +1,55 @@
+<script module lang="ts">
+  const subtexts = {
+    queued: "대기 중",
+    "download-pending": "다운로드를 기다리는 중",
+    downloading: "다운로드하는 중",
+    "encryption-pending": "암호화를 기다리는 중",
+    encrypting: "암호화하는 중",
+    "upload-pending": "업로드를 기다리는 중",
+    completed: "완료",
+    error: "실패",
+  } as const;
+</script>
+
+<script lang="ts">
+  import { ActionEntryButton } from "$lib/components/atoms";
+  import { DirectoryEntryLabel } from "$lib/components/molecules";
+  import type { FileInfo } from "$lib/modules/filesystem";
+  import { formatDateTime, formatNetworkSpeed } from "$lib/utils";
+  import type { MigrationState } from "./service.svelte";
+
+  import IconSync from "~icons/material-symbols/sync";
+
+  type FileInfoWithExists = FileInfo & { exists: true };
+
+  interface Props {
+    info: FileInfoWithExists;
+    onclick: (file: FileInfo) => void;
+    onMigrateClick: (file: FileInfoWithExists) => void;
+    state: MigrationState | undefined;
+  }
+
+  let { info, onclick, onMigrateClick, state }: Props = $props();
+
+  let subtext = $derived.by(() => {
+    if (!state) {
+      return formatDateTime(info.createdAt ?? info.lastModifiedAt);
+    }
+    if (state.status === "uploading") {
+      const progress = Math.floor((state.progress ?? 0) * 100);
+      const speed = formatNetworkSpeed((state.rate ?? 0) * 8);
+      return `전송됨 ${progress}% · ${speed}`;
+    }
+    return subtexts[state.status] ?? state.status;
+  });
+</script>
+
+<ActionEntryButton
+  class="h-14"
+  onclick={() => onclick(info)}
+  actionButtonIcon={!state || state.status === "error" ? IconSync : undefined}
+  onActionButtonClick={() => onMigrateClick(info)}
+  actionButtonClass="text-gray-800"
+>
+  <DirectoryEntryLabel type="file" name={info.name} {subtext} />
+</ActionEntryButton>

src/routes/(fullscreen)/settings/migration/service.svelte.ts

@@ -0,0 +1,165 @@
+import { limitFunction } from "p-limit";
+import { SvelteMap } from "svelte/reactivity";
+import { CHUNK_SIZE } from "$lib/constants";
+import { encodeToBase64, encryptChunk, digestMessage } from "$lib/modules/crypto";
+import { deleteFileCache } from "$lib/modules/file";
+import type { FileInfo } from "$lib/modules/filesystem";
+import { Scheduler } from "$lib/modules/scheduler";
+import { requestFileDownload } from "$lib/services/file";
+import { trpc } from "$trpc/client";
+
+export type MigrationStatus =
+  | "queued"
+  | "download-pending"
+  | "downloading"
+  | "encryption-pending"
+  | "encrypting"
+  | "upload-pending"
+  | "uploading"
+  | "completed"
+  | "error";
+
+export interface MigrationState {
+  status: MigrationStatus;
+  progress?: number;
+  rate?: number;
+}
+
+const scheduler = new Scheduler();
+const states = new SvelteMap<number, MigrationState>();
+
+const createState = (status: MigrationStatus): MigrationState => {
+  const state = $state({ status });
+  return state;
+};
+
+export const getMigrationState = (fileId: number) => {
+  return states.get(fileId);
+};
+
+export const clearMigrationStates = () => {
+  for (const [id, state] of states) {
+    if (state.status === "completed" || state.status === "error") {
+      states.delete(id);
+    }
+  }
+};
+
+const encryptChunks = async (fileBuffer: ArrayBuffer, dataKey: CryptoKey) => {
+  const chunksEncrypted: { chunkEncrypted: ArrayBuffer; chunkEncryptedHash: string }[] = [];
+  let offset = 0;
+
+  while (offset < fileBuffer.byteLength) {
+    const nextOffset = Math.min(offset + CHUNK_SIZE, fileBuffer.byteLength);
+    const chunkEncrypted = await encryptChunk(fileBuffer.slice(offset, nextOffset), dataKey);
+    chunksEncrypted.push({
+      chunkEncrypted: chunkEncrypted,
+      chunkEncryptedHash: encodeToBase64(await digestMessage(chunkEncrypted)),
+    });
+    offset = nextOffset;
+  }
+
+  return chunksEncrypted;
+};
+
+const uploadMigrationChunks = limitFunction(
+  async (
+    state: MigrationState,
+    fileId: number,
+    chunksEncrypted: { chunkEncrypted: ArrayBuffer; chunkEncryptedHash: string }[],
+  ) => {
+    state.status = "uploading";
+
+    const { uploadId } = await trpc().upload.startMigrationUpload.mutate({
+      file: fileId,
+      chunks: chunksEncrypted.length,
+    });
+
+    const totalBytes = chunksEncrypted.reduce((sum, c) => sum + c.chunkEncrypted.byteLength, 0);
+    let uploadedBytes = 0;
+    const startTime = Date.now();
+
+    for (let i = 0; i < chunksEncrypted.length; i++) {
+      const { chunkEncrypted, chunkEncryptedHash } = chunksEncrypted[i]!;
+
+      const response = await fetch(`/api/upload/${uploadId}/chunks/${i}`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/octet-stream",
+          "Content-Digest": `sha-256=:${chunkEncryptedHash}:`,
+        },
+        body: chunkEncrypted,
+      });
+
+      if (!response.ok) {
+        throw new Error(`Chunk upload failed: ${response.status} ${response.statusText}`);
+      }
+
+      uploadedBytes += chunkEncrypted.byteLength;
+
+      const elapsed = (Date.now() - startTime) / 1000;
+      const rate = uploadedBytes / elapsed;
+
+      state.progress = uploadedBytes / totalBytes;
+      state.rate = rate;
+    }
+
+    await trpc().upload.completeMigrationUpload.mutate({ uploadId });
+  },
+  { concurrency: 1 },
+);
+
+const encryptFile = limitFunction(
+  async (state: MigrationState, fileBuffer: ArrayBuffer, dataKey: CryptoKey) => {
+    state.status = "encrypting";
+    const chunksEncrypted = await encryptChunks(fileBuffer, dataKey);
+    state.status = "upload-pending";
+    return chunksEncrypted;
+  },
+  { concurrency: 4 },
+);
+
+export const requestFileMigration = async (fileInfo: FileInfo & { exists: true }) => {
+  let state = states.get(fileInfo.id);
+  if (state) {
+    if (state.status !== "error") return;
+    state.status = "queued";
+    state.progress = undefined;
+    state.rate = undefined;
+  } else {
+    state = createState("queued");
+    states.set(fileInfo.id, state);
+  }
+
+  try {
+    const dataKey = fileInfo.dataKey?.key;
+    if (!dataKey) {
+      throw new Error("Data key not available");
+    }
+
+    let fileBuffer: ArrayBuffer | undefined;
+
+    await scheduler.schedule(
+      async () => {
+        state.status = "download-pending";
+        state.status = "downloading";
+        fileBuffer = await requestFileDownload(fileInfo.id, dataKey, true);
+        return fileBuffer.byteLength;
+      },
+      async () => {
+        state.status = "encryption-pending";
+        const chunksEncrypted = await encryptFile(state, fileBuffer!, dataKey);
+
+        await uploadMigrationChunks(state, fileInfo.id, chunksEncrypted);
+
+        // Clear file cache since the file format has changed
+        await deleteFileCache(fileInfo.id);
+
+        state.status = "completed";
+      },
+    );
+  } catch (e) {
+    state.status = "error";
+    throw e;
+  }
+};

src/routes/(main)/menu/+page.svelte

@@ -5,6 +5,7 @@
 
   import IconStorage from "~icons/material-symbols/storage";
   import IconImage from "~icons/material-symbols/image";
+  import IconLockReset from "~icons/material-symbols/lock-reset";
   import IconPassword from "~icons/material-symbols/password";
   import IconLogout from "~icons/material-symbols/logout";
 
@@ -41,6 +42,13 @@
     >
       썸네일
     </MenuEntryButton>
+    <MenuEntryButton
+      onclick={() => goto("/settings/migration")}
+      icon={IconLockReset}
+      iconColor="text-teal-500"
+    >
+      암호화 마이그레이션
+    </MenuEntryButton>
   </div>
   <div class="space-y-2">
     <p class="font-semibold">보안</p>

src/trpc/routers/file.ts

@@ -100,6 +100,10 @@ const fileRouter = router({
     return await MediaRepo.getMissingFileThumbnails(ctx.session.userId);
   }),
 
+  listLegacy: roleProcedure["activeClient"].query(async ({ ctx }) => {
+    return await FileRepo.getLegacyFileIds(ctx.session.userId);
+  }),
+
   rename: roleProcedure["activeClient"]
     .input(
       z.object({

src/trpc/routers/upload.ts

@@ -250,6 +250,110 @@ const uploadRouter = router({
         sessionLocks.delete(uploadId);
       }
     }),
+
+  startMigrationUpload: roleProcedure["activeClient"]
+    .input(
+      z.object({
+        file: z.int().positive(),
+        chunks: z.int().positive(),
+      }),
+    )
+    .mutation(async ({ ctx, input }) => {
+      const { id, path } = await generateSessionId();
+
+      try {
+        await UploadRepo.createMigrationUploadSession({
+          id,
+          userId: ctx.session.userId,
+          path,
+          totalChunks: input.chunks,
+          expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours
+          fileId: input.file,
+        });
+        return { uploadId: id };
+      } catch (e) {
+        await safeRecursiveRm(path);
+
+        if (e instanceof IntegrityError) {
+          if (e.message === "File not found") {
+            throw new TRPCError({ code: "NOT_FOUND", message: "File not found" });
+          } else if (e.message === "File is not legacy") {
+            throw new TRPCError({ code: "BAD_REQUEST", message: "File is not legacy" });
+          }
+        }
+        throw e;
+      }
+    }),
+
+  completeMigrationUpload: roleProcedure["activeClient"]
+    .input(
+      z.object({
+        uploadId: z.uuidv4(),
+      }),
+    )
+    .mutation(async ({ ctx, input }) => {
+      const { uploadId } = input;
+      if (sessionLocks.has(uploadId)) {
+        throw new TRPCError({ code: "CONFLICT", message: "Completion already in progress" });
+      } else {
+        sessionLocks.add(uploadId);
+      }
+
+      let filePath = "";
+
+      try {
+        const session = await UploadRepo.getUploadSession(uploadId, ctx.session.userId);
+        if (!session || session.type !== "migration") {
+          throw new TRPCError({ code: "NOT_FOUND", message: "Invalid upload id" });
+        } else if (session.uploadedChunks.length < session.totalChunks) {
+          throw new TRPCError({ code: "BAD_REQUEST", message: "Upload not completed" });
+        }
+
+        filePath = `${env.libraryPath}/${ctx.session.userId}/${uuidv4()}`;
+        await mkdir(dirname(filePath), { recursive: true });
+
+        const hashStream = createHash("sha256");
+        const writeStream = createWriteStream(filePath, { flags: "wx", mode: 0o600 });
+
+        for (let i = 0; i < session.totalChunks; i++) {
+          for await (const chunk of createReadStream(`${session.path}/${i}`)) {
+            hashStream.update(chunk);
+            writeStream.write(chunk);
+          }
+        }
+
+        await new Promise<void>((resolve, reject) => {
+          writeStream.end((e: any) => (e ? reject(e) : resolve()));
+        });
+
+        const hash = hashStream.digest("base64");
+        const oldPath = await db.transaction().execute(async (trx) => {
+          const oldPath = await FileRepo.migrateFileContent(
+            trx,
+            ctx.session.userId,
+            session.fileId,
+            filePath,
+            hash,
+          );
+          await UploadRepo.deleteUploadSession(trx, uploadId);
+          return oldPath;
+        });
+
+        await Promise.all([safeUnlink(oldPath), safeRecursiveRm(session.path)]);
+      } catch (e) {
+        await safeUnlink(filePath);
+        if (e instanceof IntegrityError) {
+          if (e.message === "File not found") {
+            throw new TRPCError({ code: "NOT_FOUND", message: "File not found" });
+          } else if (e.message === "File is not legacy") {
+            throw new TRPCError({ code: "BAD_REQUEST", message: "File is not legacy" });
+          }
+        }
+        throw e;
+      } finally {
+        sessionLocks.delete(uploadId);
+      }
+    }),
 });
 
 export default uploadRouter;