Merge pull request #19 from kmc7468/dev

v0.8.0

.github/workflows/docker.yaml

@@ -0,0 +1,45 @@
+name: Docker Image Build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=semver,value={{version}}
+            type=raw,value=latest
+            type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

docker-compose.yaml

@@ -9,6 +9,7 @@ services:
     volumes:
       - ./data/library:/app/data/library
       - ./data/thumbnails:/app/data/thumbnails
+      - ./data/uploads:/app/data/uploads
     environment:
       # ArkVault
       - DATABASE_HOST=database

src/hooks.server.ts

@@ -7,8 +7,8 @@ import {
   cleanupExpiredSessions,
   cleanupExpiredSessionUpgradeChallenges,
 } from "$lib/server/db/session";
-import { cleanupExpiredUploadSessions } from "$lib/server/services/upload";
 import { authenticate, setAgentInfo } from "$lib/server/middlewares";
+import { cleanupExpiredUploadSessions } from "$lib/server/services/upload";
 
 export const init: ServerInit = async () => {
   await migrateDB();

src/lib/modules/crypto/hmac.worker.ts

@@ -1,30 +0,0 @@
-import { hmac } from "@noble/hashes/hmac.js";
-import { sha256 } from "@noble/hashes/sha2.js";
-
-interface ComputeMessage {
-  type: "compute";
-  file: File;
-  hmacSecret: ArrayBuffer;
-}
-
-type WorkerMessage = ComputeMessage;
-
-self.onmessage = async (event: MessageEvent<WorkerMessage>) => {
-  const { type } = event.data;
-
-  if (type === "compute") {
-    const { file, hmacSecret } = event.data;
-
-    const h = hmac.create(sha256, new Uint8Array(hmacSecret));
-    const reader = file.stream().getReader();
-
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      h.update(value);
-    }
-
-    const result = h.digest();
-    self.postMessage({ type: "result", hmac: result }, { transfer: [result.buffer] });
-  }
-};

src/lib/modules/crypto/hmacWorker.ts

@@ -1,25 +0,0 @@
-import HmacWorker from "./hmac.worker?worker";
-
-export const computeFileHmac = async (file: File, hmacSecret: CryptoKey): Promise<Uint8Array> => {
-  const worker = new HmacWorker();
-  const hmacSecretRaw = await crypto.subtle.exportKey("raw", hmacSecret);
-
-  return new Promise((resolve, reject) => {
-    worker.onmessage = (event: MessageEvent<{ type: "result"; hmac: Uint8Array }>) => {
-      if (event.data.type === "result") {
-        resolve(event.data.hmac);
-        worker.terminate();
-      }
-    };
-
-    worker.onerror = (error) => {
-      reject(error);
-      worker.terminate();
-    };
-
-    worker.postMessage(
-      { type: "compute", file, hmacSecret: hmacSecretRaw },
-      { transfer: [hmacSecretRaw] },
-    );
-  });
-};

src/lib/modules/crypto/sha.ts

@@ -1,5 +1,5 @@
-import { hmac } from "@noble/hashes/hmac.js";
-import { sha256 } from "@noble/hashes/sha2.js";
+import HmacWorker from "$workers/hmac?worker";
+import type { ComputeMessage, ResultMessage } from "$workers/hmac";
 
 export const digestMessage = async (message: BufferSource) => {
   return await crypto.subtle.digest("SHA-256", message);
@@ -18,10 +18,24 @@ export const generateHmacSecret = async () => {
   };
 };
 
-export const createHmacStream = async (hmacSecret: CryptoKey) => {
-  const h = hmac.create(sha256, new Uint8Array(await crypto.subtle.exportKey("raw", hmacSecret)));
-  return {
-    update: (data: Uint8Array) => h.update(data),
-    digest: () => h.digest(),
+export const signMessageHmac = async (message: Blob, hmacSecret: CryptoKey) => {
+  const stream = message.stream();
+  const hmacSecretRaw = new Uint8Array(await crypto.subtle.exportKey("raw", hmacSecret));
+  const worker = new HmacWorker();
+
+  return new Promise<Uint8Array>((resolve, reject) => {
+    worker.onmessage = ({ data }: MessageEvent<ResultMessage>) => {
+      resolve(data.result);
+      worker.terminate();
     };
+
+    worker.onerror = ({ error }) => {
+      reject(error);
+      worker.terminate();
+    };
+
+    worker.postMessage({ stream, key: hmacSecretRaw } satisfies ComputeMessage, {
+      transfer: [stream, hmacSecretRaw.buffer],
+    });
+  });
 };

src/lib/modules/file/download.svelte.ts

@@ -1,6 +1,6 @@
 import axios from "axios";
 import { limitFunction } from "p-limit";
-import { CHUNK_SIZE, ENCRYPTION_OVERHEAD } from "$lib/constants";
+import { ENCRYPTED_CHUNK_SIZE } from "$lib/constants";
 import { decryptChunk, concatenateBuffers } from "$lib/modules/crypto";
 
 export interface FileDownloadState {
@@ -100,7 +100,7 @@ export const downloadFile = async (id: number, dataKey: CryptoKey, isLegacy: boo
     return await decryptFile(
       state,
       fileEncrypted,
-      isLegacy ? fileEncrypted.byteLength : CHUNK_SIZE + ENCRYPTION_OVERHEAD,
+      isLegacy ? fileEncrypted.byteLength : ENCRYPTED_CHUNK_SIZE,
       dataKey,
     );
   } catch (e) {

src/lib/modules/file/thumbnail.ts

@@ -1,7 +1,7 @@
 import { LRUCache } from "lru-cache";
 import { writable, type Writable } from "svelte/store";
 import { browser } from "$app/environment";
-import { decryptData } from "$lib/modules/crypto";
+import { decryptChunk } from "$lib/modules/crypto";
 import type { SummarizedFileInfo } from "$lib/modules/filesystem";
 import { readFile, writeFile, deleteFile, deleteDirectory } from "$lib/modules/opfs";
 import { getThumbnailUrl } from "$lib/modules/thumbnail";
@@ -20,12 +20,7 @@ const fetchFromServer = async (fileId: number, dataKey: CryptoKey) => {
   const res = await fetch(`/api/file/${fileId}/thumbnail/download`);
   if (!res.ok) return null;
 
-  const thumbnailEncrypted = await res.arrayBuffer();
-  const thumbnailBuffer = await decryptData(
-    thumbnailEncrypted.slice(12),
-    thumbnailEncrypted.slice(0, 12),
-    dataKey,
-  );
+  const thumbnailBuffer = await decryptChunk(await res.arrayBuffer(), dataKey);
 
   void writeFile(`/thumbnail/file/${fileId}`, thumbnailBuffer);
   return getThumbnailUrl(thumbnailBuffer);

src/lib/modules/file/upload.svelte.ts

@@ -2,11 +2,11 @@ import ExifReader from "exifreader";
 import { limitFunction } from "p-limit";
 import { CHUNK_SIZE } from "$lib/constants";
 import { encodeToBase64, generateDataKey, wrapDataKey, encryptString } from "$lib/modules/crypto";
-import { computeFileHmac } from "$lib/modules/crypto/hmacWorker";
-import { Scheduler } from "$lib/modules/scheduler";
+import { signMessageHmac } from "$lib/modules/crypto";
 import { generateThumbnail } from "$lib/modules/thumbnail";
 import { uploadBlob } from "$lib/modules/upload";
 import type { MasterKey, HmacSecret } from "$lib/stores";
+import { Scheduler } from "$lib/utils";
 import { trpc } from "$trpc/client";
 
 export interface FileUploadState {
@@ -50,9 +50,15 @@ export const clearUploadedFiles = () => {
 };
 
 const requestDuplicateFileScan = limitFunction(
-  async (file: File, hmacSecret: HmacSecret, onDuplicate: () => Promise<boolean>) => {
-    const hmacResult = await computeFileHmac(file, hmacSecret.secret);
-    const fileSigned = encodeToBase64(hmacResult);
+  async (
+    state: FileUploadState,
+    file: File,
+    hmacSecret: HmacSecret,
+    onDuplicate: () => Promise<boolean>,
+  ) => {
+    state.status = "encryption-pending";
+
+    const fileSigned = encodeToBase64(await signMessageHmac(file, hmacSecret.secret));
     const files = await trpc().file.listByHash.query({
       hskVersion: hmacSecret.version,
       contentHmac: fileSigned,
@@ -98,16 +104,18 @@ const extractExifDateTime = (fileBuffer: ArrayBuffer) => {
   return new Date(utcDate - offsetMs);
 };
 
-const requestFileUpload2 = async (
-  state: FileUploadState,
-  file: Blob,
-  fileSigned: string,
-  fileMetadata: {
+interface FileMetadata {
   parentId: "root" | number;
   name: string;
   createdAt?: Date;
   lastModifiedAt: Date;
-  },
+}
+
+const requestFileMetadataEncryption = limitFunction(
+  async (
+    state: FileUploadState,
+    file: Blob,
+    fileMetadata: FileMetadata,
     masterKey: MasterKey,
     hmacSecret: HmacSecret,
   ) => {
@@ -119,7 +127,8 @@ const requestFileUpload2 = async (
     const [nameEncrypted, createdAtEncrypted, lastModifiedAtEncrypted, thumbnailBuffer] =
       await Promise.all([
         encryptString(fileMetadata.name, dataKey),
-      fileMetadata.createdAt && encryptString(fileMetadata.createdAt.getTime().toString(), dataKey),
+        fileMetadata.createdAt &&
+          encryptString(fileMetadata.createdAt.getTime().toString(), dataKey),
         encryptString(fileMetadata.lastModifiedAt.getTime().toString(), dataKey),
         generateThumbnail(file).then((blob) => blob?.arrayBuffer()),
       ]);
@@ -140,12 +149,28 @@ const requestFileUpload2 = async (
       lastModifiedAtIv: lastModifiedAtEncrypted.iv,
     });
 
+    state.status = "upload-pending";
+    return { uploadId, thumbnailBuffer, dataKey, dataKeyVersion };
+  },
+  { concurrency: 4 },
+);
+
+const requestFileUpload = limitFunction(
+  async (
+    state: FileUploadState,
+    uploadId: string,
+    file: Blob,
+    fileSigned: string,
+    thumbnailBuffer: ArrayBuffer | undefined,
+    dataKey: CryptoKey,
+    dataKeyVersion: Date,
+  ) => {
     state.status = "uploading";
 
     await uploadBlob(uploadId, file, dataKey, {
       onProgress(s) {
         state.progress = s.progress;
-      state.rate = s.rateBps;
+        state.rate = s.rate;
       },
     });
 
@@ -155,6 +180,7 @@ const requestFileUpload2 = async (
     });
 
     if (thumbnailBuffer) {
+      try {
         const { uploadId } = await trpc().upload.startFileThumbnailUpload.mutate({
           file: fileId,
           dekVersion: dataKeyVersion,
@@ -163,12 +189,16 @@ const requestFileUpload2 = async (
         await uploadBlob(uploadId, new Blob([thumbnailBuffer]), dataKey);
 
         await trpc().upload.completeFileThumbnailUpload.mutate({ uploadId });
+      } catch (e) {
+        console.error(e);
+      }
     }
 
     state.status = "uploaded";
-
-  return { fileId, thumbnailBuffer };
-};
+    return { fileId };
+  },
+  { concurrency: 1 },
+);
 
 export const uploadFile = async (
   file: File,
@@ -185,51 +215,44 @@ export const uploadFile = async (
   const state = uploadingFiles.at(-1)!;
 
   return await scheduler.schedule(file.size, async () => {
-    state.status = "encryption-pending";
-
     try {
-      const { fileSigned } = await requestDuplicateFileScan(file, hmacSecret, onDuplicate);
+      const { fileSigned } = await requestDuplicateFileScan(state, file, hmacSecret, onDuplicate);
+
       if (!fileSigned) {
         state.status = "canceled";
         uploadingFiles = uploadingFiles.filter((file) => file !== state);
         return;
       }
 
+      let fileBuffer;
       const fileType = getFileType(file);
-      if (fileType.startsWith("image/")) {
-        const fileBuffer = await file.arrayBuffer();
-        const fileCreatedAt = extractExifDateTime(fileBuffer);
-
-        const { fileId, thumbnailBuffer } = await requestFileUpload2(
-          state,
-          new Blob([fileBuffer], { type: fileType }),
-          fileSigned,
-          {
+      const fileMetadata: FileMetadata = {
         parentId,
         name: file.name,
-            createdAt: fileCreatedAt,
         lastModifiedAt: new Date(file.lastModified),
-          },
-          masterKey,
-          hmacSecret,
+      };
+
+      if (fileType.startsWith("image/")) {
+        fileBuffer = await file.arrayBuffer();
+        fileMetadata.createdAt = extractExifDateTime(fileBuffer);
+      }
+
+      const blob = new Blob([file], { type: fileType });
+
+      const { uploadId, thumbnailBuffer, dataKey, dataKeyVersion } =
+        await requestFileMetadataEncryption(state, blob, fileMetadata, masterKey, hmacSecret);
+
+      const { fileId } = await requestFileUpload(
+        state,
+        uploadId,
+        blob,
+        fileSigned,
+        thumbnailBuffer,
+        dataKey,
+        dataKeyVersion,
       );
 
       return { fileId, fileBuffer, thumbnailBuffer };
-      } else {
-        const { fileId, thumbnailBuffer } = await requestFileUpload2(
-          state,
-          file,
-          fileSigned,
-          {
-            parentId,
-            name: file.name,
-            lastModifiedAt: new Date(file.lastModified),
-          },
-          masterKey,
-          hmacSecret,
-        );
-        return { fileId, thumbnailBuffer };
-      }
     } catch (e) {
       state.status = "error";
       throw e;

src/lib/modules/upload.ts

@@ -2,50 +2,99 @@ import axios from "axios";
 import pLimit from "p-limit";
 import { ENCRYPTION_OVERHEAD, CHUNK_SIZE } from "$lib/constants";
 import { encryptChunk, digestMessage, encodeToBase64 } from "$lib/modules/crypto";
+import { BoundedQueue } from "$lib/utils";
 
-type UploadStats = {
-  progress: number; // 0..1 (암호화 후 기준)
-  rateBps: number; // bytes/sec
-  uploadedBytes: number;
-  totalBytes: number;
-};
+interface UploadStats {
+  progress: number;
+  rate: number;
+}
+
+interface EncryptedChunk {
+  index: number;
+  data: ArrayBuffer;
+  hash: string;
+}
+
+const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => {
+  const samples: { t: number; b: number }[] = [];
+  let lastSpeed = 0;
+  let startTime: number | null = null;
+
+  return (bytesNow?: number) => {
+    if (bytesNow === undefined) return lastSpeed;
 
-function createSpeedMeter(windowMs = 1500) {
-  const samples: Array<{ t: number; b: number }> = [];
-  return (bytesNow: number) => {
     const now = performance.now();
+
+    // Initialize start time on first call
+    if (startTime === null) {
+      startTime = now;
+    }
+
+    // Check if enough time has passed since the last sample
+    const lastSample = samples[samples.length - 1];
+    if (lastSample && now - lastSample.t < minInterval) {
+      return lastSpeed;
+    }
+
     samples.push({ t: now, b: bytesNow });
-    const cutoff = now - windowMs;
+
+    // Remove old samples outside the time window
+    const cutoff = now - timeWindow;
     while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift();
 
+    // Need at least 2 samples to calculate speed
+    if (samples.length < 2) {
+      return lastSpeed;
+    }
+
     const first = samples[0]!;
     const dt = now - first.t;
     const db = bytesNow - first.b;
-    return dt > 0 ? (db / dt) * 1000 : 0;
-  };
+
+    if (dt >= minInterval) {
+      const instantSpeed = (db / dt) * 1000;
+      // Apply EMA for smoother speed transitions
+      const alpha = 0.3;
+      const rawSpeed =
+        lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed;
+
+      // Apply warmup ramp to prevent initial overestimation
+      const elapsed = now - startTime;
+      const warmupWeight = Math.min(1, elapsed / warmupPeriod);
+      lastSpeed = rawSpeed * warmupWeight;
     }
 
-const uploadChunk = async (
-  uploadId: string,
-  chunkIndex: number,
+    return lastSpeed;
+  };
+};
+
+const encryptChunkData = async (
   chunk: Blob,
   dataKey: CryptoKey,
+): Promise<{ data: ArrayBuffer; hash: string }> => {
+  const encrypted = await encryptChunk(await chunk.arrayBuffer(), dataKey);
+  const hash = encodeToBase64(await digestMessage(encrypted));
+  return { data: encrypted, hash };
+};
+
+const uploadEncryptedChunk = async (
+  uploadId: string,
+  chunkIndex: number,
+  encrypted: ArrayBuffer,
+  hash: string,
   onChunkProgress: (chunkIndex: number, loaded: number) => void,
 ) => {
-  const chunkEncrypted = await encryptChunk(await chunk.arrayBuffer(), dataKey);
-  const chunkEncryptedHash = encodeToBase64(await digestMessage(chunkEncrypted));
-
-  await axios.post(`/api/upload/${uploadId}/chunks/${chunkIndex}`, chunkEncrypted, {
+  await axios.post(`/api/upload/${uploadId}/chunks/${chunkIndex + 1}`, encrypted, {
     headers: {
       "Content-Type": "application/octet-stream",
-      "Content-Digest": `sha-256=:${chunkEncryptedHash}:`,
+      "Content-Digest": `sha-256=:${hash}:`,
     },
     onUploadProgress(e) {
       onChunkProgress(chunkIndex, e.loaded ?? 0);
     },
   });
 
-  onChunkProgress(chunkIndex, chunkEncrypted.byteLength);
+  onChunkProgress(chunkIndex, encrypted.byteLength);
 };
 
 export const uploadBlob = async (
@@ -55,21 +104,23 @@ export const uploadBlob = async (
   options?: { concurrency?: number; onProgress?: (s: UploadStats) => void },
 ) => {
   const onProgress = options?.onProgress;
+  const networkConcurrency = options?.concurrency ?? 4;
+  const maxQueueSize = 8;
 
   const totalChunks = Math.ceil(blob.size / CHUNK_SIZE);
   const totalBytes = blob.size + totalChunks * ENCRYPTION_OVERHEAD;
 
   const uploadedByChunk = new Array<number>(totalChunks).fill(0);
-  const speedMeter = createSpeedMeter(1500);
+  const speedMeter = createSpeedMeter(3000, 200);
 
   const emit = () => {
     if (!onProgress) return;
 
     const uploadedBytes = uploadedByChunk.reduce((a, b) => a + b, 0);
-    const rateBps = speedMeter(uploadedBytes);
+    const rate = speedMeter(uploadedBytes);
     const progress = Math.min(1, uploadedBytes / totalBytes);
 
-    onProgress({ progress, rateBps, uploadedBytes, totalBytes });
+    onProgress({ progress, rate });
   };
 
   const onChunkProgress = (idx: number, loaded: number) => {
@@ -77,27 +128,56 @@ export const uploadBlob = async (
     emit();
   };
 
-  const limit = pLimit(options?.concurrency ?? 4);
+  const queue = new BoundedQueue<EncryptedChunk>(maxQueueSize);
+  let encryptionError: Error | null = null;
 
-  await Promise.all(
-    Array.from({ length: totalChunks }, (_, chunkIndex) =>
-      limit(() =>
-        uploadChunk(
-          uploadId,
-          chunkIndex,
-          blob.slice(chunkIndex * CHUNK_SIZE, (chunkIndex + 1) * CHUNK_SIZE),
-          dataKey,
-          onChunkProgress,
-        ),
-      ),
-    ),
-  );
-
-  // 완료 보정
-  onProgress?.({
-    progress: 1,
-    rateBps: 0,
-    uploadedBytes: totalBytes,
-    totalBytes,
-  });
+  // Producer: encrypt chunks and push to queue
+  const encryptionProducer = async () => {
+    try {
+      for (let i = 0; i < totalChunks; i++) {
+        const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+        const { data, hash } = await encryptChunkData(chunk, dataKey);
+        await queue.push({ index: i, data, hash });
+      }
+    } catch (e) {
+      encryptionError = e instanceof Error ? e : new Error(String(e));
+    } finally {
+      queue.close();
+    }
+  };
+
+  // Consumer: upload chunks from queue with concurrency limit
+  const uploadConsumer = async () => {
+    const limit = pLimit(networkConcurrency);
+    const activeTasks = new Set<Promise<void>>();
+
+    while (true) {
+      const item = await queue.pop();
+      if (item === null) break;
+      if (encryptionError) throw encryptionError;
+
+      const task = limit(async () => {
+        try {
+          await uploadEncryptedChunk(uploadId, item.index, item.data, item.hash, onChunkProgress);
+        } finally {
+          // @ts-ignore
+          item.data = null;
+        }
+      });
+
+      activeTasks.add(task);
+      task.finally(() => activeTasks.delete(task));
+
+      if (activeTasks.size >= networkConcurrency) {
+        await Promise.race(activeTasks);
+      }
+    }
+
+    await Promise.all(activeTasks);
+  };
+
+  // Run producer and consumer concurrently
+  await Promise.all([encryptionProducer(), uploadConsumer()]);
+
+  onProgress?.({ progress: 1, rate: speedMeter() });
 };

src/lib/server/db/file.ts

@@ -497,21 +497,22 @@ export const migrateFileContent = async (
   userId: number,
   fileId: number,
   newPath: string,
+  dekVersion: Date,
   encContentHash: string,
 ) => {
   const file = await trx
     .selectFrom("file")
-    .select(["path", "encrypted_content_iv"])
+    .select(["path", "data_encryption_key_version", "encrypted_content_iv"])
     .where("id", "=", fileId)
     .where("user_id", "=", userId)
     .limit(1)
     .forUpdate()
     .executeTakeFirst();
-
   if (!file) {
     throw new IntegrityError("File not found");
-  }
-  if (!file.encrypted_content_iv) {
+  } else if (file.data_encryption_key_version.getTime() !== dekVersion.getTime()) {
+    throw new IntegrityError("Invalid DEK version");
+  } else if (!file.encrypted_content_iv) {
     throw new IntegrityError("File is not legacy");
   }
 
@@ -525,7 +526,6 @@ export const migrateFileContent = async (
     .where("id", "=", fileId)
     .where("user_id", "=", userId)
     .execute();
-
   await trx
     .insertInto("file_log")
     .values({
@@ -534,8 +534,7 @@ export const migrateFileContent = async (
       action: "migrate",
     })
     .execute();
-
-  return file.path;
+  return { oldPath: file.path };
 };
 
 export const addFileToCategory = async (fileId: number, categoryId: number) => {

src/lib/server/db/migrations/1768062380-AddChunkedUpload.ts

@@ -21,8 +21,14 @@ export const up = async (db: Kysely<any>) => {
     .addColumn("type", "text", (col) => col.notNull())
     .addColumn("user_id", "integer", (col) => col.references("user.id").notNull())
     .addColumn("path", "text", (col) => col.notNull())
+    .addColumn("bitmap", "bytea", (col) => col.notNull())
     .addColumn("total_chunks", "integer", (col) => col.notNull())
-    .addColumn("uploaded_chunks", sql`integer[]`, (col) => col.notNull().defaultTo(sql`'{}'`))
+    .addColumn("uploaded_chunks", "integer", (col) =>
+      col
+        .generatedAlwaysAs(sql`bit_count(bitmap)`)
+        .stored()
+        .notNull(),
+    )
     .addColumn("expires_at", "timestamp(3)", (col) => col.notNull())
     .addColumn("parent_id", "integer", (col) => col.references("directory.id"))
     .addColumn("master_encryption_key_version", "integer")
@@ -46,6 +52,11 @@ export const up = async (db: Kysely<any>) => {
       "hmac_secret_key",
       ["user_id", "version"],
     )
+    .addCheckConstraint(
+      "upload_session_ck01",
+      sql`length(bitmap) = ceil(total_chunks / 8.0)::integer`,
+    )
+    .addCheckConstraint("upload_session_ck02", sql`uploaded_chunks <= total_chunks`)
     .execute();
 };
 

src/lib/server/db/schema/upload.ts

@@ -6,11 +6,11 @@ interface UploadSessionTable {
   type: "file" | "thumbnail" | "migration";
   user_id: number;
   path: string;
+  bitmap: Buffer;
   total_chunks: number;
-  uploaded_chunks: Generated<number[]>;
+  uploaded_chunks: Generated<number>;
   expires_at: Date;
 
-  // For file uploads
   parent_id: number | null;
   master_encryption_key_version: number | null;
   encrypted_data_encryption_key: string | null; // Base64
@@ -20,8 +20,6 @@ interface UploadSessionTable {
   encrypted_name: Ciphertext | null;
   encrypted_created_at: Ciphertext | null;
   encrypted_last_modified_at: Ciphertext | null;
-
-  // For thumbnail uploads
   file_id: number | null;
 }
 

src/lib/server/db/upload.ts

@@ -7,8 +7,9 @@ interface BaseUploadSession {
   id: string;
   userId: number;
   path: string;
+  bitmap: Buffer;
   totalChunks: number;
-  uploadedChunks: number[];
+  uploadedChunks: number;
   expiresAt: Date;
 }
 
@@ -25,19 +26,14 @@ interface FileUploadSession extends BaseUploadSession {
   encLastModifiedAt: Ciphertext;
 }
 
-interface ThumbnailUploadSession extends BaseUploadSession {
-  type: "thumbnail";
+interface ThumbnailOrMigrationUploadSession extends BaseUploadSession {
+  type: "thumbnail" | "migration";
   fileId: number;
   dekVersion: Date;
 }
 
-interface MigrationUploadSession extends BaseUploadSession {
-  type: "migration";
-  fileId: number;
-}
-
 export const createFileUploadSession = async (
-  params: Omit<FileUploadSession, "type" | "uploadedChunks">,
+  params: Omit<FileUploadSession, "type" | "bitmap" | "uploadedChunks">,
 ) => {
   await db.transaction().execute(async (trx) => {
     const mek = await trx
@@ -73,6 +69,7 @@ export const createFileUploadSession = async (
         type: "file",
         user_id: params.userId,
         path: params.path,
+        bitmap: Buffer.alloc(Math.ceil(params.totalChunks / 8)),
         total_chunks: params.totalChunks,
         expires_at: params.expiresAt,
         parent_id: params.parentId !== "root" ? params.parentId : null,
@@ -89,8 +86,8 @@ export const createFileUploadSession = async (
   });
 };
 
-export const createThumbnailUploadSession = async (
-  params: Omit<ThumbnailUploadSession, "type" | "uploadedChunks">,
+export const createThumbnailOrMigrationUploadSession = async (
+  params: Omit<ThumbnailOrMigrationUploadSession, "bitmap" | "uploadedChunks">,
 ) => {
   await db.transaction().execute(async (trx) => {
     const file = await trx
@@ -111,9 +108,10 @@ export const createThumbnailUploadSession = async (
       .insertInto("upload_session")
       .values({
         id: params.id,
-        type: "thumbnail",
+        type: params.type,
         user_id: params.userId,
         path: params.path,
+        bitmap: Buffer.alloc(Math.ceil(params.totalChunks / 8)),
         total_chunks: params.totalChunks,
         expires_at: params.expiresAt,
         file_id: params.fileId,
@@ -123,39 +121,6 @@ export const createThumbnailUploadSession = async (
   });
 };
 
-export const createMigrationUploadSession = async (
-  params: Omit<MigrationUploadSession, "type" | "uploadedChunks">,
-) => {
-  await db.transaction().execute(async (trx) => {
-    const file = await trx
-      .selectFrom("file")
-      .select("encrypted_content_iv")
-      .where("id", "=", params.fileId)
-      .where("user_id", "=", params.userId)
-      .limit(1)
-      .forUpdate()
-      .executeTakeFirst();
-    if (!file) {
-      throw new IntegrityError("File not found");
-    } else if (!file.encrypted_content_iv) {
-      throw new IntegrityError("File is not legacy");
-    }
-
-    await trx
-      .insertInto("upload_session")
-      .values({
-        id: params.id,
-        type: "migration",
-        user_id: params.userId,
-        path: params.path,
-        total_chunks: params.totalChunks,
-        expires_at: params.expiresAt,
-        file_id: params.fileId,
-      })
-      .execute();
-  });
-};
-
 export const getUploadSession = async (sessionId: string, userId: number) => {
   const session = await db
     .selectFrom("upload_session")
@@ -173,6 +138,7 @@ export const getUploadSession = async (sessionId: string, userId: number) => {
       id: session.id,
       userId: session.user_id,
       path: session.path,
+      bitmap: session.bitmap,
       totalChunks: session.total_chunks,
       uploadedChunks: session.uploaded_chunks,
       expiresAt: session.expires_at,
@@ -186,36 +152,28 @@ export const getUploadSession = async (sessionId: string, userId: number) => {
       encCreatedAt: session.encrypted_created_at,
       encLastModifiedAt: session.encrypted_last_modified_at!,
     } satisfies FileUploadSession;
-  } else if (session.type === "thumbnail") {
+  } else {
     return {
-      type: "thumbnail",
+      type: session.type,
       id: session.id,
       userId: session.user_id,
       path: session.path,
+      bitmap: session.bitmap,
       totalChunks: session.total_chunks,
       uploadedChunks: session.uploaded_chunks,
       expiresAt: session.expires_at,
       fileId: session.file_id!,
       dekVersion: session.data_encryption_key_version!,
-    } satisfies ThumbnailUploadSession;
-  } else {
-    return {
-      type: "migration",
-      id: session.id,
-      userId: session.user_id,
-      path: session.path,
-      totalChunks: session.total_chunks,
-      uploadedChunks: session.uploaded_chunks,
-      expiresAt: session.expires_at,
-      fileId: session.file_id!,
-    } satisfies MigrationUploadSession;
+    } satisfies ThumbnailOrMigrationUploadSession;
   }
 };
 
 export const markChunkAsUploaded = async (sessionId: string, chunkIndex: number) => {
   await db
     .updateTable("upload_session")
-    .set({ uploaded_chunks: sql`array_append(uploaded_chunks, ${chunkIndex})` })
+    .set({
+      bitmap: sql`set_bit(${sql.ref("bitmap")}, ${chunkIndex - 1}, 1)`,
+    })
     .where("id", "=", sessionId)
     .execute();
 };

src/lib/server/services/upload.ts

@@ -8,6 +8,12 @@ import { safeRecursiveRm, safeUnlink } from "$lib/server/modules/filesystem";
 
 const chunkLocks = new Set<string>();
 
+const isChunkUploaded = (bitmap: Buffer, chunkIndex: number) => {
+  chunkIndex -= 1;
+  const byte = bitmap[Math.floor(chunkIndex / 8)];
+  return !!byte && (byte & (1 << (chunkIndex % 8))) !== 0; // Postgres sucks
+};
+
 export const uploadChunk = async (
   userId: number,
   sessionId: string,
@@ -28,13 +34,13 @@ export const uploadChunk = async (
     const session = await UploadRepo.getUploadSession(sessionId, userId);
     if (!session) {
       error(404, "Invalid upload id");
-    } else if (chunkIndex >= session.totalChunks) {
+    } else if (chunkIndex > session.totalChunks) {
       error(400, "Invalid chunk index");
-    } else if (session.uploadedChunks.includes(chunkIndex)) {
+    } else if (isChunkUploaded(session.bitmap, chunkIndex)) {
       error(409, "Chunk already uploaded");
     }
 
-    const isLastChunk = chunkIndex === session.totalChunks - 1;
+    const isLastChunk = chunkIndex === session.totalChunks;
     filePath = `${session.path}/${chunkIndex}`;
 
     const hashStream = createHash("sha256");

src/lib/services/file.ts

@@ -1,5 +1,4 @@
 import { getAllFileInfos } from "$lib/indexedDB/filesystem";
-import { encodeToBase64, digestMessage } from "$lib/modules/crypto";
 import {
   getFileCache,
   storeFileCache,
@@ -7,6 +6,7 @@ import {
   downloadFile,
   deleteFileThumbnailCache,
 } from "$lib/modules/file";
+import { uploadBlob } from "$lib/modules/upload";
 import { trpc } from "$trpc/client";
 
 export const requestFileDownload = async (
@@ -24,41 +24,24 @@ export const requestFileDownload = async (
 
 export const requestFileThumbnailUpload = async (
   fileId: number,
+  thumbnail: Blob,
+  dataKey: CryptoKey,
   dataKeyVersion: Date,
-  thumbnailEncrypted: { ciphertext: ArrayBuffer; iv: ArrayBuffer },
 ) => {
+  try {
     const { uploadId } = await trpc().upload.startFileThumbnailUpload.mutate({
       file: fileId,
       dekVersion: dataKeyVersion,
     });
 
-  // Prepend IV to ciphertext (consistent with file download format)
-  const ivAndCiphertext = new Uint8Array(
-    thumbnailEncrypted.iv.byteLength + thumbnailEncrypted.ciphertext.byteLength,
-  );
-  ivAndCiphertext.set(new Uint8Array(thumbnailEncrypted.iv), 0);
-  ivAndCiphertext.set(
-    new Uint8Array(thumbnailEncrypted.ciphertext),
-    thumbnailEncrypted.iv.byteLength,
-  );
-
-  const chunkHash = encodeToBase64(await digestMessage(ivAndCiphertext));
-
-  const response = await fetch(`/api/upload/${uploadId}/chunks/0`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/octet-stream",
-      "Content-Digest": `sha-256=:${chunkHash}:`,
-    },
-    body: ivAndCiphertext,
-  });
-
-  if (!response.ok) {
-    throw new Error(`Thumbnail upload failed: ${response.status} ${response.statusText}`);
-  }
+    await uploadBlob(uploadId, thumbnail, dataKey);
 
     await trpc().upload.completeFileThumbnailUpload.mutate({ uploadId });
-  return response;
+    return true;
+  } catch {
+    // TODO: Error Handling
+    return false;
+  }
 };
 
 export const requestDeletedFilesCleanup = async () => {

src/lib/utils/concurrency/BoundedQueue.ts

@@ -0,0 +1,44 @@
+export class BoundedQueue<T> {
+  private isClosed = false;
+  private reservedCount = 0;
+  private items: T[] = [];
+
+  private waitersNotFull: (() => void)[] = [];
+  private waitersNotEmpty: (() => void)[] = [];
+
+  constructor(private readonly maxSize: number) {}
+
+  async push(item: T) {
+    if (this.isClosed) {
+      throw new Error("Queue closed");
+    }
+
+    while (this.reservedCount >= this.maxSize) {
+      await new Promise<void>((resolve) => this.waitersNotFull.push(resolve));
+      if (this.isClosed) throw new Error("Queue closed");
+    }
+
+    this.reservedCount++;
+    this.items.push(item);
+    this.waitersNotEmpty.shift()?.();
+  }
+
+  async pop() {
+    while (this.items.length === 0) {
+      if (this.isClosed) return null;
+      await new Promise<void>((resolve) => this.waitersNotEmpty.push(resolve));
+    }
+
+    const item = this.items.shift()!;
+    this.reservedCount--;
+    this.waitersNotFull.shift()?.();
+
+    return item;
+  }
+
+  close() {
+    this.isClosed = true;
+    while (this.waitersNotEmpty.length > 0) this.waitersNotEmpty.shift()!();
+    while (this.waitersNotFull.length > 0) this.waitersNotFull.shift()!();
+  }
+}

src/lib/utils/concurrency/index.ts

@@ -0,0 +1,3 @@
+export * from "./BoundedQueue";
+export * from "./HybridPromise";
+export * from "./Scheduler";

src/lib/utils/index.ts

@@ -1,4 +1,4 @@
+export * from "./concurrency";
 export * from "./format";
 export * from "./gotoStateful";
-export * from "./HybridPromise";
 export * from "./sort";

src/routes/(fullscreen)/file/[id]/service.ts

@@ -1,4 +1,3 @@
-import { encryptData } from "$lib/modules/crypto";
 import { storeFileThumbnailCache } from "$lib/modules/file";
 import { prepareFileDecryption, getDecryptedFileUrl } from "$lib/serviceWorker";
 import { requestFileThumbnailUpload } from "$lib/services/file";
@@ -33,12 +32,10 @@ export const requestThumbnailUpload = async (
   dataKey: CryptoKey,
   dataKeyVersion: Date,
 ) => {
-  const thumbnailBuffer = await thumbnail.arrayBuffer();
-  const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey);
-  const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnailEncrypted);
-  if (!res.ok) return false;
+  const res = await requestFileThumbnailUpload(fileId, thumbnail, dataKey, dataKeyVersion);
+  if (!res) return false;
 
-  storeFileThumbnailCache(fileId, thumbnailBuffer); // Intended
+  void thumbnail.arrayBuffer().then((buffer) => storeFileThumbnailCache(fileId, buffer));
   return true;
 };
 

src/routes/(fullscreen)/settings/migration/+page.svelte

@@ -18,7 +18,7 @@
         info,
         state: getMigrationState(info.id),
       }))
-      .filter((file) => file.state?.status !== "completed"),
+      .filter((file) => file.state?.status !== "uploaded"),
   );
 
   const migrateAllFiles = () => {

src/routes/(fullscreen)/settings/migration/File.svelte

@@ -1,12 +1,9 @@
 <script module lang="ts">
   const subtexts = {
     queued: "대기 중",
-    "download-pending": "다운로드를 기다리는 중",
     downloading: "다운로드하는 중",
-    "encryption-pending": "암호화를 기다리는 중",
-    encrypting: "암호화하는 중",
     "upload-pending": "업로드를 기다리는 중",
-    completed: "완료",
+    uploaded: "",
     error: "실패",
   } as const;
 </script>

src/routes/(fullscreen)/settings/migration/service.svelte.ts

@@ -2,20 +2,17 @@ import { limitFunction } from "p-limit";
 import { SvelteMap } from "svelte/reactivity";
 import { CHUNK_SIZE } from "$lib/constants";
 import type { FileInfo } from "$lib/modules/filesystem";
-import { Scheduler } from "$lib/modules/scheduler";
 import { uploadBlob } from "$lib/modules/upload";
 import { requestFileDownload } from "$lib/services/file";
+import { Scheduler } from "$lib/utils";
 import { trpc } from "$trpc/client";
 
 export type MigrationStatus =
   | "queued"
-  | "download-pending"
   | "downloading"
-  | "encryption-pending"
-  | "encrypting"
   | "upload-pending"
   | "uploading"
-  | "completed"
+  | "uploaded"
   | "error";
 
 export interface MigrationState {
@@ -38,29 +35,37 @@ export const getMigrationState = (fileId: number) => {
 
 export const clearMigrationStates = () => {
   for (const [id, state] of states) {
-    if (state.status === "completed" || state.status === "error") {
+    if (state.status === "uploaded" || state.status === "error") {
       states.delete(id);
     }
   }
 };
 
-const uploadMigrationChunks = limitFunction(
-  async (state: MigrationState, fileId: number, fileBuffer: ArrayBuffer, dataKey: CryptoKey) => {
+const requestFileUpload = limitFunction(
+  async (
+    state: MigrationState,
+    fileId: number,
+    fileBuffer: ArrayBuffer,
+    dataKey: CryptoKey,
+    dataKeyVersion: Date,
+  ) => {
     state.status = "uploading";
 
     const { uploadId } = await trpc().upload.startMigrationUpload.mutate({
       file: fileId,
       chunks: Math.ceil(fileBuffer.byteLength / CHUNK_SIZE),
+      dekVersion: dataKeyVersion,
     });
 
     await uploadBlob(uploadId, new Blob([fileBuffer]), dataKey, {
       onProgress(s) {
         state.progress = s.progress;
-        state.rate = s.rateBps;
+        state.rate = s.rate;
       },
     });
 
     await trpc().upload.completeMigrationUpload.mutate({ uploadId });
+    state.status = "uploaded";
   },
   { concurrency: 1 },
 );
@@ -78,7 +83,7 @@ export const requestFileMigration = async (fileInfo: FileInfo) => {
   }
 
   try {
-    const dataKey = fileInfo.dataKey?.key;
+    const dataKey = fileInfo.dataKey;
     if (!dataKey) {
       throw new Error("Data key not available");
     }
@@ -87,18 +92,11 @@ export const requestFileMigration = async (fileInfo: FileInfo) => {
 
     await scheduler.schedule(
       async () => {
-        state.status = "download-pending";
         state.status = "downloading";
-        fileBuffer = await requestFileDownload(fileInfo.id, dataKey, true);
+        fileBuffer = await requestFileDownload(fileInfo.id, dataKey.key, true);
         return fileBuffer.byteLength;
       },
-      async () => {
-        state.status = "encryption-pending";
-
-        await uploadMigrationChunks(state, fileInfo.id, fileBuffer!, dataKey);
-
-        state.status = "completed";
-      },
+      () => requestFileUpload(state, fileInfo.id, fileBuffer!, dataKey.key, dataKey.version),
     );
   } catch (e) {
     state.status = "error";

src/routes/(fullscreen)/settings/thumbnail/File.svelte

@@ -3,7 +3,6 @@
     queued: "대기 중",
     "generation-pending": "준비 중",
     generating: "생성하는 중",
-    "upload-pending": "업로드를 기다리는 중",
     uploading: "업로드하는 중",
     error: "실패",
   } as const;

src/routes/(fullscreen)/settings/thumbnail/service.ts

@@ -1,17 +1,15 @@
 import { limitFunction } from "p-limit";
 import { SvelteMap } from "svelte/reactivity";
-import { encryptData } from "$lib/modules/crypto";
 import { storeFileThumbnailCache } from "$lib/modules/file";
 import type { FileInfo } from "$lib/modules/filesystem";
-import { Scheduler } from "$lib/modules/scheduler";
-import { generateThumbnail as doGenerateThumbnail } from "$lib/modules/thumbnail";
+import { generateThumbnail } from "$lib/modules/thumbnail";
 import { requestFileDownload, requestFileThumbnailUpload } from "$lib/services/file";
+import { Scheduler } from "$lib/utils";
 
 export type GenerationStatus =
   | "queued"
   | "generation-pending"
   | "generating"
-  | "upload-pending"
   | "uploading"
   | "uploaded"
   | "error";
@@ -31,33 +29,27 @@ export const clearThumbnailGenerationStatuses = () => {
   }
 };
 
-const generateThumbnail = limitFunction(
-  async (fileId: number, fileBuffer: ArrayBuffer, fileType: string, dataKey: CryptoKey) => {
-    statuses.set(fileId, "generating");
-
-    const thumbnail = await doGenerateThumbnail(fileBuffer, fileType);
-    if (!thumbnail) return null;
-
-    const thumbnailBuffer = await thumbnail.arrayBuffer();
-    const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey);
-    statuses.set(fileId, "upload-pending");
-    return { plaintext: thumbnailBuffer, ...thumbnailEncrypted };
-  },
-  { concurrency: 4 },
-);
-
 const requestThumbnailUpload = limitFunction(
-  async (
-    fileId: number,
-    dataKeyVersion: Date,
-    thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: ArrayBuffer },
-  ) => {
-    statuses.set(fileId, "uploading");
+  async (fileInfo: FileInfo, fileBuffer: ArrayBuffer) => {
+    statuses.set(fileInfo.id, "generating");
 
-    const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnail);
-    if (!res.ok) return false;
-    statuses.set(fileId, "uploaded");
-    storeFileThumbnailCache(fileId, thumbnail.plaintext); // Intended
+    const thumbnail = await generateThumbnail(
+      new Blob([fileBuffer], { type: fileInfo.contentType }),
+    );
+    if (!thumbnail) return false;
+
+    statuses.set(fileInfo.id, "uploading");
+
+    const res = await requestFileThumbnailUpload(
+      fileInfo.id,
+      thumbnail,
+      fileInfo.dataKey?.key!,
+      fileInfo.dataKey?.version!,
+    );
+    if (!res) return false;
+
+    statuses.set(fileInfo.id, "uploaded");
+    void thumbnail.arrayBuffer().then((buffer) => storeFileThumbnailCache(fileInfo.id, buffer));
     return true;
   },
   { concurrency: 4 },
@@ -81,16 +73,7 @@ export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
         return file.byteLength;
       },
       async () => {
-        const thumbnail = await generateThumbnail(
-          fileInfo.id,
-          file!,
-          fileInfo.contentType,
-          fileInfo.dataKey?.key!,
-        );
-        if (
-          !thumbnail ||
-          !(await requestThumbnailUpload(fileInfo.id, fileInfo.dataKey?.version!, thumbnail))
-        ) {
+        if (!(await requestThumbnailUpload(fileInfo, file!))) {
           statuses.set(fileInfo.id, "error");
         }
       },

src/routes/api/upload/[id]/chunks/[index]/+server.ts

@@ -1,6 +1,6 @@
 import { error, text } from "@sveltejs/kit";
 import { Readable } from "stream";
-import { ReadableStream } from "stream/web";
+import type { ReadableStream } from "stream/web";
 import { z } from "zod";
 import { parseContentDigestHeader } from "$lib/modules/http";
 import { authorize } from "$lib/server/modules/auth";
@@ -13,7 +13,7 @@ export const POST: RequestHandler = async ({ locals, params, request }) => {
   const zodRes = z
     .object({
       id: z.uuidv4(),
-      index: z.coerce.number().int().nonnegative(),
+      index: z.coerce.number().int().positive(),
     })
     .safeParse(params);
   if (!zodRes.success) error(400, "Invalid path parameters");

src/service-worker/constants.ts

@@ -0,0 +1 @@
+export * from "../lib/constants";

src/service-worker/handlers/decryptFile.ts

@@ -1,4 +1,4 @@
-import { DECRYPTED_FILE_URL_PREFIX, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "../modules/constants";
+import { DECRYPTED_FILE_URL_PREFIX, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "../constants";
 import { decryptChunk, getEncryptedRange, getDecryptedSize } from "../modules/crypto";
 import { parseRangeHeader, getContentRangeHeader } from "../modules/http";
 import { getFile } from "../modules/opfs";
@@ -15,10 +15,13 @@ const createResponse = (
   const headers: Record<string, string> = {
     "Accept-Ranges": "bytes",
     "Content-Length": String(range.end - range.start + 1),
-    "Content-Type": contentType ?? "application/octet-stream",
     ...(isRangeRequest ? getContentRangeHeader(range) : {}),
   };
 
+  if (contentType) {
+    headers["Content-Type"] = contentType;
+  }
+
   if (downloadFilename) {
     headers["Content-Disposition"] =
       `attachment; filename*=UTF-8''${encodeURIComponent(downloadFilename)}`;

src/service-worker/index.ts

@@ -3,7 +3,7 @@
 /// <reference lib="webworker" />
 /// <reference types="@sveltejs/kit" />
 
-import { DECRYPTED_FILE_URL_PREFIX } from "./modules/constants";
+import { DECRYPTED_FILE_URL_PREFIX } from "./constants";
 import { decryptFile } from "./handlers";
 import { fileMetadataStore } from "./stores";
 import type { ServiceWorkerMessage, ServiceWorkerResponse } from "./types";

src/service-worker/modules/constants.ts

@@ -1 +0,0 @@
-export * from "../../lib/constants";

src/service-worker/modules/crypto.ts

@@ -1,4 +1,4 @@
-import { ENCRYPTION_OVERHEAD, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "./constants";
+import { ENCRYPTION_OVERHEAD, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "../constants";
 
 export * from "../../lib/modules/crypto";
 

src/trpc/routers/upload.ts

@@ -1,7 +1,7 @@
 import { TRPCError } from "@trpc/server";
 import { createHash } from "crypto";
 import { createReadStream, createWriteStream } from "fs";
-import { mkdir, rename } from "fs/promises";
+import { copyFile, mkdir } from "fs/promises";
 import mime from "mime";
 import { dirname } from "path";
 import { v4 as uuidv4 } from "uuid";
@@ -13,6 +13,8 @@ import env from "$lib/server/loadenv";
 import { safeRecursiveRm, safeUnlink } from "$lib/server/modules/filesystem";
 import { router, roleProcedure } from "../init.server";
 
+const UPLOADS_EXPIRES = 24 * 3600 * 1000; // 24 hours
+
 const sessionLocks = new Set<string>();
 
 const generateSessionId = async () => {
@@ -60,7 +62,7 @@ const uploadRouter = router({
           userId: ctx.session.userId,
           path,
           totalChunks: input.chunks,
-          expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours
+          expiresAt: new Date(Date.now() + UPLOADS_EXPIRES),
           parentId: input.parent,
           mekVersion: input.mekVersion,
           encDek: input.dek,
@@ -89,41 +91,6 @@ const uploadRouter = router({
       }
     }),
 
-  startFileThumbnailUpload: roleProcedure["activeClient"]
-    .input(
-      z.object({
-        file: z.int().positive(),
-        dekVersion: z.date(),
-      }),
-    )
-    .mutation(async ({ ctx, input }) => {
-      const { id, path } = await generateSessionId();
-
-      try {
-        await UploadRepo.createThumbnailUploadSession({
-          id,
-          userId: ctx.session.userId,
-          path,
-          totalChunks: 1, // Up to 4 MiB
-          expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours
-          fileId: input.file,
-          dekVersion: input.dekVersion,
-        });
-        return { uploadId: id };
-      } catch (e) {
-        await safeRecursiveRm(path);
-
-        if (e instanceof IntegrityError) {
-          if (e.message === "File not found") {
-            throw new TRPCError({ code: "NOT_FOUND", message: "File not found" });
-          } else if (e.message === "Invalid DEK version") {
-            throw new TRPCError({ code: "BAD_REQUEST", message: "Mismatched DEK version" });
-          }
-        }
-        throw e;
-      }
-    }),
-
   completeFileUpload: roleProcedure["activeClient"]
     .input(
       z.object({
@@ -143,14 +110,14 @@ const uploadRouter = router({
 
       try {
         const session = await UploadRepo.getUploadSession(uploadId, ctx.session.userId);
-        if (!session || session.type !== "file") {
+        if (session?.type !== "file") {
           throw new TRPCError({ code: "NOT_FOUND", message: "Invalid upload id" });
         } else if (
           (session.hskVersion && !input.contentHmac) ||
           (!session.hskVersion && input.contentHmac)
         ) {
           throw new TRPCError({ code: "BAD_REQUEST", message: "Invalid content HMAC" });
-        } else if (session.uploadedChunks.length < session.totalChunks) {
+        } else if (session.uploadedChunks < session.totalChunks) {
           throw new TRPCError({ code: "BAD_REQUEST", message: "Upload not completed" });
         }
 
@@ -160,7 +127,7 @@ const uploadRouter = router({
         const hashStream = createHash("sha256");
         const writeStream = createWriteStream(filePath, { flags: "wx", mode: 0o600 });
 
-        for (let i = 0; i < session.totalChunks; i++) {
+        for (let i = 1; i <= session.totalChunks; i++) {
           for await (const chunk of createReadStream(`${session.path}/${i}`)) {
             hashStream.update(chunk);
             writeStream.write(chunk);
@@ -195,6 +162,42 @@ const uploadRouter = router({
       }
     }),
 
+  startFileThumbnailUpload: roleProcedure["activeClient"]
+    .input(
+      z.object({
+        file: z.int().positive(),
+        dekVersion: z.date(),
+      }),
+    )
+    .mutation(async ({ ctx, input }) => {
+      const { id, path } = await generateSessionId();
+
+      try {
+        await UploadRepo.createThumbnailOrMigrationUploadSession({
+          id,
+          type: "thumbnail",
+          userId: ctx.session.userId,
+          path,
+          totalChunks: 1, // Up to 4 MiB
+          expiresAt: new Date(Date.now() + UPLOADS_EXPIRES),
+          fileId: input.file,
+          dekVersion: input.dekVersion,
+        });
+        return { uploadId: id };
+      } catch (e) {
+        await safeRecursiveRm(path);
+
+        if (e instanceof IntegrityError) {
+          if (e.message === "File not found") {
+            throw new TRPCError({ code: "NOT_FOUND", message: "Invalid file id" });
+          } else if (e.message === "Invalid DEK version") {
+            throw new TRPCError({ code: "BAD_REQUEST", message: e.message });
+          }
+        }
+        throw e;
+      }
+    }),
+
   completeFileThumbnailUpload: roleProcedure["activeClient"]
     .input(
       z.object({
@@ -213,15 +216,15 @@ const uploadRouter = router({
 
       try {
         const session = await UploadRepo.getUploadSession(uploadId, ctx.session.userId);
-        if (!session || session.type !== "thumbnail") {
+        if (session?.type !== "thumbnail") {
           throw new TRPCError({ code: "NOT_FOUND", message: "Invalid upload id" });
-        } else if (session.uploadedChunks.length < session.totalChunks) {
+        } else if (session.uploadedChunks < session.totalChunks) {
           throw new TRPCError({ code: "BAD_REQUEST", message: "Upload not completed" });
         }
 
         thumbnailPath = `${env.thumbnailsPath}/${ctx.session.userId}/${uploadId}`;
         await mkdir(dirname(thumbnailPath), { recursive: true });
-        await rename(`${session.path}/0`, thumbnailPath);
+        await copyFile(`${session.path}/1`, thumbnailPath);
 
         const oldThumbnailPath = await db.transaction().execute(async (trx) => {
           const oldPath = await MediaRepo.updateFileThumbnail(
@@ -238,12 +241,10 @@ const uploadRouter = router({
         await Promise.all([safeUnlink(oldThumbnailPath), safeRecursiveRm(session.path)]);
       } catch (e) {
         await safeUnlink(thumbnailPath);
-        if (e instanceof IntegrityError) {
-          if (e.message === "File not found") {
-            throw new TRPCError({ code: "NOT_FOUND", message: "File not found" });
-          } else if (e.message === "Invalid DEK version") {
-            throw new TRPCError({ code: "BAD_REQUEST", message: "Mismatched DEK version" });
-          }
+
+        if (e instanceof IntegrityError && e.message === "Invalid DEK version") {
+          // DEK rotated after this upload started
+          throw new TRPCError({ code: "CONFLICT", message: e.message });
         }
         throw e;
       } finally {
@@ -256,19 +257,22 @@ const uploadRouter = router({
       z.object({
         file: z.int().positive(),
         chunks: z.int().positive(),
+        dekVersion: z.date(),
       }),
     )
     .mutation(async ({ ctx, input }) => {
       const { id, path } = await generateSessionId();
 
       try {
-        await UploadRepo.createMigrationUploadSession({
+        await UploadRepo.createThumbnailOrMigrationUploadSession({
           id,
+          type: "migration",
           userId: ctx.session.userId,
           path,
           totalChunks: input.chunks,
-          expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours
+          expiresAt: new Date(Date.now() + UPLOADS_EXPIRES),
           fileId: input.file,
+          dekVersion: input.dekVersion,
         });
         return { uploadId: id };
       } catch (e) {
@@ -276,9 +280,9 @@ const uploadRouter = router({
 
         if (e instanceof IntegrityError) {
           if (e.message === "File not found") {
-            throw new TRPCError({ code: "NOT_FOUND", message: "File not found" });
+            throw new TRPCError({ code: "NOT_FOUND", message: "Invalid file id" });
           } else if (e.message === "File is not legacy") {
-            throw new TRPCError({ code: "BAD_REQUEST", message: "File is not legacy" });
+            throw new TRPCError({ code: "BAD_REQUEST", message: e.message });
           }
         }
         throw e;
@@ -303,9 +307,9 @@ const uploadRouter = router({
 
       try {
         const session = await UploadRepo.getUploadSession(uploadId, ctx.session.userId);
-        if (!session || session.type !== "migration") {
+        if (session?.type !== "migration") {
           throw new TRPCError({ code: "NOT_FOUND", message: "Invalid upload id" });
-        } else if (session.uploadedChunks.length < session.totalChunks) {
+        } else if (session.uploadedChunks < session.totalChunks) {
           throw new TRPCError({ code: "BAD_REQUEST", message: "Upload not completed" });
         }
 
@@ -315,7 +319,7 @@ const uploadRouter = router({
         const hashStream = createHash("sha256");
         const writeStream = createWriteStream(filePath, { flags: "wx", mode: 0o600 });
 
-        for (let i = 0; i < session.totalChunks; i++) {
+        for (let i = 1; i <= session.totalChunks; i++) {
           for await (const chunk of createReadStream(`${session.path}/${i}`)) {
             hashStream.update(chunk);
             writeStream.write(chunk);
@@ -328,11 +332,12 @@ const uploadRouter = router({
 
         const hash = hashStream.digest("base64");
         const oldPath = await db.transaction().execute(async (trx) => {
-          const oldPath = await FileRepo.migrateFileContent(
+          const { oldPath } = await FileRepo.migrateFileContent(
             trx,
             ctx.session.userId,
             session.fileId,
             filePath,
+            session.dekVersion!,
             hash,
           );
           await UploadRepo.deleteUploadSession(trx, uploadId);
@@ -342,12 +347,10 @@ const uploadRouter = router({
         await Promise.all([safeUnlink(oldPath), safeRecursiveRm(session.path)]);
       } catch (e) {
         await safeUnlink(filePath);
-        if (e instanceof IntegrityError) {
-          if (e.message === "File not found") {
-            throw new TRPCError({ code: "NOT_FOUND", message: "File not found" });
-          } else if (e.message === "File is not legacy") {
-            throw new TRPCError({ code: "BAD_REQUEST", message: "File is not legacy" });
-          }
+
+        if (e instanceof IntegrityError && e.message === "File is not legacy") {
+          // File migrated after this upload started
+          throw new TRPCError({ code: "CONFLICT", message: e.message });
         }
         throw e;
       } finally {

src/workers/hmac.ts

@@ -0,0 +1,25 @@
+import { hmac } from "@noble/hashes/hmac.js";
+import { sha256 } from "@noble/hashes/sha2.js";
+
+export interface ComputeMessage {
+  stream: ReadableStream;
+  key: Uint8Array;
+}
+
+export interface ResultMessage {
+  result: Uint8Array;
+}
+
+self.onmessage = async (event: MessageEvent<ComputeMessage>) => {
+  const h = hmac.create(sha256, event.data.key);
+  const reader = event.data.stream.getReader();
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    h.update(value);
+  }
+
+  const result = h.digest();
+  self.postMessage({ result } satisfies ResultMessage, { transfer: [result.buffer] });
+};

svelte.config.js

@@ -8,6 +8,7 @@ const config = {
     adapter: adapter(),
     alias: {
       $trpc: "./src/trpc",
+      $workers: "./src/workers",
     },
   },
 };