서버, 클라이언트, 서비스 워커 공용 코드를 lib/shared로 분리하고 브라우저 전용 모듈을 lib/client로 재구성

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/lib/client/transfer/chunkedUpload.ts

@@ -0,0 +1,177 @@
+import { sha256 } from "@noble/hashes/sha2.js";
+import axios from "axios";
+import pLimit from "p-limit";
+import { encodeToBase64, encryptChunk } from "#lib/shared/crypto/index.js";
+import { CHUNK_SIZE, getChunkCount, getEncryptedSize } from "#lib/shared/format/index.js";
+import { BoundedQueue } from "#lib/utils/concurrency/index.js";
+
+interface UploadStats {
+  progress: number;
+  rate: number;
+}
+
+interface EncryptedChunk {
+  index: number;
+  data: ArrayBuffer;
+}
+
+const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => {
+  const samples: { t: number; b: number }[] = [];
+  let lastSpeed = 0;
+  let startTime: number | null = null;
+
+  return (bytesNow?: number) => {
+    if (bytesNow === undefined) return lastSpeed;
+
+    const now = performance.now();
+
+    // Initialize start time on first call
+    if (startTime === null) {
+      startTime = now;
+    }
+
+    // Check if enough time has passed since the last sample
+    const lastSample = samples[samples.length - 1];
+    if (lastSample && now - lastSample.t < minInterval) {
+      return lastSpeed;
+    }
+
+    samples.push({ t: now, b: bytesNow });
+
+    // Remove old samples outside the time window
+    const cutoff = now - timeWindow;
+    while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift();
+
+    // Need at least 2 samples to calculate speed
+    if (samples.length < 2) {
+      return lastSpeed;
+    }
+
+    const first = samples[0]!;
+    const dt = now - first.t;
+    const db = bytesNow - first.b;
+
+    if (dt >= minInterval) {
+      const instantSpeed = (db / dt) * 1000;
+      // Apply EMA for smoother speed transitions
+      const alpha = 0.3;
+      const rawSpeed =
+        lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed;
+
+      // Apply warmup ramp to prevent initial overestimation
+      const elapsed = now - startTime;
+      const warmupWeight = Math.min(1, elapsed / warmupPeriod);
+      lastSpeed = rawSpeed * warmupWeight;
+    }
+
+    return lastSpeed;
+  };
+};
+
+const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
+  return await encryptChunk(await chunk.arrayBuffer(), dataKey);
+};
+
+const uploadEncryptedChunk = async (
+  uploadId: string,
+  chunkIndex: number,
+  encrypted: ArrayBuffer,
+  onChunkProgress: (chunkIndex: number, loaded: number) => void,
+) => {
+  await axios.post(`/api/upload/${uploadId}/chunks/${chunkIndex + 1}`, encrypted, {
+    headers: { "Content-Type": "application/octet-stream" },
+    onUploadProgress(e) {
+      onChunkProgress(chunkIndex, e.loaded ?? 0);
+    },
+  });
+
+  onChunkProgress(chunkIndex, encrypted.byteLength);
+};
+
+export const uploadBlob = async (
+  uploadId: string,
+  blob: Blob,
+  dataKey: CryptoKey,
+  options?: { concurrency?: number; onProgress?: (s: UploadStats) => void },
+) => {
+  const onProgress = options?.onProgress;
+  const networkConcurrency = options?.concurrency ?? 4;
+  const maxQueueSize = 8;
+
+  const totalChunks = getChunkCount(blob.size);
+  const totalBytes = getEncryptedSize(blob.size);
+
+  const uploadedByChunk = new Array<number>(totalChunks).fill(0);
+  const speedMeter = createSpeedMeter(3000, 200);
+  const hash = sha256.create();
+
+  const emit = () => {
+    if (!onProgress) return;
+
+    const uploadedBytes = uploadedByChunk.reduce((a, b) => a + b, 0);
+    const rate = speedMeter(uploadedBytes);
+    const progress = Math.min(1, uploadedBytes / totalBytes);
+
+    onProgress({ progress, rate });
+  };
+
+  const onChunkProgress = (idx: number, loaded: number) => {
+    uploadedByChunk[idx] = loaded;
+    emit();
+  };
+
+  const queue = new BoundedQueue<EncryptedChunk>(maxQueueSize);
+  let encryptionError: Error | null = null;
+
+  // Producer: encrypt chunks and push to queue
+  const encryptionProducer = async () => {
+    try {
+      for (let i = 0; i < totalChunks; i++) {
+        const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+        const data = await encryptChunkData(chunk, dataKey);
+        hash.update(new Uint8Array(data));
+        await queue.push({ index: i, data });
+      }
+    } catch (e) {
+      encryptionError = e instanceof Error ? e : new Error(String(e));
+    } finally {
+      queue.close();
+    }
+  };
+
+  // Consumer: upload chunks from queue with concurrency limit
+  const uploadConsumer = async () => {
+    const limit = pLimit(networkConcurrency);
+    const activeTasks = new Set<Promise<void>>();
+
+    while (true) {
+      const item = await queue.pop();
+      if (item === null) break;
+      if (encryptionError) throw encryptionError;
+
+      const task = limit(async () => {
+        try {
+          await uploadEncryptedChunk(uploadId, item.index, item.data, onChunkProgress);
+        } finally {
+          // @ts-expect-error
+          item.data = null;
+        }
+      });
+
+      activeTasks.add(task);
+      task.finally(() => activeTasks.delete(task));
+
+      if (activeTasks.size >= networkConcurrency) {
+        await Promise.race(activeTasks);
+      }
+    }
+
+    await Promise.all(activeTasks);
+  };
+
+  // Run producer and consumer concurrently
+  await Promise.all([encryptionProducer(), uploadConsumer()]);
+
+  onProgress?.({ progress: 1, rate: speedMeter() });
+  return { encContentHash: encodeToBase64(hash.digest()) };
+};

src/lib/client/transfer/download.svelte.ts

@@ -0,0 +1,110 @@
+import axios from "axios";
+import { limitFunction } from "p-limit";
+import { decryptChunk, concatenateBuffers } from "#lib/shared/crypto/index.js";
+import { ENCRYPTED_CHUNK_SIZE } from "#lib/shared/format/index.js";
+
+export interface FileDownloadState {
+  id: number;
+  status:
+    | "download-pending"
+    | "downloading"
+    | "decryption-pending"
+    | "decrypting"
+    | "decrypted"
+    | "canceled"
+    | "error";
+  progress?: number;
+  rate?: number;
+  estimated?: number;
+  result?: ArrayBuffer;
+}
+
+type LiveFileDownloadState = FileDownloadState & {
+  status: "download-pending" | "downloading" | "decryption-pending" | "decrypting";
+};
+
+let downloadingFiles: FileDownloadState[] = $state([]);
+
+export const isFileDownloading = (
+  status: FileDownloadState["status"],
+): status is LiveFileDownloadState["status"] =>
+  ["download-pending", "downloading", "decryption-pending", "decrypting"].includes(status);
+
+export const getFileDownloadState = (fileId: number) => {
+  return downloadingFiles.find((file) => file.id === fileId && isFileDownloading(file.status));
+};
+
+export const getDownloadingFiles = () => {
+  return downloadingFiles.filter((file) => isFileDownloading(file.status));
+};
+
+export const clearDownloadedFiles = () => {
+  downloadingFiles = downloadingFiles.filter((file) => isFileDownloading(file.status));
+};
+
+const requestFileDownload = limitFunction(
+  async (state: FileDownloadState, id: number) => {
+    state.status = "downloading";
+
+    const res = await axios.get(`/api/file/${id}/download`, {
+      responseType: "arraybuffer",
+      onDownloadProgress: ({ progress, rate, estimated }) => {
+        state.progress = progress;
+        state.rate = rate;
+        state.estimated = estimated;
+      },
+    });
+    const fileEncrypted: ArrayBuffer = res.data;
+
+    state.status = "decryption-pending";
+    return fileEncrypted;
+  },
+  { concurrency: 1 },
+);
+
+const decryptFile = limitFunction(
+  async (
+    state: FileDownloadState,
+    fileEncrypted: ArrayBuffer,
+    encryptedChunkSize: number,
+    dataKey: CryptoKey,
+  ) => {
+    state.status = "decrypting";
+
+    const chunks: ArrayBuffer[] = [];
+    let offset = 0;
+
+    while (offset < fileEncrypted.byteLength) {
+      const nextOffset = Math.min(offset + encryptedChunkSize, fileEncrypted.byteLength);
+      chunks.push(await decryptChunk(fileEncrypted.slice(offset, nextOffset), dataKey));
+      offset = nextOffset;
+    }
+
+    const fileBuffer = concatenateBuffers(...chunks).buffer;
+    state.status = "decrypted";
+    state.result = fileBuffer;
+    return fileBuffer;
+  },
+  { concurrency: 4 },
+);
+
+export const downloadFile = async (id: number, dataKey: CryptoKey, isLegacy: boolean) => {
+  downloadingFiles.push({
+    id,
+    status: "download-pending",
+  });
+  const state = downloadingFiles.at(-1)!;
+
+  try {
+    const fileEncrypted = await requestFileDownload(state, id);
+    return await decryptFile(
+      state,
+      fileEncrypted,
+      isLegacy ? fileEncrypted.byteLength : ENCRYPTED_CHUNK_SIZE,
+      dataKey,
+    );
+  } catch (e) {
+    state.status = "error";
+    throw e;
+  }
+};

src/lib/client/transfer/index.ts

@@ -0,0 +1,2 @@
+export * from "./download.svelte";
+export * from "./upload.svelte";

src/lib/client/transfer/upload.svelte.ts

@@ -0,0 +1,243 @@
+import { limitFunction } from "p-limit";
+import { signMessageHmac } from "#lib/client/crypto/hmac.js";
+import { extractExifDateTime } from "#lib/client/media/exif.js";
+import { generateThumbnail } from "#lib/client/media/thumbnail.js";
+import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
+import {
+  encodeToBase64,
+  generateDataKey,
+  wrapDataKey,
+  encryptString,
+} from "#lib/shared/crypto/index.js";
+import { getChunkCount } from "#lib/shared/format/index.js";
+import type { MasterKey, HmacSecret } from "#lib/stores/index.js";
+import { Scheduler } from "#lib/utils/index.js";
+import { trpc } from "#trpc/client.js";
+
+export interface FileUploadState {
+  id: string;
+  name: string;
+  parentId: DirectoryId;
+  status:
+    | "queued"
+    | "encryption-pending"
+    | "encrypting"
+    | "upload-pending"
+    | "uploading"
+    | "uploaded"
+    | "canceled"
+    | "error";
+  progress?: number;
+  rate?: number;
+  estimated?: number;
+}
+
+export type LiveFileUploadState = FileUploadState & {
+  status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
+};
+
+const scheduler = new Scheduler<
+  { fileId: number; fileBuffer?: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
+>();
+let uploadingFiles: FileUploadState[] = $state([]);
+
+const isFileUploading = (status: FileUploadState["status"]) =>
+  ["queued", "encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status);
+
+export const getUploadingFiles = (parentId?: DirectoryId) => {
+  return uploadingFiles.filter(
+    (file) =>
+      (parentId === undefined || file.parentId === parentId) && isFileUploading(file.status),
+  );
+};
+
+export const clearUploadedFiles = () => {
+  uploadingFiles = uploadingFiles.filter((file) => isFileUploading(file.status));
+};
+
+const requestDuplicateFileScan = limitFunction(
+  async (
+    state: FileUploadState,
+    file: File,
+    hmacSecret: HmacSecret,
+    onDuplicate: () => Promise<boolean>,
+  ) => {
+    state.status = "encryption-pending";
+
+    const fileSigned = encodeToBase64(await signMessageHmac(file, hmacSecret.secret));
+    const files = await trpc.file.listByHash.query({
+      hskVersion: hmacSecret.version,
+      contentHmac: fileSigned,
+    });
+    if (files.length === 0 || (await onDuplicate())) {
+      return { fileSigned };
+    } else {
+      return {};
+    }
+  },
+  { concurrency: 1 },
+);
+
+const getFileType = (file: File) => {
+  if (file.type) return file.type;
+  if (file.name.endsWith(".heic")) return "image/heic";
+  throw new Error("Unknown file type");
+};
+
+interface FileMetadata {
+  parentId: "root" | number;
+  name: string;
+  createdAt?: Date;
+  lastModifiedAt: Date;
+}
+
+const requestFileMetadataEncryption = limitFunction(
+  async (
+    state: FileUploadState,
+    file: Blob,
+    fileMetadata: FileMetadata,
+    masterKey: MasterKey,
+    hmacSecret: HmacSecret,
+  ) => {
+    state.status = "encrypting";
+
+    const { dataKey, dataKeyVersion } = await generateDataKey();
+    const dataKeyWrapped = await wrapDataKey(dataKey, masterKey.key);
+
+    const [nameEncrypted, createdAtEncrypted, lastModifiedAtEncrypted, thumbnailBuffer] =
+      await Promise.all([
+        encryptString(fileMetadata.name, dataKey),
+        fileMetadata.createdAt &&
+          encryptString(fileMetadata.createdAt.getTime().toString(), dataKey),
+        encryptString(fileMetadata.lastModifiedAt.getTime().toString(), dataKey),
+        generateThumbnail(file).then((blob) => blob?.arrayBuffer()),
+      ]);
+
+    const { uploadId } = await trpc.upload.startFileUpload.mutate({
+      chunks: getChunkCount(file.size),
+      parent: fileMetadata.parentId,
+      mekVersion: masterKey.version,
+      dek: dataKeyWrapped,
+      dekVersion: dataKeyVersion,
+      hskVersion: hmacSecret.version,
+      contentType: file.type,
+      name: nameEncrypted.ciphertext,
+      nameIv: nameEncrypted.iv,
+      createdAt: createdAtEncrypted?.ciphertext,
+      createdAtIv: createdAtEncrypted?.iv,
+      lastModifiedAt: lastModifiedAtEncrypted.ciphertext,
+      lastModifiedAtIv: lastModifiedAtEncrypted.iv,
+    });
+
+    state.status = "upload-pending";
+    return { uploadId, thumbnailBuffer, dataKey, dataKeyVersion };
+  },
+  { concurrency: 4 },
+);
+
+const requestFileUpload = limitFunction(
+  async (
+    state: FileUploadState,
+    uploadId: string,
+    file: Blob,
+    fileSigned: string,
+    thumbnailBuffer: ArrayBuffer | undefined,
+    dataKey: CryptoKey,
+    dataKeyVersion: Date,
+  ) => {
+    state.status = "uploading";
+
+    const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
+      onProgress(s) {
+        state.progress = s.progress;
+        state.rate = s.rate;
+      },
+    });
+
+    const { file: fileId } = await trpc.upload.completeFileUpload.mutate({
+      uploadId,
+      contentHmac: fileSigned,
+      encContentHash,
+    });
+
+    if (thumbnailBuffer) {
+      try {
+        const { uploadId } = await trpc.upload.startFileThumbnailUpload.mutate({
+          file: fileId,
+          dekVersion: dataKeyVersion,
+        });
+
+        await uploadBlob(uploadId, new Blob([thumbnailBuffer]), dataKey);
+
+        await trpc.upload.completeFileThumbnailUpload.mutate({ uploadId });
+      } catch (e) {
+        console.error(e);
+      }
+    }
+
+    state.status = "uploaded";
+    return { fileId };
+  },
+  { concurrency: 1 },
+);
+
+export const uploadFile = async (
+  file: File,
+  parentId: "root" | number,
+  masterKey: MasterKey,
+  hmacSecret: HmacSecret,
+  onDuplicate: () => Promise<boolean>,
+) => {
+  uploadingFiles.push({
+    id: crypto.randomUUID(),
+    name: file.name,
+    parentId,
+    status: "queued",
+  });
+  const state = uploadingFiles.at(-1)!;
+
+  return await scheduler.schedule(file.size, async () => {
+    try {
+      const { fileSigned } = await requestDuplicateFileScan(state, file, hmacSecret, onDuplicate);
+
+      if (!fileSigned) {
+        state.status = "canceled";
+        uploadingFiles = uploadingFiles.filter((file) => file !== state);
+        return;
+      }
+
+      let fileBuffer;
+      const fileType = getFileType(file);
+      const fileMetadata: FileMetadata = {
+        parentId,
+        name: file.name,
+        lastModifiedAt: new Date(file.lastModified),
+      };
+
+      if (fileType.startsWith("image/")) {
+        fileBuffer = await file.arrayBuffer();
+        fileMetadata.createdAt = extractExifDateTime(fileBuffer);
+      }
+
+      const blob = new Blob([file], { type: fileType });
+
+      const { uploadId, thumbnailBuffer, dataKey, dataKeyVersion } =
+        await requestFileMetadataEncryption(state, blob, fileMetadata, masterKey, hmacSecret);
+
+      const { fileId } = await requestFileUpload(
+        state,
+        uploadId,
+        blob,
+        fileSigned,
+        thumbnailBuffer,
+        dataKey,
+        dataKeyVersion,
+      );
+
+      return { fileId, fileBuffer, thumbnailBuffer };
+    } catch (e) {
+      state.status = "error";
+      throw e;
+    }
+  });
+};