파일을 업로드할 때 캐시에 저장하고 동영상을 스트리밍하는 과정에서도 청크 단위로 캐시에 저장하여 다음 재생에 활용하도록 개선

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/lib/client/cache/fileCache.ts

@@ -5,46 +5,181 @@ import {
   storeFileCacheIndex,
   updateFileCacheRetrievedAt,
 } from "#lib/client/storage/indexedDB/cacheIndex.js";
-import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js";
+import {
+  createWritable,
+  deleteDirectory,
+  deleteFile,
+  deleteFilesInDirectory,
+  getFile,
+  listDirectoryNames,
+  listFileNames,
+  moveFile,
+} from "#lib/client/storage/opfs.js";
+import { CHUNK_SIZE, getChunkCount } from "#lib/shared/format/index.js";
 
-// Used by both the pages and the service worker, so the index is always read from IndexedDB
+// Caches the plaintext of files in chunks, so that partially loaded files (e.g. videos which were
+// partially played) can also be cached. Chunk `i` starts at `i * CHUNK_SIZE` of the plaintext,
+// except that legacy files are cached as a single chunk. Used by both the pages and the service
+// worker, so the index is always read from IndexedDB.
 
-const getPath = (fileId: number) => `/cache/${fileId}`;
+export interface FileCacheLayout {
+  totalSize: number;
+  chunkCount: number;
+}
+
+const getDirectoryPath = (fileId: number) => `/cache/file/${fileId}`;
+const getChunkPath = (fileId: number, chunkIndex: number) =>
+  `${getDirectoryPath(fileId)}/${chunkIndex}`;
+
+const getChunkSize = ({ totalSize, chunkCount }: FileCacheLayout, chunkIndex: number) => {
+  return chunkCount === 1 ? totalSize : Math.min(CHUNK_SIZE, totalSize - chunkIndex * CHUNK_SIZE);
+};
 
 export const getFileCacheIndex = async () => {
   return await getAllFileCacheIndexes();
 };
 
-// The index is written only after the whole content is written
-export const getFileCache = async (fileId: number) => {
-  if (!(await getFileCacheIndexOf(fileId))) return null;
-
-  const file = await getFile(getPath(fileId));
-  if (file) {
-    void updateFileCacheRetrievedAt(fileId, new Date());
-  }
-  return file;
+export const getCachedChunkIndexes = async (fileId: number) => {
+  const names = await listFileNames(getDirectoryPath(fileId));
+  return new Set(names.filter((name) => /^\d+$/.test(name)).map(Number));
 };
 
-export const storeFileCache = async (fileId: number, stream: ReadableStream<Uint8Array>) => {
-  const path = getPath(fileId);
+export const readCachedChunk = async (fileId: number, chunkIndex: number) => {
+  return await getFile(getChunkPath(fileId, chunkIndex));
+};
+
+export const touchFileCache = async (fileId: number) => {
+  await updateFileCacheRetrievedAt(fileId, new Date());
+};
+
+const updateIndex = async (fileId: number, layout: FileCacheLayout) => {
+  const chunkIndexes = await getCachedChunkIndexes(fileId);
+  const size = chunkIndexes
+    .values()
+    .filter((chunkIndex) => chunkIndex < layout.chunkCount)
+    .reduce((size, chunkIndex) => size + getChunkSize(layout, chunkIndex), 0);
+  const now = new Date();
+
+  await storeFileCacheIndex({
+    fileId,
+    cachedAt: (await getFileCacheIndexOf(fileId))?.cachedAt ?? now,
+    lastRetrievedAt: now,
+    size,
+    totalSize: layout.totalSize,
+    chunkCount: layout.chunkCount,
+    isComplete: size === layout.totalSize,
+  });
+};
+
+// Writes of the same file are serialized so that the index isn't updated with a stale state
+const writeQueues = new Map<number, Promise<unknown>>();
+
+const enqueueWrite = <T>(fileId: number, write: () => Promise<T>) => {
+  const result = (writeQueues.get(fileId) ?? Promise.resolve()).catch(() => {}).then(write);
+  const settled = result.catch(() => {});
+  writeQueues.set(fileId, settled);
+  void settled.then(() => {
+    if (writeQueues.get(fileId) === settled) {
+      writeQueues.delete(fileId);
+    }
+  });
+  return result;
+};
+
+export const storeCachedChunk = (
+  fileId: number,
+  chunkIndex: number,
+  data: Uint8Array | Blob,
+  layout: FileCacheLayout,
+) => enqueueWrite(fileId, () => writeChunk(fileId, chunkIndex, data, layout));
+
+const writeChunk = async (
+  fileId: number,
+  chunkIndex: number,
+  data: Uint8Array | Blob,
+  layout: FileCacheLayout,
+) => {
+  const path = getChunkPath(fileId, chunkIndex);
+  const tempPath = `${path}.${crypto.randomUUID()}.tmp`;
+
+  // Written to a temporary file first, so that incomplete chunks are never visible
+  const writable = await createWritable(tempPath);
   try {
-    await stream.pipeTo(await createWritable(path));
+    await writable.write(data as FileSystemWriteChunkType);
+    await writable.close();
+    await moveFile(tempPath, path);
   } catch (e) {
-    await deleteFile(path).catch(() => {});
+    await writable.abort().catch(() => {});
+    await deleteFile(tempPath).catch(() => {});
     throw e;
   }
 
-  const now = new Date();
-  await storeFileCacheIndex({
-    fileId,
-    cachedAt: now,
-    lastRetrievedAt: now,
-    size: (await getFile(path))?.size ?? 0,
-  });
+  await updateIndex(fileId, layout);
+};
+
+const UPLOAD_CACHE_DIRECTORY = "/cache/upload";
+const STALE_UPLOAD_CACHE_AGE = 24 * 3600 * 1000; // 24 hours
+
+// Caches the plaintext of a file while it is being uploaded. Chunks are written to a temporary
+// directory, and moved to the cache once the id of the file is known.
+export const createUploadFileCache = (totalSize: number) => {
+  const layout = { totalSize, chunkCount: getChunkCount(totalSize) };
+  // Prefixed with the creation time to find stale ones (e.g. the tab was closed while uploading)
+  const directoryPath = `${UPLOAD_CACHE_DIRECTORY}/${Date.now()}-${crypto.randomUUID()}`;
+  const writtenChunkIndexes = new Set<number>();
+  let isFailed = false;
+
+  const discard = async () => {
+    isFailed = true;
+    await deleteDirectory(directoryPath).catch(() => {});
+  };
+
+  return {
+    // Failing to cache shouldn't fail the upload, so errors are ignored
+    async writeChunk(chunkIndex: number, data: ArrayBuffer) {
+      if (isFailed) return;
+
+      try {
+        const writable = await createWritable(`${directoryPath}/${chunkIndex}`);
+        await writable.write(data);
+        await writable.close();
+        writtenChunkIndexes.add(chunkIndex);
+      } catch (e) {
+        console.error(e);
+        await discard();
+      }
+    },
+    async commit(fileId: number) {
+      if (isFailed || writtenChunkIndexes.size !== layout.chunkCount) {
+        await discard();
+        return;
+      }
+
+      await enqueueWrite(fileId, async () => {
+        for (const chunkIndex of writtenChunkIndexes) {
+          await moveFile(`${directoryPath}/${chunkIndex}`, getChunkPath(fileId, chunkIndex));
+        }
+        await updateIndex(fileId, layout);
+      }).finally(discard);
+    },
+    discard,
+  };
 };
 
 export const deleteFileCache = async (fileId: number) => {
   await deleteFileCacheIndex(fileId);
-  await deleteFile(getPath(fileId));
+  await deleteDirectory(getDirectoryPath(fileId));
+};
+
+export const cleanupFileCaches = async () => {
+  // Files were cached as single files directly under `/cache` before
+  await deleteFilesInDirectory("/cache");
+
+  // Uploads might be in progress in other tabs, so only stale ones are deleted
+  for (const name of await listDirectoryNames(UPLOAD_CACHE_DIRECTORY)) {
+    const createdAt = parseInt(name, 10);
+    if (!(Date.now() - createdAt < STALE_UPLOAD_CACHE_AGE)) {
+      await deleteDirectory(`${UPLOAD_CACHE_DIRECTORY}/${name}`);
+    }
+  }
 };

src/lib/client/storage/indexedDB/cacheIndex.ts

@@ -4,7 +4,11 @@ export interface FileCacheIndex {
   fileId: number;
   cachedAt: Date;
   lastRetrievedAt: Date;
+  // Files are cached in chunks, possibly partially (e.g. videos which were partially played)
   size: number;
+  totalSize: number;
+  chunkCount: number;
+  isComplete: boolean;
 }
 
 const cacheIndex = new Dexie("cacheIndex") as Dexie & {
@@ -15,23 +19,23 @@ cacheIndex.version(1).stores({
   fileCache: "fileId",
 });
 
+// Files were cached as single files before version 2
+cacheIndex.version(2).upgrade(async (trx) => {
+  await trx.table("fileCache").clear();
+});
+
 export const getFileCacheIndex = async () => {
   return await cacheIndex.fileCache.toArray();
 };
 
-export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
-  await cacheIndex.fileCache.put({
-    fileId: fileCacheIndex.fileId,
-    cachedAt: fileCacheIndex.cachedAt,
-    lastRetrievedAt: fileCacheIndex.lastRetrievedAt,
-    size: fileCacheIndex.size,
-  });
-};
-
 export const getFileCacheIndexOf = async (fileId: number) => {
   return await cacheIndex.fileCache.get(fileId);
 };
 
+export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
+  await cacheIndex.fileCache.put(fileCacheIndex);
+};
+
 export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => {
   await cacheIndex.fileCache.update(fileId, { lastRetrievedAt });
 };

src/lib/client/storage/opfs.ts

@@ -94,3 +94,51 @@ export const deleteDirectory = async (path: string) => {
 
   await parentHandle.removeEntry(directoryHandle.name, { recursive: true });
 };
+
+const listEntryNames = async (path: string, kind: FileSystemHandleKind) => {
+  const { directoryHandle } = await getDirectoryHandle(path);
+  if (!directoryHandle) return [];
+
+  const names: string[] = [];
+  for await (const handle of directoryHandle.values()) {
+    if (handle.kind === kind) {
+      names.push(handle.name);
+    }
+  }
+  return names;
+};
+
+export const listFileNames = (path: string) => listEntryNames(path, "file");
+
+export const listDirectoryNames = (path: string) => listEntryNames(path, "directory");
+
+// Deletes the files (not directories) directly under the directory
+export const deleteFilesInDirectory = async (path: string) => {
+  const { directoryHandle } = await getDirectoryHandle(path);
+  if (!directoryHandle) return;
+
+  for (const name of await listFileNames(path)) {
+    await directoryHandle.removeEntry(name);
+  }
+};
+
+// Renames the file atomically
+export const moveFile = async (path: string, newPath: string) => {
+  const { fileHandle } = await getFileHandle(path, false);
+  if (!fileHandle) {
+    throw new Error("File not found");
+  }
+
+  let directoryHandle = await navigator.storage.getDirectory();
+  const parts = newPath.split("/").filter((part) => part);
+  for (const part of parts.slice(0, -1)) {
+    directoryHandle = await directoryHandle.getDirectoryHandle(part, { create: true });
+  }
+
+  // Supported by Chromium for the origin private file system
+  await (
+    fileHandle as FileSystemFileHandle & {
+      move(destination: FileSystemDirectoryHandle, name: string): Promise<void>;
+    }
+  ).move(directoryHandle, parts.at(-1)!);
+};

src/lib/client/transfer/chunkedUpload.ts

@@ -16,10 +16,6 @@ interface EncryptedChunk {
   data: ArrayBuffer;
 }
 
-const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
-  return await encryptChunk(await chunk.arrayBuffer(), dataKey);
-};
-
 const uploadEncryptedChunk = async (
   uploadId: string,
   chunkIndex: number,
@@ -40,7 +36,12 @@ export const uploadBlob = async (
   uploadId: string,
   blob: Blob,
   dataKey: CryptoKey,
-  options?: { concurrency?: number; onProgress?: (s: UploadStats) => void },
+  options?: {
+    concurrency?: number;
+    onProgress?: (s: UploadStats) => void;
+    // Receives each chunk of the plaintext before it is encrypted
+    onPlaintextChunk?: (chunkIndex: number, data: ArrayBuffer) => Promise<void>;
+  },
 ) => {
   const onProgress = options?.onProgress;
   const networkConcurrency = options?.concurrency ?? 4;
@@ -75,8 +76,9 @@ export const uploadBlob = async (
   const encryptionProducer = async () => {
     try {
       for (let i = 0; i < totalChunks; i++) {
-        const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
-        const data = await encryptChunkData(chunk, dataKey);
+        const chunk = await blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE).arrayBuffer();
+        await options?.onPlaintextChunk?.(i, chunk);
+        const data = await encryptChunk(chunk, dataKey);
         hash.update(new Uint8Array(data));
         await queue.push({ index: i, data });
       }

src/lib/client/transfer/upload.svelte.ts

@@ -6,6 +6,7 @@ import {
   type HmacSecret,
   type MasterKey,
 } from "#lib/client/keys/index.js";
+import { createUploadFileCache } from "#lib/client/cache/index.js";
 import { readExifDateTime } from "#lib/client/media/exif.js";
 import { generateThumbnail } from "#lib/client/media/thumbnail.js";
 import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
@@ -150,18 +151,30 @@ const requestFileUpload = limitFunction(
   ) => {
     state.status = "uploading";
 
-    const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
-      onProgress(s) {
-        state.progress = s.progress;
-        state.rate = s.rate;
-      },
-    });
+    // The file is cached while uploading, so that it doesn't have to be read again
+    const cache = createUploadFileCache(file.size);
 
-    const { file: fileId } = await trpc.upload.completeFileUpload.mutate({
-      uploadId,
-      contentHmac: fileSigned,
-      encContentHash,
-    });
+    let fileId;
+    try {
+      const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
+        onProgress(s) {
+          state.progress = s.progress;
+          state.rate = s.rate;
+        },
+        onPlaintextChunk: cache.writeChunk,
+      });
+
+      ({ file: fileId } = await trpc.upload.completeFileUpload.mutate({
+        uploadId,
+        contentHmac: fileSigned,
+        encContentHash,
+      }));
+    } catch (e) {
+      await cache.discard();
+      throw e;
+    }
+
+    await cache.commit(fileId).catch(console.error);
 
     if (thumbnailBuffer) {
       try {