diff --git a/src/hooks.client.ts b/src/hooks.client.ts index 3743e9d..696e3c5 100644 --- a/src/hooks.client.ts +++ b/src/hooks.client.ts @@ -1,4 +1,5 @@ import type { ClientInit } from "@sveltejs/kit/hooks"; +import { cleanupFileCaches } from "#lib/client/cache/index.js"; import { keyring } from "#lib/client/keys/index.js"; import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js"; import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js"; @@ -18,4 +19,5 @@ export const init: ClientInit = async () => { await Promise.all([requestPersistentStorage(), keyring.load()]); cleanupDanglingInfos(); // Intended + cleanupFileCaches().catch(console.error); }; diff --git a/src/lib/client/cache/fileCache.ts b/src/lib/client/cache/fileCache.ts index 33a5a34..325ebc5 100644 --- a/src/lib/client/cache/fileCache.ts +++ b/src/lib/client/cache/fileCache.ts @@ -5,46 +5,181 @@ import { storeFileCacheIndex, updateFileCacheRetrievedAt, } from "#lib/client/storage/indexedDB/cacheIndex.js"; -import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js"; +import { + createWritable, + deleteDirectory, + deleteFile, + deleteFilesInDirectory, + getFile, + listDirectoryNames, + listFileNames, + moveFile, +} from "#lib/client/storage/opfs.js"; +import { CHUNK_SIZE, getChunkCount } from "#lib/shared/format/index.js"; -// Used by both the pages and the service worker, so the index is always read from IndexedDB +// Caches the plaintext of files in chunks, so that partially loaded files (e.g. videos which were +// partially played) can also be cached. Chunk `i` starts at `i * CHUNK_SIZE` of the plaintext, +// except that legacy files are cached as a single chunk. Used by both the pages and the service +// worker, so the index is always read from IndexedDB. -const getPath = (fileId: number) => `/cache/${fileId}`; +export interface FileCacheLayout { + totalSize: number; + chunkCount: number; +} + +const getDirectoryPath = (fileId: number) => `/cache/file/${fileId}`; +const getChunkPath = (fileId: number, chunkIndex: number) => + `${getDirectoryPath(fileId)}/${chunkIndex}`; + +const getChunkSize = ({ totalSize, chunkCount }: FileCacheLayout, chunkIndex: number) => { + return chunkCount === 1 ? totalSize : Math.min(CHUNK_SIZE, totalSize - chunkIndex * CHUNK_SIZE); +}; export const getFileCacheIndex = async () => { return await getAllFileCacheIndexes(); }; -// The index is written only after the whole content is written -export const getFileCache = async (fileId: number) => { - if (!(await getFileCacheIndexOf(fileId))) return null; - - const file = await getFile(getPath(fileId)); - if (file) { - void updateFileCacheRetrievedAt(fileId, new Date()); - } - return file; +export const getCachedChunkIndexes = async (fileId: number) => { + const names = await listFileNames(getDirectoryPath(fileId)); + return new Set(names.filter((name) => /^\d+$/.test(name)).map(Number)); }; -export const storeFileCache = async (fileId: number, stream: ReadableStream) => { - const path = getPath(fileId); +export const readCachedChunk = async (fileId: number, chunkIndex: number) => { + return await getFile(getChunkPath(fileId, chunkIndex)); +}; + +export const touchFileCache = async (fileId: number) => { + await updateFileCacheRetrievedAt(fileId, new Date()); +}; + +const updateIndex = async (fileId: number, layout: FileCacheLayout) => { + const chunkIndexes = await getCachedChunkIndexes(fileId); + const size = chunkIndexes + .values() + .filter((chunkIndex) => chunkIndex < layout.chunkCount) + .reduce((size, chunkIndex) => size + getChunkSize(layout, chunkIndex), 0); + const now = new Date(); + + await storeFileCacheIndex({ + fileId, + cachedAt: (await getFileCacheIndexOf(fileId))?.cachedAt ?? now, + lastRetrievedAt: now, + size, + totalSize: layout.totalSize, + chunkCount: layout.chunkCount, + isComplete: size === layout.totalSize, + }); +}; + +// Writes of the same file are serialized so that the index isn't updated with a stale state +const writeQueues = new Map>(); + +const enqueueWrite = (fileId: number, write: () => Promise) => { + const result = (writeQueues.get(fileId) ?? Promise.resolve()).catch(() => {}).then(write); + const settled = result.catch(() => {}); + writeQueues.set(fileId, settled); + void settled.then(() => { + if (writeQueues.get(fileId) === settled) { + writeQueues.delete(fileId); + } + }); + return result; +}; + +export const storeCachedChunk = ( + fileId: number, + chunkIndex: number, + data: Uint8Array | Blob, + layout: FileCacheLayout, +) => enqueueWrite(fileId, () => writeChunk(fileId, chunkIndex, data, layout)); + +const writeChunk = async ( + fileId: number, + chunkIndex: number, + data: Uint8Array | Blob, + layout: FileCacheLayout, +) => { + const path = getChunkPath(fileId, chunkIndex); + const tempPath = `${path}.${crypto.randomUUID()}.tmp`; + + // Written to a temporary file first, so that incomplete chunks are never visible + const writable = await createWritable(tempPath); try { - await stream.pipeTo(await createWritable(path)); + await writable.write(data as FileSystemWriteChunkType); + await writable.close(); + await moveFile(tempPath, path); } catch (e) { - await deleteFile(path).catch(() => {}); + await writable.abort().catch(() => {}); + await deleteFile(tempPath).catch(() => {}); throw e; } - const now = new Date(); - await storeFileCacheIndex({ - fileId, - cachedAt: now, - lastRetrievedAt: now, - size: (await getFile(path))?.size ?? 0, - }); + await updateIndex(fileId, layout); +}; + +const UPLOAD_CACHE_DIRECTORY = "/cache/upload"; +const STALE_UPLOAD_CACHE_AGE = 24 * 3600 * 1000; // 24 hours + +// Caches the plaintext of a file while it is being uploaded. Chunks are written to a temporary +// directory, and moved to the cache once the id of the file is known. +export const createUploadFileCache = (totalSize: number) => { + const layout = { totalSize, chunkCount: getChunkCount(totalSize) }; + // Prefixed with the creation time to find stale ones (e.g. the tab was closed while uploading) + const directoryPath = `${UPLOAD_CACHE_DIRECTORY}/${Date.now()}-${crypto.randomUUID()}`; + const writtenChunkIndexes = new Set(); + let isFailed = false; + + const discard = async () => { + isFailed = true; + await deleteDirectory(directoryPath).catch(() => {}); + }; + + return { + // Failing to cache shouldn't fail the upload, so errors are ignored + async writeChunk(chunkIndex: number, data: ArrayBuffer) { + if (isFailed) return; + + try { + const writable = await createWritable(`${directoryPath}/${chunkIndex}`); + await writable.write(data); + await writable.close(); + writtenChunkIndexes.add(chunkIndex); + } catch (e) { + console.error(e); + await discard(); + } + }, + async commit(fileId: number) { + if (isFailed || writtenChunkIndexes.size !== layout.chunkCount) { + await discard(); + return; + } + + await enqueueWrite(fileId, async () => { + for (const chunkIndex of writtenChunkIndexes) { + await moveFile(`${directoryPath}/${chunkIndex}`, getChunkPath(fileId, chunkIndex)); + } + await updateIndex(fileId, layout); + }).finally(discard); + }, + discard, + }; }; export const deleteFileCache = async (fileId: number) => { await deleteFileCacheIndex(fileId); - await deleteFile(getPath(fileId)); + await deleteDirectory(getDirectoryPath(fileId)); +}; + +export const cleanupFileCaches = async () => { + // Files were cached as single files directly under `/cache` before + await deleteFilesInDirectory("/cache"); + + // Uploads might be in progress in other tabs, so only stale ones are deleted + for (const name of await listDirectoryNames(UPLOAD_CACHE_DIRECTORY)) { + const createdAt = parseInt(name, 10); + if (!(Date.now() - createdAt < STALE_UPLOAD_CACHE_AGE)) { + await deleteDirectory(`${UPLOAD_CACHE_DIRECTORY}/${name}`); + } + } }; diff --git a/src/lib/client/storage/indexedDB/cacheIndex.ts b/src/lib/client/storage/indexedDB/cacheIndex.ts index b3b2516..efaeb37 100644 --- a/src/lib/client/storage/indexedDB/cacheIndex.ts +++ b/src/lib/client/storage/indexedDB/cacheIndex.ts @@ -4,7 +4,11 @@ export interface FileCacheIndex { fileId: number; cachedAt: Date; lastRetrievedAt: Date; + // Files are cached in chunks, possibly partially (e.g. videos which were partially played) size: number; + totalSize: number; + chunkCount: number; + isComplete: boolean; } const cacheIndex = new Dexie("cacheIndex") as Dexie & { @@ -15,23 +19,23 @@ cacheIndex.version(1).stores({ fileCache: "fileId", }); +// Files were cached as single files before version 2 +cacheIndex.version(2).upgrade(async (trx) => { + await trx.table("fileCache").clear(); +}); + export const getFileCacheIndex = async () => { return await cacheIndex.fileCache.toArray(); }; -export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => { - await cacheIndex.fileCache.put({ - fileId: fileCacheIndex.fileId, - cachedAt: fileCacheIndex.cachedAt, - lastRetrievedAt: fileCacheIndex.lastRetrievedAt, - size: fileCacheIndex.size, - }); -}; - export const getFileCacheIndexOf = async (fileId: number) => { return await cacheIndex.fileCache.get(fileId); }; +export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => { + await cacheIndex.fileCache.put(fileCacheIndex); +}; + export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => { await cacheIndex.fileCache.update(fileId, { lastRetrievedAt }); }; diff --git a/src/lib/client/storage/opfs.ts b/src/lib/client/storage/opfs.ts index 5fd6f1a..2a3171e 100644 --- a/src/lib/client/storage/opfs.ts +++ b/src/lib/client/storage/opfs.ts @@ -94,3 +94,51 @@ export const deleteDirectory = async (path: string) => { await parentHandle.removeEntry(directoryHandle.name, { recursive: true }); }; + +const listEntryNames = async (path: string, kind: FileSystemHandleKind) => { + const { directoryHandle } = await getDirectoryHandle(path); + if (!directoryHandle) return []; + + const names: string[] = []; + for await (const handle of directoryHandle.values()) { + if (handle.kind === kind) { + names.push(handle.name); + } + } + return names; +}; + +export const listFileNames = (path: string) => listEntryNames(path, "file"); + +export const listDirectoryNames = (path: string) => listEntryNames(path, "directory"); + +// Deletes the files (not directories) directly under the directory +export const deleteFilesInDirectory = async (path: string) => { + const { directoryHandle } = await getDirectoryHandle(path); + if (!directoryHandle) return; + + for (const name of await listFileNames(path)) { + await directoryHandle.removeEntry(name); + } +}; + +// Renames the file atomically +export const moveFile = async (path: string, newPath: string) => { + const { fileHandle } = await getFileHandle(path, false); + if (!fileHandle) { + throw new Error("File not found"); + } + + let directoryHandle = await navigator.storage.getDirectory(); + const parts = newPath.split("/").filter((part) => part); + for (const part of parts.slice(0, -1)) { + directoryHandle = await directoryHandle.getDirectoryHandle(part, { create: true }); + } + + // Supported by Chromium for the origin private file system + await ( + fileHandle as FileSystemFileHandle & { + move(destination: FileSystemDirectoryHandle, name: string): Promise; + } + ).move(directoryHandle, parts.at(-1)!); +}; diff --git a/src/lib/client/transfer/chunkedUpload.ts b/src/lib/client/transfer/chunkedUpload.ts index f3a7175..a641ece 100644 --- a/src/lib/client/transfer/chunkedUpload.ts +++ b/src/lib/client/transfer/chunkedUpload.ts @@ -16,10 +16,6 @@ interface EncryptedChunk { data: ArrayBuffer; } -const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise => { - return await encryptChunk(await chunk.arrayBuffer(), dataKey); -}; - const uploadEncryptedChunk = async ( uploadId: string, chunkIndex: number, @@ -40,7 +36,12 @@ export const uploadBlob = async ( uploadId: string, blob: Blob, dataKey: CryptoKey, - options?: { concurrency?: number; onProgress?: (s: UploadStats) => void }, + options?: { + concurrency?: number; + onProgress?: (s: UploadStats) => void; + // Receives each chunk of the plaintext before it is encrypted + onPlaintextChunk?: (chunkIndex: number, data: ArrayBuffer) => Promise; + }, ) => { const onProgress = options?.onProgress; const networkConcurrency = options?.concurrency ?? 4; @@ -75,8 +76,9 @@ export const uploadBlob = async ( const encryptionProducer = async () => { try { for (let i = 0; i < totalChunks; i++) { - const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE); - const data = await encryptChunkData(chunk, dataKey); + const chunk = await blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE).arrayBuffer(); + await options?.onPlaintextChunk?.(i, chunk); + const data = await encryptChunk(chunk, dataKey); hash.update(new Uint8Array(data)); await queue.push({ index: i, data }); } diff --git a/src/lib/client/transfer/upload.svelte.ts b/src/lib/client/transfer/upload.svelte.ts index 7702e6b..ee79101 100644 --- a/src/lib/client/transfer/upload.svelte.ts +++ b/src/lib/client/transfer/upload.svelte.ts @@ -6,6 +6,7 @@ import { type HmacSecret, type MasterKey, } from "#lib/client/keys/index.js"; +import { createUploadFileCache } from "#lib/client/cache/index.js"; import { readExifDateTime } from "#lib/client/media/exif.js"; import { generateThumbnail } from "#lib/client/media/thumbnail.js"; import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js"; @@ -150,18 +151,30 @@ const requestFileUpload = limitFunction( ) => { state.status = "uploading"; - const { encContentHash } = await uploadBlob(uploadId, file, dataKey, { - onProgress(s) { - state.progress = s.progress; - state.rate = s.rate; - }, - }); + // The file is cached while uploading, so that it doesn't have to be read again + const cache = createUploadFileCache(file.size); - const { file: fileId } = await trpc.upload.completeFileUpload.mutate({ - uploadId, - contentHmac: fileSigned, - encContentHash, - }); + let fileId; + try { + const { encContentHash } = await uploadBlob(uploadId, file, dataKey, { + onProgress(s) { + state.progress = s.progress; + state.rate = s.rate; + }, + onPlaintextChunk: cache.writeChunk, + }); + + ({ file: fileId } = await trpc.upload.completeFileUpload.mutate({ + uploadId, + contentHmac: fileSigned, + encContentHash, + })); + } catch (e) { + await cache.discard(); + throw e; + } + + await cache.commit(fileId).catch(console.error); if (thumbnailBuffer) { try { diff --git a/src/lib/features/filesystem/content.ts b/src/lib/features/filesystem/content.ts index 5e34ad6..bd61b84 100644 --- a/src/lib/features/filesystem/content.ts +++ b/src/lib/features/filesystem/content.ts @@ -26,18 +26,7 @@ export const getFileStreamUrl = async ({ id, isLegacy, dataKey, contentType }: S throw new Error("Data key not loaded"); } - const res = await fetch(`/api/file/${id}/download`, { method: "HEAD" }); - if (!res.ok) { - throw new Error("Failed to get the file size"); - } - - const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10); - await prepareFileDecryption(id, { - isLegacy: !!isLegacy, - dataKey: dataKey.key, - encContentSize, - contentType, - }); + await prepareFileDecryption(id, { isLegacy: !!isLegacy, dataKey: dataKey.key, contentType }); return getDecryptedFileUrl(id); }; diff --git a/src/lib/features/filesystem/upload.ts b/src/lib/features/filesystem/upload.ts index 2117fd7..8b0c6d7 100644 --- a/src/lib/features/filesystem/upload.ts +++ b/src/lib/features/filesystem/upload.ts @@ -2,7 +2,7 @@ import { storeFileThumbnailCache } from "#lib/client/cache/index.js"; import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js"; import { invalidateDirectory } from "./directory"; -// Uploads the file and caches its thumbnail locally. Returns `false` if the upload is canceled. +// Uploads the file and caches it with its thumbnail locally. Returns `false` if the upload is canceled. export const uploadFile = async ( file: File, parentId: DirectoryId, diff --git a/src/lib/shared/format/stream.ts b/src/lib/shared/format/stream.ts index 07f96ea..8c2bda6 100644 --- a/src/lib/shared/format/stream.ts +++ b/src/lib/shared/format/stream.ts @@ -39,17 +39,22 @@ const createByteQueue = () => { }; // Decrypts the encrypted chunks covering the plaintext `range` (as returned by -// `getEncryptedRange`) into the plaintext of the range +// `getEncryptedRange`) into the plaintext of the range. `onChunk` receives each decrypted chunk +// as a whole, along with its index in the file. export const createDecryptionStream = ({ range, + firstChunkIndex = 0, chunkCount, dataKey, isLegacy, + onChunk, }: { range: ByteRange; + firstChunkIndex?: number; chunkCount: number; dataKey: CryptoKey; isLegacy: boolean; + onChunk?: (chunkIndex: number, chunk: Uint8Array) => void; }) => { const queue = createByteQueue(); let chunkIndex = 0; @@ -59,6 +64,8 @@ export const createDecryptionStream = ({ encryptedSize: number, ) => { const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey)); + onChunk?.(firstChunkIndex + chunkIndex, decrypted); + const isFirst = chunkIndex === 0; const isLast = chunkIndex === chunkCount - 1; chunkIndex++; diff --git a/src/lib/shared/serviceWorker.ts b/src/lib/shared/serviceWorker.ts index 0279e0a..a7afff5 100644 --- a/src/lib/shared/serviceWorker.ts +++ b/src/lib/shared/serviceWorker.ts @@ -3,7 +3,8 @@ export const DECRYPTED_FILE_URL_PREFIX = "/_internal/decryptedFile/"; export interface FileMetadata { isLegacy: boolean; dataKey: CryptoKey; - encContentSize: number; + // Fetched by the worker when needed + encContentSize?: number; contentType: string; } diff --git a/src/routes/(fullscreen)/settings/cache/File.svelte b/src/routes/(fullscreen)/settings/cache/File.svelte index 501c6db..cb38ab4 100644 --- a/src/routes/(fullscreen)/settings/cache/File.svelte +++ b/src/routes/(fullscreen)/settings/cache/File.svelte @@ -34,6 +34,9 @@ {/if}

읽음 {formatDate(index.lastRetrievedAt)} · {formatFileSize(index.size)} + {#if !index.isComplete} + · 일부 + {/if}