From 3e52fa7a7c9bfbe006a7f63412f8190e83d7a800 Mon Sep 17 00:00:00 2001 From: static Date: Mon, 12 Oct 2026 00:48:21 +0900 Subject: [PATCH] =?UTF-8?q?=ED=8C=8C=EC=9D=BC=EC=9D=84=20=EB=8B=A4?= =?UTF-8?q?=EC=9A=B4=EB=A1=9C=EB=93=9C=ED=95=A0=20=EB=95=8C=20=EC=A0=84?= =?UTF-8?q?=EC=B2=B4=20=ED=8C=8C=EC=9D=BC=EC=9D=84=20=EB=A9=94=EB=AA=A8?= =?UTF-8?q?=EB=A6=AC=EC=97=90=20=EC=98=AC=EB=A6=AC=EB=8A=94=20=EB=8C=80?= =?UTF-8?q?=EC=8B=A0=20=EC=84=9C=EB=B9=84=EC=8A=A4=20=EC=9B=8C=EC=BB=A4?= =?UTF-8?q?=EC=97=90=EC=84=9C=20=EC=8A=A4=ED=8A=B8=EB=A6=AC=EB=B0=8D?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EB=B3=B5=ED=98=B8=ED=99=94=ED=95=98?= =?UTF-8?q?=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- eslint.config.js | 3 +- src/hooks.client.ts | 7 +- src/lib/client/cache/fileCache.ts | 60 ++++---- src/lib/client/media/exif.ts | 20 +++ src/lib/client/media/thumbnail.ts | 41 +++--- src/lib/client/serviceWorker.ts | 37 +++-- .../client/storage/indexedDB/cacheIndex.ts | 8 ++ src/lib/client/storage/opfs.ts | 6 + src/lib/client/transfer/chunkedUpload.ts | 54 +------ src/lib/client/transfer/download.svelte.ts | 115 ++++----------- src/lib/client/transfer/upload.svelte.ts | 12 +- src/lib/features/filesystem/content.ts | 56 +++++--- src/lib/features/filesystem/upload.ts | 7 +- src/lib/shared/format/index.ts | 1 + src/lib/shared/format/stream.ts | 96 +++++++++++++ src/lib/shared/serviceWorker.ts | 12 +- src/lib/utils/index.ts | 1 + src/lib/utils/speedMeter.ts | 53 +++++++ .../(fullscreen)/file/[id]/+page.svelte | 106 ++++++-------- .../file/[id]/DownloadStatus.svelte | 18 +-- .../(fullscreen)/file/[id]/TopBarMenu.svelte | 26 +--- .../(fullscreen)/file/downloads/File.svelte | 23 +-- .../(fullscreen)/settings/cache/+page.svelte | 2 +- .../settings/thumbnail/+page.svelte | 1 - .../settings/thumbnail/service.ts | 35 ++--- src/service-worker/handlers/decryptFile.ts | 135 +++++++----------- src/service-worker/modules/clients.ts | 10 ++ src/service-worker/modules/progress.ts | 48 +++++++ tests/unit/decryptionStream.test.ts | 103 +++++++++++++ 29 files changed, 629 insertions(+), 467 deletions(-) create mode 100644 src/lib/shared/format/stream.ts create mode 100644 src/lib/utils/speedMeter.ts create mode 100644 src/service-worker/modules/clients.ts create mode 100644 src/service-worker/modules/progress.ts create mode 100644 tests/unit/decryptionStream.test.ts diff --git a/eslint.config.js b/eslint.config.js index 1994a63..4974f25 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -60,7 +60,8 @@ export default defineConfig( // Domain-agnostic components "src/lib/ui/**": "^(#lib/(server|client|features)/|#trpc/)", "src/lib/features/**": "^#lib/server/", - "src/service-worker/**": "^(#lib/(?!shared/|client/storage/opfs\\.js$)|#trpc/|#workers/)", + "src/service-worker/**": + "^(#lib/(?!shared/|client/storage/|client/cache/fileCache\\.js$)|#trpc/|#workers/)", }), { files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js"], diff --git a/src/hooks.client.ts b/src/hooks.client.ts index 79f2309..3743e9d 100644 --- a/src/hooks.client.ts +++ b/src/hooks.client.ts @@ -1,7 +1,6 @@ import type { ClientInit } from "@sveltejs/kit/hooks"; -import { prepareFileCache } from "#lib/client/cache/index.js"; import { keyring } from "#lib/client/keys/index.js"; -import { listenForMetadataRequests } from "#lib/client/serviceWorker.js"; +import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js"; import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js"; const requestPersistentStorage = async () => { @@ -14,9 +13,9 @@ const requestPersistentStorage = async () => { }; export const init: ClientInit = async () => { - listenForMetadataRequests(); + listenForServiceWorkerMessages(); - await Promise.all([requestPersistentStorage(), prepareFileCache(), keyring.load()]); + await Promise.all([requestPersistentStorage(), keyring.load()]); cleanupDanglingInfos(); // Intended }; diff --git a/src/lib/client/cache/fileCache.ts b/src/lib/client/cache/fileCache.ts index 487baa9..33a5a34 100644 --- a/src/lib/client/cache/fileCache.ts +++ b/src/lib/client/cache/fileCache.ts @@ -1,50 +1,50 @@ import { - getFileCacheIndex as getFileCacheIndexFromIndexedDB, - storeFileCacheIndex, deleteFileCacheIndex, - type FileCacheIndex, -} from "#lib/client/storage/indexedDB/index.js"; -import { readFile, writeFile, deleteFile } from "#lib/client/storage/opfs.js"; + getFileCacheIndex as getAllFileCacheIndexes, + getFileCacheIndexOf, + storeFileCacheIndex, + updateFileCacheRetrievedAt, +} from "#lib/client/storage/indexedDB/cacheIndex.js"; +import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js"; -const fileCacheIndex = new Map(); +// Used by both the pages and the service worker, so the index is always read from IndexedDB -export const prepareFileCache = async () => { - for (const cache of await getFileCacheIndexFromIndexedDB()) { - fileCacheIndex.set(cache.fileId, cache); - } -}; +const getPath = (fileId: number) => `/cache/${fileId}`; -export const getFileCacheIndex = () => { - return Array.from(fileCacheIndex.values()); +export const getFileCacheIndex = async () => { + return await getAllFileCacheIndexes(); }; +// The index is written only after the whole content is written export const getFileCache = async (fileId: number) => { - const cacheIndex = fileCacheIndex.get(fileId); - if (!cacheIndex) return null; + if (!(await getFileCacheIndexOf(fileId))) return null; - cacheIndex.lastRetrievedAt = new Date(); - storeFileCacheIndex(cacheIndex); // Intended - return await readFile(`/cache/${fileId}`); + const file = await getFile(getPath(fileId)); + if (file) { + void updateFileCacheRetrievedAt(fileId, new Date()); + } + return file; }; -export const storeFileCache = async (fileId: number, fileBuffer: ArrayBuffer) => { - const now = new Date(); - await writeFile(`/cache/${fileId}`, fileBuffer); +export const storeFileCache = async (fileId: number, stream: ReadableStream) => { + const path = getPath(fileId); + try { + await stream.pipeTo(await createWritable(path)); + } catch (e) { + await deleteFile(path).catch(() => {}); + throw e; + } - const cacheIndex: FileCacheIndex = { + const now = new Date(); + await storeFileCacheIndex({ fileId, cachedAt: now, lastRetrievedAt: now, - size: fileBuffer.byteLength, - }; - fileCacheIndex.set(fileId, cacheIndex); - await storeFileCacheIndex(cacheIndex); + size: (await getFile(path))?.size ?? 0, + }); }; export const deleteFileCache = async (fileId: number) => { - if (!fileCacheIndex.has(fileId)) return; - - fileCacheIndex.delete(fileId); - await deleteFile(`/cache/${fileId}`); await deleteFileCacheIndex(fileId); + await deleteFile(getPath(fileId)); }; diff --git a/src/lib/client/media/exif.ts b/src/lib/client/media/exif.ts index f6b363f..86f7f6a 100644 --- a/src/lib/client/media/exif.ts +++ b/src/lib/client/media/exif.ts @@ -25,3 +25,23 @@ export const extractExifDateTime = (fileBuffer: ArrayBuffer) => { const offsetMs = offsetSign * ((offsetHour ?? 0) * 60 + (offsetMinute ?? 0)) * 60 * 1000; return new Date(utcDate - offsetMs); }; + +const EXIF_SEARCH_SIZE = 1024 * 1024; // 1 MiB +const MAX_FULL_READ_SIZE = 64 * 1024 * 1024; // 64 MiB + +const tryExtractExifDateTime = (buffer: ArrayBuffer) => { + try { + return extractExifDateTime(buffer); + } catch { + return undefined; + } +}; + +// EXIF is usually at the beginning of the file, so the whole file is read only if not found +export const readExifDateTime = async (file: Blob) => { + const dateTime = tryExtractExifDateTime(await file.slice(0, EXIF_SEARCH_SIZE).arrayBuffer()); + if (dateTime || file.size <= EXIF_SEARCH_SIZE || file.size > MAX_FULL_READ_SIZE) { + return dateTime; + } + return tryExtractExifDateTime(await file.arrayBuffer()); +}; diff --git a/src/lib/client/media/thumbnail.ts b/src/lib/client/media/thumbnail.ts index c043267..7b409a9 100644 --- a/src/lib/client/media/thumbnail.ts +++ b/src/lib/client/media/thumbnail.ts @@ -82,35 +82,42 @@ const generateVideoThumbnail = (videoUrl: string, time = 0) => { }); }; -export const generateThumbnail = async (blob: Blob) => { - let url; +const convertHeicToPng = async (blob: Blob) => { + const { default: heic2any } = await import("heic2any"); + return (await heic2any({ blob, toType: "image/png" })) as Blob; +}; + +// Generates a thumbnail of the image or video at the URL. Returns `null` if not supported. +export const generateThumbnailFromUrl = async (url: string, contentType: string) => { try { - if (blob.type.startsWith("image/")) { - url = URL.createObjectURL(blob); + if (contentType.startsWith("image/")) { try { return await generateImageThumbnail(url); } catch { - URL.revokeObjectURL(url); - url = undefined; + if (contentType !== "image/heic") return null; - if (blob.type === "image/heic") { - const { default: heic2any } = await import("heic2any"); - url = URL.createObjectURL((await heic2any({ blob, toType: "image/png" })) as Blob); - return await generateImageThumbnail(url); - } else { - return null; + // Chromium can't decode HEIC images + const png = URL.createObjectURL(await convertHeicToPng(await (await fetch(url)).blob())); + try { + return await generateImageThumbnail(png); + } finally { + URL.revokeObjectURL(png); } } - } else if (blob.type.startsWith("video/")) { - url = URL.createObjectURL(blob); + } else if (contentType.startsWith("video/")) { return await generateVideoThumbnail(url); } return null; } catch { return null; + } +}; + +export const generateThumbnail = async (blob: Blob) => { + const url = URL.createObjectURL(blob); + try { + return await generateThumbnailFromUrl(url, blob.type); } finally { - if (url) { - URL.revokeObjectURL(url); - } + URL.revokeObjectURL(url); } }; diff --git a/src/lib/client/serviceWorker.ts b/src/lib/client/serviceWorker.ts index e4b86b6..1343358 100644 --- a/src/lib/client/serviceWorker.ts +++ b/src/lib/client/serviceWorker.ts @@ -1,8 +1,9 @@ -import { DECRYPTED_FILE_URL_PREFIX } from "#lib/shared/serviceWorker.js"; -import type { - FileMetadata, - ServiceWorkerMessage, - ServiceWorkerResponse, +import { handleDownloadProgress } from "#lib/client/transfer/download.svelte.js"; +import { + DECRYPTED_FILE_URL_PREFIX, + type FileMetadata, + type ServiceWorkerMessage, + type ServiceWorkerResponse, } from "#lib/shared/serviceWorker.js"; const PREPARE_TIMEOUT_MS = 5000; @@ -10,17 +11,22 @@ const PREPARE_TIMEOUT_MS = 5000; // The service worker may be terminated at any time, losing its in-memory metadata. // Keep a copy here so that it can be re-requested on demand. const preparedFiles = new Map(); -let isListeningForMetadataRequests = false; +let isListening = false; -export const listenForMetadataRequests = () => { - if (isListeningForMetadataRequests) return; - isListeningForMetadataRequests = true; +export const listenForServiceWorkerMessages = () => { + if (isListening) return; + isListening = true; navigator.serviceWorker.addEventListener( "message", (event: MessageEvent) => { - if (event.data.type === "decryption-metadata-request") { - event.ports[0]?.postMessage(preparedFiles.get(event.data.fileId) ?? null); + switch (event.data.type) { + case "decryption-metadata-request": + event.ports[0]?.postMessage(preparedFiles.get(event.data.fileId) ?? null); + break; + case "download-progress": + handleDownloadProgress(event.data); + break; } }, ); @@ -37,7 +43,7 @@ const getServiceWorker = async () => { export const prepareFileDecryption = async (id: number, metadata: FileMetadata) => { preparedFiles.set(id, metadata); - listenForMetadataRequests(); + listenForServiceWorkerMessages(); const sw = await getServiceWorker(); return new Promise((resolve, reject) => { @@ -63,3 +69,10 @@ export const prepareFileDecryption = async (id: number, metadata: FileMetadata) }; export const getDecryptedFileUrl = (id: number) => `${DECRYPTED_FILE_URL_PREFIX}${id}`; + +// Lets the service worker download, decrypt, and save the file +export const saveDecryptedFile = (id: number, filename: string) => { + const link = document.createElement("a"); + link.href = `${getDecryptedFileUrl(id)}?download=${encodeURIComponent(filename)}`; + link.click(); +}; diff --git a/src/lib/client/storage/indexedDB/cacheIndex.ts b/src/lib/client/storage/indexedDB/cacheIndex.ts index 1602c8c..b3b2516 100644 --- a/src/lib/client/storage/indexedDB/cacheIndex.ts +++ b/src/lib/client/storage/indexedDB/cacheIndex.ts @@ -28,6 +28,14 @@ export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => { }); }; +export const getFileCacheIndexOf = async (fileId: number) => { + return await cacheIndex.fileCache.get(fileId); +}; + +export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => { + await cacheIndex.fileCache.update(fileId, { lastRetrievedAt }); +}; + export const deleteFileCacheIndex = async (fileId: number) => { await cacheIndex.fileCache.delete(fileId); }; diff --git a/src/lib/client/storage/opfs.ts b/src/lib/client/storage/opfs.ts index a367aae..5fd6f1a 100644 --- a/src/lib/client/storage/opfs.ts +++ b/src/lib/client/storage/opfs.ts @@ -48,6 +48,12 @@ export const writeFile = async (path: string, data: ArrayBuffer) => { } }; +// Changes are applied atomically when the stream is closed, and discarded if it is aborted +export const createWritable = async (path: string) => { + const { fileHandle } = await getFileHandle(path); + return await fileHandle!.createWritable(); +}; + export const deleteFile = async (path: string) => { const { parentHandle, filename } = await getFileHandle(path, false); if (!parentHandle) return; diff --git a/src/lib/client/transfer/chunkedUpload.ts b/src/lib/client/transfer/chunkedUpload.ts index 8e96360..f3a7175 100644 --- a/src/lib/client/transfer/chunkedUpload.ts +++ b/src/lib/client/transfer/chunkedUpload.ts @@ -4,6 +4,7 @@ import pLimit from "p-limit"; import { encodeToBase64, encryptChunk } from "#lib/shared/crypto/index.js"; import { CHUNK_SIZE, getChunkCount, getEncryptedSize } from "#lib/shared/format/index.js"; import { BoundedQueue } from "#lib/utils/concurrency/index.js"; +import { createSpeedMeter } from "#lib/utils/speedMeter.js"; interface UploadStats { progress: number; @@ -15,59 +16,6 @@ interface EncryptedChunk { data: ArrayBuffer; } -const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => { - const samples: { t: number; b: number }[] = []; - let lastSpeed = 0; - let startTime: number | null = null; - - return (bytesNow?: number) => { - if (bytesNow === undefined) return lastSpeed; - - const now = performance.now(); - - // Initialize start time on first call - if (startTime === null) { - startTime = now; - } - - // Check if enough time has passed since the last sample - const lastSample = samples[samples.length - 1]; - if (lastSample && now - lastSample.t < minInterval) { - return lastSpeed; - } - - samples.push({ t: now, b: bytesNow }); - - // Remove old samples outside the time window - const cutoff = now - timeWindow; - while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift(); - - // Need at least 2 samples to calculate speed - if (samples.length < 2) { - return lastSpeed; - } - - const first = samples[0]!; - const dt = now - first.t; - const db = bytesNow - first.b; - - if (dt >= minInterval) { - const instantSpeed = (db / dt) * 1000; - // Apply EMA for smoother speed transitions - const alpha = 0.3; - const rawSpeed = - lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed; - - // Apply warmup ramp to prevent initial overestimation - const elapsed = now - startTime; - const warmupWeight = Math.min(1, elapsed / warmupPeriod); - lastSpeed = rawSpeed * warmupWeight; - } - - return lastSpeed; - }; -}; - const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise => { return await encryptChunk(await chunk.arrayBuffer(), dataKey); }; diff --git a/src/lib/client/transfer/download.svelte.ts b/src/lib/client/transfer/download.svelte.ts index f5fe935..ea163ff 100644 --- a/src/lib/client/transfer/download.svelte.ts +++ b/src/lib/client/transfer/download.svelte.ts @@ -1,34 +1,21 @@ -import axios from "axios"; -import { limitFunction } from "p-limit"; -import { decryptChunk, concatenateBuffers } from "#lib/shared/crypto/index.js"; -import { ENCRYPTED_CHUNK_SIZE } from "#lib/shared/format/index.js"; +import type { DownloadProgressMessage } from "#lib/shared/serviceWorker.js"; +import { createSpeedMeter } from "#lib/utils/speedMeter.js"; + +// Files are downloaded and decrypted by the service worker, which reports the progress export interface FileDownloadState { id: number; - status: - | "download-pending" - | "downloading" - | "decryption-pending" - | "decrypting" - | "decrypted" - | "canceled" - | "error"; + status: "downloading" | "completed" | "error"; progress?: number; rate?: number; - estimated?: number; - result?: ArrayBuffer; } -type LiveFileDownloadState = FileDownloadState & { - status: "download-pending" | "downloading" | "decryption-pending" | "decrypting"; -}; - let downloadingFiles: FileDownloadState[] = $state([]); +const speedMeters = new WeakMap>(); -export const isFileDownloading = ( - status: FileDownloadState["status"], -): status is LiveFileDownloadState["status"] => - ["download-pending", "downloading", "decryption-pending", "decrypting"].includes(status); +export const isFileDownloading = (status: FileDownloadState["status"]) => { + return status === "downloading"; +}; export const getFileDownloadState = (fileId: number) => { return downloadingFiles.find((file) => file.id === fileId && isFileDownloading(file.status)); @@ -42,69 +29,27 @@ export const clearDownloadedFiles = () => { downloadingFiles = downloadingFiles.filter((file) => isFileDownloading(file.status)); }; -const requestFileDownload = limitFunction( - async (state: FileDownloadState, id: number) => { - state.status = "downloading"; +export const handleDownloadProgress = ({ + fileId, + status, + loaded, + total, +}: DownloadProgressMessage) => { + let state = getFileDownloadState(fileId); + if (!state) { + if (status !== "downloading") return; - const res = await axios.get(`/api/file/${id}/download`, { - responseType: "arraybuffer", - onDownloadProgress: ({ progress, rate, estimated }) => { - state.progress = progress; - state.rate = rate; - state.estimated = estimated; - }, - }); - const fileEncrypted: ArrayBuffer = res.data; - - state.status = "decryption-pending"; - return fileEncrypted; - }, - { concurrency: 1 }, -); - -const decryptFile = limitFunction( - async ( - state: FileDownloadState, - fileEncrypted: ArrayBuffer, - encryptedChunkSize: number, - dataKey: CryptoKey, - ) => { - state.status = "decrypting"; - - const chunks: ArrayBuffer[] = []; - let offset = 0; - - while (offset < fileEncrypted.byteLength) { - const nextOffset = Math.min(offset + encryptedChunkSize, fileEncrypted.byteLength); - chunks.push(await decryptChunk(fileEncrypted.slice(offset, nextOffset), dataKey)); - offset = nextOffset; - } - - const fileBuffer = concatenateBuffers(...chunks).buffer; - state.status = "decrypted"; - state.result = fileBuffer; - return fileBuffer; - }, - { concurrency: 4 }, -); - -export const downloadFile = async (id: number, dataKey: CryptoKey, isLegacy: boolean) => { - downloadingFiles.push({ - id, - status: "download-pending", - }); - const state = downloadingFiles.at(-1)!; - - try { - const fileEncrypted = await requestFileDownload(state, id); - return await decryptFile( - state, - fileEncrypted, - isLegacy ? fileEncrypted.byteLength : ENCRYPTED_CHUNK_SIZE, - dataKey, - ); - } catch (e) { - state.status = "error"; - throw e; + downloadingFiles.push({ id: fileId, status }); + state = downloadingFiles.at(-1)!; + speedMeters.set(state, createSpeedMeter()); } + + if (status === "canceled") { + downloadingFiles = downloadingFiles.filter((file) => file !== state); + return; + } + + state.status = status; + state.progress = total > 0 ? loaded / total : 1; + state.rate = speedMeters.get(state)?.(loaded); }; diff --git a/src/lib/client/transfer/upload.svelte.ts b/src/lib/client/transfer/upload.svelte.ts index 0f9e005..7702e6b 100644 --- a/src/lib/client/transfer/upload.svelte.ts +++ b/src/lib/client/transfer/upload.svelte.ts @@ -6,7 +6,7 @@ import { type HmacSecret, type MasterKey, } from "#lib/client/keys/index.js"; -import { extractExifDateTime } from "#lib/client/media/exif.js"; +import { readExifDateTime } from "#lib/client/media/exif.js"; import { generateThumbnail } from "#lib/client/media/thumbnail.js"; import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js"; import { @@ -41,9 +41,7 @@ export type LiveFileUploadState = FileUploadState & { status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading"; }; -const scheduler = new Scheduler< - { fileId: number; fileBuffer?: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined ->(); +const scheduler = new Scheduler<{ fileId: number; thumbnailBuffer?: ArrayBuffer } | undefined>(); let uploadingFiles: FileUploadState[] = $state([]); const isFileUploading = (status: FileUploadState["status"]) => @@ -213,7 +211,6 @@ export const uploadFile = async ( return; } - let fileBuffer; const fileType = getFileType(file); const fileMetadata: FileMetadata = { parentId, @@ -222,8 +219,7 @@ export const uploadFile = async ( }; if (fileType.startsWith("image/")) { - fileBuffer = await file.arrayBuffer(); - fileMetadata.createdAt = extractExifDateTime(fileBuffer); + fileMetadata.createdAt = await readExifDateTime(file); } const blob = new Blob([file], { type: fileType }); @@ -241,7 +237,7 @@ export const uploadFile = async ( dataKeyVersion, ); - return { fileId, fileBuffer, thumbnailBuffer }; + return { fileId, thumbnailBuffer }; } catch (e) { state.status = "error"; throw e; diff --git a/src/lib/features/filesystem/content.ts b/src/lib/features/filesystem/content.ts index decc283..5e34ad6 100644 --- a/src/lib/features/filesystem/content.ts +++ b/src/lib/features/filesystem/content.ts @@ -1,41 +1,57 @@ import { deleteFileCache, deleteFileThumbnailCache, - getFileCache, - storeFileCache, storeFileThumbnailCache, } from "#lib/client/cache/index.js"; -import { getDecryptedFileUrl, prepareFileDecryption } from "#lib/client/serviceWorker.js"; +import { + getDecryptedFileUrl, + prepareFileDecryption, + saveDecryptedFile, +} from "#lib/client/serviceWorker.js"; import { getAllFileInfos } from "#lib/client/storage/indexedDB/index.js"; import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js"; -import { downloadFile } from "#lib/client/transfer/index.js"; import { trpc } from "#trpc/client.js"; import type { DataKey } from "./types"; -// Returns the decrypted content of the file, from the cache if available -export const getFileContent = async (fileId: number, dataKey: CryptoKey, isLegacy: boolean) => { - const cache = await getFileCache(fileId); - if (cache) return cache; +interface StreamableFile { + id: number; + isLegacy?: boolean; + dataKey?: DataKey; + contentType: string; +} - const fileBuffer = await downloadFile(fileId, dataKey, isLegacy); - void storeFileCache(fileId, fileBuffer); - return fileBuffer; -}; +// Returns a URL from which the service worker streams the decrypted content of the file +export const getFileStreamUrl = async ({ id, isLegacy, dataKey, contentType }: StreamableFile) => { + if (!dataKey) { + throw new Error("Data key not loaded"); + } -// Returns a URL from which the service worker streams the decrypted file -export const getFileStreamUrl = async (fileId: number, dataKey: CryptoKey, contentType: string) => { - const res = await fetch(`/api/file/${fileId}/download`, { method: "HEAD" }); + const res = await fetch(`/api/file/${id}/download`, { method: "HEAD" }); if (!res.ok) { throw new Error("Failed to get the file size"); } const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10); - if (encContentSize <= 0) { - throw new Error("Invalid file size"); - } + await prepareFileDecryption(id, { + isLegacy: !!isLegacy, + dataKey: dataKey.key, + encContentSize, + contentType, + }); + return getDecryptedFileUrl(id); +}; - await prepareFileDecryption(fileId, { isLegacy: false, dataKey, encContentSize, contentType }); - return getDecryptedFileUrl(fileId); +export const getFileBlob = async (file: StreamableFile) => { + const res = await fetch(await getFileStreamUrl(file)); + if (!res.ok) { + throw new Error("Failed to download the file"); + } + return await res.blob(); +}; + +export const saveFile = async (file: StreamableFile & { name: string }) => { + await getFileStreamUrl(file); + saveDecryptedFile(file.id, file.name); }; export const uploadFileThumbnail = async (fileId: number, thumbnail: Blob, dataKey: DataKey) => { diff --git a/src/lib/features/filesystem/upload.ts b/src/lib/features/filesystem/upload.ts index abfb7b2..2117fd7 100644 --- a/src/lib/features/filesystem/upload.ts +++ b/src/lib/features/filesystem/upload.ts @@ -1,8 +1,8 @@ -import { storeFileCache, storeFileThumbnailCache } from "#lib/client/cache/index.js"; +import { storeFileThumbnailCache } from "#lib/client/cache/index.js"; import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js"; import { invalidateDirectory } from "./directory"; -// Uploads the file and caches it locally. Returns `false` if the upload is canceled. +// Uploads the file and caches its thumbnail locally. Returns `false` if the upload is canceled. export const uploadFile = async ( file: File, parentId: DirectoryId, @@ -12,9 +12,6 @@ export const uploadFile = async ( if (!res) return false; invalidateDirectory(parentId); - if (res.fileBuffer) { - void storeFileCache(res.fileId, res.fileBuffer); - } if (res.thumbnailBuffer) { void storeFileThumbnailCache(res.fileId, res.thumbnailBuffer); } diff --git a/src/lib/shared/format/index.ts b/src/lib/shared/format/index.ts index 2e71d6c..2200e8a 100644 --- a/src/lib/shared/format/index.ts +++ b/src/lib/shared/format/index.ts @@ -1 +1,2 @@ export * from "./chunk"; +export * from "./stream"; diff --git a/src/lib/shared/format/stream.ts b/src/lib/shared/format/stream.ts new file mode 100644 index 0000000..07f96ea --- /dev/null +++ b/src/lib/shared/format/stream.ts @@ -0,0 +1,96 @@ +import { decryptChunk } from "#lib/shared/crypto/index.js"; +import type { ByteRange } from "#lib/shared/http.js"; +import { CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "./chunk"; + +// Accumulates stream pieces and takes them out in fixed sizes without copying them repeatedly +const createByteQueue = () => { + let pieces: Uint8Array[] = []; + let size = 0; + + return { + get size() { + return size; + }, + push(piece: Uint8Array) { + pieces.push(piece); + size += piece.byteLength; + }, + take(length: number) { + const result = new Uint8Array(length); + let offset = 0; + while (offset < length) { + const piece = pieces[0]!; + const count = Math.min(piece.byteLength, length - offset); + result.set(piece.subarray(0, count), offset); + offset += count; + if (count === piece.byteLength) { + pieces.shift(); + } else { + pieces[0] = piece.subarray(count); + } + } + size -= length; + if (size === 0) { + pieces = []; + } + return result; + }, + }; +}; + +// Decrypts the encrypted chunks covering the plaintext `range` (as returned by +// `getEncryptedRange`) into the plaintext of the range +export const createDecryptionStream = ({ + range, + chunkCount, + dataKey, + isLegacy, +}: { + range: ByteRange; + chunkCount: number; + dataKey: CryptoKey; + isLegacy: boolean; +}) => { + const queue = createByteQueue(); + let chunkIndex = 0; + + const decryptNextChunk = async ( + controller: TransformStreamDefaultController, + encryptedSize: number, + ) => { + const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey)); + const isFirst = chunkIndex === 0; + const isLast = chunkIndex === chunkCount - 1; + chunkIndex++; + + if (isLegacy) { + controller.enqueue(decrypted.subarray(range.start, range.end + 1)); + } else { + controller.enqueue( + decrypted.subarray( + isFirst ? range.start % CHUNK_SIZE : 0, + isLast ? (range.end % CHUNK_SIZE) + 1 : decrypted.byteLength, + ), + ); + } + }; + + return new TransformStream({ + async transform(piece, controller) { + queue.push(piece); + + // The last chunk may be shorter, so it is decrypted when the stream ends + while (!isLegacy && queue.size >= ENCRYPTED_CHUNK_SIZE && chunkIndex < chunkCount - 1) { + await decryptNextChunk(controller, ENCRYPTED_CHUNK_SIZE); + } + }, + async flush(controller) { + if (chunkIndex === chunkCount - 1 && queue.size > 0) { + await decryptNextChunk(controller, queue.size); + } + if (chunkIndex !== chunkCount || queue.size > 0) { + throw new Error("Unexpected size of the encrypted content"); + } + }, + }); +}; diff --git a/src/lib/shared/serviceWorker.ts b/src/lib/shared/serviceWorker.ts index 4a668de..0279e0a 100644 --- a/src/lib/shared/serviceWorker.ts +++ b/src/lib/shared/serviceWorker.ts @@ -23,5 +23,15 @@ export interface DecryptionMetadataRequestMessage { fileId: number; } +// Progress of the whole content of a file being streamed from the server +export interface DownloadProgressMessage { + type: "download-progress"; + fileId: number; + status: "downloading" | "completed" | "canceled" | "error"; + loaded: number; + total: number; +} + export type ServiceWorkerMessage = DecryptionPrepareMessage; -export type ServiceWorkerResponse = DecryptionReadyMessage | DecryptionMetadataRequestMessage; +export type ServiceWorkerResponse = + DecryptionReadyMessage | DecryptionMetadataRequestMessage | DownloadProgressMessage; diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index f860017..e830fde 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -3,3 +3,4 @@ export * from "./format"; export * from "./gotoStateful"; export * from "./search"; export * from "./sort"; +export * from "./speedMeter"; diff --git a/src/lib/utils/speedMeter.ts b/src/lib/utils/speedMeter.ts new file mode 100644 index 0000000..0c46a39 --- /dev/null +++ b/src/lib/utils/speedMeter.ts @@ -0,0 +1,53 @@ +// Estimates the transfer rate in bytes per second from the total transferred bytes +export const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => { + const samples: { t: number; b: number }[] = []; + let lastSpeed = 0; + let startTime: number | null = null; + + return (bytesNow?: number) => { + if (bytesNow === undefined) return lastSpeed; + + const now = performance.now(); + + // Initialize start time on first call + if (startTime === null) { + startTime = now; + } + + // Check if enough time has passed since the last sample + const lastSample = samples[samples.length - 1]; + if (lastSample && now - lastSample.t < minInterval) { + return lastSpeed; + } + + samples.push({ t: now, b: bytesNow }); + + // Remove old samples outside the time window + const cutoff = now - timeWindow; + while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift(); + + // Need at least 2 samples to calculate speed + if (samples.length < 2) { + return lastSpeed; + } + + const first = samples[0]!; + const dt = now - first.t; + const db = bytesNow - first.b; + + if (dt >= minInterval) { + const instantSpeed = (db / dt) * 1000; + // Apply EMA for smoother speed transitions + const alpha = 0.3; + const rawSpeed = + lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed; + + // Apply warmup ramp to prevent initial overestimation + const elapsed = now - startTime; + const warmupWeight = Math.min(1, elapsed / warmupPeriod); + lastSpeed = rawSpeed * warmupWeight; + } + + return lastSpeed; + }; +}; diff --git a/src/routes/(fullscreen)/file/[id]/+page.svelte b/src/routes/(fullscreen)/file/[id]/+page.svelte index 1afd429..9a2c318 100644 --- a/src/routes/(fullscreen)/file/[id]/+page.svelte +++ b/src/routes/(fullscreen)/file/[id]/+page.svelte @@ -1,15 +1,14 @@ @@ -172,9 +153,7 @@ ) ? info?.parentId : undefined} - {fileBlob} - downloadUrl={videoStreamUrl} - filename={info?.name} + onDownloadClick={fileUrl ? saveFile : undefined} isFavorite={info?.isFavorite} onToggleFavorite={toggleFavorite} /> @@ -192,16 +171,15 @@ {/snippet} {#if viewerType === "image"} - {#if fileBlobUrl} - {info.name} + {#if fileUrl} + {info.name} {:else} {@render viewerLoading("이미지를 불러오고 있어요.")} {/if} {:else if viewerType === "video"} - {#if videoStreamUrl || fileBlobUrl} + {#if fileUrl}
- + updateThumbnail(info!.dataKey!)} diff --git a/src/routes/(fullscreen)/file/[id]/DownloadStatus.svelte b/src/routes/(fullscreen)/file/[id]/DownloadStatus.svelte index e33139c..3bde890 100644 --- a/src/routes/(fullscreen)/file/[id]/DownloadStatus.svelte +++ b/src/routes/(fullscreen)/file/[id]/DownloadStatus.svelte @@ -11,22 +11,10 @@ {#if isFileDownloading(state.status)}
-

- {#if state.status === "download-pending"} - 다운로드를 기다리는 중 - {:else if state.status === "downloading"} - 다운로드하는 중 - {:else if state.status === "decryption-pending"} - 복호화를 기다리는 중 - {:else if state.status === "decrypting"} - 복호화하는 중 - {/if} -

+

다운로드하는 중

- {#if state.status === "downloading"} - 전송됨 - {Math.floor((state.progress ?? 0) * 100)}% · {formatNetworkSpeed((state.rate ?? 0) * 8)} - {/if} + 전송됨 + {Math.floor((state.progress ?? 0) * 100)}% · {formatNetworkSpeed((state.rate ?? 0) * 8)}

{/if} diff --git a/src/routes/(fullscreen)/file/[id]/TopBarMenu.svelte b/src/routes/(fullscreen)/file/[id]/TopBarMenu.svelte index 2479847..4383cbf 100644 --- a/src/routes/(fullscreen)/file/[id]/TopBarMenu.svelte +++ b/src/routes/(fullscreen)/file/[id]/TopBarMenu.svelte @@ -1,5 +1,4 @@ (isOpen = false)} /> -{#if isOpen && (directoryId || downloadUrl || fileBlob)} +{#if isOpen && (directoryId || onDownloadClick || isFavorite !== undefined)}
diff --git a/src/routes/(fullscreen)/file/downloads/File.svelte b/src/routes/(fullscreen)/file/downloads/File.svelte index f5494af..a6ff37f 100644 --- a/src/routes/(fullscreen)/file/downloads/File.svelte +++ b/src/routes/(fullscreen)/file/downloads/File.svelte @@ -3,10 +3,7 @@ import type { SummarizedFileInfo } from "#lib/features/filesystem/index.js"; import { formatNetworkSpeed } from "#lib/utils/index.js"; - import IconCloud from "~icons/material-symbols/cloud"; import IconCloudDownload from "~icons/material-symbols/cloud-download"; - import IconLock from "~icons/material-symbols/lock"; - import IconLockClock from "~icons/material-symbols/lock-clock"; import IconCheckCircle from "~icons/material-symbols/check-circle"; import IconError from "~icons/material-symbols/error"; @@ -20,15 +17,9 @@
- {#if state.status === "download-pending"} - - {:else if state.status === "downloading"} + {#if state.status === "downloading"} - {:else if state.status === "decryption-pending"} - - {:else if state.status === "decrypting"} - - {:else if state.status === "decrypted"} + {:else if state.status === "completed"} {:else if state.status === "error"} @@ -39,17 +30,11 @@ {info.name}

- {#if state.status === "download-pending"} - 다운로드를 기다리는 중 - {:else if state.status === "downloading"} + {#if state.status === "downloading"} 전송됨 {Math.floor((state.progress ?? 0) * 100)}% · {formatNetworkSpeed((state.rate ?? 0) * 8)} - {:else if state.status === "decryption-pending"} - 복호화를 기다리는 중 - {:else if state.status === "decrypting"} - 복호화하는 중 - {:else if state.status === "decrypted"} + {:else if state.status === "completed"} 다운로드 완료 {:else if state.status === "error"} 다운로드 실패 diff --git a/src/routes/(fullscreen)/settings/cache/+page.svelte b/src/routes/(fullscreen)/settings/cache/+page.svelte index 13db941..3510924 100644 --- a/src/routes/(fullscreen)/settings/cache/+page.svelte +++ b/src/routes/(fullscreen)/settings/cache/+page.svelte @@ -26,7 +26,7 @@ }; onMount(async () => { - const indexes = getFileCacheIndex(); + const indexes = await getFileCacheIndex(); const infos = await bulkGetFileInfo(indexes.map(({ fileId }) => fileId)); fileCache = indexes .map((index) => ({ index, info: infos.get(index.fileId)! })) diff --git a/src/routes/(fullscreen)/settings/thumbnail/+page.svelte b/src/routes/(fullscreen)/settings/thumbnail/+page.svelte index e508d89..4b9b55f 100644 --- a/src/routes/(fullscreen)/settings/thumbnail/+page.svelte +++ b/src/routes/(fullscreen)/settings/thumbnail/+page.svelte @@ -21,7 +21,6 @@ const statusTexts = { queued: "대기 중", - "generation-pending": "준비 중", generating: "생성하는 중", uploading: "업로드하는 중", error: "실패", diff --git a/src/routes/(fullscreen)/settings/thumbnail/service.ts b/src/routes/(fullscreen)/settings/thumbnail/service.ts index d4393da..2affe6b 100644 --- a/src/routes/(fullscreen)/settings/thumbnail/service.ts +++ b/src/routes/(fullscreen)/settings/thumbnail/service.ts @@ -1,19 +1,16 @@ import { limitFunction } from "p-limit"; import { SvelteMap } from "svelte/reactivity"; -import { generateThumbnail } from "#lib/client/media/thumbnail.js"; +import { generateThumbnailFromUrl } from "#lib/client/media/thumbnail.js"; import { - getFileContent, getFileInfo, + getFileStreamUrl, uploadFileThumbnail, type FileInfo, } from "#lib/features/filesystem/index.js"; -import { Scheduler } from "#lib/utils/index.js"; import type { RouterOutputs } from "#trpc/router.server.js"; -export type GenerationStatus = - "queued" | "generation-pending" | "generating" | "uploading" | "uploaded" | "error"; +export type GenerationStatus = "queued" | "generating" | "uploading" | "uploaded" | "error"; -const scheduler = new Scheduler(); const statuses = new SvelteMap(); export const getThumbnailGenerationStatus = (fileId: number) => { @@ -37,13 +34,13 @@ export const requestMissingThumbnailFiles = async ( return files; }; +// The content is streamed by the service worker, so only the decoded media occupy the memory const generateAndUploadThumbnail = limitFunction( - async (fileInfo: FileInfo, fileBuffer: ArrayBuffer) => { + async (fileInfo: FileInfo) => { statuses.set(fileInfo.id, "generating"); - const thumbnail = await generateThumbnail( - new Blob([fileBuffer], { type: fileInfo.contentType }), - ); + const url = await getFileStreamUrl(fileInfo); + const thumbnail = await generateThumbnailFromUrl(url, fileInfo.contentType); if (!thumbnail) { throw new Error("Failed to generate thumbnail"); } @@ -57,23 +54,11 @@ const generateAndUploadThumbnail = limitFunction( export const requestThumbnailGeneration = async (fileInfo: FileInfo) => { const status = statuses.get(fileInfo.id); - if (status) { - if (status !== "error") return; - } else { - statuses.set(fileInfo.id, "queued"); - } + if (status && status !== "error") return; + statuses.set(fileInfo.id, "queued"); try { - let file: ArrayBuffer | undefined; - - await scheduler.schedule( - async () => { - statuses.set(fileInfo.id, "generation-pending"); - file = await getFileContent(fileInfo.id, fileInfo.dataKey!.key, !!fileInfo.isLegacy); - return file.byteLength; - }, - () => generateAndUploadThumbnail(fileInfo, file!), - ); + await generateAndUploadThumbnail(fileInfo); } catch (e) { console.error(e); statuses.set(fileInfo.id, "error"); diff --git a/src/service-worker/handlers/decryptFile.ts b/src/service-worker/handlers/decryptFile.ts index 96fe636..f41be7b 100644 --- a/src/service-worker/handlers/decryptFile.ts +++ b/src/service-worker/handlers/decryptFile.ts @@ -1,8 +1,6 @@ -import { getFile } from "#lib/client/storage/opfs.js"; -import { decryptChunk } from "#lib/shared/crypto/index.js"; +import { getFileCache, storeFileCache } from "#lib/client/cache/fileCache.js"; import { - CHUNK_SIZE, - ENCRYPTED_CHUNK_SIZE, + createDecryptionStream, getDecryptedSize, getEncryptedRange, } from "#lib/shared/format/index.js"; @@ -15,18 +13,25 @@ import { } from "#lib/shared/http.js"; import { DECRYPTED_FILE_URL_PREFIX, type FileMetadata } from "#lib/shared/serviceWorker.js"; import { getFileMetadata } from "../modules/metadata"; +import { trackDownloadProgress } from "../modules/progress"; + +interface StreamRequest { + fileId: number; + clientId: string; + rangeRequest?: RangeRequest; + downloadFilename?: string; +} const createResponse = ( stream: ReadableStream, - isRangeRequest: boolean, + { rangeRequest, downloadFilename }: StreamRequest, range: ByteRange, contentType?: string, - downloadFilename?: string, ) => { const headers: Record = { "Accept-Ranges": "bytes", "Content-Length": String(range.end - range.start + 1), - ...(isRangeRequest ? getContentRangeHeader(range) : {}), + ...(rangeRequest ? getContentRangeHeader(range) : {}), }; if (contentType) { @@ -39,7 +44,7 @@ const createResponse = ( } return new Response(stream, { - status: isRangeRequest ? 206 : 200, + status: rangeRequest ? 206 : 200, headers, }); }; @@ -51,102 +56,58 @@ const createRangeNotSatisfiableResponse = (total: number) => { }); }; -const streamFromOpfs = async ( - file: File, - metadata?: FileMetadata, - rangeRequest?: RangeRequest, - downloadFilename?: string, -) => { - const range = resolveRange(rangeRequest, file.size); +const streamFromCache = (request: StreamRequest, cache: File, metadata?: FileMetadata) => { + const range = resolveRange(request.rangeRequest, cache.size); if (!range) { - return createRangeNotSatisfiableResponse(file.size); + return createRangeNotSatisfiableResponse(cache.size); } return createResponse( - file.slice(range.start, range.end + 1).stream(), - !!rangeRequest, + cache.slice(range.start, range.end + 1).stream(), + request, range, metadata?.contentType, - downloadFilename, ); }; -const streamFromServer = async ( - id: number, - metadata: FileMetadata, - rangeRequest?: RangeRequest, - downloadFilename?: string, -) => { +const streamFromServer = async (request: StreamRequest, metadata: FileMetadata) => { + const { fileId, clientId, rangeRequest } = request; const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy); const range = resolveRange(rangeRequest, totalSize); if (!range) { return createRangeNotSatisfiableResponse(totalSize); } - const { start, end } = range; - const encryptedRange = getEncryptedRange(start, end, metadata.encContentSize, metadata.isLegacy); - const apiResponse = await fetch(`/api/file/${id}/download`, { + const encryptedRange = getEncryptedRange( + range.start, + range.end, + metadata.encContentSize, + metadata.isLegacy, + ); + const apiResponse = await fetch(`/api/file/${fileId}/download`, { headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` }, }); if (apiResponse.status !== 206 || !apiResponse.body) { return new Response("Failed to fetch encrypted file", { status: 502 }); } - if (metadata.isLegacy) { - const fileEncrypted = await apiResponse.arrayBuffer(); - const decrypted = await decryptChunk(fileEncrypted, metadata.dataKey); - return createResponse( - new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array(decrypted.slice(start, end + 1))); - controller.close(); - }, - }), - !!rangeRequest, + let stream = apiResponse.body.pipeThrough( + createDecryptionStream({ range, - metadata.contentType, - downloadFilename, - ); + chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1, + dataKey: metadata.dataKey, + isLegacy: metadata.isLegacy, + }), + ); + + // Cache and report the progress only when the whole content is requested (i.e. not seeking) + if (!rangeRequest) { + const [responseStream, cacheStream] = stream.tee(); + storeFileCache(fileId, cacheStream).catch(console.error); + stream = trackDownloadProgress(responseStream, fileId, totalSize, clientId); } - const totalChunks = encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1; - let currentChunkIndex = 0; - let buffer = new Uint8Array(0); - - const decryptingStream = new TransformStream({ - async transform(chunk, controller) { - const newBuffer = new Uint8Array(buffer.length + chunk.length); - newBuffer.set(buffer); - newBuffer.set(chunk, buffer.length); - buffer = newBuffer; - - while (buffer.length >= ENCRYPTED_CHUNK_SIZE && currentChunkIndex < totalChunks - 1) { - const encryptedChunk = buffer.slice(0, ENCRYPTED_CHUNK_SIZE); - buffer = buffer.slice(ENCRYPTED_CHUNK_SIZE); - - const decrypted = await decryptChunk(encryptedChunk.buffer, metadata.dataKey); - const sliceStart = currentChunkIndex === 0 ? start % CHUNK_SIZE : 0; - controller.enqueue(new Uint8Array(decrypted.slice(sliceStart))); - currentChunkIndex++; - } - }, - async flush(controller) { - if (buffer.length > 0) { - const decrypted = await decryptChunk(buffer.buffer, metadata.dataKey); - const sliceStart = currentChunkIndex === 0 ? start % CHUNK_SIZE : 0; - const sliceEnd = (end % CHUNK_SIZE) + 1; - controller.enqueue(new Uint8Array(decrypted.slice(sliceStart, sliceEnd))); - } - }, - }); - - return createResponse( - apiResponse.body.pipeThrough(decryptingStream), - !!rangeRequest, - range, - metadata.contentType, - downloadFilename, - ); + return createResponse(stream, request, range, metadata.contentType); }; const decryptFileHandler = async (request: Request, clientId: string) => { @@ -156,14 +117,18 @@ const decryptFileHandler = async (request: Request, clientId: string) => { return new Response("Invalid file id", { status: 400 }); } - const downloadFilename = url.searchParams.get("download") ?? undefined; + const streamRequest: StreamRequest = { + fileId, + clientId, + rangeRequest: parseRangeHeader(request.headers.get("Range")), + downloadFilename: url.searchParams.get("download") ?? undefined, + }; const metadata = await getFileMetadata(fileId, clientId); - const range = parseRangeHeader(request.headers.get("Range")); - const cache = await getFile(`/cache/${fileId}`); + const cache = await getFileCache(fileId); if (cache) { - return streamFromOpfs(cache, metadata, range, downloadFilename); + return streamFromCache(streamRequest, cache, metadata); } else if (metadata) { - return streamFromServer(fileId, metadata, range, downloadFilename); + return streamFromServer(streamRequest, metadata); } else { return new Response("Decryption not prepared", { status: 400 }); } diff --git a/src/service-worker/modules/clients.ts b/src/service-worker/modules/clients.ts new file mode 100644 index 0000000..c142336 --- /dev/null +++ b/src/service-worker/modules/clients.ts @@ -0,0 +1,10 @@ +import { self } from "$app/service-worker"; +import type { ServiceWorkerResponse } from "#lib/shared/serviceWorker.js"; + +// Posts the message to the client that made the request, or to all windows if unknown +// (e.g. downloads by navigation) +export const notifyClients = async (clientId: string, message: ServiceWorkerResponse) => { + const client = clientId ? await self.clients.get(clientId) : undefined; + const clients = client ? [client] : await self.clients.matchAll({ type: "window" }); + clients.forEach((client) => client.postMessage(message)); +}; diff --git a/src/service-worker/modules/progress.ts b/src/service-worker/modules/progress.ts new file mode 100644 index 0000000..eb05c08 --- /dev/null +++ b/src/service-worker/modules/progress.ts @@ -0,0 +1,48 @@ +import type { DownloadProgressMessage } from "#lib/shared/serviceWorker.js"; +import { notifyClients } from "./clients"; + +const REPORT_INTERVAL_MS = 200; + +// Reports the progress of reading the stream to the clients +export const trackDownloadProgress = ( + stream: ReadableStream, + fileId: number, + total: number, + clientId: string, +) => { + const reader = stream.getReader(); + let loaded = 0; + let lastReportedAt = 0; + + const report = (status: DownloadProgressMessage["status"]) => { + lastReportedAt = Date.now(); + void notifyClients(clientId, { type: "download-progress", fileId, status, loaded, total }); + }; + + report("downloading"); + return new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + report("completed"); + controller.close(); + return; + } + + loaded += value.byteLength; + if (Date.now() - lastReportedAt >= REPORT_INTERVAL_MS) { + report("downloading"); + } + controller.enqueue(value); + } catch (e) { + report("error"); + controller.error(e); + } + }, + async cancel(reason) { + report("canceled"); + await reader.cancel(reason); + }, + }); +}; diff --git a/tests/unit/decryptionStream.test.ts b/tests/unit/decryptionStream.test.ts new file mode 100644 index 0000000..146f9dd --- /dev/null +++ b/tests/unit/decryptionStream.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import { encryptChunk, generateDataKey } from "#lib/shared/crypto/index.js"; +import { + CHUNK_SIZE, + createDecryptionStream, + getEncryptedRange, + getEncryptedSize, +} from "#lib/shared/format/index.js"; +import { concatenateBuffers } from "#lib/shared/crypto/index.js"; + +const plaintext = Uint8Array.from({ length: 2 * CHUNK_SIZE + 12345 }, (_, i) => (i * 7) % 251); + +// `toEqual` is too slow for large arrays +const expectBytes = (actual: Uint8Array, expected: Uint8Array) => { + expect(actual.byteLength).toBe(expected.byteLength); + expect(Buffer.from(actual).equals(Buffer.from(expected))).toBe(true); +}; + +const encrypt = async (data: Uint8Array, dataKey: CryptoKey, chunkSize = CHUNK_SIZE) => { + const chunks: ArrayBuffer[] = []; + for (let offset = 0; offset < data.byteLength; offset += chunkSize) { + chunks.push(await encryptChunk(data.slice(offset, offset + chunkSize).buffer, dataKey)); + } + return concatenateBuffers(...chunks); +}; + +// Feeds the input in pieces of irregular sizes like a network stream +const toStream = (data: Uint8Array) => + new ReadableStream({ + start(controller) { + let offset = 0; + for (let i = 0; offset < data.byteLength; i++) { + const size = [1, 1000, 65536, 3_000_000][i % 4]!; + controller.enqueue(data.slice(offset, offset + size)); + offset += size; + } + controller.close(); + }, + }); + +const decrypt = async ( + encrypted: Uint8Array, + dataKey: CryptoKey, + start: number, + end: number, + isLegacy = false, +) => { + const encryptedRange = getEncryptedRange(start, end, encrypted.byteLength, isLegacy); + const stream = toStream( + encrypted.slice(encryptedRange.start, encryptedRange.end + 1), + ).pipeThrough( + createDecryptionStream({ + range: { start, end, total: plaintext.byteLength }, + chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1, + dataKey, + isLegacy, + }), + ); + return new Uint8Array(await new Response(stream).arrayBuffer()); +}; + +describe("createDecryptionStream", async () => { + const { dataKey } = await generateDataKey(); + const encrypted = await encrypt(plaintext, dataKey); + + it("produces encrypted content of the expected size", () => { + expect(encrypted.byteLength).toBe(getEncryptedSize(plaintext.byteLength)); + }); + + it.each([ + [0, plaintext.byteLength - 1], + [0, 0], + [10, 20], + [CHUNK_SIZE - 5, CHUNK_SIZE + 5], + [CHUNK_SIZE, 2 * CHUNK_SIZE - 1], + [CHUNK_SIZE + 1, plaintext.byteLength - 1], + [plaintext.byteLength - 1, plaintext.byteLength - 1], + ])("decrypts the range %i-%i", async (start, end) => { + expectBytes(await decrypt(encrypted, dataKey, start, end), plaintext.slice(start, end + 1)); + }); + + it("decrypts ranges of legacy files", async () => { + const legacyEncrypted = await encrypt(plaintext, dataKey, plaintext.byteLength); + + expectBytes( + await decrypt(legacyEncrypted, dataKey, 100, CHUNK_SIZE + 100, true), + plaintext.slice(100, CHUNK_SIZE + 101), + ); + }); + + it("rejects truncated content", async () => { + const stream = toStream(encrypted.slice(0, CHUNK_SIZE)).pipeThrough( + createDecryptionStream({ + range: { start: 0, end: plaintext.byteLength - 1, total: plaintext.byteLength }, + chunkCount: 3, + dataKey, + isLegacy: false, + }), + ); + + await expect(new Response(stream).arrayBuffer()).rejects.toThrow(); + }); +});