파일을 다운로드할 때 전체 파일을 메모리에 올리는 대신 서비스 워커에서 스트리밍으로 복호화하도록 변경

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
staticandClaude Opus 5.5 committed 2026-10-12 00:48:21 +09:00
1 parent 877cf9c42e
commit 3e52fa7a7c
29 files changed
+629 -467

No files matched your search

+2 -1
View File
@@ -60,7 +60,8 @@ export default defineConfig(
// Domain-agnostic components // Domain-agnostic components
"src/lib/ui/**": "^(#lib/(server|client|features)/|#trpc/)", "src/lib/ui/**": "^(#lib/(server|client|features)/|#trpc/)",
"src/lib/features/**": "^#lib/server/", "src/lib/features/**": "^#lib/server/",
"src/service-worker/**": "^(#lib/(?!shared/|client/storage/opfs\\.js$)|#trpc/|#workers/)", "src/service-worker/**":
"^(#lib/(?!shared/|client/storage/|client/cache/fileCache\\.js$)|#trpc/|#workers/)",
}), }),
{ {
files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js"], files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js"],
+3 -4
View File
@@ -1,7 +1,6 @@
import type { ClientInit } from "@sveltejs/kit/hooks"; import type { ClientInit } from "@sveltejs/kit/hooks";
import { prepareFileCache } from "#lib/client/cache/index.js";
import { keyring } from "#lib/client/keys/index.js"; import { keyring } from "#lib/client/keys/index.js";
import { listenForMetadataRequests } from "#lib/client/serviceWorker.js"; import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js";
import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js"; import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js";
const requestPersistentStorage = async () => { const requestPersistentStorage = async () => {
@@ -14,9 +13,9 @@ const requestPersistentStorage = async () => {
}; };
export const init: ClientInit = async () => { export const init: ClientInit = async () => {
listenForMetadataRequests(); listenForServiceWorkerMessages();
await Promise.all([requestPersistentStorage(), prepareFileCache(), keyring.load()]); await Promise.all([requestPersistentStorage(), keyring.load()]);
cleanupDanglingInfos(); // Intended cleanupDanglingInfos(); // Intended
}; };
+30 -30
View File
@@ -1,50 +1,50 @@
import { import {
getFileCacheIndex as getFileCacheIndexFromIndexedDB,
storeFileCacheIndex,
deleteFileCacheIndex, deleteFileCacheIndex,
type FileCacheIndex, getFileCacheIndex as getAllFileCacheIndexes,
} from "#lib/client/storage/indexedDB/index.js"; getFileCacheIndexOf,
import { readFile, writeFile, deleteFile } from "#lib/client/storage/opfs.js"; storeFileCacheIndex,
updateFileCacheRetrievedAt,
} from "#lib/client/storage/indexedDB/cacheIndex.js";
import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js";
const fileCacheIndex = new Map<number, FileCacheIndex>(); // Used by both the pages and the service worker, so the index is always read from IndexedDB
export const prepareFileCache = async () => { const getPath = (fileId: number) => `/cache/${fileId}`;
for (const cache of await getFileCacheIndexFromIndexedDB()) {
fileCacheIndex.set(cache.fileId, cache);
}
};
export const getFileCacheIndex = () => { export const getFileCacheIndex = async () => {
return Array.from(fileCacheIndex.values()); return await getAllFileCacheIndexes();
}; };
// The index is written only after the whole content is written
export const getFileCache = async (fileId: number) => { export const getFileCache = async (fileId: number) => {
const cacheIndex = fileCacheIndex.get(fileId); if (!(await getFileCacheIndexOf(fileId))) return null;
if (!cacheIndex) return null;
cacheIndex.lastRetrievedAt = new Date(); const file = await getFile(getPath(fileId));
storeFileCacheIndex(cacheIndex); // Intended if (file) {
return await readFile(`/cache/${fileId}`); void updateFileCacheRetrievedAt(fileId, new Date());
}
return file;
}; };
export const storeFileCache = async (fileId: number, fileBuffer: ArrayBuffer) => { export const storeFileCache = async (fileId: number, stream: ReadableStream<Uint8Array>) => {
const now = new Date(); const path = getPath(fileId);
await writeFile(`/cache/${fileId}`, fileBuffer); try {
await stream.pipeTo(await createWritable(path));
} catch (e) {
await deleteFile(path).catch(() => {});
throw e;
}
const cacheIndex: FileCacheIndex = { const now = new Date();
await storeFileCacheIndex({
fileId, fileId,
cachedAt: now, cachedAt: now,
lastRetrievedAt: now, lastRetrievedAt: now,
size: fileBuffer.byteLength, size: (await getFile(path))?.size ?? 0,
}; });
fileCacheIndex.set(fileId, cacheIndex);
await storeFileCacheIndex(cacheIndex);
}; };
export const deleteFileCache = async (fileId: number) => { export const deleteFileCache = async (fileId: number) => {
if (!fileCacheIndex.has(fileId)) return;
fileCacheIndex.delete(fileId);
await deleteFile(`/cache/${fileId}`);
await deleteFileCacheIndex(fileId); await deleteFileCacheIndex(fileId);
await deleteFile(getPath(fileId));
}; };
+20
View File
@@ -25,3 +25,23 @@ export const extractExifDateTime = (fileBuffer: ArrayBuffer) => {
const offsetMs = offsetSign * ((offsetHour ?? 0) * 60 + (offsetMinute ?? 0)) * 60 * 1000; const offsetMs = offsetSign * ((offsetHour ?? 0) * 60 + (offsetMinute ?? 0)) * 60 * 1000;
return new Date(utcDate - offsetMs); return new Date(utcDate - offsetMs);
}; };
const EXIF_SEARCH_SIZE = 1024 * 1024; // 1 MiB
const MAX_FULL_READ_SIZE = 64 * 1024 * 1024; // 64 MiB
const tryExtractExifDateTime = (buffer: ArrayBuffer) => {
try {
return extractExifDateTime(buffer);
} catch {
return undefined;
}
};
// EXIF is usually at the beginning of the file, so the whole file is read only if not found
export const readExifDateTime = async (file: Blob) => {
const dateTime = tryExtractExifDateTime(await file.slice(0, EXIF_SEARCH_SIZE).arrayBuffer());
if (dateTime || file.size <= EXIF_SEARCH_SIZE || file.size > MAX_FULL_READ_SIZE) {
return dateTime;
}
return tryExtractExifDateTime(await file.arrayBuffer());
};
+24 -17
View File
@@ -82,35 +82,42 @@ const generateVideoThumbnail = (videoUrl: string, time = 0) => {
}); });
}; };
export const generateThumbnail = async (blob: Blob) => { const convertHeicToPng = async (blob: Blob) => {
let url; const { default: heic2any } = await import("heic2any");
return (await heic2any({ blob, toType: "image/png" })) as Blob;
};
// Generates a thumbnail of the image or video at the URL. Returns `null` if not supported.
export const generateThumbnailFromUrl = async (url: string, contentType: string) => {
try { try {
if (blob.type.startsWith("image/")) { if (contentType.startsWith("image/")) {
url = URL.createObjectURL(blob);
try { try {
return await generateImageThumbnail(url); return await generateImageThumbnail(url);
} catch { } catch {
URL.revokeObjectURL(url); if (contentType !== "image/heic") return null;
url = undefined;
if (blob.type === "image/heic") { // Chromium can't decode HEIC images
const { default: heic2any } = await import("heic2any"); const png = URL.createObjectURL(await convertHeicToPng(await (await fetch(url)).blob()));
url = URL.createObjectURL((await heic2any({ blob, toType: "image/png" })) as Blob); try {
return await generateImageThumbnail(url); return await generateImageThumbnail(png);
} else { } finally {
return null; URL.revokeObjectURL(png);
} }
} }
} else if (blob.type.startsWith("video/")) { } else if (contentType.startsWith("video/")) {
url = URL.createObjectURL(blob);
return await generateVideoThumbnail(url); return await generateVideoThumbnail(url);
} }
return null; return null;
} catch { } catch {
return null; return null;
}
};
export const generateThumbnail = async (blob: Blob) => {
const url = URL.createObjectURL(blob);
try {
return await generateThumbnailFromUrl(url, blob.type);
} finally { } finally {
if (url) { URL.revokeObjectURL(url);
URL.revokeObjectURL(url);
}
} }
}; };
+25 -12
View File
@@ -1,8 +1,9 @@
import { DECRYPTED_FILE_URL_PREFIX } from "#lib/shared/serviceWorker.js"; import { handleDownloadProgress } from "#lib/client/transfer/download.svelte.js";
import type { import {
FileMetadata, DECRYPTED_FILE_URL_PREFIX,
ServiceWorkerMessage, type FileMetadata,
ServiceWorkerResponse, type ServiceWorkerMessage,
type ServiceWorkerResponse,
} from "#lib/shared/serviceWorker.js"; } from "#lib/shared/serviceWorker.js";
const PREPARE_TIMEOUT_MS = 5000; const PREPARE_TIMEOUT_MS = 5000;
@@ -10,17 +11,22 @@ const PREPARE_TIMEOUT_MS = 5000;
// The service worker may be terminated at any time, losing its in-memory metadata. // The service worker may be terminated at any time, losing its in-memory metadata.
// Keep a copy here so that it can be re-requested on demand. // Keep a copy here so that it can be re-requested on demand.
const preparedFiles = new Map<number, FileMetadata>(); const preparedFiles = new Map<number, FileMetadata>();
let isListeningForMetadataRequests = false; let isListening = false;
export const listenForMetadataRequests = () => { export const listenForServiceWorkerMessages = () => {
if (isListeningForMetadataRequests) return; if (isListening) return;
isListeningForMetadataRequests = true; isListening = true;
navigator.serviceWorker.addEventListener( navigator.serviceWorker.addEventListener(
"message", "message",
(event: MessageEvent<ServiceWorkerResponse>) => { (event: MessageEvent<ServiceWorkerResponse>) => {
if (event.data.type === "decryption-metadata-request") { switch (event.data.type) {
event.ports[0]?.postMessage(preparedFiles.get(event.data.fileId) ?? null); case "decryption-metadata-request":
event.ports[0]?.postMessage(preparedFiles.get(event.data.fileId) ?? null);
break;
case "download-progress":
handleDownloadProgress(event.data);
break;
} }
}, },
); );
@@ -37,7 +43,7 @@ const getServiceWorker = async () => {
export const prepareFileDecryption = async (id: number, metadata: FileMetadata) => { export const prepareFileDecryption = async (id: number, metadata: FileMetadata) => {
preparedFiles.set(id, metadata); preparedFiles.set(id, metadata);
listenForMetadataRequests(); listenForServiceWorkerMessages();
const sw = await getServiceWorker(); const sw = await getServiceWorker();
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
@@ -63,3 +69,10 @@ export const prepareFileDecryption = async (id: number, metadata: FileMetadata)
}; };
export const getDecryptedFileUrl = (id: number) => `${DECRYPTED_FILE_URL_PREFIX}${id}`; export const getDecryptedFileUrl = (id: number) => `${DECRYPTED_FILE_URL_PREFIX}${id}`;
// Lets the service worker download, decrypt, and save the file
export const saveDecryptedFile = (id: number, filename: string) => {
const link = document.createElement("a");
link.href = `${getDecryptedFileUrl(id)}?download=${encodeURIComponent(filename)}`;
link.click();
};
@@ -28,6 +28,14 @@ export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
}); });
}; };
export const getFileCacheIndexOf = async (fileId: number) => {
return await cacheIndex.fileCache.get(fileId);
};
export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => {
await cacheIndex.fileCache.update(fileId, { lastRetrievedAt });
};
export const deleteFileCacheIndex = async (fileId: number) => { export const deleteFileCacheIndex = async (fileId: number) => {
await cacheIndex.fileCache.delete(fileId); await cacheIndex.fileCache.delete(fileId);
}; };
+6
View File
@@ -48,6 +48,12 @@ export const writeFile = async (path: string, data: ArrayBuffer) => {
} }
}; };
// Changes are applied atomically when the stream is closed, and discarded if it is aborted
export const createWritable = async (path: string) => {
const { fileHandle } = await getFileHandle(path);
return await fileHandle!.createWritable();
};
export const deleteFile = async (path: string) => { export const deleteFile = async (path: string) => {
const { parentHandle, filename } = await getFileHandle(path, false); const { parentHandle, filename } = await getFileHandle(path, false);
if (!parentHandle) return; if (!parentHandle) return;
+1 -53
View File
@@ -4,6 +4,7 @@ import pLimit from "p-limit";
import { encodeToBase64, encryptChunk } from "#lib/shared/crypto/index.js"; import { encodeToBase64, encryptChunk } from "#lib/shared/crypto/index.js";
import { CHUNK_SIZE, getChunkCount, getEncryptedSize } from "#lib/shared/format/index.js"; import { CHUNK_SIZE, getChunkCount, getEncryptedSize } from "#lib/shared/format/index.js";
import { BoundedQueue } from "#lib/utils/concurrency/index.js"; import { BoundedQueue } from "#lib/utils/concurrency/index.js";
import { createSpeedMeter } from "#lib/utils/speedMeter.js";
interface UploadStats { interface UploadStats {
progress: number; progress: number;
@@ -15,59 +16,6 @@ interface EncryptedChunk {
data: ArrayBuffer; data: ArrayBuffer;
} }
const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => {
const samples: { t: number; b: number }[] = [];
let lastSpeed = 0;
let startTime: number | null = null;
return (bytesNow?: number) => {
if (bytesNow === undefined) return lastSpeed;
const now = performance.now();
// Initialize start time on first call
if (startTime === null) {
startTime = now;
}
// Check if enough time has passed since the last sample
const lastSample = samples[samples.length - 1];
if (lastSample && now - lastSample.t < minInterval) {
return lastSpeed;
}
samples.push({ t: now, b: bytesNow });
// Remove old samples outside the time window
const cutoff = now - timeWindow;
while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift();
// Need at least 2 samples to calculate speed
if (samples.length < 2) {
return lastSpeed;
}
const first = samples[0]!;
const dt = now - first.t;
const db = bytesNow - first.b;
if (dt >= minInterval) {
const instantSpeed = (db / dt) * 1000;
// Apply EMA for smoother speed transitions
const alpha = 0.3;
const rawSpeed =
lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed;
// Apply warmup ramp to prevent initial overestimation
const elapsed = now - startTime;
const warmupWeight = Math.min(1, elapsed / warmupPeriod);
lastSpeed = rawSpeed * warmupWeight;
}
return lastSpeed;
};
};
const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => { const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
return await encryptChunk(await chunk.arrayBuffer(), dataKey); return await encryptChunk(await chunk.arrayBuffer(), dataKey);
}; };
+30 -85
View File
@@ -1,34 +1,21 @@
import axios from "axios"; import type { DownloadProgressMessage } from "#lib/shared/serviceWorker.js";
import { limitFunction } from "p-limit"; import { createSpeedMeter } from "#lib/utils/speedMeter.js";
import { decryptChunk, concatenateBuffers } from "#lib/shared/crypto/index.js";
import { ENCRYPTED_CHUNK_SIZE } from "#lib/shared/format/index.js"; // Files are downloaded and decrypted by the service worker, which reports the progress
export interface FileDownloadState { export interface FileDownloadState {
id: number; id: number;
status: status: "downloading" | "completed" | "error";
| "download-pending"
| "downloading"
| "decryption-pending"
| "decrypting"
| "decrypted"
| "canceled"
| "error";
progress?: number; progress?: number;
rate?: number; rate?: number;
estimated?: number;
result?: ArrayBuffer;
} }
type LiveFileDownloadState = FileDownloadState & {
status: "download-pending" | "downloading" | "decryption-pending" | "decrypting";
};
let downloadingFiles: FileDownloadState[] = $state([]); let downloadingFiles: FileDownloadState[] = $state([]);
const speedMeters = new WeakMap<FileDownloadState, ReturnType<typeof createSpeedMeter>>();
export const isFileDownloading = ( export const isFileDownloading = (status: FileDownloadState["status"]) => {
status: FileDownloadState["status"], return status === "downloading";
): status is LiveFileDownloadState["status"] => };
["download-pending", "downloading", "decryption-pending", "decrypting"].includes(status);
export const getFileDownloadState = (fileId: number) => { export const getFileDownloadState = (fileId: number) => {
return downloadingFiles.find((file) => file.id === fileId && isFileDownloading(file.status)); return downloadingFiles.find((file) => file.id === fileId && isFileDownloading(file.status));
@@ -42,69 +29,27 @@ export const clearDownloadedFiles = () => {
downloadingFiles = downloadingFiles.filter((file) => isFileDownloading(file.status)); downloadingFiles = downloadingFiles.filter((file) => isFileDownloading(file.status));
}; };
const requestFileDownload = limitFunction( export const handleDownloadProgress = ({
async (state: FileDownloadState, id: number) => { fileId,
state.status = "downloading"; status,
loaded,
total,
}: DownloadProgressMessage) => {
let state = getFileDownloadState(fileId);
if (!state) {
if (status !== "downloading") return;
const res = await axios.get(`/api/file/${id}/download`, { downloadingFiles.push({ id: fileId, status });
responseType: "arraybuffer", state = downloadingFiles.at(-1)!;
onDownloadProgress: ({ progress, rate, estimated }) => { speedMeters.set(state, createSpeedMeter());
state.progress = progress;
state.rate = rate;
state.estimated = estimated;
},
});
const fileEncrypted: ArrayBuffer = res.data;
state.status = "decryption-pending";
return fileEncrypted;
},
{ concurrency: 1 },
);
const decryptFile = limitFunction(
async (
state: FileDownloadState,
fileEncrypted: ArrayBuffer,
encryptedChunkSize: number,
dataKey: CryptoKey,
) => {
state.status = "decrypting";
const chunks: ArrayBuffer[] = [];
let offset = 0;
while (offset < fileEncrypted.byteLength) {
const nextOffset = Math.min(offset + encryptedChunkSize, fileEncrypted.byteLength);
chunks.push(await decryptChunk(fileEncrypted.slice(offset, nextOffset), dataKey));
offset = nextOffset;
}
const fileBuffer = concatenateBuffers(...chunks).buffer;
state.status = "decrypted";
state.result = fileBuffer;
return fileBuffer;
},
{ concurrency: 4 },
);
export const downloadFile = async (id: number, dataKey: CryptoKey, isLegacy: boolean) => {
downloadingFiles.push({
id,
status: "download-pending",
});
const state = downloadingFiles.at(-1)!;
try {
const fileEncrypted = await requestFileDownload(state, id);
return await decryptFile(
state,
fileEncrypted,
isLegacy ? fileEncrypted.byteLength : ENCRYPTED_CHUNK_SIZE,
dataKey,
);
} catch (e) {
state.status = "error";
throw e;
} }
if (status === "canceled") {
downloadingFiles = downloadingFiles.filter((file) => file !== state);
return;
}
state.status = status;
state.progress = total > 0 ? loaded / total : 1;
state.rate = speedMeters.get(state)?.(loaded);
}; };
+4 -8
View File
@@ -6,7 +6,7 @@ import {
type HmacSecret, type HmacSecret,
type MasterKey, type MasterKey,
} from "#lib/client/keys/index.js"; } from "#lib/client/keys/index.js";
import { extractExifDateTime } from "#lib/client/media/exif.js"; import { readExifDateTime } from "#lib/client/media/exif.js";
import { generateThumbnail } from "#lib/client/media/thumbnail.js"; import { generateThumbnail } from "#lib/client/media/thumbnail.js";
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js"; import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
import { import {
@@ -41,9 +41,7 @@ export type LiveFileUploadState = FileUploadState & {
status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading"; status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
}; };
const scheduler = new Scheduler< const scheduler = new Scheduler<{ fileId: number; thumbnailBuffer?: ArrayBuffer } | undefined>();
{ fileId: number; fileBuffer?: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
>();
let uploadingFiles: FileUploadState[] = $state([]); let uploadingFiles: FileUploadState[] = $state([]);
const isFileUploading = (status: FileUploadState["status"]) => const isFileUploading = (status: FileUploadState["status"]) =>
@@ -213,7 +211,6 @@ export const uploadFile = async (
return; return;
} }
let fileBuffer;
const fileType = getFileType(file); const fileType = getFileType(file);
const fileMetadata: FileMetadata = { const fileMetadata: FileMetadata = {
parentId, parentId,
@@ -222,8 +219,7 @@ export const uploadFile = async (
}; };
if (fileType.startsWith("image/")) { if (fileType.startsWith("image/")) {
fileBuffer = await file.arrayBuffer(); fileMetadata.createdAt = await readExifDateTime(file);
fileMetadata.createdAt = extractExifDateTime(fileBuffer);
} }
const blob = new Blob([file], { type: fileType }); const blob = new Blob([file], { type: fileType });
@@ -241,7 +237,7 @@ export const uploadFile = async (
dataKeyVersion, dataKeyVersion,
); );
return { fileId, fileBuffer, thumbnailBuffer }; return { fileId, thumbnailBuffer };
} catch (e) { } catch (e) {
state.status = "error"; state.status = "error";
throw e; throw e;
+36 -20
View File
@@ -1,41 +1,57 @@
import { import {
deleteFileCache, deleteFileCache,
deleteFileThumbnailCache, deleteFileThumbnailCache,
getFileCache,
storeFileCache,
storeFileThumbnailCache, storeFileThumbnailCache,
} from "#lib/client/cache/index.js"; } from "#lib/client/cache/index.js";
import { getDecryptedFileUrl, prepareFileDecryption } from "#lib/client/serviceWorker.js"; import {
getDecryptedFileUrl,
prepareFileDecryption,
saveDecryptedFile,
} from "#lib/client/serviceWorker.js";
import { getAllFileInfos } from "#lib/client/storage/indexedDB/index.js"; import { getAllFileInfos } from "#lib/client/storage/indexedDB/index.js";
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js"; import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
import { downloadFile } from "#lib/client/transfer/index.js";
import { trpc } from "#trpc/client.js"; import { trpc } from "#trpc/client.js";
import type { DataKey } from "./types"; import type { DataKey } from "./types";
// Returns the decrypted content of the file, from the cache if available interface StreamableFile {
export const getFileContent = async (fileId: number, dataKey: CryptoKey, isLegacy: boolean) => { id: number;
const cache = await getFileCache(fileId); isLegacy?: boolean;
if (cache) return cache; dataKey?: DataKey;
contentType: string;
}
const fileBuffer = await downloadFile(fileId, dataKey, isLegacy); // Returns a URL from which the service worker streams the decrypted content of the file
void storeFileCache(fileId, fileBuffer); export const getFileStreamUrl = async ({ id, isLegacy, dataKey, contentType }: StreamableFile) => {
return fileBuffer; if (!dataKey) {
}; throw new Error("Data key not loaded");
}
// Returns a URL from which the service worker streams the decrypted file const res = await fetch(`/api/file/${id}/download`, { method: "HEAD" });
export const getFileStreamUrl = async (fileId: number, dataKey: CryptoKey, contentType: string) => {
const res = await fetch(`/api/file/${fileId}/download`, { method: "HEAD" });
if (!res.ok) { if (!res.ok) {
throw new Error("Failed to get the file size"); throw new Error("Failed to get the file size");
} }
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10); const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10);
if (encContentSize <= 0) { await prepareFileDecryption(id, {
throw new Error("Invalid file size"); isLegacy: !!isLegacy,
} dataKey: dataKey.key,
encContentSize,
contentType,
});
return getDecryptedFileUrl(id);
};
await prepareFileDecryption(fileId, { isLegacy: false, dataKey, encContentSize, contentType }); export const getFileBlob = async (file: StreamableFile) => {
return getDecryptedFileUrl(fileId); const res = await fetch(await getFileStreamUrl(file));
if (!res.ok) {
throw new Error("Failed to download the file");
}
return await res.blob();
};
export const saveFile = async (file: StreamableFile & { name: string }) => {
await getFileStreamUrl(file);
saveDecryptedFile(file.id, file.name);
}; };
export const uploadFileThumbnail = async (fileId: number, thumbnail: Blob, dataKey: DataKey) => { export const uploadFileThumbnail = async (fileId: number, thumbnail: Blob, dataKey: DataKey) => {
+2 -5
View File
@@ -1,8 +1,8 @@
import { storeFileCache, storeFileThumbnailCache } from "#lib/client/cache/index.js"; import { storeFileThumbnailCache } from "#lib/client/cache/index.js";
import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js"; import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js";
import { invalidateDirectory } from "./directory"; import { invalidateDirectory } from "./directory";
// Uploads the file and caches it locally. Returns `false` if the upload is canceled. // Uploads the file and caches its thumbnail locally. Returns `false` if the upload is canceled.
export const uploadFile = async ( export const uploadFile = async (
file: File, file: File,
parentId: DirectoryId, parentId: DirectoryId,
@@ -12,9 +12,6 @@ export const uploadFile = async (
if (!res) return false; if (!res) return false;
invalidateDirectory(parentId); invalidateDirectory(parentId);
if (res.fileBuffer) {
void storeFileCache(res.fileId, res.fileBuffer);
}
if (res.thumbnailBuffer) { if (res.thumbnailBuffer) {
void storeFileThumbnailCache(res.fileId, res.thumbnailBuffer); void storeFileThumbnailCache(res.fileId, res.thumbnailBuffer);
} }
+1
View File
@@ -1 +1,2 @@
export * from "./chunk"; export * from "./chunk";
export * from "./stream";
+96
View File
@@ -0,0 +1,96 @@
import { decryptChunk } from "#lib/shared/crypto/index.js";
import type { ByteRange } from "#lib/shared/http.js";
import { CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "./chunk";
// Accumulates stream pieces and takes them out in fixed sizes without copying them repeatedly
const createByteQueue = () => {
let pieces: Uint8Array[] = [];
let size = 0;
return {
get size() {
return size;
},
push(piece: Uint8Array) {
pieces.push(piece);
size += piece.byteLength;
},
take(length: number) {
const result = new Uint8Array(length);
let offset = 0;
while (offset < length) {
const piece = pieces[0]!;
const count = Math.min(piece.byteLength, length - offset);
result.set(piece.subarray(0, count), offset);
offset += count;
if (count === piece.byteLength) {
pieces.shift();
} else {
pieces[0] = piece.subarray(count);
}
}
size -= length;
if (size === 0) {
pieces = [];
}
return result;
},
};
};
// Decrypts the encrypted chunks covering the plaintext `range` (as returned by
// `getEncryptedRange`) into the plaintext of the range
export const createDecryptionStream = ({
range,
chunkCount,
dataKey,
isLegacy,
}: {
range: ByteRange;
chunkCount: number;
dataKey: CryptoKey;
isLegacy: boolean;
}) => {
const queue = createByteQueue();
let chunkIndex = 0;
const decryptNextChunk = async (
controller: TransformStreamDefaultController<Uint8Array>,
encryptedSize: number,
) => {
const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey));
const isFirst = chunkIndex === 0;
const isLast = chunkIndex === chunkCount - 1;
chunkIndex++;
if (isLegacy) {
controller.enqueue(decrypted.subarray(range.start, range.end + 1));
} else {
controller.enqueue(
decrypted.subarray(
isFirst ? range.start % CHUNK_SIZE : 0,
isLast ? (range.end % CHUNK_SIZE) + 1 : decrypted.byteLength,
),
);
}
};
return new TransformStream<Uint8Array, Uint8Array>({
async transform(piece, controller) {
queue.push(piece);
// The last chunk may be shorter, so it is decrypted when the stream ends
while (!isLegacy && queue.size >= ENCRYPTED_CHUNK_SIZE && chunkIndex < chunkCount - 1) {
await decryptNextChunk(controller, ENCRYPTED_CHUNK_SIZE);
}
},
async flush(controller) {
if (chunkIndex === chunkCount - 1 && queue.size > 0) {
await decryptNextChunk(controller, queue.size);
}
if (chunkIndex !== chunkCount || queue.size > 0) {
throw new Error("Unexpected size of the encrypted content");
}
},
});
};
+11 -1
View File
@@ -23,5 +23,15 @@ export interface DecryptionMetadataRequestMessage {
fileId: number; fileId: number;
} }
// Progress of the whole content of a file being streamed from the server
export interface DownloadProgressMessage {
type: "download-progress";
fileId: number;
status: "downloading" | "completed" | "canceled" | "error";
loaded: number;
total: number;
}
export type ServiceWorkerMessage = DecryptionPrepareMessage; export type ServiceWorkerMessage = DecryptionPrepareMessage;
export type ServiceWorkerResponse = DecryptionReadyMessage | DecryptionMetadataRequestMessage; export type ServiceWorkerResponse =
DecryptionReadyMessage | DecryptionMetadataRequestMessage | DownloadProgressMessage;
+1
View File
@@ -3,3 +3,4 @@ export * from "./format";
export * from "./gotoStateful"; export * from "./gotoStateful";
export * from "./search"; export * from "./search";
export * from "./sort"; export * from "./sort";
export * from "./speedMeter";
+53
View File
@@ -0,0 +1,53 @@
// Estimates the transfer rate in bytes per second from the total transferred bytes
export const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => {
const samples: { t: number; b: number }[] = [];
let lastSpeed = 0;
let startTime: number | null = null;
return (bytesNow?: number) => {
if (bytesNow === undefined) return lastSpeed;
const now = performance.now();
// Initialize start time on first call
if (startTime === null) {
startTime = now;
}
// Check if enough time has passed since the last sample
const lastSample = samples[samples.length - 1];
if (lastSample && now - lastSample.t < minInterval) {
return lastSpeed;
}
samples.push({ t: now, b: bytesNow });
// Remove old samples outside the time window
const cutoff = now - timeWindow;
while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift();
// Need at least 2 samples to calculate speed
if (samples.length < 2) {
return lastSpeed;
}
const first = samples[0]!;
const dt = now - first.t;
const db = bytesNow - first.b;
if (dt >= minInterval) {
const instantSpeed = (db / dt) * 1000;
// Apply EMA for smoother speed transitions
const alpha = 0.3;
const rawSpeed =
lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed;
// Apply warmup ramp to prevent initial overestimation
const elapsed = now - startTime;
const warmupWeight = Math.min(1, elapsed / warmupPeriod);
lastSpeed = rawSpeed * warmupWeight;
}
return lastSpeed;
};
};
+42 -64
View File
@@ -1,15 +1,14 @@
<script lang="ts"> <script lang="ts">
import FileSaver from "file-saver";
import { untrack } from "svelte"; import { untrack } from "svelte";
import { goto } from "$app/navigation"; import { goto } from "$app/navigation";
import { page } from "$app/state"; import { page } from "$app/state";
import { captureVideoThumbnail } from "#lib/client/media/thumbnail.js"; import { captureVideoThumbnail } from "#lib/client/media/thumbnail.js";
import { saveDecryptedFile } from "#lib/client/serviceWorker.js";
import { getFileDownloadState } from "#lib/client/transfer/index.js"; import { getFileDownloadState } from "#lib/client/transfer/index.js";
import { showErrorToast, tryAction } from "#lib/features/errors.js"; import { showErrorToast, tryAction } from "#lib/features/errors.js";
import { Categories } from "#lib/features/filesystem/components/index.js"; import { Categories } from "#lib/features/filesystem/components/index.js";
import { import {
addFileToCategory, addFileToCategory,
getFileContent,
getFileStreamUrl, getFileStreamUrl,
removeFileFromCategory, removeFileFromCategory,
setFileFavorite, setFileFavorite,
@@ -37,29 +36,31 @@
let isMenuOpen = $state(false); let isMenuOpen = $state(false);
let isAddToCategoryBottomSheetOpen = $state(false); let isAddToCategoryBottomSheetOpen = $state(false);
let isDownloadRequested = $state(false); let isLoadRequested = $state(false);
let viewerType: "image" | "video" | undefined = $state(); let viewerType: "image" | "video" | undefined = $state();
let fileBlob: Blob | undefined = $state(); let fileUrl: string | undefined = $state();
let fileBlobUrl: string | undefined = $state(); let convertedImageUrl: string | undefined = $state();
let videoStreamUrl: string | undefined = $state();
let videoElement: HTMLVideoElement | undefined = $state(); let videoElement: HTMLVideoElement | undefined = $state();
const updateViewer = async (buffer: ArrayBuffer, contentType: string) => { // Chromium can't display HEIC images
fileBlob = new Blob([buffer], { type: contentType }); const convertHeicToJpeg = async () => {
fileBlobUrl = URL.createObjectURL(fileBlob); if (info?.contentType !== "image/heic" || !fileUrl || convertedImageUrl) return;
return fileBlob;
try {
const blob = await (await fetch(fileUrl)).blob();
const { default: heic2any } = await import("heic2any");
convertedImageUrl = URL.createObjectURL(
(await heic2any({ blob, toType: "image/jpeg" })) as Blob,
);
} catch (e) {
showErrorToast(e, "이미지를 표시하지 못했어요.");
}
}; };
const convertHeicToJpeg = async () => { const saveFile = () => {
if (fileBlob?.type !== "image/heic") return; if (info?.exists) {
saveDecryptedFile(info.id, info.name);
URL.revokeObjectURL(fileBlobUrl!); }
fileBlobUrl = undefined;
const { default: heic2any } = await import("heic2any");
fileBlobUrl = URL.createObjectURL(
(await heic2any({ blob: fileBlob, toType: "image/jpeg" })) as Blob,
);
}; };
const updateThumbnail = (dataKey: DataKey) => const updateThumbnail = (dataKey: DataKey) =>
@@ -91,16 +92,12 @@
"즐겨찾기를 변경하지 못했어요.", "즐겨찾기를 변경하지 못했어요.",
); );
const loadContent = (dataKey: CryptoKey, isLegacy: boolean) =>
getFileContent(data.id, dataKey, isLegacy).catch((e) => {
showErrorToast(e, "파일을 불러오지 못했어요.");
throw e;
});
$effect.pre(() => { $effect.pre(() => {
void data.id; // Reset the viewer when navigating to another file void data.id; // Reset the viewer when navigating to another file
isDownloadRequested = false; isLoadRequested = false;
viewerType = undefined; viewerType = undefined;
fileUrl = undefined;
convertedImageUrl = undefined;
}); });
$effect(() => { $effect(() => {
@@ -113,42 +110,26 @@
} }
untrack(() => { untrack(() => {
if (!downloadState && !isDownloadRequested) { if (isLoadRequested) return;
isDownloadRequested = true; isLoadRequested = true;
const { key } = info!.dataKey!; const isAlreadyDownloading = !!downloadState;
const isLegacy = !!info!.isLegacy; getFileStreamUrl(info as FileInfo)
if (viewerType === "video" && !isLegacy) { .then((url) => {
getFileStreamUrl(data.id, key, contentType) fileUrl = url;
.then((streamUrl) => { if (!viewerType && !isAlreadyDownloading) {
videoStreamUrl = streamUrl; saveFile();
}) }
.catch(async () => { })
// Fall back to downloading the whole file .catch((e) => showErrorToast(e, "파일을 불러오지 못했어요."));
await updateViewer(await loadContent(key, isLegacy), contentType);
});
} else {
void loadContent(key, isLegacy).then(async (buffer) => {
const blob = await updateViewer(buffer, contentType);
if (!viewerType) {
FileSaver.saveAs(blob, info!.name);
}
});
}
}
}); });
} }
}); });
$effect(() => { $effect(() => {
if (info?.exists && downloadState?.status === "decrypted") { const url = convertedImageUrl;
untrack( return () => url && URL.revokeObjectURL(url);
() => !isDownloadRequested && updateViewer(downloadState.result!, info!.contentType!),
);
}
}); });
$effect(() => () => fileBlobUrl && URL.revokeObjectURL(fileBlobUrl));
</script> </script>
<svelte:head> <svelte:head>
@@ -172,9 +153,7 @@
) )
? info?.parentId ? info?.parentId
: undefined} : undefined}
{fileBlob} onDownloadClick={fileUrl ? saveFile : undefined}
downloadUrl={videoStreamUrl}
filename={info?.name}
isFavorite={info?.isFavorite} isFavorite={info?.isFavorite}
onToggleFavorite={toggleFavorite} onToggleFavorite={toggleFavorite}
/> />
@@ -192,16 +171,15 @@
{/snippet} {/snippet}
{#if viewerType === "image"} {#if viewerType === "image"}
{#if fileBlobUrl} {#if fileUrl}
<img src={fileBlobUrl} alt={info.name} onerror={convertHeicToJpeg} /> <img src={convertedImageUrl ?? fileUrl} alt={info.name} onerror={convertHeicToJpeg} />
{:else} {:else}
{@render viewerLoading("이미지를 불러오고 있어요.")} {@render viewerLoading("이미지를 불러오고 있어요.")}
{/if} {/if}
{:else if viewerType === "video"} {:else if viewerType === "video"}
{#if videoStreamUrl || fileBlobUrl} {#if fileUrl}
<div class="flex flex-col space-y-2"> <div class="flex flex-col space-y-2">
<video bind:this={videoElement} src={videoStreamUrl ?? fileBlobUrl} controls muted <video bind:this={videoElement} src={fileUrl} controls muted></video>
></video>
<IconEntryButton <IconEntryButton
icon={IconCamera} icon={IconCamera}
onclick={() => updateThumbnail(info!.dataKey!)} onclick={() => updateThumbnail(info!.dataKey!)}
@@ -11,22 +11,10 @@
{#if isFileDownloading(state.status)} {#if isFileDownloading(state.status)}
<div class="w-full rounded-xl bg-gray-100 p-3"> <div class="w-full rounded-xl bg-gray-100 p-3">
<p class="font-medium"> <p class="font-medium">다운로드하는 중</p>
{#if state.status === "download-pending"}
다운로드를 기다리는 중
{:else if state.status === "downloading"}
다운로드하는 중
{:else if state.status === "decryption-pending"}
복호화를 기다리는 중
{:else if state.status === "decrypting"}
복호화하는 중
{/if}
</p>
<p class="text-xs"> <p class="text-xs">
{#if state.status === "downloading"} 전송됨
전송됨 {Math.floor((state.progress ?? 0) * 100)}% · {formatNetworkSpeed((state.rate ?? 0) * 8)}
{Math.floor((state.progress ?? 0) * 100)}% · {formatNetworkSpeed((state.rate ?? 0) * 8)}
{/if}
</p> </p>
</div> </div>
{/if} {/if}
@@ -1,5 +1,4 @@
<script lang="ts"> <script lang="ts">
import FileSaver from "file-saver";
import type { Component } from "svelte"; import type { Component } from "svelte";
import type { SvelteHTMLElements } from "svelte/elements"; import type { SvelteHTMLElements } from "svelte/elements";
import { fly } from "svelte/transition"; import { fly } from "svelte/transition";
@@ -12,39 +11,24 @@
interface Props { interface Props {
directoryId?: "root" | number; directoryId?: "root" | number;
downloadUrl?: string;
fileBlob?: Blob;
filename?: string;
isFavorite?: boolean; isFavorite?: boolean;
isOpen: boolean; isOpen: boolean;
onDownloadClick?: () => void;
onToggleFavorite?: () => void; onToggleFavorite?: () => void;
} }
let { let {
directoryId, directoryId,
downloadUrl,
fileBlob,
filename,
isFavorite, isFavorite,
isOpen = $bindable(), isOpen = $bindable(),
onDownloadClick,
onToggleFavorite, onToggleFavorite,
}: Props = $props(); }: Props = $props();
const handleDownload = () => {
if (fileBlob && filename) {
FileSaver.saveAs(fileBlob, filename);
} else if (downloadUrl && filename) {
// Use streaming download via Content-Disposition header
const url = new URL(downloadUrl, window.location.origin);
url.searchParams.set("download", filename);
window.open(url.toString(), "_blank");
}
};
</script> </script>
<svelte:window onclick={() => (isOpen = false)} /> <svelte:window onclick={() => (isOpen = false)} />
{#if isOpen && (directoryId || downloadUrl || fileBlob)} {#if isOpen && (directoryId || onDownloadClick || isFavorite !== undefined)}
<div <div
class="absolute top-full right-2 z-20 min-w-44 space-y-1 rounded-lg bg-white px-1 py-2 shadow-2xl" class="absolute top-full right-2 z-20 min-w-44 space-y-1 rounded-lg bg-white px-1 py-2 shadow-2xl"
transition:fly={{ y: -8, duration: 200 }} transition:fly={{ y: -8, duration: 200 }}
@@ -80,8 +64,8 @@
), ),
)} )}
{/if} {/if}
{#if fileBlob || downloadUrl} {#if onDownloadClick}
{@render menuButton(IconCloudDownload, "다운로드", handleDownload)} {@render menuButton(IconCloudDownload, "다운로드", onDownloadClick)}
{/if} {/if}
</div> </div>
</div> </div>
@@ -3,10 +3,7 @@
import type { SummarizedFileInfo } from "#lib/features/filesystem/index.js"; import type { SummarizedFileInfo } from "#lib/features/filesystem/index.js";
import { formatNetworkSpeed } from "#lib/utils/index.js"; import { formatNetworkSpeed } from "#lib/utils/index.js";
import IconCloud from "~icons/material-symbols/cloud";
import IconCloudDownload from "~icons/material-symbols/cloud-download"; import IconCloudDownload from "~icons/material-symbols/cloud-download";
import IconLock from "~icons/material-symbols/lock";
import IconLockClock from "~icons/material-symbols/lock-clock";
import IconCheckCircle from "~icons/material-symbols/check-circle"; import IconCheckCircle from "~icons/material-symbols/check-circle";
import IconError from "~icons/material-symbols/error"; import IconError from "~icons/material-symbols/error";
@@ -20,15 +17,9 @@
<div class="flex h-14 items-center gap-x-4 p-2"> <div class="flex h-14 items-center gap-x-4 p-2">
<div class="shrink-0 text-lg text-gray-600"> <div class="shrink-0 text-lg text-gray-600">
{#if state.status === "download-pending"} {#if state.status === "downloading"}
<IconCloud />
{:else if state.status === "downloading"}
<IconCloudDownload /> <IconCloudDownload />
{:else if state.status === "decryption-pending"} {:else if state.status === "completed"}
<IconLock />
{:else if state.status === "decrypting"}
<IconLockClock />
{:else if state.status === "decrypted"}
<IconCheckCircle class="text-green-500" /> <IconCheckCircle class="text-green-500" />
{:else if state.status === "error"} {:else if state.status === "error"}
<IconError class="text-red-500" /> <IconError class="text-red-500" />
@@ -39,17 +30,11 @@
{info.name} {info.name}
</p> </p>
<p class="text-xs text-gray-800"> <p class="text-xs text-gray-800">
{#if state.status === "download-pending"} {#if state.status === "downloading"}
다운로드를 기다리는 중
{:else if state.status === "downloading"}
전송됨 전송됨
{Math.floor((state.progress ?? 0) * 100)}% · {Math.floor((state.progress ?? 0) * 100)}% ·
{formatNetworkSpeed((state.rate ?? 0) * 8)} {formatNetworkSpeed((state.rate ?? 0) * 8)}
{:else if state.status === "decryption-pending"} {:else if state.status === "completed"}
복호화를 기다리는 중
{:else if state.status === "decrypting"}
복호화하는 중
{:else if state.status === "decrypted"}
다운로드 완료 다운로드 완료
{:else if state.status === "error"} {:else if state.status === "error"}
다운로드 실패 다운로드 실패
+1 -1
View File
@@ -26,7 +26,7 @@
}; };
onMount(async () => { onMount(async () => {
const indexes = getFileCacheIndex(); const indexes = await getFileCacheIndex();
const infos = await bulkGetFileInfo(indexes.map(({ fileId }) => fileId)); const infos = await bulkGetFileInfo(indexes.map(({ fileId }) => fileId));
fileCache = indexes fileCache = indexes
.map((index) => ({ index, info: infos.get(index.fileId)! })) .map((index) => ({ index, info: infos.get(index.fileId)! }))
@@ -21,7 +21,6 @@
const statusTexts = { const statusTexts = {
queued: "대기 중", queued: "대기 중",
"generation-pending": "준비 중",
generating: "생성하는 중", generating: "생성하는 중",
uploading: "업로드하는 중", uploading: "업로드하는 중",
error: "실패", error: "실패",
@@ -1,19 +1,16 @@
import { limitFunction } from "p-limit"; import { limitFunction } from "p-limit";
import { SvelteMap } from "svelte/reactivity"; import { SvelteMap } from "svelte/reactivity";
import { generateThumbnail } from "#lib/client/media/thumbnail.js"; import { generateThumbnailFromUrl } from "#lib/client/media/thumbnail.js";
import { import {
getFileContent,
getFileInfo, getFileInfo,
getFileStreamUrl,
uploadFileThumbnail, uploadFileThumbnail,
type FileInfo, type FileInfo,
} from "#lib/features/filesystem/index.js"; } from "#lib/features/filesystem/index.js";
import { Scheduler } from "#lib/utils/index.js";
import type { RouterOutputs } from "#trpc/router.server.js"; import type { RouterOutputs } from "#trpc/router.server.js";
export type GenerationStatus = export type GenerationStatus = "queued" | "generating" | "uploading" | "uploaded" | "error";
"queued" | "generation-pending" | "generating" | "uploading" | "uploaded" | "error";
const scheduler = new Scheduler();
const statuses = new SvelteMap<number, GenerationStatus>(); const statuses = new SvelteMap<number, GenerationStatus>();
export const getThumbnailGenerationStatus = (fileId: number) => { export const getThumbnailGenerationStatus = (fileId: number) => {
@@ -37,13 +34,13 @@ export const requestMissingThumbnailFiles = async (
return files; return files;
}; };
// The content is streamed by the service worker, so only the decoded media occupy the memory
const generateAndUploadThumbnail = limitFunction( const generateAndUploadThumbnail = limitFunction(
async (fileInfo: FileInfo, fileBuffer: ArrayBuffer) => { async (fileInfo: FileInfo) => {
statuses.set(fileInfo.id, "generating"); statuses.set(fileInfo.id, "generating");
const thumbnail = await generateThumbnail( const url = await getFileStreamUrl(fileInfo);
new Blob([fileBuffer], { type: fileInfo.contentType }), const thumbnail = await generateThumbnailFromUrl(url, fileInfo.contentType);
);
if (!thumbnail) { if (!thumbnail) {
throw new Error("Failed to generate thumbnail"); throw new Error("Failed to generate thumbnail");
} }
@@ -57,23 +54,11 @@ const generateAndUploadThumbnail = limitFunction(
export const requestThumbnailGeneration = async (fileInfo: FileInfo) => { export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
const status = statuses.get(fileInfo.id); const status = statuses.get(fileInfo.id);
if (status) { if (status && status !== "error") return;
if (status !== "error") return;
} else {
statuses.set(fileInfo.id, "queued");
}
statuses.set(fileInfo.id, "queued");
try { try {
let file: ArrayBuffer | undefined; await generateAndUploadThumbnail(fileInfo);
await scheduler.schedule(
async () => {
statuses.set(fileInfo.id, "generation-pending");
file = await getFileContent(fileInfo.id, fileInfo.dataKey!.key, !!fileInfo.isLegacy);
return file.byteLength;
},
() => generateAndUploadThumbnail(fileInfo, file!),
);
} catch (e) { } catch (e) {
console.error(e); console.error(e);
statuses.set(fileInfo.id, "error"); statuses.set(fileInfo.id, "error");
+50 -85
View File
@@ -1,8 +1,6 @@
import { getFile } from "#lib/client/storage/opfs.js"; import { getFileCache, storeFileCache } from "#lib/client/cache/fileCache.js";
import { decryptChunk } from "#lib/shared/crypto/index.js";
import { import {
CHUNK_SIZE, createDecryptionStream,
ENCRYPTED_CHUNK_SIZE,
getDecryptedSize, getDecryptedSize,
getEncryptedRange, getEncryptedRange,
} from "#lib/shared/format/index.js"; } from "#lib/shared/format/index.js";
@@ -15,18 +13,25 @@ import {
} from "#lib/shared/http.js"; } from "#lib/shared/http.js";
import { DECRYPTED_FILE_URL_PREFIX, type FileMetadata } from "#lib/shared/serviceWorker.js"; import { DECRYPTED_FILE_URL_PREFIX, type FileMetadata } from "#lib/shared/serviceWorker.js";
import { getFileMetadata } from "../modules/metadata"; import { getFileMetadata } from "../modules/metadata";
import { trackDownloadProgress } from "../modules/progress";
interface StreamRequest {
fileId: number;
clientId: string;
rangeRequest?: RangeRequest;
downloadFilename?: string;
}
const createResponse = ( const createResponse = (
stream: ReadableStream<Uint8Array>, stream: ReadableStream<Uint8Array>,
isRangeRequest: boolean, { rangeRequest, downloadFilename }: StreamRequest,
range: ByteRange, range: ByteRange,
contentType?: string, contentType?: string,
downloadFilename?: string,
) => { ) => {
const headers: Record<string, string> = { const headers: Record<string, string> = {
"Accept-Ranges": "bytes", "Accept-Ranges": "bytes",
"Content-Length": String(range.end - range.start + 1), "Content-Length": String(range.end - range.start + 1),
...(isRangeRequest ? getContentRangeHeader(range) : {}), ...(rangeRequest ? getContentRangeHeader(range) : {}),
}; };
if (contentType) { if (contentType) {
@@ -39,7 +44,7 @@ const createResponse = (
} }
return new Response(stream, { return new Response(stream, {
status: isRangeRequest ? 206 : 200, status: rangeRequest ? 206 : 200,
headers, headers,
}); });
}; };
@@ -51,102 +56,58 @@ const createRangeNotSatisfiableResponse = (total: number) => {
}); });
}; };
const streamFromOpfs = async ( const streamFromCache = (request: StreamRequest, cache: File, metadata?: FileMetadata) => {
file: File, const range = resolveRange(request.rangeRequest, cache.size);
metadata?: FileMetadata,
rangeRequest?: RangeRequest,
downloadFilename?: string,
) => {
const range = resolveRange(rangeRequest, file.size);
if (!range) { if (!range) {
return createRangeNotSatisfiableResponse(file.size); return createRangeNotSatisfiableResponse(cache.size);
} }
return createResponse( return createResponse(
file.slice(range.start, range.end + 1).stream(), cache.slice(range.start, range.end + 1).stream(),
!!rangeRequest, request,
range, range,
metadata?.contentType, metadata?.contentType,
downloadFilename,
); );
}; };
const streamFromServer = async ( const streamFromServer = async (request: StreamRequest, metadata: FileMetadata) => {
id: number, const { fileId, clientId, rangeRequest } = request;
metadata: FileMetadata,
rangeRequest?: RangeRequest,
downloadFilename?: string,
) => {
const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy); const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy);
const range = resolveRange(rangeRequest, totalSize); const range = resolveRange(rangeRequest, totalSize);
if (!range) { if (!range) {
return createRangeNotSatisfiableResponse(totalSize); return createRangeNotSatisfiableResponse(totalSize);
} }
const { start, end } = range; const encryptedRange = getEncryptedRange(
const encryptedRange = getEncryptedRange(start, end, metadata.encContentSize, metadata.isLegacy); range.start,
const apiResponse = await fetch(`/api/file/${id}/download`, { range.end,
metadata.encContentSize,
metadata.isLegacy,
);
const apiResponse = await fetch(`/api/file/${fileId}/download`, {
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` }, headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
}); });
if (apiResponse.status !== 206 || !apiResponse.body) { if (apiResponse.status !== 206 || !apiResponse.body) {
return new Response("Failed to fetch encrypted file", { status: 502 }); return new Response("Failed to fetch encrypted file", { status: 502 });
} }
if (metadata.isLegacy) { let stream = apiResponse.body.pipeThrough(
const fileEncrypted = await apiResponse.arrayBuffer(); createDecryptionStream({
const decrypted = await decryptChunk(fileEncrypted, metadata.dataKey);
return createResponse(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(decrypted.slice(start, end + 1)));
controller.close();
},
}),
!!rangeRequest,
range, range,
metadata.contentType, chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1,
downloadFilename, dataKey: metadata.dataKey,
); isLegacy: metadata.isLegacy,
}),
);
// Cache and report the progress only when the whole content is requested (i.e. not seeking)
if (!rangeRequest) {
const [responseStream, cacheStream] = stream.tee();
storeFileCache(fileId, cacheStream).catch(console.error);
stream = trackDownloadProgress(responseStream, fileId, totalSize, clientId);
} }
const totalChunks = encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1; return createResponse(stream, request, range, metadata.contentType);
let currentChunkIndex = 0;
let buffer = new Uint8Array(0);
const decryptingStream = new TransformStream<Uint8Array, Uint8Array>({
async transform(chunk, controller) {
const newBuffer = new Uint8Array(buffer.length + chunk.length);
newBuffer.set(buffer);
newBuffer.set(chunk, buffer.length);
buffer = newBuffer;
while (buffer.length >= ENCRYPTED_CHUNK_SIZE && currentChunkIndex < totalChunks - 1) {
const encryptedChunk = buffer.slice(0, ENCRYPTED_CHUNK_SIZE);
buffer = buffer.slice(ENCRYPTED_CHUNK_SIZE);
const decrypted = await decryptChunk(encryptedChunk.buffer, metadata.dataKey);
const sliceStart = currentChunkIndex === 0 ? start % CHUNK_SIZE : 0;
controller.enqueue(new Uint8Array(decrypted.slice(sliceStart)));
currentChunkIndex++;
}
},
async flush(controller) {
if (buffer.length > 0) {
const decrypted = await decryptChunk(buffer.buffer, metadata.dataKey);
const sliceStart = currentChunkIndex === 0 ? start % CHUNK_SIZE : 0;
const sliceEnd = (end % CHUNK_SIZE) + 1;
controller.enqueue(new Uint8Array(decrypted.slice(sliceStart, sliceEnd)));
}
},
});
return createResponse(
apiResponse.body.pipeThrough(decryptingStream),
!!rangeRequest,
range,
metadata.contentType,
downloadFilename,
);
}; };
const decryptFileHandler = async (request: Request, clientId: string) => { const decryptFileHandler = async (request: Request, clientId: string) => {
@@ -156,14 +117,18 @@ const decryptFileHandler = async (request: Request, clientId: string) => {
return new Response("Invalid file id", { status: 400 }); return new Response("Invalid file id", { status: 400 });
} }
const downloadFilename = url.searchParams.get("download") ?? undefined; const streamRequest: StreamRequest = {
fileId,
clientId,
rangeRequest: parseRangeHeader(request.headers.get("Range")),
downloadFilename: url.searchParams.get("download") ?? undefined,
};
const metadata = await getFileMetadata(fileId, clientId); const metadata = await getFileMetadata(fileId, clientId);
const range = parseRangeHeader(request.headers.get("Range")); const cache = await getFileCache(fileId);
const cache = await getFile(`/cache/${fileId}`);
if (cache) { if (cache) {
return streamFromOpfs(cache, metadata, range, downloadFilename); return streamFromCache(streamRequest, cache, metadata);
} else if (metadata) { } else if (metadata) {
return streamFromServer(fileId, metadata, range, downloadFilename); return streamFromServer(streamRequest, metadata);
} else { } else {
return new Response("Decryption not prepared", { status: 400 }); return new Response("Decryption not prepared", { status: 400 });
} }
+10
View File
@@ -0,0 +1,10 @@
import { self } from "$app/service-worker";
import type { ServiceWorkerResponse } from "#lib/shared/serviceWorker.js";
// Posts the message to the client that made the request, or to all windows if unknown
// (e.g. downloads by navigation)
export const notifyClients = async (clientId: string, message: ServiceWorkerResponse) => {
const client = clientId ? await self.clients.get(clientId) : undefined;
const clients = client ? [client] : await self.clients.matchAll({ type: "window" });
clients.forEach((client) => client.postMessage(message));
};
+48
View File
@@ -0,0 +1,48 @@
import type { DownloadProgressMessage } from "#lib/shared/serviceWorker.js";
import { notifyClients } from "./clients";
const REPORT_INTERVAL_MS = 200;
// Reports the progress of reading the stream to the clients
export const trackDownloadProgress = (
stream: ReadableStream<Uint8Array>,
fileId: number,
total: number,
clientId: string,
) => {
const reader = stream.getReader();
let loaded = 0;
let lastReportedAt = 0;
const report = (status: DownloadProgressMessage["status"]) => {
lastReportedAt = Date.now();
void notifyClients(clientId, { type: "download-progress", fileId, status, loaded, total });
};
report("downloading");
return new ReadableStream<Uint8Array>({
async pull(controller) {
try {
const { done, value } = await reader.read();
if (done) {
report("completed");
controller.close();
return;
}
loaded += value.byteLength;
if (Date.now() - lastReportedAt >= REPORT_INTERVAL_MS) {
report("downloading");
}
controller.enqueue(value);
} catch (e) {
report("error");
controller.error(e);
}
},
async cancel(reason) {
report("canceled");
await reader.cancel(reason);
},
});
};
+103
View File
@@ -0,0 +1,103 @@
import { describe, expect, it } from "vitest";
import { encryptChunk, generateDataKey } from "#lib/shared/crypto/index.js";
import {
CHUNK_SIZE,
createDecryptionStream,
getEncryptedRange,
getEncryptedSize,
} from "#lib/shared/format/index.js";
import { concatenateBuffers } from "#lib/shared/crypto/index.js";
const plaintext = Uint8Array.from({ length: 2 * CHUNK_SIZE + 12345 }, (_, i) => (i * 7) % 251);
// `toEqual` is too slow for large arrays
const expectBytes = (actual: Uint8Array, expected: Uint8Array) => {
expect(actual.byteLength).toBe(expected.byteLength);
expect(Buffer.from(actual).equals(Buffer.from(expected))).toBe(true);
};
const encrypt = async (data: Uint8Array, dataKey: CryptoKey, chunkSize = CHUNK_SIZE) => {
const chunks: ArrayBuffer[] = [];
for (let offset = 0; offset < data.byteLength; offset += chunkSize) {
chunks.push(await encryptChunk(data.slice(offset, offset + chunkSize).buffer, dataKey));
}
return concatenateBuffers(...chunks);
};
// Feeds the input in pieces of irregular sizes like a network stream
const toStream = (data: Uint8Array) =>
new ReadableStream<Uint8Array>({
start(controller) {
let offset = 0;
for (let i = 0; offset < data.byteLength; i++) {
const size = [1, 1000, 65536, 3_000_000][i % 4]!;
controller.enqueue(data.slice(offset, offset + size));
offset += size;
}
controller.close();
},
});
const decrypt = async (
encrypted: Uint8Array,
dataKey: CryptoKey,
start: number,
end: number,
isLegacy = false,
) => {
const encryptedRange = getEncryptedRange(start, end, encrypted.byteLength, isLegacy);
const stream = toStream(
encrypted.slice(encryptedRange.start, encryptedRange.end + 1),
).pipeThrough(
createDecryptionStream({
range: { start, end, total: plaintext.byteLength },
chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1,
dataKey,
isLegacy,
}),
);
return new Uint8Array(await new Response(stream).arrayBuffer());
};
describe("createDecryptionStream", async () => {
const { dataKey } = await generateDataKey();
const encrypted = await encrypt(plaintext, dataKey);
it("produces encrypted content of the expected size", () => {
expect(encrypted.byteLength).toBe(getEncryptedSize(plaintext.byteLength));
});
it.each([
[0, plaintext.byteLength - 1],
[0, 0],
[10, 20],
[CHUNK_SIZE - 5, CHUNK_SIZE + 5],
[CHUNK_SIZE, 2 * CHUNK_SIZE - 1],
[CHUNK_SIZE + 1, plaintext.byteLength - 1],
[plaintext.byteLength - 1, plaintext.byteLength - 1],
])("decrypts the range %i-%i", async (start, end) => {
expectBytes(await decrypt(encrypted, dataKey, start, end), plaintext.slice(start, end + 1));
});
it("decrypts ranges of legacy files", async () => {
const legacyEncrypted = await encrypt(plaintext, dataKey, plaintext.byteLength);
expectBytes(
await decrypt(legacyEncrypted, dataKey, 100, CHUNK_SIZE + 100, true),
plaintext.slice(100, CHUNK_SIZE + 101),
);
});
it("rejects truncated content", async () => {
const stream = toStream(encrypted.slice(0, CHUNK_SIZE)).pipeThrough(
createDecryptionStream({
range: { start: 0, end: plaintext.byteLength - 1, total: plaintext.byteLength },
chunkCount: 3,
dataKey,
isLegacy: false,
}),
);
await expect(new Response(stream).arrayBuffer()).rejects.toThrow();
});
});