파일을 업로드할 때 캐시에 저장하고 동영상을 스트리밍하는 과정에서도 청크 단위로 캐시에 저장하여 다음 재생에 활용하도록 개선

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
staticandClaude Opus 5.5 committed 2026-10-12 01:40:03 +09:00
1 parent 06fdfcc447
commit 225ad3515f
12 files changed
+449 -112

No files matched your search

+2
View File
@@ -1,4 +1,5 @@
import type { ClientInit } from "@sveltejs/kit/hooks"; import type { ClientInit } from "@sveltejs/kit/hooks";
import { cleanupFileCaches } from "#lib/client/cache/index.js";
import { keyring } from "#lib/client/keys/index.js"; import { keyring } from "#lib/client/keys/index.js";
import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js"; import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js";
import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js"; import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js";
@@ -18,4 +19,5 @@ export const init: ClientInit = async () => {
await Promise.all([requestPersistentStorage(), keyring.load()]); await Promise.all([requestPersistentStorage(), keyring.load()]);
cleanupDanglingInfos(); // Intended cleanupDanglingInfos(); // Intended
cleanupFileCaches().catch(console.error);
}; };
+159 -24
View File
@@ -5,46 +5,181 @@ import {
storeFileCacheIndex, storeFileCacheIndex,
updateFileCacheRetrievedAt, updateFileCacheRetrievedAt,
} from "#lib/client/storage/indexedDB/cacheIndex.js"; } from "#lib/client/storage/indexedDB/cacheIndex.js";
import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js"; import {
createWritable,
deleteDirectory,
deleteFile,
deleteFilesInDirectory,
getFile,
listDirectoryNames,
listFileNames,
moveFile,
} from "#lib/client/storage/opfs.js";
import { CHUNK_SIZE, getChunkCount } from "#lib/shared/format/index.js";
// Used by both the pages and the service worker, so the index is always read from IndexedDB // Caches the plaintext of files in chunks, so that partially loaded files (e.g. videos which were
// partially played) can also be cached. Chunk `i` starts at `i * CHUNK_SIZE` of the plaintext,
// except that legacy files are cached as a single chunk. Used by both the pages and the service
// worker, so the index is always read from IndexedDB.
const getPath = (fileId: number) => `/cache/${fileId}`; export interface FileCacheLayout {
totalSize: number;
chunkCount: number;
}
const getDirectoryPath = (fileId: number) => `/cache/file/${fileId}`;
const getChunkPath = (fileId: number, chunkIndex: number) =>
`${getDirectoryPath(fileId)}/${chunkIndex}`;
const getChunkSize = ({ totalSize, chunkCount }: FileCacheLayout, chunkIndex: number) => {
return chunkCount === 1 ? totalSize : Math.min(CHUNK_SIZE, totalSize - chunkIndex * CHUNK_SIZE);
};
export const getFileCacheIndex = async () => { export const getFileCacheIndex = async () => {
return await getAllFileCacheIndexes(); return await getAllFileCacheIndexes();
}; };
// The index is written only after the whole content is written export const getCachedChunkIndexes = async (fileId: number) => {
export const getFileCache = async (fileId: number) => { const names = await listFileNames(getDirectoryPath(fileId));
if (!(await getFileCacheIndexOf(fileId))) return null; return new Set(names.filter((name) => /^\d+$/.test(name)).map(Number));
const file = await getFile(getPath(fileId));
if (file) {
void updateFileCacheRetrievedAt(fileId, new Date());
}
return file;
}; };
export const storeFileCache = async (fileId: number, stream: ReadableStream<Uint8Array>) => { export const readCachedChunk = async (fileId: number, chunkIndex: number) => {
const path = getPath(fileId); return await getFile(getChunkPath(fileId, chunkIndex));
};
export const touchFileCache = async (fileId: number) => {
await updateFileCacheRetrievedAt(fileId, new Date());
};
const updateIndex = async (fileId: number, layout: FileCacheLayout) => {
const chunkIndexes = await getCachedChunkIndexes(fileId);
const size = chunkIndexes
.values()
.filter((chunkIndex) => chunkIndex < layout.chunkCount)
.reduce((size, chunkIndex) => size + getChunkSize(layout, chunkIndex), 0);
const now = new Date();
await storeFileCacheIndex({
fileId,
cachedAt: (await getFileCacheIndexOf(fileId))?.cachedAt ?? now,
lastRetrievedAt: now,
size,
totalSize: layout.totalSize,
chunkCount: layout.chunkCount,
isComplete: size === layout.totalSize,
});
};
// Writes of the same file are serialized so that the index isn't updated with a stale state
const writeQueues = new Map<number, Promise<unknown>>();
const enqueueWrite = <T>(fileId: number, write: () => Promise<T>) => {
const result = (writeQueues.get(fileId) ?? Promise.resolve()).catch(() => {}).then(write);
const settled = result.catch(() => {});
writeQueues.set(fileId, settled);
void settled.then(() => {
if (writeQueues.get(fileId) === settled) {
writeQueues.delete(fileId);
}
});
return result;
};
export const storeCachedChunk = (
fileId: number,
chunkIndex: number,
data: Uint8Array | Blob,
layout: FileCacheLayout,
) => enqueueWrite(fileId, () => writeChunk(fileId, chunkIndex, data, layout));
const writeChunk = async (
fileId: number,
chunkIndex: number,
data: Uint8Array | Blob,
layout: FileCacheLayout,
) => {
const path = getChunkPath(fileId, chunkIndex);
const tempPath = `${path}.${crypto.randomUUID()}.tmp`;
// Written to a temporary file first, so that incomplete chunks are never visible
const writable = await createWritable(tempPath);
try { try {
await stream.pipeTo(await createWritable(path)); await writable.write(data as FileSystemWriteChunkType);
await writable.close();
await moveFile(tempPath, path);
} catch (e) { } catch (e) {
await deleteFile(path).catch(() => {}); await writable.abort().catch(() => {});
await deleteFile(tempPath).catch(() => {});
throw e; throw e;
} }
const now = new Date(); await updateIndex(fileId, layout);
await storeFileCacheIndex({ };
fileId,
cachedAt: now, const UPLOAD_CACHE_DIRECTORY = "/cache/upload";
lastRetrievedAt: now, const STALE_UPLOAD_CACHE_AGE = 24 * 3600 * 1000; // 24 hours
size: (await getFile(path))?.size ?? 0,
}); // Caches the plaintext of a file while it is being uploaded. Chunks are written to a temporary
// directory, and moved to the cache once the id of the file is known.
export const createUploadFileCache = (totalSize: number) => {
const layout = { totalSize, chunkCount: getChunkCount(totalSize) };
// Prefixed with the creation time to find stale ones (e.g. the tab was closed while uploading)
const directoryPath = `${UPLOAD_CACHE_DIRECTORY}/${Date.now()}-${crypto.randomUUID()}`;
const writtenChunkIndexes = new Set<number>();
let isFailed = false;
const discard = async () => {
isFailed = true;
await deleteDirectory(directoryPath).catch(() => {});
};
return {
// Failing to cache shouldn't fail the upload, so errors are ignored
async writeChunk(chunkIndex: number, data: ArrayBuffer) {
if (isFailed) return;
try {
const writable = await createWritable(`${directoryPath}/${chunkIndex}`);
await writable.write(data);
await writable.close();
writtenChunkIndexes.add(chunkIndex);
} catch (e) {
console.error(e);
await discard();
}
},
async commit(fileId: number) {
if (isFailed || writtenChunkIndexes.size !== layout.chunkCount) {
await discard();
return;
}
await enqueueWrite(fileId, async () => {
for (const chunkIndex of writtenChunkIndexes) {
await moveFile(`${directoryPath}/${chunkIndex}`, getChunkPath(fileId, chunkIndex));
}
await updateIndex(fileId, layout);
}).finally(discard);
},
discard,
};
}; };
export const deleteFileCache = async (fileId: number) => { export const deleteFileCache = async (fileId: number) => {
await deleteFileCacheIndex(fileId); await deleteFileCacheIndex(fileId);
await deleteFile(getPath(fileId)); await deleteDirectory(getDirectoryPath(fileId));
};
export const cleanupFileCaches = async () => {
// Files were cached as single files directly under `/cache` before
await deleteFilesInDirectory("/cache");
// Uploads might be in progress in other tabs, so only stale ones are deleted
for (const name of await listDirectoryNames(UPLOAD_CACHE_DIRECTORY)) {
const createdAt = parseInt(name, 10);
if (!(Date.now() - createdAt < STALE_UPLOAD_CACHE_AGE)) {
await deleteDirectory(`${UPLOAD_CACHE_DIRECTORY}/${name}`);
}
}
}; };
+13 -9
View File
@@ -4,7 +4,11 @@ export interface FileCacheIndex {
fileId: number; fileId: number;
cachedAt: Date; cachedAt: Date;
lastRetrievedAt: Date; lastRetrievedAt: Date;
// Files are cached in chunks, possibly partially (e.g. videos which were partially played)
size: number; size: number;
totalSize: number;
chunkCount: number;
isComplete: boolean;
} }
const cacheIndex = new Dexie("cacheIndex") as Dexie & { const cacheIndex = new Dexie("cacheIndex") as Dexie & {
@@ -15,23 +19,23 @@ cacheIndex.version(1).stores({
fileCache: "fileId", fileCache: "fileId",
}); });
// Files were cached as single files before version 2
cacheIndex.version(2).upgrade(async (trx) => {
await trx.table("fileCache").clear();
});
export const getFileCacheIndex = async () => { export const getFileCacheIndex = async () => {
return await cacheIndex.fileCache.toArray(); return await cacheIndex.fileCache.toArray();
}; };
export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
await cacheIndex.fileCache.put({
fileId: fileCacheIndex.fileId,
cachedAt: fileCacheIndex.cachedAt,
lastRetrievedAt: fileCacheIndex.lastRetrievedAt,
size: fileCacheIndex.size,
});
};
export const getFileCacheIndexOf = async (fileId: number) => { export const getFileCacheIndexOf = async (fileId: number) => {
return await cacheIndex.fileCache.get(fileId); return await cacheIndex.fileCache.get(fileId);
}; };
export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
await cacheIndex.fileCache.put(fileCacheIndex);
};
export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => { export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => {
await cacheIndex.fileCache.update(fileId, { lastRetrievedAt }); await cacheIndex.fileCache.update(fileId, { lastRetrievedAt });
}; };
+48
View File
@@ -94,3 +94,51 @@ export const deleteDirectory = async (path: string) => {
await parentHandle.removeEntry(directoryHandle.name, { recursive: true }); await parentHandle.removeEntry(directoryHandle.name, { recursive: true });
}; };
const listEntryNames = async (path: string, kind: FileSystemHandleKind) => {
const { directoryHandle } = await getDirectoryHandle(path);
if (!directoryHandle) return [];
const names: string[] = [];
for await (const handle of directoryHandle.values()) {
if (handle.kind === kind) {
names.push(handle.name);
}
}
return names;
};
export const listFileNames = (path: string) => listEntryNames(path, "file");
export const listDirectoryNames = (path: string) => listEntryNames(path, "directory");
// Deletes the files (not directories) directly under the directory
export const deleteFilesInDirectory = async (path: string) => {
const { directoryHandle } = await getDirectoryHandle(path);
if (!directoryHandle) return;
for (const name of await listFileNames(path)) {
await directoryHandle.removeEntry(name);
}
};
// Renames the file atomically
export const moveFile = async (path: string, newPath: string) => {
const { fileHandle } = await getFileHandle(path, false);
if (!fileHandle) {
throw new Error("File not found");
}
let directoryHandle = await navigator.storage.getDirectory();
const parts = newPath.split("/").filter((part) => part);
for (const part of parts.slice(0, -1)) {
directoryHandle = await directoryHandle.getDirectoryHandle(part, { create: true });
}
// Supported by Chromium for the origin private file system
await (
fileHandle as FileSystemFileHandle & {
move(destination: FileSystemDirectoryHandle, name: string): Promise<void>;
}
).move(directoryHandle, parts.at(-1)!);
};
+9 -7
View File
@@ -16,10 +16,6 @@ interface EncryptedChunk {
data: ArrayBuffer; data: ArrayBuffer;
} }
const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
return await encryptChunk(await chunk.arrayBuffer(), dataKey);
};
const uploadEncryptedChunk = async ( const uploadEncryptedChunk = async (
uploadId: string, uploadId: string,
chunkIndex: number, chunkIndex: number,
@@ -40,7 +36,12 @@ export const uploadBlob = async (
uploadId: string, uploadId: string,
blob: Blob, blob: Blob,
dataKey: CryptoKey, dataKey: CryptoKey,
options?: { concurrency?: number; onProgress?: (s: UploadStats) => void }, options?: {
concurrency?: number;
onProgress?: (s: UploadStats) => void;
// Receives each chunk of the plaintext before it is encrypted
onPlaintextChunk?: (chunkIndex: number, data: ArrayBuffer) => Promise<void>;
},
) => { ) => {
const onProgress = options?.onProgress; const onProgress = options?.onProgress;
const networkConcurrency = options?.concurrency ?? 4; const networkConcurrency = options?.concurrency ?? 4;
@@ -75,8 +76,9 @@ export const uploadBlob = async (
const encryptionProducer = async () => { const encryptionProducer = async () => {
try { try {
for (let i = 0; i < totalChunks; i++) { for (let i = 0; i < totalChunks; i++) {
const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE); const chunk = await blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE).arrayBuffer();
const data = await encryptChunkData(chunk, dataKey); await options?.onPlaintextChunk?.(i, chunk);
const data = await encryptChunk(chunk, dataKey);
hash.update(new Uint8Array(data)); hash.update(new Uint8Array(data));
await queue.push({ index: i, data }); await queue.push({ index: i, data });
} }
+24 -11
View File
@@ -6,6 +6,7 @@ import {
type HmacSecret, type HmacSecret,
type MasterKey, type MasterKey,
} from "#lib/client/keys/index.js"; } from "#lib/client/keys/index.js";
import { createUploadFileCache } from "#lib/client/cache/index.js";
import { readExifDateTime } from "#lib/client/media/exif.js"; import { readExifDateTime } from "#lib/client/media/exif.js";
import { generateThumbnail } from "#lib/client/media/thumbnail.js"; import { generateThumbnail } from "#lib/client/media/thumbnail.js";
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js"; import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
@@ -150,18 +151,30 @@ const requestFileUpload = limitFunction(
) => { ) => {
state.status = "uploading"; state.status = "uploading";
const { encContentHash } = await uploadBlob(uploadId, file, dataKey, { // The file is cached while uploading, so that it doesn't have to be read again
onProgress(s) { const cache = createUploadFileCache(file.size);
state.progress = s.progress;
state.rate = s.rate;
},
});
const { file: fileId } = await trpc.upload.completeFileUpload.mutate({ let fileId;
uploadId, try {
contentHmac: fileSigned, const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
encContentHash, onProgress(s) {
}); state.progress = s.progress;
state.rate = s.rate;
},
onPlaintextChunk: cache.writeChunk,
});
({ file: fileId } = await trpc.upload.completeFileUpload.mutate({
uploadId,
contentHmac: fileSigned,
encContentHash,
}));
} catch (e) {
await cache.discard();
throw e;
}
await cache.commit(fileId).catch(console.error);
if (thumbnailBuffer) { if (thumbnailBuffer) {
try { try {
+1 -12
View File
@@ -26,18 +26,7 @@ export const getFileStreamUrl = async ({ id, isLegacy, dataKey, contentType }: S
throw new Error("Data key not loaded"); throw new Error("Data key not loaded");
} }
const res = await fetch(`/api/file/${id}/download`, { method: "HEAD" }); await prepareFileDecryption(id, { isLegacy: !!isLegacy, dataKey: dataKey.key, contentType });
if (!res.ok) {
throw new Error("Failed to get the file size");
}
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10);
await prepareFileDecryption(id, {
isLegacy: !!isLegacy,
dataKey: dataKey.key,
encContentSize,
contentType,
});
return getDecryptedFileUrl(id); return getDecryptedFileUrl(id);
}; };
+1 -1
View File
@@ -2,7 +2,7 @@ import { storeFileThumbnailCache } from "#lib/client/cache/index.js";
import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js"; import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js";
import { invalidateDirectory } from "./directory"; import { invalidateDirectory } from "./directory";
// Uploads the file and caches its thumbnail locally. Returns `false` if the upload is canceled. // Uploads the file and caches it with its thumbnail locally. Returns `false` if the upload is canceled.
export const uploadFile = async ( export const uploadFile = async (
file: File, file: File,
parentId: DirectoryId, parentId: DirectoryId,
+8 -1
View File
@@ -39,17 +39,22 @@ const createByteQueue = () => {
}; };
// Decrypts the encrypted chunks covering the plaintext `range` (as returned by // Decrypts the encrypted chunks covering the plaintext `range` (as returned by
// `getEncryptedRange`) into the plaintext of the range // `getEncryptedRange`) into the plaintext of the range. `onChunk` receives each decrypted chunk
// as a whole, along with its index in the file.
export const createDecryptionStream = ({ export const createDecryptionStream = ({
range, range,
firstChunkIndex = 0,
chunkCount, chunkCount,
dataKey, dataKey,
isLegacy, isLegacy,
onChunk,
}: { }: {
range: ByteRange; range: ByteRange;
firstChunkIndex?: number;
chunkCount: number; chunkCount: number;
dataKey: CryptoKey; dataKey: CryptoKey;
isLegacy: boolean; isLegacy: boolean;
onChunk?: (chunkIndex: number, chunk: Uint8Array) => void;
}) => { }) => {
const queue = createByteQueue(); const queue = createByteQueue();
let chunkIndex = 0; let chunkIndex = 0;
@@ -59,6 +64,8 @@ export const createDecryptionStream = ({
encryptedSize: number, encryptedSize: number,
) => { ) => {
const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey)); const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey));
onChunk?.(firstChunkIndex + chunkIndex, decrypted);
const isFirst = chunkIndex === 0; const isFirst = chunkIndex === 0;
const isLast = chunkIndex === chunkCount - 1; const isLast = chunkIndex === chunkCount - 1;
chunkIndex++; chunkIndex++;
+2 -1
View File
@@ -3,7 +3,8 @@ export const DECRYPTED_FILE_URL_PREFIX = "/_internal/decryptedFile/";
export interface FileMetadata { export interface FileMetadata {
isLegacy: boolean; isLegacy: boolean;
dataKey: CryptoKey; dataKey: CryptoKey;
encContentSize: number; // Fetched by the worker when needed
encContentSize?: number;
contentType: string; contentType: string;
} }
+3
View File
@@ -34,6 +34,9 @@
{/if} {/if}
<p class="text-xs text-gray-800"> <p class="text-xs text-gray-800">
읽음 {formatDate(index.lastRetrievedAt)} · {formatFileSize(index.size)} 읽음 {formatDate(index.lastRetrievedAt)} · {formatFileSize(index.size)}
{#if !index.isComplete}
· 일부
{/if}
</p> </p>
</div> </div>
<button <button
+179 -46
View File
@@ -1,6 +1,15 @@
import { getFileCache, storeFileCache } from "#lib/client/cache/fileCache.js";
import { import {
getCachedChunkIndexes,
readCachedChunk,
storeCachedChunk,
touchFileCache,
type FileCacheLayout,
} from "#lib/client/cache/fileCache.js";
import { getFileCacheIndexOf } from "#lib/client/storage/indexedDB/cacheIndex.js";
import {
CHUNK_SIZE,
createDecryptionStream, createDecryptionStream,
getChunkCount,
getDecryptedSize, getDecryptedSize,
getEncryptedRange, getEncryptedRange,
} from "#lib/shared/format/index.js"; } from "#lib/shared/format/index.js";
@@ -22,6 +31,13 @@ interface StreamRequest {
downloadFilename?: string; downloadFilename?: string;
} }
// A run of consecutive chunks which are either all cached or all not cached
interface Segment {
isCached: boolean;
firstChunkIndex: number;
lastChunkIndex: number;
}
const createResponse = ( const createResponse = (
stream: ReadableStream<Uint8Array>, stream: ReadableStream<Uint8Array>,
{ rangeRequest, downloadFilename }: StreamRequest, { rangeRequest, downloadFilename }: StreamRequest,
@@ -56,58 +72,183 @@ const createRangeNotSatisfiableResponse = (total: number) => {
}); });
}; };
const streamFromCache = (request: StreamRequest, cache: File, metadata?: FileMetadata) => { const splitIntoSegments = (
const range = resolveRange(request.rangeRequest, cache.size); firstChunkIndex: number,
if (!range) { lastChunkIndex: number,
return createRangeNotSatisfiableResponse(cache.size); cached: Set<number>,
) => {
const segments: Segment[] = [];
for (let chunkIndex = firstChunkIndex; chunkIndex <= lastChunkIndex; chunkIndex++) {
const isCached = cached.has(chunkIndex);
const last = segments.at(-1);
if (last?.isCached === isCached) {
last.lastChunkIndex = chunkIndex;
} else {
segments.push({ isCached, firstChunkIndex: chunkIndex, lastChunkIndex: chunkIndex });
}
} }
return segments;
return createResponse(
cache.slice(range.start, range.end + 1).stream(),
request,
range,
metadata?.contentType,
);
}; };
const streamFromServer = async (request: StreamRequest, metadata: FileMetadata) => { // Clips the range to the chunks of the segment
const getSegmentRange = (
range: ByteRange,
{ firstChunkIndex, lastChunkIndex }: Segment,
{ chunkCount }: FileCacheLayout,
) => {
if (chunkCount === 1) return range;
return {
start: Math.max(range.start, firstChunkIndex * CHUNK_SIZE),
end: Math.min(range.end, (lastChunkIndex + 1) * CHUNK_SIZE - 1),
total: range.total,
};
};
async function* readCachedSegment(
fileId: number,
layout: FileCacheLayout,
segment: Segment,
range: ByteRange,
) {
const { start, end } = getSegmentRange(range, segment, layout);
for (
let chunkIndex = segment.firstChunkIndex;
chunkIndex <= segment.lastChunkIndex;
chunkIndex++
) {
const chunk = await readCachedChunk(fileId, chunkIndex);
if (!chunk) {
throw new Error("Cached chunk not found");
}
const chunkStart = chunkIndex * CHUNK_SIZE;
yield* chunk
.slice(Math.max(start - chunkStart, 0), Math.min(end - chunkStart + 1, chunk.size))
.stream();
}
}
const getEncContentSize = async (fileId: number, metadata: FileMetadata) => {
if (metadata.encContentSize === undefined) {
const res = await fetch(`/api/file/${fileId}/download`, { method: "HEAD" });
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "", 10);
if (!res.ok || isNaN(encContentSize)) {
throw new Error("Failed to get the file size");
}
metadata.encContentSize = encContentSize;
}
return metadata.encContentSize;
};
async function* fetchSegment(
request: StreamRequest,
metadata: FileMetadata,
layout: FileCacheLayout,
segment: Segment,
range: ByteRange,
) {
const segmentRange = getSegmentRange(range, segment, layout);
const encryptedRange = getEncryptedRange(
segmentRange.start,
segmentRange.end,
await getEncContentSize(request.fileId, metadata),
metadata.isLegacy,
);
const res = await fetch(`/api/file/${request.fileId}/download`, {
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
});
if (res.status !== 206 || !res.body) {
throw new Error("Failed to fetch encrypted file");
}
yield* res.body.pipeThrough(
createDecryptionStream({
range: segmentRange,
firstChunkIndex: segment.firstChunkIndex,
chunkCount: segment.lastChunkIndex - segment.firstChunkIndex + 1,
dataKey: metadata.dataKey,
isLegacy: metadata.isLegacy,
// Chunks are written atomically, so the worker being terminated doesn't corrupt the cache
onChunk: (chunkIndex, chunk) => {
storeCachedChunk(request.fileId, chunkIndex, chunk, layout).catch(console.error);
},
}),
);
}
const toStream = (iterable: AsyncIterable<Uint8Array>) => {
const iterator = iterable[Symbol.asyncIterator]();
return new ReadableStream<Uint8Array>({
async pull(controller) {
const { done, value } = await iterator.next();
if (done) {
controller.close();
} else {
controller.enqueue(value);
}
},
async cancel() {
await iterator.return?.();
},
});
};
const streamDecryptedFile = async (request: StreamRequest, metadata?: FileMetadata) => {
const { fileId, clientId, rangeRequest } = request; const { fileId, clientId, rangeRequest } = request;
const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy); // Completely cached files can be served without the server or the metadata
const cacheIndex = await getFileCacheIndexOf(fileId);
let layout: FileCacheLayout;
if (cacheIndex?.isComplete) {
layout = { totalSize: cacheIndex.totalSize, chunkCount: cacheIndex.chunkCount };
} else if (metadata) {
try {
const encContentSize = await getEncContentSize(fileId, metadata);
const totalSize = getDecryptedSize(encContentSize, metadata.isLegacy);
layout = { totalSize, chunkCount: metadata.isLegacy ? 1 : getChunkCount(totalSize) };
} catch {
return new Response("Failed to get the file size", { status: 502 });
}
} else {
return new Response("Decryption not prepared", { status: 400 });
}
const { totalSize } = layout;
const range = resolveRange(rangeRequest, totalSize); const range = resolveRange(rangeRequest, totalSize);
if (!range) { if (!range) {
return createRangeNotSatisfiableResponse(totalSize); return createRangeNotSatisfiableResponse(totalSize);
} }
const encryptedRange = getEncryptedRange( // Legacy files consist of a single chunk
range.start, const getChunkIndex = (offset: number) =>
range.end, layout.chunkCount === 1 ? 0 : Math.floor(offset / CHUNK_SIZE);
metadata.encContentSize, const segments = splitIntoSegments(
metadata.isLegacy, getChunkIndex(range.start),
getChunkIndex(range.end),
await getCachedChunkIndexes(fileId),
); );
const apiResponse = await fetch(`/api/file/${fileId}/download`, { if (!metadata && segments.some(({ isCached }) => !isCached)) {
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` }, return new Response("Decryption not prepared", { status: 400 });
}); } else if (segments.some(({ isCached }) => isCached)) {
if (apiResponse.status !== 206 || !apiResponse.body) { void touchFileCache(fileId);
return new Response("Failed to fetch encrypted file", { status: 502 });
} }
let stream = apiResponse.body.pipeThrough( // Segments are read lazily, so that nothing is fetched if the client stops reading (e.g. seeking)
createDecryptionStream({ let stream = toStream(
range, (async function* () {
chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1, for (const segment of segments) {
dataKey: metadata.dataKey, yield* segment.isCached
isLegacy: metadata.isLegacy, ? readCachedSegment(fileId, layout, segment, range)
}), : fetchSegment(request, metadata!, layout, segment, range);
}
})(),
); );
// Cache and report the progress only when the whole content is requested (i.e. not seeking) // Report the progress only when the whole content is fetched (i.e. viewing or saving the file)
if (!rangeRequest) { if (!rangeRequest && segments.some(({ isCached }) => !isCached)) {
const [responseStream, cacheStream] = stream.tee(); stream = trackDownloadProgress(stream, fileId, totalSize, clientId);
storeFileCache(fileId, cacheStream).catch(console.error);
stream = trackDownloadProgress(responseStream, fileId, totalSize, clientId);
} }
return createResponse(stream, request, range, metadata.contentType); return createResponse(stream, request, range, metadata?.contentType);
}; };
const decryptFileHandler = async (request: Request, clientId: string) => { const decryptFileHandler = async (request: Request, clientId: string) => {
@@ -123,15 +264,7 @@ const decryptFileHandler = async (request: Request, clientId: string) => {
rangeRequest: parseRangeHeader(request.headers.get("Range")), rangeRequest: parseRangeHeader(request.headers.get("Range")),
downloadFilename: url.searchParams.get("download") ?? undefined, downloadFilename: url.searchParams.get("download") ?? undefined,
}; };
const metadata = await getFileMetadata(fileId, clientId); return streamDecryptedFile(streamRequest, await getFileMetadata(fileId, clientId));
const cache = await getFileCache(fileId);
if (cache) {
return streamFromCache(streamRequest, cache, metadata);
} else if (metadata) {
return streamFromServer(streamRequest, metadata);
} else {
return new Response("Decryption not prepared", { status: 400 });
}
}; };
export default decryptFileHandler; export default decryptFileHandler;