mirror of
https://github.com/kmc7468/arkvault.git
synced 2026-11-04 00:00:28 +09:00
파일을 업로드할 때 캐시에 저장하고 동영상을 스트리밍하는 과정에서도 청크 단위로 캐시에 저장하여 다음 재생에 활용하도록 개선
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
06fdfcc447
commit
225ad3515f
12 files changed
+449
-112
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import type { ClientInit } from "@sveltejs/kit/hooks";
|
||||
import { cleanupFileCaches } from "#lib/client/cache/index.js";
|
||||
import { keyring } from "#lib/client/keys/index.js";
|
||||
import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js";
|
||||
import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js";
|
||||
@@ -18,4 +19,5 @@ export const init: ClientInit = async () => {
|
||||
await Promise.all([requestPersistentStorage(), keyring.load()]);
|
||||
|
||||
cleanupDanglingInfos(); // Intended
|
||||
cleanupFileCaches().catch(console.error);
|
||||
};
|
||||
Vendored
+159
-24
@@ -5,46 +5,181 @@ import {
|
||||
storeFileCacheIndex,
|
||||
updateFileCacheRetrievedAt,
|
||||
} from "#lib/client/storage/indexedDB/cacheIndex.js";
|
||||
import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js";
|
||||
import {
|
||||
createWritable,
|
||||
deleteDirectory,
|
||||
deleteFile,
|
||||
deleteFilesInDirectory,
|
||||
getFile,
|
||||
listDirectoryNames,
|
||||
listFileNames,
|
||||
moveFile,
|
||||
} from "#lib/client/storage/opfs.js";
|
||||
import { CHUNK_SIZE, getChunkCount } from "#lib/shared/format/index.js";
|
||||
|
||||
// Used by both the pages and the service worker, so the index is always read from IndexedDB
|
||||
// Caches the plaintext of files in chunks, so that partially loaded files (e.g. videos which were
|
||||
// partially played) can also be cached. Chunk `i` starts at `i * CHUNK_SIZE` of the plaintext,
|
||||
// except that legacy files are cached as a single chunk. Used by both the pages and the service
|
||||
// worker, so the index is always read from IndexedDB.
|
||||
|
||||
const getPath = (fileId: number) => `/cache/${fileId}`;
|
||||
export interface FileCacheLayout {
|
||||
totalSize: number;
|
||||
chunkCount: number;
|
||||
}
|
||||
|
||||
const getDirectoryPath = (fileId: number) => `/cache/file/${fileId}`;
|
||||
const getChunkPath = (fileId: number, chunkIndex: number) =>
|
||||
`${getDirectoryPath(fileId)}/${chunkIndex}`;
|
||||
|
||||
const getChunkSize = ({ totalSize, chunkCount }: FileCacheLayout, chunkIndex: number) => {
|
||||
return chunkCount === 1 ? totalSize : Math.min(CHUNK_SIZE, totalSize - chunkIndex * CHUNK_SIZE);
|
||||
};
|
||||
|
||||
export const getFileCacheIndex = async () => {
|
||||
return await getAllFileCacheIndexes();
|
||||
};
|
||||
|
||||
// The index is written only after the whole content is written
|
||||
export const getFileCache = async (fileId: number) => {
|
||||
if (!(await getFileCacheIndexOf(fileId))) return null;
|
||||
|
||||
const file = await getFile(getPath(fileId));
|
||||
if (file) {
|
||||
void updateFileCacheRetrievedAt(fileId, new Date());
|
||||
}
|
||||
return file;
|
||||
export const getCachedChunkIndexes = async (fileId: number) => {
|
||||
const names = await listFileNames(getDirectoryPath(fileId));
|
||||
return new Set(names.filter((name) => /^\d+$/.test(name)).map(Number));
|
||||
};
|
||||
|
||||
export const storeFileCache = async (fileId: number, stream: ReadableStream<Uint8Array>) => {
|
||||
const path = getPath(fileId);
|
||||
export const readCachedChunk = async (fileId: number, chunkIndex: number) => {
|
||||
return await getFile(getChunkPath(fileId, chunkIndex));
|
||||
};
|
||||
|
||||
export const touchFileCache = async (fileId: number) => {
|
||||
await updateFileCacheRetrievedAt(fileId, new Date());
|
||||
};
|
||||
|
||||
const updateIndex = async (fileId: number, layout: FileCacheLayout) => {
|
||||
const chunkIndexes = await getCachedChunkIndexes(fileId);
|
||||
const size = chunkIndexes
|
||||
.values()
|
||||
.filter((chunkIndex) => chunkIndex < layout.chunkCount)
|
||||
.reduce((size, chunkIndex) => size + getChunkSize(layout, chunkIndex), 0);
|
||||
const now = new Date();
|
||||
|
||||
await storeFileCacheIndex({
|
||||
fileId,
|
||||
cachedAt: (await getFileCacheIndexOf(fileId))?.cachedAt ?? now,
|
||||
lastRetrievedAt: now,
|
||||
size,
|
||||
totalSize: layout.totalSize,
|
||||
chunkCount: layout.chunkCount,
|
||||
isComplete: size === layout.totalSize,
|
||||
});
|
||||
};
|
||||
|
||||
// Writes of the same file are serialized so that the index isn't updated with a stale state
|
||||
const writeQueues = new Map<number, Promise<unknown>>();
|
||||
|
||||
const enqueueWrite = <T>(fileId: number, write: () => Promise<T>) => {
|
||||
const result = (writeQueues.get(fileId) ?? Promise.resolve()).catch(() => {}).then(write);
|
||||
const settled = result.catch(() => {});
|
||||
writeQueues.set(fileId, settled);
|
||||
void settled.then(() => {
|
||||
if (writeQueues.get(fileId) === settled) {
|
||||
writeQueues.delete(fileId);
|
||||
}
|
||||
});
|
||||
return result;
|
||||
};
|
||||
|
||||
export const storeCachedChunk = (
|
||||
fileId: number,
|
||||
chunkIndex: number,
|
||||
data: Uint8Array | Blob,
|
||||
layout: FileCacheLayout,
|
||||
) => enqueueWrite(fileId, () => writeChunk(fileId, chunkIndex, data, layout));
|
||||
|
||||
const writeChunk = async (
|
||||
fileId: number,
|
||||
chunkIndex: number,
|
||||
data: Uint8Array | Blob,
|
||||
layout: FileCacheLayout,
|
||||
) => {
|
||||
const path = getChunkPath(fileId, chunkIndex);
|
||||
const tempPath = `${path}.${crypto.randomUUID()}.tmp`;
|
||||
|
||||
// Written to a temporary file first, so that incomplete chunks are never visible
|
||||
const writable = await createWritable(tempPath);
|
||||
try {
|
||||
await stream.pipeTo(await createWritable(path));
|
||||
await writable.write(data as FileSystemWriteChunkType);
|
||||
await writable.close();
|
||||
await moveFile(tempPath, path);
|
||||
} catch (e) {
|
||||
await deleteFile(path).catch(() => {});
|
||||
await writable.abort().catch(() => {});
|
||||
await deleteFile(tempPath).catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
await storeFileCacheIndex({
|
||||
fileId,
|
||||
cachedAt: now,
|
||||
lastRetrievedAt: now,
|
||||
size: (await getFile(path))?.size ?? 0,
|
||||
});
|
||||
await updateIndex(fileId, layout);
|
||||
};
|
||||
|
||||
const UPLOAD_CACHE_DIRECTORY = "/cache/upload";
|
||||
const STALE_UPLOAD_CACHE_AGE = 24 * 3600 * 1000; // 24 hours
|
||||
|
||||
// Caches the plaintext of a file while it is being uploaded. Chunks are written to a temporary
|
||||
// directory, and moved to the cache once the id of the file is known.
|
||||
export const createUploadFileCache = (totalSize: number) => {
|
||||
const layout = { totalSize, chunkCount: getChunkCount(totalSize) };
|
||||
// Prefixed with the creation time to find stale ones (e.g. the tab was closed while uploading)
|
||||
const directoryPath = `${UPLOAD_CACHE_DIRECTORY}/${Date.now()}-${crypto.randomUUID()}`;
|
||||
const writtenChunkIndexes = new Set<number>();
|
||||
let isFailed = false;
|
||||
|
||||
const discard = async () => {
|
||||
isFailed = true;
|
||||
await deleteDirectory(directoryPath).catch(() => {});
|
||||
};
|
||||
|
||||
return {
|
||||
// Failing to cache shouldn't fail the upload, so errors are ignored
|
||||
async writeChunk(chunkIndex: number, data: ArrayBuffer) {
|
||||
if (isFailed) return;
|
||||
|
||||
try {
|
||||
const writable = await createWritable(`${directoryPath}/${chunkIndex}`);
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
writtenChunkIndexes.add(chunkIndex);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
await discard();
|
||||
}
|
||||
},
|
||||
async commit(fileId: number) {
|
||||
if (isFailed || writtenChunkIndexes.size !== layout.chunkCount) {
|
||||
await discard();
|
||||
return;
|
||||
}
|
||||
|
||||
await enqueueWrite(fileId, async () => {
|
||||
for (const chunkIndex of writtenChunkIndexes) {
|
||||
await moveFile(`${directoryPath}/${chunkIndex}`, getChunkPath(fileId, chunkIndex));
|
||||
}
|
||||
await updateIndex(fileId, layout);
|
||||
}).finally(discard);
|
||||
},
|
||||
discard,
|
||||
};
|
||||
};
|
||||
|
||||
export const deleteFileCache = async (fileId: number) => {
|
||||
await deleteFileCacheIndex(fileId);
|
||||
await deleteFile(getPath(fileId));
|
||||
await deleteDirectory(getDirectoryPath(fileId));
|
||||
};
|
||||
|
||||
export const cleanupFileCaches = async () => {
|
||||
// Files were cached as single files directly under `/cache` before
|
||||
await deleteFilesInDirectory("/cache");
|
||||
|
||||
// Uploads might be in progress in other tabs, so only stale ones are deleted
|
||||
for (const name of await listDirectoryNames(UPLOAD_CACHE_DIRECTORY)) {
|
||||
const createdAt = parseInt(name, 10);
|
||||
if (!(Date.now() - createdAt < STALE_UPLOAD_CACHE_AGE)) {
|
||||
await deleteDirectory(`${UPLOAD_CACHE_DIRECTORY}/${name}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -4,7 +4,11 @@ export interface FileCacheIndex {
|
||||
fileId: number;
|
||||
cachedAt: Date;
|
||||
lastRetrievedAt: Date;
|
||||
// Files are cached in chunks, possibly partially (e.g. videos which were partially played)
|
||||
size: number;
|
||||
totalSize: number;
|
||||
chunkCount: number;
|
||||
isComplete: boolean;
|
||||
}
|
||||
|
||||
const cacheIndex = new Dexie("cacheIndex") as Dexie & {
|
||||
@@ -15,23 +19,23 @@ cacheIndex.version(1).stores({
|
||||
fileCache: "fileId",
|
||||
});
|
||||
|
||||
// Files were cached as single files before version 2
|
||||
cacheIndex.version(2).upgrade(async (trx) => {
|
||||
await trx.table("fileCache").clear();
|
||||
});
|
||||
|
||||
export const getFileCacheIndex = async () => {
|
||||
return await cacheIndex.fileCache.toArray();
|
||||
};
|
||||
|
||||
export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
|
||||
await cacheIndex.fileCache.put({
|
||||
fileId: fileCacheIndex.fileId,
|
||||
cachedAt: fileCacheIndex.cachedAt,
|
||||
lastRetrievedAt: fileCacheIndex.lastRetrievedAt,
|
||||
size: fileCacheIndex.size,
|
||||
});
|
||||
};
|
||||
|
||||
export const getFileCacheIndexOf = async (fileId: number) => {
|
||||
return await cacheIndex.fileCache.get(fileId);
|
||||
};
|
||||
|
||||
export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
|
||||
await cacheIndex.fileCache.put(fileCacheIndex);
|
||||
};
|
||||
|
||||
export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => {
|
||||
await cacheIndex.fileCache.update(fileId, { lastRetrievedAt });
|
||||
};
|
||||
|
||||
@@ -94,3 +94,51 @@ export const deleteDirectory = async (path: string) => {
|
||||
|
||||
await parentHandle.removeEntry(directoryHandle.name, { recursive: true });
|
||||
};
|
||||
|
||||
const listEntryNames = async (path: string, kind: FileSystemHandleKind) => {
|
||||
const { directoryHandle } = await getDirectoryHandle(path);
|
||||
if (!directoryHandle) return [];
|
||||
|
||||
const names: string[] = [];
|
||||
for await (const handle of directoryHandle.values()) {
|
||||
if (handle.kind === kind) {
|
||||
names.push(handle.name);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
};
|
||||
|
||||
export const listFileNames = (path: string) => listEntryNames(path, "file");
|
||||
|
||||
export const listDirectoryNames = (path: string) => listEntryNames(path, "directory");
|
||||
|
||||
// Deletes the files (not directories) directly under the directory
|
||||
export const deleteFilesInDirectory = async (path: string) => {
|
||||
const { directoryHandle } = await getDirectoryHandle(path);
|
||||
if (!directoryHandle) return;
|
||||
|
||||
for (const name of await listFileNames(path)) {
|
||||
await directoryHandle.removeEntry(name);
|
||||
}
|
||||
};
|
||||
|
||||
// Renames the file atomically
|
||||
export const moveFile = async (path: string, newPath: string) => {
|
||||
const { fileHandle } = await getFileHandle(path, false);
|
||||
if (!fileHandle) {
|
||||
throw new Error("File not found");
|
||||
}
|
||||
|
||||
let directoryHandle = await navigator.storage.getDirectory();
|
||||
const parts = newPath.split("/").filter((part) => part);
|
||||
for (const part of parts.slice(0, -1)) {
|
||||
directoryHandle = await directoryHandle.getDirectoryHandle(part, { create: true });
|
||||
}
|
||||
|
||||
// Supported by Chromium for the origin private file system
|
||||
await (
|
||||
fileHandle as FileSystemFileHandle & {
|
||||
move(destination: FileSystemDirectoryHandle, name: string): Promise<void>;
|
||||
}
|
||||
).move(directoryHandle, parts.at(-1)!);
|
||||
};
|
||||
@@ -16,10 +16,6 @@ interface EncryptedChunk {
|
||||
data: ArrayBuffer;
|
||||
}
|
||||
|
||||
const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
|
||||
return await encryptChunk(await chunk.arrayBuffer(), dataKey);
|
||||
};
|
||||
|
||||
const uploadEncryptedChunk = async (
|
||||
uploadId: string,
|
||||
chunkIndex: number,
|
||||
@@ -40,7 +36,12 @@ export const uploadBlob = async (
|
||||
uploadId: string,
|
||||
blob: Blob,
|
||||
dataKey: CryptoKey,
|
||||
options?: { concurrency?: number; onProgress?: (s: UploadStats) => void },
|
||||
options?: {
|
||||
concurrency?: number;
|
||||
onProgress?: (s: UploadStats) => void;
|
||||
// Receives each chunk of the plaintext before it is encrypted
|
||||
onPlaintextChunk?: (chunkIndex: number, data: ArrayBuffer) => Promise<void>;
|
||||
},
|
||||
) => {
|
||||
const onProgress = options?.onProgress;
|
||||
const networkConcurrency = options?.concurrency ?? 4;
|
||||
@@ -75,8 +76,9 @@ export const uploadBlob = async (
|
||||
const encryptionProducer = async () => {
|
||||
try {
|
||||
for (let i = 0; i < totalChunks; i++) {
|
||||
const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
|
||||
const data = await encryptChunkData(chunk, dataKey);
|
||||
const chunk = await blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE).arrayBuffer();
|
||||
await options?.onPlaintextChunk?.(i, chunk);
|
||||
const data = await encryptChunk(chunk, dataKey);
|
||||
hash.update(new Uint8Array(data));
|
||||
await queue.push({ index: i, data });
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type HmacSecret,
|
||||
type MasterKey,
|
||||
} from "#lib/client/keys/index.js";
|
||||
import { createUploadFileCache } from "#lib/client/cache/index.js";
|
||||
import { readExifDateTime } from "#lib/client/media/exif.js";
|
||||
import { generateThumbnail } from "#lib/client/media/thumbnail.js";
|
||||
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
|
||||
@@ -150,18 +151,30 @@ const requestFileUpload = limitFunction(
|
||||
) => {
|
||||
state.status = "uploading";
|
||||
|
||||
const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
|
||||
onProgress(s) {
|
||||
state.progress = s.progress;
|
||||
state.rate = s.rate;
|
||||
},
|
||||
});
|
||||
// The file is cached while uploading, so that it doesn't have to be read again
|
||||
const cache = createUploadFileCache(file.size);
|
||||
|
||||
const { file: fileId } = await trpc.upload.completeFileUpload.mutate({
|
||||
uploadId,
|
||||
contentHmac: fileSigned,
|
||||
encContentHash,
|
||||
});
|
||||
let fileId;
|
||||
try {
|
||||
const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
|
||||
onProgress(s) {
|
||||
state.progress = s.progress;
|
||||
state.rate = s.rate;
|
||||
},
|
||||
onPlaintextChunk: cache.writeChunk,
|
||||
});
|
||||
|
||||
({ file: fileId } = await trpc.upload.completeFileUpload.mutate({
|
||||
uploadId,
|
||||
contentHmac: fileSigned,
|
||||
encContentHash,
|
||||
}));
|
||||
} catch (e) {
|
||||
await cache.discard();
|
||||
throw e;
|
||||
}
|
||||
|
||||
await cache.commit(fileId).catch(console.error);
|
||||
|
||||
if (thumbnailBuffer) {
|
||||
try {
|
||||
|
||||
@@ -26,18 +26,7 @@ export const getFileStreamUrl = async ({ id, isLegacy, dataKey, contentType }: S
|
||||
throw new Error("Data key not loaded");
|
||||
}
|
||||
|
||||
const res = await fetch(`/api/file/${id}/download`, { method: "HEAD" });
|
||||
if (!res.ok) {
|
||||
throw new Error("Failed to get the file size");
|
||||
}
|
||||
|
||||
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10);
|
||||
await prepareFileDecryption(id, {
|
||||
isLegacy: !!isLegacy,
|
||||
dataKey: dataKey.key,
|
||||
encContentSize,
|
||||
contentType,
|
||||
});
|
||||
await prepareFileDecryption(id, { isLegacy: !!isLegacy, dataKey: dataKey.key, contentType });
|
||||
return getDecryptedFileUrl(id);
|
||||
};
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { storeFileThumbnailCache } from "#lib/client/cache/index.js";
|
||||
import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js";
|
||||
import { invalidateDirectory } from "./directory";
|
||||
|
||||
// Uploads the file and caches its thumbnail locally. Returns `false` if the upload is canceled.
|
||||
// Uploads the file and caches it with its thumbnail locally. Returns `false` if the upload is canceled.
|
||||
export const uploadFile = async (
|
||||
file: File,
|
||||
parentId: DirectoryId,
|
||||
|
||||
@@ -39,17 +39,22 @@ const createByteQueue = () => {
|
||||
};
|
||||
|
||||
// Decrypts the encrypted chunks covering the plaintext `range` (as returned by
|
||||
// `getEncryptedRange`) into the plaintext of the range
|
||||
// `getEncryptedRange`) into the plaintext of the range. `onChunk` receives each decrypted chunk
|
||||
// as a whole, along with its index in the file.
|
||||
export const createDecryptionStream = ({
|
||||
range,
|
||||
firstChunkIndex = 0,
|
||||
chunkCount,
|
||||
dataKey,
|
||||
isLegacy,
|
||||
onChunk,
|
||||
}: {
|
||||
range: ByteRange;
|
||||
firstChunkIndex?: number;
|
||||
chunkCount: number;
|
||||
dataKey: CryptoKey;
|
||||
isLegacy: boolean;
|
||||
onChunk?: (chunkIndex: number, chunk: Uint8Array) => void;
|
||||
}) => {
|
||||
const queue = createByteQueue();
|
||||
let chunkIndex = 0;
|
||||
@@ -59,6 +64,8 @@ export const createDecryptionStream = ({
|
||||
encryptedSize: number,
|
||||
) => {
|
||||
const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey));
|
||||
onChunk?.(firstChunkIndex + chunkIndex, decrypted);
|
||||
|
||||
const isFirst = chunkIndex === 0;
|
||||
const isLast = chunkIndex === chunkCount - 1;
|
||||
chunkIndex++;
|
||||
|
||||
@@ -3,7 +3,8 @@ export const DECRYPTED_FILE_URL_PREFIX = "/_internal/decryptedFile/";
|
||||
export interface FileMetadata {
|
||||
isLegacy: boolean;
|
||||
dataKey: CryptoKey;
|
||||
encContentSize: number;
|
||||
// Fetched by the worker when needed
|
||||
encContentSize?: number;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,9 @@
|
||||
{/if}
|
||||
<p class="text-xs text-gray-800">
|
||||
읽음 {formatDate(index.lastRetrievedAt)} · {formatFileSize(index.size)}
|
||||
{#if !index.isComplete}
|
||||
· 일부
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import { getFileCache, storeFileCache } from "#lib/client/cache/fileCache.js";
|
||||
import {
|
||||
getCachedChunkIndexes,
|
||||
readCachedChunk,
|
||||
storeCachedChunk,
|
||||
touchFileCache,
|
||||
type FileCacheLayout,
|
||||
} from "#lib/client/cache/fileCache.js";
|
||||
import { getFileCacheIndexOf } from "#lib/client/storage/indexedDB/cacheIndex.js";
|
||||
import {
|
||||
CHUNK_SIZE,
|
||||
createDecryptionStream,
|
||||
getChunkCount,
|
||||
getDecryptedSize,
|
||||
getEncryptedRange,
|
||||
} from "#lib/shared/format/index.js";
|
||||
@@ -22,6 +31,13 @@ interface StreamRequest {
|
||||
downloadFilename?: string;
|
||||
}
|
||||
|
||||
// A run of consecutive chunks which are either all cached or all not cached
|
||||
interface Segment {
|
||||
isCached: boolean;
|
||||
firstChunkIndex: number;
|
||||
lastChunkIndex: number;
|
||||
}
|
||||
|
||||
const createResponse = (
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
{ rangeRequest, downloadFilename }: StreamRequest,
|
||||
@@ -56,58 +72,183 @@ const createRangeNotSatisfiableResponse = (total: number) => {
|
||||
});
|
||||
};
|
||||
|
||||
const streamFromCache = (request: StreamRequest, cache: File, metadata?: FileMetadata) => {
|
||||
const range = resolveRange(request.rangeRequest, cache.size);
|
||||
if (!range) {
|
||||
return createRangeNotSatisfiableResponse(cache.size);
|
||||
const splitIntoSegments = (
|
||||
firstChunkIndex: number,
|
||||
lastChunkIndex: number,
|
||||
cached: Set<number>,
|
||||
) => {
|
||||
const segments: Segment[] = [];
|
||||
for (let chunkIndex = firstChunkIndex; chunkIndex <= lastChunkIndex; chunkIndex++) {
|
||||
const isCached = cached.has(chunkIndex);
|
||||
const last = segments.at(-1);
|
||||
if (last?.isCached === isCached) {
|
||||
last.lastChunkIndex = chunkIndex;
|
||||
} else {
|
||||
segments.push({ isCached, firstChunkIndex: chunkIndex, lastChunkIndex: chunkIndex });
|
||||
}
|
||||
}
|
||||
|
||||
return createResponse(
|
||||
cache.slice(range.start, range.end + 1).stream(),
|
||||
request,
|
||||
range,
|
||||
metadata?.contentType,
|
||||
);
|
||||
return segments;
|
||||
};
|
||||
|
||||
const streamFromServer = async (request: StreamRequest, metadata: FileMetadata) => {
|
||||
// Clips the range to the chunks of the segment
|
||||
const getSegmentRange = (
|
||||
range: ByteRange,
|
||||
{ firstChunkIndex, lastChunkIndex }: Segment,
|
||||
{ chunkCount }: FileCacheLayout,
|
||||
) => {
|
||||
if (chunkCount === 1) return range;
|
||||
return {
|
||||
start: Math.max(range.start, firstChunkIndex * CHUNK_SIZE),
|
||||
end: Math.min(range.end, (lastChunkIndex + 1) * CHUNK_SIZE - 1),
|
||||
total: range.total,
|
||||
};
|
||||
};
|
||||
|
||||
async function* readCachedSegment(
|
||||
fileId: number,
|
||||
layout: FileCacheLayout,
|
||||
segment: Segment,
|
||||
range: ByteRange,
|
||||
) {
|
||||
const { start, end } = getSegmentRange(range, segment, layout);
|
||||
for (
|
||||
let chunkIndex = segment.firstChunkIndex;
|
||||
chunkIndex <= segment.lastChunkIndex;
|
||||
chunkIndex++
|
||||
) {
|
||||
const chunk = await readCachedChunk(fileId, chunkIndex);
|
||||
if (!chunk) {
|
||||
throw new Error("Cached chunk not found");
|
||||
}
|
||||
|
||||
const chunkStart = chunkIndex * CHUNK_SIZE;
|
||||
yield* chunk
|
||||
.slice(Math.max(start - chunkStart, 0), Math.min(end - chunkStart + 1, chunk.size))
|
||||
.stream();
|
||||
}
|
||||
}
|
||||
|
||||
const getEncContentSize = async (fileId: number, metadata: FileMetadata) => {
|
||||
if (metadata.encContentSize === undefined) {
|
||||
const res = await fetch(`/api/file/${fileId}/download`, { method: "HEAD" });
|
||||
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "", 10);
|
||||
if (!res.ok || isNaN(encContentSize)) {
|
||||
throw new Error("Failed to get the file size");
|
||||
}
|
||||
metadata.encContentSize = encContentSize;
|
||||
}
|
||||
return metadata.encContentSize;
|
||||
};
|
||||
|
||||
async function* fetchSegment(
|
||||
request: StreamRequest,
|
||||
metadata: FileMetadata,
|
||||
layout: FileCacheLayout,
|
||||
segment: Segment,
|
||||
range: ByteRange,
|
||||
) {
|
||||
const segmentRange = getSegmentRange(range, segment, layout);
|
||||
const encryptedRange = getEncryptedRange(
|
||||
segmentRange.start,
|
||||
segmentRange.end,
|
||||
await getEncContentSize(request.fileId, metadata),
|
||||
metadata.isLegacy,
|
||||
);
|
||||
const res = await fetch(`/api/file/${request.fileId}/download`, {
|
||||
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
|
||||
});
|
||||
if (res.status !== 206 || !res.body) {
|
||||
throw new Error("Failed to fetch encrypted file");
|
||||
}
|
||||
|
||||
yield* res.body.pipeThrough(
|
||||
createDecryptionStream({
|
||||
range: segmentRange,
|
||||
firstChunkIndex: segment.firstChunkIndex,
|
||||
chunkCount: segment.lastChunkIndex - segment.firstChunkIndex + 1,
|
||||
dataKey: metadata.dataKey,
|
||||
isLegacy: metadata.isLegacy,
|
||||
// Chunks are written atomically, so the worker being terminated doesn't corrupt the cache
|
||||
onChunk: (chunkIndex, chunk) => {
|
||||
storeCachedChunk(request.fileId, chunkIndex, chunk, layout).catch(console.error);
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const toStream = (iterable: AsyncIterable<Uint8Array>) => {
|
||||
const iterator = iterable[Symbol.asyncIterator]();
|
||||
return new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
const { done, value } = await iterator.next();
|
||||
if (done) {
|
||||
controller.close();
|
||||
} else {
|
||||
controller.enqueue(value);
|
||||
}
|
||||
},
|
||||
async cancel() {
|
||||
await iterator.return?.();
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const streamDecryptedFile = async (request: StreamRequest, metadata?: FileMetadata) => {
|
||||
const { fileId, clientId, rangeRequest } = request;
|
||||
const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy);
|
||||
// Completely cached files can be served without the server or the metadata
|
||||
const cacheIndex = await getFileCacheIndexOf(fileId);
|
||||
let layout: FileCacheLayout;
|
||||
if (cacheIndex?.isComplete) {
|
||||
layout = { totalSize: cacheIndex.totalSize, chunkCount: cacheIndex.chunkCount };
|
||||
} else if (metadata) {
|
||||
try {
|
||||
const encContentSize = await getEncContentSize(fileId, metadata);
|
||||
const totalSize = getDecryptedSize(encContentSize, metadata.isLegacy);
|
||||
layout = { totalSize, chunkCount: metadata.isLegacy ? 1 : getChunkCount(totalSize) };
|
||||
} catch {
|
||||
return new Response("Failed to get the file size", { status: 502 });
|
||||
}
|
||||
} else {
|
||||
return new Response("Decryption not prepared", { status: 400 });
|
||||
}
|
||||
|
||||
const { totalSize } = layout;
|
||||
const range = resolveRange(rangeRequest, totalSize);
|
||||
if (!range) {
|
||||
return createRangeNotSatisfiableResponse(totalSize);
|
||||
}
|
||||
|
||||
const encryptedRange = getEncryptedRange(
|
||||
range.start,
|
||||
range.end,
|
||||
metadata.encContentSize,
|
||||
metadata.isLegacy,
|
||||
// Legacy files consist of a single chunk
|
||||
const getChunkIndex = (offset: number) =>
|
||||
layout.chunkCount === 1 ? 0 : Math.floor(offset / CHUNK_SIZE);
|
||||
const segments = splitIntoSegments(
|
||||
getChunkIndex(range.start),
|
||||
getChunkIndex(range.end),
|
||||
await getCachedChunkIndexes(fileId),
|
||||
);
|
||||
const apiResponse = await fetch(`/api/file/${fileId}/download`, {
|
||||
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
|
||||
});
|
||||
if (apiResponse.status !== 206 || !apiResponse.body) {
|
||||
return new Response("Failed to fetch encrypted file", { status: 502 });
|
||||
if (!metadata && segments.some(({ isCached }) => !isCached)) {
|
||||
return new Response("Decryption not prepared", { status: 400 });
|
||||
} else if (segments.some(({ isCached }) => isCached)) {
|
||||
void touchFileCache(fileId);
|
||||
}
|
||||
|
||||
let stream = apiResponse.body.pipeThrough(
|
||||
createDecryptionStream({
|
||||
range,
|
||||
chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1,
|
||||
dataKey: metadata.dataKey,
|
||||
isLegacy: metadata.isLegacy,
|
||||
}),
|
||||
// Segments are read lazily, so that nothing is fetched if the client stops reading (e.g. seeking)
|
||||
let stream = toStream(
|
||||
(async function* () {
|
||||
for (const segment of segments) {
|
||||
yield* segment.isCached
|
||||
? readCachedSegment(fileId, layout, segment, range)
|
||||
: fetchSegment(request, metadata!, layout, segment, range);
|
||||
}
|
||||
})(),
|
||||
);
|
||||
|
||||
// Cache and report the progress only when the whole content is requested (i.e. not seeking)
|
||||
if (!rangeRequest) {
|
||||
const [responseStream, cacheStream] = stream.tee();
|
||||
storeFileCache(fileId, cacheStream).catch(console.error);
|
||||
stream = trackDownloadProgress(responseStream, fileId, totalSize, clientId);
|
||||
// Report the progress only when the whole content is fetched (i.e. viewing or saving the file)
|
||||
if (!rangeRequest && segments.some(({ isCached }) => !isCached)) {
|
||||
stream = trackDownloadProgress(stream, fileId, totalSize, clientId);
|
||||
}
|
||||
|
||||
return createResponse(stream, request, range, metadata.contentType);
|
||||
return createResponse(stream, request, range, metadata?.contentType);
|
||||
};
|
||||
|
||||
const decryptFileHandler = async (request: Request, clientId: string) => {
|
||||
@@ -123,15 +264,7 @@ const decryptFileHandler = async (request: Request, clientId: string) => {
|
||||
rangeRequest: parseRangeHeader(request.headers.get("Range")),
|
||||
downloadFilename: url.searchParams.get("download") ?? undefined,
|
||||
};
|
||||
const metadata = await getFileMetadata(fileId, clientId);
|
||||
const cache = await getFileCache(fileId);
|
||||
if (cache) {
|
||||
return streamFromCache(streamRequest, cache, metadata);
|
||||
} else if (metadata) {
|
||||
return streamFromServer(streamRequest, metadata);
|
||||
} else {
|
||||
return new Response("Decryption not prepared", { status: 400 });
|
||||
}
|
||||
return streamDecryptedFile(streamRequest, await getFileMetadata(fileId, clientId));
|
||||
};
|
||||
|
||||
export default decryptFileHandler;
|
||||
Reference in new issue
Block a user