mirror of
https://github.com/kmc7468/arkvault.git
synced 2026-11-04 08:10:28 +09:00
파일을 업로드할 때 캐시에 저장하고 동영상을 스트리밍하는 과정에서도 청크 단위로 캐시에 저장하여 다음 재생에 활용하도록 개선
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
06fdfcc447
commit
225ad3515f
12 files changed
+449
-112
No files matched your search
@@ -1,4 +1,5 @@
|
|||||||
import type { ClientInit } from "@sveltejs/kit/hooks";
|
import type { ClientInit } from "@sveltejs/kit/hooks";
|
||||||
|
import { cleanupFileCaches } from "#lib/client/cache/index.js";
|
||||||
import { keyring } from "#lib/client/keys/index.js";
|
import { keyring } from "#lib/client/keys/index.js";
|
||||||
import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js";
|
import { listenForServiceWorkerMessages } from "#lib/client/serviceWorker.js";
|
||||||
import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js";
|
import { cleanupDanglingInfos } from "#lib/client/storage/indexedDB/index.js";
|
||||||
@@ -18,4 +19,5 @@ export const init: ClientInit = async () => {
|
|||||||
await Promise.all([requestPersistentStorage(), keyring.load()]);
|
await Promise.all([requestPersistentStorage(), keyring.load()]);
|
||||||
|
|
||||||
cleanupDanglingInfos(); // Intended
|
cleanupDanglingInfos(); // Intended
|
||||||
|
cleanupFileCaches().catch(console.error);
|
||||||
};
|
};
|
||||||
Vendored
+159
-24
@@ -5,46 +5,181 @@ import {
|
|||||||
storeFileCacheIndex,
|
storeFileCacheIndex,
|
||||||
updateFileCacheRetrievedAt,
|
updateFileCacheRetrievedAt,
|
||||||
} from "#lib/client/storage/indexedDB/cacheIndex.js";
|
} from "#lib/client/storage/indexedDB/cacheIndex.js";
|
||||||
import { createWritable, deleteFile, getFile } from "#lib/client/storage/opfs.js";
|
import {
|
||||||
|
createWritable,
|
||||||
|
deleteDirectory,
|
||||||
|
deleteFile,
|
||||||
|
deleteFilesInDirectory,
|
||||||
|
getFile,
|
||||||
|
listDirectoryNames,
|
||||||
|
listFileNames,
|
||||||
|
moveFile,
|
||||||
|
} from "#lib/client/storage/opfs.js";
|
||||||
|
import { CHUNK_SIZE, getChunkCount } from "#lib/shared/format/index.js";
|
||||||
|
|
||||||
// Used by both the pages and the service worker, so the index is always read from IndexedDB
|
// Caches the plaintext of files in chunks, so that partially loaded files (e.g. videos which were
|
||||||
|
// partially played) can also be cached. Chunk `i` starts at `i * CHUNK_SIZE` of the plaintext,
|
||||||
|
// except that legacy files are cached as a single chunk. Used by both the pages and the service
|
||||||
|
// worker, so the index is always read from IndexedDB.
|
||||||
|
|
||||||
const getPath = (fileId: number) => `/cache/${fileId}`;
|
export interface FileCacheLayout {
|
||||||
|
totalSize: number;
|
||||||
|
chunkCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const getDirectoryPath = (fileId: number) => `/cache/file/${fileId}`;
|
||||||
|
const getChunkPath = (fileId: number, chunkIndex: number) =>
|
||||||
|
`${getDirectoryPath(fileId)}/${chunkIndex}`;
|
||||||
|
|
||||||
|
const getChunkSize = ({ totalSize, chunkCount }: FileCacheLayout, chunkIndex: number) => {
|
||||||
|
return chunkCount === 1 ? totalSize : Math.min(CHUNK_SIZE, totalSize - chunkIndex * CHUNK_SIZE);
|
||||||
|
};
|
||||||
|
|
||||||
export const getFileCacheIndex = async () => {
|
export const getFileCacheIndex = async () => {
|
||||||
return await getAllFileCacheIndexes();
|
return await getAllFileCacheIndexes();
|
||||||
};
|
};
|
||||||
|
|
||||||
// The index is written only after the whole content is written
|
export const getCachedChunkIndexes = async (fileId: number) => {
|
||||||
export const getFileCache = async (fileId: number) => {
|
const names = await listFileNames(getDirectoryPath(fileId));
|
||||||
if (!(await getFileCacheIndexOf(fileId))) return null;
|
return new Set(names.filter((name) => /^\d+$/.test(name)).map(Number));
|
||||||
|
|
||||||
const file = await getFile(getPath(fileId));
|
|
||||||
if (file) {
|
|
||||||
void updateFileCacheRetrievedAt(fileId, new Date());
|
|
||||||
}
|
|
||||||
return file;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const storeFileCache = async (fileId: number, stream: ReadableStream<Uint8Array>) => {
|
export const readCachedChunk = async (fileId: number, chunkIndex: number) => {
|
||||||
const path = getPath(fileId);
|
return await getFile(getChunkPath(fileId, chunkIndex));
|
||||||
|
};
|
||||||
|
|
||||||
|
export const touchFileCache = async (fileId: number) => {
|
||||||
|
await updateFileCacheRetrievedAt(fileId, new Date());
|
||||||
|
};
|
||||||
|
|
||||||
|
const updateIndex = async (fileId: number, layout: FileCacheLayout) => {
|
||||||
|
const chunkIndexes = await getCachedChunkIndexes(fileId);
|
||||||
|
const size = chunkIndexes
|
||||||
|
.values()
|
||||||
|
.filter((chunkIndex) => chunkIndex < layout.chunkCount)
|
||||||
|
.reduce((size, chunkIndex) => size + getChunkSize(layout, chunkIndex), 0);
|
||||||
|
const now = new Date();
|
||||||
|
|
||||||
|
await storeFileCacheIndex({
|
||||||
|
fileId,
|
||||||
|
cachedAt: (await getFileCacheIndexOf(fileId))?.cachedAt ?? now,
|
||||||
|
lastRetrievedAt: now,
|
||||||
|
size,
|
||||||
|
totalSize: layout.totalSize,
|
||||||
|
chunkCount: layout.chunkCount,
|
||||||
|
isComplete: size === layout.totalSize,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
// Writes of the same file are serialized so that the index isn't updated with a stale state
|
||||||
|
const writeQueues = new Map<number, Promise<unknown>>();
|
||||||
|
|
||||||
|
const enqueueWrite = <T>(fileId: number, write: () => Promise<T>) => {
|
||||||
|
const result = (writeQueues.get(fileId) ?? Promise.resolve()).catch(() => {}).then(write);
|
||||||
|
const settled = result.catch(() => {});
|
||||||
|
writeQueues.set(fileId, settled);
|
||||||
|
void settled.then(() => {
|
||||||
|
if (writeQueues.get(fileId) === settled) {
|
||||||
|
writeQueues.delete(fileId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const storeCachedChunk = (
|
||||||
|
fileId: number,
|
||||||
|
chunkIndex: number,
|
||||||
|
data: Uint8Array | Blob,
|
||||||
|
layout: FileCacheLayout,
|
||||||
|
) => enqueueWrite(fileId, () => writeChunk(fileId, chunkIndex, data, layout));
|
||||||
|
|
||||||
|
const writeChunk = async (
|
||||||
|
fileId: number,
|
||||||
|
chunkIndex: number,
|
||||||
|
data: Uint8Array | Blob,
|
||||||
|
layout: FileCacheLayout,
|
||||||
|
) => {
|
||||||
|
const path = getChunkPath(fileId, chunkIndex);
|
||||||
|
const tempPath = `${path}.${crypto.randomUUID()}.tmp`;
|
||||||
|
|
||||||
|
// Written to a temporary file first, so that incomplete chunks are never visible
|
||||||
|
const writable = await createWritable(tempPath);
|
||||||
try {
|
try {
|
||||||
await stream.pipeTo(await createWritable(path));
|
await writable.write(data as FileSystemWriteChunkType);
|
||||||
|
await writable.close();
|
||||||
|
await moveFile(tempPath, path);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
await deleteFile(path).catch(() => {});
|
await writable.abort().catch(() => {});
|
||||||
|
await deleteFile(tempPath).catch(() => {});
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
|
||||||
const now = new Date();
|
await updateIndex(fileId, layout);
|
||||||
await storeFileCacheIndex({
|
};
|
||||||
fileId,
|
|
||||||
cachedAt: now,
|
const UPLOAD_CACHE_DIRECTORY = "/cache/upload";
|
||||||
lastRetrievedAt: now,
|
const STALE_UPLOAD_CACHE_AGE = 24 * 3600 * 1000; // 24 hours
|
||||||
size: (await getFile(path))?.size ?? 0,
|
|
||||||
});
|
// Caches the plaintext of a file while it is being uploaded. Chunks are written to a temporary
|
||||||
|
// directory, and moved to the cache once the id of the file is known.
|
||||||
|
export const createUploadFileCache = (totalSize: number) => {
|
||||||
|
const layout = { totalSize, chunkCount: getChunkCount(totalSize) };
|
||||||
|
// Prefixed with the creation time to find stale ones (e.g. the tab was closed while uploading)
|
||||||
|
const directoryPath = `${UPLOAD_CACHE_DIRECTORY}/${Date.now()}-${crypto.randomUUID()}`;
|
||||||
|
const writtenChunkIndexes = new Set<number>();
|
||||||
|
let isFailed = false;
|
||||||
|
|
||||||
|
const discard = async () => {
|
||||||
|
isFailed = true;
|
||||||
|
await deleteDirectory(directoryPath).catch(() => {});
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
// Failing to cache shouldn't fail the upload, so errors are ignored
|
||||||
|
async writeChunk(chunkIndex: number, data: ArrayBuffer) {
|
||||||
|
if (isFailed) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const writable = await createWritable(`${directoryPath}/${chunkIndex}`);
|
||||||
|
await writable.write(data);
|
||||||
|
await writable.close();
|
||||||
|
writtenChunkIndexes.add(chunkIndex);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
await discard();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
async commit(fileId: number) {
|
||||||
|
if (isFailed || writtenChunkIndexes.size !== layout.chunkCount) {
|
||||||
|
await discard();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await enqueueWrite(fileId, async () => {
|
||||||
|
for (const chunkIndex of writtenChunkIndexes) {
|
||||||
|
await moveFile(`${directoryPath}/${chunkIndex}`, getChunkPath(fileId, chunkIndex));
|
||||||
|
}
|
||||||
|
await updateIndex(fileId, layout);
|
||||||
|
}).finally(discard);
|
||||||
|
},
|
||||||
|
discard,
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export const deleteFileCache = async (fileId: number) => {
|
export const deleteFileCache = async (fileId: number) => {
|
||||||
await deleteFileCacheIndex(fileId);
|
await deleteFileCacheIndex(fileId);
|
||||||
await deleteFile(getPath(fileId));
|
await deleteDirectory(getDirectoryPath(fileId));
|
||||||
|
};
|
||||||
|
|
||||||
|
export const cleanupFileCaches = async () => {
|
||||||
|
// Files were cached as single files directly under `/cache` before
|
||||||
|
await deleteFilesInDirectory("/cache");
|
||||||
|
|
||||||
|
// Uploads might be in progress in other tabs, so only stale ones are deleted
|
||||||
|
for (const name of await listDirectoryNames(UPLOAD_CACHE_DIRECTORY)) {
|
||||||
|
const createdAt = parseInt(name, 10);
|
||||||
|
if (!(Date.now() - createdAt < STALE_UPLOAD_CACHE_AGE)) {
|
||||||
|
await deleteDirectory(`${UPLOAD_CACHE_DIRECTORY}/${name}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
@@ -4,7 +4,11 @@ export interface FileCacheIndex {
|
|||||||
fileId: number;
|
fileId: number;
|
||||||
cachedAt: Date;
|
cachedAt: Date;
|
||||||
lastRetrievedAt: Date;
|
lastRetrievedAt: Date;
|
||||||
|
// Files are cached in chunks, possibly partially (e.g. videos which were partially played)
|
||||||
size: number;
|
size: number;
|
||||||
|
totalSize: number;
|
||||||
|
chunkCount: number;
|
||||||
|
isComplete: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const cacheIndex = new Dexie("cacheIndex") as Dexie & {
|
const cacheIndex = new Dexie("cacheIndex") as Dexie & {
|
||||||
@@ -15,23 +19,23 @@ cacheIndex.version(1).stores({
|
|||||||
fileCache: "fileId",
|
fileCache: "fileId",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Files were cached as single files before version 2
|
||||||
|
cacheIndex.version(2).upgrade(async (trx) => {
|
||||||
|
await trx.table("fileCache").clear();
|
||||||
|
});
|
||||||
|
|
||||||
export const getFileCacheIndex = async () => {
|
export const getFileCacheIndex = async () => {
|
||||||
return await cacheIndex.fileCache.toArray();
|
return await cacheIndex.fileCache.toArray();
|
||||||
};
|
};
|
||||||
|
|
||||||
export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
|
|
||||||
await cacheIndex.fileCache.put({
|
|
||||||
fileId: fileCacheIndex.fileId,
|
|
||||||
cachedAt: fileCacheIndex.cachedAt,
|
|
||||||
lastRetrievedAt: fileCacheIndex.lastRetrievedAt,
|
|
||||||
size: fileCacheIndex.size,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getFileCacheIndexOf = async (fileId: number) => {
|
export const getFileCacheIndexOf = async (fileId: number) => {
|
||||||
return await cacheIndex.fileCache.get(fileId);
|
return await cacheIndex.fileCache.get(fileId);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const storeFileCacheIndex = async (fileCacheIndex: FileCacheIndex) => {
|
||||||
|
await cacheIndex.fileCache.put(fileCacheIndex);
|
||||||
|
};
|
||||||
|
|
||||||
export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => {
|
export const updateFileCacheRetrievedAt = async (fileId: number, lastRetrievedAt: Date) => {
|
||||||
await cacheIndex.fileCache.update(fileId, { lastRetrievedAt });
|
await cacheIndex.fileCache.update(fileId, { lastRetrievedAt });
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -94,3 +94,51 @@ export const deleteDirectory = async (path: string) => {
|
|||||||
|
|
||||||
await parentHandle.removeEntry(directoryHandle.name, { recursive: true });
|
await parentHandle.removeEntry(directoryHandle.name, { recursive: true });
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const listEntryNames = async (path: string, kind: FileSystemHandleKind) => {
|
||||||
|
const { directoryHandle } = await getDirectoryHandle(path);
|
||||||
|
if (!directoryHandle) return [];
|
||||||
|
|
||||||
|
const names: string[] = [];
|
||||||
|
for await (const handle of directoryHandle.values()) {
|
||||||
|
if (handle.kind === kind) {
|
||||||
|
names.push(handle.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const listFileNames = (path: string) => listEntryNames(path, "file");
|
||||||
|
|
||||||
|
export const listDirectoryNames = (path: string) => listEntryNames(path, "directory");
|
||||||
|
|
||||||
|
// Deletes the files (not directories) directly under the directory
|
||||||
|
export const deleteFilesInDirectory = async (path: string) => {
|
||||||
|
const { directoryHandle } = await getDirectoryHandle(path);
|
||||||
|
if (!directoryHandle) return;
|
||||||
|
|
||||||
|
for (const name of await listFileNames(path)) {
|
||||||
|
await directoryHandle.removeEntry(name);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Renames the file atomically
|
||||||
|
export const moveFile = async (path: string, newPath: string) => {
|
||||||
|
const { fileHandle } = await getFileHandle(path, false);
|
||||||
|
if (!fileHandle) {
|
||||||
|
throw new Error("File not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
let directoryHandle = await navigator.storage.getDirectory();
|
||||||
|
const parts = newPath.split("/").filter((part) => part);
|
||||||
|
for (const part of parts.slice(0, -1)) {
|
||||||
|
directoryHandle = await directoryHandle.getDirectoryHandle(part, { create: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Supported by Chromium for the origin private file system
|
||||||
|
await (
|
||||||
|
fileHandle as FileSystemFileHandle & {
|
||||||
|
move(destination: FileSystemDirectoryHandle, name: string): Promise<void>;
|
||||||
|
}
|
||||||
|
).move(directoryHandle, parts.at(-1)!);
|
||||||
|
};
|
||||||
@@ -16,10 +16,6 @@ interface EncryptedChunk {
|
|||||||
data: ArrayBuffer;
|
data: ArrayBuffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
|
|
||||||
return await encryptChunk(await chunk.arrayBuffer(), dataKey);
|
|
||||||
};
|
|
||||||
|
|
||||||
const uploadEncryptedChunk = async (
|
const uploadEncryptedChunk = async (
|
||||||
uploadId: string,
|
uploadId: string,
|
||||||
chunkIndex: number,
|
chunkIndex: number,
|
||||||
@@ -40,7 +36,12 @@ export const uploadBlob = async (
|
|||||||
uploadId: string,
|
uploadId: string,
|
||||||
blob: Blob,
|
blob: Blob,
|
||||||
dataKey: CryptoKey,
|
dataKey: CryptoKey,
|
||||||
options?: { concurrency?: number; onProgress?: (s: UploadStats) => void },
|
options?: {
|
||||||
|
concurrency?: number;
|
||||||
|
onProgress?: (s: UploadStats) => void;
|
||||||
|
// Receives each chunk of the plaintext before it is encrypted
|
||||||
|
onPlaintextChunk?: (chunkIndex: number, data: ArrayBuffer) => Promise<void>;
|
||||||
|
},
|
||||||
) => {
|
) => {
|
||||||
const onProgress = options?.onProgress;
|
const onProgress = options?.onProgress;
|
||||||
const networkConcurrency = options?.concurrency ?? 4;
|
const networkConcurrency = options?.concurrency ?? 4;
|
||||||
@@ -75,8 +76,9 @@ export const uploadBlob = async (
|
|||||||
const encryptionProducer = async () => {
|
const encryptionProducer = async () => {
|
||||||
try {
|
try {
|
||||||
for (let i = 0; i < totalChunks; i++) {
|
for (let i = 0; i < totalChunks; i++) {
|
||||||
const chunk = blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
|
const chunk = await blob.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE).arrayBuffer();
|
||||||
const data = await encryptChunkData(chunk, dataKey);
|
await options?.onPlaintextChunk?.(i, chunk);
|
||||||
|
const data = await encryptChunk(chunk, dataKey);
|
||||||
hash.update(new Uint8Array(data));
|
hash.update(new Uint8Array(data));
|
||||||
await queue.push({ index: i, data });
|
await queue.push({ index: i, data });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
type HmacSecret,
|
type HmacSecret,
|
||||||
type MasterKey,
|
type MasterKey,
|
||||||
} from "#lib/client/keys/index.js";
|
} from "#lib/client/keys/index.js";
|
||||||
|
import { createUploadFileCache } from "#lib/client/cache/index.js";
|
||||||
import { readExifDateTime } from "#lib/client/media/exif.js";
|
import { readExifDateTime } from "#lib/client/media/exif.js";
|
||||||
import { generateThumbnail } from "#lib/client/media/thumbnail.js";
|
import { generateThumbnail } from "#lib/client/media/thumbnail.js";
|
||||||
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
|
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
|
||||||
@@ -150,18 +151,30 @@ const requestFileUpload = limitFunction(
|
|||||||
) => {
|
) => {
|
||||||
state.status = "uploading";
|
state.status = "uploading";
|
||||||
|
|
||||||
const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
|
// The file is cached while uploading, so that it doesn't have to be read again
|
||||||
onProgress(s) {
|
const cache = createUploadFileCache(file.size);
|
||||||
state.progress = s.progress;
|
|
||||||
state.rate = s.rate;
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const { file: fileId } = await trpc.upload.completeFileUpload.mutate({
|
let fileId;
|
||||||
uploadId,
|
try {
|
||||||
contentHmac: fileSigned,
|
const { encContentHash } = await uploadBlob(uploadId, file, dataKey, {
|
||||||
encContentHash,
|
onProgress(s) {
|
||||||
});
|
state.progress = s.progress;
|
||||||
|
state.rate = s.rate;
|
||||||
|
},
|
||||||
|
onPlaintextChunk: cache.writeChunk,
|
||||||
|
});
|
||||||
|
|
||||||
|
({ file: fileId } = await trpc.upload.completeFileUpload.mutate({
|
||||||
|
uploadId,
|
||||||
|
contentHmac: fileSigned,
|
||||||
|
encContentHash,
|
||||||
|
}));
|
||||||
|
} catch (e) {
|
||||||
|
await cache.discard();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
|
||||||
|
await cache.commit(fileId).catch(console.error);
|
||||||
|
|
||||||
if (thumbnailBuffer) {
|
if (thumbnailBuffer) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -26,18 +26,7 @@ export const getFileStreamUrl = async ({ id, isLegacy, dataKey, contentType }: S
|
|||||||
throw new Error("Data key not loaded");
|
throw new Error("Data key not loaded");
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = await fetch(`/api/file/${id}/download`, { method: "HEAD" });
|
await prepareFileDecryption(id, { isLegacy: !!isLegacy, dataKey: dataKey.key, contentType });
|
||||||
if (!res.ok) {
|
|
||||||
throw new Error("Failed to get the file size");
|
|
||||||
}
|
|
||||||
|
|
||||||
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10);
|
|
||||||
await prepareFileDecryption(id, {
|
|
||||||
isLegacy: !!isLegacy,
|
|
||||||
dataKey: dataKey.key,
|
|
||||||
encContentSize,
|
|
||||||
contentType,
|
|
||||||
});
|
|
||||||
return getDecryptedFileUrl(id);
|
return getDecryptedFileUrl(id);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { storeFileThumbnailCache } from "#lib/client/cache/index.js";
|
|||||||
import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js";
|
import { uploadFile as doUploadFile } from "#lib/client/transfer/index.js";
|
||||||
import { invalidateDirectory } from "./directory";
|
import { invalidateDirectory } from "./directory";
|
||||||
|
|
||||||
// Uploads the file and caches its thumbnail locally. Returns `false` if the upload is canceled.
|
// Uploads the file and caches it with its thumbnail locally. Returns `false` if the upload is canceled.
|
||||||
export const uploadFile = async (
|
export const uploadFile = async (
|
||||||
file: File,
|
file: File,
|
||||||
parentId: DirectoryId,
|
parentId: DirectoryId,
|
||||||
|
|||||||
@@ -39,17 +39,22 @@ const createByteQueue = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Decrypts the encrypted chunks covering the plaintext `range` (as returned by
|
// Decrypts the encrypted chunks covering the plaintext `range` (as returned by
|
||||||
// `getEncryptedRange`) into the plaintext of the range
|
// `getEncryptedRange`) into the plaintext of the range. `onChunk` receives each decrypted chunk
|
||||||
|
// as a whole, along with its index in the file.
|
||||||
export const createDecryptionStream = ({
|
export const createDecryptionStream = ({
|
||||||
range,
|
range,
|
||||||
|
firstChunkIndex = 0,
|
||||||
chunkCount,
|
chunkCount,
|
||||||
dataKey,
|
dataKey,
|
||||||
isLegacy,
|
isLegacy,
|
||||||
|
onChunk,
|
||||||
}: {
|
}: {
|
||||||
range: ByteRange;
|
range: ByteRange;
|
||||||
|
firstChunkIndex?: number;
|
||||||
chunkCount: number;
|
chunkCount: number;
|
||||||
dataKey: CryptoKey;
|
dataKey: CryptoKey;
|
||||||
isLegacy: boolean;
|
isLegacy: boolean;
|
||||||
|
onChunk?: (chunkIndex: number, chunk: Uint8Array) => void;
|
||||||
}) => {
|
}) => {
|
||||||
const queue = createByteQueue();
|
const queue = createByteQueue();
|
||||||
let chunkIndex = 0;
|
let chunkIndex = 0;
|
||||||
@@ -59,6 +64,8 @@ export const createDecryptionStream = ({
|
|||||||
encryptedSize: number,
|
encryptedSize: number,
|
||||||
) => {
|
) => {
|
||||||
const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey));
|
const decrypted = new Uint8Array(await decryptChunk(queue.take(encryptedSize).buffer, dataKey));
|
||||||
|
onChunk?.(firstChunkIndex + chunkIndex, decrypted);
|
||||||
|
|
||||||
const isFirst = chunkIndex === 0;
|
const isFirst = chunkIndex === 0;
|
||||||
const isLast = chunkIndex === chunkCount - 1;
|
const isLast = chunkIndex === chunkCount - 1;
|
||||||
chunkIndex++;
|
chunkIndex++;
|
||||||
|
|||||||
@@ -3,7 +3,8 @@ export const DECRYPTED_FILE_URL_PREFIX = "/_internal/decryptedFile/";
|
|||||||
export interface FileMetadata {
|
export interface FileMetadata {
|
||||||
isLegacy: boolean;
|
isLegacy: boolean;
|
||||||
dataKey: CryptoKey;
|
dataKey: CryptoKey;
|
||||||
encContentSize: number;
|
// Fetched by the worker when needed
|
||||||
|
encContentSize?: number;
|
||||||
contentType: string;
|
contentType: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,9 @@
|
|||||||
{/if}
|
{/if}
|
||||||
<p class="text-xs text-gray-800">
|
<p class="text-xs text-gray-800">
|
||||||
읽음 {formatDate(index.lastRetrievedAt)} · {formatFileSize(index.size)}
|
읽음 {formatDate(index.lastRetrievedAt)} · {formatFileSize(index.size)}
|
||||||
|
{#if !index.isComplete}
|
||||||
|
· 일부
|
||||||
|
{/if}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
|
|||||||
@@ -1,6 +1,15 @@
|
|||||||
import { getFileCache, storeFileCache } from "#lib/client/cache/fileCache.js";
|
|
||||||
import {
|
import {
|
||||||
|
getCachedChunkIndexes,
|
||||||
|
readCachedChunk,
|
||||||
|
storeCachedChunk,
|
||||||
|
touchFileCache,
|
||||||
|
type FileCacheLayout,
|
||||||
|
} from "#lib/client/cache/fileCache.js";
|
||||||
|
import { getFileCacheIndexOf } from "#lib/client/storage/indexedDB/cacheIndex.js";
|
||||||
|
import {
|
||||||
|
CHUNK_SIZE,
|
||||||
createDecryptionStream,
|
createDecryptionStream,
|
||||||
|
getChunkCount,
|
||||||
getDecryptedSize,
|
getDecryptedSize,
|
||||||
getEncryptedRange,
|
getEncryptedRange,
|
||||||
} from "#lib/shared/format/index.js";
|
} from "#lib/shared/format/index.js";
|
||||||
@@ -22,6 +31,13 @@ interface StreamRequest {
|
|||||||
downloadFilename?: string;
|
downloadFilename?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A run of consecutive chunks which are either all cached or all not cached
|
||||||
|
interface Segment {
|
||||||
|
isCached: boolean;
|
||||||
|
firstChunkIndex: number;
|
||||||
|
lastChunkIndex: number;
|
||||||
|
}
|
||||||
|
|
||||||
const createResponse = (
|
const createResponse = (
|
||||||
stream: ReadableStream<Uint8Array>,
|
stream: ReadableStream<Uint8Array>,
|
||||||
{ rangeRequest, downloadFilename }: StreamRequest,
|
{ rangeRequest, downloadFilename }: StreamRequest,
|
||||||
@@ -56,58 +72,183 @@ const createRangeNotSatisfiableResponse = (total: number) => {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const streamFromCache = (request: StreamRequest, cache: File, metadata?: FileMetadata) => {
|
const splitIntoSegments = (
|
||||||
const range = resolveRange(request.rangeRequest, cache.size);
|
firstChunkIndex: number,
|
||||||
if (!range) {
|
lastChunkIndex: number,
|
||||||
return createRangeNotSatisfiableResponse(cache.size);
|
cached: Set<number>,
|
||||||
|
) => {
|
||||||
|
const segments: Segment[] = [];
|
||||||
|
for (let chunkIndex = firstChunkIndex; chunkIndex <= lastChunkIndex; chunkIndex++) {
|
||||||
|
const isCached = cached.has(chunkIndex);
|
||||||
|
const last = segments.at(-1);
|
||||||
|
if (last?.isCached === isCached) {
|
||||||
|
last.lastChunkIndex = chunkIndex;
|
||||||
|
} else {
|
||||||
|
segments.push({ isCached, firstChunkIndex: chunkIndex, lastChunkIndex: chunkIndex });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
return segments;
|
||||||
return createResponse(
|
|
||||||
cache.slice(range.start, range.end + 1).stream(),
|
|
||||||
request,
|
|
||||||
range,
|
|
||||||
metadata?.contentType,
|
|
||||||
);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const streamFromServer = async (request: StreamRequest, metadata: FileMetadata) => {
|
// Clips the range to the chunks of the segment
|
||||||
|
const getSegmentRange = (
|
||||||
|
range: ByteRange,
|
||||||
|
{ firstChunkIndex, lastChunkIndex }: Segment,
|
||||||
|
{ chunkCount }: FileCacheLayout,
|
||||||
|
) => {
|
||||||
|
if (chunkCount === 1) return range;
|
||||||
|
return {
|
||||||
|
start: Math.max(range.start, firstChunkIndex * CHUNK_SIZE),
|
||||||
|
end: Math.min(range.end, (lastChunkIndex + 1) * CHUNK_SIZE - 1),
|
||||||
|
total: range.total,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
async function* readCachedSegment(
|
||||||
|
fileId: number,
|
||||||
|
layout: FileCacheLayout,
|
||||||
|
segment: Segment,
|
||||||
|
range: ByteRange,
|
||||||
|
) {
|
||||||
|
const { start, end } = getSegmentRange(range, segment, layout);
|
||||||
|
for (
|
||||||
|
let chunkIndex = segment.firstChunkIndex;
|
||||||
|
chunkIndex <= segment.lastChunkIndex;
|
||||||
|
chunkIndex++
|
||||||
|
) {
|
||||||
|
const chunk = await readCachedChunk(fileId, chunkIndex);
|
||||||
|
if (!chunk) {
|
||||||
|
throw new Error("Cached chunk not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
const chunkStart = chunkIndex * CHUNK_SIZE;
|
||||||
|
yield* chunk
|
||||||
|
.slice(Math.max(start - chunkStart, 0), Math.min(end - chunkStart + 1, chunk.size))
|
||||||
|
.stream();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const getEncContentSize = async (fileId: number, metadata: FileMetadata) => {
|
||||||
|
if (metadata.encContentSize === undefined) {
|
||||||
|
const res = await fetch(`/api/file/${fileId}/download`, { method: "HEAD" });
|
||||||
|
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "", 10);
|
||||||
|
if (!res.ok || isNaN(encContentSize)) {
|
||||||
|
throw new Error("Failed to get the file size");
|
||||||
|
}
|
||||||
|
metadata.encContentSize = encContentSize;
|
||||||
|
}
|
||||||
|
return metadata.encContentSize;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function* fetchSegment(
|
||||||
|
request: StreamRequest,
|
||||||
|
metadata: FileMetadata,
|
||||||
|
layout: FileCacheLayout,
|
||||||
|
segment: Segment,
|
||||||
|
range: ByteRange,
|
||||||
|
) {
|
||||||
|
const segmentRange = getSegmentRange(range, segment, layout);
|
||||||
|
const encryptedRange = getEncryptedRange(
|
||||||
|
segmentRange.start,
|
||||||
|
segmentRange.end,
|
||||||
|
await getEncContentSize(request.fileId, metadata),
|
||||||
|
metadata.isLegacy,
|
||||||
|
);
|
||||||
|
const res = await fetch(`/api/file/${request.fileId}/download`, {
|
||||||
|
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
|
||||||
|
});
|
||||||
|
if (res.status !== 206 || !res.body) {
|
||||||
|
throw new Error("Failed to fetch encrypted file");
|
||||||
|
}
|
||||||
|
|
||||||
|
yield* res.body.pipeThrough(
|
||||||
|
createDecryptionStream({
|
||||||
|
range: segmentRange,
|
||||||
|
firstChunkIndex: segment.firstChunkIndex,
|
||||||
|
chunkCount: segment.lastChunkIndex - segment.firstChunkIndex + 1,
|
||||||
|
dataKey: metadata.dataKey,
|
||||||
|
isLegacy: metadata.isLegacy,
|
||||||
|
// Chunks are written atomically, so the worker being terminated doesn't corrupt the cache
|
||||||
|
onChunk: (chunkIndex, chunk) => {
|
||||||
|
storeCachedChunk(request.fileId, chunkIndex, chunk, layout).catch(console.error);
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const toStream = (iterable: AsyncIterable<Uint8Array>) => {
|
||||||
|
const iterator = iterable[Symbol.asyncIterator]();
|
||||||
|
return new ReadableStream<Uint8Array>({
|
||||||
|
async pull(controller) {
|
||||||
|
const { done, value } = await iterator.next();
|
||||||
|
if (done) {
|
||||||
|
controller.close();
|
||||||
|
} else {
|
||||||
|
controller.enqueue(value);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
async cancel() {
|
||||||
|
await iterator.return?.();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const streamDecryptedFile = async (request: StreamRequest, metadata?: FileMetadata) => {
|
||||||
const { fileId, clientId, rangeRequest } = request;
|
const { fileId, clientId, rangeRequest } = request;
|
||||||
const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy);
|
// Completely cached files can be served without the server or the metadata
|
||||||
|
const cacheIndex = await getFileCacheIndexOf(fileId);
|
||||||
|
let layout: FileCacheLayout;
|
||||||
|
if (cacheIndex?.isComplete) {
|
||||||
|
layout = { totalSize: cacheIndex.totalSize, chunkCount: cacheIndex.chunkCount };
|
||||||
|
} else if (metadata) {
|
||||||
|
try {
|
||||||
|
const encContentSize = await getEncContentSize(fileId, metadata);
|
||||||
|
const totalSize = getDecryptedSize(encContentSize, metadata.isLegacy);
|
||||||
|
layout = { totalSize, chunkCount: metadata.isLegacy ? 1 : getChunkCount(totalSize) };
|
||||||
|
} catch {
|
||||||
|
return new Response("Failed to get the file size", { status: 502 });
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return new Response("Decryption not prepared", { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const { totalSize } = layout;
|
||||||
const range = resolveRange(rangeRequest, totalSize);
|
const range = resolveRange(rangeRequest, totalSize);
|
||||||
if (!range) {
|
if (!range) {
|
||||||
return createRangeNotSatisfiableResponse(totalSize);
|
return createRangeNotSatisfiableResponse(totalSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
const encryptedRange = getEncryptedRange(
|
// Legacy files consist of a single chunk
|
||||||
range.start,
|
const getChunkIndex = (offset: number) =>
|
||||||
range.end,
|
layout.chunkCount === 1 ? 0 : Math.floor(offset / CHUNK_SIZE);
|
||||||
metadata.encContentSize,
|
const segments = splitIntoSegments(
|
||||||
metadata.isLegacy,
|
getChunkIndex(range.start),
|
||||||
|
getChunkIndex(range.end),
|
||||||
|
await getCachedChunkIndexes(fileId),
|
||||||
);
|
);
|
||||||
const apiResponse = await fetch(`/api/file/${fileId}/download`, {
|
if (!metadata && segments.some(({ isCached }) => !isCached)) {
|
||||||
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
|
return new Response("Decryption not prepared", { status: 400 });
|
||||||
});
|
} else if (segments.some(({ isCached }) => isCached)) {
|
||||||
if (apiResponse.status !== 206 || !apiResponse.body) {
|
void touchFileCache(fileId);
|
||||||
return new Response("Failed to fetch encrypted file", { status: 502 });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let stream = apiResponse.body.pipeThrough(
|
// Segments are read lazily, so that nothing is fetched if the client stops reading (e.g. seeking)
|
||||||
createDecryptionStream({
|
let stream = toStream(
|
||||||
range,
|
(async function* () {
|
||||||
chunkCount: encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1,
|
for (const segment of segments) {
|
||||||
dataKey: metadata.dataKey,
|
yield* segment.isCached
|
||||||
isLegacy: metadata.isLegacy,
|
? readCachedSegment(fileId, layout, segment, range)
|
||||||
}),
|
: fetchSegment(request, metadata!, layout, segment, range);
|
||||||
|
}
|
||||||
|
})(),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Cache and report the progress only when the whole content is requested (i.e. not seeking)
|
// Report the progress only when the whole content is fetched (i.e. viewing or saving the file)
|
||||||
if (!rangeRequest) {
|
if (!rangeRequest && segments.some(({ isCached }) => !isCached)) {
|
||||||
const [responseStream, cacheStream] = stream.tee();
|
stream = trackDownloadProgress(stream, fileId, totalSize, clientId);
|
||||||
storeFileCache(fileId, cacheStream).catch(console.error);
|
|
||||||
stream = trackDownloadProgress(responseStream, fileId, totalSize, clientId);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return createResponse(stream, request, range, metadata.contentType);
|
return createResponse(stream, request, range, metadata?.contentType);
|
||||||
};
|
};
|
||||||
|
|
||||||
const decryptFileHandler = async (request: Request, clientId: string) => {
|
const decryptFileHandler = async (request: Request, clientId: string) => {
|
||||||
@@ -123,15 +264,7 @@ const decryptFileHandler = async (request: Request, clientId: string) => {
|
|||||||
rangeRequest: parseRangeHeader(request.headers.get("Range")),
|
rangeRequest: parseRangeHeader(request.headers.get("Range")),
|
||||||
downloadFilename: url.searchParams.get("download") ?? undefined,
|
downloadFilename: url.searchParams.get("download") ?? undefined,
|
||||||
};
|
};
|
||||||
const metadata = await getFileMetadata(fileId, clientId);
|
return streamDecryptedFile(streamRequest, await getFileMetadata(fileId, clientId));
|
||||||
const cache = await getFileCache(fileId);
|
|
||||||
if (cache) {
|
|
||||||
return streamFromCache(streamRequest, cache, metadata);
|
|
||||||
} else if (metadata) {
|
|
||||||
return streamFromServer(streamRequest, metadata);
|
|
||||||
} else {
|
|
||||||
return new Response("Decryption not prepared", { status: 400 });
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export default decryptFileHandler;
|
export default decryptFileHandler;
|
||||||
Reference in new issue
Block a user