mirror of
https://github.com/kmc7468/arkvault.git
synced 2026-11-04 08:10:28 +09:00
파일을 다운로드할 때 전체 파일을 메모리에 올리는 대신 서비스 워커에서 스트리밍으로 복호화하도록 변경
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
877cf9c42e
commit
3e52fa7a7c
29 files changed
+629
-467
No files matched your search
@@ -4,6 +4,7 @@ import pLimit from "p-limit";
|
||||
import { encodeToBase64, encryptChunk } from "#lib/shared/crypto/index.js";
|
||||
import { CHUNK_SIZE, getChunkCount, getEncryptedSize } from "#lib/shared/format/index.js";
|
||||
import { BoundedQueue } from "#lib/utils/concurrency/index.js";
|
||||
import { createSpeedMeter } from "#lib/utils/speedMeter.js";
|
||||
|
||||
interface UploadStats {
|
||||
progress: number;
|
||||
@@ -15,59 +16,6 @@ interface EncryptedChunk {
|
||||
data: ArrayBuffer;
|
||||
}
|
||||
|
||||
const createSpeedMeter = (timeWindow = 3000, minInterval = 200, warmupPeriod = 500) => {
|
||||
const samples: { t: number; b: number }[] = [];
|
||||
let lastSpeed = 0;
|
||||
let startTime: number | null = null;
|
||||
|
||||
return (bytesNow?: number) => {
|
||||
if (bytesNow === undefined) return lastSpeed;
|
||||
|
||||
const now = performance.now();
|
||||
|
||||
// Initialize start time on first call
|
||||
if (startTime === null) {
|
||||
startTime = now;
|
||||
}
|
||||
|
||||
// Check if enough time has passed since the last sample
|
||||
const lastSample = samples[samples.length - 1];
|
||||
if (lastSample && now - lastSample.t < minInterval) {
|
||||
return lastSpeed;
|
||||
}
|
||||
|
||||
samples.push({ t: now, b: bytesNow });
|
||||
|
||||
// Remove old samples outside the time window
|
||||
const cutoff = now - timeWindow;
|
||||
while (samples.length > 2 && samples[0]!.t < cutoff) samples.shift();
|
||||
|
||||
// Need at least 2 samples to calculate speed
|
||||
if (samples.length < 2) {
|
||||
return lastSpeed;
|
||||
}
|
||||
|
||||
const first = samples[0]!;
|
||||
const dt = now - first.t;
|
||||
const db = bytesNow - first.b;
|
||||
|
||||
if (dt >= minInterval) {
|
||||
const instantSpeed = (db / dt) * 1000;
|
||||
// Apply EMA for smoother speed transitions
|
||||
const alpha = 0.3;
|
||||
const rawSpeed =
|
||||
lastSpeed === 0 ? instantSpeed : alpha * instantSpeed + (1 - alpha) * lastSpeed;
|
||||
|
||||
// Apply warmup ramp to prevent initial overestimation
|
||||
const elapsed = now - startTime;
|
||||
const warmupWeight = Math.min(1, elapsed / warmupPeriod);
|
||||
lastSpeed = rawSpeed * warmupWeight;
|
||||
}
|
||||
|
||||
return lastSpeed;
|
||||
};
|
||||
};
|
||||
|
||||
const encryptChunkData = async (chunk: Blob, dataKey: CryptoKey): Promise<ArrayBuffer> => {
|
||||
return await encryptChunk(await chunk.arrayBuffer(), dataKey);
|
||||
};
|
||||
|
||||
@@ -1,34 +1,21 @@
|
||||
import axios from "axios";
|
||||
import { limitFunction } from "p-limit";
|
||||
import { decryptChunk, concatenateBuffers } from "#lib/shared/crypto/index.js";
|
||||
import { ENCRYPTED_CHUNK_SIZE } from "#lib/shared/format/index.js";
|
||||
import type { DownloadProgressMessage } from "#lib/shared/serviceWorker.js";
|
||||
import { createSpeedMeter } from "#lib/utils/speedMeter.js";
|
||||
|
||||
// Files are downloaded and decrypted by the service worker, which reports the progress
|
||||
|
||||
export interface FileDownloadState {
|
||||
id: number;
|
||||
status:
|
||||
| "download-pending"
|
||||
| "downloading"
|
||||
| "decryption-pending"
|
||||
| "decrypting"
|
||||
| "decrypted"
|
||||
| "canceled"
|
||||
| "error";
|
||||
status: "downloading" | "completed" | "error";
|
||||
progress?: number;
|
||||
rate?: number;
|
||||
estimated?: number;
|
||||
result?: ArrayBuffer;
|
||||
}
|
||||
|
||||
type LiveFileDownloadState = FileDownloadState & {
|
||||
status: "download-pending" | "downloading" | "decryption-pending" | "decrypting";
|
||||
};
|
||||
|
||||
let downloadingFiles: FileDownloadState[] = $state([]);
|
||||
const speedMeters = new WeakMap<FileDownloadState, ReturnType<typeof createSpeedMeter>>();
|
||||
|
||||
export const isFileDownloading = (
|
||||
status: FileDownloadState["status"],
|
||||
): status is LiveFileDownloadState["status"] =>
|
||||
["download-pending", "downloading", "decryption-pending", "decrypting"].includes(status);
|
||||
export const isFileDownloading = (status: FileDownloadState["status"]) => {
|
||||
return status === "downloading";
|
||||
};
|
||||
|
||||
export const getFileDownloadState = (fileId: number) => {
|
||||
return downloadingFiles.find((file) => file.id === fileId && isFileDownloading(file.status));
|
||||
@@ -42,69 +29,27 @@ export const clearDownloadedFiles = () => {
|
||||
downloadingFiles = downloadingFiles.filter((file) => isFileDownloading(file.status));
|
||||
};
|
||||
|
||||
const requestFileDownload = limitFunction(
|
||||
async (state: FileDownloadState, id: number) => {
|
||||
state.status = "downloading";
|
||||
export const handleDownloadProgress = ({
|
||||
fileId,
|
||||
status,
|
||||
loaded,
|
||||
total,
|
||||
}: DownloadProgressMessage) => {
|
||||
let state = getFileDownloadState(fileId);
|
||||
if (!state) {
|
||||
if (status !== "downloading") return;
|
||||
|
||||
const res = await axios.get(`/api/file/${id}/download`, {
|
||||
responseType: "arraybuffer",
|
||||
onDownloadProgress: ({ progress, rate, estimated }) => {
|
||||
state.progress = progress;
|
||||
state.rate = rate;
|
||||
state.estimated = estimated;
|
||||
},
|
||||
});
|
||||
const fileEncrypted: ArrayBuffer = res.data;
|
||||
|
||||
state.status = "decryption-pending";
|
||||
return fileEncrypted;
|
||||
},
|
||||
{ concurrency: 1 },
|
||||
);
|
||||
|
||||
const decryptFile = limitFunction(
|
||||
async (
|
||||
state: FileDownloadState,
|
||||
fileEncrypted: ArrayBuffer,
|
||||
encryptedChunkSize: number,
|
||||
dataKey: CryptoKey,
|
||||
) => {
|
||||
state.status = "decrypting";
|
||||
|
||||
const chunks: ArrayBuffer[] = [];
|
||||
let offset = 0;
|
||||
|
||||
while (offset < fileEncrypted.byteLength) {
|
||||
const nextOffset = Math.min(offset + encryptedChunkSize, fileEncrypted.byteLength);
|
||||
chunks.push(await decryptChunk(fileEncrypted.slice(offset, nextOffset), dataKey));
|
||||
offset = nextOffset;
|
||||
}
|
||||
|
||||
const fileBuffer = concatenateBuffers(...chunks).buffer;
|
||||
state.status = "decrypted";
|
||||
state.result = fileBuffer;
|
||||
return fileBuffer;
|
||||
},
|
||||
{ concurrency: 4 },
|
||||
);
|
||||
|
||||
export const downloadFile = async (id: number, dataKey: CryptoKey, isLegacy: boolean) => {
|
||||
downloadingFiles.push({
|
||||
id,
|
||||
status: "download-pending",
|
||||
});
|
||||
const state = downloadingFiles.at(-1)!;
|
||||
|
||||
try {
|
||||
const fileEncrypted = await requestFileDownload(state, id);
|
||||
return await decryptFile(
|
||||
state,
|
||||
fileEncrypted,
|
||||
isLegacy ? fileEncrypted.byteLength : ENCRYPTED_CHUNK_SIZE,
|
||||
dataKey,
|
||||
);
|
||||
} catch (e) {
|
||||
state.status = "error";
|
||||
throw e;
|
||||
downloadingFiles.push({ id: fileId, status });
|
||||
state = downloadingFiles.at(-1)!;
|
||||
speedMeters.set(state, createSpeedMeter());
|
||||
}
|
||||
|
||||
if (status === "canceled") {
|
||||
downloadingFiles = downloadingFiles.filter((file) => file !== state);
|
||||
return;
|
||||
}
|
||||
|
||||
state.status = status;
|
||||
state.progress = total > 0 ? loaded / total : 1;
|
||||
state.rate = speedMeters.get(state)?.(loaded);
|
||||
};
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
type HmacSecret,
|
||||
type MasterKey,
|
||||
} from "#lib/client/keys/index.js";
|
||||
import { extractExifDateTime } from "#lib/client/media/exif.js";
|
||||
import { readExifDateTime } from "#lib/client/media/exif.js";
|
||||
import { generateThumbnail } from "#lib/client/media/thumbnail.js";
|
||||
import { uploadBlob } from "#lib/client/transfer/chunkedUpload.js";
|
||||
import {
|
||||
@@ -41,9 +41,7 @@ export type LiveFileUploadState = FileUploadState & {
|
||||
status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
|
||||
};
|
||||
|
||||
const scheduler = new Scheduler<
|
||||
{ fileId: number; fileBuffer?: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
|
||||
>();
|
||||
const scheduler = new Scheduler<{ fileId: number; thumbnailBuffer?: ArrayBuffer } | undefined>();
|
||||
let uploadingFiles: FileUploadState[] = $state([]);
|
||||
|
||||
const isFileUploading = (status: FileUploadState["status"]) =>
|
||||
@@ -213,7 +211,6 @@ export const uploadFile = async (
|
||||
return;
|
||||
}
|
||||
|
||||
let fileBuffer;
|
||||
const fileType = getFileType(file);
|
||||
const fileMetadata: FileMetadata = {
|
||||
parentId,
|
||||
@@ -222,8 +219,7 @@ export const uploadFile = async (
|
||||
};
|
||||
|
||||
if (fileType.startsWith("image/")) {
|
||||
fileBuffer = await file.arrayBuffer();
|
||||
fileMetadata.createdAt = extractExifDateTime(fileBuffer);
|
||||
fileMetadata.createdAt = await readExifDateTime(file);
|
||||
}
|
||||
|
||||
const blob = new Blob([file], { type: fileType });
|
||||
@@ -241,7 +237,7 @@ export const uploadFile = async (
|
||||
dataKeyVersion,
|
||||
);
|
||||
|
||||
return { fileId, fileBuffer, thumbnailBuffer };
|
||||
return { fileId, thumbnailBuffer };
|
||||
} catch (e) {
|
||||
state.status = "error";
|
||||
throw e;
|
||||
|
||||
Reference in new issue
Block a user