1 Commits

12 changed files with 205 additions and 217 deletions

View File

@@ -1,7 +1,7 @@
{ {
"name": "arkvault", "name": "arkvault",
"private": true, "private": true,
"version": "0.7.0", "version": "0.6.0",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite dev", "dev": "vite dev",

View File

@@ -22,6 +22,7 @@
count, count,
estimateSize: itemHeight, estimateSize: itemHeight,
gap: itemGap, gap: itemGap,
overscan: 5,
scrollMargin, scrollMargin,
}), }),
); );
@@ -34,7 +35,7 @@
if (!element) return; if (!element) return;
const observer = new ResizeObserver(() => { const observer = new ResizeObserver(() => {
scrollMargin = Math.round(element!.getBoundingClientRect().top + window.scrollY); scrollMargin = element!.getBoundingClientRect().top + window.scrollY;
}); });
observer.observe(element.parentElement!); observer.observe(element.parentElement!);
return () => observer.disconnect(); return () => observer.disconnect();

View File

@@ -49,7 +49,7 @@
<RowVirtualizer <RowVirtualizer
count={rows.length} count={rows.length}
itemHeight={(index) => itemHeight={(index) =>
rows[index]!.type === "header" ? 28 : 181 + (rows[index]!.isLast ? 16 : 4)} rows[index]!.type === "header" ? 28 : 185 + (rows[index]!.isLast ? 16 : 0)}
class="flex flex-grow flex-col" class="flex flex-grow flex-col"
> >
{#snippet item(index)} {#snippet item(index)}

View File

@@ -10,7 +10,6 @@ import {
digestMessage, digestMessage,
signMessageHmac, signMessageHmac,
} from "$lib/modules/crypto"; } from "$lib/modules/crypto";
import { Scheduler } from "$lib/modules/scheduler";
import { generateThumbnail } from "$lib/modules/thumbnail"; import { generateThumbnail } from "$lib/modules/thumbnail";
import type { import type {
FileThumbnailUploadRequest, FileThumbnailUploadRequest,
@@ -24,7 +23,6 @@ export interface FileUploadState {
name: string; name: string;
parentId: DirectoryId; parentId: DirectoryId;
status: status:
| "queued"
| "encryption-pending" | "encryption-pending"
| "encrypting" | "encrypting"
| "upload-pending" | "upload-pending"
@@ -38,16 +36,13 @@ export interface FileUploadState {
} }
export type LiveFileUploadState = FileUploadState & { export type LiveFileUploadState = FileUploadState & {
status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading"; status: "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
}; };
const scheduler = new Scheduler<
{ fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
>();
let uploadingFiles: FileUploadState[] = $state([]); let uploadingFiles: FileUploadState[] = $state([]);
const isFileUploading = (status: FileUploadState["status"]) => const isFileUploading = (status: FileUploadState["status"]) =>
["queued", "encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status); ["encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status);
export const getUploadingFiles = (parentId?: DirectoryId) => { export const getUploadingFiles = (parentId?: DirectoryId) => {
return uploadingFiles.filter( return uploadingFiles.filter(
@@ -188,19 +183,16 @@ export const uploadFile = async (
hmacSecret: HmacSecret, hmacSecret: HmacSecret,
masterKey: MasterKey, masterKey: MasterKey,
onDuplicate: () => Promise<boolean>, onDuplicate: () => Promise<boolean>,
) => { ): Promise<
{ fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
> => {
uploadingFiles.push({ uploadingFiles.push({
name: file.name, name: file.name,
parentId, parentId,
status: "queued", status: "encryption-pending",
}); });
const state = uploadingFiles.at(-1)!; const state = uploadingFiles.at(-1)!;
return await scheduler.schedule(
async () => file.size,
async () => {
state.status = "encryption-pending";
try { try {
const { fileBuffer, fileSigned } = await requestDuplicateFileScan( const { fileBuffer, fileSigned } = await requestDuplicateFileScan(
file, file,
@@ -267,6 +259,4 @@ export const uploadFile = async (
state.status = "error"; state.status = "error";
throw e; throw e;
} }
},
);
}; };

View File

@@ -1,41 +0,0 @@
export class Scheduler<T = void> {
private tasks = 0;
private memoryUsage = 0;
private queue: (() => void)[] = [];
constructor(public memoryLimit = 100 * 1024 * 1024 /* 100 MiB */) {}
private next() {
if (this.memoryUsage < this.memoryLimit) {
this.queue.shift()?.();
}
}
async schedule(estimateMemoryUsage: () => Promise<number>, task: () => Promise<T>) {
if (this.tasks++ > 0) {
await new Promise<void>((resolve) => {
this.queue.push(resolve);
});
}
while (this.memoryUsage >= this.memoryLimit) {
await new Promise<void>((resolve) => {
this.queue.unshift(resolve);
});
}
let taskMemoryUsage = 0;
try {
taskMemoryUsage = await estimateMemoryUsage();
this.memoryUsage += taskMemoryUsage;
this.next();
return await task();
} finally {
this.tasks--;
this.memoryUsage -= taskMemoryUsage;
this.next();
}
}
}

View File

@@ -18,7 +18,7 @@
<div class="flex h-14 items-center gap-x-4 p-2"> <div class="flex h-14 items-center gap-x-4 p-2">
<div class="flex-shrink-0 text-lg text-gray-600"> <div class="flex-shrink-0 text-lg text-gray-600">
{#if state.status === "queued" || state.status === "encryption-pending"} {#if state.status === "encryption-pending"}
<IconPending /> <IconPending />
{:else if state.status === "encrypting"} {:else if state.status === "encrypting"}
<IconLockClock /> <IconLockClock />
@@ -37,9 +37,7 @@
{state.name} {state.name}
</p> </p>
<p class="text-xs text-gray-800"> <p class="text-xs text-gray-800">
{#if state.status === "queued"} {#if state.status === "encryption-pending"}
대기 중
{:else if state.status === "encryption-pending"}
준비 중 준비 중
{:else if state.status === "encrypting"} {:else if state.status === "encrypting"}
암호화하는 중 암호화하는 중

View File

@@ -4,36 +4,17 @@
import { BottomDiv, Button, FullscreenDiv } from "$lib/components/atoms"; import { BottomDiv, Button, FullscreenDiv } from "$lib/components/atoms";
import { IconEntryButton, TopBar } from "$lib/components/molecules"; import { IconEntryButton, TopBar } from "$lib/components/molecules";
import { deleteAllFileThumbnailCaches } from "$lib/modules/file"; import { deleteAllFileThumbnailCaches } from "$lib/modules/file";
import { bulkGetFileInfo, type MaybeFileInfo } from "$lib/modules/filesystem"; import { bulkGetFileInfo } from "$lib/modules/filesystem";
import { masterKeyStore } from "$lib/stores"; import { masterKeyStore } from "$lib/stores";
import { sortEntries } from "$lib/utils";
import File from "./File.svelte"; import File from "./File.svelte";
import { import { persistentStates, requestThumbnailGeneration } from "./service.svelte";
getThumbnailGenerationStatus,
clearThumbnailGenerationStatuses,
requestThumbnailGeneration,
type GenerationStatus,
} from "./service.svelte";
import IconDelete from "~icons/material-symbols/delete"; import IconDelete from "~icons/material-symbols/delete";
let { data } = $props(); let { data } = $props();
let fileInfos: MaybeFileInfo[] = $state([]);
let files = $derived(
fileInfos
.map((info) => ({
info,
status: getThumbnailGenerationStatus(info.id),
}))
.filter(
(file): file is { info: MaybeFileInfo; status: Exclude<GenerationStatus, "uploaded"> } =>
file.status !== "uploaded",
),
);
const generateAllThumbnails = () => { const generateAllThumbnails = () => {
files.forEach(({ info }) => { persistentStates.files.forEach(({ info }) => {
if (info.exists) { if (info.exists) {
requestThumbnailGeneration(info); requestThumbnailGeneration(info);
} }
@@ -41,12 +22,13 @@
}; };
onMount(async () => { onMount(async () => {
fileInfos = sortEntries( const fileInfos = await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!);
Array.from((await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!)).values()), persistentStates.files = persistentStates.files.map(({ id, status }) => ({
); id,
info: fileInfos.get(id)!,
status,
}));
}); });
$effect(() => clearThumbnailGenerationStatuses);
</script> </script>
<svelte:head> <svelte:head>
@@ -61,19 +43,19 @@
저장된 썸네일 모두 삭제하기 저장된 썸네일 모두 삭제하기
</IconEntryButton> </IconEntryButton>
</div> </div>
{#if files.length > 0} {#if persistentStates.files.length > 0}
<div class="flex-grow space-y-2 bg-white p-4"> <div class="flex-grow space-y-2 bg-white p-4">
<p class="text-lg font-bold text-gray-800">썸네일이 누락된 파일</p> <p class="text-lg font-bold text-gray-800">썸네일이 누락된 파일</p>
<div class="space-y-4"> <div class="space-y-4">
<p class="break-keep text-gray-800"> <p class="break-keep text-gray-800">
{files.length}개 파일의 썸네일이 존재하지 않아요. {persistentStates.files.length}개 파일의 썸네일이 존재하지 않아요.
</p> </p>
<div class="space-y-2"> <div class="space-y-2">
{#each files as { info, status } (info.id)} {#each persistentStates.files as { info, status } (info.id)}
{#if info.exists} {#if info.exists}
<File <File
{info} {info}
{status} generationStatus={status}
onclick={({ id }) => goto(`/file/${id}`)} onclick={({ id }) => goto(`/file/${id}`)}
onGenerateThumbnailClick={requestThumbnailGeneration} onGenerateThumbnailClick={requestThumbnailGeneration}
/> />
@@ -84,7 +66,7 @@
</div> </div>
{/if} {/if}
</div> </div>
{#if files.length > 0} {#if persistentStates.files.length > 0}
<BottomDiv class="px-4"> <BottomDiv class="px-4">
<Button onclick={generateAllThumbnails} class="w-full">모두 썸네일 생성하기</Button> <Button onclick={generateAllThumbnails} class="w-full">모두 썸네일 생성하기</Button>
</BottomDiv> </BottomDiv>

View File

@@ -10,6 +10,7 @@
</script> </script>
<script lang="ts"> <script lang="ts">
import type { Writable } from "svelte/store";
import { ActionEntryButton } from "$lib/components/atoms"; import { ActionEntryButton } from "$lib/components/atoms";
import { DirectoryEntryLabel } from "$lib/components/molecules"; import { DirectoryEntryLabel } from "$lib/components/molecules";
import type { FileInfo } from "$lib/modules/filesystem"; import type { FileInfo } from "$lib/modules/filesystem";
@@ -22,21 +23,22 @@
info: FileInfo; info: FileInfo;
onclick: (file: FileInfo) => void; onclick: (file: FileInfo) => void;
onGenerateThumbnailClick: (file: FileInfo) => void; onGenerateThumbnailClick: (file: FileInfo) => void;
status: Exclude<GenerationStatus, "uploaded"> | undefined; generationStatus?: Writable<GenerationStatus>;
} }
let { info, onclick, onGenerateThumbnailClick, status }: Props = $props(); let { info, onclick, onGenerateThumbnailClick, generationStatus }: Props = $props();
</script> </script>
<ActionEntryButton <ActionEntryButton
class="h-14" class="h-14"
onclick={() => onclick(info)} onclick={() => onclick(info)}
actionButtonIcon={!status || status === "error" ? IconCamera : undefined} actionButtonIcon={!$generationStatus || $generationStatus === "error" ? IconCamera : undefined}
onActionButtonClick={() => onGenerateThumbnailClick(info)} onActionButtonClick={() => onGenerateThumbnailClick(info)}
actionButtonClass="text-gray-800" actionButtonClass="text-gray-800"
> >
{@const subtext = status {@const subtext =
? subtexts[status] $generationStatus && $generationStatus !== "uploaded"
? subtexts[$generationStatus]
: formatDateTime(info.createdAt ?? info.lastModifiedAt)} : formatDateTime(info.createdAt ?? info.lastModifiedAt)}
<DirectoryEntryLabel type="file" name={info.name} {subtext} /> <DirectoryEntryLabel type="file" name={info.name} {subtext} />
</ActionEntryButton> </ActionEntryButton>

View File

@@ -1,9 +1,8 @@
import { limitFunction } from "p-limit"; import { limitFunction } from "p-limit";
import { SvelteMap } from "svelte/reactivity"; import { get, writable, type Writable } from "svelte/store";
import { encryptData } from "$lib/modules/crypto"; import { encryptData } from "$lib/modules/crypto";
import { storeFileThumbnailCache } from "$lib/modules/file"; import { storeFileThumbnailCache } from "$lib/modules/file";
import type { FileInfo } from "$lib/modules/filesystem"; import type { FileInfo, MaybeFileInfo } from "$lib/modules/filesystem";
import { Scheduler } from "$lib/modules/scheduler";
import { generateThumbnail as doGenerateThumbnail } from "$lib/modules/thumbnail"; import { generateThumbnail as doGenerateThumbnail } from "$lib/modules/thumbnail";
import { requestFileDownload, requestFileThumbnailUpload } from "$lib/services/file"; import { requestFileDownload, requestFileThumbnailUpload } from "$lib/services/file";
@@ -16,31 +15,41 @@ export type GenerationStatus =
| "uploaded" | "uploaded"
| "error"; | "error";
const scheduler = new Scheduler(); interface File {
const statuses = new SvelteMap<number, GenerationStatus>(); id: number;
info: MaybeFileInfo;
status?: Writable<GenerationStatus>;
}
export const getThumbnailGenerationStatus = (fileId: number) => { const workingFiles = new Map<number, Writable<GenerationStatus>>();
return statuses.get(fileId);
};
export const clearThumbnailGenerationStatuses = () => { let queue: (() => void)[] = [];
for (const [id, status] of statuses) { let memoryUsage = 0;
if (status === "uploaded" || status === "error") { const memoryLimit = 100 * 1024 * 1024; // 100 MiB
statuses.delete(id);
} export const persistentStates = $state({
} files: [] as File[],
});
export const getGenerationStatus = (fileId: number) => {
return workingFiles.get(fileId);
}; };
const generateThumbnail = limitFunction( const generateThumbnail = limitFunction(
async (fileId: number, fileBuffer: ArrayBuffer, fileType: string, dataKey: CryptoKey) => { async (
statuses.set(fileId, "generating"); status: Writable<GenerationStatus>,
fileBuffer: ArrayBuffer,
fileType: string,
dataKey: CryptoKey,
) => {
status.set("generating");
const thumbnail = await doGenerateThumbnail(fileBuffer, fileType); const thumbnail = await doGenerateThumbnail(fileBuffer, fileType);
if (!thumbnail) return null; if (!thumbnail) return null;
const thumbnailBuffer = await thumbnail.arrayBuffer(); const thumbnailBuffer = await thumbnail.arrayBuffer();
const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey); const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey);
statuses.set(fileId, "upload-pending"); status.set("upload-pending");
return { plaintext: thumbnailBuffer, ...thumbnailEncrypted }; return { plaintext: thumbnailBuffer, ...thumbnailEncrypted };
}, },
{ concurrency: 4 }, { concurrency: 4 },
@@ -48,55 +57,106 @@ const generateThumbnail = limitFunction(
const requestThumbnailUpload = limitFunction( const requestThumbnailUpload = limitFunction(
async ( async (
status: Writable<GenerationStatus>,
fileId: number, fileId: number,
dataKeyVersion: Date, dataKeyVersion: Date,
thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: string }, thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: string },
) => { ) => {
statuses.set(fileId, "uploading"); status.set("uploading");
const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnail); const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnail);
if (!res.ok) return false; if (!res.ok) return false;
statuses.set(fileId, "uploaded");
status.set("uploaded");
workingFiles.delete(fileId);
persistentStates.files = persistentStates.files.filter(({ id }) => id != fileId);
storeFileThumbnailCache(fileId, thumbnail.plaintext); // Intended storeFileThumbnailCache(fileId, thumbnail.plaintext); // Intended
return true; return true;
}, },
{ concurrency: 4 }, { concurrency: 4 },
); );
export const requestThumbnailGeneration = async (fileInfo: FileInfo) => { const enqueue = async (
const status = statuses.get(fileInfo.id); status: Writable<GenerationStatus> | undefined,
fileInfo: FileInfo,
priority = false,
) => {
if (status) { if (status) {
if (status !== "error") return; status.set("queued");
} else { } else {
statuses.set(fileInfo.id, "queued"); status = writable("queued");
workingFiles.set(fileInfo.id, status);
persistentStates.files = persistentStates.files.map((file) =>
file.id === fileInfo.id ? { ...file, status } : file,
);
} }
try { let resolver;
let file: ArrayBuffer | undefined; const promise = new Promise((resolve) => {
resolver = resolve;
});
await scheduler.schedule( if (priority) {
async () => { queue = [resolver!, ...queue];
statuses.set(fileInfo.id, "generation-pending"); } else {
file = await requestFileDownload(fileInfo.id, fileInfo.contentIv!, fileInfo.dataKey?.key!); queue.push(resolver!);
return file.byteLength; }
},
async () => { await promise;
const thumbnail = await generateThumbnail( };
export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
let status = workingFiles.get(fileInfo.id);
if (status && get(status) !== "error") return;
if (workingFiles.values().some((status) => get(status) !== "error")) {
await enqueue(status, fileInfo);
}
while (memoryUsage >= memoryLimit) {
await enqueue(status, fileInfo, true);
}
if (status) {
status.set("generation-pending");
} else {
status = writable("generation-pending");
workingFiles.set(fileInfo.id, status);
persistentStates.files = persistentStates.files.map((file) =>
file.id === fileInfo.id ? { ...file, status } : file,
);
}
let fileSize = 0;
try {
const file = await requestFileDownload(
fileInfo.id, fileInfo.id,
file!, fileInfo.contentIv!,
fileInfo.dataKey?.key!,
);
fileSize = file.byteLength;
memoryUsage += fileSize;
if (memoryUsage < memoryLimit) {
queue.shift()?.();
}
const thumbnail = await generateThumbnail(
status,
file,
fileInfo.contentType, fileInfo.contentType,
fileInfo.dataKey?.key!, fileInfo.dataKey?.key!,
); );
if ( if (
!thumbnail || !thumbnail ||
!(await requestThumbnailUpload(fileInfo.id, fileInfo.dataKey?.version!, thumbnail)) !(await requestThumbnailUpload(status, fileInfo.id, fileInfo.dataKey?.version!, thumbnail))
) { ) {
statuses.set(fileInfo.id, "error"); status.set("error");
} }
}, } catch {
); status.set("error");
} catch (e) { } finally {
statuses.set(fileInfo.id, "error"); memoryUsage -= fileSize;
throw e; queue.shift()?.();
} }
}; };

View File

@@ -20,9 +20,7 @@
{state.name} {state.name}
</p> </p>
<p class="text-xs"> <p class="text-xs">
{#if state.status === "queued"} {#if state.status === "encryption-pending"}
대기 중
{:else if state.status === "encryption-pending"}
준비 중 준비 중
{:else if state.status === "encrypting"} {:else if state.status === "encrypting"}
암호화하는 중 암호화하는 중

View File

@@ -6,7 +6,6 @@ import type { RequestHandler } from "./$types";
const trpcHandler: RequestHandler = (event) => const trpcHandler: RequestHandler = (event) =>
fetchRequestHandler({ fetchRequestHandler({
endpoint: "/api/trpc", endpoint: "/api/trpc",
allowMethodOverride: true,
req: event.request, req: event.request,
router: appRouter, router: appRouter,
createContext: () => createContext(event), createContext: () => createContext(event),

View File

@@ -9,7 +9,6 @@ const createClient = (fetch: typeof globalThis.fetch) =>
httpBatchLink({ httpBatchLink({
url: "/api/trpc", url: "/api/trpc",
maxURLLength: 4096, maxURLLength: 4096,
methodOverride: "POST",
transformer: superjson, transformer: superjson,
fetch, fetch,
}), }),