1 Commits

12 changed files with 205 additions and 217 deletions

View File

@@ -1,7 +1,7 @@
{
"name": "arkvault",
"private": true,
"version": "0.7.0",
"version": "0.6.0",
"type": "module",
"scripts": {
"dev": "vite dev",

View File

@@ -22,6 +22,7 @@
count,
estimateSize: itemHeight,
gap: itemGap,
overscan: 5,
scrollMargin,
}),
);
@@ -34,7 +35,7 @@
if (!element) return;
const observer = new ResizeObserver(() => {
scrollMargin = Math.round(element!.getBoundingClientRect().top + window.scrollY);
scrollMargin = element!.getBoundingClientRect().top + window.scrollY;
});
observer.observe(element.parentElement!);
return () => observer.disconnect();

View File

@@ -49,7 +49,7 @@
<RowVirtualizer
count={rows.length}
itemHeight={(index) =>
rows[index]!.type === "header" ? 28 : 181 + (rows[index]!.isLast ? 16 : 4)}
rows[index]!.type === "header" ? 28 : 185 + (rows[index]!.isLast ? 16 : 0)}
class="flex flex-grow flex-col"
>
{#snippet item(index)}

View File

@@ -10,7 +10,6 @@ import {
digestMessage,
signMessageHmac,
} from "$lib/modules/crypto";
import { Scheduler } from "$lib/modules/scheduler";
import { generateThumbnail } from "$lib/modules/thumbnail";
import type {
FileThumbnailUploadRequest,
@@ -24,7 +23,6 @@ export interface FileUploadState {
name: string;
parentId: DirectoryId;
status:
| "queued"
| "encryption-pending"
| "encrypting"
| "upload-pending"
@@ -38,16 +36,13 @@ export interface FileUploadState {
}
export type LiveFileUploadState = FileUploadState & {
status: "queued" | "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
status: "encryption-pending" | "encrypting" | "upload-pending" | "uploading";
};
const scheduler = new Scheduler<
{ fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
>();
let uploadingFiles: FileUploadState[] = $state([]);
const isFileUploading = (status: FileUploadState["status"]) =>
["queued", "encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status);
["encryption-pending", "encrypting", "upload-pending", "uploading"].includes(status);
export const getUploadingFiles = (parentId?: DirectoryId) => {
return uploadingFiles.filter(
@@ -188,19 +183,16 @@ export const uploadFile = async (
hmacSecret: HmacSecret,
masterKey: MasterKey,
onDuplicate: () => Promise<boolean>,
) => {
): Promise<
{ fileId: number; fileBuffer: ArrayBuffer; thumbnailBuffer?: ArrayBuffer } | undefined
> => {
uploadingFiles.push({
name: file.name,
parentId,
status: "queued",
status: "encryption-pending",
});
const state = uploadingFiles.at(-1)!;
return await scheduler.schedule(
async () => file.size,
async () => {
state.status = "encryption-pending";
try {
const { fileBuffer, fileSigned } = await requestDuplicateFileScan(
file,
@@ -267,6 +259,4 @@ export const uploadFile = async (
state.status = "error";
throw e;
}
},
);
};

View File

@@ -1,41 +0,0 @@
export class Scheduler<T = void> {
private tasks = 0;
private memoryUsage = 0;
private queue: (() => void)[] = [];
constructor(public memoryLimit = 100 * 1024 * 1024 /* 100 MiB */) {}
private next() {
if (this.memoryUsage < this.memoryLimit) {
this.queue.shift()?.();
}
}
async schedule(estimateMemoryUsage: () => Promise<number>, task: () => Promise<T>) {
if (this.tasks++ > 0) {
await new Promise<void>((resolve) => {
this.queue.push(resolve);
});
}
while (this.memoryUsage >= this.memoryLimit) {
await new Promise<void>((resolve) => {
this.queue.unshift(resolve);
});
}
let taskMemoryUsage = 0;
try {
taskMemoryUsage = await estimateMemoryUsage();
this.memoryUsage += taskMemoryUsage;
this.next();
return await task();
} finally {
this.tasks--;
this.memoryUsage -= taskMemoryUsage;
this.next();
}
}
}

View File

@@ -18,7 +18,7 @@
<div class="flex h-14 items-center gap-x-4 p-2">
<div class="flex-shrink-0 text-lg text-gray-600">
{#if state.status === "queued" || state.status === "encryption-pending"}
{#if state.status === "encryption-pending"}
<IconPending />
{:else if state.status === "encrypting"}
<IconLockClock />
@@ -37,9 +37,7 @@
{state.name}
</p>
<p class="text-xs text-gray-800">
{#if state.status === "queued"}
대기 중
{:else if state.status === "encryption-pending"}
{#if state.status === "encryption-pending"}
준비 중
{:else if state.status === "encrypting"}
암호화하는 중

View File

@@ -4,36 +4,17 @@
import { BottomDiv, Button, FullscreenDiv } from "$lib/components/atoms";
import { IconEntryButton, TopBar } from "$lib/components/molecules";
import { deleteAllFileThumbnailCaches } from "$lib/modules/file";
import { bulkGetFileInfo, type MaybeFileInfo } from "$lib/modules/filesystem";
import { bulkGetFileInfo } from "$lib/modules/filesystem";
import { masterKeyStore } from "$lib/stores";
import { sortEntries } from "$lib/utils";
import File from "./File.svelte";
import {
getThumbnailGenerationStatus,
clearThumbnailGenerationStatuses,
requestThumbnailGeneration,
type GenerationStatus,
} from "./service.svelte";
import { persistentStates, requestThumbnailGeneration } from "./service.svelte";
import IconDelete from "~icons/material-symbols/delete";
let { data } = $props();
let fileInfos: MaybeFileInfo[] = $state([]);
let files = $derived(
fileInfos
.map((info) => ({
info,
status: getThumbnailGenerationStatus(info.id),
}))
.filter(
(file): file is { info: MaybeFileInfo; status: Exclude<GenerationStatus, "uploaded"> } =>
file.status !== "uploaded",
),
);
const generateAllThumbnails = () => {
files.forEach(({ info }) => {
persistentStates.files.forEach(({ info }) => {
if (info.exists) {
requestThumbnailGeneration(info);
}
@@ -41,12 +22,13 @@
};
onMount(async () => {
fileInfos = sortEntries(
Array.from((await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!)).values()),
);
const fileInfos = await bulkGetFileInfo(data.files, $masterKeyStore?.get(1)?.key!);
persistentStates.files = persistentStates.files.map(({ id, status }) => ({
id,
info: fileInfos.get(id)!,
status,
}));
});
$effect(() => clearThumbnailGenerationStatuses);
</script>
<svelte:head>
@@ -61,19 +43,19 @@
저장된 썸네일 모두 삭제하기
</IconEntryButton>
</div>
{#if files.length > 0}
{#if persistentStates.files.length > 0}
<div class="flex-grow space-y-2 bg-white p-4">
<p class="text-lg font-bold text-gray-800">썸네일이 누락된 파일</p>
<div class="space-y-4">
<p class="break-keep text-gray-800">
{files.length}개 파일의 썸네일이 존재하지 않아요.
{persistentStates.files.length}개 파일의 썸네일이 존재하지 않아요.
</p>
<div class="space-y-2">
{#each files as { info, status } (info.id)}
{#each persistentStates.files as { info, status } (info.id)}
{#if info.exists}
<File
{info}
{status}
generationStatus={status}
onclick={({ id }) => goto(`/file/${id}`)}
onGenerateThumbnailClick={requestThumbnailGeneration}
/>
@@ -84,7 +66,7 @@
</div>
{/if}
</div>
{#if files.length > 0}
{#if persistentStates.files.length > 0}
<BottomDiv class="px-4">
<Button onclick={generateAllThumbnails} class="w-full">모두 썸네일 생성하기</Button>
</BottomDiv>

View File

@@ -10,6 +10,7 @@
</script>
<script lang="ts">
import type { Writable } from "svelte/store";
import { ActionEntryButton } from "$lib/components/atoms";
import { DirectoryEntryLabel } from "$lib/components/molecules";
import type { FileInfo } from "$lib/modules/filesystem";
@@ -22,21 +23,22 @@
info: FileInfo;
onclick: (file: FileInfo) => void;
onGenerateThumbnailClick: (file: FileInfo) => void;
status: Exclude<GenerationStatus, "uploaded"> | undefined;
generationStatus?: Writable<GenerationStatus>;
}
let { info, onclick, onGenerateThumbnailClick, status }: Props = $props();
let { info, onclick, onGenerateThumbnailClick, generationStatus }: Props = $props();
</script>
<ActionEntryButton
class="h-14"
onclick={() => onclick(info)}
actionButtonIcon={!status || status === "error" ? IconCamera : undefined}
actionButtonIcon={!$generationStatus || $generationStatus === "error" ? IconCamera : undefined}
onActionButtonClick={() => onGenerateThumbnailClick(info)}
actionButtonClass="text-gray-800"
>
{@const subtext = status
? subtexts[status]
{@const subtext =
$generationStatus && $generationStatus !== "uploaded"
? subtexts[$generationStatus]
: formatDateTime(info.createdAt ?? info.lastModifiedAt)}
<DirectoryEntryLabel type="file" name={info.name} {subtext} />
</ActionEntryButton>

View File

@@ -1,9 +1,8 @@
import { limitFunction } from "p-limit";
import { SvelteMap } from "svelte/reactivity";
import { get, writable, type Writable } from "svelte/store";
import { encryptData } from "$lib/modules/crypto";
import { storeFileThumbnailCache } from "$lib/modules/file";
import type { FileInfo } from "$lib/modules/filesystem";
import { Scheduler } from "$lib/modules/scheduler";
import type { FileInfo, MaybeFileInfo } from "$lib/modules/filesystem";
import { generateThumbnail as doGenerateThumbnail } from "$lib/modules/thumbnail";
import { requestFileDownload, requestFileThumbnailUpload } from "$lib/services/file";
@@ -16,31 +15,41 @@ export type GenerationStatus =
| "uploaded"
| "error";
const scheduler = new Scheduler();
const statuses = new SvelteMap<number, GenerationStatus>();
export const getThumbnailGenerationStatus = (fileId: number) => {
return statuses.get(fileId);
};
export const clearThumbnailGenerationStatuses = () => {
for (const [id, status] of statuses) {
if (status === "uploaded" || status === "error") {
statuses.delete(id);
}
interface File {
id: number;
info: MaybeFileInfo;
status?: Writable<GenerationStatus>;
}
const workingFiles = new Map<number, Writable<GenerationStatus>>();
let queue: (() => void)[] = [];
let memoryUsage = 0;
const memoryLimit = 100 * 1024 * 1024; // 100 MiB
export const persistentStates = $state({
files: [] as File[],
});
export const getGenerationStatus = (fileId: number) => {
return workingFiles.get(fileId);
};
const generateThumbnail = limitFunction(
async (fileId: number, fileBuffer: ArrayBuffer, fileType: string, dataKey: CryptoKey) => {
statuses.set(fileId, "generating");
async (
status: Writable<GenerationStatus>,
fileBuffer: ArrayBuffer,
fileType: string,
dataKey: CryptoKey,
) => {
status.set("generating");
const thumbnail = await doGenerateThumbnail(fileBuffer, fileType);
if (!thumbnail) return null;
const thumbnailBuffer = await thumbnail.arrayBuffer();
const thumbnailEncrypted = await encryptData(thumbnailBuffer, dataKey);
statuses.set(fileId, "upload-pending");
status.set("upload-pending");
return { plaintext: thumbnailBuffer, ...thumbnailEncrypted };
},
{ concurrency: 4 },
@@ -48,55 +57,106 @@ const generateThumbnail = limitFunction(
const requestThumbnailUpload = limitFunction(
async (
status: Writable<GenerationStatus>,
fileId: number,
dataKeyVersion: Date,
thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: string },
) => {
statuses.set(fileId, "uploading");
status.set("uploading");
const res = await requestFileThumbnailUpload(fileId, dataKeyVersion, thumbnail);
if (!res.ok) return false;
statuses.set(fileId, "uploaded");
status.set("uploaded");
workingFiles.delete(fileId);
persistentStates.files = persistentStates.files.filter(({ id }) => id != fileId);
storeFileThumbnailCache(fileId, thumbnail.plaintext); // Intended
return true;
},
{ concurrency: 4 },
);
export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
const status = statuses.get(fileInfo.id);
const enqueue = async (
status: Writable<GenerationStatus> | undefined,
fileInfo: FileInfo,
priority = false,
) => {
if (status) {
if (status !== "error") return;
status.set("queued");
} else {
statuses.set(fileInfo.id, "queued");
status = writable("queued");
workingFiles.set(fileInfo.id, status);
persistentStates.files = persistentStates.files.map((file) =>
file.id === fileInfo.id ? { ...file, status } : file,
);
}
try {
let file: ArrayBuffer | undefined;
let resolver;
const promise = new Promise((resolve) => {
resolver = resolve;
});
await scheduler.schedule(
async () => {
statuses.set(fileInfo.id, "generation-pending");
file = await requestFileDownload(fileInfo.id, fileInfo.contentIv!, fileInfo.dataKey?.key!);
return file.byteLength;
},
async () => {
const thumbnail = await generateThumbnail(
if (priority) {
queue = [resolver!, ...queue];
} else {
queue.push(resolver!);
}
await promise;
};
export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
let status = workingFiles.get(fileInfo.id);
if (status && get(status) !== "error") return;
if (workingFiles.values().some((status) => get(status) !== "error")) {
await enqueue(status, fileInfo);
}
while (memoryUsage >= memoryLimit) {
await enqueue(status, fileInfo, true);
}
if (status) {
status.set("generation-pending");
} else {
status = writable("generation-pending");
workingFiles.set(fileInfo.id, status);
persistentStates.files = persistentStates.files.map((file) =>
file.id === fileInfo.id ? { ...file, status } : file,
);
}
let fileSize = 0;
try {
const file = await requestFileDownload(
fileInfo.id,
file!,
fileInfo.contentIv!,
fileInfo.dataKey?.key!,
);
fileSize = file.byteLength;
memoryUsage += fileSize;
if (memoryUsage < memoryLimit) {
queue.shift()?.();
}
const thumbnail = await generateThumbnail(
status,
file,
fileInfo.contentType,
fileInfo.dataKey?.key!,
);
if (
!thumbnail ||
!(await requestThumbnailUpload(fileInfo.id, fileInfo.dataKey?.version!, thumbnail))
!(await requestThumbnailUpload(status, fileInfo.id, fileInfo.dataKey?.version!, thumbnail))
) {
statuses.set(fileInfo.id, "error");
status.set("error");
}
},
);
} catch (e) {
statuses.set(fileInfo.id, "error");
throw e;
} catch {
status.set("error");
} finally {
memoryUsage -= fileSize;
queue.shift()?.();
}
};

View File

@@ -20,9 +20,7 @@
{state.name}
</p>
<p class="text-xs">
{#if state.status === "queued"}
대기 중
{:else if state.status === "encryption-pending"}
{#if state.status === "encryption-pending"}
준비 중
{:else if state.status === "encrypting"}
암호화하는 중

View File

@@ -6,7 +6,6 @@ import type { RequestHandler } from "./$types";
const trpcHandler: RequestHandler = (event) =>
fetchRequestHandler({
endpoint: "/api/trpc",
allowMethodOverride: true,
req: event.request,
router: appRouter,
createContext: () => createContext(event),

View File

@@ -9,7 +9,6 @@ const createClient = (fetch: typeof globalThis.fetch) =>
httpBatchLink({
url: "/api/trpc",
maxURLLength: 4096,
methodOverride: "POST",
transformer: superjson,
fetch,
}),