diff --git a/src/lib/server/db/error.ts b/src/lib/server/db/error.ts index a145f14..0d61d72 100644 --- a/src/lib/server/db/error.ts +++ b/src/lib/server/db/error.ts @@ -9,6 +9,7 @@ type IntegrityErrorMessages = // File | "Directory not found" | "File not found" + | "File is not legacy" | "File not found in category" | "File already added to category" | "Invalid DEK version" diff --git a/src/lib/server/db/file.ts b/src/lib/server/db/file.ts index 0418bc6..9314f4b 100644 --- a/src/lib/server/db/file.ts +++ b/src/lib/server/db/file.ts @@ -334,6 +334,16 @@ export const getAllFileIds = async (userId: number) => { return files.map(({ id }) => id); }; +export const getLegacyFileIds = async (userId: number) => { + const files = await db + .selectFrom("file") + .select("id") + .where("user_id", "=", userId) + .where("encrypted_content_iv", "is not", null) + .execute(); + return files.map(({ id }) => id); +}; + export const getAllFileIdsByContentHmac = async ( userId: number, hskVersion: number, @@ -482,6 +492,52 @@ export const unregisterFile = async (userId: number, fileId: number) => { }); }; +export const migrateFileContent = async ( + trx: typeof db, + userId: number, + fileId: number, + newPath: string, + encContentHash: string, +) => { + const file = await trx + .selectFrom("file") + .select(["path", "encrypted_content_iv"]) + .where("id", "=", fileId) + .where("user_id", "=", userId) + .limit(1) + .forUpdate() + .executeTakeFirst(); + + if (!file) { + throw new IntegrityError("File not found"); + } + if (!file.encrypted_content_iv) { + throw new IntegrityError("File is not legacy"); + } + + await trx + .updateTable("file") + .set({ + path: newPath, + encrypted_content_iv: null, + encrypted_content_hash: encContentHash, + }) + .where("id", "=", fileId) + .where("user_id", "=", userId) + .execute(); + + await trx + .insertInto("file_log") + .values({ + file_id: fileId, + timestamp: new Date(), + action: "migrate", + }) + .execute(); + + return file.path; +}; + export const addFileToCategory = async (fileId: number, categoryId: number) => { await db.transaction().execute(async (trx) => { try { diff --git a/src/lib/server/db/schema/file.ts b/src/lib/server/db/schema/file.ts index 0774082..3680d1d 100644 --- a/src/lib/server/db/schema/file.ts +++ b/src/lib/server/db/schema/file.ts @@ -41,7 +41,7 @@ interface FileLogTable { id: Generated; file_id: number; timestamp: ColumnType; - action: "create" | "rename" | "add-to-category" | "remove-from-category"; + action: "create" | "rename" | "migrate" | "add-to-category" | "remove-from-category"; new_name: Ciphertext | null; category_id: number | null; } diff --git a/src/lib/server/db/schema/upload.ts b/src/lib/server/db/schema/upload.ts index fccde36..e20227d 100644 --- a/src/lib/server/db/schema/upload.ts +++ b/src/lib/server/db/schema/upload.ts @@ -3,7 +3,7 @@ import type { Ciphertext } from "./utils"; interface UploadSessionTable { id: string; - type: "file" | "thumbnail"; + type: "file" | "thumbnail" | "migration"; user_id: number; path: string; total_chunks: number; diff --git a/src/lib/server/db/upload.ts b/src/lib/server/db/upload.ts index d506191..876c150 100644 --- a/src/lib/server/db/upload.ts +++ b/src/lib/server/db/upload.ts @@ -31,6 +31,11 @@ interface ThumbnailUploadSession extends BaseUploadSession { dekVersion: Date; } +interface MigrationUploadSession extends BaseUploadSession { + type: "migration"; + fileId: number; +} + export const createFileUploadSession = async ( params: Omit, ) => { @@ -118,6 +123,39 @@ export const createThumbnailUploadSession = async ( }); }; +export const createMigrationUploadSession = async ( + params: Omit, +) => { + await db.transaction().execute(async (trx) => { + const file = await trx + .selectFrom("file") + .select("encrypted_content_iv") + .where("id", "=", params.fileId) + .where("user_id", "=", params.userId) + .limit(1) + .forUpdate() + .executeTakeFirst(); + if (!file) { + throw new IntegrityError("File not found"); + } else if (!file.encrypted_content_iv) { + throw new IntegrityError("File is not legacy"); + } + + await trx + .insertInto("upload_session") + .values({ + id: params.id, + type: "migration", + user_id: params.userId, + path: params.path, + total_chunks: params.totalChunks, + expires_at: params.expiresAt, + file_id: params.fileId, + }) + .execute(); + }); +}; + export const getUploadSession = async (sessionId: string, userId: number) => { const session = await db .selectFrom("upload_session") @@ -148,7 +186,7 @@ export const getUploadSession = async (sessionId: string, userId: number) => { encCreatedAt: session.encrypted_created_at, encLastModifiedAt: session.encrypted_last_modified_at!, } satisfies FileUploadSession; - } else { + } else if (session.type === "thumbnail") { return { type: "thumbnail", id: session.id, @@ -160,6 +198,17 @@ export const getUploadSession = async (sessionId: string, userId: number) => { fileId: session.file_id!, dekVersion: session.data_encryption_key_version!, } satisfies ThumbnailUploadSession; + } else { + return { + type: "migration", + id: session.id, + userId: session.user_id, + path: session.path, + totalChunks: session.total_chunks, + uploadedChunks: session.uploaded_chunks, + expiresAt: session.expires_at, + fileId: session.file_id!, + } satisfies MigrationUploadSession; } }; diff --git a/src/routes/(fullscreen)/settings/migration/+page.server.ts b/src/routes/(fullscreen)/settings/migration/+page.server.ts new file mode 100644 index 0000000..3e1c32a --- /dev/null +++ b/src/routes/(fullscreen)/settings/migration/+page.server.ts @@ -0,0 +1,7 @@ +import { createCaller } from "$trpc/router.server"; +import type { PageServerLoad } from "./$types"; + +export const load: PageServerLoad = async (event) => { + const files = await createCaller(event).file.listLegacy(); + return { files }; +}; diff --git a/src/routes/(fullscreen)/settings/migration/+page.svelte b/src/routes/(fullscreen)/settings/migration/+page.svelte new file mode 100644 index 0000000..b4bc9cb --- /dev/null +++ b/src/routes/(fullscreen)/settings/migration/+page.svelte @@ -0,0 +1,79 @@ + + + + 암호화 마이그레이션 + + + + + {#if files.length > 0} +
+

+ 이전 버전의 ArkVault에서 업로드된 {files.length}개 파일을 다시 암호화할 수 있어요. +

+
+ {#each files as { info, state } (info.id)} + {#if info.exists} + goto(`/file/${id}`)} + onMigrateClick={requestFileMigration} + /> + {/if} + {/each} +
+
+ + + + {:else} +
+

+ {#if data.files.length === 0} + 마이그레이션할 파일이 없어요. + {:else} + 파일 목록을 불러오고 있어요. + {/if} +

+
+ {/if} + diff --git a/src/routes/(fullscreen)/settings/migration/File.svelte b/src/routes/(fullscreen)/settings/migration/File.svelte new file mode 100644 index 0000000..ec9d25b --- /dev/null +++ b/src/routes/(fullscreen)/settings/migration/File.svelte @@ -0,0 +1,55 @@ + + + + + onclick(info)} + actionButtonIcon={!state || state.status === "error" ? IconSync : undefined} + onActionButtonClick={() => onMigrateClick(info)} + actionButtonClass="text-gray-800" +> + + diff --git a/src/routes/(fullscreen)/settings/migration/service.svelte.ts b/src/routes/(fullscreen)/settings/migration/service.svelte.ts new file mode 100644 index 0000000..d59f46e --- /dev/null +++ b/src/routes/(fullscreen)/settings/migration/service.svelte.ts @@ -0,0 +1,165 @@ +import { limitFunction } from "p-limit"; +import { SvelteMap } from "svelte/reactivity"; +import { CHUNK_SIZE } from "$lib/constants"; +import { encodeToBase64, encryptChunk, digestMessage } from "$lib/modules/crypto"; +import { deleteFileCache } from "$lib/modules/file"; +import type { FileInfo } from "$lib/modules/filesystem"; +import { Scheduler } from "$lib/modules/scheduler"; +import { requestFileDownload } from "$lib/services/file"; +import { trpc } from "$trpc/client"; + +export type MigrationStatus = + | "queued" + | "download-pending" + | "downloading" + | "encryption-pending" + | "encrypting" + | "upload-pending" + | "uploading" + | "completed" + | "error"; + +export interface MigrationState { + status: MigrationStatus; + progress?: number; + rate?: number; +} + +const scheduler = new Scheduler(); +const states = new SvelteMap(); + +const createState = (status: MigrationStatus): MigrationState => { + const state = $state({ status }); + return state; +}; + +export const getMigrationState = (fileId: number) => { + return states.get(fileId); +}; + +export const clearMigrationStates = () => { + for (const [id, state] of states) { + if (state.status === "completed" || state.status === "error") { + states.delete(id); + } + } +}; + +const encryptChunks = async (fileBuffer: ArrayBuffer, dataKey: CryptoKey) => { + const chunksEncrypted: { chunkEncrypted: ArrayBuffer; chunkEncryptedHash: string }[] = []; + let offset = 0; + + while (offset < fileBuffer.byteLength) { + const nextOffset = Math.min(offset + CHUNK_SIZE, fileBuffer.byteLength); + const chunkEncrypted = await encryptChunk(fileBuffer.slice(offset, nextOffset), dataKey); + chunksEncrypted.push({ + chunkEncrypted: chunkEncrypted, + chunkEncryptedHash: encodeToBase64(await digestMessage(chunkEncrypted)), + }); + offset = nextOffset; + } + + return chunksEncrypted; +}; + +const uploadMigrationChunks = limitFunction( + async ( + state: MigrationState, + fileId: number, + chunksEncrypted: { chunkEncrypted: ArrayBuffer; chunkEncryptedHash: string }[], + ) => { + state.status = "uploading"; + + const { uploadId } = await trpc().upload.startMigrationUpload.mutate({ + file: fileId, + chunks: chunksEncrypted.length, + }); + + const totalBytes = chunksEncrypted.reduce((sum, c) => sum + c.chunkEncrypted.byteLength, 0); + let uploadedBytes = 0; + const startTime = Date.now(); + + for (let i = 0; i < chunksEncrypted.length; i++) { + const { chunkEncrypted, chunkEncryptedHash } = chunksEncrypted[i]!; + + const response = await fetch(`/api/upload/${uploadId}/chunks/${i}`, { + method: "POST", + headers: { + "Content-Type": "application/octet-stream", + "Content-Digest": `sha-256=:${chunkEncryptedHash}:`, + }, + body: chunkEncrypted, + }); + + if (!response.ok) { + throw new Error(`Chunk upload failed: ${response.status} ${response.statusText}`); + } + + uploadedBytes += chunkEncrypted.byteLength; + + const elapsed = (Date.now() - startTime) / 1000; + const rate = uploadedBytes / elapsed; + + state.progress = uploadedBytes / totalBytes; + state.rate = rate; + } + + await trpc().upload.completeMigrationUpload.mutate({ uploadId }); + }, + { concurrency: 1 }, +); + +const encryptFile = limitFunction( + async (state: MigrationState, fileBuffer: ArrayBuffer, dataKey: CryptoKey) => { + state.status = "encrypting"; + const chunksEncrypted = await encryptChunks(fileBuffer, dataKey); + state.status = "upload-pending"; + return chunksEncrypted; + }, + { concurrency: 4 }, +); + +export const requestFileMigration = async (fileInfo: FileInfo & { exists: true }) => { + let state = states.get(fileInfo.id); + if (state) { + if (state.status !== "error") return; + state.status = "queued"; + state.progress = undefined; + state.rate = undefined; + } else { + state = createState("queued"); + states.set(fileInfo.id, state); + } + + try { + const dataKey = fileInfo.dataKey?.key; + if (!dataKey) { + throw new Error("Data key not available"); + } + + let fileBuffer: ArrayBuffer | undefined; + + await scheduler.schedule( + async () => { + state.status = "download-pending"; + state.status = "downloading"; + fileBuffer = await requestFileDownload(fileInfo.id, dataKey, true); + return fileBuffer.byteLength; + }, + async () => { + state.status = "encryption-pending"; + const chunksEncrypted = await encryptFile(state, fileBuffer!, dataKey); + + await uploadMigrationChunks(state, fileInfo.id, chunksEncrypted); + + // Clear file cache since the file format has changed + await deleteFileCache(fileInfo.id); + + state.status = "completed"; + }, + ); + } catch (e) { + state.status = "error"; + throw e; + } +}; diff --git a/src/routes/(main)/menu/+page.svelte b/src/routes/(main)/menu/+page.svelte index 40f4a26..2bfd3fc 100644 --- a/src/routes/(main)/menu/+page.svelte +++ b/src/routes/(main)/menu/+page.svelte @@ -5,6 +5,7 @@ import IconStorage from "~icons/material-symbols/storage"; import IconImage from "~icons/material-symbols/image"; + import IconLockReset from "~icons/material-symbols/lock-reset"; import IconPassword from "~icons/material-symbols/password"; import IconLogout from "~icons/material-symbols/logout"; @@ -41,6 +42,13 @@ > 썸네일 + goto("/settings/migration")} + icon={IconLockReset} + iconColor="text-teal-500" + > + 암호화 마이그레이션 +

보안

diff --git a/src/trpc/routers/file.ts b/src/trpc/routers/file.ts index 294300c..d6d658c 100644 --- a/src/trpc/routers/file.ts +++ b/src/trpc/routers/file.ts @@ -100,6 +100,10 @@ const fileRouter = router({ return await MediaRepo.getMissingFileThumbnails(ctx.session.userId); }), + listLegacy: roleProcedure["activeClient"].query(async ({ ctx }) => { + return await FileRepo.getLegacyFileIds(ctx.session.userId); + }), + rename: roleProcedure["activeClient"] .input( z.object({ diff --git a/src/trpc/routers/upload.ts b/src/trpc/routers/upload.ts index 168e957..adc0a3e 100644 --- a/src/trpc/routers/upload.ts +++ b/src/trpc/routers/upload.ts @@ -250,6 +250,110 @@ const uploadRouter = router({ sessionLocks.delete(uploadId); } }), + + startMigrationUpload: roleProcedure["activeClient"] + .input( + z.object({ + file: z.int().positive(), + chunks: z.int().positive(), + }), + ) + .mutation(async ({ ctx, input }) => { + const { id, path } = await generateSessionId(); + + try { + await UploadRepo.createMigrationUploadSession({ + id, + userId: ctx.session.userId, + path, + totalChunks: input.chunks, + expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours + fileId: input.file, + }); + return { uploadId: id }; + } catch (e) { + await safeRecursiveRm(path); + + if (e instanceof IntegrityError) { + if (e.message === "File not found") { + throw new TRPCError({ code: "NOT_FOUND", message: "File not found" }); + } else if (e.message === "File is not legacy") { + throw new TRPCError({ code: "BAD_REQUEST", message: "File is not legacy" }); + } + } + throw e; + } + }), + + completeMigrationUpload: roleProcedure["activeClient"] + .input( + z.object({ + uploadId: z.uuidv4(), + }), + ) + .mutation(async ({ ctx, input }) => { + const { uploadId } = input; + if (sessionLocks.has(uploadId)) { + throw new TRPCError({ code: "CONFLICT", message: "Completion already in progress" }); + } else { + sessionLocks.add(uploadId); + } + + let filePath = ""; + + try { + const session = await UploadRepo.getUploadSession(uploadId, ctx.session.userId); + if (!session || session.type !== "migration") { + throw new TRPCError({ code: "NOT_FOUND", message: "Invalid upload id" }); + } else if (session.uploadedChunks.length < session.totalChunks) { + throw new TRPCError({ code: "BAD_REQUEST", message: "Upload not completed" }); + } + + filePath = `${env.libraryPath}/${ctx.session.userId}/${uuidv4()}`; + await mkdir(dirname(filePath), { recursive: true }); + + const hashStream = createHash("sha256"); + const writeStream = createWriteStream(filePath, { flags: "wx", mode: 0o600 }); + + for (let i = 0; i < session.totalChunks; i++) { + for await (const chunk of createReadStream(`${session.path}/${i}`)) { + hashStream.update(chunk); + writeStream.write(chunk); + } + } + + await new Promise((resolve, reject) => { + writeStream.end((e: any) => (e ? reject(e) : resolve())); + }); + + const hash = hashStream.digest("base64"); + const oldPath = await db.transaction().execute(async (trx) => { + const oldPath = await FileRepo.migrateFileContent( + trx, + ctx.session.userId, + session.fileId, + filePath, + hash, + ); + await UploadRepo.deleteUploadSession(trx, uploadId); + return oldPath; + }); + + await Promise.all([safeUnlink(oldPath), safeRecursiveRm(session.path)]); + } catch (e) { + await safeUnlink(filePath); + if (e instanceof IntegrityError) { + if (e.message === "File not found") { + throw new TRPCError({ code: "NOT_FOUND", message: "File not found" }); + } else if (e.message === "File is not legacy") { + throw new TRPCError({ code: "BAD_REQUEST", message: "File is not legacy" }); + } + } + throw e; + } finally { + sessionLocks.delete(uploadId); + } + }), }); export default uploadRouter;