4 Commits

53 changed files with 1204 additions and 425 deletions

View File

@@ -12,6 +12,7 @@ node_modules
/data /data
/library /library
/thumbnails /thumbnails
/uploads
# OS # OS
.DS_Store .DS_Store

View File

@@ -12,3 +12,4 @@ USER_CLIENT_CHALLENGE_EXPIRES=
SESSION_UPGRADE_CHALLENGE_EXPIRES= SESSION_UPGRADE_CHALLENGE_EXPIRES=
LIBRARY_PATH= LIBRARY_PATH=
THUMBNAILS_PATH= THUMBNAILS_PATH=
UPLOADS_PATH=

1
.gitignore vendored
View File

@@ -10,6 +10,7 @@ node_modules
/data /data
/library /library
/thumbnails /thumbnails
/uploads
# OS # OS
.DS_Store .DS_Store

View File

@@ -20,6 +20,7 @@ services:
- SESSION_UPGRADE_CHALLENGE_EXPIRES - SESSION_UPGRADE_CHALLENGE_EXPIRES
- LIBRARY_PATH=/app/data/library - LIBRARY_PATH=/app/data/library
- THUMBNAILS_PATH=/app/data/thumbnails - THUMBNAILS_PATH=/app/data/thumbnails
- UPLOADS_PATH=/app/data/uploads
# SvelteKit # SvelteKit
- ADDRESS_HEADER=${TRUST_PROXY:+X-Forwarded-For} - ADDRESS_HEADER=${TRUST_PROXY:+X-Forwarded-For}
- XFF_DEPTH=${TRUST_PROXY:-} - XFF_DEPTH=${TRUST_PROXY:-}

View File

@@ -1,7 +1,6 @@
import type { ClientInit } from "@sveltejs/kit"; import type { ClientInit } from "@sveltejs/kit";
import { cleanupDanglingInfos, getClientKey, getMasterKeys, getHmacSecrets } from "$lib/indexedDB"; import { cleanupDanglingInfos, getClientKey, getMasterKeys, getHmacSecrets } from "$lib/indexedDB";
import { prepareFileCache } from "$lib/modules/file"; import { prepareFileCache } from "$lib/modules/file";
import { prepareOpfs } from "$lib/modules/opfs";
import { clientKeyStore, masterKeyStore, hmacSecretStore } from "$lib/stores"; import { clientKeyStore, masterKeyStore, hmacSecretStore } from "$lib/stores";
const requestPersistentStorage = async () => { const requestPersistentStorage = async () => {
@@ -46,7 +45,6 @@ export const init: ClientInit = async () => {
prepareClientKeyStore(), prepareClientKeyStore(),
prepareMasterKeyStore(), prepareMasterKeyStore(),
prepareHmacSecretStore(), prepareHmacSecretStore(),
prepareOpfs(),
]); ]);
cleanupDanglingInfos(); // Intended cleanupDanglingInfos(); // Intended

View File

@@ -7,6 +7,7 @@ import {
cleanupExpiredSessions, cleanupExpiredSessions,
cleanupExpiredSessionUpgradeChallenges, cleanupExpiredSessionUpgradeChallenges,
} from "$lib/server/db/session"; } from "$lib/server/db/session";
import { cleanupExpiredUploadSessions } from "$lib/server/db/upload";
import { authenticate, setAgentInfo } from "$lib/server/middlewares"; import { authenticate, setAgentInfo } from "$lib/server/middlewares";
export const init: ServerInit = async () => { export const init: ServerInit = async () => {
@@ -16,6 +17,7 @@ export const init: ServerInit = async () => {
cleanupExpiredUserClientChallenges(); cleanupExpiredUserClientChallenges();
cleanupExpiredSessions(); cleanupExpiredSessions();
cleanupExpiredSessionUpgradeChallenges(); cleanupExpiredSessionUpgradeChallenges();
cleanupExpiredUploadSessions();
}); });
}; };

View File

@@ -0,0 +1,2 @@
export * from "./serviceWorker";
export * from "./upload";

View File

@@ -0,0 +1 @@
export const DECRYPTED_FILE_URL_PREFIX = "/_internal/decrypted-file/";

View File

@@ -0,0 +1,6 @@
export const AES_GCM_IV_SIZE = 12;
export const AES_GCM_TAG_SIZE = 16;
export const ENCRYPTION_OVERHEAD = AES_GCM_IV_SIZE + AES_GCM_TAG_SIZE;
export const CHUNK_SIZE = 4 * 1024 * 1024;
export const ENCRYPTED_CHUNK_SIZE = CHUNK_SIZE + ENCRYPTION_OVERHEAD;

View File

@@ -1,8 +1,15 @@
import { encodeString, decodeString, encodeToBase64, decodeFromBase64 } from "./util"; import { AES_GCM_IV_SIZE } from "$lib/constants";
import {
encodeString,
decodeString,
encodeToBase64,
decodeFromBase64,
concatenateBuffers,
} from "./util";
export const generateMasterKey = async () => { export const generateMasterKey = async () => {
return { return {
masterKey: await window.crypto.subtle.generateKey( masterKey: await crypto.subtle.generateKey(
{ {
name: "AES-KW", name: "AES-KW",
length: 256, length: 256,
@@ -15,7 +22,7 @@ export const generateMasterKey = async () => {
export const generateDataKey = async () => { export const generateDataKey = async () => {
return { return {
dataKey: await window.crypto.subtle.generateKey( dataKey: await crypto.subtle.generateKey(
{ {
name: "AES-GCM", name: "AES-GCM",
length: 256, length: 256,
@@ -28,9 +35,9 @@ export const generateDataKey = async () => {
}; };
export const makeAESKeyNonextractable = async (key: CryptoKey) => { export const makeAESKeyNonextractable = async (key: CryptoKey) => {
return await window.crypto.subtle.importKey( return await crypto.subtle.importKey(
"raw", "raw",
await window.crypto.subtle.exportKey("raw", key), await crypto.subtle.exportKey("raw", key),
key.algorithm, key.algorithm,
false, false,
key.usages, key.usages,
@@ -38,12 +45,12 @@ export const makeAESKeyNonextractable = async (key: CryptoKey) => {
}; };
export const wrapDataKey = async (dataKey: CryptoKey, masterKey: CryptoKey) => { export const wrapDataKey = async (dataKey: CryptoKey, masterKey: CryptoKey) => {
return encodeToBase64(await window.crypto.subtle.wrapKey("raw", dataKey, masterKey, "AES-KW")); return encodeToBase64(await crypto.subtle.wrapKey("raw", dataKey, masterKey, "AES-KW"));
}; };
export const unwrapDataKey = async (dataKeyWrapped: string, masterKey: CryptoKey) => { export const unwrapDataKey = async (dataKeyWrapped: string, masterKey: CryptoKey) => {
return { return {
dataKey: await window.crypto.subtle.unwrapKey( dataKey: await crypto.subtle.unwrapKey(
"raw", "raw",
decodeFromBase64(dataKeyWrapped), decodeFromBase64(dataKeyWrapped),
masterKey, masterKey,
@@ -56,12 +63,12 @@ export const unwrapDataKey = async (dataKeyWrapped: string, masterKey: CryptoKey
}; };
export const wrapHmacSecret = async (hmacSecret: CryptoKey, masterKey: CryptoKey) => { export const wrapHmacSecret = async (hmacSecret: CryptoKey, masterKey: CryptoKey) => {
return encodeToBase64(await window.crypto.subtle.wrapKey("raw", hmacSecret, masterKey, "AES-KW")); return encodeToBase64(await crypto.subtle.wrapKey("raw", hmacSecret, masterKey, "AES-KW"));
}; };
export const unwrapHmacSecret = async (hmacSecretWrapped: string, masterKey: CryptoKey) => { export const unwrapHmacSecret = async (hmacSecretWrapped: string, masterKey: CryptoKey) => {
return { return {
hmacSecret: await window.crypto.subtle.unwrapKey( hmacSecret: await crypto.subtle.unwrapKey(
"raw", "raw",
decodeFromBase64(hmacSecretWrapped), decodeFromBase64(hmacSecretWrapped),
masterKey, masterKey,
@@ -77,8 +84,8 @@ export const unwrapHmacSecret = async (hmacSecretWrapped: string, masterKey: Cry
}; };
export const encryptData = async (data: BufferSource, dataKey: CryptoKey) => { export const encryptData = async (data: BufferSource, dataKey: CryptoKey) => {
const iv = window.crypto.getRandomValues(new Uint8Array(12)); const iv = crypto.getRandomValues(new Uint8Array(12));
const ciphertext = await window.crypto.subtle.encrypt( const ciphertext = await crypto.subtle.encrypt(
{ {
name: "AES-GCM", name: "AES-GCM",
iv, iv,
@@ -86,14 +93,18 @@ export const encryptData = async (data: BufferSource, dataKey: CryptoKey) => {
dataKey, dataKey,
data, data,
); );
return { ciphertext, iv: encodeToBase64(iv.buffer) }; return { ciphertext, iv: iv.buffer };
}; };
export const decryptData = async (ciphertext: BufferSource, iv: string, dataKey: CryptoKey) => { export const decryptData = async (
return await window.crypto.subtle.decrypt( ciphertext: BufferSource,
iv: string | BufferSource,
dataKey: CryptoKey,
) => {
return await crypto.subtle.decrypt(
{ {
name: "AES-GCM", name: "AES-GCM",
iv: decodeFromBase64(iv), iv: typeof iv === "string" ? decodeFromBase64(iv) : iv,
} satisfies AesGcmParams, } satisfies AesGcmParams,
dataKey, dataKey,
ciphertext, ciphertext,
@@ -102,9 +113,22 @@ export const decryptData = async (ciphertext: BufferSource, iv: string, dataKey:
export const encryptString = async (plaintext: string, dataKey: CryptoKey) => { export const encryptString = async (plaintext: string, dataKey: CryptoKey) => {
const { ciphertext, iv } = await encryptData(encodeString(plaintext), dataKey); const { ciphertext, iv } = await encryptData(encodeString(plaintext), dataKey);
return { ciphertext: encodeToBase64(ciphertext), iv }; return { ciphertext: encodeToBase64(ciphertext), iv: encodeToBase64(iv) };
}; };
export const decryptString = async (ciphertext: string, iv: string, dataKey: CryptoKey) => { export const decryptString = async (ciphertext: string, iv: string, dataKey: CryptoKey) => {
return decodeString(await decryptData(decodeFromBase64(ciphertext), iv, dataKey)); return decodeString(await decryptData(decodeFromBase64(ciphertext), iv, dataKey));
}; };
export const encryptChunk = async (chunk: ArrayBuffer, dataKey: CryptoKey) => {
const { ciphertext, iv } = await encryptData(chunk, dataKey);
return concatenateBuffers(iv, ciphertext).buffer;
};
export const decryptChunk = async (encryptedChunk: ArrayBuffer, dataKey: CryptoKey) => {
return await decryptData(
encryptedChunk.slice(AES_GCM_IV_SIZE),
encryptedChunk.slice(0, AES_GCM_IV_SIZE),
dataKey,
);
};

View File

@@ -1,7 +1,7 @@
import { encodeString, encodeToBase64, decodeFromBase64 } from "./util"; import { encodeString, encodeToBase64, decodeFromBase64 } from "./util";
export const generateEncryptionKeyPair = async () => { export const generateEncryptionKeyPair = async () => {
const keyPair = await window.crypto.subtle.generateKey( const keyPair = await crypto.subtle.generateKey(
{ {
name: "RSA-OAEP", name: "RSA-OAEP",
modulusLength: 4096, modulusLength: 4096,
@@ -18,7 +18,7 @@ export const generateEncryptionKeyPair = async () => {
}; };
export const generateSigningKeyPair = async () => { export const generateSigningKeyPair = async () => {
const keyPair = await window.crypto.subtle.generateKey( const keyPair = await crypto.subtle.generateKey(
{ {
name: "RSA-PSS", name: "RSA-PSS",
modulusLength: 4096, modulusLength: 4096,
@@ -37,7 +37,7 @@ export const generateSigningKeyPair = async () => {
export const exportRSAKey = async (key: CryptoKey) => { export const exportRSAKey = async (key: CryptoKey) => {
const format = key.type === "public" ? ("spki" as const) : ("pkcs8" as const); const format = key.type === "public" ? ("spki" as const) : ("pkcs8" as const);
return { return {
key: await window.crypto.subtle.exportKey(format, key), key: await crypto.subtle.exportKey(format, key),
format, format,
}; };
}; };
@@ -54,14 +54,14 @@ export const importEncryptionKeyPairFromBase64 = async (
name: "RSA-OAEP", name: "RSA-OAEP",
hash: "SHA-256", hash: "SHA-256",
}; };
const encryptKey = await window.crypto.subtle.importKey( const encryptKey = await crypto.subtle.importKey(
"spki", "spki",
decodeFromBase64(encryptKeyBase64), decodeFromBase64(encryptKeyBase64),
algorithm, algorithm,
true, true,
["encrypt", "wrapKey"], ["encrypt", "wrapKey"],
); );
const decryptKey = await window.crypto.subtle.importKey( const decryptKey = await crypto.subtle.importKey(
"pkcs8", "pkcs8",
decodeFromBase64(decryptKeyBase64), decodeFromBase64(decryptKeyBase64),
algorithm, algorithm,
@@ -79,14 +79,14 @@ export const importSigningKeyPairFromBase64 = async (
name: "RSA-PSS", name: "RSA-PSS",
hash: "SHA-256", hash: "SHA-256",
}; };
const signKey = await window.crypto.subtle.importKey( const signKey = await crypto.subtle.importKey(
"pkcs8", "pkcs8",
decodeFromBase64(signKeyBase64), decodeFromBase64(signKeyBase64),
algorithm, algorithm,
true, true,
["sign"], ["sign"],
); );
const verifyKey = await window.crypto.subtle.importKey( const verifyKey = await crypto.subtle.importKey(
"spki", "spki",
decodeFromBase64(verifyKeyBase64), decodeFromBase64(verifyKeyBase64),
algorithm, algorithm,
@@ -98,17 +98,11 @@ export const importSigningKeyPairFromBase64 = async (
export const makeRSAKeyNonextractable = async (key: CryptoKey) => { export const makeRSAKeyNonextractable = async (key: CryptoKey) => {
const { key: exportedKey, format } = await exportRSAKey(key); const { key: exportedKey, format } = await exportRSAKey(key);
return await window.crypto.subtle.importKey( return await crypto.subtle.importKey(format, exportedKey, key.algorithm, false, key.usages);
format,
exportedKey,
key.algorithm,
false,
key.usages,
);
}; };
export const decryptChallenge = async (challenge: string, decryptKey: CryptoKey) => { export const decryptChallenge = async (challenge: string, decryptKey: CryptoKey) => {
return await window.crypto.subtle.decrypt( return await crypto.subtle.decrypt(
{ {
name: "RSA-OAEP", name: "RSA-OAEP",
} satisfies RsaOaepParams, } satisfies RsaOaepParams,
@@ -119,7 +113,7 @@ export const decryptChallenge = async (challenge: string, decryptKey: CryptoKey)
export const wrapMasterKey = async (masterKey: CryptoKey, encryptKey: CryptoKey) => { export const wrapMasterKey = async (masterKey: CryptoKey, encryptKey: CryptoKey) => {
return encodeToBase64( return encodeToBase64(
await window.crypto.subtle.wrapKey("raw", masterKey, encryptKey, { await crypto.subtle.wrapKey("raw", masterKey, encryptKey, {
name: "RSA-OAEP", name: "RSA-OAEP",
} satisfies RsaOaepParams), } satisfies RsaOaepParams),
); );
@@ -131,7 +125,7 @@ export const unwrapMasterKey = async (
extractable = false, extractable = false,
) => { ) => {
return { return {
masterKey: await window.crypto.subtle.unwrapKey( masterKey: await crypto.subtle.unwrapKey(
"raw", "raw",
decodeFromBase64(masterKeyWrapped), decodeFromBase64(masterKeyWrapped),
decryptKey, decryptKey,
@@ -146,7 +140,7 @@ export const unwrapMasterKey = async (
}; };
export const signMessageRSA = async (message: BufferSource, signKey: CryptoKey) => { export const signMessageRSA = async (message: BufferSource, signKey: CryptoKey) => {
return await window.crypto.subtle.sign( return await crypto.subtle.sign(
{ {
name: "RSA-PSS", name: "RSA-PSS",
saltLength: 32, // SHA-256 saltLength: 32, // SHA-256
@@ -161,7 +155,7 @@ export const verifySignatureRSA = async (
signature: BufferSource, signature: BufferSource,
verifyKey: CryptoKey, verifyKey: CryptoKey,
) => { ) => {
return await window.crypto.subtle.verify( return await crypto.subtle.verify(
{ {
name: "RSA-PSS", name: "RSA-PSS",
saltLength: 32, // SHA-256 saltLength: 32, // SHA-256

View File

@@ -1,10 +1,10 @@
export const digestMessage = async (message: BufferSource) => { export const digestMessage = async (message: BufferSource) => {
return await window.crypto.subtle.digest("SHA-256", message); return await crypto.subtle.digest("SHA-256", message);
}; };
export const generateHmacSecret = async () => { export const generateHmacSecret = async () => {
return { return {
hmacSecret: await window.crypto.subtle.generateKey( hmacSecret: await crypto.subtle.generateKey(
{ {
name: "HMAC", name: "HMAC",
hash: "SHA-256", hash: "SHA-256",
@@ -16,5 +16,5 @@ export const generateHmacSecret = async () => {
}; };
export const signMessageHmac = async (message: BufferSource, hmacSecret: CryptoKey) => { export const signMessageHmac = async (message: BufferSource, hmacSecret: CryptoKey) => {
return await window.crypto.subtle.sign("HMAC", hmacSecret, message); return await crypto.subtle.sign("HMAC", hmacSecret, message);
}; };

View File

@@ -1,6 +1,7 @@
import axios from "axios"; import axios from "axios";
import { limitFunction } from "p-limit"; import { limitFunction } from "p-limit";
import { decryptData } from "$lib/modules/crypto"; import { CHUNK_SIZE, ENCRYPTION_OVERHEAD } from "$lib/constants";
import { decryptChunk, concatenateBuffers } from "$lib/modules/crypto";
export interface FileDownloadState { export interface FileDownloadState {
id: number; id: number;
@@ -65,13 +66,21 @@ const decryptFile = limitFunction(
async ( async (
state: FileDownloadState, state: FileDownloadState,
fileEncrypted: ArrayBuffer, fileEncrypted: ArrayBuffer,
fileEncryptedIv: string, encryptedChunkSize: number,
dataKey: CryptoKey, dataKey: CryptoKey,
) => { ) => {
state.status = "decrypting"; state.status = "decrypting";
const fileBuffer = await decryptData(fileEncrypted, fileEncryptedIv, dataKey); const chunks: ArrayBuffer[] = [];
let offset = 0;
while (offset < fileEncrypted.byteLength) {
const nextOffset = Math.min(offset + encryptedChunkSize, fileEncrypted.byteLength);
chunks.push(await decryptChunk(fileEncrypted.slice(offset, nextOffset), dataKey));
offset = nextOffset;
}
const fileBuffer = concatenateBuffers(...chunks).buffer;
state.status = "decrypted"; state.status = "decrypted";
state.result = fileBuffer; state.result = fileBuffer;
return fileBuffer; return fileBuffer;
@@ -79,7 +88,7 @@ const decryptFile = limitFunction(
{ concurrency: 4 }, { concurrency: 4 },
); );
export const downloadFile = async (id: number, fileEncryptedIv: string, dataKey: CryptoKey) => { export const downloadFile = async (id: number, dataKey: CryptoKey, isLegacy: boolean) => {
downloadingFiles.push({ downloadingFiles.push({
id, id,
status: "download-pending", status: "download-pending",
@@ -87,7 +96,13 @@ export const downloadFile = async (id: number, fileEncryptedIv: string, dataKey:
const state = downloadingFiles.at(-1)!; const state = downloadingFiles.at(-1)!;
try { try {
return await decryptFile(state, await requestFileDownload(state, id), fileEncryptedIv, dataKey); const fileEncrypted = await requestFileDownload(state, id);
return await decryptFile(
state,
fileEncrypted,
isLegacy ? fileEncrypted.byteLength : CHUNK_SIZE + ENCRYPTION_OVERHEAD,
dataKey,
);
} catch (e) { } catch (e) {
state.status = "error"; state.status = "error";
throw e; throw e;

View File

@@ -5,7 +5,6 @@ import { decryptData } from "$lib/modules/crypto";
import type { SummarizedFileInfo } from "$lib/modules/filesystem"; import type { SummarizedFileInfo } from "$lib/modules/filesystem";
import { readFile, writeFile, deleteFile, deleteDirectory } from "$lib/modules/opfs"; import { readFile, writeFile, deleteFile, deleteDirectory } from "$lib/modules/opfs";
import { getThumbnailUrl } from "$lib/modules/thumbnail"; import { getThumbnailUrl } from "$lib/modules/thumbnail";
import { isTRPCClientError, trpc } from "$trpc/client";
const loadedThumbnails = new LRUCache<number, Writable<string>>({ max: 100 }); const loadedThumbnails = new LRUCache<number, Writable<string>>({ max: 100 });
const loadingThumbnails = new Map<number, Writable<string | undefined>>(); const loadingThumbnails = new Map<number, Writable<string | undefined>>();
@@ -18,25 +17,18 @@ const fetchFromOpfs = async (fileId: number) => {
}; };
const fetchFromServer = async (fileId: number, dataKey: CryptoKey) => { const fetchFromServer = async (fileId: number, dataKey: CryptoKey) => {
try { const res = await fetch(`/api/file/${fileId}/thumbnail/download`);
const [thumbnailEncrypted, { contentIv: thumbnailEncryptedIv }] = await Promise.all([ if (!res.ok) return null;
fetch(`/api/file/${fileId}/thumbnail/download`),
trpc().file.thumbnail.query({ id: fileId }),
]);
const thumbnailBuffer = await decryptData(
await thumbnailEncrypted.arrayBuffer(),
thumbnailEncryptedIv,
dataKey,
);
void writeFile(`/thumbnail/file/${fileId}`, thumbnailBuffer); const thumbnailEncrypted = await res.arrayBuffer();
return getThumbnailUrl(thumbnailBuffer); const thumbnailBuffer = await decryptData(
} catch (e) { thumbnailEncrypted.slice(12),
if (isTRPCClientError(e) && e.data?.code === "NOT_FOUND") { thumbnailEncrypted.slice(0, 12),
return null; dataKey,
} );
throw e;
} void writeFile(`/thumbnail/file/${fileId}`, thumbnailBuffer);
return getThumbnailUrl(thumbnailBuffer);
}; };
export const getFileThumbnail = (file: SummarizedFileInfo) => { export const getFileThumbnail = (file: SummarizedFileInfo) => {

View File

@@ -1,24 +1,23 @@
import axios from "axios"; import axios from "axios";
import ExifReader from "exifreader"; import ExifReader from "exifreader";
import { limitFunction } from "p-limit"; import { limitFunction } from "p-limit";
import { CHUNK_SIZE } from "$lib/constants";
import { import {
encodeToBase64, encodeToBase64,
generateDataKey, generateDataKey,
wrapDataKey, wrapDataKey,
encryptData, encryptData,
encryptString, encryptString,
encryptChunk,
digestMessage, digestMessage,
signMessageHmac, signMessageHmac,
} from "$lib/modules/crypto"; } from "$lib/modules/crypto";
import { Scheduler } from "$lib/modules/scheduler"; import { Scheduler } from "$lib/modules/scheduler";
import { generateThumbnail } from "$lib/modules/thumbnail"; import { generateThumbnail } from "$lib/modules/thumbnail";
import type { import type { FileThumbnailUploadRequest } from "$lib/server/schemas";
FileThumbnailUploadRequest,
FileUploadRequest,
FileUploadResponse,
} from "$lib/server/schemas";
import type { MasterKey, HmacSecret } from "$lib/stores"; import type { MasterKey, HmacSecret } from "$lib/stores";
import { trpc } from "$trpc/client"; import { trpc } from "$trpc/client";
import type { RouterInputs } from "$trpc/router.server";
export interface FileUploadState { export interface FileUploadState {
name: string; name: string;
@@ -110,6 +109,23 @@ const extractExifDateTime = (fileBuffer: ArrayBuffer) => {
return new Date(utcDate - offsetMs); return new Date(utcDate - offsetMs);
}; };
const encryptChunks = async (fileBuffer: ArrayBuffer, dataKey: CryptoKey) => {
const chunksEncrypted: { chunkEncrypted: ArrayBuffer; chunkEncryptedHash: string }[] = [];
let offset = 0;
while (offset < fileBuffer.byteLength) {
const nextOffset = Math.min(offset + CHUNK_SIZE, fileBuffer.byteLength);
const chunkEncrypted = await encryptChunk(fileBuffer.slice(offset, nextOffset), dataKey);
chunksEncrypted.push({
chunkEncrypted: chunkEncrypted,
chunkEncryptedHash: encodeToBase64(await digestMessage(chunkEncrypted)),
});
offset = nextOffset;
}
return chunksEncrypted;
};
const encryptFile = limitFunction( const encryptFile = limitFunction(
async (state: FileUploadState, file: File, fileBuffer: ArrayBuffer, masterKey: MasterKey) => { async (state: FileUploadState, file: File, fileBuffer: ArrayBuffer, masterKey: MasterKey) => {
state.status = "encrypting"; state.status = "encrypting";
@@ -123,9 +139,7 @@ const encryptFile = limitFunction(
const { dataKey, dataKeyVersion } = await generateDataKey(); const { dataKey, dataKeyVersion } = await generateDataKey();
const dataKeyWrapped = await wrapDataKey(dataKey, masterKey.key); const dataKeyWrapped = await wrapDataKey(dataKey, masterKey.key);
const chunksEncrypted = await encryptChunks(fileBuffer, dataKey);
const fileEncrypted = await encryptData(fileBuffer, dataKey);
const fileEncryptedHash = encodeToBase64(await digestMessage(fileEncrypted.ciphertext));
const nameEncrypted = await encryptString(file.name, dataKey); const nameEncrypted = await encryptString(file.name, dataKey);
const createdAtEncrypted = const createdAtEncrypted =
@@ -142,8 +156,7 @@ const encryptFile = limitFunction(
dataKeyWrapped, dataKeyWrapped,
dataKeyVersion, dataKeyVersion,
fileType, fileType,
fileEncrypted, chunksEncrypted,
fileEncryptedHash,
nameEncrypted, nameEncrypted,
createdAtEncrypted, createdAtEncrypted,
lastModifiedAtEncrypted, lastModifiedAtEncrypted,
@@ -154,30 +167,70 @@ const encryptFile = limitFunction(
); );
const requestFileUpload = limitFunction( const requestFileUpload = limitFunction(
async (state: FileUploadState, form: FormData, thumbnailForm: FormData | null) => { async (
state: FileUploadState,
metadata: RouterInputs["file"]["startUpload"],
chunksEncrypted: { chunkEncrypted: ArrayBuffer; chunkEncryptedHash: string }[],
fileSigned: string | undefined,
thumbnailForm: FormData | null,
) => {
state.status = "uploading"; state.status = "uploading";
const res = await axios.post("/api/file/upload", form, { const { uploadId } = await trpc().file.startUpload.mutate(metadata);
onUploadProgress: ({ progress, rate, estimated }) => {
state.progress = progress;
state.rate = rate;
state.estimated = estimated;
},
});
const { file }: FileUploadResponse = res.data;
// Upload chunks with progress tracking
const totalBytes = chunksEncrypted.reduce((sum, c) => sum + c.chunkEncrypted.byteLength, 0);
let uploadedBytes = 0;
const startTime = Date.now();
for (let i = 0; i < chunksEncrypted.length; i++) {
const { chunkEncrypted, chunkEncryptedHash } = chunksEncrypted[i]!;
const response = await fetch(`/api/file/upload/${uploadId}/chunks/${i}`, {
method: "POST",
headers: {
"Content-Type": "application/octet-stream",
"Content-Digest": `sha-256=:${chunkEncryptedHash}:`,
},
body: chunkEncrypted,
});
if (!response.ok) {
throw new Error(`Chunk upload failed: ${response.status} ${response.statusText}`);
}
uploadedBytes += chunkEncrypted.byteLength;
// Calculate progress, rate, estimated
const elapsed = (Date.now() - startTime) / 1000; // seconds
const rate = uploadedBytes / elapsed; // bytes per second
const remaining = totalBytes - uploadedBytes;
const estimated = rate > 0 ? remaining / rate : undefined;
state.progress = uploadedBytes / totalBytes;
state.rate = rate;
state.estimated = estimated;
}
// Complete upload
const { file: fileId } = await trpc().file.completeUpload.mutate({
uploadId,
contentHmac: fileSigned,
});
// Upload thumbnail if exists
if (thumbnailForm) { if (thumbnailForm) {
try { try {
await axios.post(`/api/file/${file}/thumbnail/upload`, thumbnailForm); await axios.post(`/api/file/${fileId}/thumbnail/upload`, thumbnailForm);
} catch (e) { } catch (e) {
// TODO // TODO: Error handling for thumbnail upload
console.error(e); console.error(e);
} }
} }
state.status = "uploaded"; state.status = "uploaded";
return { fileId: file }; return { fileId };
}, },
{ concurrency: 1 }, { concurrency: 1 },
); );
@@ -215,36 +268,28 @@ export const uploadFile = async (
dataKeyWrapped, dataKeyWrapped,
dataKeyVersion, dataKeyVersion,
fileType, fileType,
fileEncrypted, chunksEncrypted,
fileEncryptedHash,
nameEncrypted, nameEncrypted,
createdAtEncrypted, createdAtEncrypted,
lastModifiedAtEncrypted, lastModifiedAtEncrypted,
thumbnail, thumbnail,
} = await encryptFile(state, file, fileBuffer, masterKey); } = await encryptFile(state, file, fileBuffer, masterKey);
const form = new FormData(); const metadata = {
form.set( chunks: chunksEncrypted.length,
"metadata", parent: parentId,
JSON.stringify({ mekVersion: masterKey.version,
parent: parentId, dek: dataKeyWrapped,
mekVersion: masterKey.version, dekVersion: dataKeyVersion,
dek: dataKeyWrapped, hskVersion: hmacSecret.version,
dekVersion: dataKeyVersion.toISOString(), contentType: fileType,
hskVersion: hmacSecret.version, name: nameEncrypted.ciphertext,
contentHmac: fileSigned, nameIv: nameEncrypted.iv,
contentType: fileType, createdAt: createdAtEncrypted?.ciphertext,
contentIv: fileEncrypted.iv, createdAtIv: createdAtEncrypted?.iv,
name: nameEncrypted.ciphertext, lastModifiedAt: lastModifiedAtEncrypted.ciphertext,
nameIv: nameEncrypted.iv, lastModifiedAtIv: lastModifiedAtEncrypted.iv,
createdAt: createdAtEncrypted?.ciphertext, };
createdAtIv: createdAtEncrypted?.iv,
lastModifiedAt: lastModifiedAtEncrypted.ciphertext,
lastModifiedAtIv: lastModifiedAtEncrypted.iv,
} satisfies FileUploadRequest),
);
form.set("content", new Blob([fileEncrypted.ciphertext]));
form.set("checksum", fileEncryptedHash);
let thumbnailForm = null; let thumbnailForm = null;
if (thumbnail) { if (thumbnail) {
@@ -253,13 +298,19 @@ export const uploadFile = async (
"metadata", "metadata",
JSON.stringify({ JSON.stringify({
dekVersion: dataKeyVersion.toISOString(), dekVersion: dataKeyVersion.toISOString(),
contentIv: thumbnail.iv, contentIv: encodeToBase64(thumbnail.iv),
} satisfies FileThumbnailUploadRequest), } satisfies FileThumbnailUploadRequest),
); );
thumbnailForm.set("content", new Blob([thumbnail.ciphertext])); thumbnailForm.set("content", new Blob([thumbnail.ciphertext]));
} }
const { fileId } = await requestFileUpload(state, form, thumbnailForm); const { fileId } = await requestFileUpload(
state,
metadata,
chunksEncrypted,
fileSigned,
thumbnailForm,
);
return { fileId, fileBuffer, thumbnailBuffer: thumbnail?.plaintext }; return { fileId, fileBuffer, thumbnailBuffer: thumbnail?.plaintext };
} catch (e) { } catch (e) {
state.status = "error"; state.status = "error";

View File

@@ -47,10 +47,10 @@ const cache = new FilesystemCache<number, MaybeFileInfo>({
return storeToIndexedDB({ return storeToIndexedDB({
id, id,
isLegacy: file.isLegacy,
parentId: file.parent, parentId: file.parent,
dataKey: metadata.dataKey, dataKey: metadata.dataKey,
contentType: file.contentType, contentType: file.contentType,
contentIv: file.contentIv,
name: metadata.name, name: metadata.name,
createdAt: metadata.createdAt, createdAt: metadata.createdAt,
lastModifiedAt: metadata.lastModifiedAt, lastModifiedAt: metadata.lastModifiedAt,
@@ -116,9 +116,9 @@ const cache = new FilesystemCache<number, MaybeFileInfo>({
return { return {
id, id,
exists: true as const, exists: true as const,
isLegacy: metadataRaw.isLegacy,
parentId: metadataRaw.parent, parentId: metadataRaw.parent,
contentType: metadataRaw.contentType, contentType: metadataRaw.contentType,
contentIv: metadataRaw.contentIv,
categories, categories,
...metadata, ...metadata,
}; };

View File

@@ -28,10 +28,10 @@ export type SubDirectoryInfo = Omit<LocalDirectoryInfo, "subDirectories" | "file
export interface FileInfo { export interface FileInfo {
id: number; id: number;
isLegacy?: boolean;
parentId: DirectoryId; parentId: DirectoryId;
dataKey?: DataKey; dataKey?: DataKey;
contentType: string; contentType: string;
contentIv?: string;
name: string; name: string;
createdAt?: Date; createdAt?: Date;
lastModifiedAt: Date; lastModifiedAt: Date;
@@ -42,7 +42,7 @@ export type MaybeFileInfo =
| (FileInfo & { exists: true }) | (FileInfo & { exists: true })
| ({ id: number; exists: false } & AllUndefined<Omit<FileInfo, "id">>); | ({ id: number; exists: false } & AllUndefined<Omit<FileInfo, "id">>);
export type SummarizedFileInfo = Omit<FileInfo, "contentIv" | "categories">; export type SummarizedFileInfo = Omit<FileInfo, "categories">;
export type CategoryFileInfo = SummarizedFileInfo & { isRecursive: boolean }; export type CategoryFileInfo = SummarizedFileInfo & { isRecursive: boolean };
interface LocalCategoryInfo { interface LocalCategoryInfo {

14
src/lib/modules/http.ts Normal file
View File

@@ -0,0 +1,14 @@
export const parseRangeHeader = (rangeHeader: string | null) => {
if (!rangeHeader) return undefined;
const firstRange = rangeHeader.split(",")[0]!.trim();
const parts = firstRange.replace(/bytes=/, "").split("-");
return {
start: parts[0] ? parseInt(parts[0], 10) : undefined,
end: parts[1] ? parseInt(parts[1], 10) : undefined,
};
};
export const getContentRangeHeader = (range?: { start: number; end: number; total: number }) => {
return range && { "Content-Range": `bytes ${range.start}-${range.end}/${range.total}` };
};

View File

@@ -1,13 +1,5 @@
let rootHandle: FileSystemDirectoryHandle | null = null;
export const prepareOpfs = async () => {
rootHandle = await navigator.storage.getDirectory();
};
const getFileHandle = async (path: string, create = true) => { const getFileHandle = async (path: string, create = true) => {
if (!rootHandle) { if (path[0] !== "/") {
throw new Error("OPFS not prepared");
} else if (path[0] !== "/") {
throw new Error("Path must be absolute"); throw new Error("Path must be absolute");
} }
@@ -17,7 +9,7 @@ const getFileHandle = async (path: string, create = true) => {
} }
try { try {
let directoryHandle = rootHandle; let directoryHandle = await navigator.storage.getDirectory();
for (const part of parts.slice(0, -1)) { for (const part of parts.slice(0, -1)) {
if (!part) continue; if (!part) continue;
directoryHandle = await directoryHandle.getDirectoryHandle(part, { create }); directoryHandle = await directoryHandle.getDirectoryHandle(part, { create });
@@ -34,12 +26,15 @@ const getFileHandle = async (path: string, create = true) => {
} }
}; };
export const readFile = async (path: string) => { export const getFile = async (path: string) => {
const { fileHandle } = await getFileHandle(path, false); const { fileHandle } = await getFileHandle(path, false);
if (!fileHandle) return null; if (!fileHandle) return null;
const file = await fileHandle.getFile(); return await fileHandle.getFile();
return await file.arrayBuffer(); };
export const readFile = async (path: string) => {
return (await getFile(path))?.arrayBuffer() ?? null;
}; };
export const writeFile = async (path: string, data: ArrayBuffer) => { export const writeFile = async (path: string, data: ArrayBuffer) => {
@@ -61,9 +56,7 @@ export const deleteFile = async (path: string) => {
}; };
const getDirectoryHandle = async (path: string) => { const getDirectoryHandle = async (path: string) => {
if (!rootHandle) { if (path[0] !== "/") {
throw new Error("OPFS not prepared");
} else if (path[0] !== "/") {
throw new Error("Path must be absolute"); throw new Error("Path must be absolute");
} }
@@ -73,7 +66,7 @@ const getDirectoryHandle = async (path: string) => {
} }
try { try {
let directoryHandle = rootHandle; let directoryHandle = await navigator.storage.getDirectory();
let parentHandle; let parentHandle;
for (const part of parts.slice(1)) { for (const part of parts.slice(1)) {
if (!part) continue; if (!part) continue;

View File

@@ -15,8 +15,6 @@ interface Directory {
encName: Ciphertext; encName: Ciphertext;
} }
export type NewDirectory = Omit<Directory, "id">;
interface File { interface File {
id: number; id: number;
parentId: DirectoryId; parentId: DirectoryId;
@@ -28,15 +26,13 @@ interface File {
hskVersion: number | null; hskVersion: number | null;
contentHmac: string | null; contentHmac: string | null;
contentType: string; contentType: string;
encContentIv: string; encContentIv: string | null;
encContentHash: string; encContentHash: string;
encName: Ciphertext; encName: Ciphertext;
encCreatedAt: Ciphertext | null; encCreatedAt: Ciphertext | null;
encLastModifiedAt: Ciphertext; encLastModifiedAt: Ciphertext;
} }
export type NewFile = Omit<File, "id">;
interface FileCategory { interface FileCategory {
id: number; id: number;
parentId: CategoryId; parentId: CategoryId;
@@ -46,7 +42,7 @@ interface FileCategory {
encName: Ciphertext; encName: Ciphertext;
} }
export const registerDirectory = async (params: NewDirectory) => { export const registerDirectory = async (params: Omit<Directory, "id">) => {
await db.transaction().execute(async (trx) => { await db.transaction().execute(async (trx) => {
const mek = await trx const mek = await trx
.selectFrom("master_encryption_key") .selectFrom("master_encryption_key")
@@ -214,69 +210,41 @@ export const unregisterDirectory = async (userId: number, directoryId: number) =
}); });
}; };
export const registerFile = async (params: NewFile) => { export const registerFile = async (trx: typeof db, params: Omit<File, "id">) => {
if ((params.hskVersion && !params.contentHmac) || (!params.hskVersion && params.contentHmac)) { if ((params.hskVersion && !params.contentHmac) || (!params.hskVersion && params.contentHmac)) {
throw new Error("Invalid arguments"); throw new Error("Invalid arguments");
} }
return await db.transaction().execute(async (trx) => { const { fileId } = await trx
const mek = await trx .insertInto("file")
.selectFrom("master_encryption_key") .values({
.select("version") parent_id: params.parentId !== "root" ? params.parentId : null,
.where("user_id", "=", params.userId) user_id: params.userId,
.where("state", "=", "active") path: params.path,
.limit(1) master_encryption_key_version: params.mekVersion,
.forUpdate() encrypted_data_encryption_key: params.encDek,
.executeTakeFirst(); data_encryption_key_version: params.dekVersion,
if (mek?.version !== params.mekVersion) { hmac_secret_key_version: params.hskVersion,
throw new IntegrityError("Inactive MEK version"); content_hmac: params.contentHmac,
} content_type: params.contentType,
encrypted_content_iv: params.encContentIv,
if (params.hskVersion) { encrypted_content_hash: params.encContentHash,
const hsk = await trx encrypted_name: params.encName,
.selectFrom("hmac_secret_key") encrypted_created_at: params.encCreatedAt,
.select("version") encrypted_last_modified_at: params.encLastModifiedAt,
.where("user_id", "=", params.userId) })
.where("state", "=", "active") .returning("id as fileId")
.limit(1) .executeTakeFirstOrThrow();
.forUpdate() await trx
.executeTakeFirst(); .insertInto("file_log")
if (hsk?.version !== params.hskVersion) { .values({
throw new IntegrityError("Inactive HSK version"); file_id: fileId,
} timestamp: new Date(),
} action: "create",
new_name: params.encName,
const { fileId } = await trx })
.insertInto("file") .execute();
.values({ return { id: fileId };
parent_id: params.parentId !== "root" ? params.parentId : null,
user_id: params.userId,
path: params.path,
master_encryption_key_version: params.mekVersion,
encrypted_data_encryption_key: params.encDek,
data_encryption_key_version: params.dekVersion,
hmac_secret_key_version: params.hskVersion,
content_hmac: params.contentHmac,
content_type: params.contentType,
encrypted_content_iv: params.encContentIv,
encrypted_content_hash: params.encContentHash,
encrypted_name: params.encName,
encrypted_created_at: params.encCreatedAt,
encrypted_last_modified_at: params.encLastModifiedAt,
})
.returning("id as fileId")
.executeTakeFirstOrThrow();
await trx
.insertInto("file_log")
.values({
file_id: fileId,
timestamp: new Date(),
action: "create",
new_name: params.encName,
})
.execute();
return { id: fileId };
});
}; };
export const getAllFilesByParent = async (userId: number, parentId: DirectoryId) => { export const getAllFilesByParent = async (userId: number, parentId: DirectoryId) => {

View File

@@ -5,6 +5,7 @@ export * as HskRepo from "./hsk";
export * as MediaRepo from "./media"; export * as MediaRepo from "./media";
export * as MekRepo from "./mek"; export * as MekRepo from "./mek";
export * as SessionRepo from "./session"; export * as SessionRepo from "./session";
export * as UploadRepo from "./upload";
export * as UserRepo from "./user"; export * as UserRepo from "./user";
export * from "./error"; export * from "./error";

View File

@@ -0,0 +1,50 @@
import { Kysely, sql } from "kysely";
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const up = async (db: Kysely<any>) => {
// file.ts
await db.schema
.alterTable("file")
.alterColumn("encrypted_content_iv", (col) => col.dropNotNull())
.execute();
// upload.ts
await db.schema
.createTable("upload_session")
.addColumn("id", "uuid", (col) => col.primaryKey().defaultTo(sql`gen_random_uuid()`))
.addColumn("user_id", "integer", (col) => col.references("user.id").notNull())
.addColumn("total_chunks", "integer", (col) => col.notNull())
.addColumn("uploaded_chunks", sql`integer[]`, (col) => col.notNull().defaultTo(sql`'{}'`))
.addColumn("expires_at", "timestamp(3)", (col) => col.notNull())
.addColumn("parent_id", "integer", (col) => col.references("directory.id"))
.addColumn("master_encryption_key_version", "integer", (col) => col.notNull())
.addColumn("encrypted_data_encryption_key", "text", (col) => col.notNull())
.addColumn("data_encryption_key_version", "timestamp(3)", (col) => col.notNull())
.addColumn("hmac_secret_key_version", "integer")
.addColumn("content_type", "text", (col) => col.notNull())
.addColumn("encrypted_name", "json", (col) => col.notNull())
.addColumn("encrypted_created_at", "json")
.addColumn("encrypted_last_modified_at", "json", (col) => col.notNull())
.addForeignKeyConstraint(
"upload_session_fk01",
["user_id", "master_encryption_key_version"],
"master_encryption_key",
["user_id", "version"],
)
.addForeignKeyConstraint(
"upload_session_fk02",
["user_id", "hmac_secret_key_version"],
"hmac_secret_key",
["user_id", "version"],
)
.execute();
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const down = async (db: Kysely<any>) => {
await db.schema.dropTable("upload_session").execute();
await db.schema
.alterTable("file")
.alterColumn("encrypted_content_iv", (col) => col.setNotNull())
.execute();
};

View File

@@ -1,9 +1,11 @@
import * as Initial1737357000 from "./1737357000-Initial"; import * as Initial1737357000 from "./1737357000-Initial";
import * as AddFileCategory1737422340 from "./1737422340-AddFileCategory"; import * as AddFileCategory1737422340 from "./1737422340-AddFileCategory";
import * as AddThumbnail1738409340 from "./1738409340-AddThumbnail"; import * as AddThumbnail1738409340 from "./1738409340-AddThumbnail";
import * as AddChunkedUpload1768062380 from "./1768062380-AddChunkedUpload";
export default { export default {
"1737357000-Initial": Initial1737357000, "1737357000-Initial": Initial1737357000,
"1737422340-AddFileCategory": AddFileCategory1737422340, "1737422340-AddFileCategory": AddFileCategory1737422340,
"1738409340-AddThumbnail": AddThumbnail1738409340, "1738409340-AddThumbnail": AddThumbnail1738409340,
"1768062380-AddChunkedUpload": AddChunkedUpload1768062380,
}; };

View File

@@ -30,7 +30,7 @@ interface FileTable {
hmac_secret_key_version: number | null; hmac_secret_key_version: number | null;
content_hmac: string | null; // Base64 content_hmac: string | null; // Base64
content_type: string; content_type: string;
encrypted_content_iv: string; // Base64 encrypted_content_iv: string | null; // Base64
encrypted_content_hash: string; // Base64 encrypted_content_hash: string; // Base64
encrypted_name: Ciphertext; encrypted_name: Ciphertext;
encrypted_created_at: Ciphertext | null; encrypted_created_at: Ciphertext | null;

View File

@@ -5,6 +5,7 @@ export * from "./hsk";
export * from "./media"; export * from "./media";
export * from "./mek"; export * from "./mek";
export * from "./session"; export * from "./session";
export * from "./upload";
export * from "./user"; export * from "./user";
export * from "./util"; export * from "./util";

View File

@@ -0,0 +1,26 @@
import type { Generated } from "kysely";
import type { Ciphertext } from "./util";
interface UploadSessionTable {
id: Generated<string>;
user_id: number;
total_chunks: number;
uploaded_chunks: Generated<number[]>;
expires_at: Date;
parent_id: number | null;
master_encryption_key_version: number;
encrypted_data_encryption_key: string; // Base64
data_encryption_key_version: Date;
hmac_secret_key_version: number | null;
content_type: string;
encrypted_name: Ciphertext;
encrypted_created_at: Ciphertext | null;
encrypted_last_modified_at: Ciphertext;
}
declare module "./index" {
interface Database {
upload_session: UploadSessionTable;
}
}

122
src/lib/server/db/upload.ts Normal file
View File

@@ -0,0 +1,122 @@
import { sql } from "kysely";
import { IntegrityError } from "./error";
import db from "./kysely";
import type { Ciphertext } from "./schema";
interface UploadSession {
id: string;
userId: number;
totalChunks: number;
uploadedChunks: number[];
expiresAt: Date;
parentId: DirectoryId;
mekVersion: number;
encDek: string;
dekVersion: Date;
hskVersion: number | null;
contentType: string;
encName: Ciphertext;
encCreatedAt: Ciphertext | null;
encLastModifiedAt: Ciphertext;
}
export const createUploadSession = async (params: Omit<UploadSession, "id" | "uploadedChunks">) => {
return await db.transaction().execute(async (trx) => {
const mek = await trx
.selectFrom("master_encryption_key")
.select("version")
.where("user_id", "=", params.userId)
.where("state", "=", "active")
.limit(1)
.forUpdate()
.executeTakeFirst();
if (mek?.version !== params.mekVersion) {
throw new IntegrityError("Inactive MEK version");
}
if (params.hskVersion) {
const hsk = await trx
.selectFrom("hmac_secret_key")
.select("version")
.where("user_id", "=", params.userId)
.where("state", "=", "active")
.limit(1)
.forUpdate()
.executeTakeFirst();
if (hsk?.version !== params.hskVersion) {
throw new IntegrityError("Inactive HSK version");
}
}
const { sessionId } = await trx
.insertInto("upload_session")
.values({
user_id: params.userId,
total_chunks: params.totalChunks,
expires_at: params.expiresAt,
parent_id: params.parentId !== "root" ? params.parentId : null,
master_encryption_key_version: params.mekVersion,
encrypted_data_encryption_key: params.encDek,
data_encryption_key_version: params.dekVersion,
hmac_secret_key_version: params.hskVersion,
content_type: params.contentType,
encrypted_name: params.encName,
encrypted_created_at: params.encCreatedAt,
encrypted_last_modified_at: params.encLastModifiedAt,
})
.returning("id as sessionId")
.executeTakeFirstOrThrow();
return { id: sessionId };
});
};
export const getUploadSession = async (sessionId: string, userId: number) => {
const session = await db
.selectFrom("upload_session")
.selectAll()
.where("id", "=", sessionId)
.where("user_id", "=", userId)
.where("expires_at", ">", new Date())
.limit(1)
.executeTakeFirst();
return session
? ({
id: session.id,
userId: session.user_id,
totalChunks: session.total_chunks,
uploadedChunks: session.uploaded_chunks,
expiresAt: session.expires_at,
parentId: session.parent_id ?? "root",
mekVersion: session.master_encryption_key_version,
encDek: session.encrypted_data_encryption_key,
dekVersion: session.data_encryption_key_version,
hskVersion: session.hmac_secret_key_version,
contentType: session.content_type,
encName: session.encrypted_name,
encCreatedAt: session.encrypted_created_at,
encLastModifiedAt: session.encrypted_last_modified_at,
} satisfies UploadSession)
: null;
};
export const markChunkAsUploaded = async (sessionId: string, chunkIndex: number) => {
await db
.updateTable("upload_session")
.set({ uploaded_chunks: sql`array_append(uploaded_chunks, ${chunkIndex})` })
.where("id", "=", sessionId)
.execute();
};
export const deleteUploadSession = async (trx: typeof db, sessionId: string) => {
await trx.deleteFrom("upload_session").where("id", "=", sessionId).execute();
};
export const cleanupExpiredUploadSessions = async () => {
const sessions = await db
.deleteFrom("upload_session")
.where("expires_at", "<", new Date())
.returning("id")
.execute();
return sessions.map(({ id }) => id);
};

View File

@@ -26,4 +26,5 @@ export default {
}, },
libraryPath: env.LIBRARY_PATH || "library", libraryPath: env.LIBRARY_PATH || "library",
thumbnailsPath: env.THUMBNAILS_PATH || "thumbnails", thumbnailsPath: env.THUMBNAILS_PATH || "thumbnails",
uploadsPath: env.UPLOADS_PATH || "uploads",
}; };

View File

@@ -1,4 +1,7 @@
import { unlink } from "fs/promises"; import { unlink } from "fs/promises";
import env from "$lib/server/loadenv";
export const getChunkDirectoryPath = (sessionId: string) => `${env.uploadsPath}/${sessionId}`;
export const safeUnlink = async (path: string | null | undefined) => { export const safeUnlink = async (path: string | null | undefined) => {
if (path) { if (path) {

View File

@@ -1,36 +1,7 @@
import mime from "mime";
import { z } from "zod"; import { z } from "zod";
import { directoryIdSchema } from "./directory";
export const fileThumbnailUploadRequest = z.object({ export const fileThumbnailUploadRequest = z.object({
dekVersion: z.iso.datetime(), dekVersion: z.iso.datetime(),
contentIv: z.base64().nonempty(), contentIv: z.base64().nonempty(),
}); });
export type FileThumbnailUploadRequest = z.input<typeof fileThumbnailUploadRequest>; export type FileThumbnailUploadRequest = z.input<typeof fileThumbnailUploadRequest>;
export const fileUploadRequest = z.object({
parent: directoryIdSchema,
mekVersion: z.int().positive(),
dek: z.base64().nonempty(),
dekVersion: z.iso.datetime(),
hskVersion: z.int().positive(),
contentHmac: z.base64().nonempty(),
contentType: z
.string()
.trim()
.nonempty()
.refine((value) => mime.getExtension(value) !== null), // MIME type
contentIv: z.base64().nonempty(),
name: z.base64().nonempty(),
nameIv: z.base64().nonempty(),
createdAt: z.base64().nonempty().optional(),
createdAtIv: z.base64().nonempty().optional(),
lastModifiedAt: z.base64().nonempty(),
lastModifiedAtIv: z.base64().nonempty(),
});
export type FileUploadRequest = z.input<typeof fileUploadRequest>;
export const fileUploadResponse = z.object({
file: z.int().positive(),
});
export type FileUploadResponse = z.output<typeof fileUploadResponse>;

View File

@@ -6,34 +6,80 @@ import { dirname } from "path";
import { Readable } from "stream"; import { Readable } from "stream";
import { pipeline } from "stream/promises"; import { pipeline } from "stream/promises";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { FileRepo, MediaRepo, IntegrityError } from "$lib/server/db"; import { CHUNK_SIZE, ENCRYPTION_OVERHEAD } from "$lib/constants";
import { FileRepo, MediaRepo, UploadRepo, IntegrityError } from "$lib/server/db";
import env from "$lib/server/loadenv"; import env from "$lib/server/loadenv";
import { safeUnlink } from "$lib/server/modules/filesystem"; import { getChunkDirectoryPath, safeUnlink } from "$lib/server/modules/filesystem";
export const getFileStream = async (userId: number, fileId: number) => { const uploadLocks = new Set<string>();
const createEncContentStream = async (
path: string,
iv?: Buffer,
range?: { start?: number; end?: number },
) => {
const { size: fileSize } = await stat(path);
const ivSize = iv?.byteLength ?? 0;
const totalSize = fileSize + ivSize;
const start = range?.start ?? 0;
const end = range?.end ?? totalSize - 1;
if (start > end || start < 0 || end >= totalSize) {
error(416, "Invalid range");
}
return {
encContentStream: Readable.toWeb(
Readable.from(
(async function* () {
if (start < ivSize) {
yield iv!.subarray(start, Math.min(end + 1, ivSize));
}
if (end >= ivSize) {
yield* createReadStream(path, {
start: Math.max(0, start - ivSize),
end: end - ivSize,
});
}
})(),
),
),
range: { start, end, total: totalSize },
};
};
export const getFileStream = async (
userId: number,
fileId: number,
range?: { start?: number; end?: number },
) => {
const file = await FileRepo.getFile(userId, fileId); const file = await FileRepo.getFile(userId, fileId);
if (!file) { if (!file) {
error(404, "Invalid file id"); error(404, "Invalid file id");
} }
const { size } = await stat(file.path); return createEncContentStream(
return { file.path,
encContentStream: Readable.toWeb(createReadStream(file.path)), file.encContentIv ? Buffer.from(file.encContentIv, "base64") : undefined,
encContentSize: size, range,
}; );
}; };
export const getFileThumbnailStream = async (userId: number, fileId: number) => { export const getFileThumbnailStream = async (
userId: number,
fileId: number,
range?: { start?: number; end?: number },
) => {
const thumbnail = await MediaRepo.getFileThumbnail(userId, fileId); const thumbnail = await MediaRepo.getFileThumbnail(userId, fileId);
if (!thumbnail) { if (!thumbnail) {
error(404, "File or its thumbnail not found"); error(404, "File or its thumbnail not found");
} }
const { size } = await stat(thumbnail.path); return createEncContentStream(
return { thumbnail.path,
encContentStream: Readable.toWeb(createReadStream(thumbnail.path)), Buffer.from(thumbnail.encContentIv, "base64"),
encContentSize: size, range,
}; );
}; };
export const uploadFileThumbnail = async ( export const uploadFileThumbnail = async (
@@ -71,56 +117,70 @@ export const uploadFileThumbnail = async (
} }
}; };
export const uploadFile = async ( export const uploadChunk = async (
params: Omit<FileRepo.NewFile, "path" | "encContentHash">, userId: number,
encContentStream: Readable, sessionId: string,
encContentHash: Promise<string>, chunkIndex: number,
encChunkStream: Readable,
encChunkHash: string,
) => { ) => {
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); const lockKey = `${sessionId}/${chunkIndex}`;
const oneMinuteLater = new Date(Date.now() + 60 * 1000); if (uploadLocks.has(lockKey)) {
if (params.dekVersion <= oneDayAgo || params.dekVersion >= oneMinuteLater) { error(409, "Chunk already uploaded"); // TODO: Message
error(400, "Invalid DEK version"); } else {
uploadLocks.add(lockKey);
} }
const path = `${env.libraryPath}/${params.userId}/${uuidv4()}`; const filePath = `${getChunkDirectoryPath(sessionId)}/${chunkIndex}`;
await mkdir(dirname(path), { recursive: true });
try { try {
const hashStream = createHash("sha256"); const session = await UploadRepo.getUploadSession(sessionId, userId);
const [, hash] = await Promise.all([ if (!session) {
pipeline( error(404, "Invalid upload id");
encContentStream, } else if (chunkIndex >= session.totalChunks) {
async function* (source) { error(400, "Invalid chunk index");
for await (const chunk of source) { } else if (session.uploadedChunks.includes(chunkIndex)) {
hashStream.update(chunk); error(409, "Chunk already uploaded");
yield chunk;
}
},
createWriteStream(path, { flags: "wx", mode: 0o600 }),
),
encContentHash,
]);
if (hashStream.digest("base64") !== hash) {
throw new Error("Invalid checksum");
} }
const { id: fileId } = await FileRepo.registerFile({ const isLastChunk = chunkIndex === session.totalChunks - 1;
...params,
path,
encContentHash: hash,
});
return { fileId };
} catch (e) {
await safeUnlink(path);
if (e instanceof IntegrityError && e.message === "Inactive MEK version") { let writtenBytes = 0;
error(400, "Invalid MEK version"); const hashStream = createHash("sha256");
const writeStream = createWriteStream(filePath, { flags: "wx", mode: 0o600 });
for await (const chunk of encChunkStream) {
writtenBytes += chunk.length;
hashStream.update(chunk);
writeStream.write(chunk);
}
await new Promise<void>((resolve, reject) => {
writeStream.end((e: any) => (e ? reject(e) : resolve()));
});
if (hashStream.digest("base64") !== encChunkHash) {
throw new Error("Invalid checksum");
} else if ( } else if (
(!isLastChunk && writtenBytes !== CHUNK_SIZE + ENCRYPTION_OVERHEAD) ||
(isLastChunk &&
(writtenBytes <= ENCRYPTION_OVERHEAD || writtenBytes > CHUNK_SIZE + ENCRYPTION_OVERHEAD))
) {
throw new Error("Invalid chunk size");
}
await UploadRepo.markChunkAsUploaded(sessionId, chunkIndex);
} catch (e) {
await safeUnlink(filePath);
if (
e instanceof Error && e instanceof Error &&
(e.message === "Invalid request body" || e.message === "Invalid checksum") (e.message === "Invalid checksum" || e.message === "Invalid chunk size")
) { ) {
error(400, "Invalid request body"); error(400, "Invalid request body");
} }
throw e; throw e;
} finally {
uploadLocks.delete(lockKey);
} }
}; };

View File

@@ -0,0 +1,39 @@
import { DECRYPTED_FILE_URL_PREFIX } from "$lib/constants";
import type { FileMetadata, ServiceWorkerMessage, ServiceWorkerResponse } from "./types";
const PREPARE_TIMEOUT_MS = 5000;
const getServiceWorker = async () => {
const registration = await navigator.serviceWorker.ready;
const sw = registration.active;
if (!sw) {
throw new Error("Service worker not activated");
}
return sw;
};
export const prepareFileDecryption = async (id: number, metadata: FileMetadata) => {
const sw = await getServiceWorker();
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(
() => reject(new Error("Service worker timeout")),
PREPARE_TIMEOUT_MS,
);
const handler = (event: MessageEvent<ServiceWorkerResponse>) => {
if (event.data.type === "decryption-ready" && event.data.fileId === id) {
clearTimeout(timeout);
navigator.serviceWorker.removeEventListener("message", handler);
resolve();
}
};
navigator.serviceWorker.addEventListener("message", handler);
sw.postMessage({
type: "decryption-prepare",
fileId: id,
...metadata,
} satisfies ServiceWorkerMessage);
});
};
export const getDecryptedFileUrl = (id: number) => `${DECRYPTED_FILE_URL_PREFIX}${id}`;

View File

@@ -0,0 +1,2 @@
export * from "./client";
export * from "./types";

View File

@@ -0,0 +1,19 @@
export interface FileMetadata {
isLegacy: boolean;
dataKey: CryptoKey;
encContentSize: number;
contentType: string;
}
export interface DecryptionPrepareMessage extends FileMetadata {
type: "decryption-prepare";
fileId: number;
}
export interface DecryptionReadyMessage {
type: "decryption-ready";
fileId: number;
}
export type ServiceWorkerMessage = DecryptionPrepareMessage;
export type ServiceWorkerResponse = DecryptionReadyMessage;

View File

@@ -1,4 +1,5 @@
import { getAllFileInfos } from "$lib/indexedDB/filesystem"; import { getAllFileInfos } from "$lib/indexedDB/filesystem";
import { encodeToBase64 } from "$lib/modules/crypto";
import { import {
getFileCache, getFileCache,
storeFileCache, storeFileCache,
@@ -11,13 +12,13 @@ import { trpc } from "$trpc/client";
export const requestFileDownload = async ( export const requestFileDownload = async (
fileId: number, fileId: number,
fileEncryptedIv: string,
dataKey: CryptoKey, dataKey: CryptoKey,
isLegacy: boolean,
) => { ) => {
const cache = await getFileCache(fileId); const cache = await getFileCache(fileId);
if (cache) return cache; if (cache) return cache;
const fileBuffer = await downloadFile(fileId, fileEncryptedIv, dataKey); const fileBuffer = await downloadFile(fileId, dataKey, isLegacy);
storeFileCache(fileId, fileBuffer); // Intended storeFileCache(fileId, fileBuffer); // Intended
return fileBuffer; return fileBuffer;
}; };
@@ -25,14 +26,14 @@ export const requestFileDownload = async (
export const requestFileThumbnailUpload = async ( export const requestFileThumbnailUpload = async (
fileId: number, fileId: number,
dataKeyVersion: Date, dataKeyVersion: Date,
thumbnailEncrypted: { ciphertext: ArrayBuffer; iv: string }, thumbnailEncrypted: { ciphertext: ArrayBuffer; iv: ArrayBuffer },
) => { ) => {
const form = new FormData(); const form = new FormData();
form.set( form.set(
"metadata", "metadata",
JSON.stringify({ JSON.stringify({
dekVersion: dataKeyVersion.toISOString(), dekVersion: dataKeyVersion.toISOString(),
contentIv: thumbnailEncrypted.iv, contentIv: encodeToBase64(thumbnailEncrypted.iv),
} satisfies FileThumbnailUploadRequest), } satisfies FileThumbnailUploadRequest),
); );
form.set("content", new Blob([thumbnailEncrypted.ciphertext])); form.set("content", new Blob([thumbnailEncrypted.ciphertext]));

View File

@@ -5,7 +5,7 @@
import { page } from "$app/state"; import { page } from "$app/state";
import { FullscreenDiv } from "$lib/components/atoms"; import { FullscreenDiv } from "$lib/components/atoms";
import { Categories, IconEntryButton, TopBar } from "$lib/components/molecules"; import { Categories, IconEntryButton, TopBar } from "$lib/components/molecules";
import { getFileInfo, type FileInfo, type MaybeFileInfo } from "$lib/modules/filesystem"; import { getFileInfo, type MaybeFileInfo } from "$lib/modules/filesystem";
import { captureVideoThumbnail } from "$lib/modules/thumbnail"; import { captureVideoThumbnail } from "$lib/modules/thumbnail";
import { getFileDownloadState } from "$lib/modules/file"; import { getFileDownloadState } from "$lib/modules/file";
import { masterKeyStore } from "$lib/stores"; import { masterKeyStore } from "$lib/stores";
@@ -17,6 +17,7 @@
requestFileDownload, requestFileDownload,
requestThumbnailUpload, requestThumbnailUpload,
requestFileAdditionToCategory, requestFileAdditionToCategory,
requestVideoStream,
} from "./service"; } from "./service";
import TopBarMenu from "./TopBarMenu.svelte"; import TopBarMenu from "./TopBarMenu.svelte";
@@ -37,6 +38,7 @@
let viewerType: "image" | "video" | undefined = $state(); let viewerType: "image" | "video" | undefined = $state();
let fileBlob: Blob | undefined = $state(); let fileBlob: Blob | undefined = $state();
let fileBlobUrl: string | undefined = $state(); let fileBlobUrl: string | undefined = $state();
let videoStreamUrl: string | undefined = $state();
let videoElement: HTMLVideoElement | undefined = $state(); let videoElement: HTMLVideoElement | undefined = $state();
const updateViewer = async (buffer: ArrayBuffer, contentType: string) => { const updateViewer = async (buffer: ArrayBuffer, contentType: string) => {
@@ -95,14 +97,27 @@
untrack(() => { untrack(() => {
if (!downloadState && !isDownloadRequested) { if (!downloadState && !isDownloadRequested) {
isDownloadRequested = true; isDownloadRequested = true;
requestFileDownload(data.id, info!.contentIv!, info!.dataKey!.key).then(
async (buffer) => { if (viewerType === "video" && !info!.isLegacy) {
const blob = await updateViewer(buffer, contentType); requestVideoStream(data.id, info!.dataKey!.key, contentType).then((streamUrl) => {
if (!viewerType) { if (streamUrl) {
FileSaver.saveAs(blob, info!.name); videoStreamUrl = streamUrl;
} else {
requestFileDownload(data.id, info!.dataKey!.key, info!.isLegacy!).then((buffer) =>
updateViewer(buffer, contentType),
);
} }
}, });
); } else {
requestFileDownload(data.id, info!.dataKey!.key, info!.isLegacy!).then(
async (buffer) => {
const blob = await updateViewer(buffer, contentType);
if (!viewerType) {
FileSaver.saveAs(blob, info!.name);
}
},
);
}
} }
}); });
} }
@@ -110,7 +125,9 @@
$effect(() => { $effect(() => {
if (info?.exists && downloadState?.status === "decrypted") { if (info?.exists && downloadState?.status === "decrypted") {
untrack(() => !isDownloadRequested && updateViewer(downloadState.result!, info!.contentIv!)); untrack(
() => !isDownloadRequested && updateViewer(downloadState.result!, info!.contentType!),
);
} }
}); });
@@ -137,6 +154,7 @@
? info?.parentId ? info?.parentId
: undefined} : undefined}
{fileBlob} {fileBlob}
downloadUrl={videoStreamUrl}
filename={info?.name} filename={info?.name}
/> />
</div> </div>
@@ -159,9 +177,10 @@
{@render viewerLoading("이미지를 불러오고 있어요.")} {@render viewerLoading("이미지를 불러오고 있어요.")}
{/if} {/if}
{:else if viewerType === "video"} {:else if viewerType === "video"}
{#if fileBlobUrl} {#if videoStreamUrl || fileBlobUrl}
<div class="flex flex-col space-y-2"> <div class="flex flex-col space-y-2">
<video bind:this={videoElement} src={fileBlobUrl} controls muted></video> <video bind:this={videoElement} src={videoStreamUrl ?? fileBlobUrl} controls muted
></video>
<IconEntryButton <IconEntryButton
icon={IconCamera} icon={IconCamera}
onclick={() => updateThumbnail(info?.dataKey?.key!, info?.dataKey?.version!)} onclick={() => updateThumbnail(info?.dataKey?.key!, info?.dataKey?.version!)}

View File

@@ -10,17 +10,29 @@
interface Props { interface Props {
directoryId?: "root" | number; directoryId?: "root" | number;
downloadUrl?: string;
fileBlob?: Blob; fileBlob?: Blob;
filename?: string; filename?: string;
isOpen: boolean; isOpen: boolean;
} }
let { directoryId, fileBlob, filename, isOpen = $bindable() }: Props = $props(); let { directoryId, downloadUrl, fileBlob, filename, isOpen = $bindable() }: Props = $props();
const handleDownload = () => {
if (fileBlob && filename) {
FileSaver.saveAs(fileBlob, filename);
} else if (downloadUrl && filename) {
// Use streaming download via Content-Disposition header
const url = new URL(downloadUrl, window.location.origin);
url.searchParams.set("download", filename);
window.open(url.toString(), "_blank");
}
};
</script> </script>
<svelte:window onclick={() => (isOpen = false)} /> <svelte:window onclick={() => (isOpen = false)} />
{#if isOpen && (directoryId || fileBlob)} {#if isOpen && (directoryId || downloadUrl || fileBlob)}
<div <div
class="absolute right-2 top-full z-20 space-y-1 rounded-lg bg-white px-1 py-2 shadow-2xl" class="absolute right-2 top-full z-20 space-y-1 rounded-lg bg-white px-1 py-2 shadow-2xl"
transition:fly={{ y: -8, duration: 200 }} transition:fly={{ y: -8, duration: 200 }}
@@ -49,10 +61,8 @@
), ),
)} )}
{/if} {/if}
{#if fileBlob} {#if fileBlob || downloadUrl}
{@render menuButton(IconCloudDownload, "다운로드", () => { {@render menuButton(IconCloudDownload, "다운로드", handleDownload)}
FileSaver.saveAs(fileBlob, filename);
})}
{/if} {/if}
</div> </div>
</div> </div>

View File

@@ -1,11 +1,32 @@
import { encryptData } from "$lib/modules/crypto"; import { encryptData } from "$lib/modules/crypto";
import { storeFileThumbnailCache } from "$lib/modules/file"; import { storeFileThumbnailCache } from "$lib/modules/file";
import { prepareFileDecryption, getDecryptedFileUrl } from "$lib/serviceWorker";
import { requestFileThumbnailUpload } from "$lib/services/file"; import { requestFileThumbnailUpload } from "$lib/services/file";
import { trpc } from "$trpc/client"; import { trpc } from "$trpc/client";
export { requestCategoryCreation, requestFileRemovalFromCategory } from "$lib/services/category"; export { requestCategoryCreation, requestFileRemovalFromCategory } from "$lib/services/category";
export { requestFileDownload } from "$lib/services/file"; export { requestFileDownload } from "$lib/services/file";
export const requestVideoStream = async (
fileId: number,
dataKey: CryptoKey,
contentType: string,
) => {
const res = await fetch(`/api/file/${fileId}/download`, { method: "HEAD" });
if (!res.ok) return null;
const encContentSize = parseInt(res.headers.get("Content-Length") ?? "0", 10);
if (encContentSize <= 0) return null;
try {
await prepareFileDecryption(fileId, { isLegacy: false, dataKey, encContentSize, contentType });
return getDecryptedFileUrl(fileId);
} catch {
// TODO: Error Handling
return null;
}
};
export const requestThumbnailUpload = async ( export const requestThumbnailUpload = async (
fileId: number, fileId: number,
thumbnail: Blob, thumbnail: Blob,

View File

@@ -50,7 +50,7 @@ const requestThumbnailUpload = limitFunction(
async ( async (
fileId: number, fileId: number,
dataKeyVersion: Date, dataKeyVersion: Date,
thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: string }, thumbnail: { plaintext: ArrayBuffer; ciphertext: ArrayBuffer; iv: ArrayBuffer },
) => { ) => {
statuses.set(fileId, "uploading"); statuses.set(fileId, "uploading");
@@ -77,7 +77,7 @@ export const requestThumbnailGeneration = async (fileInfo: FileInfo) => {
await scheduler.schedule( await scheduler.schedule(
async () => { async () => {
statuses.set(fileInfo.id, "generation-pending"); statuses.set(fileInfo.id, "generation-pending");
file = await requestFileDownload(fileInfo.id, fileInfo.contentIv!, fileInfo.dataKey?.key!); file = await requestFileDownload(fileInfo.id, fileInfo.dataKey?.key!, fileInfo.isLegacy!);
return file.byteLength; return file.byteLength;
}, },
async () => { async () => {

View File

@@ -1,10 +1,15 @@
import { error } from "@sveltejs/kit"; import { error } from "@sveltejs/kit";
import { z } from "zod"; import { z } from "zod";
import { authorize } from "$lib/server/modules/auth"; import { authorize } from "$lib/server/modules/auth";
import { parseRangeHeader, getContentRangeHeader } from "$lib/modules/http";
import { getFileStream } from "$lib/server/services/file"; import { getFileStream } from "$lib/server/services/file";
import type { RequestHandler } from "./$types"; import type { RequestHandler } from "./$types";
export const GET: RequestHandler = async ({ locals, params }) => { const downloadHandler = async (
locals: App.Locals,
params: Record<string, string>,
request: Request,
) => {
const { userId } = await authorize(locals, "activeClient"); const { userId } = await authorize(locals, "activeClient");
const zodRes = z const zodRes = z
@@ -15,11 +20,29 @@ export const GET: RequestHandler = async ({ locals, params }) => {
if (!zodRes.success) error(400, "Invalid path parameters"); if (!zodRes.success) error(400, "Invalid path parameters");
const { id } = zodRes.data; const { id } = zodRes.data;
const { encContentStream, encContentSize } = await getFileStream(userId, id); const { encContentStream, range } = await getFileStream(
return new Response(encContentStream as ReadableStream, { userId,
id,
parseRangeHeader(request.headers.get("Range")),
);
return {
stream: encContentStream,
headers: { headers: {
"Accept-Ranges": "bytes",
"Content-Length": (range.end - range.start + 1).toString(),
"Content-Type": "application/octet-stream", "Content-Type": "application/octet-stream",
"Content-Length": encContentSize.toString(), ...getContentRangeHeader(range),
}, },
}); isRangeRequest: !!range,
};
};
export const GET: RequestHandler = async ({ locals, params, request }) => {
const { stream, headers, isRangeRequest } = await downloadHandler(locals, params, request);
return new Response(stream as ReadableStream, { status: isRangeRequest ? 206 : 200, headers });
};
export const HEAD: RequestHandler = async ({ locals, params, request }) => {
const { headers, isRangeRequest } = await downloadHandler(locals, params, request);
return new Response(null, { status: isRangeRequest ? 206 : 200, headers });
}; };

View File

@@ -1,10 +1,15 @@
import { error } from "@sveltejs/kit"; import { error } from "@sveltejs/kit";
import { z } from "zod"; import { z } from "zod";
import { authorize } from "$lib/server/modules/auth"; import { authorize } from "$lib/server/modules/auth";
import { parseRangeHeader, getContentRangeHeader } from "$lib/modules/http";
import { getFileThumbnailStream } from "$lib/server/services/file"; import { getFileThumbnailStream } from "$lib/server/services/file";
import type { RequestHandler } from "./$types"; import type { RequestHandler } from "./$types";
export const GET: RequestHandler = async ({ locals, params }) => { const downloadHandler = async (
locals: App.Locals,
params: Record<string, string>,
request: Request,
) => {
const { userId } = await authorize(locals, "activeClient"); const { userId } = await authorize(locals, "activeClient");
const zodRes = z const zodRes = z
@@ -15,11 +20,29 @@ export const GET: RequestHandler = async ({ locals, params }) => {
if (!zodRes.success) error(400, "Invalid path parameters"); if (!zodRes.success) error(400, "Invalid path parameters");
const { id } = zodRes.data; const { id } = zodRes.data;
const { encContentStream, encContentSize } = await getFileThumbnailStream(userId, id); const { encContentStream, range } = await getFileThumbnailStream(
return new Response(encContentStream as ReadableStream, { userId,
id,
parseRangeHeader(request.headers.get("Range")),
);
return {
stream: encContentStream,
headers: { headers: {
"Accept-Ranges": "bytes",
"Content-Length": (range.end - range.start + 1).toString(),
"Content-Type": "application/octet-stream", "Content-Type": "application/octet-stream",
"Content-Length": encContentSize.toString(), ...getContentRangeHeader(range),
}, },
}); isRangeRequest: !!range,
};
};
export const GET: RequestHandler = async ({ locals, params, request }) => {
const { stream, headers, isRangeRequest } = await downloadHandler(locals, params, request);
return new Response(stream as ReadableStream, { status: isRangeRequest ? 206 : 200, headers });
};
export const HEAD: RequestHandler = async ({ locals, params, request }) => {
const { headers, isRangeRequest } = await downloadHandler(locals, params, request);
return new Response(null, { status: isRangeRequest ? 206 : 200, headers });
}; };

View File

@@ -1,108 +0,0 @@
import Busboy from "@fastify/busboy";
import { error, json } from "@sveltejs/kit";
import { Readable, Writable } from "stream";
import { authorize } from "$lib/server/modules/auth";
import {
fileUploadRequest,
fileUploadResponse,
type FileUploadResponse,
} from "$lib/server/schemas";
import { uploadFile } from "$lib/server/services/file";
import type { RequestHandler } from "./$types";
type FileMetadata = Parameters<typeof uploadFile>[0];
const parseFileMetadata = (userId: number, json: string) => {
const zodRes = fileUploadRequest.safeParse(JSON.parse(json));
if (!zodRes.success) error(400, "Invalid request body");
const {
parent,
mekVersion,
dek,
dekVersion,
hskVersion,
contentHmac,
contentType,
contentIv,
name,
nameIv,
createdAt,
createdAtIv,
lastModifiedAt,
lastModifiedAtIv,
} = zodRes.data;
if ((createdAt && !createdAtIv) || (!createdAt && createdAtIv))
error(400, "Invalid request body");
return {
userId,
parentId: parent,
mekVersion,
encDek: dek,
dekVersion: new Date(dekVersion),
hskVersion,
contentHmac,
contentType,
encContentIv: contentIv,
encName: { ciphertext: name, iv: nameIv },
encCreatedAt: createdAt && createdAtIv ? { ciphertext: createdAt, iv: createdAtIv } : null,
encLastModifiedAt: { ciphertext: lastModifiedAt, iv: lastModifiedAtIv },
} satisfies FileMetadata;
};
export const POST: RequestHandler = async ({ locals, request }) => {
const { userId } = await authorize(locals, "activeClient");
const contentType = request.headers.get("Content-Type");
if (!contentType?.startsWith("multipart/form-data") || !request.body) {
error(400, "Invalid request body");
}
return new Promise<Response>((resolve, reject) => {
const bb = Busboy({ headers: { "content-type": contentType } });
const handler =
<T extends unknown[]>(f: (...args: T) => Promise<void>) =>
(...args: T) => {
f(...args).catch(reject);
};
let metadata: FileMetadata | null = null;
let content: Readable | null = null;
const checksum = new Promise<string>((resolveChecksum, rejectChecksum) => {
bb.on(
"field",
handler(async (fieldname, val) => {
if (fieldname === "metadata") {
// Ignore subsequent metadata fields
if (!metadata) {
metadata = parseFileMetadata(userId, val);
}
} else if (fieldname === "checksum") {
// Ignore subsequent checksum fields
resolveChecksum(val);
} else {
error(400, "Invalid request body");
}
}),
);
bb.on(
"file",
handler(async (fieldname, file) => {
if (fieldname !== "content") error(400, "Invalid request body");
if (!metadata || content) error(400, "Invalid request body");
content = file;
const { fileId } = await uploadFile(metadata, content, checksum);
resolve(json(fileUploadResponse.parse({ file: fileId } satisfies FileUploadResponse)));
}),
);
bb.on("finish", () => rejectChecksum(new Error("Invalid request body")));
bb.on("error", (e) => {
content?.emit("error", e) ?? reject(e);
rejectChecksum(e);
});
});
request.body!.pipeTo(Writable.toWeb(bb)).catch(() => {}); // busboy will handle the error
});
};

View File

@@ -0,0 +1,43 @@
import { error, text } from "@sveltejs/kit";
import { Readable } from "stream";
import { z } from "zod";
import { authorize } from "$lib/server/modules/auth";
import { uploadChunk } from "$lib/server/services/file";
import type { RequestHandler } from "./$types";
export const POST: RequestHandler = async ({ locals, params, request }) => {
const { userId } = await authorize(locals, "activeClient");
const zodRes = z
.object({
id: z.uuidv4(),
index: z.coerce.number().int().nonnegative(),
})
.safeParse(params);
if (!zodRes.success) error(400, "Invalid path parameters");
const { id: uploadId, index: chunkIndex } = zodRes.data;
// Parse Content-Digest header (RFC 9530)
// Expected format: sha-256=:base64hash:
const contentDigest = request.headers.get("Content-Digest");
if (!contentDigest) error(400, "Missing Content-Digest header");
const digestMatch = contentDigest.match(/^sha-256=:([A-Za-z0-9+/=]+):$/);
if (!digestMatch || !digestMatch[1])
error(400, "Invalid Content-Digest format, must be sha-256=:base64:");
const encChunkHash = digestMatch[1];
const contentType = request.headers.get("Content-Type");
if (contentType !== "application/octet-stream" || !request.body) {
error(400, "Invalid request body");
}
// Convert web ReadableStream to Node Readable
const nodeReadable = Readable.fromWeb(
request.body as unknown as Parameters<typeof Readable.fromWeb>[0],
);
await uploadChunk(userId, uploadId, chunkIndex, nodeReadable, encChunkHash);
return text("Chunk uploaded", { headers: { "Content-Type": "text/plain" } });
};

View File

@@ -0,0 +1,153 @@
import { DECRYPTED_FILE_URL_PREFIX, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "../modules/constants";
import { decryptChunk, getEncryptedRange, getDecryptedSize } from "../modules/crypto";
import { parseRangeHeader, getContentRangeHeader } from "../modules/http";
import { getFile } from "../modules/opfs";
import { fileMetadataStore } from "../stores";
import type { FileMetadata } from "../types";
const createResponse = (
stream: ReadableStream<Uint8Array>,
isRangeRequest: boolean,
range: { start: number; end: number; total: number },
contentType?: string,
downloadFilename?: string,
) => {
const headers: Record<string, string> = {
"Accept-Ranges": "bytes",
"Content-Length": String(range.end - range.start + 1),
"Content-Type": contentType ?? "application/octet-stream",
...(isRangeRequest ? getContentRangeHeader(range) : {}),
};
if (downloadFilename) {
headers["Content-Disposition"] =
`attachment; filename*=UTF-8''${encodeURIComponent(downloadFilename)}`;
}
return new Response(stream, {
status: isRangeRequest ? 206 : 200,
headers,
});
};
const streamFromOpfs = async (
file: File,
metadata?: FileMetadata,
range?: { start?: number; end?: number },
downloadFilename?: string,
) => {
const start = range?.start ?? 0;
const end = range?.end ?? file.size - 1;
if (start > end || start < 0 || end >= file.size) {
return new Response("Invalid range", { status: 416 });
}
return createResponse(
file.slice(start, end + 1).stream(),
!!range,
{ start, end, total: file.size },
metadata?.contentType,
downloadFilename,
);
};
const streamFromServer = async (
id: number,
metadata: FileMetadata,
range?: { start?: number; end?: number },
downloadFilename?: string,
) => {
const totalSize = getDecryptedSize(metadata.encContentSize, metadata.isLegacy);
const start = range?.start ?? 0;
const end =
range?.end ??
(range && !metadata.isLegacy ? Math.min(start + CHUNK_SIZE, totalSize) : totalSize) - 1;
if (start > end || start < 0 || end >= totalSize) {
return new Response("Invalid range", { status: 416 });
}
const encryptedRange = getEncryptedRange(start, end, metadata.encContentSize, metadata.isLegacy);
const apiResponse = await fetch(`/api/file/${id}/download`, {
headers: { Range: `bytes=${encryptedRange.start}-${encryptedRange.end}` },
});
if (apiResponse.status !== 206 || !apiResponse.body) {
return new Response("Failed to fetch encrypted file", { status: 502 });
}
if (metadata.isLegacy) {
const fileEncrypted = await apiResponse.arrayBuffer();
const decrypted = await decryptChunk(fileEncrypted, metadata.dataKey);
return createResponse(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(decrypted.slice(start, end + 1)));
controller.close();
},
}),
!!range,
{ start, end, total: totalSize },
metadata.contentType,
);
}
const totalChunks = encryptedRange.lastChunkIndex - encryptedRange.firstChunkIndex + 1;
let currentChunkIndex = 0;
let buffer = new Uint8Array(0);
const decryptingStream = new TransformStream<Uint8Array, Uint8Array>({
async transform(chunk, controller) {
const newBuffer = new Uint8Array(buffer.length + chunk.length);
newBuffer.set(buffer);
newBuffer.set(chunk, buffer.length);
buffer = newBuffer;
while (buffer.length >= ENCRYPTED_CHUNK_SIZE && currentChunkIndex < totalChunks - 1) {
const encryptedChunk = buffer.slice(0, ENCRYPTED_CHUNK_SIZE);
buffer = buffer.slice(ENCRYPTED_CHUNK_SIZE);
const decrypted = await decryptChunk(encryptedChunk.buffer, metadata.dataKey);
const sliceStart = currentChunkIndex === 0 ? start % CHUNK_SIZE : 0;
controller.enqueue(new Uint8Array(decrypted.slice(sliceStart)));
currentChunkIndex++;
}
},
async flush(controller) {
if (buffer.length > 0) {
const decrypted = await decryptChunk(buffer.buffer, metadata.dataKey);
const sliceStart = currentChunkIndex === 0 ? start % CHUNK_SIZE : 0;
const sliceEnd = (end % CHUNK_SIZE) + 1;
controller.enqueue(new Uint8Array(decrypted.slice(sliceStart, sliceEnd)));
}
},
});
return createResponse(
apiResponse.body.pipeThrough(decryptingStream),
!!range,
{ start, end, total: totalSize },
metadata.contentType,
downloadFilename,
);
};
const decryptFileHandler = async (request: Request) => {
const url = new URL(request.url);
const fileId = parseInt(url.pathname.slice(DECRYPTED_FILE_URL_PREFIX.length), 10);
if (isNaN(fileId)) {
throw new Response("Invalid file id", { status: 400 });
}
const downloadFilename = url.searchParams.get("download") ?? undefined;
const metadata = fileMetadataStore.get(fileId);
const range = parseRangeHeader(request.headers.get("Range"));
const cache = await getFile(`/cache/${fileId}`);
if (cache) {
return streamFromOpfs(cache, metadata, range, downloadFilename);
} else if (metadata) {
return streamFromServer(fileId, metadata, range, downloadFilename);
} else {
return new Response("Decryption not prepared", { status: 400 });
}
};
export default decryptFileHandler;

View File

@@ -0,0 +1 @@
export { default as decryptFile } from "./decryptFile";

View File

@@ -0,0 +1,43 @@
/// <reference no-default-lib="true"/>
/// <reference lib="esnext" />
/// <reference lib="webworker" />
/// <reference types="@sveltejs/kit" />
import { DECRYPTED_FILE_URL_PREFIX } from "./modules/constants";
import { decryptFile } from "./handlers";
import { fileMetadataStore } from "./stores";
import type { ServiceWorkerMessage, ServiceWorkerResponse } from "./types";
const self = globalThis.self as unknown as ServiceWorkerGlobalScope;
self.addEventListener("message", (event) => {
const message: ServiceWorkerMessage = event.data;
switch (message.type) {
case "decryption-prepare":
fileMetadataStore.set(message.fileId, message);
event.source?.postMessage({
type: "decryption-ready",
fileId: message.fileId,
} satisfies ServiceWorkerResponse);
break;
default: {
const exhaustive: never = message.type;
return exhaustive;
}
}
});
self.addEventListener("fetch", (event) => {
const url = new URL(event.request.url);
if (url.pathname.startsWith(DECRYPTED_FILE_URL_PREFIX)) {
event.respondWith(decryptFile(event.request));
}
});
self.addEventListener("install", () => {
self.skipWaiting();
});
self.addEventListener("activate", (event) => {
event.waitUntil(self.clients.claim());
});

View File

@@ -0,0 +1 @@
export * from "../../lib/constants";

View File

@@ -0,0 +1,40 @@
import { ENCRYPTION_OVERHEAD, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE } from "./constants";
export * from "../../lib/modules/crypto";
export const getEncryptedRange = (
start: number,
end: number,
totalEncryptedSize: number,
isLegacy: boolean,
) => {
if (isLegacy) {
return {
firstChunkIndex: 0,
lastChunkIndex: 0,
start: 0,
end: totalEncryptedSize - 1,
};
}
const firstChunkIndex = Math.floor(start / CHUNK_SIZE);
const lastChunkIndex = Math.floor(end / CHUNK_SIZE);
return {
firstChunkIndex,
lastChunkIndex,
start: firstChunkIndex * ENCRYPTED_CHUNK_SIZE,
end: Math.min((lastChunkIndex + 1) * ENCRYPTED_CHUNK_SIZE - 1, totalEncryptedSize - 1),
};
};
export const getDecryptedSize = (encryptedSize: number, isLegacy: boolean) => {
if (isLegacy) {
return encryptedSize - ENCRYPTION_OVERHEAD;
}
const fullChunks = Math.floor(encryptedSize / ENCRYPTED_CHUNK_SIZE);
const lastChunkEncSize = encryptedSize % ENCRYPTED_CHUNK_SIZE;
return (
fullChunks * CHUNK_SIZE + (lastChunkEncSize > 0 ? lastChunkEncSize - ENCRYPTION_OVERHEAD : 0)
);
};

View File

@@ -0,0 +1 @@
export * from "../../lib/modules/http";

View File

@@ -0,0 +1 @@
export * from "../../lib/modules/opfs";

View File

@@ -0,0 +1,3 @@
import type { FileMetadata } from "./types";
export const fileMetadataStore = new Map<number, FileMetadata>();

View File

@@ -0,0 +1 @@
export * from "../lib/serviceWorker/types";

View File

@@ -1,9 +1,20 @@
import { TRPCError } from "@trpc/server"; import { TRPCError } from "@trpc/server";
import { createHash } from "crypto";
import { createReadStream, createWriteStream } from "fs";
import { mkdir, rm } from "fs/promises";
import mime from "mime";
import { dirname } from "path";
import { v4 as uuidv4 } from "uuid";
import { z } from "zod"; import { z } from "zod";
import { FileRepo, MediaRepo, IntegrityError } from "$lib/server/db"; import { FileRepo, MediaRepo, UploadRepo, IntegrityError } from "$lib/server/db";
import { safeUnlink } from "$lib/server/modules/filesystem"; import db from "$lib/server/db/kysely";
import env from "$lib/server/loadenv";
import { getChunkDirectoryPath, safeUnlink } from "$lib/server/modules/filesystem";
import { directoryIdSchema } from "$lib/server/schemas";
import { router, roleProcedure } from "../init.server"; import { router, roleProcedure } from "../init.server";
const uploadLocks = new Set<string>();
const fileRouter = router({ const fileRouter = router({
get: roleProcedure["activeClient"] get: roleProcedure["activeClient"]
.input( .input(
@@ -19,12 +30,12 @@ const fileRouter = router({
const categories = await FileRepo.getAllFileCategories(input.id); const categories = await FileRepo.getAllFileCategories(input.id);
return { return {
isLegacy: !!file.encContentIv,
parent: file.parentId, parent: file.parentId,
mekVersion: file.mekVersion, mekVersion: file.mekVersion,
dek: file.encDek, dek: file.encDek,
dekVersion: file.dekVersion, dekVersion: file.dekVersion,
contentType: file.contentType, contentType: file.contentType,
contentIv: file.encContentIv,
name: file.encName.ciphertext, name: file.encName.ciphertext,
nameIv: file.encName.iv, nameIv: file.encName.iv,
createdAt: file.encCreatedAt?.ciphertext, createdAt: file.encCreatedAt?.ciphertext,
@@ -53,12 +64,12 @@ const fileRouter = router({
const files = await FileRepo.getFilesWithCategories(ctx.session.userId, input.ids); const files = await FileRepo.getFilesWithCategories(ctx.session.userId, input.ids);
return files.map((file) => ({ return files.map((file) => ({
id: file.id, id: file.id,
isLegacy: !!file.encContentIv,
parent: file.parentId, parent: file.parentId,
mekVersion: file.mekVersion, mekVersion: file.mekVersion,
dek: file.encDek, dek: file.encDek,
dekVersion: file.dekVersion, dekVersion: file.dekVersion,
contentType: file.contentType, contentType: file.contentType,
contentIv: file.encContentIv,
name: file.encName.ciphertext, name: file.encName.ciphertext,
nameIv: file.encName.iv, nameIv: file.encName.iv,
createdAt: file.encCreatedAt?.ciphertext, createdAt: file.encCreatedAt?.ciphertext,
@@ -158,7 +169,138 @@ const fileRouter = router({
throw new TRPCError({ code: "NOT_FOUND", message: "File or its thumbnail not found" }); throw new TRPCError({ code: "NOT_FOUND", message: "File or its thumbnail not found" });
} }
return { updatedAt: thumbnail.updatedAt, contentIv: thumbnail.encContentIv }; return { updatedAt: thumbnail.updatedAt };
}),
startUpload: roleProcedure["activeClient"]
.input(
z.object({
chunks: z.int().positive(),
parent: directoryIdSchema,
mekVersion: z.int().positive(),
dek: z.base64().nonempty(),
dekVersion: z.date(),
hskVersion: z.int().positive().optional(),
contentType: z
.string()
.trim()
.nonempty()
.refine((value) => mime.getExtension(value) !== null),
name: z.base64().nonempty(),
nameIv: z.base64().nonempty(),
createdAt: z.base64().nonempty().optional(),
createdAtIv: z.base64().nonempty().optional(),
lastModifiedAt: z.base64().nonempty(),
lastModifiedAtIv: z.base64().nonempty(),
}),
)
.mutation(async ({ ctx, input }) => {
const oneMinuteAgo = new Date(Date.now() - 60 * 1000);
const oneMinuteLater = new Date(Date.now() + 60 * 1000);
if (input.dekVersion <= oneMinuteAgo || input.dekVersion >= oneMinuteLater) {
throw new TRPCError({ code: "BAD_REQUEST", message: "Invalid DEK version" });
}
try {
const { id: sessionId } = await UploadRepo.createUploadSession({
userId: ctx.session.userId,
totalChunks: input.chunks,
expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours
parentId: input.parent,
mekVersion: input.mekVersion,
encDek: input.dek,
dekVersion: input.dekVersion,
hskVersion: input.hskVersion ?? null,
contentType: input.contentType,
encName: { ciphertext: input.name, iv: input.nameIv },
encCreatedAt:
input.createdAt && input.createdAtIv
? { ciphertext: input.createdAt, iv: input.createdAtIv }
: null,
encLastModifiedAt: { ciphertext: input.lastModifiedAt, iv: input.lastModifiedAtIv },
});
await mkdir(getChunkDirectoryPath(sessionId), { recursive: true });
return { uploadId: sessionId };
} catch (e) {
if (e instanceof IntegrityError) {
if (e.message === "Inactive MEK version") {
throw new TRPCError({ code: "BAD_REQUEST", message: "Invalid MEK version" });
} else if (e.message === "Inactive HSK version") {
throw new TRPCError({ code: "BAD_REQUEST", message: "Invalid HSK version" });
}
}
throw e;
}
}),
completeUpload: roleProcedure["activeClient"]
.input(
z.object({
uploadId: z.uuidv4(),
contentHmac: z.base64().nonempty().optional(),
}),
)
.mutation(async ({ ctx, input }) => {
const { uploadId } = input;
if (uploadLocks.has(uploadId)) {
throw new TRPCError({ code: "CONFLICT", message: "Upload already in progress" }); // TODO: Message
} else {
uploadLocks.add(uploadId);
}
const filePath = `${env.libraryPath}/${ctx.session.userId}/${uuidv4()}`;
await mkdir(dirname(filePath), { recursive: true });
try {
const session = await UploadRepo.getUploadSession(uploadId, ctx.session.userId);
if (!session) {
throw new TRPCError({ code: "NOT_FOUND", message: "Invalid upload id" });
} else if (
(session.hskVersion && !input.contentHmac) ||
(!session.hskVersion && input.contentHmac)
) {
throw new TRPCError({ code: "BAD_REQUEST", message: "Invalid content hmac" }); // TODO: message
} else if (session.uploadedChunks.length < session.totalChunks) {
throw new TRPCError({ code: "BAD_REQUEST", message: "Upload not complete" }); // TODO: Message
}
const chunkDirectoryPath = getChunkDirectoryPath(uploadId);
const hashStream = createHash("sha256");
const writeStream = createWriteStream(filePath, { flags: "wx", mode: 0o600 });
for (let i = 0; i < session.totalChunks; i++) {
for await (const chunk of createReadStream(`${chunkDirectoryPath}/${i}`)) {
hashStream.update(chunk);
writeStream.write(chunk);
}
}
await new Promise<void>((resolve, reject) => {
writeStream.end((e: any) => (e ? reject(e) : resolve()));
});
const hash = hashStream.digest("base64");
const fileId = await db.transaction().execute(async (trx) => {
const { id: fileId } = await FileRepo.registerFile(trx, {
...session,
userId: ctx.session.userId,
path: filePath,
contentHmac: input.contentHmac ?? null,
encContentHash: hash,
encContentIv: null,
});
await UploadRepo.deleteUploadSession(trx, uploadId);
return fileId;
});
await rm(chunkDirectoryPath, { recursive: true }).catch((e) => console.error(e));
return { file: fileId };
} catch (e) {
await safeUnlink(filePath);
throw e;
} finally {
uploadLocks.delete(uploadId);
}
}), }),
}); });