diff --git a/xftp-web/README.md b/xftp-web/README.md index 9b118b336..0ec38dccc 100644 --- a/xftp-web/README.md +++ b/xftp-web/README.md @@ -2,13 +2,116 @@ Browser-compatible XFTP file transfer client in TypeScript. -## Prerequisites +## Installation + +```bash +npm install xftp-web +``` + +## Usage + +```typescript +import { + XFTPAgent, + parseXFTPServer, + sendFile, receiveFile, deleteFile, + XFTPRetriableError, XFTPPermanentError, isRetriable, +} from "xftp-web" + +// Create agent (manages connections) +const agent = new XFTPAgent() + +const servers = [ + parseXFTPServer("xftp://server1..."), + parseXFTPServer("xftp://server2..."), + parseXFTPServer("xftp://server3..."), +] + +// Upload (from Uint8Array) +const {rcvDescriptions, sndDescription, uri} = await sendFile( + agent, servers, fileBytes, "photo.jpg", + {onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)} +) + +// Upload (streaming — constant memory, no full-file buffer) +const file = inputEl.files[0] +const result = await sendFile( + agent, servers, file.stream(), file.size, file.name, + {onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)} +) + +// Download +const {header, content} = await receiveFile(agent, uri, { + onProgress: (downloaded, total) => console.log(`${downloaded}/${total}`) +}) + +// Delete (requires sender description from upload) +await deleteFile(agent, sndDescription) + +// Cleanup +agent.close() +``` + +### Advanced usage + +For streaming encryption (avoids buffering the full encrypted file) or worker-based uploads: + +```typescript +import { + encryptFileForUpload, uploadFile, downloadFile, + decodeDescriptionURI, +} from "xftp-web" + +// Streaming encryption — encrypted slices emitted via callback +const metadata = await encryptFileForUpload(fileBytes, "photo.jpg", { + onSlice: (data) => { /* write to OPFS, IndexedDB, etc. */ }, + onProgress: (done, total) => {}, +}) +// metadata has {digest, key, nonce, chunkSizes} but no encData + +// Upload with custom chunk reader (e.g. reading from OPFS) +const result = await uploadFile(agent, servers, metadata, { + readChunk: (offset, size) => readFromStorage(offset, size), +}) + +// Download with FileDescription object +const fd = decodeDescriptionURI(uri) +const {header, content} = await downloadFile(agent, fd) +``` + +### Upload options + +```typescript +await sendFile(agent, servers, fileBytes, "photo.jpg", { + onProgress: (uploaded, total) => {}, // progress callback + auth: basicAuthBytes, // BasicAuth for auth-required servers + numRecipients: 3, // multiple independent download credentials (default: 1) +}) +``` + +### Error handling + +```typescript +try { + await sendFile(agent, servers, fileBytes, "photo.jpg") +} catch (e) { + if (e instanceof XFTPRetriableError) { + // Network/timeout/session errors — safe to retry + } else if (e instanceof XFTPPermanentError) { + // AUTH, NO_FILE, BLOCKED, etc. — do not retry + } + // or use: isRetriable(e) +} +``` + +## Development + +### Prerequisites - Haskell toolchain with `cabal` (to build `xftp-server`) - Node.js 20+ -- Chromium system dependencies (see below) -## Setup +### Setup ```bash # Build the XFTP server binary (from repo root) @@ -17,31 +120,31 @@ cabal build xftp-server # Install JS dependencies cd xftp-web npm install - -# Install Chromium for Playwright (browser tests) -npx playwright install chromium ``` -If Chromium fails to launch due to missing system libraries, install them with: +### Running tests ```bash -# Requires root -npx playwright install-deps chromium +npm run test ``` -## Running tests +The `pretest` script automatically installs Chromium and sets up the libsodium symlink. The browser test starts an `xftp-server` instance on port 7000 via `globalSetup`. + +If Chromium fails to launch due to missing system libraries: ```bash -# Browser round-trip test (vitest + Playwright headless Chromium) -npm run test +# Requires root +npx playwright install-deps chromium ``` -The browser test automatically starts an `xftp-server` instance on port 7000 via `globalSetup`, using certs from `tests/fixtures/`. - -## Build +### Build ```bash npm run build ``` Output goes to `dist/`. + +## License + +[AGPL-3.0-only](https://www.gnu.org/licenses/agpl-3.0.html) diff --git a/xftp-web/package.json b/xftp-web/package.json index 67610689d..e95280b91 100644 --- a/xftp-web/package.json +++ b/xftp-web/package.json @@ -1,12 +1,23 @@ { "name": "xftp-web", "version": "0.1.0", - "private": true, + "description": "XFTP file transfer protocol client for web/browser environments", + "license": "AGPL-3.0-only", + "repository": { + "type": "git", + "url": "https://github.com/simplex-chat/simplexmq", + "directory": "xftp-web" + }, "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", + "files": ["dist", "src"], + "publishConfig": { + "access": "public" + }, "scripts": { - "postinstall": "ln -sf ../../../libsodium-sumo/dist/modules-sumo-esm/libsodium-sumo.mjs node_modules/libsodium-wrappers-sumo/dist/modules-sumo-esm/libsodium-sumo.mjs && npx playwright install chromium", + "prepublishOnly": "npm run build", + "pretest": "ln -sf ../../../libsodium-sumo/dist/modules-sumo-esm/libsodium-sumo.mjs node_modules/libsodium-wrappers-sumo/dist/modules-sumo-esm/libsodium-sumo.mjs && npx playwright install chromium", "build": "tsc", "test": "vitest", "dev": "npx tsx test/runSetup.ts && vite --mode development", diff --git a/xftp-web/src/agent.ts b/xftp-web/src/agent.ts index 421ec5345..d54ea41be 100644 --- a/xftp-web/src/agent.ts +++ b/xftp-web/src/agent.ts @@ -4,10 +4,13 @@ // file descriptions, and DEFLATE-compressed URI encoding. import pako from "pako" -import {encryptFile, encodeFileHeader} from "./crypto/file.js" +import {encryptFileAsync, prepareEncryption} from "./crypto/file.js" +import {sbInit, sbEncryptChunk, sbAuth} from "./crypto/secretbox.js" +import {concatBytes, encodeInt64} from "./protocol/encoding.js" +export {prepareEncryption, type EncryptionParams} from "./crypto/file.js" import {generateEd25519KeyPair, encodePubKeyEd25519, encodePrivKeyEd25519, decodePrivKeyEd25519, ed25519KeyPairFromSeed} from "./crypto/keys.js" -import {sha512Streaming} from "./crypto/digest.js" -import {prepareChunkSizes, prepareChunkSpecs, getChunkDigest, fileSizeLen, authTagSize} from "./protocol/chunks.js" +import {sha512Streaming, sha512Init, sha512Update, sha512Final} from "./crypto/digest.js" +import {prepareChunkSpecs, getChunkDigest} from "./protocol/chunks.js" import { encodeFileDescription, decodeFileDescription, validateFileDescription, base64urlEncode, base64urlDecode, @@ -15,10 +18,11 @@ import { } from "./protocol/description.js" import type {FileInfo} from "./protocol/commands.js" import { - createXFTPChunk, uploadXFTPChunk, downloadXFTPChunk, downloadXFTPChunkRaw, - deleteXFTPChunk, type XFTPClientAgent + createXFTPChunk, addXFTPRecipients, uploadXFTPChunk, downloadXFTPChunk, downloadXFTPChunkRaw, + deleteXFTPChunk, ackXFTPChunk, XFTPAgent } from "./client.js" -export {newXFTPAgent, closeXFTPAgent, type XFTPClientAgent, type TransportConfig} from "./client.js" +export {XFTPAgent, type TransportConfig, + XFTPRetriableError, XFTPPermanentError, isRetriable, categorizeError, humanReadableMessage} from "./client.js" import {processDownloadedFile, decryptReceivedChunk} from "./download.js" import type {XFTPServer} from "./protocol/address.js" import {formatXFTPServer, parseXFTPServer} from "./protocol/address.js" @@ -30,8 +34,7 @@ interface SentChunk { chunkNo: number senderId: Uint8Array senderKey: Uint8Array // 64B libsodium Ed25519 private key - recipientId: Uint8Array - recipientKey: Uint8Array // 64B libsodium Ed25519 private key + recipients: {recipientId: Uint8Array, recipientKey: Uint8Array}[] chunkSize: number digest: Uint8Array // SHA-256 server: XFTPServer @@ -49,7 +52,7 @@ export interface EncryptedFileInfo extends EncryptedFileMetadata { } export interface UploadResult { - rcvDescription: FileDescription + rcvDescriptions: FileDescription[] sndDescription: FileDescription uri: string // base64url-encoded compressed YAML (no leading #) } @@ -78,37 +81,98 @@ export function decodeDescriptionURI(fragment: string): FileDescription { // -- Upload -export function encryptFileForUpload(source: Uint8Array, fileName: string): EncryptedFileInfo { - const key = new Uint8Array(32) - const nonce = new Uint8Array(24) - crypto.getRandomValues(key) - crypto.getRandomValues(nonce) - const fileHdr = encodeFileHeader({fileName, fileExtra: null}) - const fileSize = BigInt(fileHdr.length + source.length) - const payloadSize = Number(fileSize) + fileSizeLen + authTagSize - const chunkSizes = prepareChunkSizes(payloadSize) - const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0)) - const encData = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) - const digest = sha512Streaming([encData]) - console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`) - return {encData, digest, key, nonce, chunkSizes} +export interface EncryptForUploadOptions { + onProgress?: (done: number, total: number) => void + onSlice?: (data: Uint8Array) => void | Promise +} + +export async function encryptFileForUpload( + source: Uint8Array, fileName: string, + options: EncryptForUploadOptions & {onSlice: NonNullable} +): Promise +export async function encryptFileForUpload( + source: Uint8Array, fileName: string, + options?: EncryptForUploadOptions +): Promise +export async function encryptFileForUpload( + source: Uint8Array, + fileName: string, + options?: EncryptForUploadOptions +): Promise { + const {onProgress, onSlice} = options ?? {} + const {fileHdr, key, nonce, fileSize, encSize, chunkSizes} = prepareEncryption(source.length, fileName) + if (onSlice) { + const hashState = sha512Init() + await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress, (data) => { + sha512Update(hashState, data) + return onSlice(data) + }) + const digest = sha512Final(hashState) + return {digest, key, nonce, chunkSizes} + } else { + const encData = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress) + const digest = sha512Streaming([encData]) + console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`) + return {encData, digest, key, nonce, chunkSizes} + } } const DEFAULT_REDIRECT_THRESHOLD = 400 +const MAX_RECIPIENTS_PER_REQUEST = 200 // each key is ~46 bytes; 200 keys fit within 16KB XFTP block export interface UploadOptions { onProgress?: (uploaded: number, total: number) => void redirectThreshold?: number readChunk?: (offset: number, size: number) => Promise + auth?: Uint8Array + numRecipients?: number +} + +async function uploadSingleChunk( + agent: XFTPAgent, server: XFTPServer, + chunkNo: number, chunkData: Uint8Array, chunkSize: number, + numRecipients: number, auth: Uint8Array | null +): Promise { + const sndKp = generateEd25519KeyPair() + const rcvKps = Array.from({length: numRecipients}, () => generateEd25519KeyPair()) + const chunkDigest = getChunkDigest(chunkData) + const fileInfo: FileInfo = { + sndKey: encodePubKeyEd25519(sndKp.publicKey), + size: chunkSize, + digest: chunkDigest + } + const firstBatch = Math.min(numRecipients, MAX_RECIPIENTS_PER_REQUEST) + const firstBatchKeys = rcvKps.slice(0, firstBatch).map(kp => encodePubKeyEd25519(kp.publicKey)) + const {senderId, recipientIds: firstIds} = await createXFTPChunk( + agent, server, sndKp.privateKey, fileInfo, firstBatchKeys, auth + ) + const allRecipientIds = [...firstIds] + let added = firstBatch + while (added < numRecipients) { + const batchSize = Math.min(numRecipients - added, MAX_RECIPIENTS_PER_REQUEST) + const batchKeys = rcvKps.slice(added, added + batchSize).map(kp => encodePubKeyEd25519(kp.publicKey)) + const moreIds = await addXFTPRecipients(agent, server, sndKp.privateKey, senderId, batchKeys) + allRecipientIds.push(...moreIds) + added += batchSize + } + await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData) + return { + chunkNo, senderId, senderKey: sndKp.privateKey, + recipients: allRecipientIds.map((rid, ri) => ({ + recipientId: rid, recipientKey: rcvKps[ri].privateKey + })), + chunkSize, digest: chunkDigest, server + } } export async function uploadFile( - agent: XFTPClientAgent, - server: XFTPServer, + agent: XFTPAgent, + servers: XFTPServer[], encrypted: EncryptedFileMetadata, options?: UploadOptions ): Promise { - const {onProgress, redirectThreshold, readChunk: readChunkOpt} = options ?? {} + if (servers.length === 0) throw new Error("uploadFile: servers list is empty") + const {onProgress, redirectThreshold, readChunk: readChunkOpt, auth, numRecipients = 1} = options ?? {} const readChunk: (offset: number, size: number) => Promise = readChunkOpt ? readChunkOpt : ('encData' in encrypted @@ -116,48 +180,161 @@ export async function uploadFile( : () => { throw new Error("uploadFile: readChunk required when encData is absent") }) const total = encrypted.chunkSizes.reduce((a, b) => a + b, 0) const specs = prepareChunkSpecs(encrypted.chunkSizes) - const sentChunks: SentChunk[] = [] + + // Pre-assign servers and group by server (matching Haskell groupAllOn) + const chunkJobs = specs.map((spec, i) => ({ + index: i, + spec, + server: servers[Math.floor(Math.random() * servers.length)] + })) + const byServer = new Map() + for (const job of chunkJobs) { + const key = formatXFTPServer(job.server) + if (!byServer.has(key)) byServer.set(key, []) + byServer.get(key)!.push(job) + } + + // Upload groups in parallel, sequential within each group + const sentChunks: SentChunk[] = new Array(specs.length) let uploaded = 0 - for (let i = 0; i < specs.length; i++) { - const spec = specs[i] - const chunkNo = i + 1 - const sndKp = generateEd25519KeyPair() - const rcvKp = generateEd25519KeyPair() - const chunkData = await readChunk(spec.chunkOffset, spec.chunkSize) - const chunkDigest = getChunkDigest(chunkData) - console.log(`[AGENT-DBG] upload chunk=${chunkNo} offset=${spec.chunkOffset} size=${spec.chunkSize} digest=${_dbgHex(chunkDigest, 32)} data[0..8]=${_dbgHex(chunkData)} data[-8..]=${_dbgHex(chunkData.slice(-8))}`) - const fileInfo: FileInfo = { - sndKey: encodePubKeyEd25519(sndKp.publicKey), - size: spec.chunkSize, - digest: chunkDigest + await Promise.all([...byServer.values()].map(async (jobs) => { + for (const {index, spec, server} of jobs) { + const chunkNo = index + 1 + const chunkData = await readChunk(spec.chunkOffset, spec.chunkSize) + sentChunks[index] = await uploadSingleChunk(agent, server, chunkNo, chunkData, spec.chunkSize, numRecipients, auth ?? null) + uploaded += spec.chunkSize + onProgress?.(uploaded, total) } - const rcvKeysForChunk = [encodePubKeyEd25519(rcvKp.publicKey)] - const {senderId, recipientIds} = await createXFTPChunk( - agent, server, sndKp.privateKey, fileInfo, rcvKeysForChunk + })) + + const rcvDescriptions = Array.from({length: numRecipients}, (_, ri) => + buildDescription("recipient", ri, encrypted, sentChunks) + ) + const sndDescription = buildDescription("sender", 0, encrypted, sentChunks) + let uri = encodeDescriptionURI(rcvDescriptions[0]) + let finalRcvDescriptions = rcvDescriptions + const threshold = redirectThreshold ?? DEFAULT_REDIRECT_THRESHOLD + if (uri.length > threshold && sentChunks.length > 1) { + const redirected = await uploadRedirectDescription(agent, servers, rcvDescriptions[0], auth) + finalRcvDescriptions = [redirected, ...rcvDescriptions.slice(1)] + uri = encodeDescriptionURI(redirected) + } + return {rcvDescriptions: finalRcvDescriptions, sndDescription, uri} +} + +export interface SendFileOptions { + onProgress?: (uploaded: number, total: number) => void + auth?: Uint8Array + numRecipients?: number +} + +export async function sendFile( + agent: XFTPAgent, servers: XFTPServer[], + source: Uint8Array, fileName: string, + options?: SendFileOptions +): Promise +export async function sendFile( + agent: XFTPAgent, servers: XFTPServer[], + source: AsyncIterable, sourceSize: number, + fileName: string, options?: SendFileOptions +): Promise +export async function sendFile( + agent: XFTPAgent, servers: XFTPServer[], + source: Uint8Array | AsyncIterable, + fileNameOrSize: string | number, + fileNameOrOptions?: string | SendFileOptions, + maybeOptions?: SendFileOptions +): Promise { + let sourceSize: number, fileName: string, options: SendFileOptions | undefined + if (source instanceof Uint8Array) { + sourceSize = source.length + fileName = fileNameOrSize as string + options = fileNameOrOptions as SendFileOptions | undefined + } else { + sourceSize = fileNameOrSize as number + fileName = fileNameOrOptions as string + options = maybeOptions + } + if (servers.length === 0) throw new Error("sendFile: servers list is empty") + const {onProgress, auth, numRecipients = 1} = options ?? {} + const params = prepareEncryption(sourceSize, fileName) + const specs = prepareChunkSpecs(params.chunkSizes) + const total = params.chunkSizes.reduce((a, b) => a + b, 0) + + const encState = sbInit(params.key, params.nonce) + const hashState = sha512Init() + + const sentChunks: SentChunk[] = new Array(specs.length) + let specIdx = 0, chunkOff = 0, uploaded = 0 + let chunkBuf = new Uint8Array(specs[0].chunkSize) + + async function flushChunk() { + const server = servers[Math.floor(Math.random() * servers.length)] + sentChunks[specIdx] = await uploadSingleChunk( + agent, server, specIdx + 1, chunkBuf, specs[specIdx].chunkSize, numRecipients, auth ?? null ) - await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData) - sentChunks.push({ - chunkNo, senderId, senderKey: sndKp.privateKey, - recipientId: recipientIds[0], recipientKey: rcvKp.privateKey, - chunkSize: spec.chunkSize, digest: chunkDigest, server - }) - uploaded += spec.chunkSize + uploaded += specs[specIdx].chunkSize onProgress?.(uploaded, total) + specIdx++ + if (specIdx < specs.length) { + chunkBuf = new Uint8Array(specs[specIdx].chunkSize) + chunkOff = 0 + } } - const rcvDescription = buildDescription("recipient", encrypted, sentChunks) - const sndDescription = buildDescription("sender", encrypted, sentChunks) - let uri = encodeDescriptionURI(rcvDescription) - let finalRcvDescription = rcvDescription - const threshold = redirectThreshold ?? DEFAULT_REDIRECT_THRESHOLD - if (uri.length > threshold && sentChunks.length > 1) { - finalRcvDescription = await uploadRedirectDescription(agent, server, rcvDescription) - uri = encodeDescriptionURI(finalRcvDescription) + + async function feedEncrypted(data: Uint8Array) { + sha512Update(hashState, data) + let off = 0 + while (off < data.length) { + const space = specs[specIdx].chunkSize - chunkOff + const n = Math.min(space, data.length - off) + chunkBuf.set(data.subarray(off, off + n), chunkOff) + chunkOff += n + off += n + if (chunkOff === specs[specIdx].chunkSize) await flushChunk() + } + } + + await feedEncrypted(sbEncryptChunk(encState, concatBytes(encodeInt64(params.fileSize), params.fileHdr))) + + const SLICE = 65536 + if (source instanceof Uint8Array) { + for (let off = 0; off < source.length; off += SLICE) { + await feedEncrypted(sbEncryptChunk(encState, source.subarray(off, Math.min(off + SLICE, source.length)))) + } + } else { + for await (const chunk of source) { + for (let off = 0; off < chunk.length; off += SLICE) { + await feedEncrypted(sbEncryptChunk(encState, chunk.subarray(off, Math.min(off + SLICE, chunk.length)))) + } + } + } + + const padLen = Number(params.encSize - 16n - params.fileSize - 8n) + const padding = new Uint8Array(padLen) + padding.fill(0x23) + await feedEncrypted(sbEncryptChunk(encState, padding)) + await feedEncrypted(sbAuth(encState)) + + const digest = sha512Final(hashState) + const encrypted: EncryptedFileMetadata = {digest, key: params.key, nonce: params.nonce, chunkSizes: params.chunkSizes} + const rcvDescriptions = Array.from({length: numRecipients}, (_, ri) => + buildDescription("recipient", ri, encrypted, sentChunks) + ) + const sndDescription = buildDescription("sender", 0, encrypted, sentChunks) + let uri = encodeDescriptionURI(rcvDescriptions[0]) + let finalRcvDescriptions = rcvDescriptions + if (uri.length > DEFAULT_REDIRECT_THRESHOLD && sentChunks.length > 1) { + const redirected = await uploadRedirectDescription(agent, servers, rcvDescriptions[0], auth) + finalRcvDescriptions = [redirected, ...rcvDescriptions.slice(1)] + uri = encodeDescriptionURI(redirected) } - return {rcvDescription: finalRcvDescription, sndDescription, uri} + return {rcvDescriptions: finalRcvDescriptions, sndDescription, uri} } function buildDescription( party: "recipient" | "sender", + recipientIndex: number, enc: EncryptedFileMetadata, chunks: SentChunk[] ): FileDescription { @@ -175,8 +352,8 @@ function buildDescription( digest: c.digest, replicas: [{ server: formatXFTPServer(c.server), - replicaId: party === "recipient" ? c.recipientId : c.senderId, - replicaKey: encodePrivKeyEd25519(party === "recipient" ? c.recipientKey : c.senderKey) + replicaId: party === "recipient" ? c.recipients[recipientIndex].recipientId : c.senderId, + replicaKey: encodePrivKeyEd25519(party === "recipient" ? c.recipients[recipientIndex].recipientKey : c.senderKey) }] })), redirect: null @@ -184,38 +361,37 @@ function buildDescription( } async function uploadRedirectDescription( - agent: XFTPClientAgent, - server: XFTPServer, - innerFd: FileDescription + agent: XFTPAgent, + servers: XFTPServer[], + innerFd: FileDescription, + auth?: Uint8Array ): Promise { const yaml = encodeFileDescription(innerFd) const yamlBytes = new TextEncoder().encode(yaml) - const enc = encryptFileForUpload(yamlBytes, "") + const enc = await encryptFileForUpload(yamlBytes, "") const specs = prepareChunkSpecs(enc.chunkSizes) - const sentChunks: SentChunk[] = [] - for (let i = 0; i < specs.length; i++) { - const spec = specs[i] - const chunkNo = i + 1 - const sndKp = generateEd25519KeyPair() - const rcvKp = generateEd25519KeyPair() - const chunkData = enc.encData.subarray(spec.chunkOffset, spec.chunkOffset + spec.chunkSize) - const chunkDigest = getChunkDigest(chunkData) - const fileInfo: FileInfo = { - sndKey: encodePubKeyEd25519(sndKp.publicKey), - size: spec.chunkSize, - digest: chunkDigest - } - const rcvKeysForChunk = [encodePubKeyEd25519(rcvKp.publicKey)] - const {senderId, recipientIds} = await createXFTPChunk( - agent, server, sndKp.privateKey, fileInfo, rcvKeysForChunk - ) - await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData) - sentChunks.push({ - chunkNo, senderId, senderKey: sndKp.privateKey, - recipientId: recipientIds[0], recipientKey: rcvKp.privateKey, - chunkSize: spec.chunkSize, digest: chunkDigest, server - }) + + const chunkJobs = specs.map((spec, i) => ({ + index: i, + spec, + server: servers[Math.floor(Math.random() * servers.length)] + })) + const byServer = new Map() + for (const job of chunkJobs) { + const key = formatXFTPServer(job.server) + if (!byServer.has(key)) byServer.set(key, []) + byServer.get(key)!.push(job) } + + const sentChunks: SentChunk[] = new Array(specs.length) + await Promise.all([...byServer.values()].map(async (jobs) => { + for (const {index, spec, server} of jobs) { + const chunkNo = index + 1 + const chunkData = enc.encData.subarray(spec.chunkOffset, spec.chunkOffset + spec.chunkSize) + sentChunks[index] = await uploadSingleChunk(agent, server, chunkNo, chunkData, spec.chunkSize, 1, auth ?? null) + } + })) + return { party: "recipient", size: enc.chunkSizes.reduce((a, b) => a + b, 0), @@ -229,8 +405,8 @@ async function uploadRedirectDescription( digest: c.digest, replicas: [{ server: formatXFTPServer(c.server), - replicaId: c.recipientId, - replicaKey: encodePrivKeyEd25519(c.recipientKey) + replicaId: c.recipients[0].recipientId, + replicaKey: encodePrivKeyEd25519(c.recipients[0].recipientKey) }] })), redirect: {size: innerFd.size, digest: innerFd.digest} @@ -249,18 +425,17 @@ export interface RawDownloadedChunk { export interface DownloadRawOptions { onProgress?: (downloaded: number, total: number) => void - concurrency?: number } export async function downloadFileRaw( - agent: XFTPClientAgent, + agent: XFTPAgent, fd: FileDescription, onRawChunk: (chunk: RawDownloadedChunk) => Promise, options?: DownloadRawOptions ): Promise { const err = validateFileDescription(fd) if (err) throw new Error("downloadFileRaw: " + err) - const {onProgress, concurrency = 1} = options ?? {} + const {onProgress} = options ?? {} // Resolve redirect on main thread (redirect data is small) if (fd.redirect !== null) { console.log(`[AGENT-DBG] resolving redirect: outer size=${fd.size} chunks=${fd.chunks.length}`) @@ -292,6 +467,7 @@ export async function downloadFileRaw( body: raw.body, digest: chunk.digest }) + await ackXFTPChunk(agent, server, kp.privateKey, replica.replicaId) downloaded += chunk.chunkSize onProgress?.(downloaded, resolvedFd.size) } @@ -300,7 +476,7 @@ export async function downloadFileRaw( } export async function downloadFile( - agent: XFTPClientAgent, + agent: XFTPAgent, fd: FileDescription, onProgress?: (downloaded: number, total: number) => void ): Promise { @@ -317,20 +493,38 @@ export async function downloadFile( return processDownloadedFile(resolvedFd, chunks) } +export async function receiveFile( + agent: XFTPAgent, + uri: string, + options?: {onProgress?: (downloaded: number, total: number) => void} +): Promise { + const fd = decodeDescriptionURI(uri) + return downloadFile(agent, fd, options?.onProgress) +} + async function resolveRedirect( - agent: XFTPClientAgent, + agent: XFTPAgent, fd: FileDescription ): Promise { const plaintextChunks: Uint8Array[] = new Array(fd.chunks.length) + const byServer = new Map() for (const chunk of fd.chunks) { - const replica = chunk.replicas[0] - if (!replica) throw new Error("resolveRedirect: chunk has no replicas") - const server = parseXFTPServer(replica.server) - const seed = decodePrivKeyEd25519(replica.replicaKey) - const kp = ed25519KeyPairFromSeed(seed) - const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.digest) - plaintextChunks[chunk.chunkNo - 1] = data + const srv = chunk.replicas[0]?.server ?? "" + if (!byServer.has(srv)) byServer.set(srv, []) + byServer.get(srv)!.push(chunk) } + await Promise.all([...byServer.entries()].map(async ([srv, chunks]) => { + const server = parseXFTPServer(srv) + for (const chunk of chunks) { + const replica = chunk.replicas[0] + if (!replica) throw new Error("resolveRedirect: chunk has no replicas") + const seed = decodePrivKeyEd25519(replica.replicaKey) + const kp = ed25519KeyPairFromSeed(seed) + const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.digest) + plaintextChunks[chunk.chunkNo - 1] = data + await ackXFTPChunk(agent, server, kp.privateKey, replica.replicaId) + } + })) const totalSize = plaintextChunks.reduce((s, c) => s + c.length, 0) if (totalSize !== fd.size) throw new Error("resolveRedirect: redirect file size mismatch") const digest = sha512Streaming(plaintextChunks) @@ -347,15 +541,23 @@ async function resolveRedirect( // -- Delete -export async function deleteFile(agent: XFTPClientAgent, sndDescription: FileDescription): Promise { +export async function deleteFile(agent: XFTPAgent, sndDescription: FileDescription): Promise { + const byServer = new Map() for (const chunk of sndDescription.chunks) { - const replica = chunk.replicas[0] - if (!replica) throw new Error("deleteFile: chunk has no replicas") - const server = parseXFTPServer(replica.server) - const seed = decodePrivKeyEd25519(replica.replicaKey) - const kp = ed25519KeyPairFromSeed(seed) - await deleteXFTPChunk(agent, server, kp.privateKey, replica.replicaId) + const srv = chunk.replicas[0]?.server ?? "" + if (!byServer.has(srv)) byServer.set(srv, []) + byServer.get(srv)!.push(chunk) } + await Promise.all([...byServer.entries()].map(async ([srv, chunks]) => { + const server = parseXFTPServer(srv) + for (const chunk of chunks) { + const replica = chunk.replicas[0] + if (!replica) throw new Error("deleteFile: chunk has no replicas") + const seed = decodePrivKeyEd25519(replica.replicaKey) + const kp = ed25519KeyPairFromSeed(seed) + await deleteXFTPChunk(agent, server, kp.privateKey, replica.replicaId) + } + })) } // -- Internal diff --git a/xftp-web/src/client.ts b/xftp-web/src/client.ts index 1384147fb..04f1f9222 100644 --- a/xftp-web/src/client.ts +++ b/xftp-web/src/client.ts @@ -16,7 +16,7 @@ import { import {verifyIdentityProof} from "./crypto/identity.js" import {generateX25519KeyPair, encodePubKeyX25519, dh} from "./crypto/keys.js" import { - encodeFNEW, encodeFADD, encodeFPUT, encodeFGET, encodeFDEL, encodePING, + encodeFNEW, encodeFADD, encodeFPUT, encodeFGET, encodeFACK, encodeFDEL, encodePING, decodeResponse, type FileResponse, type FileInfo, type XFTPErrorType } from "./protocol/commands.js" import {decryptReceivedChunk} from "./download.js" @@ -172,17 +172,33 @@ interface ServerConnection { queue: Promise // tail of sequential command chain } -export interface XFTPClientAgent { - connections: Map +export class XFTPAgent { + connections = new Map() /** @internal Injectable for testing — defaults to connectXFTP */ _connectFn: (server: XFTPServer) => Promise -} -export function newXFTPAgent(): XFTPClientAgent { - return {connections: new Map(), _connectFn: connectXFTP} + constructor(connectFn?: (server: XFTPServer) => Promise) { + this._connectFn = connectFn ?? connectXFTP + } + + closeServer(server: XFTPServer): void { + const key = formatXFTPServer(server) + const conn = this.connections.get(key) + if (conn) { + this.connections.delete(key) + conn.client.then(c => c.transport.close(), () => {}) + } + } + + close(): void { + for (const conn of this.connections.values()) { + conn.client.then(c => c.transport.close(), () => {}) + } + this.connections.clear() + } } -export function getXFTPServerClient(agent: XFTPClientAgent, server: XFTPServer): Promise { +export function getXFTPServerClient(agent: XFTPAgent, server: XFTPServer): Promise { const key = formatXFTPServer(server) let conn = agent.connections.get(key) if (!conn) { @@ -197,7 +213,7 @@ export function getXFTPServerClient(agent: XFTPClientAgent, server: XFTPServer): return conn.client } -export function reconnectClient(agent: XFTPClientAgent, server: XFTPServer): Promise { +export function reconnectClient(agent: XFTPAgent, server: XFTPServer): Promise { const key = formatXFTPServer(server) const old = agent.connections.get(key) old?.client.then(c => c.transport.close(), () => {}) @@ -212,7 +228,7 @@ export function reconnectClient(agent: XFTPClientAgent, server: XFTPServer): Pro } export function removeStaleConnection( - agent: XFTPClientAgent, server: XFTPServer, failedP: Promise + agent: XFTPAgent, server: XFTPServer, failedP: Promise ): void { const key = formatXFTPServer(server) const conn = agent.connections.get(key) @@ -222,22 +238,6 @@ export function removeStaleConnection( } } -export function closeXFTPServerClient(agent: XFTPClientAgent, server: XFTPServer): void { - const key = formatXFTPServer(server) - const conn = agent.connections.get(key) - if (conn) { - agent.connections.delete(key) - conn.client.then(c => c.transport.close(), () => {}) - } -} - -export function closeXFTPAgent(agent: XFTPClientAgent): void { - for (const conn of agent.connections.values()) { - conn.client.then(c => c.transport.close(), () => {}) - } - agent.connections.clear() -} - // -- Connect + handshake export async function connectXFTP(server: XFTPServer, config?: Partial): Promise { @@ -342,7 +342,7 @@ function _hex(b: Uint8Array, n = 8): string { // -- Send command (with retry + reconnect) export async function sendXFTPCommand( - agent: XFTPClientAgent, + agent: XFTPAgent, server: XFTPServer, privateKey: Uint8Array, entityId: Uint8Array, @@ -375,7 +375,7 @@ export async function sendXFTPCommand( // -- Command wrappers export async function createXFTPChunk( - agent: XFTPClientAgent, server: XFTPServer, spKey: Uint8Array, file: FileInfo, + agent: XFTPAgent, server: XFTPServer, spKey: Uint8Array, file: FileInfo, rcvKeys: Uint8Array[], auth: Uint8Array | null = null ): Promise<{senderId: Uint8Array, recipientIds: Uint8Array[]}> { const {response} = await sendXFTPCommand(agent, server, spKey, new Uint8Array(0), encodeFNEW(file, rcvKeys, auth)) @@ -384,7 +384,7 @@ export async function createXFTPChunk( } export async function addXFTPRecipients( - agent: XFTPClientAgent, server: XFTPServer, spKey: Uint8Array, fId: Uint8Array, rcvKeys: Uint8Array[] + agent: XFTPAgent, server: XFTPServer, spKey: Uint8Array, fId: Uint8Array, rcvKeys: Uint8Array[] ): Promise { const {response} = await sendXFTPCommand(agent, server, spKey, fId, encodeFADD(rcvKeys)) if (response.type !== "FRRcvIds") throw new Error("unexpected response: " + response.type) @@ -392,7 +392,7 @@ export async function addXFTPRecipients( } export async function uploadXFTPChunk( - agent: XFTPClientAgent, server: XFTPServer, spKey: Uint8Array, fId: Uint8Array, chunkData: Uint8Array + agent: XFTPAgent, server: XFTPServer, spKey: Uint8Array, fId: Uint8Array, chunkData: Uint8Array ): Promise { const {response} = await sendXFTPCommand(agent, server, spKey, fId, encodeFPUT(), chunkData) if (response.type !== "FROk") throw new Error("unexpected response: " + response.type) @@ -405,7 +405,7 @@ export interface RawChunkResponse { } export async function downloadXFTPChunkRaw( - agent: XFTPClientAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array + agent: XFTPAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array ): Promise { const {publicKey, privateKey} = generateX25519KeyPair() const cmd = encodeFGET(encodePubKeyX25519(publicKey)) @@ -417,20 +417,27 @@ export async function downloadXFTPChunkRaw( } export async function downloadXFTPChunk( - agent: XFTPClientAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array, digest?: Uint8Array + agent: XFTPAgent, server: XFTPServer, rpKey: Uint8Array, fId: Uint8Array, digest?: Uint8Array ): Promise { const {dhSecret, nonce, body} = await downloadXFTPChunkRaw(agent, server, rpKey, fId) return decryptReceivedChunk(dhSecret, nonce, body, digest ?? null) } export async function deleteXFTPChunk( - agent: XFTPClientAgent, server: XFTPServer, spKey: Uint8Array, sId: Uint8Array + agent: XFTPAgent, server: XFTPServer, spKey: Uint8Array, sId: Uint8Array ): Promise { const {response} = await sendXFTPCommand(agent, server, spKey, sId, encodeFDEL()) if (response.type !== "FROk") throw new Error("unexpected response: " + response.type) } -export async function pingXFTP(agent: XFTPClientAgent, server: XFTPServer): Promise { +export async function ackXFTPChunk( + agent: XFTPAgent, server: XFTPServer, rpKey: Uint8Array, rId: Uint8Array +): Promise { + const {response} = await sendXFTPCommand(agent, server, rpKey, rId, encodeFACK()) + if (response.type !== "FROk") throw new Error("unexpected response: " + response.type) +} + +export async function pingXFTP(agent: XFTPAgent, server: XFTPServer): Promise { const client = await getXFTPServerClient(agent, server) const corrId = new Uint8Array(0) const block = encodeTransmission(client.sessionId, corrId, new Uint8Array(0), encodePING()) @@ -441,8 +448,3 @@ export async function pingXFTP(agent: XFTPClientAgent, server: XFTPServer): Prom if (response.type !== "FRPong") throw new Error("unexpected response: " + response.type) } -// -- Close - -export function closeXFTP(c: XFTPClient): void { - c.transport.close() -} diff --git a/xftp-web/src/crypto/digest.ts b/xftp-web/src/crypto/digest.ts index 7ef742c08..78702afbb 100644 --- a/xftp-web/src/crypto/digest.ts +++ b/xftp-web/src/crypto/digest.ts @@ -1,6 +1,6 @@ // Cryptographic hash functions matching Simplex.Messaging.Crypto (sha256Hash, sha512Hash). -import sodium from "libsodium-wrappers-sumo" +import sodium, {type StateAddress} from "libsodium-wrappers-sumo" // SHA-256 digest (32 bytes) -- Crypto.hs:1006 export function sha256(data: Uint8Array): Uint8Array { @@ -12,11 +12,24 @@ export function sha512(data: Uint8Array): Uint8Array { return sodium.crypto_hash_sha512(data) } +// Incremental SHA-512 — for computing digest during streaming encryption. +export function sha512Init(): StateAddress { + return sodium.crypto_hash_sha512_init() as unknown as StateAddress +} + +export function sha512Update(state: StateAddress, data: Uint8Array): void { + sodium.crypto_hash_sha512_update(state, data) +} + +export function sha512Final(state: StateAddress): Uint8Array { + return sodium.crypto_hash_sha512_final(state) +} + // Streaming SHA-512 over multiple chunks -- avoids copying large data into WASM memory at once. // Internally segments chunks larger than 4MB to limit peak WASM memory usage. export function sha512Streaming(chunks: Iterable): Uint8Array { const SEG = 4 * 1024 * 1024 - const state = sodium.crypto_hash_sha512_init() + const state = sodium.crypto_hash_sha512_init() as unknown as StateAddress for (const chunk of chunks) { for (let off = 0; off < chunk.length; off += SEG) { sodium.crypto_hash_sha512_update(state, chunk.subarray(off, Math.min(off + SEG, chunk.length))) diff --git a/xftp-web/src/crypto/file.ts b/xftp-web/src/crypto/file.ts index 0088a680b..3ae215cea 100644 --- a/xftp-web/src/crypto/file.ts +++ b/xftp-web/src/crypto/file.ts @@ -3,6 +3,7 @@ import {Decoder, concatBytes, encodeInt64, encodeString, decodeString, encodeMaybe, decodeMaybe} from "../protocol/encoding.js" import {sbInit, sbEncryptChunk, sbDecryptTailTag, sbAuth} from "./secretbox.js" +import {prepareChunkSizes, fileSizeLen, authTagSize} from "../protocol/chunks.js" const AUTH_TAG_SIZE = 16n @@ -69,6 +70,88 @@ export function encryptFile( return concatBytes(hdr, encSource, encPad, tag) } +// Async variant: encrypts source in 64KB slices, yielding between each to avoid blocking the main thread. +// Produces identical output to encryptFile. +// When onSlice is provided, encrypted data is streamed to the callback instead of buffered. +const ENCRYPT_SLICE = 65536 + +export async function encryptFileAsync( + source: Uint8Array, fileHdr: Uint8Array, + key: Uint8Array, nonce: Uint8Array, + fileSize: bigint, encSize: bigint, + onProgress?: (done: number, total: number) => void +): Promise +export async function encryptFileAsync( + source: Uint8Array, fileHdr: Uint8Array, + key: Uint8Array, nonce: Uint8Array, + fileSize: bigint, encSize: bigint, + onProgress: ((done: number, total: number) => void) | undefined, + onSlice: (data: Uint8Array) => void | Promise +): Promise +export async function encryptFileAsync( + source: Uint8Array, + fileHdr: Uint8Array, + key: Uint8Array, + nonce: Uint8Array, + fileSize: bigint, + encSize: bigint, + onProgress?: (done: number, total: number) => void, + onSlice?: (data: Uint8Array) => void | Promise +): Promise { + const state = sbInit(key, nonce) + const lenStr = encodeInt64(fileSize) + const padLen = Number(encSize - AUTH_TAG_SIZE - fileSize - 8n) + if (padLen < 0) throw new Error("encryptFile: encSize too small") + const out = onSlice ? null : new Uint8Array(Number(encSize)) + let outOff = 0 + + async function emit(data: Uint8Array) { + if (onSlice) { + await onSlice(data) + } else { + out!.set(data, outOff) + outOff += data.length + } + } + + await emit(sbEncryptChunk(state, concatBytes(lenStr, fileHdr))) + for (let off = 0; off < source.length; off += ENCRYPT_SLICE) { + const end = Math.min(off + ENCRYPT_SLICE, source.length) + await emit(sbEncryptChunk(state, source.subarray(off, end))) + onProgress?.(end, source.length) + await new Promise(r => setTimeout(r, 0)) + } + const padding = new Uint8Array(padLen) + padding.fill(0x23) + await emit(sbEncryptChunk(state, padding)) + await emit(sbAuth(state)) + if (out) return out +} + +// -- Encryption preparation (key gen + chunk sizing) + +export interface EncryptionParams { + fileHdr: Uint8Array + key: Uint8Array + nonce: Uint8Array + fileSize: bigint + encSize: bigint + chunkSizes: number[] +} + +export function prepareEncryption(sourceSize: number, fileName: string): EncryptionParams { + const key = new Uint8Array(32) + const nonce = new Uint8Array(24) + crypto.getRandomValues(key) + crypto.getRandomValues(nonce) + const fileHdr = encodeFileHeader({fileName, fileExtra: null}) + const fileSize = BigInt(fileHdr.length + sourceSize) + const payloadSize = Number(fileSize) + fileSizeLen + authTagSize + const chunkSizes = prepareChunkSizes(payloadSize) + const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0)) + return {fileHdr, key, nonce, fileSize, encSize, chunkSizes} +} + // -- Decryption (FileTransfer.Crypto:decryptChunks) // Decrypt one or more XFTP chunks into a FileHeader and file content. diff --git a/xftp-web/src/crypto/keys.ts b/xftp-web/src/crypto/keys.ts index 95664a788..21b217906 100644 --- a/xftp-web/src/crypto/keys.ts +++ b/xftp-web/src/crypto/keys.ts @@ -1,6 +1,7 @@ // Key generation, signing, DH -- Simplex.Messaging.Crypto (Ed25519/X25519/Ed448 functions). import sodium from "libsodium-wrappers-sumo" +await sodium.ready import {ed448} from "@noble/curves/ed448" import {sha256} from "./digest.js" import {concatBytes} from "../protocol/encoding.js" diff --git a/xftp-web/src/index.ts b/xftp-web/src/index.ts new file mode 100644 index 000000000..c3e99903f --- /dev/null +++ b/xftp-web/src/index.ts @@ -0,0 +1,4 @@ +export * from "./agent.js" +export {parseXFTPServer, formatXFTPServer, type XFTPServer} from "./protocol/address.js" +export {decodeFileDescription, encodeFileDescription, validateFileDescription, type FileDescription, type FileChunk, type FileChunkReplica, type FileParty, type RedirectFileInfo} from "./protocol/description.js" +export {type FileHeader} from "./crypto/file.js" diff --git a/xftp-web/src/protocol/commands.ts b/xftp-web/src/protocol/commands.ts index 3ca43541f..a49f5206e 100644 --- a/xftp-web/src/protocol/commands.ts +++ b/xftp-web/src/protocol/commands.ts @@ -22,11 +22,18 @@ export interface FileInfo { export type CommandError = "UNKNOWN" | "SYNTAX" | "PROHIBITED" | "NO_AUTH" | "HAS_AUTH" | "NO_ENTITY" +export type BlockingReason = "spam" | "content" + +export interface BlockingInfo { + reason: BlockingReason + notice: {ttl: number | null} | null +} + export type XFTPErrorType = | {type: "BLOCK"} | {type: "SESSION"} | {type: "HANDSHAKE"} | {type: "CMD", cmdErr: CommandError} | {type: "AUTH"} - | {type: "BLOCKED", blockInfo: string} + | {type: "BLOCKED", blockInfo: BlockingInfo} | {type: "SIZE"} | {type: "QUOTA"} | {type: "DIGEST"} | {type: "CRYPTO"} | {type: "NO_FILE"} | {type: "HAS_FILE"} | {type: "FILE_IO"} | {type: "TIMEOUT"} | {type: "INTERNAL"} @@ -73,6 +80,8 @@ export function encodeFPUT(): Uint8Array { return ascii("FPUT") } export function encodeFDEL(): Uint8Array { return ascii("FDEL") } +export function encodeFACK(): Uint8Array { return ascii("FACK") } + export function encodeFGET(rcvDhKey: Uint8Array): Uint8Array { return concatBytes(ascii("FGET"), SPACE, encodeBytes(rcvDhKey)) } @@ -102,6 +111,17 @@ function decodeCommandError(s: string): CommandError { throw new Error("bad CommandError: " + s) } +function decodeBlockingInfo(s: string): BlockingInfo { + const noticeIdx = s.indexOf(",notice=") + const reasonPart = noticeIdx >= 0 ? s.slice(0, noticeIdx) : s + const reason: BlockingReason = reasonPart === "reason=spam" ? "spam" : "content" + let notice: {ttl: number | null} | null = null + if (noticeIdx >= 0) { + try { notice = JSON.parse(s.slice(noticeIdx + 8)) } catch {} + } + return {reason, notice} +} + export function decodeXFTPError(d: Decoder): XFTPErrorType { const s = readTag(d) switch (s) { @@ -115,7 +135,7 @@ export function decodeXFTPError(d: Decoder): XFTPErrorType { const rest = d.takeAll() let info = "" for (let i = 0; i < rest.length; i++) info += String.fromCharCode(rest[i]) - return {type: "BLOCKED", blockInfo: info} + return {type: "BLOCKED", blockInfo: decodeBlockingInfo(info)} } case "SIZE": return {type: "SIZE"} case "QUOTA": return {type: "QUOTA"} diff --git a/xftp-web/test/browser.test.ts b/xftp-web/test/browser.test.ts index 26a9670ca..9c4c07aed 100644 --- a/xftp-web/test/browser.test.ts +++ b/xftp-web/test/browser.test.ts @@ -1,11 +1,11 @@ import {test, expect} from 'vitest' -import {encryptFileForUpload, uploadFile, downloadFile, newXFTPAgent, closeXFTPAgent} from '../src/agent.js' +import {encryptFileForUpload, uploadFile, downloadFile, XFTPAgent} from '../src/agent.js' import {parseXFTPServer} from '../src/protocol/address.js' const server = parseXFTPServer(import.meta.env.XFTP_SERVER) test('browser upload + download round-trip', async () => { - const agent = newXFTPAgent() + const agent = new XFTPAgent() try { const data = new Uint8Array(50000) crypto.getRandomValues(data) @@ -14,6 +14,6 @@ test('browser upload + download round-trip', async () => { const {content} = await downloadFile(agent, rcvDescription) expect(content).toEqual(data) } finally { - closeXFTPAgent(agent) + agent.close() } }) diff --git a/xftp-web/test/connection.node.test.ts b/xftp-web/test/connection.node.test.ts index c04b7dd5a..0748790ed 100644 --- a/xftp-web/test/connection.node.test.ts +++ b/xftp-web/test/connection.node.test.ts @@ -1,9 +1,9 @@ import {test, expect, vi, beforeEach} from 'vitest' import { - newXFTPAgent, getXFTPServerClient, reconnectClient, removeStaleConnection, + XFTPAgent, getXFTPServerClient, reconnectClient, removeStaleConnection, sendXFTPCommand, XFTPRetriableError, XFTPPermanentError, - type XFTPClient, type XFTPClientAgent + type XFTPClient } from '../src/client.js' import {formatXFTPServer, type XFTPServer} from '../src/protocol/address.js' import {blockPad} from '../src/protocol/transmission.js' @@ -26,10 +26,8 @@ function makeMockClient(overrides?: Partial): XFTPClient { } } -function makeAgent(connectFn: (s: any) => Promise): XFTPClientAgent { - const agent = newXFTPAgent() - agent._connectFn = connectFn - return agent +function makeAgent(connectFn: (s: any) => Promise): XFTPAgent { + return new XFTPAgent(connectFn) } // T4: getXFTPServerClient coalesces concurrent calls @@ -66,7 +64,7 @@ test('getXFTPServerClient auto-cleans failed connections', async () => { // T6: removeStaleConnection respects promise identity test('removeStaleConnection respects promise identity', () => { - const agent = newXFTPAgent() + const agent = new XFTPAgent() const mockClient1 = makeMockClient() const mockClient2 = makeMockClient() const p1 = Promise.resolve(mockClient1) diff --git a/xftp-web/test/crypto.node.test.ts b/xftp-web/test/crypto.node.test.ts new file mode 100644 index 000000000..78f6567ff --- /dev/null +++ b/xftp-web/test/crypto.node.test.ts @@ -0,0 +1,102 @@ +import {test, expect} from 'vitest' +import sodium from 'libsodium-wrappers-sumo' +import {encryptFile, encryptFileAsync, encodeFileHeader} from '../src/crypto/file.js' +import {prepareChunkSizes, fileSizeLen, authTagSize} from '../src/protocol/chunks.js' +import {sha512Streaming} from '../src/crypto/digest.js' +import {encryptFileForUpload} from '../src/agent.js' + +await sodium.ready + +function fillRandom(buf: Uint8Array) { + for (let off = 0; off < buf.length; off += 65536) { + crypto.getRandomValues(buf.subarray(off, Math.min(off + 65536, buf.length))) + } +} + +function makeTestParams(sourceSize: number) { + const source = new Uint8Array(sourceSize) + fillRandom(source) + const fileHdr = encodeFileHeader({fileName: 'test.bin', fileExtra: null}) + const key = new Uint8Array(32) + const nonce = new Uint8Array(24) + crypto.getRandomValues(key) + crypto.getRandomValues(nonce) + const fileSize = BigInt(fileHdr.length + source.length) + const payloadSize = Number(fileSize) + fileSizeLen + authTagSize + const chunkSizes = prepareChunkSizes(payloadSize) + const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0)) + return {source, fileHdr, key, nonce, fileSize, encSize} +} + +test('encryptFileAsync produces identical output to encryptFile', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(256 * 1024) // 256KB + const syncResult = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) + const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(asyncResult.length).toBe(syncResult.length) + expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult)) +}) + +test('encryptFileAsync matches sync for small files', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(100) // 100 bytes + const syncResult = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) + const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult)) +}) + +test('encryptFileAsync matches sync for exact slice boundary', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(65536) // exactly 1 slice + const syncResult = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) + const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult)) +}) + +test('encryptFileAsync matches sync for multi-slice boundary', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(65536 * 3) // exactly 3 slices + const syncResult = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) + const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult)) +}) + +test('encryptFileAsync calls onProgress', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(65536 * 2 + 100) // 2 full slices + partial + const calls: [number, number][] = [] + await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, (done, total) => { + calls.push([done, total]) + }) + expect(calls.length).toBe(3) // 2 full slices + 1 partial + expect(calls[calls.length - 1][0]).toBe(calls[calls.length - 1][1]) // last call: done === total +}) + +test('encryptFileAsync matches sync for empty source', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(0) + const syncResult = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) + const asyncResult = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(Buffer.from(asyncResult)).toEqual(Buffer.from(syncResult)) +}) + +test('encryptFileAsync streaming produces identical output to buffered', async () => { + const {source, fileHdr, key, nonce, fileSize, encSize} = makeTestParams(256 * 1024) + const slices: Uint8Array[] = [] + await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, undefined, (data) => { + slices.push(data) + }) + const streamed = Buffer.concat(slices) + const buffered = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize) + expect(streamed).toEqual(Buffer.from(buffered)) +}) + +test('encryptFileForUpload streaming digest matches slice data', async () => { + const source = new Uint8Array(256 * 1024) + fillRandom(source) + const slices: Uint8Array[] = [] + const result = await encryptFileForUpload(source, 'test.bin', { + onSlice: (data) => { slices.push(data) } + }) + const combined = Buffer.concat(slices) + const actualDigest = sha512Streaming([combined]) + expect(Buffer.from(result.digest)).toEqual(Buffer.from(actualDigest)) + expect(result.key.length).toBe(32) + expect(result.nonce.length).toBe(24) + expect(result.chunkSizes.length).toBeGreaterThan(0) + expect('encData' in result).toBe(false) +}) diff --git a/xftp-web/tsconfig.json b/xftp-web/tsconfig.json index e42e74895..08f709409 100644 --- a/xftp-web/tsconfig.json +++ b/xftp-web/tsconfig.json @@ -7,6 +7,7 @@ "outDir": "dist", "rootDir": "src", "declaration": true, + "declarationMap": true, "strict": true, "esModuleInterop": true, "skipLibCheck": true, diff --git a/xftp-web/web/crypto-backend.ts b/xftp-web/web/crypto-backend.ts index eb4155128..e1ed52b84 100644 --- a/xftp-web/web/crypto-backend.ts +++ b/xftp-web/web/crypto-backend.ts @@ -28,10 +28,27 @@ class WorkerBackend implements CryptoBackend { private pending = new Map() private nextId = 1 private progressCb: ((done: number, total: number) => void) | null = null + private ready: Promise constructor() { this.worker = new Worker(new URL('./crypto.worker.ts', import.meta.url), {type: 'module'}) - this.worker.onmessage = (e) => this.handleMessage(e.data) + let rejectReady: (e: Error) => void + this.ready = new Promise((resolve, reject) => { + rejectReady = reject + this.worker.onmessage = (e) => { + if (e.data?.type === 'ready') { + this.worker.onmessage = (e) => this.handleMessage(e.data) + resolve() + } else { + reject(new Error('Worker: unexpected first message')) + } + } + }) + this.worker.onerror = (e) => { + rejectReady(new Error('Worker failed to load: ' + e.message)) + for (const p of this.pending.values()) p.reject(new Error('Worker error: ' + e.message)) + this.pending.clear() + } } private handleMessage(msg: {id: number, type: string, [k: string]: any}) { @@ -49,7 +66,8 @@ class WorkerBackend implements CryptoBackend { } } - private send(msg: Record, transfer?: Transferable[]): Promise { + private async send(msg: Record, transfer?: Transferable[]): Promise { + await this.ready const id = this.nextId++ return new Promise((resolve, reject) => { this.pending.set(id, {resolve, reject}) diff --git a/xftp-web/web/crypto.worker.ts b/xftp-web/web/crypto.worker.ts index b9bc7e705..36301217e 100644 --- a/xftp-web/web/crypto.worker.ts +++ b/xftp-web/web/crypto.worker.ts @@ -1,7 +1,6 @@ import sodium from 'libsodium-wrappers-sumo' -import {encryptFile, encodeFileHeader, decryptChunks} from '../src/crypto/file.js' +import {encryptFile, prepareEncryption, decryptChunks} from '../src/crypto/file.js' import {sha512Streaming} from '../src/crypto/digest.js' -import {prepareChunkSizes, fileSizeLen, authTagSize} from '../src/protocol/chunks.js' import {decryptReceivedChunk} from '../src/download.js' // ── OPFS session management ───────────────────────────────────── @@ -40,15 +39,7 @@ async function sweepStale() { async function handleEncrypt(id: number, data: ArrayBuffer, fileName: string) { const source = new Uint8Array(data) - const key = new Uint8Array(32) - const nonce = new Uint8Array(24) - crypto.getRandomValues(key) - crypto.getRandomValues(nonce) - const fileHdr = encodeFileHeader({fileName, fileExtra: null}) - const fileSize = BigInt(fileHdr.length + source.length) - const payloadSize = Number(fileSize) + fileSizeLen + authTagSize - const chunkSizes = prepareChunkSizes(payloadSize) - const encSize = BigInt(chunkSizes.reduce((a: number, b: number) => a + b, 0)) + const {fileHdr, key, nonce, fileSize, encSize, chunkSizes} = prepareEncryption(source.length, fileName) const encData = encryptFile(source, fileHdr, key, nonce, fileSize, encSize) self.postMessage({id, type: 'progress', done: 50, total: 100}) @@ -265,9 +256,9 @@ async function handleCleanup(id: number) { // ── Message dispatch ──────────────────────────────────────────── self.onmessage = async (e: MessageEvent) => { - await initPromise const msg = e.data try { + await initPromise switch (msg.type) { case 'encrypt': await handleEncrypt(msg.id, msg.data, msg.fileName) @@ -311,3 +302,6 @@ const initPromise = (async () => { await sodium.ready await sweepStale() })() + +// Signal main thread that the worker is ready to receive messages +initPromise.then(() => self.postMessage({type: 'ready'}), () => {}) diff --git a/xftp-web/web/download.ts b/xftp-web/web/download.ts index 25443cf35..c3a1c494d 100644 --- a/xftp-web/web/download.ts +++ b/xftp-web/web/download.ts @@ -1,7 +1,7 @@ import {createCryptoBackend} from './crypto-backend.js' import {createProgressRing} from './progress.js' import { - newXFTPAgent, closeXFTPAgent, + XFTPAgent, decodeDescriptionURI, downloadFileRaw } from '../src/agent.js' import {XFTPPermanentError} from '../src/client.js' @@ -68,7 +68,7 @@ export function initDownload(app: HTMLElement, hash: string) { statusText.textContent = 'Downloading…' const backend = createCryptoBackend() - const agent = newXFTPAgent() + const agent = new XFTPAgent() try { const resolvedFd = await downloadFileRaw(agent, fd, async (raw) => { @@ -115,7 +115,7 @@ export function initDownload(app: HTMLElement, hash: string) { else retryBtn.hidden = false } finally { await backend.cleanup().catch(() => {}) - closeXFTPAgent(agent) + agent.close() } } } diff --git a/xftp-web/web/upload.ts b/xftp-web/web/upload.ts index 12a473cf9..ee39c569f 100644 --- a/xftp-web/web/upload.ts +++ b/xftp-web/web/upload.ts @@ -1,11 +1,10 @@ import {createCryptoBackend} from './crypto-backend.js' -import {getServers, pickRandomServer} from './servers.js' +import {getServers} from './servers.js' import {createProgressRing} from './progress.js' import { - newXFTPAgent, closeXFTPAgent, uploadFile, encodeDescriptionURI, - type EncryptedFileMetadata + XFTPAgent, uploadFile, encodeDescriptionURI, + XFTPPermanentError, type EncryptedFileMetadata } from '../src/agent.js' -import {XFTPPermanentError} from '../src/client.js' const MAX_SIZE = 100 * 1024 * 1024 @@ -105,12 +104,12 @@ export function initUpload(app: HTMLElement) { statusText.textContent = 'Encrypting…' const backend = createCryptoBackend() - const agent = newXFTPAgent() + const agent = new XFTPAgent() cancelBtn.onclick = () => { aborted = true backend.cleanup().catch(() => {}) - closeXFTPAgent(agent) + agent.close() showStage(dropZone) } @@ -131,8 +130,7 @@ export function initUpload(app: HTMLElement) { chunkSizes: encrypted.chunkSizes } const servers = getServers() - const server = pickRandomServer(servers) - const result = await uploadFile(agent, server, metadata, { + const result = await uploadFile(agent, servers, metadata, { readChunk: (off, sz) => backend.readChunk(off, sz), onProgress: (uploaded, total) => { ring.update(0.3 + (uploaded / total) * 0.7) @@ -159,7 +157,7 @@ export function initUpload(app: HTMLElement) { } } finally { await backend.cleanup().catch(() => {}) - closeXFTPAgent(agent) + agent.close() } } }