diff --git a/remote-store/package-lock.json b/remote-store/package-lock.json index 6649f6f6..5cc3c556 100644 --- a/remote-store/package-lock.json +++ b/remote-store/package-lock.json @@ -1649,7 +1649,7 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-GDANpXzBtdu39GJSlGvLhjgkDF+zOUNVDAz4J/+28gZlDlbMjkjBkzmlA5cMl81k7+RWPnHTRdQ3T3pTY7A0Yg==", + "integrity": "sha512-QY8xRW6sz4SUKK6jPlI/YZpkSDxyhBcdORTWwBKcMqRBapjyv1V7Amht4gFsH86kMGlh1uK0W++vUdfuDuHtjg==", "dependencies": { "ajv": "^8.12.0", "axios": "^1.6.1", diff --git a/web-app/package-lock.json b/web-app/package-lock.json index 809d8f92..9d5ad231 100644 --- a/web-app/package-lock.json +++ b/web-app/package-lock.json @@ -12,6 +12,7 @@ "@opentdf/client": "file:../lib/opentdf-client-2.0.0.tgz", "clsx": "^2.0.0", "native-file-system-adapter": "^3.0.1", + "p-limit": "^5.0.0", "react": "^18.2.0", "react-dom": "^18.2.0" }, @@ -602,7 +603,7 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-GDANpXzBtdu39GJSlGvLhjgkDF+zOUNVDAz4J/+28gZlDlbMjkjBkzmlA5cMl81k7+RWPnHTRdQ3T3pTY7A0Yg==", + "integrity": "sha512-QY8xRW6sz4SUKK6jPlI/YZpkSDxyhBcdORTWwBKcMqRBapjyv1V7Amht4gFsH86kMGlh1uK0W++vUdfuDuHtjg==", "dependencies": { "ajv": "^8.12.0", "axios": "^1.6.1", @@ -1014,17 +1015,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@vitest/runner/node_modules/yocto-queue": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@vitest/snapshot": { "version": "0.33.0", "dev": true, @@ -2703,14 +2693,14 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "dev": true, - "license": "MIT", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2730,6 +2720,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parent-module": { "version": "1.0.1", "dev": true, @@ -3734,11 +3751,11 @@ "license": "ISC" }, "node_modules/yocto-queue": { - "version": "0.1.0", - "dev": true, - "license": "MIT", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -4099,7 +4116,7 @@ }, "@opentdf/client": { "version": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-GDANpXzBtdu39GJSlGvLhjgkDF+zOUNVDAz4J/+28gZlDlbMjkjBkzmlA5cMl81k7+RWPnHTRdQ3T3pTY7A0Yg==", + "integrity": "sha512-QY8xRW6sz4SUKK6jPlI/YZpkSDxyhBcdORTWwBKcMqRBapjyv1V7Amht4gFsH86kMGlh1uK0W++vUdfuDuHtjg==", "requires": { "ajv": "^8.12.0", "axios": "^1.6.1", @@ -4344,10 +4361,6 @@ "requires": { "yocto-queue": "^1.0.0" } - }, - "yocto-queue": { - "version": "1.0.0", - "dev": true } } }, @@ -5377,10 +5390,11 @@ } }, "p-limit": { - "version": "3.1.0", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", "requires": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" } }, "p-locate": { @@ -5388,6 +5402,23 @@ "dev": true, "requires": { "p-limit": "^3.0.2" + }, + "dependencies": { + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true + } } }, "parent-module": { @@ -5933,8 +5964,9 @@ "dev": true }, "yocto-queue": { - "version": "0.1.0", - "dev": true + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==" } } } diff --git a/web-app/package.json b/web-app/package.json index 84f1ac94..a0e57958 100644 --- a/web-app/package.json +++ b/web-app/package.json @@ -18,6 +18,7 @@ "@opentdf/client": "file:../lib/opentdf-client-2.0.0.tgz", "clsx": "^2.0.0", "native-file-system-adapter": "^3.0.1", + "p-limit": "^5.0.0", "react": "^18.2.0", "react-dom": "^18.2.0" }, diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 6592bfc6..a04c13dc 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -1,10 +1,13 @@ import { clsx } from 'clsx'; -import { useState, useEffect, type ChangeEvent } from 'react'; +import { useState, useEffect, type ChangeEvent, useRef } from 'react'; import { showSaveFilePicker } from 'native-file-system-adapter'; import './App.css'; import { type Chunker, type DecryptSource, NanoTDFClient, TDF3Client } from '@opentdf/client'; import { type SessionInformation, OidcClient } from './session.js'; import { c } from './config.js'; +import pLimit from 'p-limit'; + +const limit = pLimit(16); function decryptedFileName(encryptedFileName: string): string { // Groups: 1 file 'name' bit @@ -30,6 +33,15 @@ function decryptedFileExtension(encryptedFileName: string): string { return m[2]; } +function ReadableBufferStream(ab: ArrayBuffer) { + return new ReadableStream({ + start(controller) { + controller.enqueue(ab); + controller.close(); + }, + }); +} + const oidcClient = new OidcClient(c.oidc.host, c.oidc.clientId, 'otdf-sample-web-app'); function saver(blob: Blob, name: string) { @@ -63,7 +75,7 @@ async function getNewFileHandle( } type Containers = 'html' | 'tdf' | 'nano'; -type CurrentDataController = AbortController | undefined; +type CurrentDataControllers = Record; type FileInputSource = { type: 'file'; file: File; @@ -78,22 +90,31 @@ type RandomInputSource = { length: number; }; -type InputSource = FileInputSource | UrlInputSource | RandomInputSource; -type SinkType = 'file' | 'fsapi' | 'none'; +type MemoryInputSource = { + type: 'memory'; + src: ArrayBuffer; + name: string; +}; + +type InputSource = FileInputSource | UrlInputSource | RandomInputSource | MemoryInputSource; +type SinkType = 'file' | 'fsapi' | 'memory' | 'none'; -function fileNameFor(inputSource: InputSource) { +function fileNameFor(inputSource?: InputSource) { if (!inputSource) { return 'undefined.bin'; } - if ('file' in inputSource) { - return inputSource.file.name; - } - if ('length' in inputSource) { - return `random-${inputSource.type}-${inputSource.length}-bytes`; + switch (inputSource.type) { + case 'file': + return inputSource.file.name; + case 'bytes': + return `random-${inputSource.type}-${inputSource.length}-bytes`; + case 'url': + const { pathname } = inputSource.url; + const i = pathname.lastIndexOf('/'); + return pathname.slice(i + 1); + case 'memory': + return inputSource.name; } - const { pathname } = inputSource.url; - const i = pathname.lastIndexOf('/'); - return pathname.slice(i + 1); } function drain() { @@ -207,9 +228,9 @@ function App() { const [decryptContainerType, setDecryptContainerType] = useState('tdf'); const [downloadState, setDownloadState] = useState(); const [encryptContainerType, setEncryptContainerType] = useState('tdf'); - const [inputSource, setInputSource] = useState(); + const [inputSources, setInputSources] = useState([]); const [sinkType, setSinkType] = useState('file'); - const [streamController, setStreamController] = useState(); + const streamControllers = useRef({}); const handleContainerFormatRadioChange = (handler: typeof setDecryptContainerType) => (e: ChangeEvent) => { @@ -232,26 +253,27 @@ function App() { const setFileHandler = (event: ChangeEvent) => { const target = event.target as HTMLInputElement; if (target.files?.length) { - const [file] = target.files; - setInputSource({ type: 'file', file }); + const fileArray = Array.from(target.files); + const srcs = fileArray.map((file): FileInputSource => ({ type: 'file', file })); + setInputSources(srcs); } else { - setInputSource(undefined); + setInputSources([]); } }; const setRandomHandler = (event: ChangeEvent) => { const target = event.target as HTMLInputElement; if (target.value && target.validity.valid) { - setInputSource({ type: 'bytes', length: parseInt(target.value) }); + setInputSources([{ type: 'bytes', length: parseInt(target.value) }]); } else { - setInputSource(undefined); + setInputSources([]); } }; const setUrlHandler = (event: ChangeEvent) => { const target = event.target as HTMLInputElement; if (target.value && target.validity.valid) { - setInputSource({ type: 'url', url: new URL(target.value) }); + setInputSources([{ type: 'url', url: new URL(target.value) }]); } else { - setInputSource(undefined); + setInputSources([]); } }; @@ -319,7 +341,7 @@ function App() { }; const handleEncrypt = async () => { - if (!inputSource) { + if (!inputSources.length) { console.warn('No input source selected'); return false; } @@ -328,48 +350,62 @@ function App() { console.warn('PLEASE LOG IN'); return false; } - const inputFileName = fileNameFor(inputSource); - console.log(`Encrypting [${inputFileName}] as ${encryptContainerType} to ${sinkType}`); + const memory: MemoryInputSource[] = []; + + async function encryptNano( + nanoClient: NanoTDFClient, + inputSource: InputSource, + inputFileName: string + ) { + if ('url' in inputSource) { + throw new Error('Unsupported : fetch the url I guess?'); + } + const plainText = + 'file' == inputSource.type + ? await inputSource.file.arrayBuffer() + : 'memory' == inputSource.type + ? inputSource.src + : randomArrayBuffer(inputSource); + setDownloadState('Encrypting...'); + const cipherText = await nanoClient.encrypt(plainText); + switch (sinkType) { + case 'file': + saver(new Blob([cipherText]), `${inputFileName}.ntdf`); + break; + case 'fsapi': + { + const file = await getNewFileHandle('ntdf', `${inputFileName}.ntdf`); + const writable = await file.createWritable(); + try { + await writable.write(cipherText); + } catch (e) { + setDownloadState(`Encrypt Failed: ${e}`); + } finally { + await writable.close(); + } + } + break; + case 'memory': + memory.push({ type: 'memory', name: `${inputFileName}.ntdf`, src: cipherText }); + break; + case 'none': + break; + } + setDownloadState('Encrypt Complete'); + } + let promises; switch (encryptContainerType) { case 'nano': { - if ('url' in inputSource) { - throw new Error('Unsupported : fetch the url I guess?'); - } - const plainText = - 'file' in inputSource - ? await inputSource.file.arrayBuffer() - : randomArrayBuffer(inputSource); - const nanoClient = new NanoTDFClient({ - authProvider: oidcClient, - kasEndpoint: c.kas, - dpopKeys: oidcClient.getSigningKey(), + promises = inputSources.map((inputSource): (() => Promise) => async () => { + const nanoClient = new NanoTDFClient({ + authProvider: oidcClient, + kasEndpoint: c.kas, + dpopKeys: oidcClient.getSigningKey(), + }); + const inputFileName = fileNameFor(inputSource); + console.log(`Encrypting [${inputFileName}] as ${encryptContainerType} to ${sinkType}`); + await encryptNano(nanoClient, inputSource, inputFileName); }); - setDownloadState('Encrypting...'); - switch (sinkType) { - case 'file': - { - const cipherText = await nanoClient.encrypt(plainText); - saver(new Blob([cipherText]), `${inputFileName}.ntdf`); - } - break; - case 'fsapi': - { - const file = await getNewFileHandle('ntdf', `${inputFileName}.ntdf`); - const cipherText = await nanoClient.encrypt(plainText); - const writable = await file.createWritable(); - try { - await writable.write(cipherText); - setDownloadState('Encrypt Complete'); - } catch (e) { - setDownloadState(`Encrypt Failed: ${e}`); - } finally { - await writable.close(); - } - } - break; - case 'none': - break; - } break; } case 'html': { @@ -379,70 +415,10 @@ function App() { kasEndpoint: c.kas, readerUrl: c.reader, }); - let source: ReadableStream, size: number; - const sc = new AbortController(); - setStreamController(sc); - switch (inputSource.type) { - case 'file': - size = inputSource.file.size; - source = inputSource.file.stream() as unknown as ReadableStream; - break; - case 'bytes': - size = inputSource.length; - source = randomStream(inputSource); - break; - case 'url': - // NOTE: Attaching the signal to the pipeline (in pipeTo, below) - // is insufficient (at least in Chrome) to abort the fetch itself. - // So aborting a sink in a pipeline does *NOT* cancel its sources - const fr = await fetch(inputSource.url, { signal: sc.signal }); - if (!fr.ok) { - throw Error( - `Error on fetch [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` - ); - } - if (!fr.body) { - throw Error( - `Failed to fetch input [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` - ); - } - size = parseInt(fr.headers.get('Content-Length') || '-1'); - source = fr.body; - break; - } - try { - const downloadName = `${inputFileName}.tdf.html`; - let f; - if (sinkType == 'fsapi') { - f = await getNewFileHandle('html', downloadName); - } - const progressTransformers = makeProgressPair(size, 'Encrypt'); - const cipherText = await client.encrypt({ - source: source.pipeThrough(progressTransformers.reader), - offline: true, - asHtml: true, - }); - cipherText.stream = cipherText.stream.pipeThrough(progressTransformers.writer); - switch (sinkType) { - case 'file': - await cipherText.toFile(downloadName, { signal: sc.signal }); - break; - case 'fsapi': - if (!f) { - throw new Error(); - } - const writable = await f.createWritable(); - await cipherText.stream.pipeTo(writable, { signal: sc.signal }); - break; - case 'none': - await cipherText.stream.pipeTo(drain(), { signal: sc.signal }); - break; - } - } catch (e) { - setDownloadState(`Encrypt Failed: ${e}`); - console.error('Encrypt Failed', e); - } - setStreamController(undefined); + promises = inputSources.map((inputSource): (() => Promise) => async () => { + const inputFileName = fileNameFor(inputSource); + await encryptTdfHtml(inputSource, inputFileName, client); + }); break; } case 'tdf': { @@ -451,74 +427,201 @@ function App() { dpopKeys: oidcClient.getSigningKey(), kasEndpoint: c.kas, }); - const sc = new AbortController(); - setStreamController(sc); - let source: ReadableStream, size: number; - switch (inputSource.type) { + promises = inputSources.map((inputSource): (() => Promise) => async () => { + const inputFileName = fileNameFor(inputSource); + await encryptTdf(inputSource, inputFileName, client); + }); + break; + } + } + await Promise.all(promises.map(limit)); + + if (memory.length) { + setInputSources(memory); + } + + return true; + + async function encryptTdfHtml( + inputSource: InputSource, + inputFileName: string, + client: TDF3Client + ) { + let source: ReadableStream, size: number; + const sc = new AbortController(); + streamControllers.current[inputFileName] = sc; + switch (inputSource.type) { + case 'file': + size = inputSource.file.size; + source = inputSource.file.stream() as unknown as ReadableStream; + break; + + case 'bytes': + size = inputSource.length; + source = randomStream(inputSource); + break; + + case 'memory': + size = inputSource.src.byteLength; + source = ReadableBufferStream(inputSource.src); + break; + + case 'url': + // NOTE: Attaching the signal to the pipeline (in pipeTo, below) + // is insufficient (at least in Chrome) to abort the fetch itself. + // So aborting a sink in a pipeline does *NOT* cancel its sources + const fr = await fetch(inputSource.url, { signal: sc.signal }); + if (!fr.ok) { + throw Error( + `Error on fetch [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` + ); + } + if (!fr.body) { + throw Error( + `Failed to fetch input [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` + ); + } + size = parseInt(fr.headers.get('Content-Length') || '-1'); + source = fr.body; + break; + } + try { + const downloadName = `${inputFileName}.tdf.html`; + const progressTransformers = makeProgressPair(size, 'Encrypt'); + const cipherText = await client.encrypt({ + source: source.pipeThrough(progressTransformers.reader), + offline: true, + asHtml: true, + }); + cipherText.stream = cipherText.stream.pipeThrough(progressTransformers.writer); + switch (sinkType) { case 'file': - size = inputSource.file.size; - source = inputSource.file.stream() as unknown as ReadableStream; + await cipherText.toFile(downloadName, { signal: sc.signal }); break; - case 'bytes': - size = inputSource.length; - source = randomStream(inputSource); + case 'fsapi': + const f = await getNewFileHandle('html', downloadName); + const writable = await f.createWritable(); + await cipherText.stream.pipeTo(writable, { signal: sc.signal }); break; - case 'url': - const fr = await fetch(inputSource.url, { signal: sc.signal }); - if (!fr.ok) { - throw Error( - `Error on fetch [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` - ); - } - if (!fr.body) { - throw Error( - `Failed to fetch input [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` - ); - } - size = parseInt(fr.headers.get('Content-Length') || '-1'); - source = fr.body; + case 'memory': + memory.push({ + type: 'memory', + name: downloadName, + src: await new Response(cipherText.stream).arrayBuffer(), + }); + break; + case 'none': + await cipherText.stream.pipeTo(drain(), { signal: sc.signal }); break; } - try { - let f; - const downloadName = `${inputFileName}.tdf`; - if (sinkType === 'fsapi') { - f = await getNewFileHandle('tdf', downloadName); + } finally { + delete streamControllers.current[inputFileName]; + } + } + + async function encryptTdf(inputSource: InputSource, inputFileName: string, client: TDF3Client) { + const sc = new AbortController(); + streamControllers.current[inputFileName] = sc; + let source: ReadableStream, size: number; + switch (inputSource.type) { + case 'file': + size = inputSource.file.size; + source = inputSource.file.stream() as unknown as ReadableStream; + break; + case 'bytes': + size = inputSource.length; + source = randomStream(inputSource); + break; + case 'memory': + size = inputSource.src.byteLength; + source = ReadableBufferStream(inputSource.src); + break; + case 'url': + const fr = await fetch(inputSource.url, { signal: sc.signal }); + if (!fr.ok) { + throw Error( + `Error on fetch [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` + ); } - const progressTransformers = makeProgressPair(size, 'Encrypt'); - const cipherText = await client.encrypt({ - source: source.pipeThrough(progressTransformers.reader), - offline: true, - }); - cipherText.stream = cipherText.stream.pipeThrough(progressTransformers.writer); - switch (sinkType) { - case 'file': - await cipherText.toFile(downloadName, { signal: sc.signal }); - break; - case 'fsapi': - if (!f) { - throw new Error(); - } - const writable = await f.createWritable(); - await cipherText.stream.pipeTo(writable, { signal: sc.signal }); - break; - case 'none': - await cipherText.stream.pipeTo(drain(), { signal: sc.signal }); - break; + if (!fr.body) { + throw Error( + `Failed to fetch input [${inputSource.url}]: ${fr.status} code received; [${fr.statusText}]` + ); } - } catch (e) { - setDownloadState(`Encrypt Failed: ${e}`); - console.error('Encrypt Failed', e); + size = parseInt(fr.headers.get('Content-Length') || '-1'); + source = fr.body; + break; + } + try { + const downloadName = `${inputFileName}.tdf`; + const progressTransformers = makeProgressPair(size, 'Encrypt'); + const cipherText = await client.encrypt({ + source: source.pipeThrough(progressTransformers.reader), + offline: true, + }); + cipherText.stream = cipherText.stream.pipeThrough(progressTransformers.writer); + switch (sinkType) { + case 'file': + await cipherText.toFile(downloadName, { signal: sc.signal }); + break; + case 'fsapi': + const f = await getNewFileHandle('tdf', downloadName); + const writable = await f.createWritable(); + await cipherText.stream.pipeTo(writable, { signal: sc.signal }); + break; + case 'memory': + memory.push({ + type: 'memory', + name: downloadName, + src: await new Response(cipherText.stream).arrayBuffer(), + }); + break; + case 'none': + await cipherText.stream.pipeTo(drain(), { signal: sc.signal }); + break; } - setStreamController(undefined); - break; + } finally { + delete streamControllers.current[inputFileName]; } } - return true; }; + async function decryptNano( + nanoClient: NanoTDFClient, + inputSource: FileInputSource | RandomInputSource | MemoryInputSource, + dfn: string + ) { + const cipherText = + 'file' == inputSource.type + ? await inputSource.file.arrayBuffer() + : 'memory' == inputSource.type + ? inputSource.src + : randomArrayBuffer(inputSource); + const plainText = await nanoClient.decrypt(cipherText); + switch (sinkType) { + case 'file': + saver(new Blob([plainText]), dfn); + break; + case 'fsapi': + const f = await getNewFileHandle(decryptedFileExtension(fileNameFor(inputSource)), dfn); + const writable = await f.createWritable(); + try { + await writable.write(plainText); + } finally { + await writable.close(); + } + break; + case 'memory': + memory.push({ type: 'memory', name: dfn, src: cipherText }); + break; + case 'none': + break; + } + } + let promises: (() => Promise)[]; + const memory: MemoryInputSource[] = []; const handleDecrypt = async () => { - if (!inputSource) { + if (!inputSources.length) { console.log('PLEASE SELECT FILE'); return false; } @@ -526,14 +629,7 @@ function App() { console.error('decrypt while logged out doesnt work'); return false; } - const dfn = decryptedFileName(fileNameFor(inputSource)); - console.log( - `Decrypting ${decryptContainerType} ${JSON.stringify(inputSource)} to ${sinkType} ${dfn}` - ); - let f; - if (sinkType === 'fsapi') { - f = await getNewFileHandle(decryptedFileExtension(fileNameFor(inputSource)), dfn); - } + switch (decryptContainerType) { case 'tdf': { const client = new TDF3Client({ @@ -541,98 +637,98 @@ function App() { dpopKeys: oidcClient.getSigningKey(), kasEndpoint: c.kas, }); - try { - const sc = new AbortController(); - setStreamController(sc); - let source: DecryptSource; - let size: number; - switch (inputSource.type) { - case 'file': - size = inputSource.file.size; - source = { type: 'file-browser', location: inputSource.file }; - break; - case 'bytes': - size = inputSource.length; - source = { type: 'chunker', location: randomChunker(inputSource) }; - break; - case 'url': - const hr = await fetch(inputSource.url, { method: 'HEAD' }); - size = parseInt(hr.headers.get('Content-Length') || '-1'); - source = { type: 'remote', location: inputSource.url.toString() }; - break; - } - const progressTransformers = makeProgressPair(size, 'Decrypt'); - // XXX chunker doesn't have an equivalent 'stream' interaface - // so we kinda fake it with percentages by tracking output, which should - // strictly be smaller than the input file. - const plainText = await client.decrypt({ source }); - plainText.stream = plainText.stream - .pipeThrough(progressTransformers.reader) - .pipeThrough(progressTransformers.writer); - switch (sinkType) { - case 'file': - await plainText.toFile(dfn, { signal: sc.signal }); - break; - case 'fsapi': - if (!f) { - throw new Error(); - } - const writable = await f.createWritable(); - await plainText.stream.pipeTo(writable, { signal: sc.signal }); - break; - case 'none': - await plainText.stream.pipeTo(drain(), { signal: sc.signal }); - break; - } - } catch (e) { - console.error('Decrypt Failed', e); - setDownloadState(`Decrypt Failed: ${e}`); - } - setStreamController(undefined); + promises = inputSources.map((inputSource): (() => Promise) => async () => { + const dfn = decryptedFileName(fileNameFor(inputSource)); + console.log( + `Decrypting ${decryptContainerType} ${JSON.stringify( + inputSource + )} to ${sinkType} ${dfn}` + ); + await decryptTdf(client, inputSource, dfn); + }); break; } case 'nano': { - if ('url' in inputSource) { - throw new Error('Unsupported : fetch the url I guess?'); - } const nanoClient = new NanoTDFClient({ authProvider: oidcClient, kasEndpoint: c.kas, dpopKeys: oidcClient.getSigningKey(), }); - try { - const cipherText = - 'file' in inputSource - ? await inputSource.file.arrayBuffer() - : randomArrayBuffer(inputSource); - const plainText = await nanoClient.decrypt(cipherText); - switch (sinkType) { - case 'file': - saver(new Blob([plainText]), dfn); - break; - case 'fsapi': - if (!f) { - throw new Error(); - } - const writable = await f.createWritable(); - try { - await writable.write(plainText); - setDownloadState('Decrypt Complete'); - } finally { - await writable.close(); - } - break; - case 'none': - break; + promises = inputSources.map((inputSource): (() => Promise) => async () => { + if ('url' in inputSource) { + throw new Error('Unsupported : fetch the url I guess?'); } - } catch (e) { - console.error('Decrypt Failed', e); - setDownloadState(`Decrypt Failed: ${e}`); - } + const dfn = decryptedFileName(fileNameFor(inputSource)); + await decryptNano(nanoClient, inputSource, dfn); + }); break; } } + try { + await Promise.all(promises.map(limit)); + setDownloadState('Decrypt Complete'); + } catch (e) { + console.error('Decrypt Failed', e); + setDownloadState(`Decrypt Failed: ${e}`); + } + + if (memory.length) { + setInputSources(memory); + } return false; + + async function decryptTdf(client: TDF3Client, inputSource: InputSource, dfn: string) { + let f; + if (sinkType === 'fsapi') { + f = await getNewFileHandle(decryptedFileExtension(fileNameFor(inputSource)), dfn); + } + const sc = new AbortController(); + setStreamController(sc); + let source: DecryptSource; + let size: number; + switch (inputSource.type) { + case 'file': + size = inputSource.file.size; + source = { type: 'file-browser', location: inputSource.file }; + break; + case 'bytes': + size = inputSource.length; + source = { type: 'chunker', location: randomChunker(inputSource) }; + break; + case 'memory': + size = inputSource.src.byteLength; + source = { type: 'buffer', location: new Uint8Array(inputSource.src) }; + break; + case 'url': + const hr = await fetch(inputSource.url, { method: 'HEAD' }); + size = parseInt(hr.headers.get('Content-Length') || '-1'); + source = { type: 'remote', location: inputSource.url.toString() }; + break; + } + const progressTransformers = makeProgressPair(size, 'Decrypt'); + // XXX chunker doesn't have an equivalent 'stream' interaface + // so we kinda fake it with percentages by tracking output, which should + // strictly be smaller than the input file. + const plainText = await client.decrypt({ source }); + plainText.stream = plainText.stream + .pipeThrough(progressTransformers.reader) + .pipeThrough(progressTransformers.writer); + switch (sinkType) { + case 'file': + await plainText.toFile(dfn, { signal: sc.signal }); + break; + case 'fsapi': + if (!f) { + throw new Error(); + } + const writable = await f.createWritable(); + await plainText.stream.pipeTo(writable, { signal: sc.signal }); + break; + case 'none': + await plainText.stream.pipeTo(drain(), { signal: sc.signal }); + break; + } + } }; const SessionInfo = @@ -653,7 +749,25 @@ function App() { 
{JSON.stringify(authState?.user, null, ' ')}
); - const hasFileInput = inputSource && 'file' in inputSource; + let inputDetails; + if (inputSources.length == 1) { + const inputSource = inputSources[0]; + inputDetails = ( + <> +

{fileNameFor(inputSource)}

+ {inputSource.type == 'file' && ( + <> +
Content Type: {inputSource.file.type}
+
Last Modified: {new Date(inputSource.file.lastModified).toLocaleString()}
+
Size: {new Intl.NumberFormat().format(inputSource.file.size)} bytes
+ + )} + + ); + } else { + inputDetails =

{inputSources.length} items

; + } + return (
@@ -667,35 +781,34 @@ function App() {
Source - {hasFileInput ? ( + {inputSources.length ? (
-

{'file' in inputSource ? inputSource.file.name : '[rand]'}

- {'file' in inputSource && ( - <> -
Content Type: {inputSource.file.type}
-
- Last Modified: {new Date(inputSource.file.lastModified).toLocaleString()} -
-
Size: {new Intl.NumberFormat().format(inputSource.file.size)} bytes
- - )} + {inputDetails}
) : ( <> - +
OR
-
+
OR:
-
+
{' '}
+ setSinkType(e.target.value as SinkType)} + checked={sinkType === 'memory'} + />{' '} + +
)} - {inputSource && !streamController && ( + {inputSources.length && !streamController && (

Encrypt

diff --git a/web-app/tests/bigfile.py b/web-app/tests/bigfile.py old mode 100644 new mode 100755 index 8a79e13f..f7ec4300 --- a/web-app/tests/bigfile.py +++ b/web-app/tests/bigfile.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import math import sys diff --git a/web-app/tests/smallfiles.py b/web-app/tests/smallfiles.py new file mode 100755 index 00000000..ff5317a1 --- /dev/null +++ b/web-app/tests/smallfiles.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import math +import sys + + +# Generate a lot of smallish files +# For example, to make an 1,024 file: +# python bigfile.py $(( 2 ** 10 )) +def fill(f, size): + power = math.ceil(math.log(size) / math.log(2)) + octets_per_word = max(1, power - 3) + strides = size // octets_per_word + remainder = size % octets_per_word + for x in range(strides): + f.write(f"%0{octets_per_word}x" % (x * octets_per_word)) + for x in range(remainder): + f.write(".") + + +if __name__ == "__main__": + n = int(sys.argv[1]) + digits = math.ceil(math.log(n+1) / math.log(10)) + for x in range(1, n + 1): + infix = f"{x}".rjust(digits, "0") + with open(f"s-{infix}.txt", "w") as file: + fill(file, x)