diff --git a/.gitignore b/.gitignore index 5cf886dec..fab97a001 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ packages/web/dist packages/filepicker/dist packages/features/dist packages/encryption/dist +packages/files/dist **/.pnp.* **/.yarn/* diff --git a/.yarn/cache/@standardnotes-files-npm-1.3.22-85a573b022-4bd58c1aed.zip b/.yarn/cache/@standardnotes-files-npm-1.3.22-85a573b022-4bd58c1aed.zip deleted file mode 100644 index 8842e412c..000000000 Binary files a/.yarn/cache/@standardnotes-files-npm-1.3.22-85a573b022-4bd58c1aed.zip and /dev/null differ diff --git a/.yarn/cache/@standardnotes-files-npm-1.3.23-e16ad43eb0-923dbd892e.zip b/.yarn/cache/@standardnotes-files-npm-1.3.23-e16ad43eb0-923dbd892e.zip deleted file mode 100644 index d7fa62577..000000000 Binary files a/.yarn/cache/@standardnotes-files-npm-1.3.23-e16ad43eb0-923dbd892e.zip and /dev/null differ diff --git a/packages/files/.eslintignore b/packages/files/.eslintignore new file mode 100644 index 000000000..f06235c46 --- /dev/null +++ b/packages/files/.eslintignore @@ -0,0 +1,2 @@ +node_modules +dist diff --git a/packages/files/.eslintrc b/packages/files/.eslintrc new file mode 100644 index 000000000..cb7136174 --- /dev/null +++ b/packages/files/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "./linter.tsconfig.json" + } +} diff --git a/packages/files/CHANGELOG.md b/packages/files/CHANGELOG.md new file mode 100644 index 000000000..527f5c976 --- /dev/null +++ b/packages/files/CHANGELOG.md @@ -0,0 +1,256 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. + +## [1.4.1](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.4.0...@standardnotes/files@1.4.1) (2022-07-05) + +**Note:** Version bump only for package @standardnotes/files + +# [1.4.0](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.24...@standardnotes/files@1.4.0) (2022-07-05) + +### Features + +* remove encryption package in favor of standardnotes/app repository ([f6d1c9e](https://github.com/standardnotes/snjs/commit/f6d1c9ee538bb59ee7ac28c0d49ca682d4eb4d38)) + +## [1.3.24](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.23...@standardnotes/files@1.3.24) (2022-07-04) + +### Bug Fixes + +* add missing reflect-metadata package to all packages ([ce3a5bb](https://github.com/standardnotes/snjs/commit/ce3a5bbf3f1d2276ac4abc3eec3c6a44c8c3ba9b)) + +## [1.3.23](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.22...@standardnotes/files@1.3.23) (2022-06-29) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.22](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.21...@standardnotes/files@1.3.22) (2022-06-27) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.21](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.20...@standardnotes/files@1.3.21) (2022-06-27) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.20](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.19...@standardnotes/files@1.3.20) (2022-06-22) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.19](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.18...@standardnotes/files@1.3.19) (2022-06-20) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.18](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.17...@standardnotes/files@1.3.18) (2022-06-16) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.17](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.16...@standardnotes/files@1.3.17) (2022-06-16) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.16](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.15...@standardnotes/files@1.3.16) (2022-06-15) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.15](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.14...@standardnotes/files@1.3.15) (2022-06-10) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.14](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.13...@standardnotes/files@1.3.14) (2022-06-09) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.13](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.12...@standardnotes/files@1.3.13) (2022-06-09) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.12](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.11...@standardnotes/files@1.3.12) (2022-06-09) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.11](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.10...@standardnotes/files@1.3.11) (2022-06-06) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.10](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.9...@standardnotes/files@1.3.10) (2022-06-03) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.9](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.8...@standardnotes/files@1.3.9) (2022-06-02) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.8](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.7...@standardnotes/files@1.3.8) (2022-06-02) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.7](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.6...@standardnotes/files@1.3.7) (2022-06-02) + +### Bug Fixes + +* remove isLast dep from ordered byte chunker ([3385581](https://github.com/standardnotes/snjs/commit/33855817d8d96d100b7d4f423f59f00c55834b6f)) + +## [1.3.6](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.5...@standardnotes/files@1.3.6) (2022-06-01) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.5](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.4...@standardnotes/files@1.3.5) (2022-05-30) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.4](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.3...@standardnotes/files@1.3.4) (2022-05-27) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.3](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.2...@standardnotes/files@1.3.3) (2022-05-27) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.2](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.1...@standardnotes/files@1.3.2) (2022-05-24) + +**Note:** Version bump only for package @standardnotes/files + +## [1.3.1](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.3.0...@standardnotes/files@1.3.1) (2022-05-24) + +**Note:** Version bump only for package @standardnotes/files + +# [1.3.0](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.2.1...@standardnotes/files@1.3.0) (2022-05-23) + +### Features + +* encrypted file cache ([#747](https://github.com/standardnotes/snjs/issues/747)) ([5b156a5](https://github.com/standardnotes/snjs/commit/5b156a5b4ee3365dac8e02653df129584a9dd4ef)) + +## [1.2.1](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.2.0...@standardnotes/files@1.2.1) (2022-05-22) + +**Note:** Version bump only for package @standardnotes/files + +# [1.2.0](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.13...@standardnotes/files@1.2.0) (2022-05-21) + +### Features + +* display controllers ([#743](https://github.com/standardnotes/snjs/issues/743)) ([5fadce3](https://github.com/standardnotes/snjs/commit/5fadce37f1b3f2f51b8a90c257bc666ac7710074)) + +## [1.1.13](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.12...@standardnotes/files@1.1.13) (2022-05-20) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.12](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.11...@standardnotes/files@1.1.12) (2022-05-20) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.11](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.10...@standardnotes/files@1.1.11) (2022-05-20) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.10](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.9...@standardnotes/files@1.1.10) (2022-05-18) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.9](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.8...@standardnotes/files@1.1.9) (2022-05-17) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.8](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.7...@standardnotes/files@1.1.8) (2022-05-17) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.7](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.6...@standardnotes/files@1.1.7) (2022-05-17) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.6](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.5...@standardnotes/files@1.1.6) (2022-05-16) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.5](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.4...@standardnotes/files@1.1.5) (2022-05-16) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.4](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.3...@standardnotes/files@1.1.4) (2022-05-16) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.3](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.2...@standardnotes/files@1.1.3) (2022-05-13) + +**Note:** Version bump only for package @standardnotes/files + +## [1.1.2](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.1...@standardnotes/files@1.1.2) (2022-05-13) + +### Bug Fixes + +* file backup decryption ([3f0d076](https://github.com/standardnotes/snjs/commit/3f0d076434c0dbb1827a298e302e4dc3815e501c)) + +## [1.1.1](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.1.0...@standardnotes/files@1.1.1) (2022-05-12) + +**Note:** Version bump only for package @standardnotes/files + +# [1.1.0](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.15...@standardnotes/files@1.1.0) (2022-05-12) + +### Features + +* file desktop backups ([#731](https://github.com/standardnotes/snjs/issues/731)) ([0dbce7d](https://github.com/standardnotes/snjs/commit/0dbce7dc9712fde848445b951079c81479c8bc11)) + +## [1.0.15](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.14...@standardnotes/files@1.0.15) (2022-05-12) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.14](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.13...@standardnotes/files@1.0.14) (2022-05-12) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.13](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.12...@standardnotes/files@1.0.13) (2022-05-09) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.12](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.11...@standardnotes/files@1.0.12) (2022-05-09) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.11](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.10...@standardnotes/files@1.0.11) (2022-05-06) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.10](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.9...@standardnotes/files@1.0.10) (2022-05-06) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.9](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.8...@standardnotes/files@1.0.9) (2022-05-05) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.8](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.6...@standardnotes/files@1.0.8) (2022-05-04) + +### Bug Fixes + +* config package missing dependencies ([3dec12f](https://github.com/standardnotes/snjs/commit/3dec12fa4a83a8aed8419819eafb7c34795cb09f)) + +## [1.0.7](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.6...@standardnotes/files@1.0.7) (2022-05-04) + +### Bug Fixes + +* config package missing dependencies ([3dec12f](https://github.com/standardnotes/snjs/commit/3dec12fa4a83a8aed8419819eafb7c34795cb09f)) + +## [1.0.6](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.5...@standardnotes/files@1.0.6) (2022-05-03) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.5](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.4...@standardnotes/files@1.0.5) (2022-05-02) + +### Bug Fixes + +* expose download progress ([#721](https://github.com/standardnotes/snjs/issues/721)) ([db7d205](https://github.com/standardnotes/snjs/commit/db7d20516ab918723b1a1d07e664d6f2ccccab75)) + +## [1.0.4](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.3...@standardnotes/files@1.0.4) (2022-05-02) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.3](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.2...@standardnotes/files@1.0.3) (2022-04-29) + +**Note:** Version bump only for package @standardnotes/files + +## [1.0.2](https://github.com/standardnotes/snjs/compare/@standardnotes/files@1.0.1...@standardnotes/files@1.0.2) (2022-04-28) + +**Note:** Version bump only for package @standardnotes/files + +## 1.0.1 (2022-04-28) + +**Note:** Version bump only for package @standardnotes/files diff --git a/packages/files/jest.config.js b/packages/files/jest.config.js new file mode 100644 index 000000000..ad1ceabb0 --- /dev/null +++ b/packages/files/jest.config.js @@ -0,0 +1,11 @@ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const base = require('../../node_modules/@standardnotes/config/src/jest.json'); + +module.exports = { + ...base, + globals: { + 'ts-jest': { + tsconfig: 'tsconfig.json', + }, + } +}; diff --git a/packages/files/linter.tsconfig.json b/packages/files/linter.tsconfig.json new file mode 100644 index 000000000..c1a7d22c5 --- /dev/null +++ b/packages/files/linter.tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["dist"] +} diff --git a/packages/files/package.json b/packages/files/package.json new file mode 100644 index 000000000..c61b1865f --- /dev/null +++ b/packages/files/package.json @@ -0,0 +1,45 @@ +{ + "name": "@standardnotes/files", + "version": "1.5.0", + "engines": { + "node": ">=16.0.0 <17.0.0" + }, + "description": "Client-side files library", + "main": "dist/index.js", + "author": "Standard Notes", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "license": "AGPL-3.0-or-later", + "scripts": { + "clean": "rm -fr dist", + "prestart": "yarn clean", + "start": "tsc -p tsconfig.json --watch", + "prebuild": "yarn clean", + "build": "tsc -p tsconfig.json", + "lint": "eslint . --ext .ts", + "test:unit": "jest" + }, + "devDependencies": { + "@types/jest": "^27.4.1", + "@typescript-eslint/eslint-plugin": "^5.30.0", + "eslint-plugin-prettier": "^4.2.1", + "jest": "^27.5.1", + "ts-jest": "^27.1.3" + }, + "dependencies": { + "@standardnotes/common": "^1.23.1", + "@standardnotes/encryption": "workspace:*", + "@standardnotes/filepicker": "workspace:*", + "@standardnotes/models": "^1.11.13", + "@standardnotes/responses": "^1.6.39", + "@standardnotes/services": "^1.13.23", + "@standardnotes/sncrypto-common": "^1.9.0", + "@standardnotes/utils": "^1.6.12", + "reflect-metadata": "^0.1.13" + } +} diff --git a/packages/files/src/Domain/Backups/BackupService.ts b/packages/files/src/Domain/Backups/BackupService.ts new file mode 100644 index 000000000..f557171f0 --- /dev/null +++ b/packages/files/src/Domain/Backups/BackupService.ts @@ -0,0 +1,173 @@ +import { ContentType, Uuid } from '@standardnotes/common' +import { EncryptionProvider } from '@standardnotes/encryption' +import { PayloadEmitSource, FileItem, CreateEncryptedBackupFileContextPayload } from '@standardnotes/models' +import { ClientDisplayableError } from '@standardnotes/responses' +import { + ItemManagerInterface, + FileBackupsDevice, + FileBackupsMapping, + AbstractService, + InternalEventBusInterface, + StatusServiceInterface, + FileBackupMetadataFile, + FilesApiInterface, +} from '@standardnotes/services' + +export class FilesBackupService extends AbstractService { + private itemsObserverDisposer: () => void + private pendingFiles = new Set() + + constructor( + private items: ItemManagerInterface, + private api: FilesApiInterface, + private encryptor: EncryptionProvider, + private device: FileBackupsDevice, + private status: StatusServiceInterface, + protected override internalEventBus: InternalEventBusInterface, + ) { + super(internalEventBus) + + this.itemsObserverDisposer = items.addObserver(ContentType.File, ({ changed, inserted, source }) => { + const applicableSources = [ + PayloadEmitSource.LocalDatabaseLoaded, + PayloadEmitSource.RemoteSaved, + PayloadEmitSource.RemoteRetrieved, + ] + + if (applicableSources.includes(source)) { + void this.handleChangedFiles([...changed, ...inserted]) + } + }) + } + + override deinit() { + this.itemsObserverDisposer() + } + + public isFilesBackupsEnabled(): Promise { + return this.device.isFilesBackupsEnabled() + } + + public async enableFilesBackups(): Promise { + await this.device.enableFilesBackups() + + if (!(await this.isFilesBackupsEnabled())) { + return + } + + this.backupAllFiles() + } + + private backupAllFiles(): void { + const files = this.items.getItems(ContentType.File) + + void this.handleChangedFiles(files) + } + + public disableFilesBackups(): Promise { + return this.device.disableFilesBackups() + } + + public changeFilesBackupsLocation(): Promise { + return this.device.changeFilesBackupsLocation() + } + + public getFilesBackupsLocation(): Promise { + return this.device.getFilesBackupsLocation() + } + + public openFilesBackupsLocation(): Promise { + return this.device.openFilesBackupsLocation() + } + + private async getBackupsMapping(): Promise { + return (await this.device.getFilesBackupsMappingFile()).files + } + + private async handleChangedFiles(files: FileItem[]): Promise { + if (files.length === 0) { + return + } + + if (!(await this.isFilesBackupsEnabled())) { + return + } + + const mapping = await this.getBackupsMapping() + + for (const file of files) { + if (this.pendingFiles.has(file.uuid)) { + continue + } + + const record = mapping[file.uuid] + + if (record == undefined) { + this.pendingFiles.add(file.uuid) + + await this.performBackupOperation(file) + + this.pendingFiles.delete(file.uuid) + } + } + } + + private async performBackupOperation(file: FileItem): Promise<'success' | 'failed' | 'aborted'> { + const messageId = this.status.addMessage(`Backing up file ${file.name}...`) + + const encryptedFile = await this.encryptor.encryptSplitSingle({ + usesItemsKeyWithKeyLookup: { + items: [file.payload], + }, + }) + + const itemsKey = this.items.getDisplayableItemsKeys().find((k) => k.uuid === encryptedFile.items_key_id) + + if (!itemsKey) { + return 'failed' + } + + const encryptedItemsKey = await this.encryptor.encryptSplitSingle({ + usesRootKeyWithKeyLookup: { + items: [itemsKey.payload], + }, + }) + + const token = await this.api.createFileValetToken(file.remoteIdentifier, 'read') + + if (token instanceof ClientDisplayableError) { + return 'failed' + } + + const metaFile: FileBackupMetadataFile = { + info: { + warning: 'Do not edit this file.', + information: 'The file and key data below is encrypted with your account password.', + instructions: + 'Drag and drop this metadata file into the File Backups preferences pane in the Standard Notes desktop or web application interface.', + }, + file: CreateEncryptedBackupFileContextPayload(encryptedFile.ejected()), + itemsKey: CreateEncryptedBackupFileContextPayload(encryptedItemsKey.ejected()), + version: '1.0.0', + } + + const metaFileAsString = JSON.stringify(metaFile, null, 2) + + const result = await this.device.saveFilesBackupsFile(file.uuid, metaFileAsString, { + chunkSizes: file.encryptedChunkSizes, + url: this.api.getFilesDownloadUrl(), + valetToken: token, + }) + + this.status.removeMessage(messageId) + + if (result === 'failed') { + const failMessageId = this.status.addMessage(`Failed to back up ${file.name}...`) + setTimeout(() => { + this.status.removeMessage(failMessageId) + }, 2000) + } + + return result + } +} diff --git a/packages/files/src/Domain/Operations/DownloadAndDecrypt.spec.ts b/packages/files/src/Domain/Operations/DownloadAndDecrypt.spec.ts new file mode 100644 index 000000000..c0e4f79f0 --- /dev/null +++ b/packages/files/src/Domain/Operations/DownloadAndDecrypt.spec.ts @@ -0,0 +1,113 @@ +import { sleep } from '@standardnotes/utils' +import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common' +import { FileDownloadProgress } from '../Types/FileDownloadProgress' +import { FilesApiInterface } from '@standardnotes/services' +import { DownloadAndDecryptFileOperation } from './DownloadAndDecrypt' +import { FileContent } from '@standardnotes/models' + +describe('download and decrypt', () => { + let apiService: FilesApiInterface + let operation: DownloadAndDecryptFileOperation + let file: { + encryptedChunkSizes: FileContent['encryptedChunkSizes'] + encryptionHeader: FileContent['encryptionHeader'] + remoteIdentifier: FileContent['remoteIdentifier'] + key: FileContent['key'] + } + let crypto: PureCryptoInterface + + const NumChunks = 5 + + const chunkOfSize = (size: number) => { + return new TextEncoder().encode('a'.repeat(size)) + } + + const downloadChunksOfSize = (size: number) => { + apiService.downloadFile = jest + .fn() + .mockImplementation( + ( + _file: string, + _chunkIndex: number, + _apiToken: string, + _rangeStart: number, + onBytesReceived: (bytes: Uint8Array) => void, + ) => { + const receiveFile = async () => { + for (let i = 0; i < NumChunks; i++) { + onBytesReceived(chunkOfSize(size)) + + await sleep(100, false) + } + } + + return new Promise((resolve) => { + void receiveFile().then(resolve) + }) + }, + ) + } + + beforeEach(() => { + apiService = {} as jest.Mocked + apiService.createFileValetToken = jest.fn() + downloadChunksOfSize(5) + + crypto = {} as jest.Mocked + + crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({ + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamDecryptorPush = jest.fn().mockReturnValue({ message: new Uint8Array([0xaa]), tag: 0 }) + + file = { + encryptedChunkSizes: [100_000], + remoteIdentifier: '123', + key: 'secret', + encryptionHeader: 'some-header', + } + }) + + it('run should resolve when operation is complete', async () => { + let receivedBytes = new Uint8Array() + + operation = new DownloadAndDecryptFileOperation(file, crypto, apiService) + + await operation.run(async (result) => { + if (result) { + receivedBytes = new Uint8Array([...receivedBytes, ...result.decrypted.decryptedBytes]) + } + + await Promise.resolve() + }) + + expect(receivedBytes.length).toEqual(NumChunks) + }) + + it('should correctly report progress', async () => { + file = { + encryptedChunkSizes: [100_000, 200_000, 200_000], + remoteIdentifier: '123', + key: 'secret', + encryptionHeader: 'some-header', + } + + downloadChunksOfSize(100_000) + + operation = new DownloadAndDecryptFileOperation(file, crypto, apiService) + + const progress: FileDownloadProgress = await new Promise((resolve) => { + // eslint-disable-next-line @typescript-eslint/require-await + void operation.run(async (result) => { + operation.abort() + resolve(result.progress) + }) + }) + + expect(progress.encryptedBytesDownloaded).toEqual(100_000) + expect(progress.encryptedBytesRemaining).toEqual(400_000) + expect(progress.encryptedFileSize).toEqual(500_000) + expect(progress.percentComplete).toEqual(20.0) + }) +}) diff --git a/packages/files/src/Domain/Operations/DownloadAndDecrypt.ts b/packages/files/src/Domain/Operations/DownloadAndDecrypt.ts new file mode 100644 index 000000000..a02c1b4aa --- /dev/null +++ b/packages/files/src/Domain/Operations/DownloadAndDecrypt.ts @@ -0,0 +1,75 @@ +import { ClientDisplayableError } from '@standardnotes/responses' +import { AbortFunction, FileDownloader } from '../UseCase/FileDownloader' +import { FileDecryptor } from '../UseCase/FileDecryptor' +import { FileDownloadProgress } from '../Types/FileDownloadProgress' +import { FilesApiInterface } from '@standardnotes/services' +import { PureCryptoInterface } from '@standardnotes/sncrypto-common' +import { FileContent } from '@standardnotes/models' +import { DecryptedBytes, EncryptedBytes } from '@standardnotes/filepicker' + +export type DownloadAndDecryptResult = { success: boolean; error?: ClientDisplayableError; aborted?: boolean } + +type OnBytesCallback = (results: { + decrypted: DecryptedBytes + encrypted: EncryptedBytes + progress: FileDownloadProgress +}) => Promise + +export class DownloadAndDecryptFileOperation { + private downloader: FileDownloader + + constructor( + private readonly file: { + encryptedChunkSizes: FileContent['encryptedChunkSizes'] + encryptionHeader: FileContent['encryptionHeader'] + remoteIdentifier: FileContent['remoteIdentifier'] + key: FileContent['key'] + }, + private readonly crypto: PureCryptoInterface, + private readonly api: FilesApiInterface, + ) { + this.downloader = new FileDownloader(this.file, this.api) + } + + private createDecryptor(): FileDecryptor { + return new FileDecryptor(this.file, this.crypto) + } + + public async run(onBytes: OnBytesCallback): Promise { + const decryptor = this.createDecryptor() + + let decryptError: ClientDisplayableError | undefined + + const onDownloadBytes = async ( + encryptedBytes: Uint8Array, + progress: FileDownloadProgress, + abortDownload: AbortFunction, + ) => { + const result = decryptor.decryptBytes(encryptedBytes) + + if (!result || result.decryptedBytes.length === 0) { + decryptError = new ClientDisplayableError('Failed to decrypt chunk') + + abortDownload() + + return + } + + const decryptedBytes = result.decryptedBytes + + await onBytes({ decrypted: { decryptedBytes }, encrypted: { encryptedBytes }, progress }) + } + + const downloadResult = await this.downloader.run(onDownloadBytes) + + return { + success: downloadResult instanceof ClientDisplayableError ? false : true, + error: downloadResult === 'aborted' ? undefined : downloadResult || decryptError, + aborted: downloadResult === 'aborted', + } + } + + abort(): void { + this.downloader.abort() + } +} diff --git a/packages/files/src/Domain/Operations/EncryptAndUpload.spec.ts b/packages/files/src/Domain/Operations/EncryptAndUpload.spec.ts new file mode 100644 index 000000000..be7322b8c --- /dev/null +++ b/packages/files/src/Domain/Operations/EncryptAndUpload.spec.ts @@ -0,0 +1,68 @@ +import { EncryptAndUploadFileOperation } from './EncryptAndUpload' +import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common' +import { FilesApiInterface } from '@standardnotes/services' +import { FileContent } from '@standardnotes/models' + +describe('encrypt and upload', () => { + let apiService: FilesApiInterface + let operation: EncryptAndUploadFileOperation + let file: { + decryptedSize: FileContent['decryptedSize'] + key: FileContent['key'] + remoteIdentifier: FileContent['remoteIdentifier'] + } + let crypto: PureCryptoInterface + + const chunkOfSize = (size: number) => { + return new TextEncoder().encode('a'.repeat(size)) + } + + beforeEach(() => { + apiService = {} as jest.Mocked + apiService.uploadFileBytes = jest.fn().mockReturnValue(true) + + crypto = {} as jest.Mocked + + crypto.xchacha20StreamInitEncryptor = jest.fn().mockReturnValue({ + header: 'some-header', + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamEncryptorPush = jest.fn().mockReturnValue(new Uint8Array()) + + file = { + remoteIdentifier: '123', + key: 'secret', + decryptedSize: 100, + } + }) + + it('should initialize encryption header', () => { + operation = new EncryptAndUploadFileOperation(file, 'api-token', crypto, apiService) + + expect(operation.getResult().encryptionHeader.length).toBeGreaterThan(0) + }) + + it('should return true when a chunk is uploaded', async () => { + operation = new EncryptAndUploadFileOperation(file, 'api-token', crypto, apiService) + + const bytes = new Uint8Array() + const success = await operation.pushBytes(bytes, 2, false) + + expect(success).toEqual(true) + }) + + it('should correctly report progress', async () => { + operation = new EncryptAndUploadFileOperation(file, 'api-token', crypto, apiService) + + const bytes = chunkOfSize(60) + await operation.pushBytes(bytes, 2, false) + + const progress = operation.getProgress() + + expect(progress.decryptedFileSize).toEqual(100) + expect(progress.decryptedBytesUploaded).toEqual(60) + expect(progress.decryptedBytesRemaining).toEqual(40) + expect(progress.percentComplete).toEqual(60.0) + }) +}) diff --git a/packages/files/src/Domain/Operations/EncryptAndUpload.ts b/packages/files/src/Domain/Operations/EncryptAndUpload.ts new file mode 100644 index 000000000..4d7a64d16 --- /dev/null +++ b/packages/files/src/Domain/Operations/EncryptAndUpload.ts @@ -0,0 +1,86 @@ +import { FileUploadProgress } from '../Types/FileUploadProgress' +import { FileUploadResult } from '../Types/FileUploadResult' +import { FilesApiInterface } from '@standardnotes/services' +import { FileUploader } from '../UseCase/FileUploader' +import { PureCryptoInterface } from '@standardnotes/sncrypto-common' +import { FileEncryptor } from '../UseCase/FileEncryptor' +import { FileContent } from '@standardnotes/models' + +export class EncryptAndUploadFileOperation { + public readonly encryptedChunkSizes: number[] = [] + + private readonly encryptor: FileEncryptor + private readonly uploader: FileUploader + private readonly encryptionHeader: string + + private totalBytesPushedInDecryptedTerms = 0 + private totalBytesUploadedInDecryptedTerms = 0 + + constructor( + private file: { + decryptedSize: FileContent['decryptedSize'] + key: FileContent['key'] + remoteIdentifier: FileContent['remoteIdentifier'] + }, + private apiToken: string, + private crypto: PureCryptoInterface, + private api: FilesApiInterface, + ) { + this.encryptor = new FileEncryptor(file, this.crypto) + this.uploader = new FileUploader(this.api) + + this.encryptionHeader = this.encryptor.initializeHeader() + } + + public getApiToken(): string { + return this.apiToken + } + + public getProgress(): FileUploadProgress { + const reportedDecryptedSize = this.file.decryptedSize + + return { + decryptedFileSize: reportedDecryptedSize, + decryptedBytesUploaded: this.totalBytesUploadedInDecryptedTerms, + decryptedBytesRemaining: reportedDecryptedSize - this.totalBytesUploadedInDecryptedTerms, + percentComplete: (this.totalBytesUploadedInDecryptedTerms / reportedDecryptedSize) * 100.0, + } + } + + public getResult(): FileUploadResult { + return { + encryptionHeader: this.encryptionHeader, + finalDecryptedSize: this.totalBytesPushedInDecryptedTerms, + key: this.file.key, + remoteIdentifier: this.file.remoteIdentifier, + } + } + + public async pushBytes(decryptedBytes: Uint8Array, chunkId: number, isFinalChunk: boolean): Promise { + this.totalBytesPushedInDecryptedTerms += decryptedBytes.byteLength + + const encryptedBytes = this.encryptBytes(decryptedBytes, isFinalChunk) + + this.encryptedChunkSizes.push(encryptedBytes.length) + + const uploadSuccess = await this.uploadBytes(encryptedBytes, chunkId) + + if (uploadSuccess) { + this.totalBytesUploadedInDecryptedTerms += decryptedBytes.byteLength + } + + return uploadSuccess + } + + private encryptBytes(decryptedBytes: Uint8Array, isFinalChunk: boolean): Uint8Array { + const encryptedBytes = this.encryptor.pushBytes(decryptedBytes, isFinalChunk) + + return encryptedBytes + } + + private async uploadBytes(encryptedBytes: Uint8Array, chunkId: number): Promise { + const success = await this.uploader.uploadBytes(encryptedBytes, chunkId, this.apiToken) + + return success + } +} diff --git a/packages/files/src/Domain/Service/FileService.spec.ts b/packages/files/src/Domain/Service/FileService.spec.ts new file mode 100644 index 000000000..ac924f0ed --- /dev/null +++ b/packages/files/src/Domain/Service/FileService.spec.ts @@ -0,0 +1,121 @@ +import { + InternalEventBusInterface, + SyncServiceInterface, + ItemManagerInterface, + AlertService, + ApiServiceInterface, + ChallengeServiceInterface, +} from '@standardnotes/services' +import { FileService } from './FileService' +import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common' +import { FileItem } from '@standardnotes/models' +import { EncryptionProvider } from '@standardnotes/encryption' + +describe('fileService', () => { + let apiService: ApiServiceInterface + let itemManager: ItemManagerInterface + let syncService: SyncServiceInterface + let alertService: AlertService + let crypto: PureCryptoInterface + let challengor: ChallengeServiceInterface + let fileService: FileService + let encryptor: EncryptionProvider + let internalEventBus: InternalEventBusInterface + + beforeEach(() => { + apiService = {} as jest.Mocked + apiService.addEventObserver = jest.fn() + apiService.createFileValetToken = jest.fn() + apiService.downloadFile = jest.fn() + apiService.deleteFile = jest.fn().mockReturnValue({}) + + itemManager = {} as jest.Mocked + itemManager.createItem = jest.fn() + itemManager.createTemplateItem = jest.fn().mockReturnValue({}) + itemManager.setItemToBeDeleted = jest.fn() + itemManager.addObserver = jest.fn() + itemManager.changeItem = jest.fn() + + challengor = {} as jest.Mocked + + syncService = {} as jest.Mocked + syncService.sync = jest.fn() + + encryptor = {} as jest.Mocked + + alertService = {} as jest.Mocked + alertService.confirm = jest.fn().mockReturnValue(true) + alertService.alert = jest.fn() + + crypto = {} as jest.Mocked + crypto.base64Decode = jest.fn() + internalEventBus = {} as jest.Mocked + internalEventBus.publish = jest.fn() + + fileService = new FileService( + apiService, + itemManager, + syncService, + encryptor, + challengor, + alertService, + crypto, + internalEventBus, + ) + + crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({ + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamDecryptorPush = jest.fn().mockReturnValue({ message: new Uint8Array([0xaa]), tag: 0 }) + + crypto.xchacha20StreamInitEncryptor = jest.fn().mockReturnValue({ + header: 'some-header', + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamEncryptorPush = jest.fn().mockReturnValue(new Uint8Array()) + }) + + it.only('should cache file after download', async () => { + const file = { + uuid: '1', + decryptedSize: 100_000, + encryptedSize: 101_000, + encryptedChunkSizes: [101_000], + } as jest.Mocked + + let downloadMock = apiService.downloadFile as jest.Mock + + await fileService.downloadFile(file, async () => { + return Promise.resolve() + }) + + expect(downloadMock).toHaveBeenCalledTimes(1) + + downloadMock = apiService.downloadFile = jest.fn() + + await fileService.downloadFile(file, async () => { + return Promise.resolve() + }) + + expect(downloadMock).toHaveBeenCalledTimes(0) + + expect(fileService['encryptedCache'].get(file.uuid)).toBeTruthy() + }) + + it('deleting file should remove it from cache', async () => { + const file = { + uuid: '1', + decryptedSize: 100_000, + } as jest.Mocked + + await fileService.downloadFile(file, async () => { + return Promise.resolve() + }) + + await fileService.deleteFile(file) + + expect(fileService['encryptedCache'].get(file.uuid)).toBeFalsy() + }) +}) diff --git a/packages/files/src/Domain/Service/FileService.ts b/packages/files/src/Domain/Service/FileService.ts new file mode 100644 index 000000000..0bc28b09e --- /dev/null +++ b/packages/files/src/Domain/Service/FileService.ts @@ -0,0 +1,314 @@ +import { DecryptedBytes, EncryptedBytes, FileMemoryCache, OrderedByteChunker } from '@standardnotes/filepicker' +import { ClientDisplayableError } from '@standardnotes/responses' +import { ContentType } from '@standardnotes/common' +import { DownloadAndDecryptFileOperation } from '../Operations/DownloadAndDecrypt' +import { EncryptAndUploadFileOperation } from '../Operations/EncryptAndUpload' +import { + FileItem, + FileProtocolV1Constants, + FileMetadata, + FileContentSpecialized, + FillItemContentSpecialized, + FileContent, + EncryptedPayload, + isEncryptedPayload, +} from '@standardnotes/models' +import { PureCryptoInterface } from '@standardnotes/sncrypto-common' +import { UuidGenerator } from '@standardnotes/utils' +import { + AbstractService, + InternalEventBusInterface, + ItemManagerInterface, + SyncServiceInterface, + AlertService, + FileSystemApi, + FilesApiInterface, + FileBackupMetadataFile, + FileHandleRead, + FileSystemNoSelection, + ChallengeServiceInterface, + FileBackupsConstantsV1, +} from '@standardnotes/services' +import { FilesClientInterface } from './FilesClientInterface' +import { FileDownloadProgress } from '../Types/FileDownloadProgress' +import { readAndDecryptBackupFile } from './ReadAndDecryptBackupFile' +import { DecryptItemsKeyWithUserFallback, EncryptionProvider, SNItemsKey } from '@standardnotes/encryption' +import { FileDecryptor } from '../UseCase/FileDecryptor' + +const OneHundredMb = 100 * 1_000_000 + +export class FileService extends AbstractService implements FilesClientInterface { + private encryptedCache: FileMemoryCache = new FileMemoryCache(OneHundredMb) + + constructor( + private api: FilesApiInterface, + private itemManager: ItemManagerInterface, + private syncService: SyncServiceInterface, + private encryptor: EncryptionProvider, + private challengor: ChallengeServiceInterface, + private alertService: AlertService, + private crypto: PureCryptoInterface, + protected override internalEventBus: InternalEventBusInterface, + ) { + super(internalEventBus) + } + + override deinit(): void { + super.deinit() + + this.encryptedCache.clear() + ;(this.encryptedCache as unknown) = undefined + ;(this.api as unknown) = undefined + ;(this.itemManager as unknown) = undefined + ;(this.encryptor as unknown) = undefined + ;(this.syncService as unknown) = undefined + ;(this.alertService as unknown) = undefined + ;(this.challengor as unknown) = undefined + ;(this.crypto as unknown) = undefined + } + + public minimumChunkSize(): number { + return 5_000_000 + } + + public async beginNewFileUpload( + sizeInBytes: number, + ): Promise { + const remoteIdentifier = UuidGenerator.GenerateUuid() + const tokenResult = await this.api.createFileValetToken(remoteIdentifier, 'write', sizeInBytes) + + if (tokenResult instanceof ClientDisplayableError) { + return tokenResult + } + + const key = this.crypto.generateRandomKey(FileProtocolV1Constants.KeySize) + + const fileParams = { + key, + remoteIdentifier, + decryptedSize: sizeInBytes, + } + + const uploadOperation = new EncryptAndUploadFileOperation(fileParams, tokenResult, this.crypto, this.api) + + const uploadSessionStarted = await this.api.startUploadSession(tokenResult) + + if (!uploadSessionStarted.uploadId) { + return new ClientDisplayableError('Could not start upload session') + } + + return uploadOperation + } + + public async pushBytesForUpload( + operation: EncryptAndUploadFileOperation, + bytes: Uint8Array, + chunkId: number, + isFinalChunk: boolean, + ): Promise { + const success = await operation.pushBytes(bytes, chunkId, isFinalChunk) + + if (!success) { + return new ClientDisplayableError('Failed to push file bytes to server') + } + + return undefined + } + + public async finishUpload( + operation: EncryptAndUploadFileOperation, + fileMetadata: FileMetadata, + ): Promise { + const uploadSessionClosed = await this.api.closeUploadSession(operation.getApiToken()) + + if (!uploadSessionClosed) { + return new ClientDisplayableError('Could not close upload session') + } + + const result = operation.getResult() + + const fileContent: FileContentSpecialized = { + decryptedSize: result.finalDecryptedSize, + encryptedChunkSizes: operation.encryptedChunkSizes, + encryptionHeader: result.encryptionHeader, + key: result.key, + mimeType: fileMetadata.mimeType, + name: fileMetadata.name, + remoteIdentifier: result.remoteIdentifier, + } + + const file = await this.itemManager.createItem( + ContentType.File, + FillItemContentSpecialized(fileContent), + true, + ) + + await this.syncService.sync() + + return file + } + + private async decryptCachedEntry(file: FileItem, entry: EncryptedBytes): Promise { + const decryptOperation = new FileDecryptor(file, this.crypto) + + let decryptedAggregate = new Uint8Array() + + const orderedChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (encryptedBytes) => { + const decryptedBytes = decryptOperation.decryptBytes(encryptedBytes) + + if (decryptedBytes) { + decryptedAggregate = new Uint8Array([...decryptedAggregate, ...decryptedBytes.decryptedBytes]) + } + }) + + await orderedChunker.addBytes(entry.encryptedBytes) + + return { decryptedBytes: decryptedAggregate } + } + + public async downloadFile( + file: FileItem, + onDecryptedBytes: (decryptedBytes: Uint8Array, progress?: FileDownloadProgress) => Promise, + ): Promise { + const cachedBytes = this.encryptedCache.get(file.uuid) + + if (cachedBytes) { + const decryptedBytes = await this.decryptCachedEntry(file, cachedBytes) + + await onDecryptedBytes(decryptedBytes.decryptedBytes, undefined) + + return undefined + } + + const addToCache = file.encryptedSize < this.encryptedCache.maxSize + + let cacheEntryAggregate = new Uint8Array() + + const operation = new DownloadAndDecryptFileOperation(file, this.crypto, this.api) + + const result = await operation.run(async ({ decrypted, encrypted, progress }): Promise => { + if (addToCache) { + cacheEntryAggregate = new Uint8Array([...cacheEntryAggregate, ...encrypted.encryptedBytes]) + } + return onDecryptedBytes(decrypted.decryptedBytes, progress) + }) + + if (addToCache) { + this.encryptedCache.add(file.uuid, { encryptedBytes: cacheEntryAggregate }) + } + + return result.error + } + + public async deleteFile(file: FileItem): Promise { + this.encryptedCache.remove(file.uuid) + + const tokenResult = await this.api.createFileValetToken(file.remoteIdentifier, 'delete') + + if (tokenResult instanceof ClientDisplayableError) { + return tokenResult + } + + const result = await this.api.deleteFile(tokenResult) + + if (result.error) { + return ClientDisplayableError.FromError(result.error) + } + + await this.itemManager.setItemToBeDeleted(file) + await this.syncService.sync() + + return undefined + } + + public isFileNameFileBackupRelated(name: string): 'metadata' | 'binary' | false { + if (name === FileBackupsConstantsV1.MetadataFileName) { + return 'metadata' + } else if (name === FileBackupsConstantsV1.BinaryFileName) { + return 'binary' + } + + return false + } + + public async decryptBackupMetadataFile(metdataFile: FileBackupMetadataFile): Promise { + const encryptedItemsKey = new EncryptedPayload({ + ...metdataFile.itemsKey, + waitingForKey: false, + errorDecrypting: false, + }) + + const decryptedItemsKeyResult = await DecryptItemsKeyWithUserFallback( + encryptedItemsKey, + this.encryptor, + this.challengor, + ) + + if (decryptedItemsKeyResult === 'failed' || decryptedItemsKeyResult === 'aborted') { + return undefined + } + + const encryptedFile = new EncryptedPayload({ ...metdataFile.file, waitingForKey: false, errorDecrypting: false }) + + const itemsKey = new SNItemsKey(decryptedItemsKeyResult) + + const decryptedFile = await this.encryptor.decryptSplitSingle({ + usesItemsKey: { + items: [encryptedFile], + key: itemsKey, + }, + }) + + if (isEncryptedPayload(decryptedFile)) { + return undefined + } + + return new FileItem(decryptedFile) + } + + public async selectFile(fileSystem: FileSystemApi): Promise { + const result = await fileSystem.selectFile() + + return result + } + + public async readBackupFileAndSaveDecrypted( + fileHandle: FileHandleRead, + file: FileItem, + fileSystem: FileSystemApi, + ): Promise<'success' | 'aborted' | 'failed'> { + const destinationDirectoryHandle = await fileSystem.selectDirectory() + + if (destinationDirectoryHandle === 'aborted' || destinationDirectoryHandle === 'failed') { + return destinationDirectoryHandle + } + + const destinationFileHandle = await fileSystem.createFile(destinationDirectoryHandle, file.name) + + if (destinationFileHandle === 'aborted' || destinationFileHandle === 'failed') { + return destinationFileHandle + } + + const result = await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => { + await fileSystem.saveBytes(destinationFileHandle, decryptedBytes) + }) + + await fileSystem.closeFileWriteStream(destinationFileHandle) + + return result + } + + public async readBackupFileBytesDecrypted( + fileHandle: FileHandleRead, + file: FileItem, + fileSystem: FileSystemApi, + ): Promise { + let bytes = new Uint8Array() + + await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => { + bytes = new Uint8Array([...bytes, ...decryptedBytes]) + }) + + return bytes + } +} diff --git a/packages/files/src/Domain/Service/FilesClientInterface.ts b/packages/files/src/Domain/Service/FilesClientInterface.ts new file mode 100644 index 000000000..b9605aec1 --- /dev/null +++ b/packages/files/src/Domain/Service/FilesClientInterface.ts @@ -0,0 +1,48 @@ +import { EncryptAndUploadFileOperation } from '../Operations/EncryptAndUpload' +import { FileItem, FileMetadata } from '@standardnotes/models' +import { ClientDisplayableError } from '@standardnotes/responses' +import { FileDownloadProgress } from '../Types/FileDownloadProgress' +import { FileSystemApi, FileBackupMetadataFile, FileHandleRead, FileSystemNoSelection } from '@standardnotes/services' + +export interface FilesClientInterface { + beginNewFileUpload(sizeInBytes: number): Promise + + pushBytesForUpload( + operation: EncryptAndUploadFileOperation, + bytes: Uint8Array, + chunkId: number, + isFinalChunk: boolean, + ): Promise + + finishUpload( + operation: EncryptAndUploadFileOperation, + fileMetadata: FileMetadata, + ): Promise + + downloadFile( + file: FileItem, + onDecryptedBytes: (bytes: Uint8Array, progress: FileDownloadProgress | undefined) => Promise, + ): Promise + + deleteFile(file: FileItem): Promise + + minimumChunkSize(): number + + isFileNameFileBackupRelated(name: string): 'metadata' | 'binary' | false + + decryptBackupMetadataFile(metdataFile: FileBackupMetadataFile): Promise + + selectFile(fileSystem: FileSystemApi): Promise + + readBackupFileAndSaveDecrypted( + fileHandle: FileHandleRead, + file: FileItem, + fileSystem: FileSystemApi, + ): Promise<'success' | 'aborted' | 'failed'> + + readBackupFileBytesDecrypted( + fileHandle: FileHandleRead, + file: FileItem, + fileSystem: FileSystemApi, + ): Promise +} diff --git a/packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts b/packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts new file mode 100644 index 000000000..a0f889279 --- /dev/null +++ b/packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts @@ -0,0 +1,36 @@ +import { FileContent } from '@standardnotes/models' +import { FileSystemApi, FileHandleRead } from '@standardnotes/services' +import { PureCryptoInterface } from '@standardnotes/sncrypto-common' +import { OrderedByteChunker } from '@standardnotes/filepicker' +import { FileDecryptor } from '../UseCase/FileDecryptor' + +export async function readAndDecryptBackupFile( + fileHandle: FileHandleRead, + file: { + encryptionHeader: FileContent['encryptionHeader'] + remoteIdentifier: FileContent['remoteIdentifier'] + encryptedChunkSizes: FileContent['encryptedChunkSizes'] + key: FileContent['key'] + }, + fileSystem: FileSystemApi, + crypto: PureCryptoInterface, + onDecryptedBytes: (decryptedBytes: Uint8Array) => Promise, +): Promise<'aborted' | 'failed' | 'success'> { + const decryptor = new FileDecryptor(file, crypto) + + const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (chunk: Uint8Array) => { + const decryptResult = decryptor.decryptBytes(chunk) + + if (!decryptResult) { + return + } + + await onDecryptedBytes(decryptResult.decryptedBytes) + }) + + const readResult = await fileSystem.readFile(fileHandle, async (encryptedBytes: Uint8Array) => { + await byteChunker.addBytes(encryptedBytes) + }) + + return readResult +} diff --git a/packages/files/src/Domain/Types/FileDownloadProgress.ts b/packages/files/src/Domain/Types/FileDownloadProgress.ts new file mode 100644 index 000000000..eac0067fb --- /dev/null +++ b/packages/files/src/Domain/Types/FileDownloadProgress.ts @@ -0,0 +1,6 @@ +export type FileDownloadProgress = { + encryptedFileSize: number + encryptedBytesDownloaded: number + encryptedBytesRemaining: number + percentComplete: number +} diff --git a/packages/files/src/Domain/Types/FileUploadProgress.ts b/packages/files/src/Domain/Types/FileUploadProgress.ts new file mode 100644 index 000000000..c90507799 --- /dev/null +++ b/packages/files/src/Domain/Types/FileUploadProgress.ts @@ -0,0 +1,6 @@ +export type FileUploadProgress = { + decryptedFileSize: number + decryptedBytesUploaded: number + decryptedBytesRemaining: number + percentComplete: number +} diff --git a/packages/files/src/Domain/Types/FileUploadResult.ts b/packages/files/src/Domain/Types/FileUploadResult.ts new file mode 100644 index 000000000..b8d22f638 --- /dev/null +++ b/packages/files/src/Domain/Types/FileUploadResult.ts @@ -0,0 +1,6 @@ +export type FileUploadResult = { + encryptionHeader: string + finalDecryptedSize: number + key: string + remoteIdentifier: string +} diff --git a/packages/files/src/Domain/UseCase/FileDecryptor.spec.ts b/packages/files/src/Domain/UseCase/FileDecryptor.spec.ts new file mode 100644 index 000000000..05d6c7fcf --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileDecryptor.spec.ts @@ -0,0 +1,51 @@ +import { FileDecryptor } from './FileDecryptor' +import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common' +import { FileContent } from '@standardnotes/models' +import { assert } from '@standardnotes/utils' + +describe('file decryptor', () => { + let decryptor: FileDecryptor + let file: { + encryptionHeader: FileContent['encryptionHeader'] + remoteIdentifier: FileContent['remoteIdentifier'] + key: FileContent['key'] + } + let crypto: PureCryptoInterface + + beforeEach(() => { + crypto = {} as jest.Mocked + + crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({ + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamDecryptorPush = jest.fn().mockReturnValue({ message: new Uint8Array([0xaa]), tag: 0 }) + + file = { + remoteIdentifier: '123', + encryptionHeader: 'some-header', + key: 'secret', + } + + decryptor = new FileDecryptor(file, crypto) + }) + + it('initialize', () => { + expect(crypto.xchacha20StreamInitDecryptor).toHaveBeenCalledWith(file.encryptionHeader, file.key) + }) + + it('decryptBytes should return decrypted bytes', () => { + const encryptedBytes = new Uint8Array([0xaa]) + const result = decryptor.decryptBytes(encryptedBytes) + + assert(result) + + expect(crypto.xchacha20StreamDecryptorPush).toHaveBeenCalledWith( + expect.any(Object), + encryptedBytes, + file.remoteIdentifier, + ) + + expect(result.decryptedBytes.length).toEqual(1) + }) +}) diff --git a/packages/files/src/Domain/UseCase/FileDecryptor.ts b/packages/files/src/Domain/UseCase/FileDecryptor.ts new file mode 100644 index 000000000..c5dd6baa5 --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileDecryptor.ts @@ -0,0 +1,29 @@ +import { PureCryptoInterface, StreamDecryptor, SodiumConstant } from '@standardnotes/sncrypto-common' +import { FileContent } from '@standardnotes/models' + +export class FileDecryptor { + private decryptor: StreamDecryptor + + constructor( + private file: { + encryptionHeader: FileContent['encryptionHeader'] + remoteIdentifier: FileContent['remoteIdentifier'] + key: FileContent['key'] + }, + private crypto: PureCryptoInterface, + ) { + this.decryptor = this.crypto.xchacha20StreamInitDecryptor(this.file.encryptionHeader, this.file.key) + } + + public decryptBytes(encryptedBytes: Uint8Array): { decryptedBytes: Uint8Array; isFinalChunk: boolean } | undefined { + const result = this.crypto.xchacha20StreamDecryptorPush(this.decryptor, encryptedBytes, this.file.remoteIdentifier) + + if (result === false) { + return undefined + } + + const isFinal = result.tag === SodiumConstant.CRYPTO_SECRETSTREAM_XCHACHA20POLY1305_TAG_FINAL + + return { decryptedBytes: result.message, isFinalChunk: isFinal } + } +} diff --git a/packages/files/src/Domain/UseCase/FileDownloader.spec.ts b/packages/files/src/Domain/UseCase/FileDownloader.spec.ts new file mode 100644 index 000000000..8e90dabd9 --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileDownloader.spec.ts @@ -0,0 +1,58 @@ +import { FileContent } from '@standardnotes/models' +import { FilesApiInterface } from '@standardnotes/services' +import { FileDownloader } from './FileDownloader' + +describe('file downloader', () => { + let apiService: FilesApiInterface + let downloader: FileDownloader + let file: { + encryptedChunkSizes: FileContent['encryptedChunkSizes'] + remoteIdentifier: FileContent['remoteIdentifier'] + } + + const numChunks = 5 + + beforeEach(() => { + apiService = {} as jest.Mocked + apiService.createFileValetToken = jest.fn() + apiService.downloadFile = jest + .fn() + .mockImplementation( + ( + _file: string, + _chunkIndex: number, + _apiToken: string, + _rangeStart: number, + onBytesReceived: (bytes: Uint8Array) => void, + ) => { + return new Promise((resolve) => { + for (let i = 0; i < numChunks; i++) { + onBytesReceived(Uint8Array.from([0xaa])) + } + + resolve() + }) + }, + ) + + file = { + encryptedChunkSizes: [100_000], + remoteIdentifier: '123', + } + }) + + it('should pass back bytes as they are received', async () => { + let receivedBytes = new Uint8Array() + + downloader = new FileDownloader(file, apiService) + + expect(receivedBytes.length).toBe(0) + + // eslint-disable-next-line @typescript-eslint/require-await + await downloader.run(async (encryptedBytes) => { + receivedBytes = new Uint8Array([...receivedBytes, ...encryptedBytes]) + }) + + expect(receivedBytes.length).toEqual(numChunks) + }) +}) diff --git a/packages/files/src/Domain/UseCase/FileDownloader.ts b/packages/files/src/Domain/UseCase/FileDownloader.ts new file mode 100644 index 000000000..8c60b6f5b --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileDownloader.ts @@ -0,0 +1,83 @@ +import { ClientDisplayableError } from '@standardnotes/responses' +import { FileDownloadProgress } from '../Types/FileDownloadProgress' +import { FilesApiInterface } from '@standardnotes/services' +import { Deferred } from '@standardnotes/utils' +import { FileContent } from '@standardnotes/models' + +export type AbortSignal = 'aborted' +export type AbortFunction = () => void +type OnEncryptedBytes = ( + encryptedBytes: Uint8Array, + progress: FileDownloadProgress, + abort: AbortFunction, +) => Promise + +export type FileDownloaderResult = ClientDisplayableError | AbortSignal | undefined + +export class FileDownloader { + private aborted = false + private abortDeferred = Deferred() + private totalBytesDownloaded = 0 + + constructor( + private file: { + encryptedChunkSizes: FileContent['encryptedChunkSizes'] + remoteIdentifier: FileContent['remoteIdentifier'] + }, + private readonly api: FilesApiInterface, + ) {} + + private getProgress(): FileDownloadProgress { + const encryptedSize = this.file.encryptedChunkSizes.reduce((total, chunk) => total + chunk, 0) + + return { + encryptedFileSize: encryptedSize, + encryptedBytesDownloaded: this.totalBytesDownloaded, + encryptedBytesRemaining: encryptedSize - this.totalBytesDownloaded, + percentComplete: (this.totalBytesDownloaded / encryptedSize) * 100.0, + } + } + + public async run(onEncryptedBytes: OnEncryptedBytes): Promise { + const tokenResult = await this.getValetToken() + + if (tokenResult instanceof ClientDisplayableError) { + return tokenResult + } + + return this.performDownload(tokenResult, onEncryptedBytes) + } + + private async getValetToken(): Promise { + const tokenResult = await this.api.createFileValetToken(this.file.remoteIdentifier, 'read') + + return tokenResult + } + + private async performDownload(valetToken: string, onEncryptedBytes: OnEncryptedBytes): Promise { + const chunkIndex = 0 + const startRange = 0 + + const onRemoteBytesReceived = async (bytes: Uint8Array) => { + if (this.aborted) { + return + } + + this.totalBytesDownloaded += bytes.byteLength + + await onEncryptedBytes(bytes, this.getProgress(), this.abort) + } + + const downloadPromise = this.api.downloadFile(this.file, chunkIndex, valetToken, startRange, onRemoteBytesReceived) + + const result = await Promise.race([this.abortDeferred.promise, downloadPromise]) + + return result + } + + public abort(): void { + this.aborted = true + + this.abortDeferred.resolve('aborted') + } +} diff --git a/packages/files/src/Domain/UseCase/FileEncryptor.spec.ts b/packages/files/src/Domain/UseCase/FileEncryptor.spec.ts new file mode 100644 index 000000000..5d8472b09 --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileEncryptor.spec.ts @@ -0,0 +1,65 @@ +import { FileContent } from '@standardnotes/models' +import { PureCryptoInterface, StreamEncryptor, SodiumConstant } from '@standardnotes/sncrypto-common' +import { FileEncryptor } from './FileEncryptor' + +describe('file encryptor', () => { + let encryptor: FileEncryptor + let file: { key: FileContent['key']; remoteIdentifier: FileContent['remoteIdentifier'] } + let crypto: PureCryptoInterface + + beforeEach(() => { + crypto = {} as jest.Mocked + crypto.xchacha20StreamInitEncryptor = jest.fn().mockReturnValue({ + header: 'some-header', + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamEncryptorPush = jest.fn().mockReturnValue(new Uint8Array()) + + file = { + remoteIdentifier: '123', + key: 'secret', + } + + encryptor = new FileEncryptor(file, crypto) + }) + + it('should initialize header', () => { + const header = encryptor.initializeHeader() + + expect(header.length).toBeGreaterThan(0) + }) + + it('pushBytes should return encrypted bytes', () => { + encryptor.initializeHeader() + const encryptedBytes = encryptor.pushBytes(new Uint8Array(), false) + + expect(encryptedBytes).toBeInstanceOf(Uint8Array) + }) + + it('pushBytes with last chunk should pass final tag', () => { + encryptor.initializeHeader() + const decryptedBytes = new Uint8Array() + encryptor.pushBytes(decryptedBytes, true) + + expect(crypto.xchacha20StreamEncryptorPush).toHaveBeenCalledWith( + expect.any(Object), + decryptedBytes, + file.remoteIdentifier, + SodiumConstant.CRYPTO_SECRETSTREAM_XCHACHA20POLY1305_TAG_FINAL, + ) + }) + + it('pushBytes with not last chunk should not pass final tag', () => { + encryptor.initializeHeader() + const decryptedBytes = new Uint8Array() + encryptor.pushBytes(decryptedBytes, false) + + expect(crypto.xchacha20StreamEncryptorPush).toHaveBeenCalledWith( + expect.any(Object), + decryptedBytes, + file.remoteIdentifier, + undefined, + ) + }) +}) diff --git a/packages/files/src/Domain/UseCase/FileEncryptor.ts b/packages/files/src/Domain/UseCase/FileEncryptor.ts new file mode 100644 index 000000000..9906bf724 --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileEncryptor.ts @@ -0,0 +1,34 @@ +import { FileContent } from '@standardnotes/models' +import { PureCryptoInterface, StreamEncryptor, SodiumConstant } from '@standardnotes/sncrypto-common' + +export class FileEncryptor { + private stream!: StreamEncryptor + + constructor( + private readonly file: { key: FileContent['key']; remoteIdentifier: FileContent['remoteIdentifier'] }, + private crypto: PureCryptoInterface, + ) {} + + public initializeHeader(): string { + this.stream = this.crypto.xchacha20StreamInitEncryptor(this.file.key) + + return this.stream.header + } + + public pushBytes(decryptedBytes: Uint8Array, isFinalChunk: boolean): Uint8Array { + if (!this.stream) { + throw new Error('FileEncryptor must call initializeHeader first') + } + + const tag = isFinalChunk ? SodiumConstant.CRYPTO_SECRETSTREAM_XCHACHA20POLY1305_TAG_FINAL : undefined + + const encryptedBytes = this.crypto.xchacha20StreamEncryptorPush( + this.stream, + decryptedBytes, + this.file.remoteIdentifier, + tag, + ) + + return encryptedBytes + } +} diff --git a/packages/files/src/Domain/UseCase/FileUploader.spec.ts b/packages/files/src/Domain/UseCase/FileUploader.spec.ts new file mode 100644 index 000000000..54e3102b5 --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileUploader.spec.ts @@ -0,0 +1,21 @@ +import { FilesApiInterface } from '@standardnotes/services' +import { FileUploader } from './FileUploader' + +describe('file uploader', () => { + let apiService + let uploader: FileUploader + + beforeEach(() => { + apiService = {} as jest.Mocked + apiService.uploadFileBytes = jest.fn().mockReturnValue(true) + + uploader = new FileUploader(apiService) + }) + + it('should return true when a chunk is uploaded', async () => { + const bytes = new Uint8Array() + const success = await uploader.uploadBytes(bytes, 2, 'api-token') + + expect(success).toEqual(true) + }) +}) diff --git a/packages/files/src/Domain/UseCase/FileUploader.ts b/packages/files/src/Domain/UseCase/FileUploader.ts new file mode 100644 index 000000000..a6f0dc1c9 --- /dev/null +++ b/packages/files/src/Domain/UseCase/FileUploader.ts @@ -0,0 +1,11 @@ +import { FilesApiInterface } from '@standardnotes/services' + +export class FileUploader { + constructor(private apiService: FilesApiInterface) {} + + public async uploadBytes(encryptedBytes: Uint8Array, chunkId: number, apiToken: string): Promise { + const result = await this.apiService.uploadFileBytes(apiToken, chunkId, encryptedBytes) + + return result + } +} diff --git a/packages/files/src/Domain/index.ts b/packages/files/src/Domain/index.ts new file mode 100644 index 000000000..e0cffa120 --- /dev/null +++ b/packages/files/src/Domain/index.ts @@ -0,0 +1,12 @@ +export * from './Service/FileService' +export * from './Service/FilesClientInterface' +export * from './Operations/DownloadAndDecrypt' +export * from './Operations/EncryptAndUpload' +export * from './UseCase/FileDecryptor' +export * from './UseCase/FileUploader' +export * from './UseCase/FileEncryptor' +export * from './UseCase/FileDownloader' +export * from './Types/FileDownloadProgress' +export * from './Types/FileUploadProgress' +export * from './Types/FileUploadResult' +export * from './Backups/BackupService' diff --git a/packages/files/src/index.ts b/packages/files/src/index.ts new file mode 100644 index 000000000..920deacdb --- /dev/null +++ b/packages/files/src/index.ts @@ -0,0 +1 @@ +export * from './Domain' diff --git a/packages/files/tsconfig.json b/packages/files/tsconfig.json new file mode 100644 index 000000000..f3dac14ef --- /dev/null +++ b/packages/files/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../node_modules/@standardnotes/config/src/tsconfig.json", + "compilerOptions": { + "skipLibCheck": true, + "rootDir": "./src", + "outDir": "./dist", + }, + "include": [ + "src/**/*" + ], + "references": [], + "exclude": ["**/*.spec.ts", "dist", "node_modules"] +} diff --git a/yarn.lock b/yarn.lock index 9433fedea..f08e58925 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6599,7 +6599,7 @@ __metadata: languageName: unknown linkType: soft -"@standardnotes/filepicker@^1.16.22, @standardnotes/filepicker@^1.16.23, @standardnotes/filepicker@workspace:*, @standardnotes/filepicker@workspace:packages/filepicker": +"@standardnotes/filepicker@workspace:*, @standardnotes/filepicker@workspace:packages/filepicker": version: 0.0.0-use.local resolution: "@standardnotes/filepicker@workspace:packages/filepicker" dependencies: @@ -6617,33 +6617,26 @@ __metadata: languageName: unknown linkType: soft -"@standardnotes/files@npm:^1.3.22": - version: 1.3.22 - resolution: "@standardnotes/files@npm:1.3.22" +"@standardnotes/files@^1.3.22, @standardnotes/files@^1.3.23, @standardnotes/files@workspace:packages/files": + version: 0.0.0-use.local + resolution: "@standardnotes/files@workspace:packages/files" dependencies: - "@standardnotes/encryption": ^1.8.22 - "@standardnotes/filepicker": ^1.16.22 - "@standardnotes/models": ^1.11.12 - "@standardnotes/responses": ^1.6.38 - "@standardnotes/services": ^1.13.22 - "@standardnotes/utils": ^1.6.12 - checksum: 4bd58c1aedf21892dceee7554b2f8003bca8aaa651079b0aad31bc57f160fadc455010241da1544e488c0f3c08e57549d610908b2c2f57b1d8c0991204ff2fb1 - languageName: node - linkType: hard - -"@standardnotes/files@npm:^1.3.23": - version: 1.3.23 - resolution: "@standardnotes/files@npm:1.3.23" - dependencies: - "@standardnotes/encryption": ^1.8.23 - "@standardnotes/filepicker": ^1.16.23 + "@standardnotes/common": ^1.23.1 + "@standardnotes/encryption": "workspace:*" + "@standardnotes/filepicker": "workspace:*" "@standardnotes/models": ^1.11.13 "@standardnotes/responses": ^1.6.39 "@standardnotes/services": ^1.13.23 + "@standardnotes/sncrypto-common": ^1.9.0 "@standardnotes/utils": ^1.6.12 - checksum: 923dbd892ebfe015f19a1fab0a46f0c00edecd89d92873be121fcb3b3ca71b0e9a6bd2bb69b421a5591adc7df19930ceb116f56be3dd45eca708cf0f05e1d05b - languageName: node - linkType: hard + "@types/jest": ^27.4.1 + "@typescript-eslint/eslint-plugin": ^5.30.0 + eslint-plugin-prettier: ^4.2.1 + jest: ^27.5.1 + reflect-metadata: ^0.1.13 + ts-jest: ^27.1.3 + languageName: unknown + linkType: soft "@standardnotes/filesafe-bar@workspace:packages/components/src/Packages/Deprecated/org.standardnotes.file-safe": version: 0.0.0-use.local