From 28e43d37c00c5cd97129c4b83266f7a234bd9857 Mon Sep 17 00:00:00 2001 From: Mo Date: Thu, 1 Dec 2022 11:56:28 -0600 Subject: [PATCH] feat: download and preview files from local backups automatically, if a local backup is available (#2076) --- .../Main/FileBackups/FileBackupsManager.ts | 44 ++++++- .../Main/FileBackups/FileReadOperation.ts | 58 ++++++++++ .../javascripts/Main/Remote/RemoteBridge.ts | 18 ++- .../app/javascripts/Main/Utils/FileUtils.ts | 35 +++--- .../app/javascripts/Renderer/DesktopDevice.ts | 10 ++ .../filepicker/src/Classic/ClassicReader.ts | 4 +- .../filepicker/src/Interface/FileReader.ts | 4 +- .../src/Streaming/StreamingReader.ts | 4 +- .../src/Domain/Chunker/ByteChunker.spec.ts | 16 +-- .../files/src/Domain/Chunker/ByteChunker.ts | 10 +- .../src/Domain/Chunker/OnChunkCallback.ts | 11 +- .../Domain/Chunker/OrderedByteChunker.spec.ts | 25 +++- .../src/Domain/Chunker/OrderedByteChunker.ts | 30 ++++- .../src/Domain/Device/FileBackupsDevice.ts | 6 + packages/files/src/Domain/Logging.ts | 30 +++++ .../Domain/Service/BackupServiceInterface.ts | 14 +++ .../Domain/Service/FilesClientInterface.ts | 2 +- ... ReadAndDecryptBackupFileFileSystemAPI.ts} | 6 +- ...dAndDecryptBackupFileUsingBackupService.ts | 52 +++++++++ .../src/Domain/Types/FileDownloadProgress.ts | 20 ++++ .../src/Domain/UseCase/FileDownloader.ts | 1 + packages/files/src/Domain/index.ts | 4 +- packages/services/package.json | 2 +- .../src/Domain/Backups/BackupService.spec.ts | 109 ++++++++++++++++++ .../src/Domain/Backups/BackupService.ts | 43 ++++++- .../src/Domain/Files/FileService.spec.ts | 70 ++++++++++- .../services/src/Domain/Files/FileService.ts | 87 ++++++++++---- packages/services/src/Domain/Logging.ts | 33 ++++++ packages/snjs/lib/Application/Application.ts | 17 +-- packages/utils/src/Domain/Utils/Utils.ts | 6 +- .../Application/Device/DesktopSnjsExports.ts | 3 + .../Components/FilePreview/FilePreview.tsx | 36 ++++-- .../Backups/Files/FileBackupsDesktop.tsx | 4 +- .../Controllers/FilesController.ts | 35 +++--- 34 files changed, 739 insertions(+), 110 deletions(-) create mode 100644 packages/desktop/app/javascripts/Main/FileBackups/FileReadOperation.ts create mode 100644 packages/files/src/Domain/Logging.ts create mode 100644 packages/files/src/Domain/Service/BackupServiceInterface.ts rename packages/files/src/Domain/Service/{ReadAndDecryptBackupFile.ts => ReadAndDecryptBackupFileFileSystemAPI.ts} (87%) create mode 100644 packages/files/src/Domain/Service/ReadAndDecryptBackupFileUsingBackupService.ts create mode 100644 packages/services/src/Domain/Backups/BackupService.spec.ts create mode 100644 packages/services/src/Domain/Logging.ts diff --git a/packages/desktop/app/javascripts/Main/FileBackups/FileBackupsManager.ts b/packages/desktop/app/javascripts/Main/FileBackups/FileBackupsManager.ts index 311c75ad7..5bd66fad5 100644 --- a/packages/desktop/app/javascripts/Main/FileBackups/FileBackupsManager.ts +++ b/packages/desktop/app/javascripts/Main/FileBackups/FileBackupsManager.ts @@ -1,4 +1,10 @@ -import { FileBackupRecord, FileBackupsDevice, FileBackupsMapping } from '@web/Application/Device/DesktopSnjsExports' +import { + FileBackupRecord, + FileBackupsDevice, + FileBackupsMapping, + FileBackupReadToken, + FileBackupReadChunkResponse, +} from '@web/Application/Device/DesktopSnjsExports' import { AppState } from 'app/AppState' import { shell } from 'electron' import { StoreKeys } from '../Store/StoreKeys' @@ -6,13 +12,14 @@ import path from 'path' import { deleteFile, ensureDirectoryExists, - moveFiles, + moveDirContents, openDirectoryPicker, readJSONFile, writeFile, writeJSONFile, } from '../Utils/FileUtils' import { FileDownloader } from './FileDownloader' +import { FileReadOperation } from './FileReadOperation' export const FileBackupsConstantsV1 = { Version: '1.0.0', @@ -21,6 +28,8 @@ export const FileBackupsConstantsV1 = { } export class FilesBackupManager implements FileBackupsDevice { + private readOperations: Map = new Map() + constructor(private appState: AppState) {} public isFilesBackupsEnabled(): Promise { @@ -78,8 +87,11 @@ export class FilesBackupManager implements FileBackupsDevice { } const entries = Object.values(mapping.files) - const itemFolders = entries.map((entry) => path.join(oldPath, entry.relativePath)) - await moveFiles(itemFolders, newPath) + for (const entry of entries) { + const sourcePath = path.join(oldPath, entry.relativePath) + const destinationPath = path.join(newPath, entry.relativePath) + await moveDirContents(sourcePath, destinationPath) + } for (const entry of entries) { entry.absolutePath = path.join(newPath, entry.relativePath) @@ -188,4 +200,28 @@ export class FilesBackupManager implements FileBackupsDevice { return result } + + async getFileBackupReadToken(record: FileBackupRecord): Promise { + const operation = new FileReadOperation(record) + + this.readOperations.set(operation.token, operation) + + return operation.token + } + + async readNextChunk(token: string): Promise { + const operation = this.readOperations.get(token) + + if (!operation) { + return Promise.reject(new Error('Invalid token')) + } + + const result = await operation.readNextChunk() + + if (result.isLast) { + this.readOperations.delete(token) + } + + return result + } } diff --git a/packages/desktop/app/javascripts/Main/FileBackups/FileReadOperation.ts b/packages/desktop/app/javascripts/Main/FileBackups/FileReadOperation.ts new file mode 100644 index 000000000..9f7919db0 --- /dev/null +++ b/packages/desktop/app/javascripts/Main/FileBackups/FileReadOperation.ts @@ -0,0 +1,58 @@ +import { FileBackupReadChunkResponse, FileBackupRecord } from '@web/Application/Device/DesktopSnjsExports' +import fs from 'fs' +import path from 'path' + +const ONE_MB = 1024 * 1024 +const CHUNK_LIMIT = ONE_MB * 5 + +export class FileReadOperation { + public readonly token: string + private currentChunkLocation = 0 + private localFileId: number + private fileLength: number + + constructor(backupRecord: FileBackupRecord) { + this.token = backupRecord.absolutePath + this.localFileId = fs.openSync(path.join(backupRecord.absolutePath, backupRecord.binaryFileName), 'r') + this.fileLength = fs.fstatSync(this.localFileId).size + } + + async readNextChunk(): Promise { + let isLast = false + let readUpto = this.currentChunkLocation + CHUNK_LIMIT + if (readUpto > this.fileLength) { + readUpto = this.fileLength + isLast = true + } + + const readLength = readUpto - this.currentChunkLocation + + const chunk = await this.readChunk(this.currentChunkLocation, readLength) + + this.currentChunkLocation = readUpto + + if (isLast) { + fs.close(this.localFileId) + } + + return { + chunk, + isLast, + progress: { + encryptedFileSize: this.fileLength, + encryptedBytesDownloaded: this.currentChunkLocation, + encryptedBytesRemaining: this.fileLength - this.currentChunkLocation, + percentComplete: (this.currentChunkLocation / this.fileLength) * 100.0, + source: 'local', + }, + } + } + + async readChunk(start: number, length: number): Promise { + const buffer = Buffer.alloc(length) + + fs.readSync(this.localFileId, buffer, 0, length, start) + + return buffer + } +} diff --git a/packages/desktop/app/javascripts/Main/Remote/RemoteBridge.ts b/packages/desktop/app/javascripts/Main/Remote/RemoteBridge.ts index 8a0e89eb6..a563aa472 100644 --- a/packages/desktop/app/javascripts/Main/Remote/RemoteBridge.ts +++ b/packages/desktop/app/javascripts/Main/Remote/RemoteBridge.ts @@ -5,7 +5,13 @@ import { StoreKeys } from '../Store/StoreKeys' const path = require('path') const rendererPath = path.join('file://', __dirname, '/renderer.js') -import { FileBackupsDevice, FileBackupsMapping, FileBackupRecord } from '@web/Application/Device/DesktopSnjsExports' +import { + FileBackupsDevice, + FileBackupsMapping, + FileBackupRecord, + FileBackupReadToken, + FileBackupReadChunkResponse, +} from '@web/Application/Device/DesktopSnjsExports' import { app, BrowserWindow } from 'electron' import { BackupsManagerInterface } from '../Backups/BackupsManagerInterface' import { KeychainInterface } from '../Keychain/KeychainInterface' @@ -65,6 +71,8 @@ export class RemoteBridge implements CrossProcessBridge { getFilesBackupsLocation: this.getFilesBackupsLocation.bind(this), openFilesBackupsLocation: this.openFilesBackupsLocation.bind(this), openFileBackup: this.openFileBackup.bind(this), + getFileBackupReadToken: this.getFileBackupReadToken.bind(this), + readNextChunk: this.readNextChunk.bind(this), } } @@ -180,6 +188,14 @@ export class RemoteBridge implements CrossProcessBridge { return this.fileBackups.saveFilesBackupsFile(uuid, metaFile, downloadRequest) } + getFileBackupReadToken(record: FileBackupRecord): Promise { + return this.fileBackups.getFileBackupReadToken(record) + } + + readNextChunk(nextToken: string): Promise { + return this.fileBackups.readNextChunk(nextToken) + } + public isFilesBackupsEnabled(): Promise { return this.fileBackups.isFilesBackupsEnabled() } diff --git a/packages/desktop/app/javascripts/Main/Utils/FileUtils.ts b/packages/desktop/app/javascripts/Main/Utils/FileUtils.ts index e0e2cec9d..7f2a67215 100644 --- a/packages/desktop/app/javascripts/Main/Utils/FileUtils.ts +++ b/packages/desktop/app/javascripts/Main/Utils/FileUtils.ts @@ -7,7 +7,6 @@ import { removeFromArray } from '../Utils/Utils' export const FileDoesNotExist = 'ENOENT' export const FileAlreadyExists = 'EEXIST' -const CrossDeviceLink = 'EXDEV' const OperationNotPermitted = 'EPERM' const DeviceIsBusy = 'EBUSY' @@ -152,8 +151,14 @@ function isChildOfDir(parent: string, potentialChild: string) { return relative && !relative.startsWith('..') && !path.isAbsolute(relative) } -export async function moveDirContents(srcDir: string, destDir: string): Promise { - let fileNames = await fs.promises.readdir(srcDir) +export async function moveDirContents(srcDir: string, destDir: string): Promise { + let fileNames: string[] + try { + fileNames = await fs.promises.readdir(srcDir) + } catch (error) { + console.error(error) + return + } await ensureDirectoryExists(destDir) if (isChildOfDir(srcDir, destDir)) { @@ -163,10 +168,14 @@ export async function moveDirContents(srcDir: string, destDir: string): Promise< removeFromArray(fileNames, path.basename(destDir)) } - return moveFiles( - fileNames.map((fileName) => path.join(srcDir, fileName)), - destDir, - ) + try { + await moveFiles( + fileNames.map((fileName) => path.join(srcDir, fileName)), + destDir, + ) + } catch (error) { + console.error(error) + } } export async function extractZip(source: string, dest: string): Promise { @@ -245,14 +254,10 @@ export async function moveFiles(sources: string[], destDir: string): Promise { + return this.remoteBridge.getFileBackupReadToken(record) + } + + readNextChunk(token: string): Promise { + return this.remoteBridge.readNextChunk(token) + } + async performHardReset(): Promise { console.error('performHardReset is not yet implemented') } diff --git a/packages/filepicker/src/Classic/ClassicReader.ts b/packages/filepicker/src/Classic/ClassicReader.ts index 635d02def..ae2dc365e 100644 --- a/packages/filepicker/src/Classic/ClassicReader.ts +++ b/packages/filepicker/src/Classic/ClassicReader.ts @@ -1,4 +1,4 @@ -import { ByteChunker, OnChunkCallback } from '@standardnotes/files' +import { ByteChunker, OnChunkCallbackNoProgress } from '@standardnotes/files' import { FileSelectionResponse } from '../types' import { readFile as utilsReadFile } from '../utils' import { FileReaderInterface } from '../Interface/FileReader' @@ -39,7 +39,7 @@ function selectFiles(): Promise { async function readFile( file: File, minimumChunkSize: number, - onChunk: OnChunkCallback, + onChunk: OnChunkCallbackNoProgress, ): Promise { const buffer = await utilsReadFile(file) const chunker = new ByteChunker(minimumChunkSize, onChunk) diff --git a/packages/filepicker/src/Interface/FileReader.ts b/packages/filepicker/src/Interface/FileReader.ts index 8ba8f6023..6dc5d15ad 100644 --- a/packages/filepicker/src/Interface/FileReader.ts +++ b/packages/filepicker/src/Interface/FileReader.ts @@ -1,11 +1,11 @@ -import { OnChunkCallback } from '@standardnotes/files' +import { OnChunkCallbackNoProgress } from '@standardnotes/files' import { FileSelectionResponse } from '../types' export interface FileReaderInterface { selectFiles(): Promise - readFile(file: File, minimumChunkSize: number, onChunk: OnChunkCallback): Promise + readFile(file: File, minimumChunkSize: number, onChunk: OnChunkCallbackNoProgress): Promise available(): boolean diff --git a/packages/filepicker/src/Streaming/StreamingReader.ts b/packages/filepicker/src/Streaming/StreamingReader.ts index f90e0455f..0207b86c4 100644 --- a/packages/filepicker/src/Streaming/StreamingReader.ts +++ b/packages/filepicker/src/Streaming/StreamingReader.ts @@ -1,4 +1,4 @@ -import { ByteChunker, OnChunkCallback } from '@standardnotes/files' +import { ByteChunker, OnChunkCallbackNoProgress } from '@standardnotes/files' import { FileReaderInterface } from './../Interface/FileReader' import { FileSelectionResponse } from '../types' @@ -39,7 +39,7 @@ async function selectFiles(): Promise { async function readFile( file: File, minimumChunkSize: number, - onChunk: OnChunkCallback, + onChunk: OnChunkCallbackNoProgress, ): Promise { const byteChunker = new ByteChunker(minimumChunkSize, onChunk) const stream = file.stream() as unknown as ReadableStream diff --git a/packages/files/src/Domain/Chunker/ByteChunker.spec.ts b/packages/files/src/Domain/Chunker/ByteChunker.spec.ts index cdbd60bcd..a2ba7ef73 100644 --- a/packages/files/src/Domain/Chunker/ByteChunker.spec.ts +++ b/packages/files/src/Domain/Chunker/ByteChunker.spec.ts @@ -8,9 +8,9 @@ describe('byte chunker', () => { it('should hold back small chunks until minimum size is met', async () => { let receivedBytes = new Uint8Array() let numChunks = 0 - const chunker = new ByteChunker(100, async (bytes) => { + const chunker = new ByteChunker(100, async (chunk) => { numChunks++ - receivedBytes = new Uint8Array([...receivedBytes, ...bytes]) + receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data]) }) await chunker.addBytes(chunkOfSize(50), false) @@ -25,9 +25,9 @@ describe('byte chunker', () => { it('should send back big chunks immediately', async () => { let receivedBytes = new Uint8Array() let numChunks = 0 - const chunker = new ByteChunker(100, async (bytes) => { + const chunker = new ByteChunker(100, async (chunk) => { numChunks++ - receivedBytes = new Uint8Array([...receivedBytes, ...bytes]) + receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data]) }) await chunker.addBytes(chunkOfSize(150), false) @@ -42,9 +42,9 @@ describe('byte chunker', () => { it('last chunk should be popped regardless of size', async () => { let receivedBytes = new Uint8Array() let numChunks = 0 - const chunker = new ByteChunker(100, async (bytes) => { + const chunker = new ByteChunker(100, async (chunk) => { numChunks++ - receivedBytes = new Uint8Array([...receivedBytes, ...bytes]) + receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data]) }) await chunker.addBytes(chunkOfSize(50), false) @@ -57,9 +57,9 @@ describe('byte chunker', () => { it('single chunk should be popped immediately', async () => { let receivedBytes = new Uint8Array() let numChunks = 0 - const chunker = new ByteChunker(100, async (bytes) => { + const chunker = new ByteChunker(100, async (chunk) => { numChunks++ - receivedBytes = new Uint8Array([...receivedBytes, ...bytes]) + receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data]) }) await chunker.addBytes(chunkOfSize(50), true) diff --git a/packages/files/src/Domain/Chunker/ByteChunker.ts b/packages/files/src/Domain/Chunker/ByteChunker.ts index 55deb700e..5ce675d27 100644 --- a/packages/files/src/Domain/Chunker/ByteChunker.ts +++ b/packages/files/src/Domain/Chunker/ByteChunker.ts @@ -1,11 +1,11 @@ -import { OnChunkCallback } from './OnChunkCallback' +import { OnChunkCallbackNoProgress } from './OnChunkCallback' export class ByteChunker { public loggingEnabled = false private bytes = new Uint8Array() private index = 1 - constructor(private minimumChunkSize: number, private onChunk: OnChunkCallback) {} + constructor(private minimumChunkSize: number, private onChunk: OnChunkCallbackNoProgress) {} private log(...args: any[]): void { if (!this.loggingEnabled) { @@ -27,9 +27,13 @@ export class ByteChunker { private async popBytes(isLast: boolean): Promise { const maxIndex = Math.max(this.minimumChunkSize, this.bytes.length) + const chunk = this.bytes.slice(0, maxIndex) + this.bytes = new Uint8Array([...this.bytes.slice(maxIndex)]) + this.log(`Chunker popping ${chunk.length}, total size in queue ${this.bytes.length}`) - await this.onChunk(chunk, this.index++, isLast) + + await this.onChunk({ data: chunk, index: this.index++, isLast }) } } diff --git a/packages/files/src/Domain/Chunker/OnChunkCallback.ts b/packages/files/src/Domain/Chunker/OnChunkCallback.ts index 9cac24d39..20d51c96e 100644 --- a/packages/files/src/Domain/Chunker/OnChunkCallback.ts +++ b/packages/files/src/Domain/Chunker/OnChunkCallback.ts @@ -1 +1,10 @@ -export type OnChunkCallback = (chunk: Uint8Array, index: number, isLast: boolean) => Promise +import { FileDownloadProgress } from '../Types/FileDownloadProgress' + +export type OnChunkCallback = (chunk: { + data: Uint8Array + index: number + isLast: boolean + progress: FileDownloadProgress +}) => Promise + +export type OnChunkCallbackNoProgress = (chunk: { data: Uint8Array; index: number; isLast: boolean }) => Promise diff --git a/packages/files/src/Domain/Chunker/OrderedByteChunker.spec.ts b/packages/files/src/Domain/Chunker/OrderedByteChunker.spec.ts index a51a64e92..e0b4296cc 100644 --- a/packages/files/src/Domain/Chunker/OrderedByteChunker.spec.ts +++ b/packages/files/src/Domain/Chunker/OrderedByteChunker.spec.ts @@ -10,9 +10,30 @@ describe('ordered byte chunker', () => { let receivedBytes = new Uint8Array() let numCallbacks = 0 - const chunker = new OrderedByteChunker(chunkSizes, async (bytes) => { + const chunker = new OrderedByteChunker(chunkSizes, 'network', async (chunk) => { numCallbacks++ - receivedBytes = new Uint8Array([...receivedBytes, ...bytes]) + receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data]) + }) + + await chunker.addBytes(chunkOfSize(30)) + + expect(numCallbacks).toEqual(3) + expect(receivedBytes.length).toEqual(30) + }) + + it('should correctly report progress', async () => { + const chunkSizes = [10, 10, 10] + let receivedBytes = new Uint8Array() + let numCallbacks = 0 + + const chunker = new OrderedByteChunker(chunkSizes, 'network', async (chunk) => { + numCallbacks++ + + receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data]) + + expect(chunk.progress.encryptedBytesDownloaded).toEqual(receivedBytes.length) + + expect(chunk.progress.percentComplete).toEqual((numCallbacks / chunkSizes.length) * 100.0) }) await chunker.addBytes(chunkOfSize(30)) diff --git a/packages/files/src/Domain/Chunker/OrderedByteChunker.ts b/packages/files/src/Domain/Chunker/OrderedByteChunker.ts index f7ad36e2f..d67845a4b 100644 --- a/packages/files/src/Domain/Chunker/OrderedByteChunker.ts +++ b/packages/files/src/Domain/Chunker/OrderedByteChunker.ts @@ -1,13 +1,28 @@ +import { FileDownloadProgress } from '../Types/FileDownloadProgress' +import { OnChunkCallback } from './OnChunkCallback' + export class OrderedByteChunker { private bytes = new Uint8Array() private index = 1 private remainingChunks: number[] = [] + private fileSize: number constructor( private chunkSizes: number[], - private onChunk: (chunk: Uint8Array, index: number, isLast: boolean) => Promise, + private source: FileDownloadProgress['source'], + private onChunk: OnChunkCallback, ) { this.remainingChunks = chunkSizes.slice() + + this.fileSize = chunkSizes.reduce((acc, size) => acc + size, 0) + } + + private get bytesPopped(): number { + return this.fileSize - this.bytesRemaining + } + + private get bytesRemaining(): number { + return this.remainingChunks.reduce((acc, size) => acc + size, 0) } private needsPop(): boolean { @@ -31,7 +46,18 @@ export class OrderedByteChunker { this.remainingChunks.shift() - await this.onChunk(chunk, this.index++, this.index === this.chunkSizes.length - 1) + await this.onChunk({ + data: chunk, + index: this.index++, + isLast: this.index === this.chunkSizes.length - 1, + progress: { + encryptedFileSize: this.fileSize, + encryptedBytesDownloaded: this.bytesPopped, + encryptedBytesRemaining: this.bytesRemaining, + percentComplete: (this.bytesPopped / this.fileSize) * 100.0, + source: this.source, + }, + }) if (this.needsPop()) { await this.popBytes() diff --git a/packages/files/src/Domain/Device/FileBackupsDevice.ts b/packages/files/src/Domain/Device/FileBackupsDevice.ts index 2149205b5..b192d60cc 100644 --- a/packages/files/src/Domain/Device/FileBackupsDevice.ts +++ b/packages/files/src/Domain/Device/FileBackupsDevice.ts @@ -1,6 +1,10 @@ import { Uuid } from '@standardnotes/common' +import { FileDownloadProgress } from '../Types/FileDownloadProgress' import { FileBackupRecord, FileBackupsMapping } from './FileBackupsMapping' +export type FileBackupReadToken = string +export type FileBackupReadChunkResponse = { chunk: Uint8Array; isLast: boolean; progress: FileDownloadProgress } + export interface FileBackupsDevice { getFilesBackupsMappingFile(): Promise saveFilesBackupsFile( @@ -12,6 +16,8 @@ export interface FileBackupsDevice { url: string }, ): Promise<'success' | 'failed'> + getFileBackupReadToken(record: FileBackupRecord): Promise + readNextChunk(token: string): Promise isFilesBackupsEnabled(): Promise enableFilesBackups(): Promise disableFilesBackups(): Promise diff --git a/packages/files/src/Domain/Logging.ts b/packages/files/src/Domain/Logging.ts new file mode 100644 index 000000000..0a93ff922 --- /dev/null +++ b/packages/files/src/Domain/Logging.ts @@ -0,0 +1,30 @@ +import { logWithColor } from '@standardnotes/utils' + +declare const process: { + env: { + NODE_ENV: string | null | undefined + } +} + +export const isDev = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test' + +export enum LoggingDomain { + FilesPackage, +} + +const LoggingStatus: Record = { + [LoggingDomain.FilesPackage]: false, +} + +const DomainColor: Record = { + [LoggingDomain.FilesPackage]: 'green', +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function log(domain: LoggingDomain, ...args: any[]): void { + if (!isDev || !LoggingStatus[domain]) { + return + } + + logWithColor(LoggingDomain[domain], DomainColor[domain], ...args) +} diff --git a/packages/files/src/Domain/Service/BackupServiceInterface.ts b/packages/files/src/Domain/Service/BackupServiceInterface.ts new file mode 100644 index 000000000..0ca310e3e --- /dev/null +++ b/packages/files/src/Domain/Service/BackupServiceInterface.ts @@ -0,0 +1,14 @@ +import { OnChunkCallback } from '../Chunker/OnChunkCallback' +import { FileBackupRecord } from '../Device/FileBackupsMapping' + +export interface BackupServiceInterface { + getFileBackupInfo(file: { uuid: string }): Promise + readEncryptedFileFromBackup(uuid: string, onChunk: OnChunkCallback): Promise<'success' | 'failed' | 'aborted'> + isFilesBackupsEnabled(): Promise + enableFilesBackups(): Promise + disableFilesBackups(): Promise + changeFilesBackupsLocation(): Promise + getFilesBackupsLocation(): Promise + openFilesBackupsLocation(): Promise + openFileBackup(record: FileBackupRecord): Promise +} diff --git a/packages/files/src/Domain/Service/FilesClientInterface.ts b/packages/files/src/Domain/Service/FilesClientInterface.ts index 6c7a157f1..99c41f104 100644 --- a/packages/files/src/Domain/Service/FilesClientInterface.ts +++ b/packages/files/src/Domain/Service/FilesClientInterface.ts @@ -24,7 +24,7 @@ export interface FilesClientInterface { downloadFile( file: FileItem, - onDecryptedBytes: (bytes: Uint8Array, progress: FileDownloadProgress | undefined) => Promise, + onDecryptedBytes: (bytes: Uint8Array, progress: FileDownloadProgress) => Promise, ): Promise deleteFile(file: FileItem): Promise diff --git a/packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts b/packages/files/src/Domain/Service/ReadAndDecryptBackupFileFileSystemAPI.ts similarity index 87% rename from packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts rename to packages/files/src/Domain/Service/ReadAndDecryptBackupFileFileSystemAPI.ts index 3a1e1f39b..78593d56a 100644 --- a/packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts +++ b/packages/files/src/Domain/Service/ReadAndDecryptBackupFileFileSystemAPI.ts @@ -5,7 +5,7 @@ import { FileSystemApi } from '../Api/FileSystemApi' import { FileHandleRead } from '../Api/FileHandleRead' import { OrderedByteChunker } from '../Chunker/OrderedByteChunker' -export async function readAndDecryptBackupFile( +export async function readAndDecryptBackupFileUsingFileSystemAPI( fileHandle: FileHandleRead, file: { encryptionHeader: FileContent['encryptionHeader'] @@ -19,8 +19,8 @@ export async function readAndDecryptBackupFile( ): Promise<'aborted' | 'failed' | 'success'> { const decryptor = new FileDecryptor(file, crypto) - const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (chunk: Uint8Array) => { - const decryptResult = decryptor.decryptBytes(chunk) + const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, 'local', async (chunk) => { + const decryptResult = decryptor.decryptBytes(chunk.data) if (!decryptResult) { return diff --git a/packages/files/src/Domain/Service/ReadAndDecryptBackupFileUsingBackupService.ts b/packages/files/src/Domain/Service/ReadAndDecryptBackupFileUsingBackupService.ts new file mode 100644 index 000000000..66a812461 --- /dev/null +++ b/packages/files/src/Domain/Service/ReadAndDecryptBackupFileUsingBackupService.ts @@ -0,0 +1,52 @@ +import { FileContent } from '@standardnotes/models' +import { PureCryptoInterface } from '@standardnotes/sncrypto-common' +import { FileDecryptor } from '../UseCase/FileDecryptor' +import { OrderedByteChunker } from '../Chunker/OrderedByteChunker' +import { BackupServiceInterface } from './BackupServiceInterface' +import { OnChunkCallback } from '../Chunker/OnChunkCallback' +import { log, LoggingDomain } from '../Logging' + +export async function readAndDecryptBackupFileUsingBackupService( + file: { + uuid: string + encryptionHeader: FileContent['encryptionHeader'] + remoteIdentifier: FileContent['remoteIdentifier'] + encryptedChunkSizes: FileContent['encryptedChunkSizes'] + key: FileContent['key'] + }, + backupService: BackupServiceInterface, + crypto: PureCryptoInterface, + onDecryptedBytes: OnChunkCallback, +): Promise<'aborted' | 'failed' | 'success'> { + log( + LoggingDomain.FilesPackage, + 'Reading and decrypting backup file', + file.uuid, + 'chunk sizes', + file.encryptedChunkSizes, + ) + + const decryptor = new FileDecryptor(file, crypto) + + const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, 'local', async (chunk) => { + log(LoggingDomain.FilesPackage, 'OrderedByteChunker did pop bytes', chunk.data.length, chunk.progress) + + const decryptResult = decryptor.decryptBytes(chunk.data) + + if (!decryptResult) { + return + } + + await onDecryptedBytes({ ...chunk, data: decryptResult.decryptedBytes }) + }) + + const readResult = await backupService.readEncryptedFileFromBackup(file.uuid, async (chunk) => { + log(LoggingDomain.FilesPackage, 'Got file chunk from backup service', chunk.data.length, chunk.progress) + + await byteChunker.addBytes(chunk.data) + }) + + log(LoggingDomain.FilesPackage, 'Finished reading and decrypting backup file', file.uuid) + + return readResult +} diff --git a/packages/files/src/Domain/Types/FileDownloadProgress.ts b/packages/files/src/Domain/Types/FileDownloadProgress.ts index eac0067fb..283c3b698 100644 --- a/packages/files/src/Domain/Types/FileDownloadProgress.ts +++ b/packages/files/src/Domain/Types/FileDownloadProgress.ts @@ -3,4 +3,24 @@ export type FileDownloadProgress = { encryptedBytesDownloaded: number encryptedBytesRemaining: number percentComplete: number + source: 'network' | 'local' | 'memcache' +} + +export function fileProgressToHumanReadableString( + progress: FileDownloadProgress, + fileName: string, + options: { showPercent: boolean }, +): string { + const progressPercent = Math.floor(progress.percentComplete) + + const sourceString = + progress.source === 'network' ? '' : progress.source === 'memcache' ? 'from cache' : 'from backup' + + let result = `Downloading file ${sourceString} "${fileName}"` + + if (options.showPercent) { + result += ` (${progressPercent}%)` + } + + return result } diff --git a/packages/files/src/Domain/UseCase/FileDownloader.ts b/packages/files/src/Domain/UseCase/FileDownloader.ts index 061236000..ea1c3f93c 100644 --- a/packages/files/src/Domain/UseCase/FileDownloader.ts +++ b/packages/files/src/Domain/UseCase/FileDownloader.ts @@ -35,6 +35,7 @@ export class FileDownloader { encryptedBytesDownloaded: this.totalBytesDownloaded, encryptedBytesRemaining: encryptedSize - this.totalBytesDownloaded, percentComplete: (this.totalBytesDownloaded / encryptedSize) * 100.0, + source: 'network', } } diff --git a/packages/files/src/Domain/index.ts b/packages/files/src/Domain/index.ts index 954fc6d96..d1f3bfc57 100644 --- a/packages/files/src/Domain/index.ts +++ b/packages/files/src/Domain/index.ts @@ -13,8 +13,10 @@ export * from './Device/FileBackupMetadataFile' export * from './Device/FileBackupsConstantsV1' export * from './Device/FileBackupsDevice' export * from './Device/FileBackupsMapping' +export * from './Service/BackupServiceInterface' export * from './Service/FilesClientInterface' -export * from './Service/ReadAndDecryptBackupFile' +export * from './Service/ReadAndDecryptBackupFileFileSystemAPI' +export * from './Service/ReadAndDecryptBackupFileUsingBackupService' export * from './Operations/DownloadAndDecrypt' export * from './Operations/EncryptAndUpload' export * from './UseCase/FileDecryptor' diff --git a/packages/services/package.json b/packages/services/package.json index 221051156..adf217865 100644 --- a/packages/services/package.json +++ b/packages/services/package.json @@ -13,7 +13,7 @@ "tsc": "tsc --project tsconfig.json", "lint": "eslint src --ext .ts", "lint:fix": "eslint src --ext .ts --fix", - "test": "jest spec --coverage" + "test": "jest --coverage" }, "dependencies": { "@standardnotes/api": "workspace:^", diff --git a/packages/services/src/Domain/Backups/BackupService.spec.ts b/packages/services/src/Domain/Backups/BackupService.spec.ts new file mode 100644 index 000000000..635eb8c4f --- /dev/null +++ b/packages/services/src/Domain/Backups/BackupService.spec.ts @@ -0,0 +1,109 @@ +import { StatusServiceInterface } from './../Status/StatusServiceInterface' +import { FilesBackupService } from './BackupService' +import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common' +import { EncryptionProviderInterface } from '@standardnotes/encryption' +import { ItemManagerInterface } from '../Item/ItemManagerInterface' +import { InternalEventBusInterface } from '..' +import { AlertService } from '../Alert/AlertService' +import { ApiServiceInterface } from '../Api/ApiServiceInterface' +import { SyncServiceInterface } from '../Sync/SyncServiceInterface' +import { FileBackupsDevice } from '@standardnotes/files' + +describe('backup service', () => { + let apiService: ApiServiceInterface + let itemManager: ItemManagerInterface + let syncService: SyncServiceInterface + let alertService: AlertService + let crypto: PureCryptoInterface + let status: StatusServiceInterface + let encryptor: EncryptionProviderInterface + let internalEventBus: InternalEventBusInterface + let backupService: FilesBackupService + let device: FileBackupsDevice + + beforeEach(() => { + apiService = {} as jest.Mocked + apiService.addEventObserver = jest.fn() + apiService.createFileValetToken = jest.fn() + apiService.downloadFile = jest.fn() + apiService.deleteFile = jest.fn().mockReturnValue({}) + + itemManager = {} as jest.Mocked + itemManager.createItem = jest.fn() + itemManager.createTemplateItem = jest.fn().mockReturnValue({}) + itemManager.setItemToBeDeleted = jest.fn() + itemManager.addObserver = jest.fn() + itemManager.changeItem = jest.fn() + + status = {} as jest.Mocked + + device = {} as jest.Mocked + device.getFileBackupReadToken = jest.fn() + device.readNextChunk = jest.fn() + + syncService = {} as jest.Mocked + syncService.sync = jest.fn() + + encryptor = {} as jest.Mocked + + alertService = {} as jest.Mocked + alertService.confirm = jest.fn().mockReturnValue(true) + alertService.alert = jest.fn() + + crypto = {} as jest.Mocked + crypto.base64Decode = jest.fn() + internalEventBus = {} as jest.Mocked + internalEventBus.publish = jest.fn() + + backupService = new FilesBackupService(itemManager, apiService, encryptor, device, status, crypto, internalEventBus) + + crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({ + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamDecryptorPush = jest.fn().mockReturnValue({ message: new Uint8Array([0xaa]), tag: 0 }) + + crypto.xchacha20StreamInitEncryptor = jest.fn().mockReturnValue({ + header: 'some-header', + state: {}, + } as StreamEncryptor) + + crypto.xchacha20StreamEncryptorPush = jest.fn().mockReturnValue(new Uint8Array()) + }) + + describe('readEncryptedFileFromBackup', () => { + it('return failed if no backup', async () => { + backupService.getFileBackupInfo = jest.fn().mockReturnValue(undefined) + + const result = await backupService.readEncryptedFileFromBackup('123', async () => {}) + + expect(result).toEqual('failed') + }) + + it('return success if backup', async () => { + backupService.getFileBackupInfo = jest.fn().mockReturnValue({}) + + device.readNextChunk = jest.fn().mockReturnValue({ chunk: new Uint8Array([]), isLast: true, progress: undefined }) + + const result = await backupService.readEncryptedFileFromBackup('123', async () => {}) + + expect(result).toEqual('success') + }) + + it('should loop through all chunks until last', async () => { + backupService.getFileBackupInfo = jest.fn().mockReturnValue({}) + const expectedChunkCount = 3 + let receivedChunkCount = 0 + + const mockFn = (device.readNextChunk = jest.fn().mockImplementation(() => { + receivedChunkCount++ + + return { chunk: new Uint8Array([]), isLast: receivedChunkCount === expectedChunkCount, progress: undefined } + })) + + await backupService.readEncryptedFileFromBackup('123', async () => {}) + + expect(mockFn.mock.calls.length).toEqual(expectedChunkCount) + }) + }) +}) diff --git a/packages/services/src/Domain/Backups/BackupService.ts b/packages/services/src/Domain/Backups/BackupService.ts index f2e8a1fd5..16d113eef 100644 --- a/packages/services/src/Domain/Backups/BackupService.ts +++ b/packages/services/src/Domain/Backups/BackupService.ts @@ -8,13 +8,17 @@ import { FileBackupsDevice, FileBackupsMapping, FileBackupRecord, + OnChunkCallback, + BackupServiceInterface, } from '@standardnotes/files' import { InternalEventBusInterface } from '../Internal/InternalEventBusInterface' import { ItemManagerInterface } from '../Item/ItemManagerInterface' import { AbstractService } from '../Service/AbstractService' import { StatusServiceInterface } from '../Status/StatusServiceInterface' +import { PureCryptoInterface } from '@standardnotes/sncrypto-common' +import { log, LoggingDomain } from '../Logging' -export class FilesBackupService extends AbstractService { +export class FilesBackupService extends AbstractService implements BackupServiceInterface { private itemsObserverDisposer: () => void private pendingFiles = new Set() private mappingCache?: FileBackupsMapping['files'] @@ -25,6 +29,7 @@ export class FilesBackupService extends AbstractService { private encryptor: EncryptionProviderInterface, private device: FileBackupsDevice, private status: StatusServiceInterface, + private crypto: PureCryptoInterface, protected override internalEventBus: InternalEventBusInterface, ) { super(internalEventBus) @@ -43,7 +48,14 @@ export class FilesBackupService extends AbstractService { } override deinit() { + super.deinit() this.itemsObserverDisposer() + ;(this.items as unknown) = undefined + ;(this.api as unknown) = undefined + ;(this.encryptor as unknown) = undefined + ;(this.device as unknown) = undefined + ;(this.status as unknown) = undefined + ;(this.crypto as unknown) = undefined } public isFilesBackupsEnabled(): Promise { @@ -98,7 +110,7 @@ export class FilesBackupService extends AbstractService { return this.mappingCache ?? (await this.getBackupsMappingFromDisk()) } - public async getFileBackupInfo(file: FileItem): Promise { + public async getFileBackupInfo(file: { uuid: string }): Promise { const mapping = await this.getBackupsMappingFromCache() const record = mapping[file.uuid] return record @@ -138,7 +150,34 @@ export class FilesBackupService extends AbstractService { this.invalidateMappingCache() } + async readEncryptedFileFromBackup(uuid: string, onChunk: OnChunkCallback): Promise<'success' | 'failed' | 'aborted'> { + const fileBackup = await this.getFileBackupInfo({ uuid }) + + if (!fileBackup) { + return 'failed' + } + + const token = await this.device.getFileBackupReadToken(fileBackup) + + let readMore = true + let index = 0 + + while (readMore) { + const { chunk, isLast, progress } = await this.device.readNextChunk(token) + + await onChunk({ data: chunk, index, isLast, progress }) + + readMore = !isLast + + index++ + } + + return 'success' + } + private async performBackupOperation(file: FileItem): Promise<'success' | 'failed' | 'aborted'> { + log(LoggingDomain.FilesBackups, 'Backing up file locally', file.uuid) + const messageId = this.status.addMessage(`Backing up file ${file.name}...`) const encryptedFile = await this.encryptor.encryptSplitSingle({ diff --git a/packages/services/src/Domain/Files/FileService.spec.ts b/packages/services/src/Domain/Files/FileService.spec.ts index e320de71b..08d75c7d1 100644 --- a/packages/services/src/Domain/Files/FileService.spec.ts +++ b/packages/services/src/Domain/Files/FileService.spec.ts @@ -1,15 +1,14 @@ import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common' import { FileItem } from '@standardnotes/models' import { EncryptionProviderInterface } from '@standardnotes/encryption' - import { ItemManagerInterface } from '../Item/ItemManagerInterface' import { ChallengeServiceInterface } from '../Challenge' import { InternalEventBusInterface } from '..' import { AlertService } from '../Alert/AlertService' import { ApiServiceInterface } from '../Api/ApiServiceInterface' import { SyncServiceInterface } from '../Sync/SyncServiceInterface' - import { FileService } from './FileService' +import { BackupServiceInterface } from '@standardnotes/files' describe('fileService', () => { let apiService: ApiServiceInterface @@ -21,13 +20,33 @@ describe('fileService', () => { let fileService: FileService let encryptor: EncryptionProviderInterface let internalEventBus: InternalEventBusInterface + let backupService: BackupServiceInterface beforeEach(() => { apiService = {} as jest.Mocked apiService.addEventObserver = jest.fn() apiService.createFileValetToken = jest.fn() - apiService.downloadFile = jest.fn() apiService.deleteFile = jest.fn().mockReturnValue({}) + const numChunks = 1 + apiService.downloadFile = jest + .fn() + .mockImplementation( + ( + _file: string, + _chunkIndex: number, + _apiToken: string, + _rangeStart: number, + onBytesReceived: (bytes: Uint8Array) => void, + ) => { + return new Promise((resolve) => { + for (let i = 0; i < numChunks; i++) { + onBytesReceived(Uint8Array.from([0xaa])) + } + + resolve() + }) + }, + ) itemManager = {} as jest.Mocked itemManager.createItem = jest.fn() @@ -52,6 +71,10 @@ describe('fileService', () => { internalEventBus = {} as jest.Mocked internalEventBus.publish = jest.fn() + backupService = {} as jest.Mocked + backupService.readEncryptedFileFromBackup = jest.fn() + backupService.getFileBackupInfo = jest.fn() + fileService = new FileService( apiService, itemManager, @@ -61,6 +84,7 @@ describe('fileService', () => { alertService, crypto, internalEventBus, + backupService, ) crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({ @@ -110,6 +134,8 @@ describe('fileService', () => { decryptedSize: 100_000, } as jest.Mocked + apiService.downloadFile = jest.fn() + await fileService.downloadFile(file, async () => { return Promise.resolve() }) @@ -118,4 +144,42 @@ describe('fileService', () => { expect(fileService['encryptedCache'].get(file.uuid)).toBeFalsy() }) + + it('should download file from network if no backup', async () => { + const file = { + uuid: '1', + decryptedSize: 100_000, + encryptedSize: 101_000, + encryptedChunkSizes: [101_000], + } as jest.Mocked + + backupService.getFileBackupInfo = jest.fn().mockReturnValue(undefined) + + const downloadMock = apiService.downloadFile as jest.Mock + + await fileService.downloadFile(file, async () => { + return Promise.resolve() + }) + + expect(downloadMock).toHaveBeenCalledTimes(1) + }) + + it('should download file from local backup if it exists', async () => { + const file = { + uuid: '1', + decryptedSize: 100_000, + encryptedSize: 101_000, + encryptedChunkSizes: [101_000], + } as jest.Mocked + + backupService.getFileBackupInfo = jest.fn().mockReturnValue({}) + + const downloadMock = (apiService.downloadFile = jest.fn()) + + await fileService.downloadFile(file, async () => { + return Promise.resolve() + }) + + expect(downloadMock).toHaveBeenCalledTimes(0) + }) }) diff --git a/packages/services/src/Domain/Files/FileService.ts b/packages/services/src/Domain/Files/FileService.ts index c5bf1046c..b334c64cb 100644 --- a/packages/services/src/Domain/Files/FileService.ts +++ b/packages/services/src/Domain/Files/FileService.ts @@ -19,7 +19,7 @@ import { FileDecryptor, FileDownloadProgress, FilesClientInterface, - readAndDecryptBackupFile, + readAndDecryptBackupFileUsingFileSystemAPI, FilesApiInterface, FileBackupsConstantsV1, FileBackupMetadataFile, @@ -30,8 +30,9 @@ import { DecryptedBytes, OrderedByteChunker, FileMemoryCache, + readAndDecryptBackupFileUsingBackupService, + BackupServiceInterface, } from '@standardnotes/files' - import { AlertService } from '../Alert/AlertService' import { ChallengeServiceInterface } from '../Challenge' import { InternalEventBusInterface } from '../Internal/InternalEventBusInterface' @@ -39,6 +40,7 @@ import { ItemManagerInterface } from '../Item/ItemManagerInterface' import { AbstractService } from '../Service/AbstractService' import { SyncServiceInterface } from '../Sync/SyncServiceInterface' import { DecryptItemsKeyWithUserFallback } from '../Encryption/Functions' +import { log, LoggingDomain } from '../Logging' const OneHundredMb = 100 * 1_000_000 @@ -54,6 +56,7 @@ export class FileService extends AbstractService implements FilesClientInterface private alertService: AlertService, private crypto: PureCryptoInterface, protected override internalEventBus: InternalEventBusInterface, + private backupsService?: BackupServiceInterface, ) { super(internalEventBus) } @@ -158,8 +161,8 @@ export class FileService extends AbstractService implements FilesClientInterface let decryptedAggregate = new Uint8Array() - const orderedChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (encryptedBytes) => { - const decryptedBytes = decryptOperation.decryptBytes(encryptedBytes) + const orderedChunker = new OrderedByteChunker(file.encryptedChunkSizes, 'memcache', async (chunk) => { + const decryptedBytes = decryptOperation.decryptBytes(chunk.data) if (decryptedBytes) { decryptedAggregate = new Uint8Array([...decryptedAggregate, ...decryptedBytes.decryptedBytes]) @@ -173,36 +176,60 @@ export class FileService extends AbstractService implements FilesClientInterface public async downloadFile( file: FileItem, - onDecryptedBytes: (decryptedBytes: Uint8Array, progress?: FileDownloadProgress) => Promise, + onDecryptedBytes: (decryptedBytes: Uint8Array, progress: FileDownloadProgress) => Promise, ): Promise { const cachedBytes = this.encryptedCache.get(file.uuid) if (cachedBytes) { const decryptedBytes = await this.decryptCachedEntry(file, cachedBytes) - await onDecryptedBytes(decryptedBytes.decryptedBytes, undefined) + await onDecryptedBytes(decryptedBytes.decryptedBytes, { + encryptedFileSize: cachedBytes.encryptedBytes.length, + encryptedBytesDownloaded: cachedBytes.encryptedBytes.length, + encryptedBytesRemaining: 0, + percentComplete: 100, + source: 'memcache', + }) return undefined } - const addToCache = file.encryptedSize < this.encryptedCache.maxSize + const fileBackup = await this.backupsService?.getFileBackupInfo(file) - let cacheEntryAggregate = new Uint8Array() + if (this.backupsService && fileBackup) { + log(LoggingDomain.FilesService, 'Downloading file from backup', fileBackup) - const operation = new DownloadAndDecryptFileOperation(file, this.crypto, this.api) + await readAndDecryptBackupFileUsingBackupService(file, this.backupsService, this.crypto, async (chunk) => { + log(LoggingDomain.FilesService, 'Got local file chunk', chunk.progress) - const result = await operation.run(async ({ decrypted, encrypted, progress }): Promise => { - if (addToCache) { - cacheEntryAggregate = new Uint8Array([...cacheEntryAggregate, ...encrypted.encryptedBytes]) + return onDecryptedBytes(chunk.data, chunk.progress) + }) + + log(LoggingDomain.FilesService, 'Finished downloading file from backup') + + return undefined + } else { + log(LoggingDomain.FilesService, 'Downloading file from network') + + const addToCache = file.encryptedSize < this.encryptedCache.maxSize + + let cacheEntryAggregate = new Uint8Array() + + const operation = new DownloadAndDecryptFileOperation(file, this.crypto, this.api) + + const result = await operation.run(async ({ decrypted, encrypted, progress }): Promise => { + if (addToCache) { + cacheEntryAggregate = new Uint8Array([...cacheEntryAggregate, ...encrypted.encryptedBytes]) + } + return onDecryptedBytes(decrypted.decryptedBytes, progress) + }) + + if (addToCache && cacheEntryAggregate.byteLength > 0) { + this.encryptedCache.add(file.uuid, { encryptedBytes: cacheEntryAggregate }) } - return onDecryptedBytes(decrypted.decryptedBytes, progress) - }) - if (addToCache) { - this.encryptedCache.add(file.uuid, { encryptedBytes: cacheEntryAggregate }) + return result.error } - - return result.error } public async deleteFile(file: FileItem): Promise { @@ -294,9 +321,15 @@ export class FileService extends AbstractService implements FilesClientInterface return destinationFileHandle } - const result = await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => { - await fileSystem.saveBytes(destinationFileHandle, decryptedBytes) - }) + const result = await readAndDecryptBackupFileUsingFileSystemAPI( + fileHandle, + file, + fileSystem, + this.crypto, + async (decryptedBytes) => { + await fileSystem.saveBytes(destinationFileHandle, decryptedBytes) + }, + ) await fileSystem.closeFileWriteStream(destinationFileHandle) @@ -310,9 +343,15 @@ export class FileService extends AbstractService implements FilesClientInterface ): Promise { let bytes = new Uint8Array() - await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => { - bytes = new Uint8Array([...bytes, ...decryptedBytes]) - }) + await readAndDecryptBackupFileUsingFileSystemAPI( + fileHandle, + file, + fileSystem, + this.crypto, + async (decryptedBytes) => { + bytes = new Uint8Array([...bytes, ...decryptedBytes]) + }, + ) return bytes } diff --git a/packages/services/src/Domain/Logging.ts b/packages/services/src/Domain/Logging.ts new file mode 100644 index 000000000..bafcc7c58 --- /dev/null +++ b/packages/services/src/Domain/Logging.ts @@ -0,0 +1,33 @@ +import { logWithColor } from '@standardnotes/utils' + +declare const process: { + env: { + NODE_ENV: string | null | undefined + } +} + +export const isDev = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test' + +export enum LoggingDomain { + FilesService, + FilesBackups, +} + +const LoggingStatus: Record = { + [LoggingDomain.FilesService]: false, + [LoggingDomain.FilesBackups]: false, +} + +const LoggingColor: Record = { + [LoggingDomain.FilesService]: 'blue', + [LoggingDomain.FilesBackups]: 'yellow', +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function log(domain: LoggingDomain, ...args: any[]): void { + if (!isDev || !LoggingStatus[domain]) { + return + } + + logWithColor(LoggingDomain[domain], LoggingColor[domain], ...args) +} diff --git a/packages/snjs/lib/Application/Application.ts b/packages/snjs/lib/Application/Application.ts index e904db788..e81560d15 100644 --- a/packages/snjs/lib/Application/Application.ts +++ b/packages/snjs/lib/Application/Application.ts @@ -64,7 +64,7 @@ import { SessionStrings, AccountEvent, } from '@standardnotes/services' -import { FilesClientInterface } from '@standardnotes/files' +import { BackupServiceInterface, FilesClientInterface } from '@standardnotes/files' import { ComputePrivateUsername } from '@standardnotes/encryption' import { useBoolean } from '@standardnotes/utils' import { @@ -275,7 +275,7 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli return this.statusService } - public get fileBackups(): FilesBackupService | undefined { + public get fileBackups(): BackupServiceInterface | undefined { return this.filesBackupService } @@ -1113,16 +1113,17 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli this.createComponentManager() this.createMigrationService() this.createMfaService() + + this.createStatusService() + if (isDesktopDevice(this.deviceInterface)) { + this.createFilesBackupService(this.deviceInterface) + } this.createFileService() + this.createIntegrityService() this.createMutatorService() this.createListedService() this.createActionsManager() - this.createStatusService() - - if (isDesktopDevice(this.deviceInterface)) { - this.createFilesBackupService(this.deviceInterface) - } } private clearServices() { @@ -1210,6 +1211,7 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli this.alertService, this.options.crypto, this.internalEventBus, + this.fileBackups, ) this.services.push(this.fileService) @@ -1670,6 +1672,7 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli this.protocolService, device, this.statusService, + this.options.crypto, this.internalEventBus, ) this.services.push(this.filesBackupService) diff --git a/packages/utils/src/Domain/Utils/Utils.ts b/packages/utils/src/Domain/Utils/Utils.ts index ddf8556f5..ebbefe662 100644 --- a/packages/utils/src/Domain/Utils/Utils.ts +++ b/packages/utils/src/Domain/Utils/Utils.ts @@ -655,11 +655,15 @@ export function secondHalfOfString(string: string): string { } export function log(namespace: string, ...args: any[]): void { + logWithColor(namespace, 'black', ...args) +} + +export function logWithColor(namespace: string, namespaceColor: string, ...args: any[]): void { const date = new Date() const timeString = `${date.toLocaleTimeString().replace(' PM', '').replace(' AM', '')}.${date.getMilliseconds()}` customLog( `%c${namespace}%c${timeString}`, - 'color: black; font-weight: bold; margin-right: 4px', + `color: ${namespaceColor}; font-weight: bold; margin-right: 4px`, 'color: gray', ...args, ) diff --git a/packages/web/src/javascripts/Application/Device/DesktopSnjsExports.ts b/packages/web/src/javascripts/Application/Device/DesktopSnjsExports.ts index 68bcd182a..b1aa8cce5 100644 --- a/packages/web/src/javascripts/Application/Device/DesktopSnjsExports.ts +++ b/packages/web/src/javascripts/Application/Device/DesktopSnjsExports.ts @@ -7,4 +7,7 @@ export { FileBackupsMapping, FileBackupsDevice, FileBackupRecord, + FileBackupReadToken, + FileBackupReadChunkResponse, + FileDownloadProgress, } from '@standardnotes/snjs' diff --git a/packages/web/src/javascripts/Components/FilePreview/FilePreview.tsx b/packages/web/src/javascripts/Components/FilePreview/FilePreview.tsx index c58c8a7ba..f3854356b 100644 --- a/packages/web/src/javascripts/Components/FilePreview/FilePreview.tsx +++ b/packages/web/src/javascripts/Components/FilePreview/FilePreview.tsx @@ -1,6 +1,11 @@ import { WebApplication } from '@/Application/Application' import { concatenateUint8Arrays } from '@/Utils' -import { ApplicationEvent, FileItem } from '@standardnotes/snjs' +import { + ApplicationEvent, + FileDownloadProgress, + FileItem, + fileProgressToHumanReadableString, +} from '@standardnotes/snjs' import { useEffect, useMemo, useState } from 'react' import Spinner from '@/Components/Spinner/Spinner' import FilePreviewError from './FilePreviewError' @@ -24,7 +29,7 @@ const FilePreview = ({ file, application, isEmbeddedInSuper = false, imageZoomLe }, [file.mimeType]) const [isDownloading, setIsDownloading] = useState(true) - const [downloadProgress, setDownloadProgress] = useState(0) + const [downloadProgress, setDownloadProgress] = useState() const [downloadedBytes, setDownloadedBytes] = useState() useEffect(() => { @@ -46,7 +51,7 @@ const FilePreview = ({ file, application, isEmbeddedInSuper = false, imageZoomLe useEffect(() => { if (!isFilePreviewable || !isAuthorized) { setIsDownloading(false) - setDownloadProgress(0) + setDownloadProgress(undefined) setDownloadedBytes(undefined) return } @@ -60,15 +65,18 @@ const FilePreview = ({ file, application, isEmbeddedInSuper = false, imageZoomLe try { const chunks: Uint8Array[] = [] - setDownloadProgress(0) - await application.files.downloadFile(file, async (decryptedChunk, progress) => { + setDownloadProgress(undefined) + const error = await application.files.downloadFile(file, async (decryptedChunk, progress) => { chunks.push(decryptedChunk) if (progress) { - setDownloadProgress(Math.round(progress.percentComplete)) + setDownloadProgress(progress) } }) - const finalDecryptedBytes = concatenateUint8Arrays(chunks) - setDownloadedBytes(finalDecryptedBytes) + + if (!error) { + const finalDecryptedBytes = concatenateUint8Arrays(chunks) + setDownloadedBytes(finalDecryptedBytes) + } } catch (error) { console.error(error) } finally { @@ -109,9 +117,17 @@ const FilePreview = ({ file, application, isEmbeddedInSuper = false, imageZoomLe
-
{downloadProgress}%
+ {downloadProgress && ( +
{Math.floor(downloadProgress.percentComplete)}%
+ )}
- Loading file... + {downloadProgress ? ( + + {fileProgressToHumanReadableString(downloadProgress, file.name, { showPercent: false })} + + ) : ( + Loading... + )}
) : downloadedBytes ? ( { )} - - {backupsEnabled && ( <> + + <> diff --git a/packages/web/src/javascripts/Controllers/FilesController.ts b/packages/web/src/javascripts/Controllers/FilesController.ts index 90a7139f3..fe7e18b67 100644 --- a/packages/web/src/javascripts/Controllers/FilesController.ts +++ b/packages/web/src/javascripts/Controllers/FilesController.ts @@ -1,3 +1,8 @@ +import { + FileDownloadProgress, + fileProgressToHumanReadableString, + OnChunkCallbackNoProgress, +} from '@standardnotes/files' import { FilePreviewModalController } from './FilePreviewModalController' import { PopoverFileItemAction, @@ -260,6 +265,8 @@ export class FilesController extends AbstractViewController { if (isUsingStreamingSaver) { await saver.pushBytes(decryptedBytes) @@ -267,14 +274,14 @@ export class FilesController extends AbstractViewController { - await this.application.files.pushBytesForUpload(operation, chunk, index, isLast) + const onChunk: OnChunkCallbackNoProgress = async ({ data, index, isLast }) => { + await this.application.files.pushBytesForUpload(operation, data, index, isLast) - const progress = Math.round(operation.getProgress().percentComplete) + const percentComplete = Math.round(operation.getProgress().percentComplete) updateToast(toastId, { - message: `Uploading file "${file.name}" (${progress}%)`, - progress, + message: `Uploading file "${file.name}" (${percentComplete}%)`, + progress: percentComplete, }) }