feat: download and preview files from local backups automatically, if a local backup is available (#2076)

packages/files/src/Domain/Chunker/ByteChunker.spec.ts

@@ -8,9 +8,9 @@ describe('byte chunker', () => {
   it('should hold back small chunks until minimum size is met', async () => {
     let receivedBytes = new Uint8Array()
     let numChunks = 0
-    const chunker = new ByteChunker(100, async (bytes) => {
+    const chunker = new ByteChunker(100, async (chunk) => {
       numChunks++
-      receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
+      receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data])
     })
 
     await chunker.addBytes(chunkOfSize(50), false)
@@ -25,9 +25,9 @@ describe('byte chunker', () => {
   it('should send back big chunks immediately', async () => {
     let receivedBytes = new Uint8Array()
     let numChunks = 0
-    const chunker = new ByteChunker(100, async (bytes) => {
+    const chunker = new ByteChunker(100, async (chunk) => {
       numChunks++
-      receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
+      receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data])
     })
 
     await chunker.addBytes(chunkOfSize(150), false)
@@ -42,9 +42,9 @@ describe('byte chunker', () => {
   it('last chunk should be popped regardless of size', async () => {
     let receivedBytes = new Uint8Array()
     let numChunks = 0
-    const chunker = new ByteChunker(100, async (bytes) => {
+    const chunker = new ByteChunker(100, async (chunk) => {
       numChunks++
-      receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
+      receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data])
     })
 
     await chunker.addBytes(chunkOfSize(50), false)
@@ -57,9 +57,9 @@ describe('byte chunker', () => {
   it('single chunk should be popped immediately', async () => {
     let receivedBytes = new Uint8Array()
     let numChunks = 0
-    const chunker = new ByteChunker(100, async (bytes) => {
+    const chunker = new ByteChunker(100, async (chunk) => {
       numChunks++
-      receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
+      receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data])
     })
 
     await chunker.addBytes(chunkOfSize(50), true)

packages/files/src/Domain/Chunker/ByteChunker.ts

@@ -1,11 +1,11 @@
-import { OnChunkCallback } from './OnChunkCallback'
+import { OnChunkCallbackNoProgress } from './OnChunkCallback'
 
 export class ByteChunker {
   public loggingEnabled = false
   private bytes = new Uint8Array()
   private index = 1
 
-  constructor(private minimumChunkSize: number, private onChunk: OnChunkCallback) {}
+  constructor(private minimumChunkSize: number, private onChunk: OnChunkCallbackNoProgress) {}
 
   private log(...args: any[]): void {
     if (!this.loggingEnabled) {
@@ -27,9 +27,13 @@ export class ByteChunker {
 
   private async popBytes(isLast: boolean): Promise<void> {
     const maxIndex = Math.max(this.minimumChunkSize, this.bytes.length)
+
     const chunk = this.bytes.slice(0, maxIndex)
+
     this.bytes = new Uint8Array([...this.bytes.slice(maxIndex)])
+
     this.log(`Chunker popping ${chunk.length}, total size in queue ${this.bytes.length}`)
-    await this.onChunk(chunk, this.index++, isLast)
+
+    await this.onChunk({ data: chunk, index: this.index++, isLast })
   }
 }

packages/files/src/Domain/Chunker/OnChunkCallback.ts

@@ -1 +1,10 @@
-export type OnChunkCallback = (chunk: Uint8Array, index: number, isLast: boolean) => Promise<void>
+import { FileDownloadProgress } from '../Types/FileDownloadProgress'
+
+export type OnChunkCallback = (chunk: {
+  data: Uint8Array
+  index: number
+  isLast: boolean
+  progress: FileDownloadProgress
+}) => Promise<void>
+
+export type OnChunkCallbackNoProgress = (chunk: { data: Uint8Array; index: number; isLast: boolean }) => Promise<void>

packages/files/src/Domain/Chunker/OrderedByteChunker.spec.ts

@@ -10,9 +10,30 @@ describe('ordered byte chunker', () => {
     let receivedBytes = new Uint8Array()
     let numCallbacks = 0
 
-    const chunker = new OrderedByteChunker(chunkSizes, async (bytes) => {
+    const chunker = new OrderedByteChunker(chunkSizes, 'network', async (chunk) => {
       numCallbacks++
-      receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
+      receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data])
+    })
+
+    await chunker.addBytes(chunkOfSize(30))
+
+    expect(numCallbacks).toEqual(3)
+    expect(receivedBytes.length).toEqual(30)
+  })
+
+  it('should correctly report progress', async () => {
+    const chunkSizes = [10, 10, 10]
+    let receivedBytes = new Uint8Array()
+    let numCallbacks = 0
+
+    const chunker = new OrderedByteChunker(chunkSizes, 'network', async (chunk) => {
+      numCallbacks++
+
+      receivedBytes = new Uint8Array([...receivedBytes, ...chunk.data])
+
+      expect(chunk.progress.encryptedBytesDownloaded).toEqual(receivedBytes.length)
+
+      expect(chunk.progress.percentComplete).toEqual((numCallbacks / chunkSizes.length) * 100.0)
     })
 
     await chunker.addBytes(chunkOfSize(30))

packages/files/src/Domain/Chunker/OrderedByteChunker.ts

@@ -1,13 +1,28 @@
+import { FileDownloadProgress } from '../Types/FileDownloadProgress'
+import { OnChunkCallback } from './OnChunkCallback'
+
 export class OrderedByteChunker {
   private bytes = new Uint8Array()
   private index = 1
   private remainingChunks: number[] = []
+  private fileSize: number
 
   constructor(
     private chunkSizes: number[],
-    private onChunk: (chunk: Uint8Array, index: number, isLast: boolean) => Promise<void>,
+    private source: FileDownloadProgress['source'],
+    private onChunk: OnChunkCallback,
   ) {
     this.remainingChunks = chunkSizes.slice()
+
+    this.fileSize = chunkSizes.reduce((acc, size) => acc + size, 0)
+  }
+
+  private get bytesPopped(): number {
+    return this.fileSize - this.bytesRemaining
+  }
+
+  private get bytesRemaining(): number {
+    return this.remainingChunks.reduce((acc, size) => acc + size, 0)
   }
 
   private needsPop(): boolean {
@@ -31,7 +46,18 @@ export class OrderedByteChunker {
 
     this.remainingChunks.shift()
 
-    await this.onChunk(chunk, this.index++, this.index === this.chunkSizes.length - 1)
+    await this.onChunk({
+      data: chunk,
+      index: this.index++,
+      isLast: this.index === this.chunkSizes.length - 1,
+      progress: {
+        encryptedFileSize: this.fileSize,
+        encryptedBytesDownloaded: this.bytesPopped,
+        encryptedBytesRemaining: this.bytesRemaining,
+        percentComplete: (this.bytesPopped / this.fileSize) * 100.0,
+        source: this.source,
+      },
+    })
 
     if (this.needsPop()) {
       await this.popBytes()

packages/files/src/Domain/Device/FileBackupsDevice.ts

@@ -1,6 +1,10 @@
 import { Uuid } from '@standardnotes/common'
+import { FileDownloadProgress } from '../Types/FileDownloadProgress'
 import { FileBackupRecord, FileBackupsMapping } from './FileBackupsMapping'
 
+export type FileBackupReadToken = string
+export type FileBackupReadChunkResponse = { chunk: Uint8Array; isLast: boolean; progress: FileDownloadProgress }
+
 export interface FileBackupsDevice {
   getFilesBackupsMappingFile(): Promise<FileBackupsMapping>
   saveFilesBackupsFile(
@@ -12,6 +16,8 @@ export interface FileBackupsDevice {
       url: string
     },
   ): Promise<'success' | 'failed'>
+  getFileBackupReadToken(record: FileBackupRecord): Promise<FileBackupReadToken>
+  readNextChunk(token: string): Promise<FileBackupReadChunkResponse>
   isFilesBackupsEnabled(): Promise<boolean>
   enableFilesBackups(): Promise<void>
   disableFilesBackups(): Promise<void>

packages/files/src/Domain/Logging.ts

@@ -0,0 +1,30 @@
+import { logWithColor } from '@standardnotes/utils'
+
+declare const process: {
+  env: {
+    NODE_ENV: string | null | undefined
+  }
+}
+
+export const isDev = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test'
+
+export enum LoggingDomain {
+  FilesPackage,
+}
+
+const LoggingStatus: Record<LoggingDomain, boolean> = {
+  [LoggingDomain.FilesPackage]: false,
+}
+
+const DomainColor: Record<LoggingDomain, string> = {
+  [LoggingDomain.FilesPackage]: 'green',
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function log(domain: LoggingDomain, ...args: any[]): void {
+  if (!isDev || !LoggingStatus[domain]) {
+    return
+  }
+
+  logWithColor(LoggingDomain[domain], DomainColor[domain], ...args)
+}

packages/files/src/Domain/Service/BackupServiceInterface.ts

@@ -0,0 +1,14 @@
+import { OnChunkCallback } from '../Chunker/OnChunkCallback'
+import { FileBackupRecord } from '../Device/FileBackupsMapping'
+
+export interface BackupServiceInterface {
+  getFileBackupInfo(file: { uuid: string }): Promise<FileBackupRecord | undefined>
+  readEncryptedFileFromBackup(uuid: string, onChunk: OnChunkCallback): Promise<'success' | 'failed' | 'aborted'>
+  isFilesBackupsEnabled(): Promise<boolean>
+  enableFilesBackups(): Promise<void>
+  disableFilesBackups(): Promise<void>
+  changeFilesBackupsLocation(): Promise<string | undefined>
+  getFilesBackupsLocation(): Promise<string>
+  openFilesBackupsLocation(): Promise<void>
+  openFileBackup(record: FileBackupRecord): Promise<void>
+}

packages/files/src/Domain/Service/FilesClientInterface.ts

@@ -24,7 +24,7 @@ export interface FilesClientInterface {
 
   downloadFile(
     file: FileItem,
-    onDecryptedBytes: (bytes: Uint8Array, progress: FileDownloadProgress | undefined) => Promise<void>,
+    onDecryptedBytes: (bytes: Uint8Array, progress: FileDownloadProgress) => Promise<void>,
   ): Promise<ClientDisplayableError | undefined>
 
   deleteFile(file: FileItem): Promise<ClientDisplayableError | undefined>

packages/files/src/Domain/Service/ReadAndDecryptBackupFileFileSystemAPI.ts

@@ -5,7 +5,7 @@ import { FileSystemApi } from '../Api/FileSystemApi'
 import { FileHandleRead } from '../Api/FileHandleRead'
 import { OrderedByteChunker } from '../Chunker/OrderedByteChunker'
 
-export async function readAndDecryptBackupFile(
+export async function readAndDecryptBackupFileUsingFileSystemAPI(
   fileHandle: FileHandleRead,
   file: {
     encryptionHeader: FileContent['encryptionHeader']
@@ -19,8 +19,8 @@ export async function readAndDecryptBackupFile(
 ): Promise<'aborted' | 'failed' | 'success'> {
   const decryptor = new FileDecryptor(file, crypto)
 
-  const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (chunk: Uint8Array) => {
-    const decryptResult = decryptor.decryptBytes(chunk)
+  const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, 'local', async (chunk) => {
+    const decryptResult = decryptor.decryptBytes(chunk.data)
 
     if (!decryptResult) {
       return

packages/files/src/Domain/Service/ReadAndDecryptBackupFileUsingBackupService.ts

@@ -0,0 +1,52 @@
+import { FileContent } from '@standardnotes/models'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { FileDecryptor } from '../UseCase/FileDecryptor'
+import { OrderedByteChunker } from '../Chunker/OrderedByteChunker'
+import { BackupServiceInterface } from './BackupServiceInterface'
+import { OnChunkCallback } from '../Chunker/OnChunkCallback'
+import { log, LoggingDomain } from '../Logging'
+
+export async function readAndDecryptBackupFileUsingBackupService(
+  file: {
+    uuid: string
+    encryptionHeader: FileContent['encryptionHeader']
+    remoteIdentifier: FileContent['remoteIdentifier']
+    encryptedChunkSizes: FileContent['encryptedChunkSizes']
+    key: FileContent['key']
+  },
+  backupService: BackupServiceInterface,
+  crypto: PureCryptoInterface,
+  onDecryptedBytes: OnChunkCallback,
+): Promise<'aborted' | 'failed' | 'success'> {
+  log(
+    LoggingDomain.FilesPackage,
+    'Reading and decrypting backup file',
+    file.uuid,
+    'chunk sizes',
+    file.encryptedChunkSizes,
+  )
+
+  const decryptor = new FileDecryptor(file, crypto)
+
+  const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, 'local', async (chunk) => {
+    log(LoggingDomain.FilesPackage, 'OrderedByteChunker did pop bytes', chunk.data.length, chunk.progress)
+
+    const decryptResult = decryptor.decryptBytes(chunk.data)
+
+    if (!decryptResult) {
+      return
+    }
+
+    await onDecryptedBytes({ ...chunk, data: decryptResult.decryptedBytes })
+  })
+
+  const readResult = await backupService.readEncryptedFileFromBackup(file.uuid, async (chunk) => {
+    log(LoggingDomain.FilesPackage, 'Got file chunk from backup service', chunk.data.length, chunk.progress)
+
+    await byteChunker.addBytes(chunk.data)
+  })
+
+  log(LoggingDomain.FilesPackage, 'Finished reading and decrypting backup file', file.uuid)
+
+  return readResult
+}

packages/files/src/Domain/Types/FileDownloadProgress.ts

@@ -3,4 +3,24 @@ export type FileDownloadProgress = {
   encryptedBytesDownloaded: number
   encryptedBytesRemaining: number
   percentComplete: number
+  source: 'network' | 'local' | 'memcache'
+}
+
+export function fileProgressToHumanReadableString(
+  progress: FileDownloadProgress,
+  fileName: string,
+  options: { showPercent: boolean },
+): string {
+  const progressPercent = Math.floor(progress.percentComplete)
+
+  const sourceString =
+    progress.source === 'network' ? '' : progress.source === 'memcache' ? 'from cache' : 'from backup'
+
+  let result = `Downloading file ${sourceString} "${fileName}"`
+
+  if (options.showPercent) {
+    result += ` (${progressPercent}%)`
+  }
+
+  return result
 }

packages/files/src/Domain/UseCase/FileDownloader.ts

@@ -35,6 +35,7 @@ export class FileDownloader {
       encryptedBytesDownloaded: this.totalBytesDownloaded,
       encryptedBytesRemaining: encryptedSize - this.totalBytesDownloaded,
       percentComplete: (this.totalBytesDownloaded / encryptedSize) * 100.0,
+      source: 'network',
     }
   }
 

packages/files/src/Domain/index.ts

@@ -13,8 +13,10 @@ export * from './Device/FileBackupMetadataFile'
 export * from './Device/FileBackupsConstantsV1'
 export * from './Device/FileBackupsDevice'
 export * from './Device/FileBackupsMapping'
+export * from './Service/BackupServiceInterface'
 export * from './Service/FilesClientInterface'
-export * from './Service/ReadAndDecryptBackupFile'
+export * from './Service/ReadAndDecryptBackupFileFileSystemAPI'
+export * from './Service/ReadAndDecryptBackupFileUsingBackupService'
 export * from './Operations/DownloadAndDecrypt'
 export * from './Operations/EncryptAndUpload'
 export * from './UseCase/FileDecryptor'