feat: add files package

packages/files/src/Domain/Service/FileService.spec.ts

@@ -0,0 +1,121 @@
+import {
+  InternalEventBusInterface,
+  SyncServiceInterface,
+  ItemManagerInterface,
+  AlertService,
+  ApiServiceInterface,
+  ChallengeServiceInterface,
+} from '@standardnotes/services'
+import { FileService } from './FileService'
+import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common'
+import { FileItem } from '@standardnotes/models'
+import { EncryptionProvider } from '@standardnotes/encryption'
+
+describe('fileService', () => {
+  let apiService: ApiServiceInterface
+  let itemManager: ItemManagerInterface
+  let syncService: SyncServiceInterface
+  let alertService: AlertService
+  let crypto: PureCryptoInterface
+  let challengor: ChallengeServiceInterface
+  let fileService: FileService
+  let encryptor: EncryptionProvider
+  let internalEventBus: InternalEventBusInterface
+
+  beforeEach(() => {
+    apiService = {} as jest.Mocked<ApiServiceInterface>
+    apiService.addEventObserver = jest.fn()
+    apiService.createFileValetToken = jest.fn()
+    apiService.downloadFile = jest.fn()
+    apiService.deleteFile = jest.fn().mockReturnValue({})
+
+    itemManager = {} as jest.Mocked<ItemManagerInterface>
+    itemManager.createItem = jest.fn()
+    itemManager.createTemplateItem = jest.fn().mockReturnValue({})
+    itemManager.setItemToBeDeleted = jest.fn()
+    itemManager.addObserver = jest.fn()
+    itemManager.changeItem = jest.fn()
+
+    challengor = {} as jest.Mocked<ChallengeServiceInterface>
+
+    syncService = {} as jest.Mocked<SyncServiceInterface>
+    syncService.sync = jest.fn()
+
+    encryptor = {} as jest.Mocked<EncryptionProvider>
+
+    alertService = {} as jest.Mocked<AlertService>
+    alertService.confirm = jest.fn().mockReturnValue(true)
+    alertService.alert = jest.fn()
+
+    crypto = {} as jest.Mocked<PureCryptoInterface>
+    crypto.base64Decode = jest.fn()
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+
+    fileService = new FileService(
+      apiService,
+      itemManager,
+      syncService,
+      encryptor,
+      challengor,
+      alertService,
+      crypto,
+      internalEventBus,
+    )
+
+    crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({
+      state: {},
+    } as StreamEncryptor)
+
+    crypto.xchacha20StreamDecryptorPush = jest.fn().mockReturnValue({ message: new Uint8Array([0xaa]), tag: 0 })
+
+    crypto.xchacha20StreamInitEncryptor = jest.fn().mockReturnValue({
+      header: 'some-header',
+      state: {},
+    } as StreamEncryptor)
+
+    crypto.xchacha20StreamEncryptorPush = jest.fn().mockReturnValue(new Uint8Array())
+  })
+
+  it.only('should cache file after download', async () => {
+    const file = {
+      uuid: '1',
+      decryptedSize: 100_000,
+      encryptedSize: 101_000,
+      encryptedChunkSizes: [101_000],
+    } as jest.Mocked<FileItem>
+
+    let downloadMock = apiService.downloadFile as jest.Mock
+
+    await fileService.downloadFile(file, async () => {
+      return Promise.resolve()
+    })
+
+    expect(downloadMock).toHaveBeenCalledTimes(1)
+
+    downloadMock = apiService.downloadFile = jest.fn()
+
+    await fileService.downloadFile(file, async () => {
+      return Promise.resolve()
+    })
+
+    expect(downloadMock).toHaveBeenCalledTimes(0)
+
+    expect(fileService['encryptedCache'].get(file.uuid)).toBeTruthy()
+  })
+
+  it('deleting file should remove it from cache', async () => {
+    const file = {
+      uuid: '1',
+      decryptedSize: 100_000,
+    } as jest.Mocked<FileItem>
+
+    await fileService.downloadFile(file, async () => {
+      return Promise.resolve()
+    })
+
+    await fileService.deleteFile(file)
+
+    expect(fileService['encryptedCache'].get(file.uuid)).toBeFalsy()
+  })
+})

packages/files/src/Domain/Service/FileService.ts

@@ -0,0 +1,314 @@
+import { DecryptedBytes, EncryptedBytes, FileMemoryCache, OrderedByteChunker } from '@standardnotes/filepicker'
+import { ClientDisplayableError } from '@standardnotes/responses'
+import { ContentType } from '@standardnotes/common'
+import { DownloadAndDecryptFileOperation } from '../Operations/DownloadAndDecrypt'
+import { EncryptAndUploadFileOperation } from '../Operations/EncryptAndUpload'
+import {
+  FileItem,
+  FileProtocolV1Constants,
+  FileMetadata,
+  FileContentSpecialized,
+  FillItemContentSpecialized,
+  FileContent,
+  EncryptedPayload,
+  isEncryptedPayload,
+} from '@standardnotes/models'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { UuidGenerator } from '@standardnotes/utils'
+import {
+  AbstractService,
+  InternalEventBusInterface,
+  ItemManagerInterface,
+  SyncServiceInterface,
+  AlertService,
+  FileSystemApi,
+  FilesApiInterface,
+  FileBackupMetadataFile,
+  FileHandleRead,
+  FileSystemNoSelection,
+  ChallengeServiceInterface,
+  FileBackupsConstantsV1,
+} from '@standardnotes/services'
+import { FilesClientInterface } from './FilesClientInterface'
+import { FileDownloadProgress } from '../Types/FileDownloadProgress'
+import { readAndDecryptBackupFile } from './ReadAndDecryptBackupFile'
+import { DecryptItemsKeyWithUserFallback, EncryptionProvider, SNItemsKey } from '@standardnotes/encryption'
+import { FileDecryptor } from '../UseCase/FileDecryptor'
+
+const OneHundredMb = 100 * 1_000_000
+
+export class FileService extends AbstractService implements FilesClientInterface {
+  private encryptedCache: FileMemoryCache = new FileMemoryCache(OneHundredMb)
+
+  constructor(
+    private api: FilesApiInterface,
+    private itemManager: ItemManagerInterface,
+    private syncService: SyncServiceInterface,
+    private encryptor: EncryptionProvider,
+    private challengor: ChallengeServiceInterface,
+    private alertService: AlertService,
+    private crypto: PureCryptoInterface,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  override deinit(): void {
+    super.deinit()
+
+    this.encryptedCache.clear()
+    ;(this.encryptedCache as unknown) = undefined
+    ;(this.api as unknown) = undefined
+    ;(this.itemManager as unknown) = undefined
+    ;(this.encryptor as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    ;(this.alertService as unknown) = undefined
+    ;(this.challengor as unknown) = undefined
+    ;(this.crypto as unknown) = undefined
+  }
+
+  public minimumChunkSize(): number {
+    return 5_000_000
+  }
+
+  public async beginNewFileUpload(
+    sizeInBytes: number,
+  ): Promise<EncryptAndUploadFileOperation | ClientDisplayableError> {
+    const remoteIdentifier = UuidGenerator.GenerateUuid()
+    const tokenResult = await this.api.createFileValetToken(remoteIdentifier, 'write', sizeInBytes)
+
+    if (tokenResult instanceof ClientDisplayableError) {
+      return tokenResult
+    }
+
+    const key = this.crypto.generateRandomKey(FileProtocolV1Constants.KeySize)
+
+    const fileParams = {
+      key,
+      remoteIdentifier,
+      decryptedSize: sizeInBytes,
+    }
+
+    const uploadOperation = new EncryptAndUploadFileOperation(fileParams, tokenResult, this.crypto, this.api)
+
+    const uploadSessionStarted = await this.api.startUploadSession(tokenResult)
+
+    if (!uploadSessionStarted.uploadId) {
+      return new ClientDisplayableError('Could not start upload session')
+    }
+
+    return uploadOperation
+  }
+
+  public async pushBytesForUpload(
+    operation: EncryptAndUploadFileOperation,
+    bytes: Uint8Array,
+    chunkId: number,
+    isFinalChunk: boolean,
+  ): Promise<ClientDisplayableError | undefined> {
+    const success = await operation.pushBytes(bytes, chunkId, isFinalChunk)
+
+    if (!success) {
+      return new ClientDisplayableError('Failed to push file bytes to server')
+    }
+
+    return undefined
+  }
+
+  public async finishUpload(
+    operation: EncryptAndUploadFileOperation,
+    fileMetadata: FileMetadata,
+  ): Promise<FileItem | ClientDisplayableError> {
+    const uploadSessionClosed = await this.api.closeUploadSession(operation.getApiToken())
+
+    if (!uploadSessionClosed) {
+      return new ClientDisplayableError('Could not close upload session')
+    }
+
+    const result = operation.getResult()
+
+    const fileContent: FileContentSpecialized = {
+      decryptedSize: result.finalDecryptedSize,
+      encryptedChunkSizes: operation.encryptedChunkSizes,
+      encryptionHeader: result.encryptionHeader,
+      key: result.key,
+      mimeType: fileMetadata.mimeType,
+      name: fileMetadata.name,
+      remoteIdentifier: result.remoteIdentifier,
+    }
+
+    const file = await this.itemManager.createItem<FileItem>(
+      ContentType.File,
+      FillItemContentSpecialized(fileContent),
+      true,
+    )
+
+    await this.syncService.sync()
+
+    return file
+  }
+
+  private async decryptCachedEntry(file: FileItem, entry: EncryptedBytes): Promise<DecryptedBytes> {
+    const decryptOperation = new FileDecryptor(file, this.crypto)
+
+    let decryptedAggregate = new Uint8Array()
+
+    const orderedChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (encryptedBytes) => {
+      const decryptedBytes = decryptOperation.decryptBytes(encryptedBytes)
+
+      if (decryptedBytes) {
+        decryptedAggregate = new Uint8Array([...decryptedAggregate, ...decryptedBytes.decryptedBytes])
+      }
+    })
+
+    await orderedChunker.addBytes(entry.encryptedBytes)
+
+    return { decryptedBytes: decryptedAggregate }
+  }
+
+  public async downloadFile(
+    file: FileItem,
+    onDecryptedBytes: (decryptedBytes: Uint8Array, progress?: FileDownloadProgress) => Promise<void>,
+  ): Promise<ClientDisplayableError | undefined> {
+    const cachedBytes = this.encryptedCache.get(file.uuid)
+
+    if (cachedBytes) {
+      const decryptedBytes = await this.decryptCachedEntry(file, cachedBytes)
+
+      await onDecryptedBytes(decryptedBytes.decryptedBytes, undefined)
+
+      return undefined
+    }
+
+    const addToCache = file.encryptedSize < this.encryptedCache.maxSize
+
+    let cacheEntryAggregate = new Uint8Array()
+
+    const operation = new DownloadAndDecryptFileOperation(file, this.crypto, this.api)
+
+    const result = await operation.run(async ({ decrypted, encrypted, progress }): Promise<void> => {
+      if (addToCache) {
+        cacheEntryAggregate = new Uint8Array([...cacheEntryAggregate, ...encrypted.encryptedBytes])
+      }
+      return onDecryptedBytes(decrypted.decryptedBytes, progress)
+    })
+
+    if (addToCache) {
+      this.encryptedCache.add(file.uuid, { encryptedBytes: cacheEntryAggregate })
+    }
+
+    return result.error
+  }
+
+  public async deleteFile(file: FileItem): Promise<ClientDisplayableError | undefined> {
+    this.encryptedCache.remove(file.uuid)
+
+    const tokenResult = await this.api.createFileValetToken(file.remoteIdentifier, 'delete')
+
+    if (tokenResult instanceof ClientDisplayableError) {
+      return tokenResult
+    }
+
+    const result = await this.api.deleteFile(tokenResult)
+
+    if (result.error) {
+      return ClientDisplayableError.FromError(result.error)
+    }
+
+    await this.itemManager.setItemToBeDeleted(file)
+    await this.syncService.sync()
+
+    return undefined
+  }
+
+  public isFileNameFileBackupRelated(name: string): 'metadata' | 'binary' | false {
+    if (name === FileBackupsConstantsV1.MetadataFileName) {
+      return 'metadata'
+    } else if (name === FileBackupsConstantsV1.BinaryFileName) {
+      return 'binary'
+    }
+
+    return false
+  }
+
+  public async decryptBackupMetadataFile(metdataFile: FileBackupMetadataFile): Promise<FileItem | undefined> {
+    const encryptedItemsKey = new EncryptedPayload({
+      ...metdataFile.itemsKey,
+      waitingForKey: false,
+      errorDecrypting: false,
+    })
+
+    const decryptedItemsKeyResult = await DecryptItemsKeyWithUserFallback(
+      encryptedItemsKey,
+      this.encryptor,
+      this.challengor,
+    )
+
+    if (decryptedItemsKeyResult === 'failed' || decryptedItemsKeyResult === 'aborted') {
+      return undefined
+    }
+
+    const encryptedFile = new EncryptedPayload({ ...metdataFile.file, waitingForKey: false, errorDecrypting: false })
+
+    const itemsKey = new SNItemsKey(decryptedItemsKeyResult)
+
+    const decryptedFile = await this.encryptor.decryptSplitSingle<FileContent>({
+      usesItemsKey: {
+        items: [encryptedFile],
+        key: itemsKey,
+      },
+    })
+
+    if (isEncryptedPayload(decryptedFile)) {
+      return undefined
+    }
+
+    return new FileItem(decryptedFile)
+  }
+
+  public async selectFile(fileSystem: FileSystemApi): Promise<FileHandleRead | FileSystemNoSelection> {
+    const result = await fileSystem.selectFile()
+
+    return result
+  }
+
+  public async readBackupFileAndSaveDecrypted(
+    fileHandle: FileHandleRead,
+    file: FileItem,
+    fileSystem: FileSystemApi,
+  ): Promise<'success' | 'aborted' | 'failed'> {
+    const destinationDirectoryHandle = await fileSystem.selectDirectory()
+
+    if (destinationDirectoryHandle === 'aborted' || destinationDirectoryHandle === 'failed') {
+      return destinationDirectoryHandle
+    }
+
+    const destinationFileHandle = await fileSystem.createFile(destinationDirectoryHandle, file.name)
+
+    if (destinationFileHandle === 'aborted' || destinationFileHandle === 'failed') {
+      return destinationFileHandle
+    }
+
+    const result = await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => {
+      await fileSystem.saveBytes(destinationFileHandle, decryptedBytes)
+    })
+
+    await fileSystem.closeFileWriteStream(destinationFileHandle)
+
+    return result
+  }
+
+  public async readBackupFileBytesDecrypted(
+    fileHandle: FileHandleRead,
+    file: FileItem,
+    fileSystem: FileSystemApi,
+  ): Promise<Uint8Array> {
+    let bytes = new Uint8Array()
+
+    await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => {
+      bytes = new Uint8Array([...bytes, ...decryptedBytes])
+    })
+
+    return bytes
+  }
+}

packages/files/src/Domain/Service/FilesClientInterface.ts

@@ -0,0 +1,48 @@
+import { EncryptAndUploadFileOperation } from '../Operations/EncryptAndUpload'
+import { FileItem, FileMetadata } from '@standardnotes/models'
+import { ClientDisplayableError } from '@standardnotes/responses'
+import { FileDownloadProgress } from '../Types/FileDownloadProgress'
+import { FileSystemApi, FileBackupMetadataFile, FileHandleRead, FileSystemNoSelection } from '@standardnotes/services'
+
+export interface FilesClientInterface {
+  beginNewFileUpload(sizeInBytes: number): Promise<EncryptAndUploadFileOperation | ClientDisplayableError>
+
+  pushBytesForUpload(
+    operation: EncryptAndUploadFileOperation,
+    bytes: Uint8Array,
+    chunkId: number,
+    isFinalChunk: boolean,
+  ): Promise<ClientDisplayableError | undefined>
+
+  finishUpload(
+    operation: EncryptAndUploadFileOperation,
+    fileMetadata: FileMetadata,
+  ): Promise<FileItem | ClientDisplayableError>
+
+  downloadFile(
+    file: FileItem,
+    onDecryptedBytes: (bytes: Uint8Array, progress: FileDownloadProgress | undefined) => Promise<void>,
+  ): Promise<ClientDisplayableError | undefined>
+
+  deleteFile(file: FileItem): Promise<ClientDisplayableError | undefined>
+
+  minimumChunkSize(): number
+
+  isFileNameFileBackupRelated(name: string): 'metadata' | 'binary' | false
+
+  decryptBackupMetadataFile(metdataFile: FileBackupMetadataFile): Promise<FileItem | undefined>
+
+  selectFile(fileSystem: FileSystemApi): Promise<FileHandleRead | FileSystemNoSelection>
+
+  readBackupFileAndSaveDecrypted(
+    fileHandle: FileHandleRead,
+    file: FileItem,
+    fileSystem: FileSystemApi,
+  ): Promise<'success' | 'aborted' | 'failed'>
+
+  readBackupFileBytesDecrypted(
+    fileHandle: FileHandleRead,
+    file: FileItem,
+    fileSystem: FileSystemApi,
+  ): Promise<Uint8Array>
+}

packages/files/src/Domain/Service/ReadAndDecryptBackupFile.ts

@@ -0,0 +1,36 @@
+import { FileContent } from '@standardnotes/models'
+import { FileSystemApi, FileHandleRead } from '@standardnotes/services'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { OrderedByteChunker } from '@standardnotes/filepicker'
+import { FileDecryptor } from '../UseCase/FileDecryptor'
+
+export async function readAndDecryptBackupFile(
+  fileHandle: FileHandleRead,
+  file: {
+    encryptionHeader: FileContent['encryptionHeader']
+    remoteIdentifier: FileContent['remoteIdentifier']
+    encryptedChunkSizes: FileContent['encryptedChunkSizes']
+    key: FileContent['key']
+  },
+  fileSystem: FileSystemApi,
+  crypto: PureCryptoInterface,
+  onDecryptedBytes: (decryptedBytes: Uint8Array) => Promise<void>,
+): Promise<'aborted' | 'failed' | 'success'> {
+  const decryptor = new FileDecryptor(file, crypto)
+
+  const byteChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (chunk: Uint8Array) => {
+    const decryptResult = decryptor.decryptBytes(chunk)
+
+    if (!decryptResult) {
+      return
+    }
+
+    await onDecryptedBytes(decryptResult.decryptedBytes)
+  })
+
+  const readResult = await fileSystem.readFile(fileHandle, async (encryptedBytes: Uint8Array) => {
+    await byteChunker.addBytes(encryptedBytes)
+  })
+
+  return readResult
+}