feat: Automatic plaintext backup option in Preferences > Backups will backup your notes and tags into plaintext, unencrypted folders on your computer. In addition, automatic encrypted text backups preference management has moved from the top-level menu in the desktop app to Preferences > Backups. (#2322)

2023-05-02 11:05:10 -05:00
parent 3df23cdb5c
commit 7e3db49322
76 changed files with 1526 additions and 1013 deletions
--- a/packages/services/src/Domain/Backups/BackupService.spec.ts
+++ b/packages/services/src/Domain/Backups/BackupService.spec.ts
@@ -1,3 +1,7 @@
+import { HistoryServiceInterface } from './../History/HistoryServiceInterface'
+import { PayloadManagerInterface } from './../Payloads/PayloadManagerInterface'
+import { StorageServiceInterface } from './../Storage/StorageServiceInterface'
+import { SessionsClientInterface } from './../Session/SessionsClientInterface'
 import { StatusServiceInterface } from './../Status/StatusServiceInterface'
 import { FilesBackupService } from './BackupService'
 import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common'
@@ -20,6 +24,10 @@ describe('backup service', () => {
  let internalEventBus: InternalEventBusInterface
  let backupService: FilesBackupService
  let device: FileBackupsDevice
+  let session: SessionsClientInterface
+  let storage: StorageServiceInterface
+  let payloads: PayloadManagerInterface
+  let history: HistoryServiceInterface

  beforeEach(() => {
    apiService = {} as jest.Mocked<ApiServiceInterface>
@@ -41,6 +49,8 @@ describe('backup service', () => {
    device.getFileBackupReadToken = jest.fn()
    device.readNextChunk = jest.fn()

+    session = {} as jest.Mocked<SessionsClientInterface>
+
    syncService = {} as jest.Mocked<SyncServiceInterface>
    syncService.sync = jest.fn()

@@ -55,7 +65,25 @@ describe('backup service', () => {
    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
    internalEventBus.publish = jest.fn()

-    backupService = new FilesBackupService(itemManager, apiService, encryptor, device, status, crypto, internalEventBus)
+    payloads = {} as PayloadManagerInterface
+    history = {} as HistoryServiceInterface
+
+    storage = {} as StorageServiceInterface
+    storage.getValue = jest.fn().mockReturnValue('')
+
+    backupService = new FilesBackupService(
+      itemManager,
+      apiService,
+      encryptor,
+      device,
+      status,
+      crypto,
+      storage,
+      session,
+      payloads,
+      history,
+      internalEventBus,
+    )

    crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({
      state: {},
--- a/packages/services/src/Domain/Backups/BackupService.ts
+++ b/packages/services/src/Domain/Backups/BackupService.ts
@@ -1,6 +1,15 @@
+import { ApplicationStage } from './../Application/ApplicationStage'
 import { ContentType } from '@standardnotes/common'
 import { EncryptionProviderInterface } from '@standardnotes/encryption'
-import { PayloadEmitSource, FileItem, CreateEncryptedBackupFileContextPayload } from '@standardnotes/models'
+import {
+  PayloadEmitSource,
+  FileItem,
+  CreateEncryptedBackupFileContextPayload,
+  SNNote,
+  SNTag,
+  isNote,
+  NoteContent,
+} from '@standardnotes/models'
 import { ClientDisplayableError } from '@standardnotes/responses'
 import {
  FilesApiInterface,
@@ -10,16 +19,28 @@ import {
  FileBackupRecord,
  OnChunkCallback,
  BackupServiceInterface,
+  DesktopWatchedDirectoriesChanges,
 } from '@standardnotes/files'
 import { InternalEventBusInterface } from '../Internal/InternalEventBusInterface'
 import { ItemManagerInterface } from '../Item/ItemManagerInterface'
 import { AbstractService } from '../Service/AbstractService'
 import { StatusServiceInterface } from '../Status/StatusServiceInterface'
 import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
-import { log, LoggingDomain } from '../Logging'
+import { StorageServiceInterface } from '../Storage/StorageServiceInterface'
+import { StorageKey } from '../Storage/StorageKeys'
+import { SessionsClientInterface } from '../Session/SessionsClientInterface'
+import { PayloadManagerInterface } from '../Payloads/PayloadManagerInterface'
+import { HistoryServiceInterface } from '../History/HistoryServiceInterface'
+
+const PlaintextBackupsDirectoryName = 'Plaintext Backups'
+export const TextBackupsDirectoryName = 'Text Backups'
+export const FileBackupsDirectoryName = 'File Backups'

 export class FilesBackupService extends AbstractService implements BackupServiceInterface {
-  private itemsObserverDisposer: () => void
+  private filesObserverDisposer: () => void
+  private notesObserverDisposer: () => void
+  private tagsObserverDisposer: () => void
+
  private pendingFiles = new Set<string>()
  private mappingCache?: FileBackupsMapping['files']

@@ -30,45 +51,259 @@ export class FilesBackupService extends AbstractService implements BackupService
    private device: FileBackupsDevice,
    private status: StatusServiceInterface,
    private crypto: PureCryptoInterface,
+    private storage: StorageServiceInterface,
+    private session: SessionsClientInterface,
+    private payloads: PayloadManagerInterface,
+    private history: HistoryServiceInterface,
    protected override internalEventBus: InternalEventBusInterface,
  ) {
    super(internalEventBus)

-    this.itemsObserverDisposer = items.addObserver<FileItem>(ContentType.File, ({ changed, inserted, source }) => {
+    this.filesObserverDisposer = items.addObserver<FileItem>(ContentType.File, ({ changed, inserted, source }) => {
      const applicableSources = [
        PayloadEmitSource.LocalDatabaseLoaded,
        PayloadEmitSource.RemoteSaved,
        PayloadEmitSource.RemoteRetrieved,
      ]
-
      if (applicableSources.includes(source)) {
        void this.handleChangedFiles([...changed, ...inserted])
      }
    })
+
+    const noteAndTagSources = [
+      PayloadEmitSource.RemoteSaved,
+      PayloadEmitSource.RemoteRetrieved,
+      PayloadEmitSource.OfflineSyncSaved,
+    ]
+
+    this.notesObserverDisposer = items.addObserver<SNNote>(ContentType.Note, ({ changed, inserted, source }) => {
+      if (noteAndTagSources.includes(source)) {
+        void this.handleChangedNotes([...changed, ...inserted])
+      }
+    })
+
+    this.tagsObserverDisposer = items.addObserver<SNTag>(ContentType.Tag, ({ changed, inserted, source }) => {
+      if (noteAndTagSources.includes(source)) {
+        void this.handleChangedTags([...changed, ...inserted])
+      }
+    })
+  }
+
+  async importWatchedDirectoryChanges(changes: DesktopWatchedDirectoriesChanges): Promise<void> {
+    for (const change of changes) {
+      const existingItem = this.items.findItem(change.itemUuid)
+      if (!existingItem) {
+        continue
+      }
+
+      if (!isNote(existingItem)) {
+        continue
+      }
+
+      const newContent: NoteContent = {
+        ...existingItem.payload.content,
+        preview_html: undefined,
+        preview_plain: undefined,
+        text: change.content,
+      }
+
+      const payloadCopy = existingItem.payload.copy({
+        content: newContent,
+      })
+
+      await this.payloads.importPayloads([payloadCopy], this.history.getHistoryMapCopy())
+    }
  }

  override deinit() {
    super.deinit()
-    this.itemsObserverDisposer()
+    this.filesObserverDisposer()
+    this.notesObserverDisposer()
+    this.tagsObserverDisposer()
    ;(this.items as unknown) = undefined
    ;(this.api as unknown) = undefined
    ;(this.encryptor as unknown) = undefined
    ;(this.device as unknown) = undefined
    ;(this.status as unknown) = undefined
    ;(this.crypto as unknown) = undefined
+    ;(this.storage as unknown) = undefined
+    ;(this.session as unknown) = undefined
  }

-  public isFilesBackupsEnabled(): Promise<boolean> {
-    return this.device.isFilesBackupsEnabled()
+  override async handleApplicationStage(stage: ApplicationStage): Promise<void> {
+    if (stage === ApplicationStage.Launched_10) {
+      void this.automaticallyEnableTextBackupsIfPreferenceNotSet()
+    }
+  }
+
+  private async automaticallyEnableTextBackupsIfPreferenceNotSet(): Promise<void> {
+    if (this.storage.getValue(StorageKey.TextBackupsEnabled) == undefined) {
+      this.storage.setValue(StorageKey.TextBackupsEnabled, true)
+      const location = `${await this.device.getUserDocumentsDirectory()}/${this.prependWorkspacePathForPath(
+        TextBackupsDirectoryName,
+      )}`
+      this.storage.setValue(StorageKey.TextBackupsLocation, location)
+    }
+  }
+
+  openAllDirectoriesContainingBackupFiles(): void {
+    const fileBackupsLocation = this.getFilesBackupsLocation()
+    const plaintextBackupsLocation = this.getPlaintextBackupsLocation()
+    const textBackupsLocation = this.getTextBackupsLocation()
+
+    if (fileBackupsLocation) {
+      void this.device.openLocation(fileBackupsLocation)
+    }
+
+    if (plaintextBackupsLocation) {
+      void this.device.openLocation(plaintextBackupsLocation)
+    }
+
+    if (textBackupsLocation) {
+      void this.device.openLocation(textBackupsLocation)
+    }
+  }
+
+  isFilesBackupsEnabled(): boolean {
+    return this.storage.getValue(StorageKey.FileBackupsEnabled, undefined, false)
+  }
+
+  getFilesBackupsLocation(): string | undefined {
+    return this.storage.getValue(StorageKey.FileBackupsLocation)
+  }
+
+  isTextBackupsEnabled(): boolean {
+    return this.storage.getValue(StorageKey.TextBackupsEnabled, undefined, true)
+  }
+
+  prependWorkspacePathForPath(path: string): string {
+    const workspacePath = this.session.getWorkspaceDisplayIdentifier()
+
+    return `${workspacePath}/${path}`
+  }
+
+  async enableTextBackups(): Promise<void> {
+    let location = this.getTextBackupsLocation()
+    if (!location) {
+      location = await this.device.presentDirectoryPickerForLocationChangeAndTransferOld(
+        this.prependWorkspacePathForPath(TextBackupsDirectoryName),
+      )
+      if (!location) {
+        return
+      }
+    }
+
+    this.storage.setValue(StorageKey.TextBackupsEnabled, true)
+    this.storage.setValue(StorageKey.TextBackupsLocation, location)
+  }
+
+  disableTextBackups(): void {
+    this.storage.setValue(StorageKey.TextBackupsEnabled, false)
+  }
+
+  getTextBackupsLocation(): string | undefined {
+    return this.storage.getValue(StorageKey.TextBackupsLocation)
+  }
+
+  async openTextBackupsLocation(): Promise<void> {
+    const location = this.getTextBackupsLocation()
+    if (location) {
+      void this.device.openLocation(location)
+    }
+  }
+
+  async changeTextBackupsLocation(): Promise<string | undefined> {
+    const oldLocation = this.getTextBackupsLocation()
+    const newLocation = await this.device.presentDirectoryPickerForLocationChangeAndTransferOld(
+      this.prependWorkspacePathForPath(TextBackupsDirectoryName),
+      oldLocation,
+    )
+
+    if (!newLocation) {
+      return undefined
+    }
+
+    this.storage.setValue(StorageKey.TextBackupsLocation, newLocation)
+
+    return newLocation
+  }
+
+  async saveTextBackupData(data: string): Promise<void> {
+    const location = this.getTextBackupsLocation()
+    if (!location) {
+      return
+    }
+
+    return this.device.saveTextBackupData(location, data)
+  }
+
+  isPlaintextBackupsEnabled(): boolean {
+    return this.storage.getValue(StorageKey.PlaintextBackupsEnabled, undefined, false)
+  }
+
+  public async enablePlaintextBackups(): Promise<void> {
+    let location = this.getPlaintextBackupsLocation()
+    if (!location) {
+      location = await this.device.presentDirectoryPickerForLocationChangeAndTransferOld(
+        this.prependWorkspacePathForPath(PlaintextBackupsDirectoryName),
+      )
+      if (!location) {
+        return
+      }
+    }
+
+    this.storage.setValue(StorageKey.PlaintextBackupsEnabled, true)
+    this.storage.setValue(StorageKey.PlaintextBackupsLocation, location)
+
+    void this.handleChangedNotes(this.items.getItems<SNNote>(ContentType.Note))
+  }
+
+  disablePlaintextBackups(): void {
+    this.storage.setValue(StorageKey.PlaintextBackupsEnabled, false)
+    this.storage.setValue(StorageKey.PlaintextBackupsLocation, undefined)
+  }
+
+  getPlaintextBackupsLocation(): string | undefined {
+    return this.storage.getValue(StorageKey.PlaintextBackupsLocation)
+  }
+
+  async openPlaintextBackupsLocation(): Promise<void> {
+    const location = this.getPlaintextBackupsLocation()
+    if (location) {
+      void this.device.openLocation(location)
+    }
+  }
+
+  async changePlaintextBackupsLocation(): Promise<string | undefined> {
+    const oldLocation = this.getPlaintextBackupsLocation()
+    const newLocation = await this.device.presentDirectoryPickerForLocationChangeAndTransferOld(
+      this.prependWorkspacePathForPath(PlaintextBackupsDirectoryName),
+      oldLocation,
+    )
+
+    if (!newLocation) {
+      return undefined
+    }
+
+    this.storage.setValue(StorageKey.PlaintextBackupsLocation, newLocation)
+
+    return newLocation
  }

  public async enableFilesBackups(): Promise<void> {
-    await this.device.enableFilesBackups()
-
-    if (!(await this.isFilesBackupsEnabled())) {
-      return
+    let location = this.getFilesBackupsLocation()
+    if (!location) {
+      location = await this.device.presentDirectoryPickerForLocationChangeAndTransferOld(
+        this.prependWorkspacePathForPath(FileBackupsDirectoryName),
+      )
+      if (!location) {
+        return
+      }
    }

+    this.storage.setValue(StorageKey.FileBackupsEnabled, true)
+    this.storage.setValue(StorageKey.FileBackupsLocation, location)
+
    this.backupAllFiles()
  }

@@ -78,24 +313,39 @@ export class FilesBackupService extends AbstractService implements BackupService
    void this.handleChangedFiles(files)
  }

-  public disableFilesBackups(): Promise<void> {
-    return this.device.disableFilesBackups()
+  public disableFilesBackups(): void {
+    this.storage.setValue(StorageKey.FileBackupsEnabled, false)
  }

-  public changeFilesBackupsLocation(): Promise<string | undefined> {
-    return this.device.changeFilesBackupsLocation()
+  public async changeFilesBackupsLocation(): Promise<string | undefined> {
+    const oldLocation = this.getFilesBackupsLocation()
+    const newLocation = await this.device.presentDirectoryPickerForLocationChangeAndTransferOld(
+      this.prependWorkspacePathForPath(FileBackupsDirectoryName),
+      oldLocation,
+    )
+    if (!newLocation) {
+      return undefined
+    }
+
+    this.storage.setValue(StorageKey.FileBackupsLocation, newLocation)
+
+    return newLocation
  }

-  public getFilesBackupsLocation(): Promise<string> {
-    return this.device.getFilesBackupsLocation()
+  public async openFilesBackupsLocation(): Promise<void> {
+    const location = this.getFilesBackupsLocation()
+    if (location) {
+      void this.device.openLocation(location)
+    }
  }

-  public openFilesBackupsLocation(): Promise<void> {
-    return this.device.openFilesBackupsLocation()
-  }
+  private async getBackupsMappingFromDisk(): Promise<FileBackupsMapping['files'] | undefined> {
+    const location = this.getFilesBackupsLocation()
+    if (!location) {
+      return undefined
+    }

-  private async getBackupsMappingFromDisk(): Promise<FileBackupsMapping['files']> {
-    const result = (await this.device.getFilesBackupsMappingFile()).files
+    const result = (await this.device.getFilesBackupsMappingFile(location)).files

    this.mappingCache = result

@@ -106,30 +356,39 @@ export class FilesBackupService extends AbstractService implements BackupService
    this.mappingCache = undefined
  }

-  private async getBackupsMappingFromCache(): Promise<FileBackupsMapping['files']> {
+  private async getBackupsMappingFromCache(): Promise<FileBackupsMapping['files'] | undefined> {
    return this.mappingCache ?? (await this.getBackupsMappingFromDisk())
  }

  public async getFileBackupInfo(file: { uuid: string }): Promise<FileBackupRecord | undefined> {
    const mapping = await this.getBackupsMappingFromCache()
+    if (!mapping) {
+      return undefined
+    }
+
    const record = mapping[file.uuid]
    return record
  }

+  public getFileBackupAbsolutePath(record: FileBackupRecord): string {
+    const location = this.getFilesBackupsLocation()
+    return `${location}/${record.relativePath}`
+  }
+
  public async openFileBackup(record: FileBackupRecord): Promise<void> {
-    await this.device.openFileBackup(record)
+    const location = this.getFileBackupAbsolutePath(record)
+    await this.device.openLocation(location)
  }

  private async handleChangedFiles(files: FileItem[]): Promise<void> {
-    if (files.length === 0) {
-      return
-    }
-
-    if (!(await this.isFilesBackupsEnabled())) {
+    if (files.length === 0 || !this.isFilesBackupsEnabled()) {
      return
    }

    const mapping = await this.getBackupsMappingFromDisk()
+    if (!mapping) {
+      throw new ClientDisplayableError('No backups mapping found')
+    }

    for (const file of files) {
      if (this.pendingFiles.has(file.uuid)) {
@@ -150,6 +409,36 @@ export class FilesBackupService extends AbstractService implements BackupService
    this.invalidateMappingCache()
  }

+  private async handleChangedNotes(notes: SNNote[]): Promise<void> {
+    if (notes.length === 0 || !this.isPlaintextBackupsEnabled()) {
+      return
+    }
+
+    const location = this.getPlaintextBackupsLocation()
+    if (!location) {
+      throw new ClientDisplayableError('No plaintext backups location found')
+    }
+
+    for (const note of notes) {
+      const tags = this.items.getSortedTagsForItem(note)
+      const tagNames = tags.map((tag) => this.items.getTagLongTitle(tag))
+      await this.device.savePlaintextNoteBackup(location, note.uuid, note.title, tagNames, note.text)
+    }
+
+    await this.device.persistPlaintextBackupsMappingFile(location)
+  }
+
+  private async handleChangedTags(tags: SNTag[]): Promise<void> {
+    if (tags.length === 0 || !this.isPlaintextBackupsEnabled()) {
+      return
+    }
+
+    for (const tag of tags) {
+      const notes = this.items.referencesForItem<SNNote>(tag, ContentType.Note)
+      await this.handleChangedNotes(notes)
+    }
+  }
+
  async readEncryptedFileFromBackup(uuid: string, onChunk: OnChunkCallback): Promise<'success' | 'failed' | 'aborted'> {
    const fileBackup = await this.getFileBackupInfo({ uuid })

@@ -157,7 +446,8 @@ export class FilesBackupService extends AbstractService implements BackupService
      return 'failed'
    }

-    const token = await this.device.getFileBackupReadToken(fileBackup)
+    const path = `${this.getFilesBackupsLocation()}/${fileBackup.relativePath}/${fileBackup.binaryFileName}`
+    const token = await this.device.getFileBackupReadToken(path)

    let readMore = true
    let index = 0
@@ -176,7 +466,10 @@ export class FilesBackupService extends AbstractService implements BackupService
  }

  private async performBackupOperation(file: FileItem): Promise<'success' | 'failed' | 'aborted'> {
-    log(LoggingDomain.FilesBackups, 'Backing up file locally', file.uuid)
+    const location = this.getFilesBackupsLocation()
+    if (!location) {
+      return 'failed'
+    }

    const messageId = this.status.addMessage(`Backing up file ${file.name}...`)

@@ -189,6 +482,7 @@ export class FilesBackupService extends AbstractService implements BackupService
    const itemsKey = this.items.getDisplayableItemsKeys().find((k) => k.uuid === encryptedFile.items_key_id)

    if (!itemsKey) {
+      this.status.removeMessage(messageId)
      return 'failed'
    }

@@ -201,6 +495,7 @@ export class FilesBackupService extends AbstractService implements BackupService
    const token = await this.api.createFileValetToken(file.remoteIdentifier, 'read')

    if (token instanceof ClientDisplayableError) {
+      this.status.removeMessage(messageId)
      return 'failed'
    }

@@ -218,7 +513,7 @@ export class FilesBackupService extends AbstractService implements BackupService

    const metaFileAsString = JSON.stringify(metaFile, null, 2)

-    const result = await this.device.saveFilesBackupsFile(file.uuid, metaFileAsString, {
+    const result = await this.device.saveFilesBackupsFile(location, file.uuid, metaFileAsString, {
      chunkSizes: file.encryptedChunkSizes,
      url: this.api.getFilesDownloadUrl(),
      valetToken: token,
@@ -235,4 +530,18 @@ export class FilesBackupService extends AbstractService implements BackupService

    return result
  }
+
+  /**
+   * Not presently used or enabled. It works, but presently has the following edge cases:
+   * 1. Editing the note directly in SN triggers an immediate backup which triggers a file change which triggers the observer
+   * 2. Since changes are based on filenames, a note with the same title as another may not properly map to the correct uuid
+   * 3. Opening the file triggers a watch event from Node's watch API.
+   * 4. Gives web code ability to monitor arbitrary locations. Needs whitelisting mechanism.
+   */
+  disabledExperimental_monitorPlaintextBackups(): void {
+    const location = this.getPlaintextBackupsLocation()
+    if (location) {
+      void this.device.monitorPlaintextBackupsLocationForChanges(location)
+    }
+  }
 }