feat: add snjs package

packages/snjs/.browserslistrc

@@ -0,0 +1,7 @@
+Edge 16
+Firefox 53
+Chrome 57
+Safari 11
+Opera 44
+ios 11
+ChromeAndroid 84

packages/snjs/.eslintignore

@@ -0,0 +1,5 @@
+node_modules
+dist
+test
+*.config.js
+mocha/**/*

packages/snjs/.eslintrc

@@ -0,0 +1,9 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "./linter.tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-explicit-any": ["warn", { "ignoreRestArgs": true }]
+  }
+}

packages/snjs/babel.config.js

@@ -0,0 +1,7 @@
+module.exports = function (api) {
+  api.cache.forever();
+
+  return {
+    presets: ['@babel/preset-env'],
+  };
+};

packages/snjs/jest-global.ts

@@ -0,0 +1,2 @@
+//@ts-ignore
+global['__VERSION__'] = global['SnjsVersion'] = require('./package.json').version

packages/snjs/jest.config.js

@@ -0,0 +1,37 @@
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const base = require('../../node_modules/@standardnotes/config/src/jest.json');
+
+module.exports = {
+  ...base,
+  moduleNameMapper: {
+    '@Lib/(.*)': '<rootDir>/lib/$1',
+    '@Services/(.*)': '<rootDir>/lib/Services/$1',
+  },
+  globals: {
+    'ts-jest': {
+      tsconfig: '<rootDir>/lib/tsconfig.json',
+      isolatedModules: true,
+      babelConfig: 'babel.config.js',
+    },
+  },
+  clearMocks: true,
+  collectCoverageFrom: ['lib/**/{!(index),}.ts'],
+  coverageDirectory: 'coverage',
+  coverageReporters: ['json', 'text', 'html'],
+  resetMocks: true,
+  resetModules: true,
+  roots: ['<rootDir>/lib'],
+  setupFiles: ['<rootDir>/jest-global.ts'],
+  setupFilesAfterEnv: [],
+  transform: {
+    '^.+\\.(ts|js)?$': 'ts-jest',
+  },
+  coverageThreshold: {
+    global: {
+      branches: 13,
+      functions: 22,
+      lines: 27,
+      statements: 28,
+    },
+  },
+}

packages/snjs/jsdoc.json

@@ -0,0 +1,22 @@
+{
+  "source": {
+    "includePattern": ".+\\.js(doc|x)?$",
+    "include": ["lib"],
+    "exclude": ["node_modules"]
+  },
+  "recurseDepth": 10,
+  "opts": {
+    "destination": "./docs/",
+    "recurse": true,
+    "template": "node_modules/docdash"
+  },
+  "tags": {
+    "allowUnknownTags": true,
+    "dictionaries": ["jsdoc", "closure"]
+  },
+  "docdash": {
+    "meta": {
+      "title": "SNJS Documentation"
+    }
+  }
+}

packages/snjs/lib/Application/Application.spec.ts

@@ -0,0 +1,151 @@
+import { SNLog } from './../Log'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import {
+  AlertService,
+  DeviceInterface,
+  Environment,
+  namespacedKey,
+  Platform,
+  RawStorageKey,
+} from '@standardnotes/services'
+import { SNApplication } from './Application'
+
+describe('application', () => {
+  // eslint-disable-next-line no-console
+  SNLog.onLog = console.log
+  SNLog.onError = console.error
+
+  let application: SNApplication
+  let device: DeviceInterface
+  let crypto: PureCryptoInterface
+
+  beforeEach(async () => {
+    const identifier = '123'
+
+    crypto = {} as jest.Mocked<PureCryptoInterface>
+    crypto.initialize = jest.fn()
+
+    device = {} as jest.Mocked<DeviceInterface>
+    device.openDatabase = jest.fn().mockResolvedValue(true)
+    device.getAllRawDatabasePayloads = jest.fn().mockReturnValue([])
+    device.setRawStorageValue = jest.fn()
+    device.getRawStorageValue = jest.fn().mockImplementation((key) => {
+      if (key === namespacedKey(identifier, RawStorageKey.SnjsVersion)) {
+        return '10.0.0'
+      }
+      return undefined
+    })
+    device.getDatabaseKeys = async () => {
+      return Promise.resolve(['1', '2', '3'])
+    }
+
+    application = new SNApplication({
+      environment: Environment.Mobile,
+      platform: Platform.Ios,
+      deviceInterface: device,
+      crypto: crypto,
+      alertService: {} as jest.Mocked<AlertService>,
+      identifier: identifier,
+      defaultHost: 'localhost',
+      appVersion: '1.0',
+    })
+
+    await application.prepareForLaunch({ receiveChallenge: jest.fn() })
+  })
+
+  it('diagnostics', async () => {
+    const diagnostics = await application.getDiagnostics()
+
+    expect(diagnostics).toEqual(
+      expect.objectContaining({
+        application: expect.objectContaining({
+          appVersion: '1.0',
+          environment: 3,
+          platform: 1,
+        }),
+        payloads: {
+          integrityPayloads: [],
+          nonDeletedItemCount: 0,
+          invalidPayloadsCount: 0,
+        },
+        items: { allIds: [] },
+        storage: {
+          storagePersistable: false,
+          persistencePolicy: 'Default',
+          encryptionPolicy: 'Default',
+          needsPersist: false,
+          currentPersistPromise: false,
+          isStorageWrapped: false,
+          allRawPayloadsCount: 0,
+          databaseKeys: ['1', '2', '3'],
+        },
+        encryption: expect.objectContaining({
+          getLatestVersion: '004',
+          hasAccount: false,
+          getUserVersion: undefined,
+          upgradeAvailable: false,
+          accountUpgradeAvailable: false,
+          passcodeUpgradeAvailable: false,
+          hasPasscode: false,
+          isPasscodeLocked: false,
+          itemsEncryption: expect.objectContaining({
+            itemsKeysIds: [],
+          }),
+          rootKeyEncryption: expect.objectContaining({
+            hasRootKey: false,
+            keyMode: 'RootKeyNone',
+            hasRootKeyWrapper: false,
+            hasAccount: false,
+            hasPasscode: false,
+          }),
+        }),
+        api: {
+          hasSession: false,
+          user: undefined,
+          registering: false,
+          authenticating: false,
+          changing: false,
+          refreshingSession: false,
+          filesHost: undefined,
+          host: 'localhost',
+        },
+        session: {
+          isSessionRenewChallengePresented: false,
+          online: false,
+          offline: true,
+          isSignedIn: false,
+          isSignedIntoFirstPartyServer: false,
+        },
+        sync: {
+          syncToken: undefined,
+          cursorToken: undefined,
+          lastSyncDate: undefined,
+          outOfSync: false,
+          completedOnlineDownloadFirstSync: false,
+          clientLocked: false,
+          databaseLoaded: false,
+          syncLock: false,
+          dealloced: false,
+          itemsNeedingSync: [],
+          itemsNeedingSyncCount: 0,
+          pendingRequestCount: 0,
+        },
+        protections: expect.objectContaining({
+          getLastSessionLength: undefined,
+          hasProtectionSources: false,
+          hasUnprotectedAccessSession: true,
+          hasBiometricsEnabled: false,
+        }),
+        keyRecovery: { queueLength: 0, isProcessingQueue: false },
+        features: {
+          roles: [],
+          features: [],
+          enabledExperimentalFeatures: [],
+          needsInitialFeaturesUpdate: true,
+          completedSuccessfulFeaturesRetrieval: false,
+        },
+        migrations: { activeMigrations: [] },
+      }),
+    )
+  })
+})

packages/snjs/lib/Application/Event.ts

@@ -0,0 +1,90 @@
+import { SyncEvent } from '@standardnotes/services'
+export { SyncEvent }
+
+export enum ApplicationEvent {
+  SignedIn = 2,
+  SignedOut = 3,
+
+  /** When a full, potentially multi-page sync completes */
+  CompletedFullSync = 5,
+
+  FailedSync = 6,
+  HighLatencySync = 7,
+  EnteredOutOfSync = 8,
+  ExitedOutOfSync = 9,
+
+  /**
+   * The application has finished it `prepareForLaunch` state and is now ready for unlock
+   * Called when the application has initialized and is ready for launch, but before
+   * the application has been unlocked, if applicable. Use this to do pre-launch
+   * configuration, but do not attempt to access user data like notes or tags.
+   */
+  Started = 10,
+
+  /**
+   * The applicaiton is fully unlocked and ready for i/o
+   * Called when the application has been fully decrypted and unlocked. Use this to
+   * to begin streaming data like notes and tags.
+   */
+  Launched = 11,
+  LocalDataLoaded = 12,
+
+  /**
+   * When the root key or root key wrapper changes. Includes events like account state
+   * changes (registering, signing in, changing pw, logging out) and passcode state
+   * changes (adding, removing, changing).
+   */
+  KeyStatusChanged = 13,
+
+  MajorDataChange = 14,
+  CompletedRestart = 15,
+  LocalDataIncrementalLoad = 16,
+  SyncStatusChanged = 17,
+  WillSync = 18,
+  InvalidSyncSession = 19,
+  LocalDatabaseReadError = 20,
+  LocalDatabaseWriteError = 21,
+
+  /** When a single roundtrip completes with sync, in a potentially multi-page sync request.
+   * If just a single roundtrip, this event will be triggered, along with CompletedFullSync */
+  CompletedIncrementalSync = 22,
+
+  /**
+   * The application has loaded all pending migrations (but not run any, except for the base one),
+   * and consumers may now call `hasPendingMigrations`
+   */
+  MigrationsLoaded = 23,
+
+  /** When StorageService is ready to start servicing read/write requests */
+  StorageReady = 24,
+
+  PreferencesChanged = 25,
+  UnprotectedSessionBegan = 26,
+  UserRolesChanged = 27,
+  FeaturesUpdated = 28,
+  UnprotectedSessionExpired = 29,
+  /** Called when the app first launches and after first sync request made after sign in */
+  CompletedInitialSync = 30,
+}
+
+export function applicationEventForSyncEvent(syncEvent: SyncEvent) {
+  return (
+    {
+      [SyncEvent.SyncCompletedWithAllItemsUploaded]: ApplicationEvent.CompletedFullSync,
+      [SyncEvent.SingleRoundTripSyncCompleted]: ApplicationEvent.CompletedIncrementalSync,
+      [SyncEvent.SyncError]: ApplicationEvent.FailedSync,
+      [SyncEvent.SyncTakingTooLong]: ApplicationEvent.HighLatencySync,
+      [SyncEvent.EnterOutOfSync]: ApplicationEvent.EnteredOutOfSync,
+      [SyncEvent.ExitOutOfSync]: ApplicationEvent.ExitedOutOfSync,
+      [SyncEvent.LocalDataLoaded]: ApplicationEvent.LocalDataLoaded,
+      [SyncEvent.MajorDataChange]: ApplicationEvent.MajorDataChange,
+      [SyncEvent.LocalDataIncrementalLoad]: ApplicationEvent.LocalDataIncrementalLoad,
+      [SyncEvent.StatusChanged]: ApplicationEvent.SyncStatusChanged,
+      [SyncEvent.SyncWillBegin]: ApplicationEvent.WillSync,
+      [SyncEvent.InvalidSession]: ApplicationEvent.InvalidSyncSession,
+      [SyncEvent.DatabaseReadError]: ApplicationEvent.LocalDatabaseReadError,
+      [SyncEvent.DatabaseWriteError]: ApplicationEvent.LocalDatabaseWriteError,
+      [SyncEvent.DownloadFirstSyncCompleted]: ApplicationEvent.CompletedInitialSync,
+    } as any
+  )[syncEvent]
+}

packages/snjs/lib/Application/LiveItem.ts

@@ -0,0 +1,34 @@
+import { DecryptedItemInterface } from '@standardnotes/models'
+import { SNApplication } from './Application'
+
+/** Keeps an item reference up to date with changes */
+export class LiveItem<T extends DecryptedItemInterface> {
+  public item: T
+  private removeObserver: () => void
+
+  constructor(uuid: string, application: SNApplication, onChange?: (item: T) => void) {
+    this.item = application.items.findSureItem(uuid)
+
+    onChange && onChange(this.item)
+
+    this.removeObserver = application.streamItems(this.item.content_type, ({ changed, inserted }) => {
+      const matchingItem = [...changed, ...inserted].find((item) => {
+        return item.uuid === uuid
+      })
+
+      if (matchingItem) {
+        this.item = matchingItem as T
+        onChange && onChange(this.item)
+      }
+    })
+  }
+
+  public deinit() {
+    if (!this.removeObserver) {
+      console.error('A LiveItem is attempting to be deinited more than once.')
+    } else {
+      this.removeObserver()
+      ;(this.removeObserver as unknown) = undefined
+    }
+  }
+}

packages/snjs/lib/Application/Options/ApplicationOptions.ts

@@ -0,0 +1,16 @@
+import { ApplicationOptionsWhichHaveDefaults } from './Defaults'
+import {
+  ApplicationDisplayOptions,
+  ApplicationOptionalConfiguratioOptions,
+  ApplicationSyncOptions,
+} from './OptionalOptions'
+import { RequiredApplicationOptions } from './RequiredOptions'
+
+export type ApplicationConstructorOptions = RequiredApplicationOptions &
+  Partial<ApplicationSyncOptions & ApplicationDisplayOptions & ApplicationOptionalConfiguratioOptions>
+
+export type FullyResolvedApplicationOptions = RequiredApplicationOptions &
+  ApplicationSyncOptions &
+  ApplicationDisplayOptions &
+  ApplicationOptionalConfiguratioOptions &
+  ApplicationOptionsWhichHaveDefaults

packages/snjs/lib/Application/Options/Defaults.ts

@@ -0,0 +1,11 @@
+import { ApplicationDisplayOptions, ApplicationSyncOptions } from './OptionalOptions'
+
+export interface ApplicationOptionsWhichHaveDefaults {
+  loadBatchSize: ApplicationSyncOptions['loadBatchSize']
+  supportsFileNavigation: ApplicationDisplayOptions['supportsFileNavigation']
+}
+
+export const ApplicationOptionsDefaults: ApplicationOptionsWhichHaveDefaults = {
+  loadBatchSize: 700,
+  supportsFileNavigation: false,
+}

packages/snjs/lib/Application/Options/OptionalOptions.ts

@@ -0,0 +1,25 @@
+export interface ApplicationSyncOptions {
+  /**
+   * The size of the item batch to decrypt and render upon application load.
+   */
+  loadBatchSize: number
+}
+
+export interface ApplicationDisplayOptions {
+  supportsFileNavigation: boolean
+}
+
+export interface ApplicationOptionalConfiguratioOptions {
+  /**
+   * Gives consumers the ability to provide their own custom
+   * subclass for a service. swapClasses should be an array of key/value pairs
+   * consisting of keys 'swap' and 'with'. 'swap' is the base class you wish to replace,
+   * and 'with' is the custom subclass to use.
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  swapClasses?: { swap: any; with: any }[]
+  /**
+   * URL for WebSocket providing permissions and roles information.
+   */
+  webSocketUrl?: string
+}

packages/snjs/lib/Application/Options/RequiredOptions.ts

@@ -0,0 +1,42 @@
+import { ApplicationIdentifier } from '@standardnotes/common'
+import { AlertService, DeviceInterface, Environment, Platform } from '@standardnotes/services'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+
+export interface RequiredApplicationOptions {
+  /**
+   * The Environment that identifies your application.
+   */
+  environment: Environment
+  /**
+   * The Platform that identifies your application.
+   */
+  platform: Platform
+  /**
+   * The device interface that provides platform specific
+   * utilities that are used to read/write raw values from/to the database or value storage.
+   */
+  deviceInterface: DeviceInterface
+  /**
+   * The platform-dependent implementation of SNPureCrypto to use.
+   * Web uses SNWebCrypto, mobile uses SNReactNativeCrypto.
+   */
+  crypto: PureCryptoInterface
+  /**
+   * The platform-dependent implementation of alert service.
+   */
+  alertService: AlertService
+  /**
+   * A unique persistent identifier to namespace storage and other
+   * persistent properties. For an ephemeral runtime identifier, use ephemeralIdentifier.
+   */
+  identifier: ApplicationIdentifier
+
+  /**
+   * Default host to use in ApiService.
+   */
+  defaultHost: string
+  /**
+   * Version of client application.
+   */
+  appVersion: string
+}

packages/snjs/lib/Application/Platforms.ts

@@ -0,0 +1,55 @@
+import { Environment, Platform } from '@standardnotes/services'
+
+export function platformFromString(string: string) {
+  const map: Record<string, Platform> = {
+    'mac-web': Platform.MacWeb,
+    'mac-desktop': Platform.MacDesktop,
+    'linux-web': Platform.LinuxWeb,
+    'linux-desktop': Platform.LinuxDesktop,
+    'windows-web': Platform.WindowsWeb,
+    'windows-desktop': Platform.WindowsDesktop,
+    ios: Platform.Ios,
+    android: Platform.Android,
+  }
+  return map[string]
+}
+
+export function platformToString(platform: Platform) {
+  const map = {
+    [Platform.MacWeb]: 'mac-web',
+    [Platform.MacDesktop]: 'mac-desktop',
+    [Platform.LinuxWeb]: 'linux-web',
+    [Platform.LinuxDesktop]: 'linux-desktop',
+    [Platform.WindowsWeb]: 'windows-web',
+    [Platform.WindowsDesktop]: 'windows-desktop',
+    [Platform.Ios]: 'ios',
+    [Platform.Android]: 'android',
+  }
+  return map[platform]
+}
+
+export function environmentFromString(string: string) {
+  const map: Record<string, Environment> = {
+    web: Environment.Web,
+    desktop: Environment.Desktop,
+    mobile: Environment.Mobile,
+  }
+  return map[string]
+}
+
+export function environmentToString(environment: Environment) {
+  const map = {
+    [Environment.Web]: 'web',
+    [Environment.Desktop]: 'desktop',
+    [Environment.Mobile]: 'mobile',
+  }
+  return map[environment]
+}
+
+export function isEnvironmentWebOrDesktop(environment: Environment) {
+  return environment === Environment.Web || environment === Environment.Desktop
+}
+
+export function isEnvironmentMobile(environment: Environment) {
+  return environment === Environment.Mobile
+}

packages/snjs/lib/Application/index.ts

@@ -0,0 +1,4 @@
+export * from './Application'
+export * from './Event'
+export * from './LiveItem'
+export * from './Platforms'

packages/snjs/lib/ApplicationGroup/AppGroupCallback.ts

@@ -0,0 +1,6 @@
+import { AppGroupManagedApplication, DeviceInterface } from '@standardnotes/services'
+import { ApplicationDescriptor } from './ApplicationDescriptor'
+
+export type AppGroupCallback<D extends DeviceInterface = DeviceInterface> = {
+  applicationCreator: (descriptor: ApplicationDescriptor, deviceInterface: D) => Promise<AppGroupManagedApplication>
+}

packages/snjs/lib/ApplicationGroup/ApplicationDescriptor.ts

@@ -0,0 +1,7 @@
+import { ApplicationIdentifier } from '@standardnotes/common'
+
+export type ApplicationDescriptor = {
+  identifier: ApplicationIdentifier
+  label: string
+  primary: boolean
+}

packages/snjs/lib/ApplicationGroup/ApplicationGroup.ts

@@ -0,0 +1,247 @@
+import {
+  AbstractService,
+  AppGroupManagedApplication,
+  DeinitSource,
+  DeinitCallback,
+  DeviceInterface,
+  DeinitMode,
+  InternalEventBus,
+  InternalEventBusInterface,
+  RawStorageKey,
+} from '@standardnotes/services'
+import { UuidGenerator } from '@standardnotes/utils'
+import { AppGroupCallback } from './AppGroupCallback'
+import { ApplicationGroupEvent, ApplicationGroupEventData } from './ApplicationGroupEvent'
+import { DescriptorRecord } from './DescriptorRecord'
+import { ApplicationDescriptor } from './ApplicationDescriptor'
+
+export class SNApplicationGroup<D extends DeviceInterface = DeviceInterface> extends AbstractService<
+  ApplicationGroupEvent,
+  | ApplicationGroupEventData[ApplicationGroupEvent.PrimaryApplicationSet]
+  | ApplicationGroupEventData[ApplicationGroupEvent.DeviceWillRestart]
+  | ApplicationGroupEventData[ApplicationGroupEvent.DescriptorsDataChanged]
+> {
+  public primaryApplication!: AppGroupManagedApplication
+  private descriptorRecord!: DescriptorRecord
+  callback!: AppGroupCallback<D>
+
+  constructor(public device: D, internalEventBus?: InternalEventBusInterface) {
+    if (internalEventBus === undefined) {
+      internalEventBus = new InternalEventBus()
+    }
+
+    super(internalEventBus)
+  }
+
+  override deinit() {
+    super.deinit()
+
+    this.device.deinit()
+    ;(this.device as unknown) = undefined
+    ;(this.callback as unknown) = undefined
+    ;(this.primaryApplication as unknown) = undefined
+    ;(this.onApplicationDeinit as unknown) = undefined
+  }
+
+  public async initialize(callback: AppGroupCallback<D>): Promise<void> {
+    if (this.device.isDeviceDestroyed()) {
+      throw 'Attempting to initialize new application while device is destroyed.'
+    }
+
+    this.callback = callback
+
+    this.descriptorRecord = (await this.device.getJsonParsedRawStorageValue(
+      RawStorageKey.DescriptorRecord,
+    )) as DescriptorRecord
+
+    if (!this.descriptorRecord) {
+      await this.createNewDescriptorRecord()
+    }
+
+    let primaryDescriptor = this.findPrimaryDescriptor()
+    if (!primaryDescriptor) {
+      console.error('No primary application descriptor found. Ensure migrations have been run.')
+      primaryDescriptor = this.getDescriptors()[0]
+
+      this.setDescriptorAsPrimary(primaryDescriptor)
+
+      await this.persistDescriptors()
+    }
+
+    const application = await this.buildApplication(primaryDescriptor)
+
+    this.primaryApplication = application
+
+    await this.notifyEvent(ApplicationGroupEvent.PrimaryApplicationSet, { application: application })
+  }
+
+  private async createNewDescriptorRecord() {
+    /**
+     * The identifier 'standardnotes' is used because this was the
+     * database name of Standard Notes web/desktop
+     * */
+    const identifier = 'standardnotes'
+    const descriptorRecord: DescriptorRecord = {
+      [identifier]: {
+        identifier: identifier,
+        label: 'Main Workspace',
+        primary: true,
+      },
+    }
+
+    void this.device.setRawStorageValue(RawStorageKey.DescriptorRecord, JSON.stringify(descriptorRecord))
+
+    this.descriptorRecord = descriptorRecord
+
+    await this.persistDescriptors()
+  }
+
+  public getDescriptors() {
+    return Object.values(this.descriptorRecord)
+  }
+
+  private findPrimaryDescriptor() {
+    for (const descriptor of this.getDescriptors()) {
+      if (descriptor.primary) {
+        return descriptor
+      }
+    }
+    return undefined
+  }
+
+  async signOutAllWorkspaces() {
+    await this.primaryApplication.user.signOut(false, DeinitSource.SignOutAll)
+  }
+
+  onApplicationDeinit: DeinitCallback = (
+    application: AppGroupManagedApplication,
+    mode: DeinitMode,
+    source: DeinitSource,
+  ) => {
+    if (this.primaryApplication === application) {
+      ;(this.primaryApplication as unknown) = undefined
+    }
+
+    const performSyncronously = async () => {
+      if (source === DeinitSource.SignOut) {
+        void this.removeDescriptor(this.descriptorForApplication(application))
+      }
+
+      const descriptors = this.getDescriptors()
+
+      if (descriptors.length === 0 || source === DeinitSource.SignOutAll) {
+        const identifiers = descriptors.map((d) => d.identifier)
+
+        this.descriptorRecord = {}
+
+        const { killsApplication } = await this.device.clearAllDataFromDevice(identifiers)
+
+        if (killsApplication) {
+          return
+        }
+      }
+
+      const device = this.device
+
+      void this.notifyEvent(ApplicationGroupEvent.DeviceWillRestart, { source, mode })
+
+      this.deinit()
+
+      if (mode === DeinitMode.Hard) {
+        device.performHardReset()
+      } else {
+        device.performSoftReset()
+      }
+    }
+
+    void performSyncronously()
+  }
+
+  public setDescriptorAsPrimary(primaryDescriptor: ApplicationDescriptor) {
+    for (const descriptor of this.getDescriptors()) {
+      descriptor.primary = descriptor === primaryDescriptor
+    }
+  }
+
+  private async persistDescriptors() {
+    await this.device.setRawStorageValue(RawStorageKey.DescriptorRecord, JSON.stringify(this.descriptorRecord))
+
+    void this.notifyEvent(ApplicationGroupEvent.DescriptorsDataChanged, { descriptors: this.descriptorRecord })
+  }
+
+  public renameDescriptor(descriptor: ApplicationDescriptor, label: string) {
+    descriptor.label = label
+
+    void this.persistDescriptors()
+  }
+
+  public removeDescriptor(descriptor: ApplicationDescriptor) {
+    delete this.descriptorRecord[descriptor.identifier]
+
+    const descriptors = this.getDescriptors()
+    if (descriptor.primary && descriptors.length > 0) {
+      this.setDescriptorAsPrimary(descriptors[0])
+    }
+
+    return this.persistDescriptors()
+  }
+
+  public removeAllDescriptors() {
+    this.descriptorRecord = {}
+
+    return this.persistDescriptors()
+  }
+
+  private descriptorForApplication(application: AppGroupManagedApplication) {
+    return this.descriptorRecord[application.identifier]
+  }
+
+  private createNewApplicationDescriptor(label?: string) {
+    const identifier = UuidGenerator.GenerateUuid()
+    const index = this.getDescriptors().length + 1
+
+    const descriptor: ApplicationDescriptor = {
+      identifier: identifier,
+      label: label || `Workspace ${index}`,
+      primary: false,
+    }
+
+    return descriptor
+  }
+
+  private async createNewPrimaryDescriptor(label?: string): Promise<void> {
+    const descriptor = this.createNewApplicationDescriptor(label)
+
+    this.descriptorRecord[descriptor.identifier] = descriptor
+
+    this.setDescriptorAsPrimary(descriptor)
+
+    await this.persistDescriptors()
+  }
+
+  public async unloadCurrentAndCreateNewDescriptor(label?: string): Promise<void> {
+    await this.createNewPrimaryDescriptor(label)
+
+    if (this.primaryApplication) {
+      this.primaryApplication.deinit(this.primaryApplication.getDeinitMode(), DeinitSource.SwitchWorkspace)
+    }
+  }
+
+  public async unloadCurrentAndActivateDescriptor(descriptor: ApplicationDescriptor) {
+    this.setDescriptorAsPrimary(descriptor)
+
+    await this.persistDescriptors()
+
+    if (this.primaryApplication) {
+      this.primaryApplication.deinit(this.primaryApplication.getDeinitMode(), DeinitSource.SwitchWorkspace)
+    }
+  }
+
+  private async buildApplication(descriptor: ApplicationDescriptor) {
+    const application = await this.callback.applicationCreator(descriptor, this.device)
+
+    application.setOnDeinit(this.onApplicationDeinit)
+
+    return application
+  }
+}

packages/snjs/lib/ApplicationGroup/ApplicationGroupEvent.ts

@@ -0,0 +1,21 @@
+import { ApplicationInterface, DeinitMode, DeinitSource } from '@standardnotes/services'
+import { DescriptorRecord } from './DescriptorRecord'
+
+export enum ApplicationGroupEvent {
+  PrimaryApplicationSet = 'PrimaryApplicationSet',
+  DescriptorsDataChanged = 'DescriptorsDataChanged',
+  DeviceWillRestart = 'DeviceWillRestart',
+}
+
+export interface ApplicationGroupEventData {
+  [ApplicationGroupEvent.PrimaryApplicationSet]: {
+    application: ApplicationInterface
+  }
+  [ApplicationGroupEvent.DeviceWillRestart]: {
+    source: DeinitSource
+    mode: DeinitMode
+  }
+  [ApplicationGroupEvent.DescriptorsDataChanged]: {
+    descriptors: DescriptorRecord
+  }
+}

packages/snjs/lib/ApplicationGroup/DeinitCallback.ts

@@ -0,0 +1,3 @@
+import { AppGroupManagedApplication, DeinitSource, DeinitMode } from '@standardnotes/services'
+
+export type DeinitCallback = (application: AppGroupManagedApplication, mode: DeinitMode, source: DeinitSource) => void

packages/snjs/lib/ApplicationGroup/DescriptorRecord.ts

@@ -0,0 +1,3 @@
+import { ApplicationDescriptor } from './ApplicationDescriptor'
+
+export type DescriptorRecord = Record<string, ApplicationDescriptor>

packages/snjs/lib/ApplicationGroup/index.ts

@@ -0,0 +1,5 @@
+export * from './AppGroupCallback'
+export * from './ApplicationDescriptor'
+export * from './ApplicationGroup'
+export * from './ApplicationGroupEvent'
+export * from './DescriptorRecord'

packages/snjs/lib/Client/FileViewController.ts

@@ -0,0 +1,41 @@
+import { FileItem } from '@standardnotes/models'
+import { ContentType } from '@standardnotes/common'
+import { SNApplication } from '../Application/Application'
+import { ItemViewControllerInterface } from './ItemViewControllerInterface'
+
+export class FileViewController implements ItemViewControllerInterface {
+  public dealloced = false
+  private removeStreamObserver?: () => void
+
+  constructor(private application: SNApplication, public item: FileItem) {}
+
+  deinit() {
+    this.dealloced = true
+    this.removeStreamObserver?.()
+    ;(this.removeStreamObserver as unknown) = undefined
+    ;(this.application as unknown) = undefined
+    ;(this.item as unknown) = undefined
+  }
+
+  async initialize() {
+    this.streamItems()
+  }
+
+  private streamItems() {
+    this.removeStreamObserver = this.application.streamItems<FileItem>(ContentType.File, ({ changed, inserted }) => {
+      if (this.dealloced) {
+        return
+      }
+
+      const files = changed.concat(inserted)
+
+      const matchingFile = files.find((item) => {
+        return item.uuid === this.item.uuid
+      })
+
+      if (matchingFile) {
+        this.item = matchingFile
+      }
+    })
+  }
+}

packages/snjs/lib/Client/IconsController.spec.ts

@@ -0,0 +1,69 @@
+import { IconsController } from './IconsController'
+
+describe('IconsController', () => {
+  let iconsController: IconsController
+
+  beforeEach(() => {
+    iconsController = new IconsController()
+  })
+
+  describe('getIconForFileType', () => {
+    it('should return correct icon type for supported mimetypes', () => {
+      const iconTypeForPdf = iconsController.getIconForFileType('application/pdf')
+      expect(iconTypeForPdf).toBe('file-pdf')
+
+      const iconTypeForDoc = iconsController.getIconForFileType('application/msword')
+      const iconTypeForDocx = iconsController.getIconForFileType(
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      )
+      expect(iconTypeForDoc).toBe('file-doc')
+      expect(iconTypeForDocx).toBe('file-doc')
+
+      const iconTypeForPpt = iconsController.getIconForFileType('application/vnd.ms-powerpoint')
+      const iconTypeForPptx = iconsController.getIconForFileType(
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+      )
+      expect(iconTypeForPpt).toBe('file-ppt')
+      expect(iconTypeForPptx).toBe('file-ppt')
+
+      const iconTypeForXls = iconsController.getIconForFileType('application/vnd.ms-excel')
+      const iconTypeForXlsx = iconsController.getIconForFileType(
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.spreadsheet',
+      )
+      expect(iconTypeForXls).toBe('file-xls')
+      expect(iconTypeForXlsx).toBe('file-xls')
+
+      const iconTypeForJpg = iconsController.getIconForFileType('image/jpeg')
+      const iconTypeForPng = iconsController.getIconForFileType('image/png')
+      expect(iconTypeForJpg).toBe('file-image')
+      expect(iconTypeForPng).toBe('file-image')
+
+      const iconTypeForMpeg = iconsController.getIconForFileType('video/mpeg')
+      const iconTypeForMp4 = iconsController.getIconForFileType('video/mp4')
+      expect(iconTypeForMpeg).toBe('file-mov')
+      expect(iconTypeForMp4).toBe('file-mov')
+
+      const iconTypeForWav = iconsController.getIconForFileType('audio/wav')
+      const iconTypeForMp3 = iconsController.getIconForFileType('audio/mp3')
+      expect(iconTypeForWav).toBe('file-music')
+      expect(iconTypeForMp3).toBe('file-music')
+
+      const iconTypeForZip = iconsController.getIconForFileType('application/zip')
+      const iconTypeForRar = iconsController.getIconForFileType('application/vnd.rar')
+      const iconTypeForTar = iconsController.getIconForFileType('application/x-tar')
+      const iconTypeFor7z = iconsController.getIconForFileType('application/x-7z-compressed')
+      expect(iconTypeForZip).toBe('file-zip')
+      expect(iconTypeForRar).toBe('file-zip')
+      expect(iconTypeForTar).toBe('file-zip')
+      expect(iconTypeFor7z).toBe('file-zip')
+    })
+
+    it('should return fallback icon type for unsupported mimetypes', () => {
+      const iconForBin = iconsController.getIconForFileType('application/octet-stream')
+      expect(iconForBin).toBe('file-other')
+
+      const iconForNoType = iconsController.getIconForFileType('')
+      expect(iconForNoType).toBe('file-other')
+    })
+  })
+})

packages/snjs/lib/Client/IconsController.ts

@@ -0,0 +1,61 @@
+import { NoteType } from '@standardnotes/features'
+import { IconType } from '@Lib/Types/IconType'
+
+export class IconsController {
+  getIconForFileType(type: string): IconType {
+    let iconType: IconType = 'file-other'
+
+    if (type === 'application/pdf') {
+      iconType = 'file-pdf'
+    }
+
+    if (/word/.test(type)) {
+      iconType = 'file-doc'
+    }
+
+    if (/powerpoint|presentation/.test(type)) {
+      iconType = 'file-ppt'
+    }
+
+    if (/excel|spreadsheet/.test(type)) {
+      iconType = 'file-xls'
+    }
+
+    if (/^image\//.test(type)) {
+      iconType = 'file-image'
+    }
+
+    if (/^video\//.test(type)) {
+      iconType = 'file-mov'
+    }
+
+    if (/^audio\//.test(type)) {
+      iconType = 'file-music'
+    }
+
+    if (/(zip)|([tr]ar)|(7z)/.test(type)) {
+      iconType = 'file-zip'
+    }
+
+    return iconType
+  }
+
+  getIconAndTintForNoteType(noteType?: NoteType): [IconType, number] {
+    switch (noteType) {
+      case NoteType.RichText:
+        return ['rich-text', 1]
+      case NoteType.Markdown:
+        return ['markdown', 2]
+      case NoteType.Authentication:
+        return ['authenticator', 6]
+      case NoteType.Spreadsheet:
+        return ['spreadsheets', 5]
+      case NoteType.Task:
+        return ['tasks', 3]
+      case NoteType.Code:
+        return ['code', 4]
+      default:
+        return ['plain-text', 1]
+    }
+  }
+}

packages/snjs/lib/Client/ItemGroupController.ts

@@ -0,0 +1,125 @@
+import { ApplicationEvent } from '../Application/Event'
+import { FileItem, PrefKey, SNNote } from '@standardnotes/models'
+import { removeFromArray } from '@standardnotes/utils'
+import { SNApplication } from '../Application/Application'
+import { NoteViewController } from './NoteViewController'
+import { FileViewController } from './FileViewController'
+import { TemplateNoteViewControllerOptions } from './TemplateNoteViewControllerOptions'
+
+type ItemControllerGroupChangeCallback = (activeController: NoteViewController | FileViewController | undefined) => void
+
+type CreateItemControllerOptions = FileItem | SNNote | TemplateNoteViewControllerOptions
+
+export class ItemGroupController {
+  public itemControllers: (NoteViewController | FileViewController)[] = []
+  private addTagHierarchy: boolean
+  changeObservers: ItemControllerGroupChangeCallback[] = []
+  eventObservers: (() => void)[] = []
+
+  constructor(private application: SNApplication) {
+    this.addTagHierarchy = application.getPreference(PrefKey.NoteAddToParentFolders, true)
+
+    this.eventObservers.push(
+      application.addSingleEventObserver(ApplicationEvent.PreferencesChanged, async () => {
+        this.addTagHierarchy = application.getPreference(PrefKey.NoteAddToParentFolders, true)
+      }),
+    )
+  }
+
+  public deinit(): void {
+    ;(this.application as unknown) = undefined
+
+    this.eventObservers.forEach((removeObserver) => {
+      removeObserver()
+    })
+
+    this.changeObservers.length = 0
+
+    for (const controller of this.itemControllers) {
+      this.closeItemController(controller, { notify: false })
+    }
+
+    this.itemControllers.length = 0
+  }
+
+  async createItemController(options: CreateItemControllerOptions): Promise<NoteViewController | FileViewController> {
+    if (this.activeItemViewController) {
+      this.closeItemController(this.activeItemViewController, { notify: false })
+    }
+
+    let controller!: NoteViewController | FileViewController
+
+    if (options instanceof FileItem) {
+      const file = options
+      controller = new FileViewController(this.application, file)
+    } else if (options instanceof SNNote) {
+      const note = options
+      controller = new NoteViewController(this.application, note)
+    } else {
+      controller = new NoteViewController(this.application, undefined, options)
+    }
+
+    this.itemControllers.push(controller)
+
+    await controller.initialize(this.addTagHierarchy)
+
+    this.notifyObservers()
+
+    return controller
+  }
+
+  public closeItemController(
+    controller: NoteViewController | FileViewController,
+    { notify = true }: { notify: boolean } = { notify: true },
+  ): void {
+    controller.deinit()
+
+    removeFromArray(this.itemControllers, controller)
+
+    if (notify) {
+      this.notifyObservers()
+    }
+  }
+
+  closeActiveItemController(): void {
+    const activeController = this.activeItemViewController
+
+    if (activeController) {
+      this.closeItemController(activeController, { notify: true })
+    }
+  }
+
+  closeAllItemControllers(): void {
+    for (const controller of this.itemControllers) {
+      this.closeItemController(controller, { notify: false })
+    }
+
+    this.notifyObservers()
+  }
+
+  get activeItemViewController(): NoteViewController | FileViewController | undefined {
+    return this.itemControllers[0]
+  }
+
+  /**
+   * Notifies observer when the active controller has changed.
+   */
+  public addActiveControllerChangeObserver(callback: ItemControllerGroupChangeCallback): () => void {
+    this.changeObservers.push(callback)
+
+    if (this.activeItemViewController) {
+      callback(this.activeItemViewController)
+    }
+
+    const thislessChangeObservers = this.changeObservers
+    return () => {
+      removeFromArray(thislessChangeObservers, callback)
+    }
+  }
+
+  private notifyObservers(): void {
+    for (const observer of this.changeObservers) {
+      observer(this.activeItemViewController)
+    }
+  }
+}

packages/snjs/lib/Client/ItemViewControllerInterface.ts

@@ -0,0 +1,8 @@
+import { SNNote, FileItem } from '@standardnotes/models'
+
+export interface ItemViewControllerInterface {
+  item: SNNote | FileItem
+
+  deinit: () => void
+  initialize(addTagHierarchy?: boolean): Promise<void>
+}

packages/snjs/lib/Client/NoteViewController.ts

@@ -0,0 +1,208 @@
+import {
+  NoteMutator,
+  SNNote,
+  SNTag,
+  NoteContent,
+  DecryptedItemInterface,
+  PayloadEmitSource,
+} from '@standardnotes/models'
+import { removeFromArray } from '@standardnotes/utils'
+import { ContentType } from '@standardnotes/common'
+import { UuidString } from '@Lib/Types/UuidString'
+import { SNApplication } from '../Application/Application'
+import {
+  STRING_SAVING_WHILE_DOCUMENT_HIDDEN,
+  STRING_INVALID_NOTE,
+  NOTE_PREVIEW_CHAR_LIMIT,
+  STRING_ELLIPSES,
+  SAVE_TIMEOUT_NO_DEBOUNCE,
+  SAVE_TIMEOUT_DEBOUNCE,
+} from './Types'
+import { ItemViewControllerInterface } from './ItemViewControllerInterface'
+import { TemplateNoteViewControllerOptions } from './TemplateNoteViewControllerOptions'
+
+export type EditorValues = {
+  title: string
+  text: string
+}
+
+export class NoteViewController implements ItemViewControllerInterface {
+  public item!: SNNote
+  public dealloced = false
+  private innerValueChangeObservers: ((note: SNNote, source: PayloadEmitSource) => void)[] = []
+  private removeStreamObserver?: () => void
+  public isTemplateNote = false
+  private saveTimeout?: ReturnType<typeof setTimeout>
+  private defaultTitle: string | undefined
+  private defaultTag: UuidString | undefined
+
+  constructor(
+    private application: SNApplication,
+    item?: SNNote,
+    templateNoteOptions?: TemplateNoteViewControllerOptions,
+  ) {
+    if (item) {
+      this.item = item
+    }
+
+    if (templateNoteOptions) {
+      this.defaultTitle = templateNoteOptions.title
+      this.defaultTag = templateNoteOptions.tag
+    }
+  }
+
+  deinit(): void {
+    this.dealloced = true
+    this.removeStreamObserver?.()
+    ;(this.removeStreamObserver as unknown) = undefined
+    ;(this.application as unknown) = undefined
+    ;(this.item as unknown) = undefined
+
+    this.innerValueChangeObservers.length = 0
+
+    this.saveTimeout = undefined
+  }
+
+  async initialize(addTagHierarchy: boolean): Promise<void> {
+    if (!this.item) {
+      const note = this.application.mutator.createTemplateItem<NoteContent, SNNote>(ContentType.Note, {
+        text: '',
+        title: this.defaultTitle || '',
+        references: [],
+      })
+
+      this.isTemplateNote = true
+      this.item = note
+
+      if (this.defaultTag) {
+        const tag = this.application.items.findItem(this.defaultTag) as SNTag
+        await this.application.items.addTagToNote(note, tag, addTagHierarchy)
+      }
+
+      this.notifyObservers(this.item, PayloadEmitSource.InitialObserverRegistrationPush)
+    }
+
+    this.streamItems()
+  }
+
+  private notifyObservers(note: SNNote, source: PayloadEmitSource): void {
+    for (const observer of this.innerValueChangeObservers) {
+      observer(note, source)
+    }
+  }
+
+  private streamItems() {
+    this.removeStreamObserver = this.application.streamItems<SNNote>(
+      ContentType.Note,
+      ({ changed, inserted, source }) => {
+        if (this.dealloced) {
+          return
+        }
+
+        const notes = changed.concat(inserted)
+
+        const matchingNote = notes.find((item) => {
+          return item.uuid === this.item.uuid
+        })
+
+        if (matchingNote) {
+          this.isTemplateNote = false
+          this.item = matchingNote
+          this.notifyObservers(matchingNote, source)
+        }
+      },
+    )
+  }
+
+  public insertTemplatedNote(): Promise<DecryptedItemInterface> {
+    this.isTemplateNote = false
+    return this.application.mutator.insertItem(this.item)
+  }
+
+  /**
+   * Register to be notified when the controller's note's inner values change
+   * (and thus a new object reference is created)
+   */
+  public addNoteInnerValueChangeObserver(callback: (note: SNNote, source: PayloadEmitSource) => void): () => void {
+    this.innerValueChangeObservers.push(callback)
+
+    if (this.item) {
+      callback(this.item, PayloadEmitSource.InitialObserverRegistrationPush)
+    }
+
+    const thislessChangeObservers = this.innerValueChangeObservers
+    return () => {
+      removeFromArray(thislessChangeObservers, callback)
+    }
+  }
+
+  /**
+   * @param bypassDebouncer Calling save will debounce by default. You can pass true to save
+   * immediately.
+   * @param isUserModified This field determines if the item will be saved as a user
+   * modification, thus updating the user modified date displayed in the UI
+   * @param dontUpdatePreviews Whether this change should update the note's plain and HTML
+   * preview.
+   * @param customMutate A custom mutator function.
+   */
+  public async save(dto: {
+    editorValues: EditorValues
+    bypassDebouncer?: boolean
+    isUserModified?: boolean
+    dontUpdatePreviews?: boolean
+    customMutate?: (mutator: NoteMutator) => void
+  }): Promise<void> {
+    const title = dto.editorValues.title
+    const text = dto.editorValues.text
+    const isTemplate = this.isTemplateNote
+
+    if (typeof document !== 'undefined' && document.hidden) {
+      void this.application.alertService.alert(STRING_SAVING_WHILE_DOCUMENT_HIDDEN)
+      return
+    }
+
+    if (isTemplate) {
+      await this.insertTemplatedNote()
+    }
+
+    if (!this.application.items.findItem(this.item.uuid)) {
+      void this.application.alertService.alert(STRING_INVALID_NOTE)
+      return
+    }
+
+    await this.application.mutator.changeItem(
+      this.item,
+      (mutator) => {
+        const noteMutator = mutator as NoteMutator
+        if (dto.customMutate) {
+          dto.customMutate(noteMutator)
+        }
+        noteMutator.title = title
+        noteMutator.text = text
+
+        if (!dto.dontUpdatePreviews) {
+          const noteText = text || ''
+          const truncate = noteText.length > NOTE_PREVIEW_CHAR_LIMIT
+          const substring = noteText.substring(0, NOTE_PREVIEW_CHAR_LIMIT)
+          const previewPlain = substring + (truncate ? STRING_ELLIPSES : '')
+
+          // eslint-disable-next-line camelcase
+          noteMutator.preview_plain = previewPlain
+          // eslint-disable-next-line camelcase
+          noteMutator.preview_html = undefined
+        }
+      },
+      dto.isUserModified,
+    )
+
+    if (this.saveTimeout) {
+      clearTimeout(this.saveTimeout)
+    }
+
+    const noDebounce = dto.bypassDebouncer || this.application.noAccount()
+    const syncDebouceMs = noDebounce ? SAVE_TIMEOUT_NO_DEBOUNCE : SAVE_TIMEOUT_DEBOUNCE
+    this.saveTimeout = setTimeout(() => {
+      void this.application.sync.sync()
+    }, syncDebouceMs)
+  }
+}

packages/snjs/lib/Client/TemplateNoteViewControllerOptions.ts

@@ -0,0 +1,6 @@
+import { UuidString } from '@Lib/Types/UuidString'
+
+export type TemplateNoteViewControllerOptions = {
+  title?: string
+  tag?: UuidString
+}

packages/snjs/lib/Client/Types.ts

@@ -0,0 +1,8 @@
+export const STRING_SAVING_WHILE_DOCUMENT_HIDDEN =
+  'Attempting to save an item while the application is hidden. To protect data integrity, please refresh the application window and try again.'
+export const STRING_INVALID_NOTE =
+  "The note you are attempting to save can not be found or has been deleted. Changes you make will not be synced. Please copy this note's text and start a new note."
+export const STRING_ELLIPSES = '...'
+export const NOTE_PREVIEW_CHAR_LIMIT = 80
+export const SAVE_TIMEOUT_DEBOUNCE = 350
+export const SAVE_TIMEOUT_NO_DEBOUNCE = 100

packages/snjs/lib/Client/index.ts

@@ -0,0 +1,4 @@
+export * from './IconsController'
+export * from './NoteViewController'
+export * from './FileViewController'
+export * from './ItemGroupController'

packages/snjs/lib/Hosts.ts

@@ -0,0 +1,40 @@
+export const APPLICATION_DEFAULT_HOSTS = [
+  'api.standardnotes.com',
+  'api-dev.standardnotes.com',
+  'sync.standardnotes.org',
+  'syncing-server-demo.standardnotes.com',
+]
+
+export const FILES_DEFAULT_HOSTS = ['files.standardnotes.com', 'files-dev.standardnotes.com']
+
+export const TRUSTED_FEATURE_HOSTS = [
+  'api-dev.standardnotes.com',
+  'api.standardnotes.com',
+  'extensions.standardnotes.com',
+  'extensions.standardnotes.org',
+  'extensions-server-dev.standardnotes.org',
+  'extensions-server-dev.standardnotes.com',
+  'features.standardnotes.com',
+]
+
+export enum ExtensionsServerURL {
+  Dev = 'https://extensions-server-dev.standardnotes.org',
+  Prod = 'https://extensions.standardnotes.org',
+}
+
+const LocalHost = 'localhost'
+
+export function isUrlFirstParty(url: string): boolean {
+  try {
+    const { host } = new URL(url)
+    return host.startsWith(LocalHost) || APPLICATION_DEFAULT_HOSTS.includes(host) || FILES_DEFAULT_HOSTS.includes(host)
+  } catch (_err) {
+    return false
+  }
+}
+
+export const PROD_OFFLINE_FEATURES_URL = 'https://api.standardnotes.com/v1/offline/features'
+
+export const LEGACY_PROD_EXT_ORIGIN = 'https://extensions.standardnotes.org'
+
+export const TRUSTED_CUSTOM_EXTENSIONS_HOSTS = ['listed.to']

packages/snjs/lib/Log.ts

@@ -0,0 +1,13 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+export class SNLog {
+  static log(...message: any): void {
+    this.onLog(...message)
+  }
+  static error<T extends Error>(error: T): T {
+    this.onError(error)
+    return error
+  }
+  static onLog: (...message: any) => void
+  static onError: (error: Error) => void
+}

packages/snjs/lib/Migrations/Applicators/TagsToFolders.spec.ts

@@ -0,0 +1,124 @@
+import { ItemManager } from '@Lib/Services'
+import { TagsToFoldersMigrationApplicator } from './TagsToFolders'
+
+const itemManagerMock = (tagTitles: string[]) => {
+  const mockTag = (title: string) => ({
+    title,
+    uuid: title,
+    parentId: undefined,
+  })
+
+  const mock = {
+    getItems: jest.fn().mockReturnValue(tagTitles.map(mockTag)),
+    findOrCreateTagParentChain: jest.fn(),
+    changeItem: jest.fn(),
+  }
+
+  return mock
+}
+
+describe('folders component to hierarchy', () => {
+  it('should produce a valid hierarchy in the simple case', async () => {
+    const titles = ['a', 'a.b', 'a.b.c']
+
+    const itemManager = itemManagerMock(titles)
+    await TagsToFoldersMigrationApplicator.run(itemManager as unknown as ItemManager)
+
+    const findOrCreateTagParentChainCalls = itemManager.findOrCreateTagParentChain.mock.calls
+    const changeItemCalls = itemManager.changeItem.mock.calls
+
+    expect(findOrCreateTagParentChainCalls.length).toEqual(2)
+    expect(findOrCreateTagParentChainCalls[0][0]).toEqual(['a'])
+    expect(findOrCreateTagParentChainCalls[1][0]).toEqual(['a', 'b'])
+
+    expect(changeItemCalls.length).toEqual(2)
+    expect(changeItemCalls[0][0].uuid).toEqual('a.b')
+    expect(changeItemCalls[1][0].uuid).toEqual('a.b.c')
+  })
+
+  it('should not touch flat hierarchies', async () => {
+    const titles = ['a', 'x', 'y', 'z']
+
+    const itemManager = itemManagerMock(titles)
+    await TagsToFoldersMigrationApplicator.run(itemManager as unknown as ItemManager)
+
+    const findOrCreateTagParentChainCalls = itemManager.findOrCreateTagParentChain.mock.calls
+    const changeItemCalls = itemManager.changeItem.mock.calls
+
+    expect(findOrCreateTagParentChainCalls.length).toEqual(0)
+
+    expect(changeItemCalls.length).toEqual(0)
+  })
+
+  it('should work despite cloned tags', async () => {
+    const titles = ['a.b', 'c', 'a.b']
+
+    const itemManager = itemManagerMock(titles)
+    await TagsToFoldersMigrationApplicator.run(itemManager as unknown as ItemManager)
+
+    const findOrCreateTagParentChainCalls = itemManager.findOrCreateTagParentChain.mock.calls
+    const changeItemCalls = itemManager.changeItem.mock.calls
+
+    expect(findOrCreateTagParentChainCalls.length).toEqual(2)
+    expect(findOrCreateTagParentChainCalls[0][0]).toEqual(['a'])
+    expect(findOrCreateTagParentChainCalls[1][0]).toEqual(['a'])
+
+    expect(changeItemCalls.length).toEqual(2)
+    expect(changeItemCalls[0][0].uuid).toEqual('a.b')
+    expect(changeItemCalls[0][0].uuid).toEqual('a.b')
+  })
+
+  it('should produce a valid hierarchy cases with  missing intermediate tags or unordered', async () => {
+    const titles = ['y.2', 'w.3', 'y']
+
+    const itemManager = itemManagerMock(titles)
+    await TagsToFoldersMigrationApplicator.run(itemManager as unknown as ItemManager)
+
+    const findOrCreateTagParentChainCalls = itemManager.findOrCreateTagParentChain.mock.calls
+    const changeItemCalls = itemManager.changeItem.mock.calls
+
+    expect(findOrCreateTagParentChainCalls.length).toEqual(2)
+    expect(findOrCreateTagParentChainCalls[0][0]).toEqual(['w'])
+    expect(findOrCreateTagParentChainCalls[1][0]).toEqual(['y'])
+
+    expect(changeItemCalls.length).toEqual(2)
+    expect(changeItemCalls[0][0].uuid).toEqual('w.3')
+    expect(changeItemCalls[1][0].uuid).toEqual('y.2')
+  })
+
+  it('skip prefixed names', async () => {
+    const titles = ['.something', '.something...something']
+
+    const itemManager = itemManagerMock(titles)
+    await TagsToFoldersMigrationApplicator.run(itemManager as unknown as ItemManager)
+
+    const findOrCreateTagParentChainCalls = itemManager.findOrCreateTagParentChain.mock.calls
+    const changeItemCalls = itemManager.changeItem.mock.calls
+
+    expect(findOrCreateTagParentChainCalls.length).toEqual(0)
+    expect(changeItemCalls.length).toEqual(0)
+  })
+
+  it('skip not-supported names', async () => {
+    const titles = [
+      'something.',
+      'something..',
+      'something..another.thing',
+      'a.b.c',
+      'a',
+      'something..another.thing..anyway',
+    ]
+
+    const itemManager = itemManagerMock(titles)
+    await TagsToFoldersMigrationApplicator.run(itemManager as unknown as ItemManager)
+
+    const findOrCreateTagParentChainCalls = itemManager.findOrCreateTagParentChain.mock.calls
+    const changeItemCalls = itemManager.changeItem.mock.calls
+
+    expect(findOrCreateTagParentChainCalls.length).toEqual(1)
+    expect(findOrCreateTagParentChainCalls[0][0]).toEqual(['a', 'b'])
+
+    expect(changeItemCalls.length).toEqual(1)
+    expect(changeItemCalls[0][0].uuid).toEqual('a.b.c')
+  })
+})

packages/snjs/lib/Migrations/Applicators/TagsToFolders.ts

@@ -0,0 +1,50 @@
+import { SNTag, TagMutator, TagFolderDelimitter } from '@standardnotes/models'
+import { ItemManager } from '@Lib/Services'
+import { lastElement, sortByKey, withoutLastElement } from '@standardnotes/utils'
+import { ContentType } from '@standardnotes/common'
+
+export class TagsToFoldersMigrationApplicator {
+  public static isApplicableToCurrentData(itemManager: ItemManager): boolean {
+    const tags = itemManager.getItems<SNTag>(ContentType.Tag)
+    for (const tag of tags) {
+      if (tag.title.includes(TagFolderDelimitter) && !tag.parentId) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  public static async run(itemManager: ItemManager): Promise<void> {
+    const tags = itemManager.getItems(ContentType.Tag) as SNTag[]
+    const sortedTags = sortByKey(tags, 'title')
+
+    for (const tag of sortedTags) {
+      const hierarchy = tag.title.split(TagFolderDelimitter)
+      const hasSimpleTitle = hierarchy.length === 1
+      const hasParent = !!tag.parentId
+      const hasUnsupportedTitle = hierarchy.some((title) => title.length === 0)
+
+      if (hasParent || hasSimpleTitle || hasUnsupportedTitle) {
+        continue
+      }
+
+      const parents = withoutLastElement(hierarchy)
+      const newTitle = lastElement(hierarchy)
+
+      if (!newTitle) {
+        return
+      }
+
+      const parent = await itemManager.findOrCreateTagParentChain(parents)
+
+      await itemManager.changeItem(tag, (mutator: TagMutator) => {
+        mutator.title = newTitle
+
+        if (parent) {
+          mutator.makeChildOf(parent)
+        }
+      })
+    }
+  }
+}

packages/snjs/lib/Migrations/Base.ts

@@ -0,0 +1,247 @@
+import { AnyKeyParamsContent } from '@standardnotes/common'
+import { SNLog } from '@Lib/Log'
+import { EncryptedPayload, EncryptedTransferPayload, isErrorDecryptingPayload } from '@standardnotes/models'
+import { Challenge } from '../Services/Challenge'
+import { KeychainRecoveryStrings, SessionStrings } from '../Services/Api/Messages'
+import { PreviousSnjsVersion1_0_0, PreviousSnjsVersion2_0_0, SnjsVersion } from '../Version'
+import { Migration } from '@Lib/Migrations/Migration'
+import {
+  RawStorageKey,
+  namespacedKey,
+  ApplicationStage,
+  ChallengeValidation,
+  ChallengeReason,
+  ChallengePrompt,
+} from '@standardnotes/services'
+import { isNullOrUndefined } from '@standardnotes/utils'
+import { CreateReader } from './StorageReaders/Functions'
+import { StorageReader } from './StorageReaders/Reader'
+import { ContentTypeUsesRootKeyEncryption } from '@standardnotes/encryption'
+
+/** A key that was briefly present in Snjs version 2.0.0 but removed in 2.0.1 */
+const LastMigrationTimeStampKey2_0_0 = 'last_migration_timestamp'
+
+/**
+ * The base migration always runs during app initialization. It is meant as a way
+ * to set up all other migrations.
+ */
+export class BaseMigration extends Migration {
+  private reader!: StorageReader
+  private didPreRun = false
+  private memoizedNeedsKeychainRepair?: boolean
+
+  public async preRun() {
+    await this.storeVersionNumber()
+    this.didPreRun = true
+  }
+
+  protected registerStageHandlers() {
+    this.registerStageHandler(ApplicationStage.PreparingForLaunch_0, async () => {
+      if (await this.needsKeychainRepair()) {
+        await this.repairMissingKeychain()
+      }
+      this.markDone()
+    })
+  }
+
+  private getStoredVersion() {
+    const storageKey = namespacedKey(this.services.identifier, RawStorageKey.SnjsVersion)
+    return this.services.deviceInterface.getRawStorageValue(storageKey)
+  }
+
+  /**
+   * In Snjs 1.x, and Snjs 2.0.0, version numbers were not stored (as they were introduced
+   * in 2.0.1). Because migrations can now rely on this value, we want to establish a base
+   * value if we do not find it in storage.
+   */
+  private async storeVersionNumber() {
+    const storageKey = namespacedKey(this.services.identifier, RawStorageKey.SnjsVersion)
+    const version = await this.getStoredVersion()
+    if (!version) {
+      /** Determine if we are 1.0.0 or 2.0.0 */
+      /** If any of these keys exist in raw storage, we are coming from a 1.x architecture */
+      const possibleLegacyKeys = ['migrations', 'ephemeral', 'user', 'cachedThemes', 'syncToken', 'encryptedStorage']
+      let hasLegacyValue = false
+      for (const legacyKey of possibleLegacyKeys) {
+        const value = await this.services.deviceInterface.getRawStorageValue(legacyKey)
+        if (value) {
+          hasLegacyValue = true
+          break
+        }
+      }
+      if (hasLegacyValue) {
+        /** Coming from 1.0.0 */
+        await this.services.deviceInterface.setRawStorageValue(storageKey, PreviousSnjsVersion1_0_0)
+      } else {
+        /** Coming from 2.0.0 (which did not store version) OR is brand new application */
+        const migrationKey = namespacedKey(this.services.identifier, LastMigrationTimeStampKey2_0_0)
+        const migrationValue = await this.services.deviceInterface.getRawStorageValue(migrationKey)
+        const is_2_0_0_application = !isNullOrUndefined(migrationValue)
+        if (is_2_0_0_application) {
+          await this.services.deviceInterface.setRawStorageValue(storageKey, PreviousSnjsVersion2_0_0)
+          await this.services.deviceInterface.removeRawStorageValue(LastMigrationTimeStampKey2_0_0)
+        } else {
+          /** Is new application, use current version as not to run any migrations */
+          await this.services.deviceInterface.setRawStorageValue(storageKey, SnjsVersion)
+        }
+      }
+    }
+  }
+
+  private async loadReader() {
+    if (this.reader) {
+      return
+    }
+
+    const version = (await this.getStoredVersion()) as string
+    this.reader = CreateReader(
+      version,
+      this.services.deviceInterface,
+      this.services.identifier,
+      this.services.environment,
+    )
+  }
+
+  /**
+   * If the keychain is empty, and the user does not have a passcode,
+   * AND there appear to be stored account key params, this indicates
+   * a launch where the keychain was wiped due to restoring device
+   * from cloud backup which did not include keychain. This typically occurs
+   * on mobile when restoring from iCloud, but we'll also follow this same behavior
+   * on desktop/web as well, since we recently introduced keychain to desktop.
+   *
+   * We must prompt user for account password, and validate based on ability to decrypt
+   * an item. We cannot validate based on storage because 1.x mobile applications did
+   * not use encrypted storage, although we did on 2.x. But instead of having two methods
+   * of validations best to use one that works on both.
+   *
+   * The item is randomly chosen, but for 2.x applications, it must be an items key item
+   * (since only item keys are encrypted directly with account password)
+   */
+
+  public async needsKeychainRepair() {
+    if (this.memoizedNeedsKeychainRepair != undefined) {
+      return this.memoizedNeedsKeychainRepair
+    }
+
+    if (!this.didPreRun) {
+      throw Error('Attempting to access specialized function before prerun')
+    }
+
+    if (!this.reader) {
+      await this.loadReader()
+    }
+
+    const usesKeychain = this.reader.usesKeychain
+    if (!usesKeychain) {
+      /** Doesn't apply if this version did not use a keychain to begin with */
+      this.memoizedNeedsKeychainRepair = false
+      return this.memoizedNeedsKeychainRepair
+    }
+
+    const rawAccountParams = await this.reader.getAccountKeyParams()
+    const hasAccountKeyParams = !isNullOrUndefined(rawAccountParams)
+    if (!hasAccountKeyParams) {
+      /** Doesn't apply if account is not involved */
+      this.memoizedNeedsKeychainRepair = false
+      return this.memoizedNeedsKeychainRepair
+    }
+
+    const hasPasscode = await this.reader.hasPasscode()
+    if (hasPasscode) {
+      /** Doesn't apply if using passcode, as keychain would be bypassed in that case */
+      this.memoizedNeedsKeychainRepair = false
+      return this.memoizedNeedsKeychainRepair
+    }
+
+    const accountKeysMissing = !(await this.reader.hasNonWrappedAccountKeys())
+    if (!accountKeysMissing) {
+      this.memoizedNeedsKeychainRepair = false
+      return this.memoizedNeedsKeychainRepair
+    }
+
+    this.memoizedNeedsKeychainRepair = true
+    return this.memoizedNeedsKeychainRepair
+  }
+
+  private async repairMissingKeychain() {
+    const version = (await this.getStoredVersion()) as string
+    const rawAccountParams = await this.reader.getAccountKeyParams()
+
+    /** Challenge for account password */
+    const challenge = new Challenge(
+      [new ChallengePrompt(ChallengeValidation.None, undefined, SessionStrings.PasswordInputPlaceholder, true)],
+      ChallengeReason.Custom,
+      false,
+      KeychainRecoveryStrings.Title,
+      KeychainRecoveryStrings.Text,
+    )
+
+    return new Promise((resolve) => {
+      this.services.challengeService.addChallengeObserver(challenge, {
+        onNonvalidatedSubmit: async (challengeResponse) => {
+          const password = challengeResponse.values[0].value as string
+          const accountParams = this.services.protocolService.createKeyParams(rawAccountParams as AnyKeyParamsContent)
+          const rootKey = await this.services.protocolService.computeRootKey(password, accountParams)
+
+          /** Choose an item to decrypt */
+          const allItems = (
+            await this.services.deviceInterface.getAllRawDatabasePayloads<EncryptedTransferPayload>(
+              this.services.identifier,
+            )
+          ).map((p) => new EncryptedPayload(p))
+
+          let itemToDecrypt = allItems.find((item) => {
+            return ContentTypeUsesRootKeyEncryption(item.content_type)
+          })
+
+          if (!itemToDecrypt) {
+            /** If no root key encrypted item, just choose any item */
+            itemToDecrypt = allItems[0]
+          }
+
+          if (!itemToDecrypt) {
+            throw SNLog.error(Error('Attempting keychain recovery validation but no items present.'))
+          }
+
+          const decryptedPayload = await this.services.protocolService.decryptSplitSingle({
+            usesRootKey: {
+              items: [itemToDecrypt],
+              key: rootKey,
+            },
+          })
+
+          if (isErrorDecryptingPayload(decryptedPayload)) {
+            /** Wrong password, try again */
+            this.services.challengeService.setValidationStatusForChallenge(
+              challenge,
+              challengeResponse.values[0],
+              false,
+            )
+          } else {
+            /**
+             * If decryption succeeds, store the generated account key where it is expected,
+             * either in top-level keychain in 1.0.0, and namespaced location in 2.0.0+.
+             */
+            if (version === PreviousSnjsVersion1_0_0) {
+              /** Store in top level keychain */
+              await this.services.deviceInterface.setLegacyRawKeychainValue({
+                mk: rootKey.masterKey,
+                ak: rootKey.dataAuthenticationKey as string,
+                version: accountParams.version,
+              })
+            } else {
+              /** Store in namespaced location */
+              const rawKey = rootKey.getKeychainValue()
+              await this.services.deviceInterface.setNamespacedKeychainValue(rawKey, this.services.identifier)
+            }
+            resolve(true)
+            this.services.challengeService.completeChallenge(challenge)
+          }
+        },
+      })
+
+      void this.services.challengeService.promptForChallengeResponse(challenge)
+    })
+  }
+}

packages/snjs/lib/Migrations/Migration.ts

@@ -0,0 +1,60 @@
+import { Challenge } from '../Services/Challenge'
+import { MigrationServices } from './MigrationServices'
+import { ApplicationStage, ChallengeValidation, ChallengeReason, ChallengePrompt } from '@standardnotes/services'
+
+type StageHandler = () => Promise<void>
+
+export abstract class Migration {
+  private stageHandlers: Partial<Record<ApplicationStage, StageHandler>> = {}
+  private onDoneHandler?: () => void
+
+  constructor(protected services: MigrationServices) {
+    this.registerStageHandlers()
+  }
+
+  public static version(): string {
+    throw 'Must override migration version'
+  }
+
+  protected abstract registerStageHandlers(): void
+
+  protected registerStageHandler(stage: ApplicationStage, handler: StageHandler) {
+    this.stageHandlers[stage] = handler
+  }
+
+  protected markDone() {
+    this.onDoneHandler?.()
+    this.onDoneHandler = undefined
+  }
+
+  protected async promptForPasscodeUntilCorrect(validationCallback: (passcode: string) => Promise<boolean>) {
+    const challenge = new Challenge([new ChallengePrompt(ChallengeValidation.None)], ChallengeReason.Migration, false)
+    return new Promise((resolve) => {
+      this.services.challengeService.addChallengeObserver(challenge, {
+        onNonvalidatedSubmit: async (challengeResponse) => {
+          const value = challengeResponse.values[0]
+          const passcode = value.value as string
+          const valid = await validationCallback(passcode)
+          if (valid) {
+            this.services.challengeService.completeChallenge(challenge)
+            resolve(passcode)
+          } else {
+            this.services.challengeService.setValidationStatusForChallenge(challenge, value, false)
+          }
+        },
+      })
+      void this.services.challengeService.promptForChallengeResponse(challenge)
+    })
+  }
+
+  onDone(callback: () => void) {
+    this.onDoneHandler = callback
+  }
+
+  async handleStage(stage: ApplicationStage): Promise<void> {
+    const handler = this.stageHandlers[stage]
+    if (handler) {
+      await handler()
+    }
+  }
+}

packages/snjs/lib/Migrations/MigrationServices.ts

@@ -0,0 +1,20 @@
+import { SNSessionManager } from '../Services/Session/SessionManager'
+import { ApplicationIdentifier } from '@standardnotes/common'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import { EncryptionService } from '@standardnotes/encryption'
+import { DeviceInterface, InternalEventBusInterface, Environment } from '@standardnotes/services'
+import { ChallengeService, SNSingletonManager, SNFeaturesService, DiskStorageService } from '@Lib/Services'
+
+export type MigrationServices = {
+  protocolService: EncryptionService
+  deviceInterface: DeviceInterface
+  storageService: DiskStorageService
+  challengeService: ChallengeService
+  sessionManager: SNSessionManager
+  itemManager: ItemManager
+  singletonManager: SNSingletonManager
+  featuresService: SNFeaturesService
+  environment: Environment
+  identifier: ApplicationIdentifier
+  internalEventBus: InternalEventBusInterface
+}

packages/snjs/lib/Migrations/StorageReaders/Functions.ts

@@ -0,0 +1,34 @@
+import { ApplicationIdentifier } from '@standardnotes/common'
+import { compareSemVersions, isRightVersionGreaterThanLeft } from '@Lib/Version'
+import { DeviceInterface, Environment } from '@standardnotes/services'
+import { StorageReader } from './Reader'
+import * as ReaderClasses from './Versions'
+
+function ReaderClassForVersion(
+  version: string,
+): typeof ReaderClasses.StorageReader2_0_0 | typeof ReaderClasses.StorageReader1_0_0 {
+  /** Sort readers by newest first */
+  const allReaders = Object.values(ReaderClasses).sort((a, b) => {
+    return compareSemVersions(a.version(), b.version()) * -1
+  })
+  for (const reader of allReaders) {
+    if (reader.version() === version) {
+      return reader
+    }
+    if (isRightVersionGreaterThanLeft(reader.version(), version)) {
+      return reader
+    }
+  }
+
+  throw Error(`Cannot find reader for version ${version}`)
+}
+
+export function CreateReader(
+  version: string,
+  deviceInterface: DeviceInterface,
+  identifier: ApplicationIdentifier,
+  environment: Environment,
+): StorageReader {
+  const readerClass = ReaderClassForVersion(version)
+  return new readerClass(deviceInterface, identifier, environment)
+}

packages/snjs/lib/Migrations/StorageReaders/Reader.ts

@@ -0,0 +1,31 @@
+import { ApplicationIdentifier } from '@standardnotes/common'
+import { DeviceInterface, Environment } from '@standardnotes/services'
+
+/**
+ * A storage reader reads storage via a device interface
+ * given a specific version of SNJS
+ */
+export abstract class StorageReader {
+  constructor(
+    protected deviceInterface: DeviceInterface,
+    protected identifier: ApplicationIdentifier,
+    protected environment: Environment,
+  ) {}
+
+  public static version(): string {
+    throw Error('Must override')
+  }
+
+  public abstract getAccountKeyParams(): Promise<unknown | undefined>
+
+  /**
+   * Returns true if the state of storage has account keys present
+   * in version-specific storage (either keychain or raw storage)
+   */
+  public abstract hasNonWrappedAccountKeys(): Promise<boolean>
+
+  public abstract hasPasscode(): Promise<boolean>
+
+  /** Whether this version used the keychain to store keys */
+  public abstract usesKeychain(): boolean
+}

packages/snjs/lib/Migrations/StorageReaders/Versions/Reader1_0_0.ts

@@ -0,0 +1,48 @@
+import { isNullOrUndefined } from '@standardnotes/utils'
+import { isEnvironmentMobile } from '@Lib/Application/Platforms'
+import { PreviousSnjsVersion1_0_0 } from '../../../Version'
+import { isMobileDevice, LegacyKeys1_0_0 } from '@standardnotes/services'
+import { StorageReader } from '../Reader'
+
+export class StorageReader1_0_0 extends StorageReader {
+  static override version() {
+    return PreviousSnjsVersion1_0_0
+  }
+
+  public async getAccountKeyParams() {
+    return this.deviceInterface.getJsonParsedRawStorageValue(LegacyKeys1_0_0.AllAccountKeyParamsKey)
+  }
+
+  /**
+   * In 1.0.0, web uses raw storage for unwrapped account key, and mobile uses
+   * the keychain
+   */
+  public async hasNonWrappedAccountKeys() {
+    if (isMobileDevice(this.deviceInterface)) {
+      const value = await this.deviceInterface.getRawKeychainValue()
+      return !isNullOrUndefined(value)
+    } else {
+      const value = await this.deviceInterface.getRawStorageValue('mk')
+      return !isNullOrUndefined(value)
+    }
+  }
+
+  public async hasPasscode() {
+    if (isEnvironmentMobile(this.environment)) {
+      const rawPasscodeParams = await this.deviceInterface.getJsonParsedRawStorageValue(
+        LegacyKeys1_0_0.MobilePasscodeParamsKey,
+      )
+      return !isNullOrUndefined(rawPasscodeParams)
+    } else {
+      const encryptedStorage = await this.deviceInterface.getJsonParsedRawStorageValue(
+        LegacyKeys1_0_0.WebEncryptedStorageKey,
+      )
+      return !isNullOrUndefined(encryptedStorage)
+    }
+  }
+
+  /** Keychain was not used on desktop/web in 1.0.0 */
+  public usesKeychain() {
+    return isEnvironmentMobile(this.environment) ? true : false
+  }
+}

packages/snjs/lib/Migrations/StorageReaders/Versions/Reader2_0_0.ts

@@ -0,0 +1,46 @@
+import { isNullOrUndefined } from '@standardnotes/utils'
+import { RawStorageKey, StorageKey, namespacedKey, ValueModesKeys } from '@standardnotes/services'
+import { StorageReader } from '../Reader'
+import { PreviousSnjsVersion2_0_0 } from '@Lib/Version'
+
+export class StorageReader2_0_0 extends StorageReader {
+  static override version() {
+    return PreviousSnjsVersion2_0_0
+  }
+
+  private async getStorage() {
+    const storageKey = namespacedKey(this.identifier, RawStorageKey.StorageObject)
+    const storage = await this.deviceInterface.getRawStorageValue(storageKey)
+    const values = storage ? JSON.parse(storage) : undefined
+    return values
+  }
+
+  private async getNonWrappedValue(key: string) {
+    const values = await this.getStorage()
+    if (!values) {
+      return undefined
+    }
+    return values[ValueModesKeys.Nonwrapped]?.[key]
+  }
+
+  /**
+   * In 2.0.0+, account key params are stored in NonWrapped storage
+   */
+  public async getAccountKeyParams() {
+    return this.getNonWrappedValue(StorageKey.RootKeyParams)
+  }
+
+  public async hasNonWrappedAccountKeys() {
+    const value = await this.deviceInterface.getNamespacedKeychainValue(this.identifier)
+    return !isNullOrUndefined(value)
+  }
+
+  public async hasPasscode() {
+    const wrappedRootKey = await this.getNonWrappedValue(StorageKey.WrappedRootKey)
+    return !isNullOrUndefined(wrappedRootKey)
+  }
+
+  public usesKeychain() {
+    return true
+  }
+}

packages/snjs/lib/Migrations/StorageReaders/Versions/index.ts

@@ -0,0 +1,2 @@
+export { StorageReader2_0_0 } from './Reader2_0_0'
+export { StorageReader1_0_0 } from './Reader1_0_0'

packages/snjs/lib/Migrations/Versions/2_0_0.ts

@@ -0,0 +1,725 @@
+import { AnyKeyParamsContent, ContentType, ProtocolVersion } from '@standardnotes/common'
+import { JwtSession } from '../../Services/Session/Sessions/JwtSession'
+import { Migration } from '@Lib/Migrations/Migration'
+import { MigrationServices } from '../MigrationServices'
+import { PreviousSnjsVersion2_0_0 } from '../../Version'
+import { SNRootKey, CreateNewRootKey } from '@standardnotes/encryption'
+import { DiskStorageService } from '../../Services/Storage/DiskStorageService'
+import { StorageReader1_0_0 } from '../StorageReaders/Versions/Reader1_0_0'
+import * as Models from '@standardnotes/models'
+import * as Services from '@standardnotes/services'
+import * as Utils from '@standardnotes/utils'
+import { isEnvironmentMobile, isEnvironmentWebOrDesktop } from '@Lib/Application/Platforms'
+import {
+  getIncrementedDirtyIndex,
+  LegacyMobileKeychainStructure,
+  PayloadTimestampDefaults,
+} from '@standardnotes/models'
+import { isMobileDevice } from '@standardnotes/services'
+
+interface LegacyStorageContent extends Models.ItemContent {
+  storage: unknown
+}
+
+interface LegacyAccountKeysValue {
+  ak: string
+  mk: string
+  version: string
+  jwt: string
+}
+
+interface LegacyRootKeyContent extends Models.RootKeyContent {
+  accountKeys?: LegacyAccountKeysValue
+}
+
+const LEGACY_SESSION_TOKEN_KEY = 'jwt'
+
+export class Migration2_0_0 extends Migration {
+  private legacyReader!: StorageReader1_0_0
+
+  constructor(services: MigrationServices) {
+    super(services)
+    this.legacyReader = new StorageReader1_0_0(
+      this.services.deviceInterface,
+      this.services.identifier,
+      this.services.environment,
+    )
+  }
+
+  static override version() {
+    return PreviousSnjsVersion2_0_0
+  }
+
+  protected registerStageHandlers() {
+    this.registerStageHandler(Services.ApplicationStage.PreparingForLaunch_0, async () => {
+      if (isEnvironmentWebOrDesktop(this.services.environment)) {
+        await this.migrateStorageStructureForWebDesktop()
+      } else if (isEnvironmentMobile(this.services.environment)) {
+        await this.migrateStorageStructureForMobile()
+      }
+    })
+    this.registerStageHandler(Services.ApplicationStage.StorageDecrypted_09, async () => {
+      await this.migrateArbitraryRawStorageToManagedStorageAllPlatforms()
+      if (isEnvironmentMobile(this.services.environment)) {
+        await this.migrateMobilePreferences()
+      }
+      await this.migrateSessionStorage()
+      await this.deleteLegacyStorageValues()
+    })
+    this.registerStageHandler(Services.ApplicationStage.LoadingDatabase_11, async () => {
+      await this.createDefaultItemsKeyForAllPlatforms()
+      this.markDone()
+    })
+  }
+
+  /**
+   * Web
+   * Migrates legacy storage structure into new managed format.
+   * If encrypted storage exists, we need to first decrypt it with the passcode.
+   * Then extract the account key from it. Then, encrypt storage with the
+   * account key. Then encrypt the account key with the passcode and store it
+   * within the new storage format.
+   *
+   * Generate note: We do not use the keychain if passcode is available.
+   */
+  private async migrateStorageStructureForWebDesktop() {
+    const deviceInterface = this.services.deviceInterface
+    const newStorageRawStructure: Services.StorageValuesObject = {
+      [Services.ValueModesKeys.Wrapped]: {} as Models.LocalStorageEncryptedContextualPayload,
+      [Services.ValueModesKeys.Unwrapped]: {},
+      [Services.ValueModesKeys.Nonwrapped]: {},
+    }
+    const rawAccountKeyParams = (await this.legacyReader.getAccountKeyParams()) as AnyKeyParamsContent
+    /** Could be null if no account, or if account and storage is encrypted */
+    if (rawAccountKeyParams) {
+      newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyParams] = rawAccountKeyParams
+    }
+    const encryptedStorage = (await deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.WebEncryptedStorageKey,
+    )) as Models.EncryptedTransferPayload
+
+    if (encryptedStorage) {
+      const encryptedStoragePayload = new Models.EncryptedPayload(encryptedStorage)
+
+      const passcodeResult = await this.webDesktopHelperGetPasscodeKeyAndDecryptEncryptedStorage(
+        encryptedStoragePayload,
+      )
+
+      const passcodeKey = passcodeResult.key
+      const decryptedStoragePayload = passcodeResult.decryptedStoragePayload
+      const passcodeParams = passcodeResult.keyParams
+
+      newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyWrapperKeyParams] = passcodeParams.getPortableValue()
+
+      const rawStorageValueStore = Utils.Copy(decryptedStoragePayload.content.storage)
+      const storageValueStore: Record<string, unknown> = Utils.jsonParseEmbeddedKeys(rawStorageValueStore)
+      /** Store previously encrypted auth_params into new nonwrapped value key */
+
+      const accountKeyParams = storageValueStore[Services.LegacyKeys1_0_0.AllAccountKeyParamsKey] as AnyKeyParamsContent
+      newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyParams] = accountKeyParams
+
+      let keyToEncryptStorageWith = passcodeKey
+      /** Extract account key (mk, pw, ak) if it exists */
+      const hasAccountKeys = !Utils.isNullOrUndefined(storageValueStore.mk)
+
+      if (hasAccountKeys) {
+        const { accountKey, wrappedKey } = await this.webDesktopHelperExtractAndWrapAccountKeysFromValueStore(
+          passcodeKey,
+          accountKeyParams,
+          storageValueStore,
+        )
+        keyToEncryptStorageWith = accountKey
+        newStorageRawStructure.nonwrapped[Services.StorageKey.WrappedRootKey] = wrappedKey
+      }
+
+      /** Encrypt storage with proper key */
+      newStorageRawStructure.wrapped = await this.webDesktopHelperEncryptStorage(
+        keyToEncryptStorageWith,
+        decryptedStoragePayload,
+        storageValueStore,
+      )
+    } else {
+      /**
+       * No encrypted storage, take account keys (if they exist) out of raw storage
+       * and place them in the keychain. */
+      const ak = await this.services.deviceInterface.getRawStorageValue('ak')
+      const mk = await this.services.deviceInterface.getRawStorageValue('mk')
+
+      if (ak || mk) {
+        const version = rawAccountKeyParams.version || (await this.getFallbackRootKeyVersion())
+
+        const accountKey = CreateNewRootKey({
+          masterKey: mk as string,
+          dataAuthenticationKey: ak as string,
+          version: version,
+          keyParams: rawAccountKeyParams,
+        })
+        await this.services.deviceInterface.setNamespacedKeychainValue(
+          accountKey.getKeychainValue(),
+          this.services.identifier,
+        )
+      }
+    }
+
+    /** Persist storage under new key and structure */
+    await this.allPlatformHelperSetStorageStructure(newStorageRawStructure)
+  }
+
+  /**
+   * Helper
+   * All platforms
+   */
+  private async allPlatformHelperSetStorageStructure(rawStructure: Services.StorageValuesObject) {
+    const newStructure = DiskStorageService.DefaultValuesObject(
+      rawStructure.wrapped,
+      rawStructure.unwrapped,
+      rawStructure.nonwrapped,
+    ) as Partial<Services.StorageValuesObject>
+
+    newStructure[Services.ValueModesKeys.Unwrapped] = undefined
+
+    await this.services.deviceInterface.setRawStorageValue(
+      Services.namespacedKey(this.services.identifier, Services.RawStorageKey.StorageObject),
+      JSON.stringify(newStructure),
+    )
+  }
+
+  /**
+   * Helper
+   * Web/desktop only
+   */
+  private async webDesktopHelperGetPasscodeKeyAndDecryptEncryptedStorage(
+    encryptedPayload: Models.EncryptedPayloadInterface,
+  ) {
+    const rawPasscodeParams = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.WebPasscodeParamsKey,
+    )) as AnyKeyParamsContent
+    const passcodeParams = this.services.protocolService.createKeyParams(rawPasscodeParams)
+
+    /** Decrypt it with the passcode */
+    let decryptedStoragePayload:
+      | Models.DecryptedPayloadInterface<LegacyStorageContent>
+      | Models.EncryptedPayloadInterface = encryptedPayload
+    let passcodeKey: SNRootKey | undefined
+
+    await this.promptForPasscodeUntilCorrect(async (candidate: string) => {
+      passcodeKey = await this.services.protocolService.computeRootKey(candidate, passcodeParams)
+      decryptedStoragePayload = await this.services.protocolService.decryptSplitSingle({
+        usesRootKey: {
+          items: [encryptedPayload],
+          key: passcodeKey,
+        },
+      })
+
+      return !Models.isErrorDecryptingPayload(decryptedStoragePayload)
+    })
+
+    return {
+      decryptedStoragePayload:
+        decryptedStoragePayload as unknown as Models.DecryptedPayloadInterface<LegacyStorageContent>,
+      key: passcodeKey as SNRootKey,
+      keyParams: passcodeParams,
+    }
+  }
+
+  /**
+   * Helper
+   * Web/desktop only
+   */
+  private async webDesktopHelperExtractAndWrapAccountKeysFromValueStore(
+    passcodeKey: SNRootKey,
+    accountKeyParams: AnyKeyParamsContent,
+    storageValueStore: Record<string, unknown>,
+  ) {
+    const version = accountKeyParams?.version || (await this.getFallbackRootKeyVersion())
+    const accountKey = CreateNewRootKey({
+      masterKey: storageValueStore.mk as string,
+      dataAuthenticationKey: storageValueStore.ak as string,
+      version: version,
+      keyParams: accountKeyParams,
+    })
+
+    delete storageValueStore.mk
+    delete storageValueStore.pw
+    delete storageValueStore.ak
+
+    const accountKeyPayload = accountKey.payload
+
+    /** Encrypt account key with passcode */
+    const encryptedAccountKey = await this.services.protocolService.encryptSplitSingle({
+      usesRootKey: {
+        items: [accountKeyPayload],
+        key: passcodeKey,
+      },
+    })
+    return {
+      accountKey: accountKey,
+      wrappedKey: Models.CreateEncryptedLocalStorageContextPayload(encryptedAccountKey),
+    }
+  }
+
+  /**
+   * Helper
+   * Web/desktop only
+   * Encrypt storage with account key
+   */
+  async webDesktopHelperEncryptStorage(
+    key: SNRootKey,
+    decryptedStoragePayload: Models.DecryptedPayloadInterface,
+    storageValueStore: Record<string, unknown>,
+  ) {
+    const wrapped = await this.services.protocolService.encryptSplitSingle({
+      usesRootKey: {
+        items: [
+          decryptedStoragePayload.copy({
+            content_type: ContentType.EncryptedStorage,
+            content: storageValueStore as unknown as Models.ItemContent,
+          }),
+        ],
+        key: key,
+      },
+    })
+
+    return Models.CreateEncryptedLocalStorageContextPayload(wrapped)
+  }
+
+  /**
+   * Mobile
+   * On mobile legacy structure is mostly similar to new structure,
+   * in that the account key is encrypted with the passcode. But mobile did
+   * not have encrypted storage, so we simply need to transfer all existing
+   * storage values into new managed structure.
+   *
+   * In version <= 3.0.16 on mobile, encrypted account keys were stored in the keychain
+   * under `encryptedAccountKeys`. In 3.0.17 a migration was introduced that moved this value
+   * to storage under key `encrypted_account_keys`. We need to anticipate the keys being in
+   * either location.
+   *
+   * If no account but passcode only, the only thing we stored on mobile
+   * previously was keys.offline.pw and keys.offline.timing in the keychain
+   * that we compared against for valid decryption.
+   * In the new version, we know a passcode is correct if it can decrypt storage.
+   * As part of the migration, we’ll need to request the raw passcode from user,
+   * compare it against the keychain offline.pw value, and if correct,
+   * migrate storage to new structure, and encrypt with passcode key.
+   *
+   * If account only, take the value in the keychain, and rename the values
+   * (i.e mk > masterKey).
+   * @access private
+   */
+  async migrateStorageStructureForMobile() {
+    Utils.assert(isMobileDevice(this.services.deviceInterface))
+
+    const keychainValue =
+      (await this.services.deviceInterface.getRawKeychainValue()) as unknown as LegacyMobileKeychainStructure
+
+    const wrappedAccountKey = ((await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.MobileWrappedRootKeyKey,
+    )) || keychainValue?.encryptedAccountKeys) as Models.EncryptedTransferPayload
+
+    const rawAccountKeyParams = (await this.legacyReader.getAccountKeyParams()) as AnyKeyParamsContent
+
+    const rawPasscodeParams = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.MobilePasscodeParamsKey,
+    )) as AnyKeyParamsContent
+
+    const firstRunValue = await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.NonwrappedStorageKey.MobileFirstRun,
+    )
+
+    const rawStructure: Services.StorageValuesObject = {
+      [Services.ValueModesKeys.Nonwrapped]: {
+        [Services.StorageKey.WrappedRootKey]: wrappedAccountKey,
+        /** A 'hash' key may be present from legacy versions that should be deleted */
+        [Services.StorageKey.RootKeyWrapperKeyParams]: Utils.omitByCopy(rawPasscodeParams, ['hash' as never]),
+        [Services.StorageKey.RootKeyParams]: rawAccountKeyParams,
+        [Services.NonwrappedStorageKey.MobileFirstRun]: firstRunValue,
+      },
+      [Services.ValueModesKeys.Unwrapped]: {},
+      [Services.ValueModesKeys.Wrapped]: {} as Models.LocalStorageDecryptedContextualPayload,
+    }
+
+    const biometricPrefs = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.MobileBiometricsPrefs,
+    )) as { enabled: boolean; timing: unknown }
+
+    if (biometricPrefs) {
+      rawStructure.nonwrapped[Services.StorageKey.BiometricsState] = biometricPrefs.enabled
+      rawStructure.nonwrapped[Services.StorageKey.MobileBiometricsTiming] = biometricPrefs.timing
+    }
+
+    const passcodeKeyboardType = await this.services.deviceInterface.getRawStorageValue(
+      Services.LegacyKeys1_0_0.MobilePasscodeKeyboardType,
+    )
+
+    if (passcodeKeyboardType) {
+      rawStructure.nonwrapped[Services.StorageKey.MobilePasscodeKeyboardType] = passcodeKeyboardType
+    }
+
+    if (rawPasscodeParams) {
+      const passcodeParams = this.services.protocolService.createKeyParams(rawPasscodeParams)
+      const getPasscodeKey = async () => {
+        let passcodeKey: SNRootKey | undefined
+
+        await this.promptForPasscodeUntilCorrect(async (candidate: string) => {
+          passcodeKey = await this.services.protocolService.computeRootKey(candidate, passcodeParams)
+
+          const pwHash = keychainValue?.offline?.pw
+
+          if (pwHash) {
+            return passcodeKey.serverPassword === pwHash
+          } else {
+            /**
+             * Fallback decryption if keychain is missing for some reason. If account,
+             * validate by attempting to decrypt wrapped account key. Otherwise, validate
+             * by attempting to decrypt random item.
+             * */
+            if (wrappedAccountKey) {
+              const decryptedAcctKey = await this.services.protocolService.decryptSplitSingle({
+                usesRootKey: {
+                  items: [new Models.EncryptedPayload(wrappedAccountKey)],
+                  key: passcodeKey,
+                },
+              })
+              return !Models.isErrorDecryptingPayload(decryptedAcctKey)
+            } else {
+              const item = (
+                await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier)
+              )[0] as Models.EncryptedTransferPayload
+
+              if (!item) {
+                throw Error('Passcode only migration aborting due to missing keychain.offline.pw')
+              }
+
+              const decryptedPayload = await this.services.protocolService.decryptSplitSingle({
+                usesRootKey: {
+                  items: [new Models.EncryptedPayload(item)],
+                  key: passcodeKey,
+                },
+              })
+              return !Models.isErrorDecryptingPayload(decryptedPayload)
+            }
+          }
+        })
+
+        return passcodeKey as SNRootKey
+      }
+
+      rawStructure.nonwrapped[Services.StorageKey.MobilePasscodeTiming] = keychainValue?.offline?.timing
+
+      if (wrappedAccountKey) {
+        /**
+         * Account key is encrypted with passcode. Inside, the accountKey is located inside
+         * content.accountKeys. We want to unembed these values to main content, rename
+         * with proper property names, wrap again, and store in new rawStructure.
+         */
+        const passcodeKey = await getPasscodeKey()
+        const payload = new Models.EncryptedPayload(wrappedAccountKey)
+        const unwrappedAccountKey = await this.services.protocolService.decryptSplitSingle<LegacyRootKeyContent>({
+          usesRootKey: {
+            items: [payload],
+            key: passcodeKey,
+          },
+        })
+
+        if (Models.isErrorDecryptingPayload(unwrappedAccountKey)) {
+          return
+        }
+
+        const accountKeyContent = unwrappedAccountKey.content.accountKeys as LegacyAccountKeysValue
+
+        const version =
+          accountKeyContent.version || rawAccountKeyParams?.version || (await this.getFallbackRootKeyVersion())
+
+        const newAccountKey = unwrappedAccountKey.copy({
+          content: Models.FillItemContent<LegacyRootKeyContent>({
+            masterKey: accountKeyContent.mk,
+            dataAuthenticationKey: accountKeyContent.ak,
+            version: version as ProtocolVersion,
+            keyParams: rawAccountKeyParams,
+            accountKeys: undefined,
+          }),
+        })
+
+        const newWrappedAccountKey = await this.services.protocolService.encryptSplitSingle({
+          usesRootKey: {
+            items: [newAccountKey],
+            key: passcodeKey,
+          },
+        })
+        rawStructure.nonwrapped[Services.StorageKey.WrappedRootKey] =
+          Models.CreateEncryptedLocalStorageContextPayload(newWrappedAccountKey)
+
+        if (accountKeyContent.jwt) {
+          /** Move the jwt to raw storage so that it can be migrated in `migrateSessionStorage` */
+          void this.services.deviceInterface.setRawStorageValue(LEGACY_SESSION_TOKEN_KEY, accountKeyContent.jwt)
+        }
+        await this.services.deviceInterface.clearRawKeychainValue()
+      } else if (!wrappedAccountKey) {
+        /** Passcode only, no account */
+        const passcodeKey = await getPasscodeKey()
+        const payload = new Models.DecryptedPayload({
+          uuid: Utils.UuidGenerator.GenerateUuid(),
+          content: Models.FillItemContent(rawStructure.unwrapped),
+          content_type: ContentType.EncryptedStorage,
+          ...PayloadTimestampDefaults(),
+        })
+
+        /** Encrypt new storage.unwrapped structure with passcode */
+        const wrapped = await this.services.protocolService.encryptSplitSingle({
+          usesRootKey: {
+            items: [payload],
+            key: passcodeKey,
+          },
+        })
+        rawStructure.wrapped = Models.CreateEncryptedLocalStorageContextPayload(wrapped)
+
+        await this.services.deviceInterface.clearRawKeychainValue()
+      }
+    } else {
+      /** No passcode, potentially account. Migrate keychain property keys. */
+      const hasAccount = !Utils.isNullOrUndefined(keychainValue?.mk)
+      if (hasAccount) {
+        const accountVersion =
+          (keychainValue.version as ProtocolVersion) ||
+          rawAccountKeyParams?.version ||
+          (await this.getFallbackRootKeyVersion())
+
+        const accountKey = CreateNewRootKey({
+          masterKey: keychainValue.mk,
+          dataAuthenticationKey: keychainValue.ak,
+          version: accountVersion,
+          keyParams: rawAccountKeyParams,
+        })
+
+        await this.services.deviceInterface.setNamespacedKeychainValue(
+          accountKey.getKeychainValue(),
+          this.services.identifier,
+        )
+
+        if (keychainValue.jwt) {
+          /** Move the jwt to raw storage so that it can be migrated in `migrateSessionStorage` */
+          void this.services.deviceInterface.setRawStorageValue(LEGACY_SESSION_TOKEN_KEY, keychainValue.jwt)
+        }
+      }
+    }
+
+    /** Move encrypted account key into place where it is now expected */
+    await this.allPlatformHelperSetStorageStructure(rawStructure)
+  }
+
+  /**
+   * If we are unable to determine a root key's version, due to missing version
+   * parameter from key params due to 001 or 002, we need to fallback to checking
+   * any encrypted payload and retrieving its version.
+   *
+   * If we are unable to garner any meaningful information, we will default to 002.
+   *
+   * (Previously we attempted to discern version based on presence of keys.ak; if ak,
+   * then 003, otherwise 002. However, late versions of 002 also inluded an ak, so this
+   * method can't be used. This method also didn't account for 001 versions.)
+   */
+  private async getFallbackRootKeyVersion() {
+    const anyItem = (
+      await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier)
+    )[0] as Models.EncryptedTransferPayload
+
+    if (!anyItem) {
+      return ProtocolVersion.V002
+    }
+
+    const payload = new Models.EncryptedPayload(anyItem)
+    return payload.version || ProtocolVersion.V002
+  }
+
+  /**
+   * All platforms
+   * Migrate all previously independently stored storage keys into new
+   * managed approach.
+   */
+  private async migrateArbitraryRawStorageToManagedStorageAllPlatforms() {
+    const allKeyValues = await this.services.deviceInterface.getAllRawStorageKeyValues()
+    const legacyKeys = Utils.objectToValueArray(Services.LegacyKeys1_0_0)
+
+    const tryJsonParse = (value: string) => {
+      try {
+        return JSON.parse(value)
+      } catch (e) {
+        return value
+      }
+    }
+
+    const applicationIdentifier = this.services.identifier
+
+    for (const keyValuePair of allKeyValues) {
+      const key = keyValuePair.key
+      const value = keyValuePair.value
+      const isNameSpacedKey =
+        applicationIdentifier && applicationIdentifier.length > 0 && key.startsWith(applicationIdentifier)
+      if (legacyKeys.includes(key) || isNameSpacedKey) {
+        continue
+      }
+      if (!Utils.isNullOrUndefined(value)) {
+        /**
+         * Raw values should always have been json stringified.
+         * New values should always be objects/parsed.
+         */
+        const newValue = tryJsonParse(value as string)
+        this.services.storageService.setValue(key, newValue)
+      }
+    }
+  }
+
+  /**
+   * All platforms
+   * Deletes all StorageKey and LegacyKeys1_0_0 from root raw storage.
+   * @access private
+   */
+  async deleteLegacyStorageValues() {
+    const miscKeys = [
+      'mk',
+      'ak',
+      'pw',
+      /** v1 unused key */
+      'encryptionKey',
+      /** v1 unused key */
+      'authKey',
+      'jwt',
+      'ephemeral',
+      'cachedThemes',
+    ]
+
+    const managedKeys = [
+      ...Utils.objectToValueArray(Services.StorageKey),
+      ...Utils.objectToValueArray(Services.LegacyKeys1_0_0),
+      ...miscKeys,
+    ]
+
+    for (const key of managedKeys) {
+      await this.services.deviceInterface.removeRawStorageValue(key)
+    }
+  }
+
+  /**
+   * Mobile
+   * Migrate mobile preferences
+   */
+  private async migrateMobilePreferences() {
+    const lastExportDate = await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.MobileLastExportDate,
+    )
+    const doNotWarnUnsupportedEditors = await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.MobileDoNotWarnUnsupportedEditors,
+    )
+    const legacyOptionsState = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
+      Services.LegacyKeys1_0_0.MobileOptionsState,
+    )) as Record<string, unknown>
+
+    let migratedOptionsState = {}
+
+    if (legacyOptionsState) {
+      const legacySortBy = legacyOptionsState.sortBy
+      migratedOptionsState = {
+        sortBy:
+          legacySortBy === 'updated_at' || legacySortBy === 'client_updated_at'
+            ? Models.CollectionSort.UpdatedAt
+            : legacySortBy,
+        sortReverse: legacyOptionsState.sortReverse ?? false,
+        hideNotePreview: legacyOptionsState.hidePreviews ?? false,
+        hideDate: legacyOptionsState.hideDates ?? false,
+        hideTags: legacyOptionsState.hideTags ?? false,
+      }
+    }
+    const preferences = {
+      ...migratedOptionsState,
+      lastExportDate: lastExportDate ?? undefined,
+      doNotShowAgainUnsupportedEditors: doNotWarnUnsupportedEditors ?? false,
+    }
+    await this.services.storageService.setValue(Services.StorageKey.MobilePreferences, preferences)
+  }
+
+  /**
+   * All platforms
+   * Migrate previously stored session string token into object
+   * On mobile, JWTs were previously stored in storage, inside of the user object,
+   * but then custom-migrated to be stored in the keychain. We must account for
+   * both scenarios here in case a user did not perform the custom platform migration.
+   * On desktop/web, JWT was stored in storage.
+   */
+  private migrateSessionStorage() {
+    const USER_OBJECT_KEY = 'user'
+    let currentToken = this.services.storageService.getValue<string | undefined>(LEGACY_SESSION_TOKEN_KEY)
+    const user = this.services.storageService.getValue<{ jwt: string; server: string }>(USER_OBJECT_KEY)
+
+    if (!currentToken) {
+      /** Try the user object */
+      if (user) {
+        currentToken = user.jwt
+      }
+    }
+
+    if (!currentToken) {
+      /**
+       * If we detect that a user object is present, but the jwt is missing,
+       * we'll fill the jwt value with a junk value just so we create a session.
+       * When the client attempts to talk to the server, the server will reply
+       * with invalid token error, and the client will automatically prompt to reauthenticate.
+       */
+      const hasAccount = !Utils.isNullOrUndefined(user)
+      if (hasAccount) {
+        currentToken = 'junk-value'
+      } else {
+        return
+      }
+    }
+
+    const session = new JwtSession(currentToken)
+    this.services.storageService.setValue(Services.StorageKey.Session, session)
+
+    /** Server has to be migrated separately on mobile */
+    if (isEnvironmentMobile(this.services.environment)) {
+      if (user && user.server) {
+        this.services.storageService.setValue(Services.StorageKey.ServerHost, user.server)
+      }
+    }
+  }
+
+  /**
+   * All platforms
+   * Create new default items key from root key.
+   * Otherwise, when data is loaded, we won't be able to decrypt it
+   * without existence of an item key. This will mean that if this migration
+   * is run on two different platforms for the same user, they will create
+   * two new items keys. Which one they use to decrypt past items and encrypt
+   * future items doesn't really matter.
+   * @access private
+   */
+  async createDefaultItemsKeyForAllPlatforms() {
+    const rootKey = this.services.protocolService.getRootKey()
+    if (rootKey) {
+      const rootKeyParams = await this.services.protocolService.getRootKeyParams()
+      /** If params are missing a version, it must be 001 */
+      const fallbackVersion = ProtocolVersion.V001
+
+      const payload = new Models.DecryptedPayload({
+        uuid: Utils.UuidGenerator.GenerateUuid(),
+        content_type: ContentType.ItemsKey,
+        content: Models.FillItemContentSpecialized<Models.ItemsKeyContentSpecialized, Models.ItemsKeyContent>({
+          itemsKey: rootKey.masterKey,
+          dataAuthenticationKey: rootKey.dataAuthenticationKey,
+          version: rootKeyParams?.version || fallbackVersion,
+        }),
+        dirty: true,
+        dirtyIndex: getIncrementedDirtyIndex(),
+        ...PayloadTimestampDefaults(),
+      })
+
+      const itemsKey = Models.CreateDecryptedItemFromPayload(payload)
+
+      await this.services.itemManager.emitItemFromPayload(
+        itemsKey.payloadRepresentation(),
+        Models.PayloadEmitSource.LocalChanged,
+      )
+    }
+  }
+}

packages/snjs/lib/Migrations/Versions/2_0_15.ts

@@ -0,0 +1,21 @@
+import { ApplicationStage } from '@standardnotes/services'
+import { Migration } from '@Lib/Migrations/Migration'
+
+export class Migration2_0_15 extends Migration {
+  static override version(): string {
+    return '2.0.15'
+  }
+
+  protected registerStageHandlers(): void {
+    this.registerStageHandler(ApplicationStage.LoadedDatabase_12, async () => {
+      await this.createNewDefaultItemsKeyIfNecessary()
+      this.markDone()
+    })
+  }
+
+  private async createNewDefaultItemsKeyIfNecessary() {
+    if (this.services.protocolService.needsNewRootKeyBasedItemsKey()) {
+      await this.services.protocolService.createNewDefaultItemsKey()
+    }
+  }
+}

packages/snjs/lib/Migrations/Versions/2_20_0.ts

@@ -0,0 +1,26 @@
+import { Migration } from '@Lib/Migrations/Migration'
+import { ContentType } from '@standardnotes/common'
+import { ApplicationStage } from '@standardnotes/services'
+
+export class Migration2_20_0 extends Migration {
+  static override version(): string {
+    return '2.20.0'
+  }
+
+  protected registerStageHandlers(): void {
+    this.registerStageHandler(ApplicationStage.LoadedDatabase_12, async () => {
+      await this.deleteMfaItems()
+      this.markDone()
+    })
+  }
+
+  private async deleteMfaItems(): Promise<void> {
+    const contentType = 'SF|MFA' as ContentType
+    const items = this.services.itemManager.getItems(contentType)
+
+    for (const item of items) {
+      this.services.itemManager.removeItemLocally(item)
+      await this.services.storageService.deletePayloadWithId(item.uuid)
+    }
+  }
+}

packages/snjs/lib/Migrations/Versions/2_36_0.ts

@@ -0,0 +1,26 @@
+import { Migration } from '@Lib/Migrations/Migration'
+import { ContentType } from '@standardnotes/common'
+import { ApplicationStage } from '@standardnotes/services'
+
+export class Migration2_36_0 extends Migration {
+  static override version(): string {
+    return '2.36.0'
+  }
+
+  protected registerStageHandlers(): void {
+    this.registerStageHandler(ApplicationStage.LoadedDatabase_12, async () => {
+      await this.removeServerExtensionsLocally()
+      this.markDone()
+    })
+  }
+
+  private async removeServerExtensionsLocally(): Promise<void> {
+    const contentType = 'SF|Extension' as ContentType
+    const items = this.services.itemManager.getItems(contentType)
+
+    for (const item of items) {
+      this.services.itemManager.removeItemLocally(item)
+      await this.services.storageService.deletePayloadWithId(item.uuid)
+    }
+  }
+}

packages/snjs/lib/Migrations/Versions/2_42_0.ts

@@ -0,0 +1,30 @@
+import { ContentType } from '@standardnotes/common'
+import { ApplicationStage } from '@standardnotes/services'
+import { FeatureIdentifier } from '@standardnotes/features'
+import { Migration } from '@Lib/Migrations/Migration'
+import { SNTheme } from '@standardnotes/models'
+
+const NoDistractionIdentifier = 'org.standardnotes.theme-no-distraction' as FeatureIdentifier
+
+export class Migration2_42_0 extends Migration {
+  static override version(): string {
+    return '2.42.0'
+  }
+
+  protected registerStageHandlers(): void {
+    this.registerStageHandler(ApplicationStage.FullSyncCompleted_13, async () => {
+      await this.deleteNoDistraction()
+      this.markDone()
+    })
+  }
+
+  private async deleteNoDistraction(): Promise<void> {
+    const themes = (this.services.itemManager.getItems(ContentType.Theme) as SNTheme[]).filter((theme) => {
+      return theme.identifier === NoDistractionIdentifier
+    })
+
+    for (const theme of themes) {
+      await this.services.itemManager.setItemToBeDeleted(theme)
+    }
+  }
+}

packages/snjs/lib/Migrations/Versions/2_7_0.ts

@@ -0,0 +1,32 @@
+import { CompoundPredicate, Predicate, SNComponent } from '@standardnotes/models'
+import { Migration } from '@Lib/Migrations/Migration'
+import { ContentType } from '@standardnotes/common'
+import { ApplicationStage } from '@standardnotes/services'
+
+export class Migration2_7_0 extends Migration {
+  static override version(): string {
+    return '2.7.0'
+  }
+
+  protected registerStageHandlers(): void {
+    this.registerStageHandler(ApplicationStage.FullSyncCompleted_13, async () => {
+      await this.deleteBatchManagerSingleton()
+      this.markDone()
+    })
+  }
+
+  private async deleteBatchManagerSingleton() {
+    const batchMgrId = 'org.standardnotes.batch-manager'
+
+    const batchMgrPred = new CompoundPredicate('and', [
+      new Predicate<SNComponent>('content_type', '=', ContentType.Component),
+      new Predicate<SNComponent>('identifier', '=', batchMgrId),
+    ])
+
+    const batchMgrSingleton = this.services.singletonManager.findSingleton(ContentType.Component, batchMgrPred)
+
+    if (batchMgrSingleton) {
+      await this.services.itemManager.setItemToBeDeleted(batchMgrSingleton)
+    }
+  }
+}

packages/snjs/lib/Migrations/Versions/index.ts

@@ -0,0 +1,17 @@
+import { Migration2_0_0 } from './2_0_0'
+import { Migration2_0_15 } from './2_0_15'
+import { Migration2_7_0 } from './2_7_0'
+import { Migration2_20_0 } from './2_20_0'
+import { Migration2_36_0 } from './2_36_0'
+import { Migration2_42_0 } from './2_42_0'
+
+export const MigrationClasses = [
+  Migration2_0_0,
+  Migration2_0_15,
+  Migration2_7_0,
+  Migration2_20_0,
+  Migration2_36_0,
+  Migration2_42_0,
+]
+
+export { Migration2_0_0, Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0 }

packages/snjs/lib/Migrations/index.ts

@@ -0,0 +1,2 @@
+export { BaseMigration } from './Base'
+export * from './Versions'

packages/snjs/lib/Services/Actions/ActionsService.ts

@@ -0,0 +1,324 @@
+import { EncryptionService, SNRootKey } from '@standardnotes/encryption'
+import { Challenge, ChallengeService } from '../Challenge'
+import { ListedService } from '../Listed/ListedService'
+import { ActionResponse, HttpResponse } from '@standardnotes/responses'
+import { ContentType } from '@standardnotes/common'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import {
+  SNActionsExtension,
+  Action,
+  ActionAccessType,
+  ActionsExtensionMutator,
+  MutationType,
+  CreateDecryptedItemFromPayload,
+  DecryptedItemInterface,
+  DecryptedPayloadInterface,
+  ActionExtensionContent,
+  EncryptedPayload,
+  isErrorDecryptingPayload,
+  CreateEncryptedBackupFileContextPayload,
+  EncryptedTransferPayload,
+} from '@standardnotes/models'
+import { SNSyncService } from '../Sync/SyncService'
+import { PayloadManager } from '../Payloads/PayloadManager'
+import { SNHttpService } from '../Api/HttpService'
+import {
+  AbstractService,
+  DeviceInterface,
+  InternalEventBusInterface,
+  AlertService,
+  ChallengeValidation,
+  ChallengeReason,
+  ChallengePrompt,
+} from '@standardnotes/services'
+
+/**
+ * The Actions Service allows clients to interact with action-based extensions.
+ * Action-based extensions are mostly RESTful actions that can push a local value or
+ * retrieve a remote value and act on it accordingly.
+ * There are 4 action types:
+ * `get`: performs a GET request on an endpoint to retrieve an item value, and merges the
+ *      value onto the local item value. For example, you can GET an item's older revision
+ *      value and replace the current value with the revision.
+ * `render`: performs a GET request, and displays the result in the UI. This action does not
+ *         affect data unless action is taken explicitely in the UI after the data is presented.
+ * `show`: opens the action's URL in a browser.
+ * `post`: sends an item's data to a remote service. This is used for example by Listed
+ *       to allow publishing a note to a user's blog.
+ */
+export class SNActionsService extends AbstractService {
+  private previousPasswords: string[] = []
+
+  constructor(
+    private itemManager: ItemManager,
+    private alertService: AlertService,
+    public deviceInterface: DeviceInterface,
+    private httpService: SNHttpService,
+    private payloadManager: PayloadManager,
+    private protocolService: EncryptionService,
+    private syncService: SNSyncService,
+    private challengeService: ChallengeService,
+    private listedService: ListedService,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+    this.previousPasswords = []
+  }
+
+  public override deinit(): void {
+    ;(this.itemManager as unknown) = undefined
+    ;(this.alertService as unknown) = undefined
+    ;(this.deviceInterface as unknown) = undefined
+    ;(this.httpService as unknown) = undefined
+    ;(this.payloadManager as unknown) = undefined
+    ;(this.listedService as unknown) = undefined
+    ;(this.challengeService as unknown) = undefined
+    ;(this.protocolService as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    this.previousPasswords.length = 0
+    super.deinit()
+  }
+
+  public getExtensions(): SNActionsExtension[] {
+    const extensionItems = this.itemManager.getItems<SNActionsExtension>(ContentType.ActionsExtension)
+    const excludingListed = extensionItems.filter((extension) => !extension.isListedExtension)
+    return excludingListed
+  }
+
+  public extensionsInContextOfItem(item: DecryptedItemInterface) {
+    return this.getExtensions().filter((ext) => {
+      return ext.supported_types.includes(item.content_type) || ext.actionsWithContextForItem(item).length > 0
+    })
+  }
+
+  /**
+   * Loads an extension in the context of a certain item.
+   * The server then has the chance to respond with actions that are
+   * relevant just to this item. The response extension is not saved,
+   * just displayed as a one-time thing.
+   */
+  public async loadExtensionInContextOfItem(
+    extension: SNActionsExtension,
+    item: DecryptedItemInterface,
+  ): Promise<SNActionsExtension | undefined> {
+    const params = {
+      content_type: item.content_type,
+      item_uuid: item.uuid,
+    }
+
+    const response = (await this.httpService.getAbsolute(extension.url, params).catch((response) => {
+      console.error('Error loading extension', response)
+      return undefined
+    })) as ActionResponse
+
+    if (!response) {
+      return
+    }
+
+    const description = response.description || extension.description
+    const supported_types = response.supported_types || extension.supported_types
+    const actions = (response.actions || []) as Action[]
+    const mutator = new ActionsExtensionMutator(extension, MutationType.UpdateUserTimestamps)
+
+    mutator.deprecation = response.deprecation
+    mutator.description = description
+    mutator.supported_types = supported_types
+    mutator.actions = actions
+
+    const payloadResult = mutator.getResult()
+
+    return CreateDecryptedItemFromPayload(payloadResult) as SNActionsExtension
+  }
+
+  public async runAction(action: Action, item: DecryptedItemInterface): Promise<ActionResponse | undefined> {
+    let result
+    switch (action.verb) {
+      case 'render':
+        result = await this.handleRenderAction(action)
+        break
+      case 'show':
+        result = this.handleShowAction(action)
+        break
+      case 'post':
+        result = await this.handlePostAction(action, item)
+        break
+      default:
+        break
+    }
+    return result
+  }
+
+  private async handleRenderAction(action: Action): Promise<ActionResponse | undefined> {
+    const response = await this.httpService
+      .getAbsolute(action.url)
+      .then(async (response) => {
+        const payload = await this.payloadByDecryptingResponse(response as ActionResponse)
+        if (payload) {
+          const item = CreateDecryptedItemFromPayload<ActionExtensionContent, SNActionsExtension>(payload)
+          return {
+            ...(response as ActionResponse),
+            item,
+          }
+        } else {
+          return undefined
+        }
+      })
+      .catch((response) => {
+        const error = (response && response.error) || {
+          message: 'An issue occurred while processing this action. Please try again.',
+        }
+        void this.alertService.alert(error.message)
+        return { error } as HttpResponse
+      })
+
+    return response as ActionResponse
+  }
+
+  private async payloadByDecryptingResponse(
+    response: ActionResponse,
+    rootKey?: SNRootKey,
+    triedPasswords: string[] = [],
+  ): Promise<DecryptedPayloadInterface<ActionExtensionContent> | undefined> {
+    if (!response.item || response.item.deleted || response.item.content == undefined) {
+      return undefined
+    }
+
+    const payload = new EncryptedPayload(response.item as EncryptedTransferPayload)
+
+    if (!payload.enc_item_key) {
+      void this.alertService.alert('This revision is missing its key and cannot be recovered.')
+      return
+    }
+
+    let decryptedPayload = await this.protocolService.decryptSplitSingle<ActionExtensionContent>({
+      usesItemsKeyWithKeyLookup: {
+        items: [payload],
+      },
+    })
+
+    if (!isErrorDecryptingPayload(decryptedPayload)) {
+      return decryptedPayload
+    }
+
+    if (rootKey) {
+      decryptedPayload = await this.protocolService.decryptSplitSingle({
+        usesRootKey: {
+          items: [payload],
+          key: rootKey,
+        },
+      })
+      if (!isErrorDecryptingPayload(decryptedPayload)) {
+        return decryptedPayload
+      }
+    }
+
+    for (const itemsKey of this.itemManager.getDisplayableItemsKeys()) {
+      const decryptedPayload = await this.protocolService.decryptSplitSingle<ActionExtensionContent>({
+        usesItemsKey: {
+          items: [payload],
+          key: itemsKey,
+        },
+      })
+
+      if (!isErrorDecryptingPayload(decryptedPayload)) {
+        return decryptedPayload
+      }
+    }
+
+    const keyParamsData = response.keyParams || response.auth_params
+    if (!keyParamsData) {
+      /**
+       * In some cases revisions were missing auth params.
+       * Instruct the user to email us to get this remedied.
+       */
+      void this.alertService.alert(
+        'We were unable to decrypt this revision using your current keys, ' +
+          'and this revision is missing metadata that would allow us to try different ' +
+          'keys to decrypt it. This can likely be fixed with some manual intervention. ' +
+          'Please email help@standardnotes.com for assistance.',
+      )
+      return undefined
+    }
+    const keyParams = this.protocolService.createKeyParams(keyParamsData)
+
+    /* Try previous passwords */
+    for (const passwordCandidate of this.previousPasswords) {
+      if (triedPasswords.includes(passwordCandidate)) {
+        continue
+      }
+
+      triedPasswords.push(passwordCandidate)
+
+      const key = await this.protocolService.computeRootKey(passwordCandidate, keyParams)
+      if (!key) {
+        continue
+      }
+
+      const nestedResponse = await this.payloadByDecryptingResponse(response, key, triedPasswords)
+      if (nestedResponse) {
+        return nestedResponse
+      }
+    }
+
+    /** Prompt for other passwords */
+    const password = await this.promptForLegacyPassword()
+    if (!password) {
+      return undefined
+    }
+
+    if (this.previousPasswords.includes(password)) {
+      return undefined
+    }
+
+    this.previousPasswords.push(password)
+    return this.payloadByDecryptingResponse(response, rootKey)
+  }
+
+  private async promptForLegacyPassword(): Promise<string | undefined> {
+    const challenge = new Challenge(
+      [new ChallengePrompt(ChallengeValidation.None, 'Previous Password', undefined, true)],
+      ChallengeReason.Custom,
+      true,
+      'Unable to find key for revision. Please enter the account password you may have used at the time of the revision.',
+    )
+
+    const response = await this.challengeService.promptForChallengeResponse(challenge)
+
+    return response?.getDefaultValue().value as string
+  }
+
+  private async handlePostAction(action: Action, item: DecryptedItemInterface) {
+    const decrypted = action.access_type === ActionAccessType.Decrypted
+    const itemParams = await this.outgoingPayloadForItem(item, decrypted)
+    const params = {
+      items: [itemParams],
+    }
+    return this.httpService
+      .postAbsolute(action.url, params)
+      .then((response) => {
+        return response as ActionResponse
+      })
+      .catch((response) => {
+        console.error('Action error response:', response)
+        void this.alertService.alert('An issue occurred while processing this action. Please try again.')
+        return response as ActionResponse
+      })
+  }
+
+  private handleShowAction(action: Action) {
+    void this.deviceInterface.openUrl(action.url)
+    return {} as ActionResponse
+  }
+
+  private async outgoingPayloadForItem(item: DecryptedItemInterface, decrypted = false) {
+    if (decrypted) {
+      return item.payload.ejected()
+    }
+
+    const encrypted = await this.protocolService.encryptSplitSingle({
+      usesItemsKeyWithKeyLookup: { items: [item.payload] },
+    })
+
+    return CreateEncryptedBackupFileContextPayload(encrypted)
+  }
+}

packages/snjs/lib/Services/Api/ApiService.ts

@@ -0,0 +1,903 @@
+import { FeatureDescription } from '@standardnotes/features'
+import { isNullOrUndefined, joinPaths } from '@standardnotes/utils'
+import { SettingName, SubscriptionSettingName } from '@standardnotes/settings'
+import { Uuid, ErrorTag } from '@standardnotes/common'
+import {
+  AbstractService,
+  ApiServiceInterface,
+  InternalEventBusInterface,
+  IntegrityApiInterface,
+  ItemsServerInterface,
+  StorageKey,
+  ApiServiceEvent,
+  MetaReceivedData,
+  DiagnosticInfo,
+  FilesApiInterface,
+  KeyValueStoreInterface,
+} from '@standardnotes/services'
+import { ServerSyncPushContextualPayload, SNFeatureRepo, FileContent } from '@standardnotes/models'
+import * as Responses from '@standardnotes/responses'
+import { API_MESSAGE_FAILED_OFFLINE_ACTIVATION } from '@Lib/Services/Api/Messages'
+import { HttpParams, HttpRequest, HttpVerb, SNHttpService } from './HttpService'
+import { isUrlFirstParty, TRUSTED_FEATURE_HOSTS } from '@Lib/Hosts'
+import { Paths } from './Paths'
+import { Session } from '../Session/Sessions/Session'
+import { TokenSession } from '../Session/Sessions/TokenSession'
+import { DiskStorageService } from '../Storage/DiskStorageService'
+import { UserServerInterface } from '../User/UserServerInterface'
+import { UuidString } from '../../Types/UuidString'
+import * as messages from '@Lib/Services/Api/Messages'
+import merge from 'lodash/merge'
+import { SettingsServerInterface } from '../Settings/SettingsServerInterface'
+import { Strings } from '@Lib/Strings'
+import { SNRootKeyParams } from '@standardnotes/encryption'
+import { ApiEndpointParam, ClientDisplayableError, CreateValetTokenPayload } from '@standardnotes/responses'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { HttpResponseMeta } from '@standardnotes/api'
+
+/** Legacy api version field to be specified in params when calling v0 APIs. */
+const V0_API_VERSION = '20200115'
+
+type InvalidSessionObserver = (revoked: boolean) => void
+
+export class SNApiService
+  extends AbstractService<ApiServiceEvent.MetaReceived, MetaReceivedData>
+  implements
+    ApiServiceInterface,
+    FilesApiInterface,
+    IntegrityApiInterface,
+    ItemsServerInterface,
+    UserServerInterface,
+    SettingsServerInterface
+{
+  private session?: Session
+  public user?: Responses.User
+  private registering = false
+  private authenticating = false
+  private changing = false
+  private refreshingSession = false
+  private invalidSessionObserver?: InvalidSessionObserver
+  private filesHost?: string
+
+  constructor(
+    private httpService: SNHttpService,
+    private storageService: DiskStorageService,
+    private host: string,
+    private inMemoryStore: KeyValueStoreInterface<string>,
+    private crypto: PureCryptoInterface,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  override deinit(): void {
+    ;(this.httpService as unknown) = undefined
+    ;(this.storageService as unknown) = undefined
+    this.invalidSessionObserver = undefined
+    this.session = undefined
+    super.deinit()
+  }
+
+  public setUser(user?: Responses.User): void {
+    this.user = user
+  }
+
+  /**
+   * When a we receive a 401 error from the server, we'll notify the observer.
+   * Note that this applies only to sessions that are totally invalid. Sessions that
+   * are expired but can be renewed are still considered to be valid. In those cases,
+   * the server response is 498.
+   * If the session has been revoked, then the observer will have its first
+   * argument set to true.
+   */
+  public setInvalidSessionObserver(observer: InvalidSessionObserver): void {
+    this.invalidSessionObserver = observer
+  }
+
+  public loadHost(): void {
+    const storedValue = this.storageService.getValue<string | undefined>(StorageKey.ServerHost)
+    this.host =
+      storedValue ||
+      this.host ||
+      ((
+        window as {
+          _default_sync_server?: string
+        }
+      )._default_sync_server as string)
+  }
+
+  public async setHost(host: string): Promise<void> {
+    this.host = host
+    this.storageService.setValue(StorageKey.ServerHost, host)
+  }
+
+  public getHost(): string {
+    return this.host
+  }
+
+  public isThirdPartyHostUsed(): boolean {
+    const applicationHost = this.getHost() || ''
+    return !isUrlFirstParty(applicationHost)
+  }
+
+  public getFilesHost(): string {
+    if (!this.filesHost) {
+      throw Error('Attempting to access undefined filesHost')
+    }
+    return this.filesHost
+  }
+
+  public setSession(session: Session, persist = true): void {
+    this.session = session
+    if (persist) {
+      this.storageService.setValue(StorageKey.Session, session)
+    }
+  }
+
+  public getSession(): Session | undefined {
+    return this.session
+  }
+
+  public get apiVersion() {
+    return V0_API_VERSION
+  }
+
+  private params(inParams: Record<string | number | symbol, unknown>): HttpParams {
+    const params = merge(inParams, {
+      [ApiEndpointParam.ApiVersion]: this.apiVersion,
+    })
+    return params
+  }
+
+  public createErrorResponse(message: string, status?: Responses.StatusCode): Responses.HttpResponse {
+    return { error: { message, status } } as Responses.HttpResponse
+  }
+
+  private errorResponseWithFallbackMessage(response: Responses.HttpResponse, message: string) {
+    if (!response.error?.message) {
+      response.error = {
+        ...response.error,
+        status: response.error?.status ?? Responses.StatusCode.UnknownError,
+        message,
+      }
+    }
+    return response
+  }
+
+  public processMetaObject(meta: HttpResponseMeta) {
+    if (meta.auth && meta.auth.userUuid && meta.auth.roles) {
+      void this.notifyEvent(ApiServiceEvent.MetaReceived, {
+        userUuid: meta.auth.userUuid,
+        userRoles: meta.auth.roles,
+      })
+    }
+
+    if (meta.server?.filesServerUrl) {
+      this.filesHost = meta.server?.filesServerUrl
+    }
+  }
+
+  private processResponse(response: Responses.HttpResponse) {
+    if (response.meta) {
+      this.processMetaObject(response.meta)
+    }
+  }
+
+  private async request(params: {
+    verb: HttpVerb
+    url: string
+    fallbackErrorMessage: string
+    params?: HttpParams
+    rawBytes?: Uint8Array
+    authentication?: string
+    customHeaders?: Record<string, string>[]
+    responseType?: XMLHttpRequestResponseType
+    external?: boolean
+  }) {
+    try {
+      const response = await this.httpService.runHttp(params)
+      this.processResponse(response)
+      return response
+    } catch (errorResponse) {
+      return this.errorResponseWithFallbackMessage(errorResponse as Responses.HttpResponse, params.fallbackErrorMessage)
+    }
+  }
+
+  /**
+   * @param mfaKeyPath  The params path the server expects for authentication against
+   *                    a particular mfa challenge. A value of foo would mean the server
+   *                    would receive parameters as params['foo'] with value equal to mfaCode.
+   * @param mfaCode     The mfa challenge response value.
+   */
+  async getAccountKeyParams(dto: {
+    email: string
+    mfaKeyPath?: string
+    mfaCode?: string
+  }): Promise<Responses.KeyParamsResponse | Responses.HttpResponse> {
+    const codeVerifier = this.crypto.generateRandomKey(256)
+    this.inMemoryStore.setValue(StorageKey.CodeVerifier, codeVerifier)
+
+    const codeChallenge = this.crypto.base64URLEncode(await this.crypto.sha256(codeVerifier))
+
+    const params = this.params({
+      email: dto.email,
+      code_challenge: codeChallenge,
+    })
+
+    if (dto.mfaKeyPath !== undefined && dto.mfaCode !== undefined) {
+      params[dto.mfaKeyPath] = dto.mfaCode
+    }
+
+    return this.request({
+      verb: HttpVerb.Post,
+      url: joinPaths(this.host, Paths.v2.keyParams),
+      fallbackErrorMessage: messages.API_MESSAGE_GENERIC_INVALID_LOGIN,
+      params,
+      /** A session is optional here, if valid, endpoint bypasses 2FA and returns additional params */
+      authentication: this.session?.authorizationValue,
+    })
+  }
+
+  async signIn(dto: {
+    email: string
+    serverPassword: string
+    ephemeral: boolean
+  }): Promise<Responses.SignInResponse | Responses.HttpResponse> {
+    if (this.authenticating) {
+      return this.createErrorResponse(messages.API_MESSAGE_LOGIN_IN_PROGRESS) as Responses.SignInResponse
+    }
+    this.authenticating = true
+    const url = joinPaths(this.host, Paths.v2.signIn)
+    const params = this.params({
+      email: dto.email,
+      password: dto.serverPassword,
+      ephemeral: dto.ephemeral,
+      code_verifier: this.inMemoryStore.getValue(StorageKey.CodeVerifier) as string,
+    })
+
+    const response = await this.request({
+      verb: HttpVerb.Post,
+      url,
+      params,
+      fallbackErrorMessage: messages.API_MESSAGE_GENERIC_INVALID_LOGIN,
+    })
+
+    this.authenticating = false
+
+    this.inMemoryStore.removeValue(StorageKey.CodeVerifier)
+
+    return response
+  }
+
+  signOut(): Promise<Responses.SignOutResponse> {
+    const url = joinPaths(this.host, Paths.v1.signOut)
+    return this.httpService.postAbsolute(url, undefined, this.session?.authorizationValue).catch((errorResponse) => {
+      return errorResponse
+    }) as Promise<Responses.SignOutResponse>
+  }
+
+  async changeCredentials(parameters: {
+    userUuid: UuidString
+    currentServerPassword: string
+    newServerPassword: string
+    newKeyParams: SNRootKeyParams
+    newEmail?: string
+  }): Promise<Responses.ChangeCredentialsResponse | Responses.HttpResponse> {
+    if (this.changing) {
+      return this.createErrorResponse(messages.API_MESSAGE_CHANGE_CREDENTIALS_IN_PROGRESS)
+    }
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    this.changing = true
+    const url = joinPaths(this.host, Paths.v1.changeCredentials(parameters.userUuid) as string)
+    const params = this.params({
+      current_password: parameters.currentServerPassword,
+      new_password: parameters.newServerPassword,
+      new_email: parameters.newEmail,
+      ...parameters.newKeyParams.getPortableValue(),
+    })
+    const response = await this.httpService
+      .putAbsolute(url, params, this.session?.authorizationValue)
+      .catch(async (errorResponse) => {
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Put,
+            url,
+            params,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(
+          errorResponse,
+          messages.API_MESSAGE_GENERIC_CHANGE_CREDENTIALS_FAIL,
+        )
+      })
+
+    this.processResponse(response)
+
+    this.changing = false
+    return response
+  }
+
+  public async deleteAccount(userUuid: string): Promise<Responses.HttpResponse | Responses.MinimalHttpResponse> {
+    const url = joinPaths(this.host, Paths.v1.deleteAccount(userUuid))
+    const response = await this.request({
+      verb: HttpVerb.Delete,
+      url,
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.ServerErrorStrings.DeleteAccountError,
+    })
+    return response
+  }
+
+  async sync(
+    payloads: ServerSyncPushContextualPayload[],
+    lastSyncToken: string,
+    paginationToken: string,
+    limit: number,
+  ): Promise<Responses.RawSyncResponse | Responses.HttpResponse> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    const url = joinPaths(this.host, Paths.v1.sync)
+    const params = this.params({
+      [ApiEndpointParam.SyncPayloads]: payloads,
+      [ApiEndpointParam.LastSyncToken]: lastSyncToken,
+      [ApiEndpointParam.PaginationToken]: paginationToken,
+      [ApiEndpointParam.SyncDlLimit]: limit,
+    })
+    const response = await this.httpService
+      .postAbsolute(url, params, this.session?.authorizationValue)
+      .catch<Responses.HttpResponse>(async (errorResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Post,
+            url,
+            params,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_SYNC_FAIL)
+      })
+    this.processResponse(response)
+
+    return response
+  }
+
+  private async refreshSessionThenRetryRequest(httpRequest: HttpRequest): Promise<Responses.HttpResponse> {
+    const sessionResponse = await this.refreshSession()
+    if (sessionResponse.error || isNullOrUndefined(sessionResponse.data)) {
+      return sessionResponse
+    } else {
+      return this.httpService
+        .runHttp({
+          ...httpRequest,
+          authentication: this.session?.authorizationValue,
+        })
+        .catch((errorResponse) => {
+          return errorResponse
+        })
+    }
+  }
+
+  async refreshSession(): Promise<Responses.SessionRenewalResponse | Responses.HttpResponse> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    this.refreshingSession = true
+    const url = joinPaths(this.host, Paths.v1.refreshSession)
+    const session = this.session as TokenSession
+    const params = this.params({
+      access_token: session.accessToken,
+      refresh_token: session.refreshToken,
+    })
+    const result = await this.httpService
+      .postAbsolute(url, params)
+      .then(async (response) => {
+        const session = TokenSession.FromApiResponse(response as Responses.SessionRenewalResponse)
+        await this.setSession(session)
+        this.processResponse(response)
+        return response
+      })
+      .catch((errorResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_TOKEN_REFRESH_FAIL)
+      })
+    this.refreshingSession = false
+    return result
+  }
+
+  async getSessionsList(): Promise<Responses.SessionListResponse | Responses.HttpResponse> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    const url = joinPaths(this.host, Paths.v1.sessions)
+    const response = await this.httpService
+      .getAbsolute(url, {}, this.session?.authorizationValue)
+      .catch(async (errorResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Get,
+            url,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_SYNC_FAIL)
+      })
+    this.processResponse(response)
+
+    return response
+  }
+
+  async deleteSession(sessionId: UuidString): Promise<Responses.HttpResponse> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    const url = joinPaths(this.host, <string>Paths.v1.session(sessionId))
+    const response: Responses.RevisionListResponse | Responses.HttpResponse = await this.httpService
+      .deleteAbsolute(url, { uuid: sessionId }, this.session?.authorizationValue)
+      .catch((error: Responses.HttpResponse) => {
+        const errorResponse = error as Responses.HttpResponse
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Delete,
+            url,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_SYNC_FAIL)
+      })
+    this.processResponse(response)
+    return response
+  }
+
+  async getItemRevisions(itemId: UuidString): Promise<Responses.RevisionListResponse | Responses.HttpResponse> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    const url = joinPaths(this.host, Paths.v1.itemRevisions(itemId))
+    const response: Responses.RevisionListResponse | Responses.HttpResponse = await this.httpService
+      .getAbsolute(url, undefined, this.session?.authorizationValue)
+      .catch((errorResponse: Responses.HttpResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Get,
+            url,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_SYNC_FAIL)
+      })
+    this.processResponse(response)
+    return response
+  }
+
+  async getRevision(
+    entry: Responses.RevisionListEntry,
+    itemId: UuidString,
+  ): Promise<Responses.SingleRevisionResponse | Responses.HttpResponse> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError
+    }
+    const url = joinPaths(this.host, Paths.v1.itemRevision(itemId, entry.uuid))
+    const response: Responses.SingleRevisionResponse | Responses.HttpResponse = await this.httpService
+      .getAbsolute(url, undefined, this.session?.authorizationValue)
+      .catch((errorResponse: Responses.HttpResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Get,
+            url,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_SYNC_FAIL)
+      })
+    this.processResponse(response)
+    return response
+  }
+
+  async getUserFeatures(userUuid: UuidString): Promise<Responses.HttpResponse | Responses.UserFeaturesResponse> {
+    const url = joinPaths(this.host, Paths.v1.userFeatures(userUuid))
+    const response = await this.httpService
+      .getAbsolute(url, undefined, this.session?.authorizationValue)
+      .catch((errorResponse: Responses.HttpResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest({
+            verb: HttpVerb.Get,
+            url,
+          })
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, messages.API_MESSAGE_GENERIC_SYNC_FAIL)
+      })
+    this.processResponse(response)
+    return response
+  }
+
+  private async tokenRefreshableRequest<T extends Responses.MinimalHttpResponse>(
+    params: HttpRequest & { fallbackErrorMessage: string },
+  ): Promise<T> {
+    const preprocessingError = this.preprocessingError()
+    if (preprocessingError) {
+      return preprocessingError as T
+    }
+    const response: T | Responses.HttpResponse = await this.httpService
+      .runHttp(params)
+      .catch((errorResponse: Responses.HttpResponse) => {
+        this.preprocessAuthenticatedErrorResponse(errorResponse)
+        if (Responses.isErrorResponseExpiredToken(errorResponse)) {
+          return this.refreshSessionThenRetryRequest(params)
+        }
+        return this.errorResponseWithFallbackMessage(errorResponse, params.fallbackErrorMessage)
+      })
+    this.processResponse(response)
+    return response as T
+  }
+
+  async listSettings(userUuid: UuidString): Promise<Responses.ListSettingsResponse> {
+    return await this.tokenRefreshableRequest<Responses.ListSettingsResponse>({
+      verb: HttpVerb.Get,
+      url: joinPaths(this.host, Paths.v1.settings(userUuid)),
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_GET_SETTINGS,
+      authentication: this.session?.authorizationValue,
+    })
+  }
+
+  async updateSetting(
+    userUuid: UuidString,
+    settingName: string,
+    settingValue: string | null,
+    sensitive: boolean,
+  ): Promise<Responses.UpdateSettingResponse> {
+    const params = {
+      name: settingName,
+      value: settingValue,
+      sensitive: sensitive,
+    }
+    return this.tokenRefreshableRequest<Responses.UpdateSettingResponse>({
+      verb: HttpVerb.Put,
+      url: joinPaths(this.host, Paths.v1.settings(userUuid)),
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_UPDATE_SETTINGS,
+      params,
+    })
+  }
+
+  async getSetting(userUuid: UuidString, settingName: SettingName): Promise<Responses.GetSettingResponse> {
+    return await this.tokenRefreshableRequest<Responses.GetSettingResponse>({
+      verb: HttpVerb.Get,
+      url: joinPaths(this.host, Paths.v1.setting(userUuid, settingName.toLowerCase() as SettingName)),
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_GET_SETTINGS,
+    })
+  }
+
+  async getSubscriptionSetting(
+    userUuid: UuidString,
+    settingName: SubscriptionSettingName,
+  ): Promise<Responses.GetSettingResponse> {
+    return await this.tokenRefreshableRequest<Responses.GetSettingResponse>({
+      verb: HttpVerb.Get,
+      url: joinPaths(
+        this.host,
+        Paths.v1.subscriptionSetting(userUuid, settingName.toLowerCase() as SubscriptionSettingName),
+      ),
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_GET_SETTINGS,
+    })
+  }
+
+  async deleteSetting(userUuid: UuidString, settingName: SettingName): Promise<Responses.DeleteSettingResponse> {
+    return this.tokenRefreshableRequest<Responses.DeleteSettingResponse>({
+      verb: HttpVerb.Delete,
+      url: joinPaths(this.host, Paths.v1.setting(userUuid, settingName)),
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_UPDATE_SETTINGS,
+    })
+  }
+
+  async deleteRevision(
+    itemUuid: UuidString,
+    entry: Responses.RevisionListEntry,
+  ): Promise<Responses.MinimalHttpResponse> {
+    const url = joinPaths(this.host, Paths.v1.itemRevision(itemUuid, entry.uuid))
+    const response = await this.tokenRefreshableRequest({
+      verb: HttpVerb.Delete,
+      url,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_DELETE_REVISION,
+      authentication: this.session?.authorizationValue,
+    })
+    return response
+  }
+
+  public downloadFeatureUrl(url: string): Promise<Responses.HttpResponse> {
+    return this.request({
+      verb: HttpVerb.Get,
+      url,
+      external: true,
+      fallbackErrorMessage: messages.API_MESSAGE_GENERIC_INVALID_LOGIN,
+    })
+  }
+
+  public async getSubscription(userUuid: string): Promise<Responses.HttpResponse | Responses.GetSubscriptionResponse> {
+    const url = joinPaths(this.host, Paths.v1.subscription(userUuid))
+    const response = await this.tokenRefreshableRequest({
+      verb: HttpVerb.Get,
+      url,
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_SUBSCRIPTION_INFO,
+    })
+    return response
+  }
+
+  public async getAvailableSubscriptions(): Promise<
+    Responses.HttpResponse | Responses.GetAvailableSubscriptionsResponse
+  > {
+    const url = joinPaths(this.host, Paths.v2.subscriptions)
+    const response = await this.request({
+      verb: HttpVerb.Get,
+      url,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_SUBSCRIPTION_INFO,
+    })
+    return response
+  }
+
+  public async getNewSubscriptionToken(): Promise<string | undefined> {
+    const url = joinPaths(this.host, Paths.v1.subscriptionTokens)
+    const response: Responses.HttpResponse | Responses.PostSubscriptionTokensResponse = await this.request({
+      verb: HttpVerb.Post,
+      url,
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_ACCESS_PURCHASE,
+    })
+    return (response as Responses.PostSubscriptionTokensResponse).data?.token
+  }
+
+  public async downloadOfflineFeaturesFromRepo(
+    repo: SNFeatureRepo,
+  ): Promise<{ features: FeatureDescription[] } | ClientDisplayableError> {
+    try {
+      const featuresUrl = repo.offlineFeaturesUrl
+      const extensionKey = repo.offlineKey
+      if (!featuresUrl || !extensionKey) {
+        throw Error('Cannot download offline repo without url and offlineKEy')
+      }
+
+      const { host } = new URL(featuresUrl)
+
+      if (!TRUSTED_FEATURE_HOSTS.includes(host)) {
+        return new ClientDisplayableError('This offline features host is not in the trusted allowlist.')
+      }
+
+      const response: Responses.HttpResponse | Responses.GetOfflineFeaturesResponse = await this.request({
+        verb: HttpVerb.Get,
+        url: featuresUrl,
+        fallbackErrorMessage: messages.API_MESSAGE_FAILED_OFFLINE_FEATURES,
+        customHeaders: [{ key: 'x-offline-token', value: extensionKey }],
+      })
+
+      if (response.error) {
+        return ClientDisplayableError.FromError(response.error)
+      }
+      return {
+        features: (response as Responses.GetOfflineFeaturesResponse).data?.features || [],
+      }
+    } catch {
+      return new ClientDisplayableError(API_MESSAGE_FAILED_OFFLINE_ACTIVATION)
+    }
+  }
+
+  public async registerForListedAccount(): Promise<Responses.ListedRegistrationResponse> {
+    if (!this.user) {
+      throw Error('Cannot register for Listed without user account.')
+    }
+    return await this.tokenRefreshableRequest<Responses.ListedRegistrationResponse>({
+      verb: HttpVerb.Post,
+      url: joinPaths(this.host, Paths.v1.listedRegistration(this.user.uuid)),
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_LISTED_REGISTRATION,
+      authentication: this.session?.authorizationValue,
+    })
+  }
+
+  public async createFileValetToken(
+    remoteIdentifier: string,
+    operation: 'write' | 'read' | 'delete',
+    unencryptedFileSize?: number,
+  ): Promise<string | ClientDisplayableError> {
+    const url = joinPaths(this.host, Paths.v1.createFileValetToken)
+
+    const params: CreateValetTokenPayload = {
+      operation,
+      resources: [{ remoteIdentifier, unencryptedFileSize: unencryptedFileSize || 0 }],
+    }
+
+    const response = await this.tokenRefreshableRequest<Responses.CreateValetTokenResponse>({
+      verb: HttpVerb.Post,
+      url: url,
+      authentication: this.session?.authorizationValue,
+      fallbackErrorMessage: messages.API_MESSAGE_FAILED_CREATE_FILE_TOKEN,
+      params,
+    })
+
+    if (!response.data?.success) {
+      return new ClientDisplayableError(response.data?.reason as string, undefined, response.data?.reason as string)
+    }
+
+    return response.data?.valetToken
+  }
+
+  public async startUploadSession(apiToken: string): Promise<Responses.StartUploadSessionResponse> {
+    const url = joinPaths(this.getFilesHost(), Paths.v1.startUploadSession)
+
+    const response: Responses.HttpResponse | Responses.StartUploadSessionResponse = await this.tokenRefreshableRequest({
+      verb: HttpVerb.Post,
+      url,
+      customHeaders: [{ key: 'x-valet-token', value: apiToken }],
+      fallbackErrorMessage: Strings.Network.Files.FailedStartUploadSession,
+    })
+
+    return response as Responses.StartUploadSessionResponse
+  }
+
+  public async deleteFile(apiToken: string): Promise<Responses.MinimalHttpResponse> {
+    const url = joinPaths(this.getFilesHost(), Paths.v1.deleteFile)
+
+    const response: Responses.HttpResponse | Responses.StartUploadSessionResponse = await this.tokenRefreshableRequest({
+      verb: HttpVerb.Delete,
+      url,
+      customHeaders: [{ key: 'x-valet-token', value: apiToken }],
+      fallbackErrorMessage: Strings.Network.Files.FailedDeleteFile,
+    })
+
+    return response as Responses.MinimalHttpResponse
+  }
+
+  public async uploadFileBytes(apiToken: string, chunkId: number, encryptedBytes: Uint8Array): Promise<boolean> {
+    if (chunkId === 0) {
+      throw Error('chunkId must start with 1')
+    }
+    const url = joinPaths(this.getFilesHost(), Paths.v1.uploadFileChunk)
+
+    const response: Responses.HttpResponse | Responses.UploadFileChunkResponse = await this.tokenRefreshableRequest({
+      verb: HttpVerb.Post,
+      url,
+      rawBytes: encryptedBytes,
+      customHeaders: [
+        { key: 'x-valet-token', value: apiToken },
+        { key: 'x-chunk-id', value: chunkId.toString() },
+        { key: 'Content-Type', value: 'application/octet-stream' },
+      ],
+      fallbackErrorMessage: Strings.Network.Files.FailedUploadFileChunk,
+    })
+
+    return (response as Responses.UploadFileChunkResponse).success
+  }
+
+  public async closeUploadSession(apiToken: string): Promise<boolean> {
+    const url = joinPaths(this.getFilesHost(), Paths.v1.closeUploadSession)
+
+    const response: Responses.HttpResponse | Responses.CloseUploadSessionResponse = await this.tokenRefreshableRequest({
+      verb: HttpVerb.Post,
+      url,
+      customHeaders: [{ key: 'x-valet-token', value: apiToken }],
+      fallbackErrorMessage: Strings.Network.Files.FailedCloseUploadSession,
+    })
+
+    return (response as Responses.CloseUploadSessionResponse).success
+  }
+
+  public getFilesDownloadUrl(): string {
+    return joinPaths(this.getFilesHost(), Paths.v1.downloadFileChunk)
+  }
+
+  public async downloadFile(
+    file: { encryptedChunkSizes: FileContent['encryptedChunkSizes'] },
+    chunkIndex = 0,
+    apiToken: string,
+    contentRangeStart: number,
+    onBytesReceived: (bytes: Uint8Array) => Promise<void>,
+  ): Promise<ClientDisplayableError | undefined> {
+    const url = this.getFilesDownloadUrl()
+    const pullChunkSize = file.encryptedChunkSizes[chunkIndex]
+
+    const response: Responses.HttpResponse | Responses.DownloadFileChunkResponse =
+      await this.tokenRefreshableRequest<Responses.DownloadFileChunkResponse>({
+        verb: HttpVerb.Get,
+        url,
+        customHeaders: [
+          { key: 'x-valet-token', value: apiToken },
+          {
+            key: 'x-chunk-size',
+            value: pullChunkSize.toString(),
+          },
+          { key: 'range', value: `bytes=${contentRangeStart}-` },
+        ],
+        fallbackErrorMessage: Strings.Network.Files.FailedDownloadFileChunk,
+        responseType: 'arraybuffer',
+      })
+
+    const contentRangeHeader = (<Map<string, string | null>>response.headers).get('content-range')
+    if (!contentRangeHeader) {
+      return new ClientDisplayableError('Could not obtain content-range header while downloading file chunk')
+    }
+
+    const matches = contentRangeHeader.match(/(^[a-zA-Z][\w]*)\s+(\d+)\s?-\s?(\d+)?\s?\/?\s?(\d+|\*)?/)
+    if (!matches || matches.length !== 5) {
+      return new ClientDisplayableError('Malformed content-range header in response when downloading file chunk')
+    }
+
+    const rangeStart = +matches[2]
+    const rangeEnd = +matches[3]
+    const totalSize = +matches[4]
+
+    const bytesReceived = new Uint8Array(response.data as ArrayBuffer)
+
+    await onBytesReceived(bytesReceived)
+
+    if (rangeEnd < totalSize - 1) {
+      return this.downloadFile(file, ++chunkIndex, apiToken, rangeStart + pullChunkSize, onBytesReceived)
+    }
+
+    return undefined
+  }
+
+  async checkIntegrity(integrityPayloads: Responses.IntegrityPayload[]): Promise<Responses.CheckIntegrityResponse> {
+    return await this.tokenRefreshableRequest<Responses.CheckIntegrityResponse>({
+      verb: HttpVerb.Post,
+      url: joinPaths(this.host, Paths.v1.checkIntegrity),
+      params: {
+        integrityPayloads,
+      },
+      fallbackErrorMessage: messages.API_MESSAGE_GENERIC_INTEGRITY_CHECK_FAIL,
+      authentication: this.session?.authorizationValue,
+    })
+  }
+
+  async getSingleItem(itemUuid: Uuid): Promise<Responses.GetSingleItemResponse> {
+    return await this.tokenRefreshableRequest<Responses.GetSingleItemResponse>({
+      verb: HttpVerb.Get,
+      url: joinPaths(this.host, Paths.v1.getSingleItem(itemUuid)),
+      fallbackErrorMessage: messages.API_MESSAGE_GENERIC_SINGLE_ITEM_SYNC_FAIL,
+      authentication: this.session?.authorizationValue,
+    })
+  }
+
+  private preprocessingError() {
+    if (this.refreshingSession) {
+      return this.createErrorResponse(messages.API_MESSAGE_TOKEN_REFRESH_IN_PROGRESS)
+    }
+    if (!this.session) {
+      return this.createErrorResponse(messages.API_MESSAGE_INVALID_SESSION)
+    }
+    return undefined
+  }
+
+  /** Handle errored responses to authenticated requests */
+  private preprocessAuthenticatedErrorResponse(response: Responses.HttpResponse) {
+    if (response.status === Responses.StatusCode.HttpStatusInvalidSession && this.session) {
+      this.invalidSessionObserver?.(response.error?.tag === ErrorTag.RevokedSession)
+    }
+  }
+
+  override getDiagnostics(): Promise<DiagnosticInfo | undefined> {
+    return Promise.resolve({
+      api: {
+        hasSession: this.session != undefined,
+        user: this.user,
+        registering: this.registering,
+        authenticating: this.authenticating,
+        changing: this.changing,
+        refreshingSession: this.refreshingSession,
+        filesHost: this.filesHost,
+        host: this.host,
+      },
+    })
+  }
+}

packages/snjs/lib/Services/Api/HttpService.ts

@@ -0,0 +1,211 @@
+import { API_MESSAGE_RATE_LIMITED, UNKNOWN_ERROR } from './Messages'
+import { HttpResponse, StatusCode } from '@standardnotes/responses'
+import { isString } from '@standardnotes/utils'
+import { SnjsVersion } from '@Lib/Version'
+import { AbstractService, InternalEventBusInterface, Environment } from '@standardnotes/services'
+
+export enum HttpVerb {
+  Get = 'GET',
+  Post = 'POST',
+  Put = 'PUT',
+  Patch = 'PATCH',
+  Delete = 'DELETE',
+}
+
+const REQUEST_READY_STATE_COMPLETED = 4
+
+export type HttpParams = Record<string, unknown>
+
+export type HttpRequest = {
+  url: string
+  params?: HttpParams
+  rawBytes?: Uint8Array
+  verb: HttpVerb
+  authentication?: string
+  customHeaders?: Record<string, string>[]
+  responseType?: XMLHttpRequestResponseType
+  external?: boolean
+}
+
+/**
+ * A non-SNJS specific wrapper for XMLHttpRequests
+ */
+export class SNHttpService extends AbstractService {
+  constructor(
+    private readonly environment: Environment,
+    private readonly appVersion: string,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  public async getAbsolute(url: string, params?: HttpParams, authentication?: string): Promise<HttpResponse> {
+    return this.runHttp({ url, params, verb: HttpVerb.Get, authentication })
+  }
+
+  public async postAbsolute(url: string, params?: HttpParams, authentication?: string): Promise<HttpResponse> {
+    return this.runHttp({ url, params, verb: HttpVerb.Post, authentication })
+  }
+
+  public async putAbsolute(url: string, params?: HttpParams, authentication?: string): Promise<HttpResponse> {
+    return this.runHttp({ url, params, verb: HttpVerb.Put, authentication })
+  }
+
+  public async patchAbsolute(url: string, params: HttpParams, authentication?: string): Promise<HttpResponse> {
+    return this.runHttp({ url, params, verb: HttpVerb.Patch, authentication })
+  }
+
+  public async deleteAbsolute(url: string, params?: HttpParams, authentication?: string): Promise<HttpResponse> {
+    return this.runHttp({ url, params, verb: HttpVerb.Delete, authentication })
+  }
+
+  public async runHttp(httpRequest: HttpRequest): Promise<HttpResponse> {
+    const request = this.createXmlRequest(httpRequest)
+
+    return this.runRequest(request, this.createRequestBody(httpRequest))
+  }
+
+  private createRequestBody(httpRequest: HttpRequest): string | Uint8Array | undefined {
+    if (
+      httpRequest.params !== undefined &&
+      [HttpVerb.Post, HttpVerb.Put, HttpVerb.Patch, HttpVerb.Delete].includes(httpRequest.verb)
+    ) {
+      return JSON.stringify(httpRequest.params)
+    }
+
+    return httpRequest.rawBytes
+  }
+
+  private createXmlRequest(httpRequest: HttpRequest) {
+    const request = new XMLHttpRequest()
+    if (httpRequest.params && httpRequest.verb === HttpVerb.Get && Object.keys(httpRequest.params).length > 0) {
+      httpRequest.url = this.urlForUrlAndParams(httpRequest.url, httpRequest.params)
+    }
+    request.open(httpRequest.verb, httpRequest.url, true)
+    request.responseType = httpRequest.responseType ?? ''
+
+    if (!httpRequest.external) {
+      request.setRequestHeader('X-SNJS-Version', SnjsVersion)
+
+      const appVersionHeaderValue = `${Environment[this.environment]}-${this.appVersion}`
+      request.setRequestHeader('X-Application-Version', appVersionHeaderValue)
+
+      if (httpRequest.authentication) {
+        request.setRequestHeader('Authorization', 'Bearer ' + httpRequest.authentication)
+      }
+    }
+
+    let contenTypeIsSet = false
+    if (httpRequest.customHeaders && httpRequest.customHeaders.length > 0) {
+      httpRequest.customHeaders.forEach(({ key, value }) => {
+        request.setRequestHeader(key, value)
+        if (key === 'Content-Type') {
+          contenTypeIsSet = true
+        }
+      })
+    }
+    if (!contenTypeIsSet && !httpRequest.external) {
+      request.setRequestHeader('Content-Type', 'application/json')
+    }
+
+    return request
+  }
+
+  private async runRequest(request: XMLHttpRequest, body?: string | Uint8Array): Promise<HttpResponse> {
+    return new Promise((resolve, reject) => {
+      request.onreadystatechange = () => {
+        this.stateChangeHandlerForRequest(request, resolve, reject)
+      }
+      request.send(body)
+    })
+  }
+
+  private stateChangeHandlerForRequest(
+    request: XMLHttpRequest,
+    resolve: (response: HttpResponse) => void,
+    reject: (response: HttpResponse) => void,
+  ) {
+    if (request.readyState !== REQUEST_READY_STATE_COMPLETED) {
+      return
+    }
+    const httpStatus = request.status
+    const response: HttpResponse = {
+      status: httpStatus,
+      headers: new Map<string, string | null>(),
+    }
+
+    const responseHeaderLines = request
+      .getAllResponseHeaders()
+      ?.trim()
+      .split(/[\r\n]+/)
+    responseHeaderLines?.forEach((responseHeaderLine) => {
+      const parts = responseHeaderLine.split(': ')
+      const name = parts.shift() as string
+      const value = parts.join(': ')
+
+      ;(<Map<string, string | null>>response.headers).set(name, value)
+    })
+
+    try {
+      if (httpStatus !== StatusCode.HttpStatusNoContent) {
+        let body
+
+        const contentTypeHeader = response.headers?.get('content-type') || response.headers?.get('Content-Type')
+
+        if (contentTypeHeader?.includes('application/json')) {
+          body = JSON.parse(request.responseText)
+        } else {
+          body = request.response
+        }
+        /**
+         * v0 APIs do not have a `data` top-level object. In such cases, mimic
+         * the newer response body style by putting all the top-level
+         * properties inside a `data` object.
+         */
+        if (!body.data) {
+          response.data = body
+        }
+        if (!isString(body)) {
+          Object.assign(response, body)
+        }
+      }
+    } catch (error) {
+      console.error(error)
+    }
+    if (httpStatus >= StatusCode.HttpStatusMinSuccess && httpStatus <= StatusCode.HttpStatusMaxSuccess) {
+      resolve(response)
+    } else {
+      if (httpStatus === StatusCode.HttpStatusForbidden) {
+        response.error = {
+          message: API_MESSAGE_RATE_LIMITED,
+          status: httpStatus,
+        }
+      } else if (response.error == undefined) {
+        if (response.data == undefined || response.data.error == undefined) {
+          try {
+            response.error = { message: request.responseText || UNKNOWN_ERROR, status: httpStatus }
+          } catch (error) {
+            response.error = { message: UNKNOWN_ERROR, status: httpStatus }
+          }
+        } else {
+          response.error = response.data.error
+        }
+      }
+      reject(response)
+    }
+  }
+
+  private urlForUrlAndParams(url: string, params: HttpParams) {
+    const keyValueString = Object.keys(params)
+      .map((key) => {
+        return key + '=' + encodeURIComponent(params[key] as string)
+      })
+      .join('&')
+
+    if (url.includes('?')) {
+      return url + '&' + keyValueString
+    } else {
+      return url + '?' + keyValueString
+    }
+  }
+}

packages/snjs/lib/Services/Api/Messages.ts

@@ -0,0 +1,185 @@
+import { ProtocolVersion } from '@standardnotes/common'
+
+export const API_MESSAGE_GENERIC_INVALID_LOGIN = 'A server error occurred while trying to sign in. Please try again.'
+export const API_MESSAGE_GENERIC_REGISTRATION_FAIL =
+  'A server error occurred while trying to register. Please try again.'
+export const API_MESSAGE_GENERIC_CHANGE_CREDENTIALS_FAIL =
+  'Something went wrong while changing your credentials. Your credentials were not changed. Please try again.'
+export const API_MESSAGE_GENERIC_SYNC_FAIL = 'Could not connect to server.'
+
+export const ServerErrorStrings = {
+  DeleteAccountError: 'Your account was unable to be deleted due to an error. Please try your request again.',
+}
+
+export const API_MESSAGE_GENERIC_INTEGRITY_CHECK_FAIL = 'Could not check your data integrity with the server.'
+
+export const API_MESSAGE_GENERIC_SINGLE_ITEM_SYNC_FAIL = 'Could not retrieve item.'
+
+export const API_MESSAGE_REGISTRATION_IN_PROGRESS = 'An existing registration request is already in progress.'
+export const API_MESSAGE_LOGIN_IN_PROGRESS = 'An existing sign in request is already in progress.'
+export const API_MESSAGE_CHANGE_CREDENTIALS_IN_PROGRESS =
+  'An existing change credentials request is already in progress.'
+
+export const API_MESSAGE_FALLBACK_LOGIN_FAIL = 'Invalid email or password.'
+
+export const API_MESSAGE_GENERIC_TOKEN_REFRESH_FAIL =
+  'A server error occurred while trying to refresh your session. Please try again.'
+
+export const API_MESSAGE_TOKEN_REFRESH_IN_PROGRESS =
+  'Your account session is being renewed with the server. Please try your request again.'
+
+export const API_MESSAGE_RATE_LIMITED = 'Too many successive server requests. Please wait a few minutes and try again.'
+
+export const API_MESSAGE_INVALID_SESSION = 'Please sign in to an account in order to continue with your request.'
+
+export const API_MESSAGE_FAILED_GET_SETTINGS = 'Failed to get settings.'
+export const API_MESSAGE_FAILED_UPDATE_SETTINGS = 'Failed to update settings.'
+export const API_MESSAGE_FAILED_LISTED_REGISTRATION = 'Unable to register for Listed. Please try again later.'
+
+export const API_MESSAGE_FAILED_CREATE_FILE_TOKEN = 'Failed to create file token.'
+
+export const API_MESSAGE_FAILED_SUBSCRIPTION_INFO = "Failed to get subscription's information."
+
+export const API_MESSAGE_FAILED_ACCESS_PURCHASE = 'Failed to access purchase flow.'
+
+export const API_MESSAGE_FAILED_DELETE_REVISION = 'Failed to delete revision.'
+
+export const API_MESSAGE_FAILED_OFFLINE_FEATURES = 'Failed to get offline features.'
+export const API_MESSAGE_UNTRUSTED_EXTENSIONS_WARNING = `The extension you are attempting to install comes from an
+  untrusted source. Untrusted extensions may lower the security of your data. Do you want to continue?`
+export const API_MESSAGE_FAILED_DOWNLOADING_EXTENSION = `Error downloading package details. Please check the
+  extension link and try again.`
+export const API_MESSAGE_FAILED_OFFLINE_ACTIVATION =
+  'An unknown issue occurred during offline activation. Please try again.'
+
+export const INVALID_EXTENSION_URL = 'Invalid extension URL.'
+
+export const UNSUPPORTED_PROTOCOL_VERSION =
+  'This version of the application does not support your newer account type. Please upgrade to the latest version of Standard Notes to sign in.'
+
+export const EXPIRED_PROTOCOL_VERSION =
+  'The protocol version associated with your account is outdated and no longer supported by this application. Please visit standardnotes.com/help/security for more information.'
+
+export const UNSUPPORTED_KEY_DERIVATION =
+  'Your account was created on a platform with higher security capabilities than this browser supports. If we attempted to generate your login keys here, it would take hours. Please use a browser with more up to date security capabilities, like Google Chrome or Firefox, to log in.'
+
+export const INVALID_PASSWORD_COST =
+  'Unable to sign in due to insecure password parameters. Please visit standardnotes.com/help/security for more information.'
+export const INVALID_PASSWORD = 'Invalid password.'
+
+export const OUTDATED_PROTOCOL_ALERT_IGNORE = 'Sign In'
+export const UPGRADING_ENCRYPTION = "Upgrading your account's encryption version…"
+
+export const SETTING_PASSCODE = 'Setting passcode…'
+export const CHANGING_PASSCODE = 'Changing passcode…'
+export const REMOVING_PASSCODE = 'Removing passcode…'
+
+export const DO_NOT_CLOSE_APPLICATION = 'Do not close the application until this process completes.'
+
+export const UNKNOWN_ERROR = 'Unknown error.'
+
+export function InsufficientPasswordMessage(minimum: number): string {
+  return `Your password must be at least ${minimum} characters in length. For your security, please choose a longer password or, ideally, a passphrase, and try again.`
+}
+
+export function StrictSignInFailed(current: ProtocolVersion, latest: ProtocolVersion): string {
+  return `Strict Sign In has refused the server's sign-in parameters. The latest account version is ${latest}, but the server is reporting a version of ${current} for your account. If you'd like to proceed with sign in anyway, please disable Strict Sign In and try again.`
+}
+
+export const CredentialsChangeStrings = {
+  PasscodeRequired: 'Your passcode is required to process your credentials change.',
+  Failed: 'Unable to change your credentials due to a sync error. Please try again.',
+}
+
+export const RegisterStrings = {
+  PasscodeRequired: 'Your passcode is required in order to register for an account.',
+}
+
+export const SignInStrings = {
+  PasscodeRequired: 'Your passcode is required in order to sign in to your account.',
+  IncorrectMfa: 'Incorrect two-factor authentication code. Please try again.',
+  SignInCanceledMissingMfa: 'Your sign in request has been canceled.',
+}
+
+export const ProtocolUpgradeStrings = {
+  SuccessAccount:
+    "Your encryption version has been successfully upgraded. You may be asked to enter your credentials again on other devices you're signed into.",
+  SuccessPasscodeOnly: 'Your encryption version has been successfully upgraded.',
+  Fail: 'Unable to upgrade encryption version. Please try again.',
+  UpgradingPasscode: 'Upgrading local encryption...',
+}
+
+export const ChallengeModalTitle = {
+  Generic: 'Authentication Required',
+  Migration: 'Storage Update',
+}
+
+export const SessionStrings = {
+  EnterEmailAndPassword: 'Please enter your account email and password.',
+  RecoverSession(email?: string): string {
+    return email
+      ? `Your credentials are needed for ${email} to refresh your session with the server.`
+      : 'Your credentials are needed to refresh your session with the server.'
+  },
+  SessionRestored: 'Your session has been successfully restored.',
+  EnterMfa: 'Please enter your two-factor authentication code.',
+  MfaInputPlaceholder: 'Two-factor authentication code',
+  EmailInputPlaceholder: 'Email',
+  PasswordInputPlaceholder: 'Password',
+  KeychainRecoveryErrorTitle: 'Invalid Credentials',
+  KeychainRecoveryError:
+    'The email or password you entered is incorrect.\n\nPlease note that this sign-in request is made against the default server. If you are using a custom server, you must uninstall the app then reinstall, and sign back into your account.',
+  RevokeTitle: 'Revoke this session?',
+  RevokeConfirmButton: 'Revoke',
+  RevokeCancelButton: 'Cancel',
+  RevokeText:
+    'The associated app will be signed out and all data removed ' +
+    'from the device when it is next launched. You can sign back in on that ' +
+    'device at any time.',
+  CurrentSessionRevoked: 'Your session has been revoked and all local data has been removed ' + 'from this device.',
+}
+
+export const ChallengeStrings = {
+  UnlockApplication: 'Authentication is required to unlock the application',
+  NoteAccess: 'Authentication is required to view this note',
+  FileAccess: 'Authentication is required to access this file',
+  ImportFile: 'Authentication is required to import a backup file',
+  AddPasscode: 'Authentication is required to add a passcode',
+  RemovePasscode: 'Authentication is required to remove your passcode',
+  ChangePasscode: 'Authentication is required to change your passcode',
+  ChangeAutolockInterval: 'Authentication is required to change autolock timer duration',
+  RevokeSession: 'Authentication is required to revoke a session',
+  EnterAccountPassword: 'Enter your account password',
+  EnterLocalPasscode: 'Enter your application passcode',
+  EnterPasscodeForMigration:
+    'Your application passcode is required to perform an upgrade of your local data storage structure.',
+  EnterPasscodeForRootResave: 'Enter your application passcode to continue',
+  EnterCredentialsForProtocolUpgrade: 'Enter your credentials to perform encryption upgrade',
+  EnterCredentialsForDecryptedBackupDownload: 'Enter your credentials to download a decrypted backup',
+  AccountPasswordPlaceholder: 'Account Password',
+  LocalPasscodePlaceholder: 'Application Passcode',
+  DecryptEncryptedFile: 'Enter the account password associated with the import file',
+  ExportBackup: 'Authentication is required to export a backup',
+  DisableBiometrics: 'Authentication is required to disable biometrics',
+  UnprotectNote: 'Authentication is required to unprotect a note',
+  UnprotectFile: 'Authentication is required to unprotect a file',
+  SearchProtectedNotesText: 'Authentication is required to search protected contents',
+  SelectProtectedNote: 'Authentication is required to select a protected note',
+  DisableMfa: 'Authentication is required to disable two-factor authentication',
+  DeleteAccount: 'Authentication is required to delete your account',
+}
+
+export const ErrorAlertStrings = {
+  MissingSessionTitle: 'Missing Session',
+  MissingSessionBody:
+    'We were unable to load your server session. This represents an inconsistency with your application state. Please take an opportunity to backup your data, then sign out and sign back in to resolve this issue.',
+
+  StorageDecryptErrorTitle: 'Storage Error',
+  StorageDecryptErrorBody:
+    "We were unable to decrypt your local storage. Please restart the app and try again. If you're unable to resolve this issue, and you have an account, you may try uninstalling the app then reinstalling, then signing back into your account. Otherwise, please contact help@standardnotes.org for support.",
+}
+
+export const KeychainRecoveryStrings = {
+  Title: 'Restore Keychain',
+  Text: "We've detected that your keychain has been wiped. This can happen when restoring your device from a backup. Please enter your account password to restore your account keys.",
+}

packages/snjs/lib/Services/Api/Paths.ts

@@ -0,0 +1,74 @@
+import { Uuid } from '@standardnotes/common'
+import { SettingName, SubscriptionSettingName } from '@standardnotes/settings'
+
+const FilesPaths = {
+  closeUploadSession: '/v1/files/upload/close-session',
+  createFileValetToken: '/v1/files/valet-tokens',
+  deleteFile: '/v1/files',
+  downloadFileChunk: '/v1/files',
+  startUploadSession: '/v1/files/upload/create-session',
+  uploadFileChunk: '/v1/files/upload/chunk',
+}
+
+const UserPaths = {
+  changeCredentials: (userUuid: string) => `/v1/users/${userUuid}/attributes/credentials`,
+  deleteAccount: (userUuid: Uuid) => `/v1/users/${userUuid}`,
+  keyParams: '/v1/login-params',
+  refreshSession: '/v1/sessions/refresh',
+  register: '/v1/users',
+  session: (sessionUuid: string) => `/v1/sessions/${sessionUuid}`,
+  sessions: '/v1/sessions',
+  signIn: '/v1/login',
+  signOut: '/v1/logout',
+}
+
+const ItemsPaths = {
+  checkIntegrity: '/v1/items/check-integrity',
+  getSingleItem: (uuid: Uuid) => `/v1/items/${uuid}`,
+  itemRevisions: (itemUuid: string) => `/v1/items/${itemUuid}/revisions`,
+  itemRevision: (itemUuid: string, revisionUuid: string) => `/v1/items/${itemUuid}/revisions/${revisionUuid}`,
+  sync: '/v1/items',
+}
+
+const SettingsPaths = {
+  settings: (userUuid: Uuid) => `/v1/users/${userUuid}/settings`,
+  setting: (userUuid: Uuid, settingName: SettingName) => `/v1/users/${userUuid}/settings/${settingName}`,
+  subscriptionSetting: (userUuid: Uuid, settingName: SubscriptionSettingName) =>
+    `/v1/users/${userUuid}/subscription-settings/${settingName}`,
+}
+
+const SubscriptionPaths = {
+  offlineFeatures: '/v1/offline/features',
+  purchase: '/v1/purchase',
+  subscription: (userUuid: Uuid) => `/v1/users/${userUuid}/subscription`,
+  subscriptionTokens: '/v1/subscription-tokens',
+  userFeatures: (userUuid: string) => `/v1/users/${userUuid}/features`,
+}
+
+const SubscriptionPathsV2 = {
+  subscriptions: '/v2/subscriptions',
+}
+
+const UserPathsV2 = {
+  keyParams: '/v2/login-params',
+  signIn: '/v2/login',
+}
+
+const ListedPaths = {
+  listedRegistration: (userUuid: Uuid) => `/v1/users/${userUuid}/integrations/listed`,
+}
+
+export const Paths = {
+  v1: {
+    ...FilesPaths,
+    ...ItemsPaths,
+    ...ListedPaths,
+    ...SettingsPaths,
+    ...SubscriptionPaths,
+    ...UserPaths,
+  },
+  v2: {
+    ...SubscriptionPathsV2,
+    ...UserPathsV2,
+  },
+}

packages/snjs/lib/Services/Api/WebsocketsService.spec.ts

@@ -0,0 +1,30 @@
+import { InternalEventBusInterface } from '@standardnotes/services'
+import { StorageKey, DiskStorageService } from '@Lib/index'
+import { SNWebSocketsService } from './WebsocketsService'
+
+describe('webSocketsService', () => {
+  const webSocketUrl = ''
+
+  let storageService: DiskStorageService
+  let internalEventBus: InternalEventBusInterface
+
+  const createService = () => {
+    return new SNWebSocketsService(storageService, webSocketUrl, internalEventBus)
+  }
+
+  beforeEach(() => {
+    storageService = {} as jest.Mocked<DiskStorageService>
+    storageService.setValue = jest.fn()
+
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+  })
+
+  describe('setWebSocketUrl()', () => {
+    it('saves url in local storage', async () => {
+      const webSocketUrl = 'wss://test-websocket'
+      await createService().setWebSocketUrl(webSocketUrl)
+      expect(storageService.setValue).toHaveBeenCalledWith(StorageKey.WebSocketUrl, webSocketUrl)
+    })
+  })
+})

packages/snjs/lib/Services/Api/WebsocketsService.ts

@@ -0,0 +1,67 @@
+import { UserRolesChangedEvent } from '@standardnotes/domain-events'
+import { DiskStorageService } from '../Storage/DiskStorageService'
+import { AbstractService, InternalEventBusInterface, StorageKey } from '@standardnotes/services'
+
+export enum WebSocketsServiceEvent {
+  UserRoleMessageReceived = 'WebSocketMessageReceived',
+}
+
+export class SNWebSocketsService extends AbstractService<WebSocketsServiceEvent, UserRolesChangedEvent> {
+  private webSocket?: WebSocket
+
+  constructor(
+    private storageService: DiskStorageService,
+    private webSocketUrl: string | undefined,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  public setWebSocketUrl(url: string | undefined): void {
+    this.webSocketUrl = url
+    this.storageService.setValue(StorageKey.WebSocketUrl, url)
+  }
+
+  public loadWebSocketUrl(): void {
+    const storedValue = this.storageService.getValue<string | undefined>(StorageKey.WebSocketUrl)
+    this.webSocketUrl =
+      storedValue ||
+      this.webSocketUrl ||
+      (
+        window as {
+          _websocket_url?: string
+        }
+      )._websocket_url
+  }
+
+  public startWebSocketConnection(authToken: string): void {
+    if (this.webSocketUrl) {
+      try {
+        this.webSocket = new WebSocket(`${this.webSocketUrl}?authToken=Bearer+${authToken}`)
+        this.webSocket.onmessage = this.onWebSocketMessage.bind(this)
+        this.webSocket.onclose = this.onWebSocketClose.bind(this)
+      } catch (e) {
+        console.error('Error starting WebSocket connection', e)
+      }
+    }
+  }
+
+  public closeWebSocketConnection(): void {
+    this.webSocket?.close()
+  }
+
+  private onWebSocketMessage(event: MessageEvent) {
+    const eventData: UserRolesChangedEvent = JSON.parse(event.data)
+    void this.notifyEvent(WebSocketsServiceEvent.UserRoleMessageReceived, eventData)
+  }
+
+  private onWebSocketClose() {
+    this.webSocket = undefined
+  }
+
+  override deinit(): void {
+    super.deinit()
+    ;(this.storageService as unknown) = undefined
+    this.closeWebSocketConnection()
+  }
+}

packages/snjs/lib/Services/Api/index.ts

@@ -0,0 +1,7 @@
+export * from './ApiService'
+export * from './HttpService'
+export * from './Messages'
+export * from './Paths'
+export * from '../Session/Sessions/Session'
+export * from '../Session/SessionManager'
+export * from './WebsocketsService'

packages/snjs/lib/Services/AppService/ApplicationService.ts

@@ -0,0 +1,75 @@
+import { ApplicationEvent } from '@Lib/Application/Event'
+import { AbstractService, InternalEventBusInterface } from '@standardnotes/services'
+import { SNApplication } from '../../Application/Application'
+
+export class ApplicationService extends AbstractService {
+  private unsubApp!: () => void
+
+  constructor(protected application: SNApplication, protected override internalEventBus: InternalEventBusInterface) {
+    super(internalEventBus)
+    this.addAppEventObserverAfterSubclassesFinishConstructing()
+  }
+
+  override deinit() {
+    ;(this.application as unknown) = undefined
+
+    this.unsubApp()
+    ;(this.unsubApp as unknown) = undefined
+
+    super.deinit()
+  }
+
+  addAppEventObserverAfterSubclassesFinishConstructing() {
+    setTimeout(() => {
+      this.addAppEventObserver()
+    }, 0)
+  }
+
+  addAppEventObserver() {
+    if (this.application.isStarted()) {
+      void this.onAppStart()
+    }
+    if (this.application.isLaunched()) {
+      void this.onAppLaunch()
+    }
+
+    this.unsubApp = this.application.addEventObserver(async (event: ApplicationEvent) => {
+      await this.onAppEvent(event)
+      if (event === ApplicationEvent.Started) {
+        void this.onAppStart()
+      } else if (event === ApplicationEvent.Launched) {
+        void this.onAppLaunch()
+      } else if (event === ApplicationEvent.CompletedFullSync) {
+        this.onAppFullSync()
+      } else if (event === ApplicationEvent.CompletedIncrementalSync) {
+        this.onAppIncrementalSync()
+      } else if (event === ApplicationEvent.KeyStatusChanged) {
+        void this.onAppKeyChange()
+      }
+    })
+  }
+
+  async onAppEvent(_event: ApplicationEvent) {
+    /** Optional override */
+  }
+
+  async onAppStart() {
+    /** Optional override */
+  }
+
+  async onAppLaunch() {
+    /** Optional override */
+  }
+
+  async onAppKeyChange() {
+    /** Optional override */
+  }
+
+  onAppIncrementalSync() {
+    /** Optional override */
+  }
+
+  onAppFullSync() {
+    /** Optional override */
+  }
+}

packages/snjs/lib/Services/Challenge/Challenge.ts

@@ -0,0 +1,112 @@
+import { ChallengeModalTitle, ChallengeStrings } from '../Api/Messages'
+import { assertUnreachable } from '@standardnotes/utils'
+import { ChallengeValidation, ChallengeReason, ChallengeInterface, ChallengePrompt } from '@standardnotes/services'
+
+/**
+ * A challenge is a stateless description of what the client needs to provide
+ * in order to proceed.
+ */
+export class Challenge implements ChallengeInterface {
+  public readonly id = Math.random()
+
+  constructor(
+    public readonly prompts: ChallengePrompt[],
+    public readonly reason: ChallengeReason,
+    public readonly cancelable: boolean,
+    public readonly _heading?: string,
+    public readonly _subheading?: string,
+  ) {
+    Object.freeze(this)
+  }
+
+  /** Outside of the modal, this is the title of the modal itself */
+  get modalTitle(): string {
+    switch (this.reason) {
+      case ChallengeReason.Migration:
+        return ChallengeModalTitle.Migration
+      default:
+        return ChallengeModalTitle.Generic
+    }
+  }
+
+  /** Inside of the modal, this is the H1 */
+  get heading(): string | undefined {
+    if (this._heading) {
+      return this._heading
+    } else {
+      switch (this.reason) {
+        case ChallengeReason.ApplicationUnlock:
+          return ChallengeStrings.UnlockApplication
+        case ChallengeReason.Migration:
+          return ChallengeStrings.EnterLocalPasscode
+        case ChallengeReason.ResaveRootKey:
+          return ChallengeStrings.EnterPasscodeForRootResave
+        case ChallengeReason.ProtocolUpgrade:
+          return ChallengeStrings.EnterCredentialsForProtocolUpgrade
+        case ChallengeReason.AccessProtectedNote:
+          return ChallengeStrings.NoteAccess
+        case ChallengeReason.AccessProtectedFile:
+          return ChallengeStrings.FileAccess
+        case ChallengeReason.ImportFile:
+          return ChallengeStrings.ImportFile
+        case ChallengeReason.AddPasscode:
+          return ChallengeStrings.AddPasscode
+        case ChallengeReason.RemovePasscode:
+          return ChallengeStrings.RemovePasscode
+        case ChallengeReason.ChangePasscode:
+          return ChallengeStrings.ChangePasscode
+        case ChallengeReason.ChangeAutolockInterval:
+          return ChallengeStrings.ChangeAutolockInterval
+        case ChallengeReason.CreateDecryptedBackupWithProtectedItems:
+          return ChallengeStrings.EnterCredentialsForDecryptedBackupDownload
+        case ChallengeReason.RevokeSession:
+          return ChallengeStrings.RevokeSession
+        case ChallengeReason.DecryptEncryptedFile:
+          return ChallengeStrings.DecryptEncryptedFile
+        case ChallengeReason.ExportBackup:
+          return ChallengeStrings.ExportBackup
+        case ChallengeReason.DisableBiometrics:
+          return ChallengeStrings.DisableBiometrics
+        case ChallengeReason.UnprotectNote:
+          return ChallengeStrings.UnprotectNote
+        case ChallengeReason.UnprotectFile:
+          return ChallengeStrings.UnprotectFile
+        case ChallengeReason.SearchProtectedNotesText:
+          return ChallengeStrings.SearchProtectedNotesText
+        case ChallengeReason.SelectProtectedNote:
+          return ChallengeStrings.SelectProtectedNote
+        case ChallengeReason.DisableMfa:
+          return ChallengeStrings.DisableMfa
+        case ChallengeReason.DeleteAccount:
+          return ChallengeStrings.DeleteAccount
+        case ChallengeReason.Custom:
+          return ''
+        default:
+          return assertUnreachable(this.reason)
+      }
+    }
+  }
+
+  /** Inside of the modal, this is the H2 */
+  get subheading(): string | undefined {
+    if (this._subheading) {
+      return this._subheading
+    }
+
+    switch (this.reason) {
+      case ChallengeReason.Migration:
+        return ChallengeStrings.EnterPasscodeForMigration
+      default:
+        return undefined
+    }
+  }
+
+  hasPromptForValidationType(type: ChallengeValidation): boolean {
+    for (const prompt of this.prompts) {
+      if (prompt.validation === type) {
+        return true
+      }
+    }
+    return false
+  }
+}

packages/snjs/lib/Services/Challenge/ChallengeOperation.ts

@@ -0,0 +1,108 @@
+import { Challenge } from './Challenge'
+import { ChallengeResponse } from './ChallengeResponse'
+import { removeFromArray } from '@standardnotes/utils'
+import { ValueCallback } from './ChallengeService'
+import { ChallengeValue, ChallengeArtifacts } from '@standardnotes/services'
+
+/**
+ * A challenge operation stores user-submitted values and callbacks.
+ * When its values are updated, it will trigger the associated callbacks (valid/invalid/complete)
+ */
+export class ChallengeOperation {
+  private nonvalidatedValues: ChallengeValue[] = []
+  private validValues: ChallengeValue[] = []
+  private invalidValues: ChallengeValue[] = []
+  private artifacts: ChallengeArtifacts = {}
+
+  constructor(
+    public challenge: Challenge,
+    public onValidValue: ValueCallback,
+    public onInvalidValue: ValueCallback,
+    public onNonvalidatedSubmit: (response: ChallengeResponse) => void,
+    public onComplete: (response: ChallengeResponse) => void,
+    public onCancel: () => void,
+  ) {}
+
+  deinit() {
+    ;(this.challenge as unknown) = undefined
+    ;(this.onValidValue as unknown) = undefined
+    ;(this.onInvalidValue as unknown) = undefined
+    ;(this.onNonvalidatedSubmit as unknown) = undefined
+    ;(this.onComplete as unknown) = undefined
+    ;(this.onCancel as unknown) = undefined
+    ;(this.nonvalidatedValues as unknown) = undefined
+    ;(this.validValues as unknown) = undefined
+    ;(this.invalidValues as unknown) = undefined
+    ;(this.artifacts as unknown) = undefined
+  }
+
+  /**
+   * Mark this challenge as complete, triggering the resolve function,
+   * as well as notifying the client
+   */
+  public complete(response?: ChallengeResponse) {
+    if (!response) {
+      response = new ChallengeResponse(this.challenge, this.validValues, this.artifacts)
+    }
+    this.onComplete?.(response)
+  }
+
+  public nonvalidatedSubmit() {
+    const response = new ChallengeResponse(this.challenge, this.nonvalidatedValues.slice(), this.artifacts)
+    this.onNonvalidatedSubmit?.(response)
+    /** Reset values */
+    this.nonvalidatedValues = []
+  }
+
+  public cancel() {
+    this.onCancel?.()
+  }
+
+  /**
+   * @returns Returns true if the challenge has received all valid responses
+   */
+  public isFinished() {
+    return this.validValues.length === this.challenge.prompts.length
+  }
+
+  private nonvalidatedPrompts() {
+    return this.challenge.prompts.filter((p) => !p.validates)
+  }
+
+  public addNonvalidatedValue(value: ChallengeValue) {
+    const valuesArray = this.nonvalidatedValues
+    const matching = valuesArray.find((v) => v.prompt.id === value.prompt.id)
+    if (matching) {
+      removeFromArray(valuesArray, matching)
+    }
+    valuesArray.push(value)
+    if (this.nonvalidatedValues.length === this.nonvalidatedPrompts().length) {
+      this.nonvalidatedSubmit()
+    }
+  }
+
+  /**
+   * Sets the values validation status, as well as handles subsequent actions,
+   * such as completing the operation if all valid values are supplied, as well as
+   * notifying the client of this new value's validation status.
+   */
+  public setValueStatus(value: ChallengeValue, valid: boolean, artifacts?: ChallengeArtifacts) {
+    const valuesArray = valid ? this.validValues : this.invalidValues
+    const matching = valuesArray.find((v) => v.prompt.validation === value.prompt.validation)
+    if (matching) {
+      removeFromArray(valuesArray, matching)
+    }
+    valuesArray.push(value)
+    Object.assign(this.artifacts, artifacts)
+
+    if (this.isFinished()) {
+      this.complete()
+    } else {
+      if (valid) {
+        this.onValidValue?.(value)
+      } else {
+        this.onInvalidValue?.(value)
+      }
+    }
+  }
+}

packages/snjs/lib/Services/Challenge/ChallengeResponse.ts

@@ -0,0 +1,33 @@
+import { isNullOrUndefined } from '@standardnotes/utils'
+import { Challenge } from './Challenge'
+import {
+  ChallengeResponseInterface,
+  ChallengeValidation,
+  ChallengeValue,
+  ChallengeArtifacts,
+} from '@standardnotes/services'
+
+export class ChallengeResponse implements ChallengeResponseInterface {
+  constructor(
+    public readonly challenge: Challenge,
+    public readonly values: ChallengeValue[],
+    public readonly artifacts?: ChallengeArtifacts,
+  ) {
+    Object.freeze(this)
+  }
+
+  getValueForType(type: ChallengeValidation): ChallengeValue {
+    const value = this.values.find((value) => value.prompt.validation === type)
+    if (isNullOrUndefined(value)) {
+      throw Error('Could not find value for validation type ' + type)
+    }
+    return value
+  }
+
+  getDefaultValue(): ChallengeValue {
+    if (this.values.length > 1) {
+      throw Error('Attempting to retrieve default response value when more than one value exists')
+    }
+    return this.values[0]
+  }
+}

packages/snjs/lib/Services/Challenge/ChallengeService.ts

@@ -0,0 +1,300 @@
+import { RootKeyInterface } from '@standardnotes/models'
+import { EncryptionService } from '@standardnotes/encryption'
+import { DiskStorageService } from '../Storage/DiskStorageService'
+import { removeFromArray } from '@standardnotes/utils'
+import { isValidProtectionSessionLength } from '../Protection/ProtectionService'
+import {
+  AbstractService,
+  ChallengeServiceInterface,
+  InternalEventBusInterface,
+  ChallengeArtifacts,
+  ChallengeReason,
+  ChallengeValidation,
+  ChallengeValue,
+  ChallengeInterface,
+  ChallengePromptInterface,
+  ChallengePrompt,
+} from '@standardnotes/services'
+import { ChallengeResponse } from './ChallengeResponse'
+import { ChallengeOperation } from './ChallengeOperation'
+import { Challenge } from './Challenge'
+
+type ChallengeValidationResponse = {
+  valid: boolean
+  artifacts?: ChallengeArtifacts
+}
+
+export type ValueCallback = (value: ChallengeValue) => void
+
+export type ChallengeObserver = {
+  onValidValue?: ValueCallback
+  onInvalidValue?: ValueCallback
+  onNonvalidatedSubmit?: (response: ChallengeResponse) => void
+  onComplete?: (response: ChallengeResponse) => void
+  onCancel?: () => void
+}
+
+const clearChallengeObserver = (observer: ChallengeObserver) => {
+  observer.onCancel = undefined
+  observer.onComplete = undefined
+  observer.onValidValue = undefined
+  observer.onInvalidValue = undefined
+  observer.onNonvalidatedSubmit = undefined
+}
+
+/**
+ * The challenge service creates, updates and keeps track of running challenge operations.
+ */
+export class ChallengeService extends AbstractService implements ChallengeServiceInterface {
+  private challengeOperations: Record<string, ChallengeOperation> = {}
+  public sendChallenge!: (challenge: Challenge) => void
+  private challengeObservers: Record<string, ChallengeObserver[]> = {}
+
+  constructor(
+    private storageService: DiskStorageService,
+    private protocolService: EncryptionService,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  public override deinit() {
+    ;(this.storageService as unknown) = undefined
+    ;(this.protocolService as unknown) = undefined
+    ;(this.sendChallenge as unknown) = undefined
+    ;(this.challengeOperations as unknown) = undefined
+    ;(this.challengeObservers as unknown) = undefined
+    super.deinit()
+  }
+
+  public promptForChallengeResponse(challenge: Challenge): Promise<ChallengeResponse | undefined> {
+    return new Promise<ChallengeResponse | undefined>((resolve) => {
+      this.createOrGetChallengeOperation(challenge, resolve)
+      this.sendChallenge(challenge)
+    })
+  }
+
+  public createChallenge(
+    prompts: ChallengePromptInterface[],
+    reason: ChallengeReason,
+    cancelable: boolean,
+    heading?: string,
+    subheading?: string,
+  ): ChallengeInterface {
+    return new Challenge(prompts, reason, cancelable, heading, subheading)
+  }
+
+  public async validateChallengeValue(value: ChallengeValue): Promise<ChallengeValidationResponse> {
+    switch (value.prompt.validation) {
+      case ChallengeValidation.LocalPasscode:
+        return this.protocolService.validatePasscode(value.value as string)
+      case ChallengeValidation.AccountPassword:
+        return this.protocolService.validateAccountPassword(value.value as string)
+      case ChallengeValidation.Biometric:
+        return { valid: value.value === true }
+      case ChallengeValidation.ProtectionSessionDuration:
+        return { valid: isValidProtectionSessionLength(value.value) }
+      default:
+        throw Error(`Unhandled validation mode ${value.prompt.validation}`)
+    }
+  }
+
+  public async promptForCorrectPasscode(reason: ChallengeReason): Promise<string | undefined> {
+    const challenge = new Challenge([new ChallengePrompt(ChallengeValidation.LocalPasscode)], reason, true)
+    const response = await this.promptForChallengeResponse(challenge)
+    if (!response) {
+      return undefined
+    }
+    const value = response.getValueForType(ChallengeValidation.LocalPasscode)
+    return value.value as string
+  }
+
+  /**
+   * Returns the wrapping key for operations that require resaving the root key
+   * (changing the account password, signing in, registering, or upgrading protocol)
+   * Returns empty object if no passcode is configured.
+   * Otherwise returns {cancled: true} if the operation is canceled, or
+   * {wrappingKey} with the result.
+   * @param passcode - If the consumer already has access to the passcode,
+   * they can pass it here so that the user is not prompted again.
+   */
+  async getWrappingKeyIfApplicable(passcode?: string): Promise<
+    | {
+        canceled?: undefined
+        wrappingKey?: undefined
+      }
+    | {
+        canceled: boolean
+        wrappingKey?: undefined
+      }
+    | {
+        wrappingKey: RootKeyInterface
+        canceled?: undefined
+      }
+  > {
+    if (!this.protocolService.hasPasscode()) {
+      return {}
+    }
+
+    if (!passcode) {
+      passcode = await this.promptForCorrectPasscode(ChallengeReason.ResaveRootKey)
+      if (!passcode) {
+        return { canceled: true }
+      }
+    }
+
+    const wrappingKey = await this.protocolService.computeWrappingKey(passcode)
+    return { wrappingKey }
+  }
+
+  public isPasscodeLocked() {
+    return this.protocolService.isPasscodeLocked()
+  }
+
+  public addChallengeObserver(challenge: Challenge, observer: ChallengeObserver) {
+    const observers = this.challengeObservers[challenge.id] || []
+
+    observers.push(observer)
+
+    this.challengeObservers[challenge.id] = observers
+
+    return () => {
+      clearChallengeObserver(observer)
+
+      removeFromArray(observers, observer)
+    }
+  }
+
+  private createOrGetChallengeOperation(
+    challenge: Challenge,
+    resolve: (response: ChallengeResponse | undefined) => void,
+  ): ChallengeOperation {
+    let operation = this.getChallengeOperation(challenge)
+
+    if (!operation) {
+      operation = new ChallengeOperation(
+        challenge,
+        (value: ChallengeValue) => {
+          this.onChallengeValidValue(challenge, value)
+        },
+        (value: ChallengeValue) => {
+          this.onChallengeInvalidValue(challenge, value)
+        },
+        (response: ChallengeResponse) => {
+          this.onChallengeNonvalidatedSubmit(challenge, response)
+          resolve(response)
+        },
+        (response: ChallengeResponse) => {
+          this.onChallengeComplete(challenge, response)
+          resolve(response)
+        },
+        () => {
+          this.onChallengeCancel(challenge)
+          resolve(undefined)
+        },
+      )
+
+      this.challengeOperations[challenge.id] = operation
+    }
+    return operation
+  }
+
+  private performOnObservers(challenge: Challenge, perform: (observer: ChallengeObserver) => void) {
+    const observers = this.challengeObservers[challenge.id] || []
+
+    for (const observer of observers) {
+      perform(observer)
+    }
+  }
+
+  private onChallengeValidValue(challenge: Challenge, value: ChallengeValue) {
+    this.performOnObservers(challenge, (observer) => {
+      observer.onValidValue?.(value)
+    })
+  }
+
+  private onChallengeInvalidValue(challenge: Challenge, value: ChallengeValue) {
+    this.performOnObservers(challenge, (observer) => {
+      observer.onInvalidValue?.(value)
+    })
+  }
+
+  private onChallengeNonvalidatedSubmit(challenge: Challenge, response: ChallengeResponse) {
+    this.performOnObservers(challenge, (observer) => {
+      observer.onNonvalidatedSubmit?.(response)
+    })
+  }
+
+  private onChallengeComplete(challenge: Challenge, response: ChallengeResponse) {
+    this.performOnObservers(challenge, (observer) => {
+      observer.onComplete?.(response)
+    })
+  }
+
+  private onChallengeCancel(challenge: Challenge) {
+    this.performOnObservers(challenge, (observer) => {
+      observer.onCancel?.()
+    })
+  }
+
+  private getChallengeOperation(challenge: Challenge) {
+    return this.challengeOperations[challenge.id]
+  }
+
+  private deleteChallengeOperation(operation: ChallengeOperation) {
+    const challenge = operation.challenge
+    operation.deinit()
+
+    delete this.challengeOperations[challenge.id]
+  }
+
+  public cancelChallenge(challenge: Challenge) {
+    const operation = this.challengeOperations[challenge.id]
+    operation.cancel()
+
+    this.deleteChallengeOperation(operation)
+  }
+
+  public completeChallenge(challenge: Challenge): void {
+    const operation = this.challengeOperations[challenge.id]
+    operation.complete()
+
+    this.deleteChallengeOperation(operation)
+  }
+
+  public async submitValuesForChallenge(challenge: Challenge, values: ChallengeValue[]) {
+    if (values.length === 0) {
+      throw Error('Attempting to submit 0 values for challenge')
+    }
+
+    for (const value of values) {
+      if (!value.prompt.validates) {
+        const operation = this.getChallengeOperation(challenge)
+        operation.addNonvalidatedValue(value)
+      } else {
+        const { valid, artifacts } = await this.validateChallengeValue(value)
+        this.setValidationStatusForChallenge(challenge, value, valid, artifacts)
+      }
+    }
+  }
+
+  public setValidationStatusForChallenge(
+    challenge: Challenge,
+    value: ChallengeValue,
+    valid: boolean,
+    artifacts?: ChallengeArtifacts,
+  ) {
+    const operation = this.getChallengeOperation(challenge)
+    operation.setValueStatus(value, valid, artifacts)
+
+    if (operation.isFinished()) {
+      this.deleteChallengeOperation(operation)
+
+      const observers = this.challengeObservers[challenge.id]
+      observers.forEach(clearChallengeObserver)
+      observers.length = 0
+
+      delete this.challengeObservers[challenge.id]
+    }
+  }
+}

packages/snjs/lib/Services/Challenge/index.ts

@@ -0,0 +1,4 @@
+export * from './Challenge'
+export * from './ChallengeOperation'
+export * from './ChallengeResponse'
+export * from './ChallengeService'

packages/snjs/lib/Services/ComponentManager/ComponentManager.spec.ts

@@ -0,0 +1,368 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import { SNPreferencesService } from '../Preferences/PreferencesService'
+import {
+  ComponentAction,
+  ComponentPermission,
+  FeatureDescription,
+  FindNativeFeature,
+  FeatureIdentifier,
+} from '@standardnotes/features'
+import { DesktopManagerInterface } from '@Lib/Services/ComponentManager/Types'
+import { ContentType } from '@standardnotes/common'
+import { GenericItem, SNComponent } from '@standardnotes/models'
+import { InternalEventBusInterface, Environment, Platform, AlertService } from '@standardnotes/services'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import { SNFeaturesService } from '@Lib/Services/Features/FeaturesService'
+import { SNComponentManager } from './ComponentManager'
+import { SNSyncService } from '../Sync/SyncService'
+
+describe('featuresService', () => {
+  let itemManager: ItemManager
+  let featureService: SNFeaturesService
+  let alertService: AlertService
+  let syncService: SNSyncService
+  let prefsService: SNPreferencesService
+  let internalEventBus: InternalEventBusInterface
+
+  const desktopExtHost = 'http://localhost:123'
+
+  const createManager = (environment: Environment, platform: Platform) => {
+    const desktopManager: DesktopManagerInterface = {
+      // eslint-disable-next-line @typescript-eslint/no-empty-function
+      syncComponentsInstallation() {},
+      // eslint-disable-next-line @typescript-eslint/no-empty-function
+      registerUpdateObserver() {},
+      getExtServerHost() {
+        return desktopExtHost
+      },
+    }
+
+    const manager = new SNComponentManager(
+      itemManager,
+      syncService,
+      featureService,
+      prefsService,
+      alertService,
+      environment,
+      platform,
+      internalEventBus,
+    )
+    manager.setDesktopManager(desktopManager)
+    return manager
+  }
+
+  beforeEach(() => {
+    syncService = {} as jest.Mocked<SNSyncService>
+    syncService.sync = jest.fn()
+
+    itemManager = {} as jest.Mocked<ItemManager>
+    itemManager.getItems = jest.fn().mockReturnValue([])
+    itemManager.createItem = jest.fn()
+    itemManager.changeComponent = jest.fn().mockReturnValue({} as jest.Mocked<GenericItem>)
+    itemManager.setItemsToBeDeleted = jest.fn()
+    itemManager.addObserver = jest.fn()
+    itemManager.changeItem = jest.fn()
+    itemManager.changeFeatureRepo = jest.fn()
+
+    featureService = {} as jest.Mocked<SNFeaturesService>
+
+    prefsService = {} as jest.Mocked<SNPreferencesService>
+
+    alertService = {} as jest.Mocked<AlertService>
+    alertService.confirm = jest.fn()
+    alertService.alert = jest.fn()
+
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+  })
+
+  const nativeComponent = (identifier?: FeatureIdentifier, file_type?: FeatureDescription['file_type']) => {
+    return new SNComponent({
+      uuid: '789',
+      content_type: ContentType.Component,
+      content: {
+        package_info: {
+          hosted_url: 'https://example.com/component',
+          identifier: identifier || FeatureIdentifier.PlusEditor,
+          file_type: file_type ?? 'html',
+          valid_until: new Date(),
+        },
+      },
+    } as never)
+  }
+
+  const deprecatedComponent = () => {
+    return new SNComponent({
+      uuid: '789',
+      content_type: ContentType.Component,
+      content: {
+        package_info: {
+          hosted_url: 'https://example.com/component',
+          identifier: FeatureIdentifier.DeprecatedFileSafe,
+          valid_until: new Date(),
+        },
+      },
+    } as never)
+  }
+
+  const thirdPartyComponent = () => {
+    return new SNComponent({
+      uuid: '789',
+      content_type: ContentType.Component,
+      content: {
+        local_url: 'sn://Extensions/non-native-identifier/dist/index.html',
+        hosted_url: 'https://example.com/component',
+        package_info: {
+          identifier: 'non-native-identifier',
+          valid_until: new Date(),
+        },
+      },
+    } as never)
+  }
+
+  describe('permissions', () => {
+    it('editor should be able to to stream single note', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamContextItem,
+          content_types: [ContentType.Note],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(
+        manager.areRequestedPermissionsValid(nativeComponent(FeatureIdentifier.MarkdownVisualEditor), permissions),
+      ).toEqual(true)
+    })
+
+    it('no extension should be able to stream multiple notes', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [ContentType.Note],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(manager.areRequestedPermissionsValid(nativeComponent(), permissions)).toEqual(false)
+    })
+
+    it('no extension should be able to stream multiple tags', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [ContentType.Tag],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(manager.areRequestedPermissionsValid(nativeComponent(), permissions)).toEqual(false)
+    })
+
+    it('no extension should be able to stream multiple notes or tags', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [ContentType.Tag, ContentType.Note],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(manager.areRequestedPermissionsValid(nativeComponent(), permissions)).toEqual(false)
+    })
+
+    it('some valid and some invalid permissions should still return invalid permissions', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [ContentType.Tag, ContentType.FilesafeFileMetadata],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(
+        manager.areRequestedPermissionsValid(nativeComponent(FeatureIdentifier.DeprecatedFileSafe), permissions),
+      ).toEqual(false)
+    })
+
+    it('filesafe should be able to stream its files', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [
+            ContentType.FilesafeFileMetadata,
+            ContentType.FilesafeCredentials,
+            ContentType.FilesafeIntegration,
+          ],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(
+        manager.areRequestedPermissionsValid(nativeComponent(FeatureIdentifier.DeprecatedFileSafe), permissions),
+      ).toEqual(true)
+    })
+
+    it('bold editor should be able to stream filesafe files', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [
+            ContentType.FilesafeFileMetadata,
+            ContentType.FilesafeCredentials,
+            ContentType.FilesafeIntegration,
+          ],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(
+        manager.areRequestedPermissionsValid(nativeComponent(FeatureIdentifier.DeprecatedBoldEditor), permissions),
+      ).toEqual(true)
+    })
+
+    it('non bold editor should not able to stream filesafe files', () => {
+      const permissions: ComponentPermission[] = [
+        {
+          name: ComponentAction.StreamItems,
+          content_types: [
+            ContentType.FilesafeFileMetadata,
+            ContentType.FilesafeCredentials,
+            ContentType.FilesafeIntegration,
+          ],
+        },
+      ]
+
+      const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+      expect(manager.areRequestedPermissionsValid(nativeComponent(FeatureIdentifier.PlusEditor), permissions)).toEqual(
+        false,
+      )
+    })
+  })
+
+  describe('urlForComponent', () => {
+    describe('desktop', () => {
+      it('returns native path for native component', () => {
+        const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+        const component = nativeComponent()
+        const url = manager.urlForComponent(component)
+        const feature = FindNativeFeature(component.identifier)
+        expect(url).toEqual(`${desktopExtHost}/components/${feature?.identifier}/${feature?.index_path}`)
+      })
+
+      it('returns native path for deprecated native component', () => {
+        const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+        const component = deprecatedComponent()
+        const url = manager.urlForComponent(component)
+        const feature = FindNativeFeature(component.identifier)
+        expect(url).toEqual(`${desktopExtHost}/components/${feature?.identifier}/${feature?.index_path}`)
+      })
+
+      it('returns nonnative path for third party component', () => {
+        const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+        const component = thirdPartyComponent()
+        const url = manager.urlForComponent(component)
+        expect(url).toEqual(`${desktopExtHost}/Extensions/${component.identifier}/dist/index.html`)
+      })
+
+      it('returns hosted url for third party component with no local_url', () => {
+        const manager = createManager(Environment.Desktop, Platform.MacDesktop)
+        const component = new SNComponent({
+          uuid: '789',
+          content_type: ContentType.Component,
+          content: {
+            hosted_url: 'https://example.com/component',
+            package_info: {
+              identifier: 'non-native-identifier',
+              valid_until: new Date(),
+            },
+          },
+        } as never)
+        const url = manager.urlForComponent(component)
+        expect(url).toEqual('https://example.com/component')
+      })
+    })
+
+    describe('web', () => {
+      it('returns native path for native component', () => {
+        const manager = createManager(Environment.Web, Platform.MacWeb)
+        const component = nativeComponent()
+        const url = manager.urlForComponent(component)
+        const feature = FindNativeFeature(component.identifier) as FeatureDescription
+        expect(url).toEqual(`http://localhost/components/assets/${component.identifier}/${feature.index_path}`)
+      })
+
+      it('returns hosted path for third party component', () => {
+        const manager = createManager(Environment.Web, Platform.MacWeb)
+        const component = thirdPartyComponent()
+        const url = manager.urlForComponent(component)
+        expect(url).toEqual(component.hosted_url)
+      })
+    })
+  })
+
+  describe('editor change alert', () => {
+    it('should not require alert switching from plain editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const component = nativeComponent()
+      const requiresAlert = manager.doesEditorChangeRequireAlert(undefined, component)
+      expect(requiresAlert).toBe(false)
+    })
+
+    it('should not require alert switching to plain editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const component = nativeComponent()
+      const requiresAlert = manager.doesEditorChangeRequireAlert(component, undefined)
+      expect(requiresAlert).toBe(false)
+    })
+
+    it('should not require alert switching from a markdown editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const htmlEditor = nativeComponent()
+      const markdownEditor = nativeComponent(undefined, 'md')
+      const requiresAlert = manager.doesEditorChangeRequireAlert(markdownEditor, htmlEditor)
+      expect(requiresAlert).toBe(false)
+    })
+
+    it('should not require alert switching to a markdown editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const htmlEditor = nativeComponent()
+      const markdownEditor = nativeComponent(undefined, 'md')
+      const requiresAlert = manager.doesEditorChangeRequireAlert(htmlEditor, markdownEditor)
+      expect(requiresAlert).toBe(false)
+    })
+
+    it('should not require alert switching from & to a html editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const htmlEditor = nativeComponent()
+      const requiresAlert = manager.doesEditorChangeRequireAlert(htmlEditor, htmlEditor)
+      expect(requiresAlert).toBe(false)
+    })
+
+    it('should require alert switching from a html editor to custom editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const htmlEditor = nativeComponent()
+      const customEditor = nativeComponent(undefined, 'json')
+      const requiresAlert = manager.doesEditorChangeRequireAlert(htmlEditor, customEditor)
+      expect(requiresAlert).toBe(true)
+    })
+
+    it('should require alert switching from a custom editor to html editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const htmlEditor = nativeComponent()
+      const customEditor = nativeComponent(undefined, 'json')
+      const requiresAlert = manager.doesEditorChangeRequireAlert(customEditor, htmlEditor)
+      expect(requiresAlert).toBe(true)
+    })
+
+    it('should require alert switching from a custom editor to custom editor', () => {
+      const manager = createManager(Environment.Web, Platform.MacWeb)
+      const customEditor = nativeComponent(undefined, 'json')
+      const requiresAlert = manager.doesEditorChangeRequireAlert(customEditor, customEditor)
+      expect(requiresAlert).toBe(true)
+    })
+  })
+})

packages/snjs/lib/Services/ComponentManager/ComponentManager.ts

@@ -0,0 +1,673 @@
+import { AllowedBatchStreaming } from './Types'
+import { SNPreferencesService } from '../Preferences/PreferencesService'
+import { SNFeaturesService } from '@Lib/Services/Features/FeaturesService'
+import { ContentType, DisplayStringForContentType } from '@standardnotes/common'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import { SNNote, SNTheme, SNComponent, ComponentMutator, PayloadEmitSource } from '@standardnotes/models'
+import { SNSyncService } from '@Lib/Services/Sync/SyncService'
+import find from 'lodash/find'
+import uniq from 'lodash/uniq'
+import { ComponentArea, ComponentAction, ComponentPermission, FindNativeFeature } from '@standardnotes/features'
+import { Copy, filterFromArray, removeFromArray, sleep, assert } from '@standardnotes/utils'
+import { UuidString } from '@Lib/Types/UuidString'
+import {
+  PermissionDialog,
+  DesktopManagerInterface,
+  AllowedBatchContentTypes,
+} from '@Lib/Services/ComponentManager/Types'
+import { ActionObserver, ComponentViewer } from '@Lib/Services/ComponentManager/ComponentViewer'
+import {
+  AbstractService,
+  InternalEventBusInterface,
+  Environment,
+  Platform,
+  AlertService,
+} from '@standardnotes/services'
+
+const DESKTOP_URL_PREFIX = 'sn://'
+const LOCAL_HOST = 'localhost'
+const CUSTOM_LOCAL_HOST = 'sn.local'
+const ANDROID_LOCAL_HOST = '10.0.2.2'
+
+declare global {
+  interface Window {
+    /** IE Handlers */
+    attachEvent(event: string, listener: EventListener): boolean
+    detachEvent(event: string, listener: EventListener): void
+  }
+}
+
+export enum ComponentManagerEvent {
+  ViewerDidFocus = 'ViewerDidFocus',
+}
+
+export type EventData = {
+  componentViewer?: ComponentViewer
+}
+
+/**
+ * Responsible for orchestrating component functionality, including editors, themes,
+ * and other components. The component manager primarily deals with iframes, and orchestrates
+ * sending and receiving messages to and from frames via the postMessage API.
+ */
+export class SNComponentManager extends AbstractService<ComponentManagerEvent, EventData> {
+  private desktopManager?: DesktopManagerInterface
+  private viewers: ComponentViewer[] = []
+  private removeItemObserver!: () => void
+  private permissionDialogs: PermissionDialog[] = []
+
+  constructor(
+    private itemManager: ItemManager,
+    private syncService: SNSyncService,
+    private featuresService: SNFeaturesService,
+    private preferencesSerivce: SNPreferencesService,
+    protected alertService: AlertService,
+    private environment: Environment,
+    private platform: Platform,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+    this.loggingEnabled = false
+
+    this.addItemObserver()
+
+    /* On mobile, events listeners are handled by a respective component */
+    if (environment !== Environment.Mobile) {
+      window.addEventListener
+        ? window.addEventListener('focus', this.detectFocusChange, true)
+        : window.attachEvent('onfocusout', this.detectFocusChange)
+      window.addEventListener
+        ? window.addEventListener('blur', this.detectFocusChange, true)
+        : window.attachEvent('onblur', this.detectFocusChange)
+
+      window.addEventListener('message', this.onWindowMessage, true)
+    }
+  }
+
+  get isDesktop(): boolean {
+    return this.environment === Environment.Desktop
+  }
+
+  get isMobile(): boolean {
+    return this.environment === Environment.Mobile
+  }
+
+  get components(): SNComponent[] {
+    return this.itemManager.getDisplayableComponents()
+  }
+
+  componentsForArea(area: ComponentArea): SNComponent[] {
+    return this.components.filter((component) => {
+      return component.area === area
+    })
+  }
+
+  override deinit(): void {
+    super.deinit()
+
+    for (const viewer of this.viewers) {
+      viewer.destroy()
+    }
+
+    this.viewers.length = 0
+    this.permissionDialogs.length = 0
+
+    this.desktopManager = undefined
+    ;(this.itemManager as unknown) = undefined
+    ;(this.featuresService as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    ;(this.alertService as unknown) = undefined
+    ;(this.preferencesSerivce as unknown) = undefined
+
+    this.removeItemObserver?.()
+    ;(this.removeItemObserver as unknown) = undefined
+
+    if (window && !this.isMobile) {
+      window.removeEventListener('focus', this.detectFocusChange, true)
+      window.removeEventListener('blur', this.detectFocusChange, true)
+      window.removeEventListener('message', this.onWindowMessage, true)
+    }
+
+    ;(this.detectFocusChange as unknown) = undefined
+    ;(this.onWindowMessage as unknown) = undefined
+  }
+
+  public createComponentViewer(
+    component: SNComponent,
+    contextItem?: UuidString,
+    actionObserver?: ActionObserver,
+    urlOverride?: string,
+  ): ComponentViewer {
+    const viewer = new ComponentViewer(
+      component,
+      this.itemManager,
+      this.syncService,
+      this.alertService,
+      this.preferencesSerivce,
+      this.featuresService,
+      this.environment,
+      this.platform,
+      {
+        runWithPermissions: this.runWithPermissions.bind(this),
+        urlsForActiveThemes: this.urlsForActiveThemes.bind(this),
+      },
+      urlOverride || this.urlForComponent(component),
+      contextItem,
+      actionObserver,
+    )
+    this.viewers.push(viewer)
+    return viewer
+  }
+
+  public destroyComponentViewer(viewer: ComponentViewer): void {
+    viewer.destroy()
+    removeFromArray(this.viewers, viewer)
+  }
+
+  setDesktopManager(desktopManager: DesktopManagerInterface): void {
+    this.desktopManager = desktopManager
+    this.configureForDesktop()
+  }
+
+  handleChangedComponents(components: SNComponent[], source: PayloadEmitSource): void {
+    const acceptableSources = [
+      PayloadEmitSource.LocalChanged,
+      PayloadEmitSource.RemoteRetrieved,
+      PayloadEmitSource.LocalDatabaseLoaded,
+      PayloadEmitSource.LocalInserted,
+    ]
+
+    if (components.length === 0 || !acceptableSources.includes(source)) {
+      return
+    }
+
+    if (this.isDesktop) {
+      const thirdPartyComponents = components.filter((component) => {
+        const nativeFeature = FindNativeFeature(component.identifier)
+        return nativeFeature ? false : true
+      })
+      if (thirdPartyComponents.length > 0) {
+        this.desktopManager?.syncComponentsInstallation(thirdPartyComponents)
+      }
+    }
+
+    const themes = components.filter((c) => c.isTheme())
+    if (themes.length > 0) {
+      this.postActiveThemesToAllViewers()
+    }
+  }
+
+  addItemObserver(): void {
+    this.removeItemObserver = this.itemManager.addObserver<SNComponent>(
+      [ContentType.Component, ContentType.Theme],
+      ({ changed, inserted, source }) => {
+        const items = [...changed, ...inserted]
+        this.handleChangedComponents(items, source)
+      },
+    )
+  }
+
+  detectFocusChange = (): void => {
+    const activeIframes = this.allComponentIframes()
+    for (const iframe of activeIframes) {
+      if (document.activeElement === iframe) {
+        setTimeout(() => {
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          const viewer = this.findComponentViewer(
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            iframe.dataset.componentViewerId!,
+          )!
+          void this.notifyEvent(ComponentManagerEvent.ViewerDidFocus, {
+            componentViewer: viewer,
+          })
+        })
+        return
+      }
+    }
+  }
+
+  onWindowMessage = (event: MessageEvent): void => {
+    /** Make sure this message is for us */
+    if (event.data.sessionKey) {
+      this.log('Component manager received message', event.data)
+      this.componentViewerForSessionKey(event.data.sessionKey)?.handleMessage(event.data)
+    }
+  }
+
+  configureForDesktop(): void {
+    this.desktopManager?.registerUpdateObserver((component: SNComponent) => {
+      /* Reload theme if active */
+      if (component.active && component.isTheme()) {
+        this.postActiveThemesToAllViewers()
+      }
+    })
+  }
+
+  postActiveThemesToAllViewers(): void {
+    for (const viewer of this.viewers) {
+      viewer.postActiveThemes()
+    }
+  }
+
+  getActiveThemes(): SNTheme[] {
+    if (this.environment === Environment.Mobile) {
+      throw Error('getActiveThemes must be handled separately by mobile')
+    }
+    return this.componentsForArea(ComponentArea.Themes).filter((theme) => {
+      return theme.active
+    }) as SNTheme[]
+  }
+
+  urlForComponent(component: SNComponent): string | undefined {
+    const platformSupportsOfflineOnly = this.isDesktop
+    if (component.offlineOnly && !platformSupportsOfflineOnly) {
+      return undefined
+    }
+
+    const nativeFeature = FindNativeFeature(component.identifier)
+
+    if (this.isDesktop) {
+      assert(this.desktopManager)
+
+      if (nativeFeature) {
+        return `${this.desktopManager.getExtServerHost()}/components/${component.identifier}/${
+          nativeFeature.index_path
+        }`
+      } else if (component.local_url) {
+        return component.local_url.replace(DESKTOP_URL_PREFIX, this.desktopManager.getExtServerHost() + '/')
+      } else {
+        return component.hosted_url || component.legacy_url
+      }
+    }
+
+    const isWeb = this.environment === Environment.Web
+    if (nativeFeature) {
+      if (!isWeb) {
+        throw Error('Mobile must override urlForComponent to handle native paths')
+      }
+      return `${window.location.origin}/components/assets/${component.identifier}/${nativeFeature.index_path}`
+    }
+
+    let url = component.hosted_url || component.legacy_url
+    if (!url) {
+      return undefined
+    }
+    if (this.isMobile) {
+      const localReplacement = this.platform === Platform.Ios ? LOCAL_HOST : ANDROID_LOCAL_HOST
+      url = url.replace(LOCAL_HOST, localReplacement).replace(CUSTOM_LOCAL_HOST, localReplacement)
+    }
+    return url
+  }
+
+  urlsForActiveThemes(): string[] {
+    const themes = this.getActiveThemes()
+    const urls = []
+    for (const theme of themes) {
+      const url = this.urlForComponent(theme)
+      if (url) {
+        urls.push(url)
+      }
+    }
+    return urls
+  }
+
+  private findComponent(uuid: UuidString): SNComponent | undefined {
+    return this.itemManager.findItem<SNComponent>(uuid)
+  }
+
+  findComponentViewer(identifier: string): ComponentViewer | undefined {
+    return this.viewers.find((viewer) => viewer.identifier === identifier)
+  }
+
+  componentViewerForSessionKey(key: string): ComponentViewer | undefined {
+    return this.viewers.find((viewer) => viewer.sessionKey === key)
+  }
+
+  areRequestedPermissionsValid(component: SNComponent, permissions: ComponentPermission[]): boolean {
+    for (const permission of permissions) {
+      if (permission.name === ComponentAction.StreamItems) {
+        if (!AllowedBatchStreaming.includes(component.identifier)) {
+          return false
+        }
+        const hasNonAllowedBatchPermission = permission.content_types?.some(
+          (type) => !AllowedBatchContentTypes.includes(type),
+        )
+        if (hasNonAllowedBatchPermission) {
+          return false
+        }
+      }
+    }
+
+    return true
+  }
+
+  runWithPermissions(
+    componentUuid: UuidString,
+    requiredPermissions: ComponentPermission[],
+    runFunction: () => void,
+  ): void {
+    const component = this.findComponent(componentUuid)
+
+    if (!component) {
+      void this.alertService.alert(
+        `Unable to find component with ID ${componentUuid}. Please restart the app and try again.`,
+        'An unexpected error occurred',
+      )
+
+      return
+    }
+
+    if (!this.areRequestedPermissionsValid(component, requiredPermissions)) {
+      console.error('Component is requesting invalid permissions', componentUuid, requiredPermissions)
+      return
+    }
+
+    const nativeFeature = FindNativeFeature(component.identifier)
+    const acquiredPermissions = nativeFeature?.component_permissions || component.permissions
+
+    /* Make copy as not to mutate input values */
+    requiredPermissions = Copy(requiredPermissions) as ComponentPermission[]
+    for (const required of requiredPermissions.slice()) {
+      /* Remove anything we already have */
+      const respectiveAcquired = acquiredPermissions.find((candidate) => candidate.name === required.name)
+      if (!respectiveAcquired) {
+        continue
+      }
+      /* We now match on name, lets substract from required.content_types anything we have in acquired. */
+      const requiredContentTypes = required.content_types
+      if (!requiredContentTypes) {
+        /* If this permission does not require any content types (i.e stream-context-item)
+          then we can remove this from required since we match by name (respectiveAcquired.name === required.name) */
+        filterFromArray(requiredPermissions, required)
+        continue
+      }
+      for (const acquiredContentType of respectiveAcquired.content_types!) {
+        removeFromArray(requiredContentTypes, acquiredContentType)
+      }
+      if (requiredContentTypes.length === 0) {
+        /* We've removed all acquired and end up with zero, means we already have all these permissions */
+        filterFromArray(requiredPermissions, required)
+      }
+    }
+    if (requiredPermissions.length > 0) {
+      this.promptForPermissionsWithAngularAsyncRendering(
+        component,
+        requiredPermissions,
+        // eslint-disable-next-line @typescript-eslint/require-await
+        async (approved) => {
+          if (approved) {
+            runFunction()
+          }
+        },
+      )
+    } else {
+      runFunction()
+    }
+  }
+
+  promptForPermissionsWithAngularAsyncRendering(
+    component: SNComponent,
+    permissions: ComponentPermission[],
+    callback: (approved: boolean) => Promise<void>,
+  ): void {
+    setTimeout(() => {
+      this.promptForPermissions(component, permissions, callback)
+    })
+  }
+
+  promptForPermissions(
+    component: SNComponent,
+    permissions: ComponentPermission[],
+    callback: (approved: boolean) => Promise<void>,
+  ): void {
+    const params: PermissionDialog = {
+      component: component,
+      permissions: permissions,
+      permissionsString: this.permissionsStringForPermissions(permissions, component),
+      actionBlock: callback,
+      callback: async (approved: boolean) => {
+        const latestComponent = this.findComponent(component.uuid)
+
+        if (!latestComponent) {
+          return
+        }
+
+        if (approved) {
+          this.log('Changing component to expand permissions', component)
+          const componentPermissions = Copy(latestComponent.permissions) as ComponentPermission[]
+          for (const permission of permissions) {
+            const matchingPermission = componentPermissions.find((candidate) => candidate.name === permission.name)
+            if (!matchingPermission) {
+              componentPermissions.push(permission)
+            } else {
+              /* Permission already exists, but content_types may have been expanded */
+              const contentTypes = matchingPermission.content_types || []
+              matchingPermission.content_types = uniq(contentTypes.concat(permission.content_types!))
+            }
+          }
+
+          await this.itemManager.changeItem(component, (m) => {
+            const mutator = m as ComponentMutator
+            mutator.permissions = componentPermissions
+          })
+
+          void this.syncService.sync()
+        }
+
+        this.permissionDialogs = this.permissionDialogs.filter((pendingDialog) => {
+          /* Remove self */
+          if (pendingDialog === params) {
+            pendingDialog.actionBlock && pendingDialog.actionBlock(approved)
+            return false
+          }
+          const containsObjectSubset = (source: ComponentPermission[], target: ComponentPermission[]) => {
+            return !target.some((val) => !source.find((candidate) => JSON.stringify(candidate) === JSON.stringify(val)))
+          }
+          if (pendingDialog.component === component) {
+            /* remove pending dialogs that are encapsulated by already approved permissions, and run its function */
+            if (
+              pendingDialog.permissions === permissions ||
+              containsObjectSubset(permissions, pendingDialog.permissions)
+            ) {
+              /* If approved, run the action block. Otherwise, if canceled, cancel any
+              pending ones as well, since the user was explicit in their intentions */
+              if (approved) {
+                pendingDialog.actionBlock && pendingDialog.actionBlock(approved)
+              }
+              return false
+            }
+          }
+          return true
+        })
+
+        if (this.permissionDialogs.length > 0) {
+          this.presentPermissionsDialog(this.permissionDialogs[0])
+        }
+      },
+    }
+    /**
+     * Since these calls are asyncronous, multiple dialogs may be requested at the same time.
+     * We only want to present one and trigger all callbacks based on one modal result
+     */
+    const existingDialog = find(this.permissionDialogs, {
+      component: component,
+    })
+    this.permissionDialogs.push(params)
+    if (!existingDialog) {
+      this.presentPermissionsDialog(params)
+    } else {
+      this.log('Existing dialog, not presenting.')
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  presentPermissionsDialog(_dialog: PermissionDialog): void {
+    throw 'Must override SNComponentManager.presentPermissionsDialog'
+  }
+
+  async toggleTheme(uuid: UuidString): Promise<void> {
+    this.log('Toggling theme', uuid)
+
+    const theme = this.findComponent(uuid) as SNTheme
+    if (theme.active) {
+      await this.itemManager.changeComponent(theme, (mutator) => {
+        mutator.active = false
+      })
+    } else {
+      const activeThemes = this.getActiveThemes()
+
+      /* Activate current before deactivating others, so as not to flicker */
+      await this.itemManager.changeComponent(theme, (mutator) => {
+        mutator.active = true
+      })
+
+      /* Deactive currently active theme(s) if new theme is not layerable */
+      if (!theme.isLayerable()) {
+        await sleep(10)
+        for (const candidate of activeThemes) {
+          if (candidate && !candidate.isLayerable()) {
+            await this.itemManager.changeComponent(candidate, (mutator) => {
+              mutator.active = false
+            })
+          }
+        }
+      }
+    }
+  }
+
+  async toggleComponent(uuid: UuidString): Promise<void> {
+    this.log('Toggling component', uuid)
+
+    const component = this.findComponent(uuid)
+
+    if (!component) {
+      return
+    }
+
+    await this.itemManager.changeComponent(component, (mutator) => {
+      mutator.active = !(mutator.getItem() as SNComponent).active
+    })
+  }
+
+  isComponentActive(component: SNComponent): boolean {
+    return component.active
+  }
+
+  allComponentIframes(): HTMLIFrameElement[] {
+    if (this.isMobile) {
+      /**
+       * Retrieving all iframes is typically related to lifecycle management of
+       * non-editor components. So this function is not useful to mobile.
+       */
+      return []
+    }
+    return Array.from(document.getElementsByTagName('iframe'))
+  }
+
+  iframeForComponentViewer(viewer: ComponentViewer): HTMLIFrameElement | undefined {
+    return viewer.getIframe()
+  }
+
+  editorForNote(note: SNNote): SNComponent | undefined {
+    const editors = this.componentsForArea(ComponentArea.Editor)
+    for (const editor of editors) {
+      if (editor.isExplicitlyEnabledForItem(note.uuid)) {
+        return editor
+      }
+    }
+    let defaultEditor
+    /* No editor found for note. Use default editor, if note does not prefer system editor */
+    if (this.isMobile) {
+      if (!note.mobilePrefersPlainEditor) {
+        defaultEditor = this.getDefaultEditor()
+      }
+    } else {
+      if (!note.prefersPlainEditor) {
+        defaultEditor = this.getDefaultEditor()
+      }
+    }
+    if (defaultEditor && !defaultEditor.isExplicitlyDisabledForItem(note.uuid)) {
+      return defaultEditor
+    } else {
+      return undefined
+    }
+  }
+
+  getDefaultEditor(): SNComponent {
+    const editors = this.componentsForArea(ComponentArea.Editor)
+    if (this.isMobile) {
+      return editors.filter((e) => {
+        return e.isMobileDefault
+      })[0]
+    } else {
+      return editors.filter((e) => e.isDefaultEditor())[0]
+    }
+  }
+
+  permissionsStringForPermissions(permissions: ComponentPermission[], component: SNComponent): string {
+    if (permissions.length === 0) {
+      return '.'
+    }
+
+    let contentTypeStrings: string[] = []
+    let contextAreaStrings: string[] = []
+
+    permissions.forEach((permission) => {
+      switch (permission.name) {
+        case ComponentAction.StreamItems:
+          if (!permission.content_types) {
+            return
+          }
+          permission.content_types.forEach((contentType) => {
+            const desc = DisplayStringForContentType(contentType)
+            if (desc) {
+              contentTypeStrings.push(`${desc}s`)
+            } else {
+              contentTypeStrings.push(`items of type ${contentType}`)
+            }
+          })
+          break
+        case ComponentAction.StreamContextItem:
+          {
+            const componentAreaMapping = {
+              [ComponentArea.EditorStack]: 'working note',
+              [ComponentArea.Editor]: 'working note',
+              [ComponentArea.Themes]: 'Unknown',
+            }
+            contextAreaStrings.push(componentAreaMapping[component.area])
+          }
+          break
+      }
+    })
+
+    contentTypeStrings = uniq(contentTypeStrings)
+    contextAreaStrings = uniq(contextAreaStrings)
+
+    if (contentTypeStrings.length === 0 && contextAreaStrings.length === 0) {
+      return '.'
+    }
+    return contentTypeStrings.concat(contextAreaStrings).join(', ') + '.'
+  }
+
+  doesEditorChangeRequireAlert(from: SNComponent | undefined, to: SNComponent | undefined): boolean {
+    const isEitherPlainEditor = !from || !to
+    const isEitherMarkdown = from?.package_info.file_type === 'md' || to?.package_info.file_type === 'md'
+    const areBothHtml = from?.package_info.file_type === 'html' && to?.package_info.file_type === 'html'
+
+    if (isEitherPlainEditor || isEitherMarkdown || areBothHtml) {
+      return false
+    } else {
+      return true
+    }
+  }
+
+  async showEditorChangeAlert(): Promise<boolean> {
+    const shouldChangeEditor = await this.alertService.confirm(
+      'Doing so might result in minor formatting changes.',
+      "Are you sure you want to change this note's type?",
+      'Yes, change it',
+    )
+
+    return shouldChangeEditor
+  }
+}

packages/snjs/lib/Services/ComponentManager/ComponentViewer.ts

@@ -0,0 +1,913 @@
+import { SNPreferencesService } from '../Preferences/PreferencesService'
+import { FeatureStatus, FeaturesEvent } from '@Lib/Services/Features'
+import { Environment, Platform, AlertService } from '@standardnotes/services'
+import { SNFeaturesService } from '@Lib/Services'
+import {
+  SNComponent,
+  PrefKey,
+  NoteContent,
+  MutationType,
+  CreateDecryptedItemFromPayload,
+  DecryptedItemInterface,
+  DeletedItemInterface,
+  EncryptedItemInterface,
+  isDecryptedItem,
+  isNotEncryptedItem,
+  isNote,
+  CreateComponentRetrievedContextPayload,
+  createComponentCreatedContextPayload,
+  DecryptedPayload,
+  ItemContent,
+  ComponentDataDomain,
+  PayloadEmitSource,
+  PayloadTimestampDefaults,
+} from '@standardnotes/models'
+import find from 'lodash/find'
+import uniq from 'lodash/uniq'
+import remove from 'lodash/remove'
+import { SNSyncService } from '@Lib/Services/Sync/SyncService'
+import { environmentToString, platformToString } from '@Lib/Application/Platforms'
+import {
+  ComponentMessage,
+  OutgoingItemMessagePayload,
+  MessageReply,
+  StreamItemsMessageData,
+  AllowedBatchContentTypes,
+  IncomingComponentItemPayload,
+  DeleteItemsMessageData,
+  MessageReplyData,
+} from './Types'
+import { ComponentAction, ComponentPermission, ComponentArea, FindNativeFeature } from '@standardnotes/features'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import { UuidString } from '@Lib/Types/UuidString'
+import { ContentType } from '@standardnotes/common'
+import {
+  isString,
+  extendArray,
+  Copy,
+  removeFromArray,
+  log,
+  nonSecureRandomIdentifier,
+  UuidGenerator,
+  Uuids,
+  sureSearchArray,
+  isNotUndefined,
+} from '@standardnotes/utils'
+import { MessageData } from '..'
+
+type RunWithPermissionsCallback = (
+  componentUuid: UuidString,
+  requiredPermissions: ComponentPermission[],
+  runFunction: () => void,
+) => void
+
+type ComponentManagerFunctions = {
+  runWithPermissions: RunWithPermissionsCallback
+  urlsForActiveThemes: () => string[]
+}
+
+const ReadwriteActions = [
+  ComponentAction.SaveItems,
+  ComponentAction.AssociateItem,
+  ComponentAction.DeassociateItem,
+  ComponentAction.CreateItem,
+  ComponentAction.CreateItems,
+  ComponentAction.DeleteItems,
+  ComponentAction.SetComponentData,
+]
+
+export type ActionObserver = (action: ComponentAction, messageData: MessageData) => void
+
+export enum ComponentViewerEvent {
+  FeatureStatusUpdated = 'FeatureStatusUpdated',
+}
+type EventObserver = (event: ComponentViewerEvent) => void
+
+export enum ComponentViewerError {
+  OfflineRestricted = 'OfflineRestricted',
+  MissingUrl = 'MissingUrl',
+}
+
+type Writeable<T> = { -readonly [P in keyof T]: T[P] }
+
+export class ComponentViewer {
+  private streamItems?: ContentType[]
+  private streamContextItemOriginalMessage?: ComponentMessage
+  private streamItemsOriginalMessage?: ComponentMessage
+  private removeItemObserver: () => void
+  private loggingEnabled = false
+  public identifier = nonSecureRandomIdentifier()
+  private actionObservers: ActionObserver[] = []
+  public overrideContextItem?: DecryptedItemInterface
+  private featureStatus: FeatureStatus
+  private removeFeaturesObserver: () => void
+  private eventObservers: EventObserver[] = []
+  private dealloced = false
+
+  private window?: Window
+  private hidden = false
+  private readonly = false
+  public lockReadonly = false
+  public sessionKey?: string
+
+  constructor(
+    public readonly component: SNComponent,
+    private itemManager: ItemManager,
+    private syncService: SNSyncService,
+    private alertService: AlertService,
+    private preferencesSerivce: SNPreferencesService,
+    featuresService: SNFeaturesService,
+    private environment: Environment,
+    private platform: Platform,
+    private componentManagerFunctions: ComponentManagerFunctions,
+    public readonly url?: string,
+    private contextItemUuid?: UuidString,
+    actionObserver?: ActionObserver,
+  ) {
+    this.removeItemObserver = this.itemManager.addObserver(
+      ContentType.Any,
+      ({ changed, inserted, removed, source, sourceKey }) => {
+        if (this.dealloced) {
+          return
+        }
+        const items = [...changed, ...inserted, ...removed]
+        this.handleChangesInItems(items, source, sourceKey)
+      },
+    )
+    if (actionObserver) {
+      this.actionObservers.push(actionObserver)
+    }
+
+    this.featureStatus = featuresService.getFeatureStatus(component.identifier)
+
+    this.removeFeaturesObserver = featuresService.addEventObserver((event) => {
+      if (this.dealloced) {
+        return
+      }
+      if (event === FeaturesEvent.FeaturesUpdated) {
+        const featureStatus = featuresService.getFeatureStatus(component.identifier)
+
+        if (featureStatus !== this.featureStatus) {
+          this.featureStatus = featureStatus
+          this.notifyEventObservers(ComponentViewerEvent.FeatureStatusUpdated)
+        }
+      }
+    })
+
+    this.log('Constructor', this)
+  }
+
+  get isDesktop(): boolean {
+    return this.environment === Environment.Desktop
+  }
+
+  get isMobile(): boolean {
+    return this.environment === Environment.Mobile
+  }
+
+  public destroy(): void {
+    this.log('Destroying', this)
+    this.deinit()
+  }
+
+  private deinit(): void {
+    this.dealloced = true
+    ;(this.component as unknown) = undefined
+    ;(this.itemManager as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    ;(this.alertService as unknown) = undefined
+    ;(this.preferencesSerivce as unknown) = undefined
+    ;(this.componentManagerFunctions as unknown) = undefined
+
+    this.eventObservers.length = 0
+    this.actionObservers.length = 0
+
+    this.removeFeaturesObserver()
+    ;(this.removeFeaturesObserver as unknown) = undefined
+
+    this.removeItemObserver()
+    ;(this.removeItemObserver as unknown) = undefined
+  }
+
+  public addEventObserver(observer: EventObserver): () => void {
+    this.eventObservers.push(observer)
+
+    const thislessChangeObservers = this.eventObservers
+    return () => {
+      removeFromArray(thislessChangeObservers, observer)
+    }
+  }
+
+  private notifyEventObservers(event: ComponentViewerEvent): void {
+    for (const observer of this.eventObservers) {
+      observer(event)
+    }
+  }
+
+  public addActionObserver(observer: ActionObserver): () => void {
+    this.actionObservers.push(observer)
+
+    const thislessChangeObservers = this.actionObservers
+    return () => {
+      removeFromArray(thislessChangeObservers, observer)
+    }
+  }
+
+  public setReadonly(readonly: boolean): void {
+    if (this.lockReadonly) {
+      throw Error('Attempting to set readonly on lockedReadonly component viewer')
+    }
+    this.readonly = readonly
+  }
+
+  get componentUuid(): string {
+    return this.component.uuid
+  }
+
+  public getFeatureStatus(): FeatureStatus {
+    return this.featureStatus
+  }
+
+  private isOfflineRestricted(): boolean {
+    return this.component.offlineOnly && !this.isDesktop
+  }
+
+  private isNativeFeature(): boolean {
+    return !!FindNativeFeature(this.component.identifier)
+  }
+
+  private hasUrlError(): boolean {
+    if (this.isNativeFeature()) {
+      return false
+    }
+    return this.isDesktop
+      ? !this.component.local_url && !this.component.hasValidHostedUrl()
+      : !this.component.hasValidHostedUrl()
+  }
+
+  public shouldRender(): boolean {
+    return this.getError() == undefined
+  }
+
+  public getError(): ComponentViewerError | undefined {
+    if (this.isOfflineRestricted()) {
+      return ComponentViewerError.OfflineRestricted
+    }
+    if (this.hasUrlError()) {
+      return ComponentViewerError.MissingUrl
+    }
+
+    return undefined
+  }
+
+  private updateOurComponentRefFromChangedItems(items: DecryptedItemInterface[]): void {
+    const updatedComponent = items.find((item) => item.uuid === this.component.uuid)
+    if (updatedComponent && isDecryptedItem(updatedComponent)) {
+      ;(this.component as Writeable<SNComponent>) = updatedComponent as SNComponent
+    }
+  }
+
+  handleChangesInItems(
+    items: (DecryptedItemInterface | DeletedItemInterface | EncryptedItemInterface)[],
+    source: PayloadEmitSource,
+    sourceKey?: string,
+  ): void {
+    const nonencryptedItems = items.filter(isNotEncryptedItem)
+    const nondeletedItems = nonencryptedItems.filter(isDecryptedItem)
+
+    this.updateOurComponentRefFromChangedItems(nondeletedItems)
+
+    const areWeOriginator = sourceKey && sourceKey === this.component.uuid
+    if (areWeOriginator) {
+      return
+    }
+
+    if (this.streamItems) {
+      const relevantItems = nonencryptedItems.filter((item) => {
+        return this.streamItems?.includes(item.content_type)
+      })
+
+      if (relevantItems.length > 0) {
+        this.sendManyItemsThroughBridge(relevantItems)
+      }
+    }
+
+    if (this.streamContextItemOriginalMessage) {
+      const matchingItem = find(nondeletedItems, { uuid: this.contextItemUuid })
+      if (matchingItem) {
+        this.sendContextItemThroughBridge(matchingItem, source)
+      }
+    }
+  }
+
+  sendManyItemsThroughBridge(items: (DecryptedItemInterface | DeletedItemInterface)[]): void {
+    const requiredPermissions: ComponentPermission[] = [
+      {
+        name: ComponentAction.StreamItems,
+        content_types: this.streamItems!.sort(),
+      },
+    ]
+
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, requiredPermissions, () => {
+      this.sendItemsInReply(items, this.streamItemsOriginalMessage!)
+    })
+  }
+
+  sendContextItemThroughBridge(item: DecryptedItemInterface, source?: PayloadEmitSource): void {
+    const requiredContextPermissions = [
+      {
+        name: ComponentAction.StreamContextItem,
+      },
+    ] as ComponentPermission[]
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, requiredContextPermissions, () => {
+      this.log(
+        'Send context item in reply',
+        'component:',
+        this.component,
+        'item: ',
+        item,
+        'originalMessage: ',
+        this.streamContextItemOriginalMessage,
+      )
+      const response: MessageReplyData = {
+        item: this.jsonForItem(item, source),
+      }
+      this.replyToMessage(this.streamContextItemOriginalMessage!, response)
+    })
+  }
+
+  private log(message: string, ...args: unknown[]): void {
+    if (this.loggingEnabled) {
+      log('ComponentViewer', message, args)
+    }
+  }
+
+  private sendItemsInReply(
+    items: (DecryptedItemInterface | DeletedItemInterface)[],
+    message: ComponentMessage,
+    source?: PayloadEmitSource,
+  ): void {
+    this.log('Send items in reply', this.component, items, message)
+
+    const responseData: MessageReplyData = {}
+
+    const mapped = items.map((item) => {
+      return this.jsonForItem(item, source)
+    })
+
+    responseData.items = mapped
+
+    this.replyToMessage(message, responseData)
+  }
+
+  private jsonForItem(
+    item: DecryptedItemInterface | DeletedItemInterface,
+    source?: PayloadEmitSource,
+  ): OutgoingItemMessagePayload {
+    const isMetadatUpdate =
+      source === PayloadEmitSource.RemoteSaved ||
+      source === PayloadEmitSource.OfflineSyncSaved ||
+      source === PayloadEmitSource.PreSyncSave
+
+    const params: OutgoingItemMessagePayload = {
+      uuid: item.uuid,
+      content_type: item.content_type,
+      created_at: item.created_at,
+      updated_at: item.serverUpdatedAt,
+      isMetadataUpdate: isMetadatUpdate,
+    }
+
+    if (isDecryptedItem(item)) {
+      params.content = this.contentForItem(item)
+      const globalComponentData = item.getDomainData(ComponentDataDomain) || {}
+      const thisComponentData = globalComponentData[this.component.getClientDataKey()] || {}
+      params.clientData = thisComponentData as Record<string, unknown>
+    } else {
+      params.deleted = true
+    }
+
+    return this.responseItemsByRemovingPrivateProperties([params])[0]
+  }
+
+  contentForItem(item: DecryptedItemInterface): ItemContent | undefined {
+    if (isNote(item)) {
+      const content = item.content
+      const spellcheck =
+        item.spellcheck != undefined
+          ? item.spellcheck
+          : this.preferencesSerivce.getValue(PrefKey.EditorSpellcheck, true)
+
+      return {
+        ...content,
+        spellcheck,
+      } as NoteContent
+    }
+
+    return item.content
+  }
+
+  private replyToMessage(originalMessage: ComponentMessage, replyData: MessageReplyData): void {
+    const reply: MessageReply = {
+      action: ComponentAction.Reply,
+      original: originalMessage,
+      data: replyData,
+    }
+    this.sendMessage(reply)
+  }
+
+  /**
+   * @param essential If the message is non-essential, no alert will be shown
+   *  if we can no longer find the window.
+   */
+  sendMessage(message: ComponentMessage | MessageReply, essential = true): void {
+    const permissibleActionsWhileHidden = [ComponentAction.ComponentRegistered, ComponentAction.ActivateThemes]
+
+    if (this.hidden && !permissibleActionsWhileHidden.includes(message.action)) {
+      this.log('Component disabled for current item, ignoring messages.', this.component.name)
+      return
+    }
+
+    if (!this.window && message.action === ComponentAction.Reply) {
+      this.log('Component has been deallocated in between message send and reply', this.component, message)
+      return
+    }
+    this.log('Send message to component', this.component, 'message: ', message)
+
+    let origin = this.url
+    if (!origin || !this.window) {
+      if (essential) {
+        void this.alertService.alert(
+          `Standard Notes is trying to communicate with ${this.component.name}, ` +
+            'but an error is occurring. Please restart this extension and try again.',
+        )
+      }
+      return
+    }
+
+    if (!origin.startsWith('http') && !origin.startsWith('file')) {
+      /* Native extension running in web, prefix current host */
+      origin = window.location.href + origin
+    }
+
+    /* Mobile messaging requires json */
+    this.window.postMessage(this.isMobile ? JSON.stringify(message) : message, origin)
+  }
+
+  private responseItemsByRemovingPrivateProperties<T extends OutgoingItemMessagePayload | IncomingComponentItemPayload>(
+    responseItems: T[],
+    removeUrls = false,
+  ): T[] {
+    /* Don't allow component to overwrite these properties. */
+    let privateContentProperties = ['autoupdateDisabled', 'permissions', 'active']
+    if (removeUrls) {
+      privateContentProperties = privateContentProperties.concat(['hosted_url', 'local_url'])
+    }
+
+    return responseItems.map((responseItem) => {
+      const privateProperties = privateContentProperties.slice()
+      /** Server extensions are allowed to modify url property */
+      if (removeUrls) {
+        privateProperties.push('url')
+      }
+      if (!responseItem.content || isString(responseItem.content)) {
+        return responseItem
+      }
+
+      let content: Partial<ItemContent> = {}
+      for (const [key, value] of Object.entries(responseItem.content)) {
+        if (!privateProperties.includes(key)) {
+          content = {
+            ...content,
+            [key]: value,
+          }
+        }
+      }
+
+      return {
+        ...responseItem,
+        content: content,
+      }
+    })
+  }
+
+  public getWindow(): Window | undefined {
+    return this.window
+  }
+
+  /** Called by client when the iframe is ready */
+  public setWindow(window: Window): void {
+    if (this.window) {
+      throw Error('Attempting to override component viewer window. Create a new component viewer instead.')
+    }
+
+    this.log('setWindow', 'component: ', this.component, 'window: ', window)
+
+    this.window = window
+    this.sessionKey = UuidGenerator.GenerateUuid()
+
+    this.sendMessage({
+      action: ComponentAction.ComponentRegistered,
+      sessionKey: this.sessionKey,
+      componentData: this.component.componentData,
+      data: {
+        uuid: this.component.uuid,
+        environment: environmentToString(this.environment),
+        platform: platformToString(this.platform),
+        activeThemeUrls: this.componentManagerFunctions.urlsForActiveThemes(),
+      },
+    })
+
+    this.log('setWindow got new sessionKey', this.sessionKey)
+
+    this.postActiveThemes()
+  }
+
+  postActiveThemes(): void {
+    const urls = this.componentManagerFunctions.urlsForActiveThemes()
+    const data: MessageData = {
+      themes: urls,
+    }
+
+    const message: ComponentMessage = {
+      action: ComponentAction.ActivateThemes,
+      data: data,
+    }
+
+    this.sendMessage(message, false)
+  }
+
+  /* A hidden component will not receive messages. However, when a component is unhidden,
+   * we need to send it any items it may have registered streaming for. */
+  public setHidden(hidden: boolean): void {
+    if (hidden) {
+      this.hidden = true
+    } else if (this.hidden) {
+      this.hidden = false
+
+      if (this.streamContextItemOriginalMessage) {
+        this.handleStreamContextItemMessage(this.streamContextItemOriginalMessage)
+      }
+
+      if (this.streamItems) {
+        this.handleStreamItemsMessage(this.streamItemsOriginalMessage!)
+      }
+    }
+  }
+
+  handleMessage(message: ComponentMessage): void {
+    this.log('Handle message', message, this)
+    if (!this.component) {
+      this.log('Component not defined for message, returning', message)
+      void this.alertService.alert(
+        'A component is trying to communicate with Standard Notes, ' +
+          'but there is an error establishing a bridge. Please restart the app and try again.',
+      )
+      return
+    }
+    if (this.readonly && ReadwriteActions.includes(message.action)) {
+      void this.alertService.alert(
+        `${this.component.name} is trying to save, but it is in a locked state and cannot accept changes.`,
+      )
+      return
+    }
+
+    const messageHandlers: Partial<Record<ComponentAction, (message: ComponentMessage) => void>> = {
+      [ComponentAction.StreamItems]: this.handleStreamItemsMessage.bind(this),
+      [ComponentAction.StreamContextItem]: this.handleStreamContextItemMessage.bind(this),
+      [ComponentAction.SetComponentData]: this.handleSetComponentDataMessage.bind(this),
+      [ComponentAction.DeleteItems]: this.handleDeleteItemsMessage.bind(this),
+      [ComponentAction.CreateItems]: this.handleCreateItemsMessage.bind(this),
+      [ComponentAction.CreateItem]: this.handleCreateItemsMessage.bind(this),
+      [ComponentAction.SaveItems]: this.handleSaveItemsMessage.bind(this),
+      [ComponentAction.SetSize]: this.handleSetSizeEvent.bind(this),
+    }
+
+    const handler = messageHandlers[message.action]
+    handler?.(message)
+
+    for (const observer of this.actionObservers) {
+      observer(message.action, message.data)
+    }
+  }
+
+  handleStreamItemsMessage(message: ComponentMessage): void {
+    const data = message.data as StreamItemsMessageData
+    const types = data.content_types.filter((type) => AllowedBatchContentTypes.includes(type)).sort()
+    const requiredPermissions = [
+      {
+        name: ComponentAction.StreamItems,
+        content_types: types,
+      },
+    ]
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, requiredPermissions, () => {
+      if (!this.streamItems) {
+        this.streamItems = types
+        this.streamItemsOriginalMessage = message
+      }
+      /* Push immediately now */
+      const items: DecryptedItemInterface[] = []
+      for (const contentType of types) {
+        extendArray(items, this.itemManager.getItems(contentType))
+      }
+      this.sendItemsInReply(items, message)
+    })
+  }
+
+  handleStreamContextItemMessage(message: ComponentMessage): void {
+    const requiredPermissions: ComponentPermission[] = [
+      {
+        name: ComponentAction.StreamContextItem,
+      },
+    ]
+
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, requiredPermissions, () => {
+      if (!this.streamContextItemOriginalMessage) {
+        this.streamContextItemOriginalMessage = message
+      }
+      const matchingItem = this.overrideContextItem || this.itemManager.findItem(this.contextItemUuid!)
+      if (matchingItem) {
+        this.sendContextItemThroughBridge(matchingItem)
+      }
+    })
+  }
+
+  /**
+   * Save items is capable of saving existing items, and also creating new ones
+   * if they don't exist.
+   */
+  handleSaveItemsMessage(message: ComponentMessage): void {
+    let responsePayloads = message.data.items as IncomingComponentItemPayload[]
+    const requiredPermissions = []
+
+    /* Pending as in needed to be accounted for in permissions. */
+    const pendingResponseItems = responsePayloads.slice()
+
+    for (const responseItem of responsePayloads.slice()) {
+      if (responseItem.uuid === this.contextItemUuid) {
+        requiredPermissions.push({
+          name: ComponentAction.StreamContextItem,
+        })
+        removeFromArray(pendingResponseItems, responseItem)
+        /* We break because there can only be one context item */
+        break
+      }
+    }
+
+    /* Check to see if additional privileges are required */
+    if (pendingResponseItems.length > 0) {
+      const requiredContentTypes = uniq(
+        pendingResponseItems.map((item) => {
+          return item.content_type
+        }),
+      ).sort()
+
+      requiredPermissions.push({
+        name: ComponentAction.StreamItems,
+        content_types: requiredContentTypes,
+      } as ComponentPermission)
+    }
+
+    this.componentManagerFunctions.runWithPermissions(
+      this.component.uuid,
+      requiredPermissions,
+
+      async () => {
+        responsePayloads = this.responseItemsByRemovingPrivateProperties(responsePayloads, true)
+
+        /* Filter locked items */
+        const uuids = Uuids(responsePayloads)
+        const items = this.itemManager.findItemsIncludingBlanks(uuids)
+        let lockedCount = 0
+        let lockedNoteCount = 0
+
+        for (const item of items) {
+          if (!item) {
+            continue
+          }
+
+          if (item.locked) {
+            remove(responsePayloads, { uuid: item.uuid })
+            lockedCount++
+            if (item.content_type === ContentType.Note) {
+              lockedNoteCount++
+            }
+          }
+        }
+
+        if (lockedNoteCount === 1) {
+          void this.alertService.alert(
+            'The note you are attempting to save has editing disabled',
+            'Note has Editing Disabled',
+          )
+          return
+        } else if (lockedCount > 0) {
+          const itemNoun = lockedCount === 1 ? 'item' : lockedNoteCount === lockedCount ? 'notes' : 'items'
+          const auxVerb = lockedCount === 1 ? 'has' : 'have'
+          void this.alertService.alert(
+            `${lockedCount} ${itemNoun} you are attempting to save ${auxVerb} editing disabled.`,
+            'Items have Editing Disabled',
+          )
+
+          return
+        }
+
+        const contextualPayloads = responsePayloads.map((responseItem) => {
+          return CreateComponentRetrievedContextPayload(responseItem)
+        })
+
+        for (const contextualPayload of contextualPayloads) {
+          const item = this.itemManager.findItem(contextualPayload.uuid)
+          if (!item) {
+            const payload = new DecryptedPayload({
+              ...PayloadTimestampDefaults(),
+              ...contextualPayload,
+            })
+            const template = CreateDecryptedItemFromPayload(payload)
+            await this.itemManager.insertItem(template)
+          } else {
+            if (contextualPayload.content_type !== item.content_type) {
+              throw Error('Extension is trying to modify content type of item.')
+            }
+          }
+        }
+
+        await this.itemManager.changeItems(
+          items.filter(isNotUndefined),
+          (mutator) => {
+            const contextualPayload = sureSearchArray(contextualPayloads, {
+              uuid: mutator.getUuid(),
+            })
+
+            mutator.setCustomContent(contextualPayload.content)
+
+            const responseItem = sureSearchArray(responsePayloads, {
+              uuid: mutator.getUuid(),
+            })
+
+            if (responseItem.clientData) {
+              const allComponentData = Copy(mutator.getItem().getDomainData(ComponentDataDomain) || {})
+              allComponentData[this.component.getClientDataKey()] = responseItem.clientData
+              mutator.setDomainData(allComponentData, ComponentDataDomain)
+            }
+          },
+          MutationType.UpdateUserTimestamps,
+          PayloadEmitSource.ComponentRetrieved,
+          this.component.uuid,
+        )
+
+        this.syncService
+          .sync({
+            onPresyncSave: () => {
+              this.replyToMessage(message, {})
+            },
+          })
+          .catch(() => {
+            this.replyToMessage(message, {
+              error: 'save-error',
+            })
+          })
+      },
+    )
+  }
+
+  handleCreateItemsMessage(message: ComponentMessage): void {
+    let responseItems = (message.data.item ? [message.data.item] : message.data.items) as IncomingComponentItemPayload[]
+
+    const uniqueContentTypes = uniq(
+      responseItems.map((item) => {
+        return item.content_type
+      }),
+    )
+
+    const requiredPermissions: ComponentPermission[] = [
+      {
+        name: ComponentAction.StreamItems,
+        content_types: uniqueContentTypes,
+      },
+    ]
+
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, requiredPermissions, async () => {
+      responseItems = this.responseItemsByRemovingPrivateProperties(responseItems)
+      const processedItems = []
+
+      for (const responseItem of responseItems) {
+        if (!responseItem.uuid) {
+          responseItem.uuid = UuidGenerator.GenerateUuid()
+        }
+
+        const contextualPayload = createComponentCreatedContextPayload(responseItem)
+        const payload = new DecryptedPayload({
+          ...PayloadTimestampDefaults(),
+          ...contextualPayload,
+        })
+
+        const template = CreateDecryptedItemFromPayload(payload)
+        const item = await this.itemManager.insertItem(template)
+
+        await this.itemManager.changeItem(
+          item,
+          (mutator) => {
+            if (responseItem.clientData) {
+              const allComponentData = Copy(item.getDomainData(ComponentDataDomain) || {})
+              allComponentData[this.component.getClientDataKey()] = responseItem.clientData
+              mutator.setDomainData(allComponentData, ComponentDataDomain)
+            }
+          },
+          MutationType.UpdateUserTimestamps,
+          PayloadEmitSource.ComponentCreated,
+          this.component.uuid,
+        )
+        processedItems.push(item)
+      }
+
+      void this.syncService.sync()
+
+      const reply =
+        message.action === ComponentAction.CreateItem
+          ? { item: this.jsonForItem(processedItems[0]) }
+          : {
+              items: processedItems.map((item) => {
+                return this.jsonForItem(item)
+              }),
+            }
+      this.replyToMessage(message, reply)
+    })
+  }
+
+  handleDeleteItemsMessage(message: ComponentMessage): void {
+    const data = message.data as DeleteItemsMessageData
+    const items = data.items.filter((item) => AllowedBatchContentTypes.includes(item.content_type))
+
+    const requiredContentTypes = uniq(items.map((item) => item.content_type)).sort() as ContentType[]
+
+    const requiredPermissions: ComponentPermission[] = [
+      {
+        name: ComponentAction.StreamItems,
+        content_types: requiredContentTypes,
+      },
+    ]
+
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, requiredPermissions, async () => {
+      const itemsData = items
+      const noun = itemsData.length === 1 ? 'item' : 'items'
+      let reply = null
+      const didConfirm = await this.alertService.confirm(`Are you sure you want to delete ${itemsData.length} ${noun}?`)
+
+      if (didConfirm) {
+        /* Filter for any components and deactivate before deleting */
+        for (const itemData of itemsData) {
+          const item = this.itemManager.findItem(itemData.uuid)
+          if (!item) {
+            void this.alertService.alert('The item you are trying to delete cannot be found.')
+            continue
+          }
+          await this.itemManager.setItemToBeDeleted(item, PayloadEmitSource.ComponentRetrieved)
+        }
+
+        void this.syncService.sync()
+
+        reply = { deleted: true }
+      } else {
+        /* Rejected by user */
+        reply = { deleted: false }
+      }
+
+      this.replyToMessage(message, reply)
+    })
+  }
+
+  handleSetComponentDataMessage(message: ComponentMessage): void {
+    const noPermissionsRequired: ComponentPermission[] = []
+    this.componentManagerFunctions.runWithPermissions(this.component.uuid, noPermissionsRequired, async () => {
+      await this.itemManager.changeComponent(this.component, (mutator) => {
+        mutator.componentData = message.data.componentData || {}
+      })
+
+      void this.syncService.sync()
+    })
+  }
+
+  handleSetSizeEvent(message: ComponentMessage): void {
+    if (this.component.area !== ComponentArea.EditorStack) {
+      return
+    }
+
+    const parent = this.getIframe()?.parentElement
+    if (!parent) {
+      return
+    }
+
+    const data = message.data
+    const widthString = isString(data.width) ? data.width : `${data.width}px`
+    const heightString = isString(data.height) ? data.height : `${data.height}px`
+    if (parent) {
+      parent.setAttribute('style', `width:${widthString}; height:${heightString};`)
+    }
+  }
+
+  getIframe(): HTMLIFrameElement | undefined {
+    return Array.from(document.getElementsByTagName('iframe')).find(
+      (iframe) => iframe.dataset.componentViewerId === this.identifier,
+    )
+  }
+}

packages/snjs/lib/Services/ComponentManager/Types.ts

@@ -0,0 +1,134 @@
+import {
+  ComponentArea,
+  ComponentAction,
+  ComponentPermission,
+  FeatureIdentifier,
+  LegacyFileSafeIdentifier,
+} from '@standardnotes/features'
+import { ItemContent, SNComponent, DecryptedTransferPayload } from '@standardnotes/models'
+import { UuidString } from '@Lib/Types/UuidString'
+import { ContentType } from '@standardnotes/common'
+
+export interface DesktopManagerInterface {
+  syncComponentsInstallation(components: SNComponent[]): void
+  registerUpdateObserver(callback: (component: SNComponent) => void): void
+  getExtServerHost(): string
+}
+
+export type IncomingComponentItemPayload = DecryptedTransferPayload & {
+  clientData: Record<string, unknown>
+}
+
+export type OutgoingItemMessagePayload = {
+  uuid: string
+  content_type: ContentType
+  created_at: Date
+  updated_at: Date
+  deleted?: boolean
+  content?: ItemContent
+  clientData?: Record<string, unknown>
+
+  /**
+   * isMetadataUpdate implies that the extension should make reference of updated
+   * metadata, but not update content values as they may be stale relative to what the
+   * extension currently has.
+   */
+  isMetadataUpdate: boolean
+}
+
+/**
+ * Extensions allowed to batch stream AllowedBatchContentTypes
+ */
+export const AllowedBatchStreaming = Object.freeze([
+  LegacyFileSafeIdentifier,
+  FeatureIdentifier.DeprecatedFileSafe,
+  FeatureIdentifier.DeprecatedBoldEditor,
+])
+
+/**
+ * Content types which are allowed to be managed/streamed in bulk by a component.
+ */
+export const AllowedBatchContentTypes = Object.freeze([
+  ContentType.FilesafeCredentials,
+  ContentType.FilesafeFileMetadata,
+  ContentType.FilesafeIntegration,
+])
+
+export type StreamObserver = {
+  identifier: string
+  componentUuid: UuidString
+  area: ComponentArea
+  originalMessage: ComponentMessage
+  /** contentTypes is optional in the case of a context stream observer */
+  contentTypes?: ContentType[]
+}
+
+export type PermissionDialog = {
+  component: SNComponent
+  permissions: ComponentPermission[]
+  permissionsString: string
+  actionBlock: (approved: boolean) => void
+  callback: (approved: boolean) => void
+}
+
+export enum KeyboardModifier {
+  Shift = 'Shift',
+  Ctrl = 'Control',
+  Meta = 'Meta',
+}
+
+export type MessageData = Partial<{
+  /** Related to the stream-item-context action */
+  item?: IncomingComponentItemPayload
+  /** Related to the stream-items action */
+  content_types?: ContentType[]
+  items?: IncomingComponentItemPayload[]
+  /** Related to the request-permission action */
+  permissions?: ComponentPermission[]
+  /** Related to the component-registered action */
+  componentData?: Record<string, unknown>
+  uuid?: UuidString
+  environment?: string
+  platform?: string
+  activeThemeUrls?: string[]
+  /** Related to set-size action */
+  width?: string | number
+  height?: string | number
+  type?: string
+  /** Related to themes action */
+  themes?: string[]
+  /** Related to clear-selection action */
+  content_type?: ContentType
+  /** Related to key-pressed action */
+  keyboardModifier?: KeyboardModifier
+}>
+
+export type MessageReplyData = {
+  approved?: boolean
+  deleted?: boolean
+  error?: string
+  item?: OutgoingItemMessagePayload
+  items?: OutgoingItemMessagePayload[]
+  themes?: string[]
+}
+
+export type StreamItemsMessageData = MessageData & {
+  content_types: ContentType[]
+}
+
+export type DeleteItemsMessageData = MessageData & {
+  items: OutgoingItemMessagePayload[]
+}
+
+export type ComponentMessage = {
+  action: ComponentAction
+  sessionKey?: string
+  componentData?: Record<string, unknown>
+  data: MessageData
+}
+
+export type MessageReply = {
+  action: ComponentAction
+  original: ComponentMessage
+  data: MessageReplyData
+}

packages/snjs/lib/Services/ComponentManager/index.ts

@@ -0,0 +1,3 @@
+export * from './ComponentManager'
+export * from './ComponentViewer'
+export * from './Types'

packages/snjs/lib/Services/Features/ClientInterface.ts

@@ -0,0 +1,36 @@
+import { FeatureStatus, SetOfflineFeaturesFunctionResponse } from './Types'
+import { FeatureDescription, FeatureIdentifier } from '@standardnotes/features'
+import { SNComponent } from '@standardnotes/models'
+import { RoleName } from '@standardnotes/common'
+
+export interface FeaturesClientInterface {
+  downloadExternalFeature(urlOrCode: string): Promise<SNComponent | undefined>
+
+  getUserFeature(featureId: FeatureIdentifier): FeatureDescription | undefined
+
+  getFeatureStatus(featureId: FeatureIdentifier): FeatureStatus
+
+  hasMinimumRole(role: RoleName): boolean
+
+  setOfflineFeaturesCode(code: string): Promise<SetOfflineFeaturesFunctionResponse>
+
+  hasOfflineRepo(): boolean
+
+  deleteOfflineFeatureRepo(): Promise<void>
+
+  isThirdPartyFeature(identifier: string): boolean
+
+  toggleExperimentalFeature(identifier: FeatureIdentifier): void
+
+  getExperimentalFeatures(): FeatureIdentifier[]
+
+  getEnabledExperimentalFeatures(): FeatureIdentifier[]
+
+  enableExperimentalFeature(identifier: FeatureIdentifier): void
+
+  disableExperimentalFeature(identifier: FeatureIdentifier): void
+
+  isExperimentalFeatureEnabled(identifier: FeatureIdentifier): boolean
+
+  isExperimentalFeature(identifier: FeatureIdentifier): boolean
+}

packages/snjs/lib/Services/Features/FeaturesService.spec.ts

@@ -0,0 +1,799 @@
+import { ItemInterface, SNComponent, SNFeatureRepo } from '@standardnotes/models'
+import { SNSyncService } from '../Sync/SyncService'
+import { SettingName } from '@standardnotes/settings'
+import {
+  ItemManager,
+  AlertService,
+  SNApiService,
+  UserService,
+  SNSessionManager,
+  DiskStorageService,
+  StorageKey,
+} from '@Lib/index'
+import { FeatureStatus, SNFeaturesService } from '@Lib/Services/Features'
+import { ContentType, RoleName } from '@standardnotes/common'
+import { FeatureDescription, FeatureIdentifier, GetFeatures } from '@standardnotes/features'
+import { SNWebSocketsService } from '../Api/WebsocketsService'
+import { SNSettingsService } from '../Settings'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { convertTimestampToMilliseconds } from '@standardnotes/utils'
+import { InternalEventBusInterface } from '@standardnotes/services'
+
+describe('featuresService', () => {
+  let storageService: DiskStorageService
+  let apiService: SNApiService
+  let itemManager: ItemManager
+  let webSocketsService: SNWebSocketsService
+  let settingsService: SNSettingsService
+  let userService: UserService
+  let syncService: SNSyncService
+  let alertService: AlertService
+  let sessionManager: SNSessionManager
+  let crypto: PureCryptoInterface
+  let roles: RoleName[]
+  let features: FeatureDescription[]
+  let items: ItemInterface[]
+  let now: Date
+  let tomorrow_server: number
+  let tomorrow_client: number
+  let internalEventBus: InternalEventBusInterface
+  const expiredDate = new Date(new Date().getTime() - 1000).getTime()
+
+  const createService = () => {
+    return new SNFeaturesService(
+      storageService,
+      apiService,
+      itemManager,
+      webSocketsService,
+      settingsService,
+      userService,
+      syncService,
+      alertService,
+      sessionManager,
+      crypto,
+      internalEventBus,
+    )
+  }
+
+  beforeEach(() => {
+    roles = [RoleName.CoreUser, RoleName.PlusUser]
+
+    now = new Date()
+    tomorrow_client = now.setDate(now.getDate() + 1)
+    tomorrow_server = convertTimestampToMilliseconds(tomorrow_client * 1_000)
+
+    features = [
+      {
+        ...GetFeatures().find((f) => f.identifier === FeatureIdentifier.MidnightTheme),
+        expires_at: tomorrow_server,
+      },
+      {
+        ...GetFeatures().find((f) => f.identifier === FeatureIdentifier.PlusEditor),
+        expires_at: tomorrow_server,
+      },
+    ] as jest.Mocked<FeatureDescription[]>
+
+    items = [] as jest.Mocked<ItemInterface[]>
+
+    storageService = {} as jest.Mocked<DiskStorageService>
+    storageService.setValue = jest.fn()
+    storageService.getValue = jest.fn()
+
+    apiService = {} as jest.Mocked<SNApiService>
+    apiService.addEventObserver = jest.fn()
+    apiService.getUserFeatures = jest.fn().mockReturnValue({
+      data: {
+        features,
+      },
+    })
+    apiService.downloadOfflineFeaturesFromRepo = jest.fn().mockReturnValue({
+      features,
+    })
+    apiService.isThirdPartyHostUsed = jest.fn().mockReturnValue(false)
+
+    itemManager = {} as jest.Mocked<ItemManager>
+    itemManager.getItems = jest.fn().mockReturnValue(items)
+    itemManager.createItem = jest.fn()
+    itemManager.createTemplateItem = jest.fn().mockReturnValue({})
+    itemManager.changeComponent = jest.fn().mockReturnValue({} as jest.Mocked<ItemInterface>)
+    itemManager.setItemsToBeDeleted = jest.fn()
+    itemManager.addObserver = jest.fn()
+    itemManager.changeItem = jest.fn()
+    itemManager.changeFeatureRepo = jest.fn()
+
+    webSocketsService = {} as jest.Mocked<SNWebSocketsService>
+    webSocketsService.addEventObserver = jest.fn()
+
+    settingsService = {} as jest.Mocked<SNSettingsService>
+    settingsService.updateSetting = jest.fn()
+
+    userService = {} as jest.Mocked<UserService>
+    userService.addEventObserver = jest.fn()
+
+    syncService = {} as jest.Mocked<SNSyncService>
+    syncService.sync = jest.fn()
+
+    alertService = {} as jest.Mocked<AlertService>
+    alertService.confirm = jest.fn().mockReturnValue(true)
+    alertService.alert = jest.fn()
+
+    sessionManager = {} as jest.Mocked<SNSessionManager>
+    sessionManager.isSignedIntoFirstPartyServer = jest.fn()
+    sessionManager.getUser = jest.fn()
+
+    crypto = {} as jest.Mocked<PureCryptoInterface>
+    crypto.base64Decode = jest.fn()
+
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+  })
+
+  describe('experimental features', () => {
+    it('enables/disables an experimental feature', async () => {
+      storageService.getValue = jest.fn().mockReturnValue(GetFeatures())
+
+      const featuresService = createService()
+      featuresService.getExperimentalFeatures = jest.fn().mockReturnValue([FeatureIdentifier.PlusEditor])
+      featuresService.initializeFromDisk()
+
+      featuresService.enableExperimentalFeature(FeatureIdentifier.PlusEditor)
+
+      expect(featuresService.isExperimentalFeatureEnabled(FeatureIdentifier.PlusEditor)).toEqual(true)
+
+      featuresService.disableExperimentalFeature(FeatureIdentifier.PlusEditor)
+
+      expect(featuresService.isExperimentalFeatureEnabled(FeatureIdentifier.PlusEditor)).toEqual(false)
+    })
+
+    it('does not create a component for not enabled experimental feature', async () => {
+      const features = [
+        {
+          identifier: FeatureIdentifier.PlusEditor,
+          expires_at: tomorrow_server,
+          content_type: ContentType.Component,
+        },
+      ]
+
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features,
+        },
+      })
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+
+      const featuresService = createService()
+      featuresService.getExperimentalFeatures = jest.fn().mockReturnValue([FeatureIdentifier.PlusEditor])
+
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.createItem).not.toHaveBeenCalled()
+    })
+
+    it('does create a component for enabled experimental feature', async () => {
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features: GetFeatures(),
+        },
+      })
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+
+      const featuresService = createService()
+      featuresService.getExperimentalFeatures = jest.fn().mockReturnValue([FeatureIdentifier.PlusEditor])
+
+      featuresService.getEnabledExperimentalFeatures = jest.fn().mockReturnValue([FeatureIdentifier.PlusEditor])
+
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.createItem).toHaveBeenCalled()
+    })
+  })
+
+  describe('loadUserRoles()', () => {
+    it('retrieves user roles and features from storage', async () => {
+      await createService().initializeFromDisk()
+      expect(storageService.getValue).toHaveBeenCalledWith(StorageKey.UserRoles, undefined, [])
+      expect(storageService.getValue).toHaveBeenCalledWith(StorageKey.UserFeatures, undefined, [])
+    })
+  })
+
+  describe('updateRoles()', () => {
+    it('saves new roles to storage and fetches features if a role has been added', async () => {
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(storageService.setValue).toHaveBeenCalledWith(StorageKey.UserRoles, newRoles)
+      expect(apiService.getUserFeatures).toHaveBeenCalledWith('123')
+    })
+
+    it('saves new roles to storage and fetches features if a role has been removed', async () => {
+      const newRoles = [RoleName.CoreUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(storageService.setValue).toHaveBeenCalledWith(StorageKey.UserRoles, newRoles)
+      expect(apiService.getUserFeatures).toHaveBeenCalledWith('123')
+    })
+
+    it('saves features to storage when roles change', async () => {
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(storageService.setValue).toHaveBeenCalledWith(StorageKey.UserFeatures, features)
+    })
+
+    it('creates items for non-expired features with content type if they do not exist', async () => {
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.createItem).toHaveBeenCalledTimes(2)
+      expect(itemManager.createItem).toHaveBeenCalledWith(
+        ContentType.Theme,
+        expect.objectContaining({
+          package_info: expect.objectContaining({
+            content_type: ContentType.Theme,
+            expires_at: tomorrow_client,
+            identifier: FeatureIdentifier.MidnightTheme,
+          }),
+        }),
+        true,
+      )
+      expect(itemManager.createItem).toHaveBeenCalledWith(
+        ContentType.Component,
+        expect.objectContaining({
+          package_info: expect.objectContaining({
+            content_type: ContentType.Component,
+            expires_at: tomorrow_client,
+            identifier: FeatureIdentifier.PlusEditor,
+          }),
+        }),
+        true,
+      )
+    })
+
+    it('if item for a feature exists updates its content', async () => {
+      const existingItem = new SNComponent({
+        uuid: '789',
+        content_type: ContentType.Component,
+        content: {
+          package_info: {
+            identifier: FeatureIdentifier.PlusEditor,
+            valid_until: new Date(),
+          },
+        },
+      } as never)
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      itemManager.getItems = jest.fn().mockReturnValue([existingItem])
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+
+      expect(itemManager.changeComponent).toHaveBeenCalledWith(existingItem, expect.any(Function))
+    })
+
+    it('creates items for expired components if they do not exist', async () => {
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      const now = new Date()
+      const yesterday_client = now.setDate(now.getDate() - 1)
+      const yesterday_server = yesterday_client * 1_000
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features: [
+            {
+              ...features[1],
+              expires_at: yesterday_server,
+            },
+          ],
+        },
+      })
+
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.createItem).toHaveBeenCalledWith(
+        ContentType.Component,
+        expect.objectContaining({
+          package_info: expect.objectContaining({
+            content_type: ContentType.Component,
+            expires_at: yesterday_client,
+            identifier: FeatureIdentifier.PlusEditor,
+          }),
+        }),
+        true,
+      )
+    })
+
+    it('deletes items for expired themes', async () => {
+      const existingItem = new SNComponent({
+        uuid: '456',
+        content_type: ContentType.Theme,
+        content: {
+          package_info: {
+            identifier: FeatureIdentifier.MidnightTheme,
+            valid_until: new Date(),
+          },
+        },
+      } as never)
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      const now = new Date()
+      const yesterday = now.setDate(now.getDate() - 1)
+
+      itemManager.changeComponent = jest.fn().mockReturnValue(existingItem)
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      itemManager.getItems = jest.fn().mockReturnValue([existingItem])
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features: [
+            {
+              ...features[0],
+              expires_at: yesterday,
+            },
+          ],
+        },
+      })
+
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.setItemsToBeDeleted).toHaveBeenCalledWith([existingItem])
+    })
+
+    it('does not create an item for a feature without content type', async () => {
+      const features = [
+        {
+          identifier: FeatureIdentifier.TagNesting,
+          expires_at: tomorrow_server,
+        },
+      ]
+
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features,
+        },
+      })
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.createItem).not.toHaveBeenCalled()
+    })
+
+    it('does not create an item for deprecated features', async () => {
+      const features = [
+        {
+          identifier: FeatureIdentifier.DeprecatedBoldEditor,
+          expires_at: tomorrow_server,
+        },
+      ]
+
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features,
+        },
+      })
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(itemManager.createItem).not.toHaveBeenCalled()
+    })
+
+    it('does nothing after initial update if roles have not changed', async () => {
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      const featuresService = createService()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', roles)
+      await featuresService.updateRolesAndFetchFeatures('123', roles)
+      await featuresService.updateRolesAndFetchFeatures('123', roles)
+      await featuresService.updateRolesAndFetchFeatures('123', roles)
+      expect(storageService.setValue).toHaveBeenCalledTimes(2)
+    })
+
+    it('remote native features should be swapped with compiled version', async () => {
+      const remoteFeature = {
+        identifier: FeatureIdentifier.PlusEditor,
+        content_type: ContentType.Component,
+        expires_at: tomorrow_server,
+      } as FeatureDescription
+
+      const newRoles = [...roles, RoleName.PlusUser]
+
+      storageService.getValue = jest.fn().mockReturnValue(roles)
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features: [remoteFeature],
+        },
+      })
+
+      const featuresService = createService()
+      const nativeFeature = featuresService['mapRemoteNativeFeatureToStaticFeature'](remoteFeature)
+      featuresService['mapNativeFeatureToItem'] = jest.fn()
+      featuresService.initializeFromDisk()
+      await featuresService.updateRolesAndFetchFeatures('123', newRoles)
+      expect(featuresService['mapNativeFeatureToItem']).toHaveBeenCalledWith(
+        nativeFeature,
+        expect.anything(),
+        expect.anything(),
+      )
+    })
+
+    it('feature status', async () => {
+      const featuresService = createService()
+
+      features = [
+        {
+          identifier: FeatureIdentifier.MidnightTheme,
+          content_type: ContentType.Theme,
+          expires_at: tomorrow_server,
+          role_name: RoleName.PlusUser,
+        },
+        {
+          identifier: FeatureIdentifier.PlusEditor,
+          content_type: ContentType.Component,
+          expires_at: expiredDate,
+          role_name: RoleName.ProUser,
+        },
+      ] as jest.Mocked<FeatureDescription[]>
+
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features,
+        },
+      })
+
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(true)
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser, RoleName.PlusUser])
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.PlusEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.SheetsEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser])
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.NoUserSubscription)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.PlusEditor)).toBe(FeatureStatus.NoUserSubscription)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.SheetsEditor)).toBe(FeatureStatus.NoUserSubscription)
+
+      features = [
+        {
+          identifier: FeatureIdentifier.MidnightTheme,
+          content_type: ContentType.Theme,
+          expires_at: expiredDate,
+          role_name: RoleName.PlusUser,
+        },
+        {
+          identifier: FeatureIdentifier.PlusEditor,
+          content_type: ContentType.Component,
+          expires_at: expiredDate,
+          role_name: RoleName.ProUser,
+        },
+      ] as jest.Mocked<FeatureDescription[]>
+
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features,
+        },
+      })
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.PlusUser])
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(
+        FeatureStatus.InCurrentPlanButExpired,
+      )
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.PlusEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.SheetsEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+    })
+
+    it('third party feature status', async () => {
+      const featuresService = createService()
+
+      const themeFeature = {
+        identifier: 'third-party-theme' as FeatureIdentifier,
+        content_type: ContentType.Theme,
+        expires_at: tomorrow_server,
+        role_name: RoleName.CoreUser,
+      }
+
+      const editorFeature = {
+        identifier: 'third-party-editor' as FeatureIdentifier,
+        content_type: ContentType.Component,
+        expires_at: expiredDate,
+        role_name: RoleName.PlusUser,
+      }
+
+      features = [themeFeature, editorFeature] as jest.Mocked<FeatureDescription[]>
+
+      featuresService['features'] = features
+
+      itemManager.getDisplayableComponents = jest.fn().mockReturnValue([
+        new SNComponent({
+          uuid: '123',
+          content_type: ContentType.Theme,
+          content: {
+            valid_until: themeFeature.expires_at,
+            package_info: {
+              ...themeFeature,
+            },
+          },
+        } as never),
+        new SNComponent({
+          uuid: '456',
+          content_type: ContentType.Component,
+          content: {
+            valid_until: new Date(editorFeature.expires_at),
+            package_info: {
+              ...editorFeature,
+            },
+          },
+        } as never),
+      ])
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser])
+
+      expect(featuresService.getFeatureStatus(themeFeature.identifier)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(editorFeature.identifier)).toBe(FeatureStatus.InCurrentPlanButExpired)
+      expect(featuresService.getFeatureStatus('missing-feature-identifier' as FeatureIdentifier)).toBe(
+        FeatureStatus.NoUserSubscription,
+      )
+    })
+
+    it('feature status should be not entitled if no account or offline repo', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser])
+
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(false)
+
+      featuresService['completedSuccessfulFeaturesRetrieval'] = false
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.NoUserSubscription)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(
+        FeatureStatus.NoUserSubscription,
+      )
+    })
+
+    it('feature status should be entitled for subscriber until first successful features request made if no cached features', async () => {
+      const featuresService = createService()
+
+      apiService.getUserFeatures = jest.fn().mockReturnValue({
+        data: {
+          features: [],
+        },
+      })
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser, RoleName.PlusUser])
+
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(true)
+
+      featuresService['completedSuccessfulFeaturesRetrieval'] = false
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(FeatureStatus.Entitled)
+
+      await featuresService.didDownloadFeatures(features)
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+    })
+
+    it('feature status should be dynamic for subscriber if cached features and no successful features request made yet', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser, RoleName.PlusUser])
+
+      featuresService['completedSuccessfulFeaturesRetrieval'] = false
+
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(true)
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+
+      featuresService['completedSuccessfulFeaturesRetrieval'] = false
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+    })
+
+    it('feature status for offline subscription', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser, RoleName.PlusUser])
+
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(false)
+      featuresService.hasOnlineSubscription = jest.fn().mockReturnValue(false)
+      featuresService['completedSuccessfulFeaturesRetrieval'] = true
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.NoUserSubscription)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(
+        FeatureStatus.NoUserSubscription,
+      )
+
+      featuresService.hasOfflineRepo = jest.fn().mockReturnValue(true)
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.MidnightTheme)).toBe(FeatureStatus.Entitled)
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.TokenVaultEditor)).toBe(FeatureStatus.NotInCurrentPlan)
+    })
+
+    it('feature status for deprecated feature', async () => {
+      const featuresService = createService()
+
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(true)
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.DeprecatedFileSafe as FeatureIdentifier)).toBe(
+        FeatureStatus.NoUserSubscription,
+      )
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser, RoleName.PlusUser])
+
+      expect(featuresService.getFeatureStatus(FeatureIdentifier.DeprecatedFileSafe as FeatureIdentifier)).toBe(
+        FeatureStatus.Entitled,
+      )
+    })
+
+    it('has paid subscription', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser])
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(true)
+
+      expect(featuresService.hasPaidOnlineOrOfflineSubscription()).toBeFalsy
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser, RoleName.PlusUser])
+
+      expect(featuresService.hasPaidOnlineOrOfflineSubscription()).toEqual(true)
+    })
+
+    it('has paid subscription should be true if offline repo and signed into third party server', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser])
+
+      featuresService.hasOfflineRepo = jest.fn().mockReturnValue(true)
+      sessionManager.isSignedIntoFirstPartyServer = jest.fn().mockReturnValue(false)
+
+      expect(featuresService.hasPaidOnlineOrOfflineSubscription()).toEqual(true)
+    })
+  })
+
+  describe('migrateFeatureRepoToUserSetting', () => {
+    it('should extract key from extension repo url and update user setting', async () => {
+      const extensionKey = '129b029707e3470c94a8477a437f9394'
+      const extensionRepoItem = new SNFeatureRepo({
+        uuid: '456',
+        content_type: ContentType.ExtensionRepo,
+        content: {
+          url: `https://extensions.standardnotes.org/${extensionKey}`,
+        },
+      } as never)
+
+      const featuresService = createService()
+      await featuresService.migrateFeatureRepoToUserSetting([extensionRepoItem])
+      expect(settingsService.updateSetting).toHaveBeenCalledWith(SettingName.ExtensionKey, extensionKey, true)
+    })
+  })
+
+  describe('downloadExternalFeature', () => {
+    it('should not allow if identifier matches native identifier', async () => {
+      apiService.downloadFeatureUrl = jest.fn().mockReturnValue({
+        data: {
+          identifier: 'org.standardnotes.bold-editor',
+          name: 'Bold Editor',
+          content_type: 'SN|Component',
+          area: 'editor-editor',
+          version: '1.0.0',
+          url: 'http://localhost:8005/',
+        },
+      })
+
+      const installUrl = 'http://example.com'
+      crypto.base64Decode = jest.fn().mockReturnValue(installUrl)
+
+      const featuresService = createService()
+      const result = await featuresService.downloadExternalFeature(installUrl)
+      expect(result).toBeUndefined()
+    })
+
+    it('should not allow if url matches native url', async () => {
+      apiService.downloadFeatureUrl = jest.fn().mockReturnValue({
+        data: {
+          identifier: 'org.foo.bar',
+          name: 'Bold Editor',
+          content_type: 'SN|Component',
+          area: 'editor-editor',
+          version: '1.0.0',
+          url: 'http://localhost:8005/org.standardnotes.bold-editor/index.html',
+        },
+      })
+
+      const installUrl = 'http://example.com'
+      crypto.base64Decode = jest.fn().mockReturnValue(installUrl)
+
+      const featuresService = createService()
+      const result = await featuresService.downloadExternalFeature(installUrl)
+      expect(result).toBeUndefined()
+    })
+  })
+
+  describe('sortRolesByHierarchy', () => {
+    it('should sort given roles according to role hierarchy', () => {
+      const featuresService = createService()
+
+      const sortedRoles = featuresService.rolesBySorting([RoleName.ProUser, RoleName.CoreUser, RoleName.PlusUser])
+
+      expect(sortedRoles).toStrictEqual([RoleName.CoreUser, RoleName.PlusUser, RoleName.ProUser])
+    })
+  })
+
+  describe('hasMinimumRole', () => {
+    it('should be false if core user checks for plus role', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.CoreUser])
+
+      const hasPlusUserRole = featuresService.hasMinimumRole(RoleName.PlusUser)
+
+      expect(hasPlusUserRole).toBe(false)
+    })
+
+    it('should be false if plus user checks for pro role', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.PlusUser, RoleName.CoreUser])
+
+      const hasProUserRole = featuresService.hasMinimumRole(RoleName.ProUser)
+
+      expect(hasProUserRole).toBe(false)
+    })
+
+    it('should be true if pro user checks for core user', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.ProUser, RoleName.PlusUser])
+
+      const hasCoreUserRole = featuresService.hasMinimumRole(RoleName.CoreUser)
+
+      expect(hasCoreUserRole).toBe(true)
+    })
+
+    it('should be true if pro user checks for pro user', async () => {
+      const featuresService = createService()
+
+      await featuresService.updateRolesAndFetchFeatures('123', [RoleName.ProUser, RoleName.PlusUser])
+
+      const hasProUserRole = featuresService.hasMinimumRole(RoleName.ProUser)
+
+      expect(hasProUserRole).toBe(true)
+    })
+  })
+})

packages/snjs/lib/Services/Features/FeaturesService.ts

@@ -0,0 +1,718 @@
+import { AccountEvent, UserService } from '../User/UserService'
+import { SNApiService } from '../Api/ApiService'
+import {
+  arraysEqual,
+  convertTimestampToMilliseconds,
+  removeFromArray,
+  Copy,
+  lastElement,
+  isString,
+} from '@standardnotes/utils'
+import { ClientDisplayableError, UserFeaturesResponse } from '@standardnotes/responses'
+import { ContentType, RoleName } from '@standardnotes/common'
+import { FeaturesClientInterface } from './ClientInterface'
+import { FillItemContent, PayloadEmitSource } from '@standardnotes/models'
+import { ItemManager } from '../Items/ItemManager'
+import { LEGACY_PROD_EXT_ORIGIN, PROD_OFFLINE_FEATURES_URL } from '../../Hosts'
+import { SettingName } from '@standardnotes/settings'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { SNSessionManager } from '@Lib/Services/Session/SessionManager'
+import { SNSettingsService } from '../Settings'
+import { DiskStorageService } from '../Storage/DiskStorageService'
+import { SNSyncService } from '../Sync/SyncService'
+import { SNWebSocketsService, WebSocketsServiceEvent } from '../Api/WebsocketsService'
+import { TRUSTED_CUSTOM_EXTENSIONS_HOSTS, TRUSTED_FEATURE_HOSTS } from '@Lib/Hosts'
+import { UserRolesChangedEvent } from '@standardnotes/domain-events'
+import { UuidString } from '@Lib/Types/UuidString'
+import * as FeaturesImports from '@standardnotes/features'
+import * as Messages from '@Lib/Services/Api/Messages'
+import * as Models from '@standardnotes/models'
+import * as Services from '@standardnotes/services'
+import {
+  FeaturesEvent,
+  FeatureStatus,
+  OfflineSubscriptionEntitlements,
+  SetOfflineFeaturesFunctionResponse,
+} from './Types'
+import { DiagnosticInfo } from '@standardnotes/services'
+
+type GetOfflineSubscriptionDetailsResponse = OfflineSubscriptionEntitlements | ClientDisplayableError
+
+export class SNFeaturesService
+  extends Services.AbstractService<FeaturesEvent>
+  implements FeaturesClientInterface, Services.InternalEventHandlerInterface
+{
+  private deinited = false
+  private roles: RoleName[] = []
+  private features: FeaturesImports.FeatureDescription[] = []
+  private enabledExperimentalFeatures: FeaturesImports.FeatureIdentifier[] = []
+  private removeWebSocketsServiceObserver: () => void
+  private removefeatureReposObserver: () => void
+  private removeSignInObserver: () => void
+  private needsInitialFeaturesUpdate = true
+  private completedSuccessfulFeaturesRetrieval = false
+
+  constructor(
+    private storageService: DiskStorageService,
+    private apiService: SNApiService,
+    private itemManager: ItemManager,
+    private webSocketsService: SNWebSocketsService,
+    private settingsService: SNSettingsService,
+    private userService: UserService,
+    private syncService: SNSyncService,
+    private alertService: Services.AlertService,
+    private sessionManager: SNSessionManager,
+    private crypto: PureCryptoInterface,
+    protected override internalEventBus: Services.InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+
+    this.removeWebSocketsServiceObserver = webSocketsService.addEventObserver(async (eventName, data) => {
+      if (eventName === WebSocketsServiceEvent.UserRoleMessageReceived) {
+        const {
+          payload: { userUuid, currentRoles },
+        } = data as UserRolesChangedEvent
+        await this.updateRolesAndFetchFeatures(userUuid, currentRoles)
+      }
+    })
+
+    this.removefeatureReposObserver = this.itemManager.addObserver(
+      ContentType.ExtensionRepo,
+      async ({ changed, inserted, source }) => {
+        const sources = [
+          PayloadEmitSource.InitialObserverRegistrationPush,
+          PayloadEmitSource.LocalInserted,
+          PayloadEmitSource.LocalDatabaseLoaded,
+          PayloadEmitSource.RemoteRetrieved,
+          PayloadEmitSource.FileImport,
+        ]
+
+        if (sources.includes(source)) {
+          const items = [...changed, ...inserted] as Models.SNFeatureRepo[]
+          if (this.sessionManager.isSignedIntoFirstPartyServer()) {
+            await this.migrateFeatureRepoToUserSetting(items)
+          } else {
+            await this.migrateFeatureRepoToOfflineEntitlements(items)
+          }
+        }
+      },
+    )
+
+    this.removeSignInObserver = this.userService.addEventObserver((eventName: AccountEvent) => {
+      if (eventName === AccountEvent.SignedInOrRegistered) {
+        const featureRepos = this.itemManager.getItems(ContentType.ExtensionRepo) as Models.SNFeatureRepo[]
+
+        if (!this.apiService.isThirdPartyHostUsed()) {
+          void this.migrateFeatureRepoToUserSetting(featureRepos)
+        }
+      }
+    })
+  }
+
+  async handleEvent(event: Services.InternalEventInterface): Promise<void> {
+    if (event.type === Services.ApiServiceEvent.MetaReceived) {
+      if (!this.syncService) {
+        this.log('[Features Service] Handling events interrupted. Sync service is not yet initialized.', event)
+
+        return
+      }
+
+      /**
+       * All user data must be downloaded before we map features. Otherwise, feature mapping
+       * may think a component doesn't exist and create a new one, when in reality the component
+       * already exists but hasn't been downloaded yet.
+       */
+      if (!this.syncService.completedOnlineDownloadFirstSync) {
+        return
+      }
+
+      const { userUuid, userRoles } = event.payload as Services.MetaReceivedData
+      await this.updateRolesAndFetchFeatures(
+        userUuid,
+        userRoles.map((role) => role.name),
+      )
+    }
+  }
+
+  override async handleApplicationStage(stage: Services.ApplicationStage): Promise<void> {
+    await super.handleApplicationStage(stage)
+    if (stage === Services.ApplicationStage.FullSyncCompleted_13) {
+      if (!this.hasOnlineSubscription()) {
+        const offlineRepo = this.getOfflineRepo()
+        if (offlineRepo) {
+          void this.downloadOfflineFeatures(offlineRepo)
+        }
+      }
+    }
+  }
+
+  public enableExperimentalFeature(identifier: FeaturesImports.FeatureIdentifier): void {
+    const feature = this.getUserFeature(identifier)
+    if (!feature) {
+      throw Error('Attempting to enable a feature user does not have access to.')
+    }
+
+    this.enabledExperimentalFeatures.push(identifier)
+
+    void this.storageService.setValue(Services.StorageKey.ExperimentalFeatures, this.enabledExperimentalFeatures)
+
+    void this.mapRemoteNativeFeaturesToItems([feature])
+    void this.notifyEvent(FeaturesEvent.FeaturesUpdated)
+  }
+
+  public disableExperimentalFeature(identifier: FeaturesImports.FeatureIdentifier): void {
+    const feature = this.getUserFeature(identifier)
+    if (!feature) {
+      throw Error('Attempting to disable a feature user does not have access to.')
+    }
+
+    removeFromArray(this.enabledExperimentalFeatures, identifier)
+
+    void this.storageService.setValue(Services.StorageKey.ExperimentalFeatures, this.enabledExperimentalFeatures)
+
+    const component = this.itemManager
+      .getItems<Models.SNComponent | Models.SNTheme>([ContentType.Component, ContentType.Theme])
+      .find((component) => component.identifier === identifier)
+    if (!component) {
+      return
+    }
+
+    void this.itemManager.setItemToBeDeleted(component).then(() => {
+      void this.syncService.sync()
+    })
+    void this.notifyEvent(FeaturesEvent.FeaturesUpdated)
+  }
+
+  public toggleExperimentalFeature(identifier: FeaturesImports.FeatureIdentifier): void {
+    if (this.isExperimentalFeatureEnabled(identifier)) {
+      this.disableExperimentalFeature(identifier)
+    } else {
+      this.enableExperimentalFeature(identifier)
+    }
+  }
+
+  public getExperimentalFeatures(): FeaturesImports.FeatureIdentifier[] {
+    return FeaturesImports.ExperimentalFeatures
+  }
+
+  public isExperimentalFeature(featureId: FeaturesImports.FeatureIdentifier): boolean {
+    return this.getExperimentalFeatures().includes(featureId)
+  }
+
+  public getEnabledExperimentalFeatures(): FeaturesImports.FeatureIdentifier[] {
+    return this.enabledExperimentalFeatures
+  }
+
+  public isExperimentalFeatureEnabled(featureId: FeaturesImports.FeatureIdentifier): boolean {
+    return this.enabledExperimentalFeatures.includes(featureId)
+  }
+
+  public async setOfflineFeaturesCode(code: string): Promise<SetOfflineFeaturesFunctionResponse> {
+    try {
+      const activationCodeWithoutSpaces = code.replace(/\s/g, '')
+      const decodedData = this.crypto.base64Decode(activationCodeWithoutSpaces)
+      const result = this.parseOfflineEntitlementsCode(decodedData)
+
+      if (result instanceof ClientDisplayableError) {
+        return result
+      }
+
+      const offlineRepo = (await this.itemManager.createItem(
+        ContentType.ExtensionRepo,
+        FillItemContent({
+          offlineFeaturesUrl: result.featuresUrl,
+          offlineKey: result.extensionKey,
+          migratedToOfflineEntitlements: true,
+        } as Models.FeatureRepoContent),
+        true,
+      )) as Models.SNFeatureRepo
+      void this.syncService.sync()
+      return this.downloadOfflineFeatures(offlineRepo)
+    } catch (err) {
+      return new ClientDisplayableError(Messages.API_MESSAGE_FAILED_OFFLINE_ACTIVATION)
+    }
+  }
+
+  private getOfflineRepo(): Models.SNFeatureRepo | undefined {
+    const repos = this.itemManager.getItems(ContentType.ExtensionRepo) as Models.SNFeatureRepo[]
+    return repos.filter((repo) => repo.migratedToOfflineEntitlements)[0]
+  }
+
+  public hasOfflineRepo(): boolean {
+    return this.getOfflineRepo() != undefined
+  }
+
+  public async deleteOfflineFeatureRepo(): Promise<void> {
+    const repo = this.getOfflineRepo()
+    if (repo) {
+      await this.itemManager.setItemToBeDeleted(repo)
+      void this.syncService.sync()
+    }
+    await this.storageService.removeValue(Services.StorageKey.UserFeatures)
+  }
+
+  private parseOfflineEntitlementsCode(code: string): GetOfflineSubscriptionDetailsResponse | ClientDisplayableError {
+    try {
+      const { featuresUrl, extensionKey } = JSON.parse(code)
+      return {
+        featuresUrl,
+        extensionKey,
+      }
+    } catch (error) {
+      return new ClientDisplayableError(Messages.API_MESSAGE_FAILED_OFFLINE_ACTIVATION)
+    }
+  }
+
+  private async downloadOfflineFeatures(
+    repo: Models.SNFeatureRepo,
+  ): Promise<SetOfflineFeaturesFunctionResponse | ClientDisplayableError> {
+    const result = await this.apiService.downloadOfflineFeaturesFromRepo(repo)
+    if (result instanceof ClientDisplayableError) {
+      return result
+    }
+    await this.didDownloadFeatures(result.features)
+    return undefined
+  }
+
+  public async migrateFeatureRepoToUserSetting(featureRepos: Models.SNFeatureRepo[] = []): Promise<void> {
+    for (const item of featureRepos) {
+      if (item.migratedToUserSetting) {
+        continue
+      }
+      if (item.onlineUrl) {
+        const repoUrl: string = item.onlineUrl
+        const userKeyMatch = repoUrl.match(/\w{32,64}/)
+        if (userKeyMatch && userKeyMatch.length > 0) {
+          const userKey = userKeyMatch[0]
+          await this.settingsService.updateSetting(SettingName.ExtensionKey, userKey, true)
+          await this.itemManager.changeFeatureRepo(item, (m) => {
+            m.migratedToUserSetting = true
+          })
+        }
+      }
+    }
+  }
+
+  public async migrateFeatureRepoToOfflineEntitlements(featureRepos: Models.SNFeatureRepo[] = []): Promise<void> {
+    for (const item of featureRepos) {
+      if (item.migratedToOfflineEntitlements) {
+        continue
+      }
+
+      if (item.onlineUrl) {
+        const repoUrl = item.onlineUrl
+        const { origin } = new URL(repoUrl)
+
+        if (!origin.includes(LEGACY_PROD_EXT_ORIGIN)) {
+          continue
+        }
+
+        const userKeyMatch = repoUrl.match(/\w{32,64}/)
+        if (userKeyMatch && userKeyMatch.length > 0) {
+          const userKey = userKeyMatch[0]
+          const updatedRepo = await this.itemManager.changeFeatureRepo(item, (m) => {
+            m.offlineFeaturesUrl = PROD_OFFLINE_FEATURES_URL
+            m.offlineKey = userKey
+            m.migratedToOfflineEntitlements = true
+          })
+          await this.downloadOfflineFeatures(updatedRepo)
+        }
+      }
+    }
+  }
+
+  public initializeFromDisk(): void {
+    this.roles = this.storageService.getValue<RoleName[]>(Services.StorageKey.UserRoles, undefined, [])
+
+    this.features = this.storageService.getValue(Services.StorageKey.UserFeatures, undefined, [])
+
+    this.enabledExperimentalFeatures = this.storageService.getValue(
+      Services.StorageKey.ExperimentalFeatures,
+      undefined,
+      [],
+    )
+  }
+
+  public async updateRolesAndFetchFeatures(userUuid: UuidString, roles: RoleName[]): Promise<void> {
+    const userRolesChanged = this.haveRolesChanged(roles)
+
+    if (!userRolesChanged && !this.needsInitialFeaturesUpdate) {
+      return
+    }
+
+    this.needsInitialFeaturesUpdate = false
+
+    await this.setRoles(roles)
+
+    const shouldDownloadRoleBasedFeatures = !this.hasOfflineRepo()
+
+    if (shouldDownloadRoleBasedFeatures) {
+      const featuresResponse = await this.apiService.getUserFeatures(userUuid)
+
+      if (!featuresResponse.error && featuresResponse.data && !this.deinited) {
+        const features = (featuresResponse as UserFeaturesResponse).data.features
+        await this.didDownloadFeatures(features)
+      }
+    }
+  }
+
+  private async setRoles(roles: RoleName[]): Promise<void> {
+    this.roles = roles
+    if (!arraysEqual(this.roles, roles)) {
+      void this.notifyEvent(FeaturesEvent.UserRolesChanged)
+    }
+    await this.storageService.setValue(Services.StorageKey.UserRoles, this.roles)
+  }
+
+  public async didDownloadFeatures(features: FeaturesImports.FeatureDescription[]): Promise<void> {
+    features = features
+      .filter((feature) => !!FeaturesImports.FindNativeFeature(feature.identifier))
+      .map((feature) => this.mapRemoteNativeFeatureToStaticFeature(feature))
+
+    this.features = features
+    this.completedSuccessfulFeaturesRetrieval = true
+    void this.notifyEvent(FeaturesEvent.FeaturesUpdated)
+    void this.storageService.setValue(Services.StorageKey.UserFeatures, this.features)
+
+    await this.mapRemoteNativeFeaturesToItems(features)
+  }
+
+  public isThirdPartyFeature(identifier: string): boolean {
+    const isNativeFeature = !!FeaturesImports.FindNativeFeature(identifier as FeaturesImports.FeatureIdentifier)
+    return !isNativeFeature
+  }
+
+  private mapRemoteNativeFeatureToStaticFeature(
+    remoteFeature: FeaturesImports.FeatureDescription,
+  ): FeaturesImports.FeatureDescription {
+    const remoteFields: (keyof FeaturesImports.FeatureDescription)[] = [
+      'expires_at',
+      'role_name',
+      'no_expire',
+      'permission_name',
+    ]
+
+    const nativeFeature = FeaturesImports.FindNativeFeature(remoteFeature.identifier)
+    if (!nativeFeature) {
+      throw Error(`Attempting to map remote native to unfound static feature ${remoteFeature.identifier}`)
+    }
+
+    const nativeFeatureCopy = Copy(nativeFeature) as FeaturesImports.FeatureDescription
+
+    for (const field of remoteFields) {
+      nativeFeatureCopy[field] = remoteFeature[field] as never
+    }
+
+    if (nativeFeatureCopy.expires_at) {
+      nativeFeatureCopy.expires_at = convertTimestampToMilliseconds(nativeFeatureCopy.expires_at)
+    }
+    return nativeFeatureCopy
+  }
+
+  public getUserFeature(featureId: FeaturesImports.FeatureIdentifier): FeaturesImports.FeatureDescription | undefined {
+    return this.features.find((feature) => feature.identifier === featureId)
+  }
+
+  hasOnlineSubscription(): boolean {
+    const roles = this.roles
+    const unpaidRoles = [RoleName.CoreUser]
+    return roles.some((role) => !unpaidRoles.includes(role))
+  }
+
+  public hasPaidOnlineOrOfflineSubscription(): boolean {
+    return this.hasOnlineSubscription() || this.hasOfflineRepo()
+  }
+
+  public rolesBySorting(roles: RoleName[]): RoleName[] {
+    return Object.values(RoleName).filter((role) => roles.includes(role))
+  }
+
+  public hasMinimumRole(role: RoleName): boolean {
+    const sortedAllRoles = Object.values(RoleName)
+
+    const sortedUserRoles = this.rolesBySorting(this.roles)
+
+    const highestUserRoleIndex = sortedAllRoles.indexOf(lastElement(sortedUserRoles) as RoleName)
+
+    const indexOfRoleToCheck = sortedAllRoles.indexOf(role)
+
+    return indexOfRoleToCheck <= highestUserRoleIndex
+  }
+
+  public isFeatureDeprecated(featureId: FeaturesImports.FeatureIdentifier): boolean {
+    return FeaturesImports.FindNativeFeature(featureId)?.deprecated === true
+  }
+
+  public getFeatureStatus(featureId: FeaturesImports.FeatureIdentifier): FeatureStatus {
+    const isDeprecated = this.isFeatureDeprecated(featureId)
+    if (isDeprecated) {
+      if (this.hasPaidOnlineOrOfflineSubscription()) {
+        return FeatureStatus.Entitled
+      } else {
+        return FeatureStatus.NoUserSubscription
+      }
+    }
+
+    const isThirdParty = FeaturesImports.FindNativeFeature(featureId) == undefined
+    if (isThirdParty) {
+      const component = this.itemManager
+        .getDisplayableComponents()
+        .find((candidate) => candidate.identifier === featureId)
+      if (!component) {
+        return FeatureStatus.NoUserSubscription
+      }
+      if (component.isExpired) {
+        return FeatureStatus.InCurrentPlanButExpired
+      }
+      return FeatureStatus.Entitled
+    }
+
+    if (this.hasPaidOnlineOrOfflineSubscription()) {
+      if (!this.completedSuccessfulFeaturesRetrieval) {
+        const hasCachedFeatures = this.features.length > 0
+        const temporarilyAllowUntilServerUpdates = !hasCachedFeatures
+        if (temporarilyAllowUntilServerUpdates) {
+          return FeatureStatus.Entitled
+        }
+      }
+    } else {
+      return FeatureStatus.NoUserSubscription
+    }
+
+    const feature = this.getUserFeature(featureId)
+    if (!feature) {
+      return FeatureStatus.NotInCurrentPlan
+    }
+
+    const expired = feature.expires_at && new Date(feature.expires_at).getTime() < new Date().getTime()
+    if (expired) {
+      if (!this.roles.includes(feature.role_name as RoleName)) {
+        return FeatureStatus.NotInCurrentPlan
+      } else {
+        return FeatureStatus.InCurrentPlanButExpired
+      }
+    }
+
+    return FeatureStatus.Entitled
+  }
+
+  private haveRolesChanged(roles: RoleName[]): boolean {
+    return roles.some((role) => !this.roles.includes(role)) || this.roles.some((role) => !roles.includes(role))
+  }
+
+  private componentContentForNativeFeatureDescription(feature: FeaturesImports.FeatureDescription): Models.ItemContent {
+    const componentContent: Partial<Models.ComponentContent> = {
+      area: feature.area,
+      name: feature.name,
+      package_info: feature,
+      valid_until: new Date(feature.expires_at || 0),
+    }
+    return FillItemContent(componentContent)
+  }
+
+  private async mapRemoteNativeFeaturesToItems(features: FeaturesImports.FeatureDescription[]): Promise<void> {
+    const currentItems = this.itemManager.getItems<Models.SNComponent>([ContentType.Component, ContentType.Theme])
+    const itemsToDelete: Models.SNComponent[] = []
+    let hasChanges = false
+
+    for (const feature of features) {
+      const didChange = await this.mapNativeFeatureToItem(feature, currentItems, itemsToDelete)
+      if (didChange) {
+        hasChanges = true
+      }
+    }
+
+    await this.itemManager.setItemsToBeDeleted(itemsToDelete)
+    if (hasChanges) {
+      void this.syncService.sync()
+    }
+  }
+
+  private async mapNativeFeatureToItem(
+    feature: FeaturesImports.FeatureDescription,
+    currentItems: Models.SNComponent[],
+    itemsToDelete: Models.SNComponent[],
+  ): Promise<boolean> {
+    if (!feature.content_type) {
+      return false
+    }
+
+    if (this.isExperimentalFeature(feature.identifier) && !this.isExperimentalFeatureEnabled(feature.identifier)) {
+      return false
+    }
+
+    let hasChanges = false
+    const now = new Date()
+    const expired = new Date(feature.expires_at || 0).getTime() < now.getTime()
+
+    const existingItem = currentItems.find((item) => {
+      if (item.content.package_info) {
+        const itemIdentifier = item.content.package_info.identifier
+        return itemIdentifier === feature.identifier
+      }
+      return false
+    })
+
+    if (feature.deprecated && !existingItem) {
+      return false
+    }
+
+    let resultingItem: Models.SNComponent | undefined = existingItem
+
+    if (existingItem) {
+      const featureExpiresAt = new Date(feature.expires_at || 0)
+      const hasChange =
+        JSON.stringify(feature) !== JSON.stringify(existingItem.package_info) ||
+        featureExpiresAt.getTime() !== existingItem.valid_until.getTime()
+      if (hasChange) {
+        resultingItem = await this.itemManager.changeComponent(existingItem, (mutator) => {
+          mutator.package_info = feature
+          mutator.valid_until = featureExpiresAt
+        })
+        hasChanges = true
+      } else {
+        resultingItem = existingItem
+      }
+    } else if (!expired || feature.content_type === ContentType.Component) {
+      resultingItem = (await this.itemManager.createItem(
+        feature.content_type,
+        this.componentContentForNativeFeatureDescription(feature),
+        true,
+      )) as Models.SNComponent
+      hasChanges = true
+    }
+
+    if (expired && resultingItem) {
+      if (feature.content_type !== ContentType.Component) {
+        itemsToDelete.push(resultingItem)
+        hasChanges = true
+      }
+    }
+
+    return hasChanges
+  }
+
+  public async downloadExternalFeature(urlOrCode: string): Promise<Models.SNComponent | undefined> {
+    let url = urlOrCode
+    try {
+      url = this.crypto.base64Decode(urlOrCode)
+      // eslint-disable-next-line no-empty
+    } catch (err) {}
+
+    try {
+      const trustedCustomExtensionsUrls = [...TRUSTED_FEATURE_HOSTS, ...TRUSTED_CUSTOM_EXTENSIONS_HOSTS]
+      const { host } = new URL(url)
+      if (!trustedCustomExtensionsUrls.includes(host)) {
+        const didConfirm = await this.alertService.confirm(
+          Messages.API_MESSAGE_UNTRUSTED_EXTENSIONS_WARNING,
+          'Install extension from an untrusted source?',
+          'Proceed to install',
+          Services.ButtonType.Danger,
+          'Cancel',
+        )
+        if (didConfirm) {
+          return this.performDownloadExternalFeature(url)
+        }
+      } else {
+        return this.performDownloadExternalFeature(url)
+      }
+    } catch (err) {
+      void this.alertService.alert(Messages.INVALID_EXTENSION_URL)
+    }
+
+    return undefined
+  }
+
+  private async performDownloadExternalFeature(url: string): Promise<Models.SNComponent | undefined> {
+    const response = await this.apiService.downloadFeatureUrl(url)
+    if (response.error) {
+      await this.alertService.alert(Messages.API_MESSAGE_FAILED_DOWNLOADING_EXTENSION)
+      return undefined
+    }
+
+    let rawFeature = response.data as FeaturesImports.ThirdPartyFeatureDescription
+
+    if (isString(rawFeature)) {
+      try {
+        rawFeature = JSON.parse(rawFeature)
+        // eslint-disable-next-line no-empty
+      } catch (error) {}
+    }
+
+    if (!rawFeature.content_type) {
+      return
+    }
+
+    const isValidContentType = [
+      ContentType.Component,
+      ContentType.Theme,
+      ContentType.ActionsExtension,
+      ContentType.ExtensionRepo,
+    ].includes(rawFeature.content_type)
+
+    if (!isValidContentType) {
+      return
+    }
+
+    const nativeFeature = FeaturesImports.FindNativeFeature(rawFeature.identifier)
+    if (nativeFeature) {
+      await this.alertService.alert(Messages.API_MESSAGE_FAILED_DOWNLOADING_EXTENSION)
+      return
+    }
+
+    if (rawFeature.url) {
+      for (const nativeFeature of FeaturesImports.GetFeatures()) {
+        if (rawFeature.url.includes(nativeFeature.identifier)) {
+          await this.alertService.alert(Messages.API_MESSAGE_FAILED_DOWNLOADING_EXTENSION)
+          return
+        }
+      }
+    }
+
+    const content = FillItemContent({
+      area: rawFeature.area,
+      name: rawFeature.name,
+      package_info: rawFeature,
+      valid_until: new Date(rawFeature.expires_at || 0),
+      hosted_url: rawFeature.url,
+    } as Partial<Models.ComponentContent>)
+
+    const component = this.itemManager.createTemplateItem(rawFeature.content_type, content) as Models.SNComponent
+
+    return component
+  }
+
+  override deinit(): void {
+    super.deinit()
+    this.removeSignInObserver()
+    ;(this.removeSignInObserver as unknown) = undefined
+    this.removeWebSocketsServiceObserver()
+    ;(this.removeWebSocketsServiceObserver as unknown) = undefined
+    this.removefeatureReposObserver()
+    ;(this.removefeatureReposObserver as unknown) = undefined
+    ;(this.roles as unknown) = undefined
+    ;(this.storageService as unknown) = undefined
+    ;(this.apiService as unknown) = undefined
+    ;(this.itemManager as unknown) = undefined
+    ;(this.webSocketsService as unknown) = undefined
+    ;(this.settingsService as unknown) = undefined
+    ;(this.userService as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    ;(this.alertService as unknown) = undefined
+    ;(this.sessionManager as unknown) = undefined
+    ;(this.crypto as unknown) = undefined
+    this.deinited = true
+  }
+
+  override getDiagnostics(): Promise<DiagnosticInfo | undefined> {
+    return Promise.resolve({
+      features: {
+        roles: this.roles,
+        features: this.features,
+        enabledExperimentalFeatures: this.enabledExperimentalFeatures,
+        needsInitialFeaturesUpdate: this.needsInitialFeaturesUpdate,
+        completedSuccessfulFeaturesRetrieval: this.completedSuccessfulFeaturesRetrieval,
+      },
+    })
+  }
+}

packages/snjs/lib/Services/Features/Types.ts

@@ -0,0 +1,20 @@
+import { ClientDisplayableError } from '@standardnotes/responses'
+
+export type SetOfflineFeaturesFunctionResponse = ClientDisplayableError | undefined
+
+export type OfflineSubscriptionEntitlements = {
+  featuresUrl: string
+  extensionKey: string
+}
+
+export enum FeaturesEvent {
+  UserRolesChanged = 'UserRolesChanged',
+  FeaturesUpdated = 'FeaturesUpdated',
+}
+
+export enum FeatureStatus {
+  NoUserSubscription = 'NoUserSubscription',
+  NotInCurrentPlan = 'NotInCurrentPlan',
+  InCurrentPlanButExpired = 'InCurrentPlanButExpired',
+  Entitled = 'Entitled',
+}

packages/snjs/lib/Services/Features/index.ts

@@ -0,0 +1,3 @@
+export * from './ClientInterface'
+export * from './FeaturesService'
+export * from './Types'

packages/snjs/lib/Services/History/HistoryManager.ts

@@ -0,0 +1,249 @@
+import { ContentType, Uuid } from '@standardnotes/common'
+import { EncryptionService } from '@standardnotes/encryption'
+import { isNullOrUndefined, removeFromArray } from '@standardnotes/utils'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import { SNApiService } from '@Lib/Services/Api/ApiService'
+import { DiskStorageService } from '@Lib/Services/Storage/DiskStorageService'
+import { UuidString } from '../../Types/UuidString'
+import * as Models from '@standardnotes/models'
+import * as Responses from '@standardnotes/responses'
+import * as Services from '@standardnotes/services'
+import { isErrorDecryptingPayload, PayloadTimestampDefaults, SNNote } from '@standardnotes/models'
+
+/** The amount of revisions per item above which should call for an optimization. */
+const DefaultItemRevisionsThreshold = 20
+
+/**
+ * The amount of characters added or removed that
+ * constitute a keepable entry after optimization.
+ */
+const LargeEntryDeltaThreshold = 25
+
+/**
+ * The history manager is responsible for:
+ * 1. Transient session history, which include keeping track of changes made in the
+ *    current application session. These change logs (unless otherwise configured) are
+ *    ephemeral and do not persist past application restart. Session history entries are
+ *    added via change observers that trigger when an item changes.
+ * 2. Remote server history. Entries are automatically added by the server and must be
+ *    retrieved per item via an API call.
+ */
+export class SNHistoryManager extends Services.AbstractService {
+  private removeChangeObserver: () => void
+
+  /**
+   * When no history exists for an item yet, we first put it in the staging map.
+   * Then, the next time the item changes and it has no history, we check the staging map.
+   * If the entry from the staging map differs from the incoming change, we now add the incoming
+   * change to the history map and remove it from staging. This is a way to detect when the first
+   * actual change of an item occurs (especially new items), rather than tracking a change
+   * as an item propagating through the different PayloadSource
+   * lifecycles (created, local saved, presyncsave, etc)
+   */
+  private historyStaging: Partial<Record<UuidString, Models.HistoryEntry>> = {}
+  private history: Models.HistoryMap = {}
+  private itemRevisionThreshold = DefaultItemRevisionsThreshold
+
+  constructor(
+    private itemManager: ItemManager,
+    private storageService: DiskStorageService,
+    private apiService: SNApiService,
+    private protocolService: EncryptionService,
+    public deviceInterface: Services.DeviceInterface,
+    protected override internalEventBus: Services.InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+    this.removeChangeObserver = this.itemManager.addObserver(ContentType.Note, ({ changed, inserted }) => {
+      this.recordNewHistoryForItems(changed.concat(inserted) as SNNote[])
+    })
+  }
+
+  public override deinit(): void {
+    ;(this.itemManager as unknown) = undefined
+    ;(this.storageService as unknown) = undefined
+    ;(this.history as unknown) = undefined
+    if (this.removeChangeObserver) {
+      this.removeChangeObserver()
+      ;(this.removeChangeObserver as unknown) = undefined
+    }
+    super.deinit()
+  }
+
+  private recordNewHistoryForItems(items: Models.SNNote[]) {
+    for (const item of items) {
+      const itemHistory = this.history[item.uuid] || []
+      const latestEntry = Models.historyMapFunctions.getNewestRevision(itemHistory)
+      const historyPayload = new Models.DecryptedPayload<Models.NoteContent>(item.payload)
+
+      const currentValueEntry = Models.CreateHistoryEntryForPayload(historyPayload, latestEntry)
+      if (currentValueEntry.isDiscardable()) {
+        continue
+      }
+
+      /**
+       * For every change that comes in, first add it to the staging area.
+       * Then, only on the next subsequent change do we add this previously
+       * staged entry
+       */
+      const stagedEntry = this.historyStaging[item.uuid]
+
+      /** Add prospective to staging, and consider now adding previously staged as new revision */
+      this.historyStaging[item.uuid] = currentValueEntry
+
+      if (!stagedEntry) {
+        continue
+      }
+
+      if (stagedEntry.isSameAsEntry(currentValueEntry)) {
+        continue
+      }
+
+      if (latestEntry && stagedEntry.isSameAsEntry(latestEntry)) {
+        continue
+      }
+
+      itemHistory.unshift(stagedEntry)
+      this.history[item.uuid] = itemHistory
+
+      this.optimizeHistoryForItem(item.uuid)
+    }
+  }
+
+  sessionHistoryForItem(item: Models.SNNote): Models.HistoryEntry[] {
+    return this.history[item.uuid] || []
+  }
+
+  getHistoryMapCopy(): Models.HistoryMap {
+    const copy = Object.assign({}, this.history)
+    for (const [key, value] of Object.entries(copy)) {
+      copy[key] = value.slice()
+    }
+    return Object.freeze(copy)
+  }
+
+  /**
+   * Fetches a list of revisions from the server for an item. These revisions do not
+   * include the item's content. Instead, each revision's content must be fetched
+   * individually upon selection via `fetchRemoteRevision`.
+   */
+  async remoteHistoryForItem(item: Models.SNNote): Promise<Responses.RevisionListEntry[] | undefined> {
+    const response = await this.apiService.getItemRevisions(item.uuid)
+    if (response.error || isNullOrUndefined(response.data)) {
+      return undefined
+    }
+    return (response as Responses.RevisionListResponse).data
+  }
+
+  /**
+   * Expands on a revision fetched via `remoteHistoryForItem` by getting a revision's
+   * complete fields (including encrypted content).
+   */
+  async fetchRemoteRevision(
+    note: Models.SNNote,
+    entry: Responses.RevisionListEntry,
+  ): Promise<Models.HistoryEntry | undefined> {
+    const revisionResponse = await this.apiService.getRevision(entry, note.uuid)
+    if (revisionResponse.error || isNullOrUndefined(revisionResponse.data)) {
+      return undefined
+    }
+    const revision = (revisionResponse as Responses.SingleRevisionResponse).data
+
+    const serverPayload = new Models.EncryptedPayload({
+      ...PayloadTimestampDefaults(),
+      ...revision,
+      updated_at: new Date(revision.updated_at),
+      created_at: new Date(revision.created_at),
+      waitingForKey: false,
+      errorDecrypting: false,
+    })
+
+    /**
+     * When an item is duplicated, its revisions also carry over to the newly created item.
+     * However since the new item has a different UUID than the source item, we must decrypt
+     * these olders revisions (which have not been mutated after copy) with the source item's
+     * uuid.
+     */
+    const embeddedParams = this.protocolService.getEmbeddedPayloadAuthenticatedData(serverPayload)
+    const sourceItemUuid = embeddedParams?.u as Uuid | undefined
+
+    const payload = serverPayload.copy({
+      uuid: sourceItemUuid || revision.item_uuid,
+    })
+
+    if (!Models.isRemotePayloadAllowed(payload)) {
+      console.error('Remote payload is disallowed', payload)
+      return undefined
+    }
+
+    const encryptedPayload = new Models.EncryptedPayload(payload)
+
+    const decryptedPayload = await this.protocolService.decryptSplitSingle<Models.NoteContent>({
+      usesItemsKeyWithKeyLookup: { items: [encryptedPayload] },
+    })
+
+    if (isErrorDecryptingPayload(decryptedPayload)) {
+      return undefined
+    }
+
+    return new Models.HistoryEntry(decryptedPayload)
+  }
+
+  async deleteRemoteRevision(note: SNNote, entry: Responses.RevisionListEntry): Promise<Responses.MinimalHttpResponse> {
+    const response = await this.apiService.deleteRevision(note.uuid, entry)
+    return response
+  }
+
+  /**
+   * Clean up if there are too many revisions. Note itemRevisionThreshold
+   * is the amount of revisions which above, call for an optimization. An
+   * optimization may not remove entries above this threshold. It will
+   * determine what it should keep and what it shouldn't. So, it is possible
+   * to have a threshold of 60 but have 600 entries, if the item history deems
+   * those worth keeping.
+   *
+   * Rules:
+   * - Keep an entry if it is the oldest entry
+   * - Keep an entry if it is the latest entry
+   * - Keep an entry if it is Significant
+   * - If an entry is Significant and it is a deletion change, keep the entry before this entry.
+   */
+  optimizeHistoryForItem(uuid: string): void {
+    const entries = this.history[uuid] || []
+    if (entries.length <= this.itemRevisionThreshold) {
+      return
+    }
+
+    const isEntrySignificant = (entry: Models.HistoryEntry) => {
+      return entry.deltaSize() > LargeEntryDeltaThreshold
+    }
+    const keepEntries: Models.HistoryEntry[] = []
+    const processEntry = (entry: Models.HistoryEntry, index: number, keep: boolean) => {
+      /**
+       * Entries may be processed retrospectively, meaning it can be
+       * decided to be deleted, then an upcoming processing can change that.
+       */
+      if (keep) {
+        keepEntries.unshift(entry)
+        if (isEntrySignificant(entry) && entry.operationVector() === -1) {
+          /** This is a large negative change. Hang on to the previous entry. */
+          const previousEntry = entries[index + 1]
+          if (previousEntry) {
+            keepEntries.unshift(previousEntry)
+          }
+        }
+      } else {
+        /** Don't keep, remove if in keep */
+        removeFromArray(keepEntries, entry)
+      }
+    }
+    for (let index = entries.length - 1; index >= 0; index--) {
+      const entry = entries[index]
+      const isSignificant = index === 0 || index === entries.length - 1 || isEntrySignificant(entry)
+      processEntry(entry, index, isSignificant)
+    }
+    const filtered = entries.filter((entry) => {
+      return keepEntries.includes(entry)
+    })
+    this.history[uuid] = filtered
+  }
+}

packages/snjs/lib/Services/History/index.ts

@@ -0,0 +1 @@
+export * from './HistoryManager'

packages/snjs/lib/Services/Items/ItemManager.spec.ts

@@ -0,0 +1,742 @@
+import { ContentType } from '@standardnotes/common'
+import { InternalEventBusInterface } from '@standardnotes/services'
+import { ItemManager } from './ItemManager'
+import { PayloadManager } from '../Payloads/PayloadManager'
+import { UuidGenerator } from '@standardnotes/utils'
+import * as Models from '@standardnotes/models'
+import {
+  DecryptedPayload,
+  DeletedPayload,
+  EncryptedPayload,
+  FillItemContent,
+  PayloadTimestampDefaults,
+  NoteContent,
+} from '@standardnotes/models'
+
+const setupRandomUuid = () => {
+  UuidGenerator.SetGenerator(() => String(Math.random()))
+}
+
+const VIEW_NOT_PINNED = '!["Not Pinned", "pinned", "=", false]'
+const VIEW_LAST_DAY = '!["Last Day", "updated_at", ">", "1.days.ago"]'
+const VIEW_LONG = '!["Long", "text.length", ">", 500]'
+
+const NotPinnedPredicate = Models.predicateFromJson<Models.SNTag>({
+  keypath: 'pinned',
+  operator: '=',
+  value: false,
+})
+
+const LastDayPredicate = Models.predicateFromJson<Models.SNTag>({
+  keypath: 'updated_at',
+  operator: '>',
+  value: '1.days.ago',
+})
+
+const LongTextPredicate = Models.predicateFromJson<Models.SNTag>({
+  keypath: 'text.length' as never,
+  operator: '>',
+  value: 500,
+})
+
+describe('itemManager', () => {
+  let payloadManager: PayloadManager
+  let itemManager: ItemManager
+  let items: Models.DecryptedItemInterface[]
+  let internalEventBus: InternalEventBusInterface
+
+  const createService = () => {
+    return new ItemManager(payloadManager, { supportsFileNavigation: false }, internalEventBus)
+  }
+
+  beforeEach(() => {
+    setupRandomUuid()
+
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+
+    payloadManager = new PayloadManager(internalEventBus)
+
+    items = [] as jest.Mocked<Models.DecryptedItemInterface[]>
+    itemManager = {} as jest.Mocked<ItemManager>
+    itemManager.getItems = jest.fn().mockReturnValue(items)
+    itemManager.createItem = jest.fn()
+    itemManager.changeComponent = jest.fn().mockReturnValue({} as jest.Mocked<Models.DecryptedItemInterface>)
+    itemManager.setItemsToBeDeleted = jest.fn()
+    itemManager.addObserver = jest.fn()
+    itemManager.changeItem = jest.fn()
+    itemManager.changeFeatureRepo = jest.fn()
+  })
+
+  const createTag = (title: string) => {
+    return new Models.SNTag(
+      new Models.DecryptedPayload({
+        uuid: String(Math.random()),
+        content_type: ContentType.Tag,
+        content: Models.FillItemContent<Models.TagContent>({
+          title: title,
+        }),
+        ...PayloadTimestampDefaults(),
+      }),
+    )
+  }
+
+  const createNote = (title: string) => {
+    return new Models.SNNote(
+      new Models.DecryptedPayload({
+        uuid: String(Math.random()),
+        content_type: ContentType.Note,
+        content: Models.FillItemContent<Models.NoteContent>({
+          title: title,
+        }),
+        ...PayloadTimestampDefaults(),
+      }),
+    )
+  }
+
+  const createFile = (name: string) => {
+    return new Models.FileItem(
+      new Models.DecryptedPayload({
+        uuid: String(Math.random()),
+        content_type: ContentType.File,
+        content: Models.FillItemContent<Models.FileContent>({
+          name: name,
+        }),
+        ...PayloadTimestampDefaults(),
+      }),
+    )
+  }
+
+  describe('item emit', () => {
+    it('deleted payloads should map to removed items', async () => {
+      itemManager = createService()
+
+      const payload = new DeletedPayload({
+        uuid: String(Math.random()),
+        content_type: ContentType.Note,
+        content: undefined,
+        deleted: true,
+        dirty: true,
+        ...PayloadTimestampDefaults(),
+      })
+
+      const mockFn = jest.fn()
+
+      itemManager['notifyObservers'] = mockFn
+
+      await payloadManager.emitPayload(payload, Models.PayloadEmitSource.LocalInserted)
+
+      expect(mockFn.mock.calls[0][2]).toHaveLength(1)
+    })
+
+    it('decrypted items who become encrypted should be removed from ui', async () => {
+      itemManager = createService()
+
+      const decrypted = new DecryptedPayload({
+        uuid: String(Math.random()),
+        content_type: ContentType.Note,
+        content: FillItemContent<NoteContent>({
+          title: 'foo',
+        }),
+        ...PayloadTimestampDefaults(),
+      })
+
+      await payloadManager.emitPayload(decrypted, Models.PayloadEmitSource.LocalInserted)
+
+      const encrypted = new EncryptedPayload({
+        ...decrypted,
+        content: '004:...',
+        enc_item_key: '004:...',
+        items_key_id: '123',
+        waitingForKey: true,
+        errorDecrypting: true,
+      })
+
+      const mockFn = jest.fn()
+
+      itemManager['notifyObservers'] = mockFn
+
+      await payloadManager.emitPayload(encrypted, Models.PayloadEmitSource.LocalInserted)
+
+      expect(mockFn.mock.calls[0][2]).toHaveLength(1)
+    })
+  })
+
+  describe('note display criteria', () => {
+    it('viewing notes with tag', async () => {
+      itemManager = createService()
+      const tag = createTag('parent')
+      const note = createNote('note')
+      await itemManager.insertItems([tag, note])
+      await itemManager.addTagToNote(note, tag, false)
+
+      itemManager.setPrimaryItemDisplayOptions({
+        tags: [tag],
+        sortBy: 'title',
+        sortDirection: 'asc',
+      })
+
+      const notes = itemManager.getDisplayableNotes()
+      expect(notes).toHaveLength(1)
+    })
+  })
+
+  describe('tag relationships', () => {
+    it('updates parentId of child tag', async () => {
+      itemManager = createService()
+      const parent = createTag('parent')
+      const child = createTag('child')
+      await itemManager.insertItems([parent, child])
+      await itemManager.setTagParent(parent, child)
+
+      const changedChild = itemManager.findItem(child.uuid) as Models.SNTag
+      expect(changedChild.parentId).toBe(parent.uuid)
+    })
+
+    it('forbids a tag to be its own parent', async () => {
+      itemManager = createService()
+      const tag = createTag('tag')
+      await itemManager.insertItems([tag])
+
+      expect(() => itemManager.setTagParent(tag, tag)).toThrow()
+      expect(itemManager.getTagParent(tag)).toBeUndefined()
+    })
+
+    it('forbids a tag to be its own ancestor', async () => {
+      itemManager = createService()
+      const grandParent = createTag('grandParent')
+      const parent = createTag('parent')
+      const child = createTag('child')
+
+      await itemManager.insertItems([child, parent, grandParent])
+      await itemManager.setTagParent(parent, child)
+      await itemManager.setTagParent(grandParent, parent)
+
+      expect(() => itemManager.setTagParent(child, grandParent)).toThrow()
+      expect(itemManager.getTagParent(grandParent)).toBeUndefined()
+    })
+
+    it('getTagParent', async () => {
+      itemManager = createService()
+      const parent = createTag('parent')
+      const child = createTag('child')
+      await itemManager.insertItems([parent, child])
+      await itemManager.setTagParent(parent, child)
+
+      expect(itemManager.getTagParent(child)?.uuid).toBe(parent.uuid)
+    })
+
+    it('findTagByTitleAndParent', async () => {
+      itemManager = createService()
+      const parent = createTag('name1')
+      const child = createTag('childName')
+      const duplicateNameChild = createTag('name1')
+
+      await itemManager.insertItems([parent, child, duplicateNameChild])
+      await itemManager.setTagParent(parent, child)
+      await itemManager.setTagParent(parent, duplicateNameChild)
+
+      const a = itemManager.findTagByTitleAndParent('name1', undefined)
+      const b = itemManager.findTagByTitleAndParent('name1', parent)
+      const c = itemManager.findTagByTitleAndParent('name1', child)
+
+      expect(a?.uuid).toEqual(parent.uuid)
+      expect(b?.uuid).toEqual(duplicateNameChild.uuid)
+      expect(c?.uuid).toEqual(undefined)
+    })
+
+    it('findOrCreateTagByTitle', async () => {
+      setupRandomUuid()
+      itemManager = createService()
+      const parent = createTag('parent')
+      const child = createTag('child')
+      await itemManager.insertItems([parent, child])
+      await itemManager.setTagParent(parent, child)
+
+      const childA = await itemManager.findOrCreateTagByTitle('child')
+      const childB = await itemManager.findOrCreateTagByTitle('child', parent)
+      const childC = await itemManager.findOrCreateTagByTitle('child-bis', parent)
+      const childD = await itemManager.findOrCreateTagByTitle('child-bis', parent)
+
+      expect(childA.uuid).not.toEqual(child.uuid)
+      expect(childB.uuid).toEqual(child.uuid)
+      expect(childD.uuid).toEqual(childC.uuid)
+
+      expect(itemManager.getTagParent(childA)?.uuid).toBe(undefined)
+      expect(itemManager.getTagParent(childB)?.uuid).toBe(parent.uuid)
+      expect(itemManager.getTagParent(childC)?.uuid).toBe(parent.uuid)
+      expect(itemManager.getTagParent(childD)?.uuid).toBe(parent.uuid)
+    })
+
+    it('findOrCreateTagParentChain', async () => {
+      itemManager = createService()
+      const parent = createTag('parent')
+      const child = createTag('child')
+
+      await itemManager.insertItems([parent, child])
+      await itemManager.setTagParent(parent, child)
+
+      const a = await itemManager.findOrCreateTagParentChain(['parent'])
+      const b = await itemManager.findOrCreateTagParentChain(['parent', 'child'])
+      const c = await itemManager.findOrCreateTagParentChain(['parent', 'child2'])
+      const d = await itemManager.findOrCreateTagParentChain(['parent2', 'child1'])
+
+      expect(a?.uuid).toEqual(parent.uuid)
+      expect(b?.uuid).toEqual(child.uuid)
+
+      expect(c?.uuid).not.toEqual(parent.uuid)
+      expect(c?.uuid).not.toEqual(child.uuid)
+      expect(c?.parentId).toEqual(parent.uuid)
+
+      expect(d?.uuid).not.toEqual(parent.uuid)
+      expect(d?.uuid).not.toEqual(child.uuid)
+      expect(d?.parentId).not.toEqual(parent.uuid)
+    })
+
+    it('isAncestor', async () => {
+      itemManager = createService()
+      const grandParent = createTag('grandParent')
+      const parent = createTag('parent')
+      const child = createTag('child')
+      const another = createTag('another')
+
+      await itemManager.insertItems([child, parent, grandParent, another])
+      await itemManager.setTagParent(parent, child)
+      await itemManager.setTagParent(grandParent, parent)
+
+      expect(itemManager.isTagAncestor(grandParent, parent)).toEqual(true)
+      expect(itemManager.isTagAncestor(grandParent, child)).toEqual(true)
+      expect(itemManager.isTagAncestor(parent, child)).toEqual(true)
+
+      expect(itemManager.isTagAncestor(parent, grandParent)).toBeFalsy()
+      expect(itemManager.isTagAncestor(child, grandParent)).toBeFalsy()
+      expect(itemManager.isTagAncestor(grandParent, grandParent)).toBeFalsy()
+
+      expect(itemManager.isTagAncestor(another, grandParent)).toBeFalsy()
+      expect(itemManager.isTagAncestor(child, another)).toBeFalsy()
+      expect(itemManager.isTagAncestor(grandParent, another)).toBeFalsy()
+    })
+
+    it('unsetTagRelationship', async () => {
+      itemManager = createService()
+      const parent = createTag('parent')
+      const child = createTag('child')
+      await itemManager.insertItems([parent, child])
+      await itemManager.setTagParent(parent, child)
+      expect(itemManager.getTagParent(child)?.uuid).toBe(parent.uuid)
+
+      await itemManager.unsetTagParent(child)
+
+      expect(itemManager.getTagParent(child)).toBeUndefined()
+    })
+
+    it('getTagParentChain', async () => {
+      itemManager = createService()
+      const greatGrandParent = createTag('greatGrandParent')
+      const grandParent = createTag('grandParent')
+      const parent = createTag('parent')
+      const child = createTag('child')
+      await itemManager.insertItems([greatGrandParent, grandParent, parent, child])
+      await itemManager.setTagParent(parent, child)
+      await itemManager.setTagParent(grandParent, parent)
+      await itemManager.setTagParent(greatGrandParent, grandParent)
+
+      const uuidChain = itemManager.getTagParentChain(child).map((tag) => tag.uuid)
+
+      expect(uuidChain).toHaveLength(3)
+      expect(uuidChain).toEqual([greatGrandParent.uuid, grandParent.uuid, parent.uuid])
+    })
+
+    it('viewing notes for parent tag should not display notes of children', async () => {
+      itemManager = createService()
+      const parentTag = createTag('parent')
+      const childTag = createTag('child')
+      await itemManager.insertItems([parentTag, childTag])
+      await itemManager.setTagParent(parentTag, childTag)
+
+      const parentNote = createNote('parentNote')
+      const childNote = createNote('childNote')
+      await itemManager.insertItems([parentNote, childNote])
+
+      await itemManager.addTagToNote(parentNote, parentTag, false)
+      await itemManager.addTagToNote(childNote, childTag, false)
+
+      itemManager.setPrimaryItemDisplayOptions({
+        tags: [parentTag],
+        sortBy: 'title',
+        sortDirection: 'asc',
+      })
+
+      const notes = itemManager.getDisplayableNotes()
+      expect(notes).toHaveLength(1)
+    })
+
+    it('adding a note to a tag hierarchy should add the note to its parent too', async () => {
+      itemManager = createService()
+      const parentTag = createTag('parent')
+      const childTag = createTag('child')
+      const note = createNote('note')
+
+      await itemManager.insertItems([parentTag, childTag, note])
+      await itemManager.setTagParent(parentTag, childTag)
+
+      await itemManager.addTagToNote(note, childTag, true)
+
+      const tags = itemManager.getSortedTagsForNote(note)
+
+      expect(tags).toHaveLength(2)
+      expect(tags[0].uuid).toEqual(childTag.uuid)
+      expect(tags[1].uuid).toEqual(parentTag.uuid)
+    })
+
+    it('adding a note to a tag hierarchy should not add the note to its parent if hierarchy option is disabled', async () => {
+      itemManager = createService()
+      const parentTag = createTag('parent')
+      const childTag = createTag('child')
+      const note = createNote('note')
+
+      await itemManager.insertItems([parentTag, childTag, note])
+      await itemManager.setTagParent(parentTag, childTag)
+
+      await itemManager.addTagToNote(note, childTag, false)
+
+      const tags = itemManager.getSortedTagsForNote(note)
+
+      expect(tags).toHaveLength(1)
+      expect(tags[0].uuid).toEqual(childTag.uuid)
+    })
+  })
+
+  describe('template items', () => {
+    it('create template item', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const item = await itemManager.createTemplateItem(ContentType.Note, {
+        title: 'hello',
+        references: [],
+      })
+
+      expect(!!item).toEqual(true)
+      /* Template items should never be added to the record */
+      expect(itemManager.items).toHaveLength(0)
+      expect(itemManager.getDisplayableNotes()).toHaveLength(0)
+    })
+
+    it('isTemplateItem return the correct value', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const item = await itemManager.createTemplateItem(ContentType.Note, {
+        title: 'hello',
+        references: [],
+      })
+
+      expect(itemManager.isTemplateItem(item)).toEqual(true)
+
+      await itemManager.insertItem(item)
+
+      expect(itemManager.isTemplateItem(item)).toEqual(false)
+    })
+
+    it('isTemplateItem return the correct value for system smart views', () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const [systemTag1, ...restOfSystemViews] = itemManager
+        .getSmartViews()
+        .filter((view) => Object.values(Models.SystemViewId).includes(view.uuid as Models.SystemViewId))
+
+      const isSystemTemplate = itemManager.isTemplateItem(systemTag1)
+      expect(isSystemTemplate).toEqual(false)
+
+      const areTemplates = restOfSystemViews.map((tag) => itemManager.isTemplateItem(tag)).every((value) => !!value)
+      expect(areTemplates).toEqual(false)
+    })
+  })
+
+  describe('tags', () => {
+    it('lets me create a regular tag with a clear API', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const tag = await itemManager.createTag('this is my new tag')
+
+      expect(tag).toBeTruthy()
+      expect(itemManager.isTemplateItem(tag)).toEqual(false)
+    })
+
+    it('should search tags correctly', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const foo = await itemManager.createTag('foo[')
+      const foobar = await itemManager.createTag('foo[bar]')
+      const bar = await itemManager.createTag('bar[')
+      const barfoo = await itemManager.createTag('bar[foo]')
+      const fooDelimiter = await itemManager.createTag('bar.foo')
+      const barFooDelimiter = await itemManager.createTag('baz.bar.foo')
+      const fooAttached = await itemManager.createTag('Foo')
+      const note = createNote('note')
+      await itemManager.insertItems([foo, foobar, bar, barfoo, fooDelimiter, barFooDelimiter, fooAttached, note])
+      await itemManager.addTagToNote(note, fooAttached, false)
+
+      const fooResults = itemManager.searchTags('foo')
+      expect(fooResults).toContainEqual(foo)
+      expect(fooResults).toContainEqual(foobar)
+      expect(fooResults).toContainEqual(barfoo)
+      expect(fooResults).toContainEqual(fooDelimiter)
+      expect(fooResults).toContainEqual(barFooDelimiter)
+      expect(fooResults).not.toContainEqual(bar)
+      expect(fooResults).not.toContainEqual(fooAttached)
+    })
+  })
+
+  describe('tags notes index', () => {
+    it('counts countable notes', async () => {
+      itemManager = createService()
+
+      const parentTag = createTag('parent')
+      const childTag = createTag('child')
+      await itemManager.insertItems([parentTag, childTag])
+      await itemManager.setTagParent(parentTag, childTag)
+
+      const parentNote = createNote('parentNote')
+      const childNote = createNote('childNote')
+      await itemManager.insertItems([parentNote, childNote])
+
+      await itemManager.addTagToNote(parentNote, parentTag, false)
+      await itemManager.addTagToNote(childNote, childTag, false)
+
+      expect(itemManager.countableNotesForTag(parentTag)).toBe(1)
+      expect(itemManager.countableNotesForTag(childTag)).toBe(1)
+      expect(itemManager.allCountableNotesCount()).toBe(2)
+    })
+
+    it('archiving a note should update count index', async () => {
+      itemManager = createService()
+
+      const tag1 = createTag('tag 1')
+      await itemManager.insertItems([tag1])
+
+      const note1 = createNote('note 1')
+      const note2 = createNote('note 2')
+      await itemManager.insertItems([note1, note2])
+
+      await itemManager.addTagToNote(note1, tag1, false)
+      await itemManager.addTagToNote(note2, tag1, false)
+
+      expect(itemManager.countableNotesForTag(tag1)).toBe(2)
+      expect(itemManager.allCountableNotesCount()).toBe(2)
+
+      await itemManager.changeItem<Models.NoteMutator>(note1, (m) => {
+        m.archived = true
+      })
+
+      expect(itemManager.allCountableNotesCount()).toBe(1)
+      expect(itemManager.countableNotesForTag(tag1)).toBe(1)
+
+      await itemManager.changeItem<Models.NoteMutator>(note1, (m) => {
+        m.archived = false
+      })
+
+      expect(itemManager.allCountableNotesCount()).toBe(2)
+      expect(itemManager.countableNotesForTag(tag1)).toBe(2)
+    })
+  })
+
+  describe('smart views', () => {
+    it('lets me create a smart view', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const [view1, view2, view3] = await Promise.all([
+        itemManager.createSmartView('Not Pinned', NotPinnedPredicate),
+        itemManager.createSmartView('Last Day', LastDayPredicate),
+        itemManager.createSmartView('Long', LongTextPredicate),
+      ])
+
+      expect(view1).toBeTruthy()
+      expect(view2).toBeTruthy()
+      expect(view3).toBeTruthy()
+
+      expect(view1.content_type).toEqual(ContentType.SmartView)
+      expect(view2.content_type).toEqual(ContentType.SmartView)
+      expect(view3.content_type).toEqual(ContentType.SmartView)
+    })
+
+    it('lets me use a smart view', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const view = await itemManager.createSmartView('Not Pinned', NotPinnedPredicate)
+
+      const notes = itemManager.notesMatchingSmartView(view)
+
+      expect(notes).toEqual([])
+    })
+
+    it('lets me test if a title is a smart view', () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      expect(itemManager.isSmartViewTitle(VIEW_NOT_PINNED)).toEqual(true)
+      expect(itemManager.isSmartViewTitle(VIEW_LAST_DAY)).toEqual(true)
+      expect(itemManager.isSmartViewTitle(VIEW_LONG)).toEqual(true)
+
+      expect(itemManager.isSmartViewTitle('Helloworld')).toEqual(false)
+      expect(itemManager.isSmartViewTitle('@^![ some title')).toEqual(false)
+    })
+
+    it('lets me create a smart view from the DSL', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const [tag1, tag2, tag3] = await Promise.all([
+        itemManager.createSmartViewFromDSL(VIEW_NOT_PINNED),
+        itemManager.createSmartViewFromDSL(VIEW_LAST_DAY),
+        itemManager.createSmartViewFromDSL(VIEW_LONG),
+      ])
+
+      expect(tag1).toBeTruthy()
+      expect(tag2).toBeTruthy()
+      expect(tag3).toBeTruthy()
+
+      expect(tag1.content_type).toEqual(ContentType.SmartView)
+      expect(tag2.content_type).toEqual(ContentType.SmartView)
+      expect(tag3.content_type).toEqual(ContentType.SmartView)
+    })
+
+    it('will create smart view or tags from the generic method', async () => {
+      itemManager = createService()
+      setupRandomUuid()
+
+      const someTag = await itemManager.createTagOrSmartView('some-tag')
+      const someView = await itemManager.createTagOrSmartView(VIEW_LONG)
+
+      expect(someTag.content_type).toEqual(ContentType.Tag)
+      expect(someView.content_type).toEqual(ContentType.SmartView)
+    })
+  })
+
+  it('lets me rename a smart view', async () => {
+    itemManager = createService()
+    setupRandomUuid()
+
+    const tag = await itemManager.createSmartView('Not Pinned', NotPinnedPredicate)
+
+    await itemManager.changeItem<Models.TagMutator>(tag, (m) => {
+      m.title = 'New Title'
+    })
+
+    const view = itemManager.findItem(tag.uuid) as Models.SmartView
+    const views = itemManager.getSmartViews()
+
+    expect(view.title).toEqual('New Title')
+    expect(views.some((tag: Models.SmartView) => tag.title === 'New Title')).toEqual(true)
+  })
+
+  it('lets me find a smart view', async () => {
+    itemManager = createService()
+    setupRandomUuid()
+
+    const tag = await itemManager.createSmartView('Not Pinned', NotPinnedPredicate)
+
+    const view = itemManager.findItem(tag.uuid) as Models.SmartView
+
+    expect(view).toBeDefined()
+  })
+
+  it('untagged notes smart view', async () => {
+    itemManager = createService()
+    setupRandomUuid()
+
+    const view = itemManager.untaggedNotesSmartView
+
+    const tag = createTag('tag')
+    const untaggedNote = createNote('note')
+    const taggedNote = createNote('taggedNote')
+    await itemManager.insertItems([tag, untaggedNote, taggedNote])
+
+    expect(itemManager.notesMatchingSmartView(view)).toHaveLength(2)
+
+    await itemManager.addTagToNote(taggedNote, tag, false)
+
+    expect(itemManager.notesMatchingSmartView(view)).toHaveLength(1)
+
+    expect(view).toBeDefined()
+  })
+
+  describe('files', () => {
+    it('associates with note', async () => {
+      itemManager = createService()
+      const note = createNote('invoices')
+      const file = createFile('invoice_1.pdf')
+      await itemManager.insertItems([note, file])
+
+      const resultingFile = await itemManager.associateFileWithNote(file, note)
+      const references = resultingFile.references
+
+      expect(references).toHaveLength(1)
+      expect(references[0].uuid).toEqual(note.uuid)
+    })
+
+    it('disassociates with note', async () => {
+      itemManager = createService()
+      const note = createNote('invoices')
+      const file = createFile('invoice_1.pdf')
+      await itemManager.insertItems([note, file])
+
+      const associatedFile = await itemManager.associateFileWithNote(file, note)
+      const disassociatedFile = await itemManager.disassociateFileWithNote(associatedFile, note)
+      const references = disassociatedFile.references
+
+      expect(references).toHaveLength(0)
+    })
+
+    it('should get files associated with note', async () => {
+      itemManager = createService()
+      const note = createNote('invoices')
+      const file = createFile('invoice_1.pdf')
+      const secondFile = createFile('unrelated-file.xlsx')
+      await itemManager.insertItems([note, file, secondFile])
+
+      await itemManager.associateFileWithNote(file, note)
+
+      const filesAssociatedWithNote = itemManager.getFilesForNote(note)
+
+      expect(filesAssociatedWithNote).toHaveLength(1)
+      expect(filesAssociatedWithNote[0].uuid).toBe(file.uuid)
+    })
+
+    it('should correctly rename file to filename that has extension', async () => {
+      itemManager = createService()
+      const file = createFile('initialName.ext')
+      await itemManager.insertItems([file])
+
+      const renamedFile = await itemManager.renameFile(file, 'anotherName.anotherExt')
+
+      expect(renamedFile.name).toBe('anotherName.anotherExt')
+    })
+
+    it('should correctly rename extensionless file to filename that has extension', async () => {
+      itemManager = createService()
+      const file = createFile('initialName')
+      await itemManager.insertItems([file])
+
+      const renamedFile = await itemManager.renameFile(file, 'anotherName.anotherExt')
+
+      expect(renamedFile.name).toBe('anotherName.anotherExt')
+    })
+
+    it('should correctly rename file to filename that does not have extension', async () => {
+      itemManager = createService()
+      const file = createFile('initialName.ext')
+      await itemManager.insertItems([file])
+
+      const renamedFile = await itemManager.renameFile(file, 'anotherName')
+
+      expect(renamedFile.name).toBe('anotherName')
+    })
+  })
+})

packages/snjs/lib/Services/Items/ItemsClientInterface.ts

@@ -0,0 +1,141 @@
+import { SNItemsKey } from '@standardnotes/encryption'
+import { ContentType } from '@standardnotes/common'
+import {
+  SNNote,
+  FileItem,
+  SNTag,
+  SmartView,
+  TagNoteCountChangeObserver,
+  DecryptedPayloadInterface,
+  EncryptedItemInterface,
+  DecryptedTransferPayload,
+  PredicateInterface,
+  DecryptedItemInterface,
+  SNComponent,
+  SNTheme,
+  DisplayOptions,
+} from '@standardnotes/models'
+import { UuidString } from '@Lib/Types'
+
+export interface ItemsClientInterface {
+  get invalidItems(): EncryptedItemInterface[]
+
+  associateFileWithNote(file: FileItem, note: SNNote): Promise<FileItem>
+
+  disassociateFileWithNote(file: FileItem, note: SNNote): Promise<FileItem>
+
+  getFilesForNote(note: SNNote): FileItem[]
+
+  renameFile(file: FileItem, name: string): Promise<FileItem>
+
+  addTagToNote(note: SNNote, tag: SNTag, addHierarchy: boolean): Promise<SNTag[]>
+
+  /** Creates an unmanaged, un-inserted item from a payload. */
+  createItemFromPayload(payload: DecryptedPayloadInterface): DecryptedItemInterface
+
+  createPayloadFromObject(object: DecryptedTransferPayload): DecryptedPayloadInterface
+
+  get trashedItems(): SNNote[]
+
+  setPrimaryItemDisplayOptions(options: DisplayOptions): void
+
+  getDisplayableNotes(): SNNote[]
+
+  getDisplayableTags(): SNTag[]
+
+  getDisplayableItemsKeys(): SNItemsKey[]
+
+  getDisplayableFiles(): FileItem[]
+
+  getDisplayableNotesAndFiles(): (SNNote | FileItem)[]
+
+  getDisplayableComponents(): (SNComponent | SNTheme)[]
+
+  getItems<T extends DecryptedItemInterface>(contentType: ContentType | ContentType[]): T[]
+
+  notesMatchingSmartView(view: SmartView): SNNote[]
+
+  addNoteCountChangeObserver(observer: TagNoteCountChangeObserver): () => void
+
+  allCountableNotesCount(): number
+
+  countableNotesForTag(tag: SNTag | SmartView): number
+
+  findTagByTitle(title: string): SNTag | undefined
+
+  getTagPrefixTitle(tag: SNTag): string | undefined
+
+  getTagLongTitle(tag: SNTag): string
+
+  hasTagsNeedingFoldersMigration(): boolean
+
+  referencesForItem(itemToLookupUuidFor: DecryptedItemInterface, contentType?: ContentType): DecryptedItemInterface[]
+
+  itemsReferencingItem(itemToLookupUuidFor: DecryptedItemInterface, contentType?: ContentType): DecryptedItemInterface[]
+
+  /**
+   * Finds tags with title or component starting with a search query and (optionally) not associated with a note
+   * @param searchQuery - The query string to match
+   * @param note - The note whose tags should be omitted from results
+   * @returns Array containing tags matching search query and not associated with note
+   */
+  searchTags(searchQuery: string, note?: SNNote): SNTag[]
+
+  isValidTagParent(parentTagToLookUpUuidFor: SNTag, childToLookUpUuidFor: SNTag): boolean
+
+  /**
+   * Returns the parent for a tag
+   */
+  getTagParent(itemToLookupUuidFor: SNTag): SNTag | undefined
+
+  /**
+   * Returns the hierarchy of parents for a tag
+   * @returns Array containing all parent tags
+   */
+  getTagParentChain(itemToLookupUuidFor: SNTag): SNTag[]
+
+  /**
+   * Returns all descendants for a tag
+   * @returns Array containing all descendant tags
+   */
+  getTagChildren(itemToLookupUuidFor: SNTag): SNTag[]
+
+  /**
+   * Get tags for a note sorted in natural order
+   * @param note - The note whose tags will be returned
+   * @returns Array containing tags associated with a note
+   */
+  getSortedTagsForNote(note: SNNote): SNTag[]
+
+  isSmartViewTitle(title: string): boolean
+
+  getSmartViews(): SmartView[]
+
+  getNoteCount(): number
+
+  /**
+   * Finds an item by UUID.
+   */
+  findItem<T extends DecryptedItemInterface = DecryptedItemInterface>(uuid: UuidString): T | undefined
+
+  /**
+   * Finds an item by predicate.
+   */
+  findItems<T extends DecryptedItemInterface>(uuids: UuidString[]): T[]
+
+  findSureItem<T extends DecryptedItemInterface = DecryptedItemInterface>(uuid: UuidString): T
+
+  /**
+   * Finds an item by predicate.
+   */
+  itemsMatchingPredicate<T extends DecryptedItemInterface>(
+    contentType: ContentType,
+    predicate: PredicateInterface<T>,
+  ): T[]
+
+  /**
+   * @param item item to be checked
+   * @returns Whether the item is a template (unmanaged)
+   */
+  isTemplateItem(item: DecryptedItemInterface): boolean
+}

packages/snjs/lib/Services/Items/TransactionalMutation.ts

@@ -0,0 +1,8 @@
+import * as Models from '@standardnotes/models'
+import { UuidString } from '../../Types/UuidString'
+
+export type TransactionalMutation = {
+  itemUuid: UuidString
+  mutate: (mutator: Models.ItemMutator) => void
+  mutationType?: Models.MutationType
+}

packages/snjs/lib/Services/Items/index.ts

@@ -0,0 +1,3 @@
+export * from './ItemsClientInterface'
+export * from './ItemManager'
+export * from './TransactionalMutation'

packages/snjs/lib/Services/KeyRecovery/KeyRecoveryOperation.ts

@@ -0,0 +1,65 @@
+import { ContentType } from '@standardnotes/common'
+import { ItemsKeyInterface } from '@standardnotes/models'
+import { dateSorted } from '@standardnotes/utils'
+import { SNRootKeyParams, DecryptItemsKeyByPromptingUser, EncryptionProvider } from '@standardnotes/encryption'
+import { DecryptionQueueItem, KeyRecoveryOperationResult } from './Types'
+import { serverKeyParamsAreSafe } from './Utils'
+import { ChallengeServiceInterface } from '@standardnotes/services'
+import { ItemManager } from '../Items'
+
+export class KeyRecoveryOperation {
+  constructor(
+    private queueItem: DecryptionQueueItem,
+    private itemManager: ItemManager,
+    private protocolService: EncryptionProvider,
+    private challengeService: ChallengeServiceInterface,
+    private clientParams: SNRootKeyParams | undefined,
+    private serverParams: SNRootKeyParams | undefined,
+  ) {}
+
+  public async run(): Promise<KeyRecoveryOperationResult> {
+    let replaceLocalRootKeyWithResult = false
+
+    const queueItemKeyParamsAreBetterOrEqualToClients =
+      this.serverParams &&
+      this.clientParams &&
+      !this.clientParams.compare(this.serverParams) &&
+      this.queueItem.keyParams.compare(this.serverParams) &&
+      serverKeyParamsAreSafe(this.serverParams, this.clientParams)
+
+    if (queueItemKeyParamsAreBetterOrEqualToClients) {
+      const latestDecryptedItemsKey = dateSorted(
+        this.itemManager.getItems<ItemsKeyInterface>(ContentType.ItemsKey),
+        'created_at',
+        false,
+      )[0]
+
+      if (!latestDecryptedItemsKey) {
+        replaceLocalRootKeyWithResult = true
+      } else {
+        replaceLocalRootKeyWithResult = this.queueItem.encryptedKey.created_at > latestDecryptedItemsKey.created_at
+      }
+    }
+
+    const decryptionResult = await DecryptItemsKeyByPromptingUser(
+      this.queueItem.encryptedKey,
+      this.protocolService,
+      this.challengeService,
+      this.queueItem.keyParams,
+    )
+
+    if (decryptionResult === 'aborted') {
+      return { aborted: true }
+    }
+
+    if (decryptionResult === 'failed') {
+      return { aborted: false }
+    }
+
+    return {
+      rootKey: decryptionResult.rootKey,
+      replaceLocalRootKeyWithResult,
+      decryptedItemsKey: decryptionResult.decryptedKey,
+    }
+  }
+}

packages/snjs/lib/Services/KeyRecovery/KeyRecoveryService.ts

@@ -0,0 +1,535 @@
+import { KeyRecoveryOperation } from './KeyRecoveryOperation'
+import {
+  SNRootKeyParams,
+  EncryptionService,
+  SNRootKey,
+  KeyParamsFromApiResponse,
+  KeyRecoveryStrings,
+} from '@standardnotes/encryption'
+import { UserService } from '../User/UserService'
+import {
+  isErrorDecryptingPayload,
+  EncryptedPayloadInterface,
+  EncryptedPayload,
+  isDecryptedPayload,
+  DecryptedPayloadInterface,
+  PayloadEmitSource,
+  EncryptedItemInterface,
+  getIncrementedDirtyIndex,
+} from '@standardnotes/models'
+import { SNSyncService } from '../Sync/SyncService'
+import { DiskStorageService } from '../Storage/DiskStorageService'
+import { PayloadManager } from '../Payloads/PayloadManager'
+import { Challenge, ChallengeService } from '../Challenge'
+import { SNApiService } from '@Lib/Services/Api/ApiService'
+import { ContentType, Uuid } from '@standardnotes/common'
+import { ItemManager } from '../Items/ItemManager'
+import { removeFromArray, Uuids } from '@standardnotes/utils'
+import { ClientDisplayableError, KeyParamsResponse } from '@standardnotes/responses'
+import {
+  AlertService,
+  AbstractService,
+  InternalEventBusInterface,
+  StorageValueModes,
+  ApplicationStage,
+  StorageKey,
+  DiagnosticInfo,
+  ChallengeValidation,
+  ChallengeReason,
+  ChallengePrompt,
+} from '@standardnotes/services'
+import {
+  UndecryptableItemsStorage,
+  DecryptionQueueItem,
+  KeyRecoveryEvent,
+  isSuccessResult,
+  KeyRecoveryOperationResult,
+} from './Types'
+import { serverKeyParamsAreSafe } from './Utils'
+
+/**
+ * The key recovery service listens to items key changes to detect any that cannot be decrypted.
+ * If it detects an items key that is not properly decrypted, it will present a key recovery
+ * wizard (using existing UI like Challenges and AlertService) that will attempt to recover
+ * the root key for those keys.
+ *
+ * When we encounter an items key we cannot decrypt, this is a sign that the user's password may
+ * have recently changed (even though their session is still valid). If the user has been
+ * previously signed in, we take this opportunity to reach out to the server to get the
+ * user's current key_params. We ensure these key params' version is equal to or greater than our own.
+
+ * - If this key's key params are equal to the retrieved parameters,
+    and this keys created date is greater than any existing valid items key,
+    or if we do not have any items keys:
+       1. Use the decryption of this key as a source of validation
+       2. If valid, replace our local root key with this new root key and emit the decrypted items key
+ * - Else, if the key params are not equal,
+     or its created date is less than an existing valid items key
+        1. Attempt to decrypt this key using its attached key paramas
+        2. If valid, emit decrypted items key. DO NOT replace local root key.
+ * - If by the end we did not find an items key with matching key params to the retrieved
+     key params, AND the retrieved key params are newer than what we have locally, we must
+     issue a sign in request to the server.
+
+ * If the user is not signed in and we detect an undecryptable items key, we present a detached
+ * recovery wizard that doesn't affect our local root key.
+ *
+ * When an items key is emitted, protocol service will automatically try to decrypt any
+ * related items that are in an errored state.
+ *
+ * In the item observer, `ignored` items represent items who have encrypted overwrite
+ * protection enabled (only items keys). This means that if the incoming payload is errored,
+ * but our current copy is not, we will ignore the incoming value until we can properly
+ * decrypt it.
+ */
+export class SNKeyRecoveryService extends AbstractService<KeyRecoveryEvent, DecryptedPayloadInterface[]> {
+  private removeItemObserver: () => void
+  private decryptionQueue: DecryptionQueueItem[] = []
+  private isProcessingQueue = false
+
+  constructor(
+    private itemManager: ItemManager,
+    private payloadManager: PayloadManager,
+    private apiService: SNApiService,
+    private protocolService: EncryptionService,
+    private challengeService: ChallengeService,
+    private alertService: AlertService,
+    private storageService: DiskStorageService,
+    private syncService: SNSyncService,
+    private userService: UserService,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+
+    this.removeItemObserver = this.payloadManager.addObserver(
+      [ContentType.ItemsKey],
+      ({ changed, inserted, ignored, source }) => {
+        if (source === PayloadEmitSource.LocalChanged) {
+          return
+        }
+
+        const changedOrInserted = changed.concat(inserted).filter(isErrorDecryptingPayload)
+
+        if (changedOrInserted.length > 0) {
+          void this.handleUndecryptableItemsKeys(changedOrInserted)
+        }
+
+        if (ignored.length > 0) {
+          void this.handleIgnoredItemsKeys(ignored)
+        }
+      },
+    )
+  }
+
+  public override deinit(): void {
+    ;(this.itemManager as unknown) = undefined
+    ;(this.payloadManager as unknown) = undefined
+    ;(this.apiService as unknown) = undefined
+    ;(this.protocolService as unknown) = undefined
+    ;(this.challengeService as unknown) = undefined
+    ;(this.alertService as unknown) = undefined
+    ;(this.storageService as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    ;(this.userService as unknown) = undefined
+
+    this.removeItemObserver()
+    ;(this.removeItemObserver as unknown) = undefined
+
+    super.deinit()
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  override async handleApplicationStage(stage: ApplicationStage): Promise<void> {
+    void super.handleApplicationStage(stage)
+    if (stage === ApplicationStage.LoadedDatabase_12) {
+      void this.processPersistedUndecryptables()
+    }
+  }
+
+  /**
+   * Ignored items keys are items keys which arrived from a remote source, which we were
+   * not able to decrypt, and for which we already had an existing items key that was
+   * properly decrypted. Since items keys key contents are immutable, if we already have a
+   * successfully decrypted version, yet we can't decrypt the new version, we should
+   * temporarily ignore the new version until we can properly decrypt it (through the recovery flow),
+   * and not overwrite the local copy.
+   *
+   * Ignored items are persisted to disk in isolated storage so that they may be decrypted
+   * whenever. When they are finally decryptable, we will emit them and update our database
+   * with the new decrypted value.
+   *
+   * When the app first launches, we will query the isolated storage to see if there are any
+   * keys we need to decrypt.
+   */
+  private async handleIgnoredItemsKeys(keys: EncryptedPayloadInterface[], persistIncoming = true) {
+    /**
+     * Persist the keys locally in isolated storage, so that if we don't properly decrypt
+     * them in this app session, the user has a chance to later. If there already exists
+     * the same items key in this storage, replace it with this latest incoming value.
+     */
+    if (persistIncoming) {
+      this.saveToUndecryptables(keys)
+    }
+
+    this.addKeysToQueue(keys)
+
+    await this.beginKeyRecovery()
+  }
+
+  private async handleUndecryptableItemsKeys(keys: EncryptedPayloadInterface[]) {
+    this.addKeysToQueue(keys)
+
+    await this.beginKeyRecovery()
+  }
+
+  public presentKeyRecoveryWizard(): void {
+    const invalidKeys = this.itemManager.invalidItems
+      .filter((i) => i.content_type === ContentType.ItemsKey)
+      .map((i) => i.payload)
+
+    void this.handleIgnoredItemsKeys(invalidKeys, false)
+  }
+
+  public canAttemptDecryptionOfItem(item: EncryptedItemInterface): ClientDisplayableError | true {
+    const keyId = item.payload.items_key_id
+
+    if (!keyId) {
+      return new ClientDisplayableError('This item cannot be recovered.')
+    }
+
+    const key = this.payloadManager.findOne(keyId)
+
+    if (!key) {
+      return new ClientDisplayableError(
+        `Unable to find key ${keyId} for this item. You may try signing out and back in; if that doesn't help, check your backup files for a key with this ID and import it.`,
+      )
+    }
+
+    return true
+  }
+
+  public async processPersistedUndecryptables() {
+    const record = this.getUndecryptables()
+
+    const rawPayloads = Object.values(record)
+
+    if (rawPayloads.length === 0) {
+      return
+    }
+
+    const keys = rawPayloads.map((raw) => new EncryptedPayload(raw))
+
+    return this.handleIgnoredItemsKeys(keys, false)
+  }
+
+  private getUndecryptables(): UndecryptableItemsStorage {
+    return this.storageService.getValue<UndecryptableItemsStorage>(
+      StorageKey.KeyRecoveryUndecryptableItems,
+      StorageValueModes.Default,
+      {},
+    )
+  }
+
+  private persistUndecryptables(record: UndecryptableItemsStorage) {
+    this.storageService.setValue(StorageKey.KeyRecoveryUndecryptableItems, record)
+  }
+
+  private saveToUndecryptables(keys: EncryptedPayloadInterface[]) {
+    const record = this.getUndecryptables()
+
+    for (const key of keys) {
+      record[key.uuid] = key.ejected()
+    }
+
+    this.persistUndecryptables(record)
+  }
+
+  private removeFromUndecryptables(keyIds: Uuid[]) {
+    const record = this.getUndecryptables()
+
+    for (const id of keyIds) {
+      delete record[id]
+    }
+
+    this.persistUndecryptables(record)
+  }
+
+  private getClientKeyParams() {
+    return this.protocolService.getAccountKeyParams()
+  }
+
+  private async performServerSignIn(): Promise<SNRootKey | undefined> {
+    const accountPasswordChallenge = new Challenge(
+      [new ChallengePrompt(ChallengeValidation.None, undefined, undefined, true)],
+      ChallengeReason.Custom,
+      true,
+      KeyRecoveryStrings.KeyRecoveryLoginFlowReason,
+    )
+
+    const challengeResponse = await this.challengeService.promptForChallengeResponse(accountPasswordChallenge)
+    if (!challengeResponse) {
+      return undefined
+    }
+
+    this.challengeService.completeChallenge(accountPasswordChallenge)
+
+    const password = challengeResponse.values[0].value as string
+
+    const clientParams = this.getClientKeyParams() as SNRootKeyParams
+
+    const serverParams = await this.getLatestKeyParamsFromServer(clientParams.identifier)
+
+    if (!serverParams || !serverKeyParamsAreSafe(serverParams, clientParams)) {
+      return
+    }
+
+    const rootKey = await this.protocolService.computeRootKey(password, serverParams)
+
+    const signInResponse = await this.userService.correctiveSignIn(rootKey)
+
+    if (!signInResponse.error) {
+      void this.alertService.alert(KeyRecoveryStrings.KeyRecoveryRootKeyReplaced)
+
+      return rootKey
+    } else {
+      await this.alertService.alert(KeyRecoveryStrings.KeyRecoveryLoginFlowInvalidPassword)
+
+      return this.performServerSignIn()
+    }
+  }
+
+  private async getWrappingKeyIfApplicable(): Promise<SNRootKey | undefined> {
+    if (!this.protocolService.hasPasscode()) {
+      return undefined
+    }
+    const { wrappingKey, canceled } = await this.challengeService.getWrappingKeyIfApplicable()
+    if (canceled) {
+      await this.alertService.alert(
+        KeyRecoveryStrings.KeyRecoveryPasscodeRequiredText,
+        KeyRecoveryStrings.KeyRecoveryPasscodeRequiredTitle,
+      )
+
+      return this.getWrappingKeyIfApplicable()
+    }
+    return wrappingKey
+  }
+
+  private addKeysToQueue(keys: EncryptedPayloadInterface[]) {
+    for (const key of keys) {
+      const keyParams = this.protocolService.getKeyEmbeddedKeyParams(key)
+      if (!keyParams) {
+        continue
+      }
+
+      const queueItem: DecryptionQueueItem = {
+        encryptedKey: key,
+        keyParams,
+      }
+
+      this.decryptionQueue.push(queueItem)
+    }
+  }
+
+  private readdQueueItem(queueItem: DecryptionQueueItem) {
+    this.decryptionQueue.unshift(queueItem)
+  }
+
+  private async getLatestKeyParamsFromServer(identifier: string): Promise<SNRootKeyParams | undefined> {
+    const paramsResponse = await this.apiService.getAccountKeyParams({
+      email: identifier,
+    })
+
+    if (!paramsResponse.error && paramsResponse.data) {
+      return KeyParamsFromApiResponse(paramsResponse as KeyParamsResponse)
+    } else {
+      return undefined
+    }
+  }
+
+  private async beginKeyRecovery() {
+    if (this.isProcessingQueue) {
+      return
+    }
+
+    this.isProcessingQueue = true
+
+    const clientParams = this.getClientKeyParams()
+
+    let serverParams: SNRootKeyParams | undefined = undefined
+    if (clientParams) {
+      serverParams = await this.getLatestKeyParamsFromServer(clientParams.identifier)
+    }
+
+    const deallocedAfterNetworkRequest = this.protocolService == undefined
+    if (deallocedAfterNetworkRequest) {
+      return
+    }
+
+    const credentialsMissing = !this.protocolService.hasAccount() && !this.protocolService.hasPasscode()
+
+    if (credentialsMissing) {
+      const rootKey = await this.performServerSignIn()
+
+      if (rootKey) {
+        const replaceLocalRootKeyWithResult = true
+        await this.handleDecryptionOfAllKeysMatchingCorrectRootKey(rootKey, replaceLocalRootKeyWithResult, serverParams)
+      }
+    }
+
+    await this.processQueue(serverParams)
+
+    if (serverParams) {
+      await this.potentiallyPerformFallbackSignInToUpdateOutdatedLocalRootKey(serverParams)
+    }
+
+    if (this.syncService.isOutOfSync()) {
+      void this.syncService.sync({ checkIntegrity: true })
+    }
+  }
+
+  private async potentiallyPerformFallbackSignInToUpdateOutdatedLocalRootKey(serverParams: SNRootKeyParams) {
+    const latestClientParamsAfterAllRecoveryOperations = this.getClientKeyParams()
+
+    if (!latestClientParamsAfterAllRecoveryOperations) {
+      return
+    }
+
+    const serverParamsDiffer = !serverParams.compare(latestClientParamsAfterAllRecoveryOperations)
+
+    if (serverParamsDiffer && serverKeyParamsAreSafe(serverParams, latestClientParamsAfterAllRecoveryOperations)) {
+      await this.performServerSignIn()
+    }
+  }
+
+  private async processQueue(serverParams?: SNRootKeyParams): Promise<void> {
+    let queueItem = this.decryptionQueue[0]
+
+    while (queueItem) {
+      const result = await this.processQueueItem(queueItem, serverParams)
+
+      removeFromArray(this.decryptionQueue, queueItem)
+
+      if (!isSuccessResult(result) && result.aborted) {
+        this.isProcessingQueue = false
+
+        return
+      }
+
+      queueItem = this.decryptionQueue[0]
+    }
+
+    this.isProcessingQueue = false
+  }
+
+  private async processQueueItem(
+    queueItem: DecryptionQueueItem,
+    serverParams?: SNRootKeyParams,
+  ): Promise<KeyRecoveryOperationResult> {
+    const clientParams = this.getClientKeyParams()
+
+    const operation = new KeyRecoveryOperation(
+      queueItem,
+      this.itemManager,
+      this.protocolService,
+      this.challengeService,
+      clientParams,
+      serverParams,
+    )
+
+    const result = await operation.run()
+
+    if (!isSuccessResult(result)) {
+      if (!result.aborted) {
+        await this.alertService.alert(KeyRecoveryStrings.KeyRecoveryUnableToRecover)
+        this.readdQueueItem(queueItem)
+      }
+
+      return result
+    }
+
+    await this.handleDecryptionOfAllKeysMatchingCorrectRootKey(
+      result.rootKey,
+      result.replaceLocalRootKeyWithResult,
+      serverParams,
+    )
+
+    return result
+  }
+
+  private async handleDecryptionOfAllKeysMatchingCorrectRootKey(
+    rootKey: SNRootKey,
+    replacesRootKey: boolean,
+    serverParams?: SNRootKeyParams,
+  ): Promise<void> {
+    if (replacesRootKey) {
+      const wrappingKey = await this.getWrappingKeyIfApplicable()
+
+      await this.protocolService.setRootKey(rootKey, wrappingKey)
+    }
+
+    const clientKeyParams = this.getClientKeyParams()
+
+    const clientParamsMatchServer = clientKeyParams && serverParams && clientKeyParams.compare(serverParams)
+
+    const matchingKeys = this.removeElementsFromQueueForMatchingKeyParams(rootKey.keyParams).map((qItem) => {
+      const needsResync = clientParamsMatchServer && !serverParams.compare(qItem.keyParams)
+
+      return needsResync
+        ? qItem.encryptedKey.copy({ dirty: true, dirtyIndex: getIncrementedDirtyIndex() })
+        : qItem.encryptedKey
+    })
+
+    const matchingResults = await this.protocolService.decryptSplit({
+      usesRootKey: {
+        items: matchingKeys,
+        key: rootKey,
+      },
+    })
+
+    const decryptedMatching = matchingResults.filter(isDecryptedPayload)
+
+    void this.payloadManager.emitPayloads(decryptedMatching, PayloadEmitSource.LocalChanged)
+
+    await this.storageService.savePayloads(decryptedMatching)
+
+    if (replacesRootKey) {
+      void this.alertService.alert(KeyRecoveryStrings.KeyRecoveryRootKeyReplaced)
+    } else {
+      void this.alertService.alert(KeyRecoveryStrings.KeyRecoveryKeyRecovered)
+    }
+
+    if (decryptedMatching.some((p) => p.dirty)) {
+      await this.syncService.sync()
+    }
+
+    await this.notifyEvent(KeyRecoveryEvent.KeysRecovered, decryptedMatching)
+
+    void this.removeFromUndecryptables(Uuids(decryptedMatching))
+  }
+
+  private removeElementsFromQueueForMatchingKeyParams(keyParams: SNRootKeyParams) {
+    const matching = []
+    const nonmatching = []
+
+    for (const queueItem of this.decryptionQueue) {
+      if (queueItem.keyParams.compare(keyParams)) {
+        matching.push(queueItem)
+      } else {
+        nonmatching.push(queueItem)
+      }
+    }
+
+    this.decryptionQueue = nonmatching
+
+    return matching
+  }
+
+  override getDiagnostics(): Promise<DiagnosticInfo | undefined> {
+    return Promise.resolve({
+      keyRecovery: {
+        queueLength: this.decryptionQueue.length,
+        isProcessingQueue: this.isProcessingQueue,
+      },
+    })
+  }
+}

packages/snjs/lib/Services/KeyRecovery/Types.ts

@@ -0,0 +1,36 @@
+import { SNRootKeyParams } from '@standardnotes/encryption'
+import {
+  EncryptedTransferPayload,
+  EncryptedPayloadInterface,
+  DecryptedPayloadInterface,
+  ItemsKeyContent,
+  RootKeyInterface,
+} from '@standardnotes/models'
+import { UuidString } from '@Lib/Types'
+
+export type UndecryptableItemsStorage = Record<UuidString, EncryptedTransferPayload>
+
+export type KeyRecoveryOperationSuccessResult = {
+  rootKey: RootKeyInterface
+  decryptedItemsKey: DecryptedPayloadInterface<ItemsKeyContent>
+  replaceLocalRootKeyWithResult: boolean
+}
+
+export type KeyRecoveryOperationFailResult = {
+  aborted: boolean
+}
+
+export type KeyRecoveryOperationResult = KeyRecoveryOperationSuccessResult | KeyRecoveryOperationFailResult
+
+export function isSuccessResult(x: KeyRecoveryOperationResult): x is KeyRecoveryOperationSuccessResult {
+  return 'rootKey' in x
+}
+
+export type DecryptionQueueItem = {
+  encryptedKey: EncryptedPayloadInterface
+  keyParams: SNRootKeyParams
+}
+
+export enum KeyRecoveryEvent {
+  KeysRecovered = 'KeysRecovered',
+}

packages/snjs/lib/Services/KeyRecovery/Utils.ts

@@ -0,0 +1,6 @@
+import { leftVersionGreaterThanOrEqualToRight } from '@standardnotes/common'
+import { SNRootKeyParams } from '@standardnotes/encryption'
+
+export function serverKeyParamsAreSafe(serverParams: SNRootKeyParams, clientParams: SNRootKeyParams) {
+  return leftVersionGreaterThanOrEqualToRight(serverParams.version, clientParams.version)
+}

packages/snjs/lib/Services/Listed/ListedClientInterface.ts

@@ -0,0 +1,9 @@
+import { Uuid } from '@standardnotes/common'
+import { ListedAccount, ListedAccountInfo } from '@standardnotes/responses'
+
+export interface ListedClientInterface {
+  canRegisterNewListedAccount: () => boolean
+  requestNewListedAccount: () => Promise<ListedAccount | undefined>
+  getListedAccounts(): Promise<ListedAccount[]>
+  getListedAccountInfo(account: ListedAccount, inContextOfItem?: Uuid): Promise<ListedAccountInfo | undefined>
+}

packages/snjs/lib/Services/Listed/ListedService.ts

@@ -0,0 +1,121 @@
+import { isString, lastElement, sleep } from '@standardnotes/utils'
+import { UuidString } from '@Lib/Types/UuidString'
+import { ContentType } from '@standardnotes/common'
+import { ItemManager } from '@Lib/Services/Items/ItemManager'
+import { SNHttpService } from '../Api/HttpService'
+import { SettingName } from '@standardnotes/settings'
+import { SNSettingsService } from '../Settings/SNSettingsService'
+import { ListedClientInterface } from './ListedClientInterface'
+import { SNApiService } from '../Api/ApiService'
+import { ListedAccount, ListedAccountInfo, ListedAccountInfoResponse } from '@standardnotes/responses'
+import { SNActionsExtension } from '@standardnotes/models'
+import { AbstractService, InternalEventBusInterface } from '@standardnotes/services'
+
+export class ListedService extends AbstractService implements ListedClientInterface {
+  constructor(
+    private apiService: SNApiService,
+    private itemManager: ItemManager,
+    private settingsService: SNSettingsService,
+    private httpSerivce: SNHttpService,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  override deinit() {
+    ;(this.itemManager as unknown) = undefined
+    ;(this.settingsService as unknown) = undefined
+    ;(this.apiService as unknown) = undefined
+    ;(this.httpSerivce as unknown) = undefined
+    super.deinit()
+  }
+
+  public canRegisterNewListedAccount(): boolean {
+    return this.apiService.user != undefined
+  }
+
+  /**
+   * Account creation is asyncronous on the backend due to message-based nature of architecture.
+   * In order to get the newly created account, we poll the server to check for new accounts.
+   */
+  public async requestNewListedAccount(): Promise<ListedAccount | undefined> {
+    const accountsBeforeRequest = await this.getSettingsBasedListedAccounts()
+    const response = await this.apiService.registerForListedAccount()
+    if (response.error) {
+      return undefined
+    }
+    const MaxAttempts = 4
+    const DelayBetweenRequests = 3000
+    for (let i = 0; i < MaxAttempts; i++) {
+      const accounts = await this.getSettingsBasedListedAccounts()
+      if (accounts.length > accountsBeforeRequest.length) {
+        return lastElement(accounts)
+      } else {
+        await sleep(DelayBetweenRequests, false)
+      }
+    }
+    return undefined
+  }
+
+  public async getListedAccounts(): Promise<ListedAccount[]> {
+    const settingsBasedAccounts = await this.getSettingsBasedListedAccounts()
+    const legacyAccounts = this.getLegacyListedAccounts()
+
+    return [...settingsBasedAccounts, ...legacyAccounts]
+  }
+
+  public async getListedAccountInfo(
+    account: ListedAccount,
+    inContextOfItem?: UuidString,
+  ): Promise<ListedAccountInfo | undefined> {
+    const hostUrl = account.hostUrl
+    let url = `${hostUrl}/authors/${account.authorId}/extension?secret=${account.secret}`
+    if (inContextOfItem) {
+      url += `&item_uuid=${inContextOfItem}`
+    }
+    const response = (await this.httpSerivce.getAbsolute(url)) as ListedAccountInfoResponse
+    if (response.error || !response.data || isString(response.data)) {
+      return undefined
+    }
+
+    return response.data
+  }
+
+  private async getSettingsBasedListedAccounts(): Promise<ListedAccount[]> {
+    const response = await this.settingsService.getSetting(SettingName.ListedAuthorSecrets)
+    if (!response) {
+      return []
+    }
+    const accounts = JSON.parse(response) as ListedAccount[]
+    return accounts
+  }
+
+  private getLegacyListedAccounts(): ListedAccount[] {
+    const extensions = this.itemManager
+      .getItems<SNActionsExtension>(ContentType.ActionsExtension)
+      .filter((extension) => extension.isListedExtension)
+
+    const accounts: ListedAccount[] = []
+
+    for (const extension of extensions) {
+      const urlString = extension.url
+      const url = new URL(urlString)
+
+      /** Expected path format: '/authors/647/extension/' */
+      const path = url.pathname
+      const authorId = path.split('/')[2]
+
+      /** Expected query string format: '?secret=xxx&type=sn&name=Listed' */
+      const queryString = url.search
+      const key = queryString.split('secret=')[1].split('&')[0]
+
+      accounts.push({
+        secret: key,
+        authorId,
+        hostUrl: url.origin,
+      })
+    }
+
+    return accounts
+  }
+}

packages/snjs/lib/Services/Listed/index.ts

@@ -0,0 +1,2 @@
+export * from './ListedClientInterface'
+export * from './ListedService'

packages/snjs/lib/Services/Mfa/MfaService.ts

@@ -0,0 +1,68 @@
+import { SettingName } from '@standardnotes/settings'
+
+import { SNSettingsService } from '../Settings'
+import * as messages from '../Api/Messages'
+import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
+import { SNFeaturesService } from '../Features/FeaturesService'
+import { FeatureIdentifier } from '@standardnotes/features'
+import { AbstractService, InternalEventBusInterface } from '@standardnotes/services'
+
+export class SNMfaService extends AbstractService {
+  constructor(
+    private settingsService: SNSettingsService,
+    private crypto: PureCryptoInterface,
+    private featuresService: SNFeaturesService,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  private async saveMfaSetting(secret: string): Promise<void> {
+    return await this.settingsService.updateSetting(SettingName.MfaSecret, secret, true)
+  }
+
+  async isMfaActivated(): Promise<boolean> {
+    const mfaSetting = await this.settingsService.getDoesSensitiveSettingExist(SettingName.MfaSecret)
+    return mfaSetting != false
+  }
+
+  async generateMfaSecret(): Promise<string> {
+    return this.crypto.generateOtpSecret()
+  }
+
+  async getOtpToken(secret: string): Promise<string> {
+    return this.crypto.totpToken(secret, Date.now(), 6, 30)
+  }
+
+  async enableMfa(secret: string, otpToken: string): Promise<void> {
+    const otpTokenValid = otpToken != undefined && otpToken === (await this.getOtpToken(secret))
+
+    if (!otpTokenValid) {
+      throw new Error(messages.SignInStrings.IncorrectMfa)
+    }
+
+    return this.saveMfaSetting(secret)
+  }
+
+  async disableMfa(): Promise<void> {
+    return await this.settingsService.deleteSetting(SettingName.MfaSecret)
+  }
+
+  isMfaFeatureAvailable(): boolean {
+    const feature = this.featuresService.getUserFeature(FeatureIdentifier.TwoFactorAuth)
+
+    // If the feature is not present in the collection, we don't want to block it
+    if (feature == undefined) {
+      return false
+    }
+
+    return feature.no_expire === true || (feature.expires_at ?? 0) > Date.now()
+  }
+
+  override deinit(): void {
+    ;(this.settingsService as unknown) = undefined
+    ;(this.crypto as unknown) = undefined
+    ;(this.featuresService as unknown) = undefined
+    super.deinit()
+  }
+}

packages/snjs/lib/Services/Migration/MigrationService.ts

@@ -0,0 +1,151 @@
+import { ApplicationEvent } from '../../Application/Event'
+import { BaseMigration } from '@Lib/Migrations/Base'
+import { compareSemVersions } from '@Lib/Version'
+import { lastElement } from '@standardnotes/utils'
+import { Migration } from '@Lib/Migrations/Migration'
+import { MigrationServices } from '../../Migrations/MigrationServices'
+import {
+  RawStorageKey,
+  namespacedKey,
+  ApplicationStage,
+  AbstractService,
+  DiagnosticInfo,
+} from '@standardnotes/services'
+import { SnjsVersion, isRightVersionGreaterThanLeft } from '../../Version'
+import { SNLog } from '@Lib/Log'
+import { MigrationClasses } from '@Lib/Migrations/Versions'
+
+/**
+ * The migration service orchestrates the execution of multi-stage migrations.
+ * Migrations are registered during initial application launch, and listen for application
+ * life-cycle events, and act accordingly. Migrations operate on the app-level, and not global level.
+ * For example, a single migration may perform a unique set of steps when the application
+ * first launches, and also other steps after the application is unlocked, or after the
+ * first sync completes. Migrations live under /migrations and inherit from the base Migration class.
+ */
+export class SNMigrationService extends AbstractService {
+  private activeMigrations?: Migration[]
+  private baseMigration!: BaseMigration
+
+  constructor(private services: MigrationServices) {
+    super(services.internalEventBus)
+  }
+
+  override deinit(): void {
+    ;(this.services as unknown) = undefined
+
+    if (this.activeMigrations) {
+      this.activeMigrations.length = 0
+    }
+
+    super.deinit()
+  }
+
+  public async initialize(): Promise<void> {
+    await this.runBaseMigrationPreRun()
+
+    const requiredMigrations = SNMigrationService.getRequiredMigrations(await this.getStoredSnjsVersion())
+
+    this.activeMigrations = this.instantiateMigrationClasses(requiredMigrations)
+
+    if (this.activeMigrations.length > 0) {
+      const lastMigration = lastElement(this.activeMigrations) as Migration
+      lastMigration.onDone(async () => {
+        await this.markMigrationsAsDone()
+      })
+    } else {
+      await this.services.deviceInterface.setRawStorageValue(
+        namespacedKey(this.services.identifier, RawStorageKey.SnjsVersion),
+        SnjsVersion,
+      )
+    }
+  }
+
+  private async markMigrationsAsDone() {
+    await this.services.deviceInterface.setRawStorageValue(
+      namespacedKey(this.services.identifier, RawStorageKey.SnjsVersion),
+      SnjsVersion,
+    )
+  }
+
+  private async runBaseMigrationPreRun() {
+    this.baseMigration = new BaseMigration(this.services)
+    await this.baseMigration.preRun()
+  }
+
+  /**
+   * Application instances will call this function directly when they arrive
+   * at a certain migratory state.
+   */
+  public override async handleApplicationStage(stage: ApplicationStage): Promise<void> {
+    await super.handleApplicationStage(stage)
+    await this.handleStage(stage)
+  }
+
+  /**
+   * Called by application
+   */
+  public async handleApplicationEvent(event: ApplicationEvent): Promise<void> {
+    if (event === ApplicationEvent.SignedIn) {
+      await this.handleStage(ApplicationStage.SignedIn_30)
+    }
+  }
+
+  public async hasPendingMigrations(): Promise<boolean> {
+    const requiredMigrations = SNMigrationService.getRequiredMigrations(await this.getStoredSnjsVersion())
+    return requiredMigrations.length > 0 || (await this.baseMigration.needsKeychainRepair())
+  }
+
+  public async getStoredSnjsVersion(): Promise<string> {
+    const version = await this.services.deviceInterface.getRawStorageValue(
+      namespacedKey(this.services.identifier, RawStorageKey.SnjsVersion),
+    )
+    if (!version) {
+      throw SNLog.error(Error('Snjs version missing from storage, run base migration.'))
+    }
+    return version
+  }
+
+  private static getRequiredMigrations(storedVersion: string) {
+    const resultingClasses = []
+    const sortedClasses = MigrationClasses.sort((a, b) => {
+      return compareSemVersions(a.version(), b.version())
+    })
+    for (const migrationClass of sortedClasses) {
+      const migrationVersion = migrationClass.version()
+      if (migrationVersion === storedVersion) {
+        continue
+      }
+      if (isRightVersionGreaterThanLeft(storedVersion, migrationVersion)) {
+        resultingClasses.push(migrationClass)
+      }
+    }
+    return resultingClasses
+  }
+
+  private instantiateMigrationClasses(classes: typeof MigrationClasses): Migration[] {
+    return classes.map((migrationClass) => {
+      return new migrationClass(this.services)
+    })
+  }
+
+  private async handleStage(stage: ApplicationStage) {
+    await this.baseMigration.handleStage(stage)
+
+    if (!this.activeMigrations) {
+      throw new Error('Invalid active migrations')
+    }
+
+    for (const migration of this.activeMigrations) {
+      await migration.handleStage(stage)
+    }
+  }
+
+  override getDiagnostics(): Promise<DiagnosticInfo | undefined> {
+    return Promise.resolve({
+      migrations: {
+        activeMigrations: this.activeMigrations && this.activeMigrations.map((m) => typeof m),
+      },
+    })
+  }
+}

packages/snjs/lib/Services/Mutator/MutatorClientInterface.ts

@@ -0,0 +1,183 @@
+import { ContentType } from '@standardnotes/common'
+import { ChallengeReason, SyncOptions } from '@standardnotes/services'
+import { TransactionalMutation } from '../Items'
+import * as Models from '@standardnotes/models'
+import { ClientDisplayableError } from '@standardnotes/responses'
+import { BackupFile } from '@standardnotes/encryption'
+
+export interface MutatorClientInterface {
+  /**
+   * Inserts the input item by its payload properties, and marks the item as dirty.
+   * A sync is not performed after an item is inserted. This must be handled by the caller.
+   */
+  insertItem(item: Models.DecryptedItemInterface): Promise<Models.DecryptedItemInterface>
+
+  /**
+   * Mutates a pre-existing item, marks it as dirty, and syncs it
+   */
+  changeAndSaveItem<M extends Models.DecryptedItemMutator = Models.DecryptedItemMutator>(
+    itemToLookupUuidFor: Models.DecryptedItemInterface,
+    mutate: (mutator: M) => void,
+    updateTimestamps?: boolean,
+    emitSource?: Models.PayloadEmitSource,
+    syncOptions?: SyncOptions,
+  ): Promise<Models.DecryptedItemInterface | undefined>
+
+  /**
+   * Mutates pre-existing items, marks them as dirty, and syncs
+   */
+  changeAndSaveItems<M extends Models.DecryptedItemMutator = Models.DecryptedItemMutator>(
+    itemsToLookupUuidsFor: Models.DecryptedItemInterface[],
+    mutate: (mutator: M) => void,
+    updateTimestamps?: boolean,
+    emitSource?: Models.PayloadEmitSource,
+    syncOptions?: SyncOptions,
+  ): Promise<void>
+
+  /**
+   * Mutates a pre-existing item and marks it as dirty. Does not sync changes.
+   */
+  changeItem<M extends Models.DecryptedItemMutator>(
+    itemToLookupUuidFor: Models.DecryptedItemInterface,
+    mutate: (mutator: M) => void,
+    updateTimestamps?: boolean,
+  ): Promise<Models.DecryptedItemInterface | undefined>
+
+  /**
+   * Mutates a pre-existing items and marks them as dirty. Does not sync changes.
+   */
+  changeItems<M extends Models.DecryptedItemMutator = Models.DecryptedItemMutator>(
+    itemsToLookupUuidsFor: Models.DecryptedItemInterface[],
+    mutate: (mutator: M) => void,
+    updateTimestamps?: boolean,
+  ): Promise<(Models.DecryptedItemInterface | undefined)[]>
+
+  /**
+   * Run unique mutations per each item in the array, then only propagate all changes
+   * once all mutations have been run. This differs from `changeItems` in that changeItems
+   * runs the same mutation on all items.
+   */
+  runTransactionalMutations(
+    transactions: TransactionalMutation[],
+    emitSource?: Models.PayloadEmitSource,
+    payloadSourceKey?: string,
+  ): Promise<(Models.DecryptedItemInterface | undefined)[]>
+
+  runTransactionalMutation(
+    transaction: TransactionalMutation,
+    emitSource?: Models.PayloadEmitSource,
+    payloadSourceKey?: string,
+  ): Promise<Models.DecryptedItemInterface | undefined>
+
+  protectItems<
+    _M extends Models.DecryptedItemMutator<Models.ItemContent>,
+    I extends Models.DecryptedItemInterface<Models.ItemContent>,
+  >(
+    items: I[],
+  ): Promise<I[]>
+
+  unprotectItems<
+    _M extends Models.DecryptedItemMutator<Models.ItemContent>,
+    I extends Models.DecryptedItemInterface<Models.ItemContent>,
+  >(
+    items: I[],
+    reason: ChallengeReason,
+  ): Promise<I[] | undefined>
+
+  protectNote(note: Models.SNNote): Promise<Models.SNNote>
+
+  unprotectNote(note: Models.SNNote): Promise<Models.SNNote | undefined>
+
+  protectNotes(notes: Models.SNNote[]): Promise<Models.SNNote[]>
+
+  unprotectNotes(notes: Models.SNNote[]): Promise<Models.SNNote[]>
+
+  protectFile(file: Models.FileItem): Promise<Models.FileItem>
+
+  unprotectFile(file: Models.FileItem): Promise<Models.FileItem | undefined>
+
+  /**
+   * Takes the values of the input item and emits it onto global state.
+   */
+  mergeItem(
+    item: Models.DecryptedItemInterface,
+    source: Models.PayloadEmitSource,
+  ): Promise<Models.DecryptedItemInterface>
+
+  /**
+   * Creates an unmanaged item that can be added later.
+   */
+  createTemplateItem<
+    C extends Models.ItemContent = Models.ItemContent,
+    I extends Models.DecryptedItemInterface<C> = Models.DecryptedItemInterface<C>,
+  >(
+    contentType: ContentType,
+    content?: C,
+  ): I
+
+  /**
+   * @param isUserModified  Whether to change the modified date the user
+   * sees of the item.
+   */
+  setItemNeedsSync(
+    item: Models.DecryptedItemInterface,
+    isUserModified?: boolean,
+  ): Promise<Models.DecryptedItemInterface | undefined>
+
+  setItemsNeedsSync(items: Models.DecryptedItemInterface[]): Promise<(Models.DecryptedItemInterface | undefined)[]>
+
+  deleteItem(item: Models.DecryptedItemInterface | Models.EncryptedItemInterface): Promise<void>
+
+  deleteItems(items: (Models.DecryptedItemInterface | Models.EncryptedItemInterface)[]): Promise<void>
+
+  emptyTrash(): Promise<void>
+
+  duplicateItem<T extends Models.DecryptedItemInterface>(item: T, additionalContent?: Partial<T['content']>): Promise<T>
+
+  /**
+   * Migrates any tags containing a '.' character to sa chema-based heirarchy, removing
+   * the dot from the tag's title.
+   */
+  migrateTagsToFolders(): Promise<unknown>
+
+  /**
+   * Establishes a hierarchical relationship between two tags.
+   */
+  setTagParent(parentTag: Models.SNTag, childTag: Models.SNTag): Promise<void>
+
+  /**
+   * Remove the tag parent.
+   */
+  unsetTagParent(childTag: Models.SNTag): Promise<void>
+
+  findOrCreateTag(title: string): Promise<Models.SNTag>
+
+  /** Creates and returns the tag but does not run sync. Callers must perform sync. */
+  createTagOrSmartView(title: string): Promise<Models.SNTag | Models.SmartView>
+
+  /**
+   * Activates or deactivates a component, depending on its
+   * current state, and syncs.
+   */
+  toggleComponent(component: Models.SNComponent): Promise<void>
+
+  toggleTheme(theme: Models.SNComponent): Promise<void>
+
+  /**
+   * @returns
+   * .affectedItems: Items that were either created or dirtied by this import
+   * .errorCount: The number of items that were not imported due to failure to decrypt.
+   */
+  importData(
+    data: BackupFile,
+    awaitSync?: boolean,
+  ): Promise<
+    | {
+        affectedItems: Models.DecryptedItemInterface[]
+        errorCount: number
+      }
+    | {
+        error: ClientDisplayableError
+      }
+  >
+}

packages/snjs/lib/Services/Mutator/MutatorService.spec.ts

@@ -0,0 +1,83 @@
+import { SNHistoryManager } from './../History/HistoryManager'
+import { NoteContent, SNNote, FillItemContent, DecryptedPayload, PayloadTimestampDefaults } from '@standardnotes/models'
+import { EncryptionService } from '@standardnotes/encryption'
+import { ContentType } from '@standardnotes/common'
+import { InternalEventBusInterface } from '@standardnotes/services'
+import {
+  ChallengeService,
+  MutatorService,
+  PayloadManager,
+  SNComponentManager,
+  SNProtectionService,
+  ItemManager,
+  SNSyncService,
+} from '../'
+import { UuidGenerator } from '@standardnotes/utils'
+
+const setupRandomUuid = () => {
+  UuidGenerator.SetGenerator(() => String(Math.random()))
+}
+
+describe('mutator service', () => {
+  let mutatorService: MutatorService
+  let payloadManager: PayloadManager
+  let itemManager: ItemManager
+  let syncService: SNSyncService
+  let protectionService: SNProtectionService
+  let protocolService: EncryptionService
+  let challengeService: ChallengeService
+  let componentManager: SNComponentManager
+  let historyService: SNHistoryManager
+
+  let internalEventBus: InternalEventBusInterface
+
+  beforeEach(() => {
+    setupRandomUuid()
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+
+    payloadManager = new PayloadManager(internalEventBus)
+    itemManager = new ItemManager(payloadManager, { supportsFileNavigation: false }, internalEventBus)
+
+    mutatorService = new MutatorService(
+      itemManager,
+      syncService,
+      protectionService,
+      protocolService,
+      payloadManager,
+      challengeService,
+      componentManager,
+      historyService,
+      internalEventBus,
+    )
+  })
+
+  const insertNote = (title: string) => {
+    const note = new SNNote(
+      new DecryptedPayload({
+        uuid: String(Math.random()),
+        content_type: ContentType.Note,
+        content: FillItemContent<NoteContent>({
+          title: title,
+        }),
+        ...PayloadTimestampDefaults(),
+      }),
+    )
+    return mutatorService.insertItem(note)
+  }
+
+  describe('note modifications', () => {
+    it('pinning should not update timestamps', async () => {
+      const note = await insertNote('hello')
+      const pinnedNote = await mutatorService.changeItem(
+        note,
+        (mutator) => {
+          mutator.pinned = true
+        },
+        false,
+      )
+
+      expect(note.userModifiedDate).toEqual(pinnedNote?.userModifiedDate)
+    })
+  })
+})

packages/snjs/lib/Services/Mutator/MutatorService.ts

@@ -0,0 +1,386 @@
+import { SNHistoryManager } from './../History/HistoryManager'
+import {
+  AbstractService,
+  InternalEventBusInterface,
+  SyncOptions,
+  ChallengeValidation,
+  ChallengePrompt,
+  ChallengeReason,
+} from '@standardnotes/services'
+import { BackupFile, EncryptionProvider } from '@standardnotes/encryption'
+import { ClientDisplayableError } from '@standardnotes/responses'
+import { ContentType, ProtocolVersion, compareVersions } from '@standardnotes/common'
+import { ItemManager, TransactionalMutation } from '../Items'
+import { MutatorClientInterface } from './MutatorClientInterface'
+import { PayloadManager } from '../Payloads/PayloadManager'
+import { SNComponentManager } from '../ComponentManager/ComponentManager'
+import { SNProtectionService } from '../Protection/ProtectionService'
+import { SNSyncService } from '../Sync'
+import { Strings } from '../../Strings'
+import { TagsToFoldersMigrationApplicator } from '@Lib/Migrations/Applicators/TagsToFolders'
+import * as Models from '@standardnotes/models'
+import { Challenge, ChallengeService } from '../Challenge'
+import {
+  CreateDecryptedBackupFileContextPayload,
+  CreateEncryptedBackupFileContextPayload,
+  isDecryptedPayload,
+  isEncryptedTransferPayload,
+} from '@standardnotes/models'
+
+export class MutatorService extends AbstractService implements MutatorClientInterface {
+  constructor(
+    private itemManager: ItemManager,
+    private syncService: SNSyncService,
+    private protectionService: SNProtectionService,
+    private encryption: EncryptionProvider,
+    private payloadManager: PayloadManager,
+    private challengeService: ChallengeService,
+    private componentManager: SNComponentManager,
+    private historyService: SNHistoryManager,
+    protected override internalEventBus: InternalEventBusInterface,
+  ) {
+    super(internalEventBus)
+  }
+
+  override deinit() {
+    super.deinit()
+    ;(this.itemManager as unknown) = undefined
+    ;(this.syncService as unknown) = undefined
+    ;(this.protectionService as unknown) = undefined
+    ;(this.encryption as unknown) = undefined
+    ;(this.payloadManager as unknown) = undefined
+    ;(this.challengeService as unknown) = undefined
+    ;(this.componentManager as unknown) = undefined
+    ;(this.historyService as unknown) = undefined
+  }
+
+  public async insertItem(item: Models.DecryptedItemInterface): Promise<Models.DecryptedItemInterface> {
+    const mutator = Models.CreateDecryptedMutatorForItem(item, Models.MutationType.UpdateUserTimestamps)
+    const dirtiedPayload = mutator.getResult()
+    const insertedItem = await this.itemManager.emitItemFromPayload(
+      dirtiedPayload,
+      Models.PayloadEmitSource.LocalInserted,
+    )
+    return insertedItem
+  }
+
+  public async changeAndSaveItem<M extends Models.DecryptedItemMutator = Models.DecryptedItemMutator>(
+    itemToLookupUuidFor: Models.DecryptedItemInterface,
+    mutate: (mutator: M) => void,
+    updateTimestamps = true,
+    emitSource?: Models.PayloadEmitSource,
+    syncOptions?: SyncOptions,
+  ): Promise<Models.DecryptedItemInterface | undefined> {
+    await this.itemManager.changeItems(
+      [itemToLookupUuidFor],
+      mutate,
+      updateTimestamps ? Models.MutationType.UpdateUserTimestamps : Models.MutationType.NoUpdateUserTimestamps,
+      emitSource,
+    )
+    await this.syncService.sync(syncOptions)
+    return this.itemManager.findItem(itemToLookupUuidFor.uuid)
+  }
+
+  public async changeAndSaveItems<M extends Models.DecryptedItemMutator = Models.DecryptedItemMutator>(
+    itemsToLookupUuidsFor: Models.DecryptedItemInterface[],
+    mutate: (mutator: M) => void,
+    updateTimestamps = true,
+    emitSource?: Models.PayloadEmitSource,
+    syncOptions?: SyncOptions,
+  ): Promise<void> {
+    await this.itemManager.changeItems(
+      itemsToLookupUuidsFor,
+      mutate,
+      updateTimestamps ? Models.MutationType.UpdateUserTimestamps : Models.MutationType.NoUpdateUserTimestamps,
+      emitSource,
+    )
+    await this.syncService.sync(syncOptions)
+  }
+
+  public async changeItem<M extends Models.DecryptedItemMutator>(
+    itemToLookupUuidFor: Models.DecryptedItemInterface,
+    mutate: (mutator: M) => void,
+    updateTimestamps = true,
+  ): Promise<Models.DecryptedItemInterface | undefined> {
+    await this.itemManager.changeItems(
+      [itemToLookupUuidFor],
+      mutate,
+      updateTimestamps ? Models.MutationType.UpdateUserTimestamps : Models.MutationType.NoUpdateUserTimestamps,
+    )
+    return this.itemManager.findItem(itemToLookupUuidFor.uuid)
+  }
+
+  public async changeItems<M extends Models.DecryptedItemMutator = Models.DecryptedItemMutator>(
+    itemsToLookupUuidsFor: Models.DecryptedItemInterface[],
+    mutate: (mutator: M) => void,
+    updateTimestamps = true,
+  ): Promise<(Models.DecryptedItemInterface | undefined)[]> {
+    return this.itemManager.changeItems(
+      itemsToLookupUuidsFor,
+      mutate,
+      updateTimestamps ? Models.MutationType.UpdateUserTimestamps : Models.MutationType.NoUpdateUserTimestamps,
+    )
+  }
+
+  public async runTransactionalMutations(
+    transactions: TransactionalMutation[],
+    emitSource = Models.PayloadEmitSource.LocalChanged,
+    payloadSourceKey?: string,
+  ): Promise<(Models.DecryptedItemInterface | undefined)[]> {
+    return this.itemManager.runTransactionalMutations(transactions, emitSource, payloadSourceKey)
+  }
+
+  public async runTransactionalMutation(
+    transaction: TransactionalMutation,
+    emitSource = Models.PayloadEmitSource.LocalChanged,
+    payloadSourceKey?: string,
+  ): Promise<Models.DecryptedItemInterface | undefined> {
+    return this.itemManager.runTransactionalMutation(transaction, emitSource, payloadSourceKey)
+  }
+
+  async protectItems<M extends Models.DecryptedItemMutator, I extends Models.DecryptedItemInterface>(
+    items: I[],
+  ): Promise<I[]> {
+    const protectedItems = await this.itemManager.changeItems<M, I>(
+      items,
+      (mutator) => {
+        mutator.protected = true
+      },
+      Models.MutationType.NoUpdateUserTimestamps,
+    )
+
+    void this.syncService.sync()
+    return protectedItems
+  }
+
+  async unprotectItems<M extends Models.DecryptedItemMutator, I extends Models.DecryptedItemInterface>(
+    items: I[],
+    reason: ChallengeReason,
+  ): Promise<I[] | undefined> {
+    if (!(await this.protectionService.authorizeAction(reason))) {
+      return undefined
+    }
+
+    const unprotectedItems = await this.itemManager.changeItems<M, I>(
+      items,
+      (mutator) => {
+        mutator.protected = false
+      },
+      Models.MutationType.NoUpdateUserTimestamps,
+    )
+
+    void this.syncService.sync()
+    return unprotectedItems
+  }
+
+  public async protectNote(note: Models.SNNote): Promise<Models.SNNote> {
+    const result = await this.protectItems([note])
+    return result[0]
+  }
+
+  public async unprotectNote(note: Models.SNNote): Promise<Models.SNNote | undefined> {
+    const result = await this.unprotectItems([note], ChallengeReason.UnprotectNote)
+    return result ? result[0] : undefined
+  }
+
+  public async protectNotes(notes: Models.SNNote[]): Promise<Models.SNNote[]> {
+    return this.protectItems(notes)
+  }
+
+  public async unprotectNotes(notes: Models.SNNote[]): Promise<Models.SNNote[]> {
+    const results = await this.unprotectItems(notes, ChallengeReason.UnprotectNote)
+    return results || []
+  }
+
+  async protectFile(file: Models.FileItem): Promise<Models.FileItem> {
+    const result = await this.protectItems([file])
+    return result[0]
+  }
+
+  async unprotectFile(file: Models.FileItem): Promise<Models.FileItem | undefined> {
+    const result = await this.unprotectItems([file], ChallengeReason.UnprotectFile)
+    return result ? result[0] : undefined
+  }
+
+  public async mergeItem(
+    item: Models.DecryptedItemInterface,
+    source: Models.PayloadEmitSource,
+  ): Promise<Models.DecryptedItemInterface> {
+    return this.itemManager.emitItemFromPayload(item.payloadRepresentation(), source)
+  }
+
+  public createTemplateItem<
+    C extends Models.ItemContent = Models.ItemContent,
+    I extends Models.DecryptedItemInterface<C> = Models.DecryptedItemInterface<C>,
+  >(contentType: ContentType, content?: C): I {
+    return this.itemManager.createTemplateItem(contentType, content)
+  }
+
+  public async setItemNeedsSync(
+    item: Models.DecryptedItemInterface,
+    updateTimestamps = false,
+  ): Promise<Models.DecryptedItemInterface | undefined> {
+    return this.itemManager.setItemDirty(item, updateTimestamps)
+  }
+
+  public async setItemsNeedsSync(
+    items: Models.DecryptedItemInterface[],
+  ): Promise<(Models.DecryptedItemInterface | undefined)[]> {
+    return this.itemManager.setItemsDirty(items)
+  }
+
+  public async deleteItem(item: Models.DecryptedItemInterface | Models.EncryptedItemInterface): Promise<void> {
+    return this.deleteItems([item])
+  }
+
+  public async deleteItems(items: (Models.DecryptedItemInterface | Models.EncryptedItemInterface)[]): Promise<void> {
+    await this.itemManager.setItemsToBeDeleted(items)
+    await this.syncService.sync()
+  }
+
+  public async emptyTrash(): Promise<void> {
+    await this.itemManager.emptyTrash()
+    await this.syncService.sync()
+  }
+
+  public duplicateItem<T extends Models.DecryptedItemInterface>(
+    item: T,
+    additionalContent?: Partial<T['content']>,
+  ): Promise<T> {
+    const duplicate = this.itemManager.duplicateItem<T>(item, false, additionalContent)
+    void this.syncService.sync()
+    return duplicate
+  }
+
+  public async migrateTagsToFolders(): Promise<unknown> {
+    await TagsToFoldersMigrationApplicator.run(this.itemManager)
+    return this.syncService.sync()
+  }
+
+  public async setTagParent(parentTag: Models.SNTag, childTag: Models.SNTag): Promise<void> {
+    await this.itemManager.setTagParent(parentTag, childTag)
+  }
+
+  public async unsetTagParent(childTag: Models.SNTag): Promise<void> {
+    await this.itemManager.unsetTagParent(childTag)
+  }
+
+  public async findOrCreateTag(title: string): Promise<Models.SNTag> {
+    return this.itemManager.findOrCreateTagByTitle(title)
+  }
+
+  /** Creates and returns the tag but does not run sync. Callers must perform sync. */
+  public async createTagOrSmartView(title: string): Promise<Models.SNTag | Models.SmartView> {
+    return this.itemManager.createTagOrSmartView(title)
+  }
+
+  public async toggleComponent(component: Models.SNComponent): Promise<void> {
+    await this.componentManager.toggleComponent(component.uuid)
+    await this.syncService.sync()
+  }
+
+  public async toggleTheme(theme: Models.SNComponent): Promise<void> {
+    await this.componentManager.toggleTheme(theme.uuid)
+    await this.syncService.sync()
+  }
+
+  public async importData(
+    data: BackupFile,
+    awaitSync = false,
+  ): Promise<
+    | {
+        affectedItems: Models.DecryptedItemInterface[]
+        errorCount: number
+      }
+    | {
+        error: ClientDisplayableError
+      }
+  > {
+    if (data.version) {
+      /**
+       * Prior to 003 backup files did not have a version field so we cannot
+       * stop importing if there is no backup file version, only if there is
+       * an unsupported version.
+       */
+      const version = data.version as ProtocolVersion
+
+      const supportedVersions = this.encryption.supportedVersions()
+      if (!supportedVersions.includes(version)) {
+        return { error: new ClientDisplayableError(Strings.Info.UnsupportedBackupFileVersion) }
+      }
+
+      const userVersion = this.encryption.getUserVersion()
+      if (userVersion && compareVersions(version, userVersion) === 1) {
+        /** File was made with a greater version than the user's account */
+        return { error: new ClientDisplayableError(Strings.Info.BackupFileMoreRecentThanAccount) }
+      }
+    }
+
+    let password: string | undefined
+
+    if (data.auth_params || data.keyParams) {
+      /** Get import file password. */
+      const challenge = new Challenge(
+        [new ChallengePrompt(ChallengeValidation.None, Strings.Input.FileAccountPassword, undefined, true)],
+        ChallengeReason.DecryptEncryptedFile,
+        true,
+      )
+      const passwordResponse = await this.challengeService.promptForChallengeResponse(challenge)
+      if (passwordResponse == undefined) {
+        /** Challenge was canceled */
+        return { error: new ClientDisplayableError('Import aborted') }
+      }
+      this.challengeService.completeChallenge(challenge)
+      password = passwordResponse?.values[0].value as string
+    }
+
+    if (!(await this.protectionService.authorizeFileImport())) {
+      return { error: new ClientDisplayableError('Import aborted') }
+    }
+
+    data.items = data.items.map((item) => {
+      if (isEncryptedTransferPayload(item)) {
+        return CreateEncryptedBackupFileContextPayload(item)
+      } else {
+        return CreateDecryptedBackupFileContextPayload(item as Models.BackupFileDecryptedContextualPayload)
+      }
+    })
+
+    const decryptedPayloadsOrError = await this.encryption.decryptBackupFile(data, password)
+
+    if (decryptedPayloadsOrError instanceof ClientDisplayableError) {
+      return { error: decryptedPayloadsOrError }
+    }
+
+    const validPayloads = decryptedPayloadsOrError.filter(isDecryptedPayload).map((payload) => {
+      /* Don't want to activate any components during import process in
+       * case of exceptions breaking up the import proccess */
+      if (payload.content_type === ContentType.Component && (payload.content as Models.ComponentContent).active) {
+        const typedContent = payload as Models.DecryptedPayloadInterface<Models.ComponentContent>
+        return Models.CopyPayloadWithContentOverride(typedContent, {
+          active: false,
+        })
+      } else {
+        return payload
+      }
+    })
+
+    const affectedUuids = await this.payloadManager.importPayloads(
+      validPayloads,
+      this.historyService.getHistoryMapCopy(),
+    )
+
+    const promise = this.syncService.sync()
+
+    if (awaitSync) {
+      await promise
+    }
+
+    const affectedItems = this.itemManager.findItems(affectedUuids) as Models.DecryptedItemInterface[]
+
+    return {
+      affectedItems: affectedItems,
+      errorCount: decryptedPayloadsOrError.length - validPayloads.length,
+    }
+  }
+}

packages/snjs/lib/Services/Mutator/index.ts

@@ -0,0 +1,2 @@
+export * from './MutatorClientInterface'
+export * from './MutatorService'

packages/snjs/lib/Services/Payloads/PayloadManager.spec.ts

@@ -0,0 +1,53 @@
+import {
+  DecryptedPayload,
+  FillItemContent,
+  ItemsKeyContent,
+  PayloadEmitSource,
+  PayloadTimestampDefaults,
+} from '@standardnotes/models'
+import { PayloadManager } from './PayloadManager'
+import { InternalEventBusInterface } from '@standardnotes/services'
+import { ContentType } from '@standardnotes/common'
+
+describe('payload manager', () => {
+  let payloadManager: PayloadManager
+  let internalEventBus: InternalEventBusInterface
+
+  beforeEach(() => {
+    internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
+    internalEventBus.publish = jest.fn()
+
+    payloadManager = new PayloadManager(internalEventBus)
+  })
+
+  it('emitting a payload should emit as-is and not merge on top of existing payload', async () => {
+    const decrypted = new DecryptedPayload({
+      uuid: '123',
+      content_type: ContentType.ItemsKey,
+      content: FillItemContent<ItemsKeyContent>({
+        itemsKey: 'secret',
+      }),
+      ...PayloadTimestampDefaults(),
+      updated_at_timestamp: 1,
+      dirty: true,
+    })
+
+    await payloadManager.emitPayload(decrypted, PayloadEmitSource.LocalInserted)
+
+    const nondirty = new DecryptedPayload({
+      uuid: '123',
+      content_type: ContentType.ItemsKey,
+      ...PayloadTimestampDefaults(),
+      updated_at_timestamp: 2,
+      content: FillItemContent<ItemsKeyContent>({
+        itemsKey: 'secret',
+      }),
+    })
+
+    await payloadManager.emitPayload(nondirty, PayloadEmitSource.LocalChanged)
+
+    const result = payloadManager.findOne('123')
+
+    expect(result?.dirty).toBeFalsy()
+  })
+})

packages/snjs/lib/Services/Payloads/PayloadManager.ts

@@ -0,0 +1,338 @@
+import { ContentType, Uuid } from '@standardnotes/common'
+import { PayloadsChangeObserver, QueueElement, PayloadsChangeObserverCallback, EmitQueue } from './Types'
+import { removeFromArray, Uuids } from '@standardnotes/utils'
+import {
+  DeltaFileImport,
+  isDeletedPayload,
+  ImmutablePayloadCollection,
+  EncryptedPayloadInterface,
+  PayloadSource,
+  DeletedPayloadInterface,
+  DecryptedPayloadInterface,
+  PayloadCollection,
+  PayloadEmitSource,
+  DeletedPayload,
+  FullyFormedPayloadInterface,
+  isEncryptedPayload,
+  isDecryptedPayload,
+  HistoryMap,
+  DeltaEmit,
+  getIncrementedDirtyIndex,
+} from '@standardnotes/models'
+import {
+  AbstractService,
+  PayloadManagerInterface,
+  InternalEventBusInterface,
+  DiagnosticInfo,
+} from '@standardnotes/services'
+import { IntegrityPayload } from '@standardnotes/responses'
+
+/**
+ * The payload manager is responsible for keeping state regarding what items exist in the
+ * global application state. It does so by exposing functions that allow consumers to 'map'
+ * a detached payload into global application state. Whenever a change is made or retrieved
+ * from any source, it must be mapped in order to be properly reflected in global application state.
+ * The model manager deals only with in-memory state, and does not deal directly with storage.
+ * It also serves as a query store, and can be queried for current notes, tags, etc.
+ * It exposes methods that allow consumers to listen to mapping events. This is how
+ * applications 'stream' items to display in the interface.
+ */
+export class PayloadManager extends AbstractService implements PayloadManagerInterface {
+  private changeObservers: PayloadsChangeObserver[] = []
+  public collection: PayloadCollection<FullyFormedPayloadInterface>
+  private emitQueue: EmitQueue<FullyFormedPayloadInterface> = []
+
+  constructor(protected override internalEventBus: InternalEventBusInterface) {
+    super(internalEventBus)
+    this.collection = new PayloadCollection()
+  }
+
+  /**
+   * Our payload collection keeps the latest mapped payload for every payload
+   * that passes through our mapping function. Use this to query current state
+   * as needed to make decisions, like about duplication or uuid alteration.
+   */
+  public getMasterCollection() {
+    return ImmutablePayloadCollection.FromCollection(this.collection)
+  }
+
+  public override deinit() {
+    super.deinit()
+    this.changeObservers.length = 0
+    this.resetState()
+  }
+
+  public resetState() {
+    this.collection = new PayloadCollection()
+  }
+
+  public find(uuids: Uuid[]): FullyFormedPayloadInterface[] {
+    return this.collection.findAll(uuids)
+  }
+
+  public findOne(uuid: Uuid): FullyFormedPayloadInterface | undefined {
+    return this.collection.findAll([uuid])[0]
+  }
+
+  public all(contentType: ContentType): FullyFormedPayloadInterface[] {
+    return this.collection.all(contentType)
+  }
+
+  public get integrityPayloads(): IntegrityPayload[] {
+    return this.collection.integrityPayloads()
+  }
+
+  public get nonDeletedItems(): FullyFormedPayloadInterface[] {
+    return this.collection.nondeletedElements()
+  }
+
+  public get invalidPayloads(): EncryptedPayloadInterface[] {
+    return this.collection.invalidElements()
+  }
+
+  public async emitDeltaEmit<P extends FullyFormedPayloadInterface = FullyFormedPayloadInterface>(
+    emit: DeltaEmit<P>,
+    sourceKey?: string,
+  ): Promise<P[]> {
+    if (emit.emits.length === 0 && emit.ignored?.length === 0) {
+      return []
+    }
+
+    return new Promise((resolve) => {
+      const element: QueueElement<P> = {
+        emit: emit,
+        sourceKey,
+        resolve,
+      }
+
+      this.emitQueue.push(element as unknown as QueueElement<FullyFormedPayloadInterface>)
+
+      if (this.emitQueue.length === 1) {
+        void this.popQueue()
+      }
+    })
+  }
+
+  /**
+   * One of many mapping helpers available.
+   * This function maps a payload to an item
+   * @returns every paylod altered as a result of this operation, to be
+   * saved to storage by the caller
+   */
+  public async emitPayload<P extends FullyFormedPayloadInterface = FullyFormedPayloadInterface>(
+    payload: P,
+    source: PayloadEmitSource,
+    sourceKey?: string,
+  ): Promise<P[]> {
+    return this.emitPayloads([payload], source, sourceKey)
+  }
+
+  /**
+   * This function maps multiple payloads to items, and is the authoratative mapping
+   * function that all other mapping helpers rely on
+   * @returns every paylod altered as a result of this operation, to be
+   * saved to storage by the caller
+   */
+  public async emitPayloads<P extends FullyFormedPayloadInterface = FullyFormedPayloadInterface>(
+    payloads: P[],
+    source: PayloadEmitSource,
+    sourceKey?: string,
+  ): Promise<P[]> {
+    const emit: DeltaEmit<P> = {
+      emits: payloads,
+      source: source,
+    }
+
+    return this.emitDeltaEmit(emit, sourceKey)
+  }
+
+  private popQueue() {
+    const first = this.emitQueue[0]
+
+    const { changed, inserted, discarded, unerrored } = this.applyPayloads(first.emit.emits)
+
+    this.notifyChangeObservers(
+      changed,
+      inserted,
+      discarded,
+      first.emit.ignored || [],
+      unerrored,
+      first.emit.source,
+      first.sourceKey,
+    )
+
+    removeFromArray(this.emitQueue, first)
+
+    first.resolve([...changed, ...inserted, ...discarded])
+
+    if (this.emitQueue.length > 0) {
+      void this.popQueue()
+    }
+  }
+
+  private applyPayloads(applyPayloads: FullyFormedPayloadInterface[]) {
+    const changed: FullyFormedPayloadInterface[] = []
+    const inserted: FullyFormedPayloadInterface[] = []
+    const discarded: DeletedPayloadInterface[] = []
+    const unerrored: DecryptedPayloadInterface[] = []
+
+    for (const apply of applyPayloads) {
+      if (!apply.uuid || !apply.content_type) {
+        console.error('Payload is corrupt', apply)
+
+        continue
+      }
+
+      this.log(
+        'applying payload',
+        apply.uuid,
+        'globalDirtyIndexAtLastSync',
+        apply.globalDirtyIndexAtLastSync,
+        'dirtyIndex',
+        apply.dirtyIndex,
+        'dirty',
+        apply.dirty,
+      )
+
+      const base = this.collection.find(apply.uuid)
+
+      if (isDeletedPayload(apply) && apply.discardable) {
+        this.collection.discard(apply)
+
+        discarded.push(apply)
+      } else {
+        this.collection.set(apply)
+
+        if (base) {
+          changed.push(apply)
+
+          if (isEncryptedPayload(base) && isDecryptedPayload(apply)) {
+            unerrored.push(apply)
+          }
+        } else {
+          inserted.push(apply)
+        }
+      }
+    }
+
+    return { changed, inserted, discarded, unerrored }
+  }
+
+  /**
+   * Notifies observers when an item has been mapped.
+   * @param types - An array of content types to listen for
+   * @param priority - The lower the priority, the earlier the function is called
+   *  wrt to other observers
+   */
+  public addObserver(types: ContentType | ContentType[], callback: PayloadsChangeObserverCallback, priority = 1) {
+    if (!Array.isArray(types)) {
+      types = [types]
+    }
+    const observer: PayloadsChangeObserver = {
+      types,
+      priority,
+      callback,
+    }
+    this.changeObservers.push(observer)
+
+    const thislessChangeObservers = this.changeObservers
+    return () => {
+      removeFromArray(thislessChangeObservers, observer)
+    }
+  }
+
+  /**
+   * This function is mostly for internal use, but can be used externally by consumers who
+   * explicitely understand what they are doing (want to propagate model state without mapping)
+   */
+  public notifyChangeObservers(
+    changed: FullyFormedPayloadInterface[],
+    inserted: FullyFormedPayloadInterface[],
+    discarded: DeletedPayloadInterface[],
+    ignored: EncryptedPayloadInterface[],
+    unerrored: DecryptedPayloadInterface[],
+    source: PayloadEmitSource,
+    sourceKey?: string,
+  ) {
+    /** Slice the observers array as sort modifies in-place */
+    const observers = this.changeObservers.slice().sort((a, b) => {
+      return a.priority < b.priority ? -1 : 1
+    })
+
+    const filter = <P extends FullyFormedPayloadInterface = FullyFormedPayloadInterface>(
+      payloads: P[],
+      types: ContentType[],
+    ) => {
+      return types.includes(ContentType.Any)
+        ? payloads.slice()
+        : payloads.slice().filter((payload) => {
+            return types.includes(payload.content_type)
+          })
+    }
+
+    for (const observer of observers) {
+      observer.callback({
+        changed: filter(changed, observer.types),
+        inserted: filter(inserted, observer.types),
+        discarded: filter(discarded, observer.types),
+        ignored: filter(ignored, observer.types),
+        unerrored: filter(unerrored, observer.types),
+        source,
+        sourceKey,
+      })
+    }
+  }
+
+  /**
+   * Imports an array of payloads from an external source (such as a backup file)
+   * and marks the items as dirty.
+   * @returns Resulting items
+   */
+  public async importPayloads(payloads: DecryptedPayloadInterface[], historyMap: HistoryMap): Promise<Uuid[]> {
+    const sourcedPayloads = payloads.map((p) => p.copy(undefined, PayloadSource.FileImport))
+
+    const delta = new DeltaFileImport(this.getMasterCollection(), sourcedPayloads, historyMap)
+
+    const emit = delta.result()
+
+    await this.emitDeltaEmit(emit)
+
+    return Uuids(payloads)
+  }
+
+  public removePayloadLocally(payload: FullyFormedPayloadInterface) {
+    this.collection.discard(payload)
+  }
+
+  public erroredPayloadsForContentType(contentType: ContentType): EncryptedPayloadInterface[] {
+    return this.collection.invalidElements().filter((p) => p.content_type === contentType)
+  }
+
+  public async deleteErroredPayloads(payloads: EncryptedPayloadInterface[]): Promise<void> {
+    const deleted = payloads.map(
+      (payload) =>
+        new DeletedPayload(
+          {
+            ...payload.ejected(),
+            deleted: true,
+            content: undefined,
+            dirty: true,
+            dirtyIndex: getIncrementedDirtyIndex(),
+          },
+          payload.source,
+        ),
+    )
+
+    await this.emitPayloads(deleted, PayloadEmitSource.LocalChanged)
+  }
+
+  override getDiagnostics(): Promise<DiagnosticInfo | undefined> {
+    return Promise.resolve({
+      payloads: {
+        integrityPayloads: this.integrityPayloads,
+        nonDeletedItemCount: this.nonDeletedItems.length,
+        invalidPayloadsCount: this.invalidPayloads.length,
+      },
+    })
+  }
+}