feat: improve initial load performance on mobile (#2126)

packages/services/src/Domain/Application/ApplicationInterface.ts

@@ -1,7 +1,6 @@
 import { ApplicationIdentifier, ContentType } from '@standardnotes/common'
 import { BackupFile, DecryptedItemInterface, ItemStream, Platform, PrefKey, PrefValue } from '@standardnotes/models'
 import { FilesClientInterface } from '@standardnotes/files'
-
 import { AlertService } from '../Alert/AlertService'
 import { ComponentManagerInterface } from '../Component/ComponentManagerInterface'
 import { ApplicationEvent } from '../Event/ApplicationEvent'

packages/services/src/Domain/Device/DatabaseItemMetadata.ts

@@ -0,0 +1,3 @@
+import { TransferPayload } from '@standardnotes/models'
+
+export type DatabaseItemMetadata = Pick<TransferPayload, 'uuid' | 'updated_at' | 'content_type'>

packages/services/src/Domain/Device/DatabaseLoadOptions.ts

@@ -0,0 +1,44 @@
+import { ContentType } from '@standardnotes/common'
+import { FullyFormedTransferPayload } from '@standardnotes/models'
+
+export type DatabaseKeysLoadChunk = {
+  keys: string[]
+}
+
+export type DatabaseFullEntryLoadChunk = {
+  entries: FullyFormedTransferPayload[]
+}
+
+export function isChunkFullEntry(
+  x: DatabaseKeysLoadChunk | DatabaseFullEntryLoadChunk,
+): x is DatabaseFullEntryLoadChunk {
+  return (x as DatabaseFullEntryLoadChunk).entries !== undefined
+}
+
+export type DatabaseKeysLoadChunkResponse = {
+  keys: {
+    itemsKeys: DatabaseKeysLoadChunk
+    remainingChunks: DatabaseKeysLoadChunk[]
+  }
+  remainingChunksItemCount: number
+}
+
+export type DatabaseFullEntryLoadChunkResponse = {
+  fullEntries: {
+    itemsKeys: DatabaseFullEntryLoadChunk
+    remainingChunks: DatabaseFullEntryLoadChunk[]
+  }
+  remainingChunksItemCount: number
+}
+
+export function isFullEntryLoadChunkResponse(
+  x: DatabaseKeysLoadChunkResponse | DatabaseFullEntryLoadChunkResponse,
+): x is DatabaseFullEntryLoadChunkResponse {
+  return (x as DatabaseFullEntryLoadChunkResponse).fullEntries !== undefined
+}
+
+export type DatabaseLoadOptions = {
+  contentTypePriority: ContentType[]
+  uuidPriority: string[]
+  batchSize: number
+}

packages/services/src/Domain/Device/DatabaseLoadSorter.spec.ts

@@ -0,0 +1,150 @@
+import { ContentType } from '@standardnotes/common'
+import { FullyFormedPayloadInterface } from '@standardnotes/models'
+import { GetSortedPayloadsByPriority } from './DatabaseLoadSorter'
+
+describe('GetSortedPayloadsByPriority', () => {
+  let payloads: FullyFormedPayloadInterface[] = []
+  const contentTypePriority = [ContentType.ItemsKey, ContentType.UserPrefs, ContentType.Component, ContentType.Theme]
+  let launchPriorityUuids: string[] = []
+
+  it('should sort payloads based on content type priority', () => {
+    payloads = [
+      {
+        content_type: ContentType.Theme,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.UserPrefs,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Component,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.ItemsKey,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+      } as FullyFormedPayloadInterface,
+    ]
+
+    const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(payloads, {
+      contentTypePriority,
+      uuidPriority: launchPriorityUuids,
+      batchSize: 1000,
+    })
+
+    expect(itemsKeyPayloads.length).toBe(1)
+    expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey)
+
+    expect(contentTypePriorityPayloads.length).toBe(3)
+    expect(contentTypePriorityPayloads[0].content_type).toBe(ContentType.UserPrefs)
+    expect(contentTypePriorityPayloads[1].content_type).toBe(ContentType.Component)
+    expect(contentTypePriorityPayloads[2].content_type).toBe(ContentType.Theme)
+
+    expect(remainingPayloads.length).toBe(1)
+    expect(remainingPayloads[0].content_type).toBe(ContentType.Note)
+  })
+
+  it('should sort payloads based on launch priority uuids', () => {
+    const unprioritizedNoteUuid = 'unprioritized-note'
+    const unprioritizedTagUuid = 'unprioritized-tag'
+
+    const prioritizedNoteUuid = 'prioritized-note'
+    const prioritizedTagUuid = 'prioritized-tag'
+
+    payloads = [
+      {
+        content_type: ContentType.Theme,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.UserPrefs,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Component,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.ItemsKey,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+        uuid: unprioritizedNoteUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: unprioritizedTagUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+        uuid: prioritizedNoteUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: prioritizedTagUuid,
+      } as FullyFormedPayloadInterface,
+    ]
+
+    launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid]
+
+    const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(payloads, {
+      contentTypePriority,
+      uuidPriority: launchPriorityUuids,
+      batchSize: 1000,
+    })
+
+    expect(itemsKeyPayloads.length).toBe(1)
+    expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey)
+
+    expect(contentTypePriorityPayloads.length).toBe(3)
+    expect(contentTypePriorityPayloads[0].content_type).toBe(ContentType.UserPrefs)
+    expect(contentTypePriorityPayloads[1].content_type).toBe(ContentType.Component)
+    expect(contentTypePriorityPayloads[2].content_type).toBe(ContentType.Theme)
+
+    expect(remainingPayloads.length).toBe(4)
+    expect(remainingPayloads[0].uuid).toBe(prioritizedNoteUuid)
+    expect(remainingPayloads[1].uuid).toBe(prioritizedTagUuid)
+    expect(remainingPayloads[2].uuid).toBe(unprioritizedNoteUuid)
+    expect(remainingPayloads[3].uuid).toBe(unprioritizedTagUuid)
+  })
+
+  it('should sort payloads based on server updated date if same content type', () => {
+    const unprioritizedNoteUuid = 'unprioritized-note'
+    const unprioritizedTagUuid = 'unprioritized-tag'
+
+    const prioritizedNoteUuid = 'prioritized-note'
+    const prioritizedTagUuid = 'prioritized-tag'
+
+    payloads = [
+      {
+        content_type: ContentType.Note,
+        uuid: unprioritizedNoteUuid,
+        updated_at: new Date(1),
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: unprioritizedTagUuid,
+        updated_at: new Date(2),
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+        uuid: prioritizedNoteUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: prioritizedTagUuid,
+      } as FullyFormedPayloadInterface,
+    ]
+
+    launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid]
+
+    const { remainingPayloads } = GetSortedPayloadsByPriority(payloads, {
+      contentTypePriority,
+      uuidPriority: launchPriorityUuids,
+      batchSize: 1000,
+    })
+
+    expect(remainingPayloads.length).toBe(4)
+    expect(remainingPayloads[0].uuid).toBe(prioritizedNoteUuid)
+    expect(remainingPayloads[1].uuid).toBe(prioritizedTagUuid)
+    expect(remainingPayloads[2].uuid).toBe(unprioritizedTagUuid)
+    expect(remainingPayloads[3].uuid).toBe(unprioritizedNoteUuid)
+  })
+})

packages/services/src/Domain/Device/DatabaseLoadSorter.ts

@@ -0,0 +1,113 @@
+import { DatabaseItemMetadata } from './DatabaseItemMetadata'
+import { DatabaseLoadOptions } from './DatabaseLoadOptions'
+import { ContentType, Uuid } from '@standardnotes/common'
+
+/**
+ * Sorts payloads according by most recently modified first, according to the priority,
+ * whereby the earlier a content_type appears in the priorityList,
+ * the earlier it will appear in the resulting sorted array.
+ */
+function SortPayloadsByRecentAndContentPriority<T extends DatabaseItemMetadata = DatabaseItemMetadata>(
+  payloads: T[],
+  contentTypePriorityList: ContentType[],
+): T[] {
+  return payloads.sort((a, b) => {
+    const dateResult = new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
+
+    let aPriority = 0
+    let bPriority = 0
+
+    aPriority = contentTypePriorityList.indexOf(a.content_type)
+    bPriority = contentTypePriorityList.indexOf(b.content_type)
+
+    if (aPriority === -1) {
+      aPriority = contentTypePriorityList.length
+    }
+
+    if (bPriority === -1) {
+      bPriority = contentTypePriorityList.length
+    }
+
+    if (aPriority === bPriority) {
+      return dateResult
+    }
+
+    if (aPriority < bPriority) {
+      return -1
+    } else {
+      return 1
+    }
+  })
+}
+
+/**
+ * Sorts payloads according by most recently modified first, according to the priority,
+ * whereby the earlier a uuid appears in the priorityList,
+ * the earlier it will appear in the resulting sorted array.
+ */
+function SortPayloadsByRecentAndUuidPriority<T extends DatabaseItemMetadata = DatabaseItemMetadata>(
+  payloads: T[],
+  uuidPriorityList: Uuid[],
+): T[] {
+  return payloads.sort((a, b) => {
+    const dateResult = new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
+
+    let aPriority = 0
+    let bPriority = 0
+
+    aPriority = uuidPriorityList.indexOf(a.uuid)
+    bPriority = uuidPriorityList.indexOf(b.uuid)
+
+    if (aPriority === -1) {
+      aPriority = uuidPriorityList.length
+    }
+
+    if (bPriority === -1) {
+      bPriority = uuidPriorityList.length
+    }
+
+    if (aPriority === bPriority) {
+      return dateResult
+    }
+
+    if (aPriority < bPriority) {
+      return -1
+    } else {
+      return 1
+    }
+  })
+}
+
+export function GetSortedPayloadsByPriority<T extends DatabaseItemMetadata = DatabaseItemMetadata>(
+  payloads: T[],
+  options: DatabaseLoadOptions,
+): {
+  itemsKeyPayloads: T[]
+  contentTypePriorityPayloads: T[]
+  remainingPayloads: T[]
+} {
+  const itemsKeyPayloads: T[] = []
+  const contentTypePriorityPayloads: T[] = []
+  const remainingPayloads: T[] = []
+
+  for (let index = 0; index < payloads.length; index++) {
+    const payload = payloads[index]
+
+    if (payload.content_type === ContentType.ItemsKey) {
+      itemsKeyPayloads.push(payload)
+    } else if (options.contentTypePriority.includes(payload.content_type)) {
+      contentTypePriorityPayloads.push(payload)
+    } else {
+      remainingPayloads.push(payload)
+    }
+  }
+
+  return {
+    itemsKeyPayloads,
+    contentTypePriorityPayloads: SortPayloadsByRecentAndContentPriority(
+      contentTypePriorityPayloads,
+      options.contentTypePriority,
+    ),
+    remainingPayloads: SortPayloadsByRecentAndUuidPriority(remainingPayloads, options.uuidPriority),
+  }
+}

packages/services/src/Domain/Device/DeviceInterface.ts

@@ -2,10 +2,14 @@ import { ApplicationIdentifier } from '@standardnotes/common'
 import {
   FullyFormedTransferPayload,
   TransferPayload,
-  LegacyRawKeychainValue,
   NamespacedRootKeyInKeychain,
   Environment,
 } from '@standardnotes/models'
+import {
+  DatabaseLoadOptions,
+  DatabaseKeysLoadChunkResponse,
+  DatabaseFullEntryLoadChunkResponse,
+} from './DatabaseLoadOptions'
 
 /**
  * Platforms must override this class to provide platform specific utilities
@@ -21,8 +25,6 @@ export interface DeviceInterface {
 
   getJsonParsedRawStorageValue(key: string): Promise<unknown | undefined>
 
-  getAllRawStorageKeyValues(): Promise<{ key: string; value: unknown }[]>
-
   setRawStorageValue(key: string, value: string): Promise<void>
 
   removeRawStorageValue(key: string): Promise<void>
@@ -38,10 +40,10 @@ export interface DeviceInterface {
    */
   openDatabase(identifier: ApplicationIdentifier): Promise<{ isNewDatabase?: boolean } | undefined>
 
-  /**
-   * In a key/value database, this function returns just the keys.
-   */
-  getDatabaseKeys(): Promise<string[]>
+  getDatabaseLoadChunks(
+    options: DatabaseLoadOptions,
+    identifier: ApplicationIdentifier,
+  ): Promise<DatabaseKeysLoadChunkResponse | DatabaseFullEntryLoadChunkResponse>
 
   /**
    * Remove all keychain and database data from device.
@@ -52,17 +54,22 @@ export interface DeviceInterface {
    */
   clearAllDataFromDevice(workspaceIdentifiers: ApplicationIdentifier[]): Promise<{ killsApplication: boolean }>
 
-  getAllRawDatabasePayloads<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
+  getAllDatabaseEntries<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
     identifier: ApplicationIdentifier,
   ): Promise<T[]>
 
-  saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void>
+  getDatabaseEntries<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
+    identifier: ApplicationIdentifier,
+    keys: string[],
+  ): Promise<T[]>
 
-  saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void>
+  saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void>
 
-  removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier): Promise<void>
+  saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void>
 
-  removeAllRawDatabasePayloads(identifier: ApplicationIdentifier): Promise<void>
+  removeDatabaseEntry(id: string, identifier: ApplicationIdentifier): Promise<void>
+
+  removeAllDatabaseEntries(identifier: ApplicationIdentifier): Promise<void>
 
   getNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<NamespacedRootKeyInKeychain | undefined>
 
@@ -70,8 +77,6 @@ export interface DeviceInterface {
 
   clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<void>
 
-  setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise<void>
-
   clearRawKeychainValue(): Promise<void>
 
   openUrl(url: string): void

packages/services/src/Domain/Storage/StorageServiceInterface.ts

@@ -1,7 +1,13 @@
-import { FullyFormedPayloadInterface, PayloadInterface, RootKeyInterface } from '@standardnotes/models'
+import {
+  FullyFormedPayloadInterface,
+  PayloadInterface,
+  RootKeyInterface,
+  FullyFormedTransferPayload,
+} from '@standardnotes/models'
 import { StoragePersistencePolicies, StorageValueModes } from './StorageTypes'
 
 export interface StorageServiceInterface {
+  getAllRawPayloads(): Promise<FullyFormedTransferPayload[]>
   getValue<T>(key: string, mode?: StorageValueModes, defaultValue?: T): T
   canDecryptWithKey(key: RootKeyInterface): Promise<boolean>
   savePayload(payload: PayloadInterface): Promise<void>

packages/services/src/Domain/Sync/SyncOptions.ts

@@ -11,6 +11,7 @@ export type SyncOptions = {
   checkIntegrity?: boolean
   /** Internally used to keep track of how sync requests were spawned. */
   source: SyncSource
+  sourceDescription?: string
   /** Whether to await any sync requests that may be queued from this call. */
   awaitAll?: boolean
   /**

packages/services/src/Domain/index.ts

@@ -22,6 +22,9 @@ export * from './Device/DeviceInterface'
 export * from './Device/MobileDeviceInterface'
 export * from './Device/TypeCheck'
 export * from './Device/WebOrDesktopDeviceInterface'
+export * from './Device/DatabaseLoadOptions'
+export * from './Device/DatabaseItemMetadata'
+export * from './Device/DatabaseLoadSorter'
 export * from './Diagnostics/ServiceDiagnostics'
 export * from './Encryption/BackupFileDecryptor'
 export * from './Encryption/EncryptionService'