feat: prioritize loading latest selected items (#1930)

packages/snjs/lib/Services/Preferences/PreferencesService.ts

@@ -35,7 +35,7 @@ export class SNPreferencesService
     })
 
     this.removeSyncObserver = syncService.addEventObserver((event) => {
-      if (event === SyncEvent.SyncCompletedWithAllItemsUploaded) {
+      if (event === SyncEvent.SyncCompletedWithAllItemsUploaded || event === SyncEvent.LocalDataIncrementalLoad) {
         void this.reload()
       }
     })

packages/snjs/lib/Services/Sync/SyncClientInterface.ts

@@ -2,6 +2,8 @@ import { SyncOpStatus } from './SyncOpStatus'
 import { SyncOptions } from '@standardnotes/services'
 
 export interface SyncClientInterface {
+  setLaunchPriorityUuids(launchPriorityUuids: string[]): void
+
   sync(options?: Partial<SyncOptions>): Promise<unknown>
 
   isOutOfSync(): boolean

packages/snjs/lib/Services/Sync/SyncService.ts

@@ -18,7 +18,7 @@ import { SNHistoryManager } from '../History/HistoryManager'
 import { SNLog } from '@Lib/Log'
 import { SNSessionManager } from '../Session/SessionManager'
 import { DiskStorageService } from '../Storage/DiskStorageService'
-import { SortPayloadsByRecentAndContentPriority } from '@Lib/Services/Sync/Utils'
+import { GetSortedPayloadsByPriority } from '@Lib/Services/Sync/Utils'
 import { SyncClientInterface } from './SyncClientInterface'
 import { SyncPromise } from './Types'
 import { SyncOpStatus } from '@Lib/Services/Sync/SyncOpStatus'
@@ -56,6 +56,7 @@ import {
   PayloadEmitSource,
   getIncrementedDirtyIndex,
   getCurrentDirtyIndex,
+  ItemContent,
 } from '@standardnotes/models'
 import {
   AbstractService,
@@ -158,6 +159,14 @@ export class SNSyncService
     }
   }
 
+  private get launchPriorityUuids() {
+    return this.storageService.getValue<string[]>(StorageKey.LaunchPriorityUuids) ?? []
+  }
+
+  public setLaunchPriorityUuids(launchPriorityUuids: string[]) {
+    this.storageService.setValue(StorageKey.LaunchPriorityUuids, launchPriorityUuids)
+  }
+
   public override deinit(): void {
     this.dealloced = true
     ;(this.sessionManager as unknown) = undefined
@@ -272,15 +281,15 @@ export class SNSyncService
       })
       .filter(isNotUndefined)
 
-    const payloads = SortPayloadsByRecentAndContentPriority(unsortedPayloads, this.localLoadPriorty)
+    const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(
+      unsortedPayloads,
+      this.localLoadPriorty,
+      this.launchPriorityUuids,
+    )
 
-    const itemsKeysPayloads = payloads.filter((payload) => {
-      return payload.content_type === ContentType.ItemsKey
-    })
+    await this.processItemsKeysFirstDuringDatabaseLoad(itemsKeyPayloads)
 
-    subtractFromArray(payloads, itemsKeysPayloads)
-
-    await this.processItemsKeysFirstDuringDatabaseLoad(itemsKeysPayloads)
+    await this.processPayloadBatch(contentTypePriorityPayloads)
 
     /**
      * Map in batches to give interface a chance to update. Note that total decryption
@@ -288,45 +297,55 @@ export class SNSyncService
      * batches will result in the same time spent. It's the emitting/painting/rendering
      * that requires batch size optimization.
      */
-    const payloadCount = payloads.length
+    const payloadCount = remainingPayloads.length
     const batchSize = this.options.loadBatchSize
     const numBatches = Math.ceil(payloadCount / batchSize)
 
     for (let batchIndex = 0; batchIndex < numBatches; batchIndex++) {
       const currentPosition = batchIndex * batchSize
-      const batch = payloads.slice(currentPosition, currentPosition + batchSize)
-      const encrypted: EncryptedPayloadInterface[] = []
-      const nonencrypted: (DecryptedPayloadInterface | DeletedPayloadInterface)[] = []
-
-      for (const payload of batch) {
-        if (isEncryptedPayload(payload)) {
-          encrypted.push(payload)
-        } else {
-          nonencrypted.push(payload)
-        }
-      }
-
-      const split: KeyedDecryptionSplit = {
-        usesItemsKeyWithKeyLookup: {
-          items: encrypted,
-        },
-      }
-
-      const results = await this.protocolService.decryptSplit(split)
-
-      await this.payloadManager.emitPayloads([...nonencrypted, ...results], PayloadEmitSource.LocalDatabaseLoaded)
-
-      void this.notifyEvent(SyncEvent.LocalDataIncrementalLoad)
-
-      this.opStatus.setDatabaseLoadStatus(currentPosition, payloadCount, false)
-
-      await sleep(1, false)
+      const batch = remainingPayloads.slice(currentPosition, currentPosition + batchSize)
+      await this.processPayloadBatch(batch, currentPosition, payloadCount)
     }
 
     this.databaseLoaded = true
     this.opStatus.setDatabaseLoadStatus(0, 0, true)
   }
 
+  private async processPayloadBatch(
+    batch: FullyFormedPayloadInterface<ItemContent>[],
+    currentPosition?: number,
+    payloadCount?: number,
+  ) {
+    const encrypted: EncryptedPayloadInterface[] = []
+    const nonencrypted: (DecryptedPayloadInterface | DeletedPayloadInterface)[] = []
+
+    for (const payload of batch) {
+      if (isEncryptedPayload(payload)) {
+        encrypted.push(payload)
+      } else {
+        nonencrypted.push(payload)
+      }
+    }
+
+    const split: KeyedDecryptionSplit = {
+      usesItemsKeyWithKeyLookup: {
+        items: encrypted,
+      },
+    }
+
+    const results = await this.protocolService.decryptSplit(split)
+
+    await this.payloadManager.emitPayloads([...nonencrypted, ...results], PayloadEmitSource.LocalDatabaseLoaded)
+
+    void this.notifyEvent(SyncEvent.LocalDataIncrementalLoad)
+
+    if (currentPosition != undefined && payloadCount != undefined) {
+      this.opStatus.setDatabaseLoadStatus(currentPosition, payloadCount, false)
+    }
+
+    await sleep(1, false)
+  }
+
   private setLastSyncToken(token: string) {
     this.syncToken = token
     return this.storageService.setValue(StorageKey.LastSyncToken, token)

packages/snjs/lib/Services/Sync/Utils.spec.ts

@@ -0,0 +1,146 @@
+import { ContentType } from '@standardnotes/common'
+import { FullyFormedPayloadInterface } from '@standardnotes/models'
+import { GetSortedPayloadsByPriority } from './Utils'
+
+describe('GetSortedPayloadsByPriority', () => {
+  let payloads: FullyFormedPayloadInterface[] = []
+  const contentTypePriority = [ContentType.ItemsKey, ContentType.UserPrefs, ContentType.Component, ContentType.Theme]
+  let launchPriorityUuids: string[] = []
+
+  it('should sort payloads based on content type priority', () => {
+    payloads = [
+      {
+        content_type: ContentType.Theme,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.UserPrefs,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Component,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.ItemsKey,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+      } as FullyFormedPayloadInterface,
+    ]
+
+    const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(
+      payloads,
+      contentTypePriority,
+      launchPriorityUuids,
+    )
+
+    expect(itemsKeyPayloads.length).toBe(1)
+    expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey)
+
+    expect(contentTypePriorityPayloads.length).toBe(3)
+    expect(contentTypePriorityPayloads[0].content_type).toBe(ContentType.UserPrefs)
+    expect(contentTypePriorityPayloads[1].content_type).toBe(ContentType.Component)
+    expect(contentTypePriorityPayloads[2].content_type).toBe(ContentType.Theme)
+
+    expect(remainingPayloads.length).toBe(1)
+    expect(remainingPayloads[0].content_type).toBe(ContentType.Note)
+  })
+
+  it('should sort payloads based on launch priority uuids', () => {
+    const unprioritizedNoteUuid = 'unprioritized-note'
+    const unprioritizedTagUuid = 'unprioritized-tag'
+
+    const prioritizedNoteUuid = 'prioritized-note'
+    const prioritizedTagUuid = 'prioritized-tag'
+
+    payloads = [
+      {
+        content_type: ContentType.Theme,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.UserPrefs,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Component,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.ItemsKey,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+        uuid: unprioritizedNoteUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: unprioritizedTagUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+        uuid: prioritizedNoteUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: prioritizedTagUuid,
+      } as FullyFormedPayloadInterface,
+    ]
+
+    launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid]
+
+    const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(
+      payloads,
+      contentTypePriority,
+      launchPriorityUuids,
+    )
+
+    expect(itemsKeyPayloads.length).toBe(1)
+    expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey)
+
+    expect(contentTypePriorityPayloads.length).toBe(3)
+    expect(contentTypePriorityPayloads[0].content_type).toBe(ContentType.UserPrefs)
+    expect(contentTypePriorityPayloads[1].content_type).toBe(ContentType.Component)
+    expect(contentTypePriorityPayloads[2].content_type).toBe(ContentType.Theme)
+
+    expect(remainingPayloads.length).toBe(4)
+    expect(remainingPayloads[0].uuid).toBe(prioritizedNoteUuid)
+    expect(remainingPayloads[1].uuid).toBe(prioritizedTagUuid)
+    expect(remainingPayloads[2].uuid).toBe(unprioritizedNoteUuid)
+    expect(remainingPayloads[3].uuid).toBe(unprioritizedTagUuid)
+  })
+
+  it('should sort payloads based on server updated date if same content type', () => {
+    const unprioritizedNoteUuid = 'unprioritized-note'
+    const unprioritizedTagUuid = 'unprioritized-tag'
+
+    const prioritizedNoteUuid = 'prioritized-note'
+    const prioritizedTagUuid = 'prioritized-tag'
+
+    payloads = [
+      {
+        content_type: ContentType.Note,
+        uuid: unprioritizedNoteUuid,
+        serverUpdatedAt: new Date(1),
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: unprioritizedTagUuid,
+        serverUpdatedAt: new Date(2),
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Note,
+        uuid: prioritizedNoteUuid,
+      } as FullyFormedPayloadInterface,
+      {
+        content_type: ContentType.Tag,
+        uuid: prioritizedTagUuid,
+      } as FullyFormedPayloadInterface,
+    ]
+
+    launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid]
+
+    const { remainingPayloads } = GetSortedPayloadsByPriority(payloads, contentTypePriority, launchPriorityUuids)
+
+    expect(remainingPayloads.length).toBe(4)
+    expect(remainingPayloads[0].uuid).toBe(prioritizedNoteUuid)
+    expect(remainingPayloads[1].uuid).toBe(prioritizedTagUuid)
+    expect(remainingPayloads[2].uuid).toBe(unprioritizedTagUuid)
+    expect(remainingPayloads[3].uuid).toBe(unprioritizedNoteUuid)
+  })
+})

packages/snjs/lib/Services/Sync/Utils.ts

@@ -1,3 +1,4 @@
+import { UuidString } from '@Lib/Types'
 import { ContentType } from '@standardnotes/common'
 import { FullyFormedPayloadInterface } from '@standardnotes/models'
 
@@ -6,9 +7,9 @@ import { FullyFormedPayloadInterface } from '@standardnotes/models'
  * whereby the earlier a content_type appears in the priorityList,
  * the earlier it will appear in the resulting sorted array.
  */
-export function SortPayloadsByRecentAndContentPriority(
+function SortPayloadsByRecentAndContentPriority(
   payloads: FullyFormedPayloadInterface[],
-  priorityList: ContentType[],
+  contentTypePriorityList: ContentType[],
 ): FullyFormedPayloadInterface[] {
   return payloads.sort((a, b) => {
     const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime()
@@ -16,18 +17,15 @@ export function SortPayloadsByRecentAndContentPriority(
     let aPriority = 0
     let bPriority = 0
 
-    if (priorityList) {
-      aPriority = priorityList.indexOf(a.content_type)
-      bPriority = priorityList.indexOf(b.content_type)
+    aPriority = contentTypePriorityList.indexOf(a.content_type)
+    bPriority = contentTypePriorityList.indexOf(b.content_type)
 
-      if (aPriority === -1) {
-        /** Not found in list, not prioritized. Set it to max value */
-        aPriority = priorityList.length
-      }
-      if (bPriority === -1) {
-        /** Not found in list, not prioritized. Set it to max value */
-        bPriority = priorityList.length
-      }
+    if (aPriority === -1) {
+      aPriority = contentTypePriorityList.length
+    }
+
+    if (bPriority === -1) {
+      bPriority = contentTypePriorityList.length
     }
 
     if (aPriority === bPriority) {
@@ -41,3 +39,76 @@ export function SortPayloadsByRecentAndContentPriority(
     }
   })
 }
+
+/**
+ * Sorts payloads according by most recently modified first, according to the priority,
+ * whereby the earlier a uuid appears in the priorityList,
+ * the earlier it will appear in the resulting sorted array.
+ */
+function SortPayloadsByRecentAndUuidPriority(
+  payloads: FullyFormedPayloadInterface[],
+  uuidPriorityList: UuidString[],
+): FullyFormedPayloadInterface[] {
+  return payloads.sort((a, b) => {
+    const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime()
+
+    let aPriority = 0
+    let bPriority = 0
+
+    aPriority = uuidPriorityList.indexOf(a.uuid)
+    bPriority = uuidPriorityList.indexOf(b.uuid)
+
+    if (aPriority === -1) {
+      aPriority = uuidPriorityList.length
+    }
+
+    if (bPriority === -1) {
+      bPriority = uuidPriorityList.length
+    }
+
+    if (aPriority === bPriority) {
+      return dateResult
+    }
+
+    if (aPriority < bPriority) {
+      return -1
+    } else {
+      return 1
+    }
+  })
+}
+
+export function GetSortedPayloadsByPriority(
+  payloads: FullyFormedPayloadInterface[],
+  contentTypePriorityList: ContentType[],
+  uuidPriorityList: UuidString[],
+): {
+  itemsKeyPayloads: FullyFormedPayloadInterface[]
+  contentTypePriorityPayloads: FullyFormedPayloadInterface[]
+  remainingPayloads: FullyFormedPayloadInterface[]
+} {
+  const itemsKeyPayloads: FullyFormedPayloadInterface[] = []
+  const contentTypePriorityPayloads: FullyFormedPayloadInterface[] = []
+  const remainingPayloads: FullyFormedPayloadInterface[] = []
+
+  for (let index = 0; index < payloads.length; index++) {
+    const payload = payloads[index]
+
+    if (payload.content_type === ContentType.ItemsKey) {
+      itemsKeyPayloads.push(payload)
+    } else if (contentTypePriorityList.includes(payload.content_type)) {
+      contentTypePriorityPayloads.push(payload)
+    } else {
+      remainingPayloads.push(payload)
+    }
+  }
+
+  return {
+    itemsKeyPayloads,
+    contentTypePriorityPayloads: SortPayloadsByRecentAndContentPriority(
+      contentTypePriorityPayloads,
+      contentTypePriorityList,
+    ),
+    remainingPayloads: SortPayloadsByRecentAndUuidPriority(remainingPayloads, uuidPriorityList),
+  }
+}

packages/snjs/mocha/sync_tests/online.test.js

@@ -672,10 +672,10 @@ describe('online syncing', function () {
       const payload = Factory.createStorageItemPayload(contentTypes[Math.floor(i / 2)])
       originalPayloads.push(payload)
     }
-    const sorted = SortPayloadsByRecentAndContentPriority(originalPayloads, ['C', 'A', 'B'])
-    expect(sorted[0].content_type).to.equal('C')
-    expect(sorted[2].content_type).to.equal('A')
-    expect(sorted[4].content_type).to.equal('B')
+    const { contentTypePriorityPayloads } = GetSortedPayloadsByPriority(originalPayloads, ['C', 'A', 'B'])
+    expect(contentTypePriorityPayloads[0].content_type).to.equal('C')
+    expect(contentTypePriorityPayloads[2].content_type).to.equal('A')
+    expect(contentTypePriorityPayloads[4].content_type).to.equal('B')
   })
 
   it('should sign in and retrieve large number of items', async function () {