feat: improve initial load performance on mobile (#2126)

packages/mobile/src/Lib/Database/Database.ts

@@ -0,0 +1,158 @@
+import AsyncStorage from '@react-native-community/async-storage'
+import {
+  DatabaseKeysLoadChunk,
+  DatabaseKeysLoadChunkResponse,
+  DatabaseLoadOptions,
+  GetSortedPayloadsByPriority,
+  TransferPayload,
+} from '@standardnotes/snjs'
+import { Platform } from 'react-native'
+import { DatabaseInterface } from './DatabaseInterface'
+import { DatabaseMetadata } from './DatabaseMetadata'
+import { FlashKeyValueStore } from './FlashKeyValueStore'
+import { isLegacyIdentifier } from './LegacyIdentifier'
+import { showLoadFailForItemIds } from './showLoadFailForItemIds'
+
+export class Database implements DatabaseInterface {
+  private metadataStore: DatabaseMetadata
+
+  constructor(private identifier: string) {
+    const flashStorage = new FlashKeyValueStore(identifier)
+    this.metadataStore = new DatabaseMetadata(identifier, flashStorage)
+  }
+
+  private databaseKeyForPayloadId(id: string) {
+    return `${this.getDatabaseKeyPrefix()}${id}`
+  }
+
+  private getDatabaseKeyPrefix() {
+    if (this.identifier && !isLegacyIdentifier(this.identifier)) {
+      return `${this.identifier}-Item-`
+    } else {
+      return 'Item-'
+    }
+  }
+
+  async getAllEntries<T extends TransferPayload = TransferPayload>(): Promise<T[]> {
+    const keys = await this.getAllKeys()
+    return this.multiGet(keys)
+  }
+
+  async getAllKeys(): Promise<string[]> {
+    const keys = await AsyncStorage.getAllKeys()
+    const filtered = keys.filter((key) => {
+      return key.startsWith(this.getDatabaseKeyPrefix())
+    })
+    return filtered
+  }
+
+  async multiDelete(keys: string[]): Promise<void> {
+    return AsyncStorage.multiRemove(keys)
+  }
+
+  async deleteItem(itemUuid: string): Promise<void> {
+    const key = this.databaseKeyForPayloadId(itemUuid)
+    this.metadataStore.deleteMetadataItem(itemUuid)
+    return this.multiDelete([key])
+  }
+
+  async deleteAll(): Promise<void> {
+    const keys = await this.getAllKeys()
+    return this.multiDelete(keys)
+  }
+
+  async setItems(items: TransferPayload[]): Promise<void> {
+    if (items.length === 0) {
+      return
+    }
+
+    await Promise.all(
+      items.map((item) => {
+        return Promise.all([
+          AsyncStorage.setItem(this.databaseKeyForPayloadId(item.uuid), JSON.stringify(item)),
+          this.metadataStore.setMetadataForPayloads([item]),
+        ])
+      }),
+    )
+  }
+
+  async getLoadChunks(options: DatabaseLoadOptions): Promise<DatabaseKeysLoadChunkResponse> {
+    let metadataItems = this.metadataStore.getAllMetadataItems()
+
+    if (metadataItems.length === 0) {
+      const allEntries = await this.getAllEntries()
+      metadataItems = this.metadataStore.runMigration(allEntries)
+    }
+
+    const sorted = GetSortedPayloadsByPriority(metadataItems, options)
+
+    const itemsKeysChunk: DatabaseKeysLoadChunk = {
+      keys: sorted.itemsKeyPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)),
+    }
+
+    const contentTypePriorityChunk: DatabaseKeysLoadChunk = {
+      keys: sorted.contentTypePriorityPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)),
+    }
+
+    const remainingKeys = sorted.remainingPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid))
+
+    const remainingKeysChunks: DatabaseKeysLoadChunk[] = []
+    for (let i = 0; i < remainingKeys.length; i += options.batchSize) {
+      remainingKeysChunks.push({
+        keys: remainingKeys.slice(i, i + options.batchSize),
+      })
+    }
+
+    const result: DatabaseKeysLoadChunkResponse = {
+      keys: {
+        itemsKeys: itemsKeysChunk,
+        remainingChunks: [contentTypePriorityChunk, ...remainingKeysChunks],
+      },
+      remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length,
+    }
+
+    return result
+  }
+
+  async multiGet<T>(keys: string[]): Promise<T[]> {
+    const results: T[] = []
+
+    if (Platform.OS === 'android') {
+      const failedItemIds: string[] = []
+      for (const key of keys) {
+        try {
+          const item = await AsyncStorage.getItem(key)
+          if (item) {
+            try {
+              results.push(JSON.parse(item) as T)
+            } catch (e) {
+              results.push(item as T)
+            }
+          }
+        } catch (e) {
+          console.error('Error getting item', key, e)
+          failedItemIds.push(key)
+        }
+      }
+      if (failedItemIds.length > 0) {
+        showLoadFailForItemIds(failedItemIds)
+      }
+    } else {
+      try {
+        for (const item of await AsyncStorage.multiGet(keys)) {
+          if (item[1]) {
+            try {
+              results.push(JSON.parse(item[1]))
+            } catch (e) {
+              results.push(item[1] as T)
+            }
+          }
+        }
+      } catch (e) {
+        console.error('Error getting items', e)
+      }
+    }
+
+    return results
+  }
+}

packages/mobile/src/Lib/Database/DatabaseInterface.ts

@@ -0,0 +1,10 @@
+import { TransferPayload } from '@standardnotes/snjs'
+
+export interface DatabaseInterface {
+  getAllKeys(): Promise<string[]>
+  multiDelete(keys: string[]): Promise<void>
+  deleteItem(itemUuid: string): Promise<void>
+  deleteAll(): Promise<void>
+  setItems(items: TransferPayload[]): Promise<void>
+  multiGet<T>(keys: string[]): Promise<T[]>
+}

packages/mobile/src/Lib/Database/DatabaseMetadata.ts

@@ -0,0 +1,39 @@
+import { DatabaseItemMetadata, isNotUndefined, TransferPayload } from '@standardnotes/snjs'
+import { FlashKeyValueStore } from './FlashKeyValueStore'
+
+export class DatabaseMetadata {
+  constructor(private identifier: string, private flashStorage: FlashKeyValueStore) {}
+
+  runMigration(payloads: TransferPayload[]) {
+    const metadataItems = this.setMetadataForPayloads(payloads)
+    return metadataItems
+  }
+
+  setMetadataForPayloads(payloads: TransferPayload[]) {
+    const metadataItems = []
+    for (const payload of payloads) {
+      const { uuid, content_type, updated_at } = payload
+      const key = this.keyForUuid(uuid)
+      const metadata: DatabaseItemMetadata = { uuid, content_type, updated_at }
+      this.flashStorage.set(key, metadata)
+      metadataItems.push(metadata)
+    }
+    return metadataItems
+  }
+
+  deleteMetadataItem(itemUuid: string) {
+    const key = this.keyForUuid(itemUuid)
+    this.flashStorage.delete(key)
+  }
+
+  getAllMetadataItems(): DatabaseItemMetadata[] {
+    const keys = this.flashStorage.getAllKeys()
+    const metadataKeys = keys.filter((key) => key.endsWith('-Metadata'))
+    const metadataItems = this.flashStorage.multiGet<DatabaseItemMetadata>(metadataKeys).filter(isNotUndefined)
+    return metadataItems
+  }
+
+  private keyForUuid(uuid: string) {
+    return `${this.identifier}-Item-${uuid}-Metadata`
+  }
+}

packages/mobile/src/Lib/Database/FlashKeyValueStore.ts

@@ -0,0 +1,40 @@
+import { MMKV } from 'react-native-mmkv'
+
+export class FlashKeyValueStore {
+  private storage: MMKV
+
+  constructor(identifier: string) {
+    this.storage = new MMKV({ id: identifier })
+  }
+
+  set(key: string, value: unknown): void {
+    this.storage.set(key, JSON.stringify(value))
+  }
+
+  delete(key: string): void {
+    this.storage.delete(key)
+  }
+
+  deleteAll(): void {
+    this.storage.clearAll()
+  }
+
+  getAllKeys(): string[] {
+    return this.storage.getAllKeys()
+  }
+
+  get<T>(key: string): T | undefined {
+    const item = this.storage.getString(key)
+    if (item) {
+      try {
+        return JSON.parse(item)
+      } catch (e) {
+        return item as T
+      }
+    }
+  }
+
+  multiGet<T>(keys: string[]): (T | undefined)[] {
+    return keys.map((key) => this.get<T>(key))
+  }
+}

packages/mobile/src/Lib/Database/LegacyIdentifier.ts

@@ -0,0 +1,15 @@
+import { ApplicationIdentifier } from '@standardnotes/snjs'
+
+/**
+ * This identifier was the database name used in Standard Notes web/desktop.
+ */
+const LEGACY_IDENTIFIER = 'standardnotes'
+
+/**
+ * We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not.
+ * It is also used to decide if the raw or the namespaced keychain is used.
+ * @param identifier The ApplicationIdentifier
+ */
+export const isLegacyIdentifier = function (identifier: ApplicationIdentifier) {
+  return identifier && identifier === LEGACY_IDENTIFIER
+}

packages/mobile/src/Lib/Database/LegacyKeyValueStore.ts

@@ -0,0 +1,26 @@
+import AsyncStorage from '@react-native-community/async-storage'
+
+export class LegacyKeyValueStore {
+  set(key: string, value: string): Promise<void> {
+    return AsyncStorage.setItem(key, JSON.stringify(value))
+  }
+
+  delete(key: string): Promise<void> {
+    return AsyncStorage.removeItem(key)
+  }
+
+  deleteAll(): Promise<void> {
+    return AsyncStorage.clear()
+  }
+
+  async getValue<T>(key: string): Promise<T | undefined> {
+    const item = await AsyncStorage.getItem(key)
+    if (item) {
+      try {
+        return JSON.parse(item)
+      } catch (e) {
+        return item as T
+      }
+    }
+  }
+}

packages/mobile/src/Lib/Database/showLoadFailForItemIds.ts

@@ -0,0 +1,16 @@
+import { Alert } from 'react-native'
+
+export const showLoadFailForItemIds = (failedItemIds: string[]) => {
+  let text =
+    'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n'
+  let index = 0
+  text += failedItemIds.map((id) => {
+    let result = id
+    if (index !== failedItemIds.length - 1) {
+      result += '\n'
+    }
+    index++
+    return result
+  })
+  Alert.alert('Unable to load item(s)', text)
+}