feat: improve initial load performance on mobile (#2126)

This commit is contained in:
Mo
2023-01-03 14:15:45 -06:00
committed by GitHub
parent a447fa1ad7
commit 3c332a35f6
59 changed files with 868 additions and 3003 deletions

View File

@@ -0,0 +1,158 @@
import AsyncStorage from '@react-native-community/async-storage'
import {
DatabaseKeysLoadChunk,
DatabaseKeysLoadChunkResponse,
DatabaseLoadOptions,
GetSortedPayloadsByPriority,
TransferPayload,
} from '@standardnotes/snjs'
import { Platform } from 'react-native'
import { DatabaseInterface } from './DatabaseInterface'
import { DatabaseMetadata } from './DatabaseMetadata'
import { FlashKeyValueStore } from './FlashKeyValueStore'
import { isLegacyIdentifier } from './LegacyIdentifier'
import { showLoadFailForItemIds } from './showLoadFailForItemIds'
export class Database implements DatabaseInterface {
private metadataStore: DatabaseMetadata
constructor(private identifier: string) {
const flashStorage = new FlashKeyValueStore(identifier)
this.metadataStore = new DatabaseMetadata(identifier, flashStorage)
}
private databaseKeyForPayloadId(id: string) {
return `${this.getDatabaseKeyPrefix()}${id}`
}
private getDatabaseKeyPrefix() {
if (this.identifier && !isLegacyIdentifier(this.identifier)) {
return `${this.identifier}-Item-`
} else {
return 'Item-'
}
}
async getAllEntries<T extends TransferPayload = TransferPayload>(): Promise<T[]> {
const keys = await this.getAllKeys()
return this.multiGet(keys)
}
async getAllKeys(): Promise<string[]> {
const keys = await AsyncStorage.getAllKeys()
const filtered = keys.filter((key) => {
return key.startsWith(this.getDatabaseKeyPrefix())
})
return filtered
}
async multiDelete(keys: string[]): Promise<void> {
return AsyncStorage.multiRemove(keys)
}
async deleteItem(itemUuid: string): Promise<void> {
const key = this.databaseKeyForPayloadId(itemUuid)
this.metadataStore.deleteMetadataItem(itemUuid)
return this.multiDelete([key])
}
async deleteAll(): Promise<void> {
const keys = await this.getAllKeys()
return this.multiDelete(keys)
}
async setItems(items: TransferPayload[]): Promise<void> {
if (items.length === 0) {
return
}
await Promise.all(
items.map((item) => {
return Promise.all([
AsyncStorage.setItem(this.databaseKeyForPayloadId(item.uuid), JSON.stringify(item)),
this.metadataStore.setMetadataForPayloads([item]),
])
}),
)
}
async getLoadChunks(options: DatabaseLoadOptions): Promise<DatabaseKeysLoadChunkResponse> {
let metadataItems = this.metadataStore.getAllMetadataItems()
if (metadataItems.length === 0) {
const allEntries = await this.getAllEntries()
metadataItems = this.metadataStore.runMigration(allEntries)
}
const sorted = GetSortedPayloadsByPriority(metadataItems, options)
const itemsKeysChunk: DatabaseKeysLoadChunk = {
keys: sorted.itemsKeyPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)),
}
const contentTypePriorityChunk: DatabaseKeysLoadChunk = {
keys: sorted.contentTypePriorityPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)),
}
const remainingKeys = sorted.remainingPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid))
const remainingKeysChunks: DatabaseKeysLoadChunk[] = []
for (let i = 0; i < remainingKeys.length; i += options.batchSize) {
remainingKeysChunks.push({
keys: remainingKeys.slice(i, i + options.batchSize),
})
}
const result: DatabaseKeysLoadChunkResponse = {
keys: {
itemsKeys: itemsKeysChunk,
remainingChunks: [contentTypePriorityChunk, ...remainingKeysChunks],
},
remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length,
}
return result
}
async multiGet<T>(keys: string[]): Promise<T[]> {
const results: T[] = []
if (Platform.OS === 'android') {
const failedItemIds: string[] = []
for (const key of keys) {
try {
const item = await AsyncStorage.getItem(key)
if (item) {
try {
results.push(JSON.parse(item) as T)
} catch (e) {
results.push(item as T)
}
}
} catch (e) {
console.error('Error getting item', key, e)
failedItemIds.push(key)
}
}
if (failedItemIds.length > 0) {
showLoadFailForItemIds(failedItemIds)
}
} else {
try {
for (const item of await AsyncStorage.multiGet(keys)) {
if (item[1]) {
try {
results.push(JSON.parse(item[1]))
} catch (e) {
results.push(item[1] as T)
}
}
}
} catch (e) {
console.error('Error getting items', e)
}
}
return results
}
}

View File

@@ -0,0 +1,10 @@
import { TransferPayload } from '@standardnotes/snjs'
export interface DatabaseInterface {
getAllKeys(): Promise<string[]>
multiDelete(keys: string[]): Promise<void>
deleteItem(itemUuid: string): Promise<void>
deleteAll(): Promise<void>
setItems(items: TransferPayload[]): Promise<void>
multiGet<T>(keys: string[]): Promise<T[]>
}

View File

@@ -0,0 +1,39 @@
import { DatabaseItemMetadata, isNotUndefined, TransferPayload } from '@standardnotes/snjs'
import { FlashKeyValueStore } from './FlashKeyValueStore'
export class DatabaseMetadata {
constructor(private identifier: string, private flashStorage: FlashKeyValueStore) {}
runMigration(payloads: TransferPayload[]) {
const metadataItems = this.setMetadataForPayloads(payloads)
return metadataItems
}
setMetadataForPayloads(payloads: TransferPayload[]) {
const metadataItems = []
for (const payload of payloads) {
const { uuid, content_type, updated_at } = payload
const key = this.keyForUuid(uuid)
const metadata: DatabaseItemMetadata = { uuid, content_type, updated_at }
this.flashStorage.set(key, metadata)
metadataItems.push(metadata)
}
return metadataItems
}
deleteMetadataItem(itemUuid: string) {
const key = this.keyForUuid(itemUuid)
this.flashStorage.delete(key)
}
getAllMetadataItems(): DatabaseItemMetadata[] {
const keys = this.flashStorage.getAllKeys()
const metadataKeys = keys.filter((key) => key.endsWith('-Metadata'))
const metadataItems = this.flashStorage.multiGet<DatabaseItemMetadata>(metadataKeys).filter(isNotUndefined)
return metadataItems
}
private keyForUuid(uuid: string) {
return `${this.identifier}-Item-${uuid}-Metadata`
}
}

View File

@@ -0,0 +1,40 @@
import { MMKV } from 'react-native-mmkv'
export class FlashKeyValueStore {
private storage: MMKV
constructor(identifier: string) {
this.storage = new MMKV({ id: identifier })
}
set(key: string, value: unknown): void {
this.storage.set(key, JSON.stringify(value))
}
delete(key: string): void {
this.storage.delete(key)
}
deleteAll(): void {
this.storage.clearAll()
}
getAllKeys(): string[] {
return this.storage.getAllKeys()
}
get<T>(key: string): T | undefined {
const item = this.storage.getString(key)
if (item) {
try {
return JSON.parse(item)
} catch (e) {
return item as T
}
}
}
multiGet<T>(keys: string[]): (T | undefined)[] {
return keys.map((key) => this.get<T>(key))
}
}

View File

@@ -0,0 +1,15 @@
import { ApplicationIdentifier } from '@standardnotes/snjs'
/**
* This identifier was the database name used in Standard Notes web/desktop.
*/
const LEGACY_IDENTIFIER = 'standardnotes'
/**
* We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not.
* It is also used to decide if the raw or the namespaced keychain is used.
* @param identifier The ApplicationIdentifier
*/
export const isLegacyIdentifier = function (identifier: ApplicationIdentifier) {
return identifier && identifier === LEGACY_IDENTIFIER
}

View File

@@ -0,0 +1,26 @@
import AsyncStorage from '@react-native-community/async-storage'
export class LegacyKeyValueStore {
set(key: string, value: string): Promise<void> {
return AsyncStorage.setItem(key, JSON.stringify(value))
}
delete(key: string): Promise<void> {
return AsyncStorage.removeItem(key)
}
deleteAll(): Promise<void> {
return AsyncStorage.clear()
}
async getValue<T>(key: string): Promise<T | undefined> {
const item = await AsyncStorage.getItem(key)
if (item) {
try {
return JSON.parse(item)
} catch (e) {
return item as T
}
}
}
}

View File

@@ -0,0 +1,16 @@
import { Alert } from 'react-native'
export const showLoadFailForItemIds = (failedItemIds: string[]) => {
let text =
'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n'
let index = 0
text += failedItemIds.map((id) => {
let result = id
if (index !== failedItemIds.length - 1) {
result += '\n'
}
index++
return result
})
Alert.alert('Unable to load item(s)', text)
}

View File

@@ -1,12 +1,11 @@
import AsyncStorage from '@react-native-community/async-storage'
import SNReactNative from '@standardnotes/react-native-utils'
import { AppleIAPReceipt } from '@standardnotes/services'
import {
AppleIAPProductId,
AppleIAPReceipt,
ApplicationIdentifier,
DatabaseKeysLoadChunkResponse,
DatabaseLoadOptions,
Environment,
LegacyMobileKeychainStructure,
LegacyRawKeychainValue,
MobileDeviceInterface,
NamespacedRootKeyInKeychain,
Platform as SNPlatform,
@@ -41,8 +40,11 @@ import {
import { hide, show } from 'react-native-privacy-snapshot'
import Share from 'react-native-share'
import { AndroidBackHandlerService } from '../AndroidBackHandlerService'
import { AppStateObserverService } from '../AppStateObserverService'
import { PurchaseManager } from '../PurchaseManager'
import { AppStateObserverService } from './../AppStateObserverService'
import { Database } from './Database/Database'
import { isLegacyIdentifier } from './Database/LegacyIdentifier'
import { LegacyKeyValueStore } from './Database/LegacyKeyValueStore'
import Keychain from './Keychain'
export type BiometricsType = 'Fingerprint' | 'Face ID' | 'Biometrics' | 'Touch ID'
@@ -53,41 +55,6 @@ export enum MobileDeviceEvent {
type MobileDeviceEventHandler = (event: MobileDeviceEvent) => void
/**
* This identifier was the database name used in Standard Notes web/desktop.
*/
const LEGACY_IDENTIFIER = 'standardnotes'
/**
* We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not.
* It is also used to decide if the raw or the namespaced keychain is used.
* @param identifier The ApplicationIdentifier
*/
const isLegacyIdentifier = function (identifier: ApplicationIdentifier) {
return identifier && identifier === LEGACY_IDENTIFIER
}
function isLegacyMobileKeychain(
x: LegacyMobileKeychainStructure | RawKeychainValue,
): x is LegacyMobileKeychainStructure {
return x.ak != undefined
}
const showLoadFailForItemIds = (failedItemIds: string[]) => {
let text =
'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n'
let index = 0
text += failedItemIds.map((id) => {
let result = id
if (index !== failedItemIds.length - 1) {
result += '\n'
}
index++
return result
})
Alert.alert('Unable to load item(s)', text)
}
export class MobileDevice implements MobileDeviceInterface {
environment: Environment.Mobile = Environment.Mobile
platform: SNPlatform.Ios | SNPlatform.Android = Platform.OS === 'ios' ? SNPlatform.Ios : SNPlatform.Android
@@ -95,6 +62,8 @@ export class MobileDevice implements MobileDeviceInterface {
public isDarkMode = false
public statusBarBgColor: string | undefined
private componentUrls: Map<UuidString, string> = new Map()
private keyValueStore = new LegacyKeyValueStore()
private databases = new Map<string, Database>()
constructor(
private stateObserverService?: AppStateObserverService,
@@ -106,6 +75,17 @@ export class MobileDevice implements MobileDeviceInterface {
return PurchaseManager.getInstance().purchase(plan)
}
private findOrCreateDatabase(identifier: ApplicationIdentifier): Database {
const existing = this.databases.get(identifier)
if (existing) {
return existing
}
const newDb = new Database(identifier)
this.databases.set(identifier, newDb)
return newDb
}
deinit() {
this.stateObserverService?.deinit()
;(this.stateObserverService as unknown) = undefined
@@ -120,10 +100,6 @@ export class MobileDevice implements MobileDeviceInterface {
console.log(args)
}
async setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise<void> {
await Keychain.setKeys(value)
}
public async getJsonParsedRawStorageValue(key: string): Promise<unknown | undefined> {
const value = await this.getRawStorageValue(key)
if (value == undefined) {
@@ -136,219 +112,57 @@ export class MobileDevice implements MobileDeviceInterface {
}
}
private getDatabaseKeyPrefix(identifier: ApplicationIdentifier) {
if (identifier && !isLegacyIdentifier(identifier)) {
return `${identifier}-Item-`
} else {
return 'Item-'
}
}
private keyForPayloadId(id: string, identifier: ApplicationIdentifier) {
return `${this.getDatabaseKeyPrefix(identifier)}${id}`
}
private async getAllDatabaseKeys(identifier: ApplicationIdentifier) {
const keys = await AsyncStorage.getAllKeys()
const filtered = keys.filter((key) => {
return key.startsWith(this.getDatabaseKeyPrefix(identifier))
})
return filtered
}
getDatabaseKeys(): Promise<string[]> {
return AsyncStorage.getAllKeys()
}
private async getRawStorageKeyValues(keys: string[]) {
const results: { key: string; value: unknown }[] = []
if (Platform.OS === 'android') {
for (const key of keys) {
try {
const item = await AsyncStorage.getItem(key)
if (item) {
results.push({ key, value: item })
}
} catch (e) {
console.error('Error getting item', key, e)
}
}
} else {
try {
for (const item of await AsyncStorage.multiGet(keys)) {
if (item[1]) {
results.push({ key: item[0], value: item[1] })
}
}
} catch (e) {
console.error('Error getting items', e)
}
}
return results
}
private async getDatabaseKeyValues(keys: string[]) {
const results: (TransferPayload | unknown)[] = []
if (Platform.OS === 'android') {
const failedItemIds: string[] = []
for (const key of keys) {
try {
const item = await AsyncStorage.getItem(key)
if (item) {
try {
results.push(JSON.parse(item) as TransferPayload)
} catch (e) {
results.push(item)
}
}
} catch (e) {
console.error('Error getting item', key, e)
failedItemIds.push(key)
}
}
if (failedItemIds.length > 0) {
showLoadFailForItemIds(failedItemIds)
}
} else {
try {
for (const item of await AsyncStorage.multiGet(keys)) {
if (item[1]) {
try {
results.push(JSON.parse(item[1]))
} catch (e) {
results.push(item[1])
}
}
}
} catch (e) {
console.error('Error getting items', e)
}
}
return results
}
async getRawStorageValue(key: string) {
const item = await AsyncStorage.getItem(key)
if (item) {
try {
return JSON.parse(item)
} catch (e) {
return item
}
}
}
hideMobileInterfaceFromScreenshots(): void {
hide()
this.setAndroidScreenshotPrivacy(true)
}
stopHidingMobileInterfaceFromScreenshots(): void {
show()
this.setAndroidScreenshotPrivacy(false)
}
async getAllRawStorageKeyValues() {
const keys = await AsyncStorage.getAllKeys()
return this.getRawStorageKeyValues(keys)
getRawStorageValue(key: string): Promise<string | undefined> {
return this.keyValueStore.getValue(key)
}
setRawStorageValue(key: string, value: string): Promise<void> {
return AsyncStorage.setItem(key, JSON.stringify(value))
return this.keyValueStore.set(key, value)
}
removeRawStorageValue(key: string): Promise<void> {
return AsyncStorage.removeItem(key)
return this.keyValueStore.delete(key)
}
removeAllRawStorageValues(): Promise<void> {
return AsyncStorage.clear()
return this.keyValueStore.deleteAll()
}
openDatabase(): Promise<{ isNewDatabase?: boolean | undefined } | undefined> {
return Promise.resolve({ isNewDatabase: false })
}
async getAllRawDatabasePayloads<T extends TransferPayload = TransferPayload>(
getDatabaseLoadChunks(options: DatabaseLoadOptions, identifier: string): Promise<DatabaseKeysLoadChunkResponse> {
return this.findOrCreateDatabase(identifier).getLoadChunks(options)
}
async getAllDatabaseEntries<T extends TransferPayload = TransferPayload>(
identifier: ApplicationIdentifier,
): Promise<T[]> {
const keys = await this.getAllDatabaseKeys(identifier)
return this.getDatabaseKeyValues(keys) as Promise<T[]>
return this.findOrCreateDatabase(identifier).getAllEntries()
}
saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void> {
return this.saveRawDatabasePayloads([payload], identifier)
}
async saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void> {
if (payloads.length === 0) {
return
}
await Promise.all(
payloads.map((item) => {
return AsyncStorage.setItem(this.keyForPayloadId(item.uuid, identifier), JSON.stringify(item))
}),
)
}
removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier): Promise<void> {
return this.removeRawStorageValue(this.keyForPayloadId(id, identifier))
}
async removeAllRawDatabasePayloads(identifier: ApplicationIdentifier): Promise<void> {
const keys = await this.getAllDatabaseKeys(identifier)
return AsyncStorage.multiRemove(keys)
}
async getNamespacedKeychainValue(
async getDatabaseEntries<T extends TransferPayload = TransferPayload>(
identifier: ApplicationIdentifier,
): Promise<NamespacedRootKeyInKeychain | undefined> {
const keychain = await this.getRawKeychainValue()
if (!keychain) {
return
}
const namespacedValue = keychain[identifier]
if (!namespacedValue && isLegacyIdentifier(identifier)) {
return keychain as unknown as NamespacedRootKeyInKeychain
}
return namespacedValue
keys: string[],
): Promise<T[]> {
return this.findOrCreateDatabase(identifier).multiGet<T>(keys)
}
async setNamespacedKeychainValue(
value: NamespacedRootKeyInKeychain,
identifier: ApplicationIdentifier,
): Promise<void> {
let keychain = await this.getRawKeychainValue()
if (!keychain) {
keychain = {}
}
await Keychain.setKeys({
...keychain,
[identifier]: value,
})
saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void> {
return this.saveDatabaseEntries([payload], identifier)
}
async clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<void> {
const keychain = await this.getRawKeychainValue()
async saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void> {
return this.findOrCreateDatabase(identifier).setItems(payloads)
}
if (!keychain) {
return
}
removeDatabaseEntry(id: string, identifier: ApplicationIdentifier): Promise<void> {
return this.findOrCreateDatabase(identifier).deleteItem(id)
}
if (!keychain[identifier] && isLegacyIdentifier(identifier) && isLegacyMobileKeychain(keychain)) {
await this.clearRawKeychainValue()
return
}
delete keychain[identifier]
await Keychain.setKeys(keychain)
async removeAllDatabaseEntries(identifier: ApplicationIdentifier): Promise<void> {
return this.findOrCreateDatabase(identifier).deleteAll()
}
async getDeviceBiometricsAvailability() {
@@ -413,6 +227,51 @@ export class MobileDevice implements MobileDeviceInterface {
return result
}
async getNamespacedKeychainValue(
identifier: ApplicationIdentifier,
): Promise<NamespacedRootKeyInKeychain | undefined> {
const keychain = await this.getRawKeychainValue()
if (!keychain) {
return
}
const namespacedValue = keychain[identifier]
if (!namespacedValue && isLegacyIdentifier(identifier)) {
return keychain as unknown as NamespacedRootKeyInKeychain
}
return namespacedValue
}
async setNamespacedKeychainValue(
value: NamespacedRootKeyInKeychain,
identifier: ApplicationIdentifier,
): Promise<void> {
let keychain = await this.getRawKeychainValue()
if (!keychain) {
keychain = {}
}
await Keychain.setKeys({
...keychain,
[identifier]: value,
})
}
async clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<void> {
const keychain = await this.getRawKeychainValue()
if (!keychain) {
return
}
delete keychain[identifier]
await Keychain.setKeys(keychain)
}
async getRawKeychainValue(): Promise<RawKeychainValue | undefined> {
const result = await Keychain.getKeys()
@@ -641,4 +500,14 @@ export class MobileDevice implements MobileDeviceInterface {
async getColorScheme(): Promise<ColorSchemeName> {
return Appearance.getColorScheme()
}
hideMobileInterfaceFromScreenshots(): void {
hide()
this.setAndroidScreenshotPrivacy(true)
}
stopHidingMobileInterfaceFromScreenshots(): void {
show()
this.setAndroidScreenshotPrivacy(false)
}
}

View File

@@ -7,7 +7,7 @@ import { OnShouldStartLoadWithRequest } from 'react-native-webview/lib/WebViewTy
import { AndroidBackHandlerService } from './AndroidBackHandlerService'
import { AppStateObserverService } from './AppStateObserverService'
import { ColorSchemeObserverService } from './ColorSchemeObserverService'
import { MobileDevice, MobileDeviceEvent } from './Lib/Interface'
import { MobileDevice, MobileDeviceEvent } from './Lib/MobileDevice'
import { IsDev } from './Lib/Utils'
const LoggingEnabled = IsDev
@@ -177,6 +177,10 @@ const MobileWebAppContents = ({ destroyAndReload }: { destroyAndReload: () => vo
window.ReactNativeWebView.postMessage('[web log] ' + args.join(' '));
}
console.error = (...args) => {
window.ReactNativeWebView.postMessage('[web log] ' + args.join(' '));
}
${WebProcessDeviceInterface}
${WebProcessMessageSender}