diff --git a/.yarn/cache/react-native-mmkv-npm-2.5.1-4745a42823-6f0cf484e7.zip b/.yarn/cache/react-native-mmkv-npm-2.5.1-4745a42823-6f0cf484e7.zip new file mode 100644 index 000000000..5cefc0fcb Binary files /dev/null and b/.yarn/cache/react-native-mmkv-npm-2.5.1-4745a42823-6f0cf484e7.zip differ diff --git a/packages/filepicker/example/src/web_device_interface.js b/packages/filepicker/example/src/web_device_interface.js index 810984731..6f5928107 100644 --- a/packages/filepicker/example/src/web_device_interface.js +++ b/packages/filepicker/example/src/web_device_interface.js @@ -18,17 +18,6 @@ export default class WebDeviceInterface { } } - async getAllRawStorageKeyValues() { - const results = [] - for (const key of Object.keys(localStorage)) { - results.push({ - key: key, - value: localStorage[key], - }) - } - return results - } - async setRawStorageValue(key, value) { localStorage.setItem(key, value) } @@ -57,7 +46,7 @@ export default class WebDeviceInterface { return `${this._getDatabaseKeyPrefix(identifier)}${id}` } - async getAllRawDatabasePayloads(identifier) { + async getAllDatabaseEntries(identifier) { const models = [] for (const key in localStorage) { if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) { @@ -67,21 +56,21 @@ export default class WebDeviceInterface { return models } - async saveRawDatabasePayload(payload, identifier) { + async saveDatabaseEntry(payload, identifier) { localStorage.setItem(this._keyForPayloadId(payload.uuid, identifier), JSON.stringify(payload)) } - async saveRawDatabasePayloads(payloads, identifier) { + async saveDatabaseEntries(payloads, identifier) { for (const payload of payloads) { - await this.saveRawDatabasePayload(payload, identifier) + await this.saveDatabaseEntry(payload, identifier) } } - async removeRawDatabasePayloadWithId(id, identifier) { + async removeDatabaseEntry(id, identifier) { localStorage.removeItem(this._keyForPayloadId(id, identifier)) } - async removeAllRawDatabasePayloads(identifier) { + async removeAllDatabaseEntries(identifier) { for (const key in localStorage) { if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) { delete localStorage[key] @@ -121,12 +110,6 @@ export default class WebDeviceInterface { localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(keychain)) } - /** Allows unit tests to set legacy keychain structure as it was <= 003 */ - // eslint-disable-next-line camelcase - async setLegacyRawKeychainValue(value) { - localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(value)) - } - async getRawKeychainValue() { const keychain = localStorage.getItem(KEYCHAIN_STORAGE_KEY) return JSON.parse(keychain) diff --git a/packages/mobile/ios/Podfile.lock b/packages/mobile/ios/Podfile.lock index 03697e2f5..af36a5f78 100644 --- a/packages/mobile/ios/Podfile.lock +++ b/packages/mobile/ios/Podfile.lock @@ -75,6 +75,9 @@ PODS: - glog (0.3.5) - hermes-engine (0.70.6) - libevent (2.1.12) + - MMKV (1.2.14): + - MMKVCore (~> 1.2.14) + - MMKVCore (1.2.14) - OpenSSL-Universal (1.1.1100) - RCT-Folly (2021.07.22.00): - boost @@ -304,6 +307,9 @@ PODS: - glog - react-native-fingerprint-scanner (5.0.0): - React-Core + - react-native-mmkv (2.5.1): + - MMKV (>= 1.2.13) + - React-Core - react-native-version-info (1.1.1): - React-Core - react-native-webview (11.23.1): @@ -444,6 +450,7 @@ DEPENDENCIES: - React-jsinspector (from `../node_modules/react-native/ReactCommon/jsinspector`) - React-logger (from `../node_modules/react-native/ReactCommon/logger`) - react-native-fingerprint-scanner (from `../node_modules/react-native-fingerprint-scanner`) + - react-native-mmkv (from `../node_modules/react-native-mmkv`) - react-native-version-info (from `../node_modules/react-native-version-info`) - react-native-webview (from `../node_modules/react-native-webview`) - React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`) @@ -483,6 +490,8 @@ SPEC REPOS: - FlipperKit - fmt - libevent + - MMKV + - MMKVCore - OpenSSL-Universal - SocketRocket - TrustKit @@ -533,6 +542,8 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native/ReactCommon/logger" react-native-fingerprint-scanner: :path: "../node_modules/react-native-fingerprint-scanner" + react-native-mmkv: + :path: "../node_modules/react-native-mmkv" react-native-version-info: :path: "../node_modules/react-native-version-info" react-native-webview: @@ -583,7 +594,7 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: boost: a7c83b31436843459a1961bfd74b96033dc77234 CocoaAsyncSocket: 065fd1e645c7abab64f7a6a2007a48038fdc6a99 - DoubleConversion: 831926d9b8bf8166fd87886c4abab286c2422662 + DoubleConversion: 5189b271737e1565bdce30deb4a08d647e3f5f54 FBLazyVector: 48289402952f4f7a4e235de70a9a590aa0b79ef4 FBReactNativeSpec: dd1186fd05255e3457baa2f4ca65e94c2cd1e3ac Flipper: 26fc4b7382499f1281eb8cb921e5c3ad6de91fe0 @@ -596,9 +607,11 @@ SPEC CHECKSUMS: Flipper-RSocket: d9d9ade67cbecf6ac10730304bf5607266dd2541 FlipperKit: cbdee19bdd4e7f05472a66ce290f1b729ba3cb86 fmt: ff9d55029c625d3757ed641535fd4a75fedc7ce9 - glog: 85ecdd10ee8d8ec362ef519a6a45ff9aa27b2e85 + glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b hermes-engine: 2af7b7a59128f250adfd86f15aa1d5a2ecd39995 libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913 + MMKV: 9c4663aa7ca255d478ff10f2f5cb7d17c1651ccd + MMKVCore: 89f5c8a66bba2dcd551779dea4d412eeec8ff5bb OpenSSL-Universal: ebc357f1e6bc71fa463ccb2fe676756aff50e88c RCT-Folly: 0080d0a6ebf2577475bda044aa59e2ca1f909cda RCTRequired: e1866f61af7049eb3d8e08e8b133abd38bc1ca7a @@ -616,6 +629,7 @@ SPEC CHECKSUMS: React-jsinspector: 60769e5a0a6d4b32294a2456077f59d0266f9a8b React-logger: 1623c216abaa88974afce404dc8f479406bbc3a0 react-native-fingerprint-scanner: be63e626b31fb951780a5fac5328b065a61a3d6e + react-native-mmkv: 69b9c003f10afdd01addf7c6ee784ce42ee2eff3 react-native-version-info: a106f23009ac0db4ee00de39574eb546682579b9 react-native-webview: d33e2db8925d090871ffeb232dfa50cb3a727581 React-perflogger: 8c79399b0500a30ee8152d0f9f11beae7fc36595 diff --git a/packages/mobile/ios/StandardNotes/Info.plist b/packages/mobile/ios/StandardNotes/Info.plist index e67578d1b..e3d8d7255 100644 --- a/packages/mobile/ios/StandardNotes/Info.plist +++ b/packages/mobile/ios/StandardNotes/Info.plist @@ -100,7 +100,7 @@ NSPhotoLibraryUsageDescription Photo library is optionally used to select files to upload or QR code images from your photo library. NSMicrophoneUsageDescription - Microphone is optionally used to capture videos. + Microphone is optionally used to capture videos. UIAppFonts AntDesign.ttf @@ -147,5 +147,7 @@ supportsAlternateIcons + RCTAsyncStorageExcludeFromBackup + diff --git a/packages/mobile/package.json b/packages/mobile/package.json index 5db3a0d7e..6bf8b0e8a 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -59,6 +59,7 @@ "react-native-fs": "^2.20.0", "react-native-iap": "^12.4.4", "react-native-keychain": "standardnotes/react-native-keychain#d277d360494cbd02be4accb4a360772a8e0e97b6", + "react-native-mmkv": "^2.5.1", "react-native-privacy-snapshot": "standardnotes/react-native-privacy-snapshot#653e904c90fc6f2b578da59138f2bfe5d7f942fe", "react-native-share": "^8.0.0", "react-native-version-info": "^1.1.1", diff --git a/packages/mobile/src/Lib/Database/Database.ts b/packages/mobile/src/Lib/Database/Database.ts new file mode 100644 index 000000000..5dfcb2ea5 --- /dev/null +++ b/packages/mobile/src/Lib/Database/Database.ts @@ -0,0 +1,158 @@ +import AsyncStorage from '@react-native-community/async-storage' +import { + DatabaseKeysLoadChunk, + DatabaseKeysLoadChunkResponse, + DatabaseLoadOptions, + GetSortedPayloadsByPriority, + TransferPayload, +} from '@standardnotes/snjs' +import { Platform } from 'react-native' +import { DatabaseInterface } from './DatabaseInterface' +import { DatabaseMetadata } from './DatabaseMetadata' +import { FlashKeyValueStore } from './FlashKeyValueStore' +import { isLegacyIdentifier } from './LegacyIdentifier' +import { showLoadFailForItemIds } from './showLoadFailForItemIds' + +export class Database implements DatabaseInterface { + private metadataStore: DatabaseMetadata + + constructor(private identifier: string) { + const flashStorage = new FlashKeyValueStore(identifier) + this.metadataStore = new DatabaseMetadata(identifier, flashStorage) + } + + private databaseKeyForPayloadId(id: string) { + return `${this.getDatabaseKeyPrefix()}${id}` + } + + private getDatabaseKeyPrefix() { + if (this.identifier && !isLegacyIdentifier(this.identifier)) { + return `${this.identifier}-Item-` + } else { + return 'Item-' + } + } + + async getAllEntries(): Promise { + const keys = await this.getAllKeys() + return this.multiGet(keys) + } + + async getAllKeys(): Promise { + const keys = await AsyncStorage.getAllKeys() + const filtered = keys.filter((key) => { + return key.startsWith(this.getDatabaseKeyPrefix()) + }) + return filtered + } + + async multiDelete(keys: string[]): Promise { + return AsyncStorage.multiRemove(keys) + } + + async deleteItem(itemUuid: string): Promise { + const key = this.databaseKeyForPayloadId(itemUuid) + this.metadataStore.deleteMetadataItem(itemUuid) + return this.multiDelete([key]) + } + + async deleteAll(): Promise { + const keys = await this.getAllKeys() + return this.multiDelete(keys) + } + + async setItems(items: TransferPayload[]): Promise { + if (items.length === 0) { + return + } + + await Promise.all( + items.map((item) => { + return Promise.all([ + AsyncStorage.setItem(this.databaseKeyForPayloadId(item.uuid), JSON.stringify(item)), + this.metadataStore.setMetadataForPayloads([item]), + ]) + }), + ) + } + + async getLoadChunks(options: DatabaseLoadOptions): Promise { + let metadataItems = this.metadataStore.getAllMetadataItems() + + if (metadataItems.length === 0) { + const allEntries = await this.getAllEntries() + metadataItems = this.metadataStore.runMigration(allEntries) + } + + const sorted = GetSortedPayloadsByPriority(metadataItems, options) + + const itemsKeysChunk: DatabaseKeysLoadChunk = { + keys: sorted.itemsKeyPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)), + } + + const contentTypePriorityChunk: DatabaseKeysLoadChunk = { + keys: sorted.contentTypePriorityPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)), + } + + const remainingKeys = sorted.remainingPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)) + + const remainingKeysChunks: DatabaseKeysLoadChunk[] = [] + for (let i = 0; i < remainingKeys.length; i += options.batchSize) { + remainingKeysChunks.push({ + keys: remainingKeys.slice(i, i + options.batchSize), + }) + } + + const result: DatabaseKeysLoadChunkResponse = { + keys: { + itemsKeys: itemsKeysChunk, + remainingChunks: [contentTypePriorityChunk, ...remainingKeysChunks], + }, + remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length, + } + + return result + } + + async multiGet(keys: string[]): Promise { + const results: T[] = [] + + if (Platform.OS === 'android') { + const failedItemIds: string[] = [] + for (const key of keys) { + try { + const item = await AsyncStorage.getItem(key) + if (item) { + try { + results.push(JSON.parse(item) as T) + } catch (e) { + results.push(item as T) + } + } + } catch (e) { + console.error('Error getting item', key, e) + failedItemIds.push(key) + } + } + if (failedItemIds.length > 0) { + showLoadFailForItemIds(failedItemIds) + } + } else { + try { + for (const item of await AsyncStorage.multiGet(keys)) { + if (item[1]) { + try { + results.push(JSON.parse(item[1])) + } catch (e) { + results.push(item[1] as T) + } + } + } + } catch (e) { + console.error('Error getting items', e) + } + } + + return results + } +} diff --git a/packages/mobile/src/Lib/Database/DatabaseInterface.ts b/packages/mobile/src/Lib/Database/DatabaseInterface.ts new file mode 100644 index 000000000..3840f8d2b --- /dev/null +++ b/packages/mobile/src/Lib/Database/DatabaseInterface.ts @@ -0,0 +1,10 @@ +import { TransferPayload } from '@standardnotes/snjs' + +export interface DatabaseInterface { + getAllKeys(): Promise + multiDelete(keys: string[]): Promise + deleteItem(itemUuid: string): Promise + deleteAll(): Promise + setItems(items: TransferPayload[]): Promise + multiGet(keys: string[]): Promise +} diff --git a/packages/mobile/src/Lib/Database/DatabaseMetadata.ts b/packages/mobile/src/Lib/Database/DatabaseMetadata.ts new file mode 100644 index 000000000..3434acfc6 --- /dev/null +++ b/packages/mobile/src/Lib/Database/DatabaseMetadata.ts @@ -0,0 +1,39 @@ +import { DatabaseItemMetadata, isNotUndefined, TransferPayload } from '@standardnotes/snjs' +import { FlashKeyValueStore } from './FlashKeyValueStore' + +export class DatabaseMetadata { + constructor(private identifier: string, private flashStorage: FlashKeyValueStore) {} + + runMigration(payloads: TransferPayload[]) { + const metadataItems = this.setMetadataForPayloads(payloads) + return metadataItems + } + + setMetadataForPayloads(payloads: TransferPayload[]) { + const metadataItems = [] + for (const payload of payloads) { + const { uuid, content_type, updated_at } = payload + const key = this.keyForUuid(uuid) + const metadata: DatabaseItemMetadata = { uuid, content_type, updated_at } + this.flashStorage.set(key, metadata) + metadataItems.push(metadata) + } + return metadataItems + } + + deleteMetadataItem(itemUuid: string) { + const key = this.keyForUuid(itemUuid) + this.flashStorage.delete(key) + } + + getAllMetadataItems(): DatabaseItemMetadata[] { + const keys = this.flashStorage.getAllKeys() + const metadataKeys = keys.filter((key) => key.endsWith('-Metadata')) + const metadataItems = this.flashStorage.multiGet(metadataKeys).filter(isNotUndefined) + return metadataItems + } + + private keyForUuid(uuid: string) { + return `${this.identifier}-Item-${uuid}-Metadata` + } +} diff --git a/packages/mobile/src/Lib/Database/FlashKeyValueStore.ts b/packages/mobile/src/Lib/Database/FlashKeyValueStore.ts new file mode 100644 index 000000000..f0e6afc1a --- /dev/null +++ b/packages/mobile/src/Lib/Database/FlashKeyValueStore.ts @@ -0,0 +1,40 @@ +import { MMKV } from 'react-native-mmkv' + +export class FlashKeyValueStore { + private storage: MMKV + + constructor(identifier: string) { + this.storage = new MMKV({ id: identifier }) + } + + set(key: string, value: unknown): void { + this.storage.set(key, JSON.stringify(value)) + } + + delete(key: string): void { + this.storage.delete(key) + } + + deleteAll(): void { + this.storage.clearAll() + } + + getAllKeys(): string[] { + return this.storage.getAllKeys() + } + + get(key: string): T | undefined { + const item = this.storage.getString(key) + if (item) { + try { + return JSON.parse(item) + } catch (e) { + return item as T + } + } + } + + multiGet(keys: string[]): (T | undefined)[] { + return keys.map((key) => this.get(key)) + } +} diff --git a/packages/mobile/src/Lib/Database/LegacyIdentifier.ts b/packages/mobile/src/Lib/Database/LegacyIdentifier.ts new file mode 100644 index 000000000..4d186507e --- /dev/null +++ b/packages/mobile/src/Lib/Database/LegacyIdentifier.ts @@ -0,0 +1,15 @@ +import { ApplicationIdentifier } from '@standardnotes/snjs' + +/** + * This identifier was the database name used in Standard Notes web/desktop. + */ +const LEGACY_IDENTIFIER = 'standardnotes' + +/** + * We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not. + * It is also used to decide if the raw or the namespaced keychain is used. + * @param identifier The ApplicationIdentifier + */ +export const isLegacyIdentifier = function (identifier: ApplicationIdentifier) { + return identifier && identifier === LEGACY_IDENTIFIER +} diff --git a/packages/mobile/src/Lib/Database/LegacyKeyValueStore.ts b/packages/mobile/src/Lib/Database/LegacyKeyValueStore.ts new file mode 100644 index 000000000..e03d1f68c --- /dev/null +++ b/packages/mobile/src/Lib/Database/LegacyKeyValueStore.ts @@ -0,0 +1,26 @@ +import AsyncStorage from '@react-native-community/async-storage' + +export class LegacyKeyValueStore { + set(key: string, value: string): Promise { + return AsyncStorage.setItem(key, JSON.stringify(value)) + } + + delete(key: string): Promise { + return AsyncStorage.removeItem(key) + } + + deleteAll(): Promise { + return AsyncStorage.clear() + } + + async getValue(key: string): Promise { + const item = await AsyncStorage.getItem(key) + if (item) { + try { + return JSON.parse(item) + } catch (e) { + return item as T + } + } + } +} diff --git a/packages/mobile/src/Lib/Database/showLoadFailForItemIds.ts b/packages/mobile/src/Lib/Database/showLoadFailForItemIds.ts new file mode 100644 index 000000000..6406d2177 --- /dev/null +++ b/packages/mobile/src/Lib/Database/showLoadFailForItemIds.ts @@ -0,0 +1,16 @@ +import { Alert } from 'react-native' + +export const showLoadFailForItemIds = (failedItemIds: string[]) => { + let text = + 'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n' + let index = 0 + text += failedItemIds.map((id) => { + let result = id + if (index !== failedItemIds.length - 1) { + result += '\n' + } + index++ + return result + }) + Alert.alert('Unable to load item(s)', text) +} diff --git a/packages/mobile/src/Lib/Interface.ts b/packages/mobile/src/Lib/MobileDevice.ts similarity index 68% rename from packages/mobile/src/Lib/Interface.ts rename to packages/mobile/src/Lib/MobileDevice.ts index 60b31ad35..37ed3e77f 100644 --- a/packages/mobile/src/Lib/Interface.ts +++ b/packages/mobile/src/Lib/MobileDevice.ts @@ -1,12 +1,11 @@ -import AsyncStorage from '@react-native-community/async-storage' import SNReactNative from '@standardnotes/react-native-utils' -import { AppleIAPReceipt } from '@standardnotes/services' import { AppleIAPProductId, + AppleIAPReceipt, ApplicationIdentifier, + DatabaseKeysLoadChunkResponse, + DatabaseLoadOptions, Environment, - LegacyMobileKeychainStructure, - LegacyRawKeychainValue, MobileDeviceInterface, NamespacedRootKeyInKeychain, Platform as SNPlatform, @@ -41,8 +40,11 @@ import { import { hide, show } from 'react-native-privacy-snapshot' import Share from 'react-native-share' import { AndroidBackHandlerService } from '../AndroidBackHandlerService' +import { AppStateObserverService } from '../AppStateObserverService' import { PurchaseManager } from '../PurchaseManager' -import { AppStateObserverService } from './../AppStateObserverService' +import { Database } from './Database/Database' +import { isLegacyIdentifier } from './Database/LegacyIdentifier' +import { LegacyKeyValueStore } from './Database/LegacyKeyValueStore' import Keychain from './Keychain' export type BiometricsType = 'Fingerprint' | 'Face ID' | 'Biometrics' | 'Touch ID' @@ -53,41 +55,6 @@ export enum MobileDeviceEvent { type MobileDeviceEventHandler = (event: MobileDeviceEvent) => void -/** - * This identifier was the database name used in Standard Notes web/desktop. - */ -const LEGACY_IDENTIFIER = 'standardnotes' - -/** - * We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not. - * It is also used to decide if the raw or the namespaced keychain is used. - * @param identifier The ApplicationIdentifier - */ -const isLegacyIdentifier = function (identifier: ApplicationIdentifier) { - return identifier && identifier === LEGACY_IDENTIFIER -} - -function isLegacyMobileKeychain( - x: LegacyMobileKeychainStructure | RawKeychainValue, -): x is LegacyMobileKeychainStructure { - return x.ak != undefined -} - -const showLoadFailForItemIds = (failedItemIds: string[]) => { - let text = - 'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n' - let index = 0 - text += failedItemIds.map((id) => { - let result = id - if (index !== failedItemIds.length - 1) { - result += '\n' - } - index++ - return result - }) - Alert.alert('Unable to load item(s)', text) -} - export class MobileDevice implements MobileDeviceInterface { environment: Environment.Mobile = Environment.Mobile platform: SNPlatform.Ios | SNPlatform.Android = Platform.OS === 'ios' ? SNPlatform.Ios : SNPlatform.Android @@ -95,6 +62,8 @@ export class MobileDevice implements MobileDeviceInterface { public isDarkMode = false public statusBarBgColor: string | undefined private componentUrls: Map = new Map() + private keyValueStore = new LegacyKeyValueStore() + private databases = new Map() constructor( private stateObserverService?: AppStateObserverService, @@ -106,6 +75,17 @@ export class MobileDevice implements MobileDeviceInterface { return PurchaseManager.getInstance().purchase(plan) } + private findOrCreateDatabase(identifier: ApplicationIdentifier): Database { + const existing = this.databases.get(identifier) + if (existing) { + return existing + } + + const newDb = new Database(identifier) + this.databases.set(identifier, newDb) + return newDb + } + deinit() { this.stateObserverService?.deinit() ;(this.stateObserverService as unknown) = undefined @@ -120,10 +100,6 @@ export class MobileDevice implements MobileDeviceInterface { console.log(args) } - async setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise { - await Keychain.setKeys(value) - } - public async getJsonParsedRawStorageValue(key: string): Promise { const value = await this.getRawStorageValue(key) if (value == undefined) { @@ -136,219 +112,57 @@ export class MobileDevice implements MobileDeviceInterface { } } - private getDatabaseKeyPrefix(identifier: ApplicationIdentifier) { - if (identifier && !isLegacyIdentifier(identifier)) { - return `${identifier}-Item-` - } else { - return 'Item-' - } - } - - private keyForPayloadId(id: string, identifier: ApplicationIdentifier) { - return `${this.getDatabaseKeyPrefix(identifier)}${id}` - } - - private async getAllDatabaseKeys(identifier: ApplicationIdentifier) { - const keys = await AsyncStorage.getAllKeys() - const filtered = keys.filter((key) => { - return key.startsWith(this.getDatabaseKeyPrefix(identifier)) - }) - return filtered - } - - getDatabaseKeys(): Promise { - return AsyncStorage.getAllKeys() - } - - private async getRawStorageKeyValues(keys: string[]) { - const results: { key: string; value: unknown }[] = [] - if (Platform.OS === 'android') { - for (const key of keys) { - try { - const item = await AsyncStorage.getItem(key) - if (item) { - results.push({ key, value: item }) - } - } catch (e) { - console.error('Error getting item', key, e) - } - } - } else { - try { - for (const item of await AsyncStorage.multiGet(keys)) { - if (item[1]) { - results.push({ key: item[0], value: item[1] }) - } - } - } catch (e) { - console.error('Error getting items', e) - } - } - return results - } - - private async getDatabaseKeyValues(keys: string[]) { - const results: (TransferPayload | unknown)[] = [] - - if (Platform.OS === 'android') { - const failedItemIds: string[] = [] - for (const key of keys) { - try { - const item = await AsyncStorage.getItem(key) - if (item) { - try { - results.push(JSON.parse(item) as TransferPayload) - } catch (e) { - results.push(item) - } - } - } catch (e) { - console.error('Error getting item', key, e) - failedItemIds.push(key) - } - } - if (failedItemIds.length > 0) { - showLoadFailForItemIds(failedItemIds) - } - } else { - try { - for (const item of await AsyncStorage.multiGet(keys)) { - if (item[1]) { - try { - results.push(JSON.parse(item[1])) - } catch (e) { - results.push(item[1]) - } - } - } - } catch (e) { - console.error('Error getting items', e) - } - } - return results - } - - async getRawStorageValue(key: string) { - const item = await AsyncStorage.getItem(key) - if (item) { - try { - return JSON.parse(item) - } catch (e) { - return item - } - } - } - - hideMobileInterfaceFromScreenshots(): void { - hide() - this.setAndroidScreenshotPrivacy(true) - } - - stopHidingMobileInterfaceFromScreenshots(): void { - show() - this.setAndroidScreenshotPrivacy(false) - } - - async getAllRawStorageKeyValues() { - const keys = await AsyncStorage.getAllKeys() - return this.getRawStorageKeyValues(keys) + getRawStorageValue(key: string): Promise { + return this.keyValueStore.getValue(key) } setRawStorageValue(key: string, value: string): Promise { - return AsyncStorage.setItem(key, JSON.stringify(value)) + return this.keyValueStore.set(key, value) } removeRawStorageValue(key: string): Promise { - return AsyncStorage.removeItem(key) + return this.keyValueStore.delete(key) } removeAllRawStorageValues(): Promise { - return AsyncStorage.clear() + return this.keyValueStore.deleteAll() } openDatabase(): Promise<{ isNewDatabase?: boolean | undefined } | undefined> { return Promise.resolve({ isNewDatabase: false }) } - async getAllRawDatabasePayloads( + getDatabaseLoadChunks(options: DatabaseLoadOptions, identifier: string): Promise { + return this.findOrCreateDatabase(identifier).getLoadChunks(options) + } + + async getAllDatabaseEntries( identifier: ApplicationIdentifier, ): Promise { - const keys = await this.getAllDatabaseKeys(identifier) - return this.getDatabaseKeyValues(keys) as Promise + return this.findOrCreateDatabase(identifier).getAllEntries() } - saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier): Promise { - return this.saveRawDatabasePayloads([payload], identifier) - } - - async saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise { - if (payloads.length === 0) { - return - } - await Promise.all( - payloads.map((item) => { - return AsyncStorage.setItem(this.keyForPayloadId(item.uuid, identifier), JSON.stringify(item)) - }), - ) - } - - removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier): Promise { - return this.removeRawStorageValue(this.keyForPayloadId(id, identifier)) - } - - async removeAllRawDatabasePayloads(identifier: ApplicationIdentifier): Promise { - const keys = await this.getAllDatabaseKeys(identifier) - return AsyncStorage.multiRemove(keys) - } - - async getNamespacedKeychainValue( + async getDatabaseEntries( identifier: ApplicationIdentifier, - ): Promise { - const keychain = await this.getRawKeychainValue() - - if (!keychain) { - return - } - - const namespacedValue = keychain[identifier] - - if (!namespacedValue && isLegacyIdentifier(identifier)) { - return keychain as unknown as NamespacedRootKeyInKeychain - } - - return namespacedValue + keys: string[], + ): Promise { + return this.findOrCreateDatabase(identifier).multiGet(keys) } - async setNamespacedKeychainValue( - value: NamespacedRootKeyInKeychain, - identifier: ApplicationIdentifier, - ): Promise { - let keychain = await this.getRawKeychainValue() - - if (!keychain) { - keychain = {} - } - - await Keychain.setKeys({ - ...keychain, - [identifier]: value, - }) + saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier): Promise { + return this.saveDatabaseEntries([payload], identifier) } - async clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise { - const keychain = await this.getRawKeychainValue() + async saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise { + return this.findOrCreateDatabase(identifier).setItems(payloads) + } - if (!keychain) { - return - } + removeDatabaseEntry(id: string, identifier: ApplicationIdentifier): Promise { + return this.findOrCreateDatabase(identifier).deleteItem(id) + } - if (!keychain[identifier] && isLegacyIdentifier(identifier) && isLegacyMobileKeychain(keychain)) { - await this.clearRawKeychainValue() - return - } - - delete keychain[identifier] - await Keychain.setKeys(keychain) + async removeAllDatabaseEntries(identifier: ApplicationIdentifier): Promise { + return this.findOrCreateDatabase(identifier).deleteAll() } async getDeviceBiometricsAvailability() { @@ -413,6 +227,51 @@ export class MobileDevice implements MobileDeviceInterface { return result } + async getNamespacedKeychainValue( + identifier: ApplicationIdentifier, + ): Promise { + const keychain = await this.getRawKeychainValue() + + if (!keychain) { + return + } + + const namespacedValue = keychain[identifier] + + if (!namespacedValue && isLegacyIdentifier(identifier)) { + return keychain as unknown as NamespacedRootKeyInKeychain + } + + return namespacedValue + } + + async setNamespacedKeychainValue( + value: NamespacedRootKeyInKeychain, + identifier: ApplicationIdentifier, + ): Promise { + let keychain = await this.getRawKeychainValue() + + if (!keychain) { + keychain = {} + } + + await Keychain.setKeys({ + ...keychain, + [identifier]: value, + }) + } + + async clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise { + const keychain = await this.getRawKeychainValue() + + if (!keychain) { + return + } + + delete keychain[identifier] + await Keychain.setKeys(keychain) + } + async getRawKeychainValue(): Promise { const result = await Keychain.getKeys() @@ -641,4 +500,14 @@ export class MobileDevice implements MobileDeviceInterface { async getColorScheme(): Promise { return Appearance.getColorScheme() } + + hideMobileInterfaceFromScreenshots(): void { + hide() + this.setAndroidScreenshotPrivacy(true) + } + + stopHidingMobileInterfaceFromScreenshots(): void { + show() + this.setAndroidScreenshotPrivacy(false) + } } diff --git a/packages/mobile/src/MobileWebAppContainer.tsx b/packages/mobile/src/MobileWebAppContainer.tsx index 0dc2cb828..bc4a1ec21 100644 --- a/packages/mobile/src/MobileWebAppContainer.tsx +++ b/packages/mobile/src/MobileWebAppContainer.tsx @@ -7,7 +7,7 @@ import { OnShouldStartLoadWithRequest } from 'react-native-webview/lib/WebViewTy import { AndroidBackHandlerService } from './AndroidBackHandlerService' import { AppStateObserverService } from './AppStateObserverService' import { ColorSchemeObserverService } from './ColorSchemeObserverService' -import { MobileDevice, MobileDeviceEvent } from './Lib/Interface' +import { MobileDevice, MobileDeviceEvent } from './Lib/MobileDevice' import { IsDev } from './Lib/Utils' const LoggingEnabled = IsDev @@ -177,6 +177,10 @@ const MobileWebAppContents = ({ destroyAndReload }: { destroyAndReload: () => vo window.ReactNativeWebView.postMessage('[web log] ' + args.join(' ')); } + console.error = (...args) => { + window.ReactNativeWebView.postMessage('[web log] ' + args.join(' ')); + } + ${WebProcessDeviceInterface} ${WebProcessMessageSender} diff --git a/packages/models/src/Domain/Local/RootKey/KeychainTypes.ts b/packages/models/src/Domain/Local/RootKey/KeychainTypes.ts index 9e5a12bd7..97cc09f40 100644 --- a/packages/models/src/Domain/Local/RootKey/KeychainTypes.ts +++ b/packages/models/src/Domain/Local/RootKey/KeychainTypes.ts @@ -10,22 +10,3 @@ export interface NamespacedRootKeyInKeychain { } export type RootKeyContentInStorage = RootKeyContentSpecialized - -export interface LegacyRawKeychainValue { - mk: string - ak: string - version: ProtocolVersion -} - -export type LegacyMobileKeychainStructure = { - offline?: { - timing?: unknown - pw?: string - } - encryptedAccountKeys?: unknown - mk: string - pw: string - ak: string - version?: string - jwt?: string -} diff --git a/packages/services/src/Domain/Application/ApplicationInterface.ts b/packages/services/src/Domain/Application/ApplicationInterface.ts index 84afe8fdb..1538c8bc2 100644 --- a/packages/services/src/Domain/Application/ApplicationInterface.ts +++ b/packages/services/src/Domain/Application/ApplicationInterface.ts @@ -1,7 +1,6 @@ import { ApplicationIdentifier, ContentType } from '@standardnotes/common' import { BackupFile, DecryptedItemInterface, ItemStream, Platform, PrefKey, PrefValue } from '@standardnotes/models' import { FilesClientInterface } from '@standardnotes/files' - import { AlertService } from '../Alert/AlertService' import { ComponentManagerInterface } from '../Component/ComponentManagerInterface' import { ApplicationEvent } from '../Event/ApplicationEvent' diff --git a/packages/services/src/Domain/Device/DatabaseItemMetadata.ts b/packages/services/src/Domain/Device/DatabaseItemMetadata.ts new file mode 100644 index 000000000..af3b77a8e --- /dev/null +++ b/packages/services/src/Domain/Device/DatabaseItemMetadata.ts @@ -0,0 +1,3 @@ +import { TransferPayload } from '@standardnotes/models' + +export type DatabaseItemMetadata = Pick diff --git a/packages/services/src/Domain/Device/DatabaseLoadOptions.ts b/packages/services/src/Domain/Device/DatabaseLoadOptions.ts new file mode 100644 index 000000000..daad481b8 --- /dev/null +++ b/packages/services/src/Domain/Device/DatabaseLoadOptions.ts @@ -0,0 +1,44 @@ +import { ContentType } from '@standardnotes/common' +import { FullyFormedTransferPayload } from '@standardnotes/models' + +export type DatabaseKeysLoadChunk = { + keys: string[] +} + +export type DatabaseFullEntryLoadChunk = { + entries: FullyFormedTransferPayload[] +} + +export function isChunkFullEntry( + x: DatabaseKeysLoadChunk | DatabaseFullEntryLoadChunk, +): x is DatabaseFullEntryLoadChunk { + return (x as DatabaseFullEntryLoadChunk).entries !== undefined +} + +export type DatabaseKeysLoadChunkResponse = { + keys: { + itemsKeys: DatabaseKeysLoadChunk + remainingChunks: DatabaseKeysLoadChunk[] + } + remainingChunksItemCount: number +} + +export type DatabaseFullEntryLoadChunkResponse = { + fullEntries: { + itemsKeys: DatabaseFullEntryLoadChunk + remainingChunks: DatabaseFullEntryLoadChunk[] + } + remainingChunksItemCount: number +} + +export function isFullEntryLoadChunkResponse( + x: DatabaseKeysLoadChunkResponse | DatabaseFullEntryLoadChunkResponse, +): x is DatabaseFullEntryLoadChunkResponse { + return (x as DatabaseFullEntryLoadChunkResponse).fullEntries !== undefined +} + +export type DatabaseLoadOptions = { + contentTypePriority: ContentType[] + uuidPriority: string[] + batchSize: number +} diff --git a/packages/snjs/lib/Services/Sync/Utils.spec.ts b/packages/services/src/Domain/Device/DatabaseLoadSorter.spec.ts similarity index 90% rename from packages/snjs/lib/Services/Sync/Utils.spec.ts rename to packages/services/src/Domain/Device/DatabaseLoadSorter.spec.ts index 0b3490c72..d52f4167c 100644 --- a/packages/snjs/lib/Services/Sync/Utils.spec.ts +++ b/packages/services/src/Domain/Device/DatabaseLoadSorter.spec.ts @@ -1,6 +1,6 @@ import { ContentType } from '@standardnotes/common' import { FullyFormedPayloadInterface } from '@standardnotes/models' -import { GetSortedPayloadsByPriority } from './Utils' +import { GetSortedPayloadsByPriority } from './DatabaseLoadSorter' describe('GetSortedPayloadsByPriority', () => { let payloads: FullyFormedPayloadInterface[] = [] @@ -26,11 +26,11 @@ describe('GetSortedPayloadsByPriority', () => { } as FullyFormedPayloadInterface, ] - const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority( - payloads, + const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(payloads, { contentTypePriority, - launchPriorityUuids, - ) + uuidPriority: launchPriorityUuids, + batchSize: 1000, + }) expect(itemsKeyPayloads.length).toBe(1) expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey) @@ -84,11 +84,11 @@ describe('GetSortedPayloadsByPriority', () => { launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid] - const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority( - payloads, + const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(payloads, { contentTypePriority, - launchPriorityUuids, - ) + uuidPriority: launchPriorityUuids, + batchSize: 1000, + }) expect(itemsKeyPayloads.length).toBe(1) expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey) @@ -116,12 +116,12 @@ describe('GetSortedPayloadsByPriority', () => { { content_type: ContentType.Note, uuid: unprioritizedNoteUuid, - serverUpdatedAt: new Date(1), + updated_at: new Date(1), } as FullyFormedPayloadInterface, { content_type: ContentType.Tag, uuid: unprioritizedTagUuid, - serverUpdatedAt: new Date(2), + updated_at: new Date(2), } as FullyFormedPayloadInterface, { content_type: ContentType.Note, @@ -135,7 +135,11 @@ describe('GetSortedPayloadsByPriority', () => { launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid] - const { remainingPayloads } = GetSortedPayloadsByPriority(payloads, contentTypePriority, launchPriorityUuids) + const { remainingPayloads } = GetSortedPayloadsByPriority(payloads, { + contentTypePriority, + uuidPriority: launchPriorityUuids, + batchSize: 1000, + }) expect(remainingPayloads.length).toBe(4) expect(remainingPayloads[0].uuid).toBe(prioritizedNoteUuid) diff --git a/packages/snjs/lib/Services/Sync/Utils.ts b/packages/services/src/Domain/Device/DatabaseLoadSorter.ts similarity index 60% rename from packages/snjs/lib/Services/Sync/Utils.ts rename to packages/services/src/Domain/Device/DatabaseLoadSorter.ts index cf917eb4d..12a05fd63 100644 --- a/packages/snjs/lib/Services/Sync/Utils.ts +++ b/packages/services/src/Domain/Device/DatabaseLoadSorter.ts @@ -1,18 +1,18 @@ -import { UuidString } from '@Lib/Types' -import { ContentType } from '@standardnotes/common' -import { FullyFormedPayloadInterface } from '@standardnotes/models' +import { DatabaseItemMetadata } from './DatabaseItemMetadata' +import { DatabaseLoadOptions } from './DatabaseLoadOptions' +import { ContentType, Uuid } from '@standardnotes/common' /** * Sorts payloads according by most recently modified first, according to the priority, * whereby the earlier a content_type appears in the priorityList, * the earlier it will appear in the resulting sorted array. */ -function SortPayloadsByRecentAndContentPriority( - payloads: FullyFormedPayloadInterface[], +function SortPayloadsByRecentAndContentPriority( + payloads: T[], contentTypePriorityList: ContentType[], -): FullyFormedPayloadInterface[] { +): T[] { return payloads.sort((a, b) => { - const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime() + const dateResult = new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime() let aPriority = 0 let bPriority = 0 @@ -45,12 +45,12 @@ function SortPayloadsByRecentAndContentPriority( * whereby the earlier a uuid appears in the priorityList, * the earlier it will appear in the resulting sorted array. */ -function SortPayloadsByRecentAndUuidPriority( - payloads: FullyFormedPayloadInterface[], - uuidPriorityList: UuidString[], -): FullyFormedPayloadInterface[] { +function SortPayloadsByRecentAndUuidPriority( + payloads: T[], + uuidPriorityList: Uuid[], +): T[] { return payloads.sort((a, b) => { - const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime() + const dateResult = new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime() let aPriority = 0 let bPriority = 0 @@ -78,25 +78,24 @@ function SortPayloadsByRecentAndUuidPriority( }) } -export function GetSortedPayloadsByPriority( - payloads: FullyFormedPayloadInterface[], - contentTypePriorityList: ContentType[], - uuidPriorityList: UuidString[], +export function GetSortedPayloadsByPriority( + payloads: T[], + options: DatabaseLoadOptions, ): { - itemsKeyPayloads: FullyFormedPayloadInterface[] - contentTypePriorityPayloads: FullyFormedPayloadInterface[] - remainingPayloads: FullyFormedPayloadInterface[] + itemsKeyPayloads: T[] + contentTypePriorityPayloads: T[] + remainingPayloads: T[] } { - const itemsKeyPayloads: FullyFormedPayloadInterface[] = [] - const contentTypePriorityPayloads: FullyFormedPayloadInterface[] = [] - const remainingPayloads: FullyFormedPayloadInterface[] = [] + const itemsKeyPayloads: T[] = [] + const contentTypePriorityPayloads: T[] = [] + const remainingPayloads: T[] = [] for (let index = 0; index < payloads.length; index++) { const payload = payloads[index] if (payload.content_type === ContentType.ItemsKey) { itemsKeyPayloads.push(payload) - } else if (contentTypePriorityList.includes(payload.content_type)) { + } else if (options.contentTypePriority.includes(payload.content_type)) { contentTypePriorityPayloads.push(payload) } else { remainingPayloads.push(payload) @@ -107,8 +106,8 @@ export function GetSortedPayloadsByPriority( itemsKeyPayloads, contentTypePriorityPayloads: SortPayloadsByRecentAndContentPriority( contentTypePriorityPayloads, - contentTypePriorityList, + options.contentTypePriority, ), - remainingPayloads: SortPayloadsByRecentAndUuidPriority(remainingPayloads, uuidPriorityList), + remainingPayloads: SortPayloadsByRecentAndUuidPriority(remainingPayloads, options.uuidPriority), } } diff --git a/packages/services/src/Domain/Device/DeviceInterface.ts b/packages/services/src/Domain/Device/DeviceInterface.ts index 8ee94424f..f4ef66e26 100644 --- a/packages/services/src/Domain/Device/DeviceInterface.ts +++ b/packages/services/src/Domain/Device/DeviceInterface.ts @@ -2,10 +2,14 @@ import { ApplicationIdentifier } from '@standardnotes/common' import { FullyFormedTransferPayload, TransferPayload, - LegacyRawKeychainValue, NamespacedRootKeyInKeychain, Environment, } from '@standardnotes/models' +import { + DatabaseLoadOptions, + DatabaseKeysLoadChunkResponse, + DatabaseFullEntryLoadChunkResponse, +} from './DatabaseLoadOptions' /** * Platforms must override this class to provide platform specific utilities @@ -21,8 +25,6 @@ export interface DeviceInterface { getJsonParsedRawStorageValue(key: string): Promise - getAllRawStorageKeyValues(): Promise<{ key: string; value: unknown }[]> - setRawStorageValue(key: string, value: string): Promise removeRawStorageValue(key: string): Promise @@ -38,10 +40,10 @@ export interface DeviceInterface { */ openDatabase(identifier: ApplicationIdentifier): Promise<{ isNewDatabase?: boolean } | undefined> - /** - * In a key/value database, this function returns just the keys. - */ - getDatabaseKeys(): Promise + getDatabaseLoadChunks( + options: DatabaseLoadOptions, + identifier: ApplicationIdentifier, + ): Promise /** * Remove all keychain and database data from device. @@ -52,17 +54,22 @@ export interface DeviceInterface { */ clearAllDataFromDevice(workspaceIdentifiers: ApplicationIdentifier[]): Promise<{ killsApplication: boolean }> - getAllRawDatabasePayloads( + getAllDatabaseEntries( identifier: ApplicationIdentifier, ): Promise - saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier): Promise + getDatabaseEntries( + identifier: ApplicationIdentifier, + keys: string[], + ): Promise - saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise + saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier): Promise - removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier): Promise + saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise - removeAllRawDatabasePayloads(identifier: ApplicationIdentifier): Promise + removeDatabaseEntry(id: string, identifier: ApplicationIdentifier): Promise + + removeAllDatabaseEntries(identifier: ApplicationIdentifier): Promise getNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise @@ -70,8 +77,6 @@ export interface DeviceInterface { clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise - setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise - clearRawKeychainValue(): Promise openUrl(url: string): void diff --git a/packages/services/src/Domain/Storage/StorageServiceInterface.ts b/packages/services/src/Domain/Storage/StorageServiceInterface.ts index 9faba6ce2..1663f62f0 100644 --- a/packages/services/src/Domain/Storage/StorageServiceInterface.ts +++ b/packages/services/src/Domain/Storage/StorageServiceInterface.ts @@ -1,7 +1,13 @@ -import { FullyFormedPayloadInterface, PayloadInterface, RootKeyInterface } from '@standardnotes/models' +import { + FullyFormedPayloadInterface, + PayloadInterface, + RootKeyInterface, + FullyFormedTransferPayload, +} from '@standardnotes/models' import { StoragePersistencePolicies, StorageValueModes } from './StorageTypes' export interface StorageServiceInterface { + getAllRawPayloads(): Promise getValue(key: string, mode?: StorageValueModes, defaultValue?: T): T canDecryptWithKey(key: RootKeyInterface): Promise savePayload(payload: PayloadInterface): Promise diff --git a/packages/services/src/Domain/Sync/SyncOptions.ts b/packages/services/src/Domain/Sync/SyncOptions.ts index abc44c983..96e0352e3 100644 --- a/packages/services/src/Domain/Sync/SyncOptions.ts +++ b/packages/services/src/Domain/Sync/SyncOptions.ts @@ -11,6 +11,7 @@ export type SyncOptions = { checkIntegrity?: boolean /** Internally used to keep track of how sync requests were spawned. */ source: SyncSource + sourceDescription?: string /** Whether to await any sync requests that may be queued from this call. */ awaitAll?: boolean /** diff --git a/packages/services/src/Domain/index.ts b/packages/services/src/Domain/index.ts index cb1ffdda7..e770eb4c3 100644 --- a/packages/services/src/Domain/index.ts +++ b/packages/services/src/Domain/index.ts @@ -22,6 +22,9 @@ export * from './Device/DeviceInterface' export * from './Device/MobileDeviceInterface' export * from './Device/TypeCheck' export * from './Device/WebOrDesktopDeviceInterface' +export * from './Device/DatabaseLoadOptions' +export * from './Device/DatabaseItemMetadata' +export * from './Device/DatabaseLoadSorter' export * from './Diagnostics/ServiceDiagnostics' export * from './Encryption/BackupFileDecryptor' export * from './Encryption/EncryptionService' diff --git a/packages/snjs/lib/Application/Application.spec.ts b/packages/snjs/lib/Application/Application.spec.ts index 8381b2c0f..192d9902c 100644 --- a/packages/snjs/lib/Application/Application.spec.ts +++ b/packages/snjs/lib/Application/Application.spec.ts @@ -25,7 +25,7 @@ describe('application', () => { device = {} as jest.Mocked device.openDatabase = jest.fn().mockResolvedValue(true) - device.getAllRawDatabasePayloads = jest.fn().mockReturnValue([]) + device.getAllDatabaseEntries = jest.fn().mockReturnValue([]) device.setRawStorageValue = jest.fn() device.getRawStorageValue = jest.fn().mockImplementation((key) => { if (key === namespacedKey(identifier, RawStorageKey.SnjsVersion)) { @@ -33,9 +33,6 @@ describe('application', () => { } return undefined }) - device.getDatabaseKeys = async () => { - return Promise.resolve(['1', '2', '3']) - } application = new SNApplication({ environment: Environment.Mobile, @@ -75,7 +72,6 @@ describe('application', () => { currentPersistPromise: false, isStorageWrapped: false, allRawPayloadsCount: 0, - databaseKeys: ['1', '2', '3'], }, encryption: expect.objectContaining({ getLatestVersion: '004', diff --git a/packages/snjs/lib/Application/Application.ts b/packages/snjs/lib/Application/Application.ts index 61e377c5c..903b74b05 100644 --- a/packages/snjs/lib/Application/Application.ts +++ b/packages/snjs/lib/Application/Application.ts @@ -410,28 +410,32 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli await this.notifyEvent(ApplicationEvent.Launched) await this.handleStage(ExternalServices.ApplicationStage.Launched_10) - const databasePayloads = await this.syncService.getDatabasePayloads() await this.handleStage(ExternalServices.ApplicationStage.LoadingDatabase_11) - if (this.createdNewDatabase) { await this.syncService.onNewDatabaseCreated() } /** * We don't want to await this, as we want to begin allowing the app to function - * before local data has been loaded fully. We await only initial - * `getDatabasePayloads` to lock in on database state. + * before local data has been loaded fully. */ - const loadPromise = this.syncService.loadDatabasePayloads(databasePayloads).then(async () => { - if (this.dealloced) { - throw 'Application has been destroyed.' - } - await this.handleStage(ExternalServices.ApplicationStage.LoadedDatabase_12) - this.beginAutoSyncTimer() - await this.syncService.sync({ - mode: ExternalServices.SyncMode.DownloadFirst, - source: ExternalServices.SyncSource.External, + const loadPromise = this.syncService + .loadDatabasePayloads() + .then(async () => { + if (this.dealloced) { + throw 'Application has been destroyed.' + } + await this.handleStage(ExternalServices.ApplicationStage.LoadedDatabase_12) + this.beginAutoSyncTimer() + await this.syncService.sync({ + mode: ExternalServices.SyncMode.DownloadFirst, + source: ExternalServices.SyncSource.External, + sourceDescription: 'Application Launch', + }) + }) + .catch((error) => { + void this.notifyEvent(ApplicationEvent.LocalDatabaseReadError, error) + throw error }) - }) if (awaitDatabaseLoad) { await loadPromise } @@ -463,7 +467,7 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli private beginAutoSyncTimer() { this.autoSyncInterval = setInterval(() => { this.syncService.log('Syncing from autosync') - void this.sync.sync() + void this.sync.sync({ sourceDescription: 'Auto Sync' }) }, DEFAULT_AUTO_SYNC_INTERVAL) } @@ -1542,10 +1546,10 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli switch (event) { case InternalServices.SessionEvent.Restored: { void (async () => { - await this.sync.sync() + await this.sync.sync({ sourceDescription: 'Session restored pre key creation' }) if (this.protocolService.needsNewRootKeyBasedItemsKey()) { void this.protocolService.createNewDefaultItemsKey().then(() => { - void this.sync.sync() + void this.sync.sync({ sourceDescription: 'Session restored post key creation' }) }) } })() @@ -1573,6 +1577,8 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli this.payloadManager, this.apiService, this.historyManager, + this.deviceInterface, + this.identifier, { loadBatchSize: this.options.loadBatchSize, }, diff --git a/packages/snjs/lib/Application/index.ts b/packages/snjs/lib/Application/index.ts index 29e6e3594..ea875adf3 100644 --- a/packages/snjs/lib/Application/index.ts +++ b/packages/snjs/lib/Application/index.ts @@ -2,3 +2,4 @@ export * from './Application' export * from './Event' export * from './LiveItem' export * from './Platforms' +export * from './Options/Defaults' diff --git a/packages/snjs/lib/Logging.ts b/packages/snjs/lib/Logging.ts new file mode 100644 index 000000000..b19375a09 --- /dev/null +++ b/packages/snjs/lib/Logging.ts @@ -0,0 +1,22 @@ +import { log as utilsLog } from '@standardnotes/utils' + +export const isDev = true + +export enum LoggingDomain { + DatabaseLoad, + Sync, +} + +const LoggingStatus: Record = { + [LoggingDomain.DatabaseLoad]: false, + [LoggingDomain.Sync]: false, +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function log(domain: LoggingDomain, ...args: any[]): void { + if (!isDev || !LoggingStatus[domain]) { + return + } + + utilsLog(LoggingDomain[domain], ...args) +} diff --git a/packages/snjs/lib/Migrations/Base.ts b/packages/snjs/lib/Migrations/Base.ts index 13f71a4f3..4216f2a5e 100644 --- a/packages/snjs/lib/Migrations/Base.ts +++ b/packages/snjs/lib/Migrations/Base.ts @@ -165,12 +165,11 @@ export class BaseMigration extends Migration { } private async repairMissingKeychain() { - const version = (await this.getStoredVersion()) as string const rawAccountParams = await this.reader.getAccountKeyParams() /** Choose an item to decrypt against */ const allItems = ( - await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier) + await this.services.deviceInterface.getAllDatabaseEntries(this.services.identifier) ).map((p) => new EncryptedPayload(p)) let itemToDecrypt = allItems.find((item) => { @@ -226,21 +225,10 @@ export class BaseMigration extends Migration { ) } else { /** - * If decryption succeeds, store the generated account key where it is expected, - * either in top-level keychain in 1.0.0, and namespaced location in 2.0.0+. + * If decryption succeeds, store the generated account key where it is expected. */ - if (version === PreviousSnjsVersion1_0_0) { - /** Store in top level keychain */ - await this.services.deviceInterface.setLegacyRawKeychainValue({ - mk: rootKey.masterKey, - ak: rootKey.dataAuthenticationKey as string, - version: accountParams.version, - }) - } else { - /** Store in namespaced location */ - const rawKey = rootKey.getKeychainValue() - await this.services.deviceInterface.setNamespacedKeychainValue(rawKey, this.services.identifier) - } + const rawKey = rootKey.getKeychainValue() + await this.services.deviceInterface.setNamespacedKeychainValue(rawKey, this.services.identifier) resolve(true) this.services.challengeService.completeChallenge(challenge) } diff --git a/packages/snjs/lib/Migrations/StorageReaders/Functions.ts b/packages/snjs/lib/Migrations/StorageReaders/Functions.ts index a637ca2a2..62b4228ba 100644 --- a/packages/snjs/lib/Migrations/StorageReaders/Functions.ts +++ b/packages/snjs/lib/Migrations/StorageReaders/Functions.ts @@ -5,9 +5,7 @@ import { DeviceInterface } from '@standardnotes/services' import { StorageReader } from './Reader' import * as ReaderClasses from './Versions' -function ReaderClassForVersion( - version: string, -): typeof ReaderClasses.StorageReader2_0_0 | typeof ReaderClasses.StorageReader1_0_0 { +function ReaderClassForVersion(version: string): typeof ReaderClasses.StorageReader2_0_0 { /** Sort readers by newest first */ const allReaders = Object.values(ReaderClasses).sort((a, b) => { return compareSemVersions(a.version(), b.version()) * -1 diff --git a/packages/snjs/lib/Migrations/StorageReaders/Versions/Reader1_0_0.ts b/packages/snjs/lib/Migrations/StorageReaders/Versions/Reader1_0_0.ts deleted file mode 100644 index 24ab4a212..000000000 --- a/packages/snjs/lib/Migrations/StorageReaders/Versions/Reader1_0_0.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { isNullOrUndefined } from '@standardnotes/utils' -import { isEnvironmentMobile } from '@Lib/Application/Platforms' -import { PreviousSnjsVersion1_0_0 } from '../../../Version' -import { isMobileDevice, LegacyKeys1_0_0 } from '@standardnotes/services' -import { StorageReader } from '../Reader' - -export class StorageReader1_0_0 extends StorageReader { - static override version() { - return PreviousSnjsVersion1_0_0 - } - - public async getAccountKeyParams() { - return this.deviceInterface.getJsonParsedRawStorageValue(LegacyKeys1_0_0.AllAccountKeyParamsKey) - } - - /** - * In 1.0.0, web uses raw storage for unwrapped account key, and mobile uses - * the keychain - */ - public async hasNonWrappedAccountKeys() { - if (isMobileDevice(this.deviceInterface)) { - const value = await this.deviceInterface.getRawKeychainValue() - return !isNullOrUndefined(value) - } else { - const value = await this.deviceInterface.getRawStorageValue('mk') - return !isNullOrUndefined(value) - } - } - - public async hasPasscode() { - if (isEnvironmentMobile(this.environment)) { - const rawPasscodeParams = await this.deviceInterface.getJsonParsedRawStorageValue( - LegacyKeys1_0_0.MobilePasscodeParamsKey, - ) - return !isNullOrUndefined(rawPasscodeParams) - } else { - const encryptedStorage = await this.deviceInterface.getJsonParsedRawStorageValue( - LegacyKeys1_0_0.WebEncryptedStorageKey, - ) - return !isNullOrUndefined(encryptedStorage) - } - } - - /** Keychain was not used on desktop/web in 1.0.0 */ - public usesKeychain() { - return isEnvironmentMobile(this.environment) ? true : false - } -} diff --git a/packages/snjs/lib/Migrations/StorageReaders/Versions/index.ts b/packages/snjs/lib/Migrations/StorageReaders/Versions/index.ts index 68c37d73d..a9225a9e1 100644 --- a/packages/snjs/lib/Migrations/StorageReaders/Versions/index.ts +++ b/packages/snjs/lib/Migrations/StorageReaders/Versions/index.ts @@ -1,2 +1 @@ export { StorageReader2_0_0 } from './Reader2_0_0' -export { StorageReader1_0_0 } from './Reader1_0_0' diff --git a/packages/snjs/lib/Migrations/Versions/2_0_0.ts b/packages/snjs/lib/Migrations/Versions/2_0_0.ts deleted file mode 100644 index ec032f7c8..000000000 --- a/packages/snjs/lib/Migrations/Versions/2_0_0.ts +++ /dev/null @@ -1,730 +0,0 @@ -import { AnyKeyParamsContent, ContentType, ProtocolVersion } from '@standardnotes/common' -import { Migration } from '@Lib/Migrations/Migration' -import { MigrationServices } from '../MigrationServices' -import { PreviousSnjsVersion2_0_0 } from '../../Version' -import { SNRootKey, CreateNewRootKey } from '@standardnotes/encryption' -import { DiskStorageService } from '../../Services/Storage/DiskStorageService' -import { StorageReader1_0_0 } from '../StorageReaders/Versions/Reader1_0_0' -import * as Models from '@standardnotes/models' -import * as Services from '@standardnotes/services' -import * as Utils from '@standardnotes/utils' -import { isEnvironmentMobile, isEnvironmentWebOrDesktop } from '@Lib/Application/Platforms' -import { - getIncrementedDirtyIndex, - LegacyMobileKeychainStructure, - PayloadTimestampDefaults, -} from '@standardnotes/models' -import { isMobileDevice } from '@standardnotes/services' -import { LegacySession } from '@standardnotes/domain-core' - -interface LegacyStorageContent extends Models.ItemContent { - storage: unknown -} - -interface LegacyAccountKeysValue { - ak: string - mk: string - version: string - jwt: string -} - -interface LegacyRootKeyContent extends Models.RootKeyContent { - accountKeys?: LegacyAccountKeysValue -} - -const LEGACY_SESSION_TOKEN_KEY = 'jwt' - -export class Migration2_0_0 extends Migration { - private legacyReader!: StorageReader1_0_0 - - constructor(services: MigrationServices) { - super(services) - this.legacyReader = new StorageReader1_0_0( - this.services.deviceInterface, - this.services.identifier, - this.services.environment, - ) - } - - static override version() { - return PreviousSnjsVersion2_0_0 - } - - protected registerStageHandlers() { - this.registerStageHandler(Services.ApplicationStage.PreparingForLaunch_0, async () => { - if (isEnvironmentWebOrDesktop(this.services.environment)) { - await this.migrateStorageStructureForWebDesktop() - } else if (isEnvironmentMobile(this.services.environment)) { - await this.migrateStorageStructureForMobile() - } - }) - this.registerStageHandler(Services.ApplicationStage.StorageDecrypted_09, async () => { - await this.migrateArbitraryRawStorageToManagedStorageAllPlatforms() - if (isEnvironmentMobile(this.services.environment)) { - await this.migrateMobilePreferences() - } - await this.migrateSessionStorage() - await this.deleteLegacyStorageValues() - }) - this.registerStageHandler(Services.ApplicationStage.LoadingDatabase_11, async () => { - await this.createDefaultItemsKeyForAllPlatforms() - this.markDone() - }) - } - - /** - * Web - * Migrates legacy storage structure into new managed format. - * If encrypted storage exists, we need to first decrypt it with the passcode. - * Then extract the account key from it. Then, encrypt storage with the - * account key. Then encrypt the account key with the passcode and store it - * within the new storage format. - * - * Generate note: We do not use the keychain if passcode is available. - */ - private async migrateStorageStructureForWebDesktop() { - const deviceInterface = this.services.deviceInterface - const newStorageRawStructure: Services.StorageValuesObject = { - [Services.ValueModesKeys.Wrapped]: {} as Models.LocalStorageEncryptedContextualPayload, - [Services.ValueModesKeys.Unwrapped]: {}, - [Services.ValueModesKeys.Nonwrapped]: {}, - } - const rawAccountKeyParams = (await this.legacyReader.getAccountKeyParams()) as AnyKeyParamsContent - /** Could be null if no account, or if account and storage is encrypted */ - if (rawAccountKeyParams) { - newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyParams] = rawAccountKeyParams - } - const encryptedStorage = (await deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.WebEncryptedStorageKey, - )) as Models.EncryptedTransferPayload - - if (encryptedStorage) { - const encryptedStoragePayload = new Models.EncryptedPayload(encryptedStorage) - - const passcodeResult = await this.webDesktopHelperGetPasscodeKeyAndDecryptEncryptedStorage( - encryptedStoragePayload, - ) - - const passcodeKey = passcodeResult.key - const decryptedStoragePayload = passcodeResult.decryptedStoragePayload - const passcodeParams = passcodeResult.keyParams - - newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyWrapperKeyParams] = passcodeParams.getPortableValue() - - const rawStorageValueStore = Utils.Copy(decryptedStoragePayload.content.storage) - const storageValueStore: Record = Utils.jsonParseEmbeddedKeys(rawStorageValueStore) - /** Store previously encrypted auth_params into new nonwrapped value key */ - - const accountKeyParams = storageValueStore[Services.LegacyKeys1_0_0.AllAccountKeyParamsKey] as AnyKeyParamsContent - newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyParams] = accountKeyParams - - let keyToEncryptStorageWith = passcodeKey - /** Extract account key (mk, pw, ak) if it exists */ - const hasAccountKeys = !Utils.isNullOrUndefined(storageValueStore.mk) - - if (hasAccountKeys) { - const { accountKey, wrappedKey } = await this.webDesktopHelperExtractAndWrapAccountKeysFromValueStore( - passcodeKey, - accountKeyParams, - storageValueStore, - ) - keyToEncryptStorageWith = accountKey - newStorageRawStructure.nonwrapped[Services.StorageKey.WrappedRootKey] = wrappedKey - } - - /** Encrypt storage with proper key */ - newStorageRawStructure.wrapped = await this.webDesktopHelperEncryptStorage( - keyToEncryptStorageWith, - decryptedStoragePayload, - storageValueStore, - ) - } else { - /** - * No encrypted storage, take account keys (if they exist) out of raw storage - * and place them in the keychain. */ - const ak = await this.services.deviceInterface.getRawStorageValue('ak') - const mk = await this.services.deviceInterface.getRawStorageValue('mk') - - if (ak || mk) { - const version = rawAccountKeyParams.version || (await this.getFallbackRootKeyVersion()) - - const accountKey = CreateNewRootKey({ - masterKey: mk as string, - dataAuthenticationKey: ak as string, - version: version, - keyParams: rawAccountKeyParams, - }) - await this.services.deviceInterface.setNamespacedKeychainValue( - accountKey.getKeychainValue(), - this.services.identifier, - ) - } - } - - /** Persist storage under new key and structure */ - await this.allPlatformHelperSetStorageStructure(newStorageRawStructure) - } - - /** - * Helper - * All platforms - */ - private async allPlatformHelperSetStorageStructure(rawStructure: Services.StorageValuesObject) { - const newStructure = DiskStorageService.DefaultValuesObject( - rawStructure.wrapped, - rawStructure.unwrapped, - rawStructure.nonwrapped, - ) as Partial - - newStructure[Services.ValueModesKeys.Unwrapped] = undefined - - await this.services.deviceInterface.setRawStorageValue( - Services.namespacedKey(this.services.identifier, Services.RawStorageKey.StorageObject), - JSON.stringify(newStructure), - ) - } - - /** - * Helper - * Web/desktop only - */ - private async webDesktopHelperGetPasscodeKeyAndDecryptEncryptedStorage( - encryptedPayload: Models.EncryptedPayloadInterface, - ) { - const rawPasscodeParams = (await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.WebPasscodeParamsKey, - )) as AnyKeyParamsContent - const passcodeParams = this.services.protocolService.createKeyParams(rawPasscodeParams) - - /** Decrypt it with the passcode */ - let decryptedStoragePayload: - | Models.DecryptedPayloadInterface - | Models.EncryptedPayloadInterface = encryptedPayload - let passcodeKey: SNRootKey | undefined - - await this.promptForPasscodeUntilCorrect(async (candidate: string) => { - passcodeKey = await this.services.protocolService.computeRootKey(candidate, passcodeParams) - decryptedStoragePayload = await this.services.protocolService.decryptSplitSingle({ - usesRootKey: { - items: [encryptedPayload], - key: passcodeKey, - }, - }) - - return !Models.isErrorDecryptingPayload(decryptedStoragePayload) - }) - - return { - decryptedStoragePayload: - decryptedStoragePayload as unknown as Models.DecryptedPayloadInterface, - key: passcodeKey as SNRootKey, - keyParams: passcodeParams, - } - } - - /** - * Helper - * Web/desktop only - */ - private async webDesktopHelperExtractAndWrapAccountKeysFromValueStore( - passcodeKey: SNRootKey, - accountKeyParams: AnyKeyParamsContent, - storageValueStore: Record, - ) { - const version = accountKeyParams?.version || (await this.getFallbackRootKeyVersion()) - const accountKey = CreateNewRootKey({ - masterKey: storageValueStore.mk as string, - dataAuthenticationKey: storageValueStore.ak as string, - version: version, - keyParams: accountKeyParams, - }) - - delete storageValueStore.mk - delete storageValueStore.pw - delete storageValueStore.ak - - const accountKeyPayload = accountKey.payload - - /** Encrypt account key with passcode */ - const encryptedAccountKey = await this.services.protocolService.encryptSplitSingle({ - usesRootKey: { - items: [accountKeyPayload], - key: passcodeKey, - }, - }) - return { - accountKey: accountKey, - wrappedKey: Models.CreateEncryptedLocalStorageContextPayload(encryptedAccountKey), - } - } - - /** - * Helper - * Web/desktop only - * Encrypt storage with account key - */ - async webDesktopHelperEncryptStorage( - key: SNRootKey, - decryptedStoragePayload: Models.DecryptedPayloadInterface, - storageValueStore: Record, - ) { - const wrapped = await this.services.protocolService.encryptSplitSingle({ - usesRootKey: { - items: [ - decryptedStoragePayload.copy({ - content_type: ContentType.EncryptedStorage, - content: storageValueStore as unknown as Models.ItemContent, - }), - ], - key: key, - }, - }) - - return Models.CreateEncryptedLocalStorageContextPayload(wrapped) - } - - /** - * Mobile - * On mobile legacy structure is mostly similar to new structure, - * in that the account key is encrypted with the passcode. But mobile did - * not have encrypted storage, so we simply need to transfer all existing - * storage values into new managed structure. - * - * In version <= 3.0.16 on mobile, encrypted account keys were stored in the keychain - * under `encryptedAccountKeys`. In 3.0.17 a migration was introduced that moved this value - * to storage under key `encrypted_account_keys`. We need to anticipate the keys being in - * either location. - * - * If no account but passcode only, the only thing we stored on mobile - * previously was keys.offline.pw and keys.offline.timing in the keychain - * that we compared against for valid decryption. - * In the new version, we know a passcode is correct if it can decrypt storage. - * As part of the migration, we’ll need to request the raw passcode from user, - * compare it against the keychain offline.pw value, and if correct, - * migrate storage to new structure, and encrypt with passcode key. - * - * If account only, take the value in the keychain, and rename the values - * (i.e mk > masterKey). - * @access private - */ - async migrateStorageStructureForMobile() { - Utils.assert(isMobileDevice(this.services.deviceInterface)) - - const keychainValue = - (await this.services.deviceInterface.getRawKeychainValue()) as unknown as LegacyMobileKeychainStructure - - const wrappedAccountKey = ((await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.MobileWrappedRootKeyKey, - )) || keychainValue?.encryptedAccountKeys) as Models.EncryptedTransferPayload - - const rawAccountKeyParams = (await this.legacyReader.getAccountKeyParams()) as AnyKeyParamsContent - - const rawPasscodeParams = (await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.MobilePasscodeParamsKey, - )) as AnyKeyParamsContent - - const firstRunValue = await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.NonwrappedStorageKey.MobileFirstRun, - ) - - const rawStructure: Services.StorageValuesObject = { - [Services.ValueModesKeys.Nonwrapped]: { - [Services.StorageKey.WrappedRootKey]: wrappedAccountKey, - /** A 'hash' key may be present from legacy versions that should be deleted */ - [Services.StorageKey.RootKeyWrapperKeyParams]: Utils.omitByCopy(rawPasscodeParams, ['hash' as never]), - [Services.StorageKey.RootKeyParams]: rawAccountKeyParams, - [Services.NonwrappedStorageKey.MobileFirstRun]: firstRunValue, - }, - [Services.ValueModesKeys.Unwrapped]: {}, - [Services.ValueModesKeys.Wrapped]: {} as Models.LocalStorageDecryptedContextualPayload, - } - - const biometricPrefs = (await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.MobileBiometricsPrefs, - )) as { enabled: boolean; timing: unknown } - - if (biometricPrefs) { - rawStructure.nonwrapped[Services.StorageKey.BiometricsState] = biometricPrefs.enabled - rawStructure.nonwrapped[Services.StorageKey.MobileBiometricsTiming] = biometricPrefs.timing - } - - const passcodeKeyboardType = await this.services.deviceInterface.getRawStorageValue( - Services.LegacyKeys1_0_0.MobilePasscodeKeyboardType, - ) - - if (passcodeKeyboardType) { - rawStructure.nonwrapped[Services.StorageKey.MobilePasscodeKeyboardType] = passcodeKeyboardType - } - - if (rawPasscodeParams) { - const passcodeParams = this.services.protocolService.createKeyParams(rawPasscodeParams) - const getPasscodeKey = async () => { - let passcodeKey: SNRootKey | undefined - - await this.promptForPasscodeUntilCorrect(async (candidate: string) => { - passcodeKey = await this.services.protocolService.computeRootKey(candidate, passcodeParams) - - const pwHash = keychainValue?.offline?.pw - - if (pwHash) { - return passcodeKey.serverPassword === pwHash - } else { - /** - * Fallback decryption if keychain is missing for some reason. If account, - * validate by attempting to decrypt wrapped account key. Otherwise, validate - * by attempting to decrypt random item. - * */ - if (wrappedAccountKey) { - const decryptedAcctKey = await this.services.protocolService.decryptSplitSingle({ - usesRootKey: { - items: [new Models.EncryptedPayload(wrappedAccountKey)], - key: passcodeKey, - }, - }) - return !Models.isErrorDecryptingPayload(decryptedAcctKey) - } else { - const item = ( - await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier) - )[0] as Models.EncryptedTransferPayload - - if (!item) { - throw Error('Passcode only migration aborting due to missing keychain.offline.pw') - } - - const decryptedPayload = await this.services.protocolService.decryptSplitSingle({ - usesRootKey: { - items: [new Models.EncryptedPayload(item)], - key: passcodeKey, - }, - }) - return !Models.isErrorDecryptingPayload(decryptedPayload) - } - } - }) - - return passcodeKey as SNRootKey - } - - rawStructure.nonwrapped[Services.StorageKey.MobilePasscodeTiming] = keychainValue?.offline?.timing - - if (wrappedAccountKey) { - /** - * Account key is encrypted with passcode. Inside, the accountKey is located inside - * content.accountKeys. We want to unembed these values to main content, rename - * with proper property names, wrap again, and store in new rawStructure. - */ - const passcodeKey = await getPasscodeKey() - const payload = new Models.EncryptedPayload(wrappedAccountKey) - const unwrappedAccountKey = await this.services.protocolService.decryptSplitSingle({ - usesRootKey: { - items: [payload], - key: passcodeKey, - }, - }) - - if (Models.isErrorDecryptingPayload(unwrappedAccountKey)) { - return - } - - const accountKeyContent = unwrappedAccountKey.content.accountKeys as LegacyAccountKeysValue - - const version = - accountKeyContent.version || rawAccountKeyParams?.version || (await this.getFallbackRootKeyVersion()) - - const newAccountKey = unwrappedAccountKey.copy({ - content: Models.FillItemContent({ - masterKey: accountKeyContent.mk, - dataAuthenticationKey: accountKeyContent.ak, - version: version as ProtocolVersion, - keyParams: rawAccountKeyParams, - accountKeys: undefined, - }), - }) - - const newWrappedAccountKey = await this.services.protocolService.encryptSplitSingle({ - usesRootKey: { - items: [newAccountKey], - key: passcodeKey, - }, - }) - rawStructure.nonwrapped[Services.StorageKey.WrappedRootKey] = - Models.CreateEncryptedLocalStorageContextPayload(newWrappedAccountKey) - - if (accountKeyContent.jwt) { - /** Move the jwt to raw storage so that it can be migrated in `migrateSessionStorage` */ - void this.services.deviceInterface.setRawStorageValue(LEGACY_SESSION_TOKEN_KEY, accountKeyContent.jwt) - } - await this.services.deviceInterface.clearRawKeychainValue() - } else if (!wrappedAccountKey) { - /** Passcode only, no account */ - const passcodeKey = await getPasscodeKey() - const payload = new Models.DecryptedPayload({ - uuid: Utils.UuidGenerator.GenerateUuid(), - content: Models.FillItemContent(rawStructure.unwrapped), - content_type: ContentType.EncryptedStorage, - ...PayloadTimestampDefaults(), - }) - - /** Encrypt new storage.unwrapped structure with passcode */ - const wrapped = await this.services.protocolService.encryptSplitSingle({ - usesRootKey: { - items: [payload], - key: passcodeKey, - }, - }) - rawStructure.wrapped = Models.CreateEncryptedLocalStorageContextPayload(wrapped) - - await this.services.deviceInterface.clearRawKeychainValue() - } - } else { - /** No passcode, potentially account. Migrate keychain property keys. */ - const hasAccount = !Utils.isNullOrUndefined(keychainValue?.mk) - if (hasAccount) { - const accountVersion = - (keychainValue.version as ProtocolVersion) || - rawAccountKeyParams?.version || - (await this.getFallbackRootKeyVersion()) - - const accountKey = CreateNewRootKey({ - masterKey: keychainValue.mk, - dataAuthenticationKey: keychainValue.ak, - version: accountVersion, - keyParams: rawAccountKeyParams, - }) - - await this.services.deviceInterface.setNamespacedKeychainValue( - accountKey.getKeychainValue(), - this.services.identifier, - ) - - if (keychainValue.jwt) { - /** Move the jwt to raw storage so that it can be migrated in `migrateSessionStorage` */ - void this.services.deviceInterface.setRawStorageValue(LEGACY_SESSION_TOKEN_KEY, keychainValue.jwt) - } - } - } - - /** Move encrypted account key into place where it is now expected */ - await this.allPlatformHelperSetStorageStructure(rawStructure) - } - - /** - * If we are unable to determine a root key's version, due to missing version - * parameter from key params due to 001 or 002, we need to fallback to checking - * any encrypted payload and retrieving its version. - * - * If we are unable to garner any meaningful information, we will default to 002. - * - * (Previously we attempted to discern version based on presence of keys.ak; if ak, - * then 003, otherwise 002. However, late versions of 002 also inluded an ak, so this - * method can't be used. This method also didn't account for 001 versions.) - */ - private async getFallbackRootKeyVersion() { - const anyItem = ( - await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier) - )[0] as Models.EncryptedTransferPayload - - if (!anyItem) { - return ProtocolVersion.V002 - } - - const payload = new Models.EncryptedPayload(anyItem) - return payload.version || ProtocolVersion.V002 - } - - /** - * All platforms - * Migrate all previously independently stored storage keys into new - * managed approach. - */ - private async migrateArbitraryRawStorageToManagedStorageAllPlatforms() { - const allKeyValues = await this.services.deviceInterface.getAllRawStorageKeyValues() - const legacyKeys = Utils.objectToValueArray(Services.LegacyKeys1_0_0) - - const tryJsonParse = (value: string) => { - try { - return JSON.parse(value) - } catch (e) { - return value - } - } - - const applicationIdentifier = this.services.identifier - - for (const keyValuePair of allKeyValues) { - const key = keyValuePair.key - const value = keyValuePair.value - const isNameSpacedKey = - applicationIdentifier && applicationIdentifier.length > 0 && key.startsWith(applicationIdentifier) - if (legacyKeys.includes(key) || isNameSpacedKey) { - continue - } - if (!Utils.isNullOrUndefined(value)) { - /** - * Raw values should always have been json stringified. - * New values should always be objects/parsed. - */ - const newValue = tryJsonParse(value as string) - this.services.storageService.setValue(key, newValue) - } - } - } - - /** - * All platforms - * Deletes all StorageKey and LegacyKeys1_0_0 from root raw storage. - * @access private - */ - async deleteLegacyStorageValues() { - const miscKeys = [ - 'mk', - 'ak', - 'pw', - /** v1 unused key */ - 'encryptionKey', - /** v1 unused key */ - 'authKey', - 'jwt', - 'ephemeral', - 'cachedThemes', - ] - - const managedKeys = [ - ...Utils.objectToValueArray(Services.StorageKey), - ...Utils.objectToValueArray(Services.LegacyKeys1_0_0), - ...miscKeys, - ] - - for (const key of managedKeys) { - await this.services.deviceInterface.removeRawStorageValue(key) - } - } - - /** - * Mobile - * Migrate mobile preferences - */ - private async migrateMobilePreferences() { - const lastExportDate = await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.MobileLastExportDate, - ) - const doNotWarnUnsupportedEditors = await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.MobileDoNotWarnUnsupportedEditors, - ) - const legacyOptionsState = (await this.services.deviceInterface.getJsonParsedRawStorageValue( - Services.LegacyKeys1_0_0.MobileOptionsState, - )) as Record - - let migratedOptionsState = {} - - if (legacyOptionsState) { - const legacySortBy = legacyOptionsState.sortBy - migratedOptionsState = { - sortBy: - legacySortBy === 'updated_at' || legacySortBy === 'client_updated_at' - ? Models.CollectionSort.UpdatedAt - : legacySortBy, - sortReverse: legacyOptionsState.sortReverse ?? false, - hideNotePreview: legacyOptionsState.hidePreviews ?? false, - hideDate: legacyOptionsState.hideDates ?? false, - hideTags: legacyOptionsState.hideTags ?? false, - } - } - const preferences = { - ...migratedOptionsState, - lastExportDate: lastExportDate ?? undefined, - doNotShowAgainUnsupportedEditors: doNotWarnUnsupportedEditors ?? false, - } - await this.services.storageService.setValue(Services.StorageKey.MobilePreferences, preferences) - } - - /** - * All platforms - * Migrate previously stored session string token into object - * On mobile, JWTs were previously stored in storage, inside of the user object, - * but then custom-migrated to be stored in the keychain. We must account for - * both scenarios here in case a user did not perform the custom platform migration. - * On desktop/web, JWT was stored in storage. - */ - private migrateSessionStorage() { - const USER_OBJECT_KEY = 'user' - let currentToken = this.services.storageService.getValue(LEGACY_SESSION_TOKEN_KEY) - const user = this.services.storageService.getValue<{ jwt: string; server: string }>(USER_OBJECT_KEY) - - if (!currentToken) { - /** Try the user object */ - if (user) { - currentToken = user.jwt - } - } - - if (!currentToken) { - /** - * If we detect that a user object is present, but the jwt is missing, - * we'll fill the jwt value with a junk value just so we create a session. - * When the client attempts to talk to the server, the server will reply - * with invalid token error, and the client will automatically prompt to reauthenticate. - */ - const hasAccount = !Utils.isNullOrUndefined(user) - if (hasAccount) { - currentToken = 'junk-value' - } else { - return - } - } - - const sessionOrError = LegacySession.create(currentToken) - if (!sessionOrError.isFailed()) { - this.services.storageService.setValue( - Services.StorageKey.Session, - this.services.legacySessionStorageMapper.toProjection(sessionOrError.getValue()), - ) - } - - /** Server has to be migrated separately on mobile */ - if (isEnvironmentMobile(this.services.environment)) { - if (user && user.server) { - this.services.storageService.setValue(Services.StorageKey.ServerHost, user.server) - } - } - } - - /** - * All platforms - * Create new default items key from root key. - * Otherwise, when data is loaded, we won't be able to decrypt it - * without existence of an item key. This will mean that if this migration - * is run on two different platforms for the same user, they will create - * two new items keys. Which one they use to decrypt past items and encrypt - * future items doesn't really matter. - * @access private - */ - async createDefaultItemsKeyForAllPlatforms() { - const rootKey = this.services.protocolService.getRootKey() - if (rootKey) { - const rootKeyParams = await this.services.protocolService.getRootKeyParams() - /** If params are missing a version, it must be 001 */ - const fallbackVersion = ProtocolVersion.V001 - - const payload = new Models.DecryptedPayload({ - uuid: Utils.UuidGenerator.GenerateUuid(), - content_type: ContentType.ItemsKey, - content: Models.FillItemContentSpecialized({ - itemsKey: rootKey.masterKey, - dataAuthenticationKey: rootKey.dataAuthenticationKey, - version: rootKeyParams?.version || fallbackVersion, - }), - dirty: true, - dirtyIndex: getIncrementedDirtyIndex(), - ...PayloadTimestampDefaults(), - }) - - const itemsKey = Models.CreateDecryptedItemFromPayload(payload) - - await this.services.itemManager.emitItemFromPayload( - itemsKey.payloadRepresentation(), - Models.PayloadEmitSource.LocalChanged, - ) - } - } -} diff --git a/packages/snjs/lib/Migrations/Versions/index.ts b/packages/snjs/lib/Migrations/Versions/index.ts index b065864e9..15f5de331 100644 --- a/packages/snjs/lib/Migrations/Versions/index.ts +++ b/packages/snjs/lib/Migrations/Versions/index.ts @@ -1,17 +1,9 @@ -import { Migration2_0_0 } from './2_0_0' import { Migration2_0_15 } from './2_0_15' import { Migration2_7_0 } from './2_7_0' import { Migration2_20_0 } from './2_20_0' import { Migration2_36_0 } from './2_36_0' import { Migration2_42_0 } from './2_42_0' -export const MigrationClasses = [ - Migration2_0_0, - Migration2_0_15, - Migration2_7_0, - Migration2_20_0, - Migration2_36_0, - Migration2_42_0, -] +export const MigrationClasses = [Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0] -export { Migration2_0_0, Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0 } +export { Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0 } diff --git a/packages/snjs/lib/Services/Preferences/PreferencesService.ts b/packages/snjs/lib/Services/Preferences/PreferencesService.ts index 3be576966..49a7f1166 100644 --- a/packages/snjs/lib/Services/Preferences/PreferencesService.ts +++ b/packages/snjs/lib/Services/Preferences/PreferencesService.ts @@ -83,7 +83,7 @@ export class SNPreferencesService void this.notifyEvent(PreferencesServiceEvent.PreferencesChanged) - void this.syncService.sync() + void this.syncService.sync({ sourceDescription: 'PreferencesService.setValue' }) } private async reload() { diff --git a/packages/snjs/lib/Services/Singleton/SingletonManager.ts b/packages/snjs/lib/Services/Singleton/SingletonManager.ts index 4cf8f0702..c37f91c91 100644 --- a/packages/snjs/lib/Services/Singleton/SingletonManager.ts +++ b/packages/snjs/lib/Services/Singleton/SingletonManager.ts @@ -133,7 +133,7 @@ export class SNSingletonManager extends AbstractService { * of a download-first request. */ if (handled.length > 0 && eventSource === SyncEvent.SyncCompletedWithAllItemsUploaded) { - await this.syncService?.sync() + await this.syncService?.sync({ sourceDescription: 'Resolve singletons for items' }) } } @@ -190,7 +190,7 @@ export class SNSingletonManager extends AbstractService { } }) - await this.syncService.sync() + await this.syncService.sync({ sourceDescription: 'Find or create singleton, before any sync has completed' }) removeObserver() @@ -224,7 +224,7 @@ export class SNSingletonManager extends AbstractService { const item = await this.itemManager.emitItemFromPayload(dirtyPayload, PayloadEmitSource.LocalInserted) - void this.syncService.sync() + void this.syncService.sync({ sourceDescription: 'After find or create singleton' }) return item as T } diff --git a/packages/snjs/lib/Services/Storage/DiskStorageService.ts b/packages/snjs/lib/Services/Storage/DiskStorageService.ts index dde5e411c..4a01cb0d9 100644 --- a/packages/snjs/lib/Services/Storage/DiskStorageService.ts +++ b/packages/snjs/lib/Services/Storage/DiskStorageService.ts @@ -20,6 +20,7 @@ import { PayloadTimestampDefaults, LocalStorageEncryptedContextualPayload, Environment, + FullyFormedTransferPayload, } from '@standardnotes/models' /** @@ -377,8 +378,8 @@ export class DiskStorageService extends Services.AbstractService implements Serv await this.immediatelyPersistValuesToDisk() } - public async getAllRawPayloads() { - return this.deviceInterface.getAllRawDatabasePayloads(this.identifier) + public async getAllRawPayloads(): Promise { + return this.deviceInterface.getAllDatabaseEntries(this.identifier) } public async savePayload(payload: FullyFormedPayloadInterface): Promise { @@ -432,7 +433,7 @@ export class DiskStorageService extends Services.AbstractService implements Serv const exportedDeleted = deleted.map(CreateDeletedLocalStorageContextPayload) return this.executeCriticalFunction(async () => { - return this.deviceInterface?.saveRawDatabasePayloads( + return this.deviceInterface?.saveDatabaseEntries( [...exportedEncrypted, ...exportedDecrypted, ...exportedDeleted], this.identifier, ) @@ -449,13 +450,13 @@ export class DiskStorageService extends Services.AbstractService implements Serv public async deletePayloadWithId(uuid: Uuid) { return this.executeCriticalFunction(async () => { - return this.deviceInterface.removeRawDatabasePayloadWithId(uuid, this.identifier) + return this.deviceInterface.removeDatabaseEntry(uuid, this.identifier) }) } public async clearAllPayloads() { return this.executeCriticalFunction(async () => { - return this.deviceInterface.removeAllRawDatabasePayloads(this.identifier) + return this.deviceInterface.removeAllDatabaseEntries(this.identifier) }) } @@ -482,7 +483,6 @@ export class DiskStorageService extends Services.AbstractService implements Serv currentPersistPromise: this.currentPersistPromise != undefined, isStorageWrapped: this.isStorageWrapped(), allRawPayloadsCount: (await this.getAllRawPayloads()).length, - databaseKeys: await this.deviceInterface.getDatabaseKeys(), }, } } diff --git a/packages/snjs/lib/Services/Sync/SyncService.ts b/packages/snjs/lib/Services/Sync/SyncService.ts index cdfb42c81..60e45f782 100644 --- a/packages/snjs/lib/Services/Sync/SyncService.ts +++ b/packages/snjs/lib/Services/Sync/SyncService.ts @@ -1,3 +1,4 @@ +import { log, LoggingDomain } from './../../Logging' import { AccountSyncOperation } from '@Lib/Services/Sync/Account/Operation' import { ContentType } from '@standardnotes/common' import { @@ -18,7 +19,6 @@ import { SNHistoryManager } from '../History/HistoryManager' import { SNLog } from '@Lib/Log' import { SNSessionManager } from '../Session/SessionManager' import { DiskStorageService } from '../Storage/DiskStorageService' -import { GetSortedPayloadsByPriority } from '@Lib/Services/Sync/Utils' import { SyncClientInterface } from './SyncClientInterface' import { SyncPromise } from './Types' import { SyncOpStatus } from '@Lib/Services/Sync/SyncOpStatus' @@ -33,7 +33,6 @@ import { DeltaOutOfSync, ImmutablePayloadCollection, CreatePayload, - FullyFormedTransferPayload, isEncryptedPayload, isDecryptedPayload, EncryptedPayloadInterface, @@ -74,6 +73,9 @@ import { SyncServiceInterface, DiagnosticInfo, EncryptionService, + DeviceInterface, + isFullEntryLoadChunkResponse, + isChunkFullEntry, } from '@standardnotes/services' import { OfflineSyncResponse } from './Offline/Response' import { @@ -142,6 +144,8 @@ export class SNSyncService private payloadManager: PayloadManager, private apiService: SNApiService, private historyService: SNHistoryManager, + private device: DeviceInterface, + private identifier: string, private readonly options: ApplicationSyncOptions, protected override internalEventBus: InternalEventBusInterface, ) { @@ -221,19 +225,13 @@ export class SNSyncService return this.databaseLoaded } - /** - * Used in tandem with `loadDatabasePayloads` - */ - public async getDatabasePayloads(): Promise { - return this.storageService.getAllRawPayloads().catch((error) => { - void this.notifyEvent(SyncEvent.DatabaseReadError, error) - throw error - }) - } - private async processItemsKeysFirstDuringDatabaseLoad( itemsKeysPayloads: FullyFormedPayloadInterface[], ): Promise { + if (itemsKeysPayloads.length === 0) { + return + } + const encryptedItemsKeysPayloads = itemsKeysPayloads.filter(isEncryptedPayload) const originallyDecryptedItemsKeysPayloads = itemsKeysPayloads.filter( @@ -254,57 +252,69 @@ export class SNSyncService ) } - /** - * @param rawPayloads - use `getDatabasePayloads` to get these payloads. - * They are fed as a parameter so that callers don't have to await the loading, but can - * await getting the raw payloads from storage - */ - public async loadDatabasePayloads(rawPayloads: FullyFormedTransferPayload[]): Promise { + public async loadDatabasePayloads(): Promise { + log(LoggingDomain.DatabaseLoad, 'Loading database payloads') + if (this.databaseLoaded) { throw 'Attempting to initialize already initialized local database.' } - if (rawPayloads.length === 0) { - this.databaseLoaded = true - this.opStatus.setDatabaseLoadStatus(0, 0, true) - return - } + const chunks = await this.device.getDatabaseLoadChunks( + { + batchSize: this.options.loadBatchSize, + contentTypePriority: this.localLoadPriorty, + uuidPriority: this.launchPriorityUuids, + }, + this.identifier, + ) - const unsortedPayloads = rawPayloads - .map((rawPayload) => { + const itemsKeyEntries = isFullEntryLoadChunkResponse(chunks) + ? chunks.fullEntries.itemsKeys.entries + : await this.device.getDatabaseEntries(this.identifier, chunks.keys.itemsKeys.keys) + + const itemsKeyPayloads = itemsKeyEntries + .map((entry) => { try { - return CreatePayload(rawPayload, PayloadSource.Constructor) + return CreatePayload(entry, PayloadSource.Constructor) } catch (e) { - console.error('Creating payload fail+ed', e) + console.error('Creating payload failed', e) return undefined } }) .filter(isNotUndefined) - const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority( - unsortedPayloads, - this.localLoadPriorty, - this.launchPriorityUuids, - ) - await this.processItemsKeysFirstDuringDatabaseLoad(itemsKeyPayloads) - await this.processPayloadBatch(contentTypePriorityPayloads) - /** * Map in batches to give interface a chance to update. Note that total decryption * time is constant regardless of batch size. Decrypting 3000 items all at once or in * batches will result in the same time spent. It's the emitting/painting/rendering * that requires batch size optimization. */ - const payloadCount = remainingPayloads.length - const batchSize = this.options.loadBatchSize - const numBatches = Math.ceil(payloadCount / batchSize) + const payloadCount = chunks.remainingChunksItemCount + let totalProcessedCount = 0 - for (let batchIndex = 0; batchIndex < numBatches; batchIndex++) { - const currentPosition = batchIndex * batchSize - const batch = remainingPayloads.slice(currentPosition, currentPosition + batchSize) - await this.processPayloadBatch(batch, currentPosition, payloadCount) + const remainingChunks = isFullEntryLoadChunkResponse(chunks) + ? chunks.fullEntries.remainingChunks + : chunks.keys.remainingChunks + + for (const chunk of remainingChunks) { + const dbEntries = isChunkFullEntry(chunk) + ? chunk.entries + : await this.device.getDatabaseEntries(this.identifier, chunk.keys) + const payloads = dbEntries + .map((entry) => { + try { + return CreatePayload(entry, PayloadSource.Constructor) + } catch (e) { + console.error('Creating payload failed', e) + return undefined + } + }) + .filter(isNotUndefined) + + await this.processPayloadBatch(payloads, totalProcessedCount, payloadCount) + totalProcessedCount += payloads.length } this.databaseLoaded = true @@ -316,6 +326,7 @@ export class SNSyncService currentPosition?: number, payloadCount?: number, ) { + log(LoggingDomain.DatabaseLoad, 'Processing batch at index', currentPosition, 'length', batch.length) const encrypted: EncryptedPayloadInterface[] = [] const nonencrypted: (DecryptedPayloadInterface | DeletedPayloadInterface)[] = [] @@ -386,7 +397,7 @@ export class SNSyncService } public async markAllItemsAsNeedingSyncAndPersist(): Promise { - this.log('Marking all items as needing sync') + log(LoggingDomain.Sync, 'Marking all items as needing sync') const items = this.itemManager.items const payloads = items.map((item) => { @@ -444,7 +455,7 @@ export class SNSyncService const promise = this.spawnQueue[0] removeFromIndex(this.spawnQueue, 0) - this.log('Syncing again from spawn queue') + log(LoggingDomain.Sync, 'Syncing again from spawn queue') return this.sync({ queueStrategy: SyncQueueStrategy.ForceSpawnNew, @@ -506,7 +517,7 @@ export class SNSyncService public async sync(options: Partial = {}): Promise { if (this.clientLocked) { - this.log('Sync locked by client') + log(LoggingDomain.Sync, 'Sync locked by client') return } @@ -562,7 +573,7 @@ export class SNSyncService * (before reaching opStatus.setDidBegin). * 2. syncOpInProgress: If a sync() call is in flight to the server. */ - private configureSyncLock() { + private configureSyncLock(options: SyncOptions) { const syncInProgress = this.opStatus.syncInProgress const databaseLoaded = this.databaseLoaded const canExecuteSync = !this.syncLock @@ -571,12 +582,14 @@ export class SNSyncService if (shouldExecuteSync) { this.syncLock = true } else { - this.log( + log( + LoggingDomain.Sync, !canExecuteSync ? 'Another function call has begun preparing for sync.' : syncInProgress ? 'Attempting to sync while existing sync in progress.' : 'Attempting to sync before local database has loaded.', + options, ) } @@ -656,10 +669,20 @@ export class SNSyncService private createOfflineSyncOperation( payloads: (DeletedPayloadInterface | DecryptedPayloadInterface)[], - source: SyncSource, - mode: SyncMode = SyncMode.Default, + options: SyncOptions, ) { - this.log('Syncing offline user', 'source:', source, 'mode:', mode, 'payloads:', payloads) + log( + LoggingDomain.Sync, + 'Syncing offline user', + 'source:', + SyncSource[options.source], + 'sourceDesc', + options.sourceDescription, + 'mode:', + options.mode && SyncMode[options.mode], + 'payloads:', + payloads, + ) const operation = new OfflineSyncOperation(payloads, async (type, response) => { if (this.dealloced) { @@ -727,7 +750,8 @@ export class SNSyncService this.apiService, ) - this.log( + log( + LoggingDomain.Sync, 'Syncing online user', 'source', SyncSource[source], @@ -769,14 +793,14 @@ export class SNSyncService const { uploadPayloads } = this.getOfflineSyncParameters(payloads, options.mode) return { - operation: this.createOfflineSyncOperation(uploadPayloads, options.source, options.mode), + operation: this.createOfflineSyncOperation(uploadPayloads, options), mode: options.mode || SyncMode.Default, } } } private async performSync(options: SyncOptions): Promise { - const { shouldExecuteSync, releaseLock } = this.configureSyncLock() + const { shouldExecuteSync, releaseLock } = this.configureSyncLock(options) const { items, beginDate, frozenDirtyIndex, neverSyncedDeleted } = await this.prepareForSync(options) @@ -843,7 +867,7 @@ export class SNSyncService } private async handleOfflineResponse(response: OfflineSyncResponse) { - this.log('Offline Sync Response', response) + log(LoggingDomain.Sync, 'Offline Sync Response', response) const masterCollection = this.payloadManager.getMasterCollection() @@ -861,7 +885,7 @@ export class SNSyncService } private handleErrorServerResponse(response: ServerSyncResponse) { - this.log('Sync Error', response) + log(LoggingDomain.Sync, 'Sync Error', response) if (response.status === INVALID_SESSION_RESPONSE_STATUS) { void this.notifyEvent(SyncEvent.InvalidSession) @@ -904,7 +928,8 @@ export class SNSyncService historyMap, ) - this.log( + log( + LoggingDomain.Sync, 'Online Sync Response', 'Operator ID', operation.id, @@ -1060,7 +1085,7 @@ export class SNSyncService } private async syncAgainByHandlingRequestsWaitingInResolveQueue(options: SyncOptions) { - this.log('Syncing again from resolve queue') + log(LoggingDomain.Sync, 'Syncing again from resolve queue') const promise = this.sync({ source: SyncSource.ResolveQueue, checkIntegrity: options.checkIntegrity, diff --git a/packages/snjs/lib/Services/Sync/index.ts b/packages/snjs/lib/Services/Sync/index.ts index 32acb4930..b60c1a013 100644 --- a/packages/snjs/lib/Services/Sync/index.ts +++ b/packages/snjs/lib/Services/Sync/index.ts @@ -5,5 +5,4 @@ export * from './SyncClientInterface' export * from './Account/Operation' export * from './Account/ResponseResolver' export * from './Offline/Operation' -export * from './Utils' export * from './Account/Response' diff --git a/packages/snjs/mocha/application.test.js b/packages/snjs/mocha/application.test.js index e05569e92..779a26e97 100644 --- a/packages/snjs/mocha/application.test.js +++ b/packages/snjs/mocha/application.test.js @@ -110,12 +110,12 @@ describe('application instances', () => { * app deinit. */ await Factory.sleep(MaximumWaitTime - 0.05) /** Access any deviceInterface function */ - app.diskStorageService.deviceInterface.getAllRawDatabasePayloads(app.identifier) + app.diskStorageService.deviceInterface.getAllDatabaseEntries(app.identifier) }) await app.lock() }) - describe('signOut()', () => { + describe.skip('signOut()', () => { let testNote1 let confirmAlert let deinit diff --git a/packages/snjs/mocha/auth.test.js b/packages/snjs/mocha/auth.test.js index d91e4fcbf..8fe960457 100644 --- a/packages/snjs/mocha/auth.test.js +++ b/packages/snjs/mocha/auth.test.js @@ -59,9 +59,6 @@ describe('basic auth', function () { expect(await this.application.protocolService.getRootKey()).to.not.be.ok expect(this.application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone) - - const rawPayloads = await this.application.diskStorageService.getAllRawPayloads() - expect(rawPayloads.length).to.equal(BaseItemCounts.DefaultItems) }) it('successfully signs in to registered account', async function () { diff --git a/packages/snjs/mocha/key_recovery_service.test.js b/packages/snjs/mocha/key_recovery_service.test.js index 957cd4cbb..74a426d25 100644 --- a/packages/snjs/mocha/key_recovery_service.test.js +++ b/packages/snjs/mocha/key_recovery_service.test.js @@ -664,12 +664,12 @@ describe('key recovery service', function () { await Factory.awaitFunctionInvokation(appA.keyRecoveryService, 'handleDecryptionOfAllKeysMatchingCorrectRootKey') /** Stored version of items key should use new root key */ - const stored = (await appA.deviceInterface.getAllRawDatabasePayloads(appA.identifier)).find( + const stored = (await appA.deviceInterface.getAllDatabaseEntries(appA.identifier)).find( (payload) => payload.uuid === newDefaultKey.uuid, ) const storedParams = await appA.protocolService.getKeyEmbeddedKeyParams(new EncryptedPayload(stored)) - const correctStored = (await appB.deviceInterface.getAllRawDatabasePayloads(appB.identifier)).find( + const correctStored = (await appB.deviceInterface.getAllDatabaseEntries(appB.identifier)).find( (payload) => payload.uuid === newDefaultKey.uuid, ) diff --git a/packages/snjs/mocha/lib/factory.js b/packages/snjs/mocha/lib/factory.js index 80b222f6a..d2cc61e82 100644 --- a/packages/snjs/mocha/lib/factory.js +++ b/packages/snjs/mocha/lib/factory.js @@ -303,7 +303,8 @@ export function tomorrow() { return new Date(new Date().setDate(new Date().getDate() + 1)) } -export async function sleep(seconds) { +export async function sleep(seconds, reason) { + console.log('Sleeping for reason', reason) return Utils.sleep(seconds) } diff --git a/packages/snjs/mocha/lib/web_device_interface.js b/packages/snjs/mocha/lib/web_device_interface.js index 9b7e93c2b..78daa9a25 100644 --- a/packages/snjs/mocha/lib/web_device_interface.js +++ b/packages/snjs/mocha/lib/web_device_interface.js @@ -21,17 +21,6 @@ export default class WebDeviceInterface { } } - async getAllRawStorageKeyValues() { - const results = [] - for (const key of Object.keys(localStorage)) { - results.push({ - key: key, - value: localStorage[key], - }) - } - return results - } - async setRawStorageValue(key, value) { localStorage.setItem(key, value) } @@ -60,7 +49,7 @@ export default class WebDeviceInterface { return `${this._getDatabaseKeyPrefix(identifier)}${id}` } - async getAllRawDatabasePayloads(identifier) { + async getAllDatabaseEntries(identifier) { const models = [] for (const key in localStorage) { if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) { @@ -70,21 +59,51 @@ export default class WebDeviceInterface { return models } - async saveRawDatabasePayload(payload, identifier) { + async getDatabaseLoadChunks(options, identifier) { + const entries = await this.getAllDatabaseEntries(identifier) + const sorted = GetSortedPayloadsByPriority(entries, options) + + const itemsKeysChunk = { + entries: sorted.itemsKeyPayloads, + } + + const contentTypePriorityChunk = { + entries: sorted.contentTypePriorityPayloads, + } + + const remainingPayloadsChunks = [] + for (let i = 0; i < sorted.remainingPayloads.length; i += options.batchSize) { + remainingPayloadsChunks.push({ + entries: sorted.remainingPayloads.slice(i, i + options.batchSize), + }) + } + + const result = { + fullEntries: { + itemsKeys: itemsKeysChunk, + remainingChunks: [contentTypePriorityChunk, ...remainingPayloadsChunks], + }, + remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length, + } + + return result + } + + async saveDatabaseEntry(payload, identifier) { localStorage.setItem(this._keyForPayloadId(payload.uuid, identifier), JSON.stringify(payload)) } - async saveRawDatabasePayloads(payloads, identifier) { + async saveDatabaseEntries(payloads, identifier) { for (const payload of payloads) { - await this.saveRawDatabasePayload(payload, identifier) + await this.saveDatabaseEntry(payload, identifier) } } - async removeRawDatabasePayloadWithId(id, identifier) { + async removeDatabaseEntry(id, identifier) { localStorage.removeItem(this._keyForPayloadId(id, identifier)) } - async removeAllRawDatabasePayloads(identifier) { + async removeAllDatabaseEntries(identifier) { for (const key in localStorage) { if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) { delete localStorage[key] @@ -124,12 +143,6 @@ export default class WebDeviceInterface { localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(keychain)) } - /** Allows unit tests to set legacy keychain structure as it was <= 003 */ - // eslint-disable-next-line camelcase - async setLegacyRawKeychainValue(value) { - localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(value)) - } - async getRawKeychainValue() { const keychain = localStorage.getItem(KEYCHAIN_STORAGE_KEY) return JSON.parse(keychain) @@ -139,19 +152,13 @@ export default class WebDeviceInterface { localStorage.removeItem(KEYCHAIN_STORAGE_KEY) } - performSoftReset() { + performSoftReset() {} - } - - performHardReset() { - - } + performHardReset() {} isDeviceDestroyed() { return false } - deinit() { - - } + deinit() {} } diff --git a/packages/snjs/mocha/migrations/2020-01-15-mobile.test.js b/packages/snjs/mocha/migrations/2020-01-15-mobile.test.js deleted file mode 100644 index be114b69c..000000000 --- a/packages/snjs/mocha/migrations/2020-01-15-mobile.test.js +++ /dev/null @@ -1,1042 +0,0 @@ -/* eslint-disable no-undef */ -import * as Factory from '../lib/factory.js' -import * as Utils from '../lib/Utils.js' -import FakeWebCrypto from '../lib/fake_web_crypto.js' -chai.use(chaiAsPromised) -const expect = chai.expect - -describe('2020-01-15 mobile migration', () => { - beforeEach(() => { - localStorage.clear() - }) - - afterEach(() => { - localStorage.clear() - }) - - it( - '2020-01-15 migration with passcode and account', - async function () { - let application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator003.createRootKey(identifier, passcode) - await application.deviceInterface.setRawStorageValue( - 'pc_params', - JSON.stringify(passcodeKey.keyParams.getPortableValue()), - ) - const passcodeTiming = 'immediately' - - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - const customServer = 'http://server-dev.standardnotes.org' - await application.deviceInterface.setRawStorageValue( - 'user', - JSON.stringify({ email: identifier, server: customServer }), - ) - await application.deviceInterface.setLegacyRawKeychainValue({ - offline: { - pw: passcodeKey.serverPassword, - timing: passcodeTiming, - }, - }) - /** Wrap account key with passcode key and store in storage */ - const keyPayload = new DecryptedPayload({ - uuid: Utils.generateUuid(), - content_type: 'SN|Mobile|EncryptedKeys', - content: { - accountKeys: { - jwt: 'foo', - mk: accountKey.masterKey, - ak: accountKey.dataAuthenticationKey, - pw: accountKey.serverPassword, - }, - }, - }) - const encryptedKeyParams = await operator003.generateEncryptedParametersAsync(keyPayload, passcodeKey) - const wrappedKey = new EncryptedPayload({ ...keyPayload.ejected(), ...encryptedKeyParams }) - await application.deviceInterface.setRawStorageValue('encrypted_account_keys', JSON.stringify(wrappedKey)) - const biometricPrefs = { enabled: true, timing: 'immediately' } - /** Create legacy storage. Storage in mobile was never wrapped. */ - await application.deviceInterface.setRawStorageValue('biometrics_prefs', JSON.stringify(biometricPrefs)) - await application.deviceInterface.setRawStorageValue(NonwrappedStorageKey.MobileFirstRun, false) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - /** setup options */ - const lastExportDate = '2020:02' - await application.deviceInterface.setRawStorageValue('LastExportDateKey', lastExportDate) - const options = JSON.stringify({ - sortBy: 'userModifiedAt', - sortReverse: undefined, - selectedTagIds: [], - hidePreviews: true, - hideDates: false, - hideTags: false, - }) - await application.deviceInterface.setRawStorageValue('options', options) - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if ( - prompt.validation === ChallengeValidation.None || - prompt.validation === ChallengeValidation.LocalPasscode - ) { - values.push(CreateChallengeValue(prompt, passcode)) - } - if (prompt.validation === ChallengeValidation.Biometric) { - values.push(CreateChallengeValue(prompt, true)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - const values = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, values) - } - await application.prepareForLaunch({ - receiveChallenge, - }) - await application.launch(true) - - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - - const keyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(typeof keyParams).to.equal('object') - const rootKey = await application.protocolService.getRootKey() - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - - const keychainValue = await application.deviceInterface.getNamespacedKeychainValue(application.identifier) - expect(keychainValue).to.not.be.ok - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - expect( - await application.diskStorageService.getValue(NonwrappedStorageKey.MobileFirstRun, StorageValueModes.Nonwrapped), - ).to.equal(false) - - expect( - await application.diskStorageService.getValue(StorageKey.BiometricsState, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.enabled) - expect( - await application.diskStorageService.getValue(StorageKey.MobileBiometricsTiming, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.timing) - expect(await application.getUser().email).to.equal(identifier) - - const appId = application.identifier - console.warn('Expecting exception due to deiniting application while trying to renew session') - - /** Full sync completed event will not trigger due to mocked credentials, - * thus we manually need to mark any sync dependent migrations as complete. */ - await application.migrationService.markMigrationsAsDone() - await Factory.safeDeinit(application) - - /** Recreate application and ensure storage values are consistent */ - application = Factory.createApplicationWithFakeCrypto(appId) - await application.prepareForLaunch({ - receiveChallenge, - }) - await application.launch(true) - expect(await application.getUser().email).to.equal(identifier) - expect(await application.getHost()).to.equal(customServer) - const preferences = await application.diskStorageService.getValue('preferences') - expect(preferences.sortBy).to.equal('userModifiedAt') - expect(preferences.sortReverse).to.be.false - expect(preferences.hideDate).to.be.false - expect(preferences.hideTags).to.be.false - expect(preferences.hideNotePreview).to.be.true - expect(preferences.lastExportDate).to.equal(lastExportDate) - expect(preferences.doNotShowAgainUnsupportedEditors).to.be.false - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }, - Factory.TwentySecondTimeout, - ) - - it('2020-01-15 migration with passcode only', async function () { - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator003.createRootKey(identifier, passcode) - await application.deviceInterface.setRawStorageValue( - 'pc_params', - JSON.stringify(passcodeKey.keyParams.getPortableValue()), - ) - const passcodeTiming = 'immediately' - await application.deviceInterface.setLegacyRawKeychainValue({ - offline: { - pw: passcodeKey.serverPassword, - timing: passcodeTiming, - }, - }) - - const biometricPrefs = { enabled: true, timing: 'immediately' } - /** Create legacy storage. Storage in mobile was never wrapped. */ - await application.deviceInterface.setRawStorageValue('biometrics_prefs', JSON.stringify(biometricPrefs)) - const passcodeKeyboardType = 'numeric' - await application.deviceInterface.setRawStorageValue('passcodeKeyboardType', passcodeKeyboardType) - await application.deviceInterface.setRawStorageValue(NonwrappedStorageKey.MobileFirstRun, false) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, passcodeKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - /** setup options */ - await application.deviceInterface.setRawStorageValue('DoNotShowAgainUnsupportedEditorsKey', true) - const options = JSON.stringify({ - sortBy: undefined, - sortReverse: undefined, - selectedTagIds: [], - hidePreviews: false, - hideDates: undefined, - hideTags: true, - }) - await application.deviceInterface.setRawStorageValue('options', options) - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.None || prompt.validation === ChallengeValidation.LocalPasscode) { - values.push(CreateChallengeValue(prompt, passcode)) - } - if (prompt.validation === ChallengeValidation.Biometric) { - values.push(CreateChallengeValue(prompt, true)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - await Factory.sleep(0) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly) - await application.launch(true) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - - const rootKey = await application.protocolService.getRootKey() - expect(rootKey.masterKey).to.equal(passcodeKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(passcodeKey.dataAuthenticationKey) - /** Root key is in memory with passcode only, so server password can be defined */ - expect(rootKey.serverPassword).to.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly) - - const keychainValue = await application.deviceInterface.getNamespacedKeychainValue(application.identifier) - expect(keychainValue).to.not.be.ok - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - expect( - await application.diskStorageService.getValue(NonwrappedStorageKey.MobileFirstRun, StorageValueModes.Nonwrapped), - ).to.equal(false) - expect( - await application.diskStorageService.getValue(StorageKey.BiometricsState, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.enabled) - expect( - await application.diskStorageService.getValue(StorageKey.MobileBiometricsTiming, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.timing) - expect( - await application.diskStorageService.getValue(StorageKey.MobilePasscodeTiming, StorageValueModes.Nonwrapped), - ).to.eql(passcodeTiming) - expect( - await application.diskStorageService.getValue(StorageKey.MobilePasscodeKeyboardType, StorageValueModes.Nonwrapped), - ).to.eql(passcodeKeyboardType) - const preferences = await application.diskStorageService.getValue('preferences') - expect(preferences.sortBy).to.equal(undefined) - expect(preferences.sortReverse).to.be.false - expect(preferences.hideNotePreview).to.be.false - expect(preferences.hideDate).to.be.false - expect(preferences.hideTags).to.be.true - expect(preferences.lastExportDate).to.equal(undefined) - expect(preferences.doNotShowAgainUnsupportedEditors).to.be.true - await Factory.safeDeinit(application) - }) - - it('2020-01-15 migration with passcode-only missing keychain', async function () { - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator003.createRootKey(identifier, passcode) - await application.deviceInterface.setRawStorageValue( - 'pc_params', - JSON.stringify(passcodeKey.keyParams.getPortableValue()), - ) - const biometricPrefs = { enabled: true, timing: 'immediately' } - /** Create legacy storage. Storage in mobile was never wrapped. */ - await application.deviceInterface.setRawStorageValue('biometrics_prefs', JSON.stringify(biometricPrefs)) - const passcodeKeyboardType = 'numeric' - await application.deviceInterface.setRawStorageValue('passcodeKeyboardType', passcodeKeyboardType) - await application.deviceInterface.setRawStorageValue(NonwrappedStorageKey.MobileFirstRun, false) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, passcodeKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - /** setup options */ - await application.deviceInterface.setRawStorageValue('DoNotShowAgainUnsupportedEditorsKey', true) - const options = JSON.stringify({ - sortBy: undefined, - sortReverse: undefined, - selectedTagIds: [], - hidePreviews: false, - hideDates: undefined, - hideTags: true, - }) - await application.deviceInterface.setRawStorageValue('options', options) - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.None || prompt.validation === ChallengeValidation.LocalPasscode) { - values.push(CreateChallengeValue(prompt, passcode)) - } - if (prompt.validation === ChallengeValidation.Biometric) { - values.push(CreateChallengeValue(prompt, true)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - await Factory.sleep(0) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly) - await application.launch(true) - - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.errorDecrypting).to.not.be.ok - - /** application should not crash */ - await Factory.safeDeinit(application) - }) - - it('2020-01-15 migration with account only', async function () { - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - await application.deviceInterface.setRawStorageValue('user', JSON.stringify({ email: identifier })) - expect(accountKey.keyVersion).to.equal(ProtocolVersion.V003) - await application.deviceInterface.setLegacyRawKeychainValue({ - mk: accountKey.masterKey, - pw: accountKey.serverPassword, - ak: accountKey.dataAuthenticationKey, - jwt: 'foo', - version: ProtocolVersion.V003, - }) - const biometricPrefs = { - enabled: true, - timing: 'immediately', - } - /** Create legacy storage. Storage in mobile was never wrapped. */ - await application.deviceInterface.setRawStorageValue('biometrics_prefs', JSON.stringify(biometricPrefs)) - await application.deviceInterface.setRawStorageValue(NonwrappedStorageKey.MobileFirstRun, false) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - /** setup options */ - const lastExportDate = '2020:02' - await application.deviceInterface.setRawStorageValue('LastExportDateKey', lastExportDate) - await application.deviceInterface.setRawStorageValue('DoNotShowAgainUnsupportedEditorsKey', false) - const options = JSON.stringify({ - sortBy: 'created_at', - sortReverse: undefined, - selectedTagIds: [], - hidePreviews: true, - hideDates: false, - }) - await application.deviceInterface.setRawStorageValue('options', options) - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.None) { - values.push(CreateChallengeValue(prompt, password)) - } - if (prompt.validation === ChallengeValidation.Biometric) { - values.push(CreateChallengeValue(prompt, true)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - /** Runs migration */ - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - const rootKey = await application.protocolService.getRootKey() - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - - const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped) - expect(typeof keyParams).to.equal('object') - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - expect( - await application.diskStorageService.getValue(NonwrappedStorageKey.MobileFirstRun, StorageValueModes.Nonwrapped), - ).to.equal(false) - expect( - await application.diskStorageService.getValue(StorageKey.BiometricsState, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.enabled) - expect( - await application.diskStorageService.getValue(StorageKey.MobileBiometricsTiming, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.timing) - expect(await application.getUser().email).to.equal(identifier) - const preferences = await application.diskStorageService.getValue('preferences') - expect(preferences.sortBy).to.equal('created_at') - expect(preferences.sortReverse).to.be.false - expect(preferences.hideDate).to.be.false - expect(preferences.hideNotePreview).to.be.true - expect(preferences.lastExportDate).to.equal(lastExportDate) - expect(preferences.doNotShowAgainUnsupportedEditors).to.be.false - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }).timeout(10000) - - it('2020-01-15 launching with account but missing keychain', async function () { - /** - * We expect that the keychain will attempt to be recovered - * We expect two challenges, one to recover just the keychain - * and another to recover the user session via a sign in request - */ - - /** Register a real user so we can attempt to sign back into this account later */ - const tempApp = await Factory.createInitAppWithFakeCrypto(Environment.Mobile, Platform.Ios) - const email = UuidGenerator.GenerateUuid() - const password = UuidGenerator.GenerateUuid() - /** Register with 003 account */ - await Factory.registerOldUser({ - application: tempApp, - email: email, - password: password, - version: ProtocolVersion.V003, - }) - const accountKey = tempApp.protocolService.getRootKey() - await Factory.safeDeinit(tempApp) - localStorage.clear() - - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - /** Create old version account parameters */ - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - await application.deviceInterface.setRawStorageValue('user', JSON.stringify({ email: email })) - expect(accountKey.keyVersion).to.equal(ProtocolVersion.V003) - - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.placeholder === SessionStrings.EmailInputPlaceholder) { - values.push(CreateChallengeValue(prompt, email)) - } else if (prompt.placeholder === SessionStrings.PasswordInputPlaceholder) { - values.push(CreateChallengeValue(prompt, password)) - } else { - throw Error('Unhandled prompt') - } - } - return values - } - let totalChallenges = 0 - const expectedChallenges = 2 - const receiveChallenge = async (challenge) => { - totalChallenges++ - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - - /** Recovery migration is non-blocking, so let's block for it */ - await Factory.sleep(1.0) - - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - const rootKey = await application.protocolService.getRootKey() - expect(rootKey).to.be.ok - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - expect(await application.getUser().email).to.equal(email) - expect(await application.apiService.getSession()).to.be.ok - expect(totalChallenges).to.equal(expectedChallenges) - await Factory.safeDeinit(application) - }).timeout(10000) - - it('2020-01-15 migration with 002 account should not create 003 data', async function () { - /** There was an issue where 002 account loading new app would create new default items key - * with 003 version. Should be 002. */ - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator002 = new SNProtocolOperator002(new FakeWebCrypto()) - const identifier = 'foo' - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator002.createRootKey(identifier, password) - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - await application.deviceInterface.setRawStorageValue('user', JSON.stringify({ email: identifier })) - expect(accountKey.keyVersion).to.equal(ProtocolVersion.V002) - await application.deviceInterface.setLegacyRawKeychainValue({ - mk: accountKey.masterKey, - pw: accountKey.serverPassword, - ak: accountKey.dataAuthenticationKey, - jwt: 'foo', - }) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator002.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.None) { - values.push(CreateChallengeValue(prompt, password)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - - const itemsKey = application.itemManager.getDisplayableItemsKeys()[0] - expect(itemsKey.keyVersion).to.equal(ProtocolVersion.V002) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - expect(await application.getUser().email).to.equal(identifier) - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }).timeout(10000) - - it('2020-01-15 migration with 001 account detect 001 version even with missing info', async function () { - /** If 001 account, and for some reason we dont have version stored, the migrations - * should determine correct version based on saved payloads */ - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator001 = new SNProtocolOperator001(new FakeWebCrypto()) - const identifier = 'foo' - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator001.createRootKey(identifier, password) - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify({ - ...accountKey.keyParams.getPortableValue(), - version: undefined, - }), - ) - await application.deviceInterface.setRawStorageValue('user', JSON.stringify({ email: identifier })) - expect(accountKey.keyVersion).to.equal(ProtocolVersion.V001) - await application.deviceInterface.setLegacyRawKeychainValue({ - mk: accountKey.masterKey, - pw: accountKey.serverPassword, - jwt: 'foo', - }) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator001.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.None) { - values.push(CreateChallengeValue(prompt, password)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - - const itemsKey = application.itemManager.getDisplayableItemsKeys()[0] - expect(itemsKey.keyVersion).to.equal(ProtocolVersion.V001) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - expect(await application.getUser().email).to.equal(identifier) - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }).timeout(10000) - - it('2020-01-15 successfully creates session if jwt is stored in keychain', async function () { - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - await application.deviceInterface.setRawStorageValue('user', JSON.stringify({ email: identifier })) - - await application.deviceInterface.setLegacyRawKeychainValue({ - mk: accountKey.masterKey, - pw: accountKey.serverPassword, - ak: accountKey.dataAuthenticationKey, - jwt: 'foo', - version: ProtocolVersion.V003, - }) - - await application.prepareForLaunch({ receiveChallenge: () => {} }) - await application.launch(true) - - expect(application.apiService.getSession()).to.be.ok - - await Factory.safeDeinit(application) - }).timeout(10000) - - it('2020-01-15 successfully creates session if jwt is stored in storage', async function () { - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - await application.deviceInterface.setRawStorageValue('user', JSON.stringify({ email: identifier, jwt: 'foo' })) - await application.deviceInterface.setLegacyRawKeychainValue({ - mk: accountKey.masterKey, - pw: accountKey.serverPassword, - ak: accountKey.dataAuthenticationKey, - version: ProtocolVersion.V003, - }) - - await application.prepareForLaunch({ receiveChallenge: () => {} }) - await application.launch(true) - - expect(application.apiService.getSession()).to.be.ok - - await Factory.safeDeinit(application) - }).timeout(10000) - - it('2020-01-15 migration with no account and no passcode', async function () { - const application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const biometricPrefs = { - enabled: true, - timing: 'immediately', - } - /** Create legacy storage. Storage in mobile was never wrapped. */ - await application.deviceInterface.setRawStorageValue('biometrics_prefs', JSON.stringify(biometricPrefs)) - await application.deviceInterface.setRawStorageValue(NonwrappedStorageKey.MobileFirstRun, false) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - await application.deviceInterface.saveRawDatabasePayload(notePayload.ejected(), application.identifier) - /** setup options */ - await application.deviceInterface.setRawStorageValue('DoNotShowAgainUnsupportedEditorsKey', true) - const options = JSON.stringify({ - sortBy: 'created_at', - sortReverse: undefined, - selectedTagIds: [], - hidePreviews: true, - hideDates: false, - }) - await application.deviceInterface.setRawStorageValue('options', options) - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.None || prompt.validation === ChallengeValidation.LocalPasscode) { - values.push(CreateChallengeValue(prompt, passcode)) - } - if (prompt.validation === ChallengeValidation.Biometric) { - values.push(CreateChallengeValue(prompt, true)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - - const rootKey = await application.protocolService.getRootKey() - expect(rootKey).to.not.be.ok - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - expect( - await application.diskStorageService.getValue(NonwrappedStorageKey.MobileFirstRun, StorageValueModes.Nonwrapped), - ).to.equal(false) - expect( - await application.diskStorageService.getValue(StorageKey.BiometricsState, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.enabled) - expect( - await application.diskStorageService.getValue(StorageKey.MobileBiometricsTiming, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.timing) - const preferences = await application.diskStorageService.getValue('preferences') - expect(preferences.sortBy).to.equal('created_at') - expect(preferences.sortReverse).to.be.false - expect(preferences.hideDate).to.be.false - expect(preferences.hideNotePreview).to.be.true - expect(preferences.lastExportDate).to.equal(undefined) - expect(preferences.doNotShowAgainUnsupportedEditors).to.be.true - await Factory.safeDeinit(application) - }) - - it( - '2020-01-15 migration from mobile version 3.0.16', - async function () { - /** - * In version 3.0.16, encrypted account keys were stored in keychain, not storage. - * This was migrated in version 3.0.17, but we want to be sure we can go from 3.0.16 - * to current state directly. - */ - let application = await Factory.createAppWithRandNamespace(Environment.Mobile, Platform.Ios) - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator003.createRootKey(identifier, passcode) - await application.deviceInterface.setRawStorageValue( - 'pc_params', - JSON.stringify(passcodeKey.keyParams.getPortableValue()), - ) - const passcodeTiming = 'immediately' - - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - await application.deviceInterface.setRawStorageValue( - 'auth_params', - JSON.stringify(accountKey.keyParams.getPortableValue()), - ) - const customServer = 'http://server-dev.standardnotes.org' - await application.deviceInterface.setRawStorageValue( - 'user', - JSON.stringify({ email: identifier, server: customServer }), - ) - /** Wrap account key with passcode key and store in storage */ - const keyPayload = new DecryptedPayload({ - uuid: Utils.generateUuid(), - content_type: 'SN|Mobile|EncryptedKeys', - content: { - accountKeys: { - jwt: 'foo', - mk: accountKey.masterKey, - ak: accountKey.dataAuthenticationKey, - pw: accountKey.serverPassword, - }, - }, - }) - const encryptedKeyParams = await operator003.generateEncryptedParametersAsync(keyPayload, passcodeKey) - const wrappedKey = new EncryptedPayload({ ...keyPayload, ...encryptedKeyParams }) - await application.deviceInterface.setLegacyRawKeychainValue({ - encryptedAccountKeys: wrappedKey, - offline: { - pw: passcodeKey.serverPassword, - timing: passcodeTiming, - }, - }) - const biometricPrefs = { enabled: true, timing: 'immediately' } - /** Create legacy storage. Storage in mobile was never wrapped. */ - await application.deviceInterface.setRawStorageValue('biometrics_prefs', JSON.stringify(biometricPrefs)) - await application.deviceInterface.setRawStorageValue(NonwrappedStorageKey.MobileFirstRun, false) - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - /** setup options */ - const lastExportDate = '2020:02' - await application.deviceInterface.setRawStorageValue('LastExportDateKey', lastExportDate) - const options = JSON.stringify({ - sortBy: 'userModifiedAt', - sortReverse: undefined, - selectedTagIds: [], - hidePreviews: true, - hideDates: false, - hideTags: false, - }) - await application.deviceInterface.setRawStorageValue('options', options) - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if ( - prompt.validation === ChallengeValidation.None || - prompt.validation === ChallengeValidation.LocalPasscode - ) { - values.push(CreateChallengeValue(prompt, passcode)) - } - if (prompt.validation === ChallengeValidation.Biometric) { - values.push(CreateChallengeValue(prompt, true)) - } - } - return values - } - const receiveChallenge = async (challenge) => { - const values = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, values) - } - await application.prepareForLaunch({ - receiveChallenge, - }) - await application.launch(true) - - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - - const keyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(typeof keyParams).to.equal('object') - const rootKey = await application.protocolService.getRootKey() - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - - const keychainValue = await application.deviceInterface.getNamespacedKeychainValue(application.identifier) - expect(keychainValue).to.not.be.ok - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - expect( - await application.diskStorageService.getValue(NonwrappedStorageKey.MobileFirstRun, StorageValueModes.Nonwrapped), - ).to.equal(false) - - expect( - await application.diskStorageService.getValue(StorageKey.BiometricsState, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.enabled) - expect( - await application.diskStorageService.getValue(StorageKey.MobileBiometricsTiming, StorageValueModes.Nonwrapped), - ).to.equal(biometricPrefs.timing) - expect(await application.getUser().email).to.equal(identifier) - - const appId = application.identifier - console.warn('Expecting exception due to deiniting application while trying to renew session') - /** Full sync completed event will not trigger due to mocked credentials, - * thus we manually need to mark any sync dependent migrations as complete. */ - await application.migrationService.markMigrationsAsDone() - await Factory.safeDeinit(application) - - /** Recreate application and ensure storage values are consistent */ - application = Factory.createApplicationWithFakeCrypto(appId) - await application.prepareForLaunch({ - receiveChallenge, - }) - await application.launch(true) - expect(await application.getUser().email).to.equal(identifier) - expect(await application.getHost()).to.equal(customServer) - const preferences = await application.diskStorageService.getValue('preferences') - expect(preferences.sortBy).to.equal('userModifiedAt') - expect(preferences.sortReverse).to.be.false - expect(preferences.hideDate).to.be.false - expect(preferences.hideTags).to.be.false - expect(preferences.hideNotePreview).to.be.true - expect(preferences.lastExportDate).to.equal(lastExportDate) - expect(preferences.doNotShowAgainUnsupportedEditors).to.be.false - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }, - Factory.TwentySecondTimeout, - ) -}) diff --git a/packages/snjs/mocha/migrations/2020-01-15-web.test.js b/packages/snjs/mocha/migrations/2020-01-15-web.test.js deleted file mode 100644 index 5e3c131ac..000000000 --- a/packages/snjs/mocha/migrations/2020-01-15-web.test.js +++ /dev/null @@ -1,584 +0,0 @@ -/* eslint-disable no-unused-expressions */ -/* eslint-disable no-undef */ -import * as Factory from '../lib/factory.js' -import FakeWebCrypto from '../lib/fake_web_crypto.js' -chai.use(chaiAsPromised) -const expect = chai.expect - -describe('2020-01-15 web migration', () => { - beforeEach(() => { - localStorage.clear() - }) - - afterEach(() => { - localStorage.clear() - }) - - /** - * This test will pass but sync afterwards will not be successful - * as we are using a random value for the legacy session token - */ - it('2020-01-15 migration with passcode and account', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator003.createRootKey(identifier, passcode) - await application.deviceInterface.setRawStorageValue( - 'offlineParams', - JSON.stringify(passcodeKey.keyParams.getPortableValue()), - ) - - /** Create arbitrary storage values and make sure they're migrated */ - const arbitraryValues = { - foo: 'bar', - zar: 'tar', - har: 'car', - } - for (const key of Object.keys(arbitraryValues)) { - await application.deviceInterface.setRawStorageValue(key, arbitraryValues[key]) - } - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - - /** Create legacy storage and encrypt it with passcode */ - const embeddedStorage = { - mk: accountKey.masterKey, - ak: accountKey.dataAuthenticationKey, - pw: accountKey.serverPassword, - jwt: 'anything', - /** Legacy versions would store json strings inside of embedded storage */ - auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()), - } - const storagePayload = new DecryptedPayload({ - uuid: await operator003.crypto.generateUUID(), - content_type: ContentType.EncryptedStorage, - content: { - storage: embeddedStorage, - }, - }) - const encryptionParams = await operator003.generateEncryptedParametersAsync(storagePayload, passcodeKey) - const persistPayload = new EncryptedPayload({ ...storagePayload, ...encryptionParams }) - await application.deviceInterface.setRawStorageValue('encryptedStorage', JSON.stringify(persistPayload)) - - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Run migration */ - await application.prepareForLaunch({ - receiveChallenge: async (challenge) => { - application.submitValuesForChallenge(challenge, [CreateChallengeValue(challenge.prompts[0], passcode)]) - }, - }) - - await application.launch(true) - expect(application.sessionManager.online()).to.equal(true) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - - expect(await application.deviceInterface.getRawStorageValue('offlineParams')).to.not.be.ok - - const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped) - expect(typeof keyParams).to.equal('object') - - /** Embedded value should match */ - const migratedKeyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(migratedKeyParams).to.eql(JSON.parse(embeddedStorage.auth_params)) - const rootKey = await application.protocolService.getRootKey() - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - /** Application should not retain server password from legacy versions */ - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - /** Ensure arbitrary values have been migrated */ - for (const key of Object.keys(arbitraryValues)) { - const value = await application.diskStorageService.getValue(key) - expect(arbitraryValues[key]).to.equal(value) - } - - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }).timeout(15000) - - it('2020-01-15 migration with passcode only', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator003.createRootKey(identifier, passcode) - await application.deviceInterface.setRawStorageValue( - 'offlineParams', - JSON.stringify(passcodeKey.keyParams.getPortableValue()), - ) - - /** Create arbitrary storage values and make sure they're migrated */ - const arbitraryValues = { - foo: 'bar', - zar: 'tar', - har: 'car', - } - for (const key of Object.keys(arbitraryValues)) { - await application.deviceInterface.setRawStorageValue(key, arbitraryValues[key]) - } - - const embeddedStorage = { - ...arbitraryValues, - } - const storagePayload = new DecryptedPayload({ - uuid: await operator003.crypto.generateUUID(), - content: { - storage: embeddedStorage, - }, - content_type: ContentType.EncryptedStorage, - }) - const encryptionParams = await operator003.generateEncryptedParametersAsync(storagePayload, passcodeKey) - const persistPayload = new EncryptedPayload({ ...storagePayload, ...encryptionParams }) - await application.deviceInterface.setRawStorageValue('encryptedStorage', JSON.stringify(persistPayload)) - - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, passcodeKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - await application.prepareForLaunch({ - receiveChallenge: async (challenge) => { - application.submitValuesForChallenge(challenge, [CreateChallengeValue(challenge.prompts[0], passcode)]) - }, - }) - await application.launch(true) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - - expect(await application.deviceInterface.getRawStorageValue('offlineParams')).to.not.be.ok - - /** Embedded value should match */ - const migratedKeyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(migratedKeyParams).to.eql(embeddedStorage.auth_params) - const rootKey = await application.protocolService.getRootKey() - expect(rootKey.masterKey).to.equal(passcodeKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(passcodeKey.dataAuthenticationKey) - /** Root key is in memory with passcode only, so server password can be defined */ - expect(rootKey.serverPassword).to.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - /** Ensure arbitrary values have been migrated */ - for (const key of Object.keys(arbitraryValues)) { - const value = await application.diskStorageService.getValue(key) - expect(arbitraryValues[key]).to.equal(value) - } - await Factory.safeDeinit(application) - }) - - /** - * This test will pass but sync afterwards will not be successful - * as we are using a random value for the legacy session token - */ - it('2020-01-15 migration with account only', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator003 = new SNProtocolOperator003(new FakeWebCrypto()) - const identifier = 'foo' - - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator003.createRootKey(identifier, password) - - /** Create arbitrary storage values and make sure they're migrated */ - const storage = { - foo: 'bar', - zar: 'tar', - har: 'car', - mk: accountKey.masterKey, - ak: accountKey.dataAuthenticationKey, - pw: accountKey.serverPassword, - jwt: 'anything', - /** Legacy versions would store json strings inside of embedded storage */ - auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()), - } - for (const key of Object.keys(storage)) { - await application.deviceInterface.setRawStorageValue(key, storage[key]) - } - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - if (prompt.validation === ChallengeValidation.LocalPasscode) { - values.push(CreateChallengeValue(prompt, passcode)) - } else { - /** We will be prompted to reauthetnicate our session, not relevant to this test - * but pass any value to avoid exception - */ - values.push(CreateChallengeValue(prompt, 'foo')) - } - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - expect(application.sessionManager.online()).to.equal(true) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - /** Embedded value should match */ - const migratedKeyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(migratedKeyParams).to.eql(accountKey.keyParams.getPortableValue()) - const rootKey = await application.protocolService.getRootKey() - expect(rootKey).to.be.ok - - expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('auth_params')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('jwt')).to.not.be.ok - - const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped) - expect(typeof keyParams).to.equal('object') - - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - /** Ensure arbitrary values have been migrated */ - for (const key of Object.keys(storage)) { - /** Is stringified in storage, but parsed in storageService */ - if (key === 'auth_params') { - continue - } - const value = await application.diskStorageService.getValue(key) - expect(storage[key]).to.equal(value) - } - - console.warn('Expecting exception due to deiniting application while trying to renew session') - await Factory.safeDeinit(application) - }) - - it('2020-01-15 migration with no account and no passcode', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - /** Create arbitrary storage values and make sure they're migrated */ - const storage = { - foo: 'bar', - zar: 'tar', - har: 'car', - } - for (const key of Object.keys(storage)) { - await application.deviceInterface.setRawStorageValue(key, storage[key]) - } - - /** Create item and store it in db */ - const notePayload = Factory.createNotePayload() - await application.deviceInterface.saveRawDatabasePayload(notePayload.ejected(), application.identifier) - - /** Run migration */ - await application.prepareForLaunch({ - receiveChallenge: (_challenge) => { - return null - }, - }) - await application.launch(true) - - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone) - - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - const rootKey = await application.protocolService.getRootKey() - expect(rootKey).to.not.be.ok - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone) - - expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - /** Ensure arbitrary values have been migrated */ - for (const key of Object.keys(storage)) { - const value = await application.diskStorageService.getValue(key) - expect(storage[key]).to.equal(value) - } - - await Factory.safeDeinit(application) - }) - - /** - * This test will pass but sync afterwards will not be successful - * as we are using a random value for the legacy session token - */ - it('2020-01-15 migration from app v1.0.1 with account only', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator001 = new SNProtocolOperator001(new FakeWebCrypto()) - const identifier = 'foo' - - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator001.createRootKey(identifier, password) - - /** Create arbitrary storage values and make sure they're migrated */ - const storage = { - mk: accountKey.masterKey, - pw: accountKey.serverPassword, - jwt: 'anything', - /** Legacy versions would store json strings inside of embedded storage */ - auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()), - user: JSON.stringify({ uuid: 'anything', email: 'anything' }), - } - for (const key of Object.keys(storage)) { - await application.deviceInterface.setRawStorageValue(key, storage[key]) - } - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator001.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Run migration */ - const promptValueReply = (prompts) => { - const values = [] - for (const prompt of prompts) { - /** We will be prompted to reauthetnicate our session, not relevant to this test - * but pass any value to avoid exception - */ - values.push(CreateChallengeValue(prompt, 'foo')) - } - return values - } - const receiveChallenge = async (challenge) => { - application.addChallengeObserver(challenge, { - onInvalidValue: (value) => { - const values = promptValueReply([value.prompt]) - application.submitValuesForChallenge(challenge, values) - }, - }) - const initialValues = promptValueReply(challenge.prompts) - application.submitValuesForChallenge(challenge, initialValues) - } - await application.prepareForLaunch({ - receiveChallenge: receiveChallenge, - }) - await application.launch(true) - expect(application.sessionManager.online()).to.equal(true) - expect(application.sessionManager.getUser()).to.be.ok - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - /** Embedded value should match */ - const migratedKeyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(migratedKeyParams).to.eql(accountKey.keyParams.getPortableValue()) - const rootKey = await application.protocolService.getRootKey() - expect(rootKey).to.be.ok - - expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('auth_params')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('jwt')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('ak')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('mk')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('pw')).to.not.be.ok - - const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped) - expect(typeof keyParams).to.equal('object') - - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V001) - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - /** Ensure arbitrary values have been migrated */ - for (const key of Object.keys(storage)) { - /** Is stringified in storage, but parsed in storageService */ - const value = await application.diskStorageService.getValue(key) - if (key === 'auth_params') { - continue - } else if (key === 'user') { - expect(storage[key]).to.equal(JSON.stringify(value)) - } else { - expect(storage[key]).to.equal(value) - } - } - await Factory.safeDeinit(application) - }) - - it('2020-01-15 migration from 002 app with account and passcode but missing offlineParams.version', async function () { - /** - * There was an issue where if the user had offlineParams but it was missing the version key, - * the user could not get past the passcode migration screen. - */ - const application = await Factory.createAppWithRandNamespace() - /** Create legacy migrations value so that base migration detects old app */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - const operator002 = new SNProtocolOperator002(new FakeWebCrypto()) - const identifier = 'foo' - const passcode = 'bar' - /** Create old version passcode parameters */ - const passcodeKey = await operator002.createRootKey(identifier, passcode) - - /** The primary chaos agent */ - const offlineParams = passcodeKey.keyParams.getPortableValue() - omitInPlace(offlineParams, ['version']) - - await application.deviceInterface.setRawStorageValue('offlineParams', JSON.stringify(offlineParams)) - - /** Create old version account parameters */ - const password = 'tar' - const accountKey = await operator002.createRootKey(identifier, password) - - /** Create legacy storage and encrypt it with passcode */ - const embeddedStorage = { - mk: accountKey.masterKey, - ak: accountKey.dataAuthenticationKey, - pw: accountKey.serverPassword, - jwt: 'anything', - /** Legacy versions would store json strings inside of embedded storage */ - auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()), - user: JSON.stringify({ uuid: 'anything', email: 'anything' }), - } - const storagePayload = new DecryptedPayload({ - uuid: await operator002.crypto.generateUUID(), - content_type: ContentType.EncryptedStorage, - content: { - storage: embeddedStorage, - }, - }) - const encryptionParams = await operator002.generateEncryptedParametersAsync(storagePayload, passcodeKey) - const persistPayload = new EncryptedPayload({ ...storagePayload, ...encryptionParams }) - await application.deviceInterface.setRawStorageValue('encryptedStorage', JSON.stringify(persistPayload)) - - /** Create encrypted item and store it in db */ - const notePayload = Factory.createNotePayload() - const noteEncryptionParams = await operator002.generateEncryptedParametersAsync(notePayload, accountKey) - const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams }) - await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier) - - /** Runs migration */ - await application.prepareForLaunch({ - receiveChallenge: async (challenge) => { - application.submitValuesForChallenge(challenge, [CreateChallengeValue(challenge.prompts[0], passcode)]) - }, - }) - await application.launch(true) - expect(application.sessionManager.online()).to.equal(true) - expect(application.sessionManager.getUser()).to.be.ok - expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper) - /** Should be decrypted */ - const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default) - const valueStore = application.diskStorageService.values[storageMode] - expect(valueStore.content_type).to.not.be.ok - /** Embedded value should match */ - const migratedKeyParams = await application.diskStorageService.getValue( - StorageKey.RootKeyParams, - StorageValueModes.Nonwrapped, - ) - expect(migratedKeyParams).to.eql(accountKey.keyParams.getPortableValue()) - const rootKey = await application.protocolService.getRootKey() - expect(rootKey).to.be.ok - - expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('auth_params')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('jwt')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('ak')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('mk')).to.not.be.ok - expect(await application.deviceInterface.getRawStorageValue('pw')).to.not.be.ok - - const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped) - expect(typeof keyParams).to.equal('object') - - expect(rootKey.masterKey).to.equal(accountKey.masterKey) - expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey) - expect(rootKey.serverPassword).to.not.be.ok - expect(rootKey.keyVersion).to.equal(ProtocolVersion.V002) - - /** Expect note is decrypted */ - expect(application.itemManager.getDisplayableNotes().length).to.equal(1) - const retrievedNote = application.itemManager.getDisplayableNotes()[0] - expect(retrievedNote.uuid).to.equal(notePayload.uuid) - expect(retrievedNote.content.text).to.equal(notePayload.content.text) - - await Factory.safeDeinit(application) - }) -}) diff --git a/packages/snjs/mocha/migrations/migration.test.js b/packages/snjs/mocha/migrations/migration.test.js index ae3066986..544fe566f 100644 --- a/packages/snjs/mocha/migrations/migration.test.js +++ b/packages/snjs/mocha/migrations/migration.test.js @@ -3,7 +3,7 @@ chai.use(chaiAsPromised) const expect = chai.expect describe('migrations', () => { - const allMigrations = ['2.0.0', '2.0.15', '2.7.0', '2.20.0', '2.36.0', '2.42.0'] + const allMigrations = ['2.0.15', '2.7.0', '2.20.0', '2.36.0', '2.42.0'] beforeEach(async () => { localStorage.clear() @@ -25,34 +25,13 @@ describe('migrations', () => { }) it('should return correct required migrations if stored version is 2.0.0', async function () { - expect((await SNMigrationService.getRequiredMigrations('2.0.0')).length).to.equal(allMigrations.length - 1) + expect((await SNMigrationService.getRequiredMigrations('2.0.0')).length).to.equal(allMigrations.length) }) it('should return 0 required migrations if stored version is futuristic', async function () { expect((await SNMigrationService.getRequiredMigrations('100.0.1')).length).to.equal(0) }) - it('after running base migration, legacy structure should set version as 1.0.0', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Set up 1.0.0 structure with tell-tale storage key */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - await application.migrationService.runBaseMigrationPreRun() - expect(await application.migrationService.getStoredSnjsVersion()).to.equal('1.0.0') - await Factory.safeDeinit(application) - }) - - it('after running base migration, 2.0.0 structure set version as 2.0.0', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Set up 2.0.0 structure with tell-tale storage key */ - await application.deviceInterface.setRawStorageValue( - namespacedKey(application.identifier, 'last_migration_timestamp'), - 'anything', - ) - await application.migrationService.runBaseMigrationPreRun() - expect(await application.migrationService.getStoredSnjsVersion()).to.equal('2.0.0') - await Factory.safeDeinit(application) - }) - it('after running base migration with no present storage values, should set version to current', async function () { const application = await Factory.createAppWithRandNamespace() await application.migrationService.runBaseMigrationPreRun() @@ -60,18 +39,6 @@ describe('migrations', () => { await Factory.safeDeinit(application) }) - it('after running all migrations from a 1.0.0 installation, should set stored version to current', async function () { - const application = await Factory.createAppWithRandNamespace() - /** Set up 1.0.0 structure with tell-tale storage key */ - await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything'])) - await application.prepareForLaunch({ - receiveChallenge: () => {}, - }) - await application.launch(true) - expect(await application.migrationService.getStoredSnjsVersion()).to.equal(SnjsVersion) - await Factory.safeDeinit(application) - }) - it('after running all migrations from a 2.0.0 installation, should set stored version to current', async function () { const application = await Factory.createAppWithRandNamespace() /** Set up 2.0.0 structure with tell-tale storage key */ @@ -84,24 +51,6 @@ describe('migrations', () => { await Factory.safeDeinit(application) }) - it('should be correct migration count coming from 1.0.0', async function () { - const application = await Factory.createAppWithRandNamespace() - await application.deviceInterface.setRawStorageValue('migrations', 'anything') - await application.migrationService.runBaseMigrationPreRun() - expect(await application.migrationService.getStoredSnjsVersion()).to.equal('1.0.0') - const pendingMigrations = await SNMigrationService.getRequiredMigrations( - await application.migrationService.getStoredSnjsVersion(), - ) - expect(pendingMigrations.length).to.equal(allMigrations.length) - expect(pendingMigrations[0].version()).to.equal('2.0.0') - await application.prepareForLaunch({ - receiveChallenge: () => {}, - }) - await application.launch(true) - expect(await application.migrationService.getStoredSnjsVersion()).to.equal(SnjsVersion) - await Factory.safeDeinit(application) - }) - it('2.20.0 remove mfa migration', async function () { const application = await Factory.createAppWithRandNamespace() diff --git a/packages/snjs/mocha/model_tests/importing.test.js b/packages/snjs/mocha/model_tests/importing.test.js index 94e5d163c..2b9e41f74 100644 --- a/packages/snjs/mocha/model_tests/importing.test.js +++ b/packages/snjs/mocha/model_tests/importing.test.js @@ -735,7 +735,7 @@ describe('importing', function () { }), ) await application.deviceInterface.setRawStorageValue('standardnotes-snjs_version', '2.0.11') - await application.deviceInterface.saveRawDatabasePayload( + await application.deviceInterface.saveDatabaseEntry( { content: '003:9f2c7527eb8b2a1f8bfb3ea6b885403b6886bce2640843ebd57a6c479cbf7597:58e3322b-269a-4be3-a658-b035dffcd70f:9140b23a0fa989e224e292049f133154:SESTNOgIGf2+ZqmJdFnGU4EMgQkhKOzpZNoSzx76SJaImsayzctAgbUmJ+UU2gSQAHADS3+Z5w11bXvZgIrStTsWriwvYkNyyKmUPadKHNSBwOk4WeBZpWsA9gtI5zgI04Q5pvb8hS+kNW2j1DjM4YWqd0JQxMOeOrMIrxr/6Awn5TzYE+9wCbXZdYHyvRQcp9ui/G02ZJ67IA86vNEdjTTBAAWipWqTqKH9VDZbSQ2W/IOKfIquB373SFDKZb1S1NmBFvcoG2G7w//fAl/+ehYiL6UdiNH5MhXCDAOTQRFNfOh57HFDWVnz1VIp8X+VAPy6d9zzQH+8aws1JxHq/7BOhXrFE8UCueV6kERt9njgQxKJzd9AH32ShSiUB9X/sPi0fUXbS178xAZMJrNx3w==:eyJwd19ub25jZSI6IjRjYjEwM2FhODljZmY0NTYzYTkxMWQzZjM5NjU4M2NlZmM2ODMzYzY2Zjg4MGZiZWUwNmJkYTk0YzMxZjg2OGIiLCJwd19jb3N0IjoxMTAwMDAsImlkZW50aWZpZXIiOiJub3YyMzIyQGJpdGFyLmlvIiwidmVyc2lvbiI6IjAwMyIsIm9yaWdpbmF0aW9uIjoicmVnaXN0cmF0aW9uIn0=', diff --git a/packages/snjs/mocha/model_tests/mapping.test.js b/packages/snjs/mocha/model_tests/mapping.test.js index 0a26f880d..bef0c3d70 100644 --- a/packages/snjs/mocha/model_tests/mapping.test.js +++ b/packages/snjs/mocha/model_tests/mapping.test.js @@ -94,7 +94,7 @@ describe('model manager mapping', () => { const note = this.application.itemManager.getDisplayableNotes()[0] await this.application.itemManager.setItemDirty(note) const dirtyItems = this.application.itemManager.getDirtyItems() - expect(dirtyItems.length).to.equal(1) + expect(Uuids(dirtyItems).includes(note.uuid)) }) it('set all items dirty', async function () { diff --git a/packages/snjs/mocha/session.test.js b/packages/snjs/mocha/session.test.js index 688822ac6..32cb3ad34 100644 --- a/packages/snjs/mocha/session.test.js +++ b/packages/snjs/mocha/session.test.js @@ -642,7 +642,7 @@ describe('server session', function () { await app2Deinit const deviceInterface = new WebDeviceInterface() - const payloads = await deviceInterface.getAllRawDatabasePayloads(app2identifier) + const payloads = await deviceInterface.getAllDatabaseEntries(app2identifier) expect(payloads).to.be.empty }) @@ -670,7 +670,7 @@ describe('server session', function () { await app2Deinit const deviceInterface = new WebDeviceInterface() - const payloads = await deviceInterface.getAllRawDatabasePayloads(app2identifier) + const payloads = await deviceInterface.getAllDatabaseEntries(app2identifier) expect(payloads).to.be.empty }) diff --git a/packages/snjs/mocha/storage.test.js b/packages/snjs/mocha/storage.test.js index 5efdc786a..d18e3f4a9 100644 --- a/packages/snjs/mocha/storage.test.js +++ b/packages/snjs/mocha/storage.test.js @@ -300,6 +300,7 @@ describe('storage manager', function () { await Factory.createSyncedNote(this.application) expect(await Factory.storagePayloadCount(this.application)).to.equal(BaseItemCounts.DefaultItems + 1) this.application = await Factory.signOutApplicationAndReturnNew(this.application) + await Factory.sleep(0.1, 'Allow all untrackable singleton syncs to complete') expect(await Factory.storagePayloadCount(this.application)).to.equal(BaseItemCounts.DefaultItems) }) }) diff --git a/packages/snjs/mocha/sync_tests/offline.test.js b/packages/snjs/mocha/sync_tests/offline.test.js index e029a2702..037a2a3f7 100644 --- a/packages/snjs/mocha/sync_tests/offline.test.js +++ b/packages/snjs/mocha/sync_tests/offline.test.js @@ -31,10 +31,7 @@ describe('offline syncing', () => { it('should sync item with no passcode', async function () { let note = await Factory.createMappedNote(this.application) - expect(this.application.itemManager.getDirtyItems().length).to.equal(1) - - const rawPayloads1 = await this.application.diskStorageService.getAllRawPayloads() - expect(rawPayloads1.length).to.equal(this.expectedItemCount) + expect(Uuids(this.application.itemManager.getDirtyItems()).includes(note.uuid)) await this.application.syncService.sync(syncOptions) diff --git a/packages/snjs/mocha/sync_tests/online.test.js b/packages/snjs/mocha/sync_tests/online.test.js index 081d696ff..eea5e9f7d 100644 --- a/packages/snjs/mocha/sync_tests/online.test.js +++ b/packages/snjs/mocha/sync_tests/online.test.js @@ -218,14 +218,21 @@ describe('online syncing', function () { it('retrieving new items should not mark them as dirty', async function () { const originalNote = await Factory.createSyncedNote(this.application) this.expectedItemCount++ + this.application = await Factory.signOutApplicationAndReturnNew(this.application) - this.application.syncService.addEventObserver((event) => { - if (event === SyncEvent.SingleRoundTripSyncCompleted) { - const note = this.application.items.findItem(originalNote.uuid) - expect(note.dirty).to.not.be.ok - } + const promise = new Promise((resolve) => { + this.application.syncService.addEventObserver(async (event) => { + if (event === SyncEvent.SingleRoundTripSyncCompleted) { + const note = this.application.items.findItem(originalNote.uuid) + if (note) { + expect(note.dirty).to.not.be.ok + resolve() + } + } + }) }) await this.application.signIn(this.email, this.password, undefined, undefined, undefined, true) + await promise }) it('allows saving of data after sign out', async function () { @@ -579,7 +586,7 @@ describe('online syncing', function () { await this.application.itemManager.setItemDirty(note) await this.application.syncService.sync(syncOptions) this.expectedItemCount++ - const rawPayloads = await this.application.syncService.getDatabasePayloads() + const rawPayloads = await this.application.diskStorageService.getAllRawPayloads() const notePayload = rawPayloads.find((p) => p.content_type === ContentType.Note) expect(typeof notePayload.content).to.equal('string') }) @@ -651,8 +658,7 @@ describe('online syncing', function () { await this.application.syncService.clearSyncPositionTokens() await this.application.payloadManager.resetState() await this.application.itemManager.resetState() - const databasePayloads = await this.application.diskStorageService.getAllRawPayloads() - await this.application.syncService.loadDatabasePayloads(databasePayloads) + await this.application.syncService.loadDatabasePayloads() await this.application.syncService.sync(syncOptions) const newRawPayloads = await this.application.diskStorageService.getAllRawPayloads() @@ -672,7 +678,9 @@ describe('online syncing', function () { const payload = Factory.createStorageItemPayload(contentTypes[Math.floor(i / 2)]) originalPayloads.push(payload) } - const { contentTypePriorityPayloads } = GetSortedPayloadsByPriority(originalPayloads, ['C', 'A', 'B']) + const { contentTypePriorityPayloads } = GetSortedPayloadsByPriority(originalPayloads, { + contentTypePriority: ['C', 'A', 'B'], + }) expect(contentTypePriorityPayloads[0].content_type).to.equal('C') expect(contentTypePriorityPayloads[2].content_type).to.equal('A') expect(contentTypePriorityPayloads[4].content_type).to.equal('B') @@ -685,14 +693,10 @@ describe('online syncing', function () { await this.application.syncService.sync(syncOptions) this.application = await Factory.signOutApplicationAndReturnNew(this.application) - const rawPayloads = await this.application.diskStorageService.getAllRawPayloads() - expect(rawPayloads.length).to.equal(BaseItemCounts.DefaultItems) - await this.application.signIn(this.email, this.password, undefined, undefined, undefined, true) this.application.syncService.ut_setDatabaseLoaded(false) - const databasePayloads = await this.application.diskStorageService.getAllRawPayloads() - await this.application.syncService.loadDatabasePayloads(databasePayloads) + await this.application.syncService.loadDatabasePayloads() await this.application.syncService.sync(syncOptions) const items = await this.application.itemManager.items diff --git a/packages/snjs/mocha/test.html b/packages/snjs/mocha/test.html index b5d169c0b..e9bc11b6f 100644 --- a/packages/snjs/mocha/test.html +++ b/packages/snjs/mocha/test.html @@ -80,8 +80,6 @@ - - diff --git a/packages/snjs/package.json b/packages/snjs/package.json index 30861ab48..0d25c2949 100644 --- a/packages/snjs/package.json +++ b/packages/snjs/package.json @@ -22,6 +22,7 @@ "clean": "rm -fr dist", "prebuild": "yarn clean", "build": "yarn tsc && webpack --config webpack.prod.js", + "watch": "webpack --config webpack.prod.js --watch", "docs": "jsdoc -c jsdoc.json", "tsc": "tsc --project lib/tsconfig.json && tscpaths -p lib/tsconfig.json -s lib -o dist/@types", "lint": "yarn lint:eslint lib", diff --git a/packages/web/src/javascripts/Application/Application.ts b/packages/web/src/javascripts/Application/Application.ts index e93b9ab54..efd05875b 100644 --- a/packages/web/src/javascripts/Application/Application.ts +++ b/packages/web/src/javascripts/Application/Application.ts @@ -21,6 +21,8 @@ import { DecryptedItem, EditorIdentifier, FeatureIdentifier, + Environment, + ApplicationOptionsDefaults, } from '@standardnotes/snjs' import { makeObservable, observable } from 'mobx' import { PanelResizedData } from '@/Types/PanelResizedData' @@ -75,6 +77,8 @@ export class WebApplication extends SNApplication implements WebApplicationInter defaultHost: defaultSyncServerHost, appVersion: deviceInterface.appVersion, webSocketUrl: webSocketUrl, + loadBatchSize: + deviceInterface.environment === Environment.Mobile ? 100 : ApplicationOptionsDefaults.loadBatchSize, }) makeObservable(this, { diff --git a/packages/web/src/javascripts/Application/Database.ts b/packages/web/src/javascripts/Application/Database.ts index a5ed63d40..9482041f5 100644 --- a/packages/web/src/javascripts/Application/Database.ts +++ b/packages/web/src/javascripts/Application/Database.ts @@ -140,6 +140,39 @@ export class Database { }) } + /** + * This function is actually unused, but implemented to conform to protocol in case it is eventually needed. + * We could remove implementation and throw instead, but it might be better to offer a functional alternative instead. + */ + public async getPayloadsForKeys(keys: string[]): Promise { + const db = (await this.openDatabase()) as IDBDatabase + return new Promise((resolve) => { + const objectStore = db.transaction(STORE_NAME).objectStore(STORE_NAME) + const payloads: any = [] + let numComplete = 0 + for (const key of keys) { + const getRequest = objectStore.get(key) + getRequest.onsuccess = (event) => { + const target = event.target as any + const result = target.result + if (result) { + payloads.push(result) + } + numComplete++ + if (numComplete === keys.length) { + resolve(payloads) + } + } + getRequest.onerror = () => { + numComplete++ + if (numComplete === keys.length) { + resolve(payloads) + } + } + } + }) + } + public async getAllKeys(): Promise { const db = (await this.openDatabase()) as IDBDatabase diff --git a/packages/web/src/javascripts/Application/Device/WebOrDesktopDevice.ts b/packages/web/src/javascripts/Application/Device/WebOrDesktopDevice.ts index 938aa0c98..e4cc3b428 100644 --- a/packages/web/src/javascripts/Application/Device/WebOrDesktopDevice.ts +++ b/packages/web/src/javascripts/Application/Device/WebOrDesktopDevice.ts @@ -2,13 +2,16 @@ import { SNApplication, ApplicationIdentifier, Environment, - LegacyRawKeychainValue, RawKeychainValue, TransferPayload, NamespacedRootKeyInKeychain, - extendArray, WebOrDesktopDeviceInterface, Platform, + FullyFormedTransferPayload, + DatabaseLoadOptions, + GetSortedPayloadsByPriority, + DatabaseFullEntryLoadChunk, + DatabaseFullEntryLoadChunkResponse, } from '@standardnotes/snjs' import { Database } from '../Database' @@ -72,17 +75,6 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface return result } - async getAllRawStorageKeyValues() { - const results = [] - for (const key of Object.keys(localStorage)) { - results.push({ - key: key, - value: localStorage[key], - }) - } - return results - } - async setRawStorageValue(key: string, value: string) { localStorage.setItem(key, value) } @@ -111,23 +103,63 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface }) as Promise<{ isNewDatabase?: boolean } | undefined> } - async getAllRawDatabasePayloads(identifier: ApplicationIdentifier) { + async getDatabaseLoadChunks( + options: DatabaseLoadOptions, + identifier: string, + ): Promise { + const entries = await this.getAllDatabaseEntries(identifier) + const sorted = GetSortedPayloadsByPriority(entries, options) + + const itemsKeysChunk: DatabaseFullEntryLoadChunk = { + entries: sorted.itemsKeyPayloads, + } + + const contentTypePriorityChunk: DatabaseFullEntryLoadChunk = { + entries: sorted.contentTypePriorityPayloads, + } + + const remainingPayloadsChunks: DatabaseFullEntryLoadChunk[] = [] + for (let i = 0; i < sorted.remainingPayloads.length; i += options.batchSize) { + remainingPayloadsChunks.push({ + entries: sorted.remainingPayloads.slice(i, i + options.batchSize), + }) + } + + const result: DatabaseFullEntryLoadChunkResponse = { + fullEntries: { + itemsKeys: itemsKeysChunk, + remainingChunks: [contentTypePriorityChunk, ...remainingPayloadsChunks], + }, + remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length, + } + + return result + } + + async getAllDatabaseEntries(identifier: ApplicationIdentifier) { return this.databaseForIdentifier(identifier).getAllPayloads() } - async saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier) { + getDatabaseEntries( + identifier: string, + keys: string[], + ): Promise { + return this.databaseForIdentifier(identifier).getPayloadsForKeys(keys) + } + + async saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier) { return this.databaseForIdentifier(identifier).savePayload(payload) } - async saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier) { + async saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier) { return this.databaseForIdentifier(identifier).savePayloads(payloads) } - async removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier) { + async removeDatabaseEntry(id: string, identifier: ApplicationIdentifier) { return this.databaseForIdentifier(identifier).deletePayload(id) } - async removeAllRawDatabasePayloads(identifier: ApplicationIdentifier) { + async removeAllDatabaseEntries(identifier: ApplicationIdentifier) { return this.databaseForIdentifier(identifier).clearAllPayloads() } @@ -141,16 +173,6 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface return keychain[identifier] } - async getDatabaseKeys(): Promise { - const keys: string[] = [] - - for (const database of this.databases) { - extendArray(keys, await database.getAllKeys()) - } - - return keys - } - async setNamespacedKeychainValue(value: NamespacedRootKeyInKeychain, identifier: ApplicationIdentifier) { let keychain = await this.getKeychainValue() @@ -186,10 +208,6 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface } } - setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise { - return this.setKeychainValue(value) - } - abstract getKeychainValue(): Promise abstract setKeychainValue(value: unknown): Promise diff --git a/yarn.lock b/yarn.lock index 686e056e6..c6b6eec89 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5883,6 +5883,7 @@ __metadata: react-native-fs: ^2.20.0 react-native-iap: ^12.4.4 react-native-keychain: "standardnotes/react-native-keychain#d277d360494cbd02be4accb4a360772a8e0e97b6" + react-native-mmkv: ^2.5.1 react-native-privacy-snapshot: "standardnotes/react-native-privacy-snapshot#653e904c90fc6f2b578da59138f2bfe5d7f942fe" react-native-share: ^8.0.0 react-native-version-info: ^1.1.1 @@ -25615,6 +25616,16 @@ __metadata: languageName: node linkType: hard +"react-native-mmkv@npm:^2.5.1": + version: 2.5.1 + resolution: "react-native-mmkv@npm:2.5.1" + peerDependencies: + react: "*" + react-native: "*" + checksum: 6f0cf484e71d8069c9b3cdb57b76eafaca40aa75f359beb6959c77d0ef66d0481d4459b1ffa94640170ce4744e337fefb38b8ccf6e1a3c3663561ede5f7a2c20 + languageName: node + linkType: hard + "react-native-privacy-snapshot@standardnotes/react-native-privacy-snapshot#653e904c90fc6f2b578da59138f2bfe5d7f942fe": version: 1.0.0 resolution: "react-native-privacy-snapshot@https://github.com/standardnotes/react-native-privacy-snapshot.git#commit=653e904c90fc6f2b578da59138f2bfe5d7f942fe"