feat: improve initial load performance on mobile (#2126)
This commit is contained in:
BIN
.yarn/cache/react-native-mmkv-npm-2.5.1-4745a42823-6f0cf484e7.zip
vendored
Normal file
BIN
.yarn/cache/react-native-mmkv-npm-2.5.1-4745a42823-6f0cf484e7.zip
vendored
Normal file
Binary file not shown.
@@ -18,17 +18,6 @@ export default class WebDeviceInterface {
|
||||
}
|
||||
}
|
||||
|
||||
async getAllRawStorageKeyValues() {
|
||||
const results = []
|
||||
for (const key of Object.keys(localStorage)) {
|
||||
results.push({
|
||||
key: key,
|
||||
value: localStorage[key],
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
async setRawStorageValue(key, value) {
|
||||
localStorage.setItem(key, value)
|
||||
}
|
||||
@@ -57,7 +46,7 @@ export default class WebDeviceInterface {
|
||||
return `${this._getDatabaseKeyPrefix(identifier)}${id}`
|
||||
}
|
||||
|
||||
async getAllRawDatabasePayloads(identifier) {
|
||||
async getAllDatabaseEntries(identifier) {
|
||||
const models = []
|
||||
for (const key in localStorage) {
|
||||
if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) {
|
||||
@@ -67,21 +56,21 @@ export default class WebDeviceInterface {
|
||||
return models
|
||||
}
|
||||
|
||||
async saveRawDatabasePayload(payload, identifier) {
|
||||
async saveDatabaseEntry(payload, identifier) {
|
||||
localStorage.setItem(this._keyForPayloadId(payload.uuid, identifier), JSON.stringify(payload))
|
||||
}
|
||||
|
||||
async saveRawDatabasePayloads(payloads, identifier) {
|
||||
async saveDatabaseEntries(payloads, identifier) {
|
||||
for (const payload of payloads) {
|
||||
await this.saveRawDatabasePayload(payload, identifier)
|
||||
await this.saveDatabaseEntry(payload, identifier)
|
||||
}
|
||||
}
|
||||
|
||||
async removeRawDatabasePayloadWithId(id, identifier) {
|
||||
async removeDatabaseEntry(id, identifier) {
|
||||
localStorage.removeItem(this._keyForPayloadId(id, identifier))
|
||||
}
|
||||
|
||||
async removeAllRawDatabasePayloads(identifier) {
|
||||
async removeAllDatabaseEntries(identifier) {
|
||||
for (const key in localStorage) {
|
||||
if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) {
|
||||
delete localStorage[key]
|
||||
@@ -121,12 +110,6 @@ export default class WebDeviceInterface {
|
||||
localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(keychain))
|
||||
}
|
||||
|
||||
/** Allows unit tests to set legacy keychain structure as it was <= 003 */
|
||||
// eslint-disable-next-line camelcase
|
||||
async setLegacyRawKeychainValue(value) {
|
||||
localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(value))
|
||||
}
|
||||
|
||||
async getRawKeychainValue() {
|
||||
const keychain = localStorage.getItem(KEYCHAIN_STORAGE_KEY)
|
||||
return JSON.parse(keychain)
|
||||
|
||||
@@ -75,6 +75,9 @@ PODS:
|
||||
- glog (0.3.5)
|
||||
- hermes-engine (0.70.6)
|
||||
- libevent (2.1.12)
|
||||
- MMKV (1.2.14):
|
||||
- MMKVCore (~> 1.2.14)
|
||||
- MMKVCore (1.2.14)
|
||||
- OpenSSL-Universal (1.1.1100)
|
||||
- RCT-Folly (2021.07.22.00):
|
||||
- boost
|
||||
@@ -304,6 +307,9 @@ PODS:
|
||||
- glog
|
||||
- react-native-fingerprint-scanner (5.0.0):
|
||||
- React-Core
|
||||
- react-native-mmkv (2.5.1):
|
||||
- MMKV (>= 1.2.13)
|
||||
- React-Core
|
||||
- react-native-version-info (1.1.1):
|
||||
- React-Core
|
||||
- react-native-webview (11.23.1):
|
||||
@@ -444,6 +450,7 @@ DEPENDENCIES:
|
||||
- React-jsinspector (from `../node_modules/react-native/ReactCommon/jsinspector`)
|
||||
- React-logger (from `../node_modules/react-native/ReactCommon/logger`)
|
||||
- react-native-fingerprint-scanner (from `../node_modules/react-native-fingerprint-scanner`)
|
||||
- react-native-mmkv (from `../node_modules/react-native-mmkv`)
|
||||
- react-native-version-info (from `../node_modules/react-native-version-info`)
|
||||
- react-native-webview (from `../node_modules/react-native-webview`)
|
||||
- React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`)
|
||||
@@ -483,6 +490,8 @@ SPEC REPOS:
|
||||
- FlipperKit
|
||||
- fmt
|
||||
- libevent
|
||||
- MMKV
|
||||
- MMKVCore
|
||||
- OpenSSL-Universal
|
||||
- SocketRocket
|
||||
- TrustKit
|
||||
@@ -533,6 +542,8 @@ EXTERNAL SOURCES:
|
||||
:path: "../node_modules/react-native/ReactCommon/logger"
|
||||
react-native-fingerprint-scanner:
|
||||
:path: "../node_modules/react-native-fingerprint-scanner"
|
||||
react-native-mmkv:
|
||||
:path: "../node_modules/react-native-mmkv"
|
||||
react-native-version-info:
|
||||
:path: "../node_modules/react-native-version-info"
|
||||
react-native-webview:
|
||||
@@ -583,7 +594,7 @@ EXTERNAL SOURCES:
|
||||
SPEC CHECKSUMS:
|
||||
boost: a7c83b31436843459a1961bfd74b96033dc77234
|
||||
CocoaAsyncSocket: 065fd1e645c7abab64f7a6a2007a48038fdc6a99
|
||||
DoubleConversion: 831926d9b8bf8166fd87886c4abab286c2422662
|
||||
DoubleConversion: 5189b271737e1565bdce30deb4a08d647e3f5f54
|
||||
FBLazyVector: 48289402952f4f7a4e235de70a9a590aa0b79ef4
|
||||
FBReactNativeSpec: dd1186fd05255e3457baa2f4ca65e94c2cd1e3ac
|
||||
Flipper: 26fc4b7382499f1281eb8cb921e5c3ad6de91fe0
|
||||
@@ -596,9 +607,11 @@ SPEC CHECKSUMS:
|
||||
Flipper-RSocket: d9d9ade67cbecf6ac10730304bf5607266dd2541
|
||||
FlipperKit: cbdee19bdd4e7f05472a66ce290f1b729ba3cb86
|
||||
fmt: ff9d55029c625d3757ed641535fd4a75fedc7ce9
|
||||
glog: 85ecdd10ee8d8ec362ef519a6a45ff9aa27b2e85
|
||||
glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b
|
||||
hermes-engine: 2af7b7a59128f250adfd86f15aa1d5a2ecd39995
|
||||
libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913
|
||||
MMKV: 9c4663aa7ca255d478ff10f2f5cb7d17c1651ccd
|
||||
MMKVCore: 89f5c8a66bba2dcd551779dea4d412eeec8ff5bb
|
||||
OpenSSL-Universal: ebc357f1e6bc71fa463ccb2fe676756aff50e88c
|
||||
RCT-Folly: 0080d0a6ebf2577475bda044aa59e2ca1f909cda
|
||||
RCTRequired: e1866f61af7049eb3d8e08e8b133abd38bc1ca7a
|
||||
@@ -616,6 +629,7 @@ SPEC CHECKSUMS:
|
||||
React-jsinspector: 60769e5a0a6d4b32294a2456077f59d0266f9a8b
|
||||
React-logger: 1623c216abaa88974afce404dc8f479406bbc3a0
|
||||
react-native-fingerprint-scanner: be63e626b31fb951780a5fac5328b065a61a3d6e
|
||||
react-native-mmkv: 69b9c003f10afdd01addf7c6ee784ce42ee2eff3
|
||||
react-native-version-info: a106f23009ac0db4ee00de39574eb546682579b9
|
||||
react-native-webview: d33e2db8925d090871ffeb232dfa50cb3a727581
|
||||
React-perflogger: 8c79399b0500a30ee8152d0f9f11beae7fc36595
|
||||
|
||||
@@ -100,7 +100,7 @@
|
||||
<key>NSPhotoLibraryUsageDescription</key>
|
||||
<string>Photo library is optionally used to select files to upload or QR code images from your photo library.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Microphone is optionally used to capture videos.</string>
|
||||
<string>Microphone is optionally used to capture videos.</string>
|
||||
<key>UIAppFonts</key>
|
||||
<array>
|
||||
<string>AntDesign.ttf</string>
|
||||
@@ -147,5 +147,7 @@
|
||||
<false/>
|
||||
<key>supportsAlternateIcons</key>
|
||||
<true/>
|
||||
<key>RCTAsyncStorageExcludeFromBackup</key>
|
||||
<false/>
|
||||
</dict>
|
||||
</plist>
|
||||
|
||||
@@ -59,6 +59,7 @@
|
||||
"react-native-fs": "^2.20.0",
|
||||
"react-native-iap": "^12.4.4",
|
||||
"react-native-keychain": "standardnotes/react-native-keychain#d277d360494cbd02be4accb4a360772a8e0e97b6",
|
||||
"react-native-mmkv": "^2.5.1",
|
||||
"react-native-privacy-snapshot": "standardnotes/react-native-privacy-snapshot#653e904c90fc6f2b578da59138f2bfe5d7f942fe",
|
||||
"react-native-share": "^8.0.0",
|
||||
"react-native-version-info": "^1.1.1",
|
||||
|
||||
158
packages/mobile/src/Lib/Database/Database.ts
Normal file
158
packages/mobile/src/Lib/Database/Database.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
import AsyncStorage from '@react-native-community/async-storage'
|
||||
import {
|
||||
DatabaseKeysLoadChunk,
|
||||
DatabaseKeysLoadChunkResponse,
|
||||
DatabaseLoadOptions,
|
||||
GetSortedPayloadsByPriority,
|
||||
TransferPayload,
|
||||
} from '@standardnotes/snjs'
|
||||
import { Platform } from 'react-native'
|
||||
import { DatabaseInterface } from './DatabaseInterface'
|
||||
import { DatabaseMetadata } from './DatabaseMetadata'
|
||||
import { FlashKeyValueStore } from './FlashKeyValueStore'
|
||||
import { isLegacyIdentifier } from './LegacyIdentifier'
|
||||
import { showLoadFailForItemIds } from './showLoadFailForItemIds'
|
||||
|
||||
export class Database implements DatabaseInterface {
|
||||
private metadataStore: DatabaseMetadata
|
||||
|
||||
constructor(private identifier: string) {
|
||||
const flashStorage = new FlashKeyValueStore(identifier)
|
||||
this.metadataStore = new DatabaseMetadata(identifier, flashStorage)
|
||||
}
|
||||
|
||||
private databaseKeyForPayloadId(id: string) {
|
||||
return `${this.getDatabaseKeyPrefix()}${id}`
|
||||
}
|
||||
|
||||
private getDatabaseKeyPrefix() {
|
||||
if (this.identifier && !isLegacyIdentifier(this.identifier)) {
|
||||
return `${this.identifier}-Item-`
|
||||
} else {
|
||||
return 'Item-'
|
||||
}
|
||||
}
|
||||
|
||||
async getAllEntries<T extends TransferPayload = TransferPayload>(): Promise<T[]> {
|
||||
const keys = await this.getAllKeys()
|
||||
return this.multiGet(keys)
|
||||
}
|
||||
|
||||
async getAllKeys(): Promise<string[]> {
|
||||
const keys = await AsyncStorage.getAllKeys()
|
||||
const filtered = keys.filter((key) => {
|
||||
return key.startsWith(this.getDatabaseKeyPrefix())
|
||||
})
|
||||
return filtered
|
||||
}
|
||||
|
||||
async multiDelete(keys: string[]): Promise<void> {
|
||||
return AsyncStorage.multiRemove(keys)
|
||||
}
|
||||
|
||||
async deleteItem(itemUuid: string): Promise<void> {
|
||||
const key = this.databaseKeyForPayloadId(itemUuid)
|
||||
this.metadataStore.deleteMetadataItem(itemUuid)
|
||||
return this.multiDelete([key])
|
||||
}
|
||||
|
||||
async deleteAll(): Promise<void> {
|
||||
const keys = await this.getAllKeys()
|
||||
return this.multiDelete(keys)
|
||||
}
|
||||
|
||||
async setItems(items: TransferPayload[]): Promise<void> {
|
||||
if (items.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
items.map((item) => {
|
||||
return Promise.all([
|
||||
AsyncStorage.setItem(this.databaseKeyForPayloadId(item.uuid), JSON.stringify(item)),
|
||||
this.metadataStore.setMetadataForPayloads([item]),
|
||||
])
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
async getLoadChunks(options: DatabaseLoadOptions): Promise<DatabaseKeysLoadChunkResponse> {
|
||||
let metadataItems = this.metadataStore.getAllMetadataItems()
|
||||
|
||||
if (metadataItems.length === 0) {
|
||||
const allEntries = await this.getAllEntries()
|
||||
metadataItems = this.metadataStore.runMigration(allEntries)
|
||||
}
|
||||
|
||||
const sorted = GetSortedPayloadsByPriority(metadataItems, options)
|
||||
|
||||
const itemsKeysChunk: DatabaseKeysLoadChunk = {
|
||||
keys: sorted.itemsKeyPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)),
|
||||
}
|
||||
|
||||
const contentTypePriorityChunk: DatabaseKeysLoadChunk = {
|
||||
keys: sorted.contentTypePriorityPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid)),
|
||||
}
|
||||
|
||||
const remainingKeys = sorted.remainingPayloads.map((item) => this.databaseKeyForPayloadId(item.uuid))
|
||||
|
||||
const remainingKeysChunks: DatabaseKeysLoadChunk[] = []
|
||||
for (let i = 0; i < remainingKeys.length; i += options.batchSize) {
|
||||
remainingKeysChunks.push({
|
||||
keys: remainingKeys.slice(i, i + options.batchSize),
|
||||
})
|
||||
}
|
||||
|
||||
const result: DatabaseKeysLoadChunkResponse = {
|
||||
keys: {
|
||||
itemsKeys: itemsKeysChunk,
|
||||
remainingChunks: [contentTypePriorityChunk, ...remainingKeysChunks],
|
||||
},
|
||||
remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length,
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async multiGet<T>(keys: string[]): Promise<T[]> {
|
||||
const results: T[] = []
|
||||
|
||||
if (Platform.OS === 'android') {
|
||||
const failedItemIds: string[] = []
|
||||
for (const key of keys) {
|
||||
try {
|
||||
const item = await AsyncStorage.getItem(key)
|
||||
if (item) {
|
||||
try {
|
||||
results.push(JSON.parse(item) as T)
|
||||
} catch (e) {
|
||||
results.push(item as T)
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error getting item', key, e)
|
||||
failedItemIds.push(key)
|
||||
}
|
||||
}
|
||||
if (failedItemIds.length > 0) {
|
||||
showLoadFailForItemIds(failedItemIds)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
for (const item of await AsyncStorage.multiGet(keys)) {
|
||||
if (item[1]) {
|
||||
try {
|
||||
results.push(JSON.parse(item[1]))
|
||||
} catch (e) {
|
||||
results.push(item[1] as T)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error getting items', e)
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
}
|
||||
10
packages/mobile/src/Lib/Database/DatabaseInterface.ts
Normal file
10
packages/mobile/src/Lib/Database/DatabaseInterface.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { TransferPayload } from '@standardnotes/snjs'
|
||||
|
||||
export interface DatabaseInterface {
|
||||
getAllKeys(): Promise<string[]>
|
||||
multiDelete(keys: string[]): Promise<void>
|
||||
deleteItem(itemUuid: string): Promise<void>
|
||||
deleteAll(): Promise<void>
|
||||
setItems(items: TransferPayload[]): Promise<void>
|
||||
multiGet<T>(keys: string[]): Promise<T[]>
|
||||
}
|
||||
39
packages/mobile/src/Lib/Database/DatabaseMetadata.ts
Normal file
39
packages/mobile/src/Lib/Database/DatabaseMetadata.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { DatabaseItemMetadata, isNotUndefined, TransferPayload } from '@standardnotes/snjs'
|
||||
import { FlashKeyValueStore } from './FlashKeyValueStore'
|
||||
|
||||
export class DatabaseMetadata {
|
||||
constructor(private identifier: string, private flashStorage: FlashKeyValueStore) {}
|
||||
|
||||
runMigration(payloads: TransferPayload[]) {
|
||||
const metadataItems = this.setMetadataForPayloads(payloads)
|
||||
return metadataItems
|
||||
}
|
||||
|
||||
setMetadataForPayloads(payloads: TransferPayload[]) {
|
||||
const metadataItems = []
|
||||
for (const payload of payloads) {
|
||||
const { uuid, content_type, updated_at } = payload
|
||||
const key = this.keyForUuid(uuid)
|
||||
const metadata: DatabaseItemMetadata = { uuid, content_type, updated_at }
|
||||
this.flashStorage.set(key, metadata)
|
||||
metadataItems.push(metadata)
|
||||
}
|
||||
return metadataItems
|
||||
}
|
||||
|
||||
deleteMetadataItem(itemUuid: string) {
|
||||
const key = this.keyForUuid(itemUuid)
|
||||
this.flashStorage.delete(key)
|
||||
}
|
||||
|
||||
getAllMetadataItems(): DatabaseItemMetadata[] {
|
||||
const keys = this.flashStorage.getAllKeys()
|
||||
const metadataKeys = keys.filter((key) => key.endsWith('-Metadata'))
|
||||
const metadataItems = this.flashStorage.multiGet<DatabaseItemMetadata>(metadataKeys).filter(isNotUndefined)
|
||||
return metadataItems
|
||||
}
|
||||
|
||||
private keyForUuid(uuid: string) {
|
||||
return `${this.identifier}-Item-${uuid}-Metadata`
|
||||
}
|
||||
}
|
||||
40
packages/mobile/src/Lib/Database/FlashKeyValueStore.ts
Normal file
40
packages/mobile/src/Lib/Database/FlashKeyValueStore.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { MMKV } from 'react-native-mmkv'
|
||||
|
||||
export class FlashKeyValueStore {
|
||||
private storage: MMKV
|
||||
|
||||
constructor(identifier: string) {
|
||||
this.storage = new MMKV({ id: identifier })
|
||||
}
|
||||
|
||||
set(key: string, value: unknown): void {
|
||||
this.storage.set(key, JSON.stringify(value))
|
||||
}
|
||||
|
||||
delete(key: string): void {
|
||||
this.storage.delete(key)
|
||||
}
|
||||
|
||||
deleteAll(): void {
|
||||
this.storage.clearAll()
|
||||
}
|
||||
|
||||
getAllKeys(): string[] {
|
||||
return this.storage.getAllKeys()
|
||||
}
|
||||
|
||||
get<T>(key: string): T | undefined {
|
||||
const item = this.storage.getString(key)
|
||||
if (item) {
|
||||
try {
|
||||
return JSON.parse(item)
|
||||
} catch (e) {
|
||||
return item as T
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
multiGet<T>(keys: string[]): (T | undefined)[] {
|
||||
return keys.map((key) => this.get<T>(key))
|
||||
}
|
||||
}
|
||||
15
packages/mobile/src/Lib/Database/LegacyIdentifier.ts
Normal file
15
packages/mobile/src/Lib/Database/LegacyIdentifier.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { ApplicationIdentifier } from '@standardnotes/snjs'
|
||||
|
||||
/**
|
||||
* This identifier was the database name used in Standard Notes web/desktop.
|
||||
*/
|
||||
const LEGACY_IDENTIFIER = 'standardnotes'
|
||||
|
||||
/**
|
||||
* We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not.
|
||||
* It is also used to decide if the raw or the namespaced keychain is used.
|
||||
* @param identifier The ApplicationIdentifier
|
||||
*/
|
||||
export const isLegacyIdentifier = function (identifier: ApplicationIdentifier) {
|
||||
return identifier && identifier === LEGACY_IDENTIFIER
|
||||
}
|
||||
26
packages/mobile/src/Lib/Database/LegacyKeyValueStore.ts
Normal file
26
packages/mobile/src/Lib/Database/LegacyKeyValueStore.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import AsyncStorage from '@react-native-community/async-storage'
|
||||
|
||||
export class LegacyKeyValueStore {
|
||||
set(key: string, value: string): Promise<void> {
|
||||
return AsyncStorage.setItem(key, JSON.stringify(value))
|
||||
}
|
||||
|
||||
delete(key: string): Promise<void> {
|
||||
return AsyncStorage.removeItem(key)
|
||||
}
|
||||
|
||||
deleteAll(): Promise<void> {
|
||||
return AsyncStorage.clear()
|
||||
}
|
||||
|
||||
async getValue<T>(key: string): Promise<T | undefined> {
|
||||
const item = await AsyncStorage.getItem(key)
|
||||
if (item) {
|
||||
try {
|
||||
return JSON.parse(item)
|
||||
} catch (e) {
|
||||
return item as T
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
16
packages/mobile/src/Lib/Database/showLoadFailForItemIds.ts
Normal file
16
packages/mobile/src/Lib/Database/showLoadFailForItemIds.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { Alert } from 'react-native'
|
||||
|
||||
export const showLoadFailForItemIds = (failedItemIds: string[]) => {
|
||||
let text =
|
||||
'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n'
|
||||
let index = 0
|
||||
text += failedItemIds.map((id) => {
|
||||
let result = id
|
||||
if (index !== failedItemIds.length - 1) {
|
||||
result += '\n'
|
||||
}
|
||||
index++
|
||||
return result
|
||||
})
|
||||
Alert.alert('Unable to load item(s)', text)
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
import AsyncStorage from '@react-native-community/async-storage'
|
||||
import SNReactNative from '@standardnotes/react-native-utils'
|
||||
import { AppleIAPReceipt } from '@standardnotes/services'
|
||||
import {
|
||||
AppleIAPProductId,
|
||||
AppleIAPReceipt,
|
||||
ApplicationIdentifier,
|
||||
DatabaseKeysLoadChunkResponse,
|
||||
DatabaseLoadOptions,
|
||||
Environment,
|
||||
LegacyMobileKeychainStructure,
|
||||
LegacyRawKeychainValue,
|
||||
MobileDeviceInterface,
|
||||
NamespacedRootKeyInKeychain,
|
||||
Platform as SNPlatform,
|
||||
@@ -41,8 +40,11 @@ import {
|
||||
import { hide, show } from 'react-native-privacy-snapshot'
|
||||
import Share from 'react-native-share'
|
||||
import { AndroidBackHandlerService } from '../AndroidBackHandlerService'
|
||||
import { AppStateObserverService } from '../AppStateObserverService'
|
||||
import { PurchaseManager } from '../PurchaseManager'
|
||||
import { AppStateObserverService } from './../AppStateObserverService'
|
||||
import { Database } from './Database/Database'
|
||||
import { isLegacyIdentifier } from './Database/LegacyIdentifier'
|
||||
import { LegacyKeyValueStore } from './Database/LegacyKeyValueStore'
|
||||
import Keychain from './Keychain'
|
||||
|
||||
export type BiometricsType = 'Fingerprint' | 'Face ID' | 'Biometrics' | 'Touch ID'
|
||||
@@ -53,41 +55,6 @@ export enum MobileDeviceEvent {
|
||||
|
||||
type MobileDeviceEventHandler = (event: MobileDeviceEvent) => void
|
||||
|
||||
/**
|
||||
* This identifier was the database name used in Standard Notes web/desktop.
|
||||
*/
|
||||
const LEGACY_IDENTIFIER = 'standardnotes'
|
||||
|
||||
/**
|
||||
* We use this function to decide if we need to prefix the identifier in getDatabaseKeyPrefix or not.
|
||||
* It is also used to decide if the raw or the namespaced keychain is used.
|
||||
* @param identifier The ApplicationIdentifier
|
||||
*/
|
||||
const isLegacyIdentifier = function (identifier: ApplicationIdentifier) {
|
||||
return identifier && identifier === LEGACY_IDENTIFIER
|
||||
}
|
||||
|
||||
function isLegacyMobileKeychain(
|
||||
x: LegacyMobileKeychainStructure | RawKeychainValue,
|
||||
): x is LegacyMobileKeychainStructure {
|
||||
return x.ak != undefined
|
||||
}
|
||||
|
||||
const showLoadFailForItemIds = (failedItemIds: string[]) => {
|
||||
let text =
|
||||
'The following items could not be loaded. This may happen if you are in low-memory conditions, or if the note is very large in size. We recommend breaking up large notes into smaller chunks using the desktop or web app.\n\nItems:\n'
|
||||
let index = 0
|
||||
text += failedItemIds.map((id) => {
|
||||
let result = id
|
||||
if (index !== failedItemIds.length - 1) {
|
||||
result += '\n'
|
||||
}
|
||||
index++
|
||||
return result
|
||||
})
|
||||
Alert.alert('Unable to load item(s)', text)
|
||||
}
|
||||
|
||||
export class MobileDevice implements MobileDeviceInterface {
|
||||
environment: Environment.Mobile = Environment.Mobile
|
||||
platform: SNPlatform.Ios | SNPlatform.Android = Platform.OS === 'ios' ? SNPlatform.Ios : SNPlatform.Android
|
||||
@@ -95,6 +62,8 @@ export class MobileDevice implements MobileDeviceInterface {
|
||||
public isDarkMode = false
|
||||
public statusBarBgColor: string | undefined
|
||||
private componentUrls: Map<UuidString, string> = new Map()
|
||||
private keyValueStore = new LegacyKeyValueStore()
|
||||
private databases = new Map<string, Database>()
|
||||
|
||||
constructor(
|
||||
private stateObserverService?: AppStateObserverService,
|
||||
@@ -106,6 +75,17 @@ export class MobileDevice implements MobileDeviceInterface {
|
||||
return PurchaseManager.getInstance().purchase(plan)
|
||||
}
|
||||
|
||||
private findOrCreateDatabase(identifier: ApplicationIdentifier): Database {
|
||||
const existing = this.databases.get(identifier)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
|
||||
const newDb = new Database(identifier)
|
||||
this.databases.set(identifier, newDb)
|
||||
return newDb
|
||||
}
|
||||
|
||||
deinit() {
|
||||
this.stateObserverService?.deinit()
|
||||
;(this.stateObserverService as unknown) = undefined
|
||||
@@ -120,10 +100,6 @@ export class MobileDevice implements MobileDeviceInterface {
|
||||
console.log(args)
|
||||
}
|
||||
|
||||
async setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise<void> {
|
||||
await Keychain.setKeys(value)
|
||||
}
|
||||
|
||||
public async getJsonParsedRawStorageValue(key: string): Promise<unknown | undefined> {
|
||||
const value = await this.getRawStorageValue(key)
|
||||
if (value == undefined) {
|
||||
@@ -136,219 +112,57 @@ export class MobileDevice implements MobileDeviceInterface {
|
||||
}
|
||||
}
|
||||
|
||||
private getDatabaseKeyPrefix(identifier: ApplicationIdentifier) {
|
||||
if (identifier && !isLegacyIdentifier(identifier)) {
|
||||
return `${identifier}-Item-`
|
||||
} else {
|
||||
return 'Item-'
|
||||
}
|
||||
}
|
||||
|
||||
private keyForPayloadId(id: string, identifier: ApplicationIdentifier) {
|
||||
return `${this.getDatabaseKeyPrefix(identifier)}${id}`
|
||||
}
|
||||
|
||||
private async getAllDatabaseKeys(identifier: ApplicationIdentifier) {
|
||||
const keys = await AsyncStorage.getAllKeys()
|
||||
const filtered = keys.filter((key) => {
|
||||
return key.startsWith(this.getDatabaseKeyPrefix(identifier))
|
||||
})
|
||||
return filtered
|
||||
}
|
||||
|
||||
getDatabaseKeys(): Promise<string[]> {
|
||||
return AsyncStorage.getAllKeys()
|
||||
}
|
||||
|
||||
private async getRawStorageKeyValues(keys: string[]) {
|
||||
const results: { key: string; value: unknown }[] = []
|
||||
if (Platform.OS === 'android') {
|
||||
for (const key of keys) {
|
||||
try {
|
||||
const item = await AsyncStorage.getItem(key)
|
||||
if (item) {
|
||||
results.push({ key, value: item })
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error getting item', key, e)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
for (const item of await AsyncStorage.multiGet(keys)) {
|
||||
if (item[1]) {
|
||||
results.push({ key: item[0], value: item[1] })
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error getting items', e)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
private async getDatabaseKeyValues(keys: string[]) {
|
||||
const results: (TransferPayload | unknown)[] = []
|
||||
|
||||
if (Platform.OS === 'android') {
|
||||
const failedItemIds: string[] = []
|
||||
for (const key of keys) {
|
||||
try {
|
||||
const item = await AsyncStorage.getItem(key)
|
||||
if (item) {
|
||||
try {
|
||||
results.push(JSON.parse(item) as TransferPayload)
|
||||
} catch (e) {
|
||||
results.push(item)
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error getting item', key, e)
|
||||
failedItemIds.push(key)
|
||||
}
|
||||
}
|
||||
if (failedItemIds.length > 0) {
|
||||
showLoadFailForItemIds(failedItemIds)
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
for (const item of await AsyncStorage.multiGet(keys)) {
|
||||
if (item[1]) {
|
||||
try {
|
||||
results.push(JSON.parse(item[1]))
|
||||
} catch (e) {
|
||||
results.push(item[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error getting items', e)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
async getRawStorageValue(key: string) {
|
||||
const item = await AsyncStorage.getItem(key)
|
||||
if (item) {
|
||||
try {
|
||||
return JSON.parse(item)
|
||||
} catch (e) {
|
||||
return item
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hideMobileInterfaceFromScreenshots(): void {
|
||||
hide()
|
||||
this.setAndroidScreenshotPrivacy(true)
|
||||
}
|
||||
|
||||
stopHidingMobileInterfaceFromScreenshots(): void {
|
||||
show()
|
||||
this.setAndroidScreenshotPrivacy(false)
|
||||
}
|
||||
|
||||
async getAllRawStorageKeyValues() {
|
||||
const keys = await AsyncStorage.getAllKeys()
|
||||
return this.getRawStorageKeyValues(keys)
|
||||
getRawStorageValue(key: string): Promise<string | undefined> {
|
||||
return this.keyValueStore.getValue(key)
|
||||
}
|
||||
|
||||
setRawStorageValue(key: string, value: string): Promise<void> {
|
||||
return AsyncStorage.setItem(key, JSON.stringify(value))
|
||||
return this.keyValueStore.set(key, value)
|
||||
}
|
||||
|
||||
removeRawStorageValue(key: string): Promise<void> {
|
||||
return AsyncStorage.removeItem(key)
|
||||
return this.keyValueStore.delete(key)
|
||||
}
|
||||
|
||||
removeAllRawStorageValues(): Promise<void> {
|
||||
return AsyncStorage.clear()
|
||||
return this.keyValueStore.deleteAll()
|
||||
}
|
||||
|
||||
openDatabase(): Promise<{ isNewDatabase?: boolean | undefined } | undefined> {
|
||||
return Promise.resolve({ isNewDatabase: false })
|
||||
}
|
||||
|
||||
async getAllRawDatabasePayloads<T extends TransferPayload = TransferPayload>(
|
||||
getDatabaseLoadChunks(options: DatabaseLoadOptions, identifier: string): Promise<DatabaseKeysLoadChunkResponse> {
|
||||
return this.findOrCreateDatabase(identifier).getLoadChunks(options)
|
||||
}
|
||||
|
||||
async getAllDatabaseEntries<T extends TransferPayload = TransferPayload>(
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<T[]> {
|
||||
const keys = await this.getAllDatabaseKeys(identifier)
|
||||
return this.getDatabaseKeyValues(keys) as Promise<T[]>
|
||||
return this.findOrCreateDatabase(identifier).getAllEntries()
|
||||
}
|
||||
|
||||
saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void> {
|
||||
return this.saveRawDatabasePayloads([payload], identifier)
|
||||
}
|
||||
|
||||
async saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void> {
|
||||
if (payloads.length === 0) {
|
||||
return
|
||||
}
|
||||
await Promise.all(
|
||||
payloads.map((item) => {
|
||||
return AsyncStorage.setItem(this.keyForPayloadId(item.uuid, identifier), JSON.stringify(item))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier): Promise<void> {
|
||||
return this.removeRawStorageValue(this.keyForPayloadId(id, identifier))
|
||||
}
|
||||
|
||||
async removeAllRawDatabasePayloads(identifier: ApplicationIdentifier): Promise<void> {
|
||||
const keys = await this.getAllDatabaseKeys(identifier)
|
||||
return AsyncStorage.multiRemove(keys)
|
||||
}
|
||||
|
||||
async getNamespacedKeychainValue(
|
||||
async getDatabaseEntries<T extends TransferPayload = TransferPayload>(
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<NamespacedRootKeyInKeychain | undefined> {
|
||||
const keychain = await this.getRawKeychainValue()
|
||||
|
||||
if (!keychain) {
|
||||
return
|
||||
}
|
||||
|
||||
const namespacedValue = keychain[identifier]
|
||||
|
||||
if (!namespacedValue && isLegacyIdentifier(identifier)) {
|
||||
return keychain as unknown as NamespacedRootKeyInKeychain
|
||||
}
|
||||
|
||||
return namespacedValue
|
||||
keys: string[],
|
||||
): Promise<T[]> {
|
||||
return this.findOrCreateDatabase(identifier).multiGet<T>(keys)
|
||||
}
|
||||
|
||||
async setNamespacedKeychainValue(
|
||||
value: NamespacedRootKeyInKeychain,
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<void> {
|
||||
let keychain = await this.getRawKeychainValue()
|
||||
|
||||
if (!keychain) {
|
||||
keychain = {}
|
||||
}
|
||||
|
||||
await Keychain.setKeys({
|
||||
...keychain,
|
||||
[identifier]: value,
|
||||
})
|
||||
saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void> {
|
||||
return this.saveDatabaseEntries([payload], identifier)
|
||||
}
|
||||
|
||||
async clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<void> {
|
||||
const keychain = await this.getRawKeychainValue()
|
||||
async saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void> {
|
||||
return this.findOrCreateDatabase(identifier).setItems(payloads)
|
||||
}
|
||||
|
||||
if (!keychain) {
|
||||
return
|
||||
}
|
||||
removeDatabaseEntry(id: string, identifier: ApplicationIdentifier): Promise<void> {
|
||||
return this.findOrCreateDatabase(identifier).deleteItem(id)
|
||||
}
|
||||
|
||||
if (!keychain[identifier] && isLegacyIdentifier(identifier) && isLegacyMobileKeychain(keychain)) {
|
||||
await this.clearRawKeychainValue()
|
||||
return
|
||||
}
|
||||
|
||||
delete keychain[identifier]
|
||||
await Keychain.setKeys(keychain)
|
||||
async removeAllDatabaseEntries(identifier: ApplicationIdentifier): Promise<void> {
|
||||
return this.findOrCreateDatabase(identifier).deleteAll()
|
||||
}
|
||||
|
||||
async getDeviceBiometricsAvailability() {
|
||||
@@ -413,6 +227,51 @@ export class MobileDevice implements MobileDeviceInterface {
|
||||
return result
|
||||
}
|
||||
|
||||
async getNamespacedKeychainValue(
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<NamespacedRootKeyInKeychain | undefined> {
|
||||
const keychain = await this.getRawKeychainValue()
|
||||
|
||||
if (!keychain) {
|
||||
return
|
||||
}
|
||||
|
||||
const namespacedValue = keychain[identifier]
|
||||
|
||||
if (!namespacedValue && isLegacyIdentifier(identifier)) {
|
||||
return keychain as unknown as NamespacedRootKeyInKeychain
|
||||
}
|
||||
|
||||
return namespacedValue
|
||||
}
|
||||
|
||||
async setNamespacedKeychainValue(
|
||||
value: NamespacedRootKeyInKeychain,
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<void> {
|
||||
let keychain = await this.getRawKeychainValue()
|
||||
|
||||
if (!keychain) {
|
||||
keychain = {}
|
||||
}
|
||||
|
||||
await Keychain.setKeys({
|
||||
...keychain,
|
||||
[identifier]: value,
|
||||
})
|
||||
}
|
||||
|
||||
async clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<void> {
|
||||
const keychain = await this.getRawKeychainValue()
|
||||
|
||||
if (!keychain) {
|
||||
return
|
||||
}
|
||||
|
||||
delete keychain[identifier]
|
||||
await Keychain.setKeys(keychain)
|
||||
}
|
||||
|
||||
async getRawKeychainValue(): Promise<RawKeychainValue | undefined> {
|
||||
const result = await Keychain.getKeys()
|
||||
|
||||
@@ -641,4 +500,14 @@ export class MobileDevice implements MobileDeviceInterface {
|
||||
async getColorScheme(): Promise<ColorSchemeName> {
|
||||
return Appearance.getColorScheme()
|
||||
}
|
||||
|
||||
hideMobileInterfaceFromScreenshots(): void {
|
||||
hide()
|
||||
this.setAndroidScreenshotPrivacy(true)
|
||||
}
|
||||
|
||||
stopHidingMobileInterfaceFromScreenshots(): void {
|
||||
show()
|
||||
this.setAndroidScreenshotPrivacy(false)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import { OnShouldStartLoadWithRequest } from 'react-native-webview/lib/WebViewTy
|
||||
import { AndroidBackHandlerService } from './AndroidBackHandlerService'
|
||||
import { AppStateObserverService } from './AppStateObserverService'
|
||||
import { ColorSchemeObserverService } from './ColorSchemeObserverService'
|
||||
import { MobileDevice, MobileDeviceEvent } from './Lib/Interface'
|
||||
import { MobileDevice, MobileDeviceEvent } from './Lib/MobileDevice'
|
||||
import { IsDev } from './Lib/Utils'
|
||||
|
||||
const LoggingEnabled = IsDev
|
||||
@@ -177,6 +177,10 @@ const MobileWebAppContents = ({ destroyAndReload }: { destroyAndReload: () => vo
|
||||
window.ReactNativeWebView.postMessage('[web log] ' + args.join(' '));
|
||||
}
|
||||
|
||||
console.error = (...args) => {
|
||||
window.ReactNativeWebView.postMessage('[web log] ' + args.join(' '));
|
||||
}
|
||||
|
||||
${WebProcessDeviceInterface}
|
||||
${WebProcessMessageSender}
|
||||
|
||||
|
||||
@@ -10,22 +10,3 @@ export interface NamespacedRootKeyInKeychain {
|
||||
}
|
||||
|
||||
export type RootKeyContentInStorage = RootKeyContentSpecialized
|
||||
|
||||
export interface LegacyRawKeychainValue {
|
||||
mk: string
|
||||
ak: string
|
||||
version: ProtocolVersion
|
||||
}
|
||||
|
||||
export type LegacyMobileKeychainStructure = {
|
||||
offline?: {
|
||||
timing?: unknown
|
||||
pw?: string
|
||||
}
|
||||
encryptedAccountKeys?: unknown
|
||||
mk: string
|
||||
pw: string
|
||||
ak: string
|
||||
version?: string
|
||||
jwt?: string
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { ApplicationIdentifier, ContentType } from '@standardnotes/common'
|
||||
import { BackupFile, DecryptedItemInterface, ItemStream, Platform, PrefKey, PrefValue } from '@standardnotes/models'
|
||||
import { FilesClientInterface } from '@standardnotes/files'
|
||||
|
||||
import { AlertService } from '../Alert/AlertService'
|
||||
import { ComponentManagerInterface } from '../Component/ComponentManagerInterface'
|
||||
import { ApplicationEvent } from '../Event/ApplicationEvent'
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
import { TransferPayload } from '@standardnotes/models'
|
||||
|
||||
export type DatabaseItemMetadata = Pick<TransferPayload, 'uuid' | 'updated_at' | 'content_type'>
|
||||
44
packages/services/src/Domain/Device/DatabaseLoadOptions.ts
Normal file
44
packages/services/src/Domain/Device/DatabaseLoadOptions.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { ContentType } from '@standardnotes/common'
|
||||
import { FullyFormedTransferPayload } from '@standardnotes/models'
|
||||
|
||||
export type DatabaseKeysLoadChunk = {
|
||||
keys: string[]
|
||||
}
|
||||
|
||||
export type DatabaseFullEntryLoadChunk = {
|
||||
entries: FullyFormedTransferPayload[]
|
||||
}
|
||||
|
||||
export function isChunkFullEntry(
|
||||
x: DatabaseKeysLoadChunk | DatabaseFullEntryLoadChunk,
|
||||
): x is DatabaseFullEntryLoadChunk {
|
||||
return (x as DatabaseFullEntryLoadChunk).entries !== undefined
|
||||
}
|
||||
|
||||
export type DatabaseKeysLoadChunkResponse = {
|
||||
keys: {
|
||||
itemsKeys: DatabaseKeysLoadChunk
|
||||
remainingChunks: DatabaseKeysLoadChunk[]
|
||||
}
|
||||
remainingChunksItemCount: number
|
||||
}
|
||||
|
||||
export type DatabaseFullEntryLoadChunkResponse = {
|
||||
fullEntries: {
|
||||
itemsKeys: DatabaseFullEntryLoadChunk
|
||||
remainingChunks: DatabaseFullEntryLoadChunk[]
|
||||
}
|
||||
remainingChunksItemCount: number
|
||||
}
|
||||
|
||||
export function isFullEntryLoadChunkResponse(
|
||||
x: DatabaseKeysLoadChunkResponse | DatabaseFullEntryLoadChunkResponse,
|
||||
): x is DatabaseFullEntryLoadChunkResponse {
|
||||
return (x as DatabaseFullEntryLoadChunkResponse).fullEntries !== undefined
|
||||
}
|
||||
|
||||
export type DatabaseLoadOptions = {
|
||||
contentTypePriority: ContentType[]
|
||||
uuidPriority: string[]
|
||||
batchSize: number
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { ContentType } from '@standardnotes/common'
|
||||
import { FullyFormedPayloadInterface } from '@standardnotes/models'
|
||||
import { GetSortedPayloadsByPriority } from './Utils'
|
||||
import { GetSortedPayloadsByPriority } from './DatabaseLoadSorter'
|
||||
|
||||
describe('GetSortedPayloadsByPriority', () => {
|
||||
let payloads: FullyFormedPayloadInterface[] = []
|
||||
@@ -26,11 +26,11 @@ describe('GetSortedPayloadsByPriority', () => {
|
||||
} as FullyFormedPayloadInterface,
|
||||
]
|
||||
|
||||
const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(
|
||||
payloads,
|
||||
const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(payloads, {
|
||||
contentTypePriority,
|
||||
launchPriorityUuids,
|
||||
)
|
||||
uuidPriority: launchPriorityUuids,
|
||||
batchSize: 1000,
|
||||
})
|
||||
|
||||
expect(itemsKeyPayloads.length).toBe(1)
|
||||
expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey)
|
||||
@@ -84,11 +84,11 @@ describe('GetSortedPayloadsByPriority', () => {
|
||||
|
||||
launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid]
|
||||
|
||||
const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(
|
||||
payloads,
|
||||
const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(payloads, {
|
||||
contentTypePriority,
|
||||
launchPriorityUuids,
|
||||
)
|
||||
uuidPriority: launchPriorityUuids,
|
||||
batchSize: 1000,
|
||||
})
|
||||
|
||||
expect(itemsKeyPayloads.length).toBe(1)
|
||||
expect(itemsKeyPayloads[0].content_type).toBe(ContentType.ItemsKey)
|
||||
@@ -116,12 +116,12 @@ describe('GetSortedPayloadsByPriority', () => {
|
||||
{
|
||||
content_type: ContentType.Note,
|
||||
uuid: unprioritizedNoteUuid,
|
||||
serverUpdatedAt: new Date(1),
|
||||
updated_at: new Date(1),
|
||||
} as FullyFormedPayloadInterface,
|
||||
{
|
||||
content_type: ContentType.Tag,
|
||||
uuid: unprioritizedTagUuid,
|
||||
serverUpdatedAt: new Date(2),
|
||||
updated_at: new Date(2),
|
||||
} as FullyFormedPayloadInterface,
|
||||
{
|
||||
content_type: ContentType.Note,
|
||||
@@ -135,7 +135,11 @@ describe('GetSortedPayloadsByPriority', () => {
|
||||
|
||||
launchPriorityUuids = [prioritizedNoteUuid, prioritizedTagUuid]
|
||||
|
||||
const { remainingPayloads } = GetSortedPayloadsByPriority(payloads, contentTypePriority, launchPriorityUuids)
|
||||
const { remainingPayloads } = GetSortedPayloadsByPriority(payloads, {
|
||||
contentTypePriority,
|
||||
uuidPriority: launchPriorityUuids,
|
||||
batchSize: 1000,
|
||||
})
|
||||
|
||||
expect(remainingPayloads.length).toBe(4)
|
||||
expect(remainingPayloads[0].uuid).toBe(prioritizedNoteUuid)
|
||||
@@ -1,18 +1,18 @@
|
||||
import { UuidString } from '@Lib/Types'
|
||||
import { ContentType } from '@standardnotes/common'
|
||||
import { FullyFormedPayloadInterface } from '@standardnotes/models'
|
||||
import { DatabaseItemMetadata } from './DatabaseItemMetadata'
|
||||
import { DatabaseLoadOptions } from './DatabaseLoadOptions'
|
||||
import { ContentType, Uuid } from '@standardnotes/common'
|
||||
|
||||
/**
|
||||
* Sorts payloads according by most recently modified first, according to the priority,
|
||||
* whereby the earlier a content_type appears in the priorityList,
|
||||
* the earlier it will appear in the resulting sorted array.
|
||||
*/
|
||||
function SortPayloadsByRecentAndContentPriority(
|
||||
payloads: FullyFormedPayloadInterface[],
|
||||
function SortPayloadsByRecentAndContentPriority<T extends DatabaseItemMetadata = DatabaseItemMetadata>(
|
||||
payloads: T[],
|
||||
contentTypePriorityList: ContentType[],
|
||||
): FullyFormedPayloadInterface[] {
|
||||
): T[] {
|
||||
return payloads.sort((a, b) => {
|
||||
const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime()
|
||||
const dateResult = new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
|
||||
|
||||
let aPriority = 0
|
||||
let bPriority = 0
|
||||
@@ -45,12 +45,12 @@ function SortPayloadsByRecentAndContentPriority(
|
||||
* whereby the earlier a uuid appears in the priorityList,
|
||||
* the earlier it will appear in the resulting sorted array.
|
||||
*/
|
||||
function SortPayloadsByRecentAndUuidPriority(
|
||||
payloads: FullyFormedPayloadInterface[],
|
||||
uuidPriorityList: UuidString[],
|
||||
): FullyFormedPayloadInterface[] {
|
||||
function SortPayloadsByRecentAndUuidPriority<T extends DatabaseItemMetadata = DatabaseItemMetadata>(
|
||||
payloads: T[],
|
||||
uuidPriorityList: Uuid[],
|
||||
): T[] {
|
||||
return payloads.sort((a, b) => {
|
||||
const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime()
|
||||
const dateResult = new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
|
||||
|
||||
let aPriority = 0
|
||||
let bPriority = 0
|
||||
@@ -78,25 +78,24 @@ function SortPayloadsByRecentAndUuidPriority(
|
||||
})
|
||||
}
|
||||
|
||||
export function GetSortedPayloadsByPriority(
|
||||
payloads: FullyFormedPayloadInterface[],
|
||||
contentTypePriorityList: ContentType[],
|
||||
uuidPriorityList: UuidString[],
|
||||
export function GetSortedPayloadsByPriority<T extends DatabaseItemMetadata = DatabaseItemMetadata>(
|
||||
payloads: T[],
|
||||
options: DatabaseLoadOptions,
|
||||
): {
|
||||
itemsKeyPayloads: FullyFormedPayloadInterface[]
|
||||
contentTypePriorityPayloads: FullyFormedPayloadInterface[]
|
||||
remainingPayloads: FullyFormedPayloadInterface[]
|
||||
itemsKeyPayloads: T[]
|
||||
contentTypePriorityPayloads: T[]
|
||||
remainingPayloads: T[]
|
||||
} {
|
||||
const itemsKeyPayloads: FullyFormedPayloadInterface[] = []
|
||||
const contentTypePriorityPayloads: FullyFormedPayloadInterface[] = []
|
||||
const remainingPayloads: FullyFormedPayloadInterface[] = []
|
||||
const itemsKeyPayloads: T[] = []
|
||||
const contentTypePriorityPayloads: T[] = []
|
||||
const remainingPayloads: T[] = []
|
||||
|
||||
for (let index = 0; index < payloads.length; index++) {
|
||||
const payload = payloads[index]
|
||||
|
||||
if (payload.content_type === ContentType.ItemsKey) {
|
||||
itemsKeyPayloads.push(payload)
|
||||
} else if (contentTypePriorityList.includes(payload.content_type)) {
|
||||
} else if (options.contentTypePriority.includes(payload.content_type)) {
|
||||
contentTypePriorityPayloads.push(payload)
|
||||
} else {
|
||||
remainingPayloads.push(payload)
|
||||
@@ -107,8 +106,8 @@ export function GetSortedPayloadsByPriority(
|
||||
itemsKeyPayloads,
|
||||
contentTypePriorityPayloads: SortPayloadsByRecentAndContentPriority(
|
||||
contentTypePriorityPayloads,
|
||||
contentTypePriorityList,
|
||||
options.contentTypePriority,
|
||||
),
|
||||
remainingPayloads: SortPayloadsByRecentAndUuidPriority(remainingPayloads, uuidPriorityList),
|
||||
remainingPayloads: SortPayloadsByRecentAndUuidPriority(remainingPayloads, options.uuidPriority),
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,14 @@ import { ApplicationIdentifier } from '@standardnotes/common'
|
||||
import {
|
||||
FullyFormedTransferPayload,
|
||||
TransferPayload,
|
||||
LegacyRawKeychainValue,
|
||||
NamespacedRootKeyInKeychain,
|
||||
Environment,
|
||||
} from '@standardnotes/models'
|
||||
import {
|
||||
DatabaseLoadOptions,
|
||||
DatabaseKeysLoadChunkResponse,
|
||||
DatabaseFullEntryLoadChunkResponse,
|
||||
} from './DatabaseLoadOptions'
|
||||
|
||||
/**
|
||||
* Platforms must override this class to provide platform specific utilities
|
||||
@@ -21,8 +25,6 @@ export interface DeviceInterface {
|
||||
|
||||
getJsonParsedRawStorageValue(key: string): Promise<unknown | undefined>
|
||||
|
||||
getAllRawStorageKeyValues(): Promise<{ key: string; value: unknown }[]>
|
||||
|
||||
setRawStorageValue(key: string, value: string): Promise<void>
|
||||
|
||||
removeRawStorageValue(key: string): Promise<void>
|
||||
@@ -38,10 +40,10 @@ export interface DeviceInterface {
|
||||
*/
|
||||
openDatabase(identifier: ApplicationIdentifier): Promise<{ isNewDatabase?: boolean } | undefined>
|
||||
|
||||
/**
|
||||
* In a key/value database, this function returns just the keys.
|
||||
*/
|
||||
getDatabaseKeys(): Promise<string[]>
|
||||
getDatabaseLoadChunks(
|
||||
options: DatabaseLoadOptions,
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<DatabaseKeysLoadChunkResponse | DatabaseFullEntryLoadChunkResponse>
|
||||
|
||||
/**
|
||||
* Remove all keychain and database data from device.
|
||||
@@ -52,17 +54,22 @@ export interface DeviceInterface {
|
||||
*/
|
||||
clearAllDataFromDevice(workspaceIdentifiers: ApplicationIdentifier[]): Promise<{ killsApplication: boolean }>
|
||||
|
||||
getAllRawDatabasePayloads<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
|
||||
getAllDatabaseEntries<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
|
||||
identifier: ApplicationIdentifier,
|
||||
): Promise<T[]>
|
||||
|
||||
saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void>
|
||||
getDatabaseEntries<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
|
||||
identifier: ApplicationIdentifier,
|
||||
keys: string[],
|
||||
): Promise<T[]>
|
||||
|
||||
saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void>
|
||||
saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier): Promise<void>
|
||||
|
||||
removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier): Promise<void>
|
||||
saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier): Promise<void>
|
||||
|
||||
removeAllRawDatabasePayloads(identifier: ApplicationIdentifier): Promise<void>
|
||||
removeDatabaseEntry(id: string, identifier: ApplicationIdentifier): Promise<void>
|
||||
|
||||
removeAllDatabaseEntries(identifier: ApplicationIdentifier): Promise<void>
|
||||
|
||||
getNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<NamespacedRootKeyInKeychain | undefined>
|
||||
|
||||
@@ -70,8 +77,6 @@ export interface DeviceInterface {
|
||||
|
||||
clearNamespacedKeychainValue(identifier: ApplicationIdentifier): Promise<void>
|
||||
|
||||
setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise<void>
|
||||
|
||||
clearRawKeychainValue(): Promise<void>
|
||||
|
||||
openUrl(url: string): void
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import { FullyFormedPayloadInterface, PayloadInterface, RootKeyInterface } from '@standardnotes/models'
|
||||
import {
|
||||
FullyFormedPayloadInterface,
|
||||
PayloadInterface,
|
||||
RootKeyInterface,
|
||||
FullyFormedTransferPayload,
|
||||
} from '@standardnotes/models'
|
||||
import { StoragePersistencePolicies, StorageValueModes } from './StorageTypes'
|
||||
|
||||
export interface StorageServiceInterface {
|
||||
getAllRawPayloads(): Promise<FullyFormedTransferPayload[]>
|
||||
getValue<T>(key: string, mode?: StorageValueModes, defaultValue?: T): T
|
||||
canDecryptWithKey(key: RootKeyInterface): Promise<boolean>
|
||||
savePayload(payload: PayloadInterface): Promise<void>
|
||||
|
||||
@@ -11,6 +11,7 @@ export type SyncOptions = {
|
||||
checkIntegrity?: boolean
|
||||
/** Internally used to keep track of how sync requests were spawned. */
|
||||
source: SyncSource
|
||||
sourceDescription?: string
|
||||
/** Whether to await any sync requests that may be queued from this call. */
|
||||
awaitAll?: boolean
|
||||
/**
|
||||
|
||||
@@ -22,6 +22,9 @@ export * from './Device/DeviceInterface'
|
||||
export * from './Device/MobileDeviceInterface'
|
||||
export * from './Device/TypeCheck'
|
||||
export * from './Device/WebOrDesktopDeviceInterface'
|
||||
export * from './Device/DatabaseLoadOptions'
|
||||
export * from './Device/DatabaseItemMetadata'
|
||||
export * from './Device/DatabaseLoadSorter'
|
||||
export * from './Diagnostics/ServiceDiagnostics'
|
||||
export * from './Encryption/BackupFileDecryptor'
|
||||
export * from './Encryption/EncryptionService'
|
||||
|
||||
@@ -25,7 +25,7 @@ describe('application', () => {
|
||||
|
||||
device = {} as jest.Mocked<DeviceInterface>
|
||||
device.openDatabase = jest.fn().mockResolvedValue(true)
|
||||
device.getAllRawDatabasePayloads = jest.fn().mockReturnValue([])
|
||||
device.getAllDatabaseEntries = jest.fn().mockReturnValue([])
|
||||
device.setRawStorageValue = jest.fn()
|
||||
device.getRawStorageValue = jest.fn().mockImplementation((key) => {
|
||||
if (key === namespacedKey(identifier, RawStorageKey.SnjsVersion)) {
|
||||
@@ -33,9 +33,6 @@ describe('application', () => {
|
||||
}
|
||||
return undefined
|
||||
})
|
||||
device.getDatabaseKeys = async () => {
|
||||
return Promise.resolve(['1', '2', '3'])
|
||||
}
|
||||
|
||||
application = new SNApplication({
|
||||
environment: Environment.Mobile,
|
||||
@@ -75,7 +72,6 @@ describe('application', () => {
|
||||
currentPersistPromise: false,
|
||||
isStorageWrapped: false,
|
||||
allRawPayloadsCount: 0,
|
||||
databaseKeys: ['1', '2', '3'],
|
||||
},
|
||||
encryption: expect.objectContaining({
|
||||
getLatestVersion: '004',
|
||||
|
||||
@@ -410,28 +410,32 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli
|
||||
await this.notifyEvent(ApplicationEvent.Launched)
|
||||
await this.handleStage(ExternalServices.ApplicationStage.Launched_10)
|
||||
|
||||
const databasePayloads = await this.syncService.getDatabasePayloads()
|
||||
await this.handleStage(ExternalServices.ApplicationStage.LoadingDatabase_11)
|
||||
|
||||
if (this.createdNewDatabase) {
|
||||
await this.syncService.onNewDatabaseCreated()
|
||||
}
|
||||
/**
|
||||
* We don't want to await this, as we want to begin allowing the app to function
|
||||
* before local data has been loaded fully. We await only initial
|
||||
* `getDatabasePayloads` to lock in on database state.
|
||||
* before local data has been loaded fully.
|
||||
*/
|
||||
const loadPromise = this.syncService.loadDatabasePayloads(databasePayloads).then(async () => {
|
||||
if (this.dealloced) {
|
||||
throw 'Application has been destroyed.'
|
||||
}
|
||||
await this.handleStage(ExternalServices.ApplicationStage.LoadedDatabase_12)
|
||||
this.beginAutoSyncTimer()
|
||||
await this.syncService.sync({
|
||||
mode: ExternalServices.SyncMode.DownloadFirst,
|
||||
source: ExternalServices.SyncSource.External,
|
||||
const loadPromise = this.syncService
|
||||
.loadDatabasePayloads()
|
||||
.then(async () => {
|
||||
if (this.dealloced) {
|
||||
throw 'Application has been destroyed.'
|
||||
}
|
||||
await this.handleStage(ExternalServices.ApplicationStage.LoadedDatabase_12)
|
||||
this.beginAutoSyncTimer()
|
||||
await this.syncService.sync({
|
||||
mode: ExternalServices.SyncMode.DownloadFirst,
|
||||
source: ExternalServices.SyncSource.External,
|
||||
sourceDescription: 'Application Launch',
|
||||
})
|
||||
})
|
||||
.catch((error) => {
|
||||
void this.notifyEvent(ApplicationEvent.LocalDatabaseReadError, error)
|
||||
throw error
|
||||
})
|
||||
})
|
||||
if (awaitDatabaseLoad) {
|
||||
await loadPromise
|
||||
}
|
||||
@@ -463,7 +467,7 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli
|
||||
private beginAutoSyncTimer() {
|
||||
this.autoSyncInterval = setInterval(() => {
|
||||
this.syncService.log('Syncing from autosync')
|
||||
void this.sync.sync()
|
||||
void this.sync.sync({ sourceDescription: 'Auto Sync' })
|
||||
}, DEFAULT_AUTO_SYNC_INTERVAL)
|
||||
}
|
||||
|
||||
@@ -1542,10 +1546,10 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli
|
||||
switch (event) {
|
||||
case InternalServices.SessionEvent.Restored: {
|
||||
void (async () => {
|
||||
await this.sync.sync()
|
||||
await this.sync.sync({ sourceDescription: 'Session restored pre key creation' })
|
||||
if (this.protocolService.needsNewRootKeyBasedItemsKey()) {
|
||||
void this.protocolService.createNewDefaultItemsKey().then(() => {
|
||||
void this.sync.sync()
|
||||
void this.sync.sync({ sourceDescription: 'Session restored post key creation' })
|
||||
})
|
||||
}
|
||||
})()
|
||||
@@ -1573,6 +1577,8 @@ export class SNApplication implements ApplicationInterface, AppGroupManagedAppli
|
||||
this.payloadManager,
|
||||
this.apiService,
|
||||
this.historyManager,
|
||||
this.deviceInterface,
|
||||
this.identifier,
|
||||
{
|
||||
loadBatchSize: this.options.loadBatchSize,
|
||||
},
|
||||
|
||||
@@ -2,3 +2,4 @@ export * from './Application'
|
||||
export * from './Event'
|
||||
export * from './LiveItem'
|
||||
export * from './Platforms'
|
||||
export * from './Options/Defaults'
|
||||
|
||||
22
packages/snjs/lib/Logging.ts
Normal file
22
packages/snjs/lib/Logging.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { log as utilsLog } from '@standardnotes/utils'
|
||||
|
||||
export const isDev = true
|
||||
|
||||
export enum LoggingDomain {
|
||||
DatabaseLoad,
|
||||
Sync,
|
||||
}
|
||||
|
||||
const LoggingStatus: Record<LoggingDomain, boolean> = {
|
||||
[LoggingDomain.DatabaseLoad]: false,
|
||||
[LoggingDomain.Sync]: false,
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export function log(domain: LoggingDomain, ...args: any[]): void {
|
||||
if (!isDev || !LoggingStatus[domain]) {
|
||||
return
|
||||
}
|
||||
|
||||
utilsLog(LoggingDomain[domain], ...args)
|
||||
}
|
||||
@@ -165,12 +165,11 @@ export class BaseMigration extends Migration {
|
||||
}
|
||||
|
||||
private async repairMissingKeychain() {
|
||||
const version = (await this.getStoredVersion()) as string
|
||||
const rawAccountParams = await this.reader.getAccountKeyParams()
|
||||
|
||||
/** Choose an item to decrypt against */
|
||||
const allItems = (
|
||||
await this.services.deviceInterface.getAllRawDatabasePayloads<EncryptedTransferPayload>(this.services.identifier)
|
||||
await this.services.deviceInterface.getAllDatabaseEntries<EncryptedTransferPayload>(this.services.identifier)
|
||||
).map((p) => new EncryptedPayload(p))
|
||||
|
||||
let itemToDecrypt = allItems.find((item) => {
|
||||
@@ -226,21 +225,10 @@ export class BaseMigration extends Migration {
|
||||
)
|
||||
} else {
|
||||
/**
|
||||
* If decryption succeeds, store the generated account key where it is expected,
|
||||
* either in top-level keychain in 1.0.0, and namespaced location in 2.0.0+.
|
||||
* If decryption succeeds, store the generated account key where it is expected.
|
||||
*/
|
||||
if (version === PreviousSnjsVersion1_0_0) {
|
||||
/** Store in top level keychain */
|
||||
await this.services.deviceInterface.setLegacyRawKeychainValue({
|
||||
mk: rootKey.masterKey,
|
||||
ak: rootKey.dataAuthenticationKey as string,
|
||||
version: accountParams.version,
|
||||
})
|
||||
} else {
|
||||
/** Store in namespaced location */
|
||||
const rawKey = rootKey.getKeychainValue()
|
||||
await this.services.deviceInterface.setNamespacedKeychainValue(rawKey, this.services.identifier)
|
||||
}
|
||||
const rawKey = rootKey.getKeychainValue()
|
||||
await this.services.deviceInterface.setNamespacedKeychainValue(rawKey, this.services.identifier)
|
||||
resolve(true)
|
||||
this.services.challengeService.completeChallenge(challenge)
|
||||
}
|
||||
|
||||
@@ -5,9 +5,7 @@ import { DeviceInterface } from '@standardnotes/services'
|
||||
import { StorageReader } from './Reader'
|
||||
import * as ReaderClasses from './Versions'
|
||||
|
||||
function ReaderClassForVersion(
|
||||
version: string,
|
||||
): typeof ReaderClasses.StorageReader2_0_0 | typeof ReaderClasses.StorageReader1_0_0 {
|
||||
function ReaderClassForVersion(version: string): typeof ReaderClasses.StorageReader2_0_0 {
|
||||
/** Sort readers by newest first */
|
||||
const allReaders = Object.values(ReaderClasses).sort((a, b) => {
|
||||
return compareSemVersions(a.version(), b.version()) * -1
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import { isNullOrUndefined } from '@standardnotes/utils'
|
||||
import { isEnvironmentMobile } from '@Lib/Application/Platforms'
|
||||
import { PreviousSnjsVersion1_0_0 } from '../../../Version'
|
||||
import { isMobileDevice, LegacyKeys1_0_0 } from '@standardnotes/services'
|
||||
import { StorageReader } from '../Reader'
|
||||
|
||||
export class StorageReader1_0_0 extends StorageReader {
|
||||
static override version() {
|
||||
return PreviousSnjsVersion1_0_0
|
||||
}
|
||||
|
||||
public async getAccountKeyParams() {
|
||||
return this.deviceInterface.getJsonParsedRawStorageValue(LegacyKeys1_0_0.AllAccountKeyParamsKey)
|
||||
}
|
||||
|
||||
/**
|
||||
* In 1.0.0, web uses raw storage for unwrapped account key, and mobile uses
|
||||
* the keychain
|
||||
*/
|
||||
public async hasNonWrappedAccountKeys() {
|
||||
if (isMobileDevice(this.deviceInterface)) {
|
||||
const value = await this.deviceInterface.getRawKeychainValue()
|
||||
return !isNullOrUndefined(value)
|
||||
} else {
|
||||
const value = await this.deviceInterface.getRawStorageValue('mk')
|
||||
return !isNullOrUndefined(value)
|
||||
}
|
||||
}
|
||||
|
||||
public async hasPasscode() {
|
||||
if (isEnvironmentMobile(this.environment)) {
|
||||
const rawPasscodeParams = await this.deviceInterface.getJsonParsedRawStorageValue(
|
||||
LegacyKeys1_0_0.MobilePasscodeParamsKey,
|
||||
)
|
||||
return !isNullOrUndefined(rawPasscodeParams)
|
||||
} else {
|
||||
const encryptedStorage = await this.deviceInterface.getJsonParsedRawStorageValue(
|
||||
LegacyKeys1_0_0.WebEncryptedStorageKey,
|
||||
)
|
||||
return !isNullOrUndefined(encryptedStorage)
|
||||
}
|
||||
}
|
||||
|
||||
/** Keychain was not used on desktop/web in 1.0.0 */
|
||||
public usesKeychain() {
|
||||
return isEnvironmentMobile(this.environment) ? true : false
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1 @@
|
||||
export { StorageReader2_0_0 } from './Reader2_0_0'
|
||||
export { StorageReader1_0_0 } from './Reader1_0_0'
|
||||
|
||||
@@ -1,730 +0,0 @@
|
||||
import { AnyKeyParamsContent, ContentType, ProtocolVersion } from '@standardnotes/common'
|
||||
import { Migration } from '@Lib/Migrations/Migration'
|
||||
import { MigrationServices } from '../MigrationServices'
|
||||
import { PreviousSnjsVersion2_0_0 } from '../../Version'
|
||||
import { SNRootKey, CreateNewRootKey } from '@standardnotes/encryption'
|
||||
import { DiskStorageService } from '../../Services/Storage/DiskStorageService'
|
||||
import { StorageReader1_0_0 } from '../StorageReaders/Versions/Reader1_0_0'
|
||||
import * as Models from '@standardnotes/models'
|
||||
import * as Services from '@standardnotes/services'
|
||||
import * as Utils from '@standardnotes/utils'
|
||||
import { isEnvironmentMobile, isEnvironmentWebOrDesktop } from '@Lib/Application/Platforms'
|
||||
import {
|
||||
getIncrementedDirtyIndex,
|
||||
LegacyMobileKeychainStructure,
|
||||
PayloadTimestampDefaults,
|
||||
} from '@standardnotes/models'
|
||||
import { isMobileDevice } from '@standardnotes/services'
|
||||
import { LegacySession } from '@standardnotes/domain-core'
|
||||
|
||||
interface LegacyStorageContent extends Models.ItemContent {
|
||||
storage: unknown
|
||||
}
|
||||
|
||||
interface LegacyAccountKeysValue {
|
||||
ak: string
|
||||
mk: string
|
||||
version: string
|
||||
jwt: string
|
||||
}
|
||||
|
||||
interface LegacyRootKeyContent extends Models.RootKeyContent {
|
||||
accountKeys?: LegacyAccountKeysValue
|
||||
}
|
||||
|
||||
const LEGACY_SESSION_TOKEN_KEY = 'jwt'
|
||||
|
||||
export class Migration2_0_0 extends Migration {
|
||||
private legacyReader!: StorageReader1_0_0
|
||||
|
||||
constructor(services: MigrationServices) {
|
||||
super(services)
|
||||
this.legacyReader = new StorageReader1_0_0(
|
||||
this.services.deviceInterface,
|
||||
this.services.identifier,
|
||||
this.services.environment,
|
||||
)
|
||||
}
|
||||
|
||||
static override version() {
|
||||
return PreviousSnjsVersion2_0_0
|
||||
}
|
||||
|
||||
protected registerStageHandlers() {
|
||||
this.registerStageHandler(Services.ApplicationStage.PreparingForLaunch_0, async () => {
|
||||
if (isEnvironmentWebOrDesktop(this.services.environment)) {
|
||||
await this.migrateStorageStructureForWebDesktop()
|
||||
} else if (isEnvironmentMobile(this.services.environment)) {
|
||||
await this.migrateStorageStructureForMobile()
|
||||
}
|
||||
})
|
||||
this.registerStageHandler(Services.ApplicationStage.StorageDecrypted_09, async () => {
|
||||
await this.migrateArbitraryRawStorageToManagedStorageAllPlatforms()
|
||||
if (isEnvironmentMobile(this.services.environment)) {
|
||||
await this.migrateMobilePreferences()
|
||||
}
|
||||
await this.migrateSessionStorage()
|
||||
await this.deleteLegacyStorageValues()
|
||||
})
|
||||
this.registerStageHandler(Services.ApplicationStage.LoadingDatabase_11, async () => {
|
||||
await this.createDefaultItemsKeyForAllPlatforms()
|
||||
this.markDone()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Web
|
||||
* Migrates legacy storage structure into new managed format.
|
||||
* If encrypted storage exists, we need to first decrypt it with the passcode.
|
||||
* Then extract the account key from it. Then, encrypt storage with the
|
||||
* account key. Then encrypt the account key with the passcode and store it
|
||||
* within the new storage format.
|
||||
*
|
||||
* Generate note: We do not use the keychain if passcode is available.
|
||||
*/
|
||||
private async migrateStorageStructureForWebDesktop() {
|
||||
const deviceInterface = this.services.deviceInterface
|
||||
const newStorageRawStructure: Services.StorageValuesObject = {
|
||||
[Services.ValueModesKeys.Wrapped]: {} as Models.LocalStorageEncryptedContextualPayload,
|
||||
[Services.ValueModesKeys.Unwrapped]: {},
|
||||
[Services.ValueModesKeys.Nonwrapped]: {},
|
||||
}
|
||||
const rawAccountKeyParams = (await this.legacyReader.getAccountKeyParams()) as AnyKeyParamsContent
|
||||
/** Could be null if no account, or if account and storage is encrypted */
|
||||
if (rawAccountKeyParams) {
|
||||
newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyParams] = rawAccountKeyParams
|
||||
}
|
||||
const encryptedStorage = (await deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.WebEncryptedStorageKey,
|
||||
)) as Models.EncryptedTransferPayload
|
||||
|
||||
if (encryptedStorage) {
|
||||
const encryptedStoragePayload = new Models.EncryptedPayload(encryptedStorage)
|
||||
|
||||
const passcodeResult = await this.webDesktopHelperGetPasscodeKeyAndDecryptEncryptedStorage(
|
||||
encryptedStoragePayload,
|
||||
)
|
||||
|
||||
const passcodeKey = passcodeResult.key
|
||||
const decryptedStoragePayload = passcodeResult.decryptedStoragePayload
|
||||
const passcodeParams = passcodeResult.keyParams
|
||||
|
||||
newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyWrapperKeyParams] = passcodeParams.getPortableValue()
|
||||
|
||||
const rawStorageValueStore = Utils.Copy(decryptedStoragePayload.content.storage)
|
||||
const storageValueStore: Record<string, unknown> = Utils.jsonParseEmbeddedKeys(rawStorageValueStore)
|
||||
/** Store previously encrypted auth_params into new nonwrapped value key */
|
||||
|
||||
const accountKeyParams = storageValueStore[Services.LegacyKeys1_0_0.AllAccountKeyParamsKey] as AnyKeyParamsContent
|
||||
newStorageRawStructure.nonwrapped[Services.StorageKey.RootKeyParams] = accountKeyParams
|
||||
|
||||
let keyToEncryptStorageWith = passcodeKey
|
||||
/** Extract account key (mk, pw, ak) if it exists */
|
||||
const hasAccountKeys = !Utils.isNullOrUndefined(storageValueStore.mk)
|
||||
|
||||
if (hasAccountKeys) {
|
||||
const { accountKey, wrappedKey } = await this.webDesktopHelperExtractAndWrapAccountKeysFromValueStore(
|
||||
passcodeKey,
|
||||
accountKeyParams,
|
||||
storageValueStore,
|
||||
)
|
||||
keyToEncryptStorageWith = accountKey
|
||||
newStorageRawStructure.nonwrapped[Services.StorageKey.WrappedRootKey] = wrappedKey
|
||||
}
|
||||
|
||||
/** Encrypt storage with proper key */
|
||||
newStorageRawStructure.wrapped = await this.webDesktopHelperEncryptStorage(
|
||||
keyToEncryptStorageWith,
|
||||
decryptedStoragePayload,
|
||||
storageValueStore,
|
||||
)
|
||||
} else {
|
||||
/**
|
||||
* No encrypted storage, take account keys (if they exist) out of raw storage
|
||||
* and place them in the keychain. */
|
||||
const ak = await this.services.deviceInterface.getRawStorageValue('ak')
|
||||
const mk = await this.services.deviceInterface.getRawStorageValue('mk')
|
||||
|
||||
if (ak || mk) {
|
||||
const version = rawAccountKeyParams.version || (await this.getFallbackRootKeyVersion())
|
||||
|
||||
const accountKey = CreateNewRootKey({
|
||||
masterKey: mk as string,
|
||||
dataAuthenticationKey: ak as string,
|
||||
version: version,
|
||||
keyParams: rawAccountKeyParams,
|
||||
})
|
||||
await this.services.deviceInterface.setNamespacedKeychainValue(
|
||||
accountKey.getKeychainValue(),
|
||||
this.services.identifier,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Persist storage under new key and structure */
|
||||
await this.allPlatformHelperSetStorageStructure(newStorageRawStructure)
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper
|
||||
* All platforms
|
||||
*/
|
||||
private async allPlatformHelperSetStorageStructure(rawStructure: Services.StorageValuesObject) {
|
||||
const newStructure = DiskStorageService.DefaultValuesObject(
|
||||
rawStructure.wrapped,
|
||||
rawStructure.unwrapped,
|
||||
rawStructure.nonwrapped,
|
||||
) as Partial<Services.StorageValuesObject>
|
||||
|
||||
newStructure[Services.ValueModesKeys.Unwrapped] = undefined
|
||||
|
||||
await this.services.deviceInterface.setRawStorageValue(
|
||||
Services.namespacedKey(this.services.identifier, Services.RawStorageKey.StorageObject),
|
||||
JSON.stringify(newStructure),
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper
|
||||
* Web/desktop only
|
||||
*/
|
||||
private async webDesktopHelperGetPasscodeKeyAndDecryptEncryptedStorage(
|
||||
encryptedPayload: Models.EncryptedPayloadInterface,
|
||||
) {
|
||||
const rawPasscodeParams = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.WebPasscodeParamsKey,
|
||||
)) as AnyKeyParamsContent
|
||||
const passcodeParams = this.services.protocolService.createKeyParams(rawPasscodeParams)
|
||||
|
||||
/** Decrypt it with the passcode */
|
||||
let decryptedStoragePayload:
|
||||
| Models.DecryptedPayloadInterface<LegacyStorageContent>
|
||||
| Models.EncryptedPayloadInterface = encryptedPayload
|
||||
let passcodeKey: SNRootKey | undefined
|
||||
|
||||
await this.promptForPasscodeUntilCorrect(async (candidate: string) => {
|
||||
passcodeKey = await this.services.protocolService.computeRootKey(candidate, passcodeParams)
|
||||
decryptedStoragePayload = await this.services.protocolService.decryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [encryptedPayload],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
|
||||
return !Models.isErrorDecryptingPayload(decryptedStoragePayload)
|
||||
})
|
||||
|
||||
return {
|
||||
decryptedStoragePayload:
|
||||
decryptedStoragePayload as unknown as Models.DecryptedPayloadInterface<LegacyStorageContent>,
|
||||
key: passcodeKey as SNRootKey,
|
||||
keyParams: passcodeParams,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper
|
||||
* Web/desktop only
|
||||
*/
|
||||
private async webDesktopHelperExtractAndWrapAccountKeysFromValueStore(
|
||||
passcodeKey: SNRootKey,
|
||||
accountKeyParams: AnyKeyParamsContent,
|
||||
storageValueStore: Record<string, unknown>,
|
||||
) {
|
||||
const version = accountKeyParams?.version || (await this.getFallbackRootKeyVersion())
|
||||
const accountKey = CreateNewRootKey({
|
||||
masterKey: storageValueStore.mk as string,
|
||||
dataAuthenticationKey: storageValueStore.ak as string,
|
||||
version: version,
|
||||
keyParams: accountKeyParams,
|
||||
})
|
||||
|
||||
delete storageValueStore.mk
|
||||
delete storageValueStore.pw
|
||||
delete storageValueStore.ak
|
||||
|
||||
const accountKeyPayload = accountKey.payload
|
||||
|
||||
/** Encrypt account key with passcode */
|
||||
const encryptedAccountKey = await this.services.protocolService.encryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [accountKeyPayload],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
return {
|
||||
accountKey: accountKey,
|
||||
wrappedKey: Models.CreateEncryptedLocalStorageContextPayload(encryptedAccountKey),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper
|
||||
* Web/desktop only
|
||||
* Encrypt storage with account key
|
||||
*/
|
||||
async webDesktopHelperEncryptStorage(
|
||||
key: SNRootKey,
|
||||
decryptedStoragePayload: Models.DecryptedPayloadInterface,
|
||||
storageValueStore: Record<string, unknown>,
|
||||
) {
|
||||
const wrapped = await this.services.protocolService.encryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [
|
||||
decryptedStoragePayload.copy({
|
||||
content_type: ContentType.EncryptedStorage,
|
||||
content: storageValueStore as unknown as Models.ItemContent,
|
||||
}),
|
||||
],
|
||||
key: key,
|
||||
},
|
||||
})
|
||||
|
||||
return Models.CreateEncryptedLocalStorageContextPayload(wrapped)
|
||||
}
|
||||
|
||||
/**
|
||||
* Mobile
|
||||
* On mobile legacy structure is mostly similar to new structure,
|
||||
* in that the account key is encrypted with the passcode. But mobile did
|
||||
* not have encrypted storage, so we simply need to transfer all existing
|
||||
* storage values into new managed structure.
|
||||
*
|
||||
* In version <= 3.0.16 on mobile, encrypted account keys were stored in the keychain
|
||||
* under `encryptedAccountKeys`. In 3.0.17 a migration was introduced that moved this value
|
||||
* to storage under key `encrypted_account_keys`. We need to anticipate the keys being in
|
||||
* either location.
|
||||
*
|
||||
* If no account but passcode only, the only thing we stored on mobile
|
||||
* previously was keys.offline.pw and keys.offline.timing in the keychain
|
||||
* that we compared against for valid decryption.
|
||||
* In the new version, we know a passcode is correct if it can decrypt storage.
|
||||
* As part of the migration, we’ll need to request the raw passcode from user,
|
||||
* compare it against the keychain offline.pw value, and if correct,
|
||||
* migrate storage to new structure, and encrypt with passcode key.
|
||||
*
|
||||
* If account only, take the value in the keychain, and rename the values
|
||||
* (i.e mk > masterKey).
|
||||
* @access private
|
||||
*/
|
||||
async migrateStorageStructureForMobile() {
|
||||
Utils.assert(isMobileDevice(this.services.deviceInterface))
|
||||
|
||||
const keychainValue =
|
||||
(await this.services.deviceInterface.getRawKeychainValue()) as unknown as LegacyMobileKeychainStructure
|
||||
|
||||
const wrappedAccountKey = ((await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobileWrappedRootKeyKey,
|
||||
)) || keychainValue?.encryptedAccountKeys) as Models.EncryptedTransferPayload
|
||||
|
||||
const rawAccountKeyParams = (await this.legacyReader.getAccountKeyParams()) as AnyKeyParamsContent
|
||||
|
||||
const rawPasscodeParams = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobilePasscodeParamsKey,
|
||||
)) as AnyKeyParamsContent
|
||||
|
||||
const firstRunValue = await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.NonwrappedStorageKey.MobileFirstRun,
|
||||
)
|
||||
|
||||
const rawStructure: Services.StorageValuesObject = {
|
||||
[Services.ValueModesKeys.Nonwrapped]: {
|
||||
[Services.StorageKey.WrappedRootKey]: wrappedAccountKey,
|
||||
/** A 'hash' key may be present from legacy versions that should be deleted */
|
||||
[Services.StorageKey.RootKeyWrapperKeyParams]: Utils.omitByCopy(rawPasscodeParams, ['hash' as never]),
|
||||
[Services.StorageKey.RootKeyParams]: rawAccountKeyParams,
|
||||
[Services.NonwrappedStorageKey.MobileFirstRun]: firstRunValue,
|
||||
},
|
||||
[Services.ValueModesKeys.Unwrapped]: {},
|
||||
[Services.ValueModesKeys.Wrapped]: {} as Models.LocalStorageDecryptedContextualPayload,
|
||||
}
|
||||
|
||||
const biometricPrefs = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobileBiometricsPrefs,
|
||||
)) as { enabled: boolean; timing: unknown }
|
||||
|
||||
if (biometricPrefs) {
|
||||
rawStructure.nonwrapped[Services.StorageKey.BiometricsState] = biometricPrefs.enabled
|
||||
rawStructure.nonwrapped[Services.StorageKey.MobileBiometricsTiming] = biometricPrefs.timing
|
||||
}
|
||||
|
||||
const passcodeKeyboardType = await this.services.deviceInterface.getRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobilePasscodeKeyboardType,
|
||||
)
|
||||
|
||||
if (passcodeKeyboardType) {
|
||||
rawStructure.nonwrapped[Services.StorageKey.MobilePasscodeKeyboardType] = passcodeKeyboardType
|
||||
}
|
||||
|
||||
if (rawPasscodeParams) {
|
||||
const passcodeParams = this.services.protocolService.createKeyParams(rawPasscodeParams)
|
||||
const getPasscodeKey = async () => {
|
||||
let passcodeKey: SNRootKey | undefined
|
||||
|
||||
await this.promptForPasscodeUntilCorrect(async (candidate: string) => {
|
||||
passcodeKey = await this.services.protocolService.computeRootKey(candidate, passcodeParams)
|
||||
|
||||
const pwHash = keychainValue?.offline?.pw
|
||||
|
||||
if (pwHash) {
|
||||
return passcodeKey.serverPassword === pwHash
|
||||
} else {
|
||||
/**
|
||||
* Fallback decryption if keychain is missing for some reason. If account,
|
||||
* validate by attempting to decrypt wrapped account key. Otherwise, validate
|
||||
* by attempting to decrypt random item.
|
||||
* */
|
||||
if (wrappedAccountKey) {
|
||||
const decryptedAcctKey = await this.services.protocolService.decryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [new Models.EncryptedPayload(wrappedAccountKey)],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
return !Models.isErrorDecryptingPayload(decryptedAcctKey)
|
||||
} else {
|
||||
const item = (
|
||||
await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier)
|
||||
)[0] as Models.EncryptedTransferPayload
|
||||
|
||||
if (!item) {
|
||||
throw Error('Passcode only migration aborting due to missing keychain.offline.pw')
|
||||
}
|
||||
|
||||
const decryptedPayload = await this.services.protocolService.decryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [new Models.EncryptedPayload(item)],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
return !Models.isErrorDecryptingPayload(decryptedPayload)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return passcodeKey as SNRootKey
|
||||
}
|
||||
|
||||
rawStructure.nonwrapped[Services.StorageKey.MobilePasscodeTiming] = keychainValue?.offline?.timing
|
||||
|
||||
if (wrappedAccountKey) {
|
||||
/**
|
||||
* Account key is encrypted with passcode. Inside, the accountKey is located inside
|
||||
* content.accountKeys. We want to unembed these values to main content, rename
|
||||
* with proper property names, wrap again, and store in new rawStructure.
|
||||
*/
|
||||
const passcodeKey = await getPasscodeKey()
|
||||
const payload = new Models.EncryptedPayload(wrappedAccountKey)
|
||||
const unwrappedAccountKey = await this.services.protocolService.decryptSplitSingle<LegacyRootKeyContent>({
|
||||
usesRootKey: {
|
||||
items: [payload],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
|
||||
if (Models.isErrorDecryptingPayload(unwrappedAccountKey)) {
|
||||
return
|
||||
}
|
||||
|
||||
const accountKeyContent = unwrappedAccountKey.content.accountKeys as LegacyAccountKeysValue
|
||||
|
||||
const version =
|
||||
accountKeyContent.version || rawAccountKeyParams?.version || (await this.getFallbackRootKeyVersion())
|
||||
|
||||
const newAccountKey = unwrappedAccountKey.copy({
|
||||
content: Models.FillItemContent<LegacyRootKeyContent>({
|
||||
masterKey: accountKeyContent.mk,
|
||||
dataAuthenticationKey: accountKeyContent.ak,
|
||||
version: version as ProtocolVersion,
|
||||
keyParams: rawAccountKeyParams,
|
||||
accountKeys: undefined,
|
||||
}),
|
||||
})
|
||||
|
||||
const newWrappedAccountKey = await this.services.protocolService.encryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [newAccountKey],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
rawStructure.nonwrapped[Services.StorageKey.WrappedRootKey] =
|
||||
Models.CreateEncryptedLocalStorageContextPayload(newWrappedAccountKey)
|
||||
|
||||
if (accountKeyContent.jwt) {
|
||||
/** Move the jwt to raw storage so that it can be migrated in `migrateSessionStorage` */
|
||||
void this.services.deviceInterface.setRawStorageValue(LEGACY_SESSION_TOKEN_KEY, accountKeyContent.jwt)
|
||||
}
|
||||
await this.services.deviceInterface.clearRawKeychainValue()
|
||||
} else if (!wrappedAccountKey) {
|
||||
/** Passcode only, no account */
|
||||
const passcodeKey = await getPasscodeKey()
|
||||
const payload = new Models.DecryptedPayload({
|
||||
uuid: Utils.UuidGenerator.GenerateUuid(),
|
||||
content: Models.FillItemContent(rawStructure.unwrapped),
|
||||
content_type: ContentType.EncryptedStorage,
|
||||
...PayloadTimestampDefaults(),
|
||||
})
|
||||
|
||||
/** Encrypt new storage.unwrapped structure with passcode */
|
||||
const wrapped = await this.services.protocolService.encryptSplitSingle({
|
||||
usesRootKey: {
|
||||
items: [payload],
|
||||
key: passcodeKey,
|
||||
},
|
||||
})
|
||||
rawStructure.wrapped = Models.CreateEncryptedLocalStorageContextPayload(wrapped)
|
||||
|
||||
await this.services.deviceInterface.clearRawKeychainValue()
|
||||
}
|
||||
} else {
|
||||
/** No passcode, potentially account. Migrate keychain property keys. */
|
||||
const hasAccount = !Utils.isNullOrUndefined(keychainValue?.mk)
|
||||
if (hasAccount) {
|
||||
const accountVersion =
|
||||
(keychainValue.version as ProtocolVersion) ||
|
||||
rawAccountKeyParams?.version ||
|
||||
(await this.getFallbackRootKeyVersion())
|
||||
|
||||
const accountKey = CreateNewRootKey({
|
||||
masterKey: keychainValue.mk,
|
||||
dataAuthenticationKey: keychainValue.ak,
|
||||
version: accountVersion,
|
||||
keyParams: rawAccountKeyParams,
|
||||
})
|
||||
|
||||
await this.services.deviceInterface.setNamespacedKeychainValue(
|
||||
accountKey.getKeychainValue(),
|
||||
this.services.identifier,
|
||||
)
|
||||
|
||||
if (keychainValue.jwt) {
|
||||
/** Move the jwt to raw storage so that it can be migrated in `migrateSessionStorage` */
|
||||
void this.services.deviceInterface.setRawStorageValue(LEGACY_SESSION_TOKEN_KEY, keychainValue.jwt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Move encrypted account key into place where it is now expected */
|
||||
await this.allPlatformHelperSetStorageStructure(rawStructure)
|
||||
}
|
||||
|
||||
/**
|
||||
* If we are unable to determine a root key's version, due to missing version
|
||||
* parameter from key params due to 001 or 002, we need to fallback to checking
|
||||
* any encrypted payload and retrieving its version.
|
||||
*
|
||||
* If we are unable to garner any meaningful information, we will default to 002.
|
||||
*
|
||||
* (Previously we attempted to discern version based on presence of keys.ak; if ak,
|
||||
* then 003, otherwise 002. However, late versions of 002 also inluded an ak, so this
|
||||
* method can't be used. This method also didn't account for 001 versions.)
|
||||
*/
|
||||
private async getFallbackRootKeyVersion() {
|
||||
const anyItem = (
|
||||
await this.services.deviceInterface.getAllRawDatabasePayloads(this.services.identifier)
|
||||
)[0] as Models.EncryptedTransferPayload
|
||||
|
||||
if (!anyItem) {
|
||||
return ProtocolVersion.V002
|
||||
}
|
||||
|
||||
const payload = new Models.EncryptedPayload(anyItem)
|
||||
return payload.version || ProtocolVersion.V002
|
||||
}
|
||||
|
||||
/**
|
||||
* All platforms
|
||||
* Migrate all previously independently stored storage keys into new
|
||||
* managed approach.
|
||||
*/
|
||||
private async migrateArbitraryRawStorageToManagedStorageAllPlatforms() {
|
||||
const allKeyValues = await this.services.deviceInterface.getAllRawStorageKeyValues()
|
||||
const legacyKeys = Utils.objectToValueArray(Services.LegacyKeys1_0_0)
|
||||
|
||||
const tryJsonParse = (value: string) => {
|
||||
try {
|
||||
return JSON.parse(value)
|
||||
} catch (e) {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
const applicationIdentifier = this.services.identifier
|
||||
|
||||
for (const keyValuePair of allKeyValues) {
|
||||
const key = keyValuePair.key
|
||||
const value = keyValuePair.value
|
||||
const isNameSpacedKey =
|
||||
applicationIdentifier && applicationIdentifier.length > 0 && key.startsWith(applicationIdentifier)
|
||||
if (legacyKeys.includes(key) || isNameSpacedKey) {
|
||||
continue
|
||||
}
|
||||
if (!Utils.isNullOrUndefined(value)) {
|
||||
/**
|
||||
* Raw values should always have been json stringified.
|
||||
* New values should always be objects/parsed.
|
||||
*/
|
||||
const newValue = tryJsonParse(value as string)
|
||||
this.services.storageService.setValue(key, newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* All platforms
|
||||
* Deletes all StorageKey and LegacyKeys1_0_0 from root raw storage.
|
||||
* @access private
|
||||
*/
|
||||
async deleteLegacyStorageValues() {
|
||||
const miscKeys = [
|
||||
'mk',
|
||||
'ak',
|
||||
'pw',
|
||||
/** v1 unused key */
|
||||
'encryptionKey',
|
||||
/** v1 unused key */
|
||||
'authKey',
|
||||
'jwt',
|
||||
'ephemeral',
|
||||
'cachedThemes',
|
||||
]
|
||||
|
||||
const managedKeys = [
|
||||
...Utils.objectToValueArray(Services.StorageKey),
|
||||
...Utils.objectToValueArray(Services.LegacyKeys1_0_0),
|
||||
...miscKeys,
|
||||
]
|
||||
|
||||
for (const key of managedKeys) {
|
||||
await this.services.deviceInterface.removeRawStorageValue(key)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mobile
|
||||
* Migrate mobile preferences
|
||||
*/
|
||||
private async migrateMobilePreferences() {
|
||||
const lastExportDate = await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobileLastExportDate,
|
||||
)
|
||||
const doNotWarnUnsupportedEditors = await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobileDoNotWarnUnsupportedEditors,
|
||||
)
|
||||
const legacyOptionsState = (await this.services.deviceInterface.getJsonParsedRawStorageValue(
|
||||
Services.LegacyKeys1_0_0.MobileOptionsState,
|
||||
)) as Record<string, unknown>
|
||||
|
||||
let migratedOptionsState = {}
|
||||
|
||||
if (legacyOptionsState) {
|
||||
const legacySortBy = legacyOptionsState.sortBy
|
||||
migratedOptionsState = {
|
||||
sortBy:
|
||||
legacySortBy === 'updated_at' || legacySortBy === 'client_updated_at'
|
||||
? Models.CollectionSort.UpdatedAt
|
||||
: legacySortBy,
|
||||
sortReverse: legacyOptionsState.sortReverse ?? false,
|
||||
hideNotePreview: legacyOptionsState.hidePreviews ?? false,
|
||||
hideDate: legacyOptionsState.hideDates ?? false,
|
||||
hideTags: legacyOptionsState.hideTags ?? false,
|
||||
}
|
||||
}
|
||||
const preferences = {
|
||||
...migratedOptionsState,
|
||||
lastExportDate: lastExportDate ?? undefined,
|
||||
doNotShowAgainUnsupportedEditors: doNotWarnUnsupportedEditors ?? false,
|
||||
}
|
||||
await this.services.storageService.setValue(Services.StorageKey.MobilePreferences, preferences)
|
||||
}
|
||||
|
||||
/**
|
||||
* All platforms
|
||||
* Migrate previously stored session string token into object
|
||||
* On mobile, JWTs were previously stored in storage, inside of the user object,
|
||||
* but then custom-migrated to be stored in the keychain. We must account for
|
||||
* both scenarios here in case a user did not perform the custom platform migration.
|
||||
* On desktop/web, JWT was stored in storage.
|
||||
*/
|
||||
private migrateSessionStorage() {
|
||||
const USER_OBJECT_KEY = 'user'
|
||||
let currentToken = this.services.storageService.getValue<string | undefined>(LEGACY_SESSION_TOKEN_KEY)
|
||||
const user = this.services.storageService.getValue<{ jwt: string; server: string }>(USER_OBJECT_KEY)
|
||||
|
||||
if (!currentToken) {
|
||||
/** Try the user object */
|
||||
if (user) {
|
||||
currentToken = user.jwt
|
||||
}
|
||||
}
|
||||
|
||||
if (!currentToken) {
|
||||
/**
|
||||
* If we detect that a user object is present, but the jwt is missing,
|
||||
* we'll fill the jwt value with a junk value just so we create a session.
|
||||
* When the client attempts to talk to the server, the server will reply
|
||||
* with invalid token error, and the client will automatically prompt to reauthenticate.
|
||||
*/
|
||||
const hasAccount = !Utils.isNullOrUndefined(user)
|
||||
if (hasAccount) {
|
||||
currentToken = 'junk-value'
|
||||
} else {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
const sessionOrError = LegacySession.create(currentToken)
|
||||
if (!sessionOrError.isFailed()) {
|
||||
this.services.storageService.setValue(
|
||||
Services.StorageKey.Session,
|
||||
this.services.legacySessionStorageMapper.toProjection(sessionOrError.getValue()),
|
||||
)
|
||||
}
|
||||
|
||||
/** Server has to be migrated separately on mobile */
|
||||
if (isEnvironmentMobile(this.services.environment)) {
|
||||
if (user && user.server) {
|
||||
this.services.storageService.setValue(Services.StorageKey.ServerHost, user.server)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* All platforms
|
||||
* Create new default items key from root key.
|
||||
* Otherwise, when data is loaded, we won't be able to decrypt it
|
||||
* without existence of an item key. This will mean that if this migration
|
||||
* is run on two different platforms for the same user, they will create
|
||||
* two new items keys. Which one they use to decrypt past items and encrypt
|
||||
* future items doesn't really matter.
|
||||
* @access private
|
||||
*/
|
||||
async createDefaultItemsKeyForAllPlatforms() {
|
||||
const rootKey = this.services.protocolService.getRootKey()
|
||||
if (rootKey) {
|
||||
const rootKeyParams = await this.services.protocolService.getRootKeyParams()
|
||||
/** If params are missing a version, it must be 001 */
|
||||
const fallbackVersion = ProtocolVersion.V001
|
||||
|
||||
const payload = new Models.DecryptedPayload({
|
||||
uuid: Utils.UuidGenerator.GenerateUuid(),
|
||||
content_type: ContentType.ItemsKey,
|
||||
content: Models.FillItemContentSpecialized<Models.ItemsKeyContentSpecialized, Models.ItemsKeyContent>({
|
||||
itemsKey: rootKey.masterKey,
|
||||
dataAuthenticationKey: rootKey.dataAuthenticationKey,
|
||||
version: rootKeyParams?.version || fallbackVersion,
|
||||
}),
|
||||
dirty: true,
|
||||
dirtyIndex: getIncrementedDirtyIndex(),
|
||||
...PayloadTimestampDefaults(),
|
||||
})
|
||||
|
||||
const itemsKey = Models.CreateDecryptedItemFromPayload(payload)
|
||||
|
||||
await this.services.itemManager.emitItemFromPayload(
|
||||
itemsKey.payloadRepresentation(),
|
||||
Models.PayloadEmitSource.LocalChanged,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,9 @@
|
||||
import { Migration2_0_0 } from './2_0_0'
|
||||
import { Migration2_0_15 } from './2_0_15'
|
||||
import { Migration2_7_0 } from './2_7_0'
|
||||
import { Migration2_20_0 } from './2_20_0'
|
||||
import { Migration2_36_0 } from './2_36_0'
|
||||
import { Migration2_42_0 } from './2_42_0'
|
||||
|
||||
export const MigrationClasses = [
|
||||
Migration2_0_0,
|
||||
Migration2_0_15,
|
||||
Migration2_7_0,
|
||||
Migration2_20_0,
|
||||
Migration2_36_0,
|
||||
Migration2_42_0,
|
||||
]
|
||||
export const MigrationClasses = [Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0]
|
||||
|
||||
export { Migration2_0_0, Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0 }
|
||||
export { Migration2_0_15, Migration2_7_0, Migration2_20_0, Migration2_36_0, Migration2_42_0 }
|
||||
|
||||
@@ -83,7 +83,7 @@ export class SNPreferencesService
|
||||
|
||||
void this.notifyEvent(PreferencesServiceEvent.PreferencesChanged)
|
||||
|
||||
void this.syncService.sync()
|
||||
void this.syncService.sync({ sourceDescription: 'PreferencesService.setValue' })
|
||||
}
|
||||
|
||||
private async reload() {
|
||||
|
||||
@@ -133,7 +133,7 @@ export class SNSingletonManager extends AbstractService {
|
||||
* of a download-first request.
|
||||
*/
|
||||
if (handled.length > 0 && eventSource === SyncEvent.SyncCompletedWithAllItemsUploaded) {
|
||||
await this.syncService?.sync()
|
||||
await this.syncService?.sync({ sourceDescription: 'Resolve singletons for items' })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,7 +190,7 @@ export class SNSingletonManager extends AbstractService {
|
||||
}
|
||||
})
|
||||
|
||||
await this.syncService.sync()
|
||||
await this.syncService.sync({ sourceDescription: 'Find or create singleton, before any sync has completed' })
|
||||
|
||||
removeObserver()
|
||||
|
||||
@@ -224,7 +224,7 @@ export class SNSingletonManager extends AbstractService {
|
||||
|
||||
const item = await this.itemManager.emitItemFromPayload(dirtyPayload, PayloadEmitSource.LocalInserted)
|
||||
|
||||
void this.syncService.sync()
|
||||
void this.syncService.sync({ sourceDescription: 'After find or create singleton' })
|
||||
|
||||
return item as T
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
PayloadTimestampDefaults,
|
||||
LocalStorageEncryptedContextualPayload,
|
||||
Environment,
|
||||
FullyFormedTransferPayload,
|
||||
} from '@standardnotes/models'
|
||||
|
||||
/**
|
||||
@@ -377,8 +378,8 @@ export class DiskStorageService extends Services.AbstractService implements Serv
|
||||
await this.immediatelyPersistValuesToDisk()
|
||||
}
|
||||
|
||||
public async getAllRawPayloads() {
|
||||
return this.deviceInterface.getAllRawDatabasePayloads(this.identifier)
|
||||
public async getAllRawPayloads(): Promise<FullyFormedTransferPayload[]> {
|
||||
return this.deviceInterface.getAllDatabaseEntries(this.identifier)
|
||||
}
|
||||
|
||||
public async savePayload(payload: FullyFormedPayloadInterface): Promise<void> {
|
||||
@@ -432,7 +433,7 @@ export class DiskStorageService extends Services.AbstractService implements Serv
|
||||
const exportedDeleted = deleted.map(CreateDeletedLocalStorageContextPayload)
|
||||
|
||||
return this.executeCriticalFunction(async () => {
|
||||
return this.deviceInterface?.saveRawDatabasePayloads(
|
||||
return this.deviceInterface?.saveDatabaseEntries(
|
||||
[...exportedEncrypted, ...exportedDecrypted, ...exportedDeleted],
|
||||
this.identifier,
|
||||
)
|
||||
@@ -449,13 +450,13 @@ export class DiskStorageService extends Services.AbstractService implements Serv
|
||||
|
||||
public async deletePayloadWithId(uuid: Uuid) {
|
||||
return this.executeCriticalFunction(async () => {
|
||||
return this.deviceInterface.removeRawDatabasePayloadWithId(uuid, this.identifier)
|
||||
return this.deviceInterface.removeDatabaseEntry(uuid, this.identifier)
|
||||
})
|
||||
}
|
||||
|
||||
public async clearAllPayloads() {
|
||||
return this.executeCriticalFunction(async () => {
|
||||
return this.deviceInterface.removeAllRawDatabasePayloads(this.identifier)
|
||||
return this.deviceInterface.removeAllDatabaseEntries(this.identifier)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -482,7 +483,6 @@ export class DiskStorageService extends Services.AbstractService implements Serv
|
||||
currentPersistPromise: this.currentPersistPromise != undefined,
|
||||
isStorageWrapped: this.isStorageWrapped(),
|
||||
allRawPayloadsCount: (await this.getAllRawPayloads()).length,
|
||||
databaseKeys: await this.deviceInterface.getDatabaseKeys(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { log, LoggingDomain } from './../../Logging'
|
||||
import { AccountSyncOperation } from '@Lib/Services/Sync/Account/Operation'
|
||||
import { ContentType } from '@standardnotes/common'
|
||||
import {
|
||||
@@ -18,7 +19,6 @@ import { SNHistoryManager } from '../History/HistoryManager'
|
||||
import { SNLog } from '@Lib/Log'
|
||||
import { SNSessionManager } from '../Session/SessionManager'
|
||||
import { DiskStorageService } from '../Storage/DiskStorageService'
|
||||
import { GetSortedPayloadsByPriority } from '@Lib/Services/Sync/Utils'
|
||||
import { SyncClientInterface } from './SyncClientInterface'
|
||||
import { SyncPromise } from './Types'
|
||||
import { SyncOpStatus } from '@Lib/Services/Sync/SyncOpStatus'
|
||||
@@ -33,7 +33,6 @@ import {
|
||||
DeltaOutOfSync,
|
||||
ImmutablePayloadCollection,
|
||||
CreatePayload,
|
||||
FullyFormedTransferPayload,
|
||||
isEncryptedPayload,
|
||||
isDecryptedPayload,
|
||||
EncryptedPayloadInterface,
|
||||
@@ -74,6 +73,9 @@ import {
|
||||
SyncServiceInterface,
|
||||
DiagnosticInfo,
|
||||
EncryptionService,
|
||||
DeviceInterface,
|
||||
isFullEntryLoadChunkResponse,
|
||||
isChunkFullEntry,
|
||||
} from '@standardnotes/services'
|
||||
import { OfflineSyncResponse } from './Offline/Response'
|
||||
import {
|
||||
@@ -142,6 +144,8 @@ export class SNSyncService
|
||||
private payloadManager: PayloadManager,
|
||||
private apiService: SNApiService,
|
||||
private historyService: SNHistoryManager,
|
||||
private device: DeviceInterface,
|
||||
private identifier: string,
|
||||
private readonly options: ApplicationSyncOptions,
|
||||
protected override internalEventBus: InternalEventBusInterface,
|
||||
) {
|
||||
@@ -221,19 +225,13 @@ export class SNSyncService
|
||||
return this.databaseLoaded
|
||||
}
|
||||
|
||||
/**
|
||||
* Used in tandem with `loadDatabasePayloads`
|
||||
*/
|
||||
public async getDatabasePayloads(): Promise<FullyFormedTransferPayload[]> {
|
||||
return this.storageService.getAllRawPayloads().catch((error) => {
|
||||
void this.notifyEvent(SyncEvent.DatabaseReadError, error)
|
||||
throw error
|
||||
})
|
||||
}
|
||||
|
||||
private async processItemsKeysFirstDuringDatabaseLoad(
|
||||
itemsKeysPayloads: FullyFormedPayloadInterface[],
|
||||
): Promise<void> {
|
||||
if (itemsKeysPayloads.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const encryptedItemsKeysPayloads = itemsKeysPayloads.filter(isEncryptedPayload)
|
||||
|
||||
const originallyDecryptedItemsKeysPayloads = itemsKeysPayloads.filter(
|
||||
@@ -254,57 +252,69 @@ export class SNSyncService
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param rawPayloads - use `getDatabasePayloads` to get these payloads.
|
||||
* They are fed as a parameter so that callers don't have to await the loading, but can
|
||||
* await getting the raw payloads from storage
|
||||
*/
|
||||
public async loadDatabasePayloads(rawPayloads: FullyFormedTransferPayload[]): Promise<void> {
|
||||
public async loadDatabasePayloads(): Promise<void> {
|
||||
log(LoggingDomain.DatabaseLoad, 'Loading database payloads')
|
||||
|
||||
if (this.databaseLoaded) {
|
||||
throw 'Attempting to initialize already initialized local database.'
|
||||
}
|
||||
|
||||
if (rawPayloads.length === 0) {
|
||||
this.databaseLoaded = true
|
||||
this.opStatus.setDatabaseLoadStatus(0, 0, true)
|
||||
return
|
||||
}
|
||||
const chunks = await this.device.getDatabaseLoadChunks(
|
||||
{
|
||||
batchSize: this.options.loadBatchSize,
|
||||
contentTypePriority: this.localLoadPriorty,
|
||||
uuidPriority: this.launchPriorityUuids,
|
||||
},
|
||||
this.identifier,
|
||||
)
|
||||
|
||||
const unsortedPayloads = rawPayloads
|
||||
.map((rawPayload) => {
|
||||
const itemsKeyEntries = isFullEntryLoadChunkResponse(chunks)
|
||||
? chunks.fullEntries.itemsKeys.entries
|
||||
: await this.device.getDatabaseEntries(this.identifier, chunks.keys.itemsKeys.keys)
|
||||
|
||||
const itemsKeyPayloads = itemsKeyEntries
|
||||
.map((entry) => {
|
||||
try {
|
||||
return CreatePayload(rawPayload, PayloadSource.Constructor)
|
||||
return CreatePayload(entry, PayloadSource.Constructor)
|
||||
} catch (e) {
|
||||
console.error('Creating payload fail+ed', e)
|
||||
console.error('Creating payload failed', e)
|
||||
return undefined
|
||||
}
|
||||
})
|
||||
.filter(isNotUndefined)
|
||||
|
||||
const { itemsKeyPayloads, contentTypePriorityPayloads, remainingPayloads } = GetSortedPayloadsByPriority(
|
||||
unsortedPayloads,
|
||||
this.localLoadPriorty,
|
||||
this.launchPriorityUuids,
|
||||
)
|
||||
|
||||
await this.processItemsKeysFirstDuringDatabaseLoad(itemsKeyPayloads)
|
||||
|
||||
await this.processPayloadBatch(contentTypePriorityPayloads)
|
||||
|
||||
/**
|
||||
* Map in batches to give interface a chance to update. Note that total decryption
|
||||
* time is constant regardless of batch size. Decrypting 3000 items all at once or in
|
||||
* batches will result in the same time spent. It's the emitting/painting/rendering
|
||||
* that requires batch size optimization.
|
||||
*/
|
||||
const payloadCount = remainingPayloads.length
|
||||
const batchSize = this.options.loadBatchSize
|
||||
const numBatches = Math.ceil(payloadCount / batchSize)
|
||||
const payloadCount = chunks.remainingChunksItemCount
|
||||
let totalProcessedCount = 0
|
||||
|
||||
for (let batchIndex = 0; batchIndex < numBatches; batchIndex++) {
|
||||
const currentPosition = batchIndex * batchSize
|
||||
const batch = remainingPayloads.slice(currentPosition, currentPosition + batchSize)
|
||||
await this.processPayloadBatch(batch, currentPosition, payloadCount)
|
||||
const remainingChunks = isFullEntryLoadChunkResponse(chunks)
|
||||
? chunks.fullEntries.remainingChunks
|
||||
: chunks.keys.remainingChunks
|
||||
|
||||
for (const chunk of remainingChunks) {
|
||||
const dbEntries = isChunkFullEntry(chunk)
|
||||
? chunk.entries
|
||||
: await this.device.getDatabaseEntries(this.identifier, chunk.keys)
|
||||
const payloads = dbEntries
|
||||
.map((entry) => {
|
||||
try {
|
||||
return CreatePayload(entry, PayloadSource.Constructor)
|
||||
} catch (e) {
|
||||
console.error('Creating payload failed', e)
|
||||
return undefined
|
||||
}
|
||||
})
|
||||
.filter(isNotUndefined)
|
||||
|
||||
await this.processPayloadBatch(payloads, totalProcessedCount, payloadCount)
|
||||
totalProcessedCount += payloads.length
|
||||
}
|
||||
|
||||
this.databaseLoaded = true
|
||||
@@ -316,6 +326,7 @@ export class SNSyncService
|
||||
currentPosition?: number,
|
||||
payloadCount?: number,
|
||||
) {
|
||||
log(LoggingDomain.DatabaseLoad, 'Processing batch at index', currentPosition, 'length', batch.length)
|
||||
const encrypted: EncryptedPayloadInterface[] = []
|
||||
const nonencrypted: (DecryptedPayloadInterface | DeletedPayloadInterface)[] = []
|
||||
|
||||
@@ -386,7 +397,7 @@ export class SNSyncService
|
||||
}
|
||||
|
||||
public async markAllItemsAsNeedingSyncAndPersist(): Promise<void> {
|
||||
this.log('Marking all items as needing sync')
|
||||
log(LoggingDomain.Sync, 'Marking all items as needing sync')
|
||||
|
||||
const items = this.itemManager.items
|
||||
const payloads = items.map((item) => {
|
||||
@@ -444,7 +455,7 @@ export class SNSyncService
|
||||
|
||||
const promise = this.spawnQueue[0]
|
||||
removeFromIndex(this.spawnQueue, 0)
|
||||
this.log('Syncing again from spawn queue')
|
||||
log(LoggingDomain.Sync, 'Syncing again from spawn queue')
|
||||
|
||||
return this.sync({
|
||||
queueStrategy: SyncQueueStrategy.ForceSpawnNew,
|
||||
@@ -506,7 +517,7 @@ export class SNSyncService
|
||||
|
||||
public async sync(options: Partial<SyncOptions> = {}): Promise<unknown> {
|
||||
if (this.clientLocked) {
|
||||
this.log('Sync locked by client')
|
||||
log(LoggingDomain.Sync, 'Sync locked by client')
|
||||
return
|
||||
}
|
||||
|
||||
@@ -562,7 +573,7 @@ export class SNSyncService
|
||||
* (before reaching opStatus.setDidBegin).
|
||||
* 2. syncOpInProgress: If a sync() call is in flight to the server.
|
||||
*/
|
||||
private configureSyncLock() {
|
||||
private configureSyncLock(options: SyncOptions) {
|
||||
const syncInProgress = this.opStatus.syncInProgress
|
||||
const databaseLoaded = this.databaseLoaded
|
||||
const canExecuteSync = !this.syncLock
|
||||
@@ -571,12 +582,14 @@ export class SNSyncService
|
||||
if (shouldExecuteSync) {
|
||||
this.syncLock = true
|
||||
} else {
|
||||
this.log(
|
||||
log(
|
||||
LoggingDomain.Sync,
|
||||
!canExecuteSync
|
||||
? 'Another function call has begun preparing for sync.'
|
||||
: syncInProgress
|
||||
? 'Attempting to sync while existing sync in progress.'
|
||||
: 'Attempting to sync before local database has loaded.',
|
||||
options,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -656,10 +669,20 @@ export class SNSyncService
|
||||
|
||||
private createOfflineSyncOperation(
|
||||
payloads: (DeletedPayloadInterface | DecryptedPayloadInterface)[],
|
||||
source: SyncSource,
|
||||
mode: SyncMode = SyncMode.Default,
|
||||
options: SyncOptions,
|
||||
) {
|
||||
this.log('Syncing offline user', 'source:', source, 'mode:', mode, 'payloads:', payloads)
|
||||
log(
|
||||
LoggingDomain.Sync,
|
||||
'Syncing offline user',
|
||||
'source:',
|
||||
SyncSource[options.source],
|
||||
'sourceDesc',
|
||||
options.sourceDescription,
|
||||
'mode:',
|
||||
options.mode && SyncMode[options.mode],
|
||||
'payloads:',
|
||||
payloads,
|
||||
)
|
||||
|
||||
const operation = new OfflineSyncOperation(payloads, async (type, response) => {
|
||||
if (this.dealloced) {
|
||||
@@ -727,7 +750,8 @@ export class SNSyncService
|
||||
this.apiService,
|
||||
)
|
||||
|
||||
this.log(
|
||||
log(
|
||||
LoggingDomain.Sync,
|
||||
'Syncing online user',
|
||||
'source',
|
||||
SyncSource[source],
|
||||
@@ -769,14 +793,14 @@ export class SNSyncService
|
||||
const { uploadPayloads } = this.getOfflineSyncParameters(payloads, options.mode)
|
||||
|
||||
return {
|
||||
operation: this.createOfflineSyncOperation(uploadPayloads, options.source, options.mode),
|
||||
operation: this.createOfflineSyncOperation(uploadPayloads, options),
|
||||
mode: options.mode || SyncMode.Default,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async performSync(options: SyncOptions): Promise<unknown> {
|
||||
const { shouldExecuteSync, releaseLock } = this.configureSyncLock()
|
||||
const { shouldExecuteSync, releaseLock } = this.configureSyncLock(options)
|
||||
|
||||
const { items, beginDate, frozenDirtyIndex, neverSyncedDeleted } = await this.prepareForSync(options)
|
||||
|
||||
@@ -843,7 +867,7 @@ export class SNSyncService
|
||||
}
|
||||
|
||||
private async handleOfflineResponse(response: OfflineSyncResponse) {
|
||||
this.log('Offline Sync Response', response)
|
||||
log(LoggingDomain.Sync, 'Offline Sync Response', response)
|
||||
|
||||
const masterCollection = this.payloadManager.getMasterCollection()
|
||||
|
||||
@@ -861,7 +885,7 @@ export class SNSyncService
|
||||
}
|
||||
|
||||
private handleErrorServerResponse(response: ServerSyncResponse) {
|
||||
this.log('Sync Error', response)
|
||||
log(LoggingDomain.Sync, 'Sync Error', response)
|
||||
|
||||
if (response.status === INVALID_SESSION_RESPONSE_STATUS) {
|
||||
void this.notifyEvent(SyncEvent.InvalidSession)
|
||||
@@ -904,7 +928,8 @@ export class SNSyncService
|
||||
historyMap,
|
||||
)
|
||||
|
||||
this.log(
|
||||
log(
|
||||
LoggingDomain.Sync,
|
||||
'Online Sync Response',
|
||||
'Operator ID',
|
||||
operation.id,
|
||||
@@ -1060,7 +1085,7 @@ export class SNSyncService
|
||||
}
|
||||
|
||||
private async syncAgainByHandlingRequestsWaitingInResolveQueue(options: SyncOptions) {
|
||||
this.log('Syncing again from resolve queue')
|
||||
log(LoggingDomain.Sync, 'Syncing again from resolve queue')
|
||||
const promise = this.sync({
|
||||
source: SyncSource.ResolveQueue,
|
||||
checkIntegrity: options.checkIntegrity,
|
||||
|
||||
@@ -5,5 +5,4 @@ export * from './SyncClientInterface'
|
||||
export * from './Account/Operation'
|
||||
export * from './Account/ResponseResolver'
|
||||
export * from './Offline/Operation'
|
||||
export * from './Utils'
|
||||
export * from './Account/Response'
|
||||
|
||||
@@ -110,12 +110,12 @@ describe('application instances', () => {
|
||||
* app deinit. */
|
||||
await Factory.sleep(MaximumWaitTime - 0.05)
|
||||
/** Access any deviceInterface function */
|
||||
app.diskStorageService.deviceInterface.getAllRawDatabasePayloads(app.identifier)
|
||||
app.diskStorageService.deviceInterface.getAllDatabaseEntries(app.identifier)
|
||||
})
|
||||
await app.lock()
|
||||
})
|
||||
|
||||
describe('signOut()', () => {
|
||||
describe.skip('signOut()', () => {
|
||||
let testNote1
|
||||
let confirmAlert
|
||||
let deinit
|
||||
|
||||
@@ -59,9 +59,6 @@ describe('basic auth', function () {
|
||||
|
||||
expect(await this.application.protocolService.getRootKey()).to.not.be.ok
|
||||
expect(this.application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone)
|
||||
|
||||
const rawPayloads = await this.application.diskStorageService.getAllRawPayloads()
|
||||
expect(rawPayloads.length).to.equal(BaseItemCounts.DefaultItems)
|
||||
})
|
||||
|
||||
it('successfully signs in to registered account', async function () {
|
||||
|
||||
@@ -664,12 +664,12 @@ describe('key recovery service', function () {
|
||||
await Factory.awaitFunctionInvokation(appA.keyRecoveryService, 'handleDecryptionOfAllKeysMatchingCorrectRootKey')
|
||||
|
||||
/** Stored version of items key should use new root key */
|
||||
const stored = (await appA.deviceInterface.getAllRawDatabasePayloads(appA.identifier)).find(
|
||||
const stored = (await appA.deviceInterface.getAllDatabaseEntries(appA.identifier)).find(
|
||||
(payload) => payload.uuid === newDefaultKey.uuid,
|
||||
)
|
||||
const storedParams = await appA.protocolService.getKeyEmbeddedKeyParams(new EncryptedPayload(stored))
|
||||
|
||||
const correctStored = (await appB.deviceInterface.getAllRawDatabasePayloads(appB.identifier)).find(
|
||||
const correctStored = (await appB.deviceInterface.getAllDatabaseEntries(appB.identifier)).find(
|
||||
(payload) => payload.uuid === newDefaultKey.uuid,
|
||||
)
|
||||
|
||||
|
||||
@@ -303,7 +303,8 @@ export function tomorrow() {
|
||||
return new Date(new Date().setDate(new Date().getDate() + 1))
|
||||
}
|
||||
|
||||
export async function sleep(seconds) {
|
||||
export async function sleep(seconds, reason) {
|
||||
console.log('Sleeping for reason', reason)
|
||||
return Utils.sleep(seconds)
|
||||
}
|
||||
|
||||
|
||||
@@ -21,17 +21,6 @@ export default class WebDeviceInterface {
|
||||
}
|
||||
}
|
||||
|
||||
async getAllRawStorageKeyValues() {
|
||||
const results = []
|
||||
for (const key of Object.keys(localStorage)) {
|
||||
results.push({
|
||||
key: key,
|
||||
value: localStorage[key],
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
async setRawStorageValue(key, value) {
|
||||
localStorage.setItem(key, value)
|
||||
}
|
||||
@@ -60,7 +49,7 @@ export default class WebDeviceInterface {
|
||||
return `${this._getDatabaseKeyPrefix(identifier)}${id}`
|
||||
}
|
||||
|
||||
async getAllRawDatabasePayloads(identifier) {
|
||||
async getAllDatabaseEntries(identifier) {
|
||||
const models = []
|
||||
for (const key in localStorage) {
|
||||
if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) {
|
||||
@@ -70,21 +59,51 @@ export default class WebDeviceInterface {
|
||||
return models
|
||||
}
|
||||
|
||||
async saveRawDatabasePayload(payload, identifier) {
|
||||
async getDatabaseLoadChunks(options, identifier) {
|
||||
const entries = await this.getAllDatabaseEntries(identifier)
|
||||
const sorted = GetSortedPayloadsByPriority(entries, options)
|
||||
|
||||
const itemsKeysChunk = {
|
||||
entries: sorted.itemsKeyPayloads,
|
||||
}
|
||||
|
||||
const contentTypePriorityChunk = {
|
||||
entries: sorted.contentTypePriorityPayloads,
|
||||
}
|
||||
|
||||
const remainingPayloadsChunks = []
|
||||
for (let i = 0; i < sorted.remainingPayloads.length; i += options.batchSize) {
|
||||
remainingPayloadsChunks.push({
|
||||
entries: sorted.remainingPayloads.slice(i, i + options.batchSize),
|
||||
})
|
||||
}
|
||||
|
||||
const result = {
|
||||
fullEntries: {
|
||||
itemsKeys: itemsKeysChunk,
|
||||
remainingChunks: [contentTypePriorityChunk, ...remainingPayloadsChunks],
|
||||
},
|
||||
remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length,
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async saveDatabaseEntry(payload, identifier) {
|
||||
localStorage.setItem(this._keyForPayloadId(payload.uuid, identifier), JSON.stringify(payload))
|
||||
}
|
||||
|
||||
async saveRawDatabasePayloads(payloads, identifier) {
|
||||
async saveDatabaseEntries(payloads, identifier) {
|
||||
for (const payload of payloads) {
|
||||
await this.saveRawDatabasePayload(payload, identifier)
|
||||
await this.saveDatabaseEntry(payload, identifier)
|
||||
}
|
||||
}
|
||||
|
||||
async removeRawDatabasePayloadWithId(id, identifier) {
|
||||
async removeDatabaseEntry(id, identifier) {
|
||||
localStorage.removeItem(this._keyForPayloadId(id, identifier))
|
||||
}
|
||||
|
||||
async removeAllRawDatabasePayloads(identifier) {
|
||||
async removeAllDatabaseEntries(identifier) {
|
||||
for (const key in localStorage) {
|
||||
if (key.startsWith(this._getDatabaseKeyPrefix(identifier))) {
|
||||
delete localStorage[key]
|
||||
@@ -124,12 +143,6 @@ export default class WebDeviceInterface {
|
||||
localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(keychain))
|
||||
}
|
||||
|
||||
/** Allows unit tests to set legacy keychain structure as it was <= 003 */
|
||||
// eslint-disable-next-line camelcase
|
||||
async setLegacyRawKeychainValue(value) {
|
||||
localStorage.setItem(KEYCHAIN_STORAGE_KEY, JSON.stringify(value))
|
||||
}
|
||||
|
||||
async getRawKeychainValue() {
|
||||
const keychain = localStorage.getItem(KEYCHAIN_STORAGE_KEY)
|
||||
return JSON.parse(keychain)
|
||||
@@ -139,19 +152,13 @@ export default class WebDeviceInterface {
|
||||
localStorage.removeItem(KEYCHAIN_STORAGE_KEY)
|
||||
}
|
||||
|
||||
performSoftReset() {
|
||||
performSoftReset() {}
|
||||
|
||||
}
|
||||
|
||||
performHardReset() {
|
||||
|
||||
}
|
||||
performHardReset() {}
|
||||
|
||||
isDeviceDestroyed() {
|
||||
return false
|
||||
}
|
||||
|
||||
deinit() {
|
||||
|
||||
}
|
||||
deinit() {}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,584 +0,0 @@
|
||||
/* eslint-disable no-unused-expressions */
|
||||
/* eslint-disable no-undef */
|
||||
import * as Factory from '../lib/factory.js'
|
||||
import FakeWebCrypto from '../lib/fake_web_crypto.js'
|
||||
chai.use(chaiAsPromised)
|
||||
const expect = chai.expect
|
||||
|
||||
describe('2020-01-15 web migration', () => {
|
||||
beforeEach(() => {
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
/**
|
||||
* This test will pass but sync afterwards will not be successful
|
||||
* as we are using a random value for the legacy session token
|
||||
*/
|
||||
it('2020-01-15 migration with passcode and account', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Create legacy migrations value so that base migration detects old app */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
const operator003 = new SNProtocolOperator003(new FakeWebCrypto())
|
||||
const identifier = 'foo'
|
||||
const passcode = 'bar'
|
||||
/** Create old version passcode parameters */
|
||||
const passcodeKey = await operator003.createRootKey(identifier, passcode)
|
||||
await application.deviceInterface.setRawStorageValue(
|
||||
'offlineParams',
|
||||
JSON.stringify(passcodeKey.keyParams.getPortableValue()),
|
||||
)
|
||||
|
||||
/** Create arbitrary storage values and make sure they're migrated */
|
||||
const arbitraryValues = {
|
||||
foo: 'bar',
|
||||
zar: 'tar',
|
||||
har: 'car',
|
||||
}
|
||||
for (const key of Object.keys(arbitraryValues)) {
|
||||
await application.deviceInterface.setRawStorageValue(key, arbitraryValues[key])
|
||||
}
|
||||
/** Create old version account parameters */
|
||||
const password = 'tar'
|
||||
const accountKey = await operator003.createRootKey(identifier, password)
|
||||
|
||||
/** Create legacy storage and encrypt it with passcode */
|
||||
const embeddedStorage = {
|
||||
mk: accountKey.masterKey,
|
||||
ak: accountKey.dataAuthenticationKey,
|
||||
pw: accountKey.serverPassword,
|
||||
jwt: 'anything',
|
||||
/** Legacy versions would store json strings inside of embedded storage */
|
||||
auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()),
|
||||
}
|
||||
const storagePayload = new DecryptedPayload({
|
||||
uuid: await operator003.crypto.generateUUID(),
|
||||
content_type: ContentType.EncryptedStorage,
|
||||
content: {
|
||||
storage: embeddedStorage,
|
||||
},
|
||||
})
|
||||
const encryptionParams = await operator003.generateEncryptedParametersAsync(storagePayload, passcodeKey)
|
||||
const persistPayload = new EncryptedPayload({ ...storagePayload, ...encryptionParams })
|
||||
await application.deviceInterface.setRawStorageValue('encryptedStorage', JSON.stringify(persistPayload))
|
||||
|
||||
/** Create encrypted item and store it in db */
|
||||
const notePayload = Factory.createNotePayload()
|
||||
const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey)
|
||||
const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams })
|
||||
await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier)
|
||||
|
||||
/** Run migration */
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: async (challenge) => {
|
||||
application.submitValuesForChallenge(challenge, [CreateChallengeValue(challenge.prompts[0], passcode)])
|
||||
},
|
||||
})
|
||||
|
||||
await application.launch(true)
|
||||
expect(application.sessionManager.online()).to.equal(true)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper)
|
||||
/** Should be decrypted */
|
||||
const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default)
|
||||
const valueStore = application.diskStorageService.values[storageMode]
|
||||
expect(valueStore.content_type).to.not.be.ok
|
||||
|
||||
expect(await application.deviceInterface.getRawStorageValue('offlineParams')).to.not.be.ok
|
||||
|
||||
const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped)
|
||||
expect(typeof keyParams).to.equal('object')
|
||||
|
||||
/** Embedded value should match */
|
||||
const migratedKeyParams = await application.diskStorageService.getValue(
|
||||
StorageKey.RootKeyParams,
|
||||
StorageValueModes.Nonwrapped,
|
||||
)
|
||||
expect(migratedKeyParams).to.eql(JSON.parse(embeddedStorage.auth_params))
|
||||
const rootKey = await application.protocolService.getRootKey()
|
||||
expect(rootKey.masterKey).to.equal(accountKey.masterKey)
|
||||
expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey)
|
||||
/** Application should not retain server password from legacy versions */
|
||||
expect(rootKey.serverPassword).to.not.be.ok
|
||||
expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper)
|
||||
|
||||
/** Expect note is decrypted */
|
||||
expect(application.itemManager.getDisplayableNotes().length).to.equal(1)
|
||||
const retrievedNote = application.itemManager.getDisplayableNotes()[0]
|
||||
expect(retrievedNote.uuid).to.equal(notePayload.uuid)
|
||||
expect(retrievedNote.content.text).to.equal(notePayload.content.text)
|
||||
|
||||
/** Ensure arbitrary values have been migrated */
|
||||
for (const key of Object.keys(arbitraryValues)) {
|
||||
const value = await application.diskStorageService.getValue(key)
|
||||
expect(arbitraryValues[key]).to.equal(value)
|
||||
}
|
||||
|
||||
console.warn('Expecting exception due to deiniting application while trying to renew session')
|
||||
await Factory.safeDeinit(application)
|
||||
}).timeout(15000)
|
||||
|
||||
it('2020-01-15 migration with passcode only', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Create legacy migrations value so that base migration detects old app */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
const operator003 = new SNProtocolOperator003(new FakeWebCrypto())
|
||||
const identifier = 'foo'
|
||||
const passcode = 'bar'
|
||||
/** Create old version passcode parameters */
|
||||
const passcodeKey = await operator003.createRootKey(identifier, passcode)
|
||||
await application.deviceInterface.setRawStorageValue(
|
||||
'offlineParams',
|
||||
JSON.stringify(passcodeKey.keyParams.getPortableValue()),
|
||||
)
|
||||
|
||||
/** Create arbitrary storage values and make sure they're migrated */
|
||||
const arbitraryValues = {
|
||||
foo: 'bar',
|
||||
zar: 'tar',
|
||||
har: 'car',
|
||||
}
|
||||
for (const key of Object.keys(arbitraryValues)) {
|
||||
await application.deviceInterface.setRawStorageValue(key, arbitraryValues[key])
|
||||
}
|
||||
|
||||
const embeddedStorage = {
|
||||
...arbitraryValues,
|
||||
}
|
||||
const storagePayload = new DecryptedPayload({
|
||||
uuid: await operator003.crypto.generateUUID(),
|
||||
content: {
|
||||
storage: embeddedStorage,
|
||||
},
|
||||
content_type: ContentType.EncryptedStorage,
|
||||
})
|
||||
const encryptionParams = await operator003.generateEncryptedParametersAsync(storagePayload, passcodeKey)
|
||||
const persistPayload = new EncryptedPayload({ ...storagePayload, ...encryptionParams })
|
||||
await application.deviceInterface.setRawStorageValue('encryptedStorage', JSON.stringify(persistPayload))
|
||||
|
||||
/** Create encrypted item and store it in db */
|
||||
const notePayload = Factory.createNotePayload()
|
||||
const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, passcodeKey)
|
||||
const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams })
|
||||
await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier)
|
||||
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: async (challenge) => {
|
||||
application.submitValuesForChallenge(challenge, [CreateChallengeValue(challenge.prompts[0], passcode)])
|
||||
},
|
||||
})
|
||||
await application.launch(true)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly)
|
||||
/** Should be decrypted */
|
||||
const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default)
|
||||
const valueStore = application.diskStorageService.values[storageMode]
|
||||
expect(valueStore.content_type).to.not.be.ok
|
||||
|
||||
expect(await application.deviceInterface.getRawStorageValue('offlineParams')).to.not.be.ok
|
||||
|
||||
/** Embedded value should match */
|
||||
const migratedKeyParams = await application.diskStorageService.getValue(
|
||||
StorageKey.RootKeyParams,
|
||||
StorageValueModes.Nonwrapped,
|
||||
)
|
||||
expect(migratedKeyParams).to.eql(embeddedStorage.auth_params)
|
||||
const rootKey = await application.protocolService.getRootKey()
|
||||
expect(rootKey.masterKey).to.equal(passcodeKey.masterKey)
|
||||
expect(rootKey.dataAuthenticationKey).to.equal(passcodeKey.dataAuthenticationKey)
|
||||
/** Root key is in memory with passcode only, so server password can be defined */
|
||||
expect(rootKey.serverPassword).to.be.ok
|
||||
expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.WrapperOnly)
|
||||
|
||||
/** Expect note is decrypted */
|
||||
expect(application.itemManager.getDisplayableNotes().length).to.equal(1)
|
||||
const retrievedNote = application.itemManager.getDisplayableNotes()[0]
|
||||
expect(retrievedNote.uuid).to.equal(notePayload.uuid)
|
||||
expect(retrievedNote.content.text).to.equal(notePayload.content.text)
|
||||
|
||||
/** Ensure arbitrary values have been migrated */
|
||||
for (const key of Object.keys(arbitraryValues)) {
|
||||
const value = await application.diskStorageService.getValue(key)
|
||||
expect(arbitraryValues[key]).to.equal(value)
|
||||
}
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
/**
|
||||
* This test will pass but sync afterwards will not be successful
|
||||
* as we are using a random value for the legacy session token
|
||||
*/
|
||||
it('2020-01-15 migration with account only', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Create legacy migrations value so that base migration detects old app */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
const operator003 = new SNProtocolOperator003(new FakeWebCrypto())
|
||||
const identifier = 'foo'
|
||||
|
||||
/** Create old version account parameters */
|
||||
const password = 'tar'
|
||||
const accountKey = await operator003.createRootKey(identifier, password)
|
||||
|
||||
/** Create arbitrary storage values and make sure they're migrated */
|
||||
const storage = {
|
||||
foo: 'bar',
|
||||
zar: 'tar',
|
||||
har: 'car',
|
||||
mk: accountKey.masterKey,
|
||||
ak: accountKey.dataAuthenticationKey,
|
||||
pw: accountKey.serverPassword,
|
||||
jwt: 'anything',
|
||||
/** Legacy versions would store json strings inside of embedded storage */
|
||||
auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()),
|
||||
}
|
||||
for (const key of Object.keys(storage)) {
|
||||
await application.deviceInterface.setRawStorageValue(key, storage[key])
|
||||
}
|
||||
/** Create encrypted item and store it in db */
|
||||
const notePayload = Factory.createNotePayload()
|
||||
const noteEncryptionParams = await operator003.generateEncryptedParametersAsync(notePayload, accountKey)
|
||||
const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams })
|
||||
await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier)
|
||||
|
||||
/** Run migration */
|
||||
const promptValueReply = (prompts) => {
|
||||
const values = []
|
||||
for (const prompt of prompts) {
|
||||
if (prompt.validation === ChallengeValidation.LocalPasscode) {
|
||||
values.push(CreateChallengeValue(prompt, passcode))
|
||||
} else {
|
||||
/** We will be prompted to reauthetnicate our session, not relevant to this test
|
||||
* but pass any value to avoid exception
|
||||
*/
|
||||
values.push(CreateChallengeValue(prompt, 'foo'))
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
const receiveChallenge = async (challenge) => {
|
||||
application.addChallengeObserver(challenge, {
|
||||
onInvalidValue: (value) => {
|
||||
const values = promptValueReply([value.prompt])
|
||||
application.submitValuesForChallenge(challenge, values)
|
||||
},
|
||||
})
|
||||
const initialValues = promptValueReply(challenge.prompts)
|
||||
application.submitValuesForChallenge(challenge, initialValues)
|
||||
}
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: receiveChallenge,
|
||||
})
|
||||
await application.launch(true)
|
||||
expect(application.sessionManager.online()).to.equal(true)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly)
|
||||
/** Should be decrypted */
|
||||
const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default)
|
||||
const valueStore = application.diskStorageService.values[storageMode]
|
||||
expect(valueStore.content_type).to.not.be.ok
|
||||
/** Embedded value should match */
|
||||
const migratedKeyParams = await application.diskStorageService.getValue(
|
||||
StorageKey.RootKeyParams,
|
||||
StorageValueModes.Nonwrapped,
|
||||
)
|
||||
expect(migratedKeyParams).to.eql(accountKey.keyParams.getPortableValue())
|
||||
const rootKey = await application.protocolService.getRootKey()
|
||||
expect(rootKey).to.be.ok
|
||||
|
||||
expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('auth_params')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('jwt')).to.not.be.ok
|
||||
|
||||
const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped)
|
||||
expect(typeof keyParams).to.equal('object')
|
||||
|
||||
expect(rootKey.masterKey).to.equal(accountKey.masterKey)
|
||||
expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey)
|
||||
expect(rootKey.serverPassword).to.not.be.ok
|
||||
expect(rootKey.keyVersion).to.equal(ProtocolVersion.V003)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly)
|
||||
|
||||
/** Expect note is decrypted */
|
||||
expect(application.itemManager.getDisplayableNotes().length).to.equal(1)
|
||||
const retrievedNote = application.itemManager.getDisplayableNotes()[0]
|
||||
expect(retrievedNote.uuid).to.equal(notePayload.uuid)
|
||||
expect(retrievedNote.content.text).to.equal(notePayload.content.text)
|
||||
|
||||
/** Ensure arbitrary values have been migrated */
|
||||
for (const key of Object.keys(storage)) {
|
||||
/** Is stringified in storage, but parsed in storageService */
|
||||
if (key === 'auth_params') {
|
||||
continue
|
||||
}
|
||||
const value = await application.diskStorageService.getValue(key)
|
||||
expect(storage[key]).to.equal(value)
|
||||
}
|
||||
|
||||
console.warn('Expecting exception due to deiniting application while trying to renew session')
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('2020-01-15 migration with no account and no passcode', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Create legacy migrations value so that base migration detects old app */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
/** Create arbitrary storage values and make sure they're migrated */
|
||||
const storage = {
|
||||
foo: 'bar',
|
||||
zar: 'tar',
|
||||
har: 'car',
|
||||
}
|
||||
for (const key of Object.keys(storage)) {
|
||||
await application.deviceInterface.setRawStorageValue(key, storage[key])
|
||||
}
|
||||
|
||||
/** Create item and store it in db */
|
||||
const notePayload = Factory.createNotePayload()
|
||||
await application.deviceInterface.saveRawDatabasePayload(notePayload.ejected(), application.identifier)
|
||||
|
||||
/** Run migration */
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: (_challenge) => {
|
||||
return null
|
||||
},
|
||||
})
|
||||
await application.launch(true)
|
||||
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone)
|
||||
|
||||
/** Should be decrypted */
|
||||
const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default)
|
||||
const valueStore = application.diskStorageService.values[storageMode]
|
||||
expect(valueStore.content_type).to.not.be.ok
|
||||
const rootKey = await application.protocolService.getRootKey()
|
||||
expect(rootKey).to.not.be.ok
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyNone)
|
||||
|
||||
expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok
|
||||
|
||||
/** Expect note is decrypted */
|
||||
expect(application.itemManager.getDisplayableNotes().length).to.equal(1)
|
||||
const retrievedNote = application.itemManager.getDisplayableNotes()[0]
|
||||
expect(retrievedNote.uuid).to.equal(notePayload.uuid)
|
||||
expect(retrievedNote.content.text).to.equal(notePayload.content.text)
|
||||
|
||||
/** Ensure arbitrary values have been migrated */
|
||||
for (const key of Object.keys(storage)) {
|
||||
const value = await application.diskStorageService.getValue(key)
|
||||
expect(storage[key]).to.equal(value)
|
||||
}
|
||||
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
/**
|
||||
* This test will pass but sync afterwards will not be successful
|
||||
* as we are using a random value for the legacy session token
|
||||
*/
|
||||
it('2020-01-15 migration from app v1.0.1 with account only', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Create legacy migrations value so that base migration detects old app */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
const operator001 = new SNProtocolOperator001(new FakeWebCrypto())
|
||||
const identifier = 'foo'
|
||||
|
||||
/** Create old version account parameters */
|
||||
const password = 'tar'
|
||||
const accountKey = await operator001.createRootKey(identifier, password)
|
||||
|
||||
/** Create arbitrary storage values and make sure they're migrated */
|
||||
const storage = {
|
||||
mk: accountKey.masterKey,
|
||||
pw: accountKey.serverPassword,
|
||||
jwt: 'anything',
|
||||
/** Legacy versions would store json strings inside of embedded storage */
|
||||
auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()),
|
||||
user: JSON.stringify({ uuid: 'anything', email: 'anything' }),
|
||||
}
|
||||
for (const key of Object.keys(storage)) {
|
||||
await application.deviceInterface.setRawStorageValue(key, storage[key])
|
||||
}
|
||||
/** Create encrypted item and store it in db */
|
||||
const notePayload = Factory.createNotePayload()
|
||||
const noteEncryptionParams = await operator001.generateEncryptedParametersAsync(notePayload, accountKey)
|
||||
const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams })
|
||||
await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier)
|
||||
|
||||
/** Run migration */
|
||||
const promptValueReply = (prompts) => {
|
||||
const values = []
|
||||
for (const prompt of prompts) {
|
||||
/** We will be prompted to reauthetnicate our session, not relevant to this test
|
||||
* but pass any value to avoid exception
|
||||
*/
|
||||
values.push(CreateChallengeValue(prompt, 'foo'))
|
||||
}
|
||||
return values
|
||||
}
|
||||
const receiveChallenge = async (challenge) => {
|
||||
application.addChallengeObserver(challenge, {
|
||||
onInvalidValue: (value) => {
|
||||
const values = promptValueReply([value.prompt])
|
||||
application.submitValuesForChallenge(challenge, values)
|
||||
},
|
||||
})
|
||||
const initialValues = promptValueReply(challenge.prompts)
|
||||
application.submitValuesForChallenge(challenge, initialValues)
|
||||
}
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: receiveChallenge,
|
||||
})
|
||||
await application.launch(true)
|
||||
expect(application.sessionManager.online()).to.equal(true)
|
||||
expect(application.sessionManager.getUser()).to.be.ok
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly)
|
||||
/** Should be decrypted */
|
||||
const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default)
|
||||
const valueStore = application.diskStorageService.values[storageMode]
|
||||
expect(valueStore.content_type).to.not.be.ok
|
||||
/** Embedded value should match */
|
||||
const migratedKeyParams = await application.diskStorageService.getValue(
|
||||
StorageKey.RootKeyParams,
|
||||
StorageValueModes.Nonwrapped,
|
||||
)
|
||||
expect(migratedKeyParams).to.eql(accountKey.keyParams.getPortableValue())
|
||||
const rootKey = await application.protocolService.getRootKey()
|
||||
expect(rootKey).to.be.ok
|
||||
|
||||
expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('auth_params')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('jwt')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('ak')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('mk')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('pw')).to.not.be.ok
|
||||
|
||||
const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped)
|
||||
expect(typeof keyParams).to.equal('object')
|
||||
|
||||
expect(rootKey.masterKey).to.equal(accountKey.masterKey)
|
||||
expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey)
|
||||
expect(rootKey.serverPassword).to.not.be.ok
|
||||
expect(rootKey.keyVersion).to.equal(ProtocolVersion.V001)
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyOnly)
|
||||
|
||||
/** Expect note is decrypted */
|
||||
expect(application.itemManager.getDisplayableNotes().length).to.equal(1)
|
||||
const retrievedNote = application.itemManager.getDisplayableNotes()[0]
|
||||
expect(retrievedNote.uuid).to.equal(notePayload.uuid)
|
||||
expect(retrievedNote.content.text).to.equal(notePayload.content.text)
|
||||
|
||||
/** Ensure arbitrary values have been migrated */
|
||||
for (const key of Object.keys(storage)) {
|
||||
/** Is stringified in storage, but parsed in storageService */
|
||||
const value = await application.diskStorageService.getValue(key)
|
||||
if (key === 'auth_params') {
|
||||
continue
|
||||
} else if (key === 'user') {
|
||||
expect(storage[key]).to.equal(JSON.stringify(value))
|
||||
} else {
|
||||
expect(storage[key]).to.equal(value)
|
||||
}
|
||||
}
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('2020-01-15 migration from 002 app with account and passcode but missing offlineParams.version', async function () {
|
||||
/**
|
||||
* There was an issue where if the user had offlineParams but it was missing the version key,
|
||||
* the user could not get past the passcode migration screen.
|
||||
*/
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Create legacy migrations value so that base migration detects old app */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
const operator002 = new SNProtocolOperator002(new FakeWebCrypto())
|
||||
const identifier = 'foo'
|
||||
const passcode = 'bar'
|
||||
/** Create old version passcode parameters */
|
||||
const passcodeKey = await operator002.createRootKey(identifier, passcode)
|
||||
|
||||
/** The primary chaos agent */
|
||||
const offlineParams = passcodeKey.keyParams.getPortableValue()
|
||||
omitInPlace(offlineParams, ['version'])
|
||||
|
||||
await application.deviceInterface.setRawStorageValue('offlineParams', JSON.stringify(offlineParams))
|
||||
|
||||
/** Create old version account parameters */
|
||||
const password = 'tar'
|
||||
const accountKey = await operator002.createRootKey(identifier, password)
|
||||
|
||||
/** Create legacy storage and encrypt it with passcode */
|
||||
const embeddedStorage = {
|
||||
mk: accountKey.masterKey,
|
||||
ak: accountKey.dataAuthenticationKey,
|
||||
pw: accountKey.serverPassword,
|
||||
jwt: 'anything',
|
||||
/** Legacy versions would store json strings inside of embedded storage */
|
||||
auth_params: JSON.stringify(accountKey.keyParams.getPortableValue()),
|
||||
user: JSON.stringify({ uuid: 'anything', email: 'anything' }),
|
||||
}
|
||||
const storagePayload = new DecryptedPayload({
|
||||
uuid: await operator002.crypto.generateUUID(),
|
||||
content_type: ContentType.EncryptedStorage,
|
||||
content: {
|
||||
storage: embeddedStorage,
|
||||
},
|
||||
})
|
||||
const encryptionParams = await operator002.generateEncryptedParametersAsync(storagePayload, passcodeKey)
|
||||
const persistPayload = new EncryptedPayload({ ...storagePayload, ...encryptionParams })
|
||||
await application.deviceInterface.setRawStorageValue('encryptedStorage', JSON.stringify(persistPayload))
|
||||
|
||||
/** Create encrypted item and store it in db */
|
||||
const notePayload = Factory.createNotePayload()
|
||||
const noteEncryptionParams = await operator002.generateEncryptedParametersAsync(notePayload, accountKey)
|
||||
const noteEncryptedPayload = new EncryptedPayload({ ...notePayload, ...noteEncryptionParams })
|
||||
await application.deviceInterface.saveRawDatabasePayload(noteEncryptedPayload, application.identifier)
|
||||
|
||||
/** Runs migration */
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: async (challenge) => {
|
||||
application.submitValuesForChallenge(challenge, [CreateChallengeValue(challenge.prompts[0], passcode)])
|
||||
},
|
||||
})
|
||||
await application.launch(true)
|
||||
expect(application.sessionManager.online()).to.equal(true)
|
||||
expect(application.sessionManager.getUser()).to.be.ok
|
||||
expect(application.protocolService.rootKeyEncryption.keyMode).to.equal(KeyMode.RootKeyPlusWrapper)
|
||||
/** Should be decrypted */
|
||||
const storageMode = application.diskStorageService.domainKeyForMode(StorageValueModes.Default)
|
||||
const valueStore = application.diskStorageService.values[storageMode]
|
||||
expect(valueStore.content_type).to.not.be.ok
|
||||
/** Embedded value should match */
|
||||
const migratedKeyParams = await application.diskStorageService.getValue(
|
||||
StorageKey.RootKeyParams,
|
||||
StorageValueModes.Nonwrapped,
|
||||
)
|
||||
expect(migratedKeyParams).to.eql(accountKey.keyParams.getPortableValue())
|
||||
const rootKey = await application.protocolService.getRootKey()
|
||||
expect(rootKey).to.be.ok
|
||||
|
||||
expect(await application.deviceInterface.getRawStorageValue('migrations')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('auth_params')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('jwt')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('ak')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('mk')).to.not.be.ok
|
||||
expect(await application.deviceInterface.getRawStorageValue('pw')).to.not.be.ok
|
||||
|
||||
const keyParams = await application.diskStorageService.getValue(StorageKey.RootKeyParams, StorageValueModes.Nonwrapped)
|
||||
expect(typeof keyParams).to.equal('object')
|
||||
|
||||
expect(rootKey.masterKey).to.equal(accountKey.masterKey)
|
||||
expect(rootKey.dataAuthenticationKey).to.equal(accountKey.dataAuthenticationKey)
|
||||
expect(rootKey.serverPassword).to.not.be.ok
|
||||
expect(rootKey.keyVersion).to.equal(ProtocolVersion.V002)
|
||||
|
||||
/** Expect note is decrypted */
|
||||
expect(application.itemManager.getDisplayableNotes().length).to.equal(1)
|
||||
const retrievedNote = application.itemManager.getDisplayableNotes()[0]
|
||||
expect(retrievedNote.uuid).to.equal(notePayload.uuid)
|
||||
expect(retrievedNote.content.text).to.equal(notePayload.content.text)
|
||||
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
})
|
||||
@@ -3,7 +3,7 @@ chai.use(chaiAsPromised)
|
||||
const expect = chai.expect
|
||||
|
||||
describe('migrations', () => {
|
||||
const allMigrations = ['2.0.0', '2.0.15', '2.7.0', '2.20.0', '2.36.0', '2.42.0']
|
||||
const allMigrations = ['2.0.15', '2.7.0', '2.20.0', '2.36.0', '2.42.0']
|
||||
|
||||
beforeEach(async () => {
|
||||
localStorage.clear()
|
||||
@@ -25,34 +25,13 @@ describe('migrations', () => {
|
||||
})
|
||||
|
||||
it('should return correct required migrations if stored version is 2.0.0', async function () {
|
||||
expect((await SNMigrationService.getRequiredMigrations('2.0.0')).length).to.equal(allMigrations.length - 1)
|
||||
expect((await SNMigrationService.getRequiredMigrations('2.0.0')).length).to.equal(allMigrations.length)
|
||||
})
|
||||
|
||||
it('should return 0 required migrations if stored version is futuristic', async function () {
|
||||
expect((await SNMigrationService.getRequiredMigrations('100.0.1')).length).to.equal(0)
|
||||
})
|
||||
|
||||
it('after running base migration, legacy structure should set version as 1.0.0', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Set up 1.0.0 structure with tell-tale storage key */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
await application.migrationService.runBaseMigrationPreRun()
|
||||
expect(await application.migrationService.getStoredSnjsVersion()).to.equal('1.0.0')
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('after running base migration, 2.0.0 structure set version as 2.0.0', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Set up 2.0.0 structure with tell-tale storage key */
|
||||
await application.deviceInterface.setRawStorageValue(
|
||||
namespacedKey(application.identifier, 'last_migration_timestamp'),
|
||||
'anything',
|
||||
)
|
||||
await application.migrationService.runBaseMigrationPreRun()
|
||||
expect(await application.migrationService.getStoredSnjsVersion()).to.equal('2.0.0')
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('after running base migration with no present storage values, should set version to current', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
await application.migrationService.runBaseMigrationPreRun()
|
||||
@@ -60,18 +39,6 @@ describe('migrations', () => {
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('after running all migrations from a 1.0.0 installation, should set stored version to current', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Set up 1.0.0 structure with tell-tale storage key */
|
||||
await application.deviceInterface.setRawStorageValue('migrations', JSON.stringify(['anything']))
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: () => {},
|
||||
})
|
||||
await application.launch(true)
|
||||
expect(await application.migrationService.getStoredSnjsVersion()).to.equal(SnjsVersion)
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('after running all migrations from a 2.0.0 installation, should set stored version to current', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
/** Set up 2.0.0 structure with tell-tale storage key */
|
||||
@@ -84,24 +51,6 @@ describe('migrations', () => {
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('should be correct migration count coming from 1.0.0', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
await application.deviceInterface.setRawStorageValue('migrations', 'anything')
|
||||
await application.migrationService.runBaseMigrationPreRun()
|
||||
expect(await application.migrationService.getStoredSnjsVersion()).to.equal('1.0.0')
|
||||
const pendingMigrations = await SNMigrationService.getRequiredMigrations(
|
||||
await application.migrationService.getStoredSnjsVersion(),
|
||||
)
|
||||
expect(pendingMigrations.length).to.equal(allMigrations.length)
|
||||
expect(pendingMigrations[0].version()).to.equal('2.0.0')
|
||||
await application.prepareForLaunch({
|
||||
receiveChallenge: () => {},
|
||||
})
|
||||
await application.launch(true)
|
||||
expect(await application.migrationService.getStoredSnjsVersion()).to.equal(SnjsVersion)
|
||||
await Factory.safeDeinit(application)
|
||||
})
|
||||
|
||||
it('2.20.0 remove mfa migration', async function () {
|
||||
const application = await Factory.createAppWithRandNamespace()
|
||||
|
||||
|
||||
@@ -735,7 +735,7 @@ describe('importing', function () {
|
||||
}),
|
||||
)
|
||||
await application.deviceInterface.setRawStorageValue('standardnotes-snjs_version', '2.0.11')
|
||||
await application.deviceInterface.saveRawDatabasePayload(
|
||||
await application.deviceInterface.saveDatabaseEntry(
|
||||
{
|
||||
content:
|
||||
'003:9f2c7527eb8b2a1f8bfb3ea6b885403b6886bce2640843ebd57a6c479cbf7597:58e3322b-269a-4be3-a658-b035dffcd70f:9140b23a0fa989e224e292049f133154:SESTNOgIGf2+ZqmJdFnGU4EMgQkhKOzpZNoSzx76SJaImsayzctAgbUmJ+UU2gSQAHADS3+Z5w11bXvZgIrStTsWriwvYkNyyKmUPadKHNSBwOk4WeBZpWsA9gtI5zgI04Q5pvb8hS+kNW2j1DjM4YWqd0JQxMOeOrMIrxr/6Awn5TzYE+9wCbXZdYHyvRQcp9ui/G02ZJ67IA86vNEdjTTBAAWipWqTqKH9VDZbSQ2W/IOKfIquB373SFDKZb1S1NmBFvcoG2G7w//fAl/+ehYiL6UdiNH5MhXCDAOTQRFNfOh57HFDWVnz1VIp8X+VAPy6d9zzQH+8aws1JxHq/7BOhXrFE8UCueV6kERt9njgQxKJzd9AH32ShSiUB9X/sPi0fUXbS178xAZMJrNx3w==:eyJwd19ub25jZSI6IjRjYjEwM2FhODljZmY0NTYzYTkxMWQzZjM5NjU4M2NlZmM2ODMzYzY2Zjg4MGZiZWUwNmJkYTk0YzMxZjg2OGIiLCJwd19jb3N0IjoxMTAwMDAsImlkZW50aWZpZXIiOiJub3YyMzIyQGJpdGFyLmlvIiwidmVyc2lvbiI6IjAwMyIsIm9yaWdpbmF0aW9uIjoicmVnaXN0cmF0aW9uIn0=',
|
||||
|
||||
@@ -94,7 +94,7 @@ describe('model manager mapping', () => {
|
||||
const note = this.application.itemManager.getDisplayableNotes()[0]
|
||||
await this.application.itemManager.setItemDirty(note)
|
||||
const dirtyItems = this.application.itemManager.getDirtyItems()
|
||||
expect(dirtyItems.length).to.equal(1)
|
||||
expect(Uuids(dirtyItems).includes(note.uuid))
|
||||
})
|
||||
|
||||
it('set all items dirty', async function () {
|
||||
|
||||
@@ -642,7 +642,7 @@ describe('server session', function () {
|
||||
await app2Deinit
|
||||
|
||||
const deviceInterface = new WebDeviceInterface()
|
||||
const payloads = await deviceInterface.getAllRawDatabasePayloads(app2identifier)
|
||||
const payloads = await deviceInterface.getAllDatabaseEntries(app2identifier)
|
||||
expect(payloads).to.be.empty
|
||||
})
|
||||
|
||||
@@ -670,7 +670,7 @@ describe('server session', function () {
|
||||
await app2Deinit
|
||||
|
||||
const deviceInterface = new WebDeviceInterface()
|
||||
const payloads = await deviceInterface.getAllRawDatabasePayloads(app2identifier)
|
||||
const payloads = await deviceInterface.getAllDatabaseEntries(app2identifier)
|
||||
expect(payloads).to.be.empty
|
||||
})
|
||||
|
||||
|
||||
@@ -300,6 +300,7 @@ describe('storage manager', function () {
|
||||
await Factory.createSyncedNote(this.application)
|
||||
expect(await Factory.storagePayloadCount(this.application)).to.equal(BaseItemCounts.DefaultItems + 1)
|
||||
this.application = await Factory.signOutApplicationAndReturnNew(this.application)
|
||||
await Factory.sleep(0.1, 'Allow all untrackable singleton syncs to complete')
|
||||
expect(await Factory.storagePayloadCount(this.application)).to.equal(BaseItemCounts.DefaultItems)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -31,10 +31,7 @@ describe('offline syncing', () => {
|
||||
|
||||
it('should sync item with no passcode', async function () {
|
||||
let note = await Factory.createMappedNote(this.application)
|
||||
expect(this.application.itemManager.getDirtyItems().length).to.equal(1)
|
||||
|
||||
const rawPayloads1 = await this.application.diskStorageService.getAllRawPayloads()
|
||||
expect(rawPayloads1.length).to.equal(this.expectedItemCount)
|
||||
expect(Uuids(this.application.itemManager.getDirtyItems()).includes(note.uuid))
|
||||
|
||||
await this.application.syncService.sync(syncOptions)
|
||||
|
||||
|
||||
@@ -218,14 +218,21 @@ describe('online syncing', function () {
|
||||
it('retrieving new items should not mark them as dirty', async function () {
|
||||
const originalNote = await Factory.createSyncedNote(this.application)
|
||||
this.expectedItemCount++
|
||||
|
||||
this.application = await Factory.signOutApplicationAndReturnNew(this.application)
|
||||
this.application.syncService.addEventObserver((event) => {
|
||||
if (event === SyncEvent.SingleRoundTripSyncCompleted) {
|
||||
const note = this.application.items.findItem(originalNote.uuid)
|
||||
expect(note.dirty).to.not.be.ok
|
||||
}
|
||||
const promise = new Promise((resolve) => {
|
||||
this.application.syncService.addEventObserver(async (event) => {
|
||||
if (event === SyncEvent.SingleRoundTripSyncCompleted) {
|
||||
const note = this.application.items.findItem(originalNote.uuid)
|
||||
if (note) {
|
||||
expect(note.dirty).to.not.be.ok
|
||||
resolve()
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
await this.application.signIn(this.email, this.password, undefined, undefined, undefined, true)
|
||||
await promise
|
||||
})
|
||||
|
||||
it('allows saving of data after sign out', async function () {
|
||||
@@ -579,7 +586,7 @@ describe('online syncing', function () {
|
||||
await this.application.itemManager.setItemDirty(note)
|
||||
await this.application.syncService.sync(syncOptions)
|
||||
this.expectedItemCount++
|
||||
const rawPayloads = await this.application.syncService.getDatabasePayloads()
|
||||
const rawPayloads = await this.application.diskStorageService.getAllRawPayloads()
|
||||
const notePayload = rawPayloads.find((p) => p.content_type === ContentType.Note)
|
||||
expect(typeof notePayload.content).to.equal('string')
|
||||
})
|
||||
@@ -651,8 +658,7 @@ describe('online syncing', function () {
|
||||
await this.application.syncService.clearSyncPositionTokens()
|
||||
await this.application.payloadManager.resetState()
|
||||
await this.application.itemManager.resetState()
|
||||
const databasePayloads = await this.application.diskStorageService.getAllRawPayloads()
|
||||
await this.application.syncService.loadDatabasePayloads(databasePayloads)
|
||||
await this.application.syncService.loadDatabasePayloads()
|
||||
await this.application.syncService.sync(syncOptions)
|
||||
|
||||
const newRawPayloads = await this.application.diskStorageService.getAllRawPayloads()
|
||||
@@ -672,7 +678,9 @@ describe('online syncing', function () {
|
||||
const payload = Factory.createStorageItemPayload(contentTypes[Math.floor(i / 2)])
|
||||
originalPayloads.push(payload)
|
||||
}
|
||||
const { contentTypePriorityPayloads } = GetSortedPayloadsByPriority(originalPayloads, ['C', 'A', 'B'])
|
||||
const { contentTypePriorityPayloads } = GetSortedPayloadsByPriority(originalPayloads, {
|
||||
contentTypePriority: ['C', 'A', 'B'],
|
||||
})
|
||||
expect(contentTypePriorityPayloads[0].content_type).to.equal('C')
|
||||
expect(contentTypePriorityPayloads[2].content_type).to.equal('A')
|
||||
expect(contentTypePriorityPayloads[4].content_type).to.equal('B')
|
||||
@@ -685,14 +693,10 @@ describe('online syncing', function () {
|
||||
await this.application.syncService.sync(syncOptions)
|
||||
|
||||
this.application = await Factory.signOutApplicationAndReturnNew(this.application)
|
||||
const rawPayloads = await this.application.diskStorageService.getAllRawPayloads()
|
||||
expect(rawPayloads.length).to.equal(BaseItemCounts.DefaultItems)
|
||||
|
||||
await this.application.signIn(this.email, this.password, undefined, undefined, undefined, true)
|
||||
|
||||
this.application.syncService.ut_setDatabaseLoaded(false)
|
||||
const databasePayloads = await this.application.diskStorageService.getAllRawPayloads()
|
||||
await this.application.syncService.loadDatabasePayloads(databasePayloads)
|
||||
await this.application.syncService.loadDatabasePayloads()
|
||||
await this.application.syncService.sync(syncOptions)
|
||||
|
||||
const items = await this.application.itemManager.items
|
||||
|
||||
@@ -80,8 +80,6 @@
|
||||
<script type="module" src="protection.test.js"></script>
|
||||
<script type="module" src="singletons.test.js"></script>
|
||||
<script type="module" src="migrations/migration.test.js"></script>
|
||||
<script type="module" src="migrations/2020-01-15-web.test.js"></script>
|
||||
<script type="module" src="migrations/2020-01-15-mobile.test.js"></script>
|
||||
<script type="module" src="migrations/tags-to-folders.test.js"></script>
|
||||
<script type="module" src="history.test.js"></script>
|
||||
<script type="module" src="actions.test.js"></script>
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"clean": "rm -fr dist",
|
||||
"prebuild": "yarn clean",
|
||||
"build": "yarn tsc && webpack --config webpack.prod.js",
|
||||
"watch": "webpack --config webpack.prod.js --watch",
|
||||
"docs": "jsdoc -c jsdoc.json",
|
||||
"tsc": "tsc --project lib/tsconfig.json && tscpaths -p lib/tsconfig.json -s lib -o dist/@types",
|
||||
"lint": "yarn lint:eslint lib",
|
||||
|
||||
@@ -21,6 +21,8 @@ import {
|
||||
DecryptedItem,
|
||||
EditorIdentifier,
|
||||
FeatureIdentifier,
|
||||
Environment,
|
||||
ApplicationOptionsDefaults,
|
||||
} from '@standardnotes/snjs'
|
||||
import { makeObservable, observable } from 'mobx'
|
||||
import { PanelResizedData } from '@/Types/PanelResizedData'
|
||||
@@ -75,6 +77,8 @@ export class WebApplication extends SNApplication implements WebApplicationInter
|
||||
defaultHost: defaultSyncServerHost,
|
||||
appVersion: deviceInterface.appVersion,
|
||||
webSocketUrl: webSocketUrl,
|
||||
loadBatchSize:
|
||||
deviceInterface.environment === Environment.Mobile ? 100 : ApplicationOptionsDefaults.loadBatchSize,
|
||||
})
|
||||
|
||||
makeObservable(this, {
|
||||
|
||||
@@ -140,6 +140,39 @@ export class Database {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* This function is actually unused, but implemented to conform to protocol in case it is eventually needed.
|
||||
* We could remove implementation and throw instead, but it might be better to offer a functional alternative instead.
|
||||
*/
|
||||
public async getPayloadsForKeys(keys: string[]): Promise<any[]> {
|
||||
const db = (await this.openDatabase()) as IDBDatabase
|
||||
return new Promise((resolve) => {
|
||||
const objectStore = db.transaction(STORE_NAME).objectStore(STORE_NAME)
|
||||
const payloads: any = []
|
||||
let numComplete = 0
|
||||
for (const key of keys) {
|
||||
const getRequest = objectStore.get(key)
|
||||
getRequest.onsuccess = (event) => {
|
||||
const target = event.target as any
|
||||
const result = target.result
|
||||
if (result) {
|
||||
payloads.push(result)
|
||||
}
|
||||
numComplete++
|
||||
if (numComplete === keys.length) {
|
||||
resolve(payloads)
|
||||
}
|
||||
}
|
||||
getRequest.onerror = () => {
|
||||
numComplete++
|
||||
if (numComplete === keys.length) {
|
||||
resolve(payloads)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
public async getAllKeys(): Promise<string[]> {
|
||||
const db = (await this.openDatabase()) as IDBDatabase
|
||||
|
||||
|
||||
@@ -2,13 +2,16 @@ import {
|
||||
SNApplication,
|
||||
ApplicationIdentifier,
|
||||
Environment,
|
||||
LegacyRawKeychainValue,
|
||||
RawKeychainValue,
|
||||
TransferPayload,
|
||||
NamespacedRootKeyInKeychain,
|
||||
extendArray,
|
||||
WebOrDesktopDeviceInterface,
|
||||
Platform,
|
||||
FullyFormedTransferPayload,
|
||||
DatabaseLoadOptions,
|
||||
GetSortedPayloadsByPriority,
|
||||
DatabaseFullEntryLoadChunk,
|
||||
DatabaseFullEntryLoadChunkResponse,
|
||||
} from '@standardnotes/snjs'
|
||||
import { Database } from '../Database'
|
||||
|
||||
@@ -72,17 +75,6 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface
|
||||
return result
|
||||
}
|
||||
|
||||
async getAllRawStorageKeyValues() {
|
||||
const results = []
|
||||
for (const key of Object.keys(localStorage)) {
|
||||
results.push({
|
||||
key: key,
|
||||
value: localStorage[key],
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
async setRawStorageValue(key: string, value: string) {
|
||||
localStorage.setItem(key, value)
|
||||
}
|
||||
@@ -111,23 +103,63 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface
|
||||
}) as Promise<{ isNewDatabase?: boolean } | undefined>
|
||||
}
|
||||
|
||||
async getAllRawDatabasePayloads(identifier: ApplicationIdentifier) {
|
||||
async getDatabaseLoadChunks(
|
||||
options: DatabaseLoadOptions,
|
||||
identifier: string,
|
||||
): Promise<DatabaseFullEntryLoadChunkResponse> {
|
||||
const entries = await this.getAllDatabaseEntries(identifier)
|
||||
const sorted = GetSortedPayloadsByPriority(entries, options)
|
||||
|
||||
const itemsKeysChunk: DatabaseFullEntryLoadChunk = {
|
||||
entries: sorted.itemsKeyPayloads,
|
||||
}
|
||||
|
||||
const contentTypePriorityChunk: DatabaseFullEntryLoadChunk = {
|
||||
entries: sorted.contentTypePriorityPayloads,
|
||||
}
|
||||
|
||||
const remainingPayloadsChunks: DatabaseFullEntryLoadChunk[] = []
|
||||
for (let i = 0; i < sorted.remainingPayloads.length; i += options.batchSize) {
|
||||
remainingPayloadsChunks.push({
|
||||
entries: sorted.remainingPayloads.slice(i, i + options.batchSize),
|
||||
})
|
||||
}
|
||||
|
||||
const result: DatabaseFullEntryLoadChunkResponse = {
|
||||
fullEntries: {
|
||||
itemsKeys: itemsKeysChunk,
|
||||
remainingChunks: [contentTypePriorityChunk, ...remainingPayloadsChunks],
|
||||
},
|
||||
remainingChunksItemCount: sorted.contentTypePriorityPayloads.length + sorted.remainingPayloads.length,
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async getAllDatabaseEntries(identifier: ApplicationIdentifier) {
|
||||
return this.databaseForIdentifier(identifier).getAllPayloads()
|
||||
}
|
||||
|
||||
async saveRawDatabasePayload(payload: TransferPayload, identifier: ApplicationIdentifier) {
|
||||
getDatabaseEntries<T extends FullyFormedTransferPayload = FullyFormedTransferPayload>(
|
||||
identifier: string,
|
||||
keys: string[],
|
||||
): Promise<T[]> {
|
||||
return this.databaseForIdentifier(identifier).getPayloadsForKeys(keys)
|
||||
}
|
||||
|
||||
async saveDatabaseEntry(payload: TransferPayload, identifier: ApplicationIdentifier) {
|
||||
return this.databaseForIdentifier(identifier).savePayload(payload)
|
||||
}
|
||||
|
||||
async saveRawDatabasePayloads(payloads: TransferPayload[], identifier: ApplicationIdentifier) {
|
||||
async saveDatabaseEntries(payloads: TransferPayload[], identifier: ApplicationIdentifier) {
|
||||
return this.databaseForIdentifier(identifier).savePayloads(payloads)
|
||||
}
|
||||
|
||||
async removeRawDatabasePayloadWithId(id: string, identifier: ApplicationIdentifier) {
|
||||
async removeDatabaseEntry(id: string, identifier: ApplicationIdentifier) {
|
||||
return this.databaseForIdentifier(identifier).deletePayload(id)
|
||||
}
|
||||
|
||||
async removeAllRawDatabasePayloads(identifier: ApplicationIdentifier) {
|
||||
async removeAllDatabaseEntries(identifier: ApplicationIdentifier) {
|
||||
return this.databaseForIdentifier(identifier).clearAllPayloads()
|
||||
}
|
||||
|
||||
@@ -141,16 +173,6 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface
|
||||
return keychain[identifier]
|
||||
}
|
||||
|
||||
async getDatabaseKeys(): Promise<string[]> {
|
||||
const keys: string[] = []
|
||||
|
||||
for (const database of this.databases) {
|
||||
extendArray(keys, await database.getAllKeys())
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
async setNamespacedKeychainValue(value: NamespacedRootKeyInKeychain, identifier: ApplicationIdentifier) {
|
||||
let keychain = await this.getKeychainValue()
|
||||
|
||||
@@ -186,10 +208,6 @@ export abstract class WebOrDesktopDevice implements WebOrDesktopDeviceInterface
|
||||
}
|
||||
}
|
||||
|
||||
setLegacyRawKeychainValue(value: LegacyRawKeychainValue): Promise<void> {
|
||||
return this.setKeychainValue(value)
|
||||
}
|
||||
|
||||
abstract getKeychainValue(): Promise<RawKeychainValue>
|
||||
|
||||
abstract setKeychainValue(value: unknown): Promise<void>
|
||||
|
||||
11
yarn.lock
11
yarn.lock
@@ -5883,6 +5883,7 @@ __metadata:
|
||||
react-native-fs: ^2.20.0
|
||||
react-native-iap: ^12.4.4
|
||||
react-native-keychain: "standardnotes/react-native-keychain#d277d360494cbd02be4accb4a360772a8e0e97b6"
|
||||
react-native-mmkv: ^2.5.1
|
||||
react-native-privacy-snapshot: "standardnotes/react-native-privacy-snapshot#653e904c90fc6f2b578da59138f2bfe5d7f942fe"
|
||||
react-native-share: ^8.0.0
|
||||
react-native-version-info: ^1.1.1
|
||||
@@ -25615,6 +25616,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"react-native-mmkv@npm:^2.5.1":
|
||||
version: 2.5.1
|
||||
resolution: "react-native-mmkv@npm:2.5.1"
|
||||
peerDependencies:
|
||||
react: "*"
|
||||
react-native: "*"
|
||||
checksum: 6f0cf484e71d8069c9b3cdb57b76eafaca40aa75f359beb6959c77d0ef66d0481d4459b1ffa94640170ce4744e337fefb38b8ccf6e1a3c3663561ede5f7a2c20
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"react-native-privacy-snapshot@standardnotes/react-native-privacy-snapshot#653e904c90fc6f2b578da59138f2bfe5d7f942fe":
|
||||
version: 1.0.0
|
||||
resolution: "react-native-privacy-snapshot@https://github.com/standardnotes/react-native-privacy-snapshot.git#commit=653e904c90fc6f2b578da59138f2bfe5d7f942fe"
|
||||
|
||||
Reference in New Issue
Block a user