rmorawski/standardnotes-app-web
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

internal: incomplete vault systems behind feature flag (#2340)

Browse Source
This commit is contained in:
Mo
2023-06-30 09:01:56 -05:00
committed by GitHub
parent d16e401bb9
commit b032eb9c9b
638 changed files with 20321 additions and 4813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
packages/snjs/lib/Services/Sync/Account/Operation.ts
View File

@@ -21,17 +21,15 @@ export class AccountSyncOperation {
* @param receiver A function that receives callback multiple times during the operation
*/
constructor(
private payloads: ServerSyncPushContextualPayload[],
public readonly payloads: ServerSyncPushContextualPayload[],
private receiver: ResponseSignalReceiver<ServerSyncResponse>,
private lastSyncToken: string,
private paginationToken: string,
private apiService: SNApiService,
public readonly options: {
syncToken?: string
paginationToken?: string
sharedVaultUuids?: string[]
},
) {
this.payloads = payloads
this.lastSyncToken = lastSyncToken
this.paginationToken = paginationToken
this.apiService = apiService
this.receiver = receiver
this.pendingPayloads = payloads.slice()
}
@@ -55,13 +53,19 @@ export class AccountSyncOperation {
})
const payloads = this.popPayloads(this.upLimit)
const rawResponse = await this.apiService.sync(payloads, this.lastSyncToken, this.paginationToken, this.downLimit)
const rawResponse = await this.apiService.sync(
payloads,
this.options.syncToken,
this.options.paginationToken,
this.downLimit,
this.options.sharedVaultUuids,
)
const response = new ServerSyncResponse(rawResponse)
this.responses.push(response)
this.lastSyncToken = response.lastSyncToken as string
this.paginationToken = response.paginationToken as string
this.options.syncToken = response.lastSyncToken as string
this.options.paginationToken = response.paginationToken as string
try {
await this.receiver(SyncSignal.Response, response)
@@ -75,7 +79,7 @@ export class AccountSyncOperation {
}
get done() {
return this.pendingPayloads.length === 0 && !this.paginationToken
return this.pendingPayloads.length === 0 && !this.options.paginationToken
}
private get pendingUploadCount() {

122
packages/snjs/lib/Services/Sync/Account/Response.ts
View File

@@ -1,27 +1,36 @@
import {
ApiEndpointParam,
ConflictParams,
ConflictType,
SharedVaultInviteServerHash,
SharedVaultServerHash,
HttpError,
HttpResponse,
isErrorResponse,
RawSyncResponse,
ServerItemResponse,
UserEventServerHash,
AsymmetricMessageServerHash,
} from '@standardnotes/responses'
import {
FilterDisallowedRemotePayloadsAndMap,
CreateServerSyncSavedPayload,
ServerSyncSavedContextualPayload,
FilteredServerItem,
TrustedConflictParams,
} from '@standardnotes/models'
import { deepFreeze } from '@standardnotes/utils'
import { TrustedServerConflictMap } from './ServerConflictMap'
export class ServerSyncResponse {
public readonly savedPayloads: ServerSyncSavedContextualPayload[]
public readonly retrievedPayloads: FilteredServerItem[]
public readonly uuidConflictPayloads: FilteredServerItem[]
public readonly dataConflictPayloads: FilteredServerItem[]
public readonly rejectedPayloads: FilteredServerItem[]
readonly savedPayloads: ServerSyncSavedContextualPayload[]
readonly retrievedPayloads: FilteredServerItem[]
readonly conflicts: TrustedServerConflictMap
readonly asymmetricMessages: AsymmetricMessageServerHash[]
readonly vaults: SharedVaultServerHash[]
readonly vaultInvites: SharedVaultInviteServerHash[]
readonly userEvents: UserEventServerHash[]
private readonly rawConflictObjects: ConflictParams[]
private successResponseData: RawSyncResponse | undefined
@@ -32,6 +41,10 @@ export class ServerSyncResponse {
this.successResponseData = rawResponse.data
}
const conflicts = this.successResponseData?.conflicts || []
const legacyConflicts = this.successResponseData?.unsaved || []
this.rawConflictObjects = conflicts.concat(legacyConflicts)
this.savedPayloads = FilterDisallowedRemotePayloadsAndMap(this.successResponseData?.saved_items || []).map(
(rawItem) => {
return CreateServerSyncSavedPayload(rawItem)
@@ -40,15 +53,53 @@ export class ServerSyncResponse {
this.retrievedPayloads = FilterDisallowedRemotePayloadsAndMap(this.successResponseData?.retrieved_items || [])
this.dataConflictPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawDataConflictItems)
this.conflicts = this.filterConflicts()
this.uuidConflictPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawUuidConflictItems)
this.vaults = this.successResponseData?.shared_vaults || []
this.rejectedPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawRejectedPayloads)
this.vaultInvites = this.successResponseData?.shared_vault_invites || []
this.asymmetricMessages = this.successResponseData?.asymmetric_messages || []
this.userEvents = this.successResponseData?.user_events || []
deepFreeze(this)
}
private filterConflicts(): TrustedServerConflictMap {
const conflicts = this.rawConflictObjects
const trustedConflicts: TrustedServerConflictMap = {}
for (const conflict of conflicts) {
let serverItem: FilteredServerItem | undefined
let unsavedItem: FilteredServerItem | undefined
if (conflict.unsaved_item) {
unsavedItem = FilterDisallowedRemotePayloadsAndMap([conflict.unsaved_item])[0]
}
if (conflict.server_item) {
serverItem = FilterDisallowedRemotePayloadsAndMap([conflict.server_item])[0]
}
if (!trustedConflicts[conflict.type]) {
trustedConflicts[conflict.type] = []
}
const conflictArray = trustedConflicts[conflict.type]
if (conflictArray) {
const entry: TrustedConflictParams = <TrustedConflictParams>{
type: conflict.type,
server_item: serverItem,
unsaved_item: unsavedItem,
}
conflictArray.push(entry)
}
}
return trustedConflicts
}
public get error(): HttpError | undefined {
return isErrorResponse(this.rawResponse) ? this.rawResponse.data?.error : undefined
}
@@ -66,56 +117,9 @@ export class ServerSyncResponse {
}
public get numberOfItemsInvolved(): number {
return this.allFullyFormedPayloads.length
}
const allPayloads = [...this.retrievedPayloads, ...this.rawConflictObjects]
private get allFullyFormedPayloads(): FilteredServerItem[] {
return [
...this.retrievedPayloads,
...this.dataConflictPayloads,
...this.uuidConflictPayloads,
...this.rejectedPayloads,
]
}
private get rawUuidConflictItems(): ServerItemResponse[] {
return this.rawConflictObjects
.filter((conflict) => {
return conflict.type === ConflictType.UuidConflict
})
.map((conflict) => {
return conflict.unsaved_item || conflict.item!
})
}
private get rawDataConflictItems(): ServerItemResponse[] {
return this.rawConflictObjects
.filter((conflict) => {
return conflict.type === ConflictType.ConflictingData
})
.map((conflict) => {
return conflict.server_item || conflict.item!
})
}
private get rawRejectedPayloads(): ServerItemResponse[] {
return this.rawConflictObjects
.filter((conflict) => {
return (
conflict.type === ConflictType.ContentTypeError ||
conflict.type === ConflictType.ContentError ||
conflict.type === ConflictType.ReadOnlyError
)
})
.map((conflict) => {
return conflict.unsaved_item!
})
}
private get rawConflictObjects(): ConflictParams[] {
const conflicts = this.successResponseData?.conflicts || []
const legacyConflicts = this.successResponseData?.unsaved || []
return conflicts.concat(legacyConflicts)
return allPayloads.length
}
public get hasError(): boolean {

47
packages/snjs/lib/Services/Sync/Account/ResponseResolver.ts
View File

@@ -1,3 +1,4 @@
import { ConflictParams, ConflictType } from '@standardnotes/responses'
import {
ImmutablePayloadCollection,
HistoryMap,
@@ -11,13 +12,12 @@ import {
DeltaRemoteRejected,
DeltaEmit,
} from '@standardnotes/models'
import { DecryptedServerConflictMap } from './ServerConflictMap'
type PayloadSet = {
retrievedPayloads: FullyFormedPayloadInterface[]
savedPayloads: ServerSyncSavedContextualPayload[]
uuidConflictPayloads: FullyFormedPayloadInterface[]
dataConflictPayloads: FullyFormedPayloadInterface[]
rejectedPayloads: FullyFormedPayloadInterface[]
conflicts: DecryptedServerConflictMap
}
/**
@@ -39,8 +39,8 @@ export class ServerSyncResponseResolver {
emits.push(this.processRetrievedPayloads())
emits.push(this.processSavedPayloads())
emits.push(this.processUuidConflictPayloads())
emits.push(this.processDataConflictPayloads())
emits.push(this.processUuidConflictUnsavedPayloads())
emits.push(this.processDataConflictServerPayloads())
emits.push(this.processRejectedPayloads())
return emits
@@ -60,27 +60,42 @@ export class ServerSyncResponseResolver {
return delta.result()
}
private processDataConflictPayloads(): DeltaEmit {
const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.dataConflictPayloads)
private getConflictsForType<T extends ConflictParams<FullyFormedPayloadInterface>>(type: ConflictType): T[] {
const results = this.payloadSet.conflicts[type] || []
const delta = new DeltaRemoteDataConflicts(this.baseCollection, collection, this.historyMap)
return results as T[]
}
private processDataConflictServerPayloads(): DeltaEmit {
const delta = new DeltaRemoteDataConflicts(
this.baseCollection,
this.getConflictsForType(ConflictType.ConflictingData),
this.historyMap,
)
return delta.result()
}
private processUuidConflictPayloads(): DeltaEmit {
const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.uuidConflictPayloads)
const delta = new DeltaRemoteUuidConflicts(this.baseCollection, collection)
private processUuidConflictUnsavedPayloads(): DeltaEmit {
const delta = new DeltaRemoteUuidConflicts(this.baseCollection, this.getConflictsForType(ConflictType.UuidConflict))
return delta.result()
}
private processRejectedPayloads(): DeltaEmit {
const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.rejectedPayloads)
const conflicts = [
...this.getConflictsForType(ConflictType.ContentTypeError),
...this.getConflictsForType(ConflictType.ContentError),
...this.getConflictsForType(ConflictType.ReadOnlyError),
...this.getConflictsForType(ConflictType.UuidError),
...this.getConflictsForType(ConflictType.SharedVaultSnjsVersionError),
...this.getConflictsForType(ConflictType.SharedVaultInsufficientPermissionsError),
...this.getConflictsForType(ConflictType.SharedVaultNotMemberError),
...this.getConflictsForType(ConflictType.SharedVaultInvalidState),
]
const delta = new DeltaRemoteRejected(this.baseCollection, collection)
return delta.result()
const delta = new DeltaRemoteRejected(this.baseCollection, conflicts)
const result = delta.result()
return result
}
}

5
packages/snjs/lib/Services/Sync/Account/ServerConflictMap.ts Normal file
View File

@@ -0,0 +1,5 @@
import { ConflictType, ConflictParams } from '@standardnotes/responses'
import { FullyFormedPayloadInterface, TrustedConflictParams } from '@standardnotes/models'
export type TrustedServerConflictMap = Partial<Record<ConflictType, TrustedConflictParams[]>>
export type DecryptedServerConflictMap = Partial<Record<ConflictType, ConflictParams<FullyFormedPayloadInterface>[]>>

367
packages/snjs/lib/Services/Sync/SyncService.ts
View File

@@ -1,15 +1,15 @@
import { ConflictParams, ConflictType } from '@standardnotes/responses'
import { log, LoggingDomain } from './../../Logging'
import { AccountSyncOperation } from '@Lib/Services/Sync/Account/Operation'
import { ContentType } from '@standardnotes/common'
import {
Uuids,
extendArray,
isNotUndefined,
isNullOrUndefined,
removeFromIndex,
sleep,
subtractFromArray,
useBoolean,
Uuids,
} from '@standardnotes/utils'
import { ItemManager } from '@Lib/Services/Items/ItemManager'
import { OfflineSyncOperation } from '@Lib/Services/Sync/Offline/Operation'
@@ -56,6 +56,12 @@ import {
getIncrementedDirtyIndex,
getCurrentDirtyIndex,
ItemContent,
KeySystemItemsKeyContent,
KeySystemItemsKeyInterface,
FullyFormedTransferPayload,
ItemMutator,
isDecryptedOrDeletedItem,
MutationType,
} from '@standardnotes/models'
import {
AbstractService,
@@ -71,11 +77,14 @@ import {
SyncOptions,
SyncQueueStrategy,
SyncServiceInterface,
DiagnosticInfo,
EncryptionService,
DeviceInterface,
isFullEntryLoadChunkResponse,
isChunkFullEntry,
SyncEventReceivedSharedVaultInvitesData,
SyncEventReceivedRemoteSharedVaultsData,
SyncEventReceivedUserEventsData,
SyncEventReceivedAsymmetricMessagesData,
} from '@standardnotes/services'
import { OfflineSyncResponse } from './Offline/Response'
import {
@@ -86,10 +95,23 @@ import {
} from '@standardnotes/encryption'
import { CreatePayloadFromRawServerItem } from './Account/Utilities'
import { ApplicationSyncOptions } from '@Lib/Application/Options/OptionalOptions'
import { DecryptedServerConflictMap, TrustedServerConflictMap } from './Account/ServerConflictMap'
const DEFAULT_MAJOR_CHANGE_THRESHOLD = 15
const INVALID_SESSION_RESPONSE_STATUS = 401
/** Content types appearing first are always mapped first */
const ContentTypeLocalLoadPriorty = [
ContentType.ItemsKey,
ContentType.KeySystemRootKey,
ContentType.KeySystemItemsKey,
ContentType.VaultListing,
ContentType.TrustedContact,
ContentType.UserPrefs,
ContentType.Component,
ContentType.Theme,
]
/**
* The sync service orchestrates with the model manager, api service, and storage service
* to ensure consistent state between the three. When a change is made to an item, consumers
@@ -100,7 +122,7 @@ const INVALID_SESSION_RESPONSE_STATUS = 401
* The sync service largely does not perform any task unless it is called upon.
*/
export class SNSyncService
extends AbstractService<SyncEvent, ServerSyncResponse | OfflineSyncResponse | { source: SyncSource }>
extends AbstractService<SyncEvent>
implements SyncServiceInterface, InternalEventHandlerInterface, SyncClientInterface
{
private dirtyIndexAtLastPresyncSave?: number
@@ -128,14 +150,6 @@ export class SNSyncService
public lastSyncInvokationPromise?: Promise<unknown>
public currentSyncRequestPromise?: Promise<void>
/** Content types appearing first are always mapped first */
private readonly localLoadPriorty = [
ContentType.ItemsKey,
ContentType.UserPrefs,
ContentType.Component,
ContentType.Theme,
]
constructor(
private itemManager: ItemManager,
private sessionManager: SNSessionManager,
@@ -225,29 +239,21 @@ export class SNSyncService
return this.databaseLoaded
}
private async processItemsKeysFirstDuringDatabaseLoad(
itemsKeysPayloads: FullyFormedPayloadInterface[],
): Promise<void> {
if (itemsKeysPayloads.length === 0) {
private async processPriorityItemsForDatabaseLoad(items: FullyFormedPayloadInterface[]): Promise<void> {
if (items.length === 0) {
return
}
const encryptedItemsKeysPayloads = itemsKeysPayloads.filter(isEncryptedPayload)
const encryptedPayloads = items.filter(isEncryptedPayload)
const alreadyDecryptedPayloads = items.filter(isDecryptedPayload) as DecryptedPayloadInterface<ItemsKeyContent>[]
const originallyDecryptedItemsKeysPayloads = itemsKeysPayloads.filter(
isDecryptedPayload,
) as DecryptedPayloadInterface<ItemsKeyContent>[]
const encryptionSplit = SplitPayloadsByEncryptionType(encryptedPayloads)
const decryptionSplit = CreateDecryptionSplitWithKeyLookup(encryptionSplit)
const itemsKeysSplit: KeyedDecryptionSplit = {
usesRootKeyWithKeyLookup: {
items: encryptedItemsKeysPayloads,
},
}
const newlyDecryptedItemsKeys = await this.protocolService.decryptSplit(itemsKeysSplit)
const newlyDecryptedPayloads = await this.protocolService.decryptSplit(decryptionSplit)
await this.payloadManager.emitPayloads(
[...originallyDecryptedItemsKeysPayloads, ...newlyDecryptedItemsKeys],
[...alreadyDecryptedPayloads, ...newlyDecryptedPayloads],
PayloadEmitSource.LocalDatabaseLoaded,
)
}
@@ -262,7 +268,7 @@ export class SNSyncService
const chunks = await this.device.getDatabaseLoadChunks(
{
batchSize: this.options.loadBatchSize,
contentTypePriority: this.localLoadPriorty,
contentTypePriority: ContentTypeLocalLoadPriorty,
uuidPriority: this.launchPriorityUuids,
},
this.identifier,
@@ -272,18 +278,30 @@ export class SNSyncService
? chunks.fullEntries.itemsKeys.entries
: await this.device.getDatabaseEntries(this.identifier, chunks.keys.itemsKeys.keys)
const itemsKeyPayloads = itemsKeyEntries
.map((entry) => {
try {
return CreatePayload(entry, PayloadSource.Constructor)
} catch (e) {
console.error('Creating payload failed', e)
return undefined
}
})
.filter(isNotUndefined)
const keySystemRootKeyEntries = isFullEntryLoadChunkResponse(chunks)
? chunks.fullEntries.keySystemRootKeys.entries
: await this.device.getDatabaseEntries(this.identifier, chunks.keys.keySystemRootKeys.keys)
await this.processItemsKeysFirstDuringDatabaseLoad(itemsKeyPayloads)
const keySystemItemsKeyEntries = isFullEntryLoadChunkResponse(chunks)
? chunks.fullEntries.keySystemItemsKeys.entries
: await this.device.getDatabaseEntries(this.identifier, chunks.keys.keySystemItemsKeys.keys)
const createPayloadFromEntry = (entry: FullyFormedTransferPayload) => {
try {
return CreatePayload(entry, PayloadSource.LocalDatabaseLoaded)
} catch (e) {
console.error('Creating payload failed', e)
return undefined
}
}
await this.processPriorityItemsForDatabaseLoad(itemsKeyEntries.map(createPayloadFromEntry).filter(isNotUndefined))
await this.processPriorityItemsForDatabaseLoad(
keySystemRootKeyEntries.map(createPayloadFromEntry).filter(isNotUndefined),
)
await this.processPriorityItemsForDatabaseLoad(
keySystemItemsKeyEntries.map(createPayloadFromEntry).filter(isNotUndefined),
)
/**
* Map in batches to give interface a chance to update. Note that total decryption
@@ -308,7 +326,7 @@ export class SNSyncService
const payloads = dbEntries
.map((entry) => {
try {
return CreatePayload(entry, PayloadSource.Constructor)
return CreatePayload(entry, PayloadSource.LocalDatabaseLoaded)
} catch (e) {
console.error('Creating payload failed', e)
return undefined
@@ -348,13 +366,10 @@ export class SNSyncService
}
}
const split: KeyedDecryptionSplit = {
usesItemsKeyWithKeyLookup: {
items: encrypted,
},
}
const encryptionSplit = SplitPayloadsByEncryptionType(encrypted)
const decryptionSplit = CreateDecryptionSplitWithKeyLookup(encryptionSplit)
const results = await this.protocolService.decryptSplit(split)
const results = await this.protocolService.decryptSplit(decryptionSplit)
await this.payloadManager.emitPayloads([...nonencrypted, ...results], PayloadEmitSource.LocalDatabaseLoaded)
@@ -616,11 +631,7 @@ export class SNSyncService
if (useStrategy === SyncQueueStrategy.ResolveOnNext) {
return this.queueStrategyResolveOnNext()
} else if (useStrategy === SyncQueueStrategy.ForceSpawnNew) {
return this.queueStrategyForceSpawnNew({
mode: options.mode,
checkIntegrity: options.checkIntegrity,
source: options.source,
})
return this.queueStrategyForceSpawnNew(options)
} else {
throw Error(`Unhandled timing strategy ${useStrategy}`)
}
@@ -634,7 +645,7 @@ export class SNSyncService
) {
this.opStatus.setDidBegin()
await this.notifyEvent(SyncEvent.SyncWillBegin)
await this.notifyEvent(SyncEvent.SyncDidBeginProcessing)
/**
* Subtract from array as soon as we're sure they'll be called.
@@ -647,12 +658,41 @@ export class SNSyncService
* Setting this value means the item was 100% sent to the server.
*/
if (items.length > 0) {
return this.itemManager.setLastSyncBeganForItems(items, beginDate, frozenDirtyIndex)
return this.setLastSyncBeganForItems(items, beginDate, frozenDirtyIndex)
} else {
return items
}
}
private async setLastSyncBeganForItems(
itemsToLookupUuidsFor: (DecryptedItemInterface | DeletedItemInterface)[],
date: Date,
globalDirtyIndex: number,
): Promise<(DecryptedItemInterface | DeletedItemInterface)[]> {
const uuids = Uuids(itemsToLookupUuidsFor)
const items = this.itemManager.getCollection().findAll(uuids).filter(isDecryptedOrDeletedItem)
const payloads: (DecryptedPayloadInterface | DeletedPayloadInterface)[] = []
for (const item of items) {
const mutator = new ItemMutator<DecryptedPayloadInterface | DeletedPayloadInterface>(
item,
MutationType.NonDirtying,
)
mutator.setBeginSync(date, globalDirtyIndex)
const payload = mutator.getResult()
payloads.push(payload)
}
await this.payloadManager.emitPayloads(payloads, PayloadEmitSource.PreSyncSave)
return this.itemManager.findAnyItems(uuids) as (DecryptedItemInterface | DeletedItemInterface)[]
}
/**
* The InTime resolve queue refers to any sync requests that were made while we still
* have not sent out the current request. So, anything in the InTime resolve queue
@@ -725,12 +765,15 @@ export class SNSyncService
private async createServerSyncOperation(
payloads: ServerSyncPushContextualPayload[],
checkIntegrity: boolean,
source: SyncSource,
options: SyncOptions,
mode: SyncMode = SyncMode.Default,
) {
const syncToken = await this.getLastSyncToken()
const paginationToken = await this.getPaginationToken()
const syncToken =
options.sharedVaultUuids && options.sharedVaultUuids.length > 0 && options.syncSharedVaultsFromScratch
? undefined
: await this.getLastSyncToken()
const paginationToken =
options.sharedVaultUuids && options.syncSharedVaultsFromScratch ? undefined : await this.getPaginationToken()
const operation = new AccountSyncOperation(
payloads,
@@ -753,20 +796,23 @@ export class SNSyncService
break
}
},
syncToken,
paginationToken,
this.apiService,
{
syncToken,
paginationToken,
sharedVaultUuids: options.sharedVaultUuids,
},
)
log(
LoggingDomain.Sync,
'Syncing online user',
'source',
SyncSource[source],
SyncSource[options.source],
'operation id',
operation.id,
'integrity check',
checkIntegrity,
options.checkIntegrity,
'mode',
SyncMode[mode],
'syncToken',
@@ -789,12 +835,7 @@ export class SNSyncService
const { uploadPayloads, syncMode } = await this.getOnlineSyncParameters(payloads, options.mode)
return {
operation: await this.createServerSyncOperation(
uploadPayloads,
useBoolean(options.checkIntegrity, false),
options.source,
syncMode,
),
operation: await this.createServerSyncOperation(uploadPayloads, options, syncMode),
mode: syncMode,
}
} else {
@@ -867,6 +908,7 @@ export class SNSyncService
await this.notifyEventSync(SyncEvent.SyncCompletedWithAllItemsUploadedAndDownloaded, {
source: options.source,
options,
})
this.resolvePendingSyncRequestsThatMadeItInTimeOfCurrentRequest(inTimeResolveQueue)
@@ -889,7 +931,7 @@ export class SNSyncService
this.opStatus.clearError()
await this.notifyEvent(SyncEvent.SingleRoundTripSyncCompleted, response)
await this.notifyEvent(SyncEvent.PaginatedSyncRequestCompleted, response)
}
private handleErrorServerResponse(response: ServerSyncResponse) {
@@ -917,19 +959,36 @@ export class SNSyncService
const historyMap = this.historyService.getHistoryMapCopy()
if (response.userEvents) {
await this.notifyEventSync(SyncEvent.ReceivedUserEvents, response.userEvents as SyncEventReceivedUserEventsData)
}
if (response.asymmetricMessages) {
await this.notifyEventSync(
SyncEvent.ReceivedAsymmetricMessages,
response.asymmetricMessages as SyncEventReceivedAsymmetricMessagesData,
)
}
if (response.vaults) {
await this.notifyEventSync(
SyncEvent.ReceivedRemoteSharedVaults,
response.vaults as SyncEventReceivedRemoteSharedVaultsData,
)
}
if (response.vaultInvites) {
await this.notifyEventSync(
SyncEvent.ReceivedSharedVaultInvites,
response.vaultInvites as SyncEventReceivedSharedVaultInvitesData,
)
}
const resolver = new ServerSyncResponseResolver(
{
retrievedPayloads: await this.processServerPayloads(response.retrievedPayloads, PayloadSource.RemoteRetrieved),
savedPayloads: response.savedPayloads,
uuidConflictPayloads: await this.processServerPayloads(
response.uuidConflictPayloads,
PayloadSource.RemoteRetrieved,
),
dataConflictPayloads: await this.processServerPayloads(
response.dataConflictPayloads,
PayloadSource.RemoteRetrieved,
),
rejectedPayloads: await this.processServerPayloads(response.rejectedPayloads, PayloadSource.RemoteRetrieved),
conflicts: await this.decryptServerConflicts(response.conflicts),
},
masterCollection,
operation.payloadsSavedOrSaving,
@@ -954,11 +1013,69 @@ export class SNSyncService
await this.persistPayloads(payloadsToPersist)
}
await Promise.all([
this.setLastSyncToken(response.lastSyncToken as string),
this.setPaginationToken(response.paginationToken as string),
this.notifyEvent(SyncEvent.SingleRoundTripSyncCompleted, response),
])
if (!operation.options.sharedVaultUuids) {
await Promise.all([
this.setLastSyncToken(response.lastSyncToken as string),
this.setPaginationToken(response.paginationToken as string),
])
}
await this.notifyEvent(SyncEvent.PaginatedSyncRequestCompleted, {
...response,
uploadedPayloads: operation.payloads,
options: operation.options,
})
}
private async decryptServerConflicts(conflictMap: TrustedServerConflictMap): Promise<DecryptedServerConflictMap> {
const decrypted: DecryptedServerConflictMap = {}
for (const conflictType of Object.keys(conflictMap)) {
const conflictsForType = conflictMap[conflictType as ConflictType]
if (!conflictsForType) {
continue
}
if (!decrypted[conflictType as ConflictType]) {
decrypted[conflictType as ConflictType] = []
}
const decryptedConflictsForType = decrypted[conflictType as ConflictType]
if (!decryptedConflictsForType) {
throw Error('Decrypted conflicts for type should exist')
}
for (const conflict of conflictsForType) {
const decryptedUnsavedItem = conflict.unsaved_item
? await this.processServerPayload(conflict.unsaved_item, PayloadSource.RemoteRetrieved)
: undefined
const decryptedServerItem = conflict.server_item
? await this.processServerPayload(conflict.server_item, PayloadSource.RemoteRetrieved)
: undefined
const decryptedEntry: ConflictParams<FullyFormedPayloadInterface> = <
ConflictParams<FullyFormedPayloadInterface>
>{
type: conflict.type,
unsaved_item: decryptedUnsavedItem,
server_item: decryptedServerItem,
}
decryptedConflictsForType.push(decryptedEntry)
}
}
return decrypted
}
private async processServerPayload(
item: FilteredServerItem,
source: PayloadSource,
): Promise<FullyFormedPayloadInterface> {
const result = await this.processServerPayloads([item], source)
return result[0]
}
private async processServerPayloads(
@@ -971,7 +1088,8 @@ export class SNSyncService
const results: FullyFormedPayloadInterface[] = [...deleted]
const { rootKeyEncryption, itemsKeyEncryption } = SplitPayloadsByEncryptionType(encrypted)
const { rootKeyEncryption, itemsKeyEncryption, keySystemRootKeyEncryption } =
SplitPayloadsByEncryptionType(encrypted)
const { results: rootKeyDecryptionResults, map: processedItemsKeys } = await this.decryptServerItemsKeys(
rootKeyEncryption || [],
@@ -979,8 +1097,16 @@ export class SNSyncService
extendArray(results, rootKeyDecryptionResults)
const { results: keySystemRootKeyDecryptionResults, map: processedKeySystemItemsKeys } =
await this.decryptServerKeySystemItemsKeys(keySystemRootKeyEncryption || [])
extendArray(results, keySystemRootKeyDecryptionResults)
if (itemsKeyEncryption) {
const decryptionResults = await this.decryptProcessedServerPayloads(itemsKeyEncryption, processedItemsKeys)
const decryptionResults = await this.decryptProcessedServerPayloads(itemsKeyEncryption, {
...processedItemsKeys,
...processedKeySystemItemsKeys,
})
extendArray(results, decryptionResults)
}
@@ -1017,17 +1143,53 @@ export class SNSyncService
}
}
private async decryptServerKeySystemItemsKeys(payloads: EncryptedPayloadInterface[]) {
const map: Record<UuidString, DecryptedPayloadInterface<KeySystemItemsKeyContent>> = {}
if (payloads.length === 0) {
return {
results: [],
map,
}
}
const keySystemRootKeySplit: KeyedDecryptionSplit = {
usesKeySystemRootKeyWithKeyLookup: {
items: payloads,
},
}
const results = await this.protocolService.decryptSplit<KeySystemItemsKeyContent>(keySystemRootKeySplit)
results.forEach((result) => {
if (
isDecryptedPayload<KeySystemItemsKeyContent>(result) &&
result.content_type === ContentType.KeySystemItemsKey
) {
map[result.uuid] = result
}
})
return {
results,
map,
}
}
private async decryptProcessedServerPayloads(
payloads: EncryptedPayloadInterface[],
map: Record<UuidString, DecryptedPayloadInterface<ItemsKeyContent>>,
map: Record<UuidString, DecryptedPayloadInterface<ItemsKeyContent | KeySystemItemsKeyContent>>,
): Promise<(EncryptedPayloadInterface | DecryptedPayloadInterface)[]> {
return Promise.all(
payloads.map(async (encrypted) => {
const previouslyProcessedItemsKey: DecryptedPayloadInterface<ItemsKeyContent> | undefined =
map[encrypted.items_key_id as string]
const previouslyProcessedItemsKey:
| DecryptedPayloadInterface<ItemsKeyContent | KeySystemItemsKeyContent>
| undefined = map[encrypted.items_key_id as string]
const itemsKey = previouslyProcessedItemsKey
? (CreateDecryptedItemFromPayload(previouslyProcessedItemsKey) as ItemsKeyInterface)
? (CreateDecryptedItemFromPayload(previouslyProcessedItemsKey) as
| ItemsKeyInterface
| KeySystemItemsKeyInterface)
: undefined
const keyedSplit: KeyedDecryptionSplit = {}
@@ -1251,26 +1413,13 @@ export class SNSyncService
await this.persistPayloads(emit.emits)
}
override async getDiagnostics(): Promise<DiagnosticInfo | undefined> {
const dirtyUuids = Uuids(this.itemsNeedingSync())
return {
sync: {
syncToken: await this.getLastSyncToken(),
cursorToken: await this.getPaginationToken(),
dirtyIndexAtLastPresyncSave: this.dirtyIndexAtLastPresyncSave,
lastSyncDate: this.lastSyncDate,
outOfSync: this.outOfSync,
completedOnlineDownloadFirstSync: this.completedOnlineDownloadFirstSync,
clientLocked: this.clientLocked,
databaseLoaded: this.databaseLoaded,
syncLock: this.syncLock,
dealloced: this.dealloced,
itemsNeedingSync: dirtyUuids,
itemsNeedingSyncCount: dirtyUuids.length,
pendingRequestCount: this.resolveQueue.length + this.spawnQueue.length,
},
}
async syncSharedVaultsFromScratch(sharedVaultUuids: string[]): Promise<void> {
await this.sync({
sharedVaultUuids: sharedVaultUuids,
syncSharedVaultsFromScratch: true,
queueStrategy: SyncQueueStrategy.ForceSpawnNew,
awaitAll: true,
})
}
/** @e2e_testing */