feat: add snjs package

packages/snjs/lib/Services/Sync/Account/Operation.ts

@@ -0,0 +1,110 @@
+import { ServerSyncPushContextualPayload } from '@standardnotes/models'
+import { arrayByDifference, nonSecureRandomIdentifier, subtractFromArray } from '@standardnotes/utils'
+import { ServerSyncResponse } from '@Lib/Services/Sync/Account/Response'
+import { ResponseSignalReceiver, SyncSignal } from '@Lib/Services/Sync/Signals'
+import { SNApiService } from '../../Api/ApiService'
+import { RawSyncResponse } from '@standardnotes/responses'
+
+export const SyncUpDownLimit = 150
+
+/**
+ * A long running operation that handles multiple roundtrips from a server,
+ * emitting a stream of values that should be acted upon in real time.
+ */
+export class AccountSyncOperation {
+  public readonly id = nonSecureRandomIdentifier()
+
+  private pendingPayloads: ServerSyncPushContextualPayload[]
+  private responses: ServerSyncResponse[] = []
+
+  /**
+   * @param payloads   An array of payloads to send to the server
+   * @param receiver   A function that receives callback multiple times during the operation
+   */
+  constructor(
+    private payloads: ServerSyncPushContextualPayload[],
+    private receiver: ResponseSignalReceiver<ServerSyncResponse>,
+    private lastSyncToken: string,
+    private paginationToken: string,
+    private apiService: SNApiService,
+  ) {
+    this.payloads = payloads
+    this.lastSyncToken = lastSyncToken
+    this.paginationToken = paginationToken
+    this.apiService = apiService
+    this.receiver = receiver
+    this.pendingPayloads = payloads.slice()
+  }
+
+  /**
+   * Read the payloads that have been saved, or are currently in flight.
+   */
+  get payloadsSavedOrSaving(): ServerSyncPushContextualPayload[] {
+    return arrayByDifference(this.payloads, this.pendingPayloads)
+  }
+
+  popPayloads(count: number) {
+    const payloads = this.pendingPayloads.slice(0, count)
+    subtractFromArray(this.pendingPayloads, payloads)
+    return payloads
+  }
+
+  async run(): Promise<void> {
+    await this.receiver(SyncSignal.StatusChanged, undefined, {
+      completedUploadCount: this.totalUploadCount - this.pendingUploadCount,
+      totalUploadCount: this.totalUploadCount,
+    })
+    const payloads = this.popPayloads(this.upLimit)
+
+    const rawResponse = (await this.apiService.sync(
+      payloads,
+      this.lastSyncToken,
+      this.paginationToken,
+      this.downLimit,
+    )) as RawSyncResponse
+
+    const response = new ServerSyncResponse(rawResponse)
+    this.responses.push(response)
+
+    this.lastSyncToken = response.lastSyncToken as string
+    this.paginationToken = response.paginationToken as string
+
+    try {
+      await this.receiver(SyncSignal.Response, response)
+    } catch (error) {
+      console.error('Sync handle response error', error)
+    }
+
+    if (!this.done) {
+      return this.run()
+    }
+  }
+
+  get done() {
+    return this.pendingPayloads.length === 0 && !this.paginationToken
+  }
+
+  private get pendingUploadCount() {
+    return this.pendingPayloads.length
+  }
+
+  private get totalUploadCount() {
+    return this.payloads.length
+  }
+
+  private get upLimit() {
+    return SyncUpDownLimit
+  }
+
+  private get downLimit() {
+    return SyncUpDownLimit
+  }
+
+  get numberOfItemsInvolved() {
+    let total = 0
+    for (const response of this.responses) {
+      total += response.numberOfItemsInvolved
+    }
+    return total
+  }
+}

packages/snjs/lib/Services/Sync/Account/Response.ts

@@ -0,0 +1,115 @@
+import {
+  ApiEndpointParam,
+  ConflictParams,
+  ConflictType,
+  Error,
+  RawSyncResponse,
+  ServerItemResponse,
+} from '@standardnotes/responses'
+import {
+  FilterDisallowedRemotePayloadsAndMap,
+  CreateServerSyncSavedPayload,
+  ServerSyncSavedContextualPayload,
+  FilteredServerItem,
+} from '@standardnotes/models'
+import { deepFreeze, isNullOrUndefined } from '@standardnotes/utils'
+
+export class ServerSyncResponse {
+  public readonly rawResponse: RawSyncResponse
+  public readonly savedPayloads: ServerSyncSavedContextualPayload[]
+  public readonly retrievedPayloads: FilteredServerItem[]
+  public readonly uuidConflictPayloads: FilteredServerItem[]
+  public readonly dataConflictPayloads: FilteredServerItem[]
+  public readonly rejectedPayloads: FilteredServerItem[]
+
+  constructor(rawResponse: RawSyncResponse) {
+    this.rawResponse = rawResponse
+
+    this.savedPayloads = FilterDisallowedRemotePayloadsAndMap(rawResponse.data?.saved_items || []).map((rawItem) => {
+      return CreateServerSyncSavedPayload(rawItem)
+    })
+
+    this.retrievedPayloads = FilterDisallowedRemotePayloadsAndMap(rawResponse.data?.retrieved_items || [])
+
+    this.dataConflictPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawDataConflictItems)
+
+    this.uuidConflictPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawUuidConflictItems)
+
+    this.rejectedPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawRejectedPayloads)
+
+    deepFreeze(this)
+  }
+
+  public get error(): Error | undefined {
+    return this.rawResponse.error || this.rawResponse.data?.error
+  }
+
+  public get status(): number {
+    return this.rawResponse.status as number
+  }
+
+  public get lastSyncToken(): string | undefined {
+    return this.rawResponse.data?.[ApiEndpointParam.LastSyncToken]
+  }
+
+  public get paginationToken(): string | undefined {
+    return this.rawResponse.data?.[ApiEndpointParam.PaginationToken]
+  }
+
+  public get numberOfItemsInvolved(): number {
+    return this.allFullyFormedPayloads.length
+  }
+
+  private get allFullyFormedPayloads(): FilteredServerItem[] {
+    return [
+      ...this.retrievedPayloads,
+      ...this.dataConflictPayloads,
+      ...this.uuidConflictPayloads,
+      ...this.rejectedPayloads,
+    ]
+  }
+
+  private get rawUuidConflictItems(): ServerItemResponse[] {
+    return this.rawConflictObjects
+      .filter((conflict) => {
+        return conflict.type === ConflictType.UuidConflict
+      })
+      .map((conflict) => {
+        return conflict.unsaved_item || (conflict.item as ServerItemResponse)
+      })
+  }
+
+  private get rawDataConflictItems(): ServerItemResponse[] {
+    return this.rawConflictObjects
+      .filter((conflict) => {
+        return conflict.type === ConflictType.ConflictingData
+      })
+      .map((conflict) => {
+        return conflict.server_item || (conflict.item as ServerItemResponse)
+      })
+  }
+
+  private get rawRejectedPayloads(): ServerItemResponse[] {
+    return this.rawConflictObjects
+      .filter((conflict) => {
+        return (
+          conflict.type === ConflictType.ContentTypeError ||
+          conflict.type === ConflictType.ContentError ||
+          conflict.type === ConflictType.ReadOnlyError
+        )
+      })
+      .map((conflict) => {
+        return conflict.unsaved_item as ServerItemResponse
+      })
+  }
+
+  private get rawConflictObjects(): ConflictParams[] {
+    const conflicts = this.rawResponse.data?.conflicts || []
+    const legacyConflicts = this.rawResponse.data?.unsaved || []
+    return conflicts.concat(legacyConflicts)
+  }
+
+  public get hasError(): boolean {
+    return !isNullOrUndefined(this.rawResponse.error)
+  }
+}

packages/snjs/lib/Services/Sync/Account/ResponseResolver.ts

@@ -0,0 +1,86 @@
+import {
+  ImmutablePayloadCollection,
+  HistoryMap,
+  DeltaRemoteRetrieved,
+  DeltaRemoteSaved,
+  DeltaRemoteDataConflicts,
+  FullyFormedPayloadInterface,
+  ServerSyncPushContextualPayload,
+  ServerSyncSavedContextualPayload,
+  DeltaRemoteUuidConflicts,
+  DeltaRemoteRejected,
+  DeltaEmit,
+} from '@standardnotes/models'
+
+type PayloadSet = {
+  retrievedPayloads: FullyFormedPayloadInterface[]
+  savedPayloads: ServerSyncSavedContextualPayload[]
+  uuidConflictPayloads: FullyFormedPayloadInterface[]
+  dataConflictPayloads: FullyFormedPayloadInterface[]
+  rejectedPayloads: FullyFormedPayloadInterface[]
+}
+
+/**
+ * Given a remote sync response, the resolver applies the incoming changes on top
+ * of the current base state, and returns what the new global state should look like.
+ * The response resolver is purely functional and does not modify global state, but instead
+ * offers the 'recommended' new global state given a sync response and a current base state.
+ */
+export class ServerSyncResponseResolver {
+  constructor(
+    private payloadSet: PayloadSet,
+    private baseCollection: ImmutablePayloadCollection<FullyFormedPayloadInterface>,
+    private payloadsSavedOrSaving: ServerSyncPushContextualPayload[],
+    private historyMap: HistoryMap,
+  ) {}
+
+  public result(): DeltaEmit[] {
+    const emits: DeltaEmit[] = []
+
+    emits.push(this.processRetrievedPayloads())
+    emits.push(this.processSavedPayloads())
+    emits.push(this.processUuidConflictPayloads())
+    emits.push(this.processDataConflictPayloads())
+    emits.push(this.processRejectedPayloads())
+
+    return emits
+  }
+
+  private processSavedPayloads(): DeltaEmit {
+    const delta = new DeltaRemoteSaved(this.baseCollection, this.payloadSet.savedPayloads)
+
+    return delta.result()
+  }
+
+  private processRetrievedPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.retrievedPayloads)
+
+    const delta = new DeltaRemoteRetrieved(this.baseCollection, collection, this.payloadsSavedOrSaving, this.historyMap)
+
+    return delta.result()
+  }
+
+  private processDataConflictPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.dataConflictPayloads)
+
+    const delta = new DeltaRemoteDataConflicts(this.baseCollection, collection, this.historyMap)
+
+    return delta.result()
+  }
+
+  private processUuidConflictPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.uuidConflictPayloads)
+
+    const delta = new DeltaRemoteUuidConflicts(this.baseCollection, collection)
+
+    return delta.result()
+  }
+
+  private processRejectedPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.rejectedPayloads)
+
+    const delta = new DeltaRemoteRejected(this.baseCollection, collection)
+
+    return delta.result()
+  }
+}

packages/snjs/lib/Services/Sync/Account/Utilities.ts

@@ -0,0 +1,31 @@
+import {
+  EncryptedPayloadInterface,
+  DeletedPayloadInterface,
+  PayloadSource,
+  DeletedPayload,
+  EncryptedPayload,
+  FilteredServerItem,
+} from '@standardnotes/models'
+
+export function CreatePayloadFromRawServerItem(
+  rawItem: FilteredServerItem,
+  source: PayloadSource,
+): EncryptedPayloadInterface | DeletedPayloadInterface {
+  if (rawItem.deleted) {
+    return new DeletedPayload({ ...rawItem, content: undefined, deleted: true }, source)
+  } else if (rawItem.content != undefined) {
+    return new EncryptedPayload(
+      {
+        ...rawItem,
+        items_key_id: rawItem.items_key_id,
+        content: rawItem.content,
+        deleted: false,
+        errorDecrypting: false,
+        waitingForKey: false,
+      },
+      source,
+    )
+  } else {
+    throw Error('Unhandled case in createPayloadFromRawItem')
+  }
+}