feat: add snjs package

packages/snjs/lib/Services/Sync/Account/Operation.ts

@@ -0,0 +1,110 @@
+import { ServerSyncPushContextualPayload } from '@standardnotes/models'
+import { arrayByDifference, nonSecureRandomIdentifier, subtractFromArray } from '@standardnotes/utils'
+import { ServerSyncResponse } from '@Lib/Services/Sync/Account/Response'
+import { ResponseSignalReceiver, SyncSignal } from '@Lib/Services/Sync/Signals'
+import { SNApiService } from '../../Api/ApiService'
+import { RawSyncResponse } from '@standardnotes/responses'
+
+export const SyncUpDownLimit = 150
+
+/**
+ * A long running operation that handles multiple roundtrips from a server,
+ * emitting a stream of values that should be acted upon in real time.
+ */
+export class AccountSyncOperation {
+  public readonly id = nonSecureRandomIdentifier()
+
+  private pendingPayloads: ServerSyncPushContextualPayload[]
+  private responses: ServerSyncResponse[] = []
+
+  /**
+   * @param payloads   An array of payloads to send to the server
+   * @param receiver   A function that receives callback multiple times during the operation
+   */
+  constructor(
+    private payloads: ServerSyncPushContextualPayload[],
+    private receiver: ResponseSignalReceiver<ServerSyncResponse>,
+    private lastSyncToken: string,
+    private paginationToken: string,
+    private apiService: SNApiService,
+  ) {
+    this.payloads = payloads
+    this.lastSyncToken = lastSyncToken
+    this.paginationToken = paginationToken
+    this.apiService = apiService
+    this.receiver = receiver
+    this.pendingPayloads = payloads.slice()
+  }
+
+  /**
+   * Read the payloads that have been saved, or are currently in flight.
+   */
+  get payloadsSavedOrSaving(): ServerSyncPushContextualPayload[] {
+    return arrayByDifference(this.payloads, this.pendingPayloads)
+  }
+
+  popPayloads(count: number) {
+    const payloads = this.pendingPayloads.slice(0, count)
+    subtractFromArray(this.pendingPayloads, payloads)
+    return payloads
+  }
+
+  async run(): Promise<void> {
+    await this.receiver(SyncSignal.StatusChanged, undefined, {
+      completedUploadCount: this.totalUploadCount - this.pendingUploadCount,
+      totalUploadCount: this.totalUploadCount,
+    })
+    const payloads = this.popPayloads(this.upLimit)
+
+    const rawResponse = (await this.apiService.sync(
+      payloads,
+      this.lastSyncToken,
+      this.paginationToken,
+      this.downLimit,
+    )) as RawSyncResponse
+
+    const response = new ServerSyncResponse(rawResponse)
+    this.responses.push(response)
+
+    this.lastSyncToken = response.lastSyncToken as string
+    this.paginationToken = response.paginationToken as string
+
+    try {
+      await this.receiver(SyncSignal.Response, response)
+    } catch (error) {
+      console.error('Sync handle response error', error)
+    }
+
+    if (!this.done) {
+      return this.run()
+    }
+  }
+
+  get done() {
+    return this.pendingPayloads.length === 0 && !this.paginationToken
+  }
+
+  private get pendingUploadCount() {
+    return this.pendingPayloads.length
+  }
+
+  private get totalUploadCount() {
+    return this.payloads.length
+  }
+
+  private get upLimit() {
+    return SyncUpDownLimit
+  }
+
+  private get downLimit() {
+    return SyncUpDownLimit
+  }
+
+  get numberOfItemsInvolved() {
+    let total = 0
+    for (const response of this.responses) {
+      total += response.numberOfItemsInvolved
+    }
+    return total
+  }
+}

packages/snjs/lib/Services/Sync/Account/Response.ts

@@ -0,0 +1,115 @@
+import {
+  ApiEndpointParam,
+  ConflictParams,
+  ConflictType,
+  Error,
+  RawSyncResponse,
+  ServerItemResponse,
+} from '@standardnotes/responses'
+import {
+  FilterDisallowedRemotePayloadsAndMap,
+  CreateServerSyncSavedPayload,
+  ServerSyncSavedContextualPayload,
+  FilteredServerItem,
+} from '@standardnotes/models'
+import { deepFreeze, isNullOrUndefined } from '@standardnotes/utils'
+
+export class ServerSyncResponse {
+  public readonly rawResponse: RawSyncResponse
+  public readonly savedPayloads: ServerSyncSavedContextualPayload[]
+  public readonly retrievedPayloads: FilteredServerItem[]
+  public readonly uuidConflictPayloads: FilteredServerItem[]
+  public readonly dataConflictPayloads: FilteredServerItem[]
+  public readonly rejectedPayloads: FilteredServerItem[]
+
+  constructor(rawResponse: RawSyncResponse) {
+    this.rawResponse = rawResponse
+
+    this.savedPayloads = FilterDisallowedRemotePayloadsAndMap(rawResponse.data?.saved_items || []).map((rawItem) => {
+      return CreateServerSyncSavedPayload(rawItem)
+    })
+
+    this.retrievedPayloads = FilterDisallowedRemotePayloadsAndMap(rawResponse.data?.retrieved_items || [])
+
+    this.dataConflictPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawDataConflictItems)
+
+    this.uuidConflictPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawUuidConflictItems)
+
+    this.rejectedPayloads = FilterDisallowedRemotePayloadsAndMap(this.rawRejectedPayloads)
+
+    deepFreeze(this)
+  }
+
+  public get error(): Error | undefined {
+    return this.rawResponse.error || this.rawResponse.data?.error
+  }
+
+  public get status(): number {
+    return this.rawResponse.status as number
+  }
+
+  public get lastSyncToken(): string | undefined {
+    return this.rawResponse.data?.[ApiEndpointParam.LastSyncToken]
+  }
+
+  public get paginationToken(): string | undefined {
+    return this.rawResponse.data?.[ApiEndpointParam.PaginationToken]
+  }
+
+  public get numberOfItemsInvolved(): number {
+    return this.allFullyFormedPayloads.length
+  }
+
+  private get allFullyFormedPayloads(): FilteredServerItem[] {
+    return [
+      ...this.retrievedPayloads,
+      ...this.dataConflictPayloads,
+      ...this.uuidConflictPayloads,
+      ...this.rejectedPayloads,
+    ]
+  }
+
+  private get rawUuidConflictItems(): ServerItemResponse[] {
+    return this.rawConflictObjects
+      .filter((conflict) => {
+        return conflict.type === ConflictType.UuidConflict
+      })
+      .map((conflict) => {
+        return conflict.unsaved_item || (conflict.item as ServerItemResponse)
+      })
+  }
+
+  private get rawDataConflictItems(): ServerItemResponse[] {
+    return this.rawConflictObjects
+      .filter((conflict) => {
+        return conflict.type === ConflictType.ConflictingData
+      })
+      .map((conflict) => {
+        return conflict.server_item || (conflict.item as ServerItemResponse)
+      })
+  }
+
+  private get rawRejectedPayloads(): ServerItemResponse[] {
+    return this.rawConflictObjects
+      .filter((conflict) => {
+        return (
+          conflict.type === ConflictType.ContentTypeError ||
+          conflict.type === ConflictType.ContentError ||
+          conflict.type === ConflictType.ReadOnlyError
+        )
+      })
+      .map((conflict) => {
+        return conflict.unsaved_item as ServerItemResponse
+      })
+  }
+
+  private get rawConflictObjects(): ConflictParams[] {
+    const conflicts = this.rawResponse.data?.conflicts || []
+    const legacyConflicts = this.rawResponse.data?.unsaved || []
+    return conflicts.concat(legacyConflicts)
+  }
+
+  public get hasError(): boolean {
+    return !isNullOrUndefined(this.rawResponse.error)
+  }
+}

packages/snjs/lib/Services/Sync/Account/ResponseResolver.ts

@@ -0,0 +1,86 @@
+import {
+  ImmutablePayloadCollection,
+  HistoryMap,
+  DeltaRemoteRetrieved,
+  DeltaRemoteSaved,
+  DeltaRemoteDataConflicts,
+  FullyFormedPayloadInterface,
+  ServerSyncPushContextualPayload,
+  ServerSyncSavedContextualPayload,
+  DeltaRemoteUuidConflicts,
+  DeltaRemoteRejected,
+  DeltaEmit,
+} from '@standardnotes/models'
+
+type PayloadSet = {
+  retrievedPayloads: FullyFormedPayloadInterface[]
+  savedPayloads: ServerSyncSavedContextualPayload[]
+  uuidConflictPayloads: FullyFormedPayloadInterface[]
+  dataConflictPayloads: FullyFormedPayloadInterface[]
+  rejectedPayloads: FullyFormedPayloadInterface[]
+}
+
+/**
+ * Given a remote sync response, the resolver applies the incoming changes on top
+ * of the current base state, and returns what the new global state should look like.
+ * The response resolver is purely functional and does not modify global state, but instead
+ * offers the 'recommended' new global state given a sync response and a current base state.
+ */
+export class ServerSyncResponseResolver {
+  constructor(
+    private payloadSet: PayloadSet,
+    private baseCollection: ImmutablePayloadCollection<FullyFormedPayloadInterface>,
+    private payloadsSavedOrSaving: ServerSyncPushContextualPayload[],
+    private historyMap: HistoryMap,
+  ) {}
+
+  public result(): DeltaEmit[] {
+    const emits: DeltaEmit[] = []
+
+    emits.push(this.processRetrievedPayloads())
+    emits.push(this.processSavedPayloads())
+    emits.push(this.processUuidConflictPayloads())
+    emits.push(this.processDataConflictPayloads())
+    emits.push(this.processRejectedPayloads())
+
+    return emits
+  }
+
+  private processSavedPayloads(): DeltaEmit {
+    const delta = new DeltaRemoteSaved(this.baseCollection, this.payloadSet.savedPayloads)
+
+    return delta.result()
+  }
+
+  private processRetrievedPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.retrievedPayloads)
+
+    const delta = new DeltaRemoteRetrieved(this.baseCollection, collection, this.payloadsSavedOrSaving, this.historyMap)
+
+    return delta.result()
+  }
+
+  private processDataConflictPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.dataConflictPayloads)
+
+    const delta = new DeltaRemoteDataConflicts(this.baseCollection, collection, this.historyMap)
+
+    return delta.result()
+  }
+
+  private processUuidConflictPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.uuidConflictPayloads)
+
+    const delta = new DeltaRemoteUuidConflicts(this.baseCollection, collection)
+
+    return delta.result()
+  }
+
+  private processRejectedPayloads(): DeltaEmit {
+    const collection = ImmutablePayloadCollection.WithPayloads(this.payloadSet.rejectedPayloads)
+
+    const delta = new DeltaRemoteRejected(this.baseCollection, collection)
+
+    return delta.result()
+  }
+}

packages/snjs/lib/Services/Sync/Account/Utilities.ts

@@ -0,0 +1,31 @@
+import {
+  EncryptedPayloadInterface,
+  DeletedPayloadInterface,
+  PayloadSource,
+  DeletedPayload,
+  EncryptedPayload,
+  FilteredServerItem,
+} from '@standardnotes/models'
+
+export function CreatePayloadFromRawServerItem(
+  rawItem: FilteredServerItem,
+  source: PayloadSource,
+): EncryptedPayloadInterface | DeletedPayloadInterface {
+  if (rawItem.deleted) {
+    return new DeletedPayload({ ...rawItem, content: undefined, deleted: true }, source)
+  } else if (rawItem.content != undefined) {
+    return new EncryptedPayload(
+      {
+        ...rawItem,
+        items_key_id: rawItem.items_key_id,
+        content: rawItem.content,
+        deleted: false,
+        errorDecrypting: false,
+        waitingForKey: false,
+      },
+      source,
+    )
+  } else {
+    throw Error('Unhandled case in createPayloadFromRawItem')
+  }
+}

packages/snjs/lib/Services/Sync/Offline/Operation.ts

@@ -0,0 +1,29 @@
+import {
+  CreateOfflineSyncSavedPayload,
+  DecryptedPayloadInterface,
+  DeletedPayloadInterface,
+} from '@standardnotes/models'
+import { ResponseSignalReceiver, SyncSignal } from '@Lib/Services/Sync/Signals'
+
+import { OfflineSyncResponse } from './Response'
+
+export class OfflineSyncOperation {
+  /**
+   * @param payloads  An array of payloads to sync offline
+   * @param receiver  A function that receives callback multiple times during the operation
+   */
+  constructor(
+    private payloads: (DecryptedPayloadInterface | DeletedPayloadInterface)[],
+    private receiver: ResponseSignalReceiver<OfflineSyncResponse>,
+  ) {}
+
+  async run() {
+    const responsePayloads = this.payloads.map((payload) => {
+      return CreateOfflineSyncSavedPayload(payload)
+    })
+
+    const response = new OfflineSyncResponse(responsePayloads)
+
+    await this.receiver(SyncSignal.Response, response)
+  }
+}

packages/snjs/lib/Services/Sync/Offline/Response.ts

@@ -0,0 +1,5 @@
+import { OfflineSyncSavedContextualPayload } from '@standardnotes/models'
+
+export class OfflineSyncResponse {
+  constructor(public readonly savedPayloads: OfflineSyncSavedContextualPayload[]) {}
+}

packages/snjs/lib/Services/Sync/Signals.ts

@@ -0,0 +1,18 @@
+import { ServerSyncResponse } from '@Lib/Services/Sync/Account/Response'
+import { OfflineSyncResponse } from './Offline/Response'
+
+export enum SyncSignal {
+  Response = 1,
+  StatusChanged = 2,
+}
+
+export type SyncStats = {
+  completedUploadCount: number
+  totalUploadCount: number
+}
+
+export type ResponseSignalReceiver<T extends ServerSyncResponse | OfflineSyncResponse> = (
+  signal: SyncSignal,
+  response?: T,
+  stats?: SyncStats,
+) => Promise<void>

packages/snjs/lib/Services/Sync/SyncClientInterface.ts

@@ -0,0 +1,12 @@
+import { SyncOpStatus } from './SyncOpStatus'
+import { SyncOptions } from '@standardnotes/services'
+
+export interface SyncClientInterface {
+  sync(options?: Partial<SyncOptions>): Promise<unknown>
+
+  isOutOfSync(): boolean
+
+  getLastSyncDate(): Date | undefined
+
+  getSyncStatus(): SyncOpStatus
+}

packages/snjs/lib/Services/Sync/SyncOpStatus.ts

@@ -0,0 +1,125 @@
+import { SyncEvent, SyncEventReceiver } from '@standardnotes/services'
+
+const HEALTHY_SYNC_DURATION_THRESHOLD_S = 5
+const TIMING_MONITOR_POLL_FREQUENCY_MS = 500
+
+export class SyncOpStatus {
+  error?: any
+  private interval: any
+  private receiver: SyncEventReceiver
+  private completedUpload = 0
+  private totalUpload = 0
+  private downloaded = 0
+  private databaseLoadCurrent = 0
+  private databaseLoadTotal = 0
+  private databaseLoadDone = false
+  private syncing = false
+  private syncStart!: Date
+  private timingMonitor?: any
+
+  constructor(interval: any, receiver: SyncEventReceiver) {
+    this.interval = interval
+    this.receiver = receiver
+  }
+
+  public deinit() {
+    this.stopTimingMonitor()
+  }
+
+  public setUploadStatus(completed: number, total: number) {
+    this.completedUpload = completed
+    this.totalUpload = total
+    this.receiver(SyncEvent.StatusChanged)
+  }
+
+  public setDownloadStatus(downloaded: number) {
+    this.downloaded += downloaded
+    this.receiver(SyncEvent.StatusChanged)
+  }
+
+  public setDatabaseLoadStatus(current: number, total: number, done: boolean) {
+    this.databaseLoadCurrent = current
+    this.databaseLoadTotal = total
+    this.databaseLoadDone = done
+    if (done) {
+      this.receiver(SyncEvent.LocalDataLoaded)
+    } else {
+      this.receiver(SyncEvent.LocalDataIncrementalLoad)
+    }
+  }
+
+  public getStats() {
+    return {
+      uploadCompletionCount: this.completedUpload,
+      uploadTotalCount: this.totalUpload,
+      downloadCount: this.downloaded,
+      localDataDone: this.databaseLoadDone,
+      localDataCurrent: this.databaseLoadCurrent,
+      localDataTotal: this.databaseLoadTotal,
+    }
+  }
+
+  public setDidBegin() {
+    this.syncing = true
+    this.syncStart = new Date()
+  }
+
+  public setDidEnd() {
+    this.syncing = false
+  }
+
+  get syncInProgress() {
+    return this.syncing === true
+  }
+
+  get secondsSinceSyncStart() {
+    return (new Date().getTime() - this.syncStart.getTime()) / 1000
+  }
+
+  /**
+   * Notifies receiver if current sync request is taking too long to complete.
+   */
+  startTimingMonitor(): void {
+    if (this.timingMonitor) {
+      this.stopTimingMonitor()
+    }
+
+    this.timingMonitor = this.interval(() => {
+      if (this.secondsSinceSyncStart > HEALTHY_SYNC_DURATION_THRESHOLD_S) {
+        this.receiver(SyncEvent.SyncTakingTooLong)
+        this.stopTimingMonitor()
+      }
+    }, TIMING_MONITOR_POLL_FREQUENCY_MS)
+  }
+
+  stopTimingMonitor(): void {
+    if (Object.prototype.hasOwnProperty.call(this.interval, 'cancel')) {
+      this.interval.cancel(this.timingMonitor)
+    } else {
+      clearInterval(this.timingMonitor)
+    }
+    this.timingMonitor = null
+  }
+
+  hasError(): boolean {
+    return !!this.error
+  }
+
+  setError(error: any): void {
+    this.error = error
+  }
+
+  clearError() {
+    this.error = null
+  }
+
+  reset() {
+    this.downloaded = 0
+    this.completedUpload = 0
+    this.totalUpload = 0
+    this.syncing = false
+    this.error = null
+    this.stopTimingMonitor()
+    this.receiver(SyncEvent.StatusChanged)
+  }
+}

packages/snjs/lib/Services/Sync/Types.ts

@@ -0,0 +1,7 @@
+import { SyncOptions } from '@standardnotes/services'
+
+export type SyncPromise = {
+  resolve: (value?: unknown) => void
+  reject: () => void
+  options?: SyncOptions
+}

packages/snjs/lib/Services/Sync/Utils.ts

@@ -0,0 +1,43 @@
+import { ContentType } from '@standardnotes/common'
+import { FullyFormedPayloadInterface } from '@standardnotes/models'
+
+/**
+ * Sorts payloads according by most recently modified first, according to the priority,
+ * whereby the earlier a content_type appears in the priorityList,
+ * the earlier it will appear in the resulting sorted array.
+ */
+export function SortPayloadsByRecentAndContentPriority(
+  payloads: FullyFormedPayloadInterface[],
+  priorityList: ContentType[],
+): FullyFormedPayloadInterface[] {
+  return payloads.sort((a, b) => {
+    const dateResult = new Date(b.serverUpdatedAt).getTime() - new Date(a.serverUpdatedAt).getTime()
+
+    let aPriority = 0
+    let bPriority = 0
+
+    if (priorityList) {
+      aPriority = priorityList.indexOf(a.content_type)
+      bPriority = priorityList.indexOf(b.content_type)
+
+      if (aPriority === -1) {
+        /** Not found in list, not prioritized. Set it to max value */
+        aPriority = priorityList.length
+      }
+      if (bPriority === -1) {
+        /** Not found in list, not prioritized. Set it to max value */
+        bPriority = priorityList.length
+      }
+    }
+
+    if (aPriority === bPriority) {
+      return dateResult
+    }
+
+    if (aPriority < bPriority) {
+      return -1
+    } else {
+      return 1
+    }
+  })
+}

packages/snjs/lib/Services/Sync/index.ts

@@ -0,0 +1,9 @@
+export * from './SyncService'
+export * from './Types'
+export * from './SyncOpStatus'
+export * from './SyncClientInterface'
+export * from './Account/Operation'
+export * from './Account/ResponseResolver'
+export * from './Offline/Operation'
+export * from './Utils'
+export * from './Account/Response'