chore: display shared vault file usage (#2399)

* chore: display shared vault file usage

* fix: specs

* fix: reshape filtering result

* fix: resolving invalid server items

* fix: get revisions specs

* fix: processing issue

* fix: tests

---------

Co-authored-by: Mo <mo@standardnotes.com>

packages/snjs/lib/Application/Dependencies/Dependencies.ts

@@ -140,6 +140,7 @@ import {
   IsApplicationUsingThirdPartyHost,
   CreateDecryptedBackupFile,
   CreateEncryptedBackupFile,
+  SyncLocalVaultsWithRemoteSharedVaults,
 } from '@standardnotes/services'
 import { ItemManager } from '../../Services/Items/ItemManager'
 import { PayloadManager } from '../../Services/Payloads/PayloadManager'
@@ -399,6 +400,13 @@ export class Dependencies {
       return new GetVaults(this.get<ItemManager>(TYPES.ItemManager))
     })
 
+    this.factory.set(TYPES.SyncLocalVaultsWithRemoteSharedVaults, () => {
+      return new SyncLocalVaultsWithRemoteSharedVaults(
+        this.get<SharedVaultServer>(TYPES.SharedVaultServer),
+        this.get<MutatorService>(TYPES.MutatorService),
+      )
+    })
+
     this.factory.set(TYPES.ChangeAndSaveItem, () => {
       return new ChangeAndSaveItem(
         this.get<ItemManager>(TYPES.ItemManager),
@@ -893,6 +901,7 @@ export class Dependencies {
       return new SharedVaultService(
         this.get<ItemManager>(TYPES.ItemManager),
         this.get<SessionManager>(TYPES.SessionManager),
+        this.get<SyncLocalVaultsWithRemoteSharedVaults>(TYPES.SyncLocalVaultsWithRemoteSharedVaults),
         this.get<GetVault>(TYPES.GetVault),
         this.get<GetOwnedSharedVaults>(TYPES.GetOwnedSharedVaults),
         this.get<CreateSharedVault>(TYPES.CreateSharedVault),

packages/snjs/lib/Application/Dependencies/Types.ts

@@ -98,6 +98,7 @@ export const TYPES = {
   ValidateItemSigner: Symbol.for('ValidateItemSigner'),
   GetVault: Symbol.for('GetVault'),
   GetVaults: Symbol.for('GetVaults'),
+  SyncLocalVaultsWithRemoteSharedVaults: Symbol.for('SyncLocalVaultsWithRemoteSharedVaults'),
   GetSharedVaults: Symbol.for('GetSharedVaults'),
   GetOwnedSharedVaults: Symbol.for('GetOwnedSharedVaults'),
   ChangeVaultKeyOptions: Symbol.for('ChangeVaultKeyOptions'),

packages/snjs/lib/ApplicationGroup/ApplicationDescriptor.ts

@@ -1,4 +1,4 @@
-import { ApplicationIdentifier } from '@standardnotes/common'
+import { ApplicationIdentifier } from '@standardnotes/models'
 
 export type ApplicationDescriptor = {
   identifier: ApplicationIdentifier

packages/snjs/lib/Domain/UseCase/GetRevision/GetRevision.spec.ts

@@ -5,10 +5,10 @@ jest.mock('@standardnotes/models', () => {
 
   return {
     ...original,
-    isRemotePayloadAllowed: jest.fn(),
+    checkRemotePayloadAllowed: jest.fn(),
   }
 })
-const isRemotePayloadAllowed = require('@standardnotes/models').isRemotePayloadAllowed
+const checkRemotePayloadAllowed = require('@standardnotes/models').checkRemotePayloadAllowed
 
 import { Revision } from '../../Revision/Revision'
 
@@ -44,7 +44,7 @@ describe('GetRevision', () => {
     encryptedPayload.copy = jest.fn().mockReturnValue(encryptedPayload)
     encryptionService.decryptSplitSingle = jest.fn().mockReturnValue(encryptedPayload)
 
-    isRemotePayloadAllowed.mockImplementation(() => true)
+    checkRemotePayloadAllowed.mockImplementation(() => ({ allowed: {} }))
   })
 
   it('should get revision', async () => {
@@ -145,7 +145,7 @@ describe('GetRevision', () => {
   })
 
   it('should fail if remote payload is not allowed', async () => {
-    isRemotePayloadAllowed.mockImplementation(() => false)
+    checkRemotePayloadAllowed.mockImplementation(() => ({ disallowed: {} }))
 
     const useCase = createUseCase()
 

packages/snjs/lib/Domain/UseCase/GetRevision/GetRevision.ts

@@ -5,7 +5,7 @@ import {
   EncryptedPayload,
   HistoryEntry,
   isErrorDecryptingPayload,
-  isRemotePayloadAllowed,
+  checkRemotePayloadAllowed,
   NoteContent,
   PayloadTimestampDefaults,
 } from '@standardnotes/models'
@@ -71,7 +71,8 @@ export class GetRevision implements UseCaseInterface<HistoryEntry> {
       uuid: sourceItemUuid || revision.item_uuid,
     })
 
-    if (!isRemotePayloadAllowed(payload as ServerItemResponse)) {
+    const remotePayloadAllowedResult = checkRemotePayloadAllowed(payload as ServerItemResponse)
+    if (remotePayloadAllowedResult.disallowed !== undefined) {
       return Result.fail(`Remote payload is disallowed: ${JSON.stringify(payload)}`)
     }
 

packages/snjs/lib/Migrations/MigrationServices.ts

@@ -1,5 +1,5 @@
 import { BackupServiceInterface } from '@standardnotes/files'
-import { Environment, Platform } from '@standardnotes/models'
+import { Environment, Platform, ApplicationIdentifier } from '@standardnotes/models'
 import {
   DeviceInterface,
   InternalEventBusInterface,
@@ -8,7 +8,6 @@ import {
   PreferenceServiceInterface,
 } from '@standardnotes/services'
 import { SessionManager } from '../Services/Session/SessionManager'
-import { ApplicationIdentifier } from '@standardnotes/common'
 import { ItemManager } from '@Lib/Services/Items/ItemManager'
 import { ChallengeService, SingletonManager, FeaturesService, DiskStorageService } from '@Lib/Services'
 import { LegacySession, MapperInterface } from '@standardnotes/domain-core'

packages/snjs/lib/Migrations/StorageReaders/Functions.ts

@@ -1,5 +1,4 @@
-import { ApplicationIdentifier } from '@standardnotes/common'
-import { Environment } from '@standardnotes/models'
+import { Environment, ApplicationIdentifier } from '@standardnotes/models'
 import { compareSemVersions, isRightVersionGreaterThanLeft } from '@Lib/Version'
 import { DeviceInterface } from '@standardnotes/services'
 import { StorageReader } from './Reader'

packages/snjs/lib/Migrations/StorageReaders/Reader.ts

@@ -1,5 +1,4 @@
-import { Environment } from '@standardnotes/models'
-import { ApplicationIdentifier } from '@standardnotes/common'
+import { Environment, ApplicationIdentifier } from '@standardnotes/models'
 import { DeviceInterface } from '@standardnotes/services'
 
 /**

packages/snjs/lib/Services/Sync/Account/Response.ts

@@ -10,6 +10,8 @@ import {
   NotificationServerHash,
   AsymmetricMessageServerHash,
   getErrorFromErrorResponse,
+  ConflictType,
+  ServerItemResponse,
 } from '@standardnotes/responses'
 import {
   FilterDisallowedRemotePayloadsAndMap,
@@ -44,15 +46,21 @@ export class ServerSyncResponse {
     const legacyConflicts = this.successResponseData?.unsaved || []
     this.rawConflictObjects = conflicts.concat(legacyConflicts)
 
-    this.savedPayloads = FilterDisallowedRemotePayloadsAndMap(this.successResponseData?.saved_items || []).map(
-      (rawItem) => {
-        return CreateServerSyncSavedPayload(rawItem)
-      },
+    const disallowedPayloads = []
+
+    const savedItemsFilteringResult = FilterDisallowedRemotePayloadsAndMap(this.successResponseData?.saved_items || [])
+    this.savedPayloads = savedItemsFilteringResult.filtered.map((rawItem) => {
+      return CreateServerSyncSavedPayload(rawItem)
+    })
+    disallowedPayloads.push(...savedItemsFilteringResult.disallowed)
+
+    const retrievedItemsFilteringResult = FilterDisallowedRemotePayloadsAndMap(
+      this.successResponseData?.retrieved_items || [],
     )
+    this.retrievedPayloads = retrievedItemsFilteringResult.filtered
+    disallowedPayloads.push(...retrievedItemsFilteringResult.disallowed)
 
-    this.retrievedPayloads = FilterDisallowedRemotePayloadsAndMap(this.successResponseData?.retrieved_items || [])
-
-    this.conflicts = this.filterConflicts()
+    this.conflicts = this.filterConflictsAndDisallowedPayloads(disallowedPayloads)
 
     this.vaults = this.successResponseData?.shared_vaults || []
 
@@ -65,20 +73,48 @@ export class ServerSyncResponse {
     deepFreeze(this)
   }
 
-  private filterConflicts(): TrustedServerConflictMap {
+  private filterConflictsAndDisallowedPayloads(disallowedPayloads: ServerItemResponse[]): TrustedServerConflictMap {
     const conflicts = this.rawConflictObjects
     const trustedConflicts: TrustedServerConflictMap = {}
 
+    trustedConflicts[ConflictType.InvalidServerItem] = []
+    const invalidServerConflictsArray = trustedConflicts[ConflictType.InvalidServerItem]
+
+    for (const payload of disallowedPayloads) {
+      invalidServerConflictsArray.push(<TrustedConflictParams>{
+        type: ConflictType.InvalidServerItem,
+        server_item: payload,
+      })
+    }
+
     for (const conflict of conflicts) {
       let serverItem: FilteredServerItem | undefined
       let unsavedItem: FilteredServerItem | undefined
 
       if (conflict.unsaved_item) {
-        unsavedItem = FilterDisallowedRemotePayloadsAndMap([conflict.unsaved_item])[0]
+        const unsavedItemFilteringResult = FilterDisallowedRemotePayloadsAndMap([conflict.unsaved_item])
+        if (unsavedItemFilteringResult.filtered.length === 1) {
+          unsavedItem = unsavedItemFilteringResult.filtered[0]
+        }
+        if (unsavedItemFilteringResult.disallowed.length === 1) {
+          invalidServerConflictsArray.push(<TrustedConflictParams>{
+            type: ConflictType.InvalidServerItem,
+            unsaved_item: unsavedItemFilteringResult.disallowed[0],
+          })
+        }
       }
 
       if (conflict.server_item) {
-        serverItem = FilterDisallowedRemotePayloadsAndMap([conflict.server_item])[0]
+        const serverItemFilteringResult = FilterDisallowedRemotePayloadsAndMap([conflict.server_item])
+        if (serverItemFilteringResult.filtered.length === 1) {
+          serverItem = serverItemFilteringResult.filtered[0]
+        }
+        if (serverItemFilteringResult.disallowed.length === 1) {
+          invalidServerConflictsArray.push(<TrustedConflictParams>{
+            type: ConflictType.InvalidServerItem,
+            server_item: serverItemFilteringResult.disallowed[0],
+          })
+        }
       }
 
       if (!trustedConflicts[conflict.type]) {

packages/snjs/lib/Services/Sync/Account/ResponseResolver.ts

@@ -92,6 +92,7 @@ export class ServerSyncResponseResolver {
       ...this.getConflictsForType(ConflictType.SharedVaultInsufficientPermissionsError),
       ...this.getConflictsForType(ConflictType.SharedVaultNotMemberError),
       ...this.getConflictsForType(ConflictType.SharedVaultInvalidState),
+      ...this.getConflictsForType(ConflictType.InvalidServerItem),
     ]
 
     const delta = new DeltaRemoteRejected(this.baseCollection, conflicts)

packages/snjs/lib/Services/Sync/Account/Utilities.ts

@@ -1,3 +1,4 @@
+import { Result } from '@standardnotes/domain-core'
 import {
   EncryptedPayloadInterface,
   DeletedPayloadInterface,
@@ -10,22 +11,27 @@ import {
 export function CreatePayloadFromRawServerItem(
   rawItem: FilteredServerItem,
   source: PayloadSource,
-): EncryptedPayloadInterface | DeletedPayloadInterface {
+): Result<EncryptedPayloadInterface | DeletedPayloadInterface> {
   if (rawItem.deleted) {
-    return new DeletedPayload({ ...rawItem, content: undefined, deleted: true }, source)
+    return Result.ok(new DeletedPayload({ ...rawItem, content: undefined, deleted: true }, source))
   } else if (rawItem.content != undefined) {
-    return new EncryptedPayload(
-      {
-        ...rawItem,
-        items_key_id: rawItem.items_key_id,
-        content: rawItem.content,
-        deleted: false,
-        errorDecrypting: false,
-        waitingForKey: false,
-      },
-      source,
-    )
-  } else {
-    throw Error('Unhandled case in createPayloadFromRawItem')
+    try {
+      return Result.ok(
+        new EncryptedPayload(
+          {
+            ...rawItem,
+            items_key_id: rawItem.items_key_id,
+            content: rawItem.content,
+            deleted: false,
+            errorDecrypting: false,
+            waitingForKey: false,
+          },
+          source,
+        ),
+      )
+    } catch (error) {
+      return Result.fail(JSON.stringify(error))
+    }
   }
+  return Result.fail('Unhandled case in createPayloadFromRawItem')
 }

packages/snjs/lib/Services/Sync/SyncService.ts

@@ -1110,7 +1110,12 @@ export class SyncService
     items: FilteredServerItem[],
     source: PayloadSource,
   ): Promise<FullyFormedPayloadInterface[]> {
-    const payloads = items.map((i) => CreatePayloadFromRawServerItem(i, source))
+    const payloads = items
+      .map((i) => {
+        const result = CreatePayloadFromRawServerItem(i, source)
+        return result.isFailed() ? undefined : result.getValue()
+      })
+      .filter(isNotUndefined)
 
     const { encrypted, deleted } = CreateNonDecryptedPayloadSplit(payloads)
 
@@ -1408,9 +1413,16 @@ export class SyncService
       return
     }
 
-    const receivedPayloads = FilterDisallowedRemotePayloadsAndMap(rawPayloads).map((rawPayload) => {
-      return CreatePayloadFromRawServerItem(rawPayload, PayloadSource.RemoteRetrieved)
-    })
+    const rawPayloadsFilteringResult = FilterDisallowedRemotePayloadsAndMap(rawPayloads)
+    const receivedPayloads = rawPayloadsFilteringResult.filtered
+      .map((rawPayload) => {
+        const result = CreatePayloadFromRawServerItem(rawPayload, PayloadSource.RemoteRetrieved)
+        if (result.isFailed()) {
+          return undefined
+        }
+        return result.getValue()
+      })
+      .filter(isNotUndefined)
 
     const payloadSplit = CreateNonDecryptedPayloadSplit(receivedPayloads)
 

packages/snjs/lib/index.ts

@@ -8,7 +8,7 @@ export * from './Migrations'
 export * from './Services'
 export * from './Types'
 export * from './Version'
-export * from '@standardnotes/common'
+export { KeyParamsOrigination } from '@standardnotes/common'
 export * from '@standardnotes/domain-core'
 export * from '@standardnotes/api'
 export * from '@standardnotes/encryption'

packages/snjs/mocha/lib/factory.js

@@ -1,4 +1,3 @@
-
 import FakeWebCrypto from './fake_web_crypto.js'
 import { AppContext } from './AppContext.js'
 import { VaultsContext } from './VaultsContext.js'
@@ -303,8 +302,10 @@ export function tomorrow() {
   return new Date(new Date().setDate(new Date().getDate() + 1))
 }
 
-export async function sleep(seconds, reason) {
-  console.log('[Factory] Sleeping for reason', reason)
+export async function sleep(seconds, reason, dontLog = false) {
+  if (!dontLog) {
+    console.log('[Factory] Sleeping for reason', reason)
+  }
   return Utils.sleep(seconds)
 }
 

packages/snjs/mocha/payload.test.js

@@ -74,7 +74,7 @@ describe('payload', () => {
           content_type: ContentType.TYPES.Note,
           content: '000:somebase64string',
         }),
-      'Unrecognized protocol version 000',
+      'EncryptedPayload constructor versionResult is failed',
     )
   })
 

packages/snjs/mocha/sync_tests/online.test.js

@@ -113,7 +113,7 @@ describe('online syncing', function () {
 
     for (let i = 0; i < syncCount; i++) {
       application.sync.sync(syncOptions)
-      await Factory.sleep(0.01)
+      await Factory.sleep(0.01, undefined, true)
     }
     await promise
     expect(promise).to.be.fulfilled
@@ -958,6 +958,11 @@ describe('online syncing', function () {
       },
     })
 
+    context.anticipateConsoleError(
+      'Error decrypting payload',
+      'The encrypted payload above is not a valid encrypted payload.',
+    )
+
     await application.sync.handleSuccessServerResponse({ payloadsSavedOrSaving: [], options: {} }, response)
 
     expect(application.payloads.findOne(invalidPayload.uuid)).to.not.be.ok

packages/snjs/package.json

@@ -37,7 +37,7 @@
     "@babel/preset-env": "*",
     "@standardnotes/api": "workspace:*",
     "@standardnotes/common": "^1.50.0",
-    "@standardnotes/domain-core": "^1.24.0",
+    "@standardnotes/domain-core": "^1.25.0",
     "@standardnotes/domain-events": "^2.108.1",
     "@standardnotes/encryption": "workspace:*",
     "@standardnotes/features": "workspace:*",