clipper: handle clipped note sync in background (#2367)

packages/api/src/Domain/Http/HttpService.ts

@@ -181,7 +181,7 @@ export class HttpService implements HttpServiceInterface {
     return response
   }
 
-  private async refreshSession(): Promise<boolean> {
+  async refreshSession(): Promise<boolean> {
     if (!this.session) {
       return false
     }

packages/api/src/Domain/Http/HttpServiceInterface.ts

@@ -14,6 +14,7 @@ export interface HttpServiceInterface {
   runHttp<T>(httpRequest: HttpRequest): Promise<HttpResponse<T>>
 
   setSession(session: Session | LegacySession): void
+  refreshSession(): Promise<boolean>
   setCallbacks(
     updateMetaCallback: (meta: HttpResponseMeta) => void,
     refreshSessionCallback: (session: Session) => void,

packages/api/src/Domain/Http/index.ts

@@ -1,3 +1,4 @@
 export * from './HttpService'
+export * from './FetchRequestHandler'
 export * from './HttpServiceInterface'
 export * from './XMLHttpRequestState'

packages/clipper/src/background/background.ts

@@ -1,5 +1,7 @@
 import { runtime, action, browserAction, windows, storage, tabs } from 'webextension-polyfill'
 import { ClipPayload, RuntimeMessage, RuntimeMessageTypes } from '../types/message'
+import { Environment, FetchRequestHandler, Logger, SnjsVersion } from '@standardnotes/snjs'
+import packageInfo from '../../package.json'
 
 const isFirefox = navigator.userAgent.indexOf('Firefox/') !== -1
 
@@ -22,6 +24,9 @@ const openPopupAndClipSelection = async (payload: ClipPayload) => {
   void openPopup()
 }
 
+const logger = new Logger('clipper')
+const requestHandler = new FetchRequestHandler(SnjsVersion, packageInfo.version, Environment.Clipper, logger)
+
 runtime.onMessage.addListener(async (message: RuntimeMessage) => {
   if (message.type === RuntimeMessageTypes.OpenPopupWithSelection) {
     if (!message.payload) {
@@ -32,5 +37,7 @@ runtime.onMessage.addListener(async (message: RuntimeMessage) => {
     return await tabs.captureVisibleTab(undefined, {
       format: 'png',
     })
+  } else if (message.type === RuntimeMessageTypes.RunHttpRequest) {
+    requestHandler.handleRequest(message.payload).catch(console.error)
   }
 })

packages/clipper/src/types/message.ts

@@ -1,3 +1,5 @@
+import { HttpRequest } from '@standardnotes/snjs'
+
 export const RuntimeMessageTypes = {
   GetArticle: 'get-article',
   GetSelection: 'get-selection',
@@ -7,6 +9,7 @@ export const RuntimeMessageTypes = {
   StartNodeSelection: 'start-node-selection',
   ToggleScreenshotMode: 'toggle-screenshot-mode',
   CaptureVisibleTab: 'capture-visible-tab',
+  RunHttpRequest: 'run-http-request',
 } as const
 
 export type RuntimeMessageType = (typeof RuntimeMessageTypes)[keyof typeof RuntimeMessageTypes]
@@ -29,6 +32,7 @@ export type RuntimeMessageReturnTypes = {
   [RuntimeMessageTypes.OpenPopupWithSelection]: void
   [RuntimeMessageTypes.StartNodeSelection]: void
   [RuntimeMessageTypes.ToggleScreenshotMode]: void
+  [RuntimeMessageTypes.RunHttpRequest]: void
 }
 
 export type RuntimeMessage =
@@ -36,10 +40,19 @@ export type RuntimeMessage =
       type: MessagesWithClipPayload
       payload: ClipPayload
     }
+  | {
+      type: typeof RuntimeMessageTypes.RunHttpRequest
+      payload: HttpRequest
+    }
   | {
       type: typeof RuntimeMessageTypes.ToggleScreenshotMode
       enabled: boolean
     }
   | {
-      type: Exclude<RuntimeMessageType, MessagesWithClipPayload | typeof RuntimeMessageTypes.ToggleScreenshotMode>
+      type: Exclude<
+        RuntimeMessageType,
+        | MessagesWithClipPayload
+        | typeof RuntimeMessageTypes.ToggleScreenshotMode
+        | typeof RuntimeMessageTypes.RunHttpRequest
+      >
     }

packages/models/src/Domain/Device/Environment.ts

@@ -2,4 +2,5 @@ export enum Environment {
   Web = 1,
   Desktop = 2,
   Mobile = 3,
+  Clipper = 4,
 }

packages/services/src/Domain/Api/LegacyApiServiceInterface.ts

@@ -2,8 +2,8 @@ import { FilesApiInterface } from '@standardnotes/files'
 import { AbstractService } from '../Service/AbstractService'
 import { ApiServiceEvent } from './ApiServiceEvent'
 import { ApiServiceEventData } from './ApiServiceEventData'
-import { SNFeatureRepo } from '@standardnotes/models'
+import { SNFeatureRepo, ServerSyncPushContextualPayload } from '@standardnotes/models'
-import { ClientDisplayableError, HttpResponse } from '@standardnotes/responses'
+import { ClientDisplayableError, HttpRequest, HttpResponse } from '@standardnotes/responses'
 import { AnyFeatureDescription } from '@standardnotes/features'
 
 export interface LegacyApiServiceInterface
@@ -16,4 +16,12 @@ export interface LegacyApiServiceInterface
   ): Promise<{ features: AnyFeatureDescription[]; roles: string[] } | ClientDisplayableError>
 
   downloadFeatureUrl(url: string): Promise<HttpResponse>
+
+  getSyncHttpRequest(
+    payloads: ServerSyncPushContextualPayload[],
+    lastSyncToken: string | undefined,
+    paginationToken: string | undefined,
+    limit: number,
+    sharedVaultUuids?: string[],
+  ): HttpRequest
 }

packages/services/src/Domain/Session/SessionsClientInterface.ts

@@ -25,6 +25,7 @@ export interface SessionsClientInterface {
   isSignedIntoFirstPartyServer(): boolean
 
   getSessionsList(): Promise<HttpResponse<SessionListEntry[]>>
+  refreshSessionIfExpiringSoon(): Promise<boolean>
   revokeSession(sessionId: string): Promise<HttpResponse<SessionListResponse>>
   revokeAllOtherSessions(): Promise<void>
 

packages/services/src/Domain/Sync/SyncServiceInterface.ts

@@ -1,13 +1,18 @@
 /* istanbul ignore file */
 
-import { FullyFormedPayloadInterface } from '@standardnotes/models'
+import { DecryptedItemInterface, DeletedItemInterface, FullyFormedPayloadInterface } from '@standardnotes/models'
 import { SyncOptions } from './SyncOptions'
 import { AbstractService } from '../Service/AbstractService'
 import { SyncEvent } from '../Event/SyncEvent'
 import { SyncOpStatus } from './SyncOpStatus'
+import { HttpRequest } from '@standardnotes/responses'
 
 export interface SyncServiceInterface extends AbstractService<SyncEvent> {
   sync(options?: Partial<SyncOptions>): Promise<unknown>
+  getRawSyncRequestForExternalUse(
+    items: (DecryptedItemInterface | DeletedItemInterface)[],
+  ): Promise<HttpRequest | undefined>
+
   isDatabaseLoaded(): boolean
   onNewDatabaseCreated(): Promise<void>
   loadDatabasePayloads(): Promise<void>

packages/snjs/lib/Application/Platforms.ts

@@ -33,6 +33,7 @@ export function environmentToString(environment: Environment) {
     [Environment.Web]: 'web',
     [Environment.Desktop]: 'desktop',
     [Environment.Mobile]: 'native-mobile-web',
+    [Environment.Clipper]: 'clipper',
   }
   return map[environment]
 }

packages/snjs/lib/Services/Api/ApiService.ts

@@ -374,15 +374,8 @@ export class LegacyApiService
     if (preprocessingError) {
       return preprocessingError
     }
-    const path = Paths.v1.sync
+    const request = this.getSyncHttpRequest(payloads, lastSyncToken, paginationToken, limit, sharedVaultUuids)
-    const params = this.params({
+    const response = await this.httpService.runHttp<RawSyncResponse>(request)
-      [ApiEndpointParam.SyncPayloads]: payloads,
-      [ApiEndpointParam.LastSyncToken]: lastSyncToken,
-      [ApiEndpointParam.PaginationToken]: paginationToken,
-      [ApiEndpointParam.SyncDlLimit]: limit,
-      [ApiEndpointParam.SharedVaultUuids]: sharedVaultUuids,
-    })
-    const response = await this.httpService.post<RawSyncResponse>(path, params, this.getSessionAccessToken())
 
     if (isErrorResponse(response)) {
       this.preprocessAuthenticatedErrorResponse(response)
@@ -394,6 +387,29 @@ export class LegacyApiService
     return response
   }
 
+  getSyncHttpRequest(
+    payloads: ServerSyncPushContextualPayload[],
+    lastSyncToken: string | undefined,
+    paginationToken: string | undefined,
+    limit: number,
+    sharedVaultUuids?: string[] | undefined,
+  ): HttpRequest {
+    const path = Paths.v1.sync
+    const params = this.params({
+      [ApiEndpointParam.SyncPayloads]: payloads,
+      [ApiEndpointParam.LastSyncToken]: lastSyncToken,
+      [ApiEndpointParam.PaginationToken]: paginationToken,
+      [ApiEndpointParam.SyncDlLimit]: limit,
+      [ApiEndpointParam.SharedVaultUuids]: sharedVaultUuids,
+    })
+    return {
+      url: joinPaths(this.host, path),
+      params,
+      verb: HttpVerb.Post,
+      authentication: this.getSessionAccessToken(),
+    }
+  }
+
   async refreshSession(): Promise<HttpResponse<SessionRenewalResponse>> {
     const preprocessingError = this.preprocessingError()
     if (preprocessingError) {

packages/snjs/lib/Services/Session/SessionManager.ts

@@ -71,6 +71,7 @@ import {
 
 export const MINIMUM_PASSWORD_LENGTH = 8
 export const MissingAccountParams = 'missing-params'
+const ThirtyMinutes = 30 * 60 * 1000
 
 const cleanedEmailString = (email: string) => {
   return email.trim().toLowerCase()
@@ -837,4 +838,27 @@ export class SessionManager
 
     return Result.ok(sessionOrError.getValue())
   }
+
+  async refreshSessionIfExpiringSoon(): Promise<boolean> {
+    const session = this.getSession()
+
+    if (!session) {
+      return false
+    }
+    if (session instanceof LegacySession) {
+      return false
+    }
+
+    const accessTokenExpiration = new Date(session.accessToken.expiresAt)
+    const refreshTokenExpiration = new Date(session.refreshToken.expiresAt)
+
+    const willAccessTokenExpireSoon = accessTokenExpiration.getTime() - Date.now() < ThirtyMinutes
+    const willRefreshTokenExpireSoon = refreshTokenExpiration.getTime() - Date.now() < ThirtyMinutes
+
+    if (willAccessTokenExpireSoon || willRefreshTokenExpireSoon) {
+      return this.httpService.refreshSession()
+    }
+
+    return false
+  }
 }

packages/snjs/lib/Services/Sync/SyncService.ts

@@ -1,4 +1,4 @@
-import { ConflictParams, ConflictType } from '@standardnotes/responses'
+import { ConflictParams, ConflictType, HttpRequest } from '@standardnotes/responses'
 import { AccountSyncOperation } from '@Lib/Services/Sync/Account/Operation'
 import {
   LoggerInterface,
@@ -922,6 +922,26 @@ export class SyncService
     return undefined
   }
 
+  async getRawSyncRequestForExternalUse(
+    items: (DecryptedItemInterface | DeletedItemInterface)[],
+  ): Promise<HttpRequest | undefined> {
+    if (this.dealloced) {
+      return
+    }
+
+    const online = this.sessionManager.online()
+
+    if (!online) {
+      return
+    }
+
+    const payloads = await this.payloadsByPreparingForServer(items.map((i) => i.payloadRepresentation()))
+    const syncToken = await this.getLastSyncToken()
+    const paginationToken = await this.getPaginationToken()
+
+    return this.apiService.getSyncHttpRequest(payloads, syncToken, paginationToken, 150)
+  }
+
   private async handleOfflineResponse(response: OfflineSyncResponse) {
     this.logger.debug('Offline Sync Response', response)
 

packages/snjs/mocha/sync_tests/online.test.js

@@ -1047,4 +1047,25 @@ describe('online syncing', function () {
 
     await contextB.deinit()
   })
+
+  it('should sync note when running raw sync request for external use', async function () {
+    const contextA = this.context
+    const contextB = await Factory.createAppContextWithFakeCrypto('AppB', contextA.email, contextA.password)
+
+    await contextB.launch()
+    await contextB.signIn()
+
+    const notePayload = Factory.createNote()
+
+    const rawSyncRequest = await this.application.sync.getRawSyncRequestForExternalUse([notePayload])
+    expect(rawSyncRequest).to.be.ok
+
+    const response = await this.application.http.runHttp(rawSyncRequest)
+    expect(response.status).to.equal(200)
+
+    await contextB.sync()
+
+    const note = contextB.application.items.findItem(notePayload.uuid)
+    expect(note).to.be.ok
+  })
 })

packages/web/src/javascripts/Components/ClipperView/ClipperView.tsx

@@ -7,7 +7,7 @@ import { useApplication } from '../ApplicationProvider'
 import Icon from '../Icon/Icon'
 import Menu from '../Menu/Menu'
 import MenuItem from '../Menu/MenuItem'
-import { storage as extensionStorage, windows } from 'webextension-polyfill'
+import { storage as extensionStorage, runtime, windows } from 'webextension-polyfill'
 import sendMessageToActiveTab from '@standardnotes/clipper/src/utils/sendMessageToActiveTab'
 import { ClipPayload, RuntimeMessageTypes } from '@standardnotes/clipper/src/types/message'
 import { confirmDialog } from '@standardnotes/ui-services'
@@ -22,6 +22,7 @@ import {
   PrefKey,
   SNNote,
   SNTag,
+  classNames,
 } from '@standardnotes/snjs'
 import { addToast, ToastType } from '@standardnotes/toast'
 import { getSuperJSONFromClipPayload } from './getSuperJSONFromClipHTML'
@@ -36,6 +37,7 @@ import ItemSelectionDropdown from '../ItemSelectionDropdown/ItemSelectionDropdow
 import LinkedItemBubble from '../LinkedItems/LinkedItemBubble'
 import StyledTooltip from '../StyledTooltip/StyledTooltip'
 import MenuSwitchButtonItem from '../Menu/MenuSwitchButtonItem'
+import Spinner from '../Spinner/Spinner'
 
 const ClipperView = ({
   viewControllerManager,
@@ -60,6 +62,11 @@ const ClipperView = ({
   const isFirefoxPopup = !!currentWindow && currentWindow.type === 'popup' && currentWindow.incognito === false
 
   const [user, setUser] = useState(() => application.getUser())
+  const [isSyncing, setIsSyncing] = useState(false)
+  const [hasSyncError, setHasSyncError] = useState(false)
+  useEffect(() => {
+    application.sessions.refreshSessionIfExpiringSoon().catch(console.error)
+  }, [application.sessions])
   const [isEntitledToExtension, setIsEntitled] = useState(
     () =>
       application.features.getFeatureStatus(
@@ -88,6 +95,13 @@ const ClipperView = ({
             ) === FeatureStatus.Entitled,
           )
           break
+        case ApplicationEvent.SyncStatusChanged:
+        case ApplicationEvent.FailedSync: {
+          const status = application.sync.getSyncStatus()
+          setIsSyncing(status.syncInProgress)
+          setHasSyncError(status.hasError())
+          break
+        }
       }
     })
   }, [application])
@@ -241,10 +255,19 @@ const ClipperView = ({
         message: 'Note clipped successfully',
       })
 
-      void application.sync.sync()
+      const syncRequest = await application.sync.getRawSyncRequestForExternalUse([insertedNote])
+
+      if (syncRequest) {
+        runtime
+          .sendMessage({
+            type: RuntimeMessageTypes.RunHttpRequest,
+            payload: syncRequest,
+          })
+          .catch(console.error)
+      }
     }
 
-    void createNoteFromClip()
+    createNoteFromClip().catch(console.error)
   }, [
     application.items,
     application.linkingController,
@@ -431,6 +454,22 @@ const ClipperView = ({
             <Icon type="signOut" className="text-neutral" />
           </button>
         </div>
+        {isSyncing || hasSyncError ? (
+          <div className={classNames('flex items-center border-t border-border', hasSyncError && 'text-danger')}>
+            {isSyncing && (
+              <>
+                <Spinner className="w-4 h-4 mx-2.5" />
+                <div className="flex-grow py-2 text-sm font-semibold text-info">Syncing...</div>
+              </>
+            )}
+            {hasSyncError && (
+              <>
+                <Icon type="warning" className="mx-2.5" />
+                <div className="flex-grow py-2 text-sm font-semibold">Unable to sync</div>
+              </>
+            )}
+          </div>
+        ) : null}
       </Menu>
     </div>
   )