refactor: evernote imports (#2619) [skip e2e]

2023-11-04 18:26:22 +05:30
parent 3d794bb12e
commit 20354a926a
11 changed files with 145 additions and 22 deletions
--- a/packages/files/src/Domain/Service/SuperConverterServiceInterface.ts
+++ b/packages/files/src/Domain/Service/SuperConverterServiceInterface.ts
@@ -1,6 +1,15 @@
+import { FileItem } from '@standardnotes/models'
+import { GenerateUuid } from '@standardnotes/services'
+
 export interface SuperConverterServiceInterface {
  isValidSuperString(superString: string): boolean
  convertSuperStringToOtherFormat: (superString: string, toFormat: 'txt' | 'md' | 'html' | 'json') => Promise<string>
  convertOtherFormatToSuperString: (otherFormatString: string, fromFormat: 'txt' | 'md' | 'html' | 'json') => string
  getEmbeddedFileIDsFromSuperString(superString: string): string[]
+  uploadAndReplaceInlineFilesInSuperString(
+    superString: string,
+    uploadFile: (file: File) => Promise<FileItem | undefined>,
+    linkFile: (file: FileItem) => Promise<void>,
+    generateUuid: GenerateUuid,
+  ): Promise<string>
 }
--- a/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts
+++ b/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts
@@ -3,7 +3,7 @@
 */

 import { ContentType } from '@standardnotes/domain-core'
-import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
+import { DecryptedTransferPayload, FileItem, NoteContent, TagContent } from '@standardnotes/models'
 import { EvernoteConverter, EvernoteResource } from './EvernoteConverter'
 import { createTestResourceElement, enex, enexWithNoNoteOrTag } from './testData'
 import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
@@ -33,6 +33,12 @@ describe('EvernoteConverter', () => {
    convertOtherFormatToSuperString: (data: string) => data,
    convertSuperStringToOtherFormat: async (data: string) => data,
    getEmbeddedFileIDsFromSuperString: () => [],
+    uploadAndReplaceInlineFilesInSuperString: async (
+      superString: string,
+      _uploadFile: (file: File) => Promise<FileItem | undefined>,
+      _linkFile: (file: FileItem) => Promise<void>,
+      _generateUuid: GenerateUuid,
+    ) => superString,
  }

  const generateUuid = new GenerateUuid(crypto)
--- a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts
+++ b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts
@@ -6,7 +6,7 @@ import { jsonTextContentData, htmlTestData, jsonListContentData } from './testDa
 import { GoogleKeepConverter } from './GoogleKeepConverter'
 import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
 import { GenerateUuid } from '@standardnotes/services'
-import { SuperConverterServiceInterface } from '@standardnotes/snjs'
+import { FileItem, SuperConverterServiceInterface } from '@standardnotes/snjs'

 describe('GoogleKeepConverter', () => {
  const crypto = {
@@ -18,6 +18,12 @@ describe('GoogleKeepConverter', () => {
    convertOtherFormatToSuperString: (data: string) => data,
    convertSuperStringToOtherFormat: async (data: string) => data,
    getEmbeddedFileIDsFromSuperString: () => [],
+    uploadAndReplaceInlineFilesInSuperString: async (
+      superString: string,
+      _uploadFile: (file: File) => Promise<FileItem | undefined>,
+      _linkFile: (file: FileItem) => Promise<void>,
+      _generateUuid: GenerateUuid,
+    ) => superString,
  }
  const generateUuid = new GenerateUuid(crypto)

--- a/packages/ui-services/src/Import/Importer.ts
+++ b/packages/ui-services/src/Import/Importer.ts
@@ -6,14 +6,23 @@ import {
  ItemManagerInterface,
  MutatorClientInterface,
 } from '@standardnotes/services'
-import { NativeFeatureIdentifier } from '@standardnotes/features'
+import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
 import { AegisToAuthenticatorConverter } from './AegisConverter/AegisToAuthenticatorConverter'
 import { EvernoteConverter } from './EvernoteConverter/EvernoteConverter'
 import { GoogleKeepConverter } from './GoogleKeepConverter/GoogleKeepConverter'
 import { PlaintextConverter } from './PlaintextConverter/PlaintextConverter'
 import { SimplenoteConverter } from './SimplenoteConverter/SimplenoteConverter'
 import { readFileAsText } from './Utils'
-import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
+import {
+  DecryptedItemInterface,
+  DecryptedTransferPayload,
+  FileItem,
+  ItemContent,
+  NoteContent,
+  NoteMutator,
+  SNNote,
+  isNote,
+} from '@standardnotes/models'
 import { HTMLConverter } from './HTMLConverter/HTMLConverter'
 import { SuperConverterServiceInterface } from '@standardnotes/snjs/dist/@types'
 import { SuperConverter } from './SuperConverter/SuperConverter'
@@ -34,7 +43,22 @@ export class Importer {
    private mutator: MutatorClientInterface,
    private items: ItemManagerInterface,
    private superConverterService: SuperConverterServiceInterface,
-    _generateUuid: GenerateUuid,
+    private filesController: {
+      uploadNewFile(
+        fileOrHandle: File | FileSystemFileHandle,
+        options?: {
+          showToast?: boolean
+          note?: SNNote
+        },
+      ): Promise<FileItem | undefined>
+    },
+    private linkingController: {
+      linkItems(
+        item: DecryptedItemInterface<ItemContent>,
+        itemToLink: DecryptedItemInterface<ItemContent>,
+      ): Promise<void>
+    },
+    private _generateUuid: GenerateUuid,
  ) {
    this.aegisConverter = new AegisToAuthenticatorConverter(_generateUuid)
    this.googleKeepConverter = new GoogleKeepConverter(this.superConverterService, _generateUuid)
@@ -140,6 +164,27 @@ export class Importer {
        return this.mutator.insertItem(note)
      }),
    )
+    for (const item of insertedItems) {
+      if (!isNote(item)) {
+        continue
+      }
+      if (item.noteType !== NoteType.Super) {
+        continue
+      }
+      try {
+        const text = await this.superConverterService.uploadAndReplaceInlineFilesInSuperString(
+          item.text,
+          async (file) => await this.filesController.uploadNewFile(file, { showToast: true, note: item }),
+          async (file) => await this.linkingController.linkItems(item, file),
+          this._generateUuid,
+        )
+        await this.mutator.changeItem<NoteMutator>(item, (mutator) => {
+          mutator.text = text
+        })
+      } catch (error) {
+        console.error(error)
+      }
+    }
    return insertedItems
  }
 }
--- a/packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts
+++ b/packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts
@@ -34,7 +34,7 @@ import { SubscriptionController } from '@/Controllers/Subscription/SubscriptionC
 import { PurchaseFlowController } from '@/Controllers/PurchaseFlow/PurchaseFlowController'
 import { FilesController } from '@/Controllers/FilesController'
 import { HistoryModalController } from '@/Controllers/NoteHistory/HistoryModalController'
-import { ImportModalController } from '@/Controllers/ImportModalController'
+import { ImportModalController } from '@/Components/ImportModal/ImportModalController'
 import { ApplicationEventObserver } from '@/Event/ApplicationEventObserver'
 import { SearchOptionsController } from '@/Controllers/SearchOptionsController'
 import { LinkingController } from '@/Controllers/LinkingController'
@@ -64,6 +64,8 @@ export class WebDependencies extends DependencyContainer {
        application.mutator,
        application.items,
        this.get<HeadlessSuperConverter>(Web_TYPES.SuperConverter),
+        this.get<FilesController>(Web_TYPES.FilesController),
+        this.get<LinkingController>(Web_TYPES.LinkingController),
        application.generateUuid,
      )
    })
--- a/packages/web/src/javascripts/Application/WebApplication.ts
+++ b/packages/web/src/javascripts/Application/WebApplication.ts
@@ -65,7 +65,7 @@ import { PurchaseFlowController } from '@/Controllers/PurchaseFlow/PurchaseFlowC
 import { AccountMenuController } from '@/Controllers/AccountMenu/AccountMenuController'
 import { PreferencesController } from '@/Controllers/PreferencesController'
 import { NotesController } from '@/Controllers/NotesController/NotesController'
-import { ImportModalController } from '@/Controllers/ImportModalController'
+import { ImportModalController } from '@/Components/ImportModal/ImportModalController'
 import { SyncStatusController } from '@/Controllers/SyncStatusController'
 import { HistoryModalController } from '@/Controllers/NoteHistory/HistoryModalController'
 import { NavigationController } from '@/Controllers/Navigation/NavigationController'
--- a/packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx
+++ b/packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx
@@ -4,7 +4,7 @@ import ImportModalFileItem from './ImportModalFileItem'
 import ImportModalInitialPage from './InitialPage'
 import Modal, { ModalAction } from '../Modal/Modal'
 import ModalOverlay from '../Modal/ModalOverlay'
-import { ImportModalController } from '@/Controllers/ImportModalController'
+import { ImportModalController } from '@/Components/ImportModal/ImportModalController'
 import { useApplication } from '../ApplicationProvider'
 import Switch from '../Switch/Switch'

@@ -66,7 +66,7 @@ const ImportModal = ({ importModalController }: { importModalController: ImportM
          )}
        </div>
        {files.length > 0 && (
-          <label className="py-2 px-4 flex items-center gap-2 border-t border-border">
+          <label className="flex items-center gap-2 border-t border-border px-4 py-2">
            <Switch checked={shouldCreateTag} onChange={setShouldCreateTag} />
            <span className="text-sm">Create tag with all imported notes</span>
          </label>
--- a/packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts
+++ b/packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts
@@ -8,7 +8,7 @@ import {
 } from '@standardnotes/snjs'
 import { Importer, NoteImportType } from '@standardnotes/ui-services'
 import { action, makeObservable, observable } from 'mobx'
-import { NavigationController } from './Navigation/NavigationController'
+import { NavigationController } from '../../Controllers/Navigation/NavigationController'

 type ImportModalFileCommon = {
  id: string
--- a/packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx
+++ b/packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx
@@ -1,4 +1,4 @@
-import { ImportModalController, ImportModalFile } from '@/Controllers/ImportModalController'
+import { ImportModalController, ImportModalFile } from '@/Components/ImportModal/ImportModalController'
 import { classNames, ContentType, DecryptedTransferPayload, pluralize } from '@standardnotes/snjs'
 import { Importer, NoteImportType } from '@standardnotes/ui-services'
 import { observer } from 'mobx-react-lite'
--- a/packages/web/src/javascripts/Components/ImportModal/InitialPage.tsx
+++ b/packages/web/src/javascripts/Components/ImportModal/InitialPage.tsx
@@ -1,4 +1,4 @@
-import { ImportModalController } from '@/Controllers/ImportModalController'
+import { ImportModalController } from '@/Components/ImportModal/ImportModalController'
 import { ClassicFileReader } from '@standardnotes/filepicker'
 import { NoteImportType } from '@standardnotes/ui-services'
 import { observer } from 'mobx-react-lite'
@@ -41,30 +41,30 @@ const ImportModalInitialPage = ({ setFiles }: Props) => {
        <div className="text-lg font-semibold">Drag and drop files to auto-detect and import</div>
        <div className="text-sm">Or click to open file picker</div>
      </button>
-      <div className="text-center my-4 w-full">or import from:</div>
+      <div className="my-4 w-full text-center">or import from:</div>
      <div className="flex flex-wrap items-center justify-center gap-4">
        <Button className="flex items-center !py-2" onClick={() => selectFiles('evernote')}>
-          <Icon type="evernote" className="text-[#14cc45] mr-2" />
+          <Icon type="evernote" className="mr-2 text-[#14cc45]" />
          Evernote
        </Button>
        <Button className="flex items-center !py-2" onClick={() => selectFiles('google-keep')}>
-          <Icon type="gkeep" className="text-[#fbbd00] mr-2" />
+          <Icon type="gkeep" className="mr-2 text-[#fbbd00]" />
          Google Keep
        </Button>
        <Button className="flex items-center !py-2" onClick={() => selectFiles('simplenote')}>
-          <Icon type="simplenote" className="text-[#3360cc] mr-2" />
+          <Icon type="simplenote" className="mr-2 text-[#3360cc]" />
          Simplenote
        </Button>
        <Button className="flex items-center !py-2" onClick={() => selectFiles('aegis')}>
-          <Icon type="aegis" className="bg-[#0d47a1] text-[#fff] rounded mr-2 p-1" size="normal" />
+          <Icon type="aegis" className="mr-2 rounded bg-[#0d47a1] p-1 text-[#fff]" size="normal" />
          Aegis
        </Button>
        <Button className="flex items-center !py-2" onClick={() => selectFiles('plaintext')}>
-          <Icon type="plain-text" className="text-info mr-2" />
+          <Icon type="plain-text" className="mr-2 text-info" />
          Plaintext / Markdown
        </Button>
        <Button className="flex items-center !py-2" onClick={() => selectFiles('html')}>
-          <Icon type="rich-text" className="text-accessory-tint-2 mr-2" />
+          <Icon type="rich-text" className="mr-2 text-accessory-tint-2" />
          HTML
        </Button>
        <Button
@@ -81,7 +81,7 @@ const ImportModalInitialPage = ({ setFiles }: Props) => {
            selectFiles('super').catch(console.error)
          }}
        >
-          <Icon type="file-doc" className="text-accessory-tint-1 mr-2" />
+          <Icon type="file-doc" className="mr-2 text-accessory-tint-1" />
          Super (JSON)
        </Button>
      </div>
--- a/packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx
+++ b/packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx
@@ -1,6 +1,6 @@
 import { createHeadlessEditor } from '@lexical/headless'
 import { $convertToMarkdownString, $convertFromMarkdownString } from '@lexical/markdown'
-import { FileItem, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
+import { FileItem, GenerateUuid, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
 import {
  $createParagraphNode,
  $getRoot,
@@ -16,7 +16,8 @@ import { MarkdownTransformers } from '../MarkdownTransformers'
 import { $generateHtmlFromNodes, $generateNodesFromDOM } from '@lexical/html'
 import { FileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileNode'
 import { $createFileExportNode } from '../Lexical/Nodes/FileExportNode'
-import { $createInlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
+import { $createInlineFileNode, InlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
+import { $createFileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileUtils'
 export class HeadlessSuperConverter implements SuperConverterServiceInterface {
  private importEditor: LexicalEditor
  private exportEditor: LexicalEditor
@@ -253,4 +254,58 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {

    return ids
  }
+
+  async uploadAndReplaceInlineFilesInSuperString(
+    superString: string,
+    uploadFile: (file: File) => Promise<FileItem | undefined>,
+    linkFile: (file: FileItem) => Promise<void>,
+    generateUuid: GenerateUuid,
+  ): Promise<string> {
+    if (superString.length === 0) {
+      return superString
+    }
+
+    this.importEditor.setEditorState(this.importEditor.parseEditorState(superString))
+
+    await new Promise<void>((resolve) => {
+      this.importEditor.update(
+        () => {
+          const inlineFileNodes = $nodesOfType(InlineFileNode)
+          if (inlineFileNodes.length === 0) {
+            resolve()
+            return
+          }
+          Promise.all(
+            inlineFileNodes.map(async (node) => {
+              const blob = await fetch(node.__src).then((response) => response.blob())
+              const file = new File([blob], node.__fileName || generateUuid.execute().getValue(), {
+                type: node.__mimeType,
+              })
+
+              const uploadedFile = await uploadFile(file)
+
+              if (!uploadedFile) {
+                return
+              }
+
+              this.importEditor.update(
+                () => {
+                  const fileNode = $createFileNode(uploadedFile.uuid)
+                  node.replace(fileNode)
+                },
+                { discrete: true },
+              )
+
+              await linkFile(uploadedFile)
+            }),
+          )
+            .then(() => resolve())
+            .catch(console.error)
+        },
+        { discrete: true },
+      )
+    })
+
+    return JSON.stringify(this.importEditor.getEditorState())
+  }
 }