refactor: handle larger files in importer (#2692)

packages/files/src/Domain/Service/SuperConverterServiceInterface.ts

@@ -1,15 +1,6 @@
-import { FileItem } from '@standardnotes/models'
-import { GenerateUuid } from '@standardnotes/services'
-
 export interface SuperConverterServiceInterface {
   isValidSuperString(superString: string): boolean
   convertSuperStringToOtherFormat: (superString: string, toFormat: 'txt' | 'md' | 'html' | 'json') => Promise<string>
   convertOtherFormatToSuperString: (otherFormatString: string, fromFormat: 'txt' | 'md' | 'html' | 'json') => string
   getEmbeddedFileIDsFromSuperString(superString: string): string[]
-  uploadAndReplaceInlineFilesInSuperString(
-    superString: string,
-    uploadFile: (file: File) => Promise<FileItem | undefined>,
-    linkFile: (file: FileItem) => Promise<void>,
-    generateUuid: GenerateUuid,
-  ): Promise<string>
 }

packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts

@@ -1,5 +1,6 @@
 import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
 import { Converter } from '../Converter'
+import { ConversionResult } from '../ConversionResult'
 
 type AegisData = {
   db: {
@@ -45,7 +46,7 @@ export class AegisToAuthenticatorConverter implements Converter {
     return false
   }
 
-  convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
+  convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
     const content = await readFileAsText(file)
 
     const entries = this.parseEntries(content)
@@ -59,17 +60,22 @@ export class AegisToAuthenticatorConverter implements Converter {
     const title = file.name.split('.')[0]
     const text = JSON.stringify(entries)
 
-    return [
+    const note = await insertNote({
-      createNote({
+      createdAt,
-        createdAt,
+      updatedAt,
-        updatedAt,
+      title,
-        title,
+      text,
-        text,
+      noteType: NoteType.Authentication,
-        noteType: NoteType.Authentication,
+      editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
-        editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
+      useSuperIfPossible: false,
-        useSuperIfPossible: false,
+    })
-      }),
+
-    ]
+    const successful: ConversionResult['successful'] = [note]
+
+    return {
+      successful,
+      errored: [],
+    }
   }
 
   parseEntries(data: string): AuthenticatorEntry[] | null {

packages/ui-services/src/Import/ConversionResult.ts

@@ -0,0 +1,9 @@
+import { DecryptedItemInterface } from '@standardnotes/models'
+
+export type ConversionResult = {
+  successful: DecryptedItemInterface[]
+  errored: {
+    name: string
+    error: Error
+  }[]
+}

packages/ui-services/src/Import/Converter.ts

@@ -1,5 +1,6 @@
 import { NoteType } from '@standardnotes/features'
-import { DecryptedTransferPayload, ItemContent, NoteContent, TagContent } from '@standardnotes/models'
+import { DecryptedItemInterface, FileItem, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models'
+import { ConversionResult } from './ConversionResult'
 
 export interface Converter {
   getImportType(): string
@@ -12,17 +13,24 @@ export interface Converter {
   convert(
     file: File,
     dependencies: {
-      createNote: CreateNoteFn
+      insertNote: InsertNoteFn
-      createTag: CreateTagFn
+      insertTag: InsertTagFn
+      canUploadFiles: boolean
+      uploadFile: UploadFileFn
       canUseSuper: boolean
       convertHTMLToSuper: (html: string) => string
       convertMarkdownToSuper: (markdown: string) => string
       readFileAsText: (file: File) => Promise<string>
+      linkItems(
+        item: DecryptedItemInterface<ItemContent>,
+        itemToLink: DecryptedItemInterface<ItemContent>,
+      ): Promise<void>
+      cleanupItems(items: DecryptedItemInterface<ItemContent>[]): Promise<void>
     },
-  ): Promise<DecryptedTransferPayload<ItemContent>[]>
+  ): Promise<ConversionResult>
 }
 
-export type CreateNoteFn = (options: {
+export type InsertNoteFn = (options: {
   createdAt: Date
   updatedAt: Date
   title: string
@@ -33,10 +41,20 @@ export type CreateNoteFn = (options: {
   trashed?: boolean
   editorIdentifier?: NoteContent['editorIdentifier']
   useSuperIfPossible: boolean
-}) => DecryptedTransferPayload<NoteContent>
+}) => Promise<SNNote>
 
-export type CreateTagFn = (options: {
+export type InsertTagFn = (options: {
   createdAt: Date
   updatedAt: Date
   title: string
-}) => DecryptedTransferPayload<TagContent>
+  references: SNTag['references']
+}) => Promise<SNTag>
+
+export type UploadFileFn = (file: File) => Promise<FileItem | undefined>
+
+export type LinkItemsFn = (
+  item: DecryptedItemInterface<ItemContent>,
+  itemToLink: DecryptedItemInterface<ItemContent>,
+) => Promise<void>
+
+export type CleanupItemsFn = (items: DecryptedItemInterface<ItemContent>[]) => Promise<void>

packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts

@@ -3,9 +3,9 @@
  */
 
 import { ContentType } from '@standardnotes/domain-core'
-import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
+import { SNNote, SNTag } from '@standardnotes/models'
 import { EvernoteConverter, EvernoteResource } from './EvernoteConverter'
-import { createTestResourceElement, enex, enexWithNoNoteOrTag } from './testData'
+import { createTestResourceElement, enex } from './testData'
 import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
 import { GenerateUuid } from '@standardnotes/services'
 import { Converter } from '../Converter'
@@ -33,77 +33,80 @@ describe('EvernoteConverter', () => {
   const readFileAsText = async (file: File) => file as unknown as string
 
   const dependencies: Parameters<Converter['convert']>[1] = {
-    createNote: ({ text }) =>
+    insertNote: async ({ text }) =>
       ({
         content_type: ContentType.TYPES.Note,
         content: {
           text,
           references: [],
         },
-      }) as unknown as DecryptedTransferPayload<NoteContent>,
+        uuid: generateUuid.execute().getValue(),
-    createTag: ({ title }) =>
+      }) as unknown as SNNote,
+    insertTag: async ({ title }) =>
       ({
         content_type: ContentType.TYPES.Tag,
         content: {
           title,
           references: [],
         },
-      }) as unknown as DecryptedTransferPayload<TagContent>,
+        uuid: generateUuid.execute().getValue(),
+      }) as unknown as SNTag,
     convertHTMLToSuper: (data) => data,
     convertMarkdownToSuper: jest.fn(),
     readFileAsText,
     canUseSuper: false,
+    canUploadFiles: false,
+    uploadFile: async () => void 0,
+    linkItems: async (item, itemToLink) => {
+      itemToLink.content.references.push({
+        content_type: item.content_type,
+        uuid: item.uuid,
+      })
+    },
+    cleanupItems: async () => void 0,
   }
 
-  it('should throw error if no note or tag in enex', () => {
-    const converter = new EvernoteConverter(generateUuid)
-
-    expect(converter.convert(enexWithNoNoteOrTag as unknown as File, dependencies)).rejects.toThrowError()
-  })
-
   it('should parse and strip html', async () => {
     const converter = new EvernoteConverter(generateUuid)
 
-    const result = await converter.convert(enex as unknown as File, dependencies)
+    const { successful } = await converter.convert(enex as unknown as File, dependencies)
 
-    expect(result).not.toBeNull()
+    expect(successful).not.toBeNull()
-    expect(result?.length).toBe(3)
+    expect(successful?.length).toBe(3)
-    expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
+    expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe('This is a test.\nh e ')
+    expect((successful?.[0] as SNNote).content.text).toBe('This is a test.\nh e ')
-    expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
+    expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
-    expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
+    expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
-      'Lorem ipsum dolor sit amet, consectetur adipiscing elit.',
+    expect((successful?.[1] as SNTag).content.references.length).toBe(2)
-    )
+    expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
-    expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
+    expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
+    expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
+    expect((successful?.[2] as SNNote).content.text).toBe('Lorem ipsum dolor sit amet, consectetur adipiscing elit.')
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
   })
 
   it('should parse and not strip html', async () => {
     const converter = new EvernoteConverter(generateUuid)
 
-    const result = await converter.convert(enex as unknown as File, {
+    const { successful } = await converter.convert(enex as unknown as File, {
       ...dependencies,
       canUseSuper: true,
     })
 
-    expect(result).not.toBeNull()
+    expect(successful).not.toBeNull()
-    expect(result?.length).toBe(3)
+    expect(successful?.length).toBe(3)
-    expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
+    expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
+    expect((successful?.[0] as SNNote).content.text).toBe(
       '<p>This is a test.</p><ul></ul><ol></ol><font><span>h </span><span>e </span></font>',
     )
-    expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
+    expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
-    expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
+    expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
+    expect((successful?.[1] as SNTag).content.references.length).toBe(2)
+    expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
+    expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
+    expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
+    expect((successful?.[2] as SNNote).content.text).toBe(
       '<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>',
     )
-    expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
   })
 
   it('should convert lists to super format if applicable', () => {
@@ -129,7 +132,7 @@ describe('EvernoteConverter', () => {
     expect(unorderedList2.getAttribute('__lexicallisttype')).toBeFalsy()
   })
 
-  it('should replace media elements with resources', () => {
+  it('should replace media elements with resources', async () => {
     const resources: EvernoteResource[] = [
       {
         hash: 'hash1',
@@ -152,9 +155,14 @@ describe('EvernoteConverter', () => {
     const array = [mediaElement1, mediaElement2, mediaElement3]
 
     const converter = new EvernoteConverter(generateUuid)
-    const replacedCount = converter.replaceMediaElementsWithResources(array, resources)
+    const { replacedElements } = await converter.replaceMediaElementsWithResources(
+      array,
+      resources,
+      false,
+      dependencies.uploadFile,
+    )
 
-    expect(replacedCount).toBe(1)
+    expect(replacedElements.length).toBe(1)
   })
 
   describe('getResourceFromElement', () => {

packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts

@@ -1,11 +1,12 @@
-import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
+import { FileItem, SNTag } from '@standardnotes/models'
 import dayjs from 'dayjs'
 import customParseFormat from 'dayjs/plugin/customParseFormat'
 import utc from 'dayjs/plugin/utc'
 import { GenerateUuid } from '@standardnotes/services'
 import MD5 from 'crypto-js/md5'
 import Base64 from 'crypto-js/enc-base64'
-import { Converter } from '../Converter'
+import { Converter, UploadFileFn } from '../Converter'
+import { ConversionResult } from '../ConversionResult'
 dayjs.extend(customParseFormat)
 dayjs.extend(utc)
 
@@ -35,14 +36,23 @@ export class EvernoteConverter implements Converter {
 
   convert: Converter['convert'] = async (
     file,
-    { createNote, createTag, canUseSuper, convertHTMLToSuper, readFileAsText },
+    {
+      insertNote,
+      insertTag,
+      linkItems,
+      canUploadFiles,
+      canUseSuper,
+      convertHTMLToSuper,
+      readFileAsText,
+      uploadFile,
+      cleanupItems,
+    },
   ) => {
     const content = await readFileAsText(file)
 
     const xmlDoc = this.loadXMLString(content, 'xml')
     const xmlNotes = xmlDoc.getElementsByTagName('note')
-    const notes: DecryptedTransferPayload<NoteContent>[] = []
+    const tags: SNTag[] = []
-    const tags: DecryptedTransferPayload<TagContent>[] = []
 
     function findTag(title: string | null) {
       return tags.filter(function (tag) {
@@ -50,94 +60,119 @@ export class EvernoteConverter implements Converter {
       })[0]
     }
 
+    const successful: ConversionResult['successful'] = []
+    const errored: ConversionResult['errored'] = []
+
     for (const [index, xmlNote] of Array.from(xmlNotes).entries()) {
-      const title = xmlNote.getElementsByTagName('title')[0].textContent
+      const filesToPotentiallyCleanup: FileItem[] = []
-      const created = xmlNote.getElementsByTagName('created')[0]?.textContent
+      try {
-      const updatedNodes = xmlNote.getElementsByTagName('updated')
+        const title = xmlNote.getElementsByTagName('title')[0].textContent
-      const updated = updatedNodes.length ? updatedNodes[0].textContent : null
+        const created = xmlNote.getElementsByTagName('created')[0]?.textContent
-      const resources = Array.from(xmlNote.getElementsByTagName('resource'))
+        const updatedNodes = xmlNote.getElementsByTagName('updated')
-        .map(this.getResourceFromElement)
+        const updated = updatedNodes.length ? updatedNodes[0].textContent : null
-        .filter(Boolean) as EvernoteResource[]
+        const resources = Array.from(xmlNote.getElementsByTagName('resource'))
+          .map(this.getResourceFromElement)
+          .filter(Boolean) as EvernoteResource[]
 
-      const contentNode = xmlNote.getElementsByTagName('content')[0]
+        const contentNode = xmlNote.getElementsByTagName('content')[0]
-      const contentXmlString = this.getXmlStringFromContentElement(contentNode)
+        const contentXmlString = this.getXmlStringFromContentElement(contentNode)
-      if (!contentXmlString) {
+        if (!contentXmlString) {
-        continue
+          continue
-      }
+        }
-      const contentXml = this.loadXMLString(contentXmlString, 'html')
+        const contentXml = this.loadXMLString(contentXmlString, 'html')
 
-      const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
+        const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
 
-      const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
+        const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
 
-      if (canUseSuper) {
+        if (canUseSuper) {
-        this.convertTopLevelDivsToParagraphs(noteElement)
+          this.convertTopLevelDivsToParagraphs(noteElement)
-        this.convertListsToSuperFormatIfApplicable(unorderedLists)
+          this.convertListsToSuperFormatIfApplicable(unorderedLists)
-        this.convertLeftPaddingToSuperIndent(noteElement)
+          this.convertLeftPaddingToSuperIndent(noteElement)
-      }
-
-      this.removeEmptyAndOrphanListElements(noteElement)
-      this.removeUnnecessaryTopLevelBreaks(noteElement)
-
-      const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
-      this.replaceMediaElementsWithResources(mediaElements, resources)
-
-      // Some notes have <font> tags that contain separate <span> tags with text
-      // which causes broken paragraphs in the note.
-      const fontElements = Array.from(noteElement.getElementsByTagName('font'))
-      for (const fontElement of fontElements) {
-        fontElement.childNodes.forEach((childNode) => {
-          childNode.textContent += ' '
-        })
-        fontElement.innerText = fontElement.textContent || ''
-      }
-
-      let contentHTML = noteElement.innerHTML
-      if (!canUseSuper) {
-        contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
-        contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
-        contentHTML = contentHTML.trim()
-      }
-      const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)
-
-      const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
-      const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate
-
-      const note = createNote({
-        createdAt: createdAtDate,
-        updatedAt: updatedAtDate,
-        title: !title ? `Imported note ${index + 1} from Evernote` : title,
-        text,
-        useSuperIfPossible: canUseSuper,
-      })
-
-      const xmlTags = xmlNote.getElementsByTagName('tag')
-      for (const tagXml of Array.from(xmlTags)) {
-        const tagName = tagXml.childNodes[0].nodeValue
-        let tag = findTag(tagName)
-        if (!tag) {
-          const now = new Date()
-          tag = createTag({
-            createdAt: now,
-            updatedAt: now,
-            title: tagName || `Imported tag ${index + 1} from Evernote`,
-          })
-          tags.push(tag)
         }
 
-        note.content.references.push({ content_type: tag.content_type, uuid: tag.uuid })
+        this.removeEmptyAndOrphanListElements(noteElement)
-        tag.content.references.push({ content_type: note.content_type, uuid: note.uuid })
+        this.removeUnnecessaryTopLevelBreaks(noteElement)
+
+        const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
+        const { uploadedFiles } = await this.replaceMediaElementsWithResources(
+          mediaElements,
+          resources,
+          canUploadFiles,
+          uploadFile,
+        )
+        filesToPotentiallyCleanup.push(...uploadedFiles)
+
+        // Some notes have <font> tags that contain separate <span> tags with text
+        // which causes broken paragraphs in the note.
+        const fontElements = Array.from(noteElement.getElementsByTagName('font'))
+        for (const fontElement of fontElements) {
+          fontElement.childNodes.forEach((childNode) => {
+            childNode.textContent += ' '
+          })
+          fontElement.innerText = fontElement.textContent || ''
+        }
+
+        let contentHTML = noteElement.innerHTML
+        if (!canUseSuper) {
+          contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
+          contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
+          contentHTML = contentHTML.trim()
+        }
+        const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)
+
+        const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
+        const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate
+
+        const note = await insertNote({
+          createdAt: createdAtDate,
+          updatedAt: updatedAtDate,
+          title: !title ? `Imported note ${index + 1} from Evernote` : title,
+          text,
+          useSuperIfPossible: canUseSuper,
+        })
+
+        successful.push(note)
+
+        for (const uploadedFile of uploadedFiles) {
+          await linkItems(note, uploadedFile)
+          successful.push(uploadedFile)
+        }
+
+        const xmlTags = xmlNote.getElementsByTagName('tag')
+        for (const tagXml of Array.from(xmlTags)) {
+          const tagName = tagXml.childNodes[0].nodeValue
+          let tag = findTag(tagName)
+
+          if (!tag) {
+            const now = new Date()
+            tag = await insertTag({
+              createdAt: now,
+              updatedAt: now,
+              title: tagName || `Imported tag ${index + 1} from Evernote`,
+              references: [],
+            })
+            tags.push(tag)
+            successful.push(tag)
+          }
+
+          await linkItems(note, tag)
+        }
+      } catch (error) {
+        console.error(error)
+        errored.push({
+          name: xmlNote.getElementsByTagName('title')?.[0]?.textContent || `${file.name} - Note #${index}`,
+          error: error as Error,
+        })
+        cleanupItems(filesToPotentiallyCleanup).catch(console.error)
+        continue
       }
-
-      notes.push(note)
     }
 
-    const allItems: DecryptedTransferPayload[] = [...notes, ...tags]
+    return {
-    if (allItems.length === 0) {
+      successful,
-      throw new Error('Could not parse any notes or tags from Evernote file.')
+      errored,
     }
-
-    return allItems
   }
 
   getXmlStringFromContentElement(contentElement: Element) {
@@ -259,44 +294,83 @@ export class EvernoteConverter implements Converter {
     })
   }
 
-  replaceMediaElementsWithResources(mediaElements: Element[], resources: EvernoteResource[]): number {
+  getHTMLElementFromResource(resource: EvernoteResource) {
-    let replacedElements = 0
+    let resourceElement: HTMLElement = document.createElement('object')
+    resourceElement.setAttribute('type', resource.mimeType)
+    resourceElement.setAttribute('data', resource.data)
+    if (resource.mimeType.startsWith('image/')) {
+      resourceElement = document.createElement('img')
+      resourceElement.setAttribute('src', resource.data)
+      resourceElement.setAttribute('data-mime-type', resource.mimeType)
+    } else if (resource.mimeType.startsWith('audio/')) {
+      resourceElement = document.createElement('audio')
+      resourceElement.setAttribute('controls', 'controls')
+      const sourceElement = document.createElement('source')
+      sourceElement.setAttribute('src', resource.data)
+      sourceElement.setAttribute('type', resource.mimeType)
+      resourceElement.appendChild(sourceElement)
+    } else if (resource.mimeType.startsWith('video/')) {
+      resourceElement = document.createElement('video')
+      resourceElement.setAttribute('controls', 'controls')
+      const sourceElement = document.createElement('source')
+      sourceElement.setAttribute('src', resource.data)
+      sourceElement.setAttribute('type', resource.mimeType)
+      resourceElement.appendChild(sourceElement)
+    }
+    resourceElement.setAttribute('data-filename', resource.fileName)
+    return resourceElement
+  }
+
+  async getFileFromResource(resource: EvernoteResource): Promise<File> {
+    const response = await fetch(resource.data)
+    const blob = await response.blob()
+    return new File([blob], resource.fileName, { type: resource.mimeType })
+  }
+
+  async replaceMediaElementsWithResources(
+    mediaElements: Element[],
+    resources: EvernoteResource[],
+    canUploadFiles: boolean,
+    uploadFile: UploadFileFn,
+  ): Promise<{
+    replacedElements: HTMLElement[]
+    uploadedFiles: FileItem[]
+  }> {
+    const replacedElements: HTMLElement[] = []
+    const uploadedFiles = new Map<EvernoteResource['hash'], FileItem>()
     for (const mediaElement of mediaElements) {
       const hash = mediaElement.getAttribute('hash')
       const resource = resources.find((resource) => resource && resource.hash === hash)
       if (!resource) {
         continue
       }
-      let resourceElement: HTMLElement = document.createElement('object')
-      resourceElement.setAttribute('type', resource.mimeType)
-      resourceElement.setAttribute('data', resource.data)
-      if (resource.mimeType.startsWith('image/')) {
-        resourceElement = document.createElement('img')
-        resourceElement.setAttribute('src', resource.data)
-        resourceElement.setAttribute('data-mime-type', resource.mimeType)
-      } else if (resource.mimeType.startsWith('audio/')) {
-        resourceElement = document.createElement('audio')
-        resourceElement.setAttribute('controls', 'controls')
-        const sourceElement = document.createElement('source')
-        sourceElement.setAttribute('src', resource.data)
-        sourceElement.setAttribute('type', resource.mimeType)
-        resourceElement.appendChild(sourceElement)
-      } else if (resource.mimeType.startsWith('video/')) {
-        resourceElement = document.createElement('video')
-        resourceElement.setAttribute('controls', 'controls')
-        const sourceElement = document.createElement('source')
-        sourceElement.setAttribute('src', resource.data)
-        sourceElement.setAttribute('type', resource.mimeType)
-        resourceElement.appendChild(sourceElement)
-      }
-      resourceElement.setAttribute('data-filename', resource.fileName)
       if (!mediaElement.parentNode) {
         continue
       }
+      const existingFile = uploadedFiles.get(resource.hash)
+      const fileItem = canUploadFiles
+        ? existingFile
+          ? existingFile
+          : await uploadFile(await this.getFileFromResource(resource))
+        : undefined
+      if (fileItem) {
+        const fileElement = document.createElement('div')
+        fileElement.setAttribute('data-lexical-file-uuid', fileItem.uuid)
+        mediaElement.parentNode.replaceChild(fileElement, mediaElement)
+        replacedElements.push(fileElement)
+        if (!existingFile) {
+          uploadedFiles.set(resource.hash, fileItem)
+        }
+        continue
+      }
+      const resourceElement = this.getHTMLElementFromResource(resource)
       mediaElement.parentNode.replaceChild(resourceElement, mediaElement)
-      replacedElements++
+      replacedElements.push(resourceElement)
+    }
+    return {
+      replacedElements,
+      uploadedFiles: Array.from(uploadedFiles.values()),
     }
-    return replacedElements
   }
 
   loadXMLString(string: string, type: 'html' | 'xml') {

packages/ui-services/src/Import/EvernoteConverter/testData.ts

@@ -36,11 +36,6 @@ export const enex = `<?xml version="1.0" encoding="UTF-8"?>
   </note>
 </en-export>`
 
-export const enexWithNoNoteOrTag = `<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export3.dtd">
-<en-export export-date="20210408T052957Z" application="Evernote" version="10.8.5">
-</en-export>`
-
 export function createTestResourceElement(
   shouldHaveMimeType = true,
   shouldHaveSourceUrl = false,

packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts

@@ -4,11 +4,11 @@
 
 import { jsonTextContentData, htmlTestData, jsonListContentData } from './testData'
 import { GoogleKeepConverter } from './GoogleKeepConverter'
-import { ContentType, DecryptedTransferPayload, NoteContent } from '@standardnotes/snjs'
+import { ContentType, SNNote } from '@standardnotes/snjs'
-import { CreateNoteFn } from '../Converter'
+import { InsertNoteFn } from '../Converter'
 
 describe('GoogleKeepConverter', () => {
-  const createNote: CreateNoteFn = ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) =>
+  const insertNote: InsertNoteFn = async ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) =>
     ({
       uuid: Math.random().toString(),
       created_at: createdAt,
@@ -22,12 +22,12 @@ describe('GoogleKeepConverter', () => {
         pinned,
         references: [],
       },
-    }) as unknown as DecryptedTransferPayload<NoteContent>
+    }) as unknown as SNNote
 
-  it('should parse json data', () => {
+  it('should parse json data', async () => {
     const converter = new GoogleKeepConverter()
 
-    const textContent = converter.tryParseAsJson(jsonTextContentData, createNote, (md) => md)
+    const textContent = await converter.tryParseAsJson(jsonTextContentData, insertNote, (md) => md)
 
     expect(textContent).not.toBeNull()
     expect(textContent?.created_at).toBeInstanceOf(Date)
@@ -40,7 +40,7 @@ describe('GoogleKeepConverter', () => {
     expect(textContent?.content.archived).toBe(false)
     expect(textContent?.content.pinned).toBe(false)
 
-    const listContent = converter.tryParseAsJson(jsonListContentData, createNote, (md) => md)
+    const listContent = await converter.tryParseAsJson(jsonListContentData, insertNote, (md) => md)
 
     expect(listContent).not.toBeNull()
     expect(listContent?.created_at).toBeInstanceOf(Date)
@@ -54,15 +54,15 @@ describe('GoogleKeepConverter', () => {
     expect(textContent?.content.pinned).toBe(false)
   })
 
-  it('should parse html data', () => {
+  it('should parse html data', async () => {
     const converter = new GoogleKeepConverter()
 
-    const result = converter.tryParseAsHtml(
+    const result = await converter.tryParseAsHtml(
       htmlTestData,
       {
         name: 'note-2.html',
       },
-      createNote,
+      insertNote,
       (html) => html,
       false,
     )

packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts

@@ -1,5 +1,5 @@
-import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
+import { SNNote } from '@standardnotes/models'
-import { Converter, CreateNoteFn } from '../Converter'
+import { Converter, InsertNoteFn } from '../Converter'
 
 type Content =
   | {
@@ -45,32 +45,31 @@ export class GoogleKeepConverter implements Converter {
 
   convert: Converter['convert'] = async (
     file,
-    { createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText },
+    { insertNote: createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText },
   ) => {
     const content = await readFileAsText(file)
 
-    const possiblePayloadFromJson = this.tryParseAsJson(content, createNote, convertMarkdownToSuper)
+    const note =
+      (await this.tryParseAsJson(content, createNote, convertMarkdownToSuper)) ||
+      (await this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper))
 
-    if (possiblePayloadFromJson) {
+    if (note) {
-      return [possiblePayloadFromJson]
+      return {
-    }
+        successful: [note],
-
+        errored: [],
-    const possiblePayloadFromHtml = this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper)
+      }
-
-    if (possiblePayloadFromHtml) {
-      return [possiblePayloadFromHtml]
     }
 
     throw new Error('Could not parse Google Keep backup file')
   }
 
-  tryParseAsHtml(
+  async tryParseAsHtml(
     data: string,
     file: { name: string },
-    createNote: CreateNoteFn,
+    insertNote: InsertNoteFn,
     convertHTMLToSuper: (html: string) => string,
     canUseSuper: boolean,
-  ): DecryptedTransferPayload<NoteContent> {
+  ): Promise<SNNote> {
     const rootElement = document.createElement('html')
     rootElement.innerHTML = data
 
@@ -119,7 +118,7 @@ export class GoogleKeepConverter implements Converter {
 
     const title = rootElement.getElementsByClassName('title')[0]?.textContent || file.name
 
-    return createNote({
+    return await insertNote({
       createdAt: date,
       updatedAt: date,
       title: title,
@@ -150,11 +149,11 @@ export class GoogleKeepConverter implements Converter {
     )
   }
 
-  tryParseAsJson(
+  async tryParseAsJson(
     data: string,
-    createNote: CreateNoteFn,
+    createNote: InsertNoteFn,
     convertMarkdownToSuper: (md: string) => string,
-  ): DecryptedTransferPayload<NoteContent> | null {
+  ): Promise<SNNote | null> {
     try {
       const parsed = JSON.parse(data) as GoogleKeepJsonNote
       if (!GoogleKeepConverter.isValidGoogleKeepJson(parsed)) {
@@ -172,7 +171,7 @@ export class GoogleKeepConverter implements Converter {
           .join('\n')
       }
       text = convertMarkdownToSuper(text)
-      return createNote({
+      return await createNote({
         createdAt: date,
         updatedAt: date,
         title: parsed.title,

packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts

@@ -16,7 +16,7 @@ export class HTMLConverter implements Converter {
     return true
   }
 
-  convert: Converter['convert'] = async (file, { createNote, convertHTMLToSuper, readFileAsText }) => {
+  convert: Converter['convert'] = async (file, { insertNote: createNote, convertHTMLToSuper, readFileAsText }) => {
     const content = await readFileAsText(file)
 
     const { name } = parseFileName(file.name)
@@ -26,14 +26,17 @@ export class HTMLConverter implements Converter {
 
     const text = convertHTMLToSuper(content)
 
-    return [
+    const note = await createNote({
-      createNote({
+      createdAt: createdAtDate,
-        createdAt: createdAtDate,
+      updatedAt: updatedAtDate,
-        updatedAt: updatedAtDate,
+      title: name,
-        title: name,
+      text,
-        text,
+      useSuperIfPossible: true,
-        useSuperIfPossible: true,
+    })
-      }),
+
-    ]
+    return {
+      successful: [note],
+      errored: [],
+    }
   }
 }

packages/ui-services/src/Import/Importer.ts

@@ -15,20 +15,24 @@ import { SimplenoteConverter } from './SimplenoteConverter/SimplenoteConverter'
 import { readFileAsText } from './Utils'
 import {
   DecryptedItemInterface,
-  DecryptedTransferPayload,
   FileItem,
   ItemContent,
   NoteContent,
-  NoteMutator,
   SNNote,
-  isNote,
+  SNTag,
+  TagContent,
+  isFile,
 } from '@standardnotes/models'
 import { HTMLConverter } from './HTMLConverter/HTMLConverter'
 import { SuperConverter } from './SuperConverter/SuperConverter'
-import { Converter, CreateNoteFn, CreateTagFn } from './Converter'
+import { CleanupItemsFn, Converter, InsertNoteFn, InsertTagFn, LinkItemsFn, UploadFileFn } from './Converter'
-import { SuperConverterServiceInterface } from '@standardnotes/files'
+import { ConversionResult } from './ConversionResult'
+import { FilesClientInterface, SuperConverterServiceInterface } from '@standardnotes/files'
 import { ContentType } from '@standardnotes/domain-core'
 
+const BytesInOneMegabyte = 1_000_000
+const NoteSizeThreshold = 3 * BytesInOneMegabyte
+
 export class Importer {
   converters: Set<Converter> = new Set()
 
@@ -50,9 +54,11 @@ export class Importer {
       linkItems(
         item: DecryptedItemInterface<ItemContent>,
         itemToLink: DecryptedItemInterface<ItemContent>,
+        sync: boolean,
       ): Promise<void>
     },
     private _generateUuid: GenerateUuid,
+    private files: FilesClientInterface,
   ) {
     this.registerNativeConverters()
   }
@@ -88,19 +94,19 @@ export class Importer {
     return null
   }
 
-  createNote: CreateNoteFn = ({
+  insertNote: InsertNoteFn = async ({
     createdAt,
     updatedAt,
     title,
     text,
     noteType,
     editorIdentifier,
-    trashed,
+    trashed = false,
-    archived,
+    archived = false,
-    pinned,
+    pinned = false,
     useSuperIfPossible,
   }) => {
-    if (noteType === NoteType.Super && !this.isEntitledToSuper()) {
+    if (noteType === NoteType.Super && !this.canUseSuper()) {
       noteType = undefined
     }
 
@@ -112,16 +118,17 @@ export class Importer {
       editorIdentifier = undefined
     }
 
-    const shouldUseSuper = useSuperIfPossible && this.isEntitledToSuper()
+    const shouldUseSuper = useSuperIfPossible && this.canUseSuper()
 
-    return {
+    const noteSize = new Blob([text]).size
-      created_at: createdAt,
+
-      created_at_timestamp: createdAt.getTime(),
+    if (noteSize > NoteSizeThreshold) {
-      updated_at: updatedAt,
+      throw new Error('Note is too large to import')
-      updated_at_timestamp: updatedAt.getTime(),
+    }
-      uuid: this._generateUuid.execute().getValue(),
+
-      content_type: ContentType.TYPES.Note,
+    const note = this.items.createTemplateItem<NoteContent, SNNote>(
-      content: {
+      ContentType.TYPES.Note,
+      {
         title,
         text,
         references: [],
@@ -131,27 +138,68 @@ export class Importer {
         pinned,
         editorIdentifier: shouldUseSuper ? NativeFeatureIdentifier.TYPES.SuperEditor : editorIdentifier,
       },
-    }
+      {
+        created_at: createdAt,
+        updated_at: updatedAt,
+      },
+    )
+
+    return await this.mutator.insertItem(note)
   }
 
-  createTag: CreateTagFn = ({ createdAt, updatedAt, title }) => {
+  insertTag: InsertTagFn = async ({ createdAt, updatedAt, title, references }) => {
-    return {
+    const tag = this.items.createTemplateItem<TagContent, SNTag>(
-      uuid: this._generateUuid.execute().getValue(),
+      ContentType.TYPES.Tag,
-      content_type: ContentType.TYPES.Tag,
+      {
-      created_at: createdAt,
+        title,
-      created_at_timestamp: createdAt.getTime(),
-      updated_at: updatedAt,
-      updated_at_timestamp: updatedAt.getTime(),
-      content: {
-        title: title,
         expanded: false,
         iconString: '',
-        references: [],
+        references,
       },
+      {
+        created_at: createdAt,
+        updated_at: updatedAt,
+      },
+    )
+
+    return await this.mutator.insertItem(tag)
+  }
+
+  canUploadFiles = (): boolean => {
+    const status = this.features.getFeatureStatus(
+      NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.Files).getValue(),
+    )
+
+    return status === FeatureStatus.Entitled
+  }
+
+  uploadFile: UploadFileFn = async (file) => {
+    if (!this.canUploadFiles()) {
+      return undefined
+    }
+
+    try {
+      return await this.filesController.uploadNewFile(file, { showToast: true })
+    } catch (error) {
+      console.error(error)
+      return undefined
     }
   }
 
-  isEntitledToSuper = (): boolean => {
+  linkItems: LinkItemsFn = async (item, itemToLink) => {
+    await this.linkingController.linkItems(item, itemToLink, false)
+  }
+
+  cleanupItems: CleanupItemsFn = async (items) => {
+    for (const item of items) {
+      if (isFile(item)) {
+        await this.files.deleteFile(item)
+      }
+      await this.mutator.deleteItems([item])
+    }
+  }
+
+  canUseSuper = (): boolean => {
     return (
       this.features.getFeatureStatus(
         NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.SuperEditor).getValue(),
@@ -160,7 +208,7 @@ export class Importer {
   }
 
   convertHTMLToSuper = (html: string): string => {
-    if (!this.isEntitledToSuper()) {
+    if (!this.canUseSuper()) {
       return html
     }
 
@@ -168,20 +216,23 @@ export class Importer {
   }
 
   convertMarkdownToSuper = (markdown: string): string => {
-    if (!this.isEntitledToSuper()) {
+    if (!this.canUseSuper()) {
       return markdown
     }
 
     return this.superConverterService.convertOtherFormatToSuperString(markdown, 'md')
   }
 
-  async getPayloadsFromFile(file: File, type: string): Promise<DecryptedTransferPayload[]> {
+  async importFromFile(file: File, type: string): Promise<ConversionResult> {
-    const isEntitledToSuper = this.isEntitledToSuper()
+    const canUseSuper = this.canUseSuper()
 
-    if (type === 'super' && !isEntitledToSuper) {
+    if (type === 'super' && !canUseSuper) {
       throw new Error('Importing Super notes requires a subscription')
     }
 
+    const successful: ConversionResult['successful'] = []
+    const errored: ConversionResult['errored'] = []
+
     for (const converter of this.converters) {
       const isCorrectType = converter.getImportType() === type
 
@@ -195,65 +246,28 @@ export class Importer {
         throw new Error('Content is not valid')
       }
 
-      return await converter.convert(file, {
+      const result = await converter.convert(file, {
-        createNote: this.createNote,
+        insertNote: this.insertNote,
-        createTag: this.createTag,
+        insertTag: this.insertTag,
-        canUseSuper: isEntitledToSuper,
+        canUploadFiles: this.canUploadFiles(),
+        uploadFile: this.uploadFile,
+        canUseSuper,
         convertHTMLToSuper: this.convertHTMLToSuper,
         convertMarkdownToSuper: this.convertMarkdownToSuper,
         readFileAsText,
+        linkItems: this.linkItems,
+        cleanupItems: this.cleanupItems,
       })
+
+      successful.push(...result.successful)
+      errored.push(...result.errored)
+
+      break
     }
 
-    return []
+    return {
-  }
+      successful,
-
+      errored,
-  async importFromTransferPayloads(payloads: DecryptedTransferPayload[]) {
-    const insertedItems = await Promise.all(
-      payloads.map(async (payload) => {
-        const content = payload.content as NoteContent
-        const note = this.items.createTemplateItem(
-          payload.content_type,
-          {
-            text: content.text,
-            title: content.title,
-            noteType: content.noteType,
-            editorIdentifier: content.editorIdentifier,
-            references: content.references,
-          },
-          {
-            created_at: payload.created_at,
-            updated_at: payload.updated_at,
-            uuid: payload.uuid,
-          },
-        )
-        return this.mutator.insertItem(note)
-      }),
-    )
-    return insertedItems
-  }
-
-  async uploadAndReplaceInlineFilesInInsertedItems(insertedItems: DecryptedItemInterface<ItemContent>[]) {
-    for (const item of insertedItems) {
-      if (!isNote(item)) {
-        continue
-      }
-      if (item.noteType !== NoteType.Super) {
-        continue
-      }
-      try {
-        const text = await this.superConverterService.uploadAndReplaceInlineFilesInSuperString(
-          item.text,
-          async (file) => await this.filesController.uploadNewFile(file, { showToast: true, note: item }),
-          async (file) => await this.linkingController.linkItems(item, file),
-          this._generateUuid,
-        )
-        await this.mutator.changeItem<NoteMutator>(item, (mutator) => {
-          mutator.text = text
-        })
-      } catch (error) {
-        console.error(error)
-      }
     }
   }
 }

packages/ui-services/src/Import/PlaintextConverter/PlaintextConverter.ts

@@ -20,7 +20,7 @@ export class PlaintextConverter implements Converter {
     return file.type === 'text/plain' || file.type === 'text/markdown'
   }
 
-  convert: Converter['convert'] = async (file, { createNote, convertMarkdownToSuper, readFileAsText }) => {
+  convert: Converter['convert'] = async (file, { insertNote, convertMarkdownToSuper, readFileAsText }) => {
     const content = await readFileAsText(file)
 
     const { name } = parseFileName(file.name)
@@ -28,14 +28,17 @@ export class PlaintextConverter implements Converter {
     const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
     const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
 
-    return [
+    const note = await insertNote({
-      createNote({
+      createdAt: createdAtDate,
-        createdAt: createdAtDate,
+      updatedAt: updatedAtDate,
-        updatedAt: updatedAtDate,
+      title: name,
-        title: name,
+      text: convertMarkdownToSuper(content),
-        text: convertMarkdownToSuper(content),
+      useSuperIfPossible: true,
-        useSuperIfPossible: true,
+    })
-      }),
+
-    ]
+    return {
+      successful: [note],
+      errored: [],
+    }
   }
 }

packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.spec.ts

@@ -1,11 +1,11 @@
-import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
+import { SNNote } from '@standardnotes/models'
 import { SimplenoteConverter } from './SimplenoteConverter'
 import data from './testData'
 import { ContentType } from '@standardnotes/domain-core'
-import { CreateNoteFn } from '../Converter'
+import { InsertNoteFn } from '../Converter'
 
 describe('SimplenoteConverter', () => {
-  const createNote: CreateNoteFn = ({ title, text, trashed, createdAt, updatedAt }) =>
+  const createNote: InsertNoteFn = async ({ title, text, trashed, createdAt, updatedAt }) =>
     ({
       uuid: Math.random().toString(),
       created_at: createdAt,
@@ -17,12 +17,12 @@ describe('SimplenoteConverter', () => {
         trashed,
         references: [],
       },
-    }) as unknown as DecryptedTransferPayload<NoteContent>
+    }) as unknown as SNNote
 
-  it('should parse', () => {
+  it('should parse', async () => {
     const converter = new SimplenoteConverter()
 
-    const result = converter.parse(data, createNote)
+    const result = await converter.parse(data, createNote)
 
     expect(result).not.toBeNull()
     expect(result?.length).toBe(3)

packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.ts

@@ -1,4 +1,4 @@
-import { Converter, CreateNoteFn } from '../Converter'
+import { Converter, InsertNoteFn } from '../Converter'
 
 type SimplenoteItem = {
   creationDate: string
@@ -38,19 +38,22 @@ export class SimplenoteConverter implements Converter {
     return false
   }
 
-  convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
+  convert: Converter['convert'] = async (file, { insertNote: createNote, readFileAsText }) => {
     const content = await readFileAsText(file)
 
-    const notes = this.parse(content, createNote)
+    const notes = await this.parse(content, createNote)
 
     if (!notes) {
       throw new Error('Could not parse notes')
     }
 
-    return notes
+    return {
+      successful: notes,
+      errored: [],
+    }
   }
 
-  createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: CreateNoteFn): ReturnType<CreateNoteFn> {
+  createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: InsertNoteFn): ReturnType<InsertNoteFn> {
     const createdAtDate = new Date(item.creationDate)
     const updatedAtDate = new Date(item.lastModified)
 
@@ -70,11 +73,15 @@ export class SimplenoteConverter implements Converter {
     })
   }
 
-  parse(data: string, createNote: CreateNoteFn) {
+  async parse(data: string, createNote: InsertNoteFn) {
     try {
       const parsed = JSON.parse(data) as SimplenoteData
-      const activeNotes = parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote))
+      const activeNotes = await Promise.all(
-      const trashedNotes = parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote))
+        parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote)),
+      )
+      const trashedNotes = await Promise.all(
+        parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote)),
+      )
 
       return [...activeNotes, ...trashedNotes]
     } catch (error) {

packages/ui-services/src/Import/SuperConverter/SuperConverter.ts

@@ -1,6 +1,7 @@
 import { SuperConverterServiceInterface } from '@standardnotes/files'
 import { parseFileName } from '@standardnotes/filepicker'
 import { Converter } from '../Converter'
+import { ConversionResult } from '../ConversionResult'
 
 export class SuperConverter implements Converter {
   constructor(private converterService: SuperConverterServiceInterface) {}
@@ -17,9 +18,14 @@ export class SuperConverter implements Converter {
     return this.converterService.isValidSuperString(content)
   }
 
-  convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
+  convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
     const content = await readFileAsText(file)
 
+    const result: ConversionResult = {
+      successful: [],
+      errored: [],
+    }
+
     if (!this.converterService.isValidSuperString(content)) {
       throw new Error('Content is not valid Super JSON')
     }
@@ -29,14 +35,16 @@ export class SuperConverter implements Converter {
     const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
     const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
 
-    return [
+    const note = await insertNote({
-      createNote({
+      createdAt: createdAtDate,
-        createdAt: createdAtDate,
+      updatedAt: updatedAtDate,
-        updatedAt: updatedAtDate,
+      title: name,
-        title: name,
+      text: content,
-        text: content,
+      useSuperIfPossible: true,
-        useSuperIfPossible: true,
+    })
-      }),
+
-    ]
+    result.successful.push(note)
+
+    return result
   }
 }

packages/ui-services/src/Import/index.ts

@@ -5,3 +5,4 @@ export * from './EvernoteConverter/EvernoteConverter'
 export * from './PlaintextConverter/PlaintextConverter'
 export * from './Utils'
 export * from './Importer'
+export * from './ConversionResult'

packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts

@@ -68,6 +68,7 @@ export class WebDependencies extends DependencyContainer {
         this.get<FilesController>(Web_TYPES.FilesController),
         this.get<LinkingController>(Web_TYPES.LinkingController),
         application.generateUuid,
+        application.files,
       )
     })
 

packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx

@@ -34,9 +34,9 @@ const ImportModal = ({ importModalController }: { importModalController: ImportM
     close,
   } = importModalController
 
-  const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'ready')
+  const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'pending' && file.service)
   const importSuccessOrError =
-    files.length > 0 && files.every((file) => file.status === 'success' || file.status === 'error')
+    files.length > 0 && files.every((file) => file.status === 'finished' || file.status === 'error')
 
   const modalActions: ModalAction[] = useMemo(
     () => [

packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts

@@ -1,15 +1,14 @@
-import { DecryptedTransferPayload, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
+import { DecryptedItemInterface, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
 import {
   ContentType,
   InternalEventBusInterface,
   ItemManagerInterface,
   MutatorClientInterface,
-  pluralize,
   PreferenceServiceInterface,
   PreferencesServiceEvent,
   UuidGenerator,
 } from '@standardnotes/snjs'
-import { Importer } from '@standardnotes/ui-services'
+import { ConversionResult, Importer } from '@standardnotes/ui-services'
 import { action, makeObservable, observable, runInAction } from 'mobx'
 import { NavigationController } from '../../Controllers/Navigation/NavigationController'
 import { LinkingController } from '@/Controllers/LinkingController'
@@ -23,11 +22,10 @@ type ImportModalFileCommon = {
 
 export type ImportModalFile = (
   | { status: 'pending' }
-  | { status: 'ready'; payloads?: DecryptedTransferPayload[] }
   | { status: 'parsing' }
   | { status: 'importing' }
   | { status: 'uploading-files' }
-  | { status: 'success'; successMessage: string }
+  | ({ status: 'finished' } & ConversionResult)
   | { status: 'error'; error: Error }
 ) &
   ImportModalFileCommon
@@ -112,7 +110,7 @@ export class ImportModalController extends AbstractViewController {
       id: UuidGenerator.GenerateUuid(),
       file,
       service,
-      status: service ? 'ready' : 'pending',
+      status: 'pending',
     } as ImportModalFile
   }
 
@@ -149,70 +147,30 @@ export class ImportModalController extends AbstractViewController {
     this.setImportTag(undefined)
   }
 
-  importFromPayloads = async (file: ImportModalFile, payloads: DecryptedTransferPayload[]) => {
-    this.updateFile({
-      ...file,
-      status: 'importing',
-    })
-
-    try {
-      const insertedItems = await this.importer.importFromTransferPayloads(payloads)
-
-      this.updateFile({
-        ...file,
-        status: 'uploading-files',
-      })
-
-      await this.importer.uploadAndReplaceInlineFilesInInsertedItems(insertedItems)
-
-      const notesImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
-      const tagsImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
-
-      const successMessage =
-        `Successfully imported ${notesImported.length} ` +
-        pluralize(notesImported.length, 'note', 'notes') +
-        (tagsImported.length > 0 ? ` and ${tagsImported.length} ${pluralize(tagsImported.length, 'tag', 'tags')}` : '')
-
-      this.updateFile({
-        ...file,
-        status: 'success',
-        successMessage,
-      })
-    } catch (error) {
-      this.updateFile({
-        ...file,
-        status: 'error',
-        error: error instanceof Error ? error : new Error('Could not import file'),
-      })
-      console.error(error)
-    }
-  }
-
   parseAndImport = async () => {
     if (this.files.length === 0) {
       return
     }
-    const importedPayloads: DecryptedTransferPayload[] = []
+    const importedItems: DecryptedItemInterface[] = []
     for (const file of this.files) {
       if (!file.service) {
         return
       }
 
-      if (file.status === 'ready' && file.payloads) {
-        await this.importFromPayloads(file, file.payloads)
-        importedPayloads.push(...file.payloads)
-        continue
-      }
-
       this.updateFile({
         ...file,
         status: 'parsing',
       })
 
       try {
-        const payloads = await this.importer.getPayloadsFromFile(file.file, file.service)
+        const { successful, errored } = await this.importer.importFromFile(file.file, file.service)
-        await this.importFromPayloads(file, payloads)
+        importedItems.push(...successful)
-        importedPayloads.push(...payloads)
+        this.updateFile({
+          ...file,
+          status: 'finished',
+          successful,
+          errored,
+        })
       } catch (error) {
         this.updateFile({
           ...file,
@@ -222,7 +180,7 @@ export class ImportModalController extends AbstractViewController {
         console.error(error)
       }
     }
-    if (!importedPayloads.length) {
+    if (!importedItems.length) {
       return
     }
     if (this.addImportsToTag) {
@@ -233,7 +191,7 @@ export class ImportModalController extends AbstractViewController {
           title: `Imported on ${currentDate.toLocaleString()}`,
           expanded: false,
           iconString: '',
-          references: importedPayloads
+          references: importedItems
             .filter((payload) => payload.content_type === ContentType.TYPES.Note)
             .map((payload) => ({
               content_type: ContentType.TYPES.Note,
@@ -245,11 +203,11 @@ export class ImportModalController extends AbstractViewController {
         try {
           const latestExistingTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)
           await Promise.all(
-            importedPayloads
+            importedItems
               .filter((payload) => payload.content_type === ContentType.TYPES.Note)
               .map(async (payload) => {
                 const note = this.items.findSureItem<SNNote>(payload.uuid)
-                await this.linkingController.addTagToItem(latestExistingTag, note)
+                await this.linkingController.addTagToItem(latestExistingTag, note, false)
               }),
           )
           importTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)

packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx

@@ -1,9 +1,10 @@
 import { ImportModalController, ImportModalFile } from '@/Components/ImportModal/ImportModalController'
 import { classNames, ContentType, pluralize } from '@standardnotes/snjs'
-import { Importer } from '@standardnotes/ui-services'
+import { ConversionResult, Importer } from '@standardnotes/ui-services'
 import { observer } from 'mobx-react-lite'
 import { useCallback, useEffect, useState } from 'react'
 import Icon from '../Icon/Icon'
+import { Disclosure, DisclosureContent, DisclosureProvider } from '@ariakit/react'
 
 const NoteImportTypeColors: Record<string, string> = {
   evernote: 'bg-[#14cc45] text-[#000]',
@@ -25,6 +26,75 @@ const NoteImportTypeIcons: Record<string, string> = {
   super: 'file-doc',
 }
 
+const countSuccessfulItemsByGroup = (successful: ConversionResult['successful']) => {
+  let notes = 0
+  let tags = 0
+  let files = 0
+
+  for (const item of successful) {
+    if (item.content_type === ContentType.TYPES.Note) {
+      notes++
+    } else if (item.content_type === ContentType.TYPES.Tag) {
+      tags++
+    } else if (item.content_type === ContentType.TYPES.File) {
+      files++
+    }
+  }
+
+  return {
+    notes,
+    tags,
+    files,
+  }
+}
+
+const ImportErroredAccordion = ({ errored }: { errored: ConversionResult['errored'] }) => {
+  return (
+    <DisclosureProvider>
+      <Disclosure>
+        <div className="flex items-center gap-1">
+          <Icon type="warning" className="flex-shrink-0 text-danger" size="small" />
+          Could not import {errored.length} {pluralize(errored.length, 'item', 'items')} (click for details)
+        </div>
+      </Disclosure>
+      <DisclosureContent className="w-full overflow-hidden pl-5">
+        {errored.map((item, index) => (
+          <div className="flex w-full items-center gap-1 overflow-hidden" key={index}>
+            <span>{index + 1}.</span>
+            <span className="overflow-hidden text-ellipsis whitespace-nowrap font-semibold">{item.name}:</span>
+            <span className="overflow-hidden text-ellipsis whitespace-nowrap">{item.error.message}</span>
+          </div>
+        ))}
+      </DisclosureContent>
+    </DisclosureProvider>
+  )
+}
+
+const ImportFinishedStatus = ({ file }: { file: ImportModalFile }) => {
+  if (file.status !== 'finished') {
+    return null
+  }
+
+  const { notes, tags, files } = countSuccessfulItemsByGroup(file.successful)
+
+  const notesStatus = notes > 0 ? `${notes} ${pluralize(notes, 'note', 'notes')}` : ''
+  const tagsStatus = tags > 0 ? `${tags} ${pluralize(tags, 'tag', 'tags')}` : ''
+  const filesStatus = files > 0 ? `${files} ${pluralize(files, 'file', 'files')}` : ''
+  const status = [notesStatus, tagsStatus, filesStatus].filter(Boolean).join(', ')
+
+  return (
+    <>
+      {file.successful.length > 0 && (
+        <div className="flex items-center gap-1">
+          <Icon type="check-circle-filled" className="flex-shrink-0 text-success" size="small" />
+          <span>{status} imported</span>
+        </div>
+      )}
+      {file.errored.length > 0 && <ImportErroredAccordion errored={file.errored} />}
+    </>
+  )
+}
+
 const ImportModalFileItem = ({
   file,
   updateFile,
@@ -36,6 +106,7 @@ const ImportModalFileItem = ({
   removeFile: ImportModalController['removeFile']
   importer: Importer
 }) => {
+  const [isDetectingService, setIsDetectingService] = useState(false)
   const [changingService, setChangingService] = useState(false)
 
   const setFileService = useCallback(
@@ -46,7 +117,7 @@ const ImportModalFileItem = ({
       updateFile({
         ...file,
         service,
-        status: service ? 'ready' : 'pending',
+        status: 'pending',
       })
     },
     [file, updateFile],
@@ -54,59 +125,47 @@ const ImportModalFileItem = ({
 
   useEffect(() => {
     const detect = async () => {
+      setIsDetectingService(true)
       const detectedService = await importer.detectService(file.file)
       void setFileService(detectedService)
+      setIsDetectingService(false)
     }
     if (file.service === undefined) {
       void detect()
     }
   }, [file, importer, setFileService])
 
-  const notePayloads =
-    file.status === 'ready' && file.payloads
-      ? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
-      : []
-  const tagPayloads =
-    file.status === 'ready' && file.payloads
-      ? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
-      : []
-
-  const payloadsImportMessage =
-    `Ready to import ${notePayloads.length} ` +
-    pluralize(notePayloads.length, 'note', 'notes') +
-    (tagPayloads.length > 0 ? ` and ${tagPayloads.length} ${pluralize(tagPayloads.length, 'tag', 'tags')}` : '')
-
   return (
     <div
       className={classNames(
-        'flex gap-2 px-2 py-2.5',
+        'flex gap-2 overflow-hidden px-2 py-2.5',
         file.service == null ? 'flex-col items-start md:flex-row md:items-center' : 'items-center',
       )}
     >
-      <div className="mr-auto flex items-center">
+      <div className="mr-auto flex w-full items-center">
         {file.service && (
           <div className={classNames('mr-4 rounded p-2', NoteImportTypeColors[file.service])}>
             <Icon type={NoteImportTypeIcons[file.service]} size="medium" />
           </div>
         )}
-        <div className="flex flex-col">
+        <div className="flex w-full flex-col overflow-hidden">
           <div>{file.file.name}</div>
-          <div className="line-clamp-3 text-xs opacity-75">
+          {isDetectingService ? (
-            {file.status === 'ready'
+            <div className="text-xs opacity-75">Detecting service...</div>
-              ? notePayloads.length > 1 || tagPayloads.length
+          ) : (
-                ? payloadsImportMessage
+            <div className={classNames(file.status !== 'finished' && 'line-clamp-3', 'w-full text-xs opacity-75')}>
-                : 'Ready to import'
+              {file.status === 'pending' && file.service && 'Ready to import'}
-              : null}
+              {file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
-            {file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
+              {file.status === 'parsing' && 'Parsing...'}
-            {file.status === 'parsing' && 'Parsing...'}
+              {file.status === 'importing' && 'Importing...'}
-            {file.status === 'importing' && 'Importing...'}
+              {file.status === 'uploading-files' && 'Uploading and embedding files...'}
-            {file.status === 'uploading-files' && 'Uploading and embedding files...'}
+              {file.status === 'error' && file.error.message}
-            {file.status === 'error' && file.error.message}
+              <ImportFinishedStatus file={file} />
-            {file.status === 'success' && file.successMessage}
+            </div>
-          </div>
+          )}
         </div>
       </div>
-      {(file.status === 'ready' || file.status === 'pending') && (
+      {file.status === 'pending' && (
         <div className="flex items-center">
           {changingService ? (
             <>
@@ -172,7 +231,9 @@ const ImportModalFileItem = ({
           </button>
         </div>
       )}
-      {file.status === 'success' && <Icon type="check-circle-filled" className="flex-shrink-0 text-success" />}
+      {file.status === 'finished' && file.successful.length > 0 && file.errored.length === 0 && (
+        <Icon type="check-circle-filled" className="flex-shrink-0 text-success" />
+      )}
       {file.status === 'error' && <Icon type="warning" className="flex-shrink-0 text-danger" />}
     </div>
   )

packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx

@@ -1,6 +1,6 @@
 import { createHeadlessEditor } from '@lexical/headless'
 import { $convertToMarkdownString } from '@lexical/markdown'
-import { FileItem, GenerateUuid, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
+import { FileItem, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
 import {
   $createParagraphNode,
   $getRoot,
@@ -16,9 +16,7 @@ import { MarkdownTransformers } from '../MarkdownTransformers'
 import { $generateHtmlFromNodes, $generateNodesFromDOM } from '@lexical/html'
 import { FileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileNode'
 import { $createFileExportNode } from '../Lexical/Nodes/FileExportNode'
-import { $createInlineFileNode, $isInlineFileNode, InlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
+import { $createInlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
-import { $createFileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileUtils'
-import { RemoteImageNode } from '../Plugins/RemoteImagePlugin/RemoteImageNode'
 import { $convertFromMarkdownString } from '../Lexical/Utils/MarkdownImport'
 export class HeadlessSuperConverter implements SuperConverterServiceInterface {
   private importEditor: LexicalEditor
@@ -194,7 +192,8 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
                 type === 'link' ||
                 type === 'linebreak' ||
                 type === 'unencrypted-image' ||
-                type === 'inline-file'
+                type === 'inline-file' ||
+                type === 'snfile'
               ) {
                 const paragraphNode = $createParagraphNode()
                 paragraphNode.append(node)
@@ -232,7 +231,7 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
     }
 
     if (didThrow) {
-      throw new Error('Could not import note')
+      throw new Error('Could not import note. Check error console for details.')
     }
 
     return JSON.stringify(this.importEditor.getEditorState())
@@ -256,62 +255,4 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
 
     return ids
   }
-
-  async uploadAndReplaceInlineFilesInSuperString(
-    superString: string,
-    uploadFile: (file: File) => Promise<FileItem | undefined>,
-    linkFile: (file: FileItem) => Promise<void>,
-    generateUuid: GenerateUuid,
-  ): Promise<string> {
-    if (superString.length === 0) {
-      return superString
-    }
-
-    this.importEditor.setEditorState(this.importEditor.parseEditorState(superString))
-
-    await new Promise<void>((resolve) => {
-      this.importEditor.update(
-        () => {
-          const inlineFileNodes = $nodesOfType(InlineFileNode)
-          const remoteImageNodes = $nodesOfType(RemoteImageNode).filter((node) => node.__src.startsWith('data:'))
-          const concatenatedNodes = [...inlineFileNodes, ...remoteImageNodes]
-          if (concatenatedNodes.length === 0) {
-            resolve()
-            return
-          }
-          ;(async () => {
-            for (const node of concatenatedNodes) {
-              const blob = await fetch(node.__src).then((response) => response.blob())
-              const name = $isInlineFileNode(node) ? node.__fileName : node.__alt
-              const mimeType = $isInlineFileNode(node) ? node.__mimeType : node.__src.split(';')[0].split(':')[1]
-              const file = new File([blob], name || generateUuid.execute().getValue(), {
-                type: mimeType,
-              })
-
-              const uploadedFile = await uploadFile(file)
-
-              if (!uploadedFile) {
-                return
-              }
-
-              this.importEditor.update(
-                () => {
-                  const fileNode = $createFileNode(uploadedFile.uuid)
-                  node.replace(fileNode)
-                },
-                { discrete: true },
-              )
-
-              await linkFile(uploadedFile)
-            }
-          })()
-            .then(() => resolve())
-            .catch(console.error)
-        },
-        { discrete: true },
-      )
-    })
-
-    return JSON.stringify(this.importEditor.getEditorState())
-  }
 }

packages/web/src/javascripts/Controllers/LinkingController.tsx

@@ -204,7 +204,7 @@ export class LinkingController extends AbstractViewController implements Interna
     }
   }
 
-  linkItems = async (item: LinkableItem, itemToLink: LinkableItem) => {
+  linkItems = async (item: LinkableItem, itemToLink: LinkableItem, sync = true) => {
     const linkNoteAndFile = async (note: SNNote, file: FileItem) => {
       const updatedFile = await this.mutator.associateFileWithNote(file, note)
 
@@ -231,11 +231,11 @@ export class LinkingController extends AbstractViewController implements Interna
     }
 
     const linkTagToNote = async (tag: SNTag, note: SNNote) => {
-      await this.addTagToItem(tag, note)
+      await this.addTagToItem(tag, note, sync)
     }
 
     const linkTagToFile = async (tag: SNTag, file: FileItem) => {
-      await this.addTagToItem(tag, file)
+      await this.addTagToItem(tag, file, sync)
     }
 
     if (isNote(item)) {
@@ -273,7 +273,9 @@ export class LinkingController extends AbstractViewController implements Interna
       throw new Error('First item must be a note or file')
     }
 
-    void this.sync.sync()
+    if (sync) {
+      void this.sync.sync()
+    }
   }
 
   linkItemToSelectedItem = async (itemToLink: LinkableItem): Promise<boolean> => {
@@ -323,13 +325,15 @@ export class LinkingController extends AbstractViewController implements Interna
     return newTag
   }
 
-  addTagToItem = async (tag: SNTag, item: FileItem | SNNote) => {
+  addTagToItem = async (tag: SNTag, item: FileItem | SNNote, sync = true) => {
     if (item instanceof SNNote) {
       await this.mutator.addTagToNote(item, tag, this.shouldLinkToParentFolders)
     } else if (item instanceof FileItem) {
       await this.mutator.addTagToFile(item, tag, this.shouldLinkToParentFolders)
     }
 
-    this.sync.sync().catch(console.error)
+    if (sync) {
+      this.sync.sync().catch(console.error)
+    }
   }
 }