feat: Added Super & HTML import options in Import modal. Google Keep notes will now also be imported as Super notes, with attachments if importing from HTML (#2433)

2023-08-18 16:41:25 +05:30
parent 18b7728145
commit ca9895cac1
22 changed files with 432 additions and 101 deletions
--- a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts
+++ b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts
@@ -2,37 +2,56 @@
 * @jest-environment jsdom
 */

-import { jsonTestData, htmlTestData } from './testData'
+import { jsonTextContentData, htmlTestData, jsonListContentData } from './testData'
 import { GoogleKeepConverter } from './GoogleKeepConverter'
 import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
 import { GenerateUuid } from '@standardnotes/services'
+import { SuperConverterServiceInterface } from '@standardnotes/snjs'

 describe('GoogleKeepConverter', () => {
  const crypto = {
    generateUUID: () => String(Math.random()),
  } as unknown as PureCryptoInterface

+  const superConverterService: SuperConverterServiceInterface = {
+    isValidSuperString: () => true,
+    convertOtherFormatToSuperString: (data: string) => data,
+    convertSuperStringToOtherFormat: (data: string) => data,
+  }
  const generateUuid = new GenerateUuid(crypto)

  it('should parse json data', () => {
-    const converter = new GoogleKeepConverter(generateUuid)
+    const converter = new GoogleKeepConverter(superConverterService, generateUuid)

-    const result = converter.tryParseAsJson(jsonTestData)
+    const textContent = converter.tryParseAsJson(jsonTextContentData, false)

-    expect(result).not.toBeNull()
-    expect(result?.created_at).toBeInstanceOf(Date)
-    expect(result?.updated_at).toBeInstanceOf(Date)
-    expect(result?.uuid).not.toBeNull()
-    expect(result?.content_type).toBe('Note')
-    expect(result?.content.title).toBe('Testing 1')
-    expect(result?.content.text).toBe('This is a test.')
-    expect(result?.content.trashed).toBe(false)
-    expect(result?.content.archived).toBe(false)
-    expect(result?.content.pinned).toBe(false)
+    expect(textContent).not.toBeNull()
+    expect(textContent?.created_at).toBeInstanceOf(Date)
+    expect(textContent?.updated_at).toBeInstanceOf(Date)
+    expect(textContent?.uuid).not.toBeNull()
+    expect(textContent?.content_type).toBe('Note')
+    expect(textContent?.content.title).toBe('Testing 1')
+    expect(textContent?.content.text).toBe('This is a test.')
+    expect(textContent?.content.trashed).toBe(false)
+    expect(textContent?.content.archived).toBe(false)
+    expect(textContent?.content.pinned).toBe(false)
+
+    const listContent = converter.tryParseAsJson(jsonListContentData, false)
+
+    expect(listContent).not.toBeNull()
+    expect(listContent?.created_at).toBeInstanceOf(Date)
+    expect(listContent?.updated_at).toBeInstanceOf(Date)
+    expect(listContent?.uuid).not.toBeNull()
+    expect(listContent?.content_type).toBe('Note')
+    expect(listContent?.content.title).toBe('Testing 1')
+    expect(listContent?.content.text).toBe('- [ ] Test 1\n- [x] Test 2')
+    expect(textContent?.content.trashed).toBe(false)
+    expect(textContent?.content.archived).toBe(false)
+    expect(textContent?.content.pinned).toBe(false)
  })

  it('should parse html data', () => {
-    const converter = new GoogleKeepConverter(generateUuid)
+    const converter = new GoogleKeepConverter(superConverterService, generateUuid)

    const result = converter.tryParseAsHtml(
      htmlTestData,
--- a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts
+++ b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts
@@ -2,33 +2,48 @@ import { ContentType } from '@standardnotes/domain-core'
 import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
 import { readFileAsText } from '../Utils'
 import { GenerateUuid } from '@standardnotes/services'
+import { SuperConverterServiceInterface } from '@standardnotes/files'
+import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
+
+type Content =
+  | {
+      textContent: string
+    }
+  | {
+      listContent: {
+        text: string
+        isChecked: boolean
+      }[]
+    }

 type GoogleKeepJsonNote = {
  color: string
  isTrashed: boolean
  isPinned: boolean
  isArchived: boolean
-  textContent: string
  title: string
  userEditedTimestampUsec: number
-}
+} & Content

 export class GoogleKeepConverter {
-  constructor(private _generateUuid: GenerateUuid) {}
+  constructor(
+    private superConverterService: SuperConverterServiceInterface,
+    private _generateUuid: GenerateUuid,
+  ) {}

  async convertGoogleKeepBackupFileToNote(
    file: File,
-    stripHtml: boolean,
+    isEntitledToSuper: boolean,
  ): Promise<DecryptedTransferPayload<NoteContent>> {
    const content = await readFileAsText(file)

-    const possiblePayloadFromJson = this.tryParseAsJson(content)
+    const possiblePayloadFromJson = this.tryParseAsJson(content, isEntitledToSuper)

    if (possiblePayloadFromJson) {
      return possiblePayloadFromJson
    }

-    const possiblePayloadFromHtml = this.tryParseAsHtml(content, file, stripHtml)
+    const possiblePayloadFromHtml = this.tryParseAsHtml(content, file, isEntitledToSuper)

    if (possiblePayloadFromHtml) {
      return possiblePayloadFromHtml
@@ -37,20 +52,51 @@ export class GoogleKeepConverter {
    throw new Error('Could not parse Google Keep backup file')
  }

-  tryParseAsHtml(data: string, file: { name: string }, stripHtml: boolean): DecryptedTransferPayload<NoteContent> {
+  tryParseAsHtml(
+    data: string,
+    file: { name: string },
+    isEntitledToSuper: boolean,
+  ): DecryptedTransferPayload<NoteContent> {
    const rootElement = document.createElement('html')
    rootElement.innerHTML = data

+    const headingElement = rootElement.getElementsByClassName('heading')[0]
+    const date = new Date(headingElement?.textContent || '')
+    headingElement?.remove()
+
    const contentElement = rootElement.getElementsByClassName('content')[0]
+    if (!contentElement) {
+      throw new Error('Could not parse content. Content element not found.')
+    }
+
    let content: string | null

-    // Replace <br> with \n so line breaks get recognised
-    contentElement.innerHTML = contentElement.innerHTML.replace(/<br>/g, '\n')
+    // Convert lists to readable plaintext format
+    // or Super-convertable format
+    const lists = contentElement.getElementsByTagName('ul')
+    Array.from(lists).forEach((list) => {
+      list.setAttribute('__lexicallisttype', 'check')

-    if (stripHtml) {
+      const items = list.getElementsByTagName('li')
+      Array.from(items).forEach((item) => {
+        const bulletSpan = item.getElementsByClassName('bullet')[0]
+        bulletSpan?.remove()
+
+        const checked = item.classList.contains('checked')
+        item.setAttribute('aria-checked', checked ? 'true' : 'false')
+
+        if (!isEntitledToSuper) {
+          item.textContent = `- ${checked ? '[x]' : '[ ]'} ${item.textContent?.trim()}\n`
+        }
+      })
+    })
+
+    if (!isEntitledToSuper) {
+      // Replace <br> with \n so line breaks get recognised
+      contentElement.innerHTML = contentElement.innerHTML.replace(/<br>/g, '\n')
      content = contentElement.textContent
    } else {
-      content = contentElement.innerHTML
+      content = this.superConverterService.convertOtherFormatToSuperString(rootElement.innerHTML, 'html')
    }

    if (!content) {
@@ -59,8 +105,6 @@ export class GoogleKeepConverter {

    const title = rootElement.getElementsByClassName('title')[0]?.textContent || file.name

-    const date = this.getDateFromGKeepNote(data) || new Date()
-
    return {
      created_at: date,
      created_at_timestamp: date.getTime(),
@@ -72,35 +116,30 @@ export class GoogleKeepConverter {
        title: title,
        text: content,
        references: [],
+        ...(isEntitledToSuper
+          ? {
+              noteType: NoteType.Super,
+              editorIdentifier: NativeFeatureIdentifier.TYPES.SuperEditor,
+            }
+          : {}),
      },
    }
  }

-  getDateFromGKeepNote(note: string) {
-    const regexWithTitle = /.*(?=<\/div>\n<div class="title">)/
-    const regexWithoutTitle = /.*(?=<\/div>\n\n<div class="content">)/
-    const possibleDateStringWithTitle = regexWithTitle.exec(note)?.[0]
-    const possibleDateStringWithoutTitle = regexWithoutTitle.exec(note)?.[0]
-    if (possibleDateStringWithTitle) {
-      const date = new Date(possibleDateStringWithTitle)
-      if (date.toString() !== 'Invalid Date' && date.toString() !== 'NaN') {
-        return date
-      }
-    }
-    if (possibleDateStringWithoutTitle) {
-      const date = new Date(possibleDateStringWithoutTitle)
-      if (date.toString() !== 'Invalid Date' && date.toString() !== 'NaN') {
-        return date
-      }
-    }
-    return
-  }
-
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  static isValidGoogleKeepJson(json: any): boolean {
+    if (typeof json.textContent !== 'string') {
+      if (typeof json.listContent === 'object' && Array.isArray(json.listContent)) {
+        return json.listContent.every(
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          (item: any) => typeof item.text === 'string' && typeof item.isChecked === 'boolean',
+        )
+      }
+      return false
+    }
+
    return (
      typeof json.title === 'string' &&
-      typeof json.textContent === 'string' &&
      typeof json.userEditedTimestampUsec === 'number' &&
      typeof json.isArchived === 'boolean' &&
      typeof json.isTrashed === 'boolean' &&
@@ -109,13 +148,26 @@ export class GoogleKeepConverter {
    )
  }

-  tryParseAsJson(data: string): DecryptedTransferPayload<NoteContent> | null {
+  tryParseAsJson(data: string, isEntitledToSuper: boolean): DecryptedTransferPayload<NoteContent> | null {
    try {
      const parsed = JSON.parse(data) as GoogleKeepJsonNote
      if (!GoogleKeepConverter.isValidGoogleKeepJson(parsed)) {
        return null
      }
      const date = new Date(parsed.userEditedTimestampUsec / 1000)
+      let text: string
+      if ('textContent' in parsed) {
+        text = parsed.textContent
+      } else {
+        text = parsed.listContent
+          .map((item) => {
+            return item.isChecked ? `- [x] ${item.text}` : `- [ ] ${item.text}`
+          })
+          .join('\n')
+      }
+      if (isEntitledToSuper) {
+        text = this.superConverterService.convertOtherFormatToSuperString(text, 'md')
+      }
      return {
        created_at: date,
        created_at_timestamp: date.getTime(),
@@ -125,14 +177,21 @@ export class GoogleKeepConverter {
        content_type: ContentType.TYPES.Note,
        content: {
          title: parsed.title,
-          text: parsed.textContent,
+          text,
          references: [],
          archived: Boolean(parsed.isArchived),
          trashed: Boolean(parsed.isTrashed),
          pinned: Boolean(parsed.isPinned),
+          ...(isEntitledToSuper
+            ? {
+                noteType: NoteType.Super,
+                editorIdentifier: NativeFeatureIdentifier.TYPES.SuperEditor,
+              }
+            : {}),
        },
      }
    } catch (e) {
+      console.error(e)
      return null
    }
  }
--- a/packages/ui-services/src/Import/GoogleKeepConverter/testData.ts
+++ b/packages/ui-services/src/Import/GoogleKeepConverter/testData.ts
@@ -1,4 +1,4 @@
-const json = {
+const jsonWithTextContent = {
  color: 'DEFAULT',
  isTrashed: false,
  isPinned: false,
@@ -8,7 +8,28 @@ const json = {
  userEditedTimestampUsec: 1618528050144000,
 }

-export const jsonTestData = JSON.stringify(json)
+export const jsonTextContentData = JSON.stringify(jsonWithTextContent)
+
+const jsonWithListContent = {
+  color: 'DEFAULT',
+  isTrashed: false,
+  isPinned: false,
+  isArchived: false,
+  listContent: [
+    {
+      text: 'Test 1',
+      isChecked: false,
+    },
+    {
+      text: 'Test 2',
+      isChecked: true,
+    },
+  ],
+  title: 'Testing 1',
+  userEditedTimestampUsec: 1618528050144000,
+}
+
+export const jsonListContentData = JSON.stringify(jsonWithListContent)

 export const htmlTestData = `<?xml version="1.0" ?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
--- a/packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts
+++ b/packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts
@@ -0,0 +1,51 @@
+import { ContentType } from '@standardnotes/domain-core'
+import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
+import { parseFileName } from '@standardnotes/filepicker'
+import { SuperConverterServiceInterface } from '@standardnotes/files'
+import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
+import { GenerateUuid } from '@standardnotes/services'
+import { readFileAsText } from '../Utils'
+
+export class HTMLConverter {
+  constructor(
+    private superConverterService: SuperConverterServiceInterface,
+    private _generateUuid: GenerateUuid,
+  ) {}
+
+  static isHTMLFile(file: File): boolean {
+    return file.type === 'text/html'
+  }
+
+  async convertHTMLFileToNote(file: File, isEntitledToSuper: boolean): Promise<DecryptedTransferPayload<NoteContent>> {
+    const content = await readFileAsText(file)
+
+    const { name } = parseFileName(file.name)
+
+    const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
+    const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
+
+    const text = isEntitledToSuper
+      ? this.superConverterService.convertOtherFormatToSuperString(content, 'html')
+      : content
+
+    return {
+      created_at: createdAtDate,
+      created_at_timestamp: createdAtDate.getTime(),
+      updated_at: updatedAtDate,
+      updated_at_timestamp: updatedAtDate.getTime(),
+      uuid: this._generateUuid.execute().getValue(),
+      content_type: ContentType.TYPES.Note,
+      content: {
+        title: name,
+        text,
+        references: [],
+        ...(isEntitledToSuper
+          ? {
+              noteType: NoteType.Super,
+              editorIdentifier: NativeFeatureIdentifier.TYPES.SuperEditor,
+            }
+          : {}),
+      },
+    }
+  }
+}
--- a/packages/ui-services/src/Import/Importer.ts
+++ b/packages/ui-services/src/Import/Importer.ts
@@ -14,8 +14,11 @@ import { PlaintextConverter } from './PlaintextConverter/PlaintextConverter'
 import { SimplenoteConverter } from './SimplenoteConverter/SimplenoteConverter'
 import { readFileAsText } from './Utils'
 import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
+import { HTMLConverter } from './HTMLConverter/HTMLConverter'
+import { SuperConverterServiceInterface } from '@standardnotes/snjs/dist/@types'
+import { SuperConverter } from './SuperConverter/SuperConverter'

-export type NoteImportType = 'plaintext' | 'evernote' | 'google-keep' | 'simplenote' | 'aegis'
+export type NoteImportType = 'plaintext' | 'evernote' | 'google-keep' | 'simplenote' | 'aegis' | 'html' | 'super'

 export class Importer {
  aegisConverter: AegisToAuthenticatorConverter
@@ -23,21 +26,26 @@ export class Importer {
  simplenoteConverter: SimplenoteConverter
  plaintextConverter: PlaintextConverter
  evernoteConverter: EvernoteConverter
+  htmlConverter: HTMLConverter
+  superConverter: SuperConverter

  constructor(
    private features: FeaturesClientInterface,
    private mutator: MutatorClientInterface,
    private items: ItemManagerInterface,
+    private superConverterService: SuperConverterServiceInterface,
    _generateUuid: GenerateUuid,
  ) {
    this.aegisConverter = new AegisToAuthenticatorConverter(_generateUuid)
-    this.googleKeepConverter = new GoogleKeepConverter(_generateUuid)
+    this.googleKeepConverter = new GoogleKeepConverter(this.superConverterService, _generateUuid)
    this.simplenoteConverter = new SimplenoteConverter(_generateUuid)
    this.plaintextConverter = new PlaintextConverter(_generateUuid)
    this.evernoteConverter = new EvernoteConverter(_generateUuid)
+    this.htmlConverter = new HTMLConverter(this.superConverterService, _generateUuid)
+    this.superConverter = new SuperConverter(this.superConverterService, _generateUuid)
  }

-  static detectService = async (file: File): Promise<NoteImportType | null> => {
+  detectService = async (file: File): Promise<NoteImportType | null> => {
    const content = await readFileAsText(file)

    const { ext } = parseFileName(file.name)
@@ -46,6 +54,10 @@ export class Importer {
      return 'evernote'
    }

+    if (file.type === 'application/json' && this.superConverterService.isValidSuperString(content)) {
+      return 'super'
+    }
+
    try {
      const json = JSON.parse(content)

@@ -68,24 +80,39 @@ export class Importer {
      return 'plaintext'
    }

+    if (HTMLConverter.isHTMLFile(file)) {
+      return 'html'
+    }
+
    return null
  }

  async getPayloadsFromFile(file: File, type: NoteImportType): Promise<DecryptedTransferPayload[]> {
-    if (type === 'aegis') {
+    const isEntitledToSuper =
+      this.features.getFeatureStatus(
+        NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.SuperEditor).getValue(),
+      ) === FeatureStatus.Entitled
+    if (type === 'super') {
+      if (!isEntitledToSuper) {
+        throw new Error('Importing Super notes requires a subscription.')
+      }
+      return [await this.superConverter.convertSuperFileToNote(file)]
+    } else if (type === 'aegis') {
      const isEntitledToAuthenticator =
        this.features.getFeatureStatus(
          NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.TokenVaultEditor).getValue(),
        ) === FeatureStatus.Entitled
      return [await this.aegisConverter.convertAegisBackupFileToNote(file, isEntitledToAuthenticator)]
    } else if (type === 'google-keep') {
-      return [await this.googleKeepConverter.convertGoogleKeepBackupFileToNote(file, true)]
+      return [await this.googleKeepConverter.convertGoogleKeepBackupFileToNote(file, isEntitledToSuper)]
    } else if (type === 'simplenote') {
      return await this.simplenoteConverter.convertSimplenoteBackupFileToNotes(file)
    } else if (type === 'evernote') {
      return await this.evernoteConverter.convertENEXFileToNotesAndTags(file, false)
    } else if (type === 'plaintext') {
      return [await this.plaintextConverter.convertPlaintextFileToNote(file)]
+    } else if (type === 'html') {
+      return [await this.htmlConverter.convertHTMLFileToNote(file, isEntitledToSuper)]
    }

    return []
--- a/packages/ui-services/src/Import/SuperConverter/SuperConverter.ts
+++ b/packages/ui-services/src/Import/SuperConverter/SuperConverter.ts
@@ -0,0 +1,43 @@
+import { SuperConverterServiceInterface } from '@standardnotes/files'
+import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
+import { GenerateUuid } from '@standardnotes/services'
+import { readFileAsText } from '../Utils'
+import { parseFileName } from '@standardnotes/filepicker'
+import { ContentType } from '@standardnotes/domain-core'
+import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
+
+export class SuperConverter {
+  constructor(
+    private converterService: SuperConverterServiceInterface,
+    private _generateUuid: GenerateUuid,
+  ) {}
+
+  async convertSuperFileToNote(file: File): Promise<DecryptedTransferPayload<NoteContent>> {
+    const content = await readFileAsText(file)
+
+    if (!this.converterService.isValidSuperString(content)) {
+      throw new Error('Content is not valid Super JSON')
+    }
+
+    const { name } = parseFileName(file.name)
+
+    const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
+    const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
+
+    return {
+      created_at: createdAtDate,
+      created_at_timestamp: createdAtDate.getTime(),
+      updated_at: updatedAtDate,
+      updated_at_timestamp: updatedAtDate.getTime(),
+      uuid: this._generateUuid.execute().getValue(),
+      content_type: ContentType.TYPES.Note,
+      content: {
+        title: name,
+        text: content,
+        references: [],
+        noteType: NoteType.Super,
+        editorIdentifier: NativeFeatureIdentifier.TYPES.SuperEditor,
+      },
+    }
+  }
+}