rmorawski/standardnotes-app-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: handle larger files in importer (#2692)

Browse Source
This commit is contained in:
Aman Harwara
2023-12-11 16:30:31 +05:30
committed by GitHub
parent 63e69b5e4b
commit 82d5a36932
22 changed files with 614 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts
View File

@@ -1,5 +1,6 @@
import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
import { Converter } from '../Converter'
import { ConversionResult } from '../ConversionResult'
type AegisData = {
db: {
@@ -45,7 +46,7 @@ export class AegisToAuthenticatorConverter implements Converter {
return false
}
convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
const content = await readFileAsText(file)
const entries = this.parseEntries(content)
@@ -59,17 +60,22 @@ export class AegisToAuthenticatorConverter implements Converter {
const title = file.name.split('.')[0]
const text = JSON.stringify(entries)
return [
createNote({
createdAt,
updatedAt,
title,
text,
noteType: NoteType.Authentication,
editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
useSuperIfPossible: false,
}),
]
const note = await insertNote({
createdAt,
updatedAt,
title,
text,
noteType: NoteType.Authentication,
editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
useSuperIfPossible: false,
})
const successful: ConversionResult['successful'] = [note]
return {
successful,
errored: [],
}
}
parseEntries(data: string): AuthenticatorEntry[] | null {

9
packages/ui-services/src/Import/ConversionResult.ts Normal file
View File

@@ -0,0 +1,9 @@
import { DecryptedItemInterface } from '@standardnotes/models'
export type ConversionResult = {
successful: DecryptedItemInterface[]
errored: {
name: string
error: Error
}[]
}

34
packages/ui-services/src/Import/Converter.ts
View File

@@ -1,5 +1,6 @@
import { NoteType } from '@standardnotes/features'
import { DecryptedTransferPayload, ItemContent, NoteContent, TagContent } from '@standardnotes/models'
import { DecryptedItemInterface, FileItem, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models'
import { ConversionResult } from './ConversionResult'
export interface Converter {
getImportType(): string
@@ -12,17 +13,24 @@ export interface Converter {
convert(
file: File,
dependencies: {
createNote: CreateNoteFn
createTag: CreateTagFn
insertNote: InsertNoteFn
insertTag: InsertTagFn
canUploadFiles: boolean
uploadFile: UploadFileFn
canUseSuper: boolean
convertHTMLToSuper: (html: string) => string
convertMarkdownToSuper: (markdown: string) => string
readFileAsText: (file: File) => Promise<string>
linkItems(
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
): Promise<void>
cleanupItems(items: DecryptedItemInterface<ItemContent>[]): Promise<void>
},
): Promise<DecryptedTransferPayload<ItemContent>[]>
): Promise<ConversionResult>
}
export type CreateNoteFn = (options: {
export type InsertNoteFn = (options: {
createdAt: Date
updatedAt: Date
title: string
@@ -33,10 +41,20 @@ export type CreateNoteFn = (options: {
trashed?: boolean
editorIdentifier?: NoteContent['editorIdentifier']
useSuperIfPossible: boolean
}) => DecryptedTransferPayload<NoteContent>
}) => Promise<SNNote>
export type CreateTagFn = (options: {
export type InsertTagFn = (options: {
createdAt: Date
updatedAt: Date
title: string
}) => DecryptedTransferPayload<TagContent>
references: SNTag['references']
}) => Promise<SNTag>
export type UploadFileFn = (file: File) => Promise<FileItem | undefined>
export type LinkItemsFn = (
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
) => Promise<void>
export type CleanupItemsFn = (items: DecryptedItemInterface<ItemContent>[]) => Promise<void>

90
packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts
View File

@@ -3,9 +3,9 @@
*/
import { ContentType } from '@standardnotes/domain-core'
import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
import { SNNote, SNTag } from '@standardnotes/models'
import { EvernoteConverter, EvernoteResource } from './EvernoteConverter'
import { createTestResourceElement, enex, enexWithNoNoteOrTag } from './testData'
import { createTestResourceElement, enex } from './testData'
import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
import { GenerateUuid } from '@standardnotes/services'
import { Converter } from '../Converter'
@@ -33,77 +33,80 @@ describe('EvernoteConverter', () => {
const readFileAsText = async (file: File) => file as unknown as string
const dependencies: Parameters<Converter['convert']>[1] = {
createNote: ({ text }) =>
insertNote: async ({ text }) =>
({
content_type: ContentType.TYPES.Note,
content: {
text,
references: [],
},
}) as unknown as DecryptedTransferPayload<NoteContent>,
createTag: ({ title }) =>
uuid: generateUuid.execute().getValue(),
}) as unknown as SNNote,
insertTag: async ({ title }) =>
({
content_type: ContentType.TYPES.Tag,
content: {
title,
references: [],
},
}) as unknown as DecryptedTransferPayload<TagContent>,
uuid: generateUuid.execute().getValue(),
}) as unknown as SNTag,
convertHTMLToSuper: (data) => data,
convertMarkdownToSuper: jest.fn(),
readFileAsText,
canUseSuper: false,
canUploadFiles: false,
uploadFile: async () => void 0,
linkItems: async (item, itemToLink) => {
itemToLink.content.references.push({
content_type: item.content_type,
uuid: item.uuid,
})
},
cleanupItems: async () => void 0,
}
it('should throw error if no note or tag in enex', () => {
const converter = new EvernoteConverter(generateUuid)
expect(converter.convert(enexWithNoNoteOrTag as unknown as File, dependencies)).rejects.toThrowError()
})
it('should parse and strip html', async () => {
const converter = new EvernoteConverter(generateUuid)
const result = await converter.convert(enex as unknown as File, dependencies)
const { successful } = await converter.convert(enex as unknown as File, dependencies)
expect(result).not.toBeNull()
expect(result?.length).toBe(3)
expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe('This is a test.\nh e ')
expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
'Lorem ipsum dolor sit amet, consectetur adipiscing elit.',
)
expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
expect(successful).not.toBeNull()
expect(successful?.length).toBe(3)
expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[0] as SNNote).content.text).toBe('This is a test.\nh e ')
expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
expect((successful?.[1] as SNTag).content.references.length).toBe(2)
expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[2] as SNNote).content.text).toBe('Lorem ipsum dolor sit amet, consectetur adipiscing elit.')
})
it('should parse and not strip html', async () => {
const converter = new EvernoteConverter(generateUuid)
const result = await converter.convert(enex as unknown as File, {
const { successful } = await converter.convert(enex as unknown as File, {
...dependencies,
canUseSuper: true,
})
expect(result).not.toBeNull()
expect(result?.length).toBe(3)
expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
expect(successful).not.toBeNull()
expect(successful?.length).toBe(3)
expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[0] as SNNote).content.text).toBe(
'<p>This is a test.</p><ul></ul><ol></ol><font><span>h </span><span>e </span></font>',
)
expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
expect((successful?.[1] as SNTag).content.references.length).toBe(2)
expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[2] as SNNote).content.text).toBe(
'<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>',
)
expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
})
it('should convert lists to super format if applicable', () => {
@@ -129,7 +132,7 @@ describe('EvernoteConverter', () => {
expect(unorderedList2.getAttribute('__lexicallisttype')).toBeFalsy()
})
it('should replace media elements with resources', () => {
it('should replace media elements with resources', async () => {
const resources: EvernoteResource[] = [
{
hash: 'hash1',
@@ -152,9 +155,14 @@ describe('EvernoteConverter', () => {
const array = [mediaElement1, mediaElement2, mediaElement3]
const converter = new EvernoteConverter(generateUuid)
const replacedCount = converter.replaceMediaElementsWithResources(array, resources)
const { replacedElements } = await converter.replaceMediaElementsWithResources(
array,
resources,
false,
dependencies.uploadFile,
)
expect(replacedCount).toBe(1)
expect(replacedElements.length).toBe(1)
})
describe('getResourceFromElement', () => {

292
packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts
View File

@@ -1,11 +1,12 @@
import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
import { FileItem, SNTag } from '@standardnotes/models'
import dayjs from 'dayjs'
import customParseFormat from 'dayjs/plugin/customParseFormat'
import utc from 'dayjs/plugin/utc'
import { GenerateUuid } from '@standardnotes/services'
import MD5 from 'crypto-js/md5'
import Base64 from 'crypto-js/enc-base64'
import { Converter } from '../Converter'
import { Converter, UploadFileFn } from '../Converter'
import { ConversionResult } from '../ConversionResult'
dayjs.extend(customParseFormat)
dayjs.extend(utc)
@@ -35,14 +36,23 @@ export class EvernoteConverter implements Converter {
convert: Converter['convert'] = async (
file,
{ createNote, createTag, canUseSuper, convertHTMLToSuper, readFileAsText },
{
insertNote,
insertTag,
linkItems,
canUploadFiles,
canUseSuper,
convertHTMLToSuper,
readFileAsText,
uploadFile,
cleanupItems,
},
) => {
const content = await readFileAsText(file)
const xmlDoc = this.loadXMLString(content, 'xml')
const xmlNotes = xmlDoc.getElementsByTagName('note')
const notes: DecryptedTransferPayload<NoteContent>[] = []
const tags: DecryptedTransferPayload<TagContent>[] = []
const tags: SNTag[] = []
function findTag(title: string | null) {
return tags.filter(function (tag) {
@@ -50,94 +60,119 @@ export class EvernoteConverter implements Converter {
})[0]
}
const successful: ConversionResult['successful'] = []
const errored: ConversionResult['errored'] = []
for (const [index, xmlNote] of Array.from(xmlNotes).entries()) {
const title = xmlNote.getElementsByTagName('title')[0].textContent
const created = xmlNote.getElementsByTagName('created')[0]?.textContent
const updatedNodes = xmlNote.getElementsByTagName('updated')
const updated = updatedNodes.length ? updatedNodes[0].textContent : null
const resources = Array.from(xmlNote.getElementsByTagName('resource'))
.map(this.getResourceFromElement)
.filter(Boolean) as EvernoteResource[]
const filesToPotentiallyCleanup: FileItem[] = []
try {
const title = xmlNote.getElementsByTagName('title')[0].textContent
const created = xmlNote.getElementsByTagName('created')[0]?.textContent
const updatedNodes = xmlNote.getElementsByTagName('updated')
const updated = updatedNodes.length ? updatedNodes[0].textContent : null
const resources = Array.from(xmlNote.getElementsByTagName('resource'))
.map(this.getResourceFromElement)
.filter(Boolean) as EvernoteResource[]
const contentNode = xmlNote.getElementsByTagName('content')[0]
const contentXmlString = this.getXmlStringFromContentElement(contentNode)
if (!contentXmlString) {
continue
}
const contentXml = this.loadXMLString(contentXmlString, 'html')
const contentNode = xmlNote.getElementsByTagName('content')[0]
const contentXmlString = this.getXmlStringFromContentElement(contentNode)
if (!contentXmlString) {
continue
}
const contentXml = this.loadXMLString(contentXmlString, 'html')
const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
if (canUseSuper) {
this.convertTopLevelDivsToParagraphs(noteElement)
this.convertListsToSuperFormatIfApplicable(unorderedLists)
this.convertLeftPaddingToSuperIndent(noteElement)
}
this.removeEmptyAndOrphanListElements(noteElement)
this.removeUnnecessaryTopLevelBreaks(noteElement)
const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
this.replaceMediaElementsWithResources(mediaElements, resources)
// Some notes have <font> tags that contain separate <span> tags with text
// which causes broken paragraphs in the note.
const fontElements = Array.from(noteElement.getElementsByTagName('font'))
for (const fontElement of fontElements) {
fontElement.childNodes.forEach((childNode) => {
childNode.textContent += ' '
})
fontElement.innerText = fontElement.textContent || ''
}
let contentHTML = noteElement.innerHTML
if (!canUseSuper) {
contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
contentHTML = contentHTML.trim()
}
const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)
const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate
const note = createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: !title ? `Imported note ${index + 1} from Evernote` : title,
text,
useSuperIfPossible: canUseSuper,
})
const xmlTags = xmlNote.getElementsByTagName('tag')
for (const tagXml of Array.from(xmlTags)) {
const tagName = tagXml.childNodes[0].nodeValue
let tag = findTag(tagName)
if (!tag) {
const now = new Date()
tag = createTag({
createdAt: now,
updatedAt: now,
title: tagName || `Imported tag ${index + 1} from Evernote`,
})
tags.push(tag)
if (canUseSuper) {
this.convertTopLevelDivsToParagraphs(noteElement)
this.convertListsToSuperFormatIfApplicable(unorderedLists)
this.convertLeftPaddingToSuperIndent(noteElement)
}
note.content.references.push({ content_type: tag.content_type, uuid: tag.uuid })
tag.content.references.push({ content_type: note.content_type, uuid: note.uuid })
this.removeEmptyAndOrphanListElements(noteElement)
this.removeUnnecessaryTopLevelBreaks(noteElement)
const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
const { uploadedFiles } = await this.replaceMediaElementsWithResources(
mediaElements,
resources,
canUploadFiles,
uploadFile,
)
filesToPotentiallyCleanup.push(...uploadedFiles)
// Some notes have <font> tags that contain separate <span> tags with text
// which causes broken paragraphs in the note.
const fontElements = Array.from(noteElement.getElementsByTagName('font'))
for (const fontElement of fontElements) {
fontElement.childNodes.forEach((childNode) => {
childNode.textContent += ' '
})
fontElement.innerText = fontElement.textContent || ''
}
let contentHTML = noteElement.innerHTML
if (!canUseSuper) {
contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
contentHTML = contentHTML.trim()
}
const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)
const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate
const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: !title ? `Imported note ${index + 1} from Evernote` : title,
text,
useSuperIfPossible: canUseSuper,
})
successful.push(note)
for (const uploadedFile of uploadedFiles) {
await linkItems(note, uploadedFile)
successful.push(uploadedFile)
}
const xmlTags = xmlNote.getElementsByTagName('tag')
for (const tagXml of Array.from(xmlTags)) {
const tagName = tagXml.childNodes[0].nodeValue
let tag = findTag(tagName)
if (!tag) {
const now = new Date()
tag = await insertTag({
createdAt: now,
updatedAt: now,
title: tagName || `Imported tag ${index + 1} from Evernote`,
references: [],
})
tags.push(tag)
successful.push(tag)
}
await linkItems(note, tag)
}
} catch (error) {
console.error(error)
errored.push({
name: xmlNote.getElementsByTagName('title')?.[0]?.textContent || `${file.name} - Note #${index}`,
error: error as Error,
})
cleanupItems(filesToPotentiallyCleanup).catch(console.error)
continue
}
notes.push(note)
}
const allItems: DecryptedTransferPayload[] = [...notes, ...tags]
if (allItems.length === 0) {
throw new Error('Could not parse any notes or tags from Evernote file.')
return {
successful,
errored,
}
return allItems
}
getXmlStringFromContentElement(contentElement: Element) {
@@ -259,44 +294,83 @@ export class EvernoteConverter implements Converter {
})
}
replaceMediaElementsWithResources(mediaElements: Element[], resources: EvernoteResource[]): number {
let replacedElements = 0
getHTMLElementFromResource(resource: EvernoteResource) {
let resourceElement: HTMLElement = document.createElement('object')
resourceElement.setAttribute('type', resource.mimeType)
resourceElement.setAttribute('data', resource.data)
if (resource.mimeType.startsWith('image/')) {
resourceElement = document.createElement('img')
resourceElement.setAttribute('src', resource.data)
resourceElement.setAttribute('data-mime-type', resource.mimeType)
} else if (resource.mimeType.startsWith('audio/')) {
resourceElement = document.createElement('audio')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
} else if (resource.mimeType.startsWith('video/')) {
resourceElement = document.createElement('video')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
}
resourceElement.setAttribute('data-filename', resource.fileName)
return resourceElement
}
async getFileFromResource(resource: EvernoteResource): Promise<File> {
const response = await fetch(resource.data)
const blob = await response.blob()
return new File([blob], resource.fileName, { type: resource.mimeType })
}
async replaceMediaElementsWithResources(
mediaElements: Element[],
resources: EvernoteResource[],
canUploadFiles: boolean,
uploadFile: UploadFileFn,
): Promise<{
replacedElements: HTMLElement[]
uploadedFiles: FileItem[]
}> {
const replacedElements: HTMLElement[] = []
const uploadedFiles = new Map<EvernoteResource['hash'], FileItem>()
for (const mediaElement of mediaElements) {
const hash = mediaElement.getAttribute('hash')
const resource = resources.find((resource) => resource && resource.hash === hash)
if (!resource) {
continue
}
let resourceElement: HTMLElement = document.createElement('object')
resourceElement.setAttribute('type', resource.mimeType)
resourceElement.setAttribute('data', resource.data)
if (resource.mimeType.startsWith('image/')) {
resourceElement = document.createElement('img')
resourceElement.setAttribute('src', resource.data)
resourceElement.setAttribute('data-mime-type', resource.mimeType)
} else if (resource.mimeType.startsWith('audio/')) {
resourceElement = document.createElement('audio')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
} else if (resource.mimeType.startsWith('video/')) {
resourceElement = document.createElement('video')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
}
resourceElement.setAttribute('data-filename', resource.fileName)
if (!mediaElement.parentNode) {
continue
}
const existingFile = uploadedFiles.get(resource.hash)
const fileItem = canUploadFiles
? existingFile
? existingFile
: await uploadFile(await this.getFileFromResource(resource))
: undefined
if (fileItem) {
const fileElement = document.createElement('div')
fileElement.setAttribute('data-lexical-file-uuid', fileItem.uuid)
mediaElement.parentNode.replaceChild(fileElement, mediaElement)
replacedElements.push(fileElement)
if (!existingFile) {
uploadedFiles.set(resource.hash, fileItem)
}
continue
}
const resourceElement = this.getHTMLElementFromResource(resource)
mediaElement.parentNode.replaceChild(resourceElement, mediaElement)
replacedElements++
replacedElements.push(resourceElement)
}
return {
replacedElements,
uploadedFiles: Array.from(uploadedFiles.values()),
}
return replacedElements
}
loadXMLString(string: string, type: 'html' | 'xml') {

5
packages/ui-services/src/Import/EvernoteConverter/testData.ts
View File

@@ -36,11 +36,6 @@ export const enex = `<?xml version="1.0" encoding="UTF-8"?>
</note>
</en-export>`
export const enexWithNoNoteOrTag = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export3.dtd">
<en-export export-date="20210408T052957Z" application="Evernote" version="10.8.5">
</en-export>`
export function createTestResourceElement(
shouldHaveMimeType = true,
shouldHaveSourceUrl = false,

20
packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts
View File

@@ -4,11 +4,11 @@
import { jsonTextContentData, htmlTestData, jsonListContentData } from './testData'
import { GoogleKeepConverter } from './GoogleKeepConverter'
import { ContentType, DecryptedTransferPayload, NoteContent } from '@standardnotes/snjs'
import { CreateNoteFn } from '../Converter'
import { ContentType, SNNote } from '@standardnotes/snjs'
import { InsertNoteFn } from '../Converter'
describe('GoogleKeepConverter', () => {
const createNote: CreateNoteFn = ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) =>
const insertNote: InsertNoteFn = async ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) =>
({
uuid: Math.random().toString(),
created_at: createdAt,
@@ -22,12 +22,12 @@ describe('GoogleKeepConverter', () => {
pinned,
references: [],
},
}) as unknown as DecryptedTransferPayload<NoteContent>
}) as unknown as SNNote
it('should parse json data', () => {
it('should parse json data', async () => {
const converter = new GoogleKeepConverter()
const textContent = converter.tryParseAsJson(jsonTextContentData, createNote, (md) => md)
const textContent = await converter.tryParseAsJson(jsonTextContentData, insertNote, (md) => md)
expect(textContent).not.toBeNull()
expect(textContent?.created_at).toBeInstanceOf(Date)
@@ -40,7 +40,7 @@ describe('GoogleKeepConverter', () => {
expect(textContent?.content.archived).toBe(false)
expect(textContent?.content.pinned).toBe(false)
const listContent = converter.tryParseAsJson(jsonListContentData, createNote, (md) => md)
const listContent = await converter.tryParseAsJson(jsonListContentData, insertNote, (md) => md)
expect(listContent).not.toBeNull()
expect(listContent?.created_at).toBeInstanceOf(Date)
@@ -54,15 +54,15 @@ describe('GoogleKeepConverter', () => {
expect(textContent?.content.pinned).toBe(false)
})
it('should parse html data', () => {
it('should parse html data', async () => {
const converter = new GoogleKeepConverter()
const result = converter.tryParseAsHtml(
const result = await converter.tryParseAsHtml(
htmlTestData,
{
name: 'note-2.html',
},
createNote,
insertNote,
(html) => html,
false,
)

39
packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts
View File

@@ -1,5 +1,5 @@
import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
import { Converter, CreateNoteFn } from '../Converter'
import { SNNote } from '@standardnotes/models'
import { Converter, InsertNoteFn } from '../Converter'
type Content =
| {
@@ -45,32 +45,31 @@ export class GoogleKeepConverter implements Converter {
convert: Converter['convert'] = async (
file,
{ createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText },
{ insertNote: createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText },
) => {
const content = await readFileAsText(file)
const possiblePayloadFromJson = this.tryParseAsJson(content, createNote, convertMarkdownToSuper)
const note =
(await this.tryParseAsJson(content, createNote, convertMarkdownToSuper)) ||
(await this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper))
if (possiblePayloadFromJson) {
return [possiblePayloadFromJson]
}
const possiblePayloadFromHtml = this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper)
if (possiblePayloadFromHtml) {
return [possiblePayloadFromHtml]
if (note) {
return {
successful: [note],
errored: [],
}
}
throw new Error('Could not parse Google Keep backup file')
}
tryParseAsHtml(
async tryParseAsHtml(
data: string,
file: { name: string },
createNote: CreateNoteFn,
insertNote: InsertNoteFn,
convertHTMLToSuper: (html: string) => string,
canUseSuper: boolean,
): DecryptedTransferPayload<NoteContent> {
): Promise<SNNote> {
const rootElement = document.createElement('html')
rootElement.innerHTML = data
@@ -119,7 +118,7 @@ export class GoogleKeepConverter implements Converter {
const title = rootElement.getElementsByClassName('title')[0]?.textContent || file.name
return createNote({
return await insertNote({
createdAt: date,
updatedAt: date,
title: title,
@@ -150,11 +149,11 @@ export class GoogleKeepConverter implements Converter {
)
}
tryParseAsJson(
async tryParseAsJson(
data: string,
createNote: CreateNoteFn,
createNote: InsertNoteFn,
convertMarkdownToSuper: (md: string) => string,
): DecryptedTransferPayload<NoteContent> | null {
): Promise<SNNote | null> {
try {
const parsed = JSON.parse(data) as GoogleKeepJsonNote
if (!GoogleKeepConverter.isValidGoogleKeepJson(parsed)) {
@@ -172,7 +171,7 @@ export class GoogleKeepConverter implements Converter {
.join('\n')
}
text = convertMarkdownToSuper(text)
return createNote({
return await createNote({
createdAt: date,
updatedAt: date,
title: parsed.title,

23
packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts
View File

@@ -16,7 +16,7 @@ export class HTMLConverter implements Converter {
return true
}
convert: Converter['convert'] = async (file, { createNote, convertHTMLToSuper, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote: createNote, convertHTMLToSuper, readFileAsText }) => {
const content = await readFileAsText(file)
const { name } = parseFileName(file.name)
@@ -26,14 +26,17 @@ export class HTMLConverter implements Converter {
const text = convertHTMLToSuper(content)
return [
createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text,
useSuperIfPossible: true,
}),
]
const note = await createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text,
useSuperIfPossible: true,
})
return {
successful: [note],
errored: [],
}
}
}

194
packages/ui-services/src/Import/Importer.ts
View File

@@ -15,20 +15,24 @@ import { SimplenoteConverter } from './SimplenoteConverter/SimplenoteConverter'
import { readFileAsText } from './Utils'
import {
DecryptedItemInterface,
DecryptedTransferPayload,
FileItem,
ItemContent,
NoteContent,
NoteMutator,
SNNote,
isNote,
SNTag,
TagContent,
isFile,
} from '@standardnotes/models'
import { HTMLConverter } from './HTMLConverter/HTMLConverter'
import { SuperConverter } from './SuperConverter/SuperConverter'
import { Converter, CreateNoteFn, CreateTagFn } from './Converter'
import { SuperConverterServiceInterface } from '@standardnotes/files'
import { CleanupItemsFn, Converter, InsertNoteFn, InsertTagFn, LinkItemsFn, UploadFileFn } from './Converter'
import { ConversionResult } from './ConversionResult'
import { FilesClientInterface, SuperConverterServiceInterface } from '@standardnotes/files'
import { ContentType } from '@standardnotes/domain-core'
const BytesInOneMegabyte = 1_000_000
const NoteSizeThreshold = 3 * BytesInOneMegabyte
export class Importer {
converters: Set<Converter> = new Set()
@@ -50,9 +54,11 @@ export class Importer {
linkItems(
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
sync: boolean,
): Promise<void>
},
private _generateUuid: GenerateUuid,
private files: FilesClientInterface,
) {
this.registerNativeConverters()
}
@@ -88,19 +94,19 @@ export class Importer {
return null
}
createNote: CreateNoteFn = ({
insertNote: InsertNoteFn = async ({
createdAt,
updatedAt,
title,
text,
noteType,
editorIdentifier,
trashed,
archived,
pinned,
trashed = false,
archived = false,
pinned = false,
useSuperIfPossible,
}) => {
if (noteType === NoteType.Super && !this.isEntitledToSuper()) {
if (noteType === NoteType.Super && !this.canUseSuper()) {
noteType = undefined
}
@@ -112,16 +118,17 @@ export class Importer {
editorIdentifier = undefined
}
const shouldUseSuper = useSuperIfPossible && this.isEntitledToSuper()
const shouldUseSuper = useSuperIfPossible && this.canUseSuper()
return {
created_at: createdAt,
created_at_timestamp: createdAt.getTime(),
updated_at: updatedAt,
updated_at_timestamp: updatedAt.getTime(),
uuid: this._generateUuid.execute().getValue(),
content_type: ContentType.TYPES.Note,
content: {
const noteSize = new Blob([text]).size
if (noteSize > NoteSizeThreshold) {
throw new Error('Note is too large to import')
}
const note = this.items.createTemplateItem<NoteContent, SNNote>(
ContentType.TYPES.Note,
{
title,
text,
references: [],
@@ -131,27 +138,68 @@ export class Importer {
pinned,
editorIdentifier: shouldUseSuper ? NativeFeatureIdentifier.TYPES.SuperEditor : editorIdentifier,
},
}
{
created_at: createdAt,
updated_at: updatedAt,
},
)
return await this.mutator.insertItem(note)
}
createTag: CreateTagFn = ({ createdAt, updatedAt, title }) => {
return {
uuid: this._generateUuid.execute().getValue(),
content_type: ContentType.TYPES.Tag,
created_at: createdAt,
created_at_timestamp: createdAt.getTime(),
updated_at: updatedAt,
updated_at_timestamp: updatedAt.getTime(),
content: {
title: title,
insertTag: InsertTagFn = async ({ createdAt, updatedAt, title, references }) => {
const tag = this.items.createTemplateItem<TagContent, SNTag>(
ContentType.TYPES.Tag,
{
title,
expanded: false,
iconString: '',
references: [],
references,
},
{
created_at: createdAt,
updated_at: updatedAt,
},
)
return await this.mutator.insertItem(tag)
}
canUploadFiles = (): boolean => {
const status = this.features.getFeatureStatus(
NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.Files).getValue(),
)
return status === FeatureStatus.Entitled
}
uploadFile: UploadFileFn = async (file) => {
if (!this.canUploadFiles()) {
return undefined
}
try {
return await this.filesController.uploadNewFile(file, { showToast: true })
} catch (error) {
console.error(error)
return undefined
}
}
isEntitledToSuper = (): boolean => {
linkItems: LinkItemsFn = async (item, itemToLink) => {
await this.linkingController.linkItems(item, itemToLink, false)
}
cleanupItems: CleanupItemsFn = async (items) => {
for (const item of items) {
if (isFile(item)) {
await this.files.deleteFile(item)
}
await this.mutator.deleteItems([item])
}
}
canUseSuper = (): boolean => {
return (
this.features.getFeatureStatus(
NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.SuperEditor).getValue(),
@@ -160,7 +208,7 @@ export class Importer {
}
convertHTMLToSuper = (html: string): string => {
if (!this.isEntitledToSuper()) {
if (!this.canUseSuper()) {
return html
}
@@ -168,20 +216,23 @@ export class Importer {
}
convertMarkdownToSuper = (markdown: string): string => {
if (!this.isEntitledToSuper()) {
if (!this.canUseSuper()) {
return markdown
}
return this.superConverterService.convertOtherFormatToSuperString(markdown, 'md')
}
async getPayloadsFromFile(file: File, type: string): Promise<DecryptedTransferPayload[]> {
const isEntitledToSuper = this.isEntitledToSuper()
async importFromFile(file: File, type: string): Promise<ConversionResult> {
const canUseSuper = this.canUseSuper()
if (type === 'super' && !isEntitledToSuper) {
if (type === 'super' && !canUseSuper) {
throw new Error('Importing Super notes requires a subscription')
}
const successful: ConversionResult['successful'] = []
const errored: ConversionResult['errored'] = []
for (const converter of this.converters) {
const isCorrectType = converter.getImportType() === type
@@ -195,65 +246,28 @@ export class Importer {
throw new Error('Content is not valid')
}
return await converter.convert(file, {
createNote: this.createNote,
createTag: this.createTag,
canUseSuper: isEntitledToSuper,
const result = await converter.convert(file, {
insertNote: this.insertNote,
insertTag: this.insertTag,
canUploadFiles: this.canUploadFiles(),
uploadFile: this.uploadFile,
canUseSuper,
convertHTMLToSuper: this.convertHTMLToSuper,
convertMarkdownToSuper: this.convertMarkdownToSuper,
readFileAsText,
linkItems: this.linkItems,
cleanupItems: this.cleanupItems,
})
successful.push(...result.successful)
errored.push(...result.errored)
break
}
return []
}
async importFromTransferPayloads(payloads: DecryptedTransferPayload[]) {
const insertedItems = await Promise.all(
payloads.map(async (payload) => {
const content = payload.content as NoteContent
const note = this.items.createTemplateItem(
payload.content_type,
{
text: content.text,
title: content.title,
noteType: content.noteType,
editorIdentifier: content.editorIdentifier,
references: content.references,
},
{
created_at: payload.created_at,
updated_at: payload.updated_at,
uuid: payload.uuid,
},
)
return this.mutator.insertItem(note)
}),
)
return insertedItems
}
async uploadAndReplaceInlineFilesInInsertedItems(insertedItems: DecryptedItemInterface<ItemContent>[]) {
for (const item of insertedItems) {
if (!isNote(item)) {
continue
}
if (item.noteType !== NoteType.Super) {
continue
}
try {
const text = await this.superConverterService.uploadAndReplaceInlineFilesInSuperString(
item.text,
async (file) => await this.filesController.uploadNewFile(file, { showToast: true, note: item }),
async (file) => await this.linkingController.linkItems(item, file),
this._generateUuid,
)
await this.mutator.changeItem<NoteMutator>(item, (mutator) => {
mutator.text = text
})
} catch (error) {
console.error(error)
}
return {
successful,
errored,
}
}
}

23
packages/ui-services/src/Import/PlaintextConverter/PlaintextConverter.ts
View File

@@ -20,7 +20,7 @@ export class PlaintextConverter implements Converter {
return file.type === 'text/plain' || file.type === 'text/markdown'
}
convert: Converter['convert'] = async (file, { createNote, convertMarkdownToSuper, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote, convertMarkdownToSuper, readFileAsText }) => {
const content = await readFileAsText(file)
const { name } = parseFileName(file.name)
@@ -28,14 +28,17 @@ export class PlaintextConverter implements Converter {
const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
return [
createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: convertMarkdownToSuper(content),
useSuperIfPossible: true,
}),
]
const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: convertMarkdownToSuper(content),
useSuperIfPossible: true,
})
return {
successful: [note],
errored: [],
}
}
}

12
packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.spec.ts
View File

@@ -1,11 +1,11 @@
import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
import { SNNote } from '@standardnotes/models'
import { SimplenoteConverter } from './SimplenoteConverter'
import data from './testData'
import { ContentType } from '@standardnotes/domain-core'
import { CreateNoteFn } from '../Converter'
import { InsertNoteFn } from '../Converter'
describe('SimplenoteConverter', () => {
const createNote: CreateNoteFn = ({ title, text, trashed, createdAt, updatedAt }) =>
const createNote: InsertNoteFn = async ({ title, text, trashed, createdAt, updatedAt }) =>
({
uuid: Math.random().toString(),
created_at: createdAt,
@@ -17,12 +17,12 @@ describe('SimplenoteConverter', () => {
trashed,
references: [],
},
}) as unknown as DecryptedTransferPayload<NoteContent>
}) as unknown as SNNote
it('should parse', () => {
it('should parse', async () => {
const converter = new SimplenoteConverter()
const result = converter.parse(data, createNote)
const result = await converter.parse(data, createNote)
expect(result).not.toBeNull()
expect(result?.length).toBe(3)

23
packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.ts
View File

@@ -1,4 +1,4 @@
import { Converter, CreateNoteFn } from '../Converter'
import { Converter, InsertNoteFn } from '../Converter'
type SimplenoteItem = {
creationDate: string
@@ -38,19 +38,22 @@ export class SimplenoteConverter implements Converter {
return false
}
convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote: createNote, readFileAsText }) => {
const content = await readFileAsText(file)
const notes = this.parse(content, createNote)
const notes = await this.parse(content, createNote)
if (!notes) {
throw new Error('Could not parse notes')
}
return notes
return {
successful: notes,
errored: [],
}
}
createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: CreateNoteFn): ReturnType<CreateNoteFn> {
createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: InsertNoteFn): ReturnType<InsertNoteFn> {
const createdAtDate = new Date(item.creationDate)
const updatedAtDate = new Date(item.lastModified)
@@ -70,11 +73,15 @@ export class SimplenoteConverter implements Converter {
})
}
parse(data: string, createNote: CreateNoteFn) {
async parse(data: string, createNote: InsertNoteFn) {
try {
const parsed = JSON.parse(data) as SimplenoteData
const activeNotes = parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote))
const trashedNotes = parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote))
const activeNotes = await Promise.all(
parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote)),
)
const trashedNotes = await Promise.all(
parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote)),
)
return [...activeNotes, ...trashedNotes]
} catch (error) {

28
packages/ui-services/src/Import/SuperConverter/SuperConverter.ts
View File

@@ -1,6 +1,7 @@
import { SuperConverterServiceInterface } from '@standardnotes/files'
import { parseFileName } from '@standardnotes/filepicker'
import { Converter } from '../Converter'
import { ConversionResult } from '../ConversionResult'
export class SuperConverter implements Converter {
constructor(private converterService: SuperConverterServiceInterface) {}
@@ -17,9 +18,14 @@ export class SuperConverter implements Converter {
return this.converterService.isValidSuperString(content)
}
convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
const content = await readFileAsText(file)
const result: ConversionResult = {
successful: [],
errored: [],
}
if (!this.converterService.isValidSuperString(content)) {
throw new Error('Content is not valid Super JSON')
}
@@ -29,14 +35,16 @@ export class SuperConverter implements Converter {
const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
return [
createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: content,
useSuperIfPossible: true,
}),
]
const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: content,
useSuperIfPossible: true,
})
result.successful.push(note)
return result
}
}

1
packages/ui-services/src/Import/index.ts
View File

@@ -5,3 +5,4 @@ export * from './EvernoteConverter/EvernoteConverter'
export * from './PlaintextConverter/PlaintextConverter'
export * from './Utils'
export * from './Importer'
export * from './ConversionResult'