refactor: handle larger files in importer (#2692)

This commit is contained in:
Aman Harwara
2023-12-11 16:30:31 +05:30
committed by GitHub
parent 63e69b5e4b
commit 82d5a36932
22 changed files with 614 additions and 513 deletions

View File

@@ -1,15 +1,6 @@
import { FileItem } from '@standardnotes/models'
import { GenerateUuid } from '@standardnotes/services'
export interface SuperConverterServiceInterface {
isValidSuperString(superString: string): boolean
convertSuperStringToOtherFormat: (superString: string, toFormat: 'txt' | 'md' | 'html' | 'json') => Promise<string>
convertOtherFormatToSuperString: (otherFormatString: string, fromFormat: 'txt' | 'md' | 'html' | 'json') => string
getEmbeddedFileIDsFromSuperString(superString: string): string[]
uploadAndReplaceInlineFilesInSuperString(
superString: string,
uploadFile: (file: File) => Promise<FileItem | undefined>,
linkFile: (file: FileItem) => Promise<void>,
generateUuid: GenerateUuid,
): Promise<string>
}

View File

@@ -1,5 +1,6 @@
import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
import { Converter } from '../Converter'
import { ConversionResult } from '../ConversionResult'
type AegisData = {
db: {
@@ -45,7 +46,7 @@ export class AegisToAuthenticatorConverter implements Converter {
return false
}
convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
const content = await readFileAsText(file)
const entries = this.parseEntries(content)
@@ -59,17 +60,22 @@ export class AegisToAuthenticatorConverter implements Converter {
const title = file.name.split('.')[0]
const text = JSON.stringify(entries)
return [
createNote({
createdAt,
updatedAt,
title,
text,
noteType: NoteType.Authentication,
editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
useSuperIfPossible: false,
}),
]
const note = await insertNote({
createdAt,
updatedAt,
title,
text,
noteType: NoteType.Authentication,
editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
useSuperIfPossible: false,
})
const successful: ConversionResult['successful'] = [note]
return {
successful,
errored: [],
}
}
parseEntries(data: string): AuthenticatorEntry[] | null {

View File

@@ -0,0 +1,9 @@
import { DecryptedItemInterface } from '@standardnotes/models'
export type ConversionResult = {
successful: DecryptedItemInterface[]
errored: {
name: string
error: Error
}[]
}

View File

@@ -1,5 +1,6 @@
import { NoteType } from '@standardnotes/features'
import { DecryptedTransferPayload, ItemContent, NoteContent, TagContent } from '@standardnotes/models'
import { DecryptedItemInterface, FileItem, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models'
import { ConversionResult } from './ConversionResult'
export interface Converter {
getImportType(): string
@@ -12,17 +13,24 @@ export interface Converter {
convert(
file: File,
dependencies: {
createNote: CreateNoteFn
createTag: CreateTagFn
insertNote: InsertNoteFn
insertTag: InsertTagFn
canUploadFiles: boolean
uploadFile: UploadFileFn
canUseSuper: boolean
convertHTMLToSuper: (html: string) => string
convertMarkdownToSuper: (markdown: string) => string
readFileAsText: (file: File) => Promise<string>
linkItems(
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
): Promise<void>
cleanupItems(items: DecryptedItemInterface<ItemContent>[]): Promise<void>
},
): Promise<DecryptedTransferPayload<ItemContent>[]>
): Promise<ConversionResult>
}
export type CreateNoteFn = (options: {
export type InsertNoteFn = (options: {
createdAt: Date
updatedAt: Date
title: string
@@ -33,10 +41,20 @@ export type CreateNoteFn = (options: {
trashed?: boolean
editorIdentifier?: NoteContent['editorIdentifier']
useSuperIfPossible: boolean
}) => DecryptedTransferPayload<NoteContent>
}) => Promise<SNNote>
export type CreateTagFn = (options: {
export type InsertTagFn = (options: {
createdAt: Date
updatedAt: Date
title: string
}) => DecryptedTransferPayload<TagContent>
references: SNTag['references']
}) => Promise<SNTag>
export type UploadFileFn = (file: File) => Promise<FileItem | undefined>
export type LinkItemsFn = (
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
) => Promise<void>
export type CleanupItemsFn = (items: DecryptedItemInterface<ItemContent>[]) => Promise<void>

View File

@@ -3,9 +3,9 @@
*/
import { ContentType } from '@standardnotes/domain-core'
import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
import { SNNote, SNTag } from '@standardnotes/models'
import { EvernoteConverter, EvernoteResource } from './EvernoteConverter'
import { createTestResourceElement, enex, enexWithNoNoteOrTag } from './testData'
import { createTestResourceElement, enex } from './testData'
import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
import { GenerateUuid } from '@standardnotes/services'
import { Converter } from '../Converter'
@@ -33,77 +33,80 @@ describe('EvernoteConverter', () => {
const readFileAsText = async (file: File) => file as unknown as string
const dependencies: Parameters<Converter['convert']>[1] = {
createNote: ({ text }) =>
insertNote: async ({ text }) =>
({
content_type: ContentType.TYPES.Note,
content: {
text,
references: [],
},
}) as unknown as DecryptedTransferPayload<NoteContent>,
createTag: ({ title }) =>
uuid: generateUuid.execute().getValue(),
}) as unknown as SNNote,
insertTag: async ({ title }) =>
({
content_type: ContentType.TYPES.Tag,
content: {
title,
references: [],
},
}) as unknown as DecryptedTransferPayload<TagContent>,
uuid: generateUuid.execute().getValue(),
}) as unknown as SNTag,
convertHTMLToSuper: (data) => data,
convertMarkdownToSuper: jest.fn(),
readFileAsText,
canUseSuper: false,
canUploadFiles: false,
uploadFile: async () => void 0,
linkItems: async (item, itemToLink) => {
itemToLink.content.references.push({
content_type: item.content_type,
uuid: item.uuid,
})
},
cleanupItems: async () => void 0,
}
it('should throw error if no note or tag in enex', () => {
const converter = new EvernoteConverter(generateUuid)
expect(converter.convert(enexWithNoNoteOrTag as unknown as File, dependencies)).rejects.toThrowError()
})
it('should parse and strip html', async () => {
const converter = new EvernoteConverter(generateUuid)
const result = await converter.convert(enex as unknown as File, dependencies)
const { successful } = await converter.convert(enex as unknown as File, dependencies)
expect(result).not.toBeNull()
expect(result?.length).toBe(3)
expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe('This is a test.\nh e ')
expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
'Lorem ipsum dolor sit amet, consectetur adipiscing elit.',
)
expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
expect(successful).not.toBeNull()
expect(successful?.length).toBe(3)
expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[0] as SNNote).content.text).toBe('This is a test.\nh e ')
expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
expect((successful?.[1] as SNTag).content.references.length).toBe(2)
expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[2] as SNNote).content.text).toBe('Lorem ipsum dolor sit amet, consectetur adipiscing elit.')
})
it('should parse and not strip html', async () => {
const converter = new EvernoteConverter(generateUuid)
const result = await converter.convert(enex as unknown as File, {
const { successful } = await converter.convert(enex as unknown as File, {
...dependencies,
canUseSuper: true,
})
expect(result).not.toBeNull()
expect(result?.length).toBe(3)
expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
expect(successful).not.toBeNull()
expect(successful?.length).toBe(3)
expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[0] as SNNote).content.text).toBe(
'<p>This is a test.</p><ul></ul><ol></ol><font><span>h </span><span>e </span></font>',
)
expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
expect((successful?.[1] as SNTag).content.references.length).toBe(2)
expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
expect((successful?.[2] as SNNote).content.text).toBe(
'<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>',
)
expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
})
it('should convert lists to super format if applicable', () => {
@@ -129,7 +132,7 @@ describe('EvernoteConverter', () => {
expect(unorderedList2.getAttribute('__lexicallisttype')).toBeFalsy()
})
it('should replace media elements with resources', () => {
it('should replace media elements with resources', async () => {
const resources: EvernoteResource[] = [
{
hash: 'hash1',
@@ -152,9 +155,14 @@ describe('EvernoteConverter', () => {
const array = [mediaElement1, mediaElement2, mediaElement3]
const converter = new EvernoteConverter(generateUuid)
const replacedCount = converter.replaceMediaElementsWithResources(array, resources)
const { replacedElements } = await converter.replaceMediaElementsWithResources(
array,
resources,
false,
dependencies.uploadFile,
)
expect(replacedCount).toBe(1)
expect(replacedElements.length).toBe(1)
})
describe('getResourceFromElement', () => {

View File

@@ -1,11 +1,12 @@
import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
import { FileItem, SNTag } from '@standardnotes/models'
import dayjs from 'dayjs'
import customParseFormat from 'dayjs/plugin/customParseFormat'
import utc from 'dayjs/plugin/utc'
import { GenerateUuid } from '@standardnotes/services'
import MD5 from 'crypto-js/md5'
import Base64 from 'crypto-js/enc-base64'
import { Converter } from '../Converter'
import { Converter, UploadFileFn } from '../Converter'
import { ConversionResult } from '../ConversionResult'
dayjs.extend(customParseFormat)
dayjs.extend(utc)
@@ -35,14 +36,23 @@ export class EvernoteConverter implements Converter {
convert: Converter['convert'] = async (
file,
{ createNote, createTag, canUseSuper, convertHTMLToSuper, readFileAsText },
{
insertNote,
insertTag,
linkItems,
canUploadFiles,
canUseSuper,
convertHTMLToSuper,
readFileAsText,
uploadFile,
cleanupItems,
},
) => {
const content = await readFileAsText(file)
const xmlDoc = this.loadXMLString(content, 'xml')
const xmlNotes = xmlDoc.getElementsByTagName('note')
const notes: DecryptedTransferPayload<NoteContent>[] = []
const tags: DecryptedTransferPayload<TagContent>[] = []
const tags: SNTag[] = []
function findTag(title: string | null) {
return tags.filter(function (tag) {
@@ -50,94 +60,119 @@ export class EvernoteConverter implements Converter {
})[0]
}
const successful: ConversionResult['successful'] = []
const errored: ConversionResult['errored'] = []
for (const [index, xmlNote] of Array.from(xmlNotes).entries()) {
const title = xmlNote.getElementsByTagName('title')[0].textContent
const created = xmlNote.getElementsByTagName('created')[0]?.textContent
const updatedNodes = xmlNote.getElementsByTagName('updated')
const updated = updatedNodes.length ? updatedNodes[0].textContent : null
const resources = Array.from(xmlNote.getElementsByTagName('resource'))
.map(this.getResourceFromElement)
.filter(Boolean) as EvernoteResource[]
const filesToPotentiallyCleanup: FileItem[] = []
try {
const title = xmlNote.getElementsByTagName('title')[0].textContent
const created = xmlNote.getElementsByTagName('created')[0]?.textContent
const updatedNodes = xmlNote.getElementsByTagName('updated')
const updated = updatedNodes.length ? updatedNodes[0].textContent : null
const resources = Array.from(xmlNote.getElementsByTagName('resource'))
.map(this.getResourceFromElement)
.filter(Boolean) as EvernoteResource[]
const contentNode = xmlNote.getElementsByTagName('content')[0]
const contentXmlString = this.getXmlStringFromContentElement(contentNode)
if (!contentXmlString) {
continue
}
const contentXml = this.loadXMLString(contentXmlString, 'html')
const contentNode = xmlNote.getElementsByTagName('content')[0]
const contentXmlString = this.getXmlStringFromContentElement(contentNode)
if (!contentXmlString) {
continue
}
const contentXml = this.loadXMLString(contentXmlString, 'html')
const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
if (canUseSuper) {
this.convertTopLevelDivsToParagraphs(noteElement)
this.convertListsToSuperFormatIfApplicable(unorderedLists)
this.convertLeftPaddingToSuperIndent(noteElement)
}
this.removeEmptyAndOrphanListElements(noteElement)
this.removeUnnecessaryTopLevelBreaks(noteElement)
const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
this.replaceMediaElementsWithResources(mediaElements, resources)
// Some notes have <font> tags that contain separate <span> tags with text
// which causes broken paragraphs in the note.
const fontElements = Array.from(noteElement.getElementsByTagName('font'))
for (const fontElement of fontElements) {
fontElement.childNodes.forEach((childNode) => {
childNode.textContent += ' '
})
fontElement.innerText = fontElement.textContent || ''
}
let contentHTML = noteElement.innerHTML
if (!canUseSuper) {
contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
contentHTML = contentHTML.trim()
}
const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)
const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate
const note = createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: !title ? `Imported note ${index + 1} from Evernote` : title,
text,
useSuperIfPossible: canUseSuper,
})
const xmlTags = xmlNote.getElementsByTagName('tag')
for (const tagXml of Array.from(xmlTags)) {
const tagName = tagXml.childNodes[0].nodeValue
let tag = findTag(tagName)
if (!tag) {
const now = new Date()
tag = createTag({
createdAt: now,
updatedAt: now,
title: tagName || `Imported tag ${index + 1} from Evernote`,
})
tags.push(tag)
if (canUseSuper) {
this.convertTopLevelDivsToParagraphs(noteElement)
this.convertListsToSuperFormatIfApplicable(unorderedLists)
this.convertLeftPaddingToSuperIndent(noteElement)
}
note.content.references.push({ content_type: tag.content_type, uuid: tag.uuid })
tag.content.references.push({ content_type: note.content_type, uuid: note.uuid })
this.removeEmptyAndOrphanListElements(noteElement)
this.removeUnnecessaryTopLevelBreaks(noteElement)
const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
const { uploadedFiles } = await this.replaceMediaElementsWithResources(
mediaElements,
resources,
canUploadFiles,
uploadFile,
)
filesToPotentiallyCleanup.push(...uploadedFiles)
// Some notes have <font> tags that contain separate <span> tags with text
// which causes broken paragraphs in the note.
const fontElements = Array.from(noteElement.getElementsByTagName('font'))
for (const fontElement of fontElements) {
fontElement.childNodes.forEach((childNode) => {
childNode.textContent += ' '
})
fontElement.innerText = fontElement.textContent || ''
}
let contentHTML = noteElement.innerHTML
if (!canUseSuper) {
contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
contentHTML = contentHTML.trim()
}
const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)
const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate
const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: !title ? `Imported note ${index + 1} from Evernote` : title,
text,
useSuperIfPossible: canUseSuper,
})
successful.push(note)
for (const uploadedFile of uploadedFiles) {
await linkItems(note, uploadedFile)
successful.push(uploadedFile)
}
const xmlTags = xmlNote.getElementsByTagName('tag')
for (const tagXml of Array.from(xmlTags)) {
const tagName = tagXml.childNodes[0].nodeValue
let tag = findTag(tagName)
if (!tag) {
const now = new Date()
tag = await insertTag({
createdAt: now,
updatedAt: now,
title: tagName || `Imported tag ${index + 1} from Evernote`,
references: [],
})
tags.push(tag)
successful.push(tag)
}
await linkItems(note, tag)
}
} catch (error) {
console.error(error)
errored.push({
name: xmlNote.getElementsByTagName('title')?.[0]?.textContent || `${file.name} - Note #${index}`,
error: error as Error,
})
cleanupItems(filesToPotentiallyCleanup).catch(console.error)
continue
}
notes.push(note)
}
const allItems: DecryptedTransferPayload[] = [...notes, ...tags]
if (allItems.length === 0) {
throw new Error('Could not parse any notes or tags from Evernote file.')
return {
successful,
errored,
}
return allItems
}
getXmlStringFromContentElement(contentElement: Element) {
@@ -259,44 +294,83 @@ export class EvernoteConverter implements Converter {
})
}
replaceMediaElementsWithResources(mediaElements: Element[], resources: EvernoteResource[]): number {
let replacedElements = 0
getHTMLElementFromResource(resource: EvernoteResource) {
let resourceElement: HTMLElement = document.createElement('object')
resourceElement.setAttribute('type', resource.mimeType)
resourceElement.setAttribute('data', resource.data)
if (resource.mimeType.startsWith('image/')) {
resourceElement = document.createElement('img')
resourceElement.setAttribute('src', resource.data)
resourceElement.setAttribute('data-mime-type', resource.mimeType)
} else if (resource.mimeType.startsWith('audio/')) {
resourceElement = document.createElement('audio')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
} else if (resource.mimeType.startsWith('video/')) {
resourceElement = document.createElement('video')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
}
resourceElement.setAttribute('data-filename', resource.fileName)
return resourceElement
}
async getFileFromResource(resource: EvernoteResource): Promise<File> {
const response = await fetch(resource.data)
const blob = await response.blob()
return new File([blob], resource.fileName, { type: resource.mimeType })
}
async replaceMediaElementsWithResources(
mediaElements: Element[],
resources: EvernoteResource[],
canUploadFiles: boolean,
uploadFile: UploadFileFn,
): Promise<{
replacedElements: HTMLElement[]
uploadedFiles: FileItem[]
}> {
const replacedElements: HTMLElement[] = []
const uploadedFiles = new Map<EvernoteResource['hash'], FileItem>()
for (const mediaElement of mediaElements) {
const hash = mediaElement.getAttribute('hash')
const resource = resources.find((resource) => resource && resource.hash === hash)
if (!resource) {
continue
}
let resourceElement: HTMLElement = document.createElement('object')
resourceElement.setAttribute('type', resource.mimeType)
resourceElement.setAttribute('data', resource.data)
if (resource.mimeType.startsWith('image/')) {
resourceElement = document.createElement('img')
resourceElement.setAttribute('src', resource.data)
resourceElement.setAttribute('data-mime-type', resource.mimeType)
} else if (resource.mimeType.startsWith('audio/')) {
resourceElement = document.createElement('audio')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
} else if (resource.mimeType.startsWith('video/')) {
resourceElement = document.createElement('video')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
}
resourceElement.setAttribute('data-filename', resource.fileName)
if (!mediaElement.parentNode) {
continue
}
const existingFile = uploadedFiles.get(resource.hash)
const fileItem = canUploadFiles
? existingFile
? existingFile
: await uploadFile(await this.getFileFromResource(resource))
: undefined
if (fileItem) {
const fileElement = document.createElement('div')
fileElement.setAttribute('data-lexical-file-uuid', fileItem.uuid)
mediaElement.parentNode.replaceChild(fileElement, mediaElement)
replacedElements.push(fileElement)
if (!existingFile) {
uploadedFiles.set(resource.hash, fileItem)
}
continue
}
const resourceElement = this.getHTMLElementFromResource(resource)
mediaElement.parentNode.replaceChild(resourceElement, mediaElement)
replacedElements++
replacedElements.push(resourceElement)
}
return {
replacedElements,
uploadedFiles: Array.from(uploadedFiles.values()),
}
return replacedElements
}
loadXMLString(string: string, type: 'html' | 'xml') {

View File

@@ -36,11 +36,6 @@ export const enex = `<?xml version="1.0" encoding="UTF-8"?>
</note>
</en-export>`
export const enexWithNoNoteOrTag = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export3.dtd">
<en-export export-date="20210408T052957Z" application="Evernote" version="10.8.5">
</en-export>`
export function createTestResourceElement(
shouldHaveMimeType = true,
shouldHaveSourceUrl = false,

View File

@@ -4,11 +4,11 @@
import { jsonTextContentData, htmlTestData, jsonListContentData } from './testData'
import { GoogleKeepConverter } from './GoogleKeepConverter'
import { ContentType, DecryptedTransferPayload, NoteContent } from '@standardnotes/snjs'
import { CreateNoteFn } from '../Converter'
import { ContentType, SNNote } from '@standardnotes/snjs'
import { InsertNoteFn } from '../Converter'
describe('GoogleKeepConverter', () => {
const createNote: CreateNoteFn = ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) =>
const insertNote: InsertNoteFn = async ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) =>
({
uuid: Math.random().toString(),
created_at: createdAt,
@@ -22,12 +22,12 @@ describe('GoogleKeepConverter', () => {
pinned,
references: [],
},
}) as unknown as DecryptedTransferPayload<NoteContent>
}) as unknown as SNNote
it('should parse json data', () => {
it('should parse json data', async () => {
const converter = new GoogleKeepConverter()
const textContent = converter.tryParseAsJson(jsonTextContentData, createNote, (md) => md)
const textContent = await converter.tryParseAsJson(jsonTextContentData, insertNote, (md) => md)
expect(textContent).not.toBeNull()
expect(textContent?.created_at).toBeInstanceOf(Date)
@@ -40,7 +40,7 @@ describe('GoogleKeepConverter', () => {
expect(textContent?.content.archived).toBe(false)
expect(textContent?.content.pinned).toBe(false)
const listContent = converter.tryParseAsJson(jsonListContentData, createNote, (md) => md)
const listContent = await converter.tryParseAsJson(jsonListContentData, insertNote, (md) => md)
expect(listContent).not.toBeNull()
expect(listContent?.created_at).toBeInstanceOf(Date)
@@ -54,15 +54,15 @@ describe('GoogleKeepConverter', () => {
expect(textContent?.content.pinned).toBe(false)
})
it('should parse html data', () => {
it('should parse html data', async () => {
const converter = new GoogleKeepConverter()
const result = converter.tryParseAsHtml(
const result = await converter.tryParseAsHtml(
htmlTestData,
{
name: 'note-2.html',
},
createNote,
insertNote,
(html) => html,
false,
)

View File

@@ -1,5 +1,5 @@
import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
import { Converter, CreateNoteFn } from '../Converter'
import { SNNote } from '@standardnotes/models'
import { Converter, InsertNoteFn } from '../Converter'
type Content =
| {
@@ -45,32 +45,31 @@ export class GoogleKeepConverter implements Converter {
convert: Converter['convert'] = async (
file,
{ createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText },
{ insertNote: createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText },
) => {
const content = await readFileAsText(file)
const possiblePayloadFromJson = this.tryParseAsJson(content, createNote, convertMarkdownToSuper)
const note =
(await this.tryParseAsJson(content, createNote, convertMarkdownToSuper)) ||
(await this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper))
if (possiblePayloadFromJson) {
return [possiblePayloadFromJson]
}
const possiblePayloadFromHtml = this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper)
if (possiblePayloadFromHtml) {
return [possiblePayloadFromHtml]
if (note) {
return {
successful: [note],
errored: [],
}
}
throw new Error('Could not parse Google Keep backup file')
}
tryParseAsHtml(
async tryParseAsHtml(
data: string,
file: { name: string },
createNote: CreateNoteFn,
insertNote: InsertNoteFn,
convertHTMLToSuper: (html: string) => string,
canUseSuper: boolean,
): DecryptedTransferPayload<NoteContent> {
): Promise<SNNote> {
const rootElement = document.createElement('html')
rootElement.innerHTML = data
@@ -119,7 +118,7 @@ export class GoogleKeepConverter implements Converter {
const title = rootElement.getElementsByClassName('title')[0]?.textContent || file.name
return createNote({
return await insertNote({
createdAt: date,
updatedAt: date,
title: title,
@@ -150,11 +149,11 @@ export class GoogleKeepConverter implements Converter {
)
}
tryParseAsJson(
async tryParseAsJson(
data: string,
createNote: CreateNoteFn,
createNote: InsertNoteFn,
convertMarkdownToSuper: (md: string) => string,
): DecryptedTransferPayload<NoteContent> | null {
): Promise<SNNote | null> {
try {
const parsed = JSON.parse(data) as GoogleKeepJsonNote
if (!GoogleKeepConverter.isValidGoogleKeepJson(parsed)) {
@@ -172,7 +171,7 @@ export class GoogleKeepConverter implements Converter {
.join('\n')
}
text = convertMarkdownToSuper(text)
return createNote({
return await createNote({
createdAt: date,
updatedAt: date,
title: parsed.title,

View File

@@ -16,7 +16,7 @@ export class HTMLConverter implements Converter {
return true
}
convert: Converter['convert'] = async (file, { createNote, convertHTMLToSuper, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote: createNote, convertHTMLToSuper, readFileAsText }) => {
const content = await readFileAsText(file)
const { name } = parseFileName(file.name)
@@ -26,14 +26,17 @@ export class HTMLConverter implements Converter {
const text = convertHTMLToSuper(content)
return [
createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text,
useSuperIfPossible: true,
}),
]
const note = await createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text,
useSuperIfPossible: true,
})
return {
successful: [note],
errored: [],
}
}
}

View File

@@ -15,20 +15,24 @@ import { SimplenoteConverter } from './SimplenoteConverter/SimplenoteConverter'
import { readFileAsText } from './Utils'
import {
DecryptedItemInterface,
DecryptedTransferPayload,
FileItem,
ItemContent,
NoteContent,
NoteMutator,
SNNote,
isNote,
SNTag,
TagContent,
isFile,
} from '@standardnotes/models'
import { HTMLConverter } from './HTMLConverter/HTMLConverter'
import { SuperConverter } from './SuperConverter/SuperConverter'
import { Converter, CreateNoteFn, CreateTagFn } from './Converter'
import { SuperConverterServiceInterface } from '@standardnotes/files'
import { CleanupItemsFn, Converter, InsertNoteFn, InsertTagFn, LinkItemsFn, UploadFileFn } from './Converter'
import { ConversionResult } from './ConversionResult'
import { FilesClientInterface, SuperConverterServiceInterface } from '@standardnotes/files'
import { ContentType } from '@standardnotes/domain-core'
const BytesInOneMegabyte = 1_000_000
const NoteSizeThreshold = 3 * BytesInOneMegabyte
export class Importer {
converters: Set<Converter> = new Set()
@@ -50,9 +54,11 @@ export class Importer {
linkItems(
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
sync: boolean,
): Promise<void>
},
private _generateUuid: GenerateUuid,
private files: FilesClientInterface,
) {
this.registerNativeConverters()
}
@@ -88,19 +94,19 @@ export class Importer {
return null
}
createNote: CreateNoteFn = ({
insertNote: InsertNoteFn = async ({
createdAt,
updatedAt,
title,
text,
noteType,
editorIdentifier,
trashed,
archived,
pinned,
trashed = false,
archived = false,
pinned = false,
useSuperIfPossible,
}) => {
if (noteType === NoteType.Super && !this.isEntitledToSuper()) {
if (noteType === NoteType.Super && !this.canUseSuper()) {
noteType = undefined
}
@@ -112,16 +118,17 @@ export class Importer {
editorIdentifier = undefined
}
const shouldUseSuper = useSuperIfPossible && this.isEntitledToSuper()
const shouldUseSuper = useSuperIfPossible && this.canUseSuper()
return {
created_at: createdAt,
created_at_timestamp: createdAt.getTime(),
updated_at: updatedAt,
updated_at_timestamp: updatedAt.getTime(),
uuid: this._generateUuid.execute().getValue(),
content_type: ContentType.TYPES.Note,
content: {
const noteSize = new Blob([text]).size
if (noteSize > NoteSizeThreshold) {
throw new Error('Note is too large to import')
}
const note = this.items.createTemplateItem<NoteContent, SNNote>(
ContentType.TYPES.Note,
{
title,
text,
references: [],
@@ -131,27 +138,68 @@ export class Importer {
pinned,
editorIdentifier: shouldUseSuper ? NativeFeatureIdentifier.TYPES.SuperEditor : editorIdentifier,
},
}
{
created_at: createdAt,
updated_at: updatedAt,
},
)
return await this.mutator.insertItem(note)
}
createTag: CreateTagFn = ({ createdAt, updatedAt, title }) => {
return {
uuid: this._generateUuid.execute().getValue(),
content_type: ContentType.TYPES.Tag,
created_at: createdAt,
created_at_timestamp: createdAt.getTime(),
updated_at: updatedAt,
updated_at_timestamp: updatedAt.getTime(),
content: {
title: title,
insertTag: InsertTagFn = async ({ createdAt, updatedAt, title, references }) => {
const tag = this.items.createTemplateItem<TagContent, SNTag>(
ContentType.TYPES.Tag,
{
title,
expanded: false,
iconString: '',
references: [],
references,
},
{
created_at: createdAt,
updated_at: updatedAt,
},
)
return await this.mutator.insertItem(tag)
}
canUploadFiles = (): boolean => {
const status = this.features.getFeatureStatus(
NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.Files).getValue(),
)
return status === FeatureStatus.Entitled
}
uploadFile: UploadFileFn = async (file) => {
if (!this.canUploadFiles()) {
return undefined
}
try {
return await this.filesController.uploadNewFile(file, { showToast: true })
} catch (error) {
console.error(error)
return undefined
}
}
isEntitledToSuper = (): boolean => {
linkItems: LinkItemsFn = async (item, itemToLink) => {
await this.linkingController.linkItems(item, itemToLink, false)
}
cleanupItems: CleanupItemsFn = async (items) => {
for (const item of items) {
if (isFile(item)) {
await this.files.deleteFile(item)
}
await this.mutator.deleteItems([item])
}
}
canUseSuper = (): boolean => {
return (
this.features.getFeatureStatus(
NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.SuperEditor).getValue(),
@@ -160,7 +208,7 @@ export class Importer {
}
convertHTMLToSuper = (html: string): string => {
if (!this.isEntitledToSuper()) {
if (!this.canUseSuper()) {
return html
}
@@ -168,20 +216,23 @@ export class Importer {
}
convertMarkdownToSuper = (markdown: string): string => {
if (!this.isEntitledToSuper()) {
if (!this.canUseSuper()) {
return markdown
}
return this.superConverterService.convertOtherFormatToSuperString(markdown, 'md')
}
async getPayloadsFromFile(file: File, type: string): Promise<DecryptedTransferPayload[]> {
const isEntitledToSuper = this.isEntitledToSuper()
async importFromFile(file: File, type: string): Promise<ConversionResult> {
const canUseSuper = this.canUseSuper()
if (type === 'super' && !isEntitledToSuper) {
if (type === 'super' && !canUseSuper) {
throw new Error('Importing Super notes requires a subscription')
}
const successful: ConversionResult['successful'] = []
const errored: ConversionResult['errored'] = []
for (const converter of this.converters) {
const isCorrectType = converter.getImportType() === type
@@ -195,65 +246,28 @@ export class Importer {
throw new Error('Content is not valid')
}
return await converter.convert(file, {
createNote: this.createNote,
createTag: this.createTag,
canUseSuper: isEntitledToSuper,
const result = await converter.convert(file, {
insertNote: this.insertNote,
insertTag: this.insertTag,
canUploadFiles: this.canUploadFiles(),
uploadFile: this.uploadFile,
canUseSuper,
convertHTMLToSuper: this.convertHTMLToSuper,
convertMarkdownToSuper: this.convertMarkdownToSuper,
readFileAsText,
linkItems: this.linkItems,
cleanupItems: this.cleanupItems,
})
successful.push(...result.successful)
errored.push(...result.errored)
break
}
return []
}
async importFromTransferPayloads(payloads: DecryptedTransferPayload[]) {
const insertedItems = await Promise.all(
payloads.map(async (payload) => {
const content = payload.content as NoteContent
const note = this.items.createTemplateItem(
payload.content_type,
{
text: content.text,
title: content.title,
noteType: content.noteType,
editorIdentifier: content.editorIdentifier,
references: content.references,
},
{
created_at: payload.created_at,
updated_at: payload.updated_at,
uuid: payload.uuid,
},
)
return this.mutator.insertItem(note)
}),
)
return insertedItems
}
async uploadAndReplaceInlineFilesInInsertedItems(insertedItems: DecryptedItemInterface<ItemContent>[]) {
for (const item of insertedItems) {
if (!isNote(item)) {
continue
}
if (item.noteType !== NoteType.Super) {
continue
}
try {
const text = await this.superConverterService.uploadAndReplaceInlineFilesInSuperString(
item.text,
async (file) => await this.filesController.uploadNewFile(file, { showToast: true, note: item }),
async (file) => await this.linkingController.linkItems(item, file),
this._generateUuid,
)
await this.mutator.changeItem<NoteMutator>(item, (mutator) => {
mutator.text = text
})
} catch (error) {
console.error(error)
}
return {
successful,
errored,
}
}
}

View File

@@ -20,7 +20,7 @@ export class PlaintextConverter implements Converter {
return file.type === 'text/plain' || file.type === 'text/markdown'
}
convert: Converter['convert'] = async (file, { createNote, convertMarkdownToSuper, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote, convertMarkdownToSuper, readFileAsText }) => {
const content = await readFileAsText(file)
const { name } = parseFileName(file.name)
@@ -28,14 +28,17 @@ export class PlaintextConverter implements Converter {
const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
return [
createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: convertMarkdownToSuper(content),
useSuperIfPossible: true,
}),
]
const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: convertMarkdownToSuper(content),
useSuperIfPossible: true,
})
return {
successful: [note],
errored: [],
}
}
}

View File

@@ -1,11 +1,11 @@
import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models'
import { SNNote } from '@standardnotes/models'
import { SimplenoteConverter } from './SimplenoteConverter'
import data from './testData'
import { ContentType } from '@standardnotes/domain-core'
import { CreateNoteFn } from '../Converter'
import { InsertNoteFn } from '../Converter'
describe('SimplenoteConverter', () => {
const createNote: CreateNoteFn = ({ title, text, trashed, createdAt, updatedAt }) =>
const createNote: InsertNoteFn = async ({ title, text, trashed, createdAt, updatedAt }) =>
({
uuid: Math.random().toString(),
created_at: createdAt,
@@ -17,12 +17,12 @@ describe('SimplenoteConverter', () => {
trashed,
references: [],
},
}) as unknown as DecryptedTransferPayload<NoteContent>
}) as unknown as SNNote
it('should parse', () => {
it('should parse', async () => {
const converter = new SimplenoteConverter()
const result = converter.parse(data, createNote)
const result = await converter.parse(data, createNote)
expect(result).not.toBeNull()
expect(result?.length).toBe(3)

View File

@@ -1,4 +1,4 @@
import { Converter, CreateNoteFn } from '../Converter'
import { Converter, InsertNoteFn } from '../Converter'
type SimplenoteItem = {
creationDate: string
@@ -38,19 +38,22 @@ export class SimplenoteConverter implements Converter {
return false
}
convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote: createNote, readFileAsText }) => {
const content = await readFileAsText(file)
const notes = this.parse(content, createNote)
const notes = await this.parse(content, createNote)
if (!notes) {
throw new Error('Could not parse notes')
}
return notes
return {
successful: notes,
errored: [],
}
}
createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: CreateNoteFn): ReturnType<CreateNoteFn> {
createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: InsertNoteFn): ReturnType<InsertNoteFn> {
const createdAtDate = new Date(item.creationDate)
const updatedAtDate = new Date(item.lastModified)
@@ -70,11 +73,15 @@ export class SimplenoteConverter implements Converter {
})
}
parse(data: string, createNote: CreateNoteFn) {
async parse(data: string, createNote: InsertNoteFn) {
try {
const parsed = JSON.parse(data) as SimplenoteData
const activeNotes = parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote))
const trashedNotes = parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote))
const activeNotes = await Promise.all(
parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote)),
)
const trashedNotes = await Promise.all(
parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote)),
)
return [...activeNotes, ...trashedNotes]
} catch (error) {

View File

@@ -1,6 +1,7 @@
import { SuperConverterServiceInterface } from '@standardnotes/files'
import { parseFileName } from '@standardnotes/filepicker'
import { Converter } from '../Converter'
import { ConversionResult } from '../ConversionResult'
export class SuperConverter implements Converter {
constructor(private converterService: SuperConverterServiceInterface) {}
@@ -17,9 +18,14 @@ export class SuperConverter implements Converter {
return this.converterService.isValidSuperString(content)
}
convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
const content = await readFileAsText(file)
const result: ConversionResult = {
successful: [],
errored: [],
}
if (!this.converterService.isValidSuperString(content)) {
throw new Error('Content is not valid Super JSON')
}
@@ -29,14 +35,16 @@ export class SuperConverter implements Converter {
const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date()
return [
createNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: content,
useSuperIfPossible: true,
}),
]
const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: name,
text: content,
useSuperIfPossible: true,
})
result.successful.push(note)
return result
}
}

View File

@@ -5,3 +5,4 @@ export * from './EvernoteConverter/EvernoteConverter'
export * from './PlaintextConverter/PlaintextConverter'
export * from './Utils'
export * from './Importer'
export * from './ConversionResult'

View File

@@ -68,6 +68,7 @@ export class WebDependencies extends DependencyContainer {
this.get<FilesController>(Web_TYPES.FilesController),
this.get<LinkingController>(Web_TYPES.LinkingController),
application.generateUuid,
application.files,
)
})

View File

@@ -34,9 +34,9 @@ const ImportModal = ({ importModalController }: { importModalController: ImportM
close,
} = importModalController
const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'ready')
const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'pending' && file.service)
const importSuccessOrError =
files.length > 0 && files.every((file) => file.status === 'success' || file.status === 'error')
files.length > 0 && files.every((file) => file.status === 'finished' || file.status === 'error')
const modalActions: ModalAction[] = useMemo(
() => [

View File

@@ -1,15 +1,14 @@
import { DecryptedTransferPayload, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
import { DecryptedItemInterface, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
import {
ContentType,
InternalEventBusInterface,
ItemManagerInterface,
MutatorClientInterface,
pluralize,
PreferenceServiceInterface,
PreferencesServiceEvent,
UuidGenerator,
} from '@standardnotes/snjs'
import { Importer } from '@standardnotes/ui-services'
import { ConversionResult, Importer } from '@standardnotes/ui-services'
import { action, makeObservable, observable, runInAction } from 'mobx'
import { NavigationController } from '../../Controllers/Navigation/NavigationController'
import { LinkingController } from '@/Controllers/LinkingController'
@@ -23,11 +22,10 @@ type ImportModalFileCommon = {
export type ImportModalFile = (
| { status: 'pending' }
| { status: 'ready'; payloads?: DecryptedTransferPayload[] }
| { status: 'parsing' }
| { status: 'importing' }
| { status: 'uploading-files' }
| { status: 'success'; successMessage: string }
| ({ status: 'finished' } & ConversionResult)
| { status: 'error'; error: Error }
) &
ImportModalFileCommon
@@ -112,7 +110,7 @@ export class ImportModalController extends AbstractViewController {
id: UuidGenerator.GenerateUuid(),
file,
service,
status: service ? 'ready' : 'pending',
status: 'pending',
} as ImportModalFile
}
@@ -149,70 +147,30 @@ export class ImportModalController extends AbstractViewController {
this.setImportTag(undefined)
}
importFromPayloads = async (file: ImportModalFile, payloads: DecryptedTransferPayload[]) => {
this.updateFile({
...file,
status: 'importing',
})
try {
const insertedItems = await this.importer.importFromTransferPayloads(payloads)
this.updateFile({
...file,
status: 'uploading-files',
})
await this.importer.uploadAndReplaceInlineFilesInInsertedItems(insertedItems)
const notesImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
const tagsImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
const successMessage =
`Successfully imported ${notesImported.length} ` +
pluralize(notesImported.length, 'note', 'notes') +
(tagsImported.length > 0 ? ` and ${tagsImported.length} ${pluralize(tagsImported.length, 'tag', 'tags')}` : '')
this.updateFile({
...file,
status: 'success',
successMessage,
})
} catch (error) {
this.updateFile({
...file,
status: 'error',
error: error instanceof Error ? error : new Error('Could not import file'),
})
console.error(error)
}
}
parseAndImport = async () => {
if (this.files.length === 0) {
return
}
const importedPayloads: DecryptedTransferPayload[] = []
const importedItems: DecryptedItemInterface[] = []
for (const file of this.files) {
if (!file.service) {
return
}
if (file.status === 'ready' && file.payloads) {
await this.importFromPayloads(file, file.payloads)
importedPayloads.push(...file.payloads)
continue
}
this.updateFile({
...file,
status: 'parsing',
})
try {
const payloads = await this.importer.getPayloadsFromFile(file.file, file.service)
await this.importFromPayloads(file, payloads)
importedPayloads.push(...payloads)
const { successful, errored } = await this.importer.importFromFile(file.file, file.service)
importedItems.push(...successful)
this.updateFile({
...file,
status: 'finished',
successful,
errored,
})
} catch (error) {
this.updateFile({
...file,
@@ -222,7 +180,7 @@ export class ImportModalController extends AbstractViewController {
console.error(error)
}
}
if (!importedPayloads.length) {
if (!importedItems.length) {
return
}
if (this.addImportsToTag) {
@@ -233,7 +191,7 @@ export class ImportModalController extends AbstractViewController {
title: `Imported on ${currentDate.toLocaleString()}`,
expanded: false,
iconString: '',
references: importedPayloads
references: importedItems
.filter((payload) => payload.content_type === ContentType.TYPES.Note)
.map((payload) => ({
content_type: ContentType.TYPES.Note,
@@ -245,11 +203,11 @@ export class ImportModalController extends AbstractViewController {
try {
const latestExistingTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)
await Promise.all(
importedPayloads
importedItems
.filter((payload) => payload.content_type === ContentType.TYPES.Note)
.map(async (payload) => {
const note = this.items.findSureItem<SNNote>(payload.uuid)
await this.linkingController.addTagToItem(latestExistingTag, note)
await this.linkingController.addTagToItem(latestExistingTag, note, false)
}),
)
importTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)

View File

@@ -1,9 +1,10 @@
import { ImportModalController, ImportModalFile } from '@/Components/ImportModal/ImportModalController'
import { classNames, ContentType, pluralize } from '@standardnotes/snjs'
import { Importer } from '@standardnotes/ui-services'
import { ConversionResult, Importer } from '@standardnotes/ui-services'
import { observer } from 'mobx-react-lite'
import { useCallback, useEffect, useState } from 'react'
import Icon from '../Icon/Icon'
import { Disclosure, DisclosureContent, DisclosureProvider } from '@ariakit/react'
const NoteImportTypeColors: Record<string, string> = {
evernote: 'bg-[#14cc45] text-[#000]',
@@ -25,6 +26,75 @@ const NoteImportTypeIcons: Record<string, string> = {
super: 'file-doc',
}
const countSuccessfulItemsByGroup = (successful: ConversionResult['successful']) => {
let notes = 0
let tags = 0
let files = 0
for (const item of successful) {
if (item.content_type === ContentType.TYPES.Note) {
notes++
} else if (item.content_type === ContentType.TYPES.Tag) {
tags++
} else if (item.content_type === ContentType.TYPES.File) {
files++
}
}
return {
notes,
tags,
files,
}
}
const ImportErroredAccordion = ({ errored }: { errored: ConversionResult['errored'] }) => {
return (
<DisclosureProvider>
<Disclosure>
<div className="flex items-center gap-1">
<Icon type="warning" className="flex-shrink-0 text-danger" size="small" />
Could not import {errored.length} {pluralize(errored.length, 'item', 'items')} (click for details)
</div>
</Disclosure>
<DisclosureContent className="w-full overflow-hidden pl-5">
{errored.map((item, index) => (
<div className="flex w-full items-center gap-1 overflow-hidden" key={index}>
<span>{index + 1}.</span>
<span className="overflow-hidden text-ellipsis whitespace-nowrap font-semibold">{item.name}:</span>
<span className="overflow-hidden text-ellipsis whitespace-nowrap">{item.error.message}</span>
</div>
))}
</DisclosureContent>
</DisclosureProvider>
)
}
const ImportFinishedStatus = ({ file }: { file: ImportModalFile }) => {
if (file.status !== 'finished') {
return null
}
const { notes, tags, files } = countSuccessfulItemsByGroup(file.successful)
const notesStatus = notes > 0 ? `${notes} ${pluralize(notes, 'note', 'notes')}` : ''
const tagsStatus = tags > 0 ? `${tags} ${pluralize(tags, 'tag', 'tags')}` : ''
const filesStatus = files > 0 ? `${files} ${pluralize(files, 'file', 'files')}` : ''
const status = [notesStatus, tagsStatus, filesStatus].filter(Boolean).join(', ')
return (
<>
{file.successful.length > 0 && (
<div className="flex items-center gap-1">
<Icon type="check-circle-filled" className="flex-shrink-0 text-success" size="small" />
<span>{status} imported</span>
</div>
)}
{file.errored.length > 0 && <ImportErroredAccordion errored={file.errored} />}
</>
)
}
const ImportModalFileItem = ({
file,
updateFile,
@@ -36,6 +106,7 @@ const ImportModalFileItem = ({
removeFile: ImportModalController['removeFile']
importer: Importer
}) => {
const [isDetectingService, setIsDetectingService] = useState(false)
const [changingService, setChangingService] = useState(false)
const setFileService = useCallback(
@@ -46,7 +117,7 @@ const ImportModalFileItem = ({
updateFile({
...file,
service,
status: service ? 'ready' : 'pending',
status: 'pending',
})
},
[file, updateFile],
@@ -54,59 +125,47 @@ const ImportModalFileItem = ({
useEffect(() => {
const detect = async () => {
setIsDetectingService(true)
const detectedService = await importer.detectService(file.file)
void setFileService(detectedService)
setIsDetectingService(false)
}
if (file.service === undefined) {
void detect()
}
}, [file, importer, setFileService])
const notePayloads =
file.status === 'ready' && file.payloads
? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
: []
const tagPayloads =
file.status === 'ready' && file.payloads
? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
: []
const payloadsImportMessage =
`Ready to import ${notePayloads.length} ` +
pluralize(notePayloads.length, 'note', 'notes') +
(tagPayloads.length > 0 ? ` and ${tagPayloads.length} ${pluralize(tagPayloads.length, 'tag', 'tags')}` : '')
return (
<div
className={classNames(
'flex gap-2 px-2 py-2.5',
'flex gap-2 overflow-hidden px-2 py-2.5',
file.service == null ? 'flex-col items-start md:flex-row md:items-center' : 'items-center',
)}
>
<div className="mr-auto flex items-center">
<div className="mr-auto flex w-full items-center">
{file.service && (
<div className={classNames('mr-4 rounded p-2', NoteImportTypeColors[file.service])}>
<Icon type={NoteImportTypeIcons[file.service]} size="medium" />
</div>
)}
<div className="flex flex-col">
<div className="flex w-full flex-col overflow-hidden">
<div>{file.file.name}</div>
<div className="line-clamp-3 text-xs opacity-75">
{file.status === 'ready'
? notePayloads.length > 1 || tagPayloads.length
? payloadsImportMessage
: 'Ready to import'
: null}
{file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
{file.status === 'parsing' && 'Parsing...'}
{file.status === 'importing' && 'Importing...'}
{file.status === 'uploading-files' && 'Uploading and embedding files...'}
{file.status === 'error' && file.error.message}
{file.status === 'success' && file.successMessage}
</div>
{isDetectingService ? (
<div className="text-xs opacity-75">Detecting service...</div>
) : (
<div className={classNames(file.status !== 'finished' && 'line-clamp-3', 'w-full text-xs opacity-75')}>
{file.status === 'pending' && file.service && 'Ready to import'}
{file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
{file.status === 'parsing' && 'Parsing...'}
{file.status === 'importing' && 'Importing...'}
{file.status === 'uploading-files' && 'Uploading and embedding files...'}
{file.status === 'error' && file.error.message}
<ImportFinishedStatus file={file} />
</div>
)}
</div>
</div>
{(file.status === 'ready' || file.status === 'pending') && (
{file.status === 'pending' && (
<div className="flex items-center">
{changingService ? (
<>
@@ -172,7 +231,9 @@ const ImportModalFileItem = ({
</button>
</div>
)}
{file.status === 'success' && <Icon type="check-circle-filled" className="flex-shrink-0 text-success" />}
{file.status === 'finished' && file.successful.length > 0 && file.errored.length === 0 && (
<Icon type="check-circle-filled" className="flex-shrink-0 text-success" />
)}
{file.status === 'error' && <Icon type="warning" className="flex-shrink-0 text-danger" />}
</div>
)

View File

@@ -1,6 +1,6 @@
import { createHeadlessEditor } from '@lexical/headless'
import { $convertToMarkdownString } from '@lexical/markdown'
import { FileItem, GenerateUuid, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
import { FileItem, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
import {
$createParagraphNode,
$getRoot,
@@ -16,9 +16,7 @@ import { MarkdownTransformers } from '../MarkdownTransformers'
import { $generateHtmlFromNodes, $generateNodesFromDOM } from '@lexical/html'
import { FileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileNode'
import { $createFileExportNode } from '../Lexical/Nodes/FileExportNode'
import { $createInlineFileNode, $isInlineFileNode, InlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
import { $createFileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileUtils'
import { RemoteImageNode } from '../Plugins/RemoteImagePlugin/RemoteImageNode'
import { $createInlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
import { $convertFromMarkdownString } from '../Lexical/Utils/MarkdownImport'
export class HeadlessSuperConverter implements SuperConverterServiceInterface {
private importEditor: LexicalEditor
@@ -194,7 +192,8 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
type === 'link' ||
type === 'linebreak' ||
type === 'unencrypted-image' ||
type === 'inline-file'
type === 'inline-file' ||
type === 'snfile'
) {
const paragraphNode = $createParagraphNode()
paragraphNode.append(node)
@@ -232,7 +231,7 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
}
if (didThrow) {
throw new Error('Could not import note')
throw new Error('Could not import note. Check error console for details.')
}
return JSON.stringify(this.importEditor.getEditorState())
@@ -256,62 +255,4 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
return ids
}
async uploadAndReplaceInlineFilesInSuperString(
superString: string,
uploadFile: (file: File) => Promise<FileItem | undefined>,
linkFile: (file: FileItem) => Promise<void>,
generateUuid: GenerateUuid,
): Promise<string> {
if (superString.length === 0) {
return superString
}
this.importEditor.setEditorState(this.importEditor.parseEditorState(superString))
await new Promise<void>((resolve) => {
this.importEditor.update(
() => {
const inlineFileNodes = $nodesOfType(InlineFileNode)
const remoteImageNodes = $nodesOfType(RemoteImageNode).filter((node) => node.__src.startsWith('data:'))
const concatenatedNodes = [...inlineFileNodes, ...remoteImageNodes]
if (concatenatedNodes.length === 0) {
resolve()
return
}
;(async () => {
for (const node of concatenatedNodes) {
const blob = await fetch(node.__src).then((response) => response.blob())
const name = $isInlineFileNode(node) ? node.__fileName : node.__alt
const mimeType = $isInlineFileNode(node) ? node.__mimeType : node.__src.split(';')[0].split(':')[1]
const file = new File([blob], name || generateUuid.execute().getValue(), {
type: mimeType,
})
const uploadedFile = await uploadFile(file)
if (!uploadedFile) {
return
}
this.importEditor.update(
() => {
const fileNode = $createFileNode(uploadedFile.uuid)
node.replace(fileNode)
},
{ discrete: true },
)
await linkFile(uploadedFile)
}
})()
.then(() => resolve())
.catch(console.error)
},
{ discrete: true },
)
})
return JSON.stringify(this.importEditor.getEditorState())
}
}

View File

@@ -204,7 +204,7 @@ export class LinkingController extends AbstractViewController implements Interna
}
}
linkItems = async (item: LinkableItem, itemToLink: LinkableItem) => {
linkItems = async (item: LinkableItem, itemToLink: LinkableItem, sync = true) => {
const linkNoteAndFile = async (note: SNNote, file: FileItem) => {
const updatedFile = await this.mutator.associateFileWithNote(file, note)
@@ -231,11 +231,11 @@ export class LinkingController extends AbstractViewController implements Interna
}
const linkTagToNote = async (tag: SNTag, note: SNNote) => {
await this.addTagToItem(tag, note)
await this.addTagToItem(tag, note, sync)
}
const linkTagToFile = async (tag: SNTag, file: FileItem) => {
await this.addTagToItem(tag, file)
await this.addTagToItem(tag, file, sync)
}
if (isNote(item)) {
@@ -273,7 +273,9 @@ export class LinkingController extends AbstractViewController implements Interna
throw new Error('First item must be a note or file')
}
void this.sync.sync()
if (sync) {
void this.sync.sync()
}
}
linkItemToSelectedItem = async (itemToLink: LinkableItem): Promise<boolean> => {
@@ -323,13 +325,15 @@ export class LinkingController extends AbstractViewController implements Interna
return newTag
}
addTagToItem = async (tag: SNTag, item: FileItem | SNNote) => {
addTagToItem = async (tag: SNTag, item: FileItem | SNNote, sync = true) => {
if (item instanceof SNNote) {
await this.mutator.addTagToNote(item, tag, this.shouldLinkToParentFolders)
} else if (item instanceof FileItem) {
await this.mutator.addTagToFile(item, tag, this.shouldLinkToParentFolders)
}
this.sync.sync().catch(console.error)
if (sync) {
this.sync.sync().catch(console.error)
}
}
}