diff --git a/packages/files/src/Domain/Service/SuperConverterServiceInterface.ts b/packages/files/src/Domain/Service/SuperConverterServiceInterface.ts index ef49b550f..8c9203505 100644 --- a/packages/files/src/Domain/Service/SuperConverterServiceInterface.ts +++ b/packages/files/src/Domain/Service/SuperConverterServiceInterface.ts @@ -1,15 +1,6 @@ -import { FileItem } from '@standardnotes/models' -import { GenerateUuid } from '@standardnotes/services' - export interface SuperConverterServiceInterface { isValidSuperString(superString: string): boolean convertSuperStringToOtherFormat: (superString: string, toFormat: 'txt' | 'md' | 'html' | 'json') => Promise convertOtherFormatToSuperString: (otherFormatString: string, fromFormat: 'txt' | 'md' | 'html' | 'json') => string getEmbeddedFileIDsFromSuperString(superString: string): string[] - uploadAndReplaceInlineFilesInSuperString( - superString: string, - uploadFile: (file: File) => Promise, - linkFile: (file: FileItem) => Promise, - generateUuid: GenerateUuid, - ): Promise } diff --git a/packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts b/packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts index 844af9ee9..b794a5f0c 100644 --- a/packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts +++ b/packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts @@ -1,5 +1,6 @@ import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features' import { Converter } from '../Converter' +import { ConversionResult } from '../ConversionResult' type AegisData = { db: { @@ -45,7 +46,7 @@ export class AegisToAuthenticatorConverter implements Converter { return false } - convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => { + convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => { const content = await readFileAsText(file) const entries = this.parseEntries(content) @@ -59,17 +60,22 @@ export class AegisToAuthenticatorConverter implements Converter { const title = file.name.split('.')[0] const text = JSON.stringify(entries) - return [ - createNote({ - createdAt, - updatedAt, - title, - text, - noteType: NoteType.Authentication, - editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor, - useSuperIfPossible: false, - }), - ] + const note = await insertNote({ + createdAt, + updatedAt, + title, + text, + noteType: NoteType.Authentication, + editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor, + useSuperIfPossible: false, + }) + + const successful: ConversionResult['successful'] = [note] + + return { + successful, + errored: [], + } } parseEntries(data: string): AuthenticatorEntry[] | null { diff --git a/packages/ui-services/src/Import/ConversionResult.ts b/packages/ui-services/src/Import/ConversionResult.ts new file mode 100644 index 000000000..7ed88878c --- /dev/null +++ b/packages/ui-services/src/Import/ConversionResult.ts @@ -0,0 +1,9 @@ +import { DecryptedItemInterface } from '@standardnotes/models' + +export type ConversionResult = { + successful: DecryptedItemInterface[] + errored: { + name: string + error: Error + }[] +} diff --git a/packages/ui-services/src/Import/Converter.ts b/packages/ui-services/src/Import/Converter.ts index c4d780824..93bc702d8 100644 --- a/packages/ui-services/src/Import/Converter.ts +++ b/packages/ui-services/src/Import/Converter.ts @@ -1,5 +1,6 @@ import { NoteType } from '@standardnotes/features' -import { DecryptedTransferPayload, ItemContent, NoteContent, TagContent } from '@standardnotes/models' +import { DecryptedItemInterface, FileItem, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models' +import { ConversionResult } from './ConversionResult' export interface Converter { getImportType(): string @@ -12,17 +13,24 @@ export interface Converter { convert( file: File, dependencies: { - createNote: CreateNoteFn - createTag: CreateTagFn + insertNote: InsertNoteFn + insertTag: InsertTagFn + canUploadFiles: boolean + uploadFile: UploadFileFn canUseSuper: boolean convertHTMLToSuper: (html: string) => string convertMarkdownToSuper: (markdown: string) => string readFileAsText: (file: File) => Promise + linkItems( + item: DecryptedItemInterface, + itemToLink: DecryptedItemInterface, + ): Promise + cleanupItems(items: DecryptedItemInterface[]): Promise }, - ): Promise[]> + ): Promise } -export type CreateNoteFn = (options: { +export type InsertNoteFn = (options: { createdAt: Date updatedAt: Date title: string @@ -33,10 +41,20 @@ export type CreateNoteFn = (options: { trashed?: boolean editorIdentifier?: NoteContent['editorIdentifier'] useSuperIfPossible: boolean -}) => DecryptedTransferPayload +}) => Promise -export type CreateTagFn = (options: { +export type InsertTagFn = (options: { createdAt: Date updatedAt: Date title: string -}) => DecryptedTransferPayload + references: SNTag['references'] +}) => Promise + +export type UploadFileFn = (file: File) => Promise + +export type LinkItemsFn = ( + item: DecryptedItemInterface, + itemToLink: DecryptedItemInterface, +) => Promise + +export type CleanupItemsFn = (items: DecryptedItemInterface[]) => Promise diff --git a/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts b/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts index b8ad3d3f5..47d39f9e8 100644 --- a/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts +++ b/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts @@ -3,9 +3,9 @@ */ import { ContentType } from '@standardnotes/domain-core' -import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models' +import { SNNote, SNTag } from '@standardnotes/models' import { EvernoteConverter, EvernoteResource } from './EvernoteConverter' -import { createTestResourceElement, enex, enexWithNoNoteOrTag } from './testData' +import { createTestResourceElement, enex } from './testData' import { PureCryptoInterface } from '@standardnotes/sncrypto-common' import { GenerateUuid } from '@standardnotes/services' import { Converter } from '../Converter' @@ -33,77 +33,80 @@ describe('EvernoteConverter', () => { const readFileAsText = async (file: File) => file as unknown as string const dependencies: Parameters[1] = { - createNote: ({ text }) => + insertNote: async ({ text }) => ({ content_type: ContentType.TYPES.Note, content: { text, references: [], }, - }) as unknown as DecryptedTransferPayload, - createTag: ({ title }) => + uuid: generateUuid.execute().getValue(), + }) as unknown as SNNote, + insertTag: async ({ title }) => ({ content_type: ContentType.TYPES.Tag, content: { title, references: [], }, - }) as unknown as DecryptedTransferPayload, + uuid: generateUuid.execute().getValue(), + }) as unknown as SNTag, convertHTMLToSuper: (data) => data, convertMarkdownToSuper: jest.fn(), readFileAsText, canUseSuper: false, + canUploadFiles: false, + uploadFile: async () => void 0, + linkItems: async (item, itemToLink) => { + itemToLink.content.references.push({ + content_type: item.content_type, + uuid: item.uuid, + }) + }, + cleanupItems: async () => void 0, } - it('should throw error if no note or tag in enex', () => { - const converter = new EvernoteConverter(generateUuid) - - expect(converter.convert(enexWithNoNoteOrTag as unknown as File, dependencies)).rejects.toThrowError() - }) - it('should parse and strip html', async () => { const converter = new EvernoteConverter(generateUuid) - const result = await converter.convert(enex as unknown as File, dependencies) + const { successful } = await converter.convert(enex as unknown as File, dependencies) - expect(result).not.toBeNull() - expect(result?.length).toBe(3) - expect(result?.[0].content_type).toBe(ContentType.TYPES.Note) - expect((result?.[0] as DecryptedTransferPayload).content.text).toBe('This is a test.\nh e ') - expect(result?.[1].content_type).toBe(ContentType.TYPES.Note) - expect((result?.[1] as DecryptedTransferPayload).content.text).toBe( - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', - ) - expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag) - expect((result?.[2] as DecryptedTransferPayload).content.title).toBe('distant reading') - expect((result?.[2] as DecryptedTransferPayload).content.references.length).toBe(2) - expect((result?.[2] as DecryptedTransferPayload).content.references[0].uuid).toBe(result?.[0].uuid) - expect((result?.[2] as DecryptedTransferPayload).content.references[1].uuid).toBe(result?.[1].uuid) + expect(successful).not.toBeNull() + expect(successful?.length).toBe(3) + expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note) + expect((successful?.[0] as SNNote).content.text).toBe('This is a test.\nh e ') + expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag) + expect((successful?.[1] as SNTag).content.title).toBe('distant reading') + expect((successful?.[1] as SNTag).content.references.length).toBe(2) + expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid) + expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid) + expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note) + expect((successful?.[2] as SNNote).content.text).toBe('Lorem ipsum dolor sit amet, consectetur adipiscing elit.') }) it('should parse and not strip html', async () => { const converter = new EvernoteConverter(generateUuid) - const result = await converter.convert(enex as unknown as File, { + const { successful } = await converter.convert(enex as unknown as File, { ...dependencies, canUseSuper: true, }) - expect(result).not.toBeNull() - expect(result?.length).toBe(3) - expect(result?.[0].content_type).toBe(ContentType.TYPES.Note) - expect((result?.[0] as DecryptedTransferPayload).content.text).toBe( + expect(successful).not.toBeNull() + expect(successful?.length).toBe(3) + expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note) + expect((successful?.[0] as SNNote).content.text).toBe( '

This is a test.

      h e ', ) - expect(result?.[1].content_type).toBe(ContentType.TYPES.Note) - expect((result?.[1] as DecryptedTransferPayload).content.text).toBe( + expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag) + expect((successful?.[1] as SNTag).content.title).toBe('distant reading') + expect((successful?.[1] as SNTag).content.references.length).toBe(2) + expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid) + expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid) + expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note) + expect((successful?.[2] as SNNote).content.text).toBe( '

      Lorem ipsum dolor sit amet, consectetur adipiscing elit.

      ', ) - expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag) - expect((result?.[2] as DecryptedTransferPayload).content.title).toBe('distant reading') - expect((result?.[2] as DecryptedTransferPayload).content.references.length).toBe(2) - expect((result?.[2] as DecryptedTransferPayload).content.references[0].uuid).toBe(result?.[0].uuid) - expect((result?.[2] as DecryptedTransferPayload).content.references[1].uuid).toBe(result?.[1].uuid) }) it('should convert lists to super format if applicable', () => { @@ -129,7 +132,7 @@ describe('EvernoteConverter', () => { expect(unorderedList2.getAttribute('__lexicallisttype')).toBeFalsy() }) - it('should replace media elements with resources', () => { + it('should replace media elements with resources', async () => { const resources: EvernoteResource[] = [ { hash: 'hash1', @@ -152,9 +155,14 @@ describe('EvernoteConverter', () => { const array = [mediaElement1, mediaElement2, mediaElement3] const converter = new EvernoteConverter(generateUuid) - const replacedCount = converter.replaceMediaElementsWithResources(array, resources) + const { replacedElements } = await converter.replaceMediaElementsWithResources( + array, + resources, + false, + dependencies.uploadFile, + ) - expect(replacedCount).toBe(1) + expect(replacedElements.length).toBe(1) }) describe('getResourceFromElement', () => { diff --git a/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts b/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts index bb68e8fad..98733743d 100644 --- a/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts +++ b/packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts @@ -1,11 +1,12 @@ -import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models' +import { FileItem, SNTag } from '@standardnotes/models' import dayjs from 'dayjs' import customParseFormat from 'dayjs/plugin/customParseFormat' import utc from 'dayjs/plugin/utc' import { GenerateUuid } from '@standardnotes/services' import MD5 from 'crypto-js/md5' import Base64 from 'crypto-js/enc-base64' -import { Converter } from '../Converter' +import { Converter, UploadFileFn } from '../Converter' +import { ConversionResult } from '../ConversionResult' dayjs.extend(customParseFormat) dayjs.extend(utc) @@ -35,14 +36,23 @@ export class EvernoteConverter implements Converter { convert: Converter['convert'] = async ( file, - { createNote, createTag, canUseSuper, convertHTMLToSuper, readFileAsText }, + { + insertNote, + insertTag, + linkItems, + canUploadFiles, + canUseSuper, + convertHTMLToSuper, + readFileAsText, + uploadFile, + cleanupItems, + }, ) => { const content = await readFileAsText(file) const xmlDoc = this.loadXMLString(content, 'xml') const xmlNotes = xmlDoc.getElementsByTagName('note') - const notes: DecryptedTransferPayload[] = [] - const tags: DecryptedTransferPayload[] = [] + const tags: SNTag[] = [] function findTag(title: string | null) { return tags.filter(function (tag) { @@ -50,94 +60,119 @@ export class EvernoteConverter implements Converter { })[0] } + const successful: ConversionResult['successful'] = [] + const errored: ConversionResult['errored'] = [] + for (const [index, xmlNote] of Array.from(xmlNotes).entries()) { - const title = xmlNote.getElementsByTagName('title')[0].textContent - const created = xmlNote.getElementsByTagName('created')[0]?.textContent - const updatedNodes = xmlNote.getElementsByTagName('updated') - const updated = updatedNodes.length ? updatedNodes[0].textContent : null - const resources = Array.from(xmlNote.getElementsByTagName('resource')) - .map(this.getResourceFromElement) - .filter(Boolean) as EvernoteResource[] + const filesToPotentiallyCleanup: FileItem[] = [] + try { + const title = xmlNote.getElementsByTagName('title')[0].textContent + const created = xmlNote.getElementsByTagName('created')[0]?.textContent + const updatedNodes = xmlNote.getElementsByTagName('updated') + const updated = updatedNodes.length ? updatedNodes[0].textContent : null + const resources = Array.from(xmlNote.getElementsByTagName('resource')) + .map(this.getResourceFromElement) + .filter(Boolean) as EvernoteResource[] - const contentNode = xmlNote.getElementsByTagName('content')[0] - const contentXmlString = this.getXmlStringFromContentElement(contentNode) - if (!contentXmlString) { - continue - } - const contentXml = this.loadXMLString(contentXmlString, 'html') + const contentNode = xmlNote.getElementsByTagName('content')[0] + const contentXmlString = this.getXmlStringFromContentElement(contentNode) + if (!contentXmlString) { + continue + } + const contentXml = this.loadXMLString(contentXmlString, 'html') - const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement + const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement - const unorderedLists = Array.from(noteElement.getElementsByTagName('ul')) + const unorderedLists = Array.from(noteElement.getElementsByTagName('ul')) - if (canUseSuper) { - this.convertTopLevelDivsToParagraphs(noteElement) - this.convertListsToSuperFormatIfApplicable(unorderedLists) - this.convertLeftPaddingToSuperIndent(noteElement) - } - - this.removeEmptyAndOrphanListElements(noteElement) - this.removeUnnecessaryTopLevelBreaks(noteElement) - - const mediaElements = Array.from(noteElement.getElementsByTagName('en-media')) - this.replaceMediaElementsWithResources(mediaElements, resources) - - // Some notes have tags that contain separate tags with text - // which causes broken paragraphs in the note. - const fontElements = Array.from(noteElement.getElementsByTagName('font')) - for (const fontElement of fontElements) { - fontElement.childNodes.forEach((childNode) => { - childNode.textContent += ' ' - }) - fontElement.innerText = fontElement.textContent || '' - } - - let contentHTML = noteElement.innerHTML - if (!canUseSuper) { - contentHTML = contentHTML.replace(/<\/div>/g, '\n') - contentHTML = contentHTML.replace(/]*>/g, '\n') - contentHTML = contentHTML.trim() - } - const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML) - - const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date() - const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate - - const note = createNote({ - createdAt: createdAtDate, - updatedAt: updatedAtDate, - title: !title ? `Imported note ${index + 1} from Evernote` : title, - text, - useSuperIfPossible: canUseSuper, - }) - - const xmlTags = xmlNote.getElementsByTagName('tag') - for (const tagXml of Array.from(xmlTags)) { - const tagName = tagXml.childNodes[0].nodeValue - let tag = findTag(tagName) - if (!tag) { - const now = new Date() - tag = createTag({ - createdAt: now, - updatedAt: now, - title: tagName || `Imported tag ${index + 1} from Evernote`, - }) - tags.push(tag) + if (canUseSuper) { + this.convertTopLevelDivsToParagraphs(noteElement) + this.convertListsToSuperFormatIfApplicable(unorderedLists) + this.convertLeftPaddingToSuperIndent(noteElement) } - note.content.references.push({ content_type: tag.content_type, uuid: tag.uuid }) - tag.content.references.push({ content_type: note.content_type, uuid: note.uuid }) + this.removeEmptyAndOrphanListElements(noteElement) + this.removeUnnecessaryTopLevelBreaks(noteElement) + + const mediaElements = Array.from(noteElement.getElementsByTagName('en-media')) + const { uploadedFiles } = await this.replaceMediaElementsWithResources( + mediaElements, + resources, + canUploadFiles, + uploadFile, + ) + filesToPotentiallyCleanup.push(...uploadedFiles) + + // Some notes have tags that contain separate tags with text + // which causes broken paragraphs in the note. + const fontElements = Array.from(noteElement.getElementsByTagName('font')) + for (const fontElement of fontElements) { + fontElement.childNodes.forEach((childNode) => { + childNode.textContent += ' ' + }) + fontElement.innerText = fontElement.textContent || '' + } + + let contentHTML = noteElement.innerHTML + if (!canUseSuper) { + contentHTML = contentHTML.replace(/<\/div>/g, '\n') + contentHTML = contentHTML.replace(/]*>/g, '\n') + contentHTML = contentHTML.trim() + } + const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML) + + const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date() + const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate + + const note = await insertNote({ + createdAt: createdAtDate, + updatedAt: updatedAtDate, + title: !title ? `Imported note ${index + 1} from Evernote` : title, + text, + useSuperIfPossible: canUseSuper, + }) + + successful.push(note) + + for (const uploadedFile of uploadedFiles) { + await linkItems(note, uploadedFile) + successful.push(uploadedFile) + } + + const xmlTags = xmlNote.getElementsByTagName('tag') + for (const tagXml of Array.from(xmlTags)) { + const tagName = tagXml.childNodes[0].nodeValue + let tag = findTag(tagName) + + if (!tag) { + const now = new Date() + tag = await insertTag({ + createdAt: now, + updatedAt: now, + title: tagName || `Imported tag ${index + 1} from Evernote`, + references: [], + }) + tags.push(tag) + successful.push(tag) + } + + await linkItems(note, tag) + } + } catch (error) { + console.error(error) + errored.push({ + name: xmlNote.getElementsByTagName('title')?.[0]?.textContent || `${file.name} - Note #${index}`, + error: error as Error, + }) + cleanupItems(filesToPotentiallyCleanup).catch(console.error) + continue } - - notes.push(note) } - const allItems: DecryptedTransferPayload[] = [...notes, ...tags] - if (allItems.length === 0) { - throw new Error('Could not parse any notes or tags from Evernote file.') + return { + successful, + errored, } - - return allItems } getXmlStringFromContentElement(contentElement: Element) { @@ -259,44 +294,83 @@ export class EvernoteConverter implements Converter { }) } - replaceMediaElementsWithResources(mediaElements: Element[], resources: EvernoteResource[]): number { - let replacedElements = 0 + getHTMLElementFromResource(resource: EvernoteResource) { + let resourceElement: HTMLElement = document.createElement('object') + resourceElement.setAttribute('type', resource.mimeType) + resourceElement.setAttribute('data', resource.data) + if (resource.mimeType.startsWith('image/')) { + resourceElement = document.createElement('img') + resourceElement.setAttribute('src', resource.data) + resourceElement.setAttribute('data-mime-type', resource.mimeType) + } else if (resource.mimeType.startsWith('audio/')) { + resourceElement = document.createElement('audio') + resourceElement.setAttribute('controls', 'controls') + const sourceElement = document.createElement('source') + sourceElement.setAttribute('src', resource.data) + sourceElement.setAttribute('type', resource.mimeType) + resourceElement.appendChild(sourceElement) + } else if (resource.mimeType.startsWith('video/')) { + resourceElement = document.createElement('video') + resourceElement.setAttribute('controls', 'controls') + const sourceElement = document.createElement('source') + sourceElement.setAttribute('src', resource.data) + sourceElement.setAttribute('type', resource.mimeType) + resourceElement.appendChild(sourceElement) + } + resourceElement.setAttribute('data-filename', resource.fileName) + return resourceElement + } + + async getFileFromResource(resource: EvernoteResource): Promise { + const response = await fetch(resource.data) + const blob = await response.blob() + return new File([blob], resource.fileName, { type: resource.mimeType }) + } + + async replaceMediaElementsWithResources( + mediaElements: Element[], + resources: EvernoteResource[], + canUploadFiles: boolean, + uploadFile: UploadFileFn, + ): Promise<{ + replacedElements: HTMLElement[] + uploadedFiles: FileItem[] + }> { + const replacedElements: HTMLElement[] = [] + const uploadedFiles = new Map() for (const mediaElement of mediaElements) { const hash = mediaElement.getAttribute('hash') const resource = resources.find((resource) => resource && resource.hash === hash) if (!resource) { continue } - let resourceElement: HTMLElement = document.createElement('object') - resourceElement.setAttribute('type', resource.mimeType) - resourceElement.setAttribute('data', resource.data) - if (resource.mimeType.startsWith('image/')) { - resourceElement = document.createElement('img') - resourceElement.setAttribute('src', resource.data) - resourceElement.setAttribute('data-mime-type', resource.mimeType) - } else if (resource.mimeType.startsWith('audio/')) { - resourceElement = document.createElement('audio') - resourceElement.setAttribute('controls', 'controls') - const sourceElement = document.createElement('source') - sourceElement.setAttribute('src', resource.data) - sourceElement.setAttribute('type', resource.mimeType) - resourceElement.appendChild(sourceElement) - } else if (resource.mimeType.startsWith('video/')) { - resourceElement = document.createElement('video') - resourceElement.setAttribute('controls', 'controls') - const sourceElement = document.createElement('source') - sourceElement.setAttribute('src', resource.data) - sourceElement.setAttribute('type', resource.mimeType) - resourceElement.appendChild(sourceElement) - } - resourceElement.setAttribute('data-filename', resource.fileName) if (!mediaElement.parentNode) { continue } + const existingFile = uploadedFiles.get(resource.hash) + const fileItem = canUploadFiles + ? existingFile + ? existingFile + : await uploadFile(await this.getFileFromResource(resource)) + : undefined + if (fileItem) { + const fileElement = document.createElement('div') + fileElement.setAttribute('data-lexical-file-uuid', fileItem.uuid) + mediaElement.parentNode.replaceChild(fileElement, mediaElement) + replacedElements.push(fileElement) + if (!existingFile) { + uploadedFiles.set(resource.hash, fileItem) + } + continue + } + const resourceElement = this.getHTMLElementFromResource(resource) mediaElement.parentNode.replaceChild(resourceElement, mediaElement) - replacedElements++ + replacedElements.push(resourceElement) + } + return { + replacedElements, + uploadedFiles: Array.from(uploadedFiles.values()), } - return replacedElements } loadXMLString(string: string, type: 'html' | 'xml') { diff --git a/packages/ui-services/src/Import/EvernoteConverter/testData.ts b/packages/ui-services/src/Import/EvernoteConverter/testData.ts index 72b25635b..ffb141189 100644 --- a/packages/ui-services/src/Import/EvernoteConverter/testData.ts +++ b/packages/ui-services/src/Import/EvernoteConverter/testData.ts @@ -36,11 +36,6 @@ export const enex = ` ` -export const enexWithNoNoteOrTag = ` - - -` - export function createTestResourceElement( shouldHaveMimeType = true, shouldHaveSourceUrl = false, diff --git a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts index 0fc2a9468..0a1768d36 100644 --- a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts +++ b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.spec.ts @@ -4,11 +4,11 @@ import { jsonTextContentData, htmlTestData, jsonListContentData } from './testData' import { GoogleKeepConverter } from './GoogleKeepConverter' -import { ContentType, DecryptedTransferPayload, NoteContent } from '@standardnotes/snjs' -import { CreateNoteFn } from '../Converter' +import { ContentType, SNNote } from '@standardnotes/snjs' +import { InsertNoteFn } from '../Converter' describe('GoogleKeepConverter', () => { - const createNote: CreateNoteFn = ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) => + const insertNote: InsertNoteFn = async ({ title, text, createdAt, updatedAt, trashed, archived, pinned }) => ({ uuid: Math.random().toString(), created_at: createdAt, @@ -22,12 +22,12 @@ describe('GoogleKeepConverter', () => { pinned, references: [], }, - }) as unknown as DecryptedTransferPayload + }) as unknown as SNNote - it('should parse json data', () => { + it('should parse json data', async () => { const converter = new GoogleKeepConverter() - const textContent = converter.tryParseAsJson(jsonTextContentData, createNote, (md) => md) + const textContent = await converter.tryParseAsJson(jsonTextContentData, insertNote, (md) => md) expect(textContent).not.toBeNull() expect(textContent?.created_at).toBeInstanceOf(Date) @@ -40,7 +40,7 @@ describe('GoogleKeepConverter', () => { expect(textContent?.content.archived).toBe(false) expect(textContent?.content.pinned).toBe(false) - const listContent = converter.tryParseAsJson(jsonListContentData, createNote, (md) => md) + const listContent = await converter.tryParseAsJson(jsonListContentData, insertNote, (md) => md) expect(listContent).not.toBeNull() expect(listContent?.created_at).toBeInstanceOf(Date) @@ -54,15 +54,15 @@ describe('GoogleKeepConverter', () => { expect(textContent?.content.pinned).toBe(false) }) - it('should parse html data', () => { + it('should parse html data', async () => { const converter = new GoogleKeepConverter() - const result = converter.tryParseAsHtml( + const result = await converter.tryParseAsHtml( htmlTestData, { name: 'note-2.html', }, - createNote, + insertNote, (html) => html, false, ) diff --git a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts index 440de487e..339d8f167 100644 --- a/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts +++ b/packages/ui-services/src/Import/GoogleKeepConverter/GoogleKeepConverter.ts @@ -1,5 +1,5 @@ -import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models' -import { Converter, CreateNoteFn } from '../Converter' +import { SNNote } from '@standardnotes/models' +import { Converter, InsertNoteFn } from '../Converter' type Content = | { @@ -45,32 +45,31 @@ export class GoogleKeepConverter implements Converter { convert: Converter['convert'] = async ( file, - { createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText }, + { insertNote: createNote, canUseSuper, convertHTMLToSuper, convertMarkdownToSuper, readFileAsText }, ) => { const content = await readFileAsText(file) - const possiblePayloadFromJson = this.tryParseAsJson(content, createNote, convertMarkdownToSuper) + const note = + (await this.tryParseAsJson(content, createNote, convertMarkdownToSuper)) || + (await this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper)) - if (possiblePayloadFromJson) { - return [possiblePayloadFromJson] - } - - const possiblePayloadFromHtml = this.tryParseAsHtml(content, file, createNote, convertHTMLToSuper, canUseSuper) - - if (possiblePayloadFromHtml) { - return [possiblePayloadFromHtml] + if (note) { + return { + successful: [note], + errored: [], + } } throw new Error('Could not parse Google Keep backup file') } - tryParseAsHtml( + async tryParseAsHtml( data: string, file: { name: string }, - createNote: CreateNoteFn, + insertNote: InsertNoteFn, convertHTMLToSuper: (html: string) => string, canUseSuper: boolean, - ): DecryptedTransferPayload { + ): Promise { const rootElement = document.createElement('html') rootElement.innerHTML = data @@ -119,7 +118,7 @@ export class GoogleKeepConverter implements Converter { const title = rootElement.getElementsByClassName('title')[0]?.textContent || file.name - return createNote({ + return await insertNote({ createdAt: date, updatedAt: date, title: title, @@ -150,11 +149,11 @@ export class GoogleKeepConverter implements Converter { ) } - tryParseAsJson( + async tryParseAsJson( data: string, - createNote: CreateNoteFn, + createNote: InsertNoteFn, convertMarkdownToSuper: (md: string) => string, - ): DecryptedTransferPayload | null { + ): Promise { try { const parsed = JSON.parse(data) as GoogleKeepJsonNote if (!GoogleKeepConverter.isValidGoogleKeepJson(parsed)) { @@ -172,7 +171,7 @@ export class GoogleKeepConverter implements Converter { .join('\n') } text = convertMarkdownToSuper(text) - return createNote({ + return await createNote({ createdAt: date, updatedAt: date, title: parsed.title, diff --git a/packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts b/packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts index 8219c7d1f..5aa127078 100644 --- a/packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts +++ b/packages/ui-services/src/Import/HTMLConverter/HTMLConverter.ts @@ -16,7 +16,7 @@ export class HTMLConverter implements Converter { return true } - convert: Converter['convert'] = async (file, { createNote, convertHTMLToSuper, readFileAsText }) => { + convert: Converter['convert'] = async (file, { insertNote: createNote, convertHTMLToSuper, readFileAsText }) => { const content = await readFileAsText(file) const { name } = parseFileName(file.name) @@ -26,14 +26,17 @@ export class HTMLConverter implements Converter { const text = convertHTMLToSuper(content) - return [ - createNote({ - createdAt: createdAtDate, - updatedAt: updatedAtDate, - title: name, - text, - useSuperIfPossible: true, - }), - ] + const note = await createNote({ + createdAt: createdAtDate, + updatedAt: updatedAtDate, + title: name, + text, + useSuperIfPossible: true, + }) + + return { + successful: [note], + errored: [], + } } } diff --git a/packages/ui-services/src/Import/Importer.ts b/packages/ui-services/src/Import/Importer.ts index a4fc6c87f..0e645c864 100644 --- a/packages/ui-services/src/Import/Importer.ts +++ b/packages/ui-services/src/Import/Importer.ts @@ -15,20 +15,24 @@ import { SimplenoteConverter } from './SimplenoteConverter/SimplenoteConverter' import { readFileAsText } from './Utils' import { DecryptedItemInterface, - DecryptedTransferPayload, FileItem, ItemContent, NoteContent, - NoteMutator, SNNote, - isNote, + SNTag, + TagContent, + isFile, } from '@standardnotes/models' import { HTMLConverter } from './HTMLConverter/HTMLConverter' import { SuperConverter } from './SuperConverter/SuperConverter' -import { Converter, CreateNoteFn, CreateTagFn } from './Converter' -import { SuperConverterServiceInterface } from '@standardnotes/files' +import { CleanupItemsFn, Converter, InsertNoteFn, InsertTagFn, LinkItemsFn, UploadFileFn } from './Converter' +import { ConversionResult } from './ConversionResult' +import { FilesClientInterface, SuperConverterServiceInterface } from '@standardnotes/files' import { ContentType } from '@standardnotes/domain-core' +const BytesInOneMegabyte = 1_000_000 +const NoteSizeThreshold = 3 * BytesInOneMegabyte + export class Importer { converters: Set = new Set() @@ -50,9 +54,11 @@ export class Importer { linkItems( item: DecryptedItemInterface, itemToLink: DecryptedItemInterface, + sync: boolean, ): Promise }, private _generateUuid: GenerateUuid, + private files: FilesClientInterface, ) { this.registerNativeConverters() } @@ -88,19 +94,19 @@ export class Importer { return null } - createNote: CreateNoteFn = ({ + insertNote: InsertNoteFn = async ({ createdAt, updatedAt, title, text, noteType, editorIdentifier, - trashed, - archived, - pinned, + trashed = false, + archived = false, + pinned = false, useSuperIfPossible, }) => { - if (noteType === NoteType.Super && !this.isEntitledToSuper()) { + if (noteType === NoteType.Super && !this.canUseSuper()) { noteType = undefined } @@ -112,16 +118,17 @@ export class Importer { editorIdentifier = undefined } - const shouldUseSuper = useSuperIfPossible && this.isEntitledToSuper() + const shouldUseSuper = useSuperIfPossible && this.canUseSuper() - return { - created_at: createdAt, - created_at_timestamp: createdAt.getTime(), - updated_at: updatedAt, - updated_at_timestamp: updatedAt.getTime(), - uuid: this._generateUuid.execute().getValue(), - content_type: ContentType.TYPES.Note, - content: { + const noteSize = new Blob([text]).size + + if (noteSize > NoteSizeThreshold) { + throw new Error('Note is too large to import') + } + + const note = this.items.createTemplateItem( + ContentType.TYPES.Note, + { title, text, references: [], @@ -131,27 +138,68 @@ export class Importer { pinned, editorIdentifier: shouldUseSuper ? NativeFeatureIdentifier.TYPES.SuperEditor : editorIdentifier, }, - } + { + created_at: createdAt, + updated_at: updatedAt, + }, + ) + + return await this.mutator.insertItem(note) } - createTag: CreateTagFn = ({ createdAt, updatedAt, title }) => { - return { - uuid: this._generateUuid.execute().getValue(), - content_type: ContentType.TYPES.Tag, - created_at: createdAt, - created_at_timestamp: createdAt.getTime(), - updated_at: updatedAt, - updated_at_timestamp: updatedAt.getTime(), - content: { - title: title, + insertTag: InsertTagFn = async ({ createdAt, updatedAt, title, references }) => { + const tag = this.items.createTemplateItem( + ContentType.TYPES.Tag, + { + title, expanded: false, iconString: '', - references: [], + references, }, + { + created_at: createdAt, + updated_at: updatedAt, + }, + ) + + return await this.mutator.insertItem(tag) + } + + canUploadFiles = (): boolean => { + const status = this.features.getFeatureStatus( + NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.Files).getValue(), + ) + + return status === FeatureStatus.Entitled + } + + uploadFile: UploadFileFn = async (file) => { + if (!this.canUploadFiles()) { + return undefined + } + + try { + return await this.filesController.uploadNewFile(file, { showToast: true }) + } catch (error) { + console.error(error) + return undefined } } - isEntitledToSuper = (): boolean => { + linkItems: LinkItemsFn = async (item, itemToLink) => { + await this.linkingController.linkItems(item, itemToLink, false) + } + + cleanupItems: CleanupItemsFn = async (items) => { + for (const item of items) { + if (isFile(item)) { + await this.files.deleteFile(item) + } + await this.mutator.deleteItems([item]) + } + } + + canUseSuper = (): boolean => { return ( this.features.getFeatureStatus( NativeFeatureIdentifier.create(NativeFeatureIdentifier.TYPES.SuperEditor).getValue(), @@ -160,7 +208,7 @@ export class Importer { } convertHTMLToSuper = (html: string): string => { - if (!this.isEntitledToSuper()) { + if (!this.canUseSuper()) { return html } @@ -168,20 +216,23 @@ export class Importer { } convertMarkdownToSuper = (markdown: string): string => { - if (!this.isEntitledToSuper()) { + if (!this.canUseSuper()) { return markdown } return this.superConverterService.convertOtherFormatToSuperString(markdown, 'md') } - async getPayloadsFromFile(file: File, type: string): Promise { - const isEntitledToSuper = this.isEntitledToSuper() + async importFromFile(file: File, type: string): Promise { + const canUseSuper = this.canUseSuper() - if (type === 'super' && !isEntitledToSuper) { + if (type === 'super' && !canUseSuper) { throw new Error('Importing Super notes requires a subscription') } + const successful: ConversionResult['successful'] = [] + const errored: ConversionResult['errored'] = [] + for (const converter of this.converters) { const isCorrectType = converter.getImportType() === type @@ -195,65 +246,28 @@ export class Importer { throw new Error('Content is not valid') } - return await converter.convert(file, { - createNote: this.createNote, - createTag: this.createTag, - canUseSuper: isEntitledToSuper, + const result = await converter.convert(file, { + insertNote: this.insertNote, + insertTag: this.insertTag, + canUploadFiles: this.canUploadFiles(), + uploadFile: this.uploadFile, + canUseSuper, convertHTMLToSuper: this.convertHTMLToSuper, convertMarkdownToSuper: this.convertMarkdownToSuper, readFileAsText, + linkItems: this.linkItems, + cleanupItems: this.cleanupItems, }) + + successful.push(...result.successful) + errored.push(...result.errored) + + break } - return [] - } - - async importFromTransferPayloads(payloads: DecryptedTransferPayload[]) { - const insertedItems = await Promise.all( - payloads.map(async (payload) => { - const content = payload.content as NoteContent - const note = this.items.createTemplateItem( - payload.content_type, - { - text: content.text, - title: content.title, - noteType: content.noteType, - editorIdentifier: content.editorIdentifier, - references: content.references, - }, - { - created_at: payload.created_at, - updated_at: payload.updated_at, - uuid: payload.uuid, - }, - ) - return this.mutator.insertItem(note) - }), - ) - return insertedItems - } - - async uploadAndReplaceInlineFilesInInsertedItems(insertedItems: DecryptedItemInterface[]) { - for (const item of insertedItems) { - if (!isNote(item)) { - continue - } - if (item.noteType !== NoteType.Super) { - continue - } - try { - const text = await this.superConverterService.uploadAndReplaceInlineFilesInSuperString( - item.text, - async (file) => await this.filesController.uploadNewFile(file, { showToast: true, note: item }), - async (file) => await this.linkingController.linkItems(item, file), - this._generateUuid, - ) - await this.mutator.changeItem(item, (mutator) => { - mutator.text = text - }) - } catch (error) { - console.error(error) - } + return { + successful, + errored, } } } diff --git a/packages/ui-services/src/Import/PlaintextConverter/PlaintextConverter.ts b/packages/ui-services/src/Import/PlaintextConverter/PlaintextConverter.ts index 33e0bf27f..6b4097a26 100644 --- a/packages/ui-services/src/Import/PlaintextConverter/PlaintextConverter.ts +++ b/packages/ui-services/src/Import/PlaintextConverter/PlaintextConverter.ts @@ -20,7 +20,7 @@ export class PlaintextConverter implements Converter { return file.type === 'text/plain' || file.type === 'text/markdown' } - convert: Converter['convert'] = async (file, { createNote, convertMarkdownToSuper, readFileAsText }) => { + convert: Converter['convert'] = async (file, { insertNote, convertMarkdownToSuper, readFileAsText }) => { const content = await readFileAsText(file) const { name } = parseFileName(file.name) @@ -28,14 +28,17 @@ export class PlaintextConverter implements Converter { const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date() const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date() - return [ - createNote({ - createdAt: createdAtDate, - updatedAt: updatedAtDate, - title: name, - text: convertMarkdownToSuper(content), - useSuperIfPossible: true, - }), - ] + const note = await insertNote({ + createdAt: createdAtDate, + updatedAt: updatedAtDate, + title: name, + text: convertMarkdownToSuper(content), + useSuperIfPossible: true, + }) + + return { + successful: [note], + errored: [], + } } } diff --git a/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.spec.ts b/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.spec.ts index 0d6df8603..3ab5bb378 100644 --- a/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.spec.ts +++ b/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.spec.ts @@ -1,11 +1,11 @@ -import { DecryptedTransferPayload, NoteContent } from '@standardnotes/models' +import { SNNote } from '@standardnotes/models' import { SimplenoteConverter } from './SimplenoteConverter' import data from './testData' import { ContentType } from '@standardnotes/domain-core' -import { CreateNoteFn } from '../Converter' +import { InsertNoteFn } from '../Converter' describe('SimplenoteConverter', () => { - const createNote: CreateNoteFn = ({ title, text, trashed, createdAt, updatedAt }) => + const createNote: InsertNoteFn = async ({ title, text, trashed, createdAt, updatedAt }) => ({ uuid: Math.random().toString(), created_at: createdAt, @@ -17,12 +17,12 @@ describe('SimplenoteConverter', () => { trashed, references: [], }, - }) as unknown as DecryptedTransferPayload + }) as unknown as SNNote - it('should parse', () => { + it('should parse', async () => { const converter = new SimplenoteConverter() - const result = converter.parse(data, createNote) + const result = await converter.parse(data, createNote) expect(result).not.toBeNull() expect(result?.length).toBe(3) diff --git a/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.ts b/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.ts index 9a13b4387..a6144581c 100644 --- a/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.ts +++ b/packages/ui-services/src/Import/SimplenoteConverter/SimplenoteConverter.ts @@ -1,4 +1,4 @@ -import { Converter, CreateNoteFn } from '../Converter' +import { Converter, InsertNoteFn } from '../Converter' type SimplenoteItem = { creationDate: string @@ -38,19 +38,22 @@ export class SimplenoteConverter implements Converter { return false } - convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => { + convert: Converter['convert'] = async (file, { insertNote: createNote, readFileAsText }) => { const content = await readFileAsText(file) - const notes = this.parse(content, createNote) + const notes = await this.parse(content, createNote) if (!notes) { throw new Error('Could not parse notes') } - return notes + return { + successful: notes, + errored: [], + } } - createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: CreateNoteFn): ReturnType { + createNoteFromItem(item: SimplenoteItem, trashed: boolean, createNote: InsertNoteFn): ReturnType { const createdAtDate = new Date(item.creationDate) const updatedAtDate = new Date(item.lastModified) @@ -70,11 +73,15 @@ export class SimplenoteConverter implements Converter { }) } - parse(data: string, createNote: CreateNoteFn) { + async parse(data: string, createNote: InsertNoteFn) { try { const parsed = JSON.parse(data) as SimplenoteData - const activeNotes = parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote)) - const trashedNotes = parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote)) + const activeNotes = await Promise.all( + parsed.activeNotes.reverse().map((item) => this.createNoteFromItem(item, false, createNote)), + ) + const trashedNotes = await Promise.all( + parsed.trashedNotes.reverse().map((item) => this.createNoteFromItem(item, true, createNote)), + ) return [...activeNotes, ...trashedNotes] } catch (error) { diff --git a/packages/ui-services/src/Import/SuperConverter/SuperConverter.ts b/packages/ui-services/src/Import/SuperConverter/SuperConverter.ts index 5277d50be..c81b037c4 100644 --- a/packages/ui-services/src/Import/SuperConverter/SuperConverter.ts +++ b/packages/ui-services/src/Import/SuperConverter/SuperConverter.ts @@ -1,6 +1,7 @@ import { SuperConverterServiceInterface } from '@standardnotes/files' import { parseFileName } from '@standardnotes/filepicker' import { Converter } from '../Converter' +import { ConversionResult } from '../ConversionResult' export class SuperConverter implements Converter { constructor(private converterService: SuperConverterServiceInterface) {} @@ -17,9 +18,14 @@ export class SuperConverter implements Converter { return this.converterService.isValidSuperString(content) } - convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => { + convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => { const content = await readFileAsText(file) + const result: ConversionResult = { + successful: [], + errored: [], + } + if (!this.converterService.isValidSuperString(content)) { throw new Error('Content is not valid Super JSON') } @@ -29,14 +35,16 @@ export class SuperConverter implements Converter { const createdAtDate = file.lastModified ? new Date(file.lastModified) : new Date() const updatedAtDate = file.lastModified ? new Date(file.lastModified) : new Date() - return [ - createNote({ - createdAt: createdAtDate, - updatedAt: updatedAtDate, - title: name, - text: content, - useSuperIfPossible: true, - }), - ] + const note = await insertNote({ + createdAt: createdAtDate, + updatedAt: updatedAtDate, + title: name, + text: content, + useSuperIfPossible: true, + }) + + result.successful.push(note) + + return result } } diff --git a/packages/ui-services/src/Import/index.ts b/packages/ui-services/src/Import/index.ts index 514616fe6..91fc3f8c2 100644 --- a/packages/ui-services/src/Import/index.ts +++ b/packages/ui-services/src/Import/index.ts @@ -5,3 +5,4 @@ export * from './EvernoteConverter/EvernoteConverter' export * from './PlaintextConverter/PlaintextConverter' export * from './Utils' export * from './Importer' +export * from './ConversionResult' diff --git a/packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts b/packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts index a2468cbc0..8d2ba4b80 100644 --- a/packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts +++ b/packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts @@ -68,6 +68,7 @@ export class WebDependencies extends DependencyContainer { this.get(Web_TYPES.FilesController), this.get(Web_TYPES.LinkingController), application.generateUuid, + application.files, ) }) diff --git a/packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx b/packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx index ed3f56ef3..8d197c726 100644 --- a/packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx +++ b/packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx @@ -34,9 +34,9 @@ const ImportModal = ({ importModalController }: { importModalController: ImportM close, } = importModalController - const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'ready') + const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'pending' && file.service) const importSuccessOrError = - files.length > 0 && files.every((file) => file.status === 'success' || file.status === 'error') + files.length > 0 && files.every((file) => file.status === 'finished' || file.status === 'error') const modalActions: ModalAction[] = useMemo( () => [ diff --git a/packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts b/packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts index 3d180281c..2096d0864 100644 --- a/packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts +++ b/packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts @@ -1,15 +1,14 @@ -import { DecryptedTransferPayload, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models' +import { DecryptedItemInterface, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models' import { ContentType, InternalEventBusInterface, ItemManagerInterface, MutatorClientInterface, - pluralize, PreferenceServiceInterface, PreferencesServiceEvent, UuidGenerator, } from '@standardnotes/snjs' -import { Importer } from '@standardnotes/ui-services' +import { ConversionResult, Importer } from '@standardnotes/ui-services' import { action, makeObservable, observable, runInAction } from 'mobx' import { NavigationController } from '../../Controllers/Navigation/NavigationController' import { LinkingController } from '@/Controllers/LinkingController' @@ -23,11 +22,10 @@ type ImportModalFileCommon = { export type ImportModalFile = ( | { status: 'pending' } - | { status: 'ready'; payloads?: DecryptedTransferPayload[] } | { status: 'parsing' } | { status: 'importing' } | { status: 'uploading-files' } - | { status: 'success'; successMessage: string } + | ({ status: 'finished' } & ConversionResult) | { status: 'error'; error: Error } ) & ImportModalFileCommon @@ -112,7 +110,7 @@ export class ImportModalController extends AbstractViewController { id: UuidGenerator.GenerateUuid(), file, service, - status: service ? 'ready' : 'pending', + status: 'pending', } as ImportModalFile } @@ -149,70 +147,30 @@ export class ImportModalController extends AbstractViewController { this.setImportTag(undefined) } - importFromPayloads = async (file: ImportModalFile, payloads: DecryptedTransferPayload[]) => { - this.updateFile({ - ...file, - status: 'importing', - }) - - try { - const insertedItems = await this.importer.importFromTransferPayloads(payloads) - - this.updateFile({ - ...file, - status: 'uploading-files', - }) - - await this.importer.uploadAndReplaceInlineFilesInInsertedItems(insertedItems) - - const notesImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note) - const tagsImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag) - - const successMessage = - `Successfully imported ${notesImported.length} ` + - pluralize(notesImported.length, 'note', 'notes') + - (tagsImported.length > 0 ? ` and ${tagsImported.length} ${pluralize(tagsImported.length, 'tag', 'tags')}` : '') - - this.updateFile({ - ...file, - status: 'success', - successMessage, - }) - } catch (error) { - this.updateFile({ - ...file, - status: 'error', - error: error instanceof Error ? error : new Error('Could not import file'), - }) - console.error(error) - } - } - parseAndImport = async () => { if (this.files.length === 0) { return } - const importedPayloads: DecryptedTransferPayload[] = [] + const importedItems: DecryptedItemInterface[] = [] for (const file of this.files) { if (!file.service) { return } - if (file.status === 'ready' && file.payloads) { - await this.importFromPayloads(file, file.payloads) - importedPayloads.push(...file.payloads) - continue - } - this.updateFile({ ...file, status: 'parsing', }) try { - const payloads = await this.importer.getPayloadsFromFile(file.file, file.service) - await this.importFromPayloads(file, payloads) - importedPayloads.push(...payloads) + const { successful, errored } = await this.importer.importFromFile(file.file, file.service) + importedItems.push(...successful) + this.updateFile({ + ...file, + status: 'finished', + successful, + errored, + }) } catch (error) { this.updateFile({ ...file, @@ -222,7 +180,7 @@ export class ImportModalController extends AbstractViewController { console.error(error) } } - if (!importedPayloads.length) { + if (!importedItems.length) { return } if (this.addImportsToTag) { @@ -233,7 +191,7 @@ export class ImportModalController extends AbstractViewController { title: `Imported on ${currentDate.toLocaleString()}`, expanded: false, iconString: '', - references: importedPayloads + references: importedItems .filter((payload) => payload.content_type === ContentType.TYPES.Note) .map((payload) => ({ content_type: ContentType.TYPES.Note, @@ -245,11 +203,11 @@ export class ImportModalController extends AbstractViewController { try { const latestExistingTag = this.items.findSureItem(this.existingTagForImports.uuid) await Promise.all( - importedPayloads + importedItems .filter((payload) => payload.content_type === ContentType.TYPES.Note) .map(async (payload) => { const note = this.items.findSureItem(payload.uuid) - await this.linkingController.addTagToItem(latestExistingTag, note) + await this.linkingController.addTagToItem(latestExistingTag, note, false) }), ) importTag = this.items.findSureItem(this.existingTagForImports.uuid) diff --git a/packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx b/packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx index 58ec31ff3..6f3f18ad7 100644 --- a/packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx +++ b/packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx @@ -1,9 +1,10 @@ import { ImportModalController, ImportModalFile } from '@/Components/ImportModal/ImportModalController' import { classNames, ContentType, pluralize } from '@standardnotes/snjs' -import { Importer } from '@standardnotes/ui-services' +import { ConversionResult, Importer } from '@standardnotes/ui-services' import { observer } from 'mobx-react-lite' import { useCallback, useEffect, useState } from 'react' import Icon from '../Icon/Icon' +import { Disclosure, DisclosureContent, DisclosureProvider } from '@ariakit/react' const NoteImportTypeColors: Record = { evernote: 'bg-[#14cc45] text-[#000]', @@ -25,6 +26,75 @@ const NoteImportTypeIcons: Record = { super: 'file-doc', } +const countSuccessfulItemsByGroup = (successful: ConversionResult['successful']) => { + let notes = 0 + let tags = 0 + let files = 0 + + for (const item of successful) { + if (item.content_type === ContentType.TYPES.Note) { + notes++ + } else if (item.content_type === ContentType.TYPES.Tag) { + tags++ + } else if (item.content_type === ContentType.TYPES.File) { + files++ + } + } + + return { + notes, + tags, + files, + } +} + +const ImportErroredAccordion = ({ errored }: { errored: ConversionResult['errored'] }) => { + return ( + + +
      + + Could not import {errored.length} {pluralize(errored.length, 'item', 'items')} (click for details) +
      +       + + {errored.map((item, index) => ( +
      + {index + 1}. + {item.name}: + {item.error.message} +
      + ))} +       +       + ) +} + +const ImportFinishedStatus = ({ file }: { file: ImportModalFile }) => { + if (file.status !== 'finished') { + return null + } + + const { notes, tags, files } = countSuccessfulItemsByGroup(file.successful) + + const notesStatus = notes > 0 ? `${notes} ${pluralize(notes, 'note', 'notes')}` : '' + const tagsStatus = tags > 0 ? `${tags} ${pluralize(tags, 'tag', 'tags')}` : '' + const filesStatus = files > 0 ? `${files} ${pluralize(files, 'file', 'files')}` : '' + const status = [notesStatus, tagsStatus, filesStatus].filter(Boolean).join(', ') + + return ( + <> + {file.successful.length > 0 && ( +
      + + {status} imported +
      + )} + {file.errored.length > 0 && } + + ) +} + const ImportModalFileItem = ({ file, updateFile, @@ -36,6 +106,7 @@ const ImportModalFileItem = ({ removeFile: ImportModalController['removeFile'] importer: Importer }) => { + const [isDetectingService, setIsDetectingService] = useState(false) const [changingService, setChangingService] = useState(false) const setFileService = useCallback( @@ -46,7 +117,7 @@ const ImportModalFileItem = ({ updateFile({ ...file, service, - status: service ? 'ready' : 'pending', + status: 'pending', }) }, [file, updateFile], @@ -54,59 +125,47 @@ const ImportModalFileItem = ({ useEffect(() => { const detect = async () => { + setIsDetectingService(true) const detectedService = await importer.detectService(file.file) void setFileService(detectedService) + setIsDetectingService(false) } if (file.service === undefined) { void detect() } }, [file, importer, setFileService]) - const notePayloads = - file.status === 'ready' && file.payloads - ? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note) - : [] - const tagPayloads = - file.status === 'ready' && file.payloads - ? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag) - : [] - - const payloadsImportMessage = - `Ready to import ${notePayloads.length} ` + - pluralize(notePayloads.length, 'note', 'notes') + - (tagPayloads.length > 0 ? ` and ${tagPayloads.length} ${pluralize(tagPayloads.length, 'tag', 'tags')}` : '') - return (
      -
      +
      {file.service && (
      )} -
      +
      {file.file.name}
      -
      - {file.status === 'ready' - ? notePayloads.length > 1 || tagPayloads.length - ? payloadsImportMessage - : 'Ready to import' - : null} - {file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'} - {file.status === 'parsing' && 'Parsing...'} - {file.status === 'importing' && 'Importing...'} - {file.status === 'uploading-files' && 'Uploading and embedding files...'} - {file.status === 'error' && file.error.message} - {file.status === 'success' && file.successMessage} -
      + {isDetectingService ? ( +
      Detecting service...
      + ) : ( +
      + {file.status === 'pending' && file.service && 'Ready to import'} + {file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'} + {file.status === 'parsing' && 'Parsing...'} + {file.status === 'importing' && 'Importing...'} + {file.status === 'uploading-files' && 'Uploading and embedding files...'} + {file.status === 'error' && file.error.message} + +
      + )}
      - {(file.status === 'ready' || file.status === 'pending') && ( + {file.status === 'pending' && (
      {changingService ? ( <> @@ -172,7 +231,9 @@ const ImportModalFileItem = ({
      )} - {file.status === 'success' && } + {file.status === 'finished' && file.successful.length > 0 && file.errored.length === 0 && ( + + )} {file.status === 'error' && }
      ) diff --git a/packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx b/packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx index 1fc8f83d3..cf010ff70 100644 --- a/packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx +++ b/packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx @@ -1,6 +1,6 @@ import { createHeadlessEditor } from '@lexical/headless' import { $convertToMarkdownString } from '@lexical/markdown' -import { FileItem, GenerateUuid, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs' +import { FileItem, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs' import { $createParagraphNode, $getRoot, @@ -16,9 +16,7 @@ import { MarkdownTransformers } from '../MarkdownTransformers' import { $generateHtmlFromNodes, $generateNodesFromDOM } from '@lexical/html' import { FileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileNode' import { $createFileExportNode } from '../Lexical/Nodes/FileExportNode' -import { $createInlineFileNode, $isInlineFileNode, InlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode' -import { $createFileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileUtils' -import { RemoteImageNode } from '../Plugins/RemoteImagePlugin/RemoteImageNode' +import { $createInlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode' import { $convertFromMarkdownString } from '../Lexical/Utils/MarkdownImport' export class HeadlessSuperConverter implements SuperConverterServiceInterface { private importEditor: LexicalEditor @@ -194,7 +192,8 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface { type === 'link' || type === 'linebreak' || type === 'unencrypted-image' || - type === 'inline-file' + type === 'inline-file' || + type === 'snfile' ) { const paragraphNode = $createParagraphNode() paragraphNode.append(node) @@ -232,7 +231,7 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface { } if (didThrow) { - throw new Error('Could not import note') + throw new Error('Could not import note. Check error console for details.') } return JSON.stringify(this.importEditor.getEditorState()) @@ -256,62 +255,4 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface { return ids } - - async uploadAndReplaceInlineFilesInSuperString( - superString: string, - uploadFile: (file: File) => Promise, - linkFile: (file: FileItem) => Promise, - generateUuid: GenerateUuid, - ): Promise { - if (superString.length === 0) { - return superString - } - - this.importEditor.setEditorState(this.importEditor.parseEditorState(superString)) - - await new Promise((resolve) => { - this.importEditor.update( - () => { - const inlineFileNodes = $nodesOfType(InlineFileNode) - const remoteImageNodes = $nodesOfType(RemoteImageNode).filter((node) => node.__src.startsWith('data:')) - const concatenatedNodes = [...inlineFileNodes, ...remoteImageNodes] - if (concatenatedNodes.length === 0) { - resolve() - return - } - ;(async () => { - for (const node of concatenatedNodes) { - const blob = await fetch(node.__src).then((response) => response.blob()) - const name = $isInlineFileNode(node) ? node.__fileName : node.__alt - const mimeType = $isInlineFileNode(node) ? node.__mimeType : node.__src.split(';')[0].split(':')[1] - const file = new File([blob], name || generateUuid.execute().getValue(), { - type: mimeType, - }) - - const uploadedFile = await uploadFile(file) - - if (!uploadedFile) { - return - } - - this.importEditor.update( - () => { - const fileNode = $createFileNode(uploadedFile.uuid) - node.replace(fileNode) - }, - { discrete: true }, - ) - - await linkFile(uploadedFile) - } - })() - .then(() => resolve()) - .catch(console.error) - }, - { discrete: true }, - ) - }) - - return JSON.stringify(this.importEditor.getEditorState()) - } } diff --git a/packages/web/src/javascripts/Controllers/LinkingController.tsx b/packages/web/src/javascripts/Controllers/LinkingController.tsx index b51350b6a..468896af2 100644 --- a/packages/web/src/javascripts/Controllers/LinkingController.tsx +++ b/packages/web/src/javascripts/Controllers/LinkingController.tsx @@ -204,7 +204,7 @@ export class LinkingController extends AbstractViewController implements Interna } } - linkItems = async (item: LinkableItem, itemToLink: LinkableItem) => { + linkItems = async (item: LinkableItem, itemToLink: LinkableItem, sync = true) => { const linkNoteAndFile = async (note: SNNote, file: FileItem) => { const updatedFile = await this.mutator.associateFileWithNote(file, note) @@ -231,11 +231,11 @@ export class LinkingController extends AbstractViewController implements Interna } const linkTagToNote = async (tag: SNTag, note: SNNote) => { - await this.addTagToItem(tag, note) + await this.addTagToItem(tag, note, sync) } const linkTagToFile = async (tag: SNTag, file: FileItem) => { - await this.addTagToItem(tag, file) + await this.addTagToItem(tag, file, sync) } if (isNote(item)) { @@ -273,7 +273,9 @@ export class LinkingController extends AbstractViewController implements Interna throw new Error('First item must be a note or file') } - void this.sync.sync() + if (sync) { + void this.sync.sync() + } } linkItemToSelectedItem = async (itemToLink: LinkableItem): Promise => { @@ -323,13 +325,15 @@ export class LinkingController extends AbstractViewController implements Interna return newTag } - addTagToItem = async (tag: SNTag, item: FileItem | SNNote) => { + addTagToItem = async (tag: SNTag, item: FileItem | SNNote, sync = true) => { if (item instanceof SNNote) { await this.mutator.addTagToNote(item, tag, this.shouldLinkToParentFolders) } else if (item instanceof FileItem) { await this.mutator.addTagToFile(item, tag, this.shouldLinkToParentFolders) } - this.sync.sync().catch(console.error) + if (sync) { + this.sync.sync().catch(console.error) + } } }