rmorawski/standardnotes-app-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: handle larger files in importer (#2692)

Browse Source
This commit is contained in:
Aman Harwara
2023-12-11 16:30:31 +05:30
committed by GitHub
parent 63e69b5e4b
commit 82d5a36932
22 changed files with 614 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/web/src/javascripts/Application/Dependencies/WebDependencies.ts
View File

@@ -68,6 +68,7 @@ export class WebDependencies extends DependencyContainer {
this.get<FilesController>(Web_TYPES.FilesController),
this.get<LinkingController>(Web_TYPES.LinkingController),
application.generateUuid,
application.files,
)
})

4
packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx
View File

@@ -34,9 +34,9 @@ const ImportModal = ({ importModalController }: { importModalController: ImportM
close,
} = importModalController
const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'ready')
const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'pending' && file.service)
const importSuccessOrError =
files.length > 0 && files.every((file) => file.status === 'success' || file.status === 'error')
files.length > 0 && files.every((file) => file.status === 'finished' || file.status === 'error')
const modalActions: ModalAction[] = useMemo(
() => [

76
packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts
View File

@@ -1,15 +1,14 @@
import { DecryptedTransferPayload, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
import { DecryptedItemInterface, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
import {
ContentType,
InternalEventBusInterface,
ItemManagerInterface,
MutatorClientInterface,
pluralize,
PreferenceServiceInterface,
PreferencesServiceEvent,
UuidGenerator,
} from '@standardnotes/snjs'
import { Importer } from '@standardnotes/ui-services'
import { ConversionResult, Importer } from '@standardnotes/ui-services'
import { action, makeObservable, observable, runInAction } from 'mobx'
import { NavigationController } from '../../Controllers/Navigation/NavigationController'
import { LinkingController } from '@/Controllers/LinkingController'
@@ -23,11 +22,10 @@ type ImportModalFileCommon = {
export type ImportModalFile = (
| { status: 'pending' }
| { status: 'ready'; payloads?: DecryptedTransferPayload[] }
| { status: 'parsing' }
| { status: 'importing' }
| { status: 'uploading-files' }
| { status: 'success'; successMessage: string }
| ({ status: 'finished' } & ConversionResult)
| { status: 'error'; error: Error }
) &
ImportModalFileCommon
@@ -112,7 +110,7 @@ export class ImportModalController extends AbstractViewController {
id: UuidGenerator.GenerateUuid(),
file,
service,
status: service ? 'ready' : 'pending',
status: 'pending',
} as ImportModalFile
}
@@ -149,70 +147,30 @@ export class ImportModalController extends AbstractViewController {
this.setImportTag(undefined)
}
importFromPayloads = async (file: ImportModalFile, payloads: DecryptedTransferPayload[]) => {
this.updateFile({
...file,
status: 'importing',
})
try {
const insertedItems = await this.importer.importFromTransferPayloads(payloads)
this.updateFile({
...file,
status: 'uploading-files',
})
await this.importer.uploadAndReplaceInlineFilesInInsertedItems(insertedItems)
const notesImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
const tagsImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
const successMessage =
`Successfully imported ${notesImported.length} ` +
pluralize(notesImported.length, 'note', 'notes') +
(tagsImported.length > 0 ? ` and ${tagsImported.length} ${pluralize(tagsImported.length, 'tag', 'tags')}` : '')
this.updateFile({
...file,
status: 'success',
successMessage,
})
} catch (error) {
this.updateFile({
...file,
status: 'error',
error: error instanceof Error ? error : new Error('Could not import file'),
})
console.error(error)
}
}
parseAndImport = async () => {
if (this.files.length === 0) {
return
}
const importedPayloads: DecryptedTransferPayload[] = []
const importedItems: DecryptedItemInterface[] = []
for (const file of this.files) {
if (!file.service) {
return
}
if (file.status === 'ready' && file.payloads) {
await this.importFromPayloads(file, file.payloads)
importedPayloads.push(...file.payloads)
continue
}
this.updateFile({
...file,
status: 'parsing',
})
try {
const payloads = await this.importer.getPayloadsFromFile(file.file, file.service)
await this.importFromPayloads(file, payloads)
importedPayloads.push(...payloads)
const { successful, errored } = await this.importer.importFromFile(file.file, file.service)
importedItems.push(...successful)
this.updateFile({
...file,
status: 'finished',
successful,
errored,
})
} catch (error) {
this.updateFile({
...file,
@@ -222,7 +180,7 @@ export class ImportModalController extends AbstractViewController {
console.error(error)
}
}
if (!importedPayloads.length) {
if (!importedItems.length) {
return
}
if (this.addImportsToTag) {
@@ -233,7 +191,7 @@ export class ImportModalController extends AbstractViewController {
title: `Imported on ${currentDate.toLocaleString()}`,
expanded: false,
iconString: '',
references: importedPayloads
references: importedItems
.filter((payload) => payload.content_type === ContentType.TYPES.Note)
.map((payload) => ({
content_type: ContentType.TYPES.Note,
@@ -245,11 +203,11 @@ export class ImportModalController extends AbstractViewController {
try {
const latestExistingTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)
await Promise.all(
importedPayloads
importedItems
.filter((payload) => payload.content_type === ContentType.TYPES.Note)
.map(async (payload) => {
const note = this.items.findSureItem<SNNote>(payload.uuid)
await this.linkingController.addTagToItem(latestExistingTag, note)
await this.linkingController.addTagToItem(latestExistingTag, note, false)
}),
)
importTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)

129
packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx
View File

@@ -1,9 +1,10 @@
import { ImportModalController, ImportModalFile } from '@/Components/ImportModal/ImportModalController'
import { classNames, ContentType, pluralize } from '@standardnotes/snjs'
import { Importer } from '@standardnotes/ui-services'
import { ConversionResult, Importer } from '@standardnotes/ui-services'
import { observer } from 'mobx-react-lite'
import { useCallback, useEffect, useState } from 'react'
import Icon from '../Icon/Icon'
import { Disclosure, DisclosureContent, DisclosureProvider } from '@ariakit/react'
const NoteImportTypeColors: Record<string, string> = {
evernote: 'bg-[#14cc45] text-[#000]',
@@ -25,6 +26,75 @@ const NoteImportTypeIcons: Record<string, string> = {
super: 'file-doc',
}
const countSuccessfulItemsByGroup = (successful: ConversionResult['successful']) => {
let notes = 0
let tags = 0
let files = 0
for (const item of successful) {
if (item.content_type === ContentType.TYPES.Note) {
notes++
} else if (item.content_type === ContentType.TYPES.Tag) {
tags++
} else if (item.content_type === ContentType.TYPES.File) {
files++
}
}
return {
notes,
tags,
files,
}
}
const ImportErroredAccordion = ({ errored }: { errored: ConversionResult['errored'] }) => {
return (
<DisclosureProvider>
<Disclosure>
<div className="flex items-center gap-1">
<Icon type="warning" className="flex-shrink-0 text-danger" size="small" />
Could not import {errored.length} {pluralize(errored.length, 'item', 'items')} (click for details)
</div>
</Disclosure>
<DisclosureContent className="w-full overflow-hidden pl-5">
{errored.map((item, index) => (
<div className="flex w-full items-center gap-1 overflow-hidden" key={index}>
<span>{index + 1}.</span>
<span className="overflow-hidden text-ellipsis whitespace-nowrap font-semibold">{item.name}:</span>
<span className="overflow-hidden text-ellipsis whitespace-nowrap">{item.error.message}</span>
</div>
))}
</DisclosureContent>
</DisclosureProvider>
)
}
const ImportFinishedStatus = ({ file }: { file: ImportModalFile }) => {
if (file.status !== 'finished') {
return null
}
const { notes, tags, files } = countSuccessfulItemsByGroup(file.successful)
const notesStatus = notes > 0 ? `${notes} ${pluralize(notes, 'note', 'notes')}` : ''
const tagsStatus = tags > 0 ? `${tags} ${pluralize(tags, 'tag', 'tags')}` : ''
const filesStatus = files > 0 ? `${files} ${pluralize(files, 'file', 'files')}` : ''
const status = [notesStatus, tagsStatus, filesStatus].filter(Boolean).join(', ')
return (
<>
{file.successful.length > 0 && (
<div className="flex items-center gap-1">
<Icon type="check-circle-filled" className="flex-shrink-0 text-success" size="small" />
<span>{status} imported</span>
</div>
)}
{file.errored.length > 0 && <ImportErroredAccordion errored={file.errored} />}
</>
)
}
const ImportModalFileItem = ({
file,
updateFile,
@@ -36,6 +106,7 @@ const ImportModalFileItem = ({
removeFile: ImportModalController['removeFile']
importer: Importer
}) => {
const [isDetectingService, setIsDetectingService] = useState(false)
const [changingService, setChangingService] = useState(false)
const setFileService = useCallback(
@@ -46,7 +117,7 @@ const ImportModalFileItem = ({
updateFile({
...file,
service,
status: service ? 'ready' : 'pending',
status: 'pending',
})
},
[file, updateFile],
@@ -54,59 +125,47 @@ const ImportModalFileItem = ({
useEffect(() => {
const detect = async () => {
setIsDetectingService(true)
const detectedService = await importer.detectService(file.file)
void setFileService(detectedService)
setIsDetectingService(false)
}
if (file.service === undefined) {
void detect()
}
}, [file, importer, setFileService])
const notePayloads =
file.status === 'ready' && file.payloads
? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
: []
const tagPayloads =
file.status === 'ready' && file.payloads
? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
: []
const payloadsImportMessage =
`Ready to import ${notePayloads.length} ` +
pluralize(notePayloads.length, 'note', 'notes') +
(tagPayloads.length > 0 ? ` and ${tagPayloads.length} ${pluralize(tagPayloads.length, 'tag', 'tags')}` : '')
return (
<div
className={classNames(
'flex gap-2 px-2 py-2.5',
'flex gap-2 overflow-hidden px-2 py-2.5',
file.service == null ? 'flex-col items-start md:flex-row md:items-center' : 'items-center',
)}
>
<div className="mr-auto flex items-center">
<div className="mr-auto flex w-full items-center">
{file.service && (
<div className={classNames('mr-4 rounded p-2', NoteImportTypeColors[file.service])}>
<Icon type={NoteImportTypeIcons[file.service]} size="medium" />
</div>
)}
<div className="flex flex-col">
<div className="flex w-full flex-col overflow-hidden">
<div>{file.file.name}</div>
<div className="line-clamp-3 text-xs opacity-75">
{file.status === 'ready'
? notePayloads.length > 1 || tagPayloads.length
? payloadsImportMessage
: 'Ready to import'
: null}
{file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
{file.status === 'parsing' && 'Parsing...'}
{file.status === 'importing' && 'Importing...'}
{file.status === 'uploading-files' && 'Uploading and embedding files...'}
{file.status === 'error' && file.error.message}
{file.status === 'success' && file.successMessage}
</div>
{isDetectingService ? (
<div className="text-xs opacity-75">Detecting service...</div>
) : (
<div className={classNames(file.status !== 'finished' && 'line-clamp-3', 'w-full text-xs opacity-75')}>
{file.status === 'pending' && file.service && 'Ready to import'}
{file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
{file.status === 'parsing' && 'Parsing...'}
{file.status === 'importing' && 'Importing...'}
{file.status === 'uploading-files' && 'Uploading and embedding files...'}
{file.status === 'error' && file.error.message}
<ImportFinishedStatus file={file} />
</div>
)}
</div>
</div>
{(file.status === 'ready' || file.status === 'pending') && (
{file.status === 'pending' && (
<div className="flex items-center">
{changingService ? (
<>
@@ -172,7 +231,9 @@ const ImportModalFileItem = ({
</button>
</div>
)}
{file.status === 'success' && <Icon type="check-circle-filled" className="flex-shrink-0 text-success" />}
{file.status === 'finished' && file.successful.length > 0 && file.errored.length === 0 && (
<Icon type="check-circle-filled" className="flex-shrink-0 text-success" />
)}
{file.status === 'error' && <Icon type="warning" className="flex-shrink-0 text-danger" />}
</div>
)

69
packages/web/src/javascripts/Components/SuperEditor/Tools/HeadlessSuperConverter.tsx
View File

@@ -1,6 +1,6 @@
import { createHeadlessEditor } from '@lexical/headless'
import { $convertToMarkdownString } from '@lexical/markdown'
import { FileItem, GenerateUuid, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
import { FileItem, PrefKey, PrefValue, SuperConverterServiceInterface } from '@standardnotes/snjs'
import {
$createParagraphNode,
$getRoot,
@@ -16,9 +16,7 @@ import { MarkdownTransformers } from '../MarkdownTransformers'
import { $generateHtmlFromNodes, $generateNodesFromDOM } from '@lexical/html'
import { FileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileNode'
import { $createFileExportNode } from '../Lexical/Nodes/FileExportNode'
import { $createInlineFileNode, $isInlineFileNode, InlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
import { $createFileNode } from '../Plugins/EncryptedFilePlugin/Nodes/FileUtils'
import { RemoteImageNode } from '../Plugins/RemoteImagePlugin/RemoteImageNode'
import { $createInlineFileNode } from '../Plugins/InlineFilePlugin/InlineFileNode'
import { $convertFromMarkdownString } from '../Lexical/Utils/MarkdownImport'
export class HeadlessSuperConverter implements SuperConverterServiceInterface {
private importEditor: LexicalEditor
@@ -194,7 +192,8 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
type === 'link' ||
type === 'linebreak' ||
type === 'unencrypted-image' ||
type === 'inline-file'
type === 'inline-file' ||
type === 'snfile'
) {
const paragraphNode = $createParagraphNode()
paragraphNode.append(node)
@@ -232,7 +231,7 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
}
if (didThrow) {
throw new Error('Could not import note')
throw new Error('Could not import note. Check error console for details.')
}
return JSON.stringify(this.importEditor.getEditorState())
@@ -256,62 +255,4 @@ export class HeadlessSuperConverter implements SuperConverterServiceInterface {
return ids
}
async uploadAndReplaceInlineFilesInSuperString(
superString: string,
uploadFile: (file: File) => Promise<FileItem | undefined>,
linkFile: (file: FileItem) => Promise<void>,
generateUuid: GenerateUuid,
): Promise<string> {
if (superString.length === 0) {
return superString
}
this.importEditor.setEditorState(this.importEditor.parseEditorState(superString))
await new Promise<void>((resolve) => {
this.importEditor.update(
() => {
const inlineFileNodes = $nodesOfType(InlineFileNode)
const remoteImageNodes = $nodesOfType(RemoteImageNode).filter((node) => node.__src.startsWith('data:'))
const concatenatedNodes = [...inlineFileNodes, ...remoteImageNodes]
if (concatenatedNodes.length === 0) {
resolve()
return
}
;(async () => {
for (const node of concatenatedNodes) {
const blob = await fetch(node.__src).then((response) => response.blob())
const name = $isInlineFileNode(node) ? node.__fileName : node.__alt
const mimeType = $isInlineFileNode(node) ? node.__mimeType : node.__src.split(';')[0].split(':')[1]
const file = new File([blob], name || generateUuid.execute().getValue(), {
type: mimeType,
})
const uploadedFile = await uploadFile(file)
if (!uploadedFile) {
return
}
this.importEditor.update(
() => {
const fileNode = $createFileNode(uploadedFile.uuid)
node.replace(fileNode)
},
{ discrete: true },
)
await linkFile(uploadedFile)
}
})()
.then(() => resolve())
.catch(console.error)
},
{ discrete: true },
)
})
return JSON.stringify(this.importEditor.getEditorState())
}
}

16
packages/web/src/javascripts/Controllers/LinkingController.tsx
View File

@@ -204,7 +204,7 @@ export class LinkingController extends AbstractViewController implements Interna
}
}
linkItems = async (item: LinkableItem, itemToLink: LinkableItem) => {
linkItems = async (item: LinkableItem, itemToLink: LinkableItem, sync = true) => {
const linkNoteAndFile = async (note: SNNote, file: FileItem) => {
const updatedFile = await this.mutator.associateFileWithNote(file, note)
@@ -231,11 +231,11 @@ export class LinkingController extends AbstractViewController implements Interna
}
const linkTagToNote = async (tag: SNTag, note: SNNote) => {
await this.addTagToItem(tag, note)
await this.addTagToItem(tag, note, sync)
}
const linkTagToFile = async (tag: SNTag, file: FileItem) => {
await this.addTagToItem(tag, file)
await this.addTagToItem(tag, file, sync)
}
if (isNote(item)) {
@@ -273,7 +273,9 @@ export class LinkingController extends AbstractViewController implements Interna
throw new Error('First item must be a note or file')
}
void this.sync.sync()
if (sync) {
void this.sync.sync()
}
}
linkItemToSelectedItem = async (itemToLink: LinkableItem): Promise<boolean> => {
@@ -323,13 +325,15 @@ export class LinkingController extends AbstractViewController implements Interna
return newTag
}
addTagToItem = async (tag: SNTag, item: FileItem | SNNote) => {
addTagToItem = async (tag: SNTag, item: FileItem | SNNote, sync = true) => {
if (item instanceof SNNote) {
await this.mutator.addTagToNote(item, tag, this.shouldLinkToParentFolders)
} else if (item instanceof FileItem) {
await this.mutator.addTagToFile(item, tag, this.shouldLinkToParentFolders)
}
this.sync.sync().catch(console.error)
if (sync) {
this.sync.sync().catch(console.error)
}
}
}