rmorawski/standardnotes-app-web
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: handle larger files in importer (#2692)

Browse Source
This commit is contained in:
Aman Harwara
2023-12-11 16:30:31 +05:30
committed by GitHub
parent 63e69b5e4b
commit 82d5a36932
22 changed files with 614 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/web/src/javascripts/Components/ImportModal/ImportModal.tsx
View File

@@ -34,9 +34,9 @@ const ImportModal = ({ importModalController }: { importModalController: ImportM
close,
} = importModalController
const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'ready')
const isReadyToImport = files.length > 0 && files.every((file) => file.status === 'pending' && file.service)
const importSuccessOrError =
files.length > 0 && files.every((file) => file.status === 'success' || file.status === 'error')
files.length > 0 && files.every((file) => file.status === 'finished' || file.status === 'error')
const modalActions: ModalAction[] = useMemo(
() => [

76
packages/web/src/javascripts/Components/ImportModal/ImportModalController.ts
View File

@@ -1,15 +1,14 @@
import { DecryptedTransferPayload, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
import { DecryptedItemInterface, PrefDefaults, PrefKey, SNNote, SNTag, TagContent } from '@standardnotes/models'
import {
ContentType,
InternalEventBusInterface,
ItemManagerInterface,
MutatorClientInterface,
pluralize,
PreferenceServiceInterface,
PreferencesServiceEvent,
UuidGenerator,
} from '@standardnotes/snjs'
import { Importer } from '@standardnotes/ui-services'
import { ConversionResult, Importer } from '@standardnotes/ui-services'
import { action, makeObservable, observable, runInAction } from 'mobx'
import { NavigationController } from '../../Controllers/Navigation/NavigationController'
import { LinkingController } from '@/Controllers/LinkingController'
@@ -23,11 +22,10 @@ type ImportModalFileCommon = {
export type ImportModalFile = (
| { status: 'pending' }
| { status: 'ready'; payloads?: DecryptedTransferPayload[] }
| { status: 'parsing' }
| { status: 'importing' }
| { status: 'uploading-files' }
| { status: 'success'; successMessage: string }
| ({ status: 'finished' } & ConversionResult)
| { status: 'error'; error: Error }
) &
ImportModalFileCommon
@@ -112,7 +110,7 @@ export class ImportModalController extends AbstractViewController {
id: UuidGenerator.GenerateUuid(),
file,
service,
status: service ? 'ready' : 'pending',
status: 'pending',
} as ImportModalFile
}
@@ -149,70 +147,30 @@ export class ImportModalController extends AbstractViewController {
this.setImportTag(undefined)
}
importFromPayloads = async (file: ImportModalFile, payloads: DecryptedTransferPayload[]) => {
this.updateFile({
...file,
status: 'importing',
})
try {
const insertedItems = await this.importer.importFromTransferPayloads(payloads)
this.updateFile({
...file,
status: 'uploading-files',
})
await this.importer.uploadAndReplaceInlineFilesInInsertedItems(insertedItems)
const notesImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
const tagsImported = payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
const successMessage =
`Successfully imported ${notesImported.length} ` +
pluralize(notesImported.length, 'note', 'notes') +
(tagsImported.length > 0 ? ` and ${tagsImported.length} ${pluralize(tagsImported.length, 'tag', 'tags')}` : '')
this.updateFile({
...file,
status: 'success',
successMessage,
})
} catch (error) {
this.updateFile({
...file,
status: 'error',
error: error instanceof Error ? error : new Error('Could not import file'),
})
console.error(error)
}
}
parseAndImport = async () => {
if (this.files.length === 0) {
return
}
const importedPayloads: DecryptedTransferPayload[] = []
const importedItems: DecryptedItemInterface[] = []
for (const file of this.files) {
if (!file.service) {
return
}
if (file.status === 'ready' && file.payloads) {
await this.importFromPayloads(file, file.payloads)
importedPayloads.push(...file.payloads)
continue
}
this.updateFile({
...file,
status: 'parsing',
})
try {
const payloads = await this.importer.getPayloadsFromFile(file.file, file.service)
await this.importFromPayloads(file, payloads)
importedPayloads.push(...payloads)
const { successful, errored } = await this.importer.importFromFile(file.file, file.service)
importedItems.push(...successful)
this.updateFile({
...file,
status: 'finished',
successful,
errored,
})
} catch (error) {
this.updateFile({
...file,
@@ -222,7 +180,7 @@ export class ImportModalController extends AbstractViewController {
console.error(error)
}
}
if (!importedPayloads.length) {
if (!importedItems.length) {
return
}
if (this.addImportsToTag) {
@@ -233,7 +191,7 @@ export class ImportModalController extends AbstractViewController {
title: `Imported on ${currentDate.toLocaleString()}`,
expanded: false,
iconString: '',
references: importedPayloads
references: importedItems
.filter((payload) => payload.content_type === ContentType.TYPES.Note)
.map((payload) => ({
content_type: ContentType.TYPES.Note,
@@ -245,11 +203,11 @@ export class ImportModalController extends AbstractViewController {
try {
const latestExistingTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)
await Promise.all(
importedPayloads
importedItems
.filter((payload) => payload.content_type === ContentType.TYPES.Note)
.map(async (payload) => {
const note = this.items.findSureItem<SNNote>(payload.uuid)
await this.linkingController.addTagToItem(latestExistingTag, note)
await this.linkingController.addTagToItem(latestExistingTag, note, false)
}),
)
importTag = this.items.findSureItem<SNTag>(this.existingTagForImports.uuid)

129
packages/web/src/javascripts/Components/ImportModal/ImportModalFileItem.tsx
View File

@@ -1,9 +1,10 @@
import { ImportModalController, ImportModalFile } from '@/Components/ImportModal/ImportModalController'
import { classNames, ContentType, pluralize } from '@standardnotes/snjs'
import { Importer } from '@standardnotes/ui-services'
import { ConversionResult, Importer } from '@standardnotes/ui-services'
import { observer } from 'mobx-react-lite'
import { useCallback, useEffect, useState } from 'react'
import Icon from '../Icon/Icon'
import { Disclosure, DisclosureContent, DisclosureProvider } from '@ariakit/react'
const NoteImportTypeColors: Record<string, string> = {
evernote: 'bg-[#14cc45] text-[#000]',
@@ -25,6 +26,75 @@ const NoteImportTypeIcons: Record<string, string> = {
super: 'file-doc',
}
const countSuccessfulItemsByGroup = (successful: ConversionResult['successful']) => {
let notes = 0
let tags = 0
let files = 0
for (const item of successful) {
if (item.content_type === ContentType.TYPES.Note) {
notes++
} else if (item.content_type === ContentType.TYPES.Tag) {
tags++
} else if (item.content_type === ContentType.TYPES.File) {
files++
}
}
return {
notes,
tags,
files,
}
}
const ImportErroredAccordion = ({ errored }: { errored: ConversionResult['errored'] }) => {
return (
<DisclosureProvider>
<Disclosure>
<div className="flex items-center gap-1">
<Icon type="warning" className="flex-shrink-0 text-danger" size="small" />
Could not import {errored.length} {pluralize(errored.length, 'item', 'items')} (click for details)
</div>
</Disclosure>
<DisclosureContent className="w-full overflow-hidden pl-5">
{errored.map((item, index) => (
<div className="flex w-full items-center gap-1 overflow-hidden" key={index}>
<span>{index + 1}.</span>
<span className="overflow-hidden text-ellipsis whitespace-nowrap font-semibold">{item.name}:</span>
<span className="overflow-hidden text-ellipsis whitespace-nowrap">{item.error.message}</span>
</div>
))}
</DisclosureContent>
</DisclosureProvider>
)
}
const ImportFinishedStatus = ({ file }: { file: ImportModalFile }) => {
if (file.status !== 'finished') {
return null
}
const { notes, tags, files } = countSuccessfulItemsByGroup(file.successful)
const notesStatus = notes > 0 ? `${notes} ${pluralize(notes, 'note', 'notes')}` : ''
const tagsStatus = tags > 0 ? `${tags} ${pluralize(tags, 'tag', 'tags')}` : ''
const filesStatus = files > 0 ? `${files} ${pluralize(files, 'file', 'files')}` : ''
const status = [notesStatus, tagsStatus, filesStatus].filter(Boolean).join(', ')
return (
<>
{file.successful.length > 0 && (
<div className="flex items-center gap-1">
<Icon type="check-circle-filled" className="flex-shrink-0 text-success" size="small" />
<span>{status} imported</span>
</div>
)}
{file.errored.length > 0 && <ImportErroredAccordion errored={file.errored} />}
</>
)
}
const ImportModalFileItem = ({
file,
updateFile,
@@ -36,6 +106,7 @@ const ImportModalFileItem = ({
removeFile: ImportModalController['removeFile']
importer: Importer
}) => {
const [isDetectingService, setIsDetectingService] = useState(false)
const [changingService, setChangingService] = useState(false)
const setFileService = useCallback(
@@ -46,7 +117,7 @@ const ImportModalFileItem = ({
updateFile({
...file,
service,
status: service ? 'ready' : 'pending',
status: 'pending',
})
},
[file, updateFile],
@@ -54,59 +125,47 @@ const ImportModalFileItem = ({
useEffect(() => {
const detect = async () => {
setIsDetectingService(true)
const detectedService = await importer.detectService(file.file)
void setFileService(detectedService)
setIsDetectingService(false)
}
if (file.service === undefined) {
void detect()
}
}, [file, importer, setFileService])
const notePayloads =
file.status === 'ready' && file.payloads
? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Note)
: []
const tagPayloads =
file.status === 'ready' && file.payloads
? file.payloads.filter((payload) => payload.content_type === ContentType.TYPES.Tag)
: []
const payloadsImportMessage =
`Ready to import ${notePayloads.length} ` +
pluralize(notePayloads.length, 'note', 'notes') +
(tagPayloads.length > 0 ? ` and ${tagPayloads.length} ${pluralize(tagPayloads.length, 'tag', 'tags')}` : '')
return (
<div
className={classNames(
'flex gap-2 px-2 py-2.5',
'flex gap-2 overflow-hidden px-2 py-2.5',
file.service == null ? 'flex-col items-start md:flex-row md:items-center' : 'items-center',
)}
>
<div className="mr-auto flex items-center">
<div className="mr-auto flex w-full items-center">
{file.service && (
<div className={classNames('mr-4 rounded p-2', NoteImportTypeColors[file.service])}>
<Icon type={NoteImportTypeIcons[file.service]} size="medium" />
</div>
)}
<div className="flex flex-col">
<div className="flex w-full flex-col overflow-hidden">
<div>{file.file.name}</div>
<div className="line-clamp-3 text-xs opacity-75">
{file.status === 'ready'
? notePayloads.length > 1 || tagPayloads.length
? payloadsImportMessage
: 'Ready to import'
: null}
{file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
{file.status === 'parsing' && 'Parsing...'}
{file.status === 'importing' && 'Importing...'}
{file.status === 'uploading-files' && 'Uploading and embedding files...'}
{file.status === 'error' && file.error.message}
{file.status === 'success' && file.successMessage}
</div>
{isDetectingService ? (
<div className="text-xs opacity-75">Detecting service...</div>
) : (
<div className={classNames(file.status !== 'finished' && 'line-clamp-3', 'w-full text-xs opacity-75')}>
{file.status === 'pending' && file.service && 'Ready to import'}
{file.status === 'pending' && !file.service && 'Could not auto-detect service. Please select manually.'}
{file.status === 'parsing' && 'Parsing...'}
{file.status === 'importing' && 'Importing...'}
{file.status === 'uploading-files' && 'Uploading and embedding files...'}
{file.status === 'error' && file.error.message}
<ImportFinishedStatus file={file} />
</div>
)}
</div>
</div>
{(file.status === 'ready' || file.status === 'pending') && (
{file.status === 'pending' && (
<div className="flex items-center">
{changingService ? (
<>
@@ -172,7 +231,9 @@ const ImportModalFileItem = ({
</button>
</div>
)}
{file.status === 'success' && <Icon type="check-circle-filled" className="flex-shrink-0 text-success" />}
{file.status === 'finished' && file.successful.length > 0 && file.errored.length === 0 && (
<Icon type="check-circle-filled" className="flex-shrink-0 text-success" />
)}
{file.status === 'error' && <Icon type="warning" className="flex-shrink-0 text-danger" />}
</div>
)