feat: add filepicker package

This commit is contained in:
Karol Sójko
2022-07-05 19:28:22 +02:00
parent 577da2ca84
commit d4188a3fa2
45 changed files with 5848 additions and 25 deletions

View File

@@ -0,0 +1,78 @@
import { EncryptedBytes } from './../TypedBytes'
import { FileMemoryCache } from './FileMemoryCache'
describe('file memory cache', () => {
const createBytes = (size: number): EncryptedBytes => {
return { encryptedBytes: new TextEncoder().encode('a'.repeat(size)) }
}
it('should add file', () => {
const cache = new FileMemoryCache(5)
const file = createBytes(1)
cache.add('123', file)
expect(cache.get('123')).toEqual(file)
})
it('should fail to add file if exceeds maximum', () => {
const maxSize = 5
const cache = new FileMemoryCache(maxSize)
const file = createBytes(maxSize + 1)
expect(cache.add('123', file)).toEqual(false)
})
it('should allow filling files up to limit', () => {
const cache = new FileMemoryCache(5)
cache.add('1', createBytes(3))
cache.add('2', createBytes(2))
expect(cache.get('1')).toBeTruthy()
expect(cache.get('2')).toBeTruthy()
})
it('should clear early files when adding new files above limit', () => {
const cache = new FileMemoryCache(5)
cache.add('1', createBytes(3))
cache.add('2', createBytes(2))
cache.add('3', createBytes(5))
expect(cache.get('1')).toBeFalsy()
expect(cache.get('2')).toBeFalsy()
expect(cache.get('3')).toBeTruthy()
})
it('should remove single file', () => {
const cache = new FileMemoryCache(5)
cache.add('1', createBytes(3))
cache.add('2', createBytes(2))
cache.remove('1')
expect(cache.get('1')).toBeFalsy()
expect(cache.get('2')).toBeTruthy()
})
it('should clear all files', () => {
const cache = new FileMemoryCache(5)
cache.add('1', createBytes(3))
cache.add('2', createBytes(2))
cache.clear()
expect(cache.get('1')).toBeFalsy()
expect(cache.get('2')).toBeFalsy()
})
it('should return correct size', () => {
const cache = new FileMemoryCache(20)
cache.add('1', createBytes(3))
cache.add('2', createBytes(10))
expect(cache.size).toEqual(13)
})
})

View File

@@ -0,0 +1,48 @@
import { removeFromArray } from '@standardnotes/utils'
import { Uuid } from '@standardnotes/common'
import { EncryptedBytes } from '../TypedBytes'
export class FileMemoryCache {
private cache: Record<Uuid, EncryptedBytes> = {}
private orderedQueue: Uuid[] = []
constructor(public readonly maxSize: number) {}
add(uuid: Uuid, data: EncryptedBytes): boolean {
if (data.encryptedBytes.length > this.maxSize) {
return false
}
while (this.size + data.encryptedBytes.length > this.maxSize) {
this.remove(this.orderedQueue[0])
}
this.cache[uuid] = data
this.orderedQueue.push(uuid)
return true
}
get size(): number {
return Object.values(this.cache)
.map((bytes) => bytes.encryptedBytes.length)
.reduce((total, fileLength) => total + fileLength, 0)
}
get(uuid: Uuid): EncryptedBytes | undefined {
return this.cache[uuid]
}
remove(uuid: Uuid): void {
delete this.cache[uuid]
removeFromArray(this.orderedQueue, uuid)
}
clear(): void {
this.cache = {}
this.orderedQueue = []
}
}

View File

@@ -0,0 +1,70 @@
import { ByteChunker } from './ByteChunker'
const chunkOfSize = (size: number) => {
return new TextEncoder().encode('a'.repeat(size))
}
describe('byte chunker', () => {
it('should hold back small chunks until minimum size is met', async () => {
let receivedBytes = new Uint8Array()
let numChunks = 0
const chunker = new ByteChunker(100, async (bytes) => {
numChunks++
receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
})
await chunker.addBytes(chunkOfSize(50), false)
await chunker.addBytes(chunkOfSize(50), false)
await chunker.addBytes(chunkOfSize(50), false)
await chunker.addBytes(chunkOfSize(50), true)
expect(numChunks).toEqual(2)
expect(receivedBytes.length).toEqual(200)
})
it('should send back big chunks immediately', async () => {
let receivedBytes = new Uint8Array()
let numChunks = 0
const chunker = new ByteChunker(100, async (bytes) => {
numChunks++
receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
})
await chunker.addBytes(chunkOfSize(150), false)
await chunker.addBytes(chunkOfSize(150), false)
await chunker.addBytes(chunkOfSize(150), false)
await chunker.addBytes(chunkOfSize(50), true)
expect(numChunks).toEqual(4)
expect(receivedBytes.length).toEqual(500)
})
it('last chunk should be popped regardless of size', async () => {
let receivedBytes = new Uint8Array()
let numChunks = 0
const chunker = new ByteChunker(100, async (bytes) => {
numChunks++
receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
})
await chunker.addBytes(chunkOfSize(50), false)
await chunker.addBytes(chunkOfSize(25), true)
expect(numChunks).toEqual(1)
expect(receivedBytes.length).toEqual(75)
})
it('single chunk should be popped immediately', async () => {
let receivedBytes = new Uint8Array()
let numChunks = 0
const chunker = new ByteChunker(100, async (bytes) => {
numChunks++
receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
})
await chunker.addBytes(chunkOfSize(50), true)
expect(numChunks).toEqual(1)
expect(receivedBytes.length).toEqual(50)
})
})

View File

@@ -0,0 +1,35 @@
import { OnChunkCallback } from '../types'
export class ByteChunker {
public loggingEnabled = false
private bytes = new Uint8Array()
private index = 1
constructor(private minimumChunkSize: number, private onChunk: OnChunkCallback) {}
private log(...args: any[]): void {
if (!this.loggingEnabled) {
return
}
// eslint-disable-next-line no-console
console.log(args)
}
public async addBytes(bytes: Uint8Array, isLast: boolean): Promise<void> {
this.bytes = new Uint8Array([...this.bytes, ...bytes])
this.log(`Chunker adding ${bytes.length}, total size ${this.bytes.length}`)
if (this.bytes.length >= this.minimumChunkSize || isLast) {
await this.popBytes(isLast)
}
}
private async popBytes(isLast: boolean): Promise<void> {
const maxIndex = Math.max(this.minimumChunkSize, this.bytes.length)
const chunk = this.bytes.slice(0, maxIndex)
this.bytes = new Uint8Array([...this.bytes.slice(maxIndex)])
this.log(`Chunker popping ${chunk.length}, total size in queue ${this.bytes.length}`)
await this.onChunk(chunk, this.index++, isLast)
}
}

View File

@@ -0,0 +1,23 @@
import { OrderedByteChunker } from './OrderedByteChunker'
const chunkOfSize = (size: number) => {
return new TextEncoder().encode('a'.repeat(size))
}
describe('ordered byte chunker', () => {
it('should callback multiple times if added bytes matches multiple chunk sizes', async () => {
const chunkSizes = [10, 10, 10]
let receivedBytes = new Uint8Array()
let numCallbacks = 0
const chunker = new OrderedByteChunker(chunkSizes, async (bytes) => {
numCallbacks++
receivedBytes = new Uint8Array([...receivedBytes, ...bytes])
})
await chunker.addBytes(chunkOfSize(30))
expect(numCallbacks).toEqual(3)
expect(receivedBytes.length).toEqual(30)
})
})

View File

@@ -0,0 +1,40 @@
export class OrderedByteChunker {
private bytes = new Uint8Array()
private index = 1
private remainingChunks: number[] = []
constructor(
private chunkSizes: number[],
private onChunk: (chunk: Uint8Array, index: number, isLast: boolean) => Promise<void>,
) {
this.remainingChunks = chunkSizes.slice()
}
private needsPop(): boolean {
return this.remainingChunks.length > 0 && this.bytes.length >= this.remainingChunks[0]
}
public async addBytes(bytes: Uint8Array): Promise<void> {
this.bytes = new Uint8Array([...this.bytes, ...bytes])
if (this.needsPop()) {
await this.popBytes()
}
}
private async popBytes(): Promise<void> {
const readUntil = this.remainingChunks[0]
const chunk = this.bytes.slice(0, readUntil)
this.bytes = new Uint8Array([...this.bytes.slice(readUntil)])
this.remainingChunks.shift()
await this.onChunk(chunk, this.index++, this.index === this.chunkSizes.length - 1)
if (this.needsPop()) {
await this.popBytes()
}
}
}

View File

@@ -0,0 +1,59 @@
import { ByteChunker } from './../Chunker/ByteChunker'
import { OnChunkCallback, FileSelectionResponse } from '../types'
import { readFile as utilsReadFile } from '../utils'
import { FileReaderInterface } from '../Interface/FileReader'
export const ClassicFileReader: FileReaderInterface = {
selectFiles,
readFile,
available,
maximumFileSize,
}
function available(): boolean {
return true
}
function maximumFileSize(): number {
return 50 * 1_000_000
}
function selectFiles(): Promise<File[]> {
const input = document.createElement('input') as HTMLInputElement
input.type = 'file'
input.multiple = true
return new Promise((resolve) => {
input.onchange = async (event) => {
const target = event.target as HTMLInputElement
const files = []
for (const file of target.files as FileList) {
files.push(file)
}
resolve(files)
}
input.click()
})
}
async function readFile(
file: File,
minimumChunkSize: number,
onChunk: OnChunkCallback,
): Promise<FileSelectionResponse> {
const buffer = await utilsReadFile(file)
const chunker = new ByteChunker(minimumChunkSize, onChunk)
const readSize = 2_000_000
for (let i = 0; i < buffer.length; i += readSize) {
const chunkMax = i + readSize
const chunk = buffer.slice(i, chunkMax)
const isFinalChunk = chunkMax >= buffer.length
await chunker.addBytes(chunk, isFinalChunk)
}
return {
name: file.name,
mimeType: file.type,
}
}

View File

@@ -0,0 +1,23 @@
import { saveFile } from '../utils'
export class ClassicFileSaver {
public loggingEnabled = false
private log(...args: any[]): void {
if (!this.loggingEnabled) {
return
}
// eslint-disable-next-line no-console
console.log(args)
}
static maximumFileSize(): number {
return 50 * 1_000_000
}
saveFile(name: string, bytes: Uint8Array): void {
this.log('Saving file to disk...')
saveFile(name, bytes)
this.log('Closing write stream')
}
}

View File

@@ -0,0 +1,11 @@
import { OnChunkCallback, FileSelectionResponse } from '../types'
export interface FileReaderInterface {
selectFiles(): Promise<File[]>
readFile(file: File, minimumChunkSize: number, onChunk: OnChunkCallback): Promise<FileSelectionResponse>
available(): boolean
maximumFileSize(): number | undefined
}

View File

@@ -0,0 +1,112 @@
import {
FileSystemApi,
DirectoryHandle,
FileHandleReadWrite,
FileHandleRead,
FileSystemNoSelection,
FileSystemResult,
} from '@standardnotes/services'
interface WebDirectoryHandle extends DirectoryHandle {
nativeHandle: FileSystemDirectoryHandle
}
interface WebFileHandleReadWrite extends FileHandleReadWrite {
nativeHandle: FileSystemFileHandle
writableStream: FileSystemWritableFileStream
}
interface WebFileHandleRead extends FileHandleRead {
nativeHandle: FileSystemFileHandle
}
export class StreamingFileApi implements FileSystemApi {
async selectDirectory(): Promise<DirectoryHandle | FileSystemNoSelection> {
try {
const nativeHandle = await window.showDirectoryPicker()
return { nativeHandle }
} catch (error) {
return 'aborted'
}
}
async createFile(directory: WebDirectoryHandle, name: string): Promise<WebFileHandleReadWrite> {
const nativeHandle = await directory.nativeHandle.getFileHandle(name, { create: true })
const writableStream = await nativeHandle.createWritable()
return {
nativeHandle,
writableStream,
}
}
async createDirectory(
parentDirectory: WebDirectoryHandle,
name: string,
): Promise<WebDirectoryHandle | FileSystemNoSelection> {
const nativeHandle = await parentDirectory.nativeHandle.getDirectoryHandle(name, { create: true })
return { nativeHandle }
}
async saveBytes(file: WebFileHandleReadWrite, bytes: Uint8Array): Promise<'success' | 'failed'> {
await file.writableStream.write(bytes)
return 'success'
}
async saveString(file: WebFileHandleReadWrite, contents: string): Promise<'success' | 'failed'> {
await file.writableStream.write(contents)
return 'success'
}
async closeFileWriteStream(file: WebFileHandleReadWrite): Promise<'success' | 'failed'> {
await file.writableStream.close()
return 'success'
}
async selectFile(): Promise<WebFileHandleRead | FileSystemNoSelection> {
try {
const selection = await window.showOpenFilePicker()
const file = selection[0]
return {
nativeHandle: file,
}
} catch (_) {
return 'aborted'
}
}
async readFile(
fileHandle: WebFileHandleRead,
onBytes: (bytes: Uint8Array, isLast: boolean) => Promise<void>,
): Promise<FileSystemResult> {
const file = await fileHandle.nativeHandle.getFile()
const stream = file.stream() as unknown as ReadableStream
const reader = stream.getReader()
let previousChunk: Uint8Array
const processChunk = async (result: ReadableStreamDefaultReadResult<Uint8Array>): Promise<void> => {
if (result.done) {
await onBytes(previousChunk, true)
return
}
if (previousChunk) {
await onBytes(previousChunk, false)
}
previousChunk = result.value
return reader.read().then(processChunk)
}
await reader.read().then(processChunk)
return 'success'
}
}

View File

@@ -0,0 +1,75 @@
import { FileReaderInterface } from './../Interface/FileReader'
import { ByteChunker } from '../Chunker/ByteChunker'
import { OnChunkCallback, FileSelectionResponse } from '../types'
interface StreamingFileReaderInterface {
getFilesFromHandles(handles: FileSystemFileHandle[]): Promise<File[]>
}
/**
* The File System Access API File Picker
* https://developer.mozilla.org/en-US/docs/Web/API/File_System_Access_API
*/
export const StreamingFileReader: StreamingFileReaderInterface & FileReaderInterface = {
getFilesFromHandles,
selectFiles,
readFile,
available,
maximumFileSize,
}
function maximumFileSize(): number | undefined {
return undefined
}
function getFilesFromHandles(handles: FileSystemFileHandle[]): Promise<File[]> {
return Promise.all(handles.map((handle) => handle.getFile()))
}
async function selectFiles(): Promise<File[]> {
let selectedFilesHandles: FileSystemFileHandle[]
try {
selectedFilesHandles = await window.showOpenFilePicker({ multiple: true })
} catch (error) {
selectedFilesHandles = []
}
return getFilesFromHandles(selectedFilesHandles)
}
async function readFile(
file: File,
minimumChunkSize: number,
onChunk: OnChunkCallback,
): Promise<FileSelectionResponse> {
const byteChunker = new ByteChunker(minimumChunkSize, onChunk)
const stream = file.stream() as unknown as ReadableStream
const reader = stream.getReader()
let previousChunk: Uint8Array
const processChunk = async (result: ReadableStreamDefaultReadResult<Uint8Array>): Promise<void> => {
if (result.done) {
await byteChunker.addBytes(previousChunk, true)
return
}
if (previousChunk) {
await byteChunker.addBytes(previousChunk, false)
}
previousChunk = result.value
return reader.read().then(processChunk)
}
await reader.read().then(processChunk)
return {
name: file.name,
mimeType: file.type,
}
}
function available(): boolean {
return window.showOpenFilePicker != undefined
}

View File

@@ -0,0 +1,49 @@
/**
* The File System Access API File Picker
* https://developer.mozilla.org/en-US/docs/Web/API/File_System_Access_API
*/
export class StreamingFileSaver {
public loggingEnabled = false
private writableStream!: FileSystemWritableFileStream
constructor(private name: string) {}
private log(...args: any[]): void {
if (!this.loggingEnabled) {
return
}
// eslint-disable-next-line no-console
console.log(args)
}
static available(): boolean {
return window.showSaveFilePicker != undefined
}
/** This function must be called in response to a user interaction, otherwise, it will be rejected by the browser. */
async selectFileToSaveTo(): Promise<void> {
this.log('Showing save file picker')
const downloadHandle = await window.showSaveFilePicker({
suggestedName: this.name,
})
this.writableStream = await downloadHandle.createWritable()
}
async pushBytes(bytes: Uint8Array): Promise<void> {
if (!this.writableStream) {
throw Error('Must call selectFileToSaveTo first')
}
this.log('Writing chunk to disk of size', bytes.length)
await this.writableStream.write(bytes)
}
async finish(): Promise<void> {
if (!this.writableStream) {
throw Error('Must call selectFileToSaveTo first')
}
this.log('Closing write stream')
await this.writableStream.close()
}
}

View File

@@ -0,0 +1,7 @@
export type EncryptedBytes = {
encryptedBytes: Uint8Array
}
export type DecryptedBytes = {
decryptedBytes: Uint8Array
}

View File

@@ -0,0 +1,11 @@
export * from './types'
export * from './Classic/ClassicReader'
export * from './Classic/ClassicSaver'
export * from './Streaming/StreamingReader'
export * from './Streaming/StreamingSaver'
export * from './Streaming/StreamingApi'
export * from './utils'
export * from './Chunker/ByteChunker'
export * from './Chunker/OrderedByteChunker'
export * from './Cache/FileMemoryCache'
export * from './TypedBytes'

View File

@@ -0,0 +1,6 @@
export type OnChunkCallback = (chunk: Uint8Array, index: number, isLast: boolean) => Promise<void>
export type FileSelectionResponse = {
name: string
mimeType: string
}

View File

@@ -0,0 +1,77 @@
import { formatSizeToReadableString, parseFileName } from './utils'
describe('utils', () => {
describe('parseFileName', () => {
it('should parse regular filenames', () => {
const fileName = 'test.txt'
const { name, ext } = parseFileName(fileName)
expect(name).toBe('test')
expect(ext).toBe('txt')
})
it('should parse filenames with multiple dots', () => {
const fileName = 'Screen Shot 2022-03-06 at 12.13.32 PM.png'
const { name, ext } = parseFileName(fileName)
expect(name).toBe('Screen Shot 2022-03-06 at 12.13.32 PM')
expect(ext).toBe('png')
})
it('should parse filenames without extensions', () => {
const fileName = 'extensionless'
const { name, ext } = parseFileName(fileName)
expect(name).toBe('extensionless')
expect(ext).toBe('')
})
})
describe('formatSizeToReadableString', () => {
it('should show as bytes if less than 1KB', () => {
const size = 1_023
const formattedSize = formatSizeToReadableString(size)
expect(formattedSize).toBe('1023 B')
})
it('should format as KB', () => {
const size = 1_024
const formattedSize = formatSizeToReadableString(size)
expect(formattedSize).toBe('1 KB')
})
it('should format as MB', () => {
const size = 1_048_576
const formattedSize = formatSizeToReadableString(size)
expect(formattedSize).toBe('1 MB')
})
it('should format as GB', () => {
const size = 1_073_741_824
const formattedSize = formatSizeToReadableString(size)
expect(formattedSize).toBe('1 GB')
})
it('should only show fixed-point notation if calculated size is not an integer', () => {
const size1 = 1_048_576
const size2 = 1_572_864
const formattedSize1 = formatSizeToReadableString(size1)
const formattedSize2 = formatSizeToReadableString(size2)
expect(formattedSize1).toBe('1 MB')
expect(formattedSize2).toBe('1.50 MB')
})
})
})

View File

@@ -0,0 +1,56 @@
export async function readFile(file: File): Promise<Uint8Array> {
const reader = new FileReader()
reader.readAsArrayBuffer(file)
return new Promise((resolve) => {
reader.onload = (readerEvent) => {
const target = readerEvent.target as FileReader
const content = target.result as ArrayBuffer
resolve(new Uint8Array(content))
}
})
}
export function parseFileName(fileName: string): {
name: string
ext: string
} {
const pattern = /(?:\.([^.]+))?$/
const extMatches = pattern.exec(fileName)
const ext = extMatches?.[1] || ''
const name = fileName.includes('.') ? fileName.substring(0, fileName.lastIndexOf('.')) : fileName
return { name, ext }
}
export function saveFile(name: string, bytes: Uint8Array): void {
const link = document.createElement('a')
const blob = new Blob([bytes], {
type: 'text/plain;charset=utf-8',
})
link.href = window.URL.createObjectURL(blob)
link.setAttribute('download', name)
document.body.appendChild(link)
link.click()
link.remove()
window.URL.revokeObjectURL(link.href)
}
const BYTES_IN_ONE_KILOBYTE = 1_024
const BYTES_IN_ONE_MEGABYTE = 1_048_576
const BYTES_IN_ONE_GIGABYTE = 1_073_741_824
export function formatSizeToReadableString(bytes: number): string {
let size = bytes
let unit = 'B'
if (bytes >= BYTES_IN_ONE_GIGABYTE) {
size = bytes / BYTES_IN_ONE_GIGABYTE
unit = 'GB'
} else if (bytes >= BYTES_IN_ONE_MEGABYTE) {
size = bytes / BYTES_IN_ONE_MEGABYTE
unit = 'MB'
} else if (bytes >= BYTES_IN_ONE_KILOBYTE) {
size = bytes / BYTES_IN_ONE_KILOBYTE
unit = 'KB'
}
return `${Number.isInteger(size) ? size : size.toFixed(2)} ${unit}`
}