rmorawski/standardnotes-app-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: download and preview files from local backups automatically, if a local backup is available (#2076)

Browse Source
This commit is contained in:
Mo
2022-12-01 11:56:28 -06:00
committed by GitHub
parent e07fed267f
commit 28e43d37c0
34 changed files with 739 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
packages/services/src/Domain/Backups/BackupService.spec.ts Normal file
View File

@@ -0,0 +1,109 @@
import { StatusServiceInterface } from './../Status/StatusServiceInterface'
import { FilesBackupService } from './BackupService'
import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common'
import { EncryptionProviderInterface } from '@standardnotes/encryption'
import { ItemManagerInterface } from '../Item/ItemManagerInterface'
import { InternalEventBusInterface } from '..'
import { AlertService } from '../Alert/AlertService'
import { ApiServiceInterface } from '../Api/ApiServiceInterface'
import { SyncServiceInterface } from '../Sync/SyncServiceInterface'
import { FileBackupsDevice } from '@standardnotes/files'
describe('backup service', () => {
let apiService: ApiServiceInterface
let itemManager: ItemManagerInterface
let syncService: SyncServiceInterface
let alertService: AlertService
let crypto: PureCryptoInterface
let status: StatusServiceInterface
let encryptor: EncryptionProviderInterface
let internalEventBus: InternalEventBusInterface
let backupService: FilesBackupService
let device: FileBackupsDevice
beforeEach(() => {
apiService = {} as jest.Mocked<ApiServiceInterface>
apiService.addEventObserver = jest.fn()
apiService.createFileValetToken = jest.fn()
apiService.downloadFile = jest.fn()
apiService.deleteFile = jest.fn().mockReturnValue({})
itemManager = {} as jest.Mocked<ItemManagerInterface>
itemManager.createItem = jest.fn()
itemManager.createTemplateItem = jest.fn().mockReturnValue({})
itemManager.setItemToBeDeleted = jest.fn()
itemManager.addObserver = jest.fn()
itemManager.changeItem = jest.fn()
status = {} as jest.Mocked<StatusServiceInterface>
device = {} as jest.Mocked<FileBackupsDevice>
device.getFileBackupReadToken = jest.fn()
device.readNextChunk = jest.fn()
syncService = {} as jest.Mocked<SyncServiceInterface>
syncService.sync = jest.fn()
encryptor = {} as jest.Mocked<EncryptionProviderInterface>
alertService = {} as jest.Mocked<AlertService>
alertService.confirm = jest.fn().mockReturnValue(true)
alertService.alert = jest.fn()
crypto = {} as jest.Mocked<PureCryptoInterface>
crypto.base64Decode = jest.fn()
internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
internalEventBus.publish = jest.fn()
backupService = new FilesBackupService(itemManager, apiService, encryptor, device, status, crypto, internalEventBus)
crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({
state: {},
} as StreamEncryptor)
crypto.xchacha20StreamDecryptorPush = jest.fn().mockReturnValue({ message: new Uint8Array([0xaa]), tag: 0 })
crypto.xchacha20StreamInitEncryptor = jest.fn().mockReturnValue({
header: 'some-header',
state: {},
} as StreamEncryptor)
crypto.xchacha20StreamEncryptorPush = jest.fn().mockReturnValue(new Uint8Array())
})
describe('readEncryptedFileFromBackup', () => {
it('return failed if no backup', async () => {
backupService.getFileBackupInfo = jest.fn().mockReturnValue(undefined)
const result = await backupService.readEncryptedFileFromBackup('123', async () => {})
expect(result).toEqual('failed')
})
it('return success if backup', async () => {
backupService.getFileBackupInfo = jest.fn().mockReturnValue({})
device.readNextChunk = jest.fn().mockReturnValue({ chunk: new Uint8Array([]), isLast: true, progress: undefined })
const result = await backupService.readEncryptedFileFromBackup('123', async () => {})
expect(result).toEqual('success')
})
it('should loop through all chunks until last', async () => {
backupService.getFileBackupInfo = jest.fn().mockReturnValue({})
const expectedChunkCount = 3
let receivedChunkCount = 0
const mockFn = (device.readNextChunk = jest.fn().mockImplementation(() => {
receivedChunkCount++
return { chunk: new Uint8Array([]), isLast: receivedChunkCount === expectedChunkCount, progress: undefined }
}))
await backupService.readEncryptedFileFromBackup('123', async () => {})
expect(mockFn.mock.calls.length).toEqual(expectedChunkCount)
})
})
})

43
packages/services/src/Domain/Backups/BackupService.ts
View File

@@ -8,13 +8,17 @@ import {
FileBackupsDevice,
FileBackupsMapping,
FileBackupRecord,
OnChunkCallback,
BackupServiceInterface,
} from '@standardnotes/files'
import { InternalEventBusInterface } from '../Internal/InternalEventBusInterface'
import { ItemManagerInterface } from '../Item/ItemManagerInterface'
import { AbstractService } from '../Service/AbstractService'
import { StatusServiceInterface } from '../Status/StatusServiceInterface'
import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
import { log, LoggingDomain } from '../Logging'
export class FilesBackupService extends AbstractService {
export class FilesBackupService extends AbstractService implements BackupServiceInterface {
private itemsObserverDisposer: () => void
private pendingFiles = new Set<Uuid>()
private mappingCache?: FileBackupsMapping['files']
@@ -25,6 +29,7 @@ export class FilesBackupService extends AbstractService {
private encryptor: EncryptionProviderInterface,
private device: FileBackupsDevice,
private status: StatusServiceInterface,
private crypto: PureCryptoInterface,
protected override internalEventBus: InternalEventBusInterface,
) {
super(internalEventBus)
@@ -43,7 +48,14 @@ export class FilesBackupService extends AbstractService {
}
override deinit() {
super.deinit()
this.itemsObserverDisposer()
;(this.items as unknown) = undefined
;(this.api as unknown) = undefined
;(this.encryptor as unknown) = undefined
;(this.device as unknown) = undefined
;(this.status as unknown) = undefined
;(this.crypto as unknown) = undefined
}
public isFilesBackupsEnabled(): Promise<boolean> {
@@ -98,7 +110,7 @@ export class FilesBackupService extends AbstractService {
return this.mappingCache ?? (await this.getBackupsMappingFromDisk())
}
public async getFileBackupInfo(file: FileItem): Promise<FileBackupRecord | undefined> {
public async getFileBackupInfo(file: { uuid: string }): Promise<FileBackupRecord | undefined> {
const mapping = await this.getBackupsMappingFromCache()
const record = mapping[file.uuid]
return record
@@ -138,7 +150,34 @@ export class FilesBackupService extends AbstractService {
this.invalidateMappingCache()
}
async readEncryptedFileFromBackup(uuid: string, onChunk: OnChunkCallback): Promise<'success' | 'failed' | 'aborted'> {
const fileBackup = await this.getFileBackupInfo({ uuid })
if (!fileBackup) {
return 'failed'
}
const token = await this.device.getFileBackupReadToken(fileBackup)
let readMore = true
let index = 0
while (readMore) {
const { chunk, isLast, progress } = await this.device.readNextChunk(token)
await onChunk({ data: chunk, index, isLast, progress })
readMore = !isLast
index++
}
return 'success'
}
private async performBackupOperation(file: FileItem): Promise<'success' | 'failed' | 'aborted'> {
log(LoggingDomain.FilesBackups, 'Backing up file locally', file.uuid)
const messageId = this.status.addMessage(`Backing up file ${file.name}...`)
const encryptedFile = await this.encryptor.encryptSplitSingle({

70
packages/services/src/Domain/Files/FileService.spec.ts
View File

@@ -1,15 +1,14 @@
import { PureCryptoInterface, StreamEncryptor } from '@standardnotes/sncrypto-common'
import { FileItem } from '@standardnotes/models'
import { EncryptionProviderInterface } from '@standardnotes/encryption'
import { ItemManagerInterface } from '../Item/ItemManagerInterface'
import { ChallengeServiceInterface } from '../Challenge'
import { InternalEventBusInterface } from '..'
import { AlertService } from '../Alert/AlertService'
import { ApiServiceInterface } from '../Api/ApiServiceInterface'
import { SyncServiceInterface } from '../Sync/SyncServiceInterface'
import { FileService } from './FileService'
import { BackupServiceInterface } from '@standardnotes/files'
describe('fileService', () => {
let apiService: ApiServiceInterface
@@ -21,13 +20,33 @@ describe('fileService', () => {
let fileService: FileService
let encryptor: EncryptionProviderInterface
let internalEventBus: InternalEventBusInterface
let backupService: BackupServiceInterface
beforeEach(() => {
apiService = {} as jest.Mocked<ApiServiceInterface>
apiService.addEventObserver = jest.fn()
apiService.createFileValetToken = jest.fn()
apiService.downloadFile = jest.fn()
apiService.deleteFile = jest.fn().mockReturnValue({})
const numChunks = 1
apiService.downloadFile = jest
.fn()
.mockImplementation(
(
_file: string,
_chunkIndex: number,
_apiToken: string,
_rangeStart: number,
onBytesReceived: (bytes: Uint8Array) => void,
) => {
return new Promise<void>((resolve) => {
for (let i = 0; i < numChunks; i++) {
onBytesReceived(Uint8Array.from([0xaa]))
}
resolve()
})
},
)
itemManager = {} as jest.Mocked<ItemManagerInterface>
itemManager.createItem = jest.fn()
@@ -52,6 +71,10 @@ describe('fileService', () => {
internalEventBus = {} as jest.Mocked<InternalEventBusInterface>
internalEventBus.publish = jest.fn()
backupService = {} as jest.Mocked<BackupServiceInterface>
backupService.readEncryptedFileFromBackup = jest.fn()
backupService.getFileBackupInfo = jest.fn()
fileService = new FileService(
apiService,
itemManager,
@@ -61,6 +84,7 @@ describe('fileService', () => {
alertService,
crypto,
internalEventBus,
backupService,
)
crypto.xchacha20StreamInitDecryptor = jest.fn().mockReturnValue({
@@ -110,6 +134,8 @@ describe('fileService', () => {
decryptedSize: 100_000,
} as jest.Mocked<FileItem>
apiService.downloadFile = jest.fn()
await fileService.downloadFile(file, async () => {
return Promise.resolve()
})
@@ -118,4 +144,42 @@ describe('fileService', () => {
expect(fileService['encryptedCache'].get(file.uuid)).toBeFalsy()
})
it('should download file from network if no backup', async () => {
const file = {
uuid: '1',
decryptedSize: 100_000,
encryptedSize: 101_000,
encryptedChunkSizes: [101_000],
} as jest.Mocked<FileItem>
backupService.getFileBackupInfo = jest.fn().mockReturnValue(undefined)
const downloadMock = apiService.downloadFile as jest.Mock
await fileService.downloadFile(file, async () => {
return Promise.resolve()
})
expect(downloadMock).toHaveBeenCalledTimes(1)
})
it('should download file from local backup if it exists', async () => {
const file = {
uuid: '1',
decryptedSize: 100_000,
encryptedSize: 101_000,
encryptedChunkSizes: [101_000],
} as jest.Mocked<FileItem>
backupService.getFileBackupInfo = jest.fn().mockReturnValue({})
const downloadMock = (apiService.downloadFile = jest.fn())
await fileService.downloadFile(file, async () => {
return Promise.resolve()
})
expect(downloadMock).toHaveBeenCalledTimes(0)
})
})

87
packages/services/src/Domain/Files/FileService.ts
View File

@@ -19,7 +19,7 @@ import {
FileDecryptor,
FileDownloadProgress,
FilesClientInterface,
readAndDecryptBackupFile,
readAndDecryptBackupFileUsingFileSystemAPI,
FilesApiInterface,
FileBackupsConstantsV1,
FileBackupMetadataFile,
@@ -30,8 +30,9 @@ import {
DecryptedBytes,
OrderedByteChunker,
FileMemoryCache,
readAndDecryptBackupFileUsingBackupService,
BackupServiceInterface,
} from '@standardnotes/files'
import { AlertService } from '../Alert/AlertService'
import { ChallengeServiceInterface } from '../Challenge'
import { InternalEventBusInterface } from '../Internal/InternalEventBusInterface'
@@ -39,6 +40,7 @@ import { ItemManagerInterface } from '../Item/ItemManagerInterface'
import { AbstractService } from '../Service/AbstractService'
import { SyncServiceInterface } from '../Sync/SyncServiceInterface'
import { DecryptItemsKeyWithUserFallback } from '../Encryption/Functions'
import { log, LoggingDomain } from '../Logging'
const OneHundredMb = 100 * 1_000_000
@@ -54,6 +56,7 @@ export class FileService extends AbstractService implements FilesClientInterface
private alertService: AlertService,
private crypto: PureCryptoInterface,
protected override internalEventBus: InternalEventBusInterface,
private backupsService?: BackupServiceInterface,
) {
super(internalEventBus)
}
@@ -158,8 +161,8 @@ export class FileService extends AbstractService implements FilesClientInterface
let decryptedAggregate = new Uint8Array()
const orderedChunker = new OrderedByteChunker(file.encryptedChunkSizes, async (encryptedBytes) => {
const decryptedBytes = decryptOperation.decryptBytes(encryptedBytes)
const orderedChunker = new OrderedByteChunker(file.encryptedChunkSizes, 'memcache', async (chunk) => {
const decryptedBytes = decryptOperation.decryptBytes(chunk.data)
if (decryptedBytes) {
decryptedAggregate = new Uint8Array([...decryptedAggregate, ...decryptedBytes.decryptedBytes])
@@ -173,36 +176,60 @@ export class FileService extends AbstractService implements FilesClientInterface
public async downloadFile(
file: FileItem,
onDecryptedBytes: (decryptedBytes: Uint8Array, progress?: FileDownloadProgress) => Promise<void>,
onDecryptedBytes: (decryptedBytes: Uint8Array, progress: FileDownloadProgress) => Promise<void>,
): Promise<ClientDisplayableError | undefined> {
const cachedBytes = this.encryptedCache.get(file.uuid)
if (cachedBytes) {
const decryptedBytes = await this.decryptCachedEntry(file, cachedBytes)
await onDecryptedBytes(decryptedBytes.decryptedBytes, undefined)
await onDecryptedBytes(decryptedBytes.decryptedBytes, {
encryptedFileSize: cachedBytes.encryptedBytes.length,
encryptedBytesDownloaded: cachedBytes.encryptedBytes.length,
encryptedBytesRemaining: 0,
percentComplete: 100,
source: 'memcache',
})
return undefined
}
const addToCache = file.encryptedSize < this.encryptedCache.maxSize
const fileBackup = await this.backupsService?.getFileBackupInfo(file)
let cacheEntryAggregate = new Uint8Array()
if (this.backupsService && fileBackup) {
log(LoggingDomain.FilesService, 'Downloading file from backup', fileBackup)
const operation = new DownloadAndDecryptFileOperation(file, this.crypto, this.api)
await readAndDecryptBackupFileUsingBackupService(file, this.backupsService, this.crypto, async (chunk) => {
log(LoggingDomain.FilesService, 'Got local file chunk', chunk.progress)
const result = await operation.run(async ({ decrypted, encrypted, progress }): Promise<void> => {
if (addToCache) {
cacheEntryAggregate = new Uint8Array([...cacheEntryAggregate, ...encrypted.encryptedBytes])
return onDecryptedBytes(chunk.data, chunk.progress)
})
log(LoggingDomain.FilesService, 'Finished downloading file from backup')
return undefined
} else {
log(LoggingDomain.FilesService, 'Downloading file from network')
const addToCache = file.encryptedSize < this.encryptedCache.maxSize
let cacheEntryAggregate = new Uint8Array()
const operation = new DownloadAndDecryptFileOperation(file, this.crypto, this.api)
const result = await operation.run(async ({ decrypted, encrypted, progress }): Promise<void> => {
if (addToCache) {
cacheEntryAggregate = new Uint8Array([...cacheEntryAggregate, ...encrypted.encryptedBytes])
}
return onDecryptedBytes(decrypted.decryptedBytes, progress)
})
if (addToCache && cacheEntryAggregate.byteLength > 0) {
this.encryptedCache.add(file.uuid, { encryptedBytes: cacheEntryAggregate })
}
return onDecryptedBytes(decrypted.decryptedBytes, progress)
})
if (addToCache) {
this.encryptedCache.add(file.uuid, { encryptedBytes: cacheEntryAggregate })
return result.error
}
return result.error
}
public async deleteFile(file: FileItem): Promise<ClientDisplayableError | undefined> {
@@ -294,9 +321,15 @@ export class FileService extends AbstractService implements FilesClientInterface
return destinationFileHandle
}
const result = await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => {
await fileSystem.saveBytes(destinationFileHandle, decryptedBytes)
})
const result = await readAndDecryptBackupFileUsingFileSystemAPI(
fileHandle,
file,
fileSystem,
this.crypto,
async (decryptedBytes) => {
await fileSystem.saveBytes(destinationFileHandle, decryptedBytes)
},
)
await fileSystem.closeFileWriteStream(destinationFileHandle)
@@ -310,9 +343,15 @@ export class FileService extends AbstractService implements FilesClientInterface
): Promise<Uint8Array> {
let bytes = new Uint8Array()
await readAndDecryptBackupFile(fileHandle, file, fileSystem, this.crypto, async (decryptedBytes) => {
bytes = new Uint8Array([...bytes, ...decryptedBytes])
})
await readAndDecryptBackupFileUsingFileSystemAPI(
fileHandle,
file,
fileSystem,
this.crypto,
async (decryptedBytes) => {
bytes = new Uint8Array([...bytes, ...decryptedBytes])
},
)
return bytes
}

33
packages/services/src/Domain/Logging.ts Normal file
View File

@@ -0,0 +1,33 @@
import { logWithColor } from '@standardnotes/utils'
declare const process: {
env: {
NODE_ENV: string | null | undefined
}
}
export const isDev = process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test'
export enum LoggingDomain {
FilesService,
FilesBackups,
}
const LoggingStatus: Record<LoggingDomain, boolean> = {
[LoggingDomain.FilesService]: false,
[LoggingDomain.FilesBackups]: false,
}
const LoggingColor: Record<LoggingDomain, string> = {
[LoggingDomain.FilesService]: 'blue',
[LoggingDomain.FilesBackups]: 'yellow',
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function log(domain: LoggingDomain, ...args: any[]): void {
if (!isDev || !LoggingStatus[domain]) {
return
}
logWithColor(LoggingDomain[domain], LoggingColor[domain], ...args)
}