From d0bc1155b2a62fcb6f00cb7de2f5aa0411270570 Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Wed, 13 Apr 2022 14:21:20 +0900 Subject: [PATCH] feat(files): large file downloads, improved qol, and improve extension fixing --- components/views/files/view/View.html | 20 ++--- components/views/files/view/View.vue | 81 ++++++------------- libraries/Files/Fil.ts | 47 ++++------- libraries/Files/FilSystem.ts | 10 ++- libraries/Files/TextileFileSystem.ts | 1 - libraries/Files/remote/textile/Bucket.ts | 33 +++----- .../test/__snapshots__/FilSystem.test.ts.snap | 56 ++++++------- libraries/Files/types/filesystem.ts | 1 + package.json | 2 + plugins/thirdparty/persist.ts | 1 + store/ui/__snapshots__/state.test.ts.snap | 1 + store/ui/mutations.ts | 9 +++ store/ui/state.ts | 1 + store/ui/types.ts | 1 + 14 files changed, 113 insertions(+), 151 deletions(-) diff --git a/components/views/files/view/View.html b/components/views/files/view/View.html index e23e744dcb..d8f9b0158f 100644 --- a/components/views/files/view/View.html +++ b/components/views/files/view/View.html @@ -7,14 +7,16 @@
- + @@ -35,16 +37,14 @@
- - +
+ diff --git a/components/views/files/view/View.vue b/components/views/files/view/View.vue index cdc9cf4ecf..8fa5843b95 100644 --- a/components/views/files/view/View.vue +++ b/components/views/files/view/View.vue @@ -10,7 +10,6 @@ import { XIcon, LinkIcon, } from 'satellite-lucide-icons' -import { filetypeextension } from 'magic-bytes.js' import { Fil } from '~/libraries/Files/Fil' export default Vue.extend({ @@ -25,74 +24,40 @@ export default Vue.extend({ data() { return { file: undefined as Fil | undefined, - name: '' as string, - progress: 0 as number, } }, computed: { ...mapState(['ui']), - isLoading(): boolean { - return this.progress >= 0 && this.progress < 100 + isDownloading(): boolean { + return this.ui.fileDownloadList.includes(this.file?.name) }, }, - /** - */ - async created() { + created() { this.file = this.$FileSystem.getChild(this.ui.filePreview) as Fil - this.name = this.file?.name - - // if no file data available, pull encrypted file from textile bucket - if (!this.file.file) { - const fsFil: Fil = this.$FileSystem.getChild(this.file.name) as Fil - fsFil.file = await this.$TextileManager.bucket?.pullFile( - this.file.id, - this.file.name, - this.file.type, - this.file.size, - this.setProgress, - ) - } - // file extension according to file name - const fileExt = this.file.name - .slice(((this.file.name.lastIndexOf('.') - 1) >>> 0) + 2) - .toLowerCase() - // you only need the first 256 bytes or so to confirm file type - const buffer = new Uint8Array( - await this.file.file.slice(0, 256).arrayBuffer(), - ) - // file extension according to byte data - const dataExt = filetypeextension(buffer)[0] - - // magicbytes declares svg as xml, so we need to manually check - const decodedFile = new TextDecoder().decode(buffer) - if (decodedFile.includes('xmlns="http://www.w3.org/2000/svg"')) { - // if corrupted, set .svg extension - if (fileExt !== 'svg') { - this.name += '.svg' - } - return - } - - // if corrupted txt file - if (!dataExt && fileExt !== 'txt') { - this.name += '.txt' - return - } - - // if corrupted file with wrong extension, force the correct one - if (fileExt !== dataExt && dataExt) { - this.name += `.${dataExt}` - } }, methods: { /** - * @method setProgress - * @description set progress (% out of 100) while file is being pulled from textile bucket. passed as a callback - * @param num current progress in bytes - * @param size total file size in bytes + * @method download + * @description download file using stream saver, apply original extension if it was removed + * add name to store so the user doesn't start another download of the same file + * also takes a bit to get started for large files, this adds loading indicator */ - setProgress(num: number, size: number) { - this.progress = Math.floor((num / size) * 100) + async download() { + if (this.file) { + this.$store.commit('ui/addFileDownload', this.file.name) + const fileExt = this.file.name + .slice(((this.file.name.lastIndexOf('.') - 1) >>> 0) + 2) + .toLowerCase() + + await this.$TextileManager.bucket?.pullFileStream( + this.file.id, + this.file.extension === fileExt + ? this.file.name + : (this.file.name += `.${this.file.extension}`), + this.file.size, + ) + this.$store.commit('ui/removeFileDownload', this.file.name) + } }, /** * @method share diff --git a/libraries/Files/Fil.ts b/libraries/Files/Fil.ts index 294dc4a0cf..3a8877560a 100644 --- a/libraries/Files/Fil.ts +++ b/libraries/Files/Fil.ts @@ -6,8 +6,8 @@ import { FILE_TYPE } from './types/file' export class Fil extends Item { private _description: string = '' private _size: number = 0 - private _file: File | undefined private _thumbnail: string + private _extension: string /** * @constructor @@ -17,7 +17,6 @@ export class Fil extends Item { constructor({ id, name, - file, size, liked, shared, @@ -25,10 +24,10 @@ export class Fil extends Item { description, type, thumbnail, + extension, }: { id?: string name: string - file?: File size: number liked?: boolean shared?: boolean @@ -36,15 +35,19 @@ export class Fil extends Item { description?: string type?: FILE_TYPE thumbnail?: string + extension?: string }) { if (!size) { throw new Error(FileSystemErrors.FILE_SIZE) } super({ name, liked, shared, modified, id, type }) - this._file = file || undefined this._description = description || '' this._size = size this._thumbnail = thumbnail || '' + // set original extension in case user changes it during rename + this._extension = + extension || + name.slice(((name.lastIndexOf('.') - 1) >>> 0) + 2).toLowerCase() } /** @@ -62,7 +65,6 @@ export class Fil extends Item { get copy(): Fil { return new Fil({ name: `${this.name} copy`, - file: this._file, size: this.size, modified: this.modified, liked: this.liked, @@ -70,12 +72,13 @@ export class Fil extends Item { description: this.description, type: this.type as FILE_TYPE, thumbnail: this.thumbnail, + extension: this.extension, }) } /** * @getter size - * @returns file size + * @returns {number} file size */ get size(): number { return this._size @@ -83,42 +86,26 @@ export class Fil extends Item { /** * @getter modified - * @returns last modified timestamp + * @returns {number} last modified timestamp */ get modified(): number { return this.modifiedVal } - /** - * @getter file - * @returns file object fetched from textile bucket - */ - get file(): File | undefined { - return this._file - } - - /** - * @setter file - * @param {File} file file object - */ - set file(file: File | undefined) { - this._file = file - } - /** * @getter url - * @returns link of locally stored File for image preview and downloads + * @returns {string} link of localally stored File for image preview and downloads */ - get url(): string { - return this.file ? URL.createObjectURL(this.file) : '' + get thumbnail(): string { + return this._thumbnail } /** - * @getter url - * @returns link of localally stored File for image preview and downloads + * @getter extension + * @returns {string} content the content to set the file description to */ - get thumbnail(): string { - return this._thumbnail + get extension(): string { + return this._extension } /** diff --git a/libraries/Files/FilSystem.ts b/libraries/Files/FilSystem.ts index 1890386721..6886428b78 100644 --- a/libraries/Files/FilSystem.ts +++ b/libraries/Files/FilSystem.ts @@ -157,6 +157,7 @@ export class FilSystem { description, modified, thumbnail, + extension, } = item return { id, @@ -168,6 +169,7 @@ export class FilSystem { description, modified, thumbnail, + extension, } } const { id, name, liked, shared, type, modified } = item @@ -213,6 +215,7 @@ export class FilSystem { description, modified, thumbnail, + extension, } = item as ExportFile const type = item.type as FILE_TYPE this.createFile({ @@ -225,6 +228,7 @@ export class FilSystem { type, modified, thumbnail, + extension, }) } if ((Object.values(DIRECTORY_TYPE) as string[]).includes(item.type)) { @@ -248,7 +252,6 @@ export class FilSystem { public createFile({ id, name, - file, size, liked, shared, @@ -256,10 +259,10 @@ export class FilSystem { type, modified, thumbnail, + extension, }: { id?: string name: string - file?: File size: number liked?: boolean shared?: boolean @@ -267,11 +270,11 @@ export class FilSystem { type?: FILE_TYPE modified?: number thumbnail?: string + extension?: string }): Fil | null { const newFile = new Fil({ id, name, - file, size, liked, shared, @@ -279,6 +282,7 @@ export class FilSystem { type, modified, thumbnail, + extension, }) const inserted = this.addChild(newFile) return inserted ? newFile : null diff --git a/libraries/Files/TextileFileSystem.ts b/libraries/Files/TextileFileSystem.ts index 96dbc6de05..09942be2ad 100644 --- a/libraries/Files/TextileFileSystem.ts +++ b/libraries/Files/TextileFileSystem.ts @@ -35,7 +35,6 @@ export class TextileFileSystem extends FilSystem { this.createFile({ id, name: file.name, - file, size: file.size, type: Object.values(FILE_TYPE).includes(type) ? type : FILE_TYPE.GENERIC, thumbnail: await this._createThumbnail(file, byteType), diff --git a/libraries/Files/remote/textile/Bucket.ts b/libraries/Files/remote/textile/Bucket.ts index e23e2f77dc..1513ebbdd7 100644 --- a/libraries/Files/remote/textile/Bucket.ts +++ b/libraries/Files/remote/textile/Bucket.ts @@ -1,4 +1,5 @@ import { Buckets, RemovePathResponse, Root } from '@textile/hub' +import { createWriteStream } from 'streamsaver' import { RFM } from '../abstracts/RFM.abstract' import { RFMInterface } from '../interface/RFM.interface' import { Config } from '~/config' @@ -7,7 +8,6 @@ import { FileSystemExport, FILESYSTEM_TYPE, } from '~/libraries/Files/types/filesystem' -import { FILE_TYPE } from '~/libraries/Files/types/file' export class Bucket extends RFM implements RFMInterface { private _textile: TextileInitializationData @@ -158,33 +158,24 @@ export class Bucket extends RFM implements RFMInterface { * @method pullFile * @description fetch encrypted file from bucket * @param {string} id file path in bucket - * @param {string} type file mime type - * @param {Function} progressCallback used to show progress meter in componment that calls this method + * @param {string} name file name + * @param {number} size file size to show progress in browser * @returns Promise of File */ - async pullFile( - id: string, - name: string, - type: string, - size: number, - progressCallback: Function, - ): Promise { + async pullFileStream(id: string, name: string, size: number) { if (!this.buckets || !this.key) { throw new Error('Bucket or bucket key not found') } + const fileStream = createWriteStream(name, { size }) + const writer = fileStream.getWriter() - const data = [] - for await (const bytes of this.buckets.pullPath(this.key, id, { - progress: (num) => { - progressCallback(num, size) - }, - })) { - data.push(bytes) + window.onunload = () => writer.abort() + + for await (const bytes of this.buckets.pullPath(this.key, id)) { + console.log(bytes) + writer.write(bytes) } - // if type is unknown(generic), then don't use in File constructor - return new File(data, name, { - type: type === FILE_TYPE.GENERIC ? '' : type, - }) + writer.close() } /** diff --git a/libraries/Files/test/__snapshots__/FilSystem.test.ts.snap b/libraries/Files/test/__snapshots__/FilSystem.test.ts.snap index 74b7313130..19ea3b9eff 100644 --- a/libraries/Files/test/__snapshots__/FilSystem.test.ts.snap +++ b/libraries/Files/test/__snapshots__/FilSystem.test.ts.snap @@ -25,7 +25,7 @@ Array [ "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -38,7 +38,7 @@ Array [ }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -51,7 +51,7 @@ Array [ }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -64,7 +64,7 @@ Array [ }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -129,7 +129,7 @@ Array [ "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -142,7 +142,7 @@ Array [ }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -155,7 +155,7 @@ Array [ }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -168,7 +168,7 @@ Array [ }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -214,7 +214,7 @@ Array [ "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -227,7 +227,7 @@ Array [ }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -240,7 +240,7 @@ Array [ }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -253,7 +253,7 @@ Array [ }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -346,7 +346,7 @@ Array [ "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -359,7 +359,7 @@ Array [ }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -372,7 +372,7 @@ Array [ }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -385,7 +385,7 @@ Array [ }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -449,7 +449,7 @@ FilSystem { "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -462,7 +462,7 @@ FilSystem { }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -475,7 +475,7 @@ FilSystem { }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -488,7 +488,7 @@ FilSystem { }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -554,7 +554,7 @@ FilSystem { "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -567,7 +567,7 @@ FilSystem { }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -580,7 +580,7 @@ FilSystem { }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -593,7 +593,7 @@ FilSystem { }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -659,7 +659,7 @@ FilSystem { "_children": Map { "testfile.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -672,7 +672,7 @@ FilSystem { }, "testpng2.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -685,7 +685,7 @@ FilSystem { }, "abc.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, @@ -698,7 +698,7 @@ FilSystem { }, "cc123.png" => Fil { "_description": "Test file description", - "_file": undefined, + "_extension": "png", "_id": "testid", "_liked": false, "_modified": 1645617999076, diff --git a/libraries/Files/types/filesystem.ts b/libraries/Files/types/filesystem.ts index 0b0e8edc43..7b36fb6581 100644 --- a/libraries/Files/types/filesystem.ts +++ b/libraries/Files/types/filesystem.ts @@ -17,6 +17,7 @@ export interface ExportFile extends ExportSharedProps { size: number description: string thumbnail: string + extension: string } export interface ExportDirectory extends ExportSharedProps { diff --git a/package.json b/package.json index 3f74748416..bfc3c98085 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "simple-markdown": "^0.7.3", "simple-peer": "^9.11.1", "skaler": "^1.0.7", + "streamsaver": "^2.0.6", "swiper": "^8.0.7", "uuid": "^8.3.2", "v-calendar": "^2.4.1", @@ -135,6 +136,7 @@ "@types/mousetrap": "^1.6.9", "@types/node": "^16.11.26", "@types/simple-peer": "^9.11.4", + "@types/streamsaver": "^2.0.1", "@types/uuid": "^8.3.4", "@types/vue-markdown": "^2.2.1", "@typescript-eslint/eslint-plugin": "^5.13.0", diff --git a/plugins/thirdparty/persist.ts b/plugins/thirdparty/persist.ts index d0bf7cdc46..89f1fcdc45 100644 --- a/plugins/thirdparty/persist.ts +++ b/plugins/thirdparty/persist.ts @@ -38,6 +38,7 @@ const commonProperties = [ 'ui.modals', 'ui.isScrollOver', 'ui.filePreview', + 'ui.fileDownloadList', ] const propertiesNoStorePin = [ diff --git a/store/ui/__snapshots__/state.test.ts.snap b/store/ui/__snapshots__/state.test.ts.snap index 6583de59a4..1a580ac654 100644 --- a/store/ui/__snapshots__/state.test.ts.snap +++ b/store/ui/__snapshots__/state.test.ts.snap @@ -29,6 +29,7 @@ Object { "route": "emotes", "show": false, }, + "fileDownloadList": Array [], "filePreview": undefined, "fileSort": Object { "asc": true, diff --git a/store/ui/mutations.ts b/store/ui/mutations.ts index edabb80d93..fc29610ad9 100644 --- a/store/ui/mutations.ts +++ b/store/ui/mutations.ts @@ -368,4 +368,13 @@ export default { setSwiperSlideIndex(state: UIState, index: number) { state.swiperSlideIndex = index }, + addFileDownload(state: UIState, name: string) { + state.fileDownloadList.push(name) + }, + removeFileDownload(state: UIState, name: string) { + const index = state.fileDownloadList.indexOf(name) + if (index > -1) { + state.fileDownloadList.splice(index, 1) + } + }, } diff --git a/store/ui/state.ts b/store/ui/state.ts index 59508465fa..8b35f7509e 100644 --- a/store/ui/state.ts +++ b/store/ui/state.ts @@ -76,6 +76,7 @@ const InitialUIState = (): UIState => ({ isLoadingFileIndex: false, renameCurrentName: undefined, filePreview: undefined, + fileDownloadList: [], chatImageOverlay: undefined, fileSort: { category: FileSortEnum.MODIFIED, diff --git a/store/ui/types.ts b/store/ui/types.ts index aec1b863b3..cae5a51613 100644 --- a/store/ui/types.ts +++ b/store/ui/types.ts @@ -215,6 +215,7 @@ export interface UIState { isLoadingFileIndex: boolean renameCurrentName?: string filePreview?: string + fileDownloadList: string[] chatImageOverlay?: ImageMessage fileSort: FileSort swiperSlideIndex: number