From b73aa6e643b89b4ab7e25de3aabfc5f8e80d6d0e Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Fri, 4 Mar 2022 13:31:15 +0900 Subject: [PATCH 01/15] feat(files): encrypted bucket for personal files --- components/views/files/file/File.vue | 4 +-- cypress | 2 +- libraries/Files/Fil.ts | 17 +++++++++ libraries/Files/FilSystem.ts | 24 ++++++++++--- libraries/Files/TextileFileSystem.ts | 11 +----- libraries/Files/abstracts/Item.abstract.ts | 2 -- libraries/Files/remote/textile/Bucket.ts | 41 +++++++++++++--------- store/accounts/actions.ts | 2 +- store/textile/actions.ts | 2 +- 9 files changed, 66 insertions(+), 39 deletions(-) diff --git a/components/views/files/file/File.vue b/components/views/files/file/File.vue index 5281b082f2..c5589a0914 100644 --- a/components/views/files/file/File.vue +++ b/components/views/files/file/File.vue @@ -84,9 +84,7 @@ export default Vue.extend({ * @returns path inside textile bucket */ path(): string { - return this.item instanceof Fil - ? this.$Config.textile.browser + this.item.hash - : '' + return this.item instanceof Fil ? this.item.url : '' }, }, methods: { diff --git a/cypress b/cypress index 886227fa98..580df174fc 160000 --- a/cypress +++ b/cypress @@ -1 +1 @@ -Subproject commit 886227fa98fdc9185c93ff54ccbe22c54fdbd3d8 +Subproject commit 580df174fc274843fbbc1250c5ddf0cd7cdfebf3 diff --git a/libraries/Files/Fil.ts b/libraries/Files/Fil.ts index 1a098745a6..20042ecaa8 100644 --- a/libraries/Files/Fil.ts +++ b/libraries/Files/Fil.ts @@ -7,6 +7,7 @@ export class Fil extends Item { private _hash: string = '' private _description: string = '' private _size: number = 0 + private _file: File /** * @constructor @@ -15,6 +16,7 @@ export class Fil extends Item { */ constructor({ name, + file, hash, size, liked, @@ -24,6 +26,7 @@ export class Fil extends Item { type, }: { name: string + file: File hash: string size: number liked?: boolean @@ -33,6 +36,7 @@ export class Fil extends Item { type?: FILE_TYPE }) { super({ name: name || 'un-named file', liked, shared, modified }) + this._file = file this._description = description || '' this._hash = hash || '' this._size = size || 0 @@ -70,6 +74,7 @@ export class Fil extends Item { get copy(): Fil { return new Fil({ name: `${this.name} copy`, + file: this._file, hash: this.hash, size: this.size, modified: this.modified, @@ -96,6 +101,18 @@ export class Fil extends Item { return this.modifiedVal } + /** + * @getter file + * @returns last modified timestamp + */ + get file(): File { + return this._file + } + + get url(): string { + return URL.createObjectURL(this.file) + } + /** * @setter file description text * @param {string} content the content to set the file description to diff --git a/libraries/Files/FilSystem.ts b/libraries/Files/FilSystem.ts index 58be7a7768..60af4c296a 100644 --- a/libraries/Files/FilSystem.ts +++ b/libraries/Files/FilSystem.ts @@ -1,3 +1,4 @@ +import Vue from 'vue' import { matchSorter } from 'match-sorter' import { Directory } from './Directory' import { DIRECTORY_TYPE } from './types/directory' @@ -11,6 +12,7 @@ import { ExportDirectory, } from './types/filesystem' import { FILE_TYPE } from './types/file' +import { Bucket } from './remote/textile/Bucket' import { Config } from '~/config' export class FilSystem { @@ -139,6 +141,14 @@ export class FilSystem { return (this.totalSize / Config.personalFilesLimit) * 100 } + /** + * @getter bucket + * @returns {Bucket} bucket global to upload files to textile + */ + get bucket(): Bucket { + return Vue.prototype.$TextileManager.bucket + } + /** * @method exportChildren * @param {Item} item @@ -179,10 +189,10 @@ export class FilSystem { * @param {FileSystemExport} fs * @description sets global file system based on parameter. will be fetched from Bucket */ - public import(fs: FileSystemExport) { - fs.content.forEach((item) => { + public async import(fs: FileSystemExport) { + for (const item of fs.content) { this.importChildren(item) - }) + } this._version = fs.version } @@ -191,13 +201,16 @@ export class FilSystem { * @param {FileSystemExport} fs * @description recursively adds files and directories from JSON export */ - public importChildren(item: ExportItem) { + public async importChildren(item: ExportItem) { if ((Object.values(FILE_TYPE) as string[]).includes(item.type)) { const { name, hash, size, liked, shared, description, modified } = item as ExportFile const type = item.type as FILE_TYPE + const file = await this.bucket.pullFile(name, type) + console.log(file) this.createFile({ name, + file, hash, size, liked, @@ -227,6 +240,7 @@ export class FilSystem { */ public createFile({ name, + file, hash, size, liked, @@ -236,6 +250,7 @@ export class FilSystem { modified, }: { name: string + file: File hash: string size: number liked?: boolean @@ -246,6 +261,7 @@ export class FilSystem { }): Fil | null { const newFile = new Fil({ name, + file, hash, size, liked, diff --git a/libraries/Files/TextileFileSystem.ts b/libraries/Files/TextileFileSystem.ts index 89476f416e..a28cb8c9b8 100644 --- a/libraries/Files/TextileFileSystem.ts +++ b/libraries/Files/TextileFileSystem.ts @@ -1,18 +1,8 @@ import { PushPathResult } from '@textile/hub' -import Vue from 'vue' import { FilSystem } from './FilSystem' -import { Bucket } from './remote/textile/Bucket' import { FILE_TYPE } from './types/file' export class TextileFileSystem extends FilSystem { - /** - * @getter bucket - * @returns {Bucket} bucket global to upload files to textile - */ - get bucket() { - return Vue.prototype.$TextileManager.bucket - } - /** * @method uploadFile * @description Upload file to the bucket and create in the file system afterwards @@ -22,6 +12,7 @@ export class TextileFileSystem extends FilSystem { const result: PushPathResult = await this.bucket.pushFile(file) this.createFile({ name: file.name, + file, hash: result.path.path, size: file.size, type: (Object.values(FILE_TYPE) as string[]).includes(file.type) diff --git a/libraries/Files/abstracts/Item.abstract.ts b/libraries/Files/abstracts/Item.abstract.ts index a74a3cddd2..4de53c7f4d 100644 --- a/libraries/Files/abstracts/Item.abstract.ts +++ b/libraries/Files/abstracts/Item.abstract.ts @@ -111,7 +111,6 @@ export abstract class Item implements ItemInterface { */ toggleLiked() { this._liked = !this._liked - this._modified = Date.now() } /** @@ -120,7 +119,6 @@ export abstract class Item implements ItemInterface { */ shareItem() { this._shared = true - this._modified = Date.now() } /** diff --git a/libraries/Files/remote/textile/Bucket.ts b/libraries/Files/remote/textile/Bucket.ts index 4c0a8f288e..dee5ae820a 100644 --- a/libraries/Files/remote/textile/Bucket.ts +++ b/libraries/Files/remote/textile/Bucket.ts @@ -1,10 +1,4 @@ -import { - Buckets, - Path, - PushPathResult, - RemovePathResponse, - Root, -} from '@textile/hub' +import { Buckets, PushPathResult, RemovePathResponse, Root } from '@textile/hub' import { RFM } from '../abstracts/RFM.abstract' import { RFMInterface } from '../interface/RFM.interface' import { Config } from '~/config' @@ -57,24 +51,19 @@ export class Bucket extends RFM implements RFMInterface { this.buckets = await Buckets.withKeyInfo({ key: Config.textile.key }) await this.buckets.getToken(this._textile.identity) - const result = await this.buckets.getOrCreate(name) + const result = await this.buckets.getOrCreate(name, { encrypted: true }) if (!result.root) throw new Error(`failed to open bucket ${name}`) this.key = result.root.key try { - const path: Path | void = await this.buckets.listPath( + for await (const data of this.buckets.pullPath( this.key, Config.textile.fsTable, - ) - - const hash = path?.item?.path - - this._index = await fetch(Config.textile.browser + hash).then((res) => - res.json(), - ) - + )) { + this._index = JSON.parse(new TextDecoder().decode(data)) + } if (!this._index) throw new Error('Index not found') return this._index @@ -124,6 +113,24 @@ export class Bucket extends RFM implements RFMInterface { return await this.buckets.pushPath(this.key, file.name, file) } + /** + * @method pullFile + * @description Remove file from bucket + * @param {File} file file to be pulled + * @returns Promise whether it was uploaded or not + */ + async pullFile(name: string, type: string): Promise { + if (!this.buckets || !this.key) { + throw new Error('Bucket or bucket key not found') + } + + const data = [] + for await (const bytes of this.buckets.pullPath(this.key, name)) { + data.push(bytes) + } + return new File(data, name, { type: type || '' }) + } + /** * @method removeFile * @description Remove file from bucket diff --git a/store/accounts/actions.ts b/store/accounts/actions.ts index 792bead23d..c3a277c686 100644 --- a/store/accounts/actions.ts +++ b/store/accounts/actions.ts @@ -296,7 +296,7 @@ export default { const { pin } = state if (!textileInitialized && pin) { - dispatch( + await dispatch( 'textile/initialize', { id: payerAccount?.publicKey.toBase58(), diff --git a/store/textile/actions.ts b/store/textile/actions.ts index e13ccb4525..62b22ba54a 100644 --- a/store/textile/actions.ts +++ b/store/textile/actions.ts @@ -37,7 +37,7 @@ export default { if (fsExport) { const $FileSystem: FilSystem = Vue.prototype.$FileSystem - $FileSystem.import(fsExport) + await $FileSystem.import(fsExport) } }, /** From 0299e9b6785dc52520c4e0ded2579a4364256e78 Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Fri, 4 Mar 2022 15:46:57 +0900 Subject: [PATCH 02/15] fix(files): fix links, download, previews --- components/views/files/file/File.html | 2 +- components/views/files/file/File.vue | 7 +------ components/views/files/view/View.html | 6 +++--- components/views/files/view/View.vue | 3 --- libraries/Files/FilSystem.ts | 2 +- plugins/local/classLoader.ts | 4 ++-- 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/components/views/files/file/File.html b/components/views/files/file/File.html index 7661752110..748b4a2202 100644 --- a/components/views/files/file/File.html +++ b/components/views/files/file/File.html @@ -25,7 +25,7 @@
diff --git a/components/views/files/file/File.vue b/components/views/files/file/File.vue index c5589a0914..76ccc1f484 100644 --- a/components/views/files/file/File.vue +++ b/components/views/files/file/File.vue @@ -27,6 +27,7 @@ declare module 'vue/types/vue' { share: () => void rename: () => void delete: () => void + $filesize: (item: number) => string } } @@ -80,12 +81,6 @@ export default Vue.extend({ isArchive(): boolean { return Boolean(this.item.name.match(this.$Config.regex.archive)) }, - /** - * @returns path inside textile bucket - */ - path(): string { - return this.item instanceof Fil ? this.item.url : '' - }, }, methods: { /** diff --git a/components/views/files/view/View.html b/components/views/files/view/View.html index bfef2a00ac..14ff13b2fa 100644 --- a/components/views/files/view/View.html +++ b/components/views/files/view/View.html @@ -16,7 +16,7 @@ @@ -42,13 +42,13 @@
diff --git a/components/views/files/view/View.vue b/components/views/files/view/View.vue index 4fafc2b3c1..0849bafa4b 100644 --- a/components/views/files/view/View.vue +++ b/components/views/files/view/View.vue @@ -29,9 +29,6 @@ export default Vue.extend({ }, computed: { ...mapState(['ui']), - path(): string { - return this.$Config.textile.browser + this.file.hash - }, isImage(): boolean { return Boolean(this.file.name.match(this.$Config.regex.image)) }, diff --git a/libraries/Files/FilSystem.ts b/libraries/Files/FilSystem.ts index 60af4c296a..4c58e8e0b1 100644 --- a/libraries/Files/FilSystem.ts +++ b/libraries/Files/FilSystem.ts @@ -191,7 +191,7 @@ export class FilSystem { */ public async import(fs: FileSystemExport) { for (const item of fs.content) { - this.importChildren(item) + await this.importChildren(item) } this._version = fs.version } diff --git a/plugins/local/classLoader.ts b/plugins/local/classLoader.ts index a8b084d0a5..ac0a9f775b 100644 --- a/plugins/local/classLoader.ts +++ b/plugins/local/classLoader.ts @@ -21,7 +21,7 @@ import Cursor from '~/libraries/ui/Cursor' declare module 'vue/types/vue' { interface Vue { - $Config: any + $Config: typeof Config $WebRTC: WebRTC $SolanaManager: SolanaManager $Sounds: SoundManager @@ -39,7 +39,7 @@ declare module 'vue/types/vue' { declare module '@nuxt/types' { interface Context { - $Config: any + $Config: typeof Config $WebRTC: WebRTC $SolanaManager: SolanaManager $Sounds: SoundManager From 0c2e42f4bdb205d9d76e67d0f515d80ef5749eb2 Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Fri, 4 Mar 2022 16:51:08 +0900 Subject: [PATCH 03/15] fix(files): added doc and fixed errors --- libraries/Files/Fil.ts | 6 ++++- libraries/Files/FilSystem.ts | 31 ++++++++++++------------ libraries/Files/remote/textile/Bucket.ts | 8 +++--- libraries/Files/types/filesystem.ts | 2 +- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/libraries/Files/Fil.ts b/libraries/Files/Fil.ts index 20042ecaa8..74ea8e531d 100644 --- a/libraries/Files/Fil.ts +++ b/libraries/Files/Fil.ts @@ -103,12 +103,16 @@ export class Fil extends Item { /** * @getter file - * @returns last modified timestamp + * @returns file object fetched from textile bucket */ get file(): File { return this._file } + /** + * @getter url + * @returns link of localally stored File for image preview and downloads + */ get url(): string { return URL.createObjectURL(this.file) } diff --git a/libraries/Files/FilSystem.ts b/libraries/Files/FilSystem.ts index 4c58e8e0b1..9f28e4b9cc 100644 --- a/libraries/Files/FilSystem.ts +++ b/libraries/Files/FilSystem.ts @@ -207,18 +207,19 @@ export class FilSystem { item as ExportFile const type = item.type as FILE_TYPE const file = await this.bucket.pullFile(name, type) - console.log(file) - this.createFile({ - name, - file, - hash, - size, - liked, - shared, - description, - type, - modified, - }) + if (file) { + this.createFile({ + name, + file, + hash, + size, + liked, + shared, + description, + type, + modified, + }) + } } if ((Object.values(DIRECTORY_TYPE) as string[]).includes(item.type)) { const { name, liked, shared, children, modified } = @@ -226,9 +227,9 @@ export class FilSystem { const type = item.type as DIRECTORY_TYPE this.createDirectory({ name, liked, shared, type, modified }) this.openDirectory(name) - children.forEach((item: ExportItem) => { - this.importChildren(item) - }) + for (const item of children) { + await this.importChildren(item) + } this.goBack() } } diff --git a/libraries/Files/remote/textile/Bucket.ts b/libraries/Files/remote/textile/Bucket.ts index dee5ae820a..33a4ecea4e 100644 --- a/libraries/Files/remote/textile/Bucket.ts +++ b/libraries/Files/remote/textile/Bucket.ts @@ -62,7 +62,9 @@ export class Bucket extends RFM implements RFMInterface { this.key, Config.textile.fsTable, )) { - this._index = JSON.parse(new TextDecoder().decode(data)) + this._index = JSON.parse( + new TextDecoder().decode(data, { stream: true }), + ) } if (!this._index) throw new Error('Index not found') @@ -115,9 +117,9 @@ export class Bucket extends RFM implements RFMInterface { /** * @method pullFile - * @description Remove file from bucket + * @description fetch encrypted file from bucket * @param {File} file file to be pulled - * @returns Promise whether it was uploaded or not + * @returns Promise of File */ async pullFile(name: string, type: string): Promise { if (!this.buckets || !this.key) { diff --git a/libraries/Files/types/filesystem.ts b/libraries/Files/types/filesystem.ts index 815b0379d4..aab0bba77c 100644 --- a/libraries/Files/types/filesystem.ts +++ b/libraries/Files/types/filesystem.ts @@ -20,7 +20,7 @@ export interface ExportFile extends ExportSharedProps { export interface ExportDirectory extends ExportSharedProps { // eslint-disable-next-line no-use-before-define - children: ExportItem + children: ExportItem[] } export type ExportItem = ExportFile | ExportDirectory From fcb40e0527d2974e2671a7c75ccb7de170208d29 Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Fri, 4 Mar 2022 17:30:55 +0900 Subject: [PATCH 04/15] fix(files): show loader while files are being pulled --- components/views/files/controls/Controls.html | 6 +++--- components/views/files/controls/Controls.vue | 2 +- pages/files/browse/Browse.html | 3 ++- pages/files/browse/index.vue | 2 ++ plugins/thirdparty/persist.ts | 1 + store/accounts/actions.ts | 2 +- store/textile/__snapshots__/state.test.ts.snap | 1 + store/textile/actions.ts | 3 ++- store/textile/mutations.ts | 3 +++ store/textile/state.ts | 1 + store/textile/types.ts | 1 + 11 files changed, 18 insertions(+), 7 deletions(-) diff --git a/components/views/files/controls/Controls.html b/components/views/files/controls/Controls.html index 08777ca49b..3a93072b59 100644 --- a/components/views/files/controls/Controls.html +++ b/components/views/files/controls/Controls.html @@ -23,7 +23,7 @@ :text="$t('pages.files.controls.new_file')" size="small" :action="addFile" - :loading="ui.isLoadingFileIndex" + :loading="ui.isLoadingFileIndex || textile.filesLoading" > @@ -34,8 +34,8 @@ type="primary" :action="addFolder" :placeholder="$t('pages.files.controls.name_folder')" - :loading="ui.isLoadingFileIndex" - :disabled="ui.isLoadingFileIndex" + :loading="ui.isLoadingFileIndex || textile.filesLoading" + :disabled="ui.isLoadingFileIndex || textile.filesLoading" > diff --git a/components/views/files/controls/Controls.vue b/components/views/files/controls/Controls.vue index c4ca73797b..efb79e56fd 100644 --- a/components/views/files/controls/Controls.vue +++ b/components/views/files/controls/Controls.vue @@ -28,7 +28,7 @@ export default Vue.extend({ } }, computed: { - ...mapState(['ui']), + ...mapState(['ui', 'textile']), }, methods: { /** diff --git a/pages/files/browse/Browse.html b/pages/files/browse/Browse.html index 14a175b508..6f9c62ef3c 100644 --- a/pages/files/browse/Browse.html +++ b/pages/files/browse/Browse.html @@ -12,8 +12,9 @@
+ import Vue from 'vue' +import { mapState } from 'vuex' import { Item } from '~/libraries/Files/abstracts/Item.abstract' import { Directory } from '~/libraries/Files/Directory' import { Fil } from '~/libraries/Files/Fil' @@ -27,6 +28,7 @@ export default Vue.extend({ } }, computed: { + ...mapState(['textile']), /** * @returns Current directory items * @description included counter to force rendering on Map updates diff --git a/plugins/thirdparty/persist.ts b/plugins/thirdparty/persist.ts index 92a58900f6..1fd428a875 100644 --- a/plugins/thirdparty/persist.ts +++ b/plugins/thirdparty/persist.ts @@ -21,6 +21,7 @@ const mutationsBlacklist = [ const commonProperties = [ 'webrtc.initialized', 'textile.initialized', + 'textile.filesLoading', 'accounts.initialized', 'friends.all', 'webrtc.activeStream', diff --git a/store/accounts/actions.ts b/store/accounts/actions.ts index c3a277c686..792bead23d 100644 --- a/store/accounts/actions.ts +++ b/store/accounts/actions.ts @@ -296,7 +296,7 @@ export default { const { pin } = state if (!textileInitialized && pin) { - await dispatch( + dispatch( 'textile/initialize', { id: payerAccount?.publicKey.toBase58(), diff --git a/store/textile/__snapshots__/state.test.ts.snap b/store/textile/__snapshots__/state.test.ts.snap index c8490028f0..92dbb143a2 100644 --- a/store/textile/__snapshots__/state.test.ts.snap +++ b/store/textile/__snapshots__/state.test.ts.snap @@ -4,6 +4,7 @@ exports[`init should return the initial settings state 1`] = ` Object { "conversationLoading": false, "conversations": Object {}, + "filesLoading": false, "initialized": false, "messageLoading": false, "uploadProgress": Object {}, diff --git a/store/textile/actions.ts b/store/textile/actions.ts index 62b22ba54a..73aa5ada84 100644 --- a/store/textile/actions.ts +++ b/store/textile/actions.ts @@ -36,8 +36,9 @@ export default { const fsExport = $TextileManager.bucket?.index if (fsExport) { + commit('setFilesLoading', true) const $FileSystem: FilSystem = Vue.prototype.$FileSystem - await $FileSystem.import(fsExport) + $FileSystem.import(fsExport).then(() => commit('setFilesLoading', false)) } }, /** diff --git a/store/textile/mutations.ts b/store/textile/mutations.ts index 34ace625b9..7a2576ed7f 100644 --- a/store/textile/mutations.ts +++ b/store/textile/mutations.ts @@ -147,6 +147,9 @@ const mutations = { } } }, + setFilesLoading(state: TextileState, status: boolean) { + state.filesLoading = status + }, } export default mutations diff --git a/store/textile/state.ts b/store/textile/state.ts index daafdc6b0d..f4902f8d93 100644 --- a/store/textile/state.ts +++ b/store/textile/state.ts @@ -6,6 +6,7 @@ const InitialTextileState = (): TextileState => ({ conversationLoading: false, messageLoading: false, uploadProgress: {}, + filesLoading: false, }) export default InitialTextileState diff --git a/store/textile/types.ts b/store/textile/types.ts index 2acd4b9089..ebb344fd12 100644 --- a/store/textile/types.ts +++ b/store/textile/types.ts @@ -27,6 +27,7 @@ export interface TextileState { name: string } } + filesLoading: boolean } export enum TextileError { From 81b661333e5e458d3c8f4c20a362eb25dd2d8f4d Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Mon, 7 Mar 2022 14:16:35 +0900 Subject: [PATCH 05/15] fix(files): check for thumbnail rather than load every file --- components/views/files/controls/Controls.html | 6 +- components/views/files/controls/Controls.vue | 2 +- components/views/files/file/File.html | 2 +- libraries/Files/Fil.ts | 22 +++++-- libraries/Files/FilSystem.ts | 58 ++++++++++++------- libraries/Files/TextileFileSystem.ts | 32 ++++++++++ libraries/Files/types/filesystem.ts | 1 + pages/files/browse/Browse.html | 3 +- pages/files/browse/index.vue | 2 - plugins/thirdparty/persist.ts | 1 - .../textile/__snapshots__/state.test.ts.snap | 1 - store/textile/actions.ts | 3 +- store/textile/mutations.ts | 3 - store/textile/state.ts | 1 - store/textile/types.ts | 1 - 15 files changed, 95 insertions(+), 43 deletions(-) diff --git a/components/views/files/controls/Controls.html b/components/views/files/controls/Controls.html index 3a93072b59..08777ca49b 100644 --- a/components/views/files/controls/Controls.html +++ b/components/views/files/controls/Controls.html @@ -23,7 +23,7 @@ :text="$t('pages.files.controls.new_file')" size="small" :action="addFile" - :loading="ui.isLoadingFileIndex || textile.filesLoading" + :loading="ui.isLoadingFileIndex" > @@ -34,8 +34,8 @@ type="primary" :action="addFolder" :placeholder="$t('pages.files.controls.name_folder')" - :loading="ui.isLoadingFileIndex || textile.filesLoading" - :disabled="ui.isLoadingFileIndex || textile.filesLoading" + :loading="ui.isLoadingFileIndex" + :disabled="ui.isLoadingFileIndex" > diff --git a/components/views/files/controls/Controls.vue b/components/views/files/controls/Controls.vue index efb79e56fd..c4ca73797b 100644 --- a/components/views/files/controls/Controls.vue +++ b/components/views/files/controls/Controls.vue @@ -28,7 +28,7 @@ export default Vue.extend({ } }, computed: { - ...mapState(['ui', 'textile']), + ...mapState(['ui']), }, methods: { /** diff --git a/components/views/files/file/File.html b/components/views/files/file/File.html index 748b4a2202..750de90d1c 100644 --- a/components/views/files/file/File.html +++ b/components/views/files/file/File.html @@ -25,7 +25,7 @@
diff --git a/libraries/Files/Fil.ts b/libraries/Files/Fil.ts index 74ea8e531d..d5f992f068 100644 --- a/libraries/Files/Fil.ts +++ b/libraries/Files/Fil.ts @@ -7,7 +7,8 @@ export class Fil extends Item { private _hash: string = '' private _description: string = '' private _size: number = 0 - private _file: File + private _file: File | undefined + private _thumbnail: string /** * @constructor @@ -24,9 +25,10 @@ export class Fil extends Item { modified, description, type, + thumbnail, }: { name: string - file: File + file?: File hash: string size: number liked?: boolean @@ -34,13 +36,15 @@ export class Fil extends Item { modified?: number description?: string type?: FILE_TYPE + thumbnail?: string }) { super({ name: name || 'un-named file', liked, shared, modified }) - this._file = file + this._file = file || undefined this._description = description || '' this._hash = hash || '' this._size = size || 0 this._type = type || FILE_TYPE.GENERIC + this._thumbnail = thumbnail || '' } /** @@ -105,7 +109,7 @@ export class Fil extends Item { * @getter file * @returns file object fetched from textile bucket */ - get file(): File { + get file(): File | undefined { return this._file } @@ -114,7 +118,7 @@ export class Fil extends Item { * @returns link of localally stored File for image preview and downloads */ get url(): string { - return URL.createObjectURL(this.file) + return this.file ? URL.createObjectURL(this.file) : '' } /** @@ -124,4 +128,12 @@ export class Fil extends Item { set description(content: string) { this._description = `${content || ''}` } + + /** + * @getter url + * @returns link of localally stored File for image preview and downloads + */ + get thumbnail(): string { + return this._thumbnail + } } diff --git a/libraries/Files/FilSystem.ts b/libraries/Files/FilSystem.ts index 9f28e4b9cc..604730244f 100644 --- a/libraries/Files/FilSystem.ts +++ b/libraries/Files/FilSystem.ts @@ -157,8 +157,17 @@ export class FilSystem { */ exportChildren(item: Item): ExportItem { if (item instanceof Fil) { - const { name, liked, shared, type, hash, size, description, modified } = - item + const { + name, + liked, + shared, + type, + hash, + size, + description, + modified, + thumbnail, + } = item return { name, liked, @@ -168,6 +177,7 @@ export class FilSystem { size, description, modified, + thumbnail, } } const { name, liked, shared, type, modified } = item @@ -198,28 +208,33 @@ export class FilSystem { /** * @method importChildren - * @param {FileSystemExport} fs + * @param {ExportItem} item * @description recursively adds files and directories from JSON export */ public async importChildren(item: ExportItem) { if ((Object.values(FILE_TYPE) as string[]).includes(item.type)) { - const { name, hash, size, liked, shared, description, modified } = - item as ExportFile + const { + name, + hash, + size, + liked, + shared, + description, + modified, + thumbnail, + } = item as ExportFile const type = item.type as FILE_TYPE - const file = await this.bucket.pullFile(name, type) - if (file) { - this.createFile({ - name, - file, - hash, - size, - liked, - shared, - description, - type, - modified, - }) - } + this.createFile({ + name, + hash, + size, + liked, + shared, + description, + type, + modified, + thumbnail, + }) } if ((Object.values(DIRECTORY_TYPE) as string[]).includes(item.type)) { const { name, liked, shared, children, modified } = @@ -249,9 +264,10 @@ export class FilSystem { description, type, modified, + thumbnail, }: { name: string - file: File + file?: File hash: string size: number liked?: boolean @@ -259,6 +275,7 @@ export class FilSystem { description?: string type?: FILE_TYPE modified?: number + thumbnail?: string }): Fil | null { const newFile = new Fil({ name, @@ -270,6 +287,7 @@ export class FilSystem { description, type, modified, + thumbnail, }) const inserted = this.addChild(newFile) return inserted ? newFile : null diff --git a/libraries/Files/TextileFileSystem.ts b/libraries/Files/TextileFileSystem.ts index a28cb8c9b8..752319a745 100644 --- a/libraries/Files/TextileFileSystem.ts +++ b/libraries/Files/TextileFileSystem.ts @@ -1,6 +1,8 @@ import { PushPathResult } from '@textile/hub' import { FilSystem } from './FilSystem' import { FILE_TYPE } from './types/file' +import { isHeic } from '~/utilities/Heic' +const convert = require('heic-convert') export class TextileFileSystem extends FilSystem { /** @@ -10,6 +12,7 @@ export class TextileFileSystem extends FilSystem { */ async uploadFile(file: File) { const result: PushPathResult = await this.bucket.pushFile(file) + this.createFile({ name: file.name, file, @@ -18,6 +21,7 @@ export class TextileFileSystem extends FilSystem { type: (Object.values(FILE_TYPE) as string[]).includes(file.type) ? (file.type as FILE_TYPE) : FILE_TYPE.GENERIC, + thumbnail: await this._createThumbnail(file), }) } @@ -29,4 +33,32 @@ export class TextileFileSystem extends FilSystem { async removeFile(name: string) { await this.bucket.removeFile(name) } + + /** + * @method _createThumbnail + * @description create thumbnail if embeddable image format, otherwise return '' + * @param {File} file + */ + private async _createThumbnail(file: File): Promise { + const buffer = new Uint8Array(await file.arrayBuffer()) + if (isHeic(buffer)) { + const outputBuffer = await convert({ + buffer, + format: 'JPEG', + quality: 1, + }) + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.readAsArrayBuffer(outputBuffer) + reader.onload = () => resolve(reader.result?.toString() || '') + reader.onerror = (error) => reject(error) + }) + } + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.readAsDataURL(file) + reader.onload = () => resolve(reader.result?.toString() || '') + reader.onerror = (error) => reject(error) + }) + } } diff --git a/libraries/Files/types/filesystem.ts b/libraries/Files/types/filesystem.ts index aab0bba77c..1cc7355b57 100644 --- a/libraries/Files/types/filesystem.ts +++ b/libraries/Files/types/filesystem.ts @@ -16,6 +16,7 @@ export interface ExportFile extends ExportSharedProps { hash: string size: number description: string + thumbnail: string } export interface ExportDirectory extends ExportSharedProps { diff --git a/pages/files/browse/Browse.html b/pages/files/browse/Browse.html index 6f9c62ef3c..14a175b508 100644 --- a/pages/files/browse/Browse.html +++ b/pages/files/browse/Browse.html @@ -12,9 +12,8 @@
- import Vue from 'vue' -import { mapState } from 'vuex' import { Item } from '~/libraries/Files/abstracts/Item.abstract' import { Directory } from '~/libraries/Files/Directory' import { Fil } from '~/libraries/Files/Fil' @@ -28,7 +27,6 @@ export default Vue.extend({ } }, computed: { - ...mapState(['textile']), /** * @returns Current directory items * @description included counter to force rendering on Map updates diff --git a/plugins/thirdparty/persist.ts b/plugins/thirdparty/persist.ts index 1fd428a875..92a58900f6 100644 --- a/plugins/thirdparty/persist.ts +++ b/plugins/thirdparty/persist.ts @@ -21,7 +21,6 @@ const mutationsBlacklist = [ const commonProperties = [ 'webrtc.initialized', 'textile.initialized', - 'textile.filesLoading', 'accounts.initialized', 'friends.all', 'webrtc.activeStream', diff --git a/store/textile/__snapshots__/state.test.ts.snap b/store/textile/__snapshots__/state.test.ts.snap index 92dbb143a2..c8490028f0 100644 --- a/store/textile/__snapshots__/state.test.ts.snap +++ b/store/textile/__snapshots__/state.test.ts.snap @@ -4,7 +4,6 @@ exports[`init should return the initial settings state 1`] = ` Object { "conversationLoading": false, "conversations": Object {}, - "filesLoading": false, "initialized": false, "messageLoading": false, "uploadProgress": Object {}, diff --git a/store/textile/actions.ts b/store/textile/actions.ts index 73aa5ada84..62b22ba54a 100644 --- a/store/textile/actions.ts +++ b/store/textile/actions.ts @@ -36,9 +36,8 @@ export default { const fsExport = $TextileManager.bucket?.index if (fsExport) { - commit('setFilesLoading', true) const $FileSystem: FilSystem = Vue.prototype.$FileSystem - $FileSystem.import(fsExport).then(() => commit('setFilesLoading', false)) + await $FileSystem.import(fsExport) } }, /** diff --git a/store/textile/mutations.ts b/store/textile/mutations.ts index 7a2576ed7f..34ace625b9 100644 --- a/store/textile/mutations.ts +++ b/store/textile/mutations.ts @@ -147,9 +147,6 @@ const mutations = { } } }, - setFilesLoading(state: TextileState, status: boolean) { - state.filesLoading = status - }, } export default mutations diff --git a/store/textile/state.ts b/store/textile/state.ts index f4902f8d93..daafdc6b0d 100644 --- a/store/textile/state.ts +++ b/store/textile/state.ts @@ -6,7 +6,6 @@ const InitialTextileState = (): TextileState => ({ conversationLoading: false, messageLoading: false, uploadProgress: {}, - filesLoading: false, }) export default InitialTextileState diff --git a/store/textile/types.ts b/store/textile/types.ts index ebb344fd12..2acd4b9089 100644 --- a/store/textile/types.ts +++ b/store/textile/types.ts @@ -27,7 +27,6 @@ export interface TextileState { name: string } } - filesLoading: boolean } export enum TextileError { From f1945e44c7bcf53bc467405e2555159e2faaf439 Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Tue, 8 Mar 2022 09:48:38 +0900 Subject: [PATCH 06/15] fix(files): shrink thumbnail using skaler package --- components/views/files/file/File.html | 4 +-- components/views/files/file/File.vue | 3 -- components/views/files/view/View.html | 4 +-- components/views/files/view/View.vue | 9 ++++-- cypress | 2 +- libraries/Files/FilSystem.ts | 10 ------- libraries/Files/TextileFileSystem.ts | 35 +++++++++++++++++++----- libraries/Files/remote/textile/Bucket.ts | 23 +++++++--------- package.json | 2 +- plugins/local/classLoader.ts | 1 - 10 files changed, 50 insertions(+), 43 deletions(-) diff --git a/components/views/files/file/File.html b/components/views/files/file/File.html index 750de90d1c..ac6ee82927 100644 --- a/components/views/files/file/File.html +++ b/components/views/files/file/File.html @@ -23,7 +23,7 @@
@@ -31,7 +31,7 @@ diff --git a/components/views/files/file/File.vue b/components/views/files/file/File.vue index 76ccc1f484..e64c9c0529 100644 --- a/components/views/files/file/File.vue +++ b/components/views/files/file/File.vue @@ -75,9 +75,6 @@ export default Vue.extend({ ? this.item.content.length + ' items' : this.$filesize((this.item as Fil).size) }, - isImage(): boolean { - return Boolean(this.item.name.match(this.$Config.regex.image)) - }, isArchive(): boolean { return Boolean(this.item.name.match(this.$Config.regex.archive)) }, diff --git a/components/views/files/view/View.html b/components/views/files/view/View.html index 14ff13b2fa..d5284b010b 100644 --- a/components/views/files/view/View.html +++ b/components/views/files/view/View.html @@ -40,9 +40,9 @@
{ + private async _createThumbnail(file: File): Promise { + if (!file.name.match(Config.regex.image)) { + return + } const buffer = new Uint8Array(await file.arrayBuffer()) if (isHeic(buffer)) { const outputBuffer = await convert({ @@ -47,13 +62,19 @@ export class TextileFileSystem extends FilSystem { format: 'JPEG', quality: 1, }) - return new Promise((resolve, reject) => { - const reader = new FileReader() - reader.readAsArrayBuffer(outputBuffer) - reader.onload = () => resolve(reader.result?.toString() || '') - reader.onerror = (error) => reject(error) - }) + return await this._fileToData( + await skaler( + new File([outputBuffer.buffer], file.name, { + type: 'image/jpeg', + }), + { width: 400 }, + ), + ) } + return await this._fileToData(await skaler(file, { width: 400 })) + } + + private _fileToData(file: File) { return new Promise((resolve, reject) => { const reader = new FileReader() reader.readAsDataURL(file) diff --git a/libraries/Files/remote/textile/Bucket.ts b/libraries/Files/remote/textile/Bucket.ts index 33a4ecea4e..f89fd158c7 100644 --- a/libraries/Files/remote/textile/Bucket.ts +++ b/libraries/Files/remote/textile/Bucket.ts @@ -21,14 +21,6 @@ export class Bucket extends RFM implements RFMInterface { this.key = null } - /** - * @getter - * @returns textile data - */ - get textile(): TextileInitializationData | null { - return this._textile - } - /** * @getter * @returns file system export data @@ -40,7 +32,7 @@ export class Bucket extends RFM implements RFMInterface { /** * @method init * @description Initializes bucket - * @param param0 Bucket Configuration that includes id, password, SolanaWallet instance, and bucket name + * @param name bucket name * @returns a promise that resolves when the initialization completes */ async init(name: string): Promise { @@ -58,14 +50,19 @@ export class Bucket extends RFM implements RFMInterface { this.key = result.root.key try { - for await (const data of this.buckets.pullPath( + const data = [] + for await (const bytes of this.buckets.pullPath( this.key, Config.textile.fsTable, )) { - this._index = JSON.parse( - new TextDecoder().decode(data, { stream: true }), - ) + data.push(bytes) } + this._index = JSON.parse( + await new Blob(data, { + type: 'application/json', + }).text(), + ) + if (!this._index) throw new Error('Index not found') return this._index diff --git a/package.json b/package.json index 2b4e17b640..79369faf3e 100644 --- a/package.json +++ b/package.json @@ -67,9 +67,9 @@ "qrcode.vue": "^1.7.0", "remarkable": "^2.0.1", "satellite-lucide-icons": "git+https://git@github.com/Satellite-im/satellite-lucide-vue", - "sharp": "^0.30.1", "simple-markdown": "^0.7.3", "simple-peer": "^9.11.1", + "skaler": "^1.0.7", "swiper": "5.x", "uuid": "^8.3.2", "v-calendar": "^2.4.1", diff --git a/plugins/local/classLoader.ts b/plugins/local/classLoader.ts index ac0a9f775b..7a483c33ac 100644 --- a/plugins/local/classLoader.ts +++ b/plugins/local/classLoader.ts @@ -11,7 +11,6 @@ import Security from '~/libraries/Security/Security' import { RootStore } from '~/types/store/store' import TextileManager from '~/libraries/Textile/TextileManager' import { Alerts } from '~/libraries/ui/Alerts' -import { Bucket } from '~/libraries/Files/remote/textile/Bucket' import { TextileFileSystem } from '~/libraries/Files/TextileFileSystem' // Utils import Hounddog from '~/utilities/Hounddog' From b15d4793544712a7db657ee22ac44bd1df8a4d02 Mon Sep 17 00:00:00 2001 From: Joe McGrath Date: Tue, 8 Mar 2022 11:46:41 +0900 Subject: [PATCH 07/15] feat(files): pull file from textile on full page view --- components/views/files/file/File.html | 13 ++-------- components/views/files/file/File.vue | 10 ++++++-- components/views/files/filepath/Filepath.html | 10 +++++--- components/views/files/filepath/Filepath.less | 23 +++++++++++------- components/views/files/filepath/Filepath.vue | 2 ++ components/views/files/view/View.html | 18 +++++++++----- components/views/files/view/View.less | 3 +++ components/views/files/view/View.vue | 14 +++++++++-- config.ts | 2 +- libraries/Files/Fil.ts | 24 ++++++++++++------- libraries/Files/FilSystem.ts | 2 +- libraries/Files/TextileFileSystem.ts | 6 ++++- libraries/Files/remote/textile/Bucket.ts | 3 ++- tsconfig.json | 2 +- types/modules/skaler.d.ts | 1 + 15 files changed, 88 insertions(+), 45 deletions(-) create mode 100644 types/modules/skaler.d.ts diff --git a/components/views/files/file/File.html b/components/views/files/file/File.html index ac6ee82927..96cb7b7aa2 100644 --- a/components/views/files/file/File.html +++ b/components/views/files/file/File.html @@ -21,20 +21,11 @@ @mouseleave="heartHover=false" />
- - +
- +
diff --git a/components/views/files/file/File.vue b/components/views/files/file/File.vue index e64c9c0529..168e0da386 100644 --- a/components/views/files/file/File.vue +++ b/components/views/files/file/File.vue @@ -68,13 +68,19 @@ export default Vue.extend({ computed: { ...mapState(['ui']), /** - * @returns if directory, child count. if file, size + * @returns {string} if directory, child count. if file, size */ getSubtext(): string { return this.item instanceof Directory ? this.item.content.length + ' items' : this.$filesize((this.item as Fil).size) }, + /** + * @returns {boolean} if item has discrete MIME type of image + */ + isImage(): boolean { + return this.item.type.split('/')[0] === 'image' + }, isArchive(): boolean { return Boolean(this.item.name.match(this.$Config.regex.archive)) }, @@ -123,11 +129,11 @@ export default Vue.extend({ this.item.shareItem() await this.$TextileManager.bucket?.updateIndex(this.$FileSystem.export) this.$store.commit('ui/setIsLoadingFileIndex', false) + this.$emit('forceRender') } navigator.clipboard.writeText(this.path).then(() => { this.$toast.show(this.$t('pages.files.link_copied') as string) }) - this.$emit('forceRender') }, /** * @method rename diff --git a/components/views/files/filepath/Filepath.html b/components/views/files/filepath/Filepath.html index 6293464bf3..8bb6112706 100644 --- a/components/views/files/filepath/Filepath.html +++ b/components/views/files/filepath/Filepath.html @@ -1,14 +1,18 @@