fix(files): nsfw scan memory limit (#2655)

components/views/files/controls/Controls.html

@@ -48,6 +48,8 @@
       multiple
     />
   </div>
+  <TypographyText v-if="status" class="status" :text="status" :size="6" />
+  <UiProgress v-if="progress < 100" :progress="progress" />
   <div class="error-container" v-if="errors.length">
     <alert-triangle-icon size="1.3x" />
     <div>

components/views/files/controls/Controls.less

@@ -22,6 +22,11 @@
     }
   }
 
+  .status,
+  .progress {
+    margin-bottom: @normal-spacing;
+  }
+
   .error-container {
     &:extend(.round-corners);
     display: flex;

components/views/files/controls/Controls.vue

@@ -33,6 +33,8 @@ export default Vue.extend({
     return {
       text: '' as string,
       errors: [] as Array<string | TranslateResult>,
+      status: '' as string | TranslateResult,
+      progress: 100 as number,
     }
   },
   computed: {
@@ -114,10 +116,6 @@ export default Vue.extend({
         sameNameResults.map(async (file: File) => {
           // convert heic to jpg for scan. return original heic if sfw
           if (await isHeic(file)) {
-            // prevent crash in case of larger than 2GB heic files. could possibly be broken up into multiple buffers
-            if (file.size >= this.$Config.arrayBufferLimit) {
-              return { file, nsfw: false }
-            }
             const buffer = new Uint8Array(await file.arrayBuffer())
             const outputBuffer = await convert({
               buffer,
@@ -148,22 +146,25 @@ export default Vue.extend({
           files.push(el.file)
         }
       }
-
       for (const file of files) {
         try {
-          await this.$FileSystem.uploadFile(file)
+          this.status = this.$t('pages.files.controls.upload', [file.name])
+          await this.$FileSystem.uploadFile(file, this.setProgress)
         } catch (e: any) {
           this.errors.push(e?.message ?? '')
         }
       }
 
       // only update index if files have been updated
       if (files.length) {
+        this.status = this.$t('pages.files.controls.index')
         await this.$TextileManager.bucket?.updateIndex(this.$FileSystem.export)
       }
 
       this.$store.commit('ui/setIsLoadingFileIndex', false)
+      this.status = ''
 
+      // re-render so new files show up
       this.$emit('forceRender')
 
       if (originalFiles.length !== invalidNameResults.length) {
@@ -179,6 +180,15 @@ export default Vue.extend({
         this.errors.push(this.$t('errors.chat.contains_nsfw'))
       }
     },
+    /**
+     * @method setProgress
+     * @description set progress (% out of 100) while file is being pushed to textile bucket. passed as a callback
+     * @param num current progress in bytes
+     * @param size total file size in bytes
+     */
+    setProgress(num: number, size: number) {
+      this.progress = Math.floor((num / size) * 100)
+    },
   },
 })
 </script>

components/views/files/upload/Upload.vue

@@ -148,7 +148,7 @@ export default Vue.extend({
         )
 
         for (const uploadFile of filesToUpload) {
-          if (uploadFile.file.size <= Config.nsfwByteLimit) {
+          if (uploadFile.file.size <= Config.nsfwPictureLimit) {
             uploadFile.nsfw.checking = true
             try {
               uploadFile.nsfw.status = await this.$Security.isNSFW(

components/views/files/view/View.html

@@ -7,7 +7,7 @@
       <TypographyText :text="$dayjs(file.modified).fromNow()" :size="6" />
     </div>
     <div class="controls">
-      <UiLoadersSpinner v-if="load" class="control disabled" spinning />
+      <UiLoadersSpinner v-if="isLoading" class="control disabled" spinning />
       <a
         v-else
         :data-tooltip="$t('controls.download')"
@@ -35,14 +35,13 @@
       </div>
     </div>
   </div>
-  <img v-if="file.thumbnail" class="file-image" :src="file.thumbnail" />
+  <UiProgress v-if="isLoading" :progress="progress" />
+  <img v-else-if="file.thumbnail" class="file-image" :src="file.thumbnail" />
   <div v-else class="no-preview">
-    <UiLoadersSpinner v-if="load" class="file-icon" spinning />
     <a
-      v-else
       :data-tooltip="$t('controls.download')"
       class="has-tooltip has-tooltip-primary has-tooltip-top"
-      :class="{'disabled' : load}"
+      :class="{'disabled' : isLoading}"
       :href="file.url"
       target="_blank"
       :download="name"

components/views/files/view/View.less

@@ -4,6 +4,7 @@
   inset: 0;
   background: rgba(0, 0, 0, 0.75);
   z-index: @base-z-index + 35;
+  padding: @normal-spacing;
 
   .file-image {
     display: flex;
@@ -28,28 +29,29 @@
       margin-bottom: 0;
     }
   }
+  .progress {
+    display: flex;
+    margin: auto;
+    max-width: 1000px;
+  }
 
   .file-nav {
-    width: @full;
+    width: calc(@full - @xlarge-spacing); //to account for margin
     display: flex;
     position: absolute;
-    margin: @normal-spacing;
 
     .file-info {
       max-width: 50vw;
       .title {
         &:extend(.font-secondary);
-        overflow: hidden;
-        white-space: nowrap;
-        text-overflow: ellipsis;
+        &:extend(.ellipsis);
       }
     }
 
     .controls {
       display: flex;
       height: fit-content;
       margin-left: auto;
-      margin-right: @large-spacing;
       gap: @normal-spacing;
 
       .control {

components/views/files/view/View.vue

@@ -24,19 +24,20 @@ export default Vue.extend({
   },
   data() {
     return {
-      load: false as boolean,
       file: undefined as Fil | undefined,
       name: '' as string,
+      progress: 0 as number,
     }
   },
   computed: {
     ...mapState(['ui']),
+    isLoading(): boolean {
+      return this.progress >= 0 && this.progress < 100
+    },
   },
   /**
    */
   async created() {
-    this.load = true
-
     this.file = this.$FileSystem.getChild(this.ui.filePreview) as Fil
     this.name = this.file?.name
 
@@ -47,6 +48,8 @@ export default Vue.extend({
         this.file.id,
         this.file.name,
         this.file.type,
+        this.file.size,
+        this.setProgress,
       )
     }
     // file extension according to file name
@@ -67,24 +70,30 @@ export default Vue.extend({
       if (fileExt !== 'svg') {
         this.name += '.svg'
       }
-      this.load = false
       return
     }
 
     // if corrupted txt file
     if (!dataExt && fileExt !== 'txt') {
       this.name += '.txt'
-      this.load = false
       return
     }
 
     // if corrupted file with wrong extension, force the correct one
     if (fileExt !== dataExt && dataExt) {
       this.name += `.${dataExt}`
     }
-    this.load = false
   },
   methods: {
+    /**
+     * @method setProgress
+     * @description set progress (% out of 100) while file is being pulled from textile bucket. passed as a callback
+     * @param num current progress in bytes
+     * @param size total file size in bytes
+     */
+    setProgress(num: number, size: number) {
+      this.progress = Math.floor((num / size) * 100)
+    },
     /**
      * @method share
      * @description Emit to share item - pages/files/browse/index.vue

components/views/user/create/Create.vue

@@ -78,9 +78,8 @@ export default Vue.extend({
         return
       }
 
-      // stop upload if picture is too large for nsfw scan.
-      // Leaving this in place since nsfw profile pictures would be bad
-      if (files[0].size > this.$Config.nsfwByteLimit) {
+      // stop upload if picture is too large for nsfw scan
+      if (files[0].size > this.$Config.nsfwPictureLimit) {
         this.error = this.$t('errors.accounts.file_too_large') as string
         this.isLoading = false
         return

config.ts

@@ -112,9 +112,9 @@ export const Config = {
   routingMiddleware: {
     prerequisitesCheckBypass: ['auth', 'setup'],
   },
-  nsfwByteLimit: 1048576 * 8, // 8MB - arbitrary image limit for nsfw scan - binary
-  personalFilesLimit: 1000000000 * 4, // 4GB - free tier limit - decimal because stoarge systems typically are
-  arrayBufferLimit: 1073741824 * 2, // 2GB - array buffers larger than this crash - binary
+  nsfwPictureLimit: 1048576 * 8, // 8MB - images will be scaled down to this value if possible to prevent memory issues - binary
+  personalFilesLimit: 1000000000 * 4, // 4GB - free tier limit - decimal
+  nsfwVideoLimit: 1073741824 * 2, // 2GB - videos larger than this crash - binary
   regex: {
     // identify if a file type is embeddable image
     image: '^.*.(apng|avif|gif|jpg|jpeg|jfif|pjpeg|pjp|png|svg|webp)$',

layouts/files.vue

@@ -39,6 +39,7 @@
         </swiper-slide>
         <swiper-slide class="dynamic-content">
           <menu-icon
+            v-if="!showSidebar || $device.isMobile"
             class="toggle--sidebar"
             size="1.2x"
             full-width

libraries/Files/Directory.ts

@@ -142,7 +142,6 @@ export class Directory extends Item {
     }
 
     let parent = this.parent
-
     while (!isEqual(parent, null)) {
       if (isEqual(parent, child)) {
         throw new Error(FileSystemErrors.DIR_PARENT_PARADOX)

libraries/Files/TextileFileSystem.ts

@@ -23,10 +23,11 @@ export class TextileFileSystem extends FilSystem {
    * @description Upload file to the bucket and create in the file system afterwards
    * use uuid as bucket path so files can be renamed freely
    * @param {File} file file to be uploaded
+   * @param {Function} progressCallback used to show progress meter in componment that calls this method
    */
-  async uploadFile(file: File) {
+  async uploadFile(file: File, progressCallback: Function) {
     const id = uuidv4()
-    await this.bucket.pushFile(file, id)
+    await this.bucket.pushFile(file, id, progressCallback)
     // read magic byte type, use metadata as backup
     const byteType = (await mimeType(file)) as FILE_TYPE
     const type = byteType || file.type
@@ -61,10 +62,6 @@ export class TextileFileSystem extends FilSystem {
     type: FILE_TYPE,
   ): Promise<string | undefined> {
     if (await isHeic(file)) {
-      // prevent crash in case of larger than 2GB heic files. could possibly be broken up into multiple buffers
-      if (file.size >= Config.arrayBufferLimit) {
-        return
-      }
       const buffer = new Uint8Array(await file.arrayBuffer())
       const outputBuffer = await convert({
         buffer,

libraries/Files/remote/textile/Bucket.ts

@@ -1,4 +1,4 @@
-import { Buckets, PushPathResult, RemovePathResponse, Root } from '@textile/hub'
+import { Buckets, RemovePathResponse, Root } from '@textile/hub'
 import { RFM } from '../abstracts/RFM.abstract'
 import { RFMInterface } from '../interface/RFM.interface'
 import { Config } from '~/config'
@@ -107,34 +107,78 @@ export class Bucket extends RFM implements RFMInterface {
   /**
    * @method pushFile
    * @description Add file to bucket
+   * stream upload syntax - https://textileio.github.io/js-textile/docs/hub.buckets.pushpath#example-2
    * @param {File} file file to be uploaded
-   * @returns Promise whether it was uploaded or not
+   * @param {string} path uuid to maintain unique bucket paths
+   * @param {Function} progressCallback used to show progress meter in componment that calls this method
    */
-  async pushFile(file: File, id: string): Promise<PushPathResult> {
+  async pushFile(file: File, path: string, progressCallback: Function) {
     if (!this.buckets || !this.key) {
       throw new Error('Bucket or bucket key not found')
     }
-    return await this.buckets.pushPath(this.key, id, file)
+
+    await this.buckets.pushPath(
+      this.key,
+      path,
+      {
+        path,
+        content: this._getStream(file),
+      },
+      {
+        progress: (num) => {
+          progressCallback(num, file.size)
+        },
+      },
+    )
+  }
+
+  private _getStream(file: File) {
+    const reader = file.stream().getReader()
+    const stream = new ReadableStream({
+      start(controller) {
+        function push() {
+          return reader
+            .read()
+            .then(({ done, value }: { done: boolean; value: Uint8Array }) => {
+              if (done) {
+                controller.close()
+                return
+              }
+              controller.enqueue(value)
+              push()
+            })
+        }
+        push()
+      },
+    })
+    return stream
   }
 
   /**
    * @method pullFile
    * @description fetch encrypted file from bucket
    * @param {string} id file path in bucket
    * @param {string} type file mime type
+   * @param {Function} progressCallback used to show progress meter in componment that calls this method
    * @returns Promise of File
    */
   async pullFile(
     id: string,
     name: string,
     type: string,
-  ): Promise<File | undefined> {
+    size: number,
+    progressCallback: Function,
+  ): Promise<File> {
     if (!this.buckets || !this.key) {
       throw new Error('Bucket or bucket key not found')
     }
 
     const data = []
-    for await (const bytes of this.buckets.pullPath(this.key, id)) {
+    for await (const bytes of this.buckets.pullPath(this.key, id, {
+      progress: (num) => {
+        progressCallback(num, size)
+      },
+    })) {
       data.push(bytes)
     }
     // if type is unknown(generic), then don't use in File constructor

locales/en-US.js

@@ -134,6 +134,8 @@ export default {
       controls: {
         new_file: 'New File',
         name_folder: 'Name Folder...',
+        upload: 'Uploading {0}',
+        index: 'Updating remote index',
       },
       browse: {
         files: 'Files',