Add a progress indicator

package.json

@@ -20,7 +20,8 @@
     "file-saver": "^2.0.5",
     "jquery": "^3.7.1",
     "js-cookie": "^3.0.1",
-    "lodash-es": "^4.17.21"
+    "lodash-es": "^4.17.21",
+    "pretty-bytes": "^6.1.1"
   },
   "devDependencies": {
     "@types/file-saver": "^2.0.7",

src/common/file.ts

@@ -3,16 +3,20 @@
 File loader
 ===========
 
-Copyright (c) 2023 Dannii Willis
+Copyright (c) 2024 Dannii Willis
 MIT licenced
 https://github.com/curiousdannii/parchment
 
 */
 
 import {ParchmentOptions} from './options.js'
 
-// Fetch a storyfile, using the proxy if necessary, and handling JSified stories
-export async function fetch_storyfile(options: ParchmentOptions, url: string) {
+export type ProgressCallback = (bytes: number) => void
+
+const utf8decoder = new TextDecoder()
+
+/** Fetch a storyfile, using the proxy if necessary, and handling JSified stories */
+export async function fetch_storyfile(options: ParchmentOptions, url: string, progress_callback?: ProgressCallback): Promise<Uint8Array> {
     // Handle a relative URL
     const story_url = new URL(url, document.URL)
     const story_domain = story_url.hostname
@@ -23,8 +27,7 @@ export async function fetch_storyfile(options: ParchmentOptions, url: string) {
     // Load an embedded storyfile
     if (story_url.protocol === 'embedded:') {
         const data = (document.getElementById(story_url.pathname) as HTMLScriptElement).text
-        const buffer = await parse_base64(data)
-        return new Uint8Array(buffer)
+        return parse_base64(data)
     }
 
     // Only directly access files same origin files or those from the list of reliable domains
@@ -64,23 +67,24 @@ export async function fetch_storyfile(options: ParchmentOptions, url: string) {
 
     // It would be nice to check here if the file was compressed, but we can only read the Content-Encoding header for same domain files
 
+    const data = await read_response(response, progress_callback)
+
     // Handle JSified stories
     if (url.endsWith('.js')) {
-        const text = await response.text()
+        const text = utf8decoder.decode(data)
         const matched = /processBase64Zcode\(['"]([a-zA-Z0-9+/=]+)['"]\)/.exec(text)
         if (!matched) {
             throw new Error('Abnormal JSified story')
         }
 
-        const buffer = await parse_base64(matched[1])
-        return new Uint8Array(buffer)
+        return parse_base64(matched[1])
     }
 
-    const buffer = await response.arrayBuffer()
-    return new Uint8Array(buffer)
+    return data
 }
 
-export async function fetch_vm_resource(options: ParchmentOptions, path: string) {
+/** Fetch a VM resource */
+export async function fetch_vm_resource(options: ParchmentOptions, path: string, progress_callback?: ProgressCallback) {
     // Handle embedded resources in single file mode
     if (options.single_file) {
         const data = (document.getElementById(path) as HTMLScriptElement).text
@@ -107,28 +111,57 @@ export async function fetch_vm_resource(options: ParchmentOptions, path: string)
         url = options.lib_path + path
     }
     const response = await fetch(url)
-    if (!response.ok) {
-        throw new Error(`Could not fetch ${path}, got ${response.status}`)
-    }
-    return response.arrayBuffer()
+    return read_response(response, progress_callback)
 }
 
-// Parse Base 64 into an ArrayBuffer
-export async function parse_base64(data: string, data_type = 'octet-binary'): Promise<ArrayBuffer> {
+/** Parse Base 64 into a Uint8Array */
+export async function parse_base64(data: string, data_type = 'octet-binary'): Promise<Uint8Array> {
     // Parse base64 using a trick from https://stackoverflow.com/a/54123275/2854284
     const response = await fetch(`data:application/${data_type};base64,${data}`)
     if (!response.ok) {
         throw new Error(`Could not parse base64: ${response.status}`)
     }
-    return response.arrayBuffer()
+    return new Uint8Array(await response.arrayBuffer())
 }
 
-// Read an uploaded file and return it as a Uint8Array
+/** Read an uploaded file and return it as a Uint8Array */
 export function read_uploaded_file(file: File): Promise<Uint8Array> {
     return new Promise((resolve, reject) => {
         const reader = new FileReader()
         reader.onerror = () => reject(reader.error)
         reader.onload = () => resolve(new Uint8Array(reader.result as ArrayBuffer))
         reader.readAsArrayBuffer(file)
     })
+}
+
+/** Read a response, with optional progress notifications */
+async function read_response(response: Response, progress_callback?: ProgressCallback): Promise<Uint8Array> {
+    if (!response.ok) {
+        throw new Error(`Could not fetch ${response.url}, got ${response.status}`)
+    }
+
+    if (!progress_callback) {
+        return new Uint8Array(await response.arrayBuffer())
+    }
+
+    // Read the response, calling the callback with each chunk
+    const chunks: Array<[number, Uint8Array]> = []
+    let length = 0
+    const reader = response.body!.getReader()
+    for (;;) {
+        const {done, value} = await reader.read()
+        if (done) {
+            break
+        }
+        chunks.push([length, value])
+        progress_callback(value.length)
+        length += value.length
+    }
+
+    // Join the chunks together
+    const result = new Uint8Array(length)
+    for (const [offset, chunk] of chunks) {
+        result.set(chunk, offset)
+    }
+    return result
 }

src/common/formats.ts

@@ -25,12 +25,12 @@ export interface Format {
     id: string
 }
 
-async function generic_emglken_vm(options: ParchmentOptions, requires: [Uint8Array, any, ArrayBuffer])
+async function generic_emglken_vm(options: ParchmentOptions, requires: [Uint8Array, any, Uint8Array])
 {
     const [file_data, engine, wasmBinary] = requires
 
     const vm_options = Object.assign({}, options, {
-        wasmBinary,
+        wasmBinary: wasmBinary.buffer,
     })
 
     const vm = new engine.default()

src/common/launcher.ts

@@ -10,11 +10,13 @@ https://github.com/curiousdannii/parchment
 */
 
 import Cookies from 'js-cookie'
+import prettyBytes from 'pretty-bytes'
 
 import {AsyncGlk, Blorb, FileView} from '../upstream/asyncglk/src/index-browser.js'
-import {get_default_options, get_query_options, ParchmentOptions, StoryOptions} from './options.js'
-import {fetch_storyfile, fetch_vm_resource, read_uploaded_file} from './file.js'
+
+import {fetch_storyfile, fetch_vm_resource, ProgressCallback, read_uploaded_file} from './file.js'
 import {find_format, identify_blorb_storyfile_format} from './formats.js'
+import {get_default_options, get_query_options, ParchmentOptions, StoryOptions} from './options.js'
 
 interface ParchmentWindow extends Window {
     parchment: ParchmentLauncher
@@ -117,11 +119,28 @@ class ParchmentLauncher
             // Identify the format
             let format = find_format(story.format, story.url)
 
+            // Look for the progress bar
+            let progress = 0
+            const progress_bar = $('#loading_progress')
+            let progress_callback: ProgressCallback | undefined
+            if (progress_bar.length) {
+                progress_callback = chunk_length => {
+                    progress += chunk_length
+                    progress_bar.val(progress)
+                }
+                if (story.filesize_gz) {
+                    $('#loading_size').text(prettyBytes(story.filesize_gz, {
+                        maximumFractionDigits: 1,
+                        minimumFractionDigits: 1,
+                    }))
+                }
+            }
+
             // If a blorb file extension is generic, we must download it first to identify its format
             let blorb: Blorb | undefined
             if (format.id === 'blorb') {
                 if (!story.data) {
-                    story.data = await fetch_storyfile(this.options, story.url!)
+                    story.data = await fetch_storyfile(this.options, story.url!, progress_callback)
                 }
                 blorb = new Blorb(story.data)
                 format = identify_blorb_storyfile_format(blorb)
@@ -130,7 +149,7 @@ class ParchmentLauncher
             const engine = format.engines![0]
 
             const requires = await Promise.all([
-                story.data || fetch_storyfile(this.options, story.url!),
+                story.data || fetch_storyfile(this.options, story.url!, progress_callback),
                 ...engine.load.map(path => fetch_vm_resource(this.options, path))
             ])
             story.data = requires[0]

src/common/options.ts

@@ -19,6 +19,8 @@ export interface StoryOptions {
     data?: Uint8Array
     /** Size of storyfile in bytes, uncompressed */
     filesize?: number
+    /** Size of storyfile in bytes, gzip compressed (doesn't need to be exact) */
+    filesize_gz?: number
     /** Format ID, matching formats.js */
     format?: string
     url?: string

src/iplayif.com/app/src/front-page.ts

@@ -59,6 +59,7 @@ export default class FrontPage {
                 cover: !!data.cover,
                 description: data.description,
                 filesize: data.filesize,
+                filesize_gz: data.filesize_gz,
                 format: data.format,
                 ifid: data.ifid,
                 title: data.title,

src/iplayif.com/app/src/metadata.ts

@@ -26,6 +26,7 @@ import sharp from 'sharp'
 import {flatten_query, SiteOptions} from './common.js'
 import {SUPPORTED_TYPES} from './common.js'
 
+const exec = util.promisify(child_process.exec)
 const execFile = util.promisify(child_process.execFile)
 
 export class FileMetadata {
@@ -36,6 +37,7 @@ export class FileMetadata {
     }
     description?: string
     filesize: number
+    filesize_gz?: number
     format: string
     ifid: string
     title: string
@@ -128,6 +130,12 @@ export async function get_metadata(file_name: string, file_path: string) {
         result.title = author_data[0].replace(/^[\s"]+|[\s"]+$/g, '')
     }
 
+    // Estimate the gzipped size
+    const gzip_results = await exec(`gzip -c ${file_path} | wc -c`)
+    if (!gzip_results.stderr.length) {
+        result.filesize_gz = parseInt(gzip_results.stdout, 10)
+    }
+
     // Extract a cover
     let cover: Uint8Array | undefined
     if (!identify_data[2].includes('no cover')) {

src/tools/index-processing.ts

@@ -23,6 +23,7 @@ export interface Story {
     description?: string
     filename?: string
     filesize?: number
+    filesize_gz?: number
     format?: string
     ifid?: string
     title?: string
@@ -100,6 +101,9 @@ export async function process_index_html(options: SingleFileOptions, files: Map<
         if (story.filesize) {
             parchment_options.story.filesize = story.filesize
         }
+        if (story.filesize_gz) {
+            parchment_options.story.filesize_gz = story.filesize_gz
+        }
         if (story.format) {
             parchment_options.story.format = story.format
         }
@@ -198,6 +202,13 @@ export async function process_index_html(options: SingleFileOptions, files: Map<
             <p>Parchment requires Javascript. Please enable it in your browser.</p>
         </noscript>`)
             .replace('<div id="loadingpane" style="display:none;">', '<div id="loadingpane">')
+
+        // Add a progress indicator
+        if (story.filesize) {
+            indexhtml = indexhtml.replace('<em>&nbsp;&nbsp;&nbsp;Loading...</em>', `<em>&nbsp;&nbsp;&nbsp;Loading...</em><br>
+            <progress id="loading_progress" max="${story.filesize}" value="0"></progress><br>
+            <span id="loading_size"></span>`)
+        }
     }
 
     // Replace the cover image

src/web/parchment.css

@@ -3,7 +3,7 @@
 Parchment
 =========
 
-Copyright (c) 2020 Dannii Willis
+Copyright (c) 2024 Dannii Willis
 MIT licenced
 https://github.com/curiousdannii/parchment
 
@@ -50,4 +50,8 @@ body {
   bottom: 0;
   position: absolute;
   width: 100%;
+}
+
+#loading_progress {
+  width: 250px;
 }