Unify the single-file processing code

src/.dockerignore

@@ -0,0 +1,11 @@
+# The Docker context of iplayif.com/app needs to be this top-level src folder so that files from common etc can be imported
+# That means the .dockerignore must be here, and folders must have relative paths
+
+fonts
+inform7
+iplayif.com/app/build
+iplayif.com/app/node_modules
+iplayif.com/app/npm-debug.log
+iplayif.com/data
+upstream
+web

src/common/options.ts

@@ -13,7 +13,7 @@ import {ClassicSyncDialog, GlkApi, GlkOte, GlkOteOptions, WebGlkOte} from '../up
 import WebDialog from '../upstream/glkote/dialog.js'
 import GlkOte_GlkApi from '../upstream/glkote/glkapi.js'
 
-type ParchmentTruthy = boolean | number
+export type ParchmentTruthy = boolean | number
 
 export interface ParchmentOptions extends Partial<GlkOteOptions> {
     // Parchment options

src/iplayif.com/app/Dockerfile

@@ -10,20 +10,21 @@ RUN apk add --no-cache build-base curl && \
     make
 
 # Stage 2: The app itself
+# Note that the Docker context is the top level src/ folder
 
 FROM node:20-alpine
 
-WORKDIR /home/app
+EXPOSE 8080
+
+WORKDIR /home/iplayif.com/app
 
 COPY --from=build-babel /home/babel /usr/local/bin/
 
-COPY package*.json ./
+COPY iplayif.com/app/package*.json ./
 
 RUN npm ci --production
 
-COPY . .
-
-EXPOSE 8080
+COPY . ../../
 
 RUN npm run build
 

src/iplayif.com/app/src/sitegen.ts

@@ -11,20 +11,18 @@ https://github.com/curiousdannii/parchment
 
 import fs from 'fs/promises'
 import child_process from 'child_process'
+import path from 'path'
 import util from 'util'
 
 import Koa from 'koa'
-import {escape} from 'lodash-es'
 
-import {ParchmentOptions} from '../../../common/options.js'
+import {process_index_html, SingleFileOptions} from '../../../tools/index-processing.js'
 
 import FrontPage from './front-page.js'
 import {flatten_query, SiteOptions} from './common.js'
 
 const execFile = util.promisify(child_process.execFile)
 
-const utf8decoder = new TextDecoder()
-
 const parchment_formats: Record<string, string> = {
     adrift: 'adrift4',
     'blorbed glulx': 'glulx',
@@ -44,100 +42,6 @@ const format_terp_files: Record<string, string[]> = {
     zcode: ['zvm.js'],
 }
 
-interface SiteGenOptions {
-    cdn_domain: string
-    font?: boolean
-    format?: string
-    ifid?: string
-    remote?: boolean
-    single_file?: boolean
-    story_file?: Buffer
-    title: string
-}
-
-// taken from https://github.com/curiousdannii/parchment/blob/master/src/tools/single-file.js
-async function generate(options: SiteGenOptions, indexhtml: string, files: Record<string, Buffer>) {
-    const inclusions: string[] = []
-    if (options.ifid) {
-        inclusions.push(`<meta prefix="ifiction: http://babel.ifarchive.org/protocol/iFiction/" property="ifiction:ifid" content="${options.ifid}">`)
-    }
-    if (options.single_file) {
-        const parchment_options: Partial<ParchmentOptions> = { single_file: 1 }
-        if (options.format) {
-            parchment_options.format = options.format
-        }
-        if (options.story_file) {
-            parchment_options.story = `data:application/octet-stream;base64,` + options.story_file.toString('base64')
-        }
-        inclusions.push(`<script>parchment_options = ${JSON.stringify(parchment_options, null, 2)}</script>`)
-    }
-
-    // Process the files
-    for (const [filename, raw_data] of Object.entries(files)) {
-        let data: Buffer | string = raw_data
-        if (/\.(css|js|html)$/.test(filename)) {
-            data = utf8decoder.decode(raw_data)
-                .replace(/(\/\/|\/\*)# sourceMappingURL.+/, '')
-                .trim()
-        }
-        if (filename === 'ie.js') {
-            inclusions.push(`<script nomodule>${data}</script>`)
-        }
-        else if (filename === 'jquery.min.js') {
-            if (options.remote) {
-                inclusions.push(`<script src="https://${options.cdn_domain}/dist/web/${filename}"></script>`)
-            } else {
-                inclusions.push(`<script>${data}</script>`)
-            }
-        }
-        else if (filename === 'main.js') {
-            if (options.remote) {
-                inclusions.push(`<script src="https://${options.cdn_domain}/dist/web/${filename}" type="module"></script>`)
-            } else {
-                inclusions.push(`<script type="module">${data}</script>`)
-            }
-        }
-        else if (filename.endsWith('.css')) {
-            // Only include a single font, the browser can fake bold and italics
-            const fontfile = files['../fonts/iosevka/iosevka-extended.woff2'].toString('base64')
-            data = (data as string).replace(/@font-face{font-family:([' \w]+);font-style:(\w+);font-weight:(\d+);src:url\([^)]+\) format\(['"]woff2['"]\)}/g, (_, font: string, style: string, weight: string) => {
-                if (font === 'Iosevka' && style === 'normal' && weight === '400' && options.font) {
-                    return `@font-face{font-family:Iosevka;font-style:normal;font-weight:400;src:url(data:font/woff2;base64,${fontfile}) format('woff2')}`
-                }
-                return ''
-            })
-                .replace(/Iosevka Narrow/g, 'Iosevka')
-            if (!options.font) {
-                data = data.replace(/--glkote(-grid)?-mono-family: "Iosevka", monospace;?/g, '')
-            }
-            inclusions.push(`<style>${data}</style>`)
-        }
-        else if (filename.endsWith('.js')) {
-            inclusions.push(`<script type="text/plain" id="./${filename}">${data}</script>`)
-        }
-        else if (filename.endsWith('.wasm')) {
-            inclusions.push(`<script type="text/plain" id="./${filename}">${data.toString('base64')}</script>`)
-        }
-    }
-
-    // Inject into index.html
-    const gif = files['waiting.gif'].toString('base64')
-    indexhtml = indexhtml
-        .replace(/<script.+?\/script>/g, '')
-        .replace(/<link rel="stylesheet".+?>/g, '')
-        .replace(/<img src="dist\/web\/waiting.gif"/, `<img src="data:image/gif;base64,${gif}"`)
-        .replace(/<title>.+?\/title>/, `<title>${escape(options.title)}</title>`)
-        .replace(/^\s+$/gm, '')
-
-    // Add the inclusions
-    const parts = indexhtml.split(/\s*<\/head>/)
-    indexhtml = `${parts[0]}
-    ${inclusions.join('\n')}
-    </head>${parts[1]}`
-
-    return indexhtml
-}
-
 export default class SiteGenerator {
     // TODO: replace with a proper CDN cache thingy
     front_page: FrontPage
@@ -161,8 +65,9 @@ export default class SiteGenerator {
             </form>`
             return
         }
-        const story_file_path = flatten_query(ctx.request.files!.story_file)!.filepath
-        const identify_results = await execFile('babel', ['-identify', story_file_path])
+        const story_file = flatten_query(ctx.request.files!.story_file)!
+        const filename = story_file.originalFilename!
+        const identify_results = await execFile('babel', ['-identify', story_file.filepath])
         if (identify_results.stderr) {
             ctx.throw(400, 'Unsupported file type')
         }
@@ -190,7 +95,8 @@ export default class SiteGenerator {
 
         const terp_files = format_terp_files[parchment_format]
 
-        const urls = [
+        const paths = [
+            '../../index.html',
             'jquery.min.js',
             'main.js',
             'waiting.gif',
@@ -199,25 +105,26 @@ export default class SiteGenerator {
             ...terp_files,
         ]
 
-        const files = Object.fromEntries(await Promise.all(urls.map(async url =>
-            [url, await fetch(`https://${this.options.cdn_domain}/dist/web/${url}`).then(r => r.arrayBuffer()).then(b => Buffer.from(b))]
-        )))
+        // Get all the files
+        const files: Map<string, Uint8Array> = new Map()
+        for (const file of paths) {
+            files.set(path.basename(file), await fetch(`https://${this.options.cdn_domain}/dist/web/${file}`).then(r => r.arrayBuffer()).then(b => new Uint8Array(b)))
+        }
 
-        const options: SiteGenOptions = {
-            cdn_domain: this.options.cdn_domain,
+        const options: SingleFileOptions = {
             font: true,
-            format: parchment_format,
-            ifid,
-            remote: false,
             single_file: true,
-            story_file: await fs.readFile(story_file_path),
-            title: bibliographic,
+            story: {
+                data: await fs.readFile(story_file.filepath),
+                filename,
+                format: parchment_format,
+                ifid,
+                title: bibliographic,
+            },
         }
 
-        const html = await generate(options, this.front_page.index_html, files)
-        const outfileName = bibliographic.replace(/"/g, '').replace(/[^"\\A-Za-z0-9'-]+/g, '-')
         ctx.type = 'text/html; charset=UTF-8'
-        ctx.set('Content-Disposition', `attachment; filename="${outfileName}.html"`)
-        ctx.body = html
+        ctx.set('Content-Disposition', `attachment; filename="${filename}.html"`)
+        ctx.body = await process_index_html(options, files)
     }
 }

src/iplayif.com/docker-compose.yml

@@ -1,6 +1,8 @@
 services:
   app:
-    build: ./app
+    build:
+      context: ../
+      dockerfile: ./iplayif.com/app/Dockerfile
     environment:
       DATA_DIR: /home/data
     logging:

src/iplayif.com/nginx/nginx.sh

@@ -78,6 +78,8 @@ server {
     $SSL
     $GZIP
 
+    client_max_body_size 100M;
+
     location = / {
         proxy_pass http://app:8080;
     }

src/tools/index-processing.ts

@@ -3,37 +3,41 @@
 Common index.html processing
 ============================
 
-Copyright (c) 2023 Dannii Willis
+Copyright (c) 2024 Dannii Willis
 MIT licenced
 https://github.com/curiousdannii/parchment
 
 */
 
-import {ParchmentOptions} from '../common/options.js'
+import {escape} from 'lodash-es'
+
+import {ParchmentTruthy, ParchmentOptions} from '../common/options.js'
 
 // Is ASCII really okay here?
 const utf8decoder = new TextDecoder('ascii', {fatal: true})
 
 export interface Story {
+    // TODO use author and title
     author?: string
     cover?: Uint8Array
     data?: Uint8Array
     description?: string
     filename?: string
+    format?: string
+    ifid?: string
     title?: string
 }
 
 export interface SingleFileOptions {
-    date?: boolean | number
-    font?: boolean | number
-    format?: string
-    single_file?: boolean | number
-    story_file?: Story
+    date?: ParchmentTruthy
+    font?: ParchmentTruthy
+    single_file?: ParchmentTruthy
+    story?: Story
 }
 
-async function Uint8Array_to_base64(data: Buffer | Uint8Array): Promise<string> {
+async function Uint8Array_to_base64(data: Uint8Array): Promise<string> {
     if (typeof Buffer !== 'undefined') {
-        return data.toString('base64')
+        return Buffer.from(data.buffer).toString('base64')
     }
     // From https://stackoverflow.com/a/66046176/2854284
     else if (typeof FileReader !== 'undefined') {
@@ -47,9 +51,17 @@ async function Uint8Array_to_base64(data: Buffer | Uint8Array): Promise<string>
     throw new Error('Cannot encode base64')
 }
 
-export async function make_single_file(options: SingleFileOptions, files: Map<string, Uint8Array>): Promise<string> {
+export async function process_index_html(options: SingleFileOptions, files: Map<string, Uint8Array>): Promise<string> {
+    const story = options.story
     const inclusions: string[] = []
 
+    // Metadata
+    if (story) {
+        if (story.ifid) {
+            inclusions.push(`<meta prefix="ifiction: http://babel.ifarchive.org/protocol/iFiction/" property="ifiction:ifid" content="${story?.ifid}">`)
+        }
+    }
+
     // Process the files
     let indexhtml = ''
     for (const [filename, data] of files) {
@@ -102,15 +114,17 @@ export async function make_single_file(options: SingleFileOptions, files: Map<st
 
     // Parchment options
     const parchment_options: Partial<ParchmentOptions> = {}
-    if (options.format) {
-        parchment_options.format = options.format
-    }
     if (options.single_file) {
         parchment_options.single_file = 1
     }
-    if (options.story_file) {
-        parchment_options.story = 'embedded:' + options.story_file.filename
-        inclusions.push(`<script type="text/plain" id="${options.story_file.filename}">${await Uint8Array_to_base64(options.story_file.data!)}</script>`)
+    if (story) {
+        if (story.format) {
+            parchment_options.format = story?.format
+        }
+        if (story.data) {
+            parchment_options.story = 'embedded:' + story.filename!
+            inclusions.push(`<script type="text/plain" id="${story.filename!}">${await Uint8Array_to_base64(story.data)}</script>`)
+        }
     }
     inclusions.push(`<script>parchment_options = ${JSON.stringify(parchment_options, null, 2)}</script>`)
 
@@ -127,7 +141,7 @@ export async function make_single_file(options: SingleFileOptions, files: Map<st
         const today = new Date()
         const title = `Parchment ${today.getFullYear()}.${today.getMonth() + 1}.${today.getDate()}`
         indexhtml = indexhtml
-            .replace(/<title.+?\/title>/, `<title>${title}</title>`)
+            .replace(/<title.+?\/title>/, `<title>${escape(title)}</title>`)
             .replace(/<\/noscript>/, `</noscript>\n<p id="footer-date">${title}</p>`)
     }