Unify front-page and index-processing

Unify sitegen and metadata
curiousdannii · Jan 24, 2024 · 1f046a1 · 1f046a1
1 parent 36d3261
commit 1f046a1
Show file tree

Hide file tree

Showing 7 changed files with 240 additions and 190 deletions.
diff --git a/src/common/launcher.ts b/src/common/launcher.ts
@@ -14,7 +14,7 @@ import Cookies from 'js-cookie'
 import {AsyncGlk, Blorb, FileView} from '../upstream/asyncglk/src/index-browser.js'
 import {get_default_options, get_query_options, ParchmentOptions, StoryOptions} from './options.js'
 import {fetch_storyfile, fetch_vm_resource, read_uploaded_file} from './file.js'
-import {find_format, Format, identify_blorb_storyfile_format} from './formats.js'
+import {find_format, identify_blorb_storyfile_format} from './formats.js'
 
 interface ParchmentWindow extends Window {
     parchment: ParchmentLauncher
@@ -27,7 +27,12 @@ class ParchmentLauncher
     options: ParchmentOptions
 
     constructor(parchment_options?: ParchmentOptions) {
-        this.options = Object.assign({}, get_default_options(), parchment_options, get_query_options(['do_vm_autosave', 'story', 'use_asyncglk']))
+        // Only get story from the URL if there is no story already in the parchment_options
+        const query_options: Array<keyof ParchmentOptions> = ['do_vm_autosave', 'use_asyncglk']
+        if (!parchment_options?.story) {
+            query_options.push('story')
+        }
+        this.options = Object.assign({}, get_default_options(), parchment_options, get_query_options(query_options))
         // Use AsyncGlk if requested
         if (this.options.use_asyncglk) {
             this.options.Glk = new AsyncGlk()

diff --git a/src/common/options.ts b/src/common/options.ts
@@ -17,6 +17,8 @@ export type ParchmentTruthy = boolean | number
 
 export interface StoryOptions {
     data?: Uint8Array
+    /** Size of storyfile in bytes, uncompressed */
+    filesize?: number
     /** Format ID, matching formats.js */
     format?: string
     url?: string

diff --git a/src/iplayif.com/app/src/common.ts b/src/iplayif.com/app/src/common.ts
@@ -34,4 +34,6 @@ export interface SiteOptions {
     proxy: {
         max_size: number
     }
-}
+}
+
+export const utf8encoder = new TextEncoder()
diff --git a/src/iplayif.com/app/src/front-page.ts b/src/iplayif.com/app/src/front-page.ts
@@ -1,6 +1,6 @@
 /*
 
-Dynamic iplayid.com front page
+Dynamic iplayif.com front page
 ==============================
 
 Copyright (c) 2024 Dannii Willis
@@ -10,10 +10,11 @@ https://github.com/curiousdannii/parchment
 */
 
 import Koa from 'koa'
-import {escape, truncate} from 'lodash-es'
 
-import {flatten_query, SiteOptions, SUPPORTED_TYPES} from './common.js'
-import {MetadataCache} from './metadata.js'
+import {process_index_html, SingleFileOptions} from '../../../tools/index-processing.js'
+
+import {flatten_query, SiteOptions, SUPPORTED_TYPES, utf8encoder} from './common.js'
+import {FileMetadata, MetadataCache} from './metadata.js'
 
 export default class FrontPage {
     index_html: string = 'ERROR: front page index.html not yet loaded'
@@ -36,63 +37,39 @@ export default class FrontPage {
             return
         }
 
-        // Check it's actually a URL
+        // Check it's actually a URL and that we have data
+        let data: FileMetadata
         try {
             new URL(story_url)
+            data = (await this.metadata.get(story_url))!
+            //console.log(data)
+            if (!data) {
+                throw null
+            }
         }
         catch (_) {
             ctx.body = this.index_html
             return
         }
 
-        const protocol_domain = `http${this.options.https ? 's' : ''}://${this.options.domain}`
-
-        const data = (await this.metadata.get(story_url))!
-        //console.log(data)
-
-        // Embed the metadata into the page title
-        let body = this.index_html
-            .replace('<title>Parchment</title>', `<title>${escape(data.title)} - Parchment</title>`)
-
-        // Open Graph meta data
-        const open_graph: string[] = []
-        if (data.title && data.author) {
-            open_graph.push(
-                `<meta property="og:site_name" content="Parchment"/>`,
-                `<meta property="og:title" content="${escape(data.title)} by ${escape(data.author)}"/>`,
-                `<meta property="og:type" content="website"/>`,
-                `<meta property="og:url" content="${protocol_domain}/?story=${encodeURIComponent(story_url)}"/>`,
-
-            )
-            if (data.description) {
-                open_graph.push(`<meta property="og:description" content="${escape(truncate(data.description, {
-                    length: 1000,
-                    separator: /[,.]? +/,
-                }))}"/>`)
-            }
-            if (data.cover) {
-                open_graph.push(`<meta property="og:image" content="${protocol_domain}/metadata/cover/?url=${encodeURIComponent(story_url)}&maxh=630"/>`)
-            }
-        }
-        if (open_graph.length) {
-            body = body.replace(/<\/head>/, `    ${open_graph.join('\n    ')}
-</head>`)
-        }
-
-        // Use the story cover
-        if (data.cover) {
-            body = body.replace(/<img src="dist\/web\/waiting\.gif"/, `<img src="${protocol_domain}/metadata/cover/?url=${encodeURIComponent(story_url)}&maxh=250"`)
+        const options: SingleFileOptions = {
+            domain: `http${this.options.https ? 's' : ''}://${this.options.domain}`,
+            story: {
+                author: data.author,
+                cover: !!data.cover,
+                description: data.description,
+                filesize: data.filesize,
+                format: data.format,
+                ifid: data.ifid,
+                title: data.title,
+                url: story_url,
+            },
         }
 
-        // And simplify the HTML a little
-        body = body.replace(/<div id="about">.+<\/noscript>\s+<\/div>/s, `<noscript>
-            <h1>Parchment</h1>
-            <p>is an interpreter for Interactive Fiction. <a href="https://github.com/curiousdannii/parchment">Find out more.</a></p>
-            <p>Parchment requires Javascript. Please enable it in your browser.</p>
-        </noscript>`)
-            .replace('<div id="loadingpane" style="display:none;">', '<div id="loadingpane">')
+        const files: Map<string, Uint8Array> = new Map()
+        files.set('index.html', utf8encoder.encode(this.index_html))
 
-        ctx.body = body
+        ctx.body = await process_index_html(options, files)
     }
 
     async update_index_html() {

diff --git a/src/iplayif.com/app/src/metadata.ts b/src/iplayif.com/app/src/metadata.ts
@@ -28,22 +28,28 @@ import {SUPPORTED_TYPES} from './common.js'
 
 const execFile = util.promisify(child_process.execFile)
 
-class File {
+export class FileMetadata {
     author?: string
     cover?: null | {
         data: Uint8Array
         type: string
     }
     description?: string
+    filesize: number
+    format: string
+    ifid: string
     title: string
 
-    constructor(title: string) {
+    constructor(title: string, filesize: number, format: string, ifid: string) {
         this.title = title
+        this.filesize = filesize
+        this.format = format
+        this.ifid = ifid
     }
 }
 
 export class MetadataCache {
-    lru: LRUCache<string, File>
+    lru: LRUCache<string, FileMetadata>
     temp: string
 
     constructor(options: SiteOptions) {
@@ -60,97 +66,119 @@ export class MetadataCache {
 
     async fetch(url: string) {
         const file_name = /([^/=]+)$/.exec(url)![1]
-        const result = new File(file_name)
+        const file_path = `${this.temp}/${Math.floor(Math.random() * 1000000000).toString().padStart(9, '0')}${path.extname(file_name)}`
 
         // Absorb all errors here
         try {
             console.log(`Metadata server fetching ${url}`)
             const response = await fetch(url)
             if (!response.ok) {
-                return result
+                return
             }
 
             // Write the file stream to temp
-            const file_path = `${this.temp}/${Math.floor(Math.random() * 1000000000).toString().padStart(9, '0')}${path.extname(file_name)}`
             // TODO: fix type
             await pipeline(response.body as any, fs_sync.createWriteStream(file_path))
 
-            // Run babel -identify
-            const identify_results = await execFile('babel', ['-identify', file_path])
-            if (identify_results.stderr.length) {
-                // If there was an error do nothing for now
-            }
-            else {
-                const identify_data = identify_results.stdout.split('\n')
-                if (identify_data[0] !== 'No bibliographic data') {
-                    const author_data = identify_data[0].split(' by ')
-                    result.author = author_data[1].trim()
-                    result.title = author_data[0].replace(/^[\s"]+|[\s"]+$/g, '')
-                }
+            return await get_metadata(file_name, file_path)
+        }
+        finally {
+            // Clean up
+            await fs.rm(file_path, {force: true})
+        }
+    }
 
-                // Extract a cover
-                let cover: Uint8Array | undefined
-                if (!identify_data[2].includes('no cover')) {
-                    const extract_cover_results = await execFile('babel', ['-cover', file_path])
-                    if (!extract_cover_results.stderr.length) {
-                        const cover_path = /Extracted ([-\w.]+)/.exec(extract_cover_results.stdout)![1]
-                        cover = await fs.readFile(cover_path)
-                        await fs.rm(cover_path)
-                    }
-                }
+    get(url: string) {
+        return this.lru.fetch(url)
+    }
+}
 
-                // See if there's a description
-                const description_results = await execFile('babel', ['-meta', file_path])
-                if (!description_results.stderr.length) {
-                    result.description = extract_description(description_results.stdout)
-                }
+export const parchment_formats: Record<string, string> = {
+    adrift: 'adrift4',
+    'blorbed glulx': 'glulx',
+    'blorbed zcode': 'zcode',
+    glulx: 'glulx',
+    hugo: 'hugo',
+    tads2: 'tads',
+    tads3: 'tads',
+    zcode: 'zcode',
+}
 
-                // Check the IFDB for details
-                if (!result.author || !result.cover || !result.description) {
-                    const ifid = /IFID: ([-\w]+)/i.exec(identify_data[1])![1]
-                    const ifdb_response = await fetch(`https://ifdb.org/viewgame?ifiction&ifid=${ifid}`)
-                    if (ifdb_response.ok) {
-                        const ifdb_xml = await ifdb_response.text()
-                        if (!result.author) {
-                            result.author = he.decode(/<author>(.+?)<\/author>/.exec(ifdb_xml)![1])
-                        }
-                        if (result.title === file_name) {
-                            result.title = he.decode(/<title>(.+?)<\/title>/.exec(ifdb_xml)![1])
-                        }
-                        if (!result.cover) {
-                            const cover_url = /<coverart><url>(.+?)<\/url><\/coverart>/.exec(ifdb_xml)?.[1]
-                            if (cover_url) {
-                                const cover_response = await fetch(he.decode(cover_url))
-                                if (cover_response.ok) {
-                                    cover = Buffer.from(await cover_response.arrayBuffer())
-                                }
-                            }
-                        }
-                        if (!result.description) {
-                            result.description = extract_description(ifdb_xml)
-                        }
-                    }
-                }
+export async function get_metadata(file_name: string, file_path: string) {
+    // Get filesize
+    const stats = await fs.stat(file_path)
+
+    // Run babel -identify
+    const identify_results = await execFile('babel', ['-identify', file_path])
+    //console.log(identify_results.stdout)
+
+    if (identify_results.stderr.length) {
+        // If there was an error we can't really do anything
+        return
+    }
 
-                if (cover) {
-                    result.cover = {
-                        data: cover,
-                        type: (await sharp(cover).metadata())!.format!
+    const identify_data = identify_results.stdout.split('\n')
+
+    const [babel_format] = identify_data[2].split(',')
+    const result = new FileMetadata(file_name, stats.size, parchment_formats[babel_format], /IFID: ([-\w]+)/i.exec(identify_data[1])![1])
+
+    if (identify_data[0] !== 'No bibliographic data') {
+        const author_data = identify_data[0].split(' by ')
+        result.author = author_data[1].trim()
+        result.title = author_data[0].replace(/^[\s"]+|[\s"]+$/g, '')
+    }
+
+    // Extract a cover
+    let cover: Uint8Array | undefined
+    if (!identify_data[2].includes('no cover')) {
+        const extract_cover_results = await execFile('babel', ['-cover', file_path])
+        if (!extract_cover_results.stderr.length) {
+            const cover_path = /Extracted ([-\w.]+)/.exec(extract_cover_results.stdout)![1]
+            cover = await fs.readFile(cover_path)
+            await fs.rm(cover_path)
+        }
+    }
+
+    // See if there's a description
+    const description_results = await execFile('babel', ['-meta', file_path])
+    if (!description_results.stderr.length) {
+        result.description = extract_description(description_results.stdout)
+    }
+
+    // Check the IFDB for details
+    if (!result.author || !result.cover || !result.description) {
+        const ifdb_response = await fetch(`https://ifdb.org/viewgame?ifiction&ifid=${result.ifid}`)
+        if (ifdb_response.ok) {
+            const ifdb_xml = await ifdb_response.text()
+            if (!result.author) {
+                result.author = he.decode(/<author>(.+?)<\/author>/.exec(ifdb_xml)![1])
+            }
+            if (result.title === file_name) {
+                result.title = he.decode(/<title>(.+?)<\/title>/.exec(ifdb_xml)![1])
+            }
+            if (!result.cover) {
+                const cover_url = /<coverart><url>(.+?)<\/url><\/coverart>/.exec(ifdb_xml)?.[1]
+                if (cover_url) {
+                    const cover_response = await fetch(he.decode(cover_url))
+                    if (cover_response.ok) {
+                        cover = Buffer.from(await cover_response.arrayBuffer())
                     }
                 }
             }
-
-            // Clean up and return
-            await fs.rm(file_path)
+            if (!result.description) {
+                result.description = extract_description(ifdb_xml)
+            }
         }
-        catch (_) {}
-
-        return result
     }
 
-    get(url: string) {
-        return this.lru.fetch(url)
+    if (cover) {
+        result.cover = {
+            data: cover,
+            type: (await sharp(cover).metadata())!.format!
+        }
     }
+
+    return result
 }
 
 export async function metadata_cover(cache: MetadataCache, ctx: Koa.Context) {