Add support for a persisted disk image

Add a mode where we persist the changes made to a chunked disk image in
the origin private file system feature that is supported by modern browsers.
They also support being able to get a synchronous file handle, which allows
us to directly intercept read/write calls from the emulator with no complications.

The persistence is handled as an overlay on top of the chunked data, we have a
separate dirty chunks bitmask that stores which chunks have been modified. When
we're about to read from a chunk, we first load it from the persisted disk.
Similarly, when a change to a chunk is made, we mark it as dirty and write it
to the persisted disk. This does mean that there is some write amplification
(any change writes a whole 256K chunk), but in practice performance appears to
be OK (the native file system cache kicks in presumably).

Since the disk image is mostly empty, we stored it compressed in Git (to avoid
having a large file). We also optimize empty chunks, so that we don't need to
store a signature for them, or load them over the network.

To feature detect if the browser supports the origin private file system and
synchronous file handles, we need to create a temporary worker (FileSystemSyncAccessHandle
is only ever defined in workers). We cache the result of this check in
localStorage so that we don't need to do it too often.

Currently disabled by default, but accessible via the custom instance
dialog or the ?saved_hd=true query parameter

Updates #152

Author: mihaip

.gitignore

@@ -33,3 +33,7 @@ public/Covers
 
 # Cloudflare Worker wrangler CLI state
 .wrangler
+
+# We only commit the compressed version, but having the uncompressed version
+# around is useful for local development.
+Images/Saved HD.dsk

Art/Saved HD Icon.acorn

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e273bbffd601335ea6d1b0f5d98dc6b87726ed2a11646f40afe04f2e227317
+size 1126897

Images/Saved HD.dsk.zip

@@ -384,6 +384,9 @@ def write_image_def(image: bytes, name: str, dest_dir: str) -> ImageDef:
     return ImageDef(name, image_path)
 
 
+ZERO_CHUNK = b"\0" * CHUNK_SIZE
+
+
 def write_chunked_image(image: ImageDef) -> None:
     total_size = 0
     chunks = []
@@ -397,6 +400,11 @@ def write_chunked_image(image: ImageDef) -> None:
                          (image.name, ((i + CHUNK_SIZE) / disk_size) * 100))
         chunk = image_bytes[i:i + CHUNK_SIZE]
         total_size += len(chunk)
+        # Don't bother storing zero-ed out chunks (common for the saved HD),
+        # the signature takes up space, and we don't need to load them
+        if chunk == ZERO_CHUNK:
+            chunks.append("")
+            continue
         chunk_signature = hashlib.blake2b(chunk, digest_size=16,
                                           salt=salt).hexdigest()
         chunks.append(chunk_signature)
@@ -519,6 +527,39 @@ def build_passthrough_image(base_name: str, dest_dir: str) -> ImageDef:
     return write_image_def(image_data, base_name, dest_dir)
 
 
+def build_saved_hd_image(base_name: str, dest_dir: str) -> ImageDef:
+    # The disk image is compressed since it's mostly empty space and we don't
+    # want to pay for a lot of Git LFS storage.
+    with zipfile.ZipFile(os.path.join(paths.IMAGES_DIR, base_name + ".zip"),
+                         "r") as zip:
+        image_data = zip.read(base_name)
+
+    # Also use the Stickies placeholder file to inject the Saved HD Read Me
+    readme_placeholder = stickies.generate_ttxt_placeholder()
+    readme_index = image_data.find(readme_placeholder)
+
+    if readme_index != -1:
+        readme_data = "TODO: explain what this is for".encode("macroman")
+
+        if len(readme_data) > len(readme_placeholder):
+            logging.warning(
+                "Read Me file is too large (%d, placeholder is only %d), "
+                "skipping customization for %s", len(readme_data),
+                len(readme_placeholder), base_name)
+        else:
+            # Replace the leftover placeholder data, so that TextText does not
+            # render.
+            image_data = image_data[:readme_index] + readme_data + \
+                b"\x00" * (len(readme_placeholder) - len(readme_data)) + \
+                image_data[readme_index + len(readme_placeholder):]
+    else:
+        logging.warning(
+            "Placeholder file not found in disk image %s, skipping Read Me",
+            base_name)
+
+    return write_image_def(image_data, base_name, dest_dir)
+
+
 def build_desktop_db(images: typing.List[ImageDef]) -> bytes:
     sys.stderr.write("Rebuilding Desktop DB for %s...\n" %
                      ",".join([i.name for i in images]))
@@ -635,6 +676,7 @@ def build_desktop_db(images: typing.List[ImageDef]) -> bytes:
             images.append(
                 build_passthrough_image("Infinite HD (MFS).dsk",
                                         dest_dir=temp_dir))
+        images.append(build_saved_hd_image("Saved HD.dsk", dest_dir=temp_dir))
 
         for image in images:
             write_chunked_image(image)

scripts/import-disks.py

@@ -240,6 +240,7 @@ function DiskContents({disk, onRun, setBezelStyle}: DiskContentsProps) {
         const runDef = {
             disks: [disk],
             includeInfiniteHD: true,
+            includeSavedHD: true,
             machine: disk.machines[0],
             cdromURLs: [],
             debugAudio: false,

src/Browser.tsx

@@ -15,6 +15,11 @@
     display: inline;
 }
 
+.Custom-Dialog-Row label.disabled {
+    color: #999;
+    pointer-events: none;
+}
+
 .Custom-Dialog-Row label:last-of-type {
     margin-bottom: 4px;
 }

src/Custom.css

@@ -21,6 +21,8 @@ import {Select} from "./controls/Select";
 import {Checkbox} from "./controls/Checkbox";
 import {emulatorSupportsAppleTalk} from "./emulator/emulator-common-emulators";
 import {CloudflareWorkerEthernetProvider} from "./CloudflareWorkerEthernetProvider";
+import classNames from "classnames";
+import {canSaveDisks} from "./canSaveDisks";
 
 export function Custom({
     defaultDisk = SYSTEM_DISKS_BY_NAME["System 7.1"],
@@ -41,6 +43,7 @@ export function Custom({
         disks: [defaultDisk],
         cdromURLs: [],
         includeInfiniteHD: true,
+        includeSavedHD: canSaveDisks() && false, // TODO: enable by default
         debugFallback: false,
         debugAudio: false,
     });
@@ -285,6 +288,31 @@ export function Custom({
                 </div>
             </div>
 
+            <div className="Custom-Dialog-Row">
+                <span className="Custom-Dialog-Label" />
+                <label className={classNames({"disabled": !canSaveDisks()})}>
+                    <Checkbox
+                        appearance={appearance}
+                        disabled={!canSaveDisks()}
+                        checked={runDef.includeSavedHD}
+                        onChange={e =>
+                            setRunDef({
+                                ...runDef,
+                                includeSavedHD: e.target.checked,
+                            })
+                        }
+                    />
+                    Saved HD
+                </label>
+                <div className="Custom-Dialog-Description Dialog-Description">
+                    Include the Saved HD disk, an empty 1 GB disk whose contents
+                    are preserved across visits to this site (best effort).
+                    {!canSaveDisks() && (
+                        <div>Not supported by this browser</div>
+                    )}
+                </div>
+            </div>
+
             {appleTalkSupported && (
                 <div className="Custom-Dialog-Row">
                     <span className="Custom-Dialog-Label" />

src/Custom.tsx

@@ -31,6 +31,7 @@ import {
     type SystemDiskDef,
     INFINITE_HD,
     INFINITE_HD_MFS,
+    SAVED_HD,
 } from "./disks";
 import {type MachineDefRAMSize, type MachineDef} from "./machines";
 import classNames from "classnames";
@@ -39,10 +40,12 @@ import {type Appearance} from "./controls/Appearance";
 import {Select} from "./controls/Select";
 import {Checkbox} from "./controls/Checkbox";
 import {fetchCDROMInfo} from "./cdroms";
+import {canSaveDisks} from "./canSaveDisks";
 
 export type MacProps = {
     disks: SystemDiskDef[];
     includeInfiniteHD: boolean;
+    includeSavedHD: boolean;
     cdroms: EmulatorCDROM[];
     initialErrorText?: string;
     machine: MachineDef;
@@ -57,6 +60,7 @@ export type MacProps = {
 export default function Mac({
     disks,
     includeInfiniteHD,
+    includeSavedHD,
     cdroms,
     initialErrorText,
     machine,
@@ -124,6 +128,9 @@ export default function Mac({
                 emulatorDisks.push(infiniteHd);
             }
         }
+        if (includeSavedHD && canSaveDisks()) {
+            emulatorDisks.push(SAVED_HD);
+        }
         const useSharedMemory =
             typeof SharedArrayBuffer !== "undefined" && !debugFallback;
         const emulator = new Emulator(
@@ -182,6 +189,8 @@ export default function Mac({
                 emulatorDidHaveError(emulator: Emulator, error: string) {
                     if (error.includes("load") && error.includes("/CD-ROM")) {
                         varz.increment("emulator_error:cdrom_chunk_load");
+                    } else if (error.includes("saved disk")) {
+                        varz.increment("emulator_error:saved_disk");
                     } else {
                         varz.increment("emulator_error:other");
                     }

src/Mac.tsx

@@ -36,6 +36,7 @@ export default function RunDefMac({
         <Mac
             disks={runDef.disks}
             includeInfiniteHD={runDef.includeInfiniteHD}
+            includeSavedHD={runDef.includeSavedHD}
             cdroms={cdroms}
             initialErrorText={cdromErrorText}
             machine={runDef.machine}

src/RunDefMac.tsx

@@ -0,0 +1,5 @@
+const supported =
+    typeof FileSystemFileHandle.prototype.createSyncAccessHandle ===
+        "function" && typeof FileSystemSyncAccessHandle === "function";
+
+postMessage(supported);

src/canSaveDisks-worker.ts

@@ -0,0 +1,46 @@
+import Worker from "./canSaveDisks-worker?worker&inline";
+
+export function canSaveDisks() {
+    if (!navigator.storage?.getDirectory) {
+        return false;
+    }
+    if (typeof FileSystemFileHandle !== "function") {
+        return false;
+    }
+
+    if (workerSyncFileSupport !== "unknown") {
+        return workerSyncFileSupport === "supported";
+    }
+    function update(support: "supported" | "unsupported") {
+        if (workerSyncFileSupport === "unsupported") {
+            // Ignore updates if one mechanism determined that it's unsupported.
+            return;
+        }
+        workerSyncFileSupport = support;
+        localStorage["workerSyncFileSupport"] = support;
+    }
+
+    // We can't check for the presence of FileSystemSyncAccessHandle in the
+    // main window, it's only available in workers. So we have to create a
+    // temporary one to check, but we cache the result in localStorage so that
+    // we can avoid doing this every time.
+    if (!worker) {
+        worker = new Worker();
+        worker.addEventListener("message", e => {
+            update(e.data ? "supported" : "unsupported");
+            worker?.terminate();
+        });
+    }
+
+    // In private contexts Safari exposes the storage API, but calls to it will
+    // fail, check for that too.
+    navigator.storage.getDirectory().catch(() => update("unsupported"));
+
+    // Assume supported until we hear otherwise.
+    return true;
+}
+
+let workerSyncFileSupport: "unknown" | "supported" | "unsupported" =
+    localStorage["workerSyncFileSupport"] ?? "unknown";
+
+let worker: Worker | undefined;

src/canSaveDisks.ts

@@ -25,6 +25,9 @@ export type EmulatorDiskDef = {
     // prefetchChunks are semi-automatically generated -- we will get a
     // warning via validateSpecPrefetchChunks() if these are incorrect.
     prefetchChunks: number[];
+    // Changes to this file will be persisted in the origin private file
+    // system.
+    persistent?: boolean;
 };
 
 export type SystemDiskDef = EmulatorDiskDef & {
@@ -765,3 +768,9 @@ export const INFINITE_HD_MFS: EmulatorDiskDef = {
     prefetchChunks: [0, 1, 2],
     generatedSpec: () => import("./Data/Infinite HD (MFS).dsk.json"),
 };
+
+export const SAVED_HD: EmulatorDiskDef = {
+    prefetchChunks: [0],
+    generatedSpec: () => import("./Data/Saved HD.dsk.json"),
+    persistent: true,
+};

src/disks.ts

@@ -74,6 +74,7 @@ export type EmulatorChunkedFileSpec = {
     chunks: string[];
     chunkSize: number;
     prefetchChunks: number[];
+    persistent?: boolean;
 };
 
 export function generateChunkUrl(
@@ -97,16 +98,21 @@ export function generateNextChunkUrl(
         return undefined;
     }
     const [chunkSignature, chunkStr] = match.slice(1);
-    const chunk = parseInt(chunkStr, 10);
-    const spec = specs.find(spec => spec.chunks[chunk] === chunkSignature);
+    const chunkIndex = parseInt(chunkStr, 10);
+    const spec = specs.find(spec => spec.chunks[chunkIndex] === chunkSignature);
     if (!spec) {
         console.warn(`Could not find spec that served ${url}`);
         return undefined;
     }
-    if (chunk + 1 >= spec.chunks.length) {
+    if (chunkIndex + 1 >= spec.chunks.length) {
         return undefined;
     }
-    return generateChunkUrl(spec, chunk + 1);
+    const nextChunkSignature = spec.chunks[chunkIndex + 1];
+    if (!nextChunkSignature) {
+        // Zero-ed out chunk, we don't need to load it.
+        return undefined;
+    }
+    return generateChunkUrl(spec, chunkIndex + 1);
 }
 
 export type EmulatorWorkerConfig = {

src/emulator/emulator-common.ts

@@ -33,8 +33,13 @@ self.addEventListener("message", event => {
             (async function () {
                 const cache = await caches.open(DISK_CACHE_NAME);
                 const prefetchChunkUrls = [];
-                for (const chunk of diskFileSpec.prefetchChunks) {
-                    const chunkUrl = generateChunkUrl(diskFileSpec, chunk);
+                for (const chunkIndex of diskFileSpec.prefetchChunks) {
+                    const chunkSignature = diskFileSpec.chunks[chunkIndex];
+                    if (!chunkSignature) {
+                        // Zero-ed out chunk, skip it.
+                        continue;
+                    }
+                    const chunkUrl = generateChunkUrl(diskFileSpec, chunkIndex);
                     const cachedResponse = await cache.match(
                         new Request(chunkUrl)
                     );

src/emulator/emulator-service-worker.ts

@@ -287,6 +287,7 @@ export class Emulator {
                 ...diskSpecs[i],
                 baseUrl: "/Disk",
                 prefetchChunks: d.prefetchChunks,
+                persistent: d.persistent,
             }));
         }
         const disks = await loadDisks(this.#config.disks);