Reduce number of files in Infinite HD for older OSes

System 6 was having trouble opening some folders in Infinite HD. With recent additions, rebuilding the desktop database would crash Mini vMac. Unclear if this is due to malformed HFS data structures generated by machfs or System 6 limitations, but as a workaround, generate a smaller subset of files for older OSes (by excluding things that won't run on them). This does complicate the disk image process somewhat and increase the hosted size (since the two images will not share chunks), but it does provide a lot of breathing room (the subsetted image only has 590 MB of data/4,6008 files vs 1,063 MB/8,793 files for the full one). Fixes #247
mihaip · Nov 17, 2023 · 60f23d3 · 60f23d3
1 parent f81b3f6
commit 60f23d3
Show file tree

Hide file tree

Showing 31 changed files with 160 additions and 90 deletions.
diff --git a/Library/Control Panels & Extensions/Aaron.json b/Library/Control Panels & Extensions/Aaron.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/aaron-161-fat.hqx"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/aaron-161-fat.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Control Panels & Extensions/BeView.json b/Library/Control Panels & Extensions/BeView.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/beview_1.1.sit_.hqx"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/beview_1.1.sit_.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Control Panels & Extensions/Greg's Buttons.json b/Library/Control Panels & Extensions/Greg's Buttons.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Gregs_Buttons_3.8.sit_.hqx"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Gregs_Buttons_3.8.sit_.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Control Panels & Extensions/Power Windows.json b/Library/Control Panels & Extensions/Power Windows.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Power_Windows_245.sit"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Power_Windows_245.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Control Panels & Extensions/SmoothType.json b/Library/Control Panels & Extensions/SmoothType.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/smoothtype_2.3.1.sit"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/smoothtype_2.3.1.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Developer/Appearance SDK.json b/Library/Developer/Appearance SDK.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Appearance_SDK_1.0.4.sit_.hqx"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Appearance_SDK_1.0.4.sit_.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Developer/CodeWarrior 4.json b/Library/Developer/CodeWarrior 4.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/CodeWarrior_Gold_68k.sit"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/CodeWarrior_Gold_68k.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Developer/Multiprocessing SDK.json b/Library/Developer/Multiprocessing SDK.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Multiprocessing_2.1v2_SDK.sit_.bin"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Multiprocessing_2.1v2_SDK.sit_.bin",
+    "needs_system_7": true
 }
diff --git a/Library/Games/Bungie/Marathon 2.json b/Library/Games/Bungie/Marathon 2.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "http://files3.bungie.org/trilogy/marathon_2.sit"
+    "src_url": "http://files3.bungie.org/trilogy/marathon_2.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Games/Bungie/Marathon.json b/Library/Games/Bungie/Marathon.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "http://files3.bungie.org/trilogy/marathon.sit"
+    "src_url": "http://files3.bungie.org/trilogy/marathon.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Games/F-A-18 Hornet.json b/Library/Games/F-A-18 Hornet.json
@@ -1,5 +1,6 @@
 {
     "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/games/FA18Hornet_Classic.img_.sit",
     "src_image": "F:A 18.img",
-    "src_folder": "F/A-18 Hornet Classic"
+    "src_folder": "F/A-18 Hornet Classic",
+    "needs_system_7": true
 }
diff --git a/Library/Games/Ultimate Doom.json b/Library/Games/Ultimate Doom.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/games/Ultimate_Doom.sit"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/games/Ultimate_Doom.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Games/Warcraft - Orcs & Humans.json b/Library/Games/Warcraft - Orcs & Humans.json
@@ -1,4 +1,5 @@
 {
     "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/games/Warcraft1.0.8.sit_.bin",
-    "src_image": "Warcraft(1.0.8).img"
+    "src_image": "Warcraft(1.0.8).img",
+    "needs_system_7": true
 }
diff --git a/Library/Games/Warcraft II - Tides of Darkness.json b/Library/Games/Warcraft II - Tides of Darkness.json
@@ -1,4 +1,5 @@
 {
     "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/games/WarcraftII.sit",
-    "src_image": "Warcraft II.img"
+    "src_image": "Warcraft II.img",
+    "needs_system_7": true
 }
diff --git a/Library/Graphics/Adobe Photoshop 1.0.json b/Library/Graphics/Adobe Photoshop 1.0.json
@@ -0,0 +1,4 @@
+{
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Adobe_Photoshop1.sit",
+    "src_image": "Adobe_Photoshop_1.img"
+}
diff --git a/Library/Graphics/Adobe Photoshop.zip → Library/Graphics/Adobe Photoshop 3.0.zip b/Library/Graphics/Adobe Photoshop.zip → Library/Graphics/Adobe Photoshop 3.0.zip
diff --git a/Library/Graphics/KPT Bryce.json b/Library/Graphics/KPT Bryce.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/KPT_Bryce101.sit"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/KPT_Bryce101.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Multimedia/Audion 1.0.json b/Library/Multimedia/Audion 1.0.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://download.panic.com/audion/audion_1.0.sit.bin"
+    "src_url": "https://download.panic.com/audion/audion_1.0.sit.bin",
+    "needs_system_7": true
 }
diff --git a/Library/Multimedia/Audion 2.6.1.json b/Library/Multimedia/Audion 2.6.1.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://download.panic.com/audion/Audion%202.6.1.sit.bin"
+    "src_url": "https://download.panic.com/audion/Audion%202.6.1.sit.bin",
+    "needs_system_7": true
 }
diff --git a/Library/Multimedia/Audion 3.0.2.json b/Library/Multimedia/Audion 3.0.2.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://download.panic.com/audion/Audion%203.0.2%209.sit.bin"
+    "src_url": "https://download.panic.com/audion/Audion%203.0.2%209.sit.bin",
+    "needs_system_7": true
 }
diff --git a/Library/Multimedia/SoundApp (PPC).json b/Library/Multimedia/SoundApp (PPC).json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/SoundAppPPC273.sit_.bin"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/SoundAppPPC273.sit_.bin",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/Glypher.json b/Library/Utilities/Glypher.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://mscape.com/downloads/glypher.sit.hqx"
+    "src_url": "https://mscape.com/downloads/glypher.sit.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/Graphing Calculator.json b/Library/Utilities/Graphing Calculator.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/gc68kfpu.sit_.bin"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/gc68kfpu.sit_.bin",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/Greg's Browser.json b/Library/Utilities/Greg's Browser.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/gregs-browser-27.hqx"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/gregs-browser-27.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/Iconographer.json b/Library/Utilities/Iconographer.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://mscape.com/downloads/iconographer.sit"
+    "src_url": "https://mscape.com/downloads/iconographer.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/Snapz Pro.json b/Library/Utilities/Snapz Pro.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Snapz-Pro-v111.sit"
+    "src_url": "https://macintoshgarden.org/sites/macintoshgarden.org/files/apps/Snapz-Pro-v111.sit",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/clip2cicn.json b/Library/Utilities/clip2cicn.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://mscape.com/downloads/clip2cicn.sit.hqx"
+    "src_url": "https://mscape.com/downloads/clip2cicn.sit.hqx",
+    "needs_system_7": true
 }
diff --git a/Library/Utilities/clip2icns.json b/Library/Utilities/clip2icns.json
@@ -1,3 +1,4 @@
 {
-    "src_url": "https://mscape.com/downloads/clip2icns.sit.hqx"
+    "src_url": "https://mscape.com/downloads/clip2icns.sit.hqx",
+    "needs_system_7": true
 }
diff --git a/scripts/import-disks.py b/scripts/import-disks.py
@@ -27,17 +27,21 @@
 
 CHUNK_SIZE = 256 * 1024
 
+ImportFolders = typing.Tuple[
+    typing.Dict[str, machfs.Folder], # Universal folders
+    typing.Dict[str, machfs.Folder], # Folders that need System 7
+]
 
-def get_import_folders() -> typing.Dict[str, machfs.Folder]:
-    import_folders = {}
-    import_folders.update(import_manifests())
+def get_import_folders() -> ImportFolders:
+    import_folders, import_folders7 = import_manifests()
     import_folders.update(import_zips())
-    return import_folders
+    return import_folders, import_folders7
 
 
-def import_manifests() -> typing.Dict[str, machfs.Folder]:
+def import_manifests() -> ImportFolders:
     sys.stderr.write("Importing other images\n")
     import_folders = {}
+    import_folders7 = {}
     debug_filter = os.getenv("DEBUG_LIBRARY_FILTER")
 
     for manifest_path in glob.iglob(os.path.join(paths.LIBRARY_DIR, "**",
@@ -62,8 +66,12 @@ def import_manifests() -> typing.Dict[str, machfs.Folder]:
         else:
             assert False, "Unexpected manifest URL extension: %s" % src_ext
 
-        import_folders[folder_path] = folder
-    return import_folders
+        if manifest_json.get("needs_system_7"):
+            import_folders7[folder_path] = folder
+        else:
+            import_folders[folder_path] = folder
+
+    return import_folders, import_folders7
 
 
 def import_disk_image(
@@ -508,33 +516,45 @@ def build_system_image(
     return write_image_def(image_data, disk.name, dest_dir)
 
 
-def build_library_image(base_name: str, dest_dir: str) -> ImageDef:
-    import_folders = get_import_folders()
+def build_library_images(dest_dir: str) -> typing.Tuple[ImageDef, ImageDef]:
+    import_folders, import_folders7 = get_import_folders()
 
     v = machfs.Volume()
-    with open(os.path.join(paths.IMAGES_DIR, base_name), "rb") as base:
+    with open(os.path.join(paths.IMAGES_DIR, "Infinite HD.dsk"), "rb") as base:
         v.read(base.read())
     v.name = "Infinite HD"
 
-    for folder_path, folder in import_folders.items():
-        parent_folder_path, folder_name = os.path.split(folder_path)
-        parent = traverse_folders(v, parent_folder_path)
-        folder_name = fix_name(folder_name)
-        if folder_name in parent:
-            sys.stderr.write(
-                "  Skipping %s, already installed in the image\n" %
-                folder_path)
-            continue
-        parent[folder_name] = folder
+    def add_folders(folders: typing.Dict[str, machfs.Folder]) -> None:
+        for folder_path, folder in folders.items():
+            parent_folder_path, folder_name = os.path.split(folder_path)
+            parent = traverse_folders(v, parent_folder_path)
+            folder_name = fix_name(folder_name)
+            if folder_name in parent:
+                sys.stderr.write(
+                    "  Skipping %s, already installed in the image\n" %
+                    folder_path)
+                continue
+            parent[folder_name] = folder
+    add_folders(import_folders)
+
+    image6 = v.write(
+        size=1000 * 1024 * 1024,
+        align=512,
+        desktopdb=False,
+        bootable=False,
+    )
+    image6_def = write_image_def(image6, "Infinite HD6.dsk", dest_dir)
 
+    add_folders(import_folders7)
     image = v.write(
         size=2000 * 1024 * 1024,
         align=512,
         desktopdb=False,
         bootable=False,
     )
+    image_def = write_image_def(image, "Infinite HD.dsk", dest_dir)
 
-    return write_image_def(image, base_name, dest_dir)
+    return image6_def, image_def
 
 
 def build_passthrough_image(base_name: str, dest_dir: str) -> ImageDef:
@@ -577,19 +597,15 @@ def build_saved_hd_image(base_name: str, dest_dir: str) -> ImageDef:
     return write_image_def(image_data, base_name, dest_dir)
 
 
-def build_desktop_db(images: typing.List[ImageDef]) -> bytes:
-    sys.stderr.write("Rebuilding Desktop DB for %s...\n" %
+def build_desktop_db6(images: typing.List[ImageDef]) -> bytes:
+    sys.stderr.write("Building System 6 Desktop for %s...\n" %
                      ",".join([i.name for i in images]))
-    # System 6 (and earlier) use a different "Desktop" file to store the
-    # database. We need to do this first, otherwise it will clobber the
-    # "Desktop DB" file generated by System 7/Mac OS 8 (the Desktop Mgr INIT
-    # is supposed to prevent this, but it also prevents the Desktop file from
-    # being built at all).
-    # We also need to ensure that Finder has an increased preferred memory
-    # size (512K instead of 160K), otherwise it will run out of memory when
-    # doing the rebuild.
     minivmac.run([disks.SYSTEM_608.path()] + [i.path for i in images])
 
+
+def build_desktop_db(images: typing.List[ImageDef]) -> bytes:
+    sys.stderr.write("Rebuilding Desktop DB for %s...\n" %
+                     ",".join([i.name for i in images]))
     basilisk.run(
         # Boot from Mac OS 8.1 to ensure that the Desktop database that's
         # created is acceptable to all classic Mac OS versions (one generated by
@@ -604,53 +620,53 @@ def read_strings(name: str) -> str:
 
 STICKIES = [
     stickies.Sticky(
-        top=242,
-        left=210,
-        bottom=436,
-        right=390,
+        top=238,
+        left=462,
+        bottom=442,
+        right=642,
         color=stickies.Color.GRAY,
         text="CHANGELOG",
     ),
     stickies.Sticky(
-        top=255,
-        left=387,
-        bottom=510,
-        right=592,
+        top=316,
+        left=22,
+        bottom=553,
+        right=280,
         color=stickies.Color.PURPLE,
         skip_in_ttxt=True,
         text=read_strings("tips.txt"),
     ),
     stickies.Sticky(
-        top=255,
-        left=387,
-        bottom=510,
-        right=592,
+        top=316,
+        left=22,
+        bottom=553,
+        right=280,
         color=stickies.Color.PURPLE,
         skip_in_stickies=True,
         text=read_strings("tips-no-outside-world.txt"),
     ),
     stickies.Sticky(
-        top=425,
-        left=24,
+        top=352,
+        left=296,
         bottom=532,
-        right=389,
+        right=482,
         color=stickies.Color.PINK,
         text=read_strings("networking.txt"),
         skip_in_ttxt=True,
     ),
     stickies.Sticky(
-        top=310,
-        left=35,
-        bottom=426,
-        right=212,
+        top=119,
+        left=494,
+        bottom=235,
+        right=671,
         color=stickies.Color.GREEN,
         text=read_strings("intro.txt"),
     ),
     stickies.Sticky(
-        top=253,
-        left=28,
-        bottom=313,
-        right=214,
+        top=66,
+        left=454,
+        bottom=126,
+        right=640,
         font=stickies.Font.HELVETICA,
         size=18,
         style={stickies.Style.BOLD},
@@ -670,12 +686,13 @@ def read_strings(name: str) -> str:
             for disk in disks.ALL_DISKS:
                 if system_filter and system_filter not in disk.name:
                     continue
-                images.append(build_system_image(disk, dest_dir=temp_dir))
+                images.append(build_system_image(disk, temp_dir))
         if not system_filter:
-            infinite_hd_image = build_library_image("Infinite HD.dsk",
-                                                    dest_dir=temp_dir)
+            infinite_hd6_image, infinite_hd_image = build_library_images(temp_dir)
+            images.append(infinite_hd6_image)
             images.append(infinite_hd_image)
             if not library_filter:
+                build_desktop_db6([infinite_hd6_image])
                 build_desktop_db([infinite_hd_image])
 
             images.append(