From 09b55d76faa5360dff05606a9915c37cbecb1b63 Mon Sep 17 00:00:00 2001 From: devkyato <95612413+devkyato@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:04:44 +0800 Subject: [PATCH] feat: add safe archive migration tooling --- CHANGELOG.md | 14 + README.md | 54 +++- docs/architecture.md | 4 + docs/compatibility.md | 61 +++++ docs/format.md | 6 + docs/getting-started.md | 81 ++++++ docs/limitations.md | 8 +- docs/security.md | 7 + pyproject.toml | 2 +- src/lowpack/__init__.py | 15 +- src/lowpack/cli.py | 44 ++++ src/lowpack/migration.py | 364 ++++++++++++++++++++++++++ src/lowpack/models.py | 26 ++ tests/fixtures/format-1.0-minimal.b64 | 1 + tests/legacy_archive_factory.py | 97 +++++++ tests/test_migration.py | 215 +++++++++++++++ 16 files changed, 985 insertions(+), 14 deletions(-) create mode 100644 docs/compatibility.md create mode 100644 docs/getting-started.md create mode 100644 src/lowpack/migration.py create mode 100644 tests/fixtures/format-1.0-minimal.b64 create mode 100644 tests/legacy_archive_factory.py create mode 100644 tests/test_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 23abb02..d97b768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.2.1 - 2026-07-29 + +- Add a read-only `lowpack compatibility` probe plus an authenticated, atomic + `lowpack migrate` command and typed Python APIs for format 1.0 archives + created by LowPack 0.1.x. +- Preserve legacy chunk payloads while converting embedded dictionaries, + selection goals, and codec decisions into strict format 1.1 manifest schema + 2.0 records. +- Fully reconstruct and verify migrated archives before publishing the output, + leaving the source untouched and requiring explicit destination overwrite. +- Separate end-user wheel installation from contributor setup, with a + cross-platform getting-started guide, checksum guidance, upgrade steps, and + compatibility reference. + ## 0.2.0 - 2026-07-29 - Harden input collection against duplicate names and identities, symlink diff --git a/README.md b/README.md index 7c09b87..0421cf6 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ it will actually be stored and used, then hands it to proven lossless codecs. It is a terminal tool and Python library that stays completely on your laptop. > **A quick, honest alpha note:** the `.lpk` format is still experimental. -> Archives from alpha releases may need migration, so please do not make -> LowPack the only copy of important data. +> LowPack 0.2.1 can migrate format 1.0 archives made by the 0.1 releases, but +> please do not make any alpha archive the only copy of important data. Oh! One point I care about being clear on: LowPack does **not** universally outperform Zstandard, gzip, ZIP, Brotli, LZ4, or anything else. Results depend @@ -24,16 +24,28 @@ indexing, deduplication, explainable selection, and reversible preparation. ## Install -Python 3.9 through 3.14 is supported. For development, I usually start with a -fresh environment so the checks describe the project instead of my machine: +Python 3.9 through 3.14 is supported. To use LowPack today, install the wheel +attached to the GitHub release: ```powershell python -m venv .venv .venv\Scripts\Activate.ps1 -python -m pip install -e ".[dev]" +python -m pip install "https://github.com/devkyato/Lowpack/releases/download/v0.2.1/lowpack-0.2.1-py3-none-any.whl" lowpack --version +lowpack doctor ``` +On macOS or Linux, activation is `source .venv/bin/activate`; the install +command is otherwise the same. If I want the terminal command isolated from a +project, I use `pipx install` with that wheel URL instead. Each release also +includes `SHA256SUMS`, the source archive, and exact verification notes. The +[getting-started guide](docs/getting-started.md) covers installation, +upgrades, a first round trip, and common command-not-found fixes. + +For development, clone the repository and use +`python -m pip install -e ".[dev]"` in a fresh environment. That keeps the +published install path separate from contributor tooling. + That is it. LowPack needs no account, website, cloud service, daemon, database, analytics, or background network access. @@ -54,6 +66,20 @@ Selective extraction is `lowpack extract project.lpk project/src -o selected`. I made `--overwrite` explicit on purpose, and `lowpack doctor` is there when you want a quick check of the local environment. +If an archive came from LowPack 0.1, migrate it without touching the original: + +```powershell +lowpack compatibility old-project.lpk +lowpack migrate old-project.lpk -o project-1.1.lpk +lowpack verify project-1.1.lpk --full +``` + +Oh! On this part I thought the safest upgrade was the least surprising one: +LowPack authenticates the old archive, rewrites only the framing and manifest, +fully reconstructs and verifies the migrated temporary archive, and moves it +into place only after all of that succeeds. See the +[compatibility guide](docs/compatibility.md). + Codec selection uses deterministic policy names: `balanced`, `smallest`, `prefer-store`, `prefer-zstd-low`, and `avoid-zlib`. They describe stable preferences rather than claiming to measure whole-machine speed or memory. @@ -146,12 +172,23 @@ the identified corpus and environment. ## Python API ```python -from lowpack import inspect_archive, pack, unpack, verify_archive +from lowpack import ( + inspect_archive, + migrate_archive, + pack, + probe_compatibility, + unpack, + verify_archive, +) pack(["project"], "project.lpk", profile="source", goal="balanced") info = inspect_archive("project.lpk") assert verify_archive("project.lpk", full=True).valid unpack("project.lpk", output="restored") + +# For a 0.1 archive: +probe_compatibility("old-project.lpk") +migrate_archive("old-project.lpk", "project-1.1.lpk") ``` Functions return typed frozen result models. @@ -159,8 +196,9 @@ Functions return typed frozen result models. ## Limitations This is where I would rather be specific than sound finished too early. The -format has no pre-1.0 forward-compatibility promise. Version 0.2 deliberately -introduces manifest schema 2 after the 0.1 security review. Source dictionaries +format has no forward-compatibility promise during alpha. Version 0.2 +deliberately introduced manifest schema 2 after the 0.1 security review; +0.2.1 provides a checked migration from format 1.0. Source dictionaries use bounded deterministic samples and only apply to Zstandard chunks. Telemetry canonical mode stores exact IEEE-754 values, but only exact mode preserves the original decimal spelling. Canonical transforms have a 64 MiB diff --git a/docs/architecture.md b/docs/architecture.md index e3a6e04..fed97f1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,6 +9,10 @@ review. extraction. Its individual stages are deliberately isolated as helpers with one-way data flow; `format.py` handles fixed binary framing and `manifest.py` handles executable-free canonical JSON plus total JSON limits. +`migration.py` is a deliberately narrow bridge from format 1.0 to 1.1: it +authenticates the source, normalizes its manifest into schema 2, preserves the +chunk payload area, and accepts the output only after current full +verification. `codecs` exposes the experimental `Codec` protocol. `profiles` detects or reversibly prepares application data without owning compression. `selection` measures a bounded sample and stores all candidate measurements. The CLI is a diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..776d70c --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,61 @@ +# Archive compatibility and migration + +LowPack 0.2 writes binary format 1.1 with manifest schema 2.0. LowPack 0.1.x +wrote format 1.0. I did not want a security-driven schema change to leave the +earliest test archives stranded, so 0.2.1 adds one explicit bridge: + +```powershell +lowpack compatibility old-project.lpk +lowpack migrate old-project.lpk -o project-1.1.lpk +lowpack verify project-1.1.lpk --full +``` + +The command never edits `old-project.lpk`. An existing output is refused unless +you pass `--overwrite`. + +`lowpack compatibility` is the read-only half of this workflow. It checks +framing, canonical manifest structure, the manifest hash, the complete body +hash, and current schema relationships, then reports either `current` or +`migration-available`. It does not decompress chunks, extract files, or write +an output: + +```powershell +lowpack compatibility old-project.lpk --json +``` + +## What the migration does + +Oh! On this part, “migration” does not mean extracting files somewhere and +packing them again with a possibly different policy. LowPack: + +1. checks the 1.0 header, footer, canonical manifest hash, and complete body + hash; +2. converts embedded compression dictionaries into the authenticated schema 2 + catalog; +3. maps the old selection goal names to their deterministic policy names and + records preferred-versus-actual chunk decisions; +4. validates paths, sizes, offsets, dictionaries, transforms, permissions, + references, and payload boundaries against the current strict schema; +5. preserves the original compressed chunk payload area; +6. writes a sibling temporary 1.1 archive and fully decompresses, + reconstructs, and hashes every file; and +7. atomically publishes the destination only after all checks pass. + +Stored permission values are reduced to ordinary rwx bits during migration. +Restoring even those bits remains opt-in during extraction. + +## What it deliberately does not do + +Migration supports format 1.0 only. It does not guess at unknown future +formats, repair a corrupt body, bypass current safety limits, or overwrite the +source. If a 0.1 archive contains a relationship that the current validator +cannot prove safe, migration fails without publishing a partial destination. + +Use `--json` when another local tool needs a stable result record: + +```powershell +lowpack migrate old.lpk -o migrated.lpk --json +``` + +For the precise framing and schema, see the [format reference](format.md). For +the trust model, see [extraction security](security.md). diff --git a/docs/format.md b/docs/format.md index 93f5a6a..c68b81f 100644 --- a/docs/format.md +++ b/docs/format.md @@ -58,6 +58,12 @@ Every file records a preferred codec policy plus one `chunk_decisions` entry per reference. Those entries name the actual stored codec/dictionary and say whether content-addressed deduplication reused an earlier representation. +Format 1.0, written by LowPack 0.1.x, used embedded per-chunk dictionary bytes +and a less explicit codec-decision record. `lowpack migrate` is the supported +bridge to format 1.1 and manifest schema 2.0. The +[compatibility guide](compatibility.md) describes exactly what is preserved +and checked. + A zero-byte file has an empty chunk list and SHA-256 `e3b0c44298fc1c149afbf4c8996fb924...`. diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..cdd06a6 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,81 @@ +# Getting started + +I wanted the first LowPack run to answer two questions quickly: “Did I install +the real release?” and “Can I restore exactly what I packed?” This page does +both without needing an account or network connection after installation. + +## Install the release + +LowPack supports Python 3.9 through 3.14. On Windows PowerShell: + +```powershell +python -m venv .venv +.venv\Scripts\Activate.ps1 +python -m pip install "https://github.com/devkyato/Lowpack/releases/download/v0.2.1/lowpack-0.2.1-py3-none-any.whl" +lowpack --version +lowpack doctor +``` + +On macOS or Linux: + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install "https://github.com/devkyato/Lowpack/releases/download/v0.2.1/lowpack-0.2.1-py3-none-any.whl" +lowpack --version +lowpack doctor +``` + +The release page includes the wheel, source archive, and `SHA256SUMS`. Download +the wheel first if you want to compare its SHA-256 before installing it. I +publish the exact expected digests in the release description as well. + +For a globally available but isolated terminal command, replace the virtual +environment steps with `pipx install `. LowPack itself does not +contact GitHub or any other service while packing, inspecting, verifying, or +extracting. + +## Make and restore an archive + +```powershell +lowpack pack my-project -o my-project.lpk --profile source +lowpack inspect my-project.lpk +lowpack verify my-project.lpk --full +lowpack unpack my-project.lpk -o restored +``` + +I thought too on the point that a successful pack should not be treated as a +successful restore. `verify --full` decompresses each stored representation +and checks reconstructed file hashes. For important data, still test the +restored directory and keep another copy outside the archive. + +The `source` profile excludes documented cache/build paths. LowPack prints and +records every exclusion; use `--include-all` when the directory should be +literal. Use the `general` profile for an ordinary tree with no profile +exclusions. + +## Upgrade or remove + +Upgrade to a newer release by activating the same environment and installing +its wheel URL with `--upgrade`. Remove LowPack with: + +```powershell +python -m pip uninstall lowpack +``` + +Uninstalling never removes `.lpk` files or extracted data. + +If you have a 0.1.x archive, install 0.2.1 and follow the +[compatibility guide](compatibility.md). Installing a new LowPack version does +not silently rewrite existing archives. + +## If `lowpack` is not found + +First run `python -m lowpack --version`. If that works, LowPack is installed +for that interpreter and the environment's scripts directory is simply not on +the current shell path. Activate the virtual environment again, or use +`python -m lowpack` in place of `lowpack`. + +If neither command works, compare `python -m pip --version` and +`python --version`; they should point to the environment where you intended to +install LowPack. diff --git a/docs/limitations.md b/docs/limitations.md index 90b9569..eed17b9 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -3,7 +3,9 @@ I would rather keep this list direct than hide unfinished edges behind an “alpha” label. These are the boundaries I know about today: -- Format compatibility is not promised across alpha releases. +- Forward compatibility is not promised across alpha releases. LowPack 0.2.1 + provides a checked migration for format 1.0 archives created by 0.1.x; it + does not guess at unknown formats or repair corrupted archives. - Chunk boundaries are fixed-size, not content-defined. - Source dictionary training is active, bounded, deterministic, and authenticated in the manifest, but it currently applies only to Zstandard @@ -23,5 +25,5 @@ I would rather keep this list direct than hide unfinished edges behind an - General safety limits are constants in the 0.2 library API. I am tracking the larger follow-ups in the -[v0.2.0 milestone](https://github.com/devkyato/Lowpack/milestone/1), so this -page and the issue tracker should tell the same story. +[issue tracker](https://github.com/devkyato/Lowpack/issues), so this page and +the roadmap should tell the same story. diff --git a/docs/security.md b/docs/security.md index 26ba51f..169b356 100644 --- a/docs/security.md +++ b/docs/security.md @@ -34,3 +34,10 @@ privately owned output directory plus OS quotas or sandboxing. Archived permissions are ignored by default. `--restore-permissions` opts in to ordinary user/group/other rwx bits; setuid, setgid, and sticky bits are masked during packing and extraction. + +Migration uses the same trust boundary. `lowpack migrate` first authenticates +the complete format 1.0 body, converts its manifest into the strict current +schema, rejects unsafe paths and inconsistent relationships, and writes to a +sibling temporary file. It then performs a full decompression and +reconstruction check before atomic replacement. The source archive is never +modified, and an existing destination still requires `--overwrite`. diff --git a/pyproject.toml b/pyproject.toml index 3f09e4f..eefd90e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "lowpack" -version = "0.2.0" +version = "0.2.1" description = "Application-aware, lossless local-first packing built on proven codecs." readme = "README.md" requires-python = ">=3.9" diff --git a/src/lowpack/__init__.py b/src/lowpack/__init__.py index b29f973..93e7cc2 100644 --- a/src/lowpack/__init__.py +++ b/src/lowpack/__init__.py @@ -1,16 +1,27 @@ """LowPack public API.""" -__version__ = "0.2.0" +__version__ = "0.2.1" from .archive import inspect_archive, pack, unpack, verify_archive -from .models import ArchiveInfo, PackResult, VerificationResult +from .migration import migrate_archive, probe_compatibility +from .models import ( + ArchiveInfo, + CompatibilityResult, + MigrationResult, + PackResult, + VerificationResult, +) __all__ = [ "ArchiveInfo", + "CompatibilityResult", + "MigrationResult", "PackResult", "VerificationResult", "inspect_archive", + "migrate_archive", "pack", + "probe_compatibility", "unpack", "verify_archive", ] diff --git a/src/lowpack/cli.py b/src/lowpack/cli.py index 6830e46..0a9121b 100644 --- a/src/lowpack/cli.py +++ b/src/lowpack/cli.py @@ -21,6 +21,7 @@ from .codecs import CODECS from .explain import explain_manifest from .format import FormatError +from .migration import migrate_archive, probe_compatibility def _size(value: str) -> int: @@ -105,6 +106,21 @@ def build_parser() -> argparse.ArgumentParser: extracting.add_argument("--max-chunks", type=int, default=1_000_000) _add_json(extracting) + migrating = subparsers.add_parser( + "migrate", help="safely migrate a format 1.0 archive to format 1.1" + ) + migrating.add_argument("archive") + migrating.add_argument("-o", "--output", required=True) + migrating.add_argument("--overwrite", action="store_true") + _add_json(migrating) + + compatibility = subparsers.add_parser( + "compatibility", + help="authenticate and classify an archive without decompressing it", + ) + compatibility.add_argument("archive") + _add_json(compatibility) + verifying = subparsers.add_parser("verify", help="verify archive integrity") verifying.add_argument("archive") mode = verifying.add_mutually_exclusive_group() @@ -278,6 +294,34 @@ def run(args: argparse.Namespace) -> int: _json([str(path) for path in paths]) if args.json else f"Extracted {len(paths)} files" ) return 0 + if command == "migrate": + migration_result = migrate_archive( + args.archive, + args.output, + overwrite=args.overwrite, + ) + print( + _json(asdict(migration_result)) + if args.json + else ( + f"Migrated {migration_result.source} from " + f"{migration_result.source_format} to " + f"{migration_result.archive} ({migration_result.target_format})" + ) + ) + return 0 + if command == "compatibility": + compatibility_result = probe_compatibility(args.archive) + print( + _json(asdict(compatibility_result)) + if args.json + else ( + f"{compatibility_result.archive}: " + f"format {compatibility_result.format_version}, " + f"{compatibility_result.status}" + ) + ) + return 0 if command == "list": rows = _list_rows(_load_manifest(args.archive)) if args.json: diff --git a/src/lowpack/migration.py b/src/lowpack/migration.py new file mode 100644 index 0000000..b6c7c15 --- /dev/null +++ b/src/lowpack/migration.py @@ -0,0 +1,364 @@ +"""Authenticated, atomic migration of earlier LowPack archives.""" + +from __future__ import annotations + +import base64 +import copy +import os +import tempfile +from pathlib import Path +from typing import Any, BinaryIO + +from . import __version__ +from .archive import ( + MANIFEST_SCHEMA_MAJOR, + MANIFEST_SCHEMA_MINOR, + _hash_body, + _read_manifest, + _validate_manifest, + verify_archive, +) +from .format import ( + FOOTER, + FOOTER_MAGIC, + HEADER, + MAGIC, + VERSION_MAJOR, + VERSION_MINOR, + FormatError, + read_header, +) +from .hashing import sha256, sha256_hex +from .manifest import encode_manifest +from .models import CompatibilityResult, MigrationResult + +LEGACY_FORMAT = (1, 0) +GOAL_MIGRATIONS = { + "balanced": "balanced", + "smallest": "smallest", + "fastest": "prefer-store", + "fastest-decode": "prefer-zstd-low", + "low-memory": "avoid-zlib", +} + + +def _copy_exact(source: BinaryIO, destination: BinaryIO, length: int) -> None: + remaining = length + while remaining: + block = source.read(min(1024 * 1024, remaining)) + if not block: + raise FormatError("archive payload was truncated during migration") + destination.write(block) + remaining -= len(block) + + +def _framing_versions(stream: BinaryIO) -> tuple[tuple[int, int], tuple[int, int]]: + stream.seek(0) + header_version = read_header(stream)[:2] + stream.seek(-FOOTER.size, os.SEEK_END) + raw_footer = stream.read(FOOTER.size) + if len(raw_footer) != FOOTER.size: + raise FormatError("truncated LowPack footer") + footer_fields = FOOTER.unpack(raw_footer) + footer_version = footer_fields[1:3] + stream.seek(0) + return header_version, footer_version + + +def _require_legacy_framing(stream: BinaryIO) -> None: + header_version, footer_version = _framing_versions(stream) + current_version = (VERSION_MAJOR, VERSION_MINOR) + if header_version == current_version and footer_version == current_version: + raise FormatError("archive is already in the current 1.1 format") + if header_version != LEGACY_FORMAT or footer_version != LEGACY_FORMAT: + raise FormatError( + "migration requires matching format 1.0 header and footer" + ) + + +def _normalize_legacy_manifest( + legacy: dict[str, Any], *, manifest_offset: int +) -> dict[str, Any]: + format_record = legacy.get("format") + if not isinstance(format_record, dict): + raise FormatError("legacy manifest has no format record") + source_version = (format_record.get("major"), format_record.get("minor")) + if source_version == (VERSION_MAJOR, VERSION_MINOR): + raise FormatError("archive is already in the current 1.1 format") + if source_version != LEGACY_FORMAT: + raise FormatError( + f"migration supports format 1.0, not " + f"{format_record.get('major')}.{format_record.get('minor')}" + ) + if "manifest_schema" in legacy: + raise FormatError("format 1.0 archive has an unexpected manifest schema") + + manifest = copy.deepcopy(legacy) + chunks = manifest.get("chunks") + files = manifest.get("files") + legacy_sources = manifest.get("source_dictionaries") + if ( + not isinstance(chunks, dict) + or not isinstance(files, list) + or not isinstance(legacy_sources, dict) + ): + raise FormatError("legacy manifest is missing required collections") + + dictionaries: dict[str, dict[str, Any]] = {} + normalized_chunks: dict[str, dict[str, Any]] = {} + for chunk_id, value in chunks.items(): + if not isinstance(chunk_id, str) or not isinstance(value, dict): + raise FormatError("invalid legacy chunk index record") + record = copy.deepcopy(value) + if "dictionary_id" in record: + raise FormatError("format 1.0 chunk has an unexpected dictionary ID") + encoded_dictionary = record.pop("dictionary", None) + if encoded_dictionary is not None: + if record.get("codec") != "zstd" or not isinstance(encoded_dictionary, str): + raise FormatError("invalid legacy compression dictionary") + try: + dictionary = base64.b64decode(encoded_dictionary, validate=True) + except (TypeError, ValueError) as exc: + raise FormatError("invalid legacy compression dictionary encoding") from exc + if not dictionary or len(dictionary) > 64 * 1024: + raise FormatError("invalid legacy compression dictionary size") + dictionary_id = sha256_hex(dictionary) + record["dictionary_id"] = dictionary_id + dictionaries[dictionary_id] = { + "data": encoded_dictionary, + "hash": dictionary_id, + "size": len(dictionary), + } + normalized_chunks[chunk_id] = record + + seen_chunks: set[str] = set() + normalized_files: list[dict[str, Any]] = [] + for value in files: + if not isinstance(value, dict): + raise FormatError("invalid legacy file record") + item = copy.deepcopy(value) + refs = item.get("chunks") + decision = item.get("decision") + if not isinstance(refs, list) or not isinstance(decision, dict): + raise FormatError("invalid legacy file decision or chunk references") + if "preferred_codec" in decision or "final_codec" not in decision: + raise FormatError("invalid format 1.0 codec decision") + decision["preferred_codec"] = decision.pop("final_codec") + item["decision"] = decision + chunk_decisions: list[dict[str, Any]] = [] + for ref in refs: + if not isinstance(ref, str) or ref not in normalized_chunks: + raise FormatError("legacy file references an unknown chunk") + chunk = normalized_chunks[ref] + reused = ref in seen_chunks + chunk_decisions.append( + { + "actual_codec": chunk.get("codec"), + "actual_level": chunk.get("level"), + "chunk": ref, + "dictionary_id": chunk.get("dictionary_id"), + "reason": ( + "reused an existing content-addressed representation" + if reused + else "preserved the representation stored by LowPack 0.1" + ), + "reused": reused, + } + ) + seen_chunks.add(ref) + item["chunk_decisions"] = chunk_decisions + if "mode" in item: + mode = item["mode"] + if isinstance(mode, bool) or not isinstance(mode, int) or mode < 0: + raise FormatError("invalid legacy permission mode") + item["mode"] = mode & 0o777 + normalized_files.append(item) + + profile = manifest.get("profile") + source_dictionaries: dict[str, dict[str, str]] = {} + used_dictionary_ids = { + record["dictionary_id"] + for record in normalized_chunks.values() + if "dictionary_id" in record + } + if profile == "source": + for group, value in legacy_sources.items(): + if not isinstance(group, str) or not isinstance(value, dict): + raise FormatError("invalid legacy source dictionary record") + size = value.get("bytes") + source_dictionary_id = value.get("hash") + if ( + isinstance(size, bool) + or not isinstance(size, int) + or size <= 0 + or not isinstance(source_dictionary_id, str) + ): + raise FormatError("invalid legacy source dictionary metadata") + if source_dictionary_id in used_dictionary_ids: + catalog_record = dictionaries.get(source_dictionary_id) + if catalog_record is None or catalog_record["size"] != size: + raise FormatError("legacy source dictionary hash or size mismatch") + source_dictionaries[group] = { + "dictionary_id": source_dictionary_id + } + if { + record["dictionary_id"] for record in source_dictionaries.values() + } != used_dictionary_ids: + raise FormatError("legacy source dictionaries do not cover stored chunks") + elif legacy_sources: + raise FormatError("non-source legacy archive contains source dictionaries") + + goal = manifest.get("goal") + if not isinstance(goal, str) or goal not in GOAL_MIGRATIONS: + raise FormatError(f"unsupported legacy selection goal: {goal}") + manifest["chunks"] = normalized_chunks + manifest["dictionaries"] = dictionaries + manifest["files"] = normalized_files + manifest["format"] = {"major": VERSION_MAJOR, "minor": VERSION_MINOR} + manifest["goal"] = GOAL_MIGRATIONS[goal] + manifest["lowpack_version"] = __version__ + manifest["manifest_schema"] = { + "major": MANIFEST_SCHEMA_MAJOR, + "minor": MANIFEST_SCHEMA_MINOR, + } + manifest["source_dictionaries"] = source_dictionaries + _validate_manifest(manifest, manifest_offset=manifest_offset) + return manifest + + +def probe_compatibility( + archive: os.PathLike[str] | str, +) -> CompatibilityResult: + """Authenticate and classify an archive without decompressing its payload.""" + + path = Path(archive) + with path.open("rb") as stream: + header_version, footer_version = _framing_versions(stream) + if header_version != footer_version: + raise FormatError("archive header and footer versions disagree") + manifest, _data, footer = _read_manifest(stream) + actual_body_hash = _hash_body( + stream, footer.manifest_offset + footer.manifest_size + ) + if actual_body_hash != footer.body_hash: + raise FormatError("archive body hash mismatch") + format_record = manifest.get("format") + if not isinstance(format_record, dict): + raise FormatError("manifest has no format record") + manifest_version = ( + format_record.get("major"), + format_record.get("minor"), + ) + if manifest_version != header_version: + raise FormatError("framing and manifest format versions disagree") + if header_version == LEGACY_FORMAT: + _normalize_legacy_manifest( + manifest, manifest_offset=footer.manifest_offset + ) + return CompatibilityResult( + archive=path, + format_version="1.0", + status="migration-available", + current=False, + migration_supported=True, + target_format=f"{VERSION_MAJOR}.{VERSION_MINOR}", + ) + current_version = (VERSION_MAJOR, VERSION_MINOR) + if header_version == current_version: + _validate_manifest( + manifest, manifest_offset=footer.manifest_offset + ) + return CompatibilityResult( + archive=path, + format_version=f"{VERSION_MAJOR}.{VERSION_MINOR}", + status="current", + current=True, + migration_supported=False, + ) + raise FormatError( + f"unsupported LowPack format {header_version[0]}.{header_version[1]}" + ) + + +def migrate_archive( + source: os.PathLike[str] | str, + archive: os.PathLike[str] | str, + *, + overwrite: bool = False, +) -> MigrationResult: + """Migrate an authenticated format 1.0 archive to current format 1.1.""" + + source_path = Path(source) + target = Path(archive) + source_identity = os.path.normcase(str(source_path.resolve())) + target_identity = os.path.normcase(str(target.resolve())) + if source_identity == target_identity: + raise ValueError("migration output must differ from the source archive") + if target.exists() and not overwrite: + raise FileExistsError(target) + target.parent.mkdir(parents=True, exist_ok=True) + + with source_path.open("rb") as source_stream: + _require_legacy_framing(source_stream) + legacy, _data, footer = _read_manifest(source_stream) + actual_body_hash = _hash_body( + source_stream, footer.manifest_offset + footer.manifest_size + ) + if actual_body_hash != footer.body_hash: + raise FormatError("legacy archive body hash mismatch") + manifest = _normalize_legacy_manifest( + legacy, manifest_offset=footer.manifest_offset + ) + + encoded = encode_manifest(manifest) + handle, temporary_name = tempfile.mkstemp( + prefix=f".{target.name}.", suffix=".tmp", dir=target.parent + ) + os.close(handle) + temporary = Path(temporary_name) + try: + with source_path.open("rb") as source_stream, temporary.open("w+b") as output: + output.write(HEADER.pack(MAGIC, VERSION_MAJOR, VERSION_MINOR, 0)) + source_stream.seek(HEADER.size) + _copy_exact( + source_stream, + output, + footer.manifest_offset - HEADER.size, + ) + manifest_offset = output.tell() + if manifest_offset != footer.manifest_offset: + raise FormatError("migration changed the chunk payload layout") + output.write(encoded) + body_length = output.tell() + body_hash = _hash_body(output, body_length) + output.seek(body_length) + output.write( + FOOTER.pack( + FOOTER_MAGIC, + VERSION_MAJOR, + VERSION_MINOR, + manifest_offset, + len(encoded), + sha256(encoded), + body_hash, + ) + ) + output.flush() + os.fsync(output.fileno()) + verification = verify_archive(temporary, full=True) + if not verification.valid: + details = "; ".join(verification.errors) or "unknown verification error" + raise FormatError(f"migrated archive failed full verification: {details}") + os.replace(temporary, target) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + return MigrationResult( + source=source_path, + archive=target, + source_format="1.0", + target_format=f"{VERSION_MAJOR}.{VERSION_MINOR}", + file_count=len(manifest["files"]), + chunk_count=len(manifest["chunks"]), + ) diff --git a/src/lowpack/models.py b/src/lowpack/models.py index a78e050..107b7d3 100644 --- a/src/lowpack/models.py +++ b/src/lowpack/models.py @@ -35,6 +35,32 @@ def to_dict(self) -> dict[str, Any]: return asdict(self) +@dataclass(frozen=True) +class MigrationResult: + source: Path + archive: Path + source_format: str + target_format: str + file_count: int + chunk_count: int + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class CompatibilityResult: + archive: Path + format_version: str + status: str + current: bool + migration_supported: bool + target_format: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @dataclass(frozen=True) class ArchiveInfo: path: Path diff --git a/tests/fixtures/format-1.0-minimal.b64 b/tests/fixtures/format-1.0-minimal.b64 new file mode 100644 index 0000000..436ca32 --- /dev/null +++ b/tests/fixtures/format-1.0-minimal.b64 @@ -0,0 +1 @@ +TE9XUEFDSwAAAQAAAAAAAExQQ0sAAAAAAAAAAAAAADoAAAAAAAAAOn0YDhr7d3etIklZlrXA0/o8+N+eQg9ADi3HeC3PrskbZGVmIGdyZWV0KG5hbWU6IHN0cikgLT4gc3RyOgogICAgcmV0dXJuIGYiSGVsbG8sIHtuYW1lfSEiCnsiY2h1bmtfc2l6ZSI6MTA0ODU3NiwiY2h1bmtzIjp7IjdkMTgwZTFhZmI3Nzc3YWQyMjQ5NTk5NmI1YzBkM2ZhM2NmOGRmOWU0MjBmNDAwZTJkYzc3ODJkY2ZhZWM5MWIiOnsiY29kZWMiOiJzdG9yZSIsImxldmVsIjpudWxsLCJvZmZzZXQiOjE2LCJwYWNrZWRfc2l6ZSI6NTgsInJhd19zaXplIjo1OH19LCJjb2RlY192ZXJzaW9ucyI6eyJzdG9yZSI6IjEiLCJ6bGliIjoiMS4zLjEuemxpYi1uZyIsInpzdGQiOiIwLjI1LjAifSwiZGlyZWN0b3JpZXMiOltdLCJleGNsdWRlZCI6W10sImZpbGVzIjpbeyJjaHVua3MiOlsiN2QxODBlMWFmYjc3NzdhZDIyNDk1OTk2YjVjMGQzZmEzY2Y4ZGY5ZTQyMGY0MDBlMmRjNzc4MmRjZmFlYzkxYiJdLCJkZWNpc2lvbiI6eyJjYW5kaWRhdGVzIjpbeyJjb2RlYyI6InN0b3JlIiwibGV2ZWwiOm51bGwsInBhY2tlZF9ieXRlcyI6NTgsInNhbXBsZV9ieXRlcyI6NTgsInNjb3JlIjo1OH1dLCJmaW5hbF9jb2RlYyI6InN0b3JlIiwibGV2ZWwiOm51bGwsInJlYXNvbiI6ImJlc3QgYmFsYW5jZWQgc2NvcmUgb24gYSBib3VuZGVkIDU4LWJ5dGUgc2FtcGxlIn0sImVuY29kZWRfaGFzaCI6IjdkMTgwZTFhZmI3Nzc3YWQyMjQ5NTk5NmI1YzBkM2ZhM2NmOGRmOWU0MjBmNDAwZTJkYzc3ODJkY2ZhZWM5MWIiLCJlbmNvZGVkX3NpemUiOjU4LCJoYXNoIjoiN2QxODBlMWFmYjc3NzdhZDIyNDk1OTk2YjVjMGQzZmEzY2Y4ZGY5ZTQyMGY0MDBlMmRjNzc4MmRjZmFlYzkxYiIsInBhdGgiOiJoZWxsby5weSIsInNpemUiOjU4LCJzb3VyY2Vfc2l6ZSI6NTgsInRyYW5zZm9ybSI6eyJpZCI6Im5vbmUiLCJtb2RlIjoiZXhhY3QifX1dLCJmb3JtYXQiOnsibWFqb3IiOjEsIm1pbm9yIjowfSwiZ29hbCI6ImJhbGFuY2VkIiwibG93cGFja192ZXJzaW9uIjoiMC4xLjIiLCJwcm9maWxlIjoiZ2VuZXJhbCIsInNvdXJjZV9kaWN0aW9uYXJpZXMiOnt9LCJzdG9yZV9wZXJtaXNzaW9ucyI6ZmFsc2V9TFBLRk9PVAAAAQAAAAAAAAAAAIIAAAAAAAADy+bdWM6ozKhDqL5jqhd7UBbkegPBs6QDVjd1Hn4uV0sd+A8iUtN57SJGOhYy0qqVSGby1bgYmb8LLztOPTD12eM= diff --git a/tests/legacy_archive_factory.py b/tests/legacy_archive_factory.py new file mode 100644 index 0000000..ae7c468 --- /dev/null +++ b/tests/legacy_archive_factory.py @@ -0,0 +1,97 @@ +"""Build format 1.0 fixtures from valid current archives.""" + +from __future__ import annotations + +import base64 +import copy +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from lowpack.archive import _read_manifest +from lowpack.format import CHUNK_HEADER, FOOTER, FOOTER_MAGIC, HEADER, MAGIC +from lowpack.hashing import sha256 +from lowpack.manifest import encode_manifest + +REVERSE_GOALS = { + "balanced": "balanced", + "smallest": "smallest", + "prefer-store": "fastest", + "prefer-zstd-low": "fastest-decode", + "avoid-zlib": "low-memory", +} + + +def make_legacy_archive( + current: Path, + destination: Path, + mutate: Callable[[dict[str, Any]], None] | None = None, +) -> Path: + """Rewrite a current archive as an authenticated format 1.0 archive.""" + + data = current.read_bytes() + with current.open("rb") as stream: + manifest, _, footer = _read_manifest(stream) + legacy = copy.deepcopy(manifest) + dictionaries = legacy.pop("dictionaries") + legacy.pop("manifest_schema") + legacy["format"] = {"major": 1, "minor": 0} + legacy["goal"] = REVERSE_GOALS[legacy["goal"]] + legacy["lowpack_version"] = "0.1.2" + for record in legacy["chunks"].values(): + dictionary_id = record.pop("dictionary_id", None) + if dictionary_id is not None: + record["dictionary"] = dictionaries[dictionary_id]["data"] + legacy["source_dictionaries"] = { + group: { + "bytes": dictionaries[record["dictionary_id"]]["size"], + "hash": record["dictionary_id"], + } + for group, record in legacy["source_dictionaries"].items() + } + for item in legacy["files"]: + item.pop("chunk_decisions") + item["decision"]["final_codec"] = item["decision"].pop("preferred_codec") + if mutate is not None: + mutate(legacy) + encoded = encode_manifest(legacy) + header = HEADER.pack(MAGIC, 1, 0, 0) + payload = data[HEADER.size : footer.manifest_offset] + body = header + payload + encoded + destination.write_bytes( + body + + FOOTER.pack( + FOOTER_MAGIC, + 1, + 0, + footer.manifest_offset, + len(encoded), + sha256(encoded), + sha256(body), + ) + ) + return destination + + +def corrupt_payload(archive: Path, *, reauthenticate: bool = False) -> None: + with archive.open("rb") as stream: + _, _, footer = _read_manifest(stream) + data = bytearray(archive.read_bytes()) + data[HEADER.size + CHUNK_HEADER.size] ^= 0x01 + if reauthenticate: + footer_start = len(data) - FOOTER.size + fields = list(FOOTER.unpack(data[footer_start:])) + fields[-1] = sha256( + data[: footer.manifest_offset + footer.manifest_size] + ) + data[footer_start:] = FOOTER.pack(*fields) + archive.write_bytes(data) + + +def dictionary_bytes(manifest: dict[str, Any]) -> dict[str, bytes]: + """Decode a manifest dictionary catalog for focused assertions.""" + + return { + identifier: base64.b64decode(record["data"], validate=True) + for identifier, record in manifest["dictionaries"].items() + } diff --git a/tests/test_migration.py b/tests/test_migration.py new file mode 100644 index 0000000..08b5e74 --- /dev/null +++ b/tests/test_migration.py @@ -0,0 +1,215 @@ +import base64 +import hashlib +import json +from pathlib import Path + +import pytest +from legacy_archive_factory import corrupt_payload, make_legacy_archive + +from lowpack import ( + migrate_archive, + pack, + probe_compatibility, + unpack, + verify_archive, +) +from lowpack.archive import _read_manifest +from lowpack.cli import main +from lowpack.format import FormatError + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_golden_format_1_0_archive(tmp_path: Path) -> None: + encoded = (FIXTURES / "format-1.0-minimal.b64").read_text( + encoding="ascii" + ).strip() + archive = tmp_path / "golden-1.0.lpk" + archive.write_bytes(base64.b64decode(encoded, validate=True)) + assert hashlib.sha256(archive.read_bytes()).hexdigest() == ( + "cf8ce945907c089b490496f8d15eb371e82f20e09a4c77eedbf2e29667c6ec21" + ) + assert probe_compatibility(archive).migration_supported + + migrated = tmp_path / "golden-1.1.lpk" + migrate_archive(archive, migrated) + restored = tmp_path / "golden-restored" + unpack(migrated, output=restored) + assert (restored / "hello.py").read_text(encoding="utf-8") == ( + 'def greet(name: str) -> str:\n' + ' return f"Hello, {name}!"\n' + ) + + +def test_migrate_legacy_archive_and_preserve_content(tmp_path: Path) -> None: + source = tmp_path / "input" + source.mkdir() + (source / "hello.txt").write_text("hello from 0.1\n" * 200, encoding="utf-8") + (source / "empty.bin").write_bytes(b"") + current = tmp_path / "current.lpk" + legacy = tmp_path / "legacy.lpk" + migrated = tmp_path / "migrated.lpk" + pack([source], current, goal="prefer-store") + make_legacy_archive( + current, + legacy, + lambda manifest: manifest.__setitem__( + "legacy_note", {"kept": "when practical"} + ), + ) + + result = migrate_archive(legacy, migrated) + + assert result.source_format == "1.0" + assert result.target_format == "1.1" + assert result.file_count == 2 + assert verify_archive(migrated, full=True).valid + with migrated.open("rb") as stream: + manifest, _, _ = _read_manifest(stream) + assert manifest["format"] == {"major": 1, "minor": 1} + assert manifest["manifest_schema"] == {"major": 2, "minor": 0} + assert manifest["goal"] == "prefer-store" + assert manifest["legacy_note"] == {"kept": "when practical"} + assert all("preferred_codec" in item["decision"] for item in manifest["files"]) + assert all( + len(item["chunk_decisions"]) == len(item["chunks"]) + for item in manifest["files"] + ) + restored = tmp_path / "restored" + unpack(migrated, output=restored) + assert (restored / "input" / "hello.txt").read_bytes() == ( + source / "hello.txt" + ).read_bytes() + assert (restored / "input" / "empty.bin").read_bytes() == b"" + + +def test_read_only_compatibility_probe(tmp_path: Path) -> None: + source = tmp_path / "input.txt" + source.write_text("compatibility", encoding="utf-8") + current = tmp_path / "current.lpk" + legacy = tmp_path / "legacy.lpk" + pack([source], current) + make_legacy_archive(current, legacy) + + current_result = probe_compatibility(current) + assert current_result.current is True + assert current_result.status == "current" + assert current_result.migration_supported is False + legacy_result = probe_compatibility(legacy) + assert legacy_result.current is False + assert legacy_result.status == "migration-available" + assert legacy_result.migration_supported is True + assert legacy_result.target_format == "1.1" + + +def test_migrate_source_dictionary_archive(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + for index in range(10): + (source / f"module_{index}.py").write_text( + f"def value_{index}():\n" + f" return {' + '.join(str(number) for number in range(200))}\n", + encoding="utf-8", + ) + current = tmp_path / "source-current.lpk" + legacy = tmp_path / "source-legacy.lpk" + migrated = tmp_path / "source-migrated.lpk" + pack([source], current, profile="source", codec="zstd") + make_legacy_archive(current, legacy) + + migrate_archive(legacy, migrated) + + with migrated.open("rb") as stream: + manifest, _, _ = _read_manifest(stream) + assert manifest["dictionaries"] + assert { + record["dictionary_id"] + for record in manifest["source_dictionaries"].values() + } == set(manifest["dictionaries"]) + assert verify_archive(migrated, full=True).valid + + +def test_migration_rejects_corruption_and_unsafe_manifest(tmp_path: Path) -> None: + source = tmp_path / "file.txt" + source.write_text("payload", encoding="utf-8") + current = tmp_path / "current.lpk" + pack([source], current) + + corrupt = make_legacy_archive(current, tmp_path / "corrupt.lpk") + corrupt_payload(corrupt) + with pytest.raises(FormatError, match="body hash mismatch"): + probe_compatibility(corrupt) + with pytest.raises(FormatError, match="body hash mismatch"): + migrate_archive(corrupt, tmp_path / "corrupt-output.lpk") + + authenticated_corrupt = make_legacy_archive( + current, tmp_path / "authenticated-corrupt.lpk" + ) + corrupt_payload(authenticated_corrupt, reauthenticate=True) + authenticated_output = tmp_path / "authenticated-output.lpk" + with pytest.raises(FormatError, match="failed full verification"): + migrate_archive(authenticated_corrupt, authenticated_output) + assert not authenticated_output.exists() + + hostile = make_legacy_archive( + current, + tmp_path / "hostile.lpk", + lambda manifest: manifest["files"][0].__setitem__("path", "../escape"), + ) + with pytest.raises(FormatError): + migrate_archive(hostile, tmp_path / "hostile-output.lpk") + + +def test_migration_output_rules_and_current_archive_rejection( + tmp_path: Path, +) -> None: + source = tmp_path / "file.txt" + source.write_text("payload", encoding="utf-8") + current = tmp_path / "current.lpk" + legacy = tmp_path / "legacy.lpk" + output = tmp_path / "output.lpk" + pack([source], current) + make_legacy_archive(current, legacy) + + with pytest.raises(ValueError, match="must differ"): + migrate_archive(legacy, legacy) + with pytest.raises(FormatError, match="already"): + migrate_archive(current, output) + output.write_bytes(b"occupied") + with pytest.raises(FileExistsError): + migrate_archive(legacy, output) + result = migrate_archive(legacy, output, overwrite=True) + assert result.archive == output + assert verify_archive(output, full=True).valid + + +def test_migrate_cli_json(tmp_path: Path, capsys) -> None: + source = tmp_path / "file.txt" + source.write_text("payload", encoding="utf-8") + current = tmp_path / "current.lpk" + legacy = tmp_path / "legacy.lpk" + migrated = tmp_path / "migrated.lpk" + pack([source], current) + make_legacy_archive(current, legacy) + + assert ( + main( + [ + "migrate", + str(legacy), + "-o", + str(migrated), + "--json", + ] + ) + == 0 + ) + value = json.loads(capsys.readouterr().out) + assert value["source_format"] == "1.0" + assert value["target_format"] == "1.1" + assert verify_archive(migrated, full=True).valid + + assert main(["compatibility", str(migrated), "--json"]) == 0 + compatibility = json.loads(capsys.readouterr().out) + assert compatibility["current"] is True + assert compatibility["status"] == "current"