From c8de6aa649700ae290e5d14bcf198b3be072d3cd Mon Sep 17 00:00:00 2001 From: Nejc Stebe Date: Fri, 21 Aug 2026 10:51:14 +0200 Subject: [PATCH] Enable requiring zipped modules ENG-168 --- module_renderer.py | 6 + partial_rendering.py | 39 ++- plain2code.py | 16 +- plain2code_exceptions.py | 8 + plain_modules.py | 204 ++++++++++++- .../actions/prepare_repositories.py | 5 + render_machine/conformance_tests.py | 43 ++- render_machine/implementation_code_helpers.py | 6 +- render_machine/render_context.py | 14 +- tests/test_plain_modules.py | 268 +++++++++++++++++- tests/test_prepare_repositories_layout.py | 158 +++++++++++ tui/plain_module_render_choice_tui.py | 33 ++- 12 files changed, 774 insertions(+), 26 deletions(-) diff --git a/module_renderer.py b/module_renderer.py index 6640d619..4ca655a2 100644 --- a/module_renderer.py +++ b/module_renderer.py @@ -111,6 +111,12 @@ def _render_module( ): return False, False + # We are going to (re)render this module. If it currently exists only as a + # ".module" archive, unpack it into the real plain_modules// folder now, + # before RenderContext snapshots the build folder path and before the git repos, metadata, + # and memory folder are touched. + plain_module.ensure_module_unpacked() + memory_manager = MemoryManager( self.codeplainAPI, plain_module.module_memory_folder, diff --git a/partial_rendering.py b/partial_rendering.py index 202bff87..40fccdda 100644 --- a/partial_rendering.py +++ b/partial_rendering.py @@ -12,7 +12,7 @@ class PlainModuleRenderState: last_render_module: PlainModule last_render_frid: str | None change: PlainModule | None = None - change_type: Literal["spec_change", "code_change"] | None = None + change_type: Literal["spec_change", "code_change", "missing_conformance_tests"] | None = None @dataclass @@ -56,6 +56,19 @@ def code_change(plain_module: PlainModule) -> PlainModule | None: return None +def archive_missing_conformance_tests(plain_module: PlainModule, render_conformance_tests: bool) -> PlainModule | None: + """Return the first module available only as a ".module" archive that has no conformance + tests while conformance testing is enabled. Such an archive cannot be consumed as-is (regression + needs the tests), so the module must be re-rendered.""" + if not render_conformance_tests: + return None + all_modules = plain_module.all_required_modules + [plain_module] + for _module in all_modules: + if _module.is_archived_only() and not _module.archive_has_conformance_tests(): + return _module + return None + + def module_comes_before_or_equal( all_required_modules: list[PlainModule], module1: PlainModule, @@ -70,15 +83,24 @@ def module_comes_before_or_equal( raise ValueError(f"Module {module1.module_name} and {module2.module_name} not found in {all_required_modules}") -def get_plain_module_render_state(plain_module: PlainModule) -> PlainModuleRenderState | None: +def get_plain_module_render_state( + plain_module: PlainModule, render_conformance_tests: bool = False +) -> PlainModuleRenderState | None: sc = spec_change(plain_module) cc = code_change(plain_module) + mt = archive_missing_conformance_tests(plain_module, render_conformance_tests) all_required_modules = plain_module.all_required_modules last_rendered_module_name, last_rendered_frid = plain_module.get_module_render_status() - if last_rendered_module_name is None and last_rendered_frid is None: + if last_rendered_module_name is None and last_rendered_frid is None and mt is None: return None - if last_rendered_module_name == plain_module.module_name: + if last_rendered_module_name is None: + # Nothing has been rendered yet, but an archived module blocks consumption (no tests). + # Anchor the state on that module so the user is prompted to re-render it. + assert mt is not None # guaranteed by the early return above + module = mt + last_rendered_frid = None + elif last_rendered_module_name == plain_module.module_name: module = plain_module else: found_module: PlainModule | None = None @@ -99,6 +121,13 @@ def get_plain_module_render_state(plain_module: PlainModule) -> PlainModuleRende change_type=None, ) + # An archive that lacks conformance tests (while testing is enabled) is a hard blocker: it cannot + # be consumed as-is, so it takes precedence over spec/code changes. + if mt is not None: + pr.change = mt + pr.change_type = "missing_conformance_tests" + return pr + if sc is None and cc is None: return pr @@ -129,6 +158,8 @@ def get_all_affected_modules_from_change( start_module = plain_module.get_next_module(plain_module_render_state.change.module_name) else: start_module = plain_module_render_state.change + elif plain_module_render_state.change_type == "missing_conformance_tests": + start_module = plain_module_render_state.change else: raise ValueError(f"Unknown change type: {plain_module_render_state.change_type}") diff --git a/plain2code.py b/plain2code.py index ef60ddf4..3c30deda 100644 --- a/plain2code.py +++ b/plain2code.py @@ -28,6 +28,7 @@ ImportedModuleWithFunctionalitiesError, InvalidAPIKey, InvalidFridArgument, + InvalidModuleArchiveError, MissingAPIKey, MissingFunctionalitiesError, MissingPreviousFunctionalitiesError, @@ -79,6 +80,7 @@ UnsupportedResourceType, UnsupportedBase64Content, GitNotInstalledError, + InvalidModuleArchiveError, SystemExit, ) @@ -212,6 +214,15 @@ def render( # noqa: C901 warn_if_acceptance_tests_without_conformance_script(plain_module, args) + # A built module can be distributed as a ".module" zip archive instead of an unpacked directory. + # Extract any such module to a scratch location up front. Scratch dirs are removed in main()'s finally. + # Skip an archive-only module that lacks conformance tests while testing is enabled: it cannot be + # consumed as-is, so leave it unmaterialized so it is detected below as needing a re-render. + for module in plain_module.all_required_modules + [plain_module]: + if args.render_conformance_tests and module.is_archived_only() and not module.archive_has_conformance_tests(): + continue + module.materialize() + # The module_metadata file lives outside the code git repo. That means that a crash mid-render can leave it # claiming a functionality was implemented even thought it wasn't yet committed (because of the crash). # Out of precaution, this reconciles every module_metadata against the code repo. @@ -220,7 +231,7 @@ def render( # noqa: C901 render_choice = None if render_range is None: - plain_module_render_state = get_plain_module_render_state(plain_module) + plain_module_render_state = get_plain_module_render_state(plain_module, args.render_conformance_tests) if plain_module_render_state is not None: render_choices = get_render_choices(plain_module, plain_module_render_state, args.force_render) ask_user = True @@ -417,6 +428,9 @@ def main(): # noqa: C901 args.filename, error_message=error_message, ) + # Remove any scratch extractions created for archive-only (".module") modules. + for module in plain_module.all_required_modules + [plain_module]: + module.cleanup_scratch() if args.headless and (exc_info is not None or not run_state.render_succeeded): sys.exit(1) diff --git a/plain2code_exceptions.py b/plain2code_exceptions.py index b29bea98..10d672bd 100644 --- a/plain2code_exceptions.py +++ b/plain2code_exceptions.py @@ -117,3 +117,11 @@ class GitNotInstalledError(Exception): """Raised when git is not installed or not found on PATH.""" pass + + +class InvalidModuleArchiveError(Exception): + """Raised when a ``.module`` archive is missing, corrupt, or has an + unexpected layout (not a zip, missing ``code/``/``tests/``, ``.git`` not a real + directory, detached HEAD, or an unsafe member path).""" + + pass diff --git a/plain_modules.py b/plain_modules.py index cda701ca..053ca95e 100644 --- a/plain_modules.py +++ b/plain_modules.py @@ -2,15 +2,24 @@ import os import shutil +import tempfile +import zipfile from functools import cached_property -from plain2code_exceptions import GitNotInstalledError, MissingPreviousFunctionalitiesError, ModuleDoesNotExistError +from plain2code_exceptions import ( + GitNotInstalledError, + InvalidModuleArchiveError, + MissingPreviousFunctionalitiesError, + ModuleDoesNotExistError, +) try: + from git import Repo from git.exc import NoSuchPathError except ImportError: raise GitNotInstalledError("git is not installed. Please install git and try again.") +import file_utils import git_utils import metadata_utils import plain_file @@ -28,6 +37,11 @@ MODULE_CODE_SUBFOLDER = "code" MODULE_TESTS_SUBFOLDER = "tests" +# A module's build output may be shipped as a single zip archive named +# ".module" instead of an unpacked "/" folder. See PlainModule.materialize +# (read/consume) and PlainModule.ensure_module_unpacked (unpack-on-change). +MODULE_ARCHIVE_EXTENSION = ".module" + def get_module_code_folder(modules_base_folder: str, module_name: str) -> str: return os.path.join(modules_base_folder, module_name, MODULE_CODE_SUBFOLDER) @@ -37,6 +51,81 @@ def get_module_tests_folder(modules_base_folder: str, module_name: str) -> str: return os.path.join(modules_base_folder, module_name, MODULE_TESTS_SUBFOLDER) +def _validate_module_repo(repo_path: str, subfolder: str, archive_path: str) -> None: + """Validate one extracted git repo (code/ or tests/) inside a module archive.""" + if not os.path.isdir(os.path.join(repo_path, ".git")): + raise InvalidModuleArchiveError( + f"Module archive '{archive_path}' has no git repository in '{subfolder}/' " + "(the archive must include the '.git' directory)." + ) + try: + repo = Repo(repo_path) + if repo.bare or repo.head.is_detached: + raise InvalidModuleArchiveError( + f"The git repository in '{subfolder}/' of module archive '{archive_path}' " + "must be a working tree checked out on a branch." + ) + except InvalidModuleArchiveError: + raise + except Exception as e: + raise InvalidModuleArchiveError( + f"The git repository in '{subfolder}/' of module archive '{archive_path}' is invalid: {e}" + ) from e + + +def _validate_module_tree(root: str, archive_path: str) -> None: + """Verify an extracted module tree. ``code/`` is required. ``tests/`` is optional: a module + rendered without a conformance-tests script has no tests folder, so its archive has none either. + When ``tests/`` is present it must be a valid git repo.""" + code_path = os.path.join(root, MODULE_CODE_SUBFOLDER) + if not os.path.isdir(code_path): + raise InvalidModuleArchiveError( + f"Module archive '{archive_path}' is missing the '{MODULE_CODE_SUBFOLDER}/' folder at its root. " + "The archive must contain the module folder's contents (code/, optionally tests/, ...) at its " + "root, not nested under a top-level directory." + ) + _validate_module_repo(code_path, MODULE_CODE_SUBFOLDER, archive_path) + + tests_path = os.path.join(root, MODULE_TESTS_SUBFOLDER) + if os.path.isdir(tests_path): + _validate_module_repo(tests_path, MODULE_TESTS_SUBFOLDER, archive_path) + + +def _extract_module_archive(archive_path: str, dest: str) -> None: + """Extract a ".module" zip into dest and validate the module layout. + + Guards against zip-slip, restores unix mode bits (zipfile drops them), and validates + that dest contains valid code/ and tests/ git repositories. Raises + InvalidModuleArchiveError on any problem. + """ + if not zipfile.is_zipfile(archive_path): + raise InvalidModuleArchiveError(f"Module archive '{archive_path}' is not a valid zip file.") + + os.makedirs(dest, exist_ok=True) + dest_root = os.path.realpath(dest) + + try: + with zipfile.ZipFile(archive_path) as zf: + for member in zf.namelist(): + target = os.path.realpath(os.path.join(dest, member)) + if target != dest_root and not target.startswith(dest_root + os.sep): + raise InvalidModuleArchiveError( + f"Module archive '{archive_path}' contains an unsafe path: {member}" + ) + zf.extractall(dest) + for info in zf.infolist(): + mode = (info.external_attr >> 16) & 0o777 + if not mode: + continue + member_path = os.path.join(dest, info.filename) + if os.path.exists(member_path) and not os.path.islink(member_path): + os.chmod(member_path, mode) + except zipfile.BadZipFile as e: + raise InvalidModuleArchiveError(f"Module archive '{archive_path}' is corrupt: {e}") from e + + _validate_module_tree(dest, archive_path) + + def _strip_functional_requirements(plain_source_tree: dict) -> dict: stripped = {k: v for k, v in plain_source_tree.items() if k != plain_spec.FUNCTIONAL_REQUIREMENTS} if "sections" in stripped: @@ -49,6 +138,10 @@ def __init__(self, filename: str, build_folder: str, template_dirs: list[str]): self.filename = filename self.build_folder = build_folder self.template_dirs = template_dirs + # When the module exists only as a ".module" archive, these hold the + # scratch extraction used for read-only consumption. See materialize(). + self._resolved_module_folder: str | None = None + self._scratch_dir: str | None = None module_name, plain_source, required_modules_names = plain_file.plain_file_parser( self.filename, self.template_dirs ) @@ -81,16 +174,28 @@ def all_required_modules(self) -> list[PlainModule]: return all_required_modules @property - def module_folder(self): + def _default_module_folder(self) -> str: return os.path.join(self.build_folder, self.module_name) + @property + def module_folder(self): + # When the module was materialized from a ".module" archive, all subpaths + # resolve against the scratch extraction; otherwise against the real folder. + if self._resolved_module_folder is not None: + return self._resolved_module_folder + return self._default_module_folder + + @property + def module_archive_path(self) -> str: + return self._default_module_folder + MODULE_ARCHIVE_EXTENSION + @property def module_conformance_tests_folder(self): - return get_module_tests_folder(self.build_folder, self.module_name) + return os.path.join(self.module_folder, MODULE_TESTS_SUBFOLDER) @property def module_build_folder(self): - return get_module_code_folder(self.build_folder, self.module_name) + return os.path.join(self.module_folder, MODULE_CODE_SUBFOLDER) @property def module_memory_folder(self): @@ -99,6 +204,84 @@ def module_memory_folder(self): def get_codeplain_folder(self): return os.path.join(self.module_folder, CODEPLAIN_METADATA_FOLDER) + def has_module_archive(self) -> bool: + return os.path.isfile(self.module_archive_path) + + def is_archived_only(self) -> bool: + return not os.path.isdir(self._default_module_folder) and self.has_module_archive() + + def archive_has_conformance_tests(self) -> bool: + """True if the ".module" archive contains a tests/ folder. A module rendered without + a conformance-tests script has no tests/, so its archive has none either.""" + if not self.has_module_archive() or not zipfile.is_zipfile(self.module_archive_path): + return False + prefix = MODULE_TESTS_SUBFOLDER + "/" + with zipfile.ZipFile(self.module_archive_path) as zf: + return any(name.startswith(prefix) for name in zf.namelist()) + + def materialize(self) -> None: + """Make an archive-only module readable without populating plain_modules//. + + If the real folder is absent but a ".module" archive exists, extract it to a + scratch directory and point this module's paths there. No-op if the real folder exists + or the module is already materialized. The archive file is preserved. + """ + if self._resolved_module_folder is not None: + return + if os.path.isdir(self._default_module_folder): + return + if not self.has_module_archive(): + return + + scratch_dir = tempfile.mkdtemp(prefix=f"codeplain-module-{self.module_name}-") + try: + _extract_module_archive(self.module_archive_path, scratch_dir) + except BaseException: + shutil.rmtree(scratch_dir, ignore_errors=True) + raise + self._scratch_dir = scratch_dir + self._resolved_module_folder = scratch_dir + + def ensure_module_unpacked(self) -> None: + """Unpack an archive-only module into the real plain_modules// in place. + + Called before a module is (re)rendered. Idempotent. If the real folder already exists, + only a stray archive is removed. Extraction is atomic (extract to a temp sibling under + the build folder, then os.replace), and the archive is deleted only after success. + """ + if os.path.isdir(self._default_module_folder): + if self.has_module_archive(): + os.remove(self.module_archive_path) + self._reset_scratch() + self._resolved_module_folder = None + return + + if not self.has_module_archive(): + return + + os.makedirs(self.build_folder, exist_ok=True) + staging_dir = tempfile.mkdtemp(prefix=f".{self.module_name}-unpacking-", dir=self.build_folder) + try: + _extract_module_archive(self.module_archive_path, staging_dir) + os.replace(staging_dir, self._default_module_folder) + except BaseException: + shutil.rmtree(staging_dir, ignore_errors=True) + raise + + os.remove(self.module_archive_path) + self._reset_scratch() + self._resolved_module_folder = None + + def _reset_scratch(self) -> None: + if self._scratch_dir is not None: + shutil.rmtree(self._scratch_dir, ignore_errors=True) + self._scratch_dir = None + + def cleanup_scratch(self) -> None: + """Remove any scratch extraction created by materialize(). Safe to call multiple times.""" + self._reset_scratch() + self._resolved_module_folder = None + def get_module_render_status(self) -> tuple[str | None, str | None]: module_name, frid = git_utils.get_last_rendered_functionality(self.module_build_folder) if module_name is not None and module_name == self.module_name: @@ -149,6 +332,9 @@ def get_module_non_functional_source_hash(self) -> str: return plain_spec.get_hash_value([stripped] + self.resources_list) def get_module_code_hash(self) -> str: + # Content-only hash (see calculate_build_folder_hash): reading from the resolved (possibly + # scratch) folder yields the same hash as the in-place folder and the same hash across + # locations, so archived modules stay portable. return ImplementationCodeHelpers.calculate_build_folder_hash(self.module_build_folder) def has_required_modules_code_changed( @@ -408,6 +594,10 @@ def has_no_rendered_functionality(self) -> bool: return False def wipe_module(self) -> None: - if os.path.exists(self.module_folder): - console.warning(f"Wiping module {self.module_folder}...") - shutil.rmtree(self.module_folder) + if os.path.isdir(self._default_module_folder): + console.warning(f"Wiping module {self._default_module_folder}...") + file_utils.delete_folder(self._default_module_folder) + if self.has_module_archive(): + os.remove(self.module_archive_path) + self._reset_scratch() + self._resolved_module_folder = None diff --git a/render_machine/actions/prepare_repositories.py b/render_machine/actions/prepare_repositories.py index a98e2cc1..e06cf55a 100644 --- a/render_machine/actions/prepare_repositories.py +++ b/render_machine/actions/prepare_repositories.py @@ -32,6 +32,11 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any | ) else: + if render_context.required_modules: + # If the required module exists only as a ".module" archive, extract it to scratch first + # (no-op if it is already an unpacked folder or already materialized). + render_context.required_modules[-1].materialize() + file_utils.delete_folder(render_context.plain_module.module_folder) render_context.plain_module.seed_module_metadata() diff --git a/render_machine/conformance_tests.py b/render_machine/conformance_tests.py index d43f3bf4..981ce4ec 100644 --- a/render_machine/conformance_tests.py +++ b/render_machine/conformance_tests.py @@ -1,5 +1,6 @@ import json import os +from typing import Callable, Optional import file_utils from plain2code_console import console @@ -16,11 +17,20 @@ def __init__( self, modules_base_folder: str, conformance_tests_definition_file_name: str, + resolve_module_tests_folder: Optional[Callable[[str], Optional[str]]] = None, ): self.modules_base_folder = modules_base_folder self.conformance_tests_definition_file_name = conformance_tests_definition_file_name + # Optional resolver mapping a module name to its resolved tests folder. Lets an + # archive-only (".module") required module resolve to its scratch extraction + # instead of the (non-existent) default plain_modules//tests path. + self._resolve_module_tests_folder = resolve_module_tests_folder def get_module_conformance_tests_folder(self, module_name: str) -> str: + if self._resolve_module_tests_folder is not None: + resolved = self._resolve_module_tests_folder(module_name) + if resolved is not None: + return resolved return get_module_tests_folder(self.modules_base_folder, module_name) def _get_full_conformance_tests_definition_file_name(self, module_name: str) -> str: @@ -29,21 +39,48 @@ def _get_full_conformance_tests_definition_file_name(self, module_name: str) -> self.conformance_tests_definition_file_name, ) + def _resolve_folder_names(self, module_name: str, conformance_tests_json: dict) -> dict: + """Turn each entry's on-disk relative ``folder_name`` into an absolute path rooted at the + module's (resolved) tests folder. An already-absolute value is left as-is, so archives + written by older builds still load.""" + base = self.get_module_conformance_tests_folder(module_name) + resolved: dict = {} + for frid, entry in conformance_tests_json.items(): + if isinstance(entry, dict) and "folder_name" in entry: + entry = {**entry, "folder_name": os.path.join(base, entry["folder_name"])} + resolved[frid] = entry + return resolved + + def _relativize_folder_names(self, module_name: str, conformance_tests_json: dict) -> dict: + """Turn each entry's absolute in-memory ``folder_name`` into a path relative to the module's + tests folder, so the stored definition is location-independent (portable across projects and + usable from a ".module" archive). Does not mutate the input dict.""" + base = self.get_module_conformance_tests_folder(module_name) + serializable: dict = {} + for frid, entry in conformance_tests_json.items(): + if isinstance(entry, dict) and "folder_name" in entry: + entry = {**entry, "folder_name": os.path.relpath(entry["folder_name"], base)} + serializable[frid] = entry + return serializable + def get_conformance_tests_json(self, module_name: str) -> dict: try: with open(self._get_full_conformance_tests_definition_file_name(module_name), "r") as f: - return json.load(f) + conformance_tests_json = json.load(f) except FileNotFoundError: return {} + return self._resolve_folder_names(module_name, conformance_tests_json) def dump_conformance_tests_json(self, module_name: str, conformance_tests_json: dict) -> None: - """Dump the conformance tests definition to the file.""" + """Dump the conformance tests definition to the file. Folder names are stored relative to the + module's tests folder so the definition is portable and works from a scratch extraction.""" if os.path.exists(self.get_module_conformance_tests_folder(module_name)): console.debug( f"Storing conformance tests definition to {self._get_full_conformance_tests_definition_file_name(module_name)}" ) + serializable = self._relativize_folder_names(module_name, conformance_tests_json) with open(self._get_full_conformance_tests_definition_file_name(module_name), "w") as f: - json.dump(conformance_tests_json, f, indent=4) + json.dump(serializable, f, indent=4) def fetch_existing_conformance_test_folder_names(self, module_name: str) -> list[str]: if os.path.isdir(self.get_module_conformance_tests_folder(module_name)): diff --git a/render_machine/implementation_code_helpers.py b/render_machine/implementation_code_helpers.py index 219f29a8..d33df1af 100644 --- a/render_machine/implementation_code_helpers.py +++ b/render_machine/implementation_code_helpers.py @@ -8,8 +8,12 @@ class ImplementationCodeHelpers: @staticmethod def calculate_build_folder_hash(build_folder: str) -> str: + # Hash the code content only (relative paths + file contents), NOT the build folder's + # absolute path, so the hash is stable across directories and machines. This is required for + # distributable ".module" archives: a module's code hash must match regardless of + # where plain_modules/ lives, or consuming an archive elsewhere falsely reports a code change. _, existing_files_content = ImplementationCodeHelpers.fetch_existing_files(build_folder) - return plain_spec.hash_text(f"folder={build_folder}|{json.dumps(existing_files_content)}") + return plain_spec.hash_text(json.dumps(existing_files_content)) @staticmethod def fetch_existing_files(build_folder: str): diff --git a/render_machine/render_context.py b/render_machine/render_context.py index 2ea7e806..75652953 100644 --- a/render_machine/render_context.py +++ b/render_machine/render_context.py @@ -87,10 +87,22 @@ def __init__( self.conformance_tests_running_context: Optional[ConformanceTestsRunningContext] = None # Constants that should remain for a single frid, but possible over multiple rerenderings of the same frid self.functional_requirements_render_attempts_failed_unit_during_conformance_tests = 0 - # Initialize conformance tests utilities + + # Initialize conformance tests utilities. The resolver lets a required module that ships + # as a ".module" archive resolve to its scratch extraction (via materialize()) + # rather than the non-existent default plain_modules//tests path. + def _resolve_module_tests_folder(module_name: str) -> Optional[str]: + if module_name == plain_module.module_name: + return plain_module.module_conformance_tests_folder + for required_module in plain_module.all_required_modules: + if required_module.module_name == module_name: + return required_module.module_conformance_tests_folder + return None + self.conformance_tests = ConformanceTests( modules_base_folder=plain_module.build_folder, conformance_tests_definition_file_name=CONFORMANCE_TESTS_DEFINITION_FILE_NAME, + resolve_module_tests_folder=_resolve_module_tests_folder, ) self.machine = None diff --git a/tests/test_plain_modules.py b/tests/test_plain_modules.py index 094a857e..b3dff68c 100644 --- a/tests/test_plain_modules.py +++ b/tests/test_plain_modules.py @@ -7,14 +7,16 @@ import json import os +import shutil import tempfile +import zipfile from pathlib import Path import pytest from change_detection import determine_partial_render_start from git_utils import FUNCTIONAL_REQUIREMENT_FINISHED_COMMIT_MESSAGE, add_all_files_and_commit, init_git_repo -from plain2code_exceptions import ModuleDoesNotExistError +from plain2code_exceptions import InvalidModuleArchiveError, ModuleDoesNotExistError from plain_modules import MODULE_METADATA_FILENAME, PlainModule # -------------------------------------------------------------------------- @@ -361,6 +363,270 @@ def test_wipe_module_removes_whole_module_folder(solo_module): assert not os.path.exists(solo_module.module_folder) +# -------------------------------------------------------------------------- +# Zipped modules (.module archives) +# -------------------------------------------------------------------------- + + +def _init_repo_with_finished_frid(repo_path: str, module_name: str, frid: str) -> None: + """Init a git repo at ``repo_path`` and commit a FUNCTIONAL_REQUIREMENT_FINISHED checkpoint.""" + os.makedirs(repo_path, exist_ok=True) + init_git_repo(repo_path, module_name=module_name) + marker = Path(repo_path) / f"frid_{frid}.txt" + marker.write_text(f"frid {frid}\n") + add_all_files_and_commit( + repo_path, + FUNCTIONAL_REQUIREMENT_FINISHED_COMMIT_MESSAGE.format(frid), + module_name=module_name, + frid=frid, + ) + + +def _setup_full_module(module: PlainModule, frid: str = "1") -> None: + """Create a realistic on-disk module: code/ and tests/ git repos plus metadata.""" + _init_repo_with_finished_frid(module.module_build_folder, module.module_name, frid) + _init_repo_with_finished_frid(module.module_conformance_tests_folder, module.module_name, frid) + _write_metadata(module, {"source_hash": "seed", "functionalities": ["fr1"]}) + + +def _zip_module_flat(module_folder: str, archive_path: str) -> None: + """Zip the *contents* of ``module_folder`` (code/, tests/, .codeplain/ at the archive + root, including .git) into ``archive_path``.""" + with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zf: + for root, _dirs, files in os.walk(module_folder): + for name in files: + full = os.path.join(root, name) + zf.write(full, os.path.relpath(full, module_folder)) + + +def _archive_module(module: PlainModule) -> str: + """Turn an on-disk module into an archive-only module: zip it, then remove the folder.""" + archive_path = module.module_archive_path + _zip_module_flat(module.module_folder, archive_path) + shutil.rmtree(module.module_folder) + return archive_path + + +def test_folder_takes_precedence_over_archive(solo_module): + """When both the folder and the archive exist, the folder is used and the archive untouched.""" + _setup_full_module(solo_module) + # Create a stray archive alongside the real folder. + with zipfile.ZipFile(solo_module.module_archive_path, "w") as zf: + zf.writestr("code/marker.txt", "x") + + default = solo_module._default_module_folder + solo_module.materialize() + + assert solo_module.module_folder == default + assert solo_module.module_build_folder == os.path.join(default, "code") + assert os.path.isfile(solo_module.module_archive_path) # untouched + + +def test_materialize_resolves_paths_to_scratch(solo_module): + _setup_full_module(solo_module) + default = solo_module._default_module_folder + _archive_module(solo_module) + + assert solo_module.is_archived_only() + solo_module.materialize() + + # Paths now resolve into a scratch extraction, not the real (absent) folder. + assert solo_module.module_folder != default + assert not os.path.exists(default) # real folder NOT created + assert os.path.isfile(solo_module.module_archive_path) # archive preserved + assert os.path.isdir(os.path.join(solo_module.module_build_folder, ".git")) + assert os.path.isdir(os.path.join(solo_module.module_conformance_tests_folder, ".git")) + # Git-backed reads work against the scratch tree. + name, frid = solo_module.get_module_render_status() + assert name == solo_module.module_name + assert frid == "1" + + solo_module.cleanup_scratch() + assert solo_module.module_folder == default # resolution reset + + +def test_get_repo_none_before_materialize_and_present_after(solo_module): + """Guards the skip-detection bug: an archive-only module reads as "no repo" until materialized, + so the orchestrator's `get_repo() is not None` check only passes after materialize().""" + _setup_full_module(solo_module) + _archive_module(solo_module) + + assert solo_module.get_repo() is None # would wrongly force a re-render if consumed as-is + solo_module.materialize() + assert solo_module.get_repo() is not None # now recognised as already rendered + + solo_module.cleanup_scratch() + + +def test_materialize_is_idempotent(solo_module): + _setup_full_module(solo_module) + _archive_module(solo_module) + solo_module.materialize() + first = solo_module.module_folder + solo_module.materialize() + assert solo_module.module_folder == first + solo_module.cleanup_scratch() + + +def test_ensure_module_unpacked_from_archive(solo_module): + _setup_full_module(solo_module) + default = solo_module._default_module_folder + _archive_module(solo_module) + + solo_module.ensure_module_unpacked() + + assert solo_module.module_folder == default + assert os.path.isdir(os.path.join(default, "code", ".git")) + assert os.path.isdir(os.path.join(default, "tests", ".git")) + assert not os.path.exists(solo_module.module_archive_path) # archive removed + # Idempotent: a second call is a no-op. + solo_module.ensure_module_unpacked() + assert os.path.isdir(default) + + +def test_ensure_module_unpacked_after_materialize(solo_module): + """materialize() then ensure_module_unpacked() lands content in the real folder.""" + _setup_full_module(solo_module) + default = solo_module._default_module_folder + _archive_module(solo_module) + + solo_module.materialize() + solo_module.ensure_module_unpacked() + + assert solo_module.module_folder == default + assert os.path.isdir(os.path.join(default, "code", ".git")) + assert not os.path.exists(solo_module.module_archive_path) + + +def test_ensure_module_unpacked_noop_when_folder_exists(solo_module): + """A stray archive next to an existing folder is removed; the folder is kept.""" + _setup_full_module(solo_module) + with zipfile.ZipFile(solo_module.module_archive_path, "w") as zf: + zf.writestr("code/marker.txt", "x") + + solo_module.ensure_module_unpacked() + + assert os.path.isdir(solo_module.module_build_folder) + assert not os.path.exists(solo_module.module_archive_path) + + +def test_wipe_module_removes_archive_and_scratch(solo_module): + _setup_full_module(solo_module) + _archive_module(solo_module) + solo_module.materialize() + scratch = solo_module._scratch_dir + + solo_module.wipe_module() + + assert not os.path.exists(solo_module.module_archive_path) + assert scratch is not None and not os.path.exists(scratch) + assert solo_module.module_folder == solo_module._default_module_folder + + +def test_render_status_parity_folder_vs_archive(fixtures_dir, tmp_build_folder): + """The same module (same logical path) reports the same status and code hash whether it + is read as an unpacked folder or from a ".module" archive.""" + module = PlainModule("pr_solo.plain", tmp_build_folder, [fixtures_dir]) + _setup_full_module(module, frid="1") + folder_status = module.get_module_render_status() + folder_hash = module.get_module_code_hash() + + _archive_module(module) # zip in place, remove the folder + module.materialize() + + assert module.get_module_render_status() == folder_status + assert module.get_module_code_hash() == folder_hash + module.cleanup_scratch() + + +def test_code_hash_is_location_independent(fixtures_dir): + """The code hash depends only on content, not the build folder's absolute path, so it is stable + across directories/machines. This is what makes .module archives distributable: consuming the + same code at a different location must not report a spurious 'required module code changed'.""" + with tempfile.TemporaryDirectory() as build_a, tempfile.TemporaryDirectory() as build_b: + mod_a = PlainModule("pr_solo.plain", build_a, [fixtures_dir]) + mod_b = PlainModule("pr_solo.plain", build_b, [fixtures_dir]) + for mod in (mod_a, mod_b): + os.makedirs(mod.module_build_folder, exist_ok=True) + (Path(mod.module_build_folder) / "main.py").write_text("print('hi')\n") + + assert mod_a.module_build_folder != mod_b.module_build_folder + assert mod_a.get_module_code_hash() == mod_b.get_module_code_hash() + + +def test_archive_has_conformance_tests(solo_module): + _setup_full_module(solo_module) # code/ + tests/ + _archive_module(solo_module) + assert solo_module.archive_has_conformance_tests() is True + + +def test_code_only_archive_materializes_without_tests(solo_module): + """A module rendered with conformance tests off has no tests/, so its archive has none. It must + still validate and materialize (tests/ is optional).""" + _init_repo_with_finished_frid(solo_module.module_build_folder, solo_module.module_name, "1") + _write_metadata(solo_module, {"source_hash": "seed"}) + _archive_module(solo_module) # code/ + .codeplain, no tests/ + + assert solo_module.archive_has_conformance_tests() is False + assert solo_module.is_archived_only() + + solo_module.materialize() # must not raise + assert os.path.isdir(os.path.join(solo_module.module_build_folder, ".git")) + assert not os.path.isdir(solo_module.module_conformance_tests_folder) + solo_module.cleanup_scratch() + + +def test_ensure_module_unpacked_code_only_archive(solo_module): + _init_repo_with_finished_frid(solo_module.module_build_folder, solo_module.module_name, "1") + _archive_module(solo_module) # code/ only + + solo_module.ensure_module_unpacked() + assert os.path.isdir(os.path.join(solo_module._default_module_folder, "code", ".git")) + assert not os.path.isdir(os.path.join(solo_module._default_module_folder, "tests")) + assert not os.path.exists(solo_module.module_archive_path) + + +def test_materialize_rejects_non_zip(solo_module): + with open(solo_module.module_archive_path, "w") as f: + f.write("not a zip") + with pytest.raises(InvalidModuleArchiveError): + solo_module.materialize() + assert solo_module._scratch_dir is None + + +def test_materialize_rejects_missing_subfolders(solo_module): + with zipfile.ZipFile(solo_module.module_archive_path, "w") as zf: + zf.writestr("code/marker.txt", "x") # tests/ missing, and no .git + with pytest.raises(InvalidModuleArchiveError): + solo_module.materialize() + + +def test_materialize_rejects_missing_git_repo(solo_module): + """code/ and tests/ present but without a .git directory is rejected.""" + with zipfile.ZipFile(solo_module.module_archive_path, "w") as zf: + zf.writestr("code/main.py", "print('x')") + zf.writestr("tests/test_main.py", "def test_x(): pass") + with pytest.raises(InvalidModuleArchiveError): + solo_module.materialize() + + +def test_materialize_rejects_zip_slip(solo_module): + with zipfile.ZipFile(solo_module.module_archive_path, "w") as zf: + zf.writestr("../escape.txt", "x") + with pytest.raises(InvalidModuleArchiveError): + solo_module.materialize() + + +def test_ensure_module_unpacked_leaves_no_partial_on_invalid_archive(solo_module): + default = solo_module._default_module_folder + with zipfile.ZipFile(solo_module.module_archive_path, "w") as zf: + zf.writestr("code/marker.txt", "x") # invalid layout + with pytest.raises(InvalidModuleArchiveError): + solo_module.ensure_module_unpacked() + assert not os.path.exists(default) # no half-written module folder + assert os.path.isfile(solo_module.module_archive_path) # archive preserved on failure + + # -------------------------------------------------------------------------- # seed_module_metadata # -------------------------------------------------------------------------- diff --git a/tests/test_prepare_repositories_layout.py b/tests/test_prepare_repositories_layout.py index 0ea658d3..165febeb 100644 --- a/tests/test_prepare_repositories_layout.py +++ b/tests/test_prepare_repositories_layout.py @@ -8,13 +8,18 @@ //tests/ git repo with the conformance tests """ +import json import os +import shutil import tempfile +import zipfile from pathlib import Path from types import SimpleNamespace import pytest +from git_utils import FUNCTIONAL_REQUIREMENT_FINISHED_COMMIT_MESSAGE, add_all_files_and_commit, init_git_repo +from partial_rendering import archive_missing_conformance_tests, get_plain_module_render_state, get_render_choices from plain_modules import PlainModule from render_machine.actions.prepare_repositories import PrepareRepositories from render_machine.conformance_tests import CONFORMANCE_TESTS_DEFINITION_FILE_NAME, ConformanceTests @@ -49,6 +54,29 @@ def _make_render_context(module: PlainModule, render_conformance_tests: bool) -> ) +def _init_repo_with_finished_frid(repo_path: str, module_name: str, frid: str = "1") -> None: + os.makedirs(repo_path, exist_ok=True) + init_git_repo(repo_path, module_name=module_name) + (Path(repo_path) / f"frid_{frid}.txt").write_text(f"frid {frid}\n") + add_all_files_and_commit( + repo_path, + FUNCTIONAL_REQUIREMENT_FINISHED_COMMIT_MESSAGE.format(frid), + module_name=module_name, + frid=frid, + ) + + +def _archive_module(module: PlainModule) -> None: + """Zip the module's folder contents (code/, tests/ incl. .git) into .module, then + remove the folder, so the module exists only as an archive.""" + with zipfile.ZipFile(module.module_archive_path, "w", zipfile.ZIP_DEFLATED) as zf: + for root, _dirs, files in os.walk(module.module_folder): + for name in files: + full = os.path.join(root, name) + zf.write(full, os.path.relpath(full, module.module_folder)) + shutil.rmtree(module.module_folder) + + # -------------------------------------------------------------------------- # PrepareRepositories — fresh render # -------------------------------------------------------------------------- @@ -117,3 +145,133 @@ def test_cross_module_copy_lands_in_hidden_folder_under_tests(tmp_build_folder): expected = os.path.join(tmp_build_folder, "top_module", "tests", ".required_module", "1_frid_feature") assert new_folder == expected assert source_folder == expected + + +# -------------------------------------------------------------------------- +# Zipped modules — consuming a required module shipped as ".module" +# -------------------------------------------------------------------------- + + +@pytest.fixture +def root_module(get_test_data_path, tmp_build_folder): + """pr_root -> pr_middle -> pr_leaf; root.required_modules[-1] == pr_middle.""" + return PlainModule("pr_root.plain", tmp_build_folder, [get_test_data_path("data/partial_rendering")]) + + +def test_full_render_clones_from_archived_required_module(root_module): + """A required module shipped only as a ".module" archive is materialized and used as + the clone starting point, and its code hash is recorded correctly in the module metadata.""" + previous = root_module.required_modules[-1] + _init_repo_with_finished_frid(previous.module_build_folder, previous.module_name) + _init_repo_with_finished_frid(previous.module_conformance_tests_folder, previous.module_name) + _archive_module(previous) + assert previous.is_archived_only() + + render_context = _make_render_context(root_module, render_conformance_tests=False) + PrepareRepositories().execute(render_context, None) + + # The root code repo was cloned (from the materialized archive), and the required module was + # materialized rather than left as an unusable archive. + assert os.path.isdir(os.path.join(root_module.module_build_folder, ".git")) + assert previous._resolved_module_folder is not None + assert not os.path.exists(previous._default_module_folder) # archive not unpacked in place + assert os.path.isfile(previous.module_archive_path) # archive preserved + + # seed_module_metadata recorded the required module's real code hash (ordering guard). + metadata = root_module.load_module_metadata() + assert metadata["required_modules_code_hash"] == previous.get_module_code_hash() + + previous.cleanup_scratch() + + +def test_archive_missing_conformance_tests_detects_code_only_required_module(root_module): + """A required module shipped as a code-only .module (no tests/) is flagged only when conformance + testing is enabled.""" + previous = root_module.required_modules[-1] + _init_repo_with_finished_frid(previous.module_build_folder, previous.module_name) + _archive_module(previous) # code/ only, no tests/ + + assert archive_missing_conformance_tests(root_module, True) is previous + assert archive_missing_conformance_tests(root_module, False) is None + + +def test_archive_with_tests_not_flagged(root_module): + previous = root_module.required_modules[-1] + _init_repo_with_finished_frid(previous.module_build_folder, previous.module_name) + _init_repo_with_finished_frid(previous.module_conformance_tests_folder, previous.module_name) + _archive_module(previous) # code/ + tests/ + + assert archive_missing_conformance_tests(root_module, True) is None + + +def test_render_state_flags_missing_tests_and_offers_rerender(root_module): + """A code-only archive (conformance on) surfaces a 'missing_conformance_tests' render state with + a rerender choice and a quit choice.""" + previous = root_module.required_modules[-1] + _init_repo_with_finished_frid(previous.module_build_folder, previous.module_name) + _archive_module(previous) # code/ only + + state = get_plain_module_render_state(root_module, render_conformance_tests=True) + assert state is not None + assert state.change_type == "missing_conformance_tests" + assert state.change.module_name == previous.module_name + + choice_types = {c.choice_type for c in get_render_choices(root_module, state).values()} + assert "quit" in choice_types + assert "rerender_affected" in choice_types + + +def test_conformance_tests_resolver_overrides_default(tmp_build_folder): + """A resolver maps an archived required module's name to its resolved (scratch) tests folder; + unknown names fall back to the default base-folder path.""" + resolved = {"req": "/scratch/req/tests"} + conformance_tests = ConformanceTests( + tmp_build_folder, + CONFORMANCE_TESTS_DEFINITION_FILE_NAME, + resolve_module_tests_folder=lambda name: resolved.get(name), + ) + assert conformance_tests.get_module_conformance_tests_folder("req") == "/scratch/req/tests" + assert conformance_tests.get_module_conformance_tests_folder("other") == os.path.join( + tmp_build_folder, "other", "tests" + ) + + +def test_conformance_tests_json_stored_relative_read_absolute(tmp_build_folder): + """folder_name is stored relative to the module's tests folder on disk, but presented as an + absolute path in memory — and the input dict is not mutated.""" + conformance_tests = ConformanceTests(tmp_build_folder, CONFORMANCE_TESTS_DEFINITION_FILE_NAME) + tests_folder = conformance_tests.get_module_conformance_tests_folder("m") + os.makedirs(tests_folder) + + absolute_folder = os.path.join(tests_folder, "1_feature") + json_in = {"1": {"folder_name": absolute_folder, "functional_requirement": "do a thing"}} + conformance_tests.dump_conformance_tests_json("m", json_in) + + # On disk: relative. + with open(os.path.join(tests_folder, CONFORMANCE_TESTS_DEFINITION_FILE_NAME)) as f: + raw = json.load(f) + assert raw["1"]["folder_name"] == "1_feature" + # Input dict not mutated. + assert json_in["1"]["folder_name"] == absolute_folder + + # On read: absolute again. + loaded = conformance_tests.get_conformance_tests_json("m") + assert loaded["1"]["folder_name"] == absolute_folder + assert loaded["1"]["functional_requirement"] == "do a thing" + + +def test_conformance_tests_json_resolves_against_scratch_base(tmp_build_folder): + """A conformance_tests.json shipped in an archive (relative folder_name) resolves against the + module's scratch tests folder when the module is materialized.""" + scratch_tests = os.path.join(tmp_build_folder, "scratch_extract", "tests") + os.makedirs(scratch_tests) + with open(os.path.join(scratch_tests, CONFORMANCE_TESTS_DEFINITION_FILE_NAME), "w") as f: + json.dump({"1": {"folder_name": "1_feature", "functional_requirement": "x"}}, f) + + conformance_tests = ConformanceTests( + tmp_build_folder, + CONFORMANCE_TESTS_DEFINITION_FILE_NAME, + resolve_module_tests_folder=lambda name: scratch_tests if name == "m" else None, + ) + loaded = conformance_tests.get_conformance_tests_json("m") + assert loaded["1"]["folder_name"] == os.path.join(scratch_tests, "1_feature") diff --git a/tui/plain_module_render_choice_tui.py b/tui/plain_module_render_choice_tui.py index 4e7f986b..0325672b 100644 --- a/tui/plain_module_render_choice_tui.py +++ b/tui/plain_module_render_choice_tui.py @@ -118,21 +118,38 @@ def on_mount(self) -> None: info_panel.mount(change_box) if pr.change: - title_start = "Spec changes" if pr.change_type == "spec_change" else "Code changes" is_required_module = pr.change.module_name != self.plain_module.module_name - change_box.mount( - Label( - f"--- {title_start} detected in {'required ' if is_required_module else 'current '}module [#5593FF]{pr.change.module_name}[/] ---", - classes="rendering-info-row", + module_kind = "required " if is_required_module else "current " + if pr.change_type == "missing_conformance_tests": + change_box.mount( + Label( + f"--- Conformance tests missing from the archive of {module_kind}module " + f"[#5593FF]{pr.change.module_name}[/] ---", + classes="rendering-info-row", + ) ) - ) - if is_required_module: change_box.mount( Label( - f"{title_start} in a required module may affect the current module", + "The .module archive has no conformance tests but testing is enabled; " + "the module must be re-rendered (its .module file will be removed).", classes="rendering-info-title", ) ) + else: + title_start = "Spec changes" if pr.change_type == "spec_change" else "Code changes" + change_box.mount( + Label( + f"--- {title_start} detected in {module_kind}module [#5593FF]{pr.change.module_name}[/] ---", + classes="rendering-info-row", + ) + ) + if is_required_module: + change_box.mount( + Label( + f"{title_start} in a required module may affect the current module", + classes="rendering-info-title", + ) + ) elif pr.last_render_module.is_module_fully_rendered(): change_box.mount(Label("The current module is fully rendered.", classes="rendering-info-title"))