From 502c9dae87f64ad9ca05034b7a3a8f8406e0dccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 11:16:51 +0300 Subject: [PATCH 01/78] Add conditional SPSS transformation profile --- CHANGELOG.md | 14 + README.md | 25 +- VERSIONING.md | 9 + conformance/in-place-transformation-0.2.json | 188 +++++++++ conformance/spss-syntax-frontend-0.2.json | 93 +++++ conformance/transformation-plan-0.2.json | 316 +++++++++++++++ docs/spss-syntax-frontend-profile-0.2.md | 103 +++++ docs/transformation-plan-profile-0.2.md | 120 ++++++ docs/transformation-plan-sql-binding-0.2.md | 76 ++++ examples/spss-syntax-transformation-0.2.md | 30 ++ requirements-validation.txt | 1 + tools/validate_transformation_plan.py | 3 + tools/validate_transformation_plan_0_2.py | 365 ++++++++++++++++++ transformation/plan-0.2.schema.json | 139 +++++++ .../spss-syntax-frontend-0.2.schema.json | 57 +++ 15 files changed, 1528 insertions(+), 11 deletions(-) create mode 100644 conformance/in-place-transformation-0.2.json create mode 100644 conformance/spss-syntax-frontend-0.2.json create mode 100644 conformance/transformation-plan-0.2.json create mode 100644 docs/spss-syntax-frontend-profile-0.2.md create mode 100644 docs/transformation-plan-profile-0.2.md create mode 100644 docs/transformation-plan-sql-binding-0.2.md create mode 100644 examples/spss-syntax-transformation-0.2.md create mode 100644 requirements-validation.txt create mode 100755 tools/validate_transformation_plan_0_2.py create mode 100644 transformation/plan-0.2.schema.json create mode 100644 transformation/spss-syntax-frontend-0.2.schema.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e72d6d..6dc7b96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog + +## v0.3.0 - 2026-07-31 + +- Added backward-compatible Transformation Plan and SPSS Syntax Frontend 0.2 + contracts for sequential numeric `COMPUTE` and conditional `IF`, bounded + comparison/`AND`/`OR` expressions, `FORMATS`, `VARIABLE LEVEL`, and + `EXECUTE`. +- Defined exact SQL three-valued missing semantics and deterministic complete + plan JSON hashing while preserving every 0.1 contract, fixture, and hash. +- Kept create-target apply fail-closed on MySQL, MariaDB, and Dolt; these + profiles require a separately versioned physical-and-catalog provisioning + action followed by an existing-target apply with no automatic Dolt commit. +- Added independent synthetic golden fixtures, hashes, examples, and repository + validation gates for the new contracts. ## v0.2.0 - 2026-07-31 - Added canonical Transformation Plan and SPSS Syntax Frontend profiles for a diff --git a/README.md b/README.md index c6202a1..853bcb4 100644 --- a/README.md +++ b/README.md @@ -37,15 +37,18 @@ permits a transformation to mutate a core import. That separate derived-data profile is not used by the SPSS-like in-place frontend below. -The [Transformation Plan Profile 0.1](docs/transformation-plan-profile-0.1.md) -adds a canonical, language-neutral operation plan. The -[SPSS Syntax Frontend Profile 0.1](docs/spss-syntax-frontend-profile-0.1.md) -lowers a deliberately small `RECODE`, `VARIABLE LABELS`, and `VALUE LABELS` -subset into that plan. The -[in-place binding](docs/transformation-plan-sql-binding-0.1.md) applies the plan +The [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) +adds sequential bounded numeric assignment and conditional assignment to the +unchanged 0.1 operations. The +[SPSS Syntax Frontend Profile 0.2](docs/spss-syntax-frontend-profile-0.2.md) +lowers `COMPUTE`, `IF`, `FORMATS`, `VARIABLE LEVEL`, and `EXECUTE` +alongside the 0.1 `RECODE`, `VARIABLE LABELS`, and `VALUE LABELS` subset. +Programs using only the 0.1 subset retain exact 0.1 plan identity and hash. The +[in-place binding](docs/transformation-plan-sql-binding-0.2.md) applies the plan to the same dataset and same physical wide table on supported SQL profiles. It creates no derived dataset, data copy, or OpenStatSpec undo layer; Dolt-specific -history and commits remain Dolt's. +history and commits remain Dolt's. MySQL, MariaDB, and Dolt require a new target +to be provisioned separately before an in-place transformation apply. ## Repository layout @@ -62,11 +65,11 @@ history and commits remain Dolt's. - `transformation/plan-0.1.schema.json` — canonical transformation-plan schema. - `sql/transformation-plan-profile-schema.sql` — compact in-place apply audit; it is not a dataset-version catalog. -- `conformance/transformation-plan-0.1.json` and - `conformance/spss-syntax-frontend-0.1.json` — plan and frontend conformance - cases with canonical hashes; `conformance/in-place-transformation-0.1.json` +- `conformance/transformation-plan-0.2.json` and + `conformance/spss-syntax-frontend-0.2.json` — additive plan and frontend + conformance cases with independent golden hashes; `conformance/in-place-transformation-0.2.json` fixes the same-dataset/same-table execution invariants and the additional - controlled Dolt context. + controlled Dolt context. The 0.1 schemas and fixtures remain unchanged. - `examples/` — small illustrative mapping fixtures. ## Conformance principle diff --git a/VERSIONING.md b/VERSIONING.md index 5d946e4..938359b 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -11,6 +11,15 @@ core source profile. Adding or revising an optional profile does not authorize changes to core datasets. A breaking workflow-profile change increments that profile's major version even when the core version is unchanged. +Transformation Plan and frontend contracts are selected by their exact +contract identifiers. An implementation may support multiple versions +simultaneously. It MUST preserve the canonical bytes and hashes of accepted +older plans and MUST NOT silently reinterpret or rewrite an older plan under a +newer schema. A frontend that accepts a newer request contract but receives a +program entirely inside the 0.1 command subset emits the exact 0.1 plan +contract. Explicit recompilation to a newer plan is a new identified operation, +not migration of the old plan or audit row. + The SPSS SAV/ZSAV profile is versioned independently in its document and manifest. A conforming implementation must publish the exact profile version, immutable specification commit, specification status, supported directions, supported SQL profiles, and engine capability declaration it tested. The immutable commit is mandatory for both release candidates and stable releases. `specification_release` is additive provenance, not an alternative to the commit: diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json new file mode 100644 index 0000000..d4cf877 --- /dev/null +++ b/conformance/in-place-transformation-0.2.json @@ -0,0 +1,188 @@ +{ + "manifest_version": "0.2", + "profile": "OpenStatSpec In-Place Transformation Binding 0.2", + "contract": "openstatspec-in-place-transformation-v0.2", + "cases": [ + { + "id": "dolt-preprovisioned-target-sequential-null-semantics", + "database_profile": "dolt", + "before": { + "dataset_id": "11111111-1111-4111-8111-111111111111", + "physical_table_name": "data_synthetic", + "physical_table_schema": null, + "dataset_count": 1, + "persistent_data_table_count": 1, + "case_count": 4, + "target_provisioning": { + "physical_column": true, + "catalog_variable": true, + "dolt_commit": "provisioning-commit" + }, + "dolt_branch": "feature/recode", + "dolt_head": "provisioning-commit", + "working_set_clean": true, + "rows": [ + { + "__case_ordinal": 1, + "source_a": 1, + "source_b": 1, + "target": null + }, + { + "__case_ordinal": 2, + "source_a": 1, + "source_b": null, + "target": null + }, + { + "__case_ordinal": 3, + "source_a": null, + "source_b": 1, + "target": null + }, + { + "__case_ordinal": 4, + "source_a": 0, + "source_b": 1, + "target": null + } + ] + }, + "after": { + "dataset_id": "11111111-1111-4111-8111-111111111111", + "physical_table_name": "data_synthetic", + "dataset_count": 1, + "persistent_data_table_count": 1, + "physical_table_schema": null, + "dolt_branch": "feature/recode", + "dolt_head": "provisioning-commit", + "case_count": 4, + "dolt_commit_performed": false, + "working_set_clean": false, + "rows": [ + { + "__case_ordinal": 1, + "target": 1 + }, + { + "__case_ordinal": 2, + "target": 0 + }, + { + "__case_ordinal": 3, + "target": 0 + }, + { + "__case_ordinal": 4, + "target": 0 + } + ], + "target_metadata": { + "variable_label": "Synthetic conjunction", + "value_labels": [ + [ + 0, + "No" + ], + [ + 1, + "Yes" + ] + ], + "format": "F1.0", + "measurement_level": "nominal" + } + }, + "expected_audit": { + "contract_id": "openstatspec-in-place-transformation-v0.2", + "operation_count": 7, + "dolt_branch": "feature/recode", + "dolt_head_before": "provisioning-commit", + "dolt_head_after": "provisioning-commit" + }, + "forbidden_artifacts": [ + "derived_dataset_row", + "persistent_output_table", + "full_table_copy", + "staging_dataset", + "staging_relation", + "snapshot_table", + "rollback_table", + "dataset_version_row", + "retirement_or_recovery_row", + "temporary_object_residue" + ], + "required_audit_fields": [ + "apply_id", + "contract_id", + "database_profile", + "dataset_id", + "physical_table_schema", + "physical_table_name", + "plan_hash", + "source_hash", + "actor", + "status", + "dolt_branch", + "dolt_head_before", + "dolt_head_after", + "operation_count", + "started_at", + "completed_at" + ], + "expected_error": null + }, + { + "id": "dolt-preprovisioned-target-or-null-semantics", + "database_profile": "dolt", + "target_preprovisioned": true, + "condition": "source_a = 1 OR source_b = 1", + "before_rows": [ + { + "__case_ordinal": 1, + "source_a": null, + "source_b": 1, + "target": 0 + }, + { + "__case_ordinal": 2, + "source_a": null, + "source_b": 0, + "target": 0 + } + ], + "after_rows": [ + { + "__case_ordinal": 1, + "target": 1 + }, + { + "__case_ordinal": 2, + "target": 0 + } + ], + "expected_error": null + }, + { + "id": "dolt-create-target-fails-before-mutation", + "database_profile": "dolt", + "target_mode": "create", + "expected_error": "schema_change_not_atomic", + "mutation_started": false + }, + { + "id": "mysql-create-target-fails-before-mutation", + "database_profile": "mysql", + "target_mode": "create", + "expected_error": "schema_change_not_atomic", + "mutation_started": false + }, + { + "id": "mariadb-create-target-fails-before-mutation", + "database_profile": "mariadb", + "target_mode": "create", + "expected_error": "schema_change_not_atomic", + "mutation_started": false + } + ] +} diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json new file mode 100644 index 0000000..4cdd71d --- /dev/null +++ b/conformance/spss-syntax-frontend-0.2.json @@ -0,0 +1,93 @@ +{ + "manifest_version": "0.2", + "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.2", + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "plan_contracts": [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2" + ], + "request_schema": "../transformation/spss-syntax-frontend-0.2.schema.json", + "plan_schemas": { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json" + }, + "cases": [ + { + "id": "compute-if-labels-format-level-execute-existing-target", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = 0.\nIF (source_a = 1 AND source_b = 1) target = 1.\nVARIABLE LABELS target 'Synthetic conjunction'.\nVALUE LABELS target 0 'No' 1 'Yes'.\nFORMATS target (F1.0).\nVARIABLE LEVEL target (NOMINAL).\nEXECUTE." + }, + "expected_plan_case": "sequential-conditional-binary-existing-target", + "expected_plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", + "expected_source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481", + "expected_error": null + }, + { + "id": "old-subset-retains-v0.1-plan", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "q1", + "storage_kind": "numeric" + } + ] + }, + "source_text": "RECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "68c05b0cae01476642c70956ddcaa693311be1a653ead1d1caf8b006843ed55b", + "expected_error": null + }, + { + "id": "nested-or-inequality-variable-operands-create", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = source_a.\nIF (source_a >= 0 OR (source_b < 5 AND source_c <= 9)) target = source_b.\nEXECUTE." + }, + "expected_plan_case": "nested-or-inequalities-variable-operands-create", + "expected_plan_hash": "926f2cd93ed0bfa330bdb2d0b31b241f552c86c116dd35a315a20408bf997daf", + "expected_source_hash": "4b3b35660b1ec57de0b8b0485dcc85bb51a05796df9e9624f4f1790d78e2bfeb", + "expected_error": null + } + ] +} diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json new file mode 100644 index 0000000..a8df9ab --- /dev/null +++ b/conformance/transformation-plan-0.2.json @@ -0,0 +1,316 @@ +{ + "manifest_version": "0.2", + "profile": "OpenStatSpec Transformation Plan 0.2", + "contract": "openstatspec-transformation-plan-v0.2", + "schema": "../transformation/plan-0.2.schema.json", + "canonicalization": "restricted-rfc8785-utf8-sha256", + "cases": [ + { + "id": "sequential-conditional-binary-existing-target", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "0000000000000000" + } + } + }, + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "set_variable_label", + "variable": "target", + "label": "Synthetic conjunction" + }, + { + "op": "replace_value_labels", + "variable": "target", + "labels": [ + { + "value": { + "type": "binary64", + "bits": "0000000000000000" + }, + "label": "No" + }, + { + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + }, + "label": "Yes" + } + ] + }, + { + "op": "set_format", + "variable": "target", + "family": "F", + "width": 1, + "decimals": 0 + }, + { + "op": "set_measurement_level", + "variable": "target", + "level": "nominal" + }, + { + "op": "execute" + } + ] + }, + "expected_plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", + "expected_error": null + }, + { + "id": "reject-string-predicate", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "color" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "string", + "value": "R" + } + } + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "expected_plan_hash": null, + "expected_error": "expression_type_unsupported" + }, + { + "id": "nested-or-inequalities-variable-operands-create", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "target", + "target_mode": "create", + "value": { + "kind": "variable", + "variable": "source_a" + } + }, + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "or", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": ">=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "0000000000000000" + } + } + }, + { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "<", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "4014000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_c" + }, + "operator": "<=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "4022000000000000" + } + } + } + ] + } + ] + }, + "target": "target", + "value": { + "kind": "variable", + "variable": "source_b" + } + }, + { + "op": "execute" + } + ] + }, + "expected_plan_hash": "926f2cd93ed0bfa330bdb2d0b31b241f552c86c116dd35a315a20408bf997daf", + "expected_error": null + }, + { + "id": "reject-invalid-format", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "set_format", + "variable": "target", + "family": "F", + "width": 2, + "decimals": 2 + } + ] + }, + "expected_plan_hash": null, + "expected_error": "invalid_format" + }, + { + "id": "reject-reserved-assignment-target", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "__target", + "target_mode": "create", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "0000000000000000" + } + } + } + ] + }, + "expected_plan_hash": null, + "expected_error": "reserved_target_name" + }, + { + "id": "reject-duplicate-value-label", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "replace_value_labels", + "variable": "target", + "labels": [ + { + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + }, + "label": "One" + }, + { + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + }, + "label": "Duplicate one" + } + ] + } + ] + }, + "expected_plan_hash": null, + "expected_error": "duplicate_value_label" + } + ] +} diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md new file mode 100644 index 0000000..f35f935 --- /dev/null +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -0,0 +1,103 @@ +# OpenStatSpec SPSS-like Syntax Frontend Profile 0.2 + +## Status and boundary + +This optional frontend maps a bounded SPSS-like command language to +Transformation Plan 0.1 or 0.2. Its request contract is +`openstatspec-spss-syntax-frontend-v0.2`. It is not full IBM SPSS Statistics +syntax and MUST reject every command, expression, or coercion not defined here. + +All lexical rules, command termination, exact source hashing, case-insensitive +name resolution, quote handling, and failure atomicity from Frontend 0.1 remain +in force. The input request conforms to +[`../transformation/spss-syntax-frontend-0.2.schema.json`](../transformation/spss-syntax-frontend-0.2.schema.json). + +The 0.1 commands `RECODE`, `VARIABLE LABELS`, and `VALUE LABELS` are +unchanged. A program containing only that subset MUST emit an exact Plan 0.1 +object. A program containing any command introduced below emits Plan 0.2. + +## Numeric expression grammar + +The bounded grammar is: + +```text +operand := numeric-variable | finite-number +comparison := operand ("=" | "<" | "<=" | ">" | ">=") operand +predicate := comparison | "(" predicate ")" + | predicate AND predicate + | predicate OR predicate +``` + +Comparison binds tighter than `AND`; `AND` binds tighter than `OR`. +Parentheses override precedence and are preserved as the equivalent nested +plan shape. Keywords are ASCII case-insensitive. Unary operators, arithmetic, +functions, strings, `NOT`, `~=`, `<>`, `MISSING`, locale-dependent +numbers, and implicit numeric/string coercion are unsupported. + +A numeric variable is resolved at the point its command is lowered, including +variables created by preceding commands. All referenced variables and assigned +values in this subset MUST be numeric. + +## Added commands + +`COMPUTE target = operand.` lowers to one ordered `assign`. If `target` +does not yet exist, it uses `target_mode=create`; otherwise it uses +`target_mode=replace`. No expression beyond the bounded operand form is +accepted in 0.2. + +`IF (predicate) target = operand.` lowers to one `conditional_assign`. +Parentheses around the complete predicate are required. The target MUST exist +at that point in command order. There is no ELSE form: a row is changed only +when the predicate is TRUE. FALSE and UNKNOWN retain the previous target value. + +`FORMATS variable (Fwidth.decimals) [variable +(Fwidth.decimals) ...].` lowers each pair in source order to `set_format`. +Only the numeric `F` family is supported. Family is canonicalized to uppercase; +width and decimals are canonical decimal integers. + +`VARIABLE LEVEL variable-list (NOMINAL|ORDINAL|SCALE) +[/ variable-list (level) ...].` lowers one `set_measurement_level` operation +per resolved variable, preserving group and variable order. Levels are +canonicalized to lowercase. + +`EXECUTE.` lowers to `{"op":"execute"}`. It marks source intent and ordered +identity but does not perform or authorize a database commit. + +For example: + +```spss +COMPUTE target = 0. +IF (source_a = 1 AND source_b = 1) target = 1. +VARIABLE LABELS target 'Example label'. +VALUE LABELS target 0 'No' 1 'Yes'. +FORMATS target (F1.0). +VARIABLE LEVEL target (NOMINAL). +EXECUTE. +``` + +lowers to sequential initialization, conditional replacement, label, value +labels, format, measurement level, and execution-boundary operations. No +command may be dropped. + +## System missing + +A variable operand may evaluate to numeric system missing. `COMPUTE` copies +that missing value. Comparisons involving missing are UNKNOWN, boolean +predicates use SQL three-valued truth, and `IF` writes only for TRUE. The +frontend MUST NOT silently turn missing into zero or FALSE. + +## Failure contract + +The frontend returns no partial plan on error. In addition to 0.1 diagnostics, +stable diagnostics include `expression_type_unsupported`, +`conditional_target_missing`, and `invalid_format`. Unsupported commands or +expression productions return `unsupported_spss_command` or +`spss_syntax_error` as appropriate. Diagnostics identify source spans without +including credentials or unrelated row values. + +The source hash remains SHA-256 of UTF-8 source after CRLF and CR are converted +to LF. The complete emitted plan is hashed independently under its declared +plan contract. + +Machine-readable golden cases are in +[`../conformance/spss-syntax-frontend-0.2.json`](../conformance/spss-syntax-frontend-0.2.json). diff --git a/docs/transformation-plan-profile-0.2.md b/docs/transformation-plan-profile-0.2.md new file mode 100644 index 0000000..af64c7f --- /dev/null +++ b/docs/transformation-plan-profile-0.2.md @@ -0,0 +1,120 @@ +# OpenStatSpec Transformation Plan Profile 0.2 + +## Status and scope + +This optional profile is the backward-compatible successor to Transformation +Plan 0.1. Its contract identifier is +`openstatspec-transformation-plan-v0.2`. The key words **MUST**, **MUST NOT**, +**SHOULD**, **SHOULD NOT**, and **MAY** are normative. + +Version 0.2 retains every 0.1 operation unchanged and adds bounded numeric +assignment, conditional assignment, display-format, measurement-level, and +execution-boundary operations. It remains a language-neutral plan rather than +an arbitrary expression language, SQL fragment, or execution program. A plan +MUST conform to +[`../transformation/plan-0.2.schema.json`](../transformation/plan-0.2.schema.json). + +## Canonical identity and backward compatibility + +A 0.2 plan contains exactly `contract`, `input_alias`, and a non-empty ordered +`operations` array. Canonical JSON and `plan_hash` use the same restricted RFC +8785 and SHA-256 rules as Plan 0.1. The hash covers the complete exact plan +object, including every expression node and metadata operation. Object member +order is immaterial; array order, expression shape, typed binary64 bits, and +exact unnormalized Unicode strings are material. + +The 0.1 schemas, contract identifiers, fixtures, canonical bytes, and hashes are +immutable. A consumer MUST validate a plan against the schema named by its +contract and MUST NOT silently reinterpret, normalize, or upgrade a 0.1 plan as +0.2. An implementation that migrates source text recompiles it explicitly, +produces a separately identified plan, and preserves the prior plan and audit. +A 0.2-capable SPSS frontend emits a 0.1 plan for a program containing only the +0.1 command subset; this preserves old plan hashes. + +Typed values, finite binary64 requirements, positive-zero canonicalization, +exact strings, reserved identifiers, and existing `recode`, +`set_variable_label`, and `replace_value_labels` semantics are unchanged +from Plan 0.1. + +## Ordered operations + +Operations execute strictly in array order. Each operation observes data, +schema, and metadata resulting from all preceding operations in the same plan. +This sequential rule is material: an `assign` may initialize a variable and a +following `conditional_assign` may selectively replace its initialized value. + +### Numeric operands and predicates + +Version 0.2 operands are either: + +- `{"kind":"variable","variable":"name"}`, resolved to one numeric variable; or +- `{"kind":"literal","value":typed-value}`, where the bounded assignment and + predicate subset requires a finite numeric binary64 value. + +String operands and string predicates are outside 0.2. They fail before +mutation with `expression_type_unsupported`. This avoids implicit SQL +collation or coercion semantics. + +A comparison node has `expression=comparison`, two operands, and one of +`=`, `<`, `<=`, `>`, or `>=`. A boolean node has +`expression=boolean`, `operator=and|or`, and at least two ordered predicate +operands. Nested boolean nodes preserve parentheses and associativity from the +frontend. Implementations MUST NOT simplify, reorder, deduplicate, distribute, +or constant-fold nodes before canonical hashing. + +Predicates use SQL three-valued truth semantics. A comparison with numeric +system missing (SQL `NULL`) is UNKNOWN. `AND` and `OR` use their standard +three-valued truth tables. A conditional assignment writes only where its +predicate is TRUE; FALSE and UNKNOWN both retain the current target value. + +### assign + +`assign` contains `target`, `target_mode=create|replace`, and one numeric +operand. It evaluates that operand against the pre-operation row and writes the +result to every case. A variable operand that is system missing writes system +missing. `create` appends a fresh numeric variable; `replace` requires an +existing numeric target and retains identity, ordinal, and metadata. + +### conditional_assign + +`conditional_assign` contains a predicate, an existing numeric `target`, and +one numeric operand. It evaluates the predicate and value against the +pre-operation row, writes only when the predicate is TRUE, and otherwise leaves +the target unchanged. The target MUST already exist when this operation binds. +It has no implicit ELSE branch and never creates a variable. + +### metadata and execute + +`set_format` replaces only the numeric variable's print/write format with +family `F`, width 1 through 40, and decimals 0 through 16. Width and decimals +MUST form a valid SPSS F format; when decimals are nonzero, width MUST provide +space for a sign, decimal separator, and the requested decimals. Invalid forms +fail with `invalid_format`. + +`set_measurement_level` replaces only the variable measurement level with +`nominal`, `ordinal`, or `scale`. + +`execute` is an explicit ordered execution boundary retained in the canonical +plan. In one atomic public apply it does not commit, start a second apply, or +publish intermediate state. It performs no row or metadata mutation by itself. + +Newly created variables begin without labels, value labels, missing-value +metadata, or measurement metadata. Later ordered metadata operations establish +those properties. Existing targets retain metadata except for the exact +metadata component explicitly replaced by a later operation. + +## Binding, validation, and diagnostics + +Structural validation, name resolution, type checking, target-state checking, +predicate validation, and backend capability checks all precede mutation. +Stable 0.2 diagnostics add `expression_type_unsupported`, +`conditional_target_missing`, `invalid_format`, and retain the 0.1 +diagnostics. + +The in-place SQL binding is +[`transformation-plan-sql-binding-0.2.md`](transformation-plan-sql-binding-0.2.md). +A fixed plan, bound schema, implementation version, and target profile MUST +compile deterministically. Generated SQL is not canonical identity. + +Machine-readable independent golden cases are in +[`../conformance/transformation-plan-0.2.json`](../conformance/transformation-plan-0.2.json). diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md new file mode 100644 index 0000000..e8a2634 --- /dev/null +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -0,0 +1,76 @@ +# In-Place Transformation Binding 0.2 + +## Normative decision + +This binding extends In-Place Transformation Binding 0.1 for Transformation +Plan 0.2. It preserves one logical dataset, one physical wide table, dataset +identity, physical table identity, case order, and case count. It creates no +derived dataset, persistent copy, snapshot, OpenStatSpec rollback layer, or +automatic database commit. + +Existing-target `assign` compiles to a direct all-row `UPDATE`. +`conditional_assign` compiles to a direct `UPDATE ... WHERE predicate`; +only SQL TRUE rows change. Metadata operations update only their normative +catalog fields. `execute` compiles to no SQL statement. + +## Target creation by SQL profile + +SQLite and PostgreSQL MAY execute `target_mode=create` only where the adapter +has test evidence that schema, data, catalogs, and compact audit are one native +transaction. + +MySQL, MariaDB, and Dolt MUST reject `target_mode=create` before mutation with +`schema_change_not_atomic`. Version 0.2 does not claim atomic create-target +apply on these profiles and MUST NOT use a copied table or compensating +recovery artifact to simulate it. + +A caller that needs a new target on MySQL, MariaDB, or Dolt performs a separate, +explicit, versioned provisioning action before the transformation apply. That +action MUST create both: + +- one nullable numeric physical column in the existing wide table; and +- one matching normative catalog variable with the same physical binding, + unique variable identity, and next ordinal. + +Provisioning is not a transformation apply and MUST NOT be hidden inside one. +It must leave no uncataloged physical column or catalog-only variable. Before a +later apply, the target resolves as an existing numeric variable, so +`COMPUTE target = ...` lowers to `assign target_mode=replace`. The apply then +contains only DML, metadata changes, and compact audit. + +For Dolt, the provisioning action is committed separately by the caller. The +subsequent transformation apply begins from that clean committed HEAD and uses +that exact commit as `expected_head`. OpenStatSpec never calls `DOLT_COMMIT` +for either action. + +## Controlled Dolt context + +Every apply requires a non-empty actor. Dolt additionally requires +`expected_branch`, `expected_head`, and a clean working set. Branch, HEAD, +and cleanliness are checked before mutation. Branch and HEAD are checked again +before successful completion. Stable diagnostics are +`dolt_context_required`, `dolt_branch_mismatch`, `dolt_head_mismatch`, +`dolt_working_set_dirty`, and `dolt_context_changed`. + +The apply remains on the same branch and MUST NOT commit, switch, merge, reset, +tag, or create a persistent recovery object. A successful apply leaves one +inspectable Dolt working-set diff. The audit's before and after HEAD identities +are equal. + +## Atomicity and failure + +All plan operations, normative and compatibility metadata changes, and the one +compact audit row belong to one public apply boundary. For an existing target, +the adapter uses the engine's native transaction semantics. A failure leaves +data, metadata, and audit unchanged. + +Capability and target checks happen before mutation. A profile that cannot +provide the required boundary fails closed. Pre-existing unrelated state is +never treated as apply-owned cleanup state. + +The compact audit and forbidden-artifact requirements from Binding 0.1 are +unchanged. It records exact plan/source hashes and operation count, not row +values or copied state. + +Machine-readable binding cases are in +[`../conformance/in-place-transformation-0.2.json`](../conformance/in-place-transformation-0.2.json). diff --git a/examples/spss-syntax-transformation-0.2.md b/examples/spss-syntax-transformation-0.2.md new file mode 100644 index 0000000..d12bd86 --- /dev/null +++ b/examples/spss-syntax-transformation-0.2.md @@ -0,0 +1,30 @@ +# Conditional SPSS-like transformation on an existing target + +Transformation Plan and SPSS Syntax Frontend 0.2 support bounded sequential +numeric assignment. Assume `target` already exists as a nullable numeric +physical column and normative catalog variable: + +```spss +COMPUTE target = 0. +IF (source_a = 1 AND source_b = 1) target = 1. +VARIABLE LABELS target 'Synthetic conjunction'. +VALUE LABELS target 0 'No' 1 'Yes'. +FORMATS target (F1.0). +VARIABLE LEVEL target (NOMINAL). +EXECUTE. +``` + +`COMPUTE` initializes every case. `IF` then changes only rows whose complete +predicate is TRUE. If either comparison is UNKNOWN because a source is numeric +system missing, the row retains zero. The remaining commands establish the +label, complete value-label map, F1.0 format, and nominal measurement level. + +On SQLite or PostgreSQL an implementation may create `target` in the apply +only when its declared profile proves one native transaction covers schema, +data, catalogs, and audit. MySQL, MariaDB, and Dolt reject create mode with +`schema_change_not_atomic`. + +For Dolt, provision the physical and catalog target together in a separate +caller-owned commit. Begin the transformation from that clean HEAD. The exact +syntax above then binds `COMPUTE` as replacement of the existing target, +leaves an inspectable working-set diff, and does not call `DOLT_COMMIT`. diff --git a/requirements-validation.txt b/requirements-validation.txt new file mode 100644 index 0000000..d9bf673 --- /dev/null +++ b/requirements-validation.txt @@ -0,0 +1 @@ +jsonschema==4.10.3 diff --git a/tools/validate_transformation_plan.py b/tools/validate_transformation_plan.py index 405306b..256508f 100644 --- a/tools/validate_transformation_plan.py +++ b/tools/validate_transformation_plan.py @@ -450,6 +450,9 @@ def validate_all() -> tuple[int, int]: plan_cases = validate_plan_manifest() frontend_count = validate_frontend_manifest(plan_cases) validate_repository_links() + from validate_transformation_plan_0_2 import validate_all as validate_0_2 + + validate_0_2() return len(plan_cases), frontend_count diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py new file mode 100755 index 0000000..6ed4662 --- /dev/null +++ b/tools/validate_transformation_plan_0_2.py @@ -0,0 +1,365 @@ +"""Validate the additive Transformation Plan and SPSS frontend 0.2 fixtures.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from jsonschema import Draft202012Validator + + +from validate_repository import canonical_hash, require, require_string +from validate_transformation_plan import validate_plan as validate_plan_0_1 + + +ROOT = Path(__file__).resolve().parents[1] +PLAN_0_1 = ROOT / "conformance/transformation-plan-0.1.json" +FRONTEND_0_1 = ROOT / "conformance/spss-syntax-frontend-0.1.json" +PLAN_SCHEMA_0_1 = ROOT / "transformation/plan-0.1.schema.json" +PLAN = ROOT / "conformance/transformation-plan-0.2.json" +FRONTEND = ROOT / "conformance/spss-syntax-frontend-0.2.json" +IN_PLACE = ROOT / "conformance/in-place-transformation-0.2.json" +PLAN_SCHEMA = ROOT / "transformation/plan-0.2.schema.json" +FRONTEND_SCHEMA = ROOT / "transformation/spss-syntax-frontend-0.2.schema.json" + + +def typed_value(value: object, context: str) -> str: + require(isinstance(value, dict), f"{context}: typed value must be an object.") + if value.get("type") == "binary64": + require(set(value) == {"type", "bits"}, f"{context}: binary64 fields differ.") + bits = value["bits"] + require( + isinstance(bits, str) + and len(bits) == 16 + and all(character in "0123456789abcdef" for character in bits), + f"{context}: binary64 bits differ.", + ) + exponent = (int(bits, 16) >> 52) & 0x7FF + require(exponent != 0x7FF and bits != "8000000000000000", f"{context}: noncanonical binary64.") + return "numeric" + require( + value.get("type") == "string" + and set(value) == {"type", "value"} + and isinstance(value.get("value"), str), + f"{context}: string value differs.", + ) + return "string" + + +def typed_value_key(value: object, context: str) -> tuple[str, str]: + kind = typed_value(value, context) + return (kind, value["bits"] if kind == "numeric" else value["value"]) + + +def operand(value: object, context: str) -> str: + require(isinstance(value, dict), f"{context}: operand must be an object.") + if value.get("kind") == "variable": + require(set(value) == {"kind", "variable"}, f"{context}: variable operand fields differ.") + require_string(value["variable"], context + ".variable") + return "variable" + require(value.get("kind") == "literal" and set(value) == {"kind", "value"}, f"{context}: operand differs.") + return typed_value(value["value"], context + ".value") + + +def predicate(value: object, context: str) -> bool: + require(isinstance(value, dict), f"{context}: predicate must be an object.") + if value.get("expression") == "comparison": + require(set(value) == {"expression", "left", "operator", "right"}, f"{context}: comparison fields differ.") + require(value["operator"] in {"=", "<", "<=", ">", ">="}, f"{context}: comparison operator differs.") + kinds = {operand(value["left"], context + ".left"), operand(value["right"], context + ".right")} + return "string" in kinds + require(value.get("expression") == "boolean", f"{context}: expression kind differs.") + require(set(value) == {"expression", "operator", "operands"}, f"{context}: boolean fields differ.") + require(value["operator"] in {"and", "or"}, f"{context}: boolean operator differs.") + children = value["operands"] + require(isinstance(children, list) and len(children) >= 2, f"{context}: boolean operands differ.") + return any(predicate(child, f"{context}.operands[{index}]") for index, child in enumerate(children)) + + +def validate_plan(plan: object, context: str) -> str | None: + require(isinstance(plan, dict) and set(plan) == {"contract", "input_alias", "operations"}, f"{context}: plan fields differ.") + require(plan["contract"] == "openstatspec-transformation-plan-v0.2", f"{context}: contract differs.") + require_string(plan["input_alias"], context + ".input_alias") + operations = plan["operations"] + require(isinstance(operations, list) and operations, f"{context}: operations missing.") + semantic_error = None + allowed = { + "assign", "conditional_assign", "set_variable_label", "replace_value_labels", + "set_format", "set_measurement_level", "execute", "recode", + } + for index, operation in enumerate(operations): + op_context = f"{context}.operations[{index}]" + require(isinstance(operation, dict) and operation.get("op") in allowed, f"{op_context}: operation differs.") + op = operation["op"] + if op == "assign": + require(set(operation) == {"op", "target", "target_mode", "value"}, f"{op_context}: assign fields differ.") + target = require_string(operation["target"], op_context + ".target") + require(operation["target_mode"] in {"create", "replace"}, f"{op_context}: target mode differs.") + if target.startswith("__"): + semantic_error = semantic_error or "reserved_target_name" + if operand(operation["value"], op_context + ".value") == "string": + semantic_error = semantic_error or "expression_type_unsupported" + elif op == "conditional_assign": + require(set(operation) == {"op", "condition", "target", "value"}, f"{op_context}: conditional fields differ.") + target = require_string(operation["target"], op_context + ".target") + string_predicate = predicate(operation["condition"], op_context + ".condition") + if target.startswith("__"): + semantic_error = semantic_error or "reserved_target_name" + string_value = operand(operation["value"], op_context + ".value") == "string" + if string_predicate or string_value: + semantic_error = semantic_error or "expression_type_unsupported" + elif op == "set_variable_label": + require(set(operation) == {"op", "variable", "label"} and isinstance(operation["label"], str), f"{op_context}: label fields differ.") + elif op == "replace_value_labels": + require(set(operation) == {"op", "variable", "labels"} and isinstance(operation["labels"], list), f"{op_context}: value labels differ.") + require_string(operation["variable"], op_context + ".variable") + require(operation["labels"], f"{op_context}: labels missing.") + seen: set[tuple[str, str]] = set() + for label_index, label in enumerate(operation["labels"]): + require(set(label) == {"value", "label"} and isinstance(label["label"], str), f"{op_context}.labels[{label_index}]: fields differ.") + key = typed_value_key(label["value"], f"{op_context}.labels[{label_index}].value") + if key in seen: + semantic_error = semantic_error or "duplicate_value_label" + seen.add(key) + elif op == "set_format": + require(set(operation) == {"op", "variable", "family", "width", "decimals"}, f"{op_context}: format fields differ.") + require(operation["family"] == "F", f"{op_context}: format family differs.") + width, decimals = operation["width"], operation["decimals"] + require(isinstance(width, int) and 1 <= width <= 40, f"{op_context}: width differs.") + require(isinstance(decimals, int) and 0 <= decimals <= 16, f"{op_context}: decimals differ.") + if decimals and width < decimals + 2: + semantic_error = semantic_error or "invalid_format" + elif op == "set_measurement_level": + require(set(operation) == {"op", "variable", "level"}, f"{op_context}: level fields differ.") + require(operation["level"] in {"nominal", "ordinal", "scale"}, f"{op_context}: level differs.") + elif op == "execute": + require(set(operation) == {"op"}, f"{op_context}: execute fields differ.") + else: + require(op == "recode", f"{op_context}: unknown operation.") + legacy_error = validate_plan_0_1( + {"contract": "openstatspec-transformation-plan-v0.1", + "input_alias": plan["input_alias"], + "operations": [operation]}, + op_context, + ) + semantic_error = semantic_error or legacy_error + return semantic_error + + +def validate_plan_manifest() -> dict[str, dict[str, object]]: + manifest = json.loads(PLAN.read_text(encoding="utf-8")) + require(set(manifest) == {"manifest_version", "profile", "contract", "schema", "canonicalization", "cases"}, "0.2 plan manifest fields differ.") + require(manifest["manifest_version"] == "0.2", "0.2 plan manifest version differs.") + require(manifest["contract"] == "openstatspec-transformation-plan-v0.2", "0.2 plan contract differs.") + require(manifest["canonicalization"] == "restricted-rfc8785-utf8-sha256", "0.2 canonicalization differs.") + cases = {} + expected = { + "sequential-conditional-binary-existing-target": None, + "reject-string-predicate": "expression_type_unsupported", + "nested-or-inequalities-variable-operands-create": None, + "reject-invalid-format": "invalid_format", + "reject-reserved-assignment-target": "reserved_target_name", + "reject-duplicate-value-label": "duplicate_value_label", + } + for case in manifest["cases"]: + require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") + identifier = require_string(case["id"], "0.2 plan case id") + require(identifier not in cases, f"Duplicate 0.2 plan case: {identifier}") + cases[identifier] = case + error = validate_plan(case["plan"], identifier) + require(error == case["expected_error"] == expected.get(identifier), f"{identifier}: semantic result differs.") + if error is None: + require(case["expected_plan_hash"] == canonical_hash(case["plan"]), f"{identifier}: golden plan hash differs.") + else: + require(case["expected_plan_hash"] is None, f"{identifier}: invalid plan claims a hash.") + require(set(cases) == set(expected), "0.2 plan case set differs.") + return cases + + +def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: + manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) + require(set(manifest) == {"manifest_version", "profile", "contract", "plan_contracts", "request_schema", "plan_schemas", "cases"}, "0.2 frontend manifest fields differ.") + require(manifest["contract"] == "openstatspec-spss-syntax-frontend-v0.2", "0.2 frontend contract differs.") + require(manifest["plan_contracts"] == [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2", + ], "0.2 frontend plan contracts differ.") + require(manifest["plan_schemas"] == { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json", + }, "0.2 frontend plan-schema mapping differs.") + legacy_manifest = json.loads(FRONTEND_0_1.read_text(encoding="utf-8")) + legacy_cases = {case["id"]: case for case in legacy_manifest["cases"]} + identifiers = set() + for case in manifest["cases"]: + identifier = require_string(case["id"], "0.2 frontend case id") + require(identifier not in identifiers, f"Duplicate 0.2 frontend case: {identifier}") + identifiers.add(identifier) + request = case["request"] + require(request["contract"] == manifest["contract"], f"{identifier}: request contract differs.") + source = request["source_text"].replace("\r\n", "\n").replace("\r", "\n") + require(source == request["source_text"], f"{identifier}: source is not normalized.") + require(case["expected_source_hash"] == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: source hash differs.") + if "expected_plan_case" in case: + require(set(case) == {"id", "request", "expected_plan_case", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: frontend success fields differ.") + require(case["expected_plan_case"] in plan_cases, f"{identifier}: plan fixture missing.") + require(case["expected_plan_hash"] == plan_cases[case["expected_plan_case"]]["expected_plan_hash"], f"{identifier}: plan hash link differs.") + required_tokens = ( + ("COMPUTE ", "IF (", " AND ", "FORMATS ", "VARIABLE LEVEL ", "EXECUTE.") + if identifier == "compute-if-labels-format-level-execute-existing-target" + else ("COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.") + ) + for token in required_tokens: + require(token in source, f"{identifier}: bounded command coverage missing: {token}") + else: + require(identifier == "old-subset-retains-v0.1-plan", f"Unexpected 0.2 frontend case: {identifier}") + require(set(case) == {"id", "request", "expected_plan_contract", "expected_plan_case_0_1", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: 0.1 compatibility fields differ.") + legacy_case = legacy_cases.get(case["expected_plan_case_0_1"]) + require(legacy_case is not None and "expected_plan" in legacy_case, f"{identifier}: immutable 0.1 plan fixture missing.") + legacy_plan = legacy_case["expected_plan"] + require(validate_plan_0_1(legacy_plan, identifier + ".0.1_plan") is None, f"{identifier}: immutable 0.1 plan is invalid.") + require(case["expected_plan_contract"] == legacy_plan["contract"], f"{identifier}: 0.1 contract differs.") + require(case["expected_plan_hash"] == legacy_case["expected_plan_hash"], f"{identifier}: 0.1 fixture hash differs.") + require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") + require(identifiers == { + "compute-if-labels-format-level-execute-existing-target", + "nested-or-inequality-variable-operands-create", + "old-subset-retains-v0.1-plan", + }, "0.2 frontend case set differs.") + return len(identifiers) + + +def validate_in_place() -> None: + manifest = json.loads(IN_PLACE.read_text(encoding="utf-8")) + require(set(manifest) == {"manifest_version", "profile", "contract", "cases"}, "0.2 binding manifest fields differ.") + require(manifest["manifest_version"] == "0.2", "0.2 binding manifest version differs.") + require(manifest["contract"] == "openstatspec-in-place-transformation-v0.2", "0.2 binding contract differs.") + case_list = manifest["cases"] + require(isinstance(case_list, list), "0.2 binding cases must be an array.") + cases = {require_string(case.get("id"), "0.2 binding case id"): case for case in case_list} + require(len(cases) == len(case_list), "Duplicate 0.2 binding case id.") + expected_ids = { + "dolt-preprovisioned-target-sequential-null-semantics", + "dolt-preprovisioned-target-or-null-semantics", + } | { + f"{profile}-create-target-fails-before-mutation" + for profile in ("dolt", "mysql", "mariadb") + } + require(set(cases) == expected_ids, "0.2 binding case set differs.") + + success = cases["dolt-preprovisioned-target-sequential-null-semantics"] + require(set(success) == { + "id", "database_profile", "before", "after", "expected_audit", + "forbidden_artifacts", "required_audit_fields", "expected_error", + }, "0.2 binding success fields differ.") + require(success["database_profile"] == "dolt" and success["expected_error"] is None, "Dolt success identity differs.") + before, after = success["before"], success["after"] + for field in ( + "dataset_id", "physical_table_schema", "physical_table_name", + "dataset_count", "persistent_data_table_count", "case_count", + "dolt_branch", "dolt_head", + ): + require(before[field] == after[field], f"Dolt success changes {field}.") + require(before["case_count"] == len(before["rows"]) == len(after["rows"]) == 4, "Dolt case count differs.") + before_ordinals = [row["__case_ordinal"] for row in before["rows"]] + after_ordinals = [row["__case_ordinal"] for row in after["rows"]] + require(before_ordinals == after_ordinals == [1, 2, 3, 4], "Dolt case order differs.") + require(before["target_provisioning"] == { + "physical_column": True, + "catalog_variable": True, + "dolt_commit": before["dolt_head"], + }, "Dolt physical/catalog provisioning is not coupled to HEAD.") + require(before["working_set_clean"] is True and after["working_set_clean"] is False, "Dolt working-set state differs.") + require([row["target"] for row in after["rows"]] == [1, 0, 0, 0], "Three-valued IF result differs.") + require(after["dolt_commit_performed"] is False, "Apply performs DOLT_COMMIT.") + require(success["expected_audit"] == { + "contract_id": manifest["contract"], + "operation_count": 7, + "dolt_branch": before["dolt_branch"], + "dolt_head_before": before["dolt_head"], + "dolt_head_after": after["dolt_head"], + }, "Dolt expected audit identity differs.") + require(set(success["forbidden_artifacts"]) == { + "derived_dataset_row", "persistent_output_table", "full_table_copy", + "staging_dataset", "staging_relation", "snapshot_table", "rollback_table", + "dataset_version_row", "retirement_or_recovery_row", "temporary_object_residue", + }, "Dolt forbidden-artifact proof differs.") + require(set(success["required_audit_fields"]) == { + "apply_id", "contract_id", "database_profile", "dataset_id", + "physical_table_schema", "physical_table_name", "plan_hash", "source_hash", + "actor", "status", "dolt_branch", "dolt_head_before", "dolt_head_after", + "operation_count", "started_at", "completed_at", + }, "Dolt required audit fields differ.") + + or_case = cases["dolt-preprovisioned-target-or-null-semantics"] + require(set(or_case) == { + "id", "database_profile", "target_preprovisioned", "condition", + "before_rows", "after_rows", "expected_error", + }, "Dolt OR-null case fields differ.") + require(or_case["database_profile"] == "dolt" and or_case["target_preprovisioned"] is True, "Dolt OR-null identity differs.") + require(or_case["condition"] == "source_a = 1 OR source_b = 1", "Dolt OR-null condition differs.") + require(or_case["before_rows"] == [ + {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 0}, + {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 0}, + ], "Dolt OR-null inputs differ.") + require(or_case["after_rows"] == [ + {"__case_ordinal": 1, "target": 1}, + {"__case_ordinal": 2, "target": 0}, + ], "Dolt OR three-valued outputs differ.") + require(or_case["expected_error"] is None, "Dolt OR-null case unexpectedly fails.") + + for profile in ("dolt", "mysql", "mariadb"): + case = cases[f"{profile}-create-target-fails-before-mutation"] + require(set(case) == {"id", "database_profile", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") + require(case["database_profile"] == profile and case["target_mode"] == "create", f"{profile}: failure identity differs.") + require(case["expected_error"] == "schema_change_not_atomic" and case["mutation_started"] is False, f"{profile}: create-target gate differs.") + + +def validate_links() -> None: + plan_schema = json.loads(PLAN_SCHEMA.read_text(encoding="utf-8")) + legacy_plan_schema = json.loads(PLAN_SCHEMA_0_1.read_text(encoding="utf-8")) + frontend_schema = json.loads(FRONTEND_SCHEMA.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(plan_schema) + Draft202012Validator.check_schema(legacy_plan_schema) + Draft202012Validator.check_schema(frontend_schema) + plan_validator = Draft202012Validator(plan_schema) + legacy_plan_validator = Draft202012Validator(legacy_plan_schema) + frontend_validator = Draft202012Validator(frontend_schema) + plan_manifest = json.loads(PLAN.read_text(encoding="utf-8")) + legacy_plan_manifest = json.loads(PLAN_0_1.read_text(encoding="utf-8")) + legacy_frontend_manifest = json.loads(FRONTEND_0_1.read_text(encoding="utf-8")) + frontend_manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) + for case in plan_manifest["cases"]: + plan_validator.validate(case["plan"]) + for case in legacy_plan_manifest["cases"]: + legacy_plan_validator.validate(case["plan"]) + for case in legacy_frontend_manifest["cases"]: + if "expected_plan" in case: + legacy_plan_validator.validate(case["expected_plan"]) + for case in frontend_manifest["cases"]: + frontend_validator.validate(case["request"]) + require(plan_schema["$id"].endswith("transformation-plan-0.2.schema.json"), "0.2 plan schema ID differs.") + require(frontend_schema["$id"].endswith("spss-syntax-frontend-0.2.schema.json"), "0.2 frontend schema ID differs.") + binding = (ROOT / "docs/transformation-plan-sql-binding-0.2.md").read_text(encoding="utf-8") + require(legacy_plan_schema["$id"].endswith("transformation-plan-0.1.schema.json"), "0.1 plan schema ID differs.") + binding_normalized = " ".join(binding.split()) + require("MySQL, MariaDB, and Dolt MUST reject" in binding_normalized, "Non-atomic profile gate missing.") + require("separate, explicit, versioned provisioning action" in binding_normalized, "Provisioning contract missing.") + require("MUST NOT commit, switch, merge, reset" in binding_normalized, "Dolt no-commit contract missing.") + + +def validate_all() -> tuple[int, int]: + plan_cases = validate_plan_manifest() + frontend_count = validate_frontend(plan_cases) + validate_in_place() + validate_links() + return len(plan_cases), frontend_count + + +def main() -> None: + plan_count, frontend_count = validate_all() + print(f"Validated {plan_count} Transformation Plan 0.2 and {frontend_count} SPSS frontend 0.2 cases.") + + +if __name__ == "__main__": + main() diff --git a/transformation/plan-0.2.schema.json b/transformation/plan-0.2.schema.json new file mode 100644 index 0000000..7f2b185 --- /dev/null +++ b/transformation/plan-0.2.schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openstatspec.org/schemas/transformation-plan-0.2.schema.json", + "title": "OpenStatSpec Transformation Plan 0.2", + "type": "object", + "additionalProperties": false, + "required": ["contract", "input_alias", "operations"], + "properties": { + "contract": {"const": "openstatspec-transformation-plan-v0.2"}, + "input_alias": {"type": "string", "minLength": 1}, + "operations": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/operation"}} + }, + "$defs": { + "binary64_value": { + "type": "object", "additionalProperties": false, "required": ["type", "bits"], + "properties": { + "type": {"const": "binary64"}, + "bits": {"type": "string", "pattern": "^(?!8000000000000000$)(?!(?:7ff|fff)[0-9a-f]{13}$)[0-9a-f]{16}$"} + } + }, + "string_value": { + "type": "object", "additionalProperties": false, "required": ["type", "value"], + "properties": {"type": {"const": "string"}, "value": {"type": "string"}} + }, + "typed_value": {"oneOf": [{"$ref": "#/$defs/binary64_value"}, {"$ref": "#/$defs/string_value"}]}, + "literal": { + "type": "object", "additionalProperties": false, "required": ["kind", "value"], + "properties": {"kind": {"const": "literal"}, "value": {"$ref": "#/$defs/typed_value"}} + }, + "system_missing": { + "type": "object", "additionalProperties": false, "required": ["kind"], + "properties": {"kind": {"const": "system_missing"}} + }, + "copy": { + "type": "object", "additionalProperties": false, "required": ["kind"], + "properties": {"kind": {"const": "copy"}} + }, + "result": {"oneOf": [{"$ref": "#/$defs/literal"}, {"$ref": "#/$defs/system_missing"}, {"$ref": "#/$defs/copy"}]}, + "variable_operand": { + "type": "object", "additionalProperties": false, "required": ["kind", "variable"], + "properties": {"kind": {"const": "variable"}, "variable": {"type": "string", "minLength": 1}} + }, + "operand": {"oneOf": [{"$ref": "#/$defs/variable_operand"}, {"$ref": "#/$defs/literal"}]}, + "comparison": { + "type": "object", "additionalProperties": false, + "required": ["expression", "left", "operator", "right"], + "properties": { + "expression": {"const": "comparison"}, + "left": {"$ref": "#/$defs/operand"}, + "operator": {"enum": ["=", "<", "<=", ">", ">="]}, + "right": {"$ref": "#/$defs/operand"} + } + }, + "boolean": { + "type": "object", "additionalProperties": false, + "required": ["expression", "operator", "operands"], + "properties": { + "expression": {"const": "boolean"}, + "operator": {"enum": ["and", "or"]}, + "operands": {"type": "array", "minItems": 2, "items": {"$ref": "#/$defs/predicate"}} + } + }, + "predicate": {"oneOf": [{"$ref": "#/$defs/comparison"}, {"$ref": "#/$defs/boolean"}]}, + "match": { + "oneOf": [ + {"type": "object", "additionalProperties": false, "required": ["kind", "values"], "properties": {"kind": {"const": "values"}, "values": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/typed_value"}}}}, + {"type": "object", "additionalProperties": false, "required": ["kind", "lower", "upper"], "properties": {"kind": {"const": "range"}, "lower": {"$ref": "#/$defs/binary64_value"}, "upper": {"$ref": "#/$defs/binary64_value"}}}, + {"$ref": "#/$defs/system_missing"} + ] + }, + "rule": { + "type": "object", "additionalProperties": false, "required": ["match", "result"], + "properties": {"match": {"$ref": "#/$defs/match"}, "result": {"$ref": "#/$defs/result"}} + }, + "recode": { + "type": "object", "additionalProperties": false, + "required": ["op", "source", "target", "target_mode", "rules", "unmatched"], + "properties": { + "op": {"const": "recode"}, "source": {"type": "string", "minLength": 1}, + "target": {"type": "string", "minLength": 1}, "target_mode": {"enum": ["create", "replace"]}, + "rules": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/rule"}}, + "unmatched": {"$ref": "#/$defs/result"} + } + }, + "assign": { + "type": "object", "additionalProperties": false, "required": ["op", "target", "target_mode", "value"], + "properties": { + "op": {"const": "assign"}, "target": {"type": "string", "minLength": 1}, + "target_mode": {"enum": ["create", "replace"]}, "value": {"$ref": "#/$defs/operand"} + } + }, + "conditional_assign": { + "type": "object", "additionalProperties": false, "required": ["op", "condition", "target", "value"], + "properties": { + "op": {"const": "conditional_assign"}, "condition": {"$ref": "#/$defs/predicate"}, + "target": {"type": "string", "minLength": 1}, "value": {"$ref": "#/$defs/operand"} + } + }, + "set_variable_label": { + "type": "object", "additionalProperties": false, "required": ["op", "variable", "label"], + "properties": {"op": {"const": "set_variable_label"}, "variable": {"type": "string", "minLength": 1}, "label": {"type": "string"}} + }, + "value_label": { + "type": "object", "additionalProperties": false, "required": ["value", "label"], + "properties": {"value": {"$ref": "#/$defs/typed_value"}, "label": {"type": "string"}} + }, + "replace_value_labels": { + "type": "object", "additionalProperties": false, "required": ["op", "variable", "labels"], + "properties": { + "op": {"const": "replace_value_labels"}, "variable": {"type": "string", "minLength": 1}, + "labels": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/value_label"}} + } + }, + "set_format": { + "type": "object", "additionalProperties": false, + "required": ["op", "variable", "family", "width", "decimals"], + "properties": { + "op": {"const": "set_format"}, "variable": {"type": "string", "minLength": 1}, + "family": {"const": "F"}, "width": {"type": "integer", "minimum": 1, "maximum": 40}, + "decimals": {"type": "integer", "minimum": 0, "maximum": 16} + } + }, + "set_measurement_level": { + "type": "object", "additionalProperties": false, "required": ["op", "variable", "level"], + "properties": {"op": {"const": "set_measurement_level"}, "variable": {"type": "string", "minLength": 1}, "level": {"enum": ["nominal", "ordinal", "scale"]}} + }, + "execute": { + "type": "object", "additionalProperties": false, "required": ["op"], + "properties": {"op": {"const": "execute"}} + }, + "operation": { + "oneOf": [ + {"$ref": "#/$defs/recode"}, {"$ref": "#/$defs/assign"}, {"$ref": "#/$defs/conditional_assign"}, + {"$ref": "#/$defs/set_variable_label"}, {"$ref": "#/$defs/replace_value_labels"}, + {"$ref": "#/$defs/set_format"}, {"$ref": "#/$defs/set_measurement_level"}, {"$ref": "#/$defs/execute"} + ] + } + } +} diff --git a/transformation/spss-syntax-frontend-0.2.schema.json b/transformation/spss-syntax-frontend-0.2.schema.json new file mode 100644 index 0000000..3e6d3f3 --- /dev/null +++ b/transformation/spss-syntax-frontend-0.2.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openstatspec.org/schemas/spss-syntax-frontend-0.2.schema.json", + "title": "OpenStatSpec SPSS-like Syntax Frontend Request 0.2", + "type": "object", + "additionalProperties": false, + "required": ["contract", "input_alias", "input_schema", "source_text"], + "properties": { + "contract": {"const": "openstatspec-spss-syntax-frontend-v0.2"}, + "input_alias": {"type": "string", "minLength": 1}, + "input_schema": { + "type": "object", "additionalProperties": false, "required": ["variables"], + "properties": { + "variables": { + "type": "array", "minItems": 1, + "items": { + "type": "object", "additionalProperties": false, "required": ["name", "storage_kind"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "storage_kind": {"enum": ["numeric", "string"]}, + "variable_label": {"type": ["string", "null"]}, + "value_labels": {"type": "array", "items": {"$ref": "#/$defs/value_label"}}, + "format_family": {"type": ["string", "null"]}, + "width": {"type": ["integer", "null"], "minimum": 1}, + "decimals": {"type": ["integer", "null"], "minimum": 0}, + "measurement_level": {"enum": ["nominal", "ordinal", "scale", null]} + } + } + } + } + }, + "source_text": {"type": "string", "minLength": 1} + }, + "$defs": { + "binary64_value": { + "type": "object", "additionalProperties": false, "required": ["type", "bits"], + "properties": { + "type": {"const": "binary64"}, + "bits": {"type": "string", "pattern": "^(?!8000000000000000$)(?!(?:7ff|fff)[0-9a-f]{13}$)[0-9a-f]{16}$"} + } + }, + "string_value": { + "type": "object", "additionalProperties": false, "required": ["type", "value"], + "properties": {"type": {"const": "string"}, "value": {"type": "string"}} + }, + "typed_value": {"oneOf": [{"$ref": "#/$defs/binary64_value"}, {"$ref": "#/$defs/string_value"}]}, + "value_label": { + "type": "object", + "additionalProperties": false, + "required": ["value", "label"], + "properties": { + "value": {"$ref": "#/$defs/typed_value"}, + "label": {"type": "string"} + } + } + } +} From 1c2871d3c71c16bf60a60af2d28c12ed9515f66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 16:39:10 +0300 Subject: [PATCH 02/78] Install transformation validator dependencies --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd1ff1..1e6feee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,4 +12,5 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.13" + - run: python -m pip install --requirement requirements-validation.txt - run: python tools/validate_repository.py From 1372fd757a90e50883b56dbf08d6bc72d850d6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:15:28 +0300 Subject: [PATCH 03/78] Keep conditional profile on the unreleased v0.3.0 line --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc7b96..d7d429c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # Changelog -## v0.3.0 - 2026-07-31 +## Unreleased + +Planned specification release: `v0.3.0`, after the reference-adapter conformance gates pass. - Added backward-compatible Transformation Plan and SPSS Syntax Frontend 0.2 contracts for sequential numeric `COMPUTE` and conditional `IF`, bounded From 52760a525f15274cbb3806686f374067d4f54779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:18:02 +0300 Subject: [PATCH 04/78] Clarify v0.3.0 release-candidate provenance [skip ci] --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d429c..e69358f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,9 @@ ## Unreleased -Planned specification release: `v0.3.0`, after the reference-adapter conformance gates pass. +Target stable specification release: `v0.3.0` (not published). + +Until a protected or signed `v0.3.0` tag targets this work, implementations MUST identify it with `specification_status: release_candidate`, `specification_release: null`, and the exact immutable specification commit. The entries below do not constitute a stable release. - Added backward-compatible Transformation Plan and SPSS Syntax Frontend 0.2 contracts for sequential numeric `COMPUTE` and conditional `IF`, bounded From fbd1a2c8e22e35088c1a0bc28bada994f0b88f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:18:07 +0300 Subject: [PATCH 05/78] Mark transformation profile 0.2 as release-candidate [skip ci] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 853bcb4..6a83877 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ permits a transformation to mutate a core import. That separate derived-data profile is not used by the SPSS-like in-place frontend below. -The [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) +The release-candidate [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) adds sequential bounded numeric assignment and conditional assignment to the unchanged 0.1 operations. The [SPSS Syntax Frontend Profile 0.2](docs/spss-syntax-frontend-profile-0.2.md) From b5276067af5b6c3789d7b7e31fe2c09e4e26d828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:18:12 +0300 Subject: [PATCH 06/78] Clarify plan 0.2 release-candidate status [skip ci] --- docs/transformation-plan-profile-0.2.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/transformation-plan-profile-0.2.md b/docs/transformation-plan-profile-0.2.md index af64c7f..79f32fe 100644 --- a/docs/transformation-plan-profile-0.2.md +++ b/docs/transformation-plan-profile-0.2.md @@ -2,6 +2,8 @@ ## Status and scope +Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. + This optional profile is the backward-compatible successor to Transformation Plan 0.1. Its contract identifier is `openstatspec-transformation-plan-v0.2`. The key words **MUST**, **MUST NOT**, From 82550617e69da89bdb1e4ff48b22a19f8d181303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:18:21 +0300 Subject: [PATCH 07/78] Clarify frontend 0.2 release-candidate status [skip ci] --- docs/spss-syntax-frontend-profile-0.2.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md index f35f935..9c1de02 100644 --- a/docs/spss-syntax-frontend-profile-0.2.md +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -2,6 +2,8 @@ ## Status and boundary +Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. + This optional frontend maps a bounded SPSS-like command language to Transformation Plan 0.1 or 0.2. Its request contract is `openstatspec-spss-syntax-frontend-v0.2`. It is not full IBM SPSS Statistics From b56eb9a3161eb6cb732eaac98c684934c84e31f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:18:28 +0300 Subject: [PATCH 08/78] Make profile 0.2 release status explicit --- docs/transformation-plan-sql-binding-0.2.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md index e8a2634..e3ef2b0 100644 --- a/docs/transformation-plan-sql-binding-0.2.md +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -1,5 +1,9 @@ # In-Place Transformation Binding 0.2 +## Status + +Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. + ## Normative decision This binding extends In-Place Transformation Binding 0.1 for Transformation From 85567a7c2bec74b495eac34af1897bab64e81b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:06:20 +0300 Subject: [PATCH 09/78] Allow in-place transformation audit contract 0.2 --- sql/transformation-plan-profile-schema.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/transformation-plan-profile-schema.sql b/sql/transformation-plan-profile-schema.sql index ac7f317..bb761f1 100644 --- a/sql/transformation-plan-profile-schema.sql +++ b/sql/transformation-plan-profile-schema.sql @@ -1,4 +1,4 @@ --- OpenStatSpec In-Place Transformation Binding 0.1 compact operation audit. +-- OpenStatSpec In-Place Transformation Bindings 0.1 and 0.2 compact operation audit. -- This table is not a dataset-version catalog and stores no row data or copies. CREATE TABLE transformation_apply ( @@ -19,7 +19,10 @@ CREATE TABLE transformation_apply ( operation_count INTEGER NOT NULL, started_at TIMESTAMP NOT NULL, completed_at TIMESTAMP NOT NULL, - CHECK (contract_id = 'openstatspec-in-place-transformation-v0.1'), + CHECK (contract_id IN ( + 'openstatspec-in-place-transformation-v0.1', + 'openstatspec-in-place-transformation-v0.2' + )), CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), CHECK (status IN ('succeeded', 'failed')), CHECK (operation_count > 0), From 9bc635f5a088feea37ab1a7181bbad89265bf104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:06:25 +0300 Subject: [PATCH 10/78] Link binding fixture to compatible audit schema --- conformance/in-place-transformation-0.2.json | 1 + 1 file changed, 1 insertion(+) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index d4cf877..5ae2817 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -2,6 +2,7 @@ "manifest_version": "0.2", "profile": "OpenStatSpec In-Place Transformation Binding 0.2", "contract": "openstatspec-in-place-transformation-v0.2", + "audit_schema": "../sql/transformation-plan-profile-schema.sql", "cases": [ { "id": "dolt-preprovisioned-target-sequential-null-semantics", From ec9e17f0681a5ec6778782bff1db6115c4f6760a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:06:30 +0300 Subject: [PATCH 11/78] Define in-place audit schema migration for binding 0.2 --- docs/transformation-plan-sql-binding-0.2.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md index e3ef2b0..d3e1531 100644 --- a/docs/transformation-plan-sql-binding-0.2.md +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -76,5 +76,17 @@ The compact audit and forbidden-artifact requirements from Binding 0.1 are unchanged. It records exact plan/source hashes and operation count, not row values or copied state. +## Audit schema compatibility + +The versioned `sql/transformation-plan-profile-schema.sql` DDL accepts both the +0.1 and 0.2 in-place binding contract identifiers in the same existing +`transformation_apply` table. Before the first 0.2 apply, an installation whose +0.1 table still has the single-contract check MUST migrate that check in place, +using its database profile's transactional DDL procedure where available. The +migration preserves every existing compact audit row and MUST NOT create a +second audit table, dataset, data-table copy, snapshot, or rollback artifact. +It is a schema migration completed before an apply; it is not part of the +transformation apply boundary. + Machine-readable binding cases are in [`../conformance/in-place-transformation-0.2.json`](../conformance/in-place-transformation-0.2.json). From 9faa5c065adc2ecf42297603080e29ea2a20a759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:06:35 +0300 Subject: [PATCH 12/78] Define canonical boolean-chain lowering --- docs/spss-syntax-frontend-profile-0.2.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md index 9c1de02..3f8f5f6 100644 --- a/docs/spss-syntax-frontend-profile-0.2.md +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -31,8 +31,13 @@ predicate := comparison | "(" predicate ")" ``` Comparison binds tighter than `AND`; `AND` binds tighter than `OR`. -Parentheses override precedence and are preserved as the equivalent nested -plan shape. Keywords are ASCII case-insensitive. Unary operators, arithmetic, +A maximal chain of the same boolean operator MUST lower to one n-ary boolean +plan node whose operands remain in source order. Thus `a AND b AND c` lowers +to one `and` node with three operands, never left- or right-nested binary +`and` nodes. Parentheses override precedence across different operators; +parentheses around a same-operator subexpression do not prevent this required +flattening. Mixed operators remain nested according to precedence and explicit +parentheses. Keywords are ASCII case-insensitive. Unary operators, arithmetic, functions, strings, `NOT`, `~=`, `<>`, `MISSING`, locale-dependent numbers, and implicit numeric/string coercion are unsupported. From 481ac6789f31acb639a5e872fd8d2ce424f12979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:07:19 +0300 Subject: [PATCH 13/78] Add canonical three-term boolean plan fixture --- conformance/transformation-plan-0.2.json | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index a8df9ab..a0b83cb 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -115,6 +115,82 @@ "expected_plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", "expected_error": null }, + { + "id": "three-term-and-flattens-source-order", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_c" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_plan_hash": "d6f94edd5de261552e79157cd29ff7668e824476827fae7f0fe9f56ba45704ea", + "expected_error": null + }, { "id": "reject-string-predicate", "plan": { From 540f11451d809839e7d0e735a1dcefd4956b39a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:07:25 +0300 Subject: [PATCH 14/78] Add three-term AND frontend conformance case --- conformance/spss-syntax-frontend-0.2.json | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 4cdd71d..82fe4b6 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -40,6 +40,38 @@ "expected_source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481", "expected_error": null }, + { + "id": "three-term-and-flattens-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a = 1 AND source_b = 1 AND source_c = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "three-term-and-flattens-source-order", + "expected_plan_hash": "d6f94edd5de261552e79157cd29ff7668e824476827fae7f0fe9f56ba45704ea", + "expected_source_hash": "518cb0fa9537e7dff65f43c7a8de2d7e51c1e15b8c856c57e1875d2cfeb5646e", + "expected_error": null + }, { "id": "old-subset-retains-v0.1-plan", "request": { From bddd110325737f40c564acaefd15755a94572800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:07:59 +0300 Subject: [PATCH 15/78] Validate audit DDL and canonical boolean chains --- tools/validate_transformation_plan_0_2.py | 46 ++++++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 6ed4662..272001a 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -19,6 +19,7 @@ PLAN = ROOT / "conformance/transformation-plan-0.2.json" FRONTEND = ROOT / "conformance/spss-syntax-frontend-0.2.json" IN_PLACE = ROOT / "conformance/in-place-transformation-0.2.json" +AUDIT_SCHEMA = ROOT / "sql/transformation-plan-profile-schema.sql" PLAN_SCHEMA = ROOT / "transformation/plan-0.2.schema.json" FRONTEND_SCHEMA = ROOT / "transformation/spss-syntax-frontend-0.2.schema.json" @@ -73,6 +74,15 @@ def predicate(value: object, context: str) -> bool: require(value["operator"] in {"and", "or"}, f"{context}: boolean operator differs.") children = value["operands"] require(isinstance(children, list) and len(children) >= 2, f"{context}: boolean operands differ.") + for index, child in enumerate(children): + require( + not ( + isinstance(child, dict) + and child.get("expression") == "boolean" + and child.get("operator") == value["operator"] + ), + f"{context}.operands[{index}]: same-operator boolean chain must be flattened.", + ) return any(predicate(child, f"{context}.operands[{index}]") for index, child in enumerate(children)) @@ -155,6 +165,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: cases = {} expected = { "sequential-conditional-binary-existing-target": None, + "three-term-and-flattens-source-order": None, "reject-string-predicate": "expression_type_unsupported", "nested-or-inequalities-variable-operands-create": None, "reject-invalid-format": "invalid_format", @@ -204,11 +215,17 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: require(set(case) == {"id", "request", "expected_plan_case", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: frontend success fields differ.") require(case["expected_plan_case"] in plan_cases, f"{identifier}: plan fixture missing.") require(case["expected_plan_hash"] == plan_cases[case["expected_plan_case"]]["expected_plan_hash"], f"{identifier}: plan hash link differs.") - required_tokens = ( - ("COMPUTE ", "IF (", " AND ", "FORMATS ", "VARIABLE LEVEL ", "EXECUTE.") - if identifier == "compute-if-labels-format-level-execute-existing-target" - else ("COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.") - ) + required_tokens = { + "compute-if-labels-format-level-execute-existing-target": ( + "COMPUTE ", "IF (", " AND ", "FORMATS ", "VARIABLE LEVEL ", "EXECUTE.", + ), + "three-term-and-flattens-source-order": ( + "IF (", "source_a = 1 AND source_b = 1 AND source_c = 1", "EXECUTE.", + ), + "nested-or-inequality-variable-operands-create": ( + "COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.", + ), + }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") else: @@ -223,6 +240,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") require(identifiers == { "compute-if-labels-format-level-execute-existing-target", + "three-term-and-flattens-source-order", "nested-or-inequality-variable-operands-create", "old-subset-retains-v0.1-plan", }, "0.2 frontend case set differs.") @@ -231,9 +249,25 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: def validate_in_place() -> None: manifest = json.loads(IN_PLACE.read_text(encoding="utf-8")) - require(set(manifest) == {"manifest_version", "profile", "contract", "cases"}, "0.2 binding manifest fields differ.") + require( + set(manifest) == {"manifest_version", "profile", "contract", "audit_schema", "cases"}, + "0.2 binding manifest fields differ.", + ) require(manifest["manifest_version"] == "0.2", "0.2 binding manifest version differs.") require(manifest["contract"] == "openstatspec-in-place-transformation-v0.2", "0.2 binding contract differs.") + require( + manifest["audit_schema"] == "../sql/transformation-plan-profile-schema.sql", + "0.2 binding audit schema link differs.", + ) + audit_schema_path = (IN_PLACE.parent / manifest["audit_schema"]).resolve() + require(audit_schema_path == AUDIT_SCHEMA.resolve(), "0.2 binding audit schema path differs.") + audit_schema = audit_schema_path.read_text(encoding="utf-8") + require( + "contract_id IN (" in audit_schema + and "'openstatspec-in-place-transformation-v0.1'" in audit_schema + and f"'{manifest['contract']}'" in audit_schema, + "In-place audit schema does not accept both 0.1 and 0.2 contracts.", + ) case_list = manifest["cases"] require(isinstance(case_list, list), "0.2 binding cases must be an array.") cases = {require_string(case.get("id"), "0.2 binding case id"): case for case in case_list} From 5a8c2ff978737515aff240bbbe19e6a3cf5d009b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:20:15 +0300 Subject: [PATCH 16/78] Align Plan 0.2 boolean canonicalization --- docs/transformation-plan-profile-0.2.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/transformation-plan-profile-0.2.md b/docs/transformation-plan-profile-0.2.md index 79f32fe..619fad5 100644 --- a/docs/transformation-plan-profile-0.2.md +++ b/docs/transformation-plan-profile-0.2.md @@ -60,9 +60,12 @@ collation or coercion semantics. A comparison node has `expression=comparison`, two operands, and one of `=`, `<`, `<=`, `>`, or `>=`. A boolean node has `expression=boolean`, `operator=and|or`, and at least two ordered predicate -operands. Nested boolean nodes preserve parentheses and associativity from the -frontend. Implementations MUST NOT simplify, reorder, deduplicate, distribute, -or constant-fold nodes before canonical hashing. +operands. A maximal same-operator chain MUST be represented by one n-ary node +whose operands remain in source order; a boolean node MUST NOT have a direct +boolean child with the same operator. Parentheses preserve grouping across +different operators, but same-operator parentheses do not prevent the required +flattening. Implementations MUST NOT otherwise simplify, reorder, deduplicate, +distribute, or constant-fold nodes before canonical hashing. Predicates use SQL three-valued truth semantics. A comparison with numeric system missing (SQL `NULL`) is UNKNOWN. `AND` and `OR` use their standard From 31545c76a2376fd8a686f37113a22c05fb6356d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:27:18 +0300 Subject: [PATCH 17/78] Permit transactional SQLite audit schema rebuild --- docs/transformation-plan-sql-binding-0.2.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md index d3e1531..0ca22c8 100644 --- a/docs/transformation-plan-sql-binding-0.2.md +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -79,14 +79,21 @@ values or copied state. ## Audit schema compatibility The versioned `sql/transformation-plan-profile-schema.sql` DDL accepts both the -0.1 and 0.2 in-place binding contract identifiers in the same existing +0.1 and 0.2 in-place binding contract identifiers in the same logical `transformation_apply` table. Before the first 0.2 apply, an installation whose -0.1 table still has the single-contract check MUST migrate that check in place, -using its database profile's transactional DDL procedure where available. The -migration preserves every existing compact audit row and MUST NOT create a -second audit table, dataset, data-table copy, snapshot, or rollback artifact. -It is a schema migration completed before an apply; it is not part of the -transformation apply boundary. +0.1 table still has the single-contract check MUST migrate that check using its +database profile's transactional DDL procedure where available. The migration +preserves every existing compact audit row and logical audit-table identity. + +SQLite, whose `ALTER TABLE` cannot drop the existing table-level check, MUST +perform the standard create/copy/drop/rename table rebuild inside one native +transaction. The replacement table is strictly ephemeral migration state: it +must use the versioned DDL, receive only the existing compact audit rows, and be +renamed to `transformation_apply` before commit. After successful migration +there MUST be exactly one persistent audit table, with no dataset, data-table +copy, snapshot, rollback artifact, or recovery-version layer added. A failure +rolls back the entire rebuild. This schema migration is completed before an +apply and is not part of the transformation apply boundary. Machine-readable binding cases are in [`../conformance/in-place-transformation-0.2.json`](../conformance/in-place-transformation-0.2.json). From d0d79dae68a887e808621cb5433c1f42892da980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:27:53 +0300 Subject: [PATCH 18/78] Add frontend failure conformance fixtures --- conformance/spss-syntax-frontend-0.2.json | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 82fe4b6..1b5516d 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -120,6 +120,64 @@ "expected_plan_hash": "926f2cd93ed0bfa330bdb2d0b31b241f552c86c116dd35a315a20408bf997daf", "expected_source_hash": "4b3b35660b1ec57de0b8b0485dcc85bb51a05796df9e9624f4f1790d78e2bfeb", "expected_error": null + }, + { + "id": "reject-string-expression", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "color", + "storage_kind": "string" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = color." + }, + "expected_source_hash": "9d44d6aa5554a716d55ecbcf04e9744f59a9a4ecf0281652ab713a7482d581c4", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-missing-conditional-target", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a = 1) missing_target = 1." + }, + "expected_source_hash": "7a7956e282fb7e2de12daa1f1a3af5cd89e20193e0c0269ba5c63a27ac81e06f", + "expected_error": "conditional_target_missing" + }, + { + "id": "reject-invalid-format", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "FORMATS target (F2.2)." + }, + "expected_source_hash": "307aee8e2ca8a03939d4d487a3c9303d78abfc82f55f6434cab92e710defe8d5", + "expected_error": "invalid_format" } ] } From 4d2e5b2cf107e43e1eedeb3e8877319092d3ce65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:28:14 +0300 Subject: [PATCH 19/78] Enforce frontend failure conformance cases --- tools/validate_transformation_plan_0_2.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 272001a..a7150e2 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -202,6 +202,11 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: legacy_manifest = json.loads(FRONTEND_0_1.read_text(encoding="utf-8")) legacy_cases = {case["id"]: case for case in legacy_manifest["cases"]} identifiers = set() + expected_failures = { + "reject-string-expression": "expression_type_unsupported", + "reject-missing-conditional-target": "conditional_target_missing", + "reject-invalid-format": "invalid_format", + } for case in manifest["cases"]: identifier = require_string(case["id"], "0.2 frontend case id") require(identifier not in identifiers, f"Duplicate 0.2 frontend case: {identifier}") @@ -211,7 +216,17 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: source = request["source_text"].replace("\r\n", "\n").replace("\r", "\n") require(source == request["source_text"], f"{identifier}: source is not normalized.") require(case["expected_source_hash"] == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: source hash differs.") - if "expected_plan_case" in case: + if identifier in expected_failures: + require( + set(case) == {"id", "request", "expected_source_hash", "expected_error"}, + f"{identifier}: frontend failure fields differ.", + ) + require( + case["expected_error"] == expected_failures[identifier], + f"{identifier}: frontend diagnostic differs.", + ) + elif "expected_plan_case" in case: + require(case["expected_error"] is None, f"{identifier}: frontend success claims an error.") require(set(case) == {"id", "request", "expected_plan_case", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: frontend success fields differ.") require(case["expected_plan_case"] in plan_cases, f"{identifier}: plan fixture missing.") require(case["expected_plan_hash"] == plan_cases[case["expected_plan_case"]]["expected_plan_hash"], f"{identifier}: plan hash link differs.") @@ -243,6 +258,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "three-term-and-flattens-source-order", "nested-or-inequality-variable-operands-create", "old-subset-retains-v0.1-plan", + "reject-string-expression", + "reject-missing-conditional-target", + "reject-invalid-format", }, "0.2 frontend case set differs.") return len(identifiers) From 07cb990ee9f25ab016e4f31eab87979f8210f046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:34:50 +0300 Subject: [PATCH 20/78] Define deterministic finite-number conversion --- docs/spss-syntax-frontend-profile-0.2.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md index 3f8f5f6..0782356 100644 --- a/docs/spss-syntax-frontend-profile-0.2.md +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -41,6 +41,27 @@ parentheses. Keywords are ASCII case-insensitive. Unary operators, arithmetic, functions, strings, `NOT`, `~=`, `<>`, `MISSING`, locale-dependent numbers, and implicit numeric/string coercion are unsupported. +The `finite-number` token has this exact ASCII grammar: + +```text +finite-number := "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? + ([eE] [+-]? [0-9]+)? +``` + +Whitespace is not allowed inside the token. A leading plus, leading decimal +point, trailing decimal point, leading zero before another integer digit, +`NaN`, and `Infinity` are rejected. Thus `-1` and `1E2` are accepted, +while `.5` and `1.` are rejected. + +The frontend MUST interpret the token as an exact base-ten value, independent +of host locale, then correctly round it to the nearest IEEE 754 binary64 value +using ties-to-even. It MUST NOT first round through a host decimal or extended +binary floating-point type. A result whose magnitude overflows to infinity is +rejected with `spss_syntax_error`; a zero result, including signed or +underflowed zero, is canonicalized to positive-zero bits +`0000000000000000`. Emitted bits use exactly 16 lowercase hexadecimal digits. +NaN and infinity can never be emitted. + A numeric variable is resolved at the point its command is lowered, including variables created by preceding commands. All referenced variables and assigned values in this subset MUST be numeric. From 20ed939be08f4b5a4f6960a9bbd0f5a5b19352dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:35:30 +0300 Subject: [PATCH 21/78] Cover parenthesized same-operator flattening --- conformance/spss-syntax-frontend-0.2.json | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 1b5516d..ed771d1 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -72,6 +72,38 @@ "expected_source_hash": "518cb0fa9537e7dff65f43c7a8de2d7e51c1e15b8c856c57e1875d2cfeb5646e", "expected_error": null }, + { + "id": "parenthesized-three-term-and-flattens-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF ((source_a = 1 AND source_b = 1) AND source_c = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "three-term-and-flattens-source-order", + "expected_plan_hash": "d6f94edd5de261552e79157cd29ff7668e824476827fae7f0fe9f56ba45704ea", + "expected_source_hash": "fdad0805734da0d8b356d67ed561c3ddfa48f0c4a942a60b1d6ca98dab3321d6", + "expected_error": null + }, { "id": "old-subset-retains-v0.1-plan", "request": { From 15d8911e37fe4e542b45f07cbcff8fbf38bd23dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:35:51 +0300 Subject: [PATCH 22/78] Enforce parenthesized flattening fixture --- tools/validate_transformation_plan_0_2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index a7150e2..ca72616 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -237,6 +237,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "three-term-and-flattens-source-order": ( "IF (", "source_a = 1 AND source_b = 1 AND source_c = 1", "EXECUTE.", ), + "parenthesized-three-term-and-flattens-source-order": ( + "IF ((source_a = 1 AND source_b = 1) AND source_c = 1)", "EXECUTE.", + ), "nested-or-inequality-variable-operands-create": ( "COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.", ), @@ -256,6 +259,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: require(identifiers == { "compute-if-labels-format-level-execute-existing-target", "three-term-and-flattens-source-order", + "parenthesized-three-term-and-flattens-source-order", "nested-or-inequality-variable-operands-create", "old-subset-retains-v0.1-plan", "reject-string-expression", From a22b647120a6982d8c9330cb396dd9c7a595b554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:53:36 +0300 Subject: [PATCH 23/78] Add mixed-precedence and numeric conversion plans --- conformance/transformation-plan-0.2.json | 226 +++++++++++++++++++++++ 1 file changed, 226 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index a0b83cb..e4fc3d9 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -387,6 +387,232 @@ }, "expected_plan_hash": null, "expected_error": "duplicate_value_label" + }, + { + "id": "mixed-default-and-precedence", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "or", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_c" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "5fc4c6e5c41661f6b0f7806d3e4b5054e8282d0f4f3f4ae9da60eae68b7f9d43" + }, + { + "id": "mixed-parentheses-override-precedence", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "boolean", + "operator": "or", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_c" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "dd283eacad64052470bc44b6fb69817db94a95aa6391b34b93f522dcbc63a279" + }, + { + "id": "finite-number-rounding-exponent-underflow", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000002" + } + } + }, + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "c05f400000000000" + } + } + }, + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "0000000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "5abd09242071c929f3fe7e597ad605c9e2cf06a6e2c72fe2b62584dd4ccccb05" } ] } From dcefb7c76c52040686c1636bd999638e510a656c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:54:15 +0300 Subject: [PATCH 24/78] Add mixed-precedence and numeric frontend cases --- conformance/spss-syntax-frontend-0.2.json | 186 ++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index ed771d1..d129a55 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -210,6 +210,192 @@ }, "expected_source_hash": "307aee8e2ca8a03939d4d487a3c9303d78abfc82f55f6434cab92e710defe8d5", "expected_error": "invalid_format" + }, + { + "id": "unparenthesized-mixed-default-precedence", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a = 1 OR source_b = 1 AND source_c = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "mixed-default-and-precedence", + "expected_plan_hash": "5fc4c6e5c41661f6b0f7806d3e4b5054e8282d0f4f3f4ae9da60eae68b7f9d43", + "expected_source_hash": "5dff61244ba2461bc64b1cbef374ebcbb751e03792cf92efcca439ad693f8943", + "expected_error": null + }, + { + "id": "parenthesized-mixed-override-precedence", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF ((source_a = 1 OR source_b = 1) AND source_c = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "mixed-parentheses-override-precedence", + "expected_plan_hash": "dd283eacad64052470bc44b6fb69817db94a95aa6391b34b93f522dcbc63a279", + "expected_source_hash": "e655fe1d0aa6829dde510bdb1e3253110c723e1fdf23c539cd20bee4b3e07660", + "expected_error": null + }, + { + "id": "finite-number-rounding-exponent-underflow", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = 1.00000000000000011102230246251565404236316680908203125.\nCOMPUTE target = 1.00000000000000033306690738754696212708950042724609375.\nCOMPUTE target = -1.25e+2.\nCOMPUTE target = 1e-4000.\nEXECUTE." + }, + "expected_plan_case": "finite-number-rounding-exponent-underflow", + "expected_plan_hash": "5abd09242071c929f3fe7e597ad605c9e2cf06a6e2c72fe2b62584dd4ccccb05", + "expected_source_hash": "bf1cbb16ed58a0fb37ac437aee7008d336d8efd024a35c654d31698cabaca21d", + "expected_error": null + }, + { + "id": "reject-numeric-overflow", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = 1e4000." + }, + "expected_source_hash": "9cd7389c9841eb18f95a2a377cc05e1f31a8d9212ed9afa92cce6d9da6b7c0f1", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-leading-decimal-point", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = .5." + }, + "expected_source_hash": "b89b9a5cbdd4ff8c6e4a00926788b5b9fc79d0124d7a5bdf913e6382933bbee0", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-trailing-decimal-point", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = 1.." + }, + "expected_source_hash": "a2d5ced231ee88930405e9e36f8f68ab86f57ba720fd02579e28616d9d71f1cf", + "expected_error": "spss_syntax_error" } ] } From 27b8e4c29bf63bee6868316fb5d76c042470b2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:54:44 +0300 Subject: [PATCH 25/78] Enforce precedence and numeric conversion goldens --- tools/validate_transformation_plan_0_2.py | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index ca72616..f8fc427 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -171,6 +171,9 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "reject-invalid-format": "invalid_format", "reject-reserved-assignment-target": "reserved_target_name", "reject-duplicate-value-label": "duplicate_value_label", + "mixed-default-and-precedence": None, + "mixed-parentheses-override-precedence": None, + "finite-number-rounding-exponent-underflow": None, } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -183,6 +186,19 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: require(case["expected_plan_hash"] == canonical_hash(case["plan"]), f"{identifier}: golden plan hash differs.") else: require(case["expected_plan_hash"] is None, f"{identifier}: invalid plan claims a hash.") + if identifier == "finite-number-rounding-exponent-underflow": + require( + [ + operation["value"]["value"]["bits"] + for operation in case["plan"]["operations"][:-1] + ] == [ + "3ff0000000000000", + "3ff0000000000002", + "c05f400000000000", + "0000000000000000", + ], + "Finite-number conversion golden bits differ.", + ) require(set(cases) == set(expected), "0.2 plan case set differs.") return cases @@ -206,6 +222,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-string-expression": "expression_type_unsupported", "reject-missing-conditional-target": "conditional_target_missing", "reject-invalid-format": "invalid_format", + "reject-numeric-overflow": "spss_syntax_error", + "reject-leading-decimal-point": "spss_syntax_error", + "reject-trailing-decimal-point": "spss_syntax_error", } for case in manifest["cases"]: identifier = require_string(case["id"], "0.2 frontend case id") @@ -243,6 +262,17 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "nested-or-inequality-variable-operands-create": ( "COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.", ), + "unparenthesized-mixed-default-precedence": ( + "IF (source_a = 1 OR source_b = 1 AND source_c = 1)", "EXECUTE.", + ), + "parenthesized-mixed-override-precedence": ( + "IF ((source_a = 1 OR source_b = 1) AND source_c = 1)", "EXECUTE.", + ), + "finite-number-rounding-exponent-underflow": ( + "1.00000000000000011102230246251565404236316680908203125", + "1.00000000000000033306690738754696212708950042724609375", + "-1.25e+2", "1e-4000", "EXECUTE.", + ), }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") @@ -265,6 +295,12 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-string-expression", "reject-missing-conditional-target", "reject-invalid-format", + "unparenthesized-mixed-default-precedence", + "parenthesized-mixed-override-precedence", + "finite-number-rounding-exponent-underflow", + "reject-numeric-overflow", + "reject-leading-decimal-point", + "reject-trailing-decimal-point", }, "0.2 frontend case set differs.") return len(identifiers) From 0fb3fd5b10248a6d4b027a7584f2cdb4a630723b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:03:30 +0300 Subject: [PATCH 26/78] Add three-term OR and strict greater-than plans --- conformance/transformation-plan-0.2.json | 116 +++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index e4fc3d9..bd51767 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -613,6 +613,122 @@ }, "expected_error": null, "expected_plan_hash": "5abd09242071c929f3fe7e597ad605c9e2cf06a6e2c72fe2b62584dd4ccccb05" + }, + { + "id": "three-term-or-flattens-source-order", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "or", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_c" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "e61850fd0e65726dbb13af184091b726ed70ead948300ae1d6163df2c23663f9" + }, + { + "id": "strict-greater-than", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": ">", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "9fd5d95f078924be3ce0809c985f47ffedb1c5fbbecadc739591d37df6297625" } ] } From 32c21158108797832e07a31b81259f66132d305e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:04:05 +0300 Subject: [PATCH 27/78] Add OR-chain and strict greater-than frontend cases --- conformance/spss-syntax-frontend-0.2.json | 96 +++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index d129a55..c9e6ba8 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -396,6 +396,102 @@ }, "expected_source_hash": "a2d5ced231ee88930405e9e36f8f68ab86f57ba720fd02579e28616d9d71f1cf", "expected_error": "spss_syntax_error" + }, + { + "id": "three-term-or-flattens-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a = 1 OR source_b = 1 OR source_c = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "three-term-or-flattens-source-order", + "expected_plan_hash": "e61850fd0e65726dbb13af184091b726ed70ead948300ae1d6163df2c23663f9", + "expected_source_hash": "6aeb46935565cb16b84c503e6df4481d0542b095a68b9788e1c0f0a7a9fcf318", + "expected_error": null + }, + { + "id": "parenthesized-three-term-or-flattens-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF ((source_a = 1 OR source_b = 1) OR source_c = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "three-term-or-flattens-source-order", + "expected_plan_hash": "e61850fd0e65726dbb13af184091b726ed70ead948300ae1d6163df2c23663f9", + "expected_source_hash": "26f683849987d5489d66822c8315f1b7be07c6583cfd88c8a8449cfe3ac9c3ea", + "expected_error": null + }, + { + "id": "strict-greater-than", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a > 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "strict-greater-than", + "expected_plan_hash": "9fd5d95f078924be3ce0809c985f47ffedb1c5fbbecadc739591d37df6297625", + "expected_source_hash": "f8e21c1e96b872c65395a7319dcfb4c93b60798a3e1c578fb7ba63825391afcf", + "expected_error": null } ] } From bb9327db1a1b3eb8f0b303952216ce0f7ec38e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:04:30 +0300 Subject: [PATCH 28/78] Enforce OR-chain and greater-than goldens --- tools/validate_transformation_plan_0_2.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index f8fc427..0c57a73 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -174,6 +174,8 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "mixed-default-and-precedence": None, "mixed-parentheses-override-precedence": None, "finite-number-rounding-exponent-underflow": None, + "three-term-or-flattens-source-order": None, + "strict-greater-than": None, } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -273,6 +275,13 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "1.00000000000000033306690738754696212708950042724609375", "-1.25e+2", "1e-4000", "EXECUTE.", ), + "three-term-or-flattens-source-order": ( + "source_a = 1 OR source_b = 1 OR source_c = 1", "EXECUTE.", + ), + "parenthesized-three-term-or-flattens-source-order": ( + "((source_a = 1 OR source_b = 1) OR source_c = 1)", "EXECUTE.", + ), + "strict-greater-than": ("source_a > 1", "EXECUTE."), }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") @@ -301,6 +310,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-numeric-overflow", "reject-leading-decimal-point", "reject-trailing-decimal-point", + "three-term-or-flattens-source-order", + "parenthesized-three-term-or-flattens-source-order", + "strict-greater-than", }, "0.2 frontend case set differs.") return len(identifiers) From 6afaa3d68848b800a8ab0e3d0db25607ea05a4bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:12:25 +0300 Subject: [PATCH 29/78] Add comparison operand permutation plan --- conformance/transformation-plan-0.2.json | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index bd51767..8797ee9 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -729,6 +729,82 @@ }, "expected_error": null, "expected_plan_hash": "9fd5d95f078924be3ce0809c985f47ffedb1c5fbbecadc739591d37df6297625" + }, + { + "id": "comparison-operand-permutations", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "variable", + "variable": "source_b" + } + }, + { + "expression": "comparison", + "left": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + }, + "operator": "<", + "right": { + "kind": "variable", + "variable": "source_c" + } + }, + { + "expression": "comparison", + "left": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_plan_hash": "cd8e04360419bc5b5d6db9a8800f4c58e267d2a29f0c2926cfba8d266892f021", + "expected_error": null } ] } From 0a3a79156c41c3bdd2810278dd02580abc5f7db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:13:04 +0300 Subject: [PATCH 30/78] Complete operand and diagnostic frontend coverage --- conformance/spss-syntax-frontend-0.2.json | 234 ++++++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index c9e6ba8..f4361a8 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -492,6 +492,240 @@ "expected_plan_hash": "9fd5d95f078924be3ce0809c985f47ffedb1c5fbbecadc739591d37df6297625", "expected_source_hash": "f8e21c1e96b872c65395a7319dcfb4c93b60798a3e1c578fb7ba63825391afcf", "expected_error": null + }, + { + "id": "comparison-operand-permutations", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a = source_b AND 1 < source_c AND 1 = 1) target = 1.\nEXECUTE." + }, + "expected_plan_case": "comparison-operand-permutations", + "expected_plan_hash": "cd8e04360419bc5b5d6db9a8800f4c58e267d2a29f0c2926cfba8d266892f021", + "expected_source_hash": "7d2a5903474103a5c317e0bf2e9eb4a1710bb5ca504b8083be6249bb8df323c6", + "expected_error": null + }, + { + "id": "reject-string-predicate", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "color", + "storage_kind": "string" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (color = 'R') target = 1." + }, + "expected_source_hash": "97dfc44da12bff2666ecc2c0d91cffa8142e3789e6f11bfe47c00c537f116920", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-arithmetic-expression", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = source_a + 1." + }, + "expected_source_hash": "eeeb5f24bc311284afb72a350b58a2787f2634465b9d6f453bd2c8817784484a", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-unsupported-command", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "SORT CASES BY source_a." + }, + "expected_source_hash": "d0aedc01794f58e2233d2ae6207490b62edc4af7f80ff53cd41845d6a38925f5", + "expected_error": "unsupported_spss_command" + }, + { + "id": "reject-leading-plus", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = +1." + }, + "expected_source_hash": "7c54709d7262ef4a751c1b8b2b4909ae7b803f4f9e50d5b269d17deff572a9ba", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-leading-zero", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = 01." + }, + "expected_source_hash": "f2513185d45ff2051dc6b1184df1a7d61abdd12aee2772975b0d0a94e00f76f2", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-nan-token", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = NaN." + }, + "expected_source_hash": "73519477685fa98ea68625926a28d7dd4cde76587a755bb754bd8983ccd09ad2", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-infinity-token", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = Infinity." + }, + "expected_source_hash": "940c30da3b6232307e976bc96a0ea64056dc9605cf717faae3580e5b4f3f0752", + "expected_error": "spss_syntax_error" } ] } From cfc62c6fe9f1f82ff3d6645880d9ead3c6103b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:13:23 +0300 Subject: [PATCH 31/78] Enforce operand and diagnostic frontend coverage --- tools/validate_transformation_plan_0_2.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 0c57a73..8de935e 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -176,6 +176,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "finite-number-rounding-exponent-underflow": None, "three-term-or-flattens-source-order": None, "strict-greater-than": None, + "comparison-operand-permutations": None, } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -227,6 +228,13 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-numeric-overflow": "spss_syntax_error", "reject-leading-decimal-point": "spss_syntax_error", "reject-trailing-decimal-point": "spss_syntax_error", + "reject-string-predicate": "expression_type_unsupported", + "reject-arithmetic-expression": "expression_type_unsupported", + "reject-unsupported-command": "unsupported_spss_command", + "reject-leading-plus": "spss_syntax_error", + "reject-leading-zero": "spss_syntax_error", + "reject-nan-token": "spss_syntax_error", + "reject-infinity-token": "spss_syntax_error", } for case in manifest["cases"]: identifier = require_string(case["id"], "0.2 frontend case id") @@ -282,6 +290,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "((source_a = 1 OR source_b = 1) OR source_c = 1)", "EXECUTE.", ), "strict-greater-than": ("source_a > 1", "EXECUTE."), + "comparison-operand-permutations": ( + "source_a = source_b", "1 < source_c", "1 = 1", "EXECUTE.", + ), }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") @@ -313,6 +324,14 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "three-term-or-flattens-source-order", "parenthesized-three-term-or-flattens-source-order", "strict-greater-than", + "comparison-operand-permutations", + "reject-string-predicate", + "reject-arithmetic-expression", + "reject-unsupported-command", + "reject-leading-plus", + "reject-leading-zero", + "reject-nan-token", + "reject-infinity-token", }, "0.2 frontend case set differs.") return len(identifiers) From d2380d0ced4058066b57f78d546b77d6ddf3b003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:20:49 +0300 Subject: [PATCH 32/78] Add subnormal and grouped metadata plans --- conformance/transformation-plan-0.2.json | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index 8797ee9..8a6f798 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -805,6 +805,80 @@ }, "expected_plan_hash": "cd8e04360419bc5b5d6db9a8800f4c58e267d2a29f0c2926cfba8d266892f021", "expected_error": null + }, + { + "id": "minimum-nonzero-subnormal", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "0000000000000001" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "fec5cad101adf699cfac0cbfbc1e0371f512f8e674345269d1de8d3a8a4538de" + }, + { + "id": "grouped-format-and-measurement-level-order", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "set_format", + "variable": "source_a", + "family": "F", + "width": 8, + "decimals": 2 + }, + { + "op": "set_format", + "variable": "source_b", + "family": "F", + "width": 10, + "decimals": 3 + }, + { + "op": "set_measurement_level", + "variable": "source_a", + "level": "ordinal" + }, + { + "op": "set_measurement_level", + "variable": "source_b", + "level": "ordinal" + }, + { + "op": "set_measurement_level", + "variable": "source_c", + "level": "scale" + }, + { + "op": "set_measurement_level", + "variable": "target", + "level": "scale" + }, + { + "op": "execute" + } + ] + }, + "expected_error": null, + "expected_plan_hash": "4b5e8d6c2e8043e25c1483b455dc497910cb66835916f6cdf7afa1eba8d334b8" } ] } From 6291af24916e584681eac3d0e906b6bcd18d5d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:21:39 +0300 Subject: [PATCH 33/78] Add subnormal and grouped metadata frontend cases --- conformance/spss-syntax-frontend-0.2.json | 64 +++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index f4361a8..e9908dc 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -726,6 +726,70 @@ }, "expected_source_hash": "940c30da3b6232307e976bc96a0ea64056dc9605cf717faae3580e5b4f3f0752", "expected_error": "spss_syntax_error" + }, + { + "id": "minimum-nonzero-subnormal", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = 5e-324.\nEXECUTE." + }, + "expected_plan_case": "minimum-nonzero-subnormal", + "expected_plan_hash": "fec5cad101adf699cfac0cbfbc1e0371f512f8e674345269d1de8d3a8a4538de", + "expected_source_hash": "d64f70960fbc1e42014c4049fdddf9c466c77e3dc896447de7a02ccaa2a8c191", + "expected_error": null + }, + { + "id": "grouped-format-and-measurement-level-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "source_b", + "storage_kind": "numeric" + }, + { + "name": "source_c", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "FORMATS source_a (F8.2) source_b (F10.3).\nVARIABLE LEVEL source_a source_b (ORDINAL) / source_c target (SCALE).\nEXECUTE." + }, + "expected_plan_case": "grouped-format-and-measurement-level-order", + "expected_plan_hash": "4b5e8d6c2e8043e25c1483b455dc497910cb66835916f6cdf7afa1eba8d334b8", + "expected_source_hash": "c9ba1298688876632af6cb435c8f363ae9d56f2475f4adf3d8ad63862e7aae20", + "expected_error": null } ] } From d22b04abba57b87dd9e6f2a67acc9a73a5e44088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:22:19 +0300 Subject: [PATCH 34/78] Cover unsupported command-form variants --- conformance/spss-syntax-frontend-0.2.json | 88 +++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index e9908dc..a71974e 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -790,6 +790,94 @@ "expected_plan_hash": "4b5e8d6c2e8043e25c1483b455dc497910cb66835916f6cdf7afa1eba8d334b8", "expected_source_hash": "c9ba1298688876632af6cb435c8f363ae9d56f2475f4adf3d8ad63862e7aae20", "expected_error": null + }, + { + "id": "reject-if-without-parentheses", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF source_a = 1 target = 1." + }, + "expected_source_hash": "543af0de6ddedd40be3b64e372a28fc897bb80db7cc7b0ec883dd24aa3de5eae", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-if-else-form", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "IF (source_a = 1) target = 1 ELSE target = 0." + }, + "expected_source_hash": "ea118668e92255019e7f70ea60c7c6aee2e1a45333d424b48a9f6c3ed3b8660a", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-non-f-format", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "FORMATS target (A8)." + }, + "expected_source_hash": "37a5fd0884cddbba67253ed76b5bc3f47a1dcd6b6acd75f207aa7f2bec40232b", + "expected_error": "invalid_format" + }, + { + "id": "reject-invalid-measurement-level", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "VARIABLE LEVEL target (UNKNOWN)." + }, + "expected_source_hash": "2bf027ec059026ac05177003ae52f9a4fb1bb994d16ba899dcebcb2df347ee61", + "expected_error": "spss_syntax_error" } ] } From c80b725a61cf3908d1bb2a48a0e4f48de6f1df68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:22:44 +0300 Subject: [PATCH 35/78] Enforce subnormal and command-form coverage --- tools/validate_transformation_plan_0_2.py | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 8de935e..dcac0b7 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -177,6 +177,8 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "three-term-or-flattens-source-order": None, "strict-greater-than": None, "comparison-operand-permutations": None, + "minimum-nonzero-subnormal": None, + "grouped-format-and-measurement-level-order": None, } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -202,6 +204,12 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: ], "Finite-number conversion golden bits differ.", ) + elif identifier == "minimum-nonzero-subnormal": + require( + case["plan"]["operations"][0]["value"]["value"]["bits"] + == "0000000000000001", + "Minimum-subnormal conversion golden bits differ.", + ) require(set(cases) == set(expected), "0.2 plan case set differs.") return cases @@ -235,6 +243,10 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-leading-zero": "spss_syntax_error", "reject-nan-token": "spss_syntax_error", "reject-infinity-token": "spss_syntax_error", + "reject-if-without-parentheses": "spss_syntax_error", + "reject-if-else-form": "spss_syntax_error", + "reject-non-f-format": "invalid_format", + "reject-invalid-measurement-level": "spss_syntax_error", } for case in manifest["cases"]: identifier = require_string(case["id"], "0.2 frontend case id") @@ -293,6 +305,12 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "comparison-operand-permutations": ( "source_a = source_b", "1 < source_c", "1 = 1", "EXECUTE.", ), + "minimum-nonzero-subnormal": ("5e-324", "EXECUTE."), + "grouped-format-and-measurement-level-order": ( + "FORMATS source_a (F8.2) source_b (F10.3).", + "VARIABLE LEVEL source_a source_b (ORDINAL) / source_c target (SCALE).", + "EXECUTE.", + ), }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") @@ -332,6 +350,12 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-leading-zero", "reject-nan-token", "reject-infinity-token", + "minimum-nonzero-subnormal", + "grouped-format-and-measurement-level-order", + "reject-if-without-parentheses", + "reject-if-else-form", + "reject-non-f-format", + "reject-invalid-measurement-level", }, "0.2 frontend case set differs.") return len(identifiers) From dd69d3e6ae38b9e1a20ef926a531c203fb589b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:29:09 +0300 Subject: [PATCH 36/78] Add negative-zero canonical plan --- conformance/transformation-plan-0.2.json | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index 8a6f798..e0867b5 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -879,6 +879,32 @@ }, "expected_error": null, "expected_plan_hash": "4b5e8d6c2e8043e25c1483b455dc497910cb66835916f6cdf7afa1eba8d334b8" + }, + { + "id": "negative-zero-canonical-positive-zero", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "0000000000000000" + } + } + }, + { + "op": "execute" + } + ] + }, + "expected_plan_hash": "0e881516aef24bbd5e3c6e2afe12a2f7f712d58f33fcba2bce2b74691bfe5a9c", + "expected_error": null } ] } From 1bea0a29d1f8baeae42741d35e84e206f5aa7434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:29:40 +0300 Subject: [PATCH 37/78] Add negative-zero frontend golden --- conformance/spss-syntax-frontend-0.2.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index a71974e..0538426 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -878,6 +878,26 @@ }, "expected_source_hash": "2bf027ec059026ac05177003ae52f9a4fb1bb994d16ba899dcebcb2df347ee61", "expected_error": "spss_syntax_error" + }, + { + "id": "negative-zero-canonical-positive-zero", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "target", + "storage_kind": "numeric" + } + ] + }, + "source_text": "COMPUTE target = -0.\nEXECUTE." + }, + "expected_plan_case": "negative-zero-canonical-positive-zero", + "expected_plan_hash": "0e881516aef24bbd5e3c6e2afe12a2f7f712d58f33fcba2bce2b74691bfe5a9c", + "expected_source_hash": "3bce3721dc895b4f58cf180cc72792494936cbaf4e1995b8c09a5ab50fee98a5", + "expected_error": null } ] } From 3a8d49d3d35362e8d0727e01116ac65f6ab34d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:30:04 +0300 Subject: [PATCH 38/78] Validate frontend identity and negative zero --- tools/validate_transformation_plan_0_2.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index dcac0b7..155daa9 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -179,6 +179,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "comparison-operand-permutations": None, "minimum-nonzero-subnormal": None, "grouped-format-and-measurement-level-order": None, + "negative-zero-canonical-positive-zero": None, } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -210,6 +211,12 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: == "0000000000000001", "Minimum-subnormal conversion golden bits differ.", ) + elif identifier == "negative-zero-canonical-positive-zero": + require( + case["plan"]["operations"][0]["value"]["value"]["bits"] + == "0000000000000000", + "Negative-zero canonical conversion bits differ.", + ) require(set(cases) == set(expected), "0.2 plan case set differs.") return cases @@ -217,7 +224,18 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) require(set(manifest) == {"manifest_version", "profile", "contract", "plan_contracts", "request_schema", "plan_schemas", "cases"}, "0.2 frontend manifest fields differ.") + require(manifest["manifest_version"] == "0.2", "0.2 frontend manifest version differs.") + require(manifest["profile"] == "OpenStatSpec SPSS-like Syntax Frontend 0.2", "0.2 frontend profile differs.") require(manifest["contract"] == "openstatspec-spss-syntax-frontend-v0.2", "0.2 frontend contract differs.") + require( + manifest["request_schema"] == "../transformation/spss-syntax-frontend-0.2.schema.json", + "0.2 frontend request-schema link differs.", + ) + request_schema_path = (FRONTEND.parent / manifest["request_schema"]).resolve() + require( + request_schema_path == FRONTEND_SCHEMA.resolve() and request_schema_path.is_file(), + "0.2 frontend request schema is missing or mismatched.", + ) require(manifest["plan_contracts"] == [ "openstatspec-transformation-plan-v0.1", "openstatspec-transformation-plan-v0.2", @@ -311,6 +329,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "VARIABLE LEVEL source_a source_b (ORDINAL) / source_c target (SCALE).", "EXECUTE.", ), + "negative-zero-canonical-positive-zero": ("COMPUTE target = -0.", "EXECUTE."), }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") @@ -356,6 +375,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-if-else-form", "reject-non-f-format", "reject-invalid-measurement-level", + "negative-zero-canonical-positive-zero", }, "0.2 frontend case set differs.") return len(identifiers) From 97d12f4484b293accf8581171515f4663b2cbb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:37:20 +0300 Subject: [PATCH 39/78] Add string assignment rejection plans --- conformance/transformation-plan-0.2.json | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index e0867b5..8c29bed 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -905,6 +905,66 @@ }, "expected_plan_hash": "0e881516aef24bbd5e3c6e2afe12a2f7f712d58f33fcba2bce2b74691bfe5a9c", "expected_error": null + }, + { + "id": "reject-string-assign-value", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "assign", + "target": "target", + "target_mode": "replace", + "value": { + "kind": "literal", + "value": { + "type": "string", + "value": "R" + } + } + } + ] + }, + "expected_plan_hash": null, + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-string-conditional-assign-value", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "string", + "value": "R" + } + } + } + ] + }, + "expected_plan_hash": null, + "expected_error": "expression_type_unsupported" } ] } From b3de86c4c7fe30495fcc22f155a40c84f96ba497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:38:02 +0300 Subject: [PATCH 40/78] Validate plan schema identity and string values --- tools/validate_transformation_plan_0_2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 155daa9..17c012a 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -160,7 +160,14 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: manifest = json.loads(PLAN.read_text(encoding="utf-8")) require(set(manifest) == {"manifest_version", "profile", "contract", "schema", "canonicalization", "cases"}, "0.2 plan manifest fields differ.") require(manifest["manifest_version"] == "0.2", "0.2 plan manifest version differs.") + require(manifest["profile"] == "OpenStatSpec Transformation Plan 0.2", "0.2 plan profile differs.") require(manifest["contract"] == "openstatspec-transformation-plan-v0.2", "0.2 plan contract differs.") + require(manifest["schema"] == "../transformation/plan-0.2.schema.json", "0.2 plan schema link differs.") + plan_schema_path = (PLAN.parent / manifest["schema"]).resolve() + require( + plan_schema_path == PLAN_SCHEMA.resolve() and plan_schema_path.is_file(), + "0.2 plan schema is missing or mismatched.", + ) require(manifest["canonicalization"] == "restricted-rfc8785-utf8-sha256", "0.2 canonicalization differs.") cases = {} expected = { @@ -180,6 +187,8 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "minimum-nonzero-subnormal": None, "grouped-format-and-measurement-level-order": None, "negative-zero-canonical-positive-zero": None, + "reject-string-assign-value": "expression_type_unsupported", + "reject-string-conditional-assign-value": "expression_type_unsupported", } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") From c128e1464fb837baba71ab4c5bd05cf6644929ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:43:13 +0300 Subject: [PATCH 41/78] Add string target rejection frontends --- conformance/spss-syntax-frontend-0.2.json | 66 +++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 0538426..233990f 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -898,6 +898,72 @@ "expected_plan_hash": "0e881516aef24bbd5e3c6e2afe12a2f7f712d58f33fcba2bce2b74691bfe5a9c", "expected_source_hash": "3bce3721dc895b4f58cf180cc72792494936cbaf4e1995b8c09a5ab50fee98a5", "expected_error": null + }, + { + "id": "reject-string-target-compute", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "color", + "storage_kind": "string" + } + ] + }, + "source_text": "COMPUTE color = 1." + }, + "expected_source_hash": "9e4d4b5f23e024b0d1dd70e03465a964f38aa17879e9701fe537c553645dda89", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-string-target-conditional", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "color", + "storage_kind": "string" + } + ] + }, + "source_text": "IF (source_a = 1) color = 1." + }, + "expected_source_hash": "a0c3d748f308793c5c529e5b7b67743979bd1a9ae5e2c0235a66951202930eb1", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-string-target-format", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "source_a", + "storage_kind": "numeric" + }, + { + "name": "color", + "storage_kind": "string" + } + ] + }, + "source_text": "FORMATS color (F8.2)." + }, + "expected_source_hash": "84514670d918d0bb42619c32c75aa85225ad9e4f24ac2a0df1cd584f1deb7d97", + "expected_error": "expression_type_unsupported" } ] } From eb9ef8baa59b7e03d9d9291395dcbabe086203e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:43:29 +0300 Subject: [PATCH 42/78] Enforce string target rejection coverage --- tools/validate_transformation_plan_0_2.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 17c012a..b2c0a2f 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -274,6 +274,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-if-else-form": "spss_syntax_error", "reject-non-f-format": "invalid_format", "reject-invalid-measurement-level": "spss_syntax_error", + "reject-string-target-compute": "expression_type_unsupported", + "reject-string-target-conditional": "expression_type_unsupported", + "reject-string-target-format": "expression_type_unsupported", } for case in manifest["cases"]: identifier = require_string(case["id"], "0.2 frontend case id") @@ -385,6 +388,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-non-f-format", "reject-invalid-measurement-level", "negative-zero-canonical-positive-zero", + "reject-string-target-compute", + "reject-string-target-conditional", + "reject-string-target-format", }, "0.2 frontend case set differs.") return len(identifiers) From ebbf668a01849b0b3b1c4037ebfb574fbca539de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:49:45 +0300 Subject: [PATCH 43/78] Add metadata-only v0.1 compatibility case --- conformance/spss-syntax-frontend-0.2.json | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 233990f..284275a 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -964,6 +964,35 @@ }, "expected_source_hash": "84514670d918d0bb42619c32c75aa85225ad9e4f24ac2a0df1cd584f1deb7d97", "expected_error": "expression_type_unsupported" + }, + { + "id": "metadata-only-value-labels-retains-v0.1-plan", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "a", + "storage_kind": "numeric" + }, + { + "name": "b", + "storage_kind": "numeric" + }, + { + "name": "color", + "storage_kind": "string" + } + ] + }, + "source_text": "VALUE LABELS a b 0 'No' 1 'Yes' / color 'R' 'Red'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "value-label-varlists-and-groups", + "expected_plan_hash": "632e2075874257f7bf2400d6a2642f9336a9fc4ddf9664a20150a333d32414ad", + "expected_source_hash": "94954e5cdeada2bdee1f862078cb89e620bea3a7cd7c0e24d8c735786d469174", + "expected_error": null } ] } From 621ed024013b6e4811180d0b5677c8e35cb74c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:49:59 +0300 Subject: [PATCH 44/78] Add variable missing propagation binding case --- conformance/in-place-transformation-0.2.json | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 5ae2817..7f0328a 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -184,6 +184,38 @@ "target_mode": "create", "expected_error": "schema_change_not_atomic", "mutation_started": false + }, + { + "id": "dolt-preprovisioned-target-variable-missing-propagation", + "database_profile": "dolt", + "target_preprovisioned": true, + "assign": { + "target": "target", + "value_variable": "source_a" + }, + "before_rows": [ + { + "__case_ordinal": 1, + "source_a": null, + "target": 7 + }, + { + "__case_ordinal": 2, + "source_a": 2, + "target": 7 + } + ], + "after_rows": [ + { + "__case_ordinal": 1, + "target": null + }, + { + "__case_ordinal": 2, + "target": 2 + } + ], + "expected_error": null } ] } From 1f3625e2ae85b477b5d9b3f14dce5e4edde581ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:50:24 +0300 Subject: [PATCH 45/78] Validate binding identity and compatibility cases --- tools/validate_transformation_plan_0_2.py | 38 ++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index b2c0a2f..068a19b 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -346,7 +346,13 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") else: - require(identifier == "old-subset-retains-v0.1-plan", f"Unexpected 0.2 frontend case: {identifier}") + require( + identifier in { + "old-subset-retains-v0.1-plan", + "metadata-only-value-labels-retains-v0.1-plan", + }, + f"Unexpected 0.2 frontend case: {identifier}", + ) require(set(case) == {"id", "request", "expected_plan_contract", "expected_plan_case_0_1", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: 0.1 compatibility fields differ.") legacy_case = legacy_cases.get(case["expected_plan_case_0_1"]) require(legacy_case is not None and "expected_plan" in legacy_case, f"{identifier}: immutable 0.1 plan fixture missing.") @@ -391,6 +397,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-string-target-compute", "reject-string-target-conditional", "reject-string-target-format", + "metadata-only-value-labels-retains-v0.1-plan", }, "0.2 frontend case set differs.") return len(identifiers) @@ -402,6 +409,10 @@ def validate_in_place() -> None: "0.2 binding manifest fields differ.", ) require(manifest["manifest_version"] == "0.2", "0.2 binding manifest version differs.") + require( + manifest["profile"] == "OpenStatSpec In-Place Transformation Binding 0.2", + "0.2 binding profile differs.", + ) require(manifest["contract"] == "openstatspec-in-place-transformation-v0.2", "0.2 binding contract differs.") require( manifest["audit_schema"] == "../sql/transformation-plan-profile-schema.sql", @@ -423,6 +434,7 @@ def validate_in_place() -> None: expected_ids = { "dolt-preprovisioned-target-sequential-null-semantics", "dolt-preprovisioned-target-or-null-semantics", + "dolt-preprovisioned-target-variable-missing-propagation", } | { f"{profile}-create-target-fails-before-mutation" for profile in ("dolt", "mysql", "mariadb") @@ -490,6 +502,30 @@ def validate_in_place() -> None: ], "Dolt OR three-valued outputs differ.") require(or_case["expected_error"] is None, "Dolt OR-null case unexpectedly fails.") + missing_case = cases["dolt-preprovisioned-target-variable-missing-propagation"] + require(set(missing_case) == { + "id", "database_profile", "target_preprovisioned", "assign", + "before_rows", "after_rows", "expected_error", + }, "Dolt missing-propagation case fields differ.") + require( + missing_case["database_profile"] == "dolt" + and missing_case["target_preprovisioned"] is True, + "Dolt missing-propagation identity differs.", + ) + require( + missing_case["assign"] == {"target": "target", "value_variable": "source_a"}, + "Dolt missing-propagation assignment differs.", + ) + require(missing_case["before_rows"] == [ + {"__case_ordinal": 1, "source_a": None, "target": 7}, + {"__case_ordinal": 2, "source_a": 2, "target": 7}, + ], "Dolt missing-propagation inputs differ.") + require(missing_case["after_rows"] == [ + {"__case_ordinal": 1, "target": None}, + {"__case_ordinal": 2, "target": 2}, + ], "Dolt variable assignment does not preserve missing.") + require(missing_case["expected_error"] is None, "Dolt missing-propagation case unexpectedly fails.") + for profile in ("dolt", "mysql", "mariadb"): case = cases[f"{profile}-create-target-fails-before-mutation"] require(set(case) == {"id", "database_profile", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") From 6f55049bb5338898110d59bb08bc1c2d046648a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:51:50 +0300 Subject: [PATCH 46/78] Add variable-label-only v0.1 compatibility golden --- conformance/spss-syntax-frontend-0.2.json | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 284275a..6ef69ae 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -993,6 +993,46 @@ "expected_plan_hash": "632e2075874257f7bf2400d6a2642f9336a9fc4ddf9664a20150a333d32414ad", "expected_source_hash": "94954e5cdeada2bdee1f862078cb89e620bea3a7cd7c0e24d8c735786d469174", "expected_error": null + }, + { + "id": "metadata-only-variable-labels-retains-v0.1-plan", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "a", + "storage_kind": "numeric" + }, + { + "name": "b", + "storage_kind": "numeric" + } + ] + }, + "source_text": "VARIABLE LABELS a 'Alpha' b 'Beta'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + { + "op": "set_variable_label", + "variable": "a", + "label": "Alpha" + }, + { + "op": "set_variable_label", + "variable": "b", + "label": "Beta" + } + ] + }, + "expected_plan_hash": "a663a7160957447ab125cbe8a106ea3419b6eb0855ea0287dea5ce71b6b08f20", + "expected_source_hash": "b70575af343e328875753476282962fca4224de98e46c656f84a415dc033b162", + "expected_error": null } ] } From 73079981dddb65dc764685ccd943ee5a1e2140f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:53:07 +0300 Subject: [PATCH 47/78] Enforce variable-label v0.1 compatibility --- tools/validate_transformation_plan_0_2.py | 43 +++++++++++++++-------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 068a19b..b54741d 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -346,21 +346,31 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") else: - require( - identifier in { - "old-subset-retains-v0.1-plan", - "metadata-only-value-labels-retains-v0.1-plan", - }, - f"Unexpected 0.2 frontend case: {identifier}", - ) - require(set(case) == {"id", "request", "expected_plan_contract", "expected_plan_case_0_1", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: 0.1 compatibility fields differ.") - legacy_case = legacy_cases.get(case["expected_plan_case_0_1"]) - require(legacy_case is not None and "expected_plan" in legacy_case, f"{identifier}: immutable 0.1 plan fixture missing.") - legacy_plan = legacy_case["expected_plan"] - require(validate_plan_0_1(legacy_plan, identifier + ".0.1_plan") is None, f"{identifier}: immutable 0.1 plan is invalid.") - require(case["expected_plan_contract"] == legacy_plan["contract"], f"{identifier}: 0.1 contract differs.") - require(case["expected_plan_hash"] == legacy_case["expected_plan_hash"], f"{identifier}: 0.1 fixture hash differs.") - require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") + if identifier == "metadata-only-variable-labels-retains-v0.1-plan": + require(set(case) == { + "id", "request", "expected_plan_contract", "expected_plan_0_1", + "expected_plan_hash", "expected_source_hash", "expected_error", + }, f"{identifier}: embedded 0.1 compatibility fields differ.") + legacy_plan = case["expected_plan_0_1"] + require(validate_plan_0_1(legacy_plan, identifier + ".0.1_plan") is None, f"{identifier}: embedded 0.1 plan is invalid.") + require(case["expected_plan_contract"] == legacy_plan["contract"], f"{identifier}: 0.1 contract differs.") + require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") + else: + require( + identifier in { + "old-subset-retains-v0.1-plan", + "metadata-only-value-labels-retains-v0.1-plan", + }, + f"Unexpected 0.2 frontend case: {identifier}", + ) + require(set(case) == {"id", "request", "expected_plan_contract", "expected_plan_case_0_1", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: 0.1 compatibility fields differ.") + legacy_case = legacy_cases.get(case["expected_plan_case_0_1"]) + require(legacy_case is not None and "expected_plan" in legacy_case, f"{identifier}: immutable 0.1 plan fixture missing.") + legacy_plan = legacy_case["expected_plan"] + require(validate_plan_0_1(legacy_plan, identifier + ".0.1_plan") is None, f"{identifier}: immutable 0.1 plan is invalid.") + require(case["expected_plan_contract"] == legacy_plan["contract"], f"{identifier}: 0.1 contract differs.") + require(case["expected_plan_hash"] == legacy_case["expected_plan_hash"], f"{identifier}: 0.1 fixture hash differs.") + require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") require(identifiers == { "compute-if-labels-format-level-execute-existing-target", "three-term-and-flattens-source-order", @@ -398,6 +408,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-string-target-conditional", "reject-string-target-format", "metadata-only-value-labels-retains-v0.1-plan", + "metadata-only-variable-labels-retains-v0.1-plan", }, "0.2 frontend case set differs.") return len(identifiers) @@ -556,6 +567,8 @@ def validate_links() -> None: legacy_plan_validator.validate(case["expected_plan"]) for case in frontend_manifest["cases"]: frontend_validator.validate(case["request"]) + if "expected_plan_0_1" in case: + legacy_plan_validator.validate(case["expected_plan_0_1"]) require(plan_schema["$id"].endswith("transformation-plan-0.2.schema.json"), "0.2 plan schema ID differs.") require(frontend_schema["$id"].endswith("spss-syntax-frontend-0.2.schema.json"), "0.2 frontend schema ID differs.") binding = (ROOT / "docs/transformation-plan-sql-binding-0.2.md").read_text(encoding="utf-8") From 57df5d3affb04e53f173bbdfc76fb602fb3def67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:00:58 +0300 Subject: [PATCH 48/78] Clarify string measurement-level targets --- docs/spss-syntax-frontend-profile-0.2.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md index 0782356..cab875c 100644 --- a/docs/spss-syntax-frontend-profile-0.2.md +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -63,8 +63,9 @@ underflowed zero, is canonicalized to positive-zero bits NaN and infinity can never be emitted. A numeric variable is resolved at the point its command is lowered, including -variables created by preceding commands. All referenced variables and assigned -values in this subset MUST be numeric. +variables created by preceding commands. All variables used as expression operands, assignment targets, or `FORMATS` +targets MUST be numeric. `VARIABLE LEVEL` targets MAY be numeric or string +variables; measurement level is metadata and does not coerce stored values. ## Added commands From de7ecb65c2b669987e8d1b9306554ec05f0adcf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:01:36 +0300 Subject: [PATCH 49/78] Add string measurement-level plan --- conformance/transformation-plan-0.2.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index 8c29bed..96a49e8 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -965,6 +965,25 @@ }, "expected_plan_hash": null, "expected_error": "expression_type_unsupported" + }, + { + "id": "string-measurement-level-target", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "set_measurement_level", + "variable": "color", + "level": "nominal" + }, + { + "op": "execute" + } + ] + }, + "expected_plan_hash": "5693460d700531e6e9162a9cf112bd276b475f9d414bd58c283c6f2db8fdbf9d", + "expected_error": null } ] } From c4f5f26d41f063d10f43953957ab201316a2733e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:01:41 +0300 Subject: [PATCH 50/78] Add string measurement-level frontend --- conformance/spss-syntax-frontend-0.2.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 6ef69ae..0dca41b 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -1033,6 +1033,26 @@ "expected_plan_hash": "a663a7160957447ab125cbe8a106ea3419b6eb0855ea0287dea5ce71b6b08f20", "expected_source_hash": "b70575af343e328875753476282962fca4224de98e46c656f84a415dc033b162", "expected_error": null + }, + { + "id": "string-measurement-level-target", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.2", + "input_alias": "parent", + "input_schema": { + "variables": [ + { + "name": "color", + "storage_kind": "string" + } + ] + }, + "source_text": "VARIABLE LEVEL color (NOMINAL).\nEXECUTE." + }, + "expected_plan_case": "string-measurement-level-target", + "expected_plan_hash": "5693460d700531e6e9162a9cf112bd276b475f9d414bd58c283c6f2db8fdbf9d", + "expected_source_hash": "0887befe1d3a49a4e196976eb5925688cd402bbbfb6efae602a27a1fe0c8196e", + "expected_error": null } ] } From 097cf5990671d8329ba9acdac6d39d7281ba5f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:02:04 +0300 Subject: [PATCH 51/78] Enforce string measurement-level behavior --- tools/validate_transformation_plan_0_2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index b54741d..dc7599a 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -189,6 +189,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "negative-zero-canonical-positive-zero": None, "reject-string-assign-value": "expression_type_unsupported", "reject-string-conditional-assign-value": "expression_type_unsupported", + "string-measurement-level-target": None, } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -342,6 +343,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "EXECUTE.", ), "negative-zero-canonical-positive-zero": ("COMPUTE target = -0.", "EXECUTE."), + "string-measurement-level-target": ( + "VARIABLE LEVEL color (NOMINAL).", "EXECUTE.", + ), }[identifier] for token in required_tokens: require(token in source, f"{identifier}: bounded command coverage missing: {token}") @@ -409,6 +413,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "reject-string-target-format", "metadata-only-value-labels-retains-v0.1-plan", "metadata-only-variable-labels-retains-v0.1-plan", + "string-measurement-level-target", }, "0.2 frontend case set differs.") return len(identifiers) From 65c9b698be235006f23823152c3f28d5bd44e990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:09:26 +0300 Subject: [PATCH 52/78] Add nested boolean rejection plan --- conformance/transformation-plan-0.2.json | 79 ++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index 96a49e8..5b7cfd7 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -984,6 +984,85 @@ }, "expected_plan_hash": "5693460d700531e6e9162a9cf112bd276b475f9d414bd58c283c6f2db8fdbf9d", "expected_error": null + }, + { + "id": "reject-nested-same-operator-boolean", + "plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + { + "op": "conditional_assign", + "condition": { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "boolean", + "operator": "and", + "operands": [ + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_a" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_b" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + { + "expression": "comparison", + "left": { + "kind": "variable", + "variable": "source_c" + }, + "operator": "=", + "right": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "target": "target", + "value": { + "kind": "literal", + "value": { + "type": "binary64", + "bits": "3ff0000000000000" + } + } + } + ] + }, + "expected_plan_hash": null, + "expected_error": "noncanonical_boolean_shape" } ] } From 6bb8c2aac1451a31a525f3aad232daefa8f45ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:09:41 +0300 Subject: [PATCH 53/78] Enforce non-empty binding actor --- conformance/in-place-transformation-0.2.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 7f0328a..5920097 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -99,7 +99,8 @@ "operation_count": 7, "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", - "dolt_head_after": "provisioning-commit" + "dolt_head_after": "provisioning-commit", + "actor": "conformance-runner" }, "forbidden_artifacts": [ "derived_dataset_row", @@ -216,6 +217,13 @@ } ], "expected_error": null + }, + { + "id": "dolt-empty-actor-fails-before-mutation", + "database_profile": "dolt", + "actor": "", + "expected_error": "actor_required", + "mutation_started": false } ] } From 527a0e71d8826da6080ed331f194722ee0059270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:10:21 +0300 Subject: [PATCH 54/78] Enforce actor and canonical boolean rejection --- tools/validate_transformation_plan_0_2.py | 37 +++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index dc7599a..fa85d40 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -74,18 +74,23 @@ def predicate(value: object, context: str) -> bool: require(value["operator"] in {"and", "or"}, f"{context}: boolean operator differs.") children = value["operands"] require(isinstance(children, list) and len(children) >= 2, f"{context}: boolean operands differ.") - for index, child in enumerate(children): - require( - not ( - isinstance(child, dict) - and child.get("expression") == "boolean" - and child.get("operator") == value["operator"] - ), - f"{context}.operands[{index}]: same-operator boolean chain must be flattened.", - ) return any(predicate(child, f"{context}.operands[{index}]") for index, child in enumerate(children)) +def has_nested_same_operator(value: object) -> bool: + if not isinstance(value, dict) or value.get("expression") != "boolean": + return False + operator = value.get("operator") + children = value.get("operands", []) + return any( + isinstance(child, dict) + and child.get("expression") == "boolean" + and child.get("operator") == operator + or has_nested_same_operator(child) + for child in children + ) + + def validate_plan(plan: object, context: str) -> str | None: require(isinstance(plan, dict) and set(plan) == {"contract", "input_alias", "operations"}, f"{context}: plan fields differ.") require(plan["contract"] == "openstatspec-transformation-plan-v0.2", f"{context}: contract differs.") @@ -113,6 +118,8 @@ def validate_plan(plan: object, context: str) -> str | None: require(set(operation) == {"op", "condition", "target", "value"}, f"{op_context}: conditional fields differ.") target = require_string(operation["target"], op_context + ".target") string_predicate = predicate(operation["condition"], op_context + ".condition") + if has_nested_same_operator(operation["condition"]): + semantic_error = semantic_error or "noncanonical_boolean_shape" if target.startswith("__"): semantic_error = semantic_error or "reserved_target_name" string_value = operand(operation["value"], op_context + ".value") == "string" @@ -190,6 +197,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "reject-string-assign-value": "expression_type_unsupported", "reject-string-conditional-assign-value": "expression_type_unsupported", "string-measurement-level-target": None, + "reject-nested-same-operator-boolean": "noncanonical_boolean_shape", } for case in manifest["cases"]: require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") @@ -451,6 +459,7 @@ def validate_in_place() -> None: "dolt-preprovisioned-target-sequential-null-semantics", "dolt-preprovisioned-target-or-null-semantics", "dolt-preprovisioned-target-variable-missing-propagation", + "dolt-empty-actor-fails-before-mutation", } | { f"{profile}-create-target-fails-before-mutation" for profile in ("dolt", "mysql", "mariadb") @@ -484,6 +493,7 @@ def validate_in_place() -> None: require(after["dolt_commit_performed"] is False, "Apply performs DOLT_COMMIT.") require(success["expected_audit"] == { "contract_id": manifest["contract"], + "actor": "conformance-runner", "operation_count": 7, "dolt_branch": before["dolt_branch"], "dolt_head_before": before["dolt_head"], @@ -542,6 +552,15 @@ def validate_in_place() -> None: ], "Dolt variable assignment does not preserve missing.") require(missing_case["expected_error"] is None, "Dolt missing-propagation case unexpectedly fails.") + actor_case = cases["dolt-empty-actor-fails-before-mutation"] + require(actor_case == { + "id": "dolt-empty-actor-fails-before-mutation", + "database_profile": "dolt", + "actor": "", + "expected_error": "actor_required", + "mutation_started": False, + }, "Dolt empty-actor rejection differs.") + for profile in ("dolt", "mysql", "mariadb"): case = cases[f"{profile}-create-target-fails-before-mutation"] require(set(case) == {"id", "database_profile", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") From ac066c7ddd3de75a8854b6f749c1e08e00b7ac02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:19:25 +0300 Subject: [PATCH 55/78] Validate successful plan links and atomic create binding --- tools/validate_transformation_plan_0_2.py | 72 ++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index fa85d40..9b3246f 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -309,7 +309,14 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: require(case["expected_error"] is None, f"{identifier}: frontend success claims an error.") require(set(case) == {"id", "request", "expected_plan_case", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: frontend success fields differ.") require(case["expected_plan_case"] in plan_cases, f"{identifier}: plan fixture missing.") - require(case["expected_plan_hash"] == plan_cases[case["expected_plan_case"]]["expected_plan_hash"], f"{identifier}: plan hash link differs.") + plan_case = plan_cases[case["expected_plan_case"]] + require( + plan_case["expected_error"] is None + and isinstance(plan_case["expected_plan_hash"], str) + and len(plan_case["expected_plan_hash"]) == 64, + f"{identifier}: frontend success links an invalid plan fixture.", + ) + require(case["expected_plan_hash"] == plan_case["expected_plan_hash"], f"{identifier}: plan hash link differs.") required_tokens = { "compute-if-labels-format-level-execute-existing-target": ( "COMPUTE ", "IF (", " AND ", "FORMATS ", "VARIABLE LEVEL ", "EXECUTE.", @@ -460,6 +467,7 @@ def validate_in_place() -> None: "dolt-preprovisioned-target-or-null-semantics", "dolt-preprovisioned-target-variable-missing-propagation", "dolt-empty-actor-fails-before-mutation", + "sqlite-create-target-atomic-success", } | { f"{profile}-create-target-fails-before-mutation" for profile in ("dolt", "mysql", "mariadb") @@ -561,6 +569,68 @@ def validate_in_place() -> None: "mutation_started": False, }, "Dolt empty-actor rejection differs.") + create_case = cases["sqlite-create-target-atomic-success"] + require(set(create_case) == { + "id", "database_profile", "target_mode", "operation", "before", "after", + "transaction_boundary", "rollback_probe", "expected_error", + }, "SQLite create-target success fields differ.") + require( + create_case["database_profile"] == "sqlite" + and create_case["target_mode"] == "create" + and create_case["expected_error"] is None, + "SQLite create-target success identity differs.", + ) + require(create_case["operation"] == { + "op": "assign", + "target": "target", + "value_variable": "source_a", + }, "SQLite create-target operation differs.") + create_before, create_after = create_case["before"], create_case["after"] + for field in ( + "dataset_id", "physical_table_schema", "physical_table_name", + "dataset_count", "persistent_data_table_count", "case_count", + ): + require(create_before[field] == create_after[field], f"SQLite create-target changes {field}.") + require( + create_before["dataset_count"] == create_after["dataset_count"] == 1 + and create_before["persistent_data_table_count"] == create_after["persistent_data_table_count"] == 1, + "SQLite create-target creates a dataset or persistent data table.", + ) + require(create_before["variable_count"] == 2 and create_after["variable_count"] == 3, "SQLite create-target variable count differs.") + require(create_before["target"] == { + "physical_column_present": False, + "catalog_variable_present": False, + }, "SQLite create-target precondition differs.") + require(create_after["target"] == { + "physical_column_present": True, + "catalog_variable_present": True, + "ordinal": 3, + "initial_metadata": { + "variable_label": None, + "value_labels": [], + "format": None, + "measurement_level": None, + }, + }, "SQLite create-target column/catalog result differs.") + require(create_before["rows"] == [ + {"__case_ordinal": 1, "source_a": 2}, + {"__case_ordinal": 2, "source_a": None}, + ], "SQLite create-target inputs differ.") + require(create_after["rows"] == [ + {"__case_ordinal": 1, "source_a": 2, "target": 2}, + {"__case_ordinal": 2, "source_a": None, "target": None}, + ], "SQLite create-target values or case order differ.") + require(create_case["transaction_boundary"] == [ + "physical_schema", "row_values", "catalog", "compact_audit", + ], "SQLite create-target transaction boundary differs.") + require(create_case["rollback_probe"] == { + "failure_point": "after_catalog_create_before_audit", + "physical_column_present": False, + "catalog_variable_present": False, + "row_values_changed": False, + "audit_row_present": False, + }, "SQLite create-target rollback proof differs.") + for profile in ("dolt", "mysql", "mariadb"): case = cases[f"{profile}-create-target-fails-before-mutation"] require(set(case) == {"id", "database_profile", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") From d40bed1342adc78a0677fae9af67bd50880316ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:19:47 +0300 Subject: [PATCH 56/78] Add SQLite atomic create-target conformance --- conformance/in-place-transformation-0.2.json | 79 ++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 5920097..24ff09c 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -224,6 +224,85 @@ "actor": "", "expected_error": "actor_required", "mutation_started": false + }, + { + "id": "sqlite-create-target-atomic-success", + "database_profile": "sqlite", + "target_mode": "create", + "operation": { + "op": "assign", + "target": "target", + "value_variable": "source_a" + }, + "before": { + "dataset_id": "22222222-2222-4222-8222-222222222222", + "physical_table_schema": null, + "physical_table_name": "data_atomic_create", + "dataset_count": 1, + "persistent_data_table_count": 1, + "variable_count": 2, + "case_count": 2, + "target": { + "physical_column_present": false, + "catalog_variable_present": false + }, + "rows": [ + { + "__case_ordinal": 1, + "source_a": 2 + }, + { + "__case_ordinal": 2, + "source_a": null + } + ] + }, + "after": { + "dataset_id": "22222222-2222-4222-8222-222222222222", + "physical_table_schema": null, + "physical_table_name": "data_atomic_create", + "dataset_count": 1, + "persistent_data_table_count": 1, + "variable_count": 3, + "case_count": 2, + "target": { + "physical_column_present": true, + "catalog_variable_present": true, + "ordinal": 3, + "initial_metadata": { + "variable_label": null, + "value_labels": [], + "format": null, + "measurement_level": null + } + }, + "rows": [ + { + "__case_ordinal": 1, + "source_a": 2, + "target": 2 + }, + { + "__case_ordinal": 2, + "source_a": null, + "target": null + } + ] + }, + "transaction_boundary": [ + "physical_schema", + "row_values", + "catalog", + "compact_audit" + ], + "rollback_probe": { + "failure_point": "after_catalog_create_before_audit", + "physical_column_present": false, + "catalog_variable_present": false, + "row_values_changed": false, + "audit_row_present": false + }, + "expected_error": null } ] } From 2e7e25ba0a894da8074727e3ca4f39422c86b668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:28:06 +0300 Subject: [PATCH 57/78] Cover actor and conditional missing semantics --- tools/validate_transformation_plan_0_2.py | 39 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 9b3246f..51fdbe3 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -466,6 +466,7 @@ def validate_in_place() -> None: "dolt-preprovisioned-target-sequential-null-semantics", "dolt-preprovisioned-target-or-null-semantics", "dolt-preprovisioned-target-variable-missing-propagation", + "dolt-preprovisioned-target-conditional-variable-missing-propagation", "dolt-empty-actor-fails-before-mutation", "sqlite-create-target-atomic-success", } | { @@ -560,6 +561,31 @@ def validate_in_place() -> None: ], "Dolt variable assignment does not preserve missing.") require(missing_case["expected_error"] is None, "Dolt missing-propagation case unexpectedly fails.") + conditional_missing_case = cases["dolt-preprovisioned-target-conditional-variable-missing-propagation"] + require(set(conditional_missing_case) == { + "id", "database_profile", "target_preprovisioned", "conditional_assign", + "before_rows", "after_rows", "expected_error", + }, "Dolt conditional missing-propagation fields differ.") + require( + conditional_missing_case["database_profile"] == "dolt" + and conditional_missing_case["target_preprovisioned"] is True, + "Dolt conditional missing-propagation identity differs.", + ) + require(conditional_missing_case["conditional_assign"] == { + "target": "target", + "value_variable": "source_a", + "predicate": "source_b = 1", + }, "Dolt conditional missing-propagation operation differs.") + require(conditional_missing_case["before_rows"] == [ + {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 7}, + {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 7}, + ], "Dolt conditional missing-propagation inputs differ.") + require(conditional_missing_case["after_rows"] == [ + {"__case_ordinal": 1, "target": None}, + {"__case_ordinal": 2, "target": 7}, + ], "Dolt conditional assignment does not preserve a missing RHS.") + require(conditional_missing_case["expected_error"] is None, "Dolt conditional missing-propagation case unexpectedly fails.") + actor_case = cases["dolt-empty-actor-fails-before-mutation"] require(actor_case == { "id": "dolt-empty-actor-fails-before-mutation", @@ -571,12 +597,13 @@ def validate_in_place() -> None: create_case = cases["sqlite-create-target-atomic-success"] require(set(create_case) == { - "id", "database_profile", "target_mode", "operation", "before", "after", - "transaction_boundary", "rollback_probe", "expected_error", + "id", "database_profile", "target_mode", "actor", "operation", "before", "after", + "transaction_boundary", "rollback_probe", "expected_audit", "expected_error", }, "SQLite create-target success fields differ.") require( create_case["database_profile"] == "sqlite" and create_case["target_mode"] == "create" + and create_case["actor"] == "conformance-runner" and create_case["expected_error"] is None, "SQLite create-target success identity differs.", ) @@ -608,6 +635,7 @@ def validate_in_place() -> None: "initial_metadata": { "variable_label": None, "value_labels": [], + "missing_values": [], "format": None, "measurement_level": None, }, @@ -623,6 +651,13 @@ def validate_in_place() -> None: require(create_case["transaction_boundary"] == [ "physical_schema", "row_values", "catalog", "compact_audit", ], "SQLite create-target transaction boundary differs.") + require(create_case["expected_audit"] == { + "contract_id": manifest["contract"], + "database_profile": "sqlite", + "actor": create_case["actor"], + "status": "succeeded", + "operation_count": 1, + }, "SQLite create-target audit expectation differs.") require(create_case["rollback_probe"] == { "failure_point": "after_catalog_create_before_audit", "physical_column_present": False, From 5ded53eb02725954e0aa4aefba30e7b411bcd0e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:28:24 +0300 Subject: [PATCH 58/78] Add actor and missing-value binding goldens --- conformance/in-place-transformation-0.2.json | 48 +++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 24ff09c..1a222eb 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -218,6 +218,41 @@ ], "expected_error": null }, + { + "id": "dolt-preprovisioned-target-conditional-variable-missing-propagation", + "database_profile": "dolt", + "target_preprovisioned": true, + "conditional_assign": { + "target": "target", + "value_variable": "source_a", + "predicate": "source_b = 1" + }, + "before_rows": [ + { + "__case_ordinal": 1, + "source_a": null, + "source_b": 1, + "target": 7 + }, + { + "__case_ordinal": 2, + "source_a": null, + "source_b": 0, + "target": 7 + } + ], + "after_rows": [ + { + "__case_ordinal": 1, + "target": null + }, + { + "__case_ordinal": 2, + "target": 7 + } + ], + "expected_error": null + }, { "id": "dolt-empty-actor-fails-before-mutation", "database_profile": "dolt", @@ -273,7 +308,8 @@ "variable_label": null, "value_labels": [], "format": null, - "measurement_level": null + "measurement_level": null, + "missing_values": [] } }, "rows": [ @@ -302,7 +338,15 @@ "row_values_changed": false, "audit_row_present": false }, - "expected_error": null + "expected_error": null, + "actor": "conformance-runner", + "expected_audit": { + "contract_id": "openstatspec-in-place-transformation-v0.2", + "database_profile": "sqlite", + "actor": "conformance-runner", + "status": "succeeded", + "operation_count": 1 + } } ] } From 8514d8a6dd9ba66ffe40c73a8e31b597bea2ab82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:48:03 +0300 Subject: [PATCH 59/78] Bind Dolt audit hashes to applied fixtures --- conformance/in-place-transformation-0.2.json | 4 ++++ tools/validate_transformation_plan_0_2.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 1a222eb..7e2b697 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -7,6 +7,8 @@ { "id": "dolt-preprovisioned-target-sequential-null-semantics", "database_profile": "dolt", + "applied_plan_case": "sequential-conditional-binary-existing-target", + "applied_frontend_case": "compute-if-labels-format-level-execute-existing-target", "before": { "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_name": "data_synthetic", @@ -96,6 +98,8 @@ }, "expected_audit": { "contract_id": "openstatspec-in-place-transformation-v0.2", + "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", + "source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481", "operation_count": 7, "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 51fdbe3..622385f 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -433,8 +433,10 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: return len(identifiers) -def validate_in_place() -> None: +def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: manifest = json.loads(IN_PLACE.read_text(encoding="utf-8")) + frontend_manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) + frontend_cases = {case["id"]: case for case in frontend_manifest["cases"]} require( set(manifest) == {"manifest_version", "profile", "contract", "audit_schema", "cases"}, "0.2 binding manifest fields differ.", @@ -477,10 +479,12 @@ def validate_in_place() -> None: success = cases["dolt-preprovisioned-target-sequential-null-semantics"] require(set(success) == { - "id", "database_profile", "before", "after", "expected_audit", - "forbidden_artifacts", "required_audit_fields", "expected_error", + "id", "database_profile", "applied_plan_case", "applied_frontend_case", "before", "after", + "expected_audit", "forbidden_artifacts", "required_audit_fields", "expected_error", }, "0.2 binding success fields differ.") require(success["database_profile"] == "dolt" and success["expected_error"] is None, "Dolt success identity differs.") + require(success["applied_plan_case"] in plan_cases, "Dolt applied plan fixture missing.") + require(success["applied_frontend_case"] in frontend_cases, "Dolt applied frontend fixture missing.") before, after = success["before"], success["after"] for field in ( "dataset_id", "physical_table_schema", "physical_table_name", @@ -504,6 +508,8 @@ def validate_in_place() -> None: "contract_id": manifest["contract"], "actor": "conformance-runner", "operation_count": 7, + "plan_hash": plan_cases[success["applied_plan_case"]]["expected_plan_hash"], + "source_hash": frontend_cases[success["applied_frontend_case"]]["expected_source_hash"], "dolt_branch": before["dolt_branch"], "dolt_head_before": before["dolt_head"], "dolt_head_after": after["dolt_head"], @@ -711,7 +717,7 @@ def validate_links() -> None: def validate_all() -> tuple[int, int]: plan_cases = validate_plan_manifest() frontend_count = validate_frontend(plan_cases) - validate_in_place() + validate_in_place(plan_cases) validate_links() return len(plan_cases), frontend_count From 6cfa4f7d40a13baf2b8da7cf02b0502f05c08152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:09:02 +0300 Subject: [PATCH 60/78] Specify SQLite audit links and boolean diagnostic --- conformance/in-place-transformation-0.2.json | 6 +++++- docs/transformation-plan-profile-0.2.md | 6 ++++-- tools/validate_transformation_plan_0_2.py | 6 +++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 7e2b697..c0231f3 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -267,6 +267,8 @@ { "id": "sqlite-create-target-atomic-success", "database_profile": "sqlite", + "applied_plan_case": "sequential-conditional-binary-existing-target", + "applied_frontend_case": "compute-if-labels-format-level-execute-existing-target", "target_mode": "create", "operation": { "op": "assign", @@ -349,7 +351,9 @@ "database_profile": "sqlite", "actor": "conformance-runner", "status": "succeeded", - "operation_count": 1 + "operation_count": 1, + "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", + "source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481" } } ] diff --git a/docs/transformation-plan-profile-0.2.md b/docs/transformation-plan-profile-0.2.md index 619fad5..f3380fc 100644 --- a/docs/transformation-plan-profile-0.2.md +++ b/docs/transformation-plan-profile-0.2.md @@ -113,8 +113,10 @@ metadata component explicitly replaced by a later operation. Structural validation, name resolution, type checking, target-state checking, predicate validation, and backend capability checks all precede mutation. Stable 0.2 diagnostics add `expression_type_unsupported`, -`conditional_target_missing`, `invalid_format`, and retain the 0.1 -diagnostics. +`conditional_target_missing`, `invalid_format`, and +`noncanonical_boolean_shape`, and retain the 0.1 diagnostics. A canonical +boolean expression MUST NOT nest the same `and` or `or` operator; it is +flattened in source order instead. The in-place SQL binding is [`transformation-plan-sql-binding-0.2.md`](transformation-plan-sql-binding-0.2.md). diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 622385f..20af79a 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -603,7 +603,7 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: create_case = cases["sqlite-create-target-atomic-success"] require(set(create_case) == { - "id", "database_profile", "target_mode", "actor", "operation", "before", "after", + "id", "database_profile", "applied_plan_case", "applied_frontend_case", "target_mode", "actor", "operation", "before", "after", "transaction_boundary", "rollback_probe", "expected_audit", "expected_error", }, "SQLite create-target success fields differ.") require( @@ -613,6 +613,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: and create_case["expected_error"] is None, "SQLite create-target success identity differs.", ) + require(create_case["applied_plan_case"] in plan_cases, "SQLite applied plan fixture missing.") + require(create_case["applied_frontend_case"] in frontend_cases, "SQLite applied frontend fixture missing.") require(create_case["operation"] == { "op": "assign", "target": "target", @@ -663,6 +665,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: "actor": create_case["actor"], "status": "succeeded", "operation_count": 1, + "plan_hash": plan_cases[create_case["applied_plan_case"]]["expected_plan_hash"], + "source_hash": frontend_cases[create_case["applied_frontend_case"]]["expected_source_hash"], }, "SQLite create-target audit expectation differs.") require(create_case["rollback_probe"] == { "failure_point": "after_catalog_create_before_audit", From 3e25276c5978974db9c36d80355cac3e788a5db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:33:32 +0300 Subject: [PATCH 61/78] fix: complete transformation binding review cases --- conformance/in-place-transformation-0.2.json | 9 +++++---- conformance/spss-syntax-frontend-0.2.json | 8 ++++++++ conformance/transformation-plan-0.2.json | 6 ++++++ docs/transformation-plan-sql-binding-0.2.md | 3 ++- tools/validate_transformation_plan_0_2.py | 4 ++++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index c0231f3..19751f9 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -98,6 +98,7 @@ }, "expected_audit": { "contract_id": "openstatspec-in-place-transformation-v0.2", + "status": "succeeded", "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", "source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481", "operation_count": 7, @@ -267,8 +268,8 @@ { "id": "sqlite-create-target-atomic-success", "database_profile": "sqlite", - "applied_plan_case": "sequential-conditional-binary-existing-target", - "applied_frontend_case": "compute-if-labels-format-level-execute-existing-target", + "applied_plan_case": "sqlite-create-target-source-copy", + "applied_frontend_case": "sqlite-create-target-source-copy", "target_mode": "create", "operation": { "op": "assign", @@ -352,8 +353,8 @@ "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, - "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", - "source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481" + "plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", + "source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15" } } ] diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index 0dca41b..d013d71 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -153,6 +153,14 @@ "expected_source_hash": "4b3b35660b1ec57de0b8b0485dcc85bb51a05796df9e9624f4f1790d78e2bfeb", "expected_error": null }, + { + "id": "sqlite-create-target-source-copy", + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, "source_text": "COMPUTE target = source_a."}, + "expected_plan_case": "sqlite-create-target-source-copy", + "expected_plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", + "expected_source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", + "expected_error": null + }, { "id": "reject-string-expression", "request": { diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index 5b7cfd7..9c7eaf7 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -316,6 +316,12 @@ "expected_plan_hash": "926f2cd93ed0bfa330bdb2d0b31b241f552c86c116dd35a315a20408bf997daf", "expected_error": null }, + { + "id": "sqlite-create-target-source-copy", + "plan": {"contract": "openstatspec-transformation-plan-v0.2", "input_alias": "parent", "operations": [{"op": "assign", "target": "target", "target_mode": "create", "value": {"kind": "variable", "variable": "source_a"}}]}, + "expected_plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", + "expected_error": null + }, { "id": "reject-invalid-format", "plan": { diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md index 0ca22c8..b268c87 100644 --- a/docs/transformation-plan-sql-binding-0.2.md +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -49,7 +49,8 @@ for either action. ## Controlled Dolt context -Every apply requires a non-empty actor. Dolt additionally requires +Every apply requires a non-empty actor. An absent or empty actor MUST fail +before mutation with the stable actor_required diagnostic. Dolt additionally requires `expected_branch`, `expected_head`, and a clean working set. Branch, HEAD, and cleanliness are checked before mutation. Branch and HEAD are checked again before successful completion. Stable diagnostics are diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 20af79a..68bd214 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -182,6 +182,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "three-term-and-flattens-source-order": None, "reject-string-predicate": "expression_type_unsupported", "nested-or-inequalities-variable-operands-create": None, + "sqlite-create-target-source-copy": None, "reject-invalid-format": "invalid_format", "reject-reserved-assignment-target": "reserved_target_name", "reject-duplicate-value-label": "duplicate_value_label", @@ -330,6 +331,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "nested-or-inequality-variable-operands-create": ( "COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.", ), + "sqlite-create-target-source-copy": ("COMPUTE target = source_a.",), "unparenthesized-mixed-default-precedence": ( "IF (source_a = 1 OR source_b = 1 AND source_c = 1)", "EXECUTE.", ), @@ -395,6 +397,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "three-term-and-flattens-source-order", "parenthesized-three-term-and-flattens-source-order", "nested-or-inequality-variable-operands-create", + "sqlite-create-target-source-copy", "old-subset-retains-v0.1-plan", "reject-string-expression", "reject-missing-conditional-target", @@ -506,6 +509,7 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: require(after["dolt_commit_performed"] is False, "Apply performs DOLT_COMMIT.") require(success["expected_audit"] == { "contract_id": manifest["contract"], + "status": "succeeded", "actor": "conformance-runner", "operation_count": 7, "plan_hash": plan_cases[success["applied_plan_case"]]["expected_plan_hash"], From ecd258a2b1737b58e01b1520d1410a76a5ef2faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:41:27 +0300 Subject: [PATCH 62/78] fix: bind SQLite audit identity --- conformance/in-place-transformation-0.2.json | 3 +++ tools/validate_transformation_plan_0_2.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 19751f9..d01675c 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -350,6 +350,9 @@ "expected_audit": { "contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "sqlite", + "dataset_id": "22222222-2222-4222-8222-222222222222", + "physical_table_schema": null, + "physical_table_name": "data_atomic_create", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 68bd214..0a4d019 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -666,6 +666,9 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: require(create_case["expected_audit"] == { "contract_id": manifest["contract"], "database_profile": "sqlite", + "dataset_id": create_before["dataset_id"], + "physical_table_schema": create_before["physical_table_schema"], + "physical_table_name": create_before["physical_table_name"], "actor": create_case["actor"], "status": "succeeded", "operation_count": 1, From f62b36ca21cb8faf4ab974fb149b0dc6aca1fddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:50:54 +0300 Subject: [PATCH 63/78] fix: strengthen transformation binding fixtures --- conformance/in-place-transformation-0.2.json | 11 +++++++++++ conformance/spss-syntax-frontend-0.2.json | 2 +- tools/validate_transformation_plan_0_2.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index d01675c..1a6b78b 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -65,18 +65,26 @@ "rows": [ { "__case_ordinal": 1, + "source_a": 1, + "source_b": 1, "target": 1 }, { "__case_ordinal": 2, + "source_a": 1, + "source_b": null, "target": 0 }, { "__case_ordinal": 3, + "source_a": null, + "source_b": 1, "target": 0 }, { "__case_ordinal": 4, + "source_a": 0, + "source_b": 1, "target": 0 } ], @@ -99,6 +107,9 @@ "expected_audit": { "contract_id": "openstatspec-in-place-transformation-v0.2", "status": "succeeded", + "dataset_id": "11111111-1111-4111-8111-111111111111", + "physical_table_schema": null, + "physical_table_name": "data_synthetic", "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", "source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481", "operation_count": 7, diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index d013d71..f4f1ce8 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -155,7 +155,7 @@ }, { "id": "sqlite-create-target-source-copy", - "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, "source_text": "COMPUTE target = source_a."}, + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}, {"name": "source_b", "storage_kind": "numeric"}]}, "source_text": "COMPUTE target = source_a."}, "expected_plan_case": "sqlite-create-target-source-copy", "expected_plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", "expected_source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 0a4d019..6ec9d5f 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -506,9 +506,16 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: }, "Dolt physical/catalog provisioning is not coupled to HEAD.") require(before["working_set_clean"] is True and after["working_set_clean"] is False, "Dolt working-set state differs.") require([row["target"] for row in after["rows"]] == [1, 0, 0, 0], "Three-valued IF result differs.") + require( + [[row["source_a"], row["source_b"]] for row in after["rows"]] + == [[row["source_a"], row["source_b"]] for row in before["rows"]], + "Dolt success changes source values.") require(after["dolt_commit_performed"] is False, "Apply performs DOLT_COMMIT.") require(success["expected_audit"] == { "contract_id": manifest["contract"], + "dataset_id": before["dataset_id"], + "physical_table_schema": before["physical_table_schema"], + "physical_table_name": before["physical_table_name"], "status": "succeeded", "actor": "conformance-runner", "operation_count": 7, @@ -619,6 +626,11 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: ) require(create_case["applied_plan_case"] in plan_cases, "SQLite applied plan fixture missing.") require(create_case["applied_frontend_case"] in frontend_cases, "SQLite applied frontend fixture missing.") + create_frontend = frontend_cases[create_case["applied_frontend_case"]] + require(create_frontend["request"]["input_schema"]["variables"] == [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "source_b", "storage_kind": "numeric"}, + ], "SQLite create-target frontend schema differs.") require(create_case["operation"] == { "op": "assign", "target": "target", From 28f801cf4c6deefbcb809b099c88e3f2614d999a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 00:06:18 +0300 Subject: [PATCH 64/78] fix: cover binding edge semantics --- conformance/in-place-transformation-0.2.json | 43 +++++++++++++++++++- tools/validate_transformation_plan_0_2.py | 38 +++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 1a6b78b..c6e411d 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -21,6 +21,16 @@ "catalog_variable": true, "dolt_commit": "provisioning-commit" }, + "target_metadata": { + "missing_values": [ + { + "ordinal": 1, + "rule_kind": "discrete", + "code_kind": "numeric", + "numeric_value": -9 + } + ] + }, "dolt_branch": "feature/recode", "dolt_head": "provisioning-commit", "working_set_clean": true, @@ -101,11 +111,20 @@ ] ], "format": "F1.0", - "measurement_level": "nominal" + "measurement_level": "nominal", + "missing_values": [ + { + "ordinal": 1, + "rule_kind": "discrete", + "code_kind": "numeric", + "numeric_value": -9 + } + ] } }, "expected_audit": { "contract_id": "openstatspec-in-place-transformation-v0.2", + "database_profile": "dolt", "status": "succeeded", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, @@ -269,6 +288,28 @@ ], "expected_error": null }, + { + "id": "sqlite-inequality-boundary-semantics", + "database_profile": "sqlite", + "target_preprovisioned": true, + "boundary": 1, + "before_rows": [{"__case_ordinal": 1, "source": 0, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}], + "operations": [{"op": "conditional_assign", "condition": "source < 1", "target": "target_lt", "value": 1}, {"op": "conditional_assign", "condition": "source <= 1", "target": "target_le", "value": 1}, {"op": "conditional_assign", "condition": "source > 1", "target": "target_gt", "value": 1}, {"op": "conditional_assign", "condition": "source >= 1", "target": "target_ge", "value": 1}], + "after_rows": [{"__case_ordinal": 1, "source": 0, "target_lt": 1, "target_le": 1, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 1, "target_gt": 0, "target_ge": 1}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 1, "target_ge": 1}], + "expected_error": null + }, + { + "id": "dolt-context-changed-after-mutation-rolls-back", + "database_profile": "dolt", + "target_preprovisioned": true, + "expected_context": {"branch": "feature/recode", "head": "provisioning-commit"}, + "completion_context": {"branch": "feature/recode", "head": "concurrent-commit"}, + "failure_point": "after_data_and_metadata_before_audit", + "before": {"rows": [{"__case_ordinal": 1, "target": 0}], "target_metadata": {"variable_label": "Before"}, "audit_row_count": 0}, + "after_failure": {"rows": [{"__case_ordinal": 1, "target": 0}], "target_metadata": {"variable_label": "Before"}, "audit_row_count": 0}, + "mutation_started": true, + "expected_error": "dolt_context_changed" + }, { "id": "dolt-empty-actor-fails-before-mutation", "database_profile": "dolt", diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 6ec9d5f..b45d03f 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -474,6 +474,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: "dolt-preprovisioned-target-conditional-variable-missing-propagation", "dolt-empty-actor-fails-before-mutation", "sqlite-create-target-atomic-success", + "sqlite-inequality-boundary-semantics", + "dolt-context-changed-after-mutation-rolls-back", } | { f"{profile}-create-target-fails-before-mutation" for profile in ("dolt", "mysql", "mariadb") @@ -504,6 +506,17 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: "catalog_variable": True, "dolt_commit": before["dolt_head"], }, "Dolt physical/catalog provisioning is not coupled to HEAD.") + missing_values = [ + {"ordinal": 1, "rule_kind": "discrete", "code_kind": "numeric", "numeric_value": -9}, + ] + require(before["target_metadata"] == {"missing_values": missing_values}, "Dolt pre-apply missing values differ.") + require(after["target_metadata"] == { + "variable_label": "Synthetic conjunction", + "value_labels": [[0, "No"], [1, "Yes"]], + "format": "F1.0", + "measurement_level": "nominal", + "missing_values": missing_values, + }, "Dolt target metadata result or preserved missing values differ.") require(before["working_set_clean"] is True and after["working_set_clean"] is False, "Dolt working-set state differs.") require([row["target"] for row in after["rows"]] == [1, 0, 0, 0], "Three-valued IF result differs.") require( @@ -513,6 +526,7 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: require(after["dolt_commit_performed"] is False, "Apply performs DOLT_COMMIT.") require(success["expected_audit"] == { "contract_id": manifest["contract"], + "database_profile": "dolt", "dataset_id": before["dataset_id"], "physical_table_schema": before["physical_table_schema"], "physical_table_name": before["physical_table_name"], @@ -602,6 +616,30 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: {"__case_ordinal": 2, "target": 7}, ], "Dolt conditional assignment does not preserve a missing RHS.") require(conditional_missing_case["expected_error"] is None, "Dolt conditional missing-propagation case unexpectedly fails.") + inequality_case = cases["sqlite-inequality-boundary-semantics"] + require(set(inequality_case) == {"id", "database_profile", "target_preprovisioned", "boundary", "before_rows", "operations", "after_rows", "expected_error"}, "Inequality-boundary fields differ.") + require(inequality_case["database_profile"] == "sqlite" and inequality_case["target_preprovisioned"] is True and inequality_case["boundary"] == 1, "Inequality-boundary identity differs.") + require(inequality_case["before_rows"] == [{"__case_ordinal": 1, "source": 0, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}], "Inequality-boundary inputs differ.") + require(inequality_case["operations"] == [ + {"op": "conditional_assign", "condition": "source < 1", "target": "target_lt", "value": 1}, + {"op": "conditional_assign", "condition": "source <= 1", "target": "target_le", "value": 1}, + {"op": "conditional_assign", "condition": "source > 1", "target": "target_gt", "value": 1}, + {"op": "conditional_assign", "condition": "source >= 1", "target": "target_ge", "value": 1}, + ], "Inequality-boundary operations differ.") + require(inequality_case["after_rows"] == [ + {"__case_ordinal": 1, "source": 0, "target_lt": 1, "target_le": 1, "target_gt": 0, "target_ge": 0}, + {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 1, "target_gt": 0, "target_ge": 1}, + {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 1, "target_ge": 1}, + ], "Strict and non-strict inequality results differ.") + require(inequality_case["expected_error"] is None, "Inequality-boundary apply unexpectedly fails.") + context_case = cases["dolt-context-changed-after-mutation-rolls-back"] + require(set(context_case) == {"id", "database_profile", "target_preprovisioned", "expected_context", "completion_context", "failure_point", "before", "after_failure", "mutation_started", "expected_error"}, "Dolt context-change fields differ.") + require(context_case["database_profile"] == "dolt" and context_case["target_preprovisioned"] is True, "Dolt context-change identity differs.") + require(context_case["expected_context"] == {"branch": "feature/recode", "head": "provisioning-commit"}, "Dolt expected context differs.") + require(context_case["completion_context"] == {"branch": "feature/recode", "head": "concurrent-commit"}, "Dolt changed context differs.") + require(context_case["failure_point"] == "after_data_and_metadata_before_audit" and context_case["mutation_started"] is True and context_case["expected_error"] == "dolt_context_changed", "Dolt post-mutation context-change diagnostic differs.") + require(context_case["before"] == {"rows": [{"__case_ordinal": 1, "target": 0}], "target_metadata": {"variable_label": "Before"}, "audit_row_count": 0}, "Dolt context-change pre-state differs.") + require(context_case["after_failure"] == context_case["before"], "Dolt context-change failure does not roll back data, metadata, and audit.") actor_case = cases["dolt-empty-actor-fails-before-mutation"] require(actor_case == { From 8b5bf07cf85df36ad4e6521077d32b12edeae7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 00:15:27 +0300 Subject: [PATCH 65/78] fix: strengthen context and rebuild fixtures --- conformance/in-place-transformation-0.2.json | 9 +++++++-- tools/validate_transformation_plan_0_2.py | 13 +++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index c6e411d..dbad7bc 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -302,6 +302,7 @@ "id": "dolt-context-changed-after-mutation-rolls-back", "database_profile": "dolt", "target_preprovisioned": true, + "actor": "conformance-runner", "expected_context": {"branch": "feature/recode", "head": "provisioning-commit"}, "completion_context": {"branch": "feature/recode", "head": "concurrent-commit"}, "failure_point": "after_data_and_metadata_before_audit", @@ -343,11 +344,13 @@ "rows": [ { "__case_ordinal": 1, - "source_a": 2 + "source_a": 2, + "source_b": 11 }, { "__case_ordinal": 2, - "source_a": null + "source_a": null, + "source_b": 22 } ] }, @@ -375,11 +378,13 @@ { "__case_ordinal": 1, "source_a": 2, + "source_b": 11, "target": 2 }, { "__case_ordinal": 2, "source_a": null, + "source_b": 22, "target": null } ] diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index b45d03f..40d1cd3 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -633,8 +633,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: ], "Strict and non-strict inequality results differ.") require(inequality_case["expected_error"] is None, "Inequality-boundary apply unexpectedly fails.") context_case = cases["dolt-context-changed-after-mutation-rolls-back"] - require(set(context_case) == {"id", "database_profile", "target_preprovisioned", "expected_context", "completion_context", "failure_point", "before", "after_failure", "mutation_started", "expected_error"}, "Dolt context-change fields differ.") - require(context_case["database_profile"] == "dolt" and context_case["target_preprovisioned"] is True, "Dolt context-change identity differs.") + require(set(context_case) == {"id", "database_profile", "target_preprovisioned", "actor", "expected_context", "completion_context", "failure_point", "before", "after_failure", "mutation_started", "expected_error"}, "Dolt context-change fields differ.") + require(context_case["database_profile"] == "dolt" and context_case["target_preprovisioned"] is True and context_case["actor"] == "conformance-runner", "Dolt context-change identity or actor differs.") require(context_case["expected_context"] == {"branch": "feature/recode", "head": "provisioning-commit"}, "Dolt expected context differs.") require(context_case["completion_context"] == {"branch": "feature/recode", "head": "concurrent-commit"}, "Dolt changed context differs.") require(context_case["failure_point"] == "after_data_and_metadata_before_audit" and context_case["mutation_started"] is True and context_case["expected_error"] == "dolt_context_changed", "Dolt post-mutation context-change diagnostic differs.") @@ -703,13 +703,14 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: }, }, "SQLite create-target column/catalog result differs.") require(create_before["rows"] == [ - {"__case_ordinal": 1, "source_a": 2}, - {"__case_ordinal": 2, "source_a": None}, + {"__case_ordinal": 1, "source_a": 2, "source_b": 11}, + {"__case_ordinal": 2, "source_a": None, "source_b": 22}, ], "SQLite create-target inputs differ.") require(create_after["rows"] == [ - {"__case_ordinal": 1, "source_a": 2, "target": 2}, - {"__case_ordinal": 2, "source_a": None, "target": None}, + {"__case_ordinal": 1, "source_a": 2, "source_b": 11, "target": 2}, + {"__case_ordinal": 2, "source_a": None, "source_b": 22, "target": None}, ], "SQLite create-target values or case order differ.") + require([row["source_b"] for row in create_before["rows"]] == [row["source_b"] for row in create_after["rows"]] == [11, 22], "SQLite create-target rebuild changes source_b.") require(create_case["transaction_boundary"] == [ "physical_schema", "row_values", "catalog", "compact_audit", ], "SQLite create-target transaction boundary differs.") From f381a46f58cd0793f4f8607c39ab95932587a72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 00:23:02 +0300 Subject: [PATCH 66/78] fix: supply inequality apply actor --- conformance/in-place-transformation-0.2.json | 1 + tools/validate_transformation_plan_0_2.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index dbad7bc..2b371cd 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -291,6 +291,7 @@ { "id": "sqlite-inequality-boundary-semantics", "database_profile": "sqlite", + "actor": "conformance-runner", "target_preprovisioned": true, "boundary": 1, "before_rows": [{"__case_ordinal": 1, "source": 0, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}], diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 40d1cd3..083cc44 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -617,8 +617,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: ], "Dolt conditional assignment does not preserve a missing RHS.") require(conditional_missing_case["expected_error"] is None, "Dolt conditional missing-propagation case unexpectedly fails.") inequality_case = cases["sqlite-inequality-boundary-semantics"] - require(set(inequality_case) == {"id", "database_profile", "target_preprovisioned", "boundary", "before_rows", "operations", "after_rows", "expected_error"}, "Inequality-boundary fields differ.") - require(inequality_case["database_profile"] == "sqlite" and inequality_case["target_preprovisioned"] is True and inequality_case["boundary"] == 1, "Inequality-boundary identity differs.") + require(set(inequality_case) == {"id", "database_profile", "target_preprovisioned", "actor", "boundary", "before_rows", "operations", "after_rows", "expected_error"}, "Inequality-boundary fields differ.") + require(inequality_case["database_profile"] == "sqlite" and inequality_case["target_preprovisioned"] is True and inequality_case["actor"] == "conformance-runner" and inequality_case["boundary"] == 1, "Inequality-boundary identity or actor differs.") require(inequality_case["before_rows"] == [{"__case_ordinal": 1, "source": 0, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}], "Inequality-boundary inputs differ.") require(inequality_case["operations"] == [ {"op": "conditional_assign", "condition": "source < 1", "target": "target_lt", "value": 1}, From 308b883882ef2b4466eb9c1949398b828613416e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 09:33:08 +0300 Subject: [PATCH 67/78] fix: complete binding apply inputs --- conformance/in-place-transformation-0.2.json | 19 ++++++++++ tools/validate_transformation_plan_0_2.py | 38 +++++++++++++------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 2b371cd..ffa0269 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -7,6 +7,10 @@ { "id": "dolt-preprovisioned-target-sequential-null-semantics", "database_profile": "dolt", + "actor": "conformance-runner", + "expected_branch": "feature/recode", + "expected_head": "provisioning-commit", + "working_set_clean": true, "applied_plan_case": "sequential-conditional-binary-existing-target", "applied_frontend_case": "compute-if-labels-format-level-execute-existing-target", "before": { @@ -172,6 +176,10 @@ { "id": "dolt-preprovisioned-target-or-null-semantics", "database_profile": "dolt", + "actor": "conformance-runner", + "expected_branch": "feature/recode", + "expected_head": "provisioning-commit", + "working_set_clean": true, "target_preprovisioned": true, "condition": "source_a = 1 OR source_b = 1", "before_rows": [ @@ -203,6 +211,7 @@ { "id": "dolt-create-target-fails-before-mutation", "database_profile": "dolt", + "actor": "conformance-runner", "target_mode": "create", "expected_error": "schema_change_not_atomic", "mutation_started": false @@ -210,6 +219,7 @@ { "id": "mysql-create-target-fails-before-mutation", "database_profile": "mysql", + "actor": "conformance-runner", "target_mode": "create", "expected_error": "schema_change_not_atomic", "mutation_started": false @@ -217,6 +227,7 @@ { "id": "mariadb-create-target-fails-before-mutation", "database_profile": "mariadb", + "actor": "conformance-runner", "target_mode": "create", "expected_error": "schema_change_not_atomic", "mutation_started": false @@ -224,6 +235,10 @@ { "id": "dolt-preprovisioned-target-variable-missing-propagation", "database_profile": "dolt", + "actor": "conformance-runner", + "expected_branch": "feature/recode", + "expected_head": "provisioning-commit", + "working_set_clean": true, "target_preprovisioned": true, "assign": { "target": "target", @@ -256,6 +271,10 @@ { "id": "dolt-preprovisioned-target-conditional-variable-missing-propagation", "database_profile": "dolt", + "actor": "conformance-runner", + "expected_branch": "feature/recode", + "expected_head": "provisioning-commit", + "working_set_clean": true, "target_preprovisioned": true, "conditional_assign": { "target": "target", diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 083cc44..08b4c78 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -485,12 +485,15 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: success = cases["dolt-preprovisioned-target-sequential-null-semantics"] require(set(success) == { "id", "database_profile", "applied_plan_case", "applied_frontend_case", "before", "after", - "expected_audit", "forbidden_artifacts", "required_audit_fields", "expected_error", + "actor", "expected_branch", "expected_head", "working_set_clean", "expected_audit", "forbidden_artifacts", "required_audit_fields", "expected_error", }, "0.2 binding success fields differ.") - require(success["database_profile"] == "dolt" and success["expected_error"] is None, "Dolt success identity differs.") + require(success["database_profile"] == "dolt" and success["actor"] == "conformance-runner" and success["expected_error"] is None, "Dolt success identity or actor differs.") require(success["applied_plan_case"] in plan_cases, "Dolt applied plan fixture missing.") require(success["applied_frontend_case"] in frontend_cases, "Dolt applied frontend fixture missing.") before, after = success["before"], success["after"] + success_frontend = frontend_cases[success["applied_frontend_case"]] + require(success_frontend.get("expected_error") is None and success_frontend.get("expected_plan_case") == success["applied_plan_case"], "Dolt frontend does not link the applied successful plan.") + require(success["expected_branch"] == before["dolt_branch"] and success["expected_head"] == before["dolt_head"] and success["working_set_clean"] is before["working_set_clean"] is True, "Dolt success apply context inputs differ.") for field in ( "dataset_id", "physical_table_schema", "physical_table_name", "dataset_count", "persistent_data_table_count", "case_count", @@ -553,10 +556,10 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: or_case = cases["dolt-preprovisioned-target-or-null-semantics"] require(set(or_case) == { - "id", "database_profile", "target_preprovisioned", "condition", - "before_rows", "after_rows", "expected_error", + "id", "database_profile", "target_preprovisioned", "actor", "expected_branch", "expected_head", + "working_set_clean", "condition", "before_rows", "after_rows", "expected_error", }, "Dolt OR-null case fields differ.") - require(or_case["database_profile"] == "dolt" and or_case["target_preprovisioned"] is True, "Dolt OR-null identity differs.") + require(or_case["database_profile"] == "dolt" and or_case["target_preprovisioned"] is True and or_case["actor"] == "conformance-runner" and or_case["expected_branch"] == "feature/recode" and or_case["expected_head"] == "provisioning-commit" and or_case["working_set_clean"] is True, "Dolt OR-null identity or apply context differs.") require(or_case["condition"] == "source_a = 1 OR source_b = 1", "Dolt OR-null condition differs.") require(or_case["before_rows"] == [ {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 0}, @@ -570,12 +573,16 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: missing_case = cases["dolt-preprovisioned-target-variable-missing-propagation"] require(set(missing_case) == { - "id", "database_profile", "target_preprovisioned", "assign", - "before_rows", "after_rows", "expected_error", + "id", "database_profile", "target_preprovisioned", "actor", "expected_branch", "expected_head", + "working_set_clean", "assign", "before_rows", "after_rows", "expected_error", }, "Dolt missing-propagation case fields differ.") require( missing_case["database_profile"] == "dolt" - and missing_case["target_preprovisioned"] is True, + and missing_case["actor"] == "conformance-runner" + and missing_case["target_preprovisioned"] is True + and missing_case["expected_branch"] == "feature/recode" + and missing_case["expected_head"] == "provisioning-commit" + and missing_case["working_set_clean"] is True, "Dolt missing-propagation identity differs.", ) require( @@ -594,12 +601,16 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: conditional_missing_case = cases["dolt-preprovisioned-target-conditional-variable-missing-propagation"] require(set(conditional_missing_case) == { - "id", "database_profile", "target_preprovisioned", "conditional_assign", - "before_rows", "after_rows", "expected_error", + "id", "database_profile", "target_preprovisioned", "actor", "expected_branch", "expected_head", + "working_set_clean", "conditional_assign", "before_rows", "after_rows", "expected_error", }, "Dolt conditional missing-propagation fields differ.") require( conditional_missing_case["database_profile"] == "dolt" - and conditional_missing_case["target_preprovisioned"] is True, + and conditional_missing_case["actor"] == "conformance-runner" + and conditional_missing_case["target_preprovisioned"] is True + and conditional_missing_case["expected_branch"] == "feature/recode" + and conditional_missing_case["expected_head"] == "provisioning-commit" + and conditional_missing_case["working_set_clean"] is True, "Dolt conditional missing-propagation identity differs.", ) require(conditional_missing_case["conditional_assign"] == { @@ -665,6 +676,7 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: require(create_case["applied_plan_case"] in plan_cases, "SQLite applied plan fixture missing.") require(create_case["applied_frontend_case"] in frontend_cases, "SQLite applied frontend fixture missing.") create_frontend = frontend_cases[create_case["applied_frontend_case"]] + require(create_frontend.get("expected_error") is None and create_frontend.get("expected_plan_case") == create_case["applied_plan_case"], "SQLite frontend does not link the applied successful plan.") require(create_frontend["request"]["input_schema"]["variables"] == [ {"name": "source_a", "storage_kind": "numeric"}, {"name": "source_b", "storage_kind": "numeric"}, @@ -736,8 +748,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: for profile in ("dolt", "mysql", "mariadb"): case = cases[f"{profile}-create-target-fails-before-mutation"] - require(set(case) == {"id", "database_profile", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") - require(case["database_profile"] == profile and case["target_mode"] == "create", f"{profile}: failure identity differs.") + require(set(case) == {"id", "database_profile", "actor", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") + require(case["database_profile"] == profile and case["actor"] == "conformance-runner" and case["target_mode"] == "create", f"{profile}: failure identity or actor differs.") require(case["expected_error"] == "schema_change_not_atomic" and case["mutation_started"] is False, f"{profile}: create-target gate differs.") From 6fa20659fa2069f6b49b4f18facc6f769b876b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 09:44:50 +0300 Subject: [PATCH 68/78] fix: supply Dolt rejection context --- conformance/in-place-transformation-0.2.json | 3 +++ tools/validate_transformation_plan_0_2.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index ffa0269..9c689ea 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -212,6 +212,9 @@ "id": "dolt-create-target-fails-before-mutation", "database_profile": "dolt", "actor": "conformance-runner", + "expected_branch": "feature/recode", + "expected_head": "provisioning-commit", + "working_set_clean": true, "target_mode": "create", "expected_error": "schema_change_not_atomic", "mutation_started": false diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 08b4c78..0c9876c 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -748,8 +748,13 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: for profile in ("dolt", "mysql", "mariadb"): case = cases[f"{profile}-create-target-fails-before-mutation"] - require(set(case) == {"id", "database_profile", "actor", "target_mode", "expected_error", "mutation_started"}, f"{profile}: failure fields differ.") + expected_fields = {"id", "database_profile", "actor", "target_mode", "expected_error", "mutation_started"} + if profile == "dolt": + expected_fields |= {"expected_branch", "expected_head", "working_set_clean"} + require(set(case) == expected_fields, f"{profile}: failure fields differ.") require(case["database_profile"] == profile and case["actor"] == "conformance-runner" and case["target_mode"] == "create", f"{profile}: failure identity or actor differs.") + if profile == "dolt": + require(case["expected_branch"] == "feature/recode" and case["expected_head"] == "provisioning-commit" and case["working_set_clean"] is True, "dolt: create-target context inputs differ.") require(case["expected_error"] == "schema_change_not_atomic" and case["mutation_started"] is False, f"{profile}: create-target gate differs.") From 6ba206ae4d86683a8f6b21edf9c11adc07104522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 10:02:53 +0300 Subject: [PATCH 69/78] fix: cover mixed frontend promotion --- conformance/in-place-transformation-0.2.json | 10 ++++++++++ conformance/spss-syntax-frontend-0.2.json | 8 ++++++++ conformance/transformation-plan-0.2.json | 6 ++++++ tools/validate_transformation_plan_0_2.py | 21 ++++++++++++++------ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 9c689ea..912fbf1 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -199,10 +199,14 @@ "after_rows": [ { "__case_ordinal": 1, + "source_a": null, + "source_b": 1, "target": 1 }, { "__case_ordinal": 2, + "source_a": null, + "source_b": 0, "target": 0 } ], @@ -262,10 +266,12 @@ "after_rows": [ { "__case_ordinal": 1, + "source_a": null, "target": null }, { "__case_ordinal": 2, + "source_a": 2, "target": 2 } ], @@ -301,10 +307,14 @@ "after_rows": [ { "__case_ordinal": 1, + "source_a": null, + "source_b": 1, "target": null }, { "__case_ordinal": 2, + "source_a": null, + "source_b": 0, "target": 7 } ], diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index f4f1ce8..c8be977 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -161,6 +161,14 @@ "expected_source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", "expected_error": null }, + { + "id": "mixed-recode-and-compute-promotes-v0.2", + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, "source_text": "RECODE q1 (1 = 0).\nCOMPUTE target = q1."}, + "expected_plan_case": "mixed-recode-and-compute-promotes-v0.2", + "expected_plan_hash": "c90b19239ea7c816e19575a16e4ea14186a2c73762d6d447c369a4b9d2f1e35f", + "expected_source_hash": "80e14ec0c4dd0e5013099ed12e47a2dd4d6b0ab0e93f3f611168744699a7e688", + "expected_error": null + }, { "id": "reject-string-expression", "request": { diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index 9c7eaf7..e143a7a 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -322,6 +322,12 @@ "expected_plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", "expected_error": null }, + { + "id": "mixed-recode-and-compute-promotes-v0.2", + "plan": {"contract": "openstatspec-transformation-plan-v0.2", "input_alias": "parent", "operations": [{"op": "recode", "source": "q1", "target": "q1", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}, {"op": "assign", "target": "target", "target_mode": "create", "value": {"kind": "variable", "variable": "q1"}}]}, + "expected_plan_hash": "c90b19239ea7c816e19575a16e4ea14186a2c73762d6d447c369a4b9d2f1e35f", + "expected_error": null + }, { "id": "reject-invalid-format", "plan": { diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py index 0c9876c..56ee641 100755 --- a/tools/validate_transformation_plan_0_2.py +++ b/tools/validate_transformation_plan_0_2.py @@ -183,6 +183,7 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: "reject-string-predicate": "expression_type_unsupported", "nested-or-inequalities-variable-operands-create": None, "sqlite-create-target-source-copy": None, + "mixed-recode-and-compute-promotes-v0.2": None, "reject-invalid-format": "invalid_format", "reject-reserved-assignment-target": "reserved_target_name", "reject-duplicate-value-label": "duplicate_value_label", @@ -236,6 +237,10 @@ def validate_plan_manifest() -> dict[str, dict[str, object]]: == "0000000000000000", "Negative-zero canonical conversion bits differ.", ) + elif identifier == "mixed-recode-and-compute-promotes-v0.2": + require( + [operation["op"] for operation in case["plan"]["operations"]] == ["recode", "assign"], + "Mixed legacy/0.2 plan operation coverage differs.") require(set(cases) == set(expected), "0.2 plan case set differs.") return cases @@ -332,6 +337,9 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.", ), "sqlite-create-target-source-copy": ("COMPUTE target = source_a.",), + "mixed-recode-and-compute-promotes-v0.2": ( + "RECODE q1 (1 = 0).", "COMPUTE target = q1.", + ), "unparenthesized-mixed-default-precedence": ( "IF (source_a = 1 OR source_b = 1 AND source_c = 1)", "EXECUTE.", ), @@ -397,6 +405,7 @@ def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: "three-term-and-flattens-source-order", "parenthesized-three-term-and-flattens-source-order", "nested-or-inequality-variable-operands-create", + "mixed-recode-and-compute-promotes-v0.2", "sqlite-create-target-source-copy", "old-subset-retains-v0.1-plan", "reject-string-expression", @@ -566,8 +575,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 0}, ], "Dolt OR-null inputs differ.") require(or_case["after_rows"] == [ - {"__case_ordinal": 1, "target": 1}, - {"__case_ordinal": 2, "target": 0}, + {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 1}, + {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 0}, ], "Dolt OR three-valued outputs differ.") require(or_case["expected_error"] is None, "Dolt OR-null case unexpectedly fails.") @@ -594,8 +603,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: {"__case_ordinal": 2, "source_a": 2, "target": 7}, ], "Dolt missing-propagation inputs differ.") require(missing_case["after_rows"] == [ - {"__case_ordinal": 1, "target": None}, - {"__case_ordinal": 2, "target": 2}, + {"__case_ordinal": 1, "source_a": None, "target": None}, + {"__case_ordinal": 2, "source_a": 2, "target": 2}, ], "Dolt variable assignment does not preserve missing.") require(missing_case["expected_error"] is None, "Dolt missing-propagation case unexpectedly fails.") @@ -623,8 +632,8 @@ def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 7}, ], "Dolt conditional missing-propagation inputs differ.") require(conditional_missing_case["after_rows"] == [ - {"__case_ordinal": 1, "target": None}, - {"__case_ordinal": 2, "target": 7}, + {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": None}, + {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 7}, ], "Dolt conditional assignment does not preserve a missing RHS.") require(conditional_missing_case["expected_error"] is None, "Dolt conditional missing-propagation case unexpectedly fails.") inequality_case = cases["sqlite-inequality-boundary-semantics"] From 61e2ed387fc19f80d12f884ae55e9f9347668787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 10:47:42 +0300 Subject: [PATCH 70/78] Refocus transformation conformance on artifact integrity --- .github/workflows/ci.yml | 1 - README.md | 2 + conformance/in-place-transformation-0.2.json | 36 +- conformance/spss-syntax-frontend-0.2.json | 32 + conformance/transformation-plan-0.2.json | 24 + requirements-validation.txt | 1 - tools/validate_repository.py | 104 ++- tools/validate_transformation_plan.py | 468 ----------- tools/validate_transformation_plan_0_2.py | 819 ------------------- 9 files changed, 191 insertions(+), 1296 deletions(-) delete mode 100644 requirements-validation.txt delete mode 100644 tools/validate_transformation_plan.py delete mode 100755 tools/validate_transformation_plan_0_2.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e6feee..8cd1ff1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,5 +12,4 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.13" - - run: python -m pip install --requirement requirements-validation.txt - run: python tools/validate_repository.py diff --git a/README.md b/README.md index 6a83877..22e3345 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ to be provisioned separately before an in-place transformation apply. Implementations must preflight target capabilities before import. If the target cannot faithfully create one wide table because of column, identifier, string, or row limits, import must fail atomically with a machine-readable capability diagnostic. It must never silently truncate, drop, split, transpose, pivot, or transform source data. +Validation of Transformation Plan and SPSS syntax frontend artifacts in this specification repository is limited to structural and integrity checks of normative schemas and declarative fixtures. Their executable behavioral conformance, including parser, SQL-engine, transaction, and adapter behavior, belongs in each implementation or adapter repository. + ## Who it is for OpenStatSpec is for people and projects that need to move statistical datasets between SPSS and relational databases without turning the data into a different model. It is especially relevant to: diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 912fbf1..17e00e1 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -25,6 +25,7 @@ "catalog_variable": true, "dolt_commit": "provisioning-commit" }, + "target_identity": {"variable_id": "33333333-3333-4333-8333-333333333333", "ordinal": 3}, "target_metadata": { "missing_values": [ { @@ -71,6 +72,7 @@ "dataset_count": 1, "persistent_data_table_count": 1, "physical_table_schema": null, + "target_identity": {"variable_id": "33333333-3333-4333-8333-333333333333", "ordinal": 3}, "dolt_branch": "feature/recode", "dolt_head": "provisioning-commit", "case_count": 4, @@ -180,6 +182,11 @@ "expected_branch": "feature/recode", "expected_head": "provisioning-commit", "working_set_clean": true, + "applied_plan_case": "binding-or-null-existing-target", + "applied_frontend_case": "binding-or-null-existing-target", + "dataset_id": "11111111-1111-4111-8111-111111111111", + "physical_table_schema": null, + "physical_table_name": "data_synthetic", "target_preprovisioned": true, "condition": "source_a = 1 OR source_b = 1", "before_rows": [ @@ -210,7 +217,8 @@ "target": 0 } ], - "expected_error": null + "expected_error": null, + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "7fa8a017ed2476606c20f7b6bc9df09b7bbff208891e03335dfa393c7c260742", "source_hash": "dc936264d68bdf22741a8476d650a4e58fc03a63ac47f86093cbbb732f31594a", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} }, { "id": "dolt-create-target-fails-before-mutation", @@ -246,6 +254,11 @@ "expected_branch": "feature/recode", "expected_head": "provisioning-commit", "working_set_clean": true, + "applied_plan_case": "binding-variable-missing-existing-target", + "applied_frontend_case": "binding-variable-missing-existing-target", + "dataset_id": "11111111-1111-4111-8111-111111111111", + "physical_table_schema": null, + "physical_table_name": "data_synthetic", "target_preprovisioned": true, "assign": { "target": "target", @@ -275,7 +288,8 @@ "target": 2 } ], - "expected_error": null + "expected_error": null, + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "2b52cbd2989c99aac69dbd10e0f3de225ca84f3902193e05931bc3bf9c3e8649", "source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} }, { "id": "dolt-preprovisioned-target-conditional-variable-missing-propagation", @@ -284,6 +298,11 @@ "expected_branch": "feature/recode", "expected_head": "provisioning-commit", "working_set_clean": true, + "applied_plan_case": "binding-conditional-missing-existing-target", + "applied_frontend_case": "binding-conditional-missing-existing-target", + "dataset_id": "11111111-1111-4111-8111-111111111111", + "physical_table_schema": null, + "physical_table_name": "data_synthetic", "target_preprovisioned": true, "conditional_assign": { "target": "target", @@ -318,25 +337,32 @@ "target": 7 } ], - "expected_error": null + "expected_error": null, + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "aa42695ecf8d84d487fe1cdc1b11dd9d9e5d048df670877ed562e9b9fbd23053", "source_hash": "9fb6357cd8b436b69c523d86be0ce1415dc0fded15a425f737e6f3e37bd08b76", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} }, { "id": "sqlite-inequality-boundary-semantics", "database_profile": "sqlite", "actor": "conformance-runner", + "applied_plan_case": "binding-inequality-boundaries-existing-targets", + "applied_frontend_case": "binding-inequality-boundaries-existing-targets", + "dataset_id": "44444444-4444-4444-8444-444444444444", + "physical_table_schema": null, + "physical_table_name": "data_inequality", "target_preprovisioned": true, "boundary": 1, "before_rows": [{"__case_ordinal": 1, "source": 0, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}], "operations": [{"op": "conditional_assign", "condition": "source < 1", "target": "target_lt", "value": 1}, {"op": "conditional_assign", "condition": "source <= 1", "target": "target_le", "value": 1}, {"op": "conditional_assign", "condition": "source > 1", "target": "target_gt", "value": 1}, {"op": "conditional_assign", "condition": "source >= 1", "target": "target_ge", "value": 1}], "after_rows": [{"__case_ordinal": 1, "source": 0, "target_lt": 1, "target_le": 1, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 1, "target_gt": 0, "target_ge": 1}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 1, "target_ge": 1}], - "expected_error": null + "expected_error": null, + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "sqlite", "dataset_id": "44444444-4444-4444-8444-444444444444", "physical_table_schema": null, "physical_table_name": "data_inequality", "actor": "conformance-runner", "status": "succeeded", "operation_count": 4, "plan_hash": "cc0bde9b2adbbe76614c51a6122a5c9c17c9e610a63241eb1769956c2e543690", "source_hash": "1208507803e9b69cb523585e83dd60643e61d3082f9b1d76fe4bca33b1db0011"} }, { "id": "dolt-context-changed-after-mutation-rolls-back", "database_profile": "dolt", "target_preprovisioned": true, "actor": "conformance-runner", - "expected_context": {"branch": "feature/recode", "head": "provisioning-commit"}, + "expected_context": {"branch": "feature/recode", "head": "provisioning-commit", "working_set_clean": true}, "completion_context": {"branch": "feature/recode", "head": "concurrent-commit"}, "failure_point": "after_data_and_metadata_before_audit", "before": {"rows": [{"__case_ordinal": 1, "target": 0}], "target_metadata": {"variable_label": "Before"}, "audit_row_count": 0}, diff --git a/conformance/spss-syntax-frontend-0.2.json b/conformance/spss-syntax-frontend-0.2.json index c8be977..553b9d0 100644 --- a/conformance/spss-syntax-frontend-0.2.json +++ b/conformance/spss-syntax-frontend-0.2.json @@ -169,6 +169,38 @@ "expected_source_hash": "80e14ec0c4dd0e5013099ed12e47a2dd4d6b0ab0e93f3f611168744699a7e688", "expected_error": null }, + { + "id": "binding-or-null-existing-target", + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}, {"name": "source_b", "storage_kind": "numeric"}, {"name": "target", "storage_kind": "numeric"}]}, "source_text": "IF (source_a = 1 OR source_b = 1) target = 1."}, + "expected_plan_case": "binding-or-null-existing-target", + "expected_plan_hash": "7fa8a017ed2476606c20f7b6bc9df09b7bbff208891e03335dfa393c7c260742", + "expected_source_hash": "dc936264d68bdf22741a8476d650a4e58fc03a63ac47f86093cbbb732f31594a", + "expected_error": null + }, + { + "id": "binding-variable-missing-existing-target", + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}, {"name": "target", "storage_kind": "numeric"}]}, "source_text": "COMPUTE target = source_a."}, + "expected_plan_case": "binding-variable-missing-existing-target", + "expected_plan_hash": "2b52cbd2989c99aac69dbd10e0f3de225ca84f3902193e05931bc3bf9c3e8649", + "expected_source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", + "expected_error": null + }, + { + "id": "binding-conditional-missing-existing-target", + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}, {"name": "source_b", "storage_kind": "numeric"}, {"name": "target", "storage_kind": "numeric"}]}, "source_text": "IF (source_b = 1) target = source_a."}, + "expected_plan_case": "binding-conditional-missing-existing-target", + "expected_plan_hash": "aa42695ecf8d84d487fe1cdc1b11dd9d9e5d048df670877ed562e9b9fbd23053", + "expected_source_hash": "9fb6357cd8b436b69c523d86be0ce1415dc0fded15a425f737e6f3e37bd08b76", + "expected_error": null + }, + { + "id": "binding-inequality-boundaries-existing-targets", + "request": {"contract": "openstatspec-spss-syntax-frontend-v0.2", "input_alias": "parent", "input_schema": {"variables": [{"name": "source", "storage_kind": "numeric"}, {"name": "target_lt", "storage_kind": "numeric"}, {"name": "target_le", "storage_kind": "numeric"}, {"name": "target_gt", "storage_kind": "numeric"}, {"name": "target_ge", "storage_kind": "numeric"}]}, "source_text": "IF (source < 1) target_lt = 1.\nIF (source <= 1) target_le = 1.\nIF (source > 1) target_gt = 1.\nIF (source >= 1) target_ge = 1."}, + "expected_plan_case": "binding-inequality-boundaries-existing-targets", + "expected_plan_hash": "cc0bde9b2adbbe76614c51a6122a5c9c17c9e610a63241eb1769956c2e543690", + "expected_source_hash": "1208507803e9b69cb523585e83dd60643e61d3082f9b1d76fe4bca33b1db0011", + "expected_error": null + }, { "id": "reject-string-expression", "request": { diff --git a/conformance/transformation-plan-0.2.json b/conformance/transformation-plan-0.2.json index e143a7a..734c56f 100644 --- a/conformance/transformation-plan-0.2.json +++ b/conformance/transformation-plan-0.2.json @@ -328,6 +328,30 @@ "expected_plan_hash": "c90b19239ea7c816e19575a16e4ea14186a2c73762d6d447c369a4b9d2f1e35f", "expected_error": null }, + { + "id": "binding-or-null-existing-target", + "plan": {"contract": "openstatspec-transformation-plan-v0.2", "input_alias": "parent", "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [{"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"expression": "comparison", "left": {"kind": "variable", "variable": "source_b"}, "operator": "=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}]}, + "expected_plan_hash": "7fa8a017ed2476606c20f7b6bc9df09b7bbff208891e03335dfa393c7c260742", + "expected_error": null + }, + { + "id": "binding-variable-missing-existing-target", + "plan": {"contract": "openstatspec-transformation-plan-v0.2", "input_alias": "parent", "operations": [{"op": "assign", "target": "target", "target_mode": "replace", "value": {"kind": "variable", "variable": "source_a"}}]}, + "expected_plan_hash": "2b52cbd2989c99aac69dbd10e0f3de225ca84f3902193e05931bc3bf9c3e8649", + "expected_error": null + }, + { + "id": "binding-conditional-missing-existing-target", + "plan": {"contract": "openstatspec-transformation-plan-v0.2", "input_alias": "parent", "operations": [{"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source_b"}, "operator": "=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target", "value": {"kind": "variable", "variable": "source_a"}}]}, + "expected_plan_hash": "aa42695ecf8d84d487fe1cdc1b11dd9d9e5d048df670877ed562e9b9fbd23053", + "expected_error": null + }, + { + "id": "binding-inequality-boundaries-existing-targets", + "plan": {"contract": "openstatspec-transformation-plan-v0.2", "input_alias": "parent", "operations": [{"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target_lt", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source"}, "operator": "<=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target_le", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target_gt", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source"}, "operator": ">=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target_ge", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}]}, + "expected_plan_hash": "cc0bde9b2adbbe76614c51a6122a5c9c17c9e610a63241eb1769956c2e543690", + "expected_error": null + }, { "id": "reject-invalid-format", "plan": { diff --git a/requirements-validation.txt b/requirements-validation.txt deleted file mode 100644 index d9bf673..0000000 --- a/requirements-validation.txt +++ /dev/null @@ -1 +0,0 @@ -jsonschema==4.10.3 diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 44f1345..68caf3b 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -19,6 +19,107 @@ def require(condition: bool, message: str) -> None: if not condition: raise SystemExit(message) +def validate_transformation_integrity() -> None: + manifest_paths = { + "plan": ROOT / "conformance/transformation-plan-0.2.json", + "frontend": ROOT / "conformance/spss-syntax-frontend-0.2.json", + "binding": ROOT / "conformance/in-place-transformation-0.2.json", + "plan_0_1": ROOT / "conformance/transformation-plan-0.1.json", + "frontend_0_1": ROOT / "conformance/spss-syntax-frontend-0.1.json", + "binding_0_1": ROOT / "conformance/in-place-transformation-0.1.json", + } + manifests: dict[str, dict[str, object]] = {} + for name, path in manifest_paths.items(): + document = json.loads(path.read_text(encoding="utf-8")) + require(isinstance(document, dict), f"{path.name}: manifest must be an object.") + require(isinstance(document.get("cases"), list), f"{path.name}: cases must be an array.") + manifests[name] = document + + links = [ + (manifest_paths["plan"], manifests["plan"]["schema"]), + (manifest_paths["frontend"], manifests["frontend"]["request_schema"]), + *[(manifest_paths["frontend"], link) for link in manifests["frontend"]["plan_schemas"].values()], + (manifest_paths["binding"], manifests["binding"]["audit_schema"]), + (manifest_paths["plan_0_1"], manifests["plan_0_1"]["schema"]), + (manifest_paths["frontend_0_1"], manifests["frontend_0_1"]["request_schema"]), + (manifest_paths["frontend_0_1"], manifests["frontend_0_1"]["plan_schema"]), + ] + for manifest_path, link in links: + require(isinstance(link, str) and link and not Path(link).is_absolute(), f"{manifest_path.name}: schema link must be relative.") + schema_path = (manifest_path.parent / link).resolve() + require(schema_path.is_file(), f"{manifest_path.name}: schema link does not exist: {link}") + if schema_path.suffix == ".json": + require(isinstance(json.loads(schema_path.read_text(encoding="utf-8")), dict), f"{schema_path.name}: schema must be an object.") + + def canonical_hash(value: object) -> str: + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + plan_hashes: dict[str, str] = {} + for case in manifests["plan"]["cases"]: + require(isinstance(case, dict), "0.2 plan case must be an object.") + identifier = require_string(case.get("id"), "0.2 plan case id") + require(identifier not in plan_hashes, f"Duplicate successful 0.2 plan case: {identifier}") + if case.get("expected_error") is None: + require(isinstance(case.get("plan"), dict), f"{identifier}: successful plan is missing.") + digest = canonical_hash(case["plan"]) + require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical plan hash differs.") + plan_hashes[identifier] = digest + + legacy_plan_hashes: dict[str, str] = {} + for case in manifests["plan_0_1"]["cases"]: + if case.get("expected_error") is None: + identifier = require_string(case.get("id"), "0.1 plan case id") + digest = canonical_hash(case.get("plan")) + require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical 0.1 plan hash differs.") + legacy_plan_hashes[identifier] = digest + + for case in manifests["frontend_0_1"]["cases"]: + if case.get("expected_error") is not None: + continue + identifier = require_string(case.get("id"), "0.1 frontend case id") + source = case.get("request", {}).get("source_text") + require(isinstance(source, str) and case.get("expected_source_hash") == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: 0.1 source hash differs.") + if "expected_plan_case" in case: + reference = case["expected_plan_case"] + require(reference in legacy_plan_hashes, f"{identifier}: referenced 0.1 plan is missing.") + else: + require("expected_plan" in case, f"{identifier}: embedded 0.1 plan is missing.") + digest = canonical_hash(case["expected_plan"]) + require(case.get("expected_plan_hash") == digest, f"{identifier}: embedded 0.1 plan hash differs.") + legacy_hashes = { + case["id"]: case["expected_plan_hash"] + for case in manifests["frontend_0_1"]["cases"] + if case.get("expected_error") is None and "expected_plan_hash" in case + } + frontend_hashes: dict[str, tuple[str, str]] = {} + for case in manifests["frontend"]["cases"]: + require(isinstance(case, dict), "0.2 frontend case must be an object.") + if case.get("expected_error") is not None: + continue + identifier = require_string(case.get("id"), "0.2 frontend case id") + source = case.get("request", {}).get("source_text") + require(isinstance(source, str), f"{identifier}: successful frontend source is missing.") + source_hash = hashlib.sha256(source.encode("utf-8")).hexdigest() + require(case.get("expected_source_hash") == source_hash, f"{identifier}: source hash differs.") + if "expected_plan_case" in case: + reference = case["expected_plan_case"] + require(reference in plan_hashes and case.get("expected_plan_hash") == plan_hashes[reference], f"{identifier}: referenced 0.2 plan hash differs.") + elif "expected_plan_case_0_1" in case: + reference = case["expected_plan_case_0_1"] + require(reference in legacy_hashes and case.get("expected_plan_hash") == legacy_hashes[reference], f"{identifier}: referenced 0.1 plan hash differs.") + else: + require("expected_plan_0_1" in case and case.get("expected_plan_hash") == canonical_hash(case["expected_plan_0_1"]), f"{identifier}: embedded plan hash differs.") + frontend_hashes[identifier] = (case["expected_plan_hash"], source_hash) + + for case in manifests["binding"]["cases"]: + if not isinstance(case, dict) or "applied_plan_case" not in case: + continue + plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") + require(plan_id in plan_hashes and frontend_id in frontend_hashes, f"{case.get('id')}: applied fixture reference is missing.") + audit = case.get("expected_audit") + require(isinstance(audit, dict) and audit.get("plan_hash") == plan_hashes[plan_id] and audit.get("source_hash") == frontend_hashes[frontend_id][1], f"{case.get('id')}: applied fixture audit hashes differ.") + def require_well_formed_create_table_blocks( schema: str, @@ -924,9 +1025,8 @@ def main() -> None: require(f"CREATE TABLE {table} (" in schema, f"Schema table is missing: {table}") validate_transformation_profile() - from validate_transformation_plan import validate_all as validate_transformation_plan + validate_transformation_integrity() - validate_transformation_plan() validate_dialect_baseline() json.loads(MANIFEST.read_text(encoding="utf-8")) diff --git a/tools/validate_transformation_plan.py b/tools/validate_transformation_plan.py deleted file mode 100644 index 256508f..0000000 --- a/tools/validate_transformation_plan.py +++ /dev/null @@ -1,468 +0,0 @@ -"""Validate Transformation Plan 0.1 and its SPSS-like frontend fixtures.""" - -from __future__ import annotations - -import hashlib -import json -import struct -from pathlib import Path - -from validate_repository import canonical_hash, require, require_string - - -ROOT = Path(__file__).resolve().parents[1] -PLAN_MANIFEST = ROOT / "conformance/transformation-plan-0.1.json" -FRONTEND_MANIFEST = ROOT / "conformance/spss-syntax-frontend-0.1.json" -PLAN_SCHEMA = ROOT / "transformation/plan-0.1.schema.json" -FRONTEND_SCHEMA = ROOT / "transformation/spss-syntax-frontend-0.1.schema.json" -IN_PLACE_MANIFEST = ROOT / "conformance/in-place-transformation-0.1.json" - -IN_PLACE_SUCCESS_IDS = { - "mysql-recode-and-labels-preserve-dataset-and-table-identity", - "dolt-recode-and-labels-preserve-controlled-context", -} -IN_PLACE_EXPECTED_ERRORS = { - "reject-dolt-branch-mismatch": "dolt_branch_mismatch", - "reject-dolt-head-mismatch": "dolt_head_mismatch", - "reject-dolt-dirty-working-set": "dolt_working_set_dirty", - "reject-mysql-nontransactional-create-target": "schema_change_not_atomic", -} -IN_PLACE_FORBIDDEN_ARTIFACTS = { - "derived_dataset_row", - "persistent_output_table", - "full_table_copy", - "staging_dataset", - "staging_relation", - "snapshot_table", - "rollback_table", - "dataset_version_row", - "retirement_or_recovery_row", - "temporary_object_residue", -} -IN_PLACE_AUDIT_FIELDS = { - "apply_id", - "contract_id", - "database_profile", - "dataset_id", - "physical_table_schema", - "physical_table_name", - "plan_hash", - "source_hash", - "actor", - "status", - "dolt_branch", - "dolt_head_before", - "dolt_head_after", - "operation_count", - "started_at", - "completed_at", -} - - -def binary64_value(value: object, context: str) -> tuple[str, str]: - require(isinstance(value, dict), f"{context}: typed value must be an object.") - require(set(value) in ({"type", "bits"}, {"type", "value"}), f"{context}: typed value fields are invalid.") - kind = value.get("type") - if kind == "binary64": - bits = value.get("bits") - require( - isinstance(bits, str) - and len(bits) == 16 - and all(character in "0123456789abcdef" for character in bits), - f"{context}: binary64 bits must be 16 lowercase hex digits.", - ) - raw = int(bits, 16) - require((raw >> 52) & 0x7FF != 0x7FF, f"{context}: non-finite binary64 is forbidden.") - require(bits != "8000000000000000", f"{context}: negative zero must canonicalize to positive zero.") - return kind, bits - require(kind == "string" and isinstance(value.get("value"), str), f"{context}: typed value kind is invalid.") - return "string", value["value"] - - -def binary64_number(bits: str) -> float: - return struct.unpack(">d", bytes.fromhex(bits))[0] - - -def validate_result(result: object, context: str) -> set[str]: - require(isinstance(result, dict), f"{context}: result must be an object.") - kind = result.get("kind") - if kind == "literal": - require(set(result) == {"kind", "value"}, f"{context}: literal result fields are invalid.") - return {binary64_value(result["value"], context + ".value")[0]} - require(kind in {"system_missing", "copy"} and set(result) == {"kind"}, f"{context}: result is invalid.") - return {kind} - - -def validate_plan(plan: object, context: str) -> str | None: - require(isinstance(plan, dict) and set(plan) == {"contract", "input_alias", "operations"}, f"{context}: plan fields are invalid.") - require(plan["contract"] == "openstatspec-transformation-plan-v0.1", f"{context}: contract is invalid.") - require_string(plan["input_alias"], context + ".input_alias") - operations = plan["operations"] - require(isinstance(operations, list) and operations, f"{context}: operations are missing.") - semantic_error: str | None = None - for operation_index, operation in enumerate(operations): - op_context = f"{context}.operations[{operation_index}]" - require(isinstance(operation, dict), f"{op_context}: operation must be an object.") - op = operation.get("op") - if op == "recode": - require( - set(operation) == {"op", "source", "target", "target_mode", "rules", "unmatched"}, - f"{op_context}: recode fields are invalid.", - ) - source = require_string(operation["source"], op_context + ".source") - target = require_string(operation["target"], op_context + ".target") - require(not target.startswith("__"), f"{op_context}: reserved target name.") - mode = operation["target_mode"] - require(mode in {"create", "replace"}, f"{op_context}: target mode is invalid.") - require(mode != "replace" or source == target, f"{op_context}: replace target must equal source.") - rules = operation["rules"] - require(isinstance(rules, list) and rules, f"{op_context}: rules are missing.") - result_kinds: set[str] = set() - for rule_index, rule in enumerate(rules): - rule_context = f"{op_context}.rules[{rule_index}]" - require(isinstance(rule, dict) and set(rule) == {"match", "result"}, f"{rule_context}: fields are invalid.") - match = rule["match"] - require(isinstance(match, dict), f"{rule_context}.match must be an object.") - match_kind = match.get("kind") - if match_kind == "values": - require(set(match) == {"kind", "values"}, f"{rule_context}.match fields are invalid.") - require(isinstance(match["values"], list) and match["values"], f"{rule_context}: values are missing.") - for value_index, value in enumerate(match["values"]): - binary64_value(value, f"{rule_context}.match.values[{value_index}]") - elif match_kind == "range": - require(set(match) == {"kind", "lower", "upper"}, f"{rule_context}.match fields are invalid.") - lower_kind, lower_bits = binary64_value(match["lower"], rule_context + ".match.lower") - upper_kind, upper_bits = binary64_value(match["upper"], rule_context + ".match.upper") - require(lower_kind == upper_kind == "binary64", f"{rule_context}: range endpoints must be binary64.") - if binary64_number(lower_bits) > binary64_number(upper_bits): - semantic_error = semantic_error or "invalid_numeric_range" - else: - require(match_kind == "system_missing" and set(match) == {"kind"}, f"{rule_context}.match is invalid.") - result_kinds.update(validate_result(rule["result"], rule_context + ".result")) - result_kinds.update(validate_result(operation["unmatched"], op_context + ".unmatched")) - literal_kinds = result_kinds & {"binary64", "string"} - if len(literal_kinds) > 1: - semantic_error = semantic_error or "mixed_result_types" - elif op == "set_variable_label": - require(set(operation) == {"op", "variable", "label"}, f"{op_context}: variable-label fields are invalid.") - require_string(operation["variable"], op_context + ".variable") - require(isinstance(operation["label"], str), f"{op_context}.label must be a string.") - elif op == "replace_value_labels": - require(set(operation) == {"op", "variable", "labels"}, f"{op_context}: value-label fields are invalid.") - require_string(operation["variable"], op_context + ".variable") - labels = operation["labels"] - require(isinstance(labels, list) and labels, f"{op_context}: labels are missing.") - seen: set[tuple[str, str]] = set() - for label_index, label in enumerate(labels): - label_context = f"{op_context}.labels[{label_index}]" - require(isinstance(label, dict) and set(label) == {"value", "label"}, f"{label_context}: fields are invalid.") - key = binary64_value(label["value"], label_context + ".value") - if key in seen: - semantic_error = semantic_error or "duplicate_value_label" - seen.add(key) - require(isinstance(label["label"], str), f"{label_context}.label must be a string.") - else: - require(False, f"{op_context}: unknown operation.") - return semantic_error - - -def validate_plan_manifest() -> dict[str, dict[str, object]]: - manifest = json.loads(PLAN_MANIFEST.read_text(encoding="utf-8")) - require( - isinstance(manifest, dict) - and set(manifest) == {"manifest_version", "profile", "contract", "schema", "cases"}, - "Transformation Plan manifest fields are invalid.", - ) - require(manifest["manifest_version"] == "0.1", "Unexpected Transformation Plan manifest version.") - require(manifest["profile"] == "OpenStatSpec Transformation Plan 0.1", "Unexpected Transformation Plan profile.") - require(manifest["contract"] == "openstatspec-transformation-plan-v0.1", "Unexpected Transformation Plan contract.") - require(manifest["schema"] == "../transformation/plan-0.1.schema.json", "Unexpected Transformation Plan schema path.") - expected_cases = { - "numeric-recode-and-declared-labels": None, - "string-value-label-replacement": None, - "reject-descending-range": "invalid_numeric_range", - "reject-duplicate-value-label": "duplicate_value_label", - } - case_map: dict[str, dict[str, object]] = {} - for case in manifest["cases"]: - require(isinstance(case, dict) and set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "Transformation Plan case fields are invalid.") - identifier = require_string(case["id"], "Transformation Plan case id") - require(identifier not in case_map, f"Duplicate Transformation Plan case: {identifier}") - case_map[identifier] = case - error = validate_plan(case["plan"], identifier) - require(error == case["expected_error"] == expected_cases.get(identifier), f"{identifier}: semantic result differs.") - if error is None: - require(case["expected_plan_hash"] == canonical_hash(case["plan"]), f"{identifier}: plan hash differs.") - else: - require(case["expected_plan_hash"] is None, f"{identifier}: invalid plan must not claim a hash.") - require(set(case_map) == set(expected_cases), "Transformation Plan conformance case set is incomplete.") - return case_map - - -def validate_frontend_manifest(plan_cases: dict[str, dict[str, object]]) -> int: - manifest = json.loads(FRONTEND_MANIFEST.read_text(encoding="utf-8")) - require( - isinstance(manifest, dict) - and set(manifest) - == {"manifest_version", "profile", "contract", "plan_contract", "request_schema", "plan_schema", "cases"}, - "SPSS frontend manifest fields are invalid.", - ) - require(manifest["manifest_version"] == "0.1", "Unexpected SPSS frontend manifest version.") - require(manifest["profile"] == "OpenStatSpec SPSS-like Syntax Frontend 0.1", "Unexpected SPSS frontend profile.") - require(manifest["contract"] == "openstatspec-spss-syntax-frontend-v0.1", "Unexpected SPSS frontend contract.") - require(manifest["plan_contract"] == "openstatspec-transformation-plan-v0.1", "Unexpected frontend plan contract.") - expected_errors = { - "recode-labels-and-value-labels": None, - "in-place-recode-default-copy": None, - "string-value-labels": None, - "recode-varlists-positionally": None, - "value-label-varlists-and-groups": None, - "reject-unknown-command": "unsupported_spss_command", - "reject-unknown-variable": "unknown_variable", - "reject-nonfinal-else": "else_not_last", - "reject-string-system-missing": "system_missing_for_string", - "reject-duplicate-value-label": "duplicate_value_label", - "reject-string-target-without-declaration": "string_target_requires_declaration", - "reject-comment-command": "unsupported_spss_command", - "reject-inline-comment": "spss_syntax_error", - } - identifiers: set[str] = set() - for case in manifest["cases"]: - require(isinstance(case, dict), "SPSS frontend case must be an object.") - identifier = require_string(case.get("id"), "SPSS frontend case id") - require(identifier not in identifiers, f"Duplicate SPSS frontend case: {identifier}") - identifiers.add(identifier) - request = case.get("request") - require(isinstance(request, dict) and set(request) == {"contract", "input_alias", "input_schema", "source_text"}, f"{identifier}: request fields are invalid.") - require(request["contract"] == manifest["contract"], f"{identifier}: request contract differs.") - source = require_string(request["source_text"], f"{identifier}.source_text") - require(source == source.replace("\r\n", "\n").replace("\r", "\n"), f"{identifier}: source is not LF-normalized.") - source_hash = hashlib.sha256(source.encode("utf-8")).hexdigest() - require(case.get("expected_source_hash") == source_hash, f"{identifier}: source hash differs.") - require(case.get("expected_error") == expected_errors.get(identifier), f"{identifier}: expected diagnostic differs.") - if case["expected_error"] is None: - if "expected_plan_case" in case: - allowed = {"id", "request", "expected_plan_case", "expected_source_hash", "expected_error"} - require(set(case) == allowed | ({"expected_output_metadata"} if "expected_output_metadata" in case else set()), f"{identifier}: fields are invalid.") - plan_case = plan_cases.get(case["expected_plan_case"]) - require(plan_case is not None and plan_case["expected_error"] is None, f"{identifier}: referenced plan case is invalid.") - else: - allowed = {"id", "request", "expected_plan", "expected_plan_hash", "expected_source_hash", "expected_error"} - require(set(case) == allowed | ({"expected_output_metadata"} if "expected_output_metadata" in case else set()), f"{identifier}: fields are invalid.") - require(validate_plan(case["expected_plan"], identifier + ".expected_plan") is None, f"{identifier}: expected plan is invalid.") - require(case["expected_plan_hash"] == canonical_hash(case["expected_plan"]), f"{identifier}: expected plan hash differs.") - if identifier == "in-place-recode-default-copy": - require(case.get("expected_output_metadata") == { - "q1": { - "variable_label": "Original score", - "value_labels": [{ - "value": {"type": "binary64", "bits": "3ff0000000000000"}, - "label": "Original one", - }], - }, - }, f"{identifier}: metadata preservation proof differs.") - else: - require(set(case) == {"id", "request", "expected_source_hash", "expected_error"}, f"{identifier}: failure fields are invalid.") - require(identifiers == set(expected_errors), "SPSS frontend conformance case set is incomplete.") - return len(identifiers) - - -def validate_in_place_failure_case(case: dict[str, object], identifier: str) -> None: - if identifier == "reject-mysql-nontransactional-create-target": - require( - set(case) - == {"id", "database_profile", "source_text", "expected_error", - "mutation_started"}, - f"{identifier}: failure fields are invalid.", - ) - require(case["database_profile"] == "mysql", f"{identifier}: profile differs.") - require(" INTO " in case["source_text"].upper(), f"{identifier}: create target is missing.") - require( - case["expected_error"] == IN_PLACE_EXPECTED_ERRORS[identifier], - f"{identifier}: diagnostic differs.", - ) - require(case["mutation_started"] is False, f"{identifier}: mutation starts.") - return - require( - set(case) - == {"id", "database_profile", "expected_context", "observed_context", - "expected_error", "mutation_started"}, - f"{identifier}: failure fields are invalid.", - ) - require(case["database_profile"] == "dolt", f"{identifier}: failure must use Dolt.") - expected, observed = case["expected_context"], case["observed_context"] - require( - isinstance(expected, dict) - and set(expected) == {"branch", "head"} - and isinstance(observed, dict) - and set(observed) == {"branch", "head", "working_set_clean"}, - f"{identifier}: controlled context is invalid.", - ) - require( - case["expected_error"] == IN_PLACE_EXPECTED_ERRORS[identifier], - f"{identifier}: diagnostic differs.", - ) - require(case["mutation_started"] is False, f"{identifier}: mutation starts.") - if identifier == "reject-dolt-branch-mismatch": - valid = ( - observed["branch"] != expected["branch"] - and observed["head"] == expected["head"] - and observed["working_set_clean"] is True - ) - elif identifier == "reject-dolt-head-mismatch": - valid = ( - observed["branch"] == expected["branch"] - and observed["head"] != expected["head"] - and observed["working_set_clean"] is True - ) - else: - valid = ( - observed["branch"] == expected["branch"] - and observed["head"] == expected["head"] - and observed["working_set_clean"] is False - ) - require(valid, f"{identifier}: fixture does not isolate its declared failure.") - - -def validate_in_place_success_case(case: dict[str, object], identifier: str) -> None: - require( - set(case) - == {"id", "database_profile", "source_text", "before", "after", - "forbidden_artifacts", "required_audit_fields"}, - f"{identifier}: success fields are invalid.", - ) - before, after = case["before"], case["after"] - require( - isinstance(before, dict) and isinstance(after, dict), - f"{identifier}: identity snapshots are invalid.", - ) - for field in ("dataset_id", "physical_table_schema", "physical_table_name"): - require(before[field] == after[field], f"{identifier}: {field} changes.") - require( - before["dataset_count"] == after["dataset_count"], - f"{identifier}: dataset count grows.", - ) - require( - before["persistent_data_table_count"] == after["persistent_data_table_count"], - f"{identifier}: persistent data-table count grows.", - ) - require( - set(case["forbidden_artifacts"]) == IN_PLACE_FORBIDDEN_ARTIFACTS, - f"{identifier}: forbidden-artifact proof is incomplete.", - ) - require( - set(case["required_audit_fields"]) == IN_PLACE_AUDIT_FIELDS, - f"{identifier}: compact audit fields are incomplete.", - ) - require( - " INTO " not in case["source_text"].upper(), - f"{identifier}: implicit-commit success creates a target.", - ) - require( - after.get("same_table_recode_target") == "score" - and "same_table_has_column" not in after, - f"{identifier}: success does not prove an existing-target recode.", - ) - if case["database_profile"] == "mysql": - require( - not any(key.startswith("dolt_") for key in before | after), - f"{identifier}: non-Dolt case contains Dolt context.", - ) - else: - require(case["database_profile"] == "dolt", f"{identifier}: unexpected profile.") - require( - before["dolt_branch"] == after["dolt_branch"] - and before["dolt_head"] == after["dolt_head"], - f"{identifier}: Dolt branch/HEAD changes.", - ) - require(after["dolt_commit_performed"] is False, f"{identifier}: Dolt commit.") - - -def validate_in_place_manifest() -> None: - manifest = json.loads(IN_PLACE_MANIFEST.read_text(encoding="utf-8")) - require( - isinstance(manifest, dict) - and set(manifest) == {"manifest_version", "profile", "contract", "cases"}, - "In-place manifest fields are invalid.", - ) - require(manifest["manifest_version"] == "0.1", "Unexpected in-place manifest version.") - require( - manifest["profile"] == "OpenStatSpec In-Place Transformation Binding 0.1", - "Unexpected in-place profile.", - ) - require( - manifest["contract"] == "openstatspec-in-place-transformation-v0.1", - "Unexpected in-place contract.", - ) - cases = manifest["cases"] - require(isinstance(cases, list), "In-place cases must be an array.") - case_map = {require_string(case.get("id"), "In-place case id"): case for case in cases} - require(len(case_map) == len(cases), "Duplicate in-place case.") - require( - set(case_map) == IN_PLACE_SUCCESS_IDS | set(IN_PLACE_EXPECTED_ERRORS), - "In-place conformance case set is incomplete.", - ) - for identifier in IN_PLACE_SUCCESS_IDS: - validate_in_place_success_case(case_map[identifier], identifier) - for identifier in IN_PLACE_EXPECTED_ERRORS: - validate_in_place_failure_case(case_map[identifier], identifier) - - -def validate_repository_links() -> None: - plan_schema = json.loads(PLAN_SCHEMA.read_text(encoding="utf-8")) - frontend_schema = json.loads(FRONTEND_SCHEMA.read_text(encoding="utf-8")) - require(plan_schema["$id"].endswith("transformation-plan-0.1.schema.json"), "Transformation Plan schema ID is invalid.") - require(frontend_schema["$id"].endswith("spss-syntax-frontend-0.1.schema.json"), "SPSS frontend schema ID is invalid.") - plan_sql = (ROOT / "sql/transformation-plan-profile-schema.sql").read_text(encoding="utf-8") - require("CREATE TABLE transformation_apply (" in plan_sql, "Compact apply audit table is missing.") - require( - "openstatspec-in-place-transformation-v0.1" in plan_sql - and "openstatspec-dolt-in-place-transformation-v0.1" not in plan_sql, - "Compact apply audit does not use the generic in-place contract.", - ) - for field in ("database_profile", "physical_table_schema", "physical_table_name"): - require(field in plan_sql, f"Compact apply audit field is missing: {field}") - for forbidden in ( - "transformation_plan_version", - "transformation_plan_compilation", - "transformation_plan_run_binding", - "transformation_run_artifact", - ): - require(forbidden not in plan_sql, f"Forbidden history/copy audit relation remains: {forbidden}") - binding = (ROOT / "docs/transformation-plan-sql-binding-0.1.md").read_text(encoding="utf-8") - binding_normalized = " ".join(binding.split()) - for phrase in ( - "dataset_id`, physical schema, and physical table name remain unchanged", - "MUST NOT publish a derived dataset", - "does not version successful data states", - "MUST NOT call `DOLT_COMMIT`", - "clean Dolt working set", - "For a non-Dolt profile every Dolt-specific field is NULL", - "MySQL-family implicit-commit DDL", - "schema_change_not_atomic", - ): - require(phrase in binding_normalized, f"In-place binding requirement is missing: {phrase}") - - validate_in_place_manifest() - - -def validate_all() -> tuple[int, int]: - plan_cases = validate_plan_manifest() - frontend_count = validate_frontend_manifest(plan_cases) - validate_repository_links() - from validate_transformation_plan_0_2 import validate_all as validate_0_2 - - validate_0_2() - return len(plan_cases), frontend_count - - -def main() -> None: - plan_count, frontend_count = validate_all() - print( - f"Validated {plan_count} Transformation Plan and " - f"{frontend_count} SPSS frontend conformance cases." - ) - - -if __name__ == "__main__": - main() diff --git a/tools/validate_transformation_plan_0_2.py b/tools/validate_transformation_plan_0_2.py deleted file mode 100755 index 56ee641..0000000 --- a/tools/validate_transformation_plan_0_2.py +++ /dev/null @@ -1,819 +0,0 @@ -"""Validate the additive Transformation Plan and SPSS frontend 0.2 fixtures.""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -from jsonschema import Draft202012Validator - - -from validate_repository import canonical_hash, require, require_string -from validate_transformation_plan import validate_plan as validate_plan_0_1 - - -ROOT = Path(__file__).resolve().parents[1] -PLAN_0_1 = ROOT / "conformance/transformation-plan-0.1.json" -FRONTEND_0_1 = ROOT / "conformance/spss-syntax-frontend-0.1.json" -PLAN_SCHEMA_0_1 = ROOT / "transformation/plan-0.1.schema.json" -PLAN = ROOT / "conformance/transformation-plan-0.2.json" -FRONTEND = ROOT / "conformance/spss-syntax-frontend-0.2.json" -IN_PLACE = ROOT / "conformance/in-place-transformation-0.2.json" -AUDIT_SCHEMA = ROOT / "sql/transformation-plan-profile-schema.sql" -PLAN_SCHEMA = ROOT / "transformation/plan-0.2.schema.json" -FRONTEND_SCHEMA = ROOT / "transformation/spss-syntax-frontend-0.2.schema.json" - - -def typed_value(value: object, context: str) -> str: - require(isinstance(value, dict), f"{context}: typed value must be an object.") - if value.get("type") == "binary64": - require(set(value) == {"type", "bits"}, f"{context}: binary64 fields differ.") - bits = value["bits"] - require( - isinstance(bits, str) - and len(bits) == 16 - and all(character in "0123456789abcdef" for character in bits), - f"{context}: binary64 bits differ.", - ) - exponent = (int(bits, 16) >> 52) & 0x7FF - require(exponent != 0x7FF and bits != "8000000000000000", f"{context}: noncanonical binary64.") - return "numeric" - require( - value.get("type") == "string" - and set(value) == {"type", "value"} - and isinstance(value.get("value"), str), - f"{context}: string value differs.", - ) - return "string" - - -def typed_value_key(value: object, context: str) -> tuple[str, str]: - kind = typed_value(value, context) - return (kind, value["bits"] if kind == "numeric" else value["value"]) - - -def operand(value: object, context: str) -> str: - require(isinstance(value, dict), f"{context}: operand must be an object.") - if value.get("kind") == "variable": - require(set(value) == {"kind", "variable"}, f"{context}: variable operand fields differ.") - require_string(value["variable"], context + ".variable") - return "variable" - require(value.get("kind") == "literal" and set(value) == {"kind", "value"}, f"{context}: operand differs.") - return typed_value(value["value"], context + ".value") - - -def predicate(value: object, context: str) -> bool: - require(isinstance(value, dict), f"{context}: predicate must be an object.") - if value.get("expression") == "comparison": - require(set(value) == {"expression", "left", "operator", "right"}, f"{context}: comparison fields differ.") - require(value["operator"] in {"=", "<", "<=", ">", ">="}, f"{context}: comparison operator differs.") - kinds = {operand(value["left"], context + ".left"), operand(value["right"], context + ".right")} - return "string" in kinds - require(value.get("expression") == "boolean", f"{context}: expression kind differs.") - require(set(value) == {"expression", "operator", "operands"}, f"{context}: boolean fields differ.") - require(value["operator"] in {"and", "or"}, f"{context}: boolean operator differs.") - children = value["operands"] - require(isinstance(children, list) and len(children) >= 2, f"{context}: boolean operands differ.") - return any(predicate(child, f"{context}.operands[{index}]") for index, child in enumerate(children)) - - -def has_nested_same_operator(value: object) -> bool: - if not isinstance(value, dict) or value.get("expression") != "boolean": - return False - operator = value.get("operator") - children = value.get("operands", []) - return any( - isinstance(child, dict) - and child.get("expression") == "boolean" - and child.get("operator") == operator - or has_nested_same_operator(child) - for child in children - ) - - -def validate_plan(plan: object, context: str) -> str | None: - require(isinstance(plan, dict) and set(plan) == {"contract", "input_alias", "operations"}, f"{context}: plan fields differ.") - require(plan["contract"] == "openstatspec-transformation-plan-v0.2", f"{context}: contract differs.") - require_string(plan["input_alias"], context + ".input_alias") - operations = plan["operations"] - require(isinstance(operations, list) and operations, f"{context}: operations missing.") - semantic_error = None - allowed = { - "assign", "conditional_assign", "set_variable_label", "replace_value_labels", - "set_format", "set_measurement_level", "execute", "recode", - } - for index, operation in enumerate(operations): - op_context = f"{context}.operations[{index}]" - require(isinstance(operation, dict) and operation.get("op") in allowed, f"{op_context}: operation differs.") - op = operation["op"] - if op == "assign": - require(set(operation) == {"op", "target", "target_mode", "value"}, f"{op_context}: assign fields differ.") - target = require_string(operation["target"], op_context + ".target") - require(operation["target_mode"] in {"create", "replace"}, f"{op_context}: target mode differs.") - if target.startswith("__"): - semantic_error = semantic_error or "reserved_target_name" - if operand(operation["value"], op_context + ".value") == "string": - semantic_error = semantic_error or "expression_type_unsupported" - elif op == "conditional_assign": - require(set(operation) == {"op", "condition", "target", "value"}, f"{op_context}: conditional fields differ.") - target = require_string(operation["target"], op_context + ".target") - string_predicate = predicate(operation["condition"], op_context + ".condition") - if has_nested_same_operator(operation["condition"]): - semantic_error = semantic_error or "noncanonical_boolean_shape" - if target.startswith("__"): - semantic_error = semantic_error or "reserved_target_name" - string_value = operand(operation["value"], op_context + ".value") == "string" - if string_predicate or string_value: - semantic_error = semantic_error or "expression_type_unsupported" - elif op == "set_variable_label": - require(set(operation) == {"op", "variable", "label"} and isinstance(operation["label"], str), f"{op_context}: label fields differ.") - elif op == "replace_value_labels": - require(set(operation) == {"op", "variable", "labels"} and isinstance(operation["labels"], list), f"{op_context}: value labels differ.") - require_string(operation["variable"], op_context + ".variable") - require(operation["labels"], f"{op_context}: labels missing.") - seen: set[tuple[str, str]] = set() - for label_index, label in enumerate(operation["labels"]): - require(set(label) == {"value", "label"} and isinstance(label["label"], str), f"{op_context}.labels[{label_index}]: fields differ.") - key = typed_value_key(label["value"], f"{op_context}.labels[{label_index}].value") - if key in seen: - semantic_error = semantic_error or "duplicate_value_label" - seen.add(key) - elif op == "set_format": - require(set(operation) == {"op", "variable", "family", "width", "decimals"}, f"{op_context}: format fields differ.") - require(operation["family"] == "F", f"{op_context}: format family differs.") - width, decimals = operation["width"], operation["decimals"] - require(isinstance(width, int) and 1 <= width <= 40, f"{op_context}: width differs.") - require(isinstance(decimals, int) and 0 <= decimals <= 16, f"{op_context}: decimals differ.") - if decimals and width < decimals + 2: - semantic_error = semantic_error or "invalid_format" - elif op == "set_measurement_level": - require(set(operation) == {"op", "variable", "level"}, f"{op_context}: level fields differ.") - require(operation["level"] in {"nominal", "ordinal", "scale"}, f"{op_context}: level differs.") - elif op == "execute": - require(set(operation) == {"op"}, f"{op_context}: execute fields differ.") - else: - require(op == "recode", f"{op_context}: unknown operation.") - legacy_error = validate_plan_0_1( - {"contract": "openstatspec-transformation-plan-v0.1", - "input_alias": plan["input_alias"], - "operations": [operation]}, - op_context, - ) - semantic_error = semantic_error or legacy_error - return semantic_error - - -def validate_plan_manifest() -> dict[str, dict[str, object]]: - manifest = json.loads(PLAN.read_text(encoding="utf-8")) - require(set(manifest) == {"manifest_version", "profile", "contract", "schema", "canonicalization", "cases"}, "0.2 plan manifest fields differ.") - require(manifest["manifest_version"] == "0.2", "0.2 plan manifest version differs.") - require(manifest["profile"] == "OpenStatSpec Transformation Plan 0.2", "0.2 plan profile differs.") - require(manifest["contract"] == "openstatspec-transformation-plan-v0.2", "0.2 plan contract differs.") - require(manifest["schema"] == "../transformation/plan-0.2.schema.json", "0.2 plan schema link differs.") - plan_schema_path = (PLAN.parent / manifest["schema"]).resolve() - require( - plan_schema_path == PLAN_SCHEMA.resolve() and plan_schema_path.is_file(), - "0.2 plan schema is missing or mismatched.", - ) - require(manifest["canonicalization"] == "restricted-rfc8785-utf8-sha256", "0.2 canonicalization differs.") - cases = {} - expected = { - "sequential-conditional-binary-existing-target": None, - "three-term-and-flattens-source-order": None, - "reject-string-predicate": "expression_type_unsupported", - "nested-or-inequalities-variable-operands-create": None, - "sqlite-create-target-source-copy": None, - "mixed-recode-and-compute-promotes-v0.2": None, - "reject-invalid-format": "invalid_format", - "reject-reserved-assignment-target": "reserved_target_name", - "reject-duplicate-value-label": "duplicate_value_label", - "mixed-default-and-precedence": None, - "mixed-parentheses-override-precedence": None, - "finite-number-rounding-exponent-underflow": None, - "three-term-or-flattens-source-order": None, - "strict-greater-than": None, - "comparison-operand-permutations": None, - "minimum-nonzero-subnormal": None, - "grouped-format-and-measurement-level-order": None, - "negative-zero-canonical-positive-zero": None, - "reject-string-assign-value": "expression_type_unsupported", - "reject-string-conditional-assign-value": "expression_type_unsupported", - "string-measurement-level-target": None, - "reject-nested-same-operator-boolean": "noncanonical_boolean_shape", - } - for case in manifest["cases"]: - require(set(case) == {"id", "plan", "expected_plan_hash", "expected_error"}, "0.2 plan case fields differ.") - identifier = require_string(case["id"], "0.2 plan case id") - require(identifier not in cases, f"Duplicate 0.2 plan case: {identifier}") - cases[identifier] = case - error = validate_plan(case["plan"], identifier) - require(error == case["expected_error"] == expected.get(identifier), f"{identifier}: semantic result differs.") - if error is None: - require(case["expected_plan_hash"] == canonical_hash(case["plan"]), f"{identifier}: golden plan hash differs.") - else: - require(case["expected_plan_hash"] is None, f"{identifier}: invalid plan claims a hash.") - if identifier == "finite-number-rounding-exponent-underflow": - require( - [ - operation["value"]["value"]["bits"] - for operation in case["plan"]["operations"][:-1] - ] == [ - "3ff0000000000000", - "3ff0000000000002", - "c05f400000000000", - "0000000000000000", - ], - "Finite-number conversion golden bits differ.", - ) - elif identifier == "minimum-nonzero-subnormal": - require( - case["plan"]["operations"][0]["value"]["value"]["bits"] - == "0000000000000001", - "Minimum-subnormal conversion golden bits differ.", - ) - elif identifier == "negative-zero-canonical-positive-zero": - require( - case["plan"]["operations"][0]["value"]["value"]["bits"] - == "0000000000000000", - "Negative-zero canonical conversion bits differ.", - ) - elif identifier == "mixed-recode-and-compute-promotes-v0.2": - require( - [operation["op"] for operation in case["plan"]["operations"]] == ["recode", "assign"], - "Mixed legacy/0.2 plan operation coverage differs.") - require(set(cases) == set(expected), "0.2 plan case set differs.") - return cases - - -def validate_frontend(plan_cases: dict[str, dict[str, object]]) -> int: - manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) - require(set(manifest) == {"manifest_version", "profile", "contract", "plan_contracts", "request_schema", "plan_schemas", "cases"}, "0.2 frontend manifest fields differ.") - require(manifest["manifest_version"] == "0.2", "0.2 frontend manifest version differs.") - require(manifest["profile"] == "OpenStatSpec SPSS-like Syntax Frontend 0.2", "0.2 frontend profile differs.") - require(manifest["contract"] == "openstatspec-spss-syntax-frontend-v0.2", "0.2 frontend contract differs.") - require( - manifest["request_schema"] == "../transformation/spss-syntax-frontend-0.2.schema.json", - "0.2 frontend request-schema link differs.", - ) - request_schema_path = (FRONTEND.parent / manifest["request_schema"]).resolve() - require( - request_schema_path == FRONTEND_SCHEMA.resolve() and request_schema_path.is_file(), - "0.2 frontend request schema is missing or mismatched.", - ) - require(manifest["plan_contracts"] == [ - "openstatspec-transformation-plan-v0.1", - "openstatspec-transformation-plan-v0.2", - ], "0.2 frontend plan contracts differ.") - require(manifest["plan_schemas"] == { - "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", - "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json", - }, "0.2 frontend plan-schema mapping differs.") - legacy_manifest = json.loads(FRONTEND_0_1.read_text(encoding="utf-8")) - legacy_cases = {case["id"]: case for case in legacy_manifest["cases"]} - identifiers = set() - expected_failures = { - "reject-string-expression": "expression_type_unsupported", - "reject-missing-conditional-target": "conditional_target_missing", - "reject-invalid-format": "invalid_format", - "reject-numeric-overflow": "spss_syntax_error", - "reject-leading-decimal-point": "spss_syntax_error", - "reject-trailing-decimal-point": "spss_syntax_error", - "reject-string-predicate": "expression_type_unsupported", - "reject-arithmetic-expression": "expression_type_unsupported", - "reject-unsupported-command": "unsupported_spss_command", - "reject-leading-plus": "spss_syntax_error", - "reject-leading-zero": "spss_syntax_error", - "reject-nan-token": "spss_syntax_error", - "reject-infinity-token": "spss_syntax_error", - "reject-if-without-parentheses": "spss_syntax_error", - "reject-if-else-form": "spss_syntax_error", - "reject-non-f-format": "invalid_format", - "reject-invalid-measurement-level": "spss_syntax_error", - "reject-string-target-compute": "expression_type_unsupported", - "reject-string-target-conditional": "expression_type_unsupported", - "reject-string-target-format": "expression_type_unsupported", - } - for case in manifest["cases"]: - identifier = require_string(case["id"], "0.2 frontend case id") - require(identifier not in identifiers, f"Duplicate 0.2 frontend case: {identifier}") - identifiers.add(identifier) - request = case["request"] - require(request["contract"] == manifest["contract"], f"{identifier}: request contract differs.") - source = request["source_text"].replace("\r\n", "\n").replace("\r", "\n") - require(source == request["source_text"], f"{identifier}: source is not normalized.") - require(case["expected_source_hash"] == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: source hash differs.") - if identifier in expected_failures: - require( - set(case) == {"id", "request", "expected_source_hash", "expected_error"}, - f"{identifier}: frontend failure fields differ.", - ) - require( - case["expected_error"] == expected_failures[identifier], - f"{identifier}: frontend diagnostic differs.", - ) - elif "expected_plan_case" in case: - require(case["expected_error"] is None, f"{identifier}: frontend success claims an error.") - require(set(case) == {"id", "request", "expected_plan_case", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: frontend success fields differ.") - require(case["expected_plan_case"] in plan_cases, f"{identifier}: plan fixture missing.") - plan_case = plan_cases[case["expected_plan_case"]] - require( - plan_case["expected_error"] is None - and isinstance(plan_case["expected_plan_hash"], str) - and len(plan_case["expected_plan_hash"]) == 64, - f"{identifier}: frontend success links an invalid plan fixture.", - ) - require(case["expected_plan_hash"] == plan_case["expected_plan_hash"], f"{identifier}: plan hash link differs.") - required_tokens = { - "compute-if-labels-format-level-execute-existing-target": ( - "COMPUTE ", "IF (", " AND ", "FORMATS ", "VARIABLE LEVEL ", "EXECUTE.", - ), - "three-term-and-flattens-source-order": ( - "IF (", "source_a = 1 AND source_b = 1 AND source_c = 1", "EXECUTE.", - ), - "parenthesized-three-term-and-flattens-source-order": ( - "IF ((source_a = 1 AND source_b = 1) AND source_c = 1)", "EXECUTE.", - ), - "nested-or-inequality-variable-operands-create": ( - "COMPUTE ", "IF (", " OR ", " AND ", ">=", "<", "<=", "EXECUTE.", - ), - "sqlite-create-target-source-copy": ("COMPUTE target = source_a.",), - "mixed-recode-and-compute-promotes-v0.2": ( - "RECODE q1 (1 = 0).", "COMPUTE target = q1.", - ), - "unparenthesized-mixed-default-precedence": ( - "IF (source_a = 1 OR source_b = 1 AND source_c = 1)", "EXECUTE.", - ), - "parenthesized-mixed-override-precedence": ( - "IF ((source_a = 1 OR source_b = 1) AND source_c = 1)", "EXECUTE.", - ), - "finite-number-rounding-exponent-underflow": ( - "1.00000000000000011102230246251565404236316680908203125", - "1.00000000000000033306690738754696212708950042724609375", - "-1.25e+2", "1e-4000", "EXECUTE.", - ), - "three-term-or-flattens-source-order": ( - "source_a = 1 OR source_b = 1 OR source_c = 1", "EXECUTE.", - ), - "parenthesized-three-term-or-flattens-source-order": ( - "((source_a = 1 OR source_b = 1) OR source_c = 1)", "EXECUTE.", - ), - "strict-greater-than": ("source_a > 1", "EXECUTE."), - "comparison-operand-permutations": ( - "source_a = source_b", "1 < source_c", "1 = 1", "EXECUTE.", - ), - "minimum-nonzero-subnormal": ("5e-324", "EXECUTE."), - "grouped-format-and-measurement-level-order": ( - "FORMATS source_a (F8.2) source_b (F10.3).", - "VARIABLE LEVEL source_a source_b (ORDINAL) / source_c target (SCALE).", - "EXECUTE.", - ), - "negative-zero-canonical-positive-zero": ("COMPUTE target = -0.", "EXECUTE."), - "string-measurement-level-target": ( - "VARIABLE LEVEL color (NOMINAL).", "EXECUTE.", - ), - }[identifier] - for token in required_tokens: - require(token in source, f"{identifier}: bounded command coverage missing: {token}") - else: - if identifier == "metadata-only-variable-labels-retains-v0.1-plan": - require(set(case) == { - "id", "request", "expected_plan_contract", "expected_plan_0_1", - "expected_plan_hash", "expected_source_hash", "expected_error", - }, f"{identifier}: embedded 0.1 compatibility fields differ.") - legacy_plan = case["expected_plan_0_1"] - require(validate_plan_0_1(legacy_plan, identifier + ".0.1_plan") is None, f"{identifier}: embedded 0.1 plan is invalid.") - require(case["expected_plan_contract"] == legacy_plan["contract"], f"{identifier}: 0.1 contract differs.") - require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") - else: - require( - identifier in { - "old-subset-retains-v0.1-plan", - "metadata-only-value-labels-retains-v0.1-plan", - }, - f"Unexpected 0.2 frontend case: {identifier}", - ) - require(set(case) == {"id", "request", "expected_plan_contract", "expected_plan_case_0_1", "expected_plan_hash", "expected_source_hash", "expected_error"}, f"{identifier}: 0.1 compatibility fields differ.") - legacy_case = legacy_cases.get(case["expected_plan_case_0_1"]) - require(legacy_case is not None and "expected_plan" in legacy_case, f"{identifier}: immutable 0.1 plan fixture missing.") - legacy_plan = legacy_case["expected_plan"] - require(validate_plan_0_1(legacy_plan, identifier + ".0.1_plan") is None, f"{identifier}: immutable 0.1 plan is invalid.") - require(case["expected_plan_contract"] == legacy_plan["contract"], f"{identifier}: 0.1 contract differs.") - require(case["expected_plan_hash"] == legacy_case["expected_plan_hash"], f"{identifier}: 0.1 fixture hash differs.") - require(case["expected_plan_hash"] == canonical_hash(legacy_plan), f"{identifier}: 0.1 canonical hash differs.") - require(identifiers == { - "compute-if-labels-format-level-execute-existing-target", - "three-term-and-flattens-source-order", - "parenthesized-three-term-and-flattens-source-order", - "nested-or-inequality-variable-operands-create", - "mixed-recode-and-compute-promotes-v0.2", - "sqlite-create-target-source-copy", - "old-subset-retains-v0.1-plan", - "reject-string-expression", - "reject-missing-conditional-target", - "reject-invalid-format", - "unparenthesized-mixed-default-precedence", - "parenthesized-mixed-override-precedence", - "finite-number-rounding-exponent-underflow", - "reject-numeric-overflow", - "reject-leading-decimal-point", - "reject-trailing-decimal-point", - "three-term-or-flattens-source-order", - "parenthesized-three-term-or-flattens-source-order", - "strict-greater-than", - "comparison-operand-permutations", - "reject-string-predicate", - "reject-arithmetic-expression", - "reject-unsupported-command", - "reject-leading-plus", - "reject-leading-zero", - "reject-nan-token", - "reject-infinity-token", - "minimum-nonzero-subnormal", - "grouped-format-and-measurement-level-order", - "reject-if-without-parentheses", - "reject-if-else-form", - "reject-non-f-format", - "reject-invalid-measurement-level", - "negative-zero-canonical-positive-zero", - "reject-string-target-compute", - "reject-string-target-conditional", - "reject-string-target-format", - "metadata-only-value-labels-retains-v0.1-plan", - "metadata-only-variable-labels-retains-v0.1-plan", - "string-measurement-level-target", - }, "0.2 frontend case set differs.") - return len(identifiers) - - -def validate_in_place(plan_cases: dict[str, dict[str, object]]) -> None: - manifest = json.loads(IN_PLACE.read_text(encoding="utf-8")) - frontend_manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) - frontend_cases = {case["id"]: case for case in frontend_manifest["cases"]} - require( - set(manifest) == {"manifest_version", "profile", "contract", "audit_schema", "cases"}, - "0.2 binding manifest fields differ.", - ) - require(manifest["manifest_version"] == "0.2", "0.2 binding manifest version differs.") - require( - manifest["profile"] == "OpenStatSpec In-Place Transformation Binding 0.2", - "0.2 binding profile differs.", - ) - require(manifest["contract"] == "openstatspec-in-place-transformation-v0.2", "0.2 binding contract differs.") - require( - manifest["audit_schema"] == "../sql/transformation-plan-profile-schema.sql", - "0.2 binding audit schema link differs.", - ) - audit_schema_path = (IN_PLACE.parent / manifest["audit_schema"]).resolve() - require(audit_schema_path == AUDIT_SCHEMA.resolve(), "0.2 binding audit schema path differs.") - audit_schema = audit_schema_path.read_text(encoding="utf-8") - require( - "contract_id IN (" in audit_schema - and "'openstatspec-in-place-transformation-v0.1'" in audit_schema - and f"'{manifest['contract']}'" in audit_schema, - "In-place audit schema does not accept both 0.1 and 0.2 contracts.", - ) - case_list = manifest["cases"] - require(isinstance(case_list, list), "0.2 binding cases must be an array.") - cases = {require_string(case.get("id"), "0.2 binding case id"): case for case in case_list} - require(len(cases) == len(case_list), "Duplicate 0.2 binding case id.") - expected_ids = { - "dolt-preprovisioned-target-sequential-null-semantics", - "dolt-preprovisioned-target-or-null-semantics", - "dolt-preprovisioned-target-variable-missing-propagation", - "dolt-preprovisioned-target-conditional-variable-missing-propagation", - "dolt-empty-actor-fails-before-mutation", - "sqlite-create-target-atomic-success", - "sqlite-inequality-boundary-semantics", - "dolt-context-changed-after-mutation-rolls-back", - } | { - f"{profile}-create-target-fails-before-mutation" - for profile in ("dolt", "mysql", "mariadb") - } - require(set(cases) == expected_ids, "0.2 binding case set differs.") - - success = cases["dolt-preprovisioned-target-sequential-null-semantics"] - require(set(success) == { - "id", "database_profile", "applied_plan_case", "applied_frontend_case", "before", "after", - "actor", "expected_branch", "expected_head", "working_set_clean", "expected_audit", "forbidden_artifacts", "required_audit_fields", "expected_error", - }, "0.2 binding success fields differ.") - require(success["database_profile"] == "dolt" and success["actor"] == "conformance-runner" and success["expected_error"] is None, "Dolt success identity or actor differs.") - require(success["applied_plan_case"] in plan_cases, "Dolt applied plan fixture missing.") - require(success["applied_frontend_case"] in frontend_cases, "Dolt applied frontend fixture missing.") - before, after = success["before"], success["after"] - success_frontend = frontend_cases[success["applied_frontend_case"]] - require(success_frontend.get("expected_error") is None and success_frontend.get("expected_plan_case") == success["applied_plan_case"], "Dolt frontend does not link the applied successful plan.") - require(success["expected_branch"] == before["dolt_branch"] and success["expected_head"] == before["dolt_head"] and success["working_set_clean"] is before["working_set_clean"] is True, "Dolt success apply context inputs differ.") - for field in ( - "dataset_id", "physical_table_schema", "physical_table_name", - "dataset_count", "persistent_data_table_count", "case_count", - "dolt_branch", "dolt_head", - ): - require(before[field] == after[field], f"Dolt success changes {field}.") - require(before["case_count"] == len(before["rows"]) == len(after["rows"]) == 4, "Dolt case count differs.") - before_ordinals = [row["__case_ordinal"] for row in before["rows"]] - after_ordinals = [row["__case_ordinal"] for row in after["rows"]] - require(before_ordinals == after_ordinals == [1, 2, 3, 4], "Dolt case order differs.") - require(before["target_provisioning"] == { - "physical_column": True, - "catalog_variable": True, - "dolt_commit": before["dolt_head"], - }, "Dolt physical/catalog provisioning is not coupled to HEAD.") - missing_values = [ - {"ordinal": 1, "rule_kind": "discrete", "code_kind": "numeric", "numeric_value": -9}, - ] - require(before["target_metadata"] == {"missing_values": missing_values}, "Dolt pre-apply missing values differ.") - require(after["target_metadata"] == { - "variable_label": "Synthetic conjunction", - "value_labels": [[0, "No"], [1, "Yes"]], - "format": "F1.0", - "measurement_level": "nominal", - "missing_values": missing_values, - }, "Dolt target metadata result or preserved missing values differ.") - require(before["working_set_clean"] is True and after["working_set_clean"] is False, "Dolt working-set state differs.") - require([row["target"] for row in after["rows"]] == [1, 0, 0, 0], "Three-valued IF result differs.") - require( - [[row["source_a"], row["source_b"]] for row in after["rows"]] - == [[row["source_a"], row["source_b"]] for row in before["rows"]], - "Dolt success changes source values.") - require(after["dolt_commit_performed"] is False, "Apply performs DOLT_COMMIT.") - require(success["expected_audit"] == { - "contract_id": manifest["contract"], - "database_profile": "dolt", - "dataset_id": before["dataset_id"], - "physical_table_schema": before["physical_table_schema"], - "physical_table_name": before["physical_table_name"], - "status": "succeeded", - "actor": "conformance-runner", - "operation_count": 7, - "plan_hash": plan_cases[success["applied_plan_case"]]["expected_plan_hash"], - "source_hash": frontend_cases[success["applied_frontend_case"]]["expected_source_hash"], - "dolt_branch": before["dolt_branch"], - "dolt_head_before": before["dolt_head"], - "dolt_head_after": after["dolt_head"], - }, "Dolt expected audit identity differs.") - require(set(success["forbidden_artifacts"]) == { - "derived_dataset_row", "persistent_output_table", "full_table_copy", - "staging_dataset", "staging_relation", "snapshot_table", "rollback_table", - "dataset_version_row", "retirement_or_recovery_row", "temporary_object_residue", - }, "Dolt forbidden-artifact proof differs.") - require(set(success["required_audit_fields"]) == { - "apply_id", "contract_id", "database_profile", "dataset_id", - "physical_table_schema", "physical_table_name", "plan_hash", "source_hash", - "actor", "status", "dolt_branch", "dolt_head_before", "dolt_head_after", - "operation_count", "started_at", "completed_at", - }, "Dolt required audit fields differ.") - - or_case = cases["dolt-preprovisioned-target-or-null-semantics"] - require(set(or_case) == { - "id", "database_profile", "target_preprovisioned", "actor", "expected_branch", "expected_head", - "working_set_clean", "condition", "before_rows", "after_rows", "expected_error", - }, "Dolt OR-null case fields differ.") - require(or_case["database_profile"] == "dolt" and or_case["target_preprovisioned"] is True and or_case["actor"] == "conformance-runner" and or_case["expected_branch"] == "feature/recode" and or_case["expected_head"] == "provisioning-commit" and or_case["working_set_clean"] is True, "Dolt OR-null identity or apply context differs.") - require(or_case["condition"] == "source_a = 1 OR source_b = 1", "Dolt OR-null condition differs.") - require(or_case["before_rows"] == [ - {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 0}, - {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 0}, - ], "Dolt OR-null inputs differ.") - require(or_case["after_rows"] == [ - {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 1}, - {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 0}, - ], "Dolt OR three-valued outputs differ.") - require(or_case["expected_error"] is None, "Dolt OR-null case unexpectedly fails.") - - missing_case = cases["dolt-preprovisioned-target-variable-missing-propagation"] - require(set(missing_case) == { - "id", "database_profile", "target_preprovisioned", "actor", "expected_branch", "expected_head", - "working_set_clean", "assign", "before_rows", "after_rows", "expected_error", - }, "Dolt missing-propagation case fields differ.") - require( - missing_case["database_profile"] == "dolt" - and missing_case["actor"] == "conformance-runner" - and missing_case["target_preprovisioned"] is True - and missing_case["expected_branch"] == "feature/recode" - and missing_case["expected_head"] == "provisioning-commit" - and missing_case["working_set_clean"] is True, - "Dolt missing-propagation identity differs.", - ) - require( - missing_case["assign"] == {"target": "target", "value_variable": "source_a"}, - "Dolt missing-propagation assignment differs.", - ) - require(missing_case["before_rows"] == [ - {"__case_ordinal": 1, "source_a": None, "target": 7}, - {"__case_ordinal": 2, "source_a": 2, "target": 7}, - ], "Dolt missing-propagation inputs differ.") - require(missing_case["after_rows"] == [ - {"__case_ordinal": 1, "source_a": None, "target": None}, - {"__case_ordinal": 2, "source_a": 2, "target": 2}, - ], "Dolt variable assignment does not preserve missing.") - require(missing_case["expected_error"] is None, "Dolt missing-propagation case unexpectedly fails.") - - conditional_missing_case = cases["dolt-preprovisioned-target-conditional-variable-missing-propagation"] - require(set(conditional_missing_case) == { - "id", "database_profile", "target_preprovisioned", "actor", "expected_branch", "expected_head", - "working_set_clean", "conditional_assign", "before_rows", "after_rows", "expected_error", - }, "Dolt conditional missing-propagation fields differ.") - require( - conditional_missing_case["database_profile"] == "dolt" - and conditional_missing_case["actor"] == "conformance-runner" - and conditional_missing_case["target_preprovisioned"] is True - and conditional_missing_case["expected_branch"] == "feature/recode" - and conditional_missing_case["expected_head"] == "provisioning-commit" - and conditional_missing_case["working_set_clean"] is True, - "Dolt conditional missing-propagation identity differs.", - ) - require(conditional_missing_case["conditional_assign"] == { - "target": "target", - "value_variable": "source_a", - "predicate": "source_b = 1", - }, "Dolt conditional missing-propagation operation differs.") - require(conditional_missing_case["before_rows"] == [ - {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": 7}, - {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 7}, - ], "Dolt conditional missing-propagation inputs differ.") - require(conditional_missing_case["after_rows"] == [ - {"__case_ordinal": 1, "source_a": None, "source_b": 1, "target": None}, - {"__case_ordinal": 2, "source_a": None, "source_b": 0, "target": 7}, - ], "Dolt conditional assignment does not preserve a missing RHS.") - require(conditional_missing_case["expected_error"] is None, "Dolt conditional missing-propagation case unexpectedly fails.") - inequality_case = cases["sqlite-inequality-boundary-semantics"] - require(set(inequality_case) == {"id", "database_profile", "target_preprovisioned", "actor", "boundary", "before_rows", "operations", "after_rows", "expected_error"}, "Inequality-boundary fields differ.") - require(inequality_case["database_profile"] == "sqlite" and inequality_case["target_preprovisioned"] is True and inequality_case["actor"] == "conformance-runner" and inequality_case["boundary"] == 1, "Inequality-boundary identity or actor differs.") - require(inequality_case["before_rows"] == [{"__case_ordinal": 1, "source": 0, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 0, "target_ge": 0}], "Inequality-boundary inputs differ.") - require(inequality_case["operations"] == [ - {"op": "conditional_assign", "condition": "source < 1", "target": "target_lt", "value": 1}, - {"op": "conditional_assign", "condition": "source <= 1", "target": "target_le", "value": 1}, - {"op": "conditional_assign", "condition": "source > 1", "target": "target_gt", "value": 1}, - {"op": "conditional_assign", "condition": "source >= 1", "target": "target_ge", "value": 1}, - ], "Inequality-boundary operations differ.") - require(inequality_case["after_rows"] == [ - {"__case_ordinal": 1, "source": 0, "target_lt": 1, "target_le": 1, "target_gt": 0, "target_ge": 0}, - {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 1, "target_gt": 0, "target_ge": 1}, - {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 1, "target_ge": 1}, - ], "Strict and non-strict inequality results differ.") - require(inequality_case["expected_error"] is None, "Inequality-boundary apply unexpectedly fails.") - context_case = cases["dolt-context-changed-after-mutation-rolls-back"] - require(set(context_case) == {"id", "database_profile", "target_preprovisioned", "actor", "expected_context", "completion_context", "failure_point", "before", "after_failure", "mutation_started", "expected_error"}, "Dolt context-change fields differ.") - require(context_case["database_profile"] == "dolt" and context_case["target_preprovisioned"] is True and context_case["actor"] == "conformance-runner", "Dolt context-change identity or actor differs.") - require(context_case["expected_context"] == {"branch": "feature/recode", "head": "provisioning-commit"}, "Dolt expected context differs.") - require(context_case["completion_context"] == {"branch": "feature/recode", "head": "concurrent-commit"}, "Dolt changed context differs.") - require(context_case["failure_point"] == "after_data_and_metadata_before_audit" and context_case["mutation_started"] is True and context_case["expected_error"] == "dolt_context_changed", "Dolt post-mutation context-change diagnostic differs.") - require(context_case["before"] == {"rows": [{"__case_ordinal": 1, "target": 0}], "target_metadata": {"variable_label": "Before"}, "audit_row_count": 0}, "Dolt context-change pre-state differs.") - require(context_case["after_failure"] == context_case["before"], "Dolt context-change failure does not roll back data, metadata, and audit.") - - actor_case = cases["dolt-empty-actor-fails-before-mutation"] - require(actor_case == { - "id": "dolt-empty-actor-fails-before-mutation", - "database_profile": "dolt", - "actor": "", - "expected_error": "actor_required", - "mutation_started": False, - }, "Dolt empty-actor rejection differs.") - - create_case = cases["sqlite-create-target-atomic-success"] - require(set(create_case) == { - "id", "database_profile", "applied_plan_case", "applied_frontend_case", "target_mode", "actor", "operation", "before", "after", - "transaction_boundary", "rollback_probe", "expected_audit", "expected_error", - }, "SQLite create-target success fields differ.") - require( - create_case["database_profile"] == "sqlite" - and create_case["target_mode"] == "create" - and create_case["actor"] == "conformance-runner" - and create_case["expected_error"] is None, - "SQLite create-target success identity differs.", - ) - require(create_case["applied_plan_case"] in plan_cases, "SQLite applied plan fixture missing.") - require(create_case["applied_frontend_case"] in frontend_cases, "SQLite applied frontend fixture missing.") - create_frontend = frontend_cases[create_case["applied_frontend_case"]] - require(create_frontend.get("expected_error") is None and create_frontend.get("expected_plan_case") == create_case["applied_plan_case"], "SQLite frontend does not link the applied successful plan.") - require(create_frontend["request"]["input_schema"]["variables"] == [ - {"name": "source_a", "storage_kind": "numeric"}, - {"name": "source_b", "storage_kind": "numeric"}, - ], "SQLite create-target frontend schema differs.") - require(create_case["operation"] == { - "op": "assign", - "target": "target", - "value_variable": "source_a", - }, "SQLite create-target operation differs.") - create_before, create_after = create_case["before"], create_case["after"] - for field in ( - "dataset_id", "physical_table_schema", "physical_table_name", - "dataset_count", "persistent_data_table_count", "case_count", - ): - require(create_before[field] == create_after[field], f"SQLite create-target changes {field}.") - require( - create_before["dataset_count"] == create_after["dataset_count"] == 1 - and create_before["persistent_data_table_count"] == create_after["persistent_data_table_count"] == 1, - "SQLite create-target creates a dataset or persistent data table.", - ) - require(create_before["variable_count"] == 2 and create_after["variable_count"] == 3, "SQLite create-target variable count differs.") - require(create_before["target"] == { - "physical_column_present": False, - "catalog_variable_present": False, - }, "SQLite create-target precondition differs.") - require(create_after["target"] == { - "physical_column_present": True, - "catalog_variable_present": True, - "ordinal": 3, - "initial_metadata": { - "variable_label": None, - "value_labels": [], - "missing_values": [], - "format": None, - "measurement_level": None, - }, - }, "SQLite create-target column/catalog result differs.") - require(create_before["rows"] == [ - {"__case_ordinal": 1, "source_a": 2, "source_b": 11}, - {"__case_ordinal": 2, "source_a": None, "source_b": 22}, - ], "SQLite create-target inputs differ.") - require(create_after["rows"] == [ - {"__case_ordinal": 1, "source_a": 2, "source_b": 11, "target": 2}, - {"__case_ordinal": 2, "source_a": None, "source_b": 22, "target": None}, - ], "SQLite create-target values or case order differ.") - require([row["source_b"] for row in create_before["rows"]] == [row["source_b"] for row in create_after["rows"]] == [11, 22], "SQLite create-target rebuild changes source_b.") - require(create_case["transaction_boundary"] == [ - "physical_schema", "row_values", "catalog", "compact_audit", - ], "SQLite create-target transaction boundary differs.") - require(create_case["expected_audit"] == { - "contract_id": manifest["contract"], - "database_profile": "sqlite", - "dataset_id": create_before["dataset_id"], - "physical_table_schema": create_before["physical_table_schema"], - "physical_table_name": create_before["physical_table_name"], - "actor": create_case["actor"], - "status": "succeeded", - "operation_count": 1, - "plan_hash": plan_cases[create_case["applied_plan_case"]]["expected_plan_hash"], - "source_hash": frontend_cases[create_case["applied_frontend_case"]]["expected_source_hash"], - }, "SQLite create-target audit expectation differs.") - require(create_case["rollback_probe"] == { - "failure_point": "after_catalog_create_before_audit", - "physical_column_present": False, - "catalog_variable_present": False, - "row_values_changed": False, - "audit_row_present": False, - }, "SQLite create-target rollback proof differs.") - - for profile in ("dolt", "mysql", "mariadb"): - case = cases[f"{profile}-create-target-fails-before-mutation"] - expected_fields = {"id", "database_profile", "actor", "target_mode", "expected_error", "mutation_started"} - if profile == "dolt": - expected_fields |= {"expected_branch", "expected_head", "working_set_clean"} - require(set(case) == expected_fields, f"{profile}: failure fields differ.") - require(case["database_profile"] == profile and case["actor"] == "conformance-runner" and case["target_mode"] == "create", f"{profile}: failure identity or actor differs.") - if profile == "dolt": - require(case["expected_branch"] == "feature/recode" and case["expected_head"] == "provisioning-commit" and case["working_set_clean"] is True, "dolt: create-target context inputs differ.") - require(case["expected_error"] == "schema_change_not_atomic" and case["mutation_started"] is False, f"{profile}: create-target gate differs.") - - -def validate_links() -> None: - plan_schema = json.loads(PLAN_SCHEMA.read_text(encoding="utf-8")) - legacy_plan_schema = json.loads(PLAN_SCHEMA_0_1.read_text(encoding="utf-8")) - frontend_schema = json.loads(FRONTEND_SCHEMA.read_text(encoding="utf-8")) - Draft202012Validator.check_schema(plan_schema) - Draft202012Validator.check_schema(legacy_plan_schema) - Draft202012Validator.check_schema(frontend_schema) - plan_validator = Draft202012Validator(plan_schema) - legacy_plan_validator = Draft202012Validator(legacy_plan_schema) - frontend_validator = Draft202012Validator(frontend_schema) - plan_manifest = json.loads(PLAN.read_text(encoding="utf-8")) - legacy_plan_manifest = json.loads(PLAN_0_1.read_text(encoding="utf-8")) - legacy_frontend_manifest = json.loads(FRONTEND_0_1.read_text(encoding="utf-8")) - frontend_manifest = json.loads(FRONTEND.read_text(encoding="utf-8")) - for case in plan_manifest["cases"]: - plan_validator.validate(case["plan"]) - for case in legacy_plan_manifest["cases"]: - legacy_plan_validator.validate(case["plan"]) - for case in legacy_frontend_manifest["cases"]: - if "expected_plan" in case: - legacy_plan_validator.validate(case["expected_plan"]) - for case in frontend_manifest["cases"]: - frontend_validator.validate(case["request"]) - if "expected_plan_0_1" in case: - legacy_plan_validator.validate(case["expected_plan_0_1"]) - require(plan_schema["$id"].endswith("transformation-plan-0.2.schema.json"), "0.2 plan schema ID differs.") - require(frontend_schema["$id"].endswith("spss-syntax-frontend-0.2.schema.json"), "0.2 frontend schema ID differs.") - binding = (ROOT / "docs/transformation-plan-sql-binding-0.2.md").read_text(encoding="utf-8") - require(legacy_plan_schema["$id"].endswith("transformation-plan-0.1.schema.json"), "0.1 plan schema ID differs.") - binding_normalized = " ".join(binding.split()) - require("MySQL, MariaDB, and Dolt MUST reject" in binding_normalized, "Non-atomic profile gate missing.") - require("separate, explicit, versioned provisioning action" in binding_normalized, "Provisioning contract missing.") - require("MUST NOT commit, switch, merge, reset" in binding_normalized, "Dolt no-commit contract missing.") - - -def validate_all() -> tuple[int, int]: - plan_cases = validate_plan_manifest() - frontend_count = validate_frontend(plan_cases) - validate_in_place(plan_cases) - validate_links() - return len(plan_cases), frontend_count - - -def main() -> None: - plan_count, frontend_count = validate_all() - print(f"Validated {plan_count} Transformation Plan 0.2 and {frontend_count} SPSS frontend 0.2 cases.") - - -if __name__ == "__main__": - main() From 1de465a4a9197b8641e9d40c9b0e03255e8eaee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 11:02:18 +0300 Subject: [PATCH 71/78] Strengthen transformation artifact integrity checks --- .github/workflows/ci.yml | 1 + conformance/spss-syntax-frontend-0.1.json | 2 + requirements-validation.txt | 1 + tools/validate_repository.py | 52 +++++++++++++++++------ 4 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 requirements-validation.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd1ff1..1e6feee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,4 +12,5 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.13" + - run: python -m pip install --requirement requirements-validation.txt - run: python tools/validate_repository.py diff --git a/conformance/spss-syntax-frontend-0.1.json b/conformance/spss-syntax-frontend-0.1.json index adfe8d0..64a8563 100644 --- a/conformance/spss-syntax-frontend-0.1.json +++ b/conformance/spss-syntax-frontend-0.1.json @@ -20,6 +20,7 @@ "source_text": "RECODE q1 (1,2 = 0) (3 THRU 5 = 1) (ELSE = SYSMIS) INTO q1_binary.\nVARIABLE LABELS q1_binary 'Positive response'.\nVALUE LABELS q1_binary 0 'No' 1 'Yes'." }, "expected_plan_case": "numeric-recode-and-declared-labels", + "expected_plan_hash": "075571b4f84b0f60f289aebb6ec990ad61d34a96b46655802ea8f8bc07a1a405", "expected_source_hash": "f1fecad0279dd633564b36fbdffe6f6960290d220a0ae3ebb6b532615a4b5b45", "expected_error": null }, @@ -64,6 +65,7 @@ "source_text": "VALUE LABELS color 'R' 'Punane' 'B' 'Sinine'." }, "expected_plan_case": "string-value-label-replacement", + "expected_plan_hash": "dae504208847790e782b2b730a2cc258a5c4c89be470a30f17272187c3aeb101", "expected_source_hash": "6bf99a32ed5365a10625f9a88b7d2c3b88d072ce12ecdc8e3459c1b4b5e1eb6a", "expected_error": null }, diff --git a/requirements-validation.txt b/requirements-validation.txt new file mode 100644 index 0000000..d9bf673 --- /dev/null +++ b/requirements-validation.txt @@ -0,0 +1 @@ +jsonschema==4.10.3 diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 68caf3b..8f41a4e 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -10,6 +10,8 @@ from pathlib import Path from uuid import UUID +from jsonschema import Draft202012Validator + ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / "conformance/spss-sav-zsav-1.0.json" @@ -33,6 +35,12 @@ def validate_transformation_integrity() -> None: document = json.loads(path.read_text(encoding="utf-8")) require(isinstance(document, dict), f"{path.name}: manifest must be an object.") require(isinstance(document.get("cases"), list), f"{path.name}: cases must be an array.") + identifiers: set[str] = set() + for case in document["cases"]: + require(isinstance(case, dict), f"{path.name}: case must be an object.") + identifier = require_string(case.get("id"), f"{path.name}: case id") + require(identifier not in identifiers, f"{path.name}: duplicate case id: {identifier}") + identifiers.add(identifier) manifests[name] = document links = [ @@ -44,12 +52,20 @@ def validate_transformation_integrity() -> None: (manifest_paths["frontend_0_1"], manifests["frontend_0_1"]["request_schema"]), (manifest_paths["frontend_0_1"], manifests["frontend_0_1"]["plan_schema"]), ] + schema_validators: dict[Path, Draft202012Validator] = {} for manifest_path, link in links: require(isinstance(link, str) and link and not Path(link).is_absolute(), f"{manifest_path.name}: schema link must be relative.") schema_path = (manifest_path.parent / link).resolve() require(schema_path.is_file(), f"{manifest_path.name}: schema link does not exist: {link}") if schema_path.suffix == ".json": - require(isinstance(json.loads(schema_path.read_text(encoding="utf-8")), dict), f"{schema_path.name}: schema must be an object.") + schema = json.loads(schema_path.read_text(encoding="utf-8")) + require(isinstance(schema, dict), f"{schema_path.name}: schema must be an object.") + Draft202012Validator.check_schema(schema) + schema_validators[schema_path] = Draft202012Validator(schema) + plan_validator = schema_validators[(manifest_paths["plan"].parent / manifests["plan"]["schema"]).resolve()] + legacy_plan_validator = schema_validators[(manifest_paths["plan_0_1"].parent / manifests["plan_0_1"]["schema"]).resolve()] + request_validator = schema_validators[(manifest_paths["frontend"].parent / manifests["frontend"]["request_schema"]).resolve()] + legacy_request_validator = schema_validators[(manifest_paths["frontend_0_1"].parent / manifests["frontend_0_1"]["request_schema"]).resolve()] def canonical_hash(value: object) -> str: encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") @@ -59,9 +75,9 @@ def canonical_hash(value: object) -> str: for case in manifests["plan"]["cases"]: require(isinstance(case, dict), "0.2 plan case must be an object.") identifier = require_string(case.get("id"), "0.2 plan case id") - require(identifier not in plan_hashes, f"Duplicate successful 0.2 plan case: {identifier}") if case.get("expected_error") is None: require(isinstance(case.get("plan"), dict), f"{identifier}: successful plan is missing.") + require(plan_validator.is_valid(case["plan"]), f"{identifier}: successful plan violates its declared schema.") digest = canonical_hash(case["plan"]) require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical plan hash differs.") plan_hashes[identifier] = digest @@ -71,20 +87,25 @@ def canonical_hash(value: object) -> str: if case.get("expected_error") is None: identifier = require_string(case.get("id"), "0.1 plan case id") digest = canonical_hash(case.get("plan")) + require(legacy_plan_validator.is_valid(case.get("plan")), f"{identifier}: successful 0.1 plan violates its declared schema.") require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical 0.1 plan hash differs.") legacy_plan_hashes[identifier] = digest for case in manifests["frontend_0_1"]["cases"]: + identifier = require_string(case.get("id"), "0.1 frontend case id") + request = case.get("request") + require(legacy_request_validator.is_valid(request), f"{identifier}: 0.1 request violates its declared schema.") + source = request.get("source_text") + if isinstance(source, str): + require(case.get("expected_source_hash") == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: 0.1 source hash differs.") if case.get("expected_error") is not None: continue - identifier = require_string(case.get("id"), "0.1 frontend case id") - source = case.get("request", {}).get("source_text") - require(isinstance(source, str) and case.get("expected_source_hash") == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: 0.1 source hash differs.") if "expected_plan_case" in case: reference = case["expected_plan_case"] - require(reference in legacy_plan_hashes, f"{identifier}: referenced 0.1 plan is missing.") + require(reference in legacy_plan_hashes and case.get("expected_plan_hash") == legacy_plan_hashes[reference], f"{identifier}: referenced 0.1 plan hash differs.") else: require("expected_plan" in case, f"{identifier}: embedded 0.1 plan is missing.") + require(legacy_plan_validator.is_valid(case["expected_plan"]), f"{identifier}: embedded 0.1 plan violates its declared schema.") digest = canonical_hash(case["expected_plan"]) require(case.get("expected_plan_hash") == digest, f"{identifier}: embedded 0.1 plan hash differs.") legacy_hashes = { @@ -95,13 +116,14 @@ def canonical_hash(value: object) -> str: frontend_hashes: dict[str, tuple[str, str]] = {} for case in manifests["frontend"]["cases"]: require(isinstance(case, dict), "0.2 frontend case must be an object.") + identifier = require_string(case.get("id"), "0.2 frontend case id") + request = case.get("request") + require(request_validator.is_valid(request), f"{identifier}: request violates its declared schema.") + source = request.get("source_text") + if isinstance(source, str): + require(case.get("expected_source_hash") == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: source hash differs.") if case.get("expected_error") is not None: continue - identifier = require_string(case.get("id"), "0.2 frontend case id") - source = case.get("request", {}).get("source_text") - require(isinstance(source, str), f"{identifier}: successful frontend source is missing.") - source_hash = hashlib.sha256(source.encode("utf-8")).hexdigest() - require(case.get("expected_source_hash") == source_hash, f"{identifier}: source hash differs.") if "expected_plan_case" in case: reference = case["expected_plan_case"] require(reference in plan_hashes and case.get("expected_plan_hash") == plan_hashes[reference], f"{identifier}: referenced 0.2 plan hash differs.") @@ -109,8 +131,10 @@ def canonical_hash(value: object) -> str: reference = case["expected_plan_case_0_1"] require(reference in legacy_hashes and case.get("expected_plan_hash") == legacy_hashes[reference], f"{identifier}: referenced 0.1 plan hash differs.") else: - require("expected_plan_0_1" in case and case.get("expected_plan_hash") == canonical_hash(case["expected_plan_0_1"]), f"{identifier}: embedded plan hash differs.") - frontend_hashes[identifier] = (case["expected_plan_hash"], source_hash) + require("expected_plan_0_1" in case, f"{identifier}: embedded plan is missing.") + require(legacy_plan_validator.is_valid(case["expected_plan_0_1"]), f"{identifier}: embedded plan violates its declared schema.") + require(case.get("expected_plan_hash") == canonical_hash(case["expected_plan_0_1"]), f"{identifier}: embedded plan hash differs.") + frontend_hashes[identifier] = (case["expected_plan_hash"], case["expected_source_hash"]) for case in manifests["binding"]["cases"]: if not isinstance(case, dict) or "applied_plan_case" not in case: @@ -118,7 +142,7 @@ def canonical_hash(value: object) -> str: plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") require(plan_id in plan_hashes and frontend_id in frontend_hashes, f"{case.get('id')}: applied fixture reference is missing.") audit = case.get("expected_audit") - require(isinstance(audit, dict) and audit.get("plan_hash") == plan_hashes[plan_id] and audit.get("source_hash") == frontend_hashes[frontend_id][1], f"{case.get('id')}: applied fixture audit hashes differ.") + require(frontend_hashes[frontend_id][0] == plan_hashes[plan_id] and isinstance(audit, dict) and audit.get("plan_hash") == plan_hashes[plan_id] and audit.get("source_hash") == frontend_hashes[frontend_id][1], f"{case.get('id')}: applied fixture hashes differ.") def require_well_formed_create_table_blocks( From 0981434b843fcb296d64a514c5fdb9b5be4a428d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 11:14:54 +0300 Subject: [PATCH 72/78] Link legacy binding conformance fixtures --- conformance/in-place-transformation-0.1.json | 8 +++++++ conformance/spss-syntax-frontend-0.1.json | 1 + conformance/transformation-plan-0.1.json | 1 + tools/validate_repository.py | 23 +++++++++++++++----- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/conformance/in-place-transformation-0.1.json b/conformance/in-place-transformation-0.1.json index 5eaa6f3..fbd3bb9 100644 --- a/conformance/in-place-transformation-0.1.json +++ b/conformance/in-place-transformation-0.1.json @@ -7,6 +7,10 @@ "id": "mysql-recode-and-labels-preserve-dataset-and-table-identity", "database_profile": "mysql", "source_text": "RECODE score (1,2 = 0) (3 = 1).\nVARIABLE LABELS score 'Score band'.\nVALUE LABELS score 0 'Lower' 1 'Upper'.", + "applied_plan_case": "binding-recode-and-labels-existing-target", + "applied_frontend_case": "binding-recode-and-labels-existing-target", + "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", + "expected_source_hash": "d9408f01082c1e160ad69ed710286ee56da757846a2132ca95b38764293267c6", "before": { "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": "survey_data", @@ -59,6 +63,10 @@ "id": "dolt-recode-and-labels-preserve-controlled-context", "database_profile": "dolt", "source_text": "RECODE score (1,2 = 0) (3 = 1).\nVARIABLE LABELS score 'Score band'.\nVALUE LABELS score 0 'Lower' 1 'Upper'.", + "applied_plan_case": "binding-recode-and-labels-existing-target", + "applied_frontend_case": "binding-recode-and-labels-existing-target", + "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", + "expected_source_hash": "d9408f01082c1e160ad69ed710286ee56da757846a2132ca95b38764293267c6", "before": { "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, diff --git a/conformance/spss-syntax-frontend-0.1.json b/conformance/spss-syntax-frontend-0.1.json index 64a8563..621b1c0 100644 --- a/conformance/spss-syntax-frontend-0.1.json +++ b/conformance/spss-syntax-frontend-0.1.json @@ -69,6 +69,7 @@ "expected_source_hash": "6bf99a32ed5365a10625f9a88b7d2c3b88d072ce12ecdc8e3459c1b4b5e1eb6a", "expected_error": null }, + {"id": "binding-recode-and-labels-existing-target", "request": {"contract": "openstatspec-spss-syntax-frontend-v0.1", "input_alias": "parent", "input_schema": {"variables": [{"name": "score", "storage_kind": "numeric"}]}, "source_text": "RECODE score (1,2 = 0) (3 = 1).\nVARIABLE LABELS score 'Score band'.\nVALUE LABELS score 0 'Lower' 1 'Upper'."}, "expected_plan_case": "binding-recode-and-labels-existing-target", "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", "expected_source_hash": "d9408f01082c1e160ad69ed710286ee56da757846a2132ca95b38764293267c6", "expected_error": null}, { "id": "recode-varlists-positionally", "request": { diff --git a/conformance/transformation-plan-0.1.json b/conformance/transformation-plan-0.1.json index 297e722..1f07be8 100644 --- a/conformance/transformation-plan-0.1.json +++ b/conformance/transformation-plan-0.1.json @@ -86,6 +86,7 @@ "expected_plan_hash": "dae504208847790e782b2b730a2cc258a5c4c89be470a30f17272187c3aeb101", "expected_error": null }, + {"id": "binding-recode-and-labels-existing-target", "plan": {"contract": "openstatspec-transformation-plan-v0.1", "input_alias": "parent", "operations": [{"op": "recode", "source": "score", "target": "score", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}, {"type": "binary64", "bits": "4000000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}, {"match": {"kind": "values", "values": [{"type": "binary64", "bits": "4008000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}], "unmatched": {"kind": "copy"}}, {"op": "set_variable_label", "variable": "score", "label": "Score band"}, {"op": "replace_value_labels", "variable": "score", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "Lower"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Upper"}]}]}, "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", "expected_error": null}, { "id": "reject-descending-range", "plan": { diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 8f41a4e..b68688b 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -75,19 +75,19 @@ def canonical_hash(value: object) -> str: for case in manifests["plan"]["cases"]: require(isinstance(case, dict), "0.2 plan case must be an object.") identifier = require_string(case.get("id"), "0.2 plan case id") + require(plan_validator.is_valid(case.get("plan")), f"{identifier}: plan violates its declared schema.") if case.get("expected_error") is None: require(isinstance(case.get("plan"), dict), f"{identifier}: successful plan is missing.") - require(plan_validator.is_valid(case["plan"]), f"{identifier}: successful plan violates its declared schema.") digest = canonical_hash(case["plan"]) require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical plan hash differs.") plan_hashes[identifier] = digest legacy_plan_hashes: dict[str, str] = {} for case in manifests["plan_0_1"]["cases"]: + identifier = require_string(case.get("id"), "0.1 plan case id") + require(legacy_plan_validator.is_valid(case.get("plan")), f"{identifier}: 0.1 plan violates its declared schema.") if case.get("expected_error") is None: - identifier = require_string(case.get("id"), "0.1 plan case id") digest = canonical_hash(case.get("plan")) - require(legacy_plan_validator.is_valid(case.get("plan")), f"{identifier}: successful 0.1 plan violates its declared schema.") require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical 0.1 plan hash differs.") legacy_plan_hashes[identifier] = digest @@ -108,11 +108,22 @@ def canonical_hash(value: object) -> str: require(legacy_plan_validator.is_valid(case["expected_plan"]), f"{identifier}: embedded 0.1 plan violates its declared schema.") digest = canonical_hash(case["expected_plan"]) require(case.get("expected_plan_hash") == digest, f"{identifier}: embedded 0.1 plan hash differs.") - legacy_hashes = { - case["id"]: case["expected_plan_hash"] + legacy_frontend_hashes = { + case["id"]: (case["expected_plan_hash"], case["expected_source_hash"]) for case in manifests["frontend_0_1"]["cases"] if case.get("expected_error") is None and "expected_plan_hash" in case } + for case in manifests["binding_0_1"]["cases"]: + if "applied_plan_case" not in case: + continue + plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") + require(plan_id in legacy_plan_hashes and frontend_id in legacy_frontend_hashes, f"{case.get('id')}: applied 0.1 fixture reference is missing.") + frontend_plan_hash, source_hash = legacy_frontend_hashes[frontend_id] + require(case.get("expected_plan_hash") == legacy_plan_hashes[plan_id] == frontend_plan_hash, f"{case.get('id')}: applied 0.1 plan hashes differ.") + require(case.get("expected_source_hash") == source_hash, f"{case.get('id')}: applied 0.1 source hashes differ.") + source = case.get("source_text") + if isinstance(source, str): + require(hashlib.sha256(source.encode("utf-8")).hexdigest() == source_hash, f"{case.get('id')}: binding source hash differs.") frontend_hashes: dict[str, tuple[str, str]] = {} for case in manifests["frontend"]["cases"]: require(isinstance(case, dict), "0.2 frontend case must be an object.") @@ -129,7 +140,7 @@ def canonical_hash(value: object) -> str: require(reference in plan_hashes and case.get("expected_plan_hash") == plan_hashes[reference], f"{identifier}: referenced 0.2 plan hash differs.") elif "expected_plan_case_0_1" in case: reference = case["expected_plan_case_0_1"] - require(reference in legacy_hashes and case.get("expected_plan_hash") == legacy_hashes[reference], f"{identifier}: referenced 0.1 plan hash differs.") + require(reference in legacy_frontend_hashes and case.get("expected_plan_hash") == legacy_frontend_hashes[reference][0], f"{identifier}: referenced 0.1 plan hash differs.") else: require("expected_plan_0_1" in case, f"{identifier}: embedded plan is missing.") require(legacy_plan_validator.is_valid(case["expected_plan_0_1"]), f"{identifier}: embedded plan violates its declared schema.") From 8d47ce2c152bcc19b4281280c4ca517d26661f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 11:17:51 +0300 Subject: [PATCH 73/78] Tighten generic transformation integrity invariants --- tools/validate_repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/validate_repository.py b/tools/validate_repository.py index b68688b..2dca6ab 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -81,6 +81,8 @@ def canonical_hash(value: object) -> str: digest = canonical_hash(case["plan"]) require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical plan hash differs.") plan_hashes[identifier] = digest + else: + require(case.get("expected_plan_hash") is None, f"{identifier}: rejected plan hash must be null.") legacy_plan_hashes: dict[str, str] = {} for case in manifests["plan_0_1"]["cases"]: @@ -90,6 +92,8 @@ def canonical_hash(value: object) -> str: digest = canonical_hash(case.get("plan")) require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical 0.1 plan hash differs.") legacy_plan_hashes[identifier] = digest + else: + require(case.get("expected_plan_hash") is None, f"{identifier}: rejected 0.1 plan hash must be null.") for case in manifests["frontend_0_1"]["cases"]: identifier = require_string(case.get("id"), "0.1 frontend case id") @@ -122,8 +126,8 @@ def canonical_hash(value: object) -> str: require(case.get("expected_plan_hash") == legacy_plan_hashes[plan_id] == frontend_plan_hash, f"{case.get('id')}: applied 0.1 plan hashes differ.") require(case.get("expected_source_hash") == source_hash, f"{case.get('id')}: applied 0.1 source hashes differ.") source = case.get("source_text") - if isinstance(source, str): - require(hashlib.sha256(source.encode("utf-8")).hexdigest() == source_hash, f"{case.get('id')}: binding source hash differs.") + require(isinstance(source, str), f"{case.get('id')}: linked binding source must be a string.") + require(hashlib.sha256(source.encode("utf-8")).hexdigest() == source_hash, f"{case.get('id')}: binding source hash differs.") frontend_hashes: dict[str, tuple[str, str]] = {} for case in manifests["frontend"]["cases"]: require(isinstance(case, dict), "0.2 frontend case must be an object.") From f09cac0882362caeafee1e46042138ea9370848c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 11:31:52 +0300 Subject: [PATCH 74/78] Validate transformation manifest declarations --- tools/validate_repository.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 2dca6ab..bf28b72 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -30,11 +30,21 @@ def validate_transformation_integrity() -> None: "frontend_0_1": ROOT / "conformance/spss-syntax-frontend-0.1.json", "binding_0_1": ROOT / "conformance/in-place-transformation-0.1.json", } + expected_metadata = { + "plan": {"manifest_version": "0.2", "profile": "OpenStatSpec Transformation Plan 0.2", "contract": "openstatspec-transformation-plan-v0.2", "canonicalization": "restricted-rfc8785-utf8-sha256"}, + "frontend": {"manifest_version": "0.2", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.2", "contract": "openstatspec-spss-syntax-frontend-v0.2"}, + "binding": {"manifest_version": "0.2", "profile": "OpenStatSpec In-Place Transformation Binding 0.2", "contract": "openstatspec-in-place-transformation-v0.2"}, + "plan_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec Transformation Plan 0.1", "contract": "openstatspec-transformation-plan-v0.1"}, + "frontend_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.1", "contract": "openstatspec-spss-syntax-frontend-v0.1"}, + "binding_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec In-Place Transformation Binding 0.1", "contract": "openstatspec-in-place-transformation-v0.1"}, + } manifests: dict[str, dict[str, object]] = {} for name, path in manifest_paths.items(): document = json.loads(path.read_text(encoding="utf-8")) require(isinstance(document, dict), f"{path.name}: manifest must be an object.") require(isinstance(document.get("cases"), list), f"{path.name}: cases must be an array.") + for field, expected in expected_metadata[name].items(): + require(document.get(field) == expected, f"{path.name}: {field} declaration differs.") identifiers: set[str] = set() for case in document["cases"]: require(isinstance(case, dict), f"{path.name}: case must be an object.") @@ -71,6 +81,14 @@ def canonical_hash(value: object) -> str: encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") return hashlib.sha256(encoded).hexdigest() + def normalized_source_hash(source: str) -> str: + normalized = source.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + frontend_plan_fields = { + "expected_plan_case", "expected_plan_case_0_1", "expected_plan", + "expected_plan_0_1", "expected_plan_contract", + } plan_hashes: dict[str, str] = {} for case in manifests["plan"]["cases"]: require(isinstance(case, dict), "0.2 plan case must be an object.") @@ -101,8 +119,10 @@ def canonical_hash(value: object) -> str: require(legacy_request_validator.is_valid(request), f"{identifier}: 0.1 request violates its declared schema.") source = request.get("source_text") if isinstance(source, str): - require(case.get("expected_source_hash") == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: 0.1 source hash differs.") + require(case.get("expected_source_hash") == normalized_source_hash(source), f"{identifier}: 0.1 source hash differs.") if case.get("expected_error") is not None: + require(frontend_plan_fields.isdisjoint(case), f"{identifier}: failed frontend case contains plan output fields.") + require(case.get("expected_plan_hash") is None, f"{identifier}: failed frontend plan hash must be absent or null.") continue if "expected_plan_case" in case: reference = case["expected_plan_case"] @@ -127,7 +147,7 @@ def canonical_hash(value: object) -> str: require(case.get("expected_source_hash") == source_hash, f"{case.get('id')}: applied 0.1 source hashes differ.") source = case.get("source_text") require(isinstance(source, str), f"{case.get('id')}: linked binding source must be a string.") - require(hashlib.sha256(source.encode("utf-8")).hexdigest() == source_hash, f"{case.get('id')}: binding source hash differs.") + require(normalized_source_hash(source) == source_hash, f"{case.get('id')}: binding source hash differs.") frontend_hashes: dict[str, tuple[str, str]] = {} for case in manifests["frontend"]["cases"]: require(isinstance(case, dict), "0.2 frontend case must be an object.") @@ -136,8 +156,10 @@ def canonical_hash(value: object) -> str: require(request_validator.is_valid(request), f"{identifier}: request violates its declared schema.") source = request.get("source_text") if isinstance(source, str): - require(case.get("expected_source_hash") == hashlib.sha256(source.encode("utf-8")).hexdigest(), f"{identifier}: source hash differs.") + require(case.get("expected_source_hash") == normalized_source_hash(source), f"{identifier}: source hash differs.") if case.get("expected_error") is not None: + require(frontend_plan_fields.isdisjoint(case), f"{identifier}: failed frontend case contains plan output fields.") + require(case.get("expected_plan_hash") is None, f"{identifier}: failed frontend plan hash must be absent or null.") continue if "expected_plan_case" in case: reference = case["expected_plan_case"] From 2a5713ab15d72cf18521b965b1698d48d36cc3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 11:44:18 +0300 Subject: [PATCH 75/78] Preserve published 0.1 conformance manifests --- conformance/in-place-transformation-0.1.json | 8 -------- conformance/spss-syntax-frontend-0.1.json | 3 --- conformance/transformation-plan-0.1.json | 1 - tools/validate_repository.py | 13 +------------ 4 files changed, 1 insertion(+), 24 deletions(-) diff --git a/conformance/in-place-transformation-0.1.json b/conformance/in-place-transformation-0.1.json index fbd3bb9..5eaa6f3 100644 --- a/conformance/in-place-transformation-0.1.json +++ b/conformance/in-place-transformation-0.1.json @@ -7,10 +7,6 @@ "id": "mysql-recode-and-labels-preserve-dataset-and-table-identity", "database_profile": "mysql", "source_text": "RECODE score (1,2 = 0) (3 = 1).\nVARIABLE LABELS score 'Score band'.\nVALUE LABELS score 0 'Lower' 1 'Upper'.", - "applied_plan_case": "binding-recode-and-labels-existing-target", - "applied_frontend_case": "binding-recode-and-labels-existing-target", - "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", - "expected_source_hash": "d9408f01082c1e160ad69ed710286ee56da757846a2132ca95b38764293267c6", "before": { "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": "survey_data", @@ -63,10 +59,6 @@ "id": "dolt-recode-and-labels-preserve-controlled-context", "database_profile": "dolt", "source_text": "RECODE score (1,2 = 0) (3 = 1).\nVARIABLE LABELS score 'Score band'.\nVALUE LABELS score 0 'Lower' 1 'Upper'.", - "applied_plan_case": "binding-recode-and-labels-existing-target", - "applied_frontend_case": "binding-recode-and-labels-existing-target", - "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", - "expected_source_hash": "d9408f01082c1e160ad69ed710286ee56da757846a2132ca95b38764293267c6", "before": { "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, diff --git a/conformance/spss-syntax-frontend-0.1.json b/conformance/spss-syntax-frontend-0.1.json index 621b1c0..adfe8d0 100644 --- a/conformance/spss-syntax-frontend-0.1.json +++ b/conformance/spss-syntax-frontend-0.1.json @@ -20,7 +20,6 @@ "source_text": "RECODE q1 (1,2 = 0) (3 THRU 5 = 1) (ELSE = SYSMIS) INTO q1_binary.\nVARIABLE LABELS q1_binary 'Positive response'.\nVALUE LABELS q1_binary 0 'No' 1 'Yes'." }, "expected_plan_case": "numeric-recode-and-declared-labels", - "expected_plan_hash": "075571b4f84b0f60f289aebb6ec990ad61d34a96b46655802ea8f8bc07a1a405", "expected_source_hash": "f1fecad0279dd633564b36fbdffe6f6960290d220a0ae3ebb6b532615a4b5b45", "expected_error": null }, @@ -65,11 +64,9 @@ "source_text": "VALUE LABELS color 'R' 'Punane' 'B' 'Sinine'." }, "expected_plan_case": "string-value-label-replacement", - "expected_plan_hash": "dae504208847790e782b2b730a2cc258a5c4c89be470a30f17272187c3aeb101", "expected_source_hash": "6bf99a32ed5365a10625f9a88b7d2c3b88d072ce12ecdc8e3459c1b4b5e1eb6a", "expected_error": null }, - {"id": "binding-recode-and-labels-existing-target", "request": {"contract": "openstatspec-spss-syntax-frontend-v0.1", "input_alias": "parent", "input_schema": {"variables": [{"name": "score", "storage_kind": "numeric"}]}, "source_text": "RECODE score (1,2 = 0) (3 = 1).\nVARIABLE LABELS score 'Score band'.\nVALUE LABELS score 0 'Lower' 1 'Upper'."}, "expected_plan_case": "binding-recode-and-labels-existing-target", "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", "expected_source_hash": "d9408f01082c1e160ad69ed710286ee56da757846a2132ca95b38764293267c6", "expected_error": null}, { "id": "recode-varlists-positionally", "request": { diff --git a/conformance/transformation-plan-0.1.json b/conformance/transformation-plan-0.1.json index 1f07be8..297e722 100644 --- a/conformance/transformation-plan-0.1.json +++ b/conformance/transformation-plan-0.1.json @@ -86,7 +86,6 @@ "expected_plan_hash": "dae504208847790e782b2b730a2cc258a5c4c89be470a30f17272187c3aeb101", "expected_error": null }, - {"id": "binding-recode-and-labels-existing-target", "plan": {"contract": "openstatspec-transformation-plan-v0.1", "input_alias": "parent", "operations": [{"op": "recode", "source": "score", "target": "score", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}, {"type": "binary64", "bits": "4000000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}, {"match": {"kind": "values", "values": [{"type": "binary64", "bits": "4008000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}], "unmatched": {"kind": "copy"}}, {"op": "set_variable_label", "variable": "score", "label": "Score band"}, {"op": "replace_value_labels", "variable": "score", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "Lower"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Upper"}]}]}, "expected_plan_hash": "9a34e76e97040da4c36b14d57516000d1a803de77140facc56836c344c9dcbd2", "expected_error": null}, { "id": "reject-descending-range", "plan": { diff --git a/tools/validate_repository.py b/tools/validate_repository.py index bf28b72..6a6fac3 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -126,7 +126,7 @@ def normalized_source_hash(source: str) -> str: continue if "expected_plan_case" in case: reference = case["expected_plan_case"] - require(reference in legacy_plan_hashes and case.get("expected_plan_hash") == legacy_plan_hashes[reference], f"{identifier}: referenced 0.1 plan hash differs.") + require(reference in legacy_plan_hashes, f"{identifier}: referenced 0.1 plan is missing.") else: require("expected_plan" in case, f"{identifier}: embedded 0.1 plan is missing.") require(legacy_plan_validator.is_valid(case["expected_plan"]), f"{identifier}: embedded 0.1 plan violates its declared schema.") @@ -137,17 +137,6 @@ def normalized_source_hash(source: str) -> str: for case in manifests["frontend_0_1"]["cases"] if case.get("expected_error") is None and "expected_plan_hash" in case } - for case in manifests["binding_0_1"]["cases"]: - if "applied_plan_case" not in case: - continue - plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") - require(plan_id in legacy_plan_hashes and frontend_id in legacy_frontend_hashes, f"{case.get('id')}: applied 0.1 fixture reference is missing.") - frontend_plan_hash, source_hash = legacy_frontend_hashes[frontend_id] - require(case.get("expected_plan_hash") == legacy_plan_hashes[plan_id] == frontend_plan_hash, f"{case.get('id')}: applied 0.1 plan hashes differ.") - require(case.get("expected_source_hash") == source_hash, f"{case.get('id')}: applied 0.1 source hashes differ.") - source = case.get("source_text") - require(isinstance(source, str), f"{case.get('id')}: linked binding source must be a string.") - require(normalized_source_hash(source) == source_hash, f"{case.get('id')}: binding source hash differs.") frontend_hashes: dict[str, tuple[str, str]] = {} for case in manifests["frontend"]["cases"]: require(isinstance(case, dict), "0.2 frontend case must be an object.") From 26a001b743410e4bf2cbf471c1fddf3e0510a9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 11:59:25 +0300 Subject: [PATCH 76/78] Complete transformation audit integrity fixtures --- conformance/in-place-transformation-0.2.json | 13 +++++++------ tools/validate_repository.py | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/conformance/in-place-transformation-0.2.json b/conformance/in-place-transformation-0.2.json index 17e00e1..370c5e4 100644 --- a/conformance/in-place-transformation-0.2.json +++ b/conformance/in-place-transformation-0.2.json @@ -135,7 +135,7 @@ "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", - "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", + "plan_hash": "8e9f72759f6ed53aa6957dc288bdc49894ca2e08a8313ca957da6b4dfd11acbd", "canonical_plan_json": "{\"contract\":\"openstatspec-transformation-plan-v0.2\",\"input_alias\":\"parent\",\"operations\":[{\"op\":\"assign\",\"target\":\"target\",\"target_mode\":\"replace\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"0000000000000000\",\"type\":\"binary64\"}}},{\"condition\":{\"expression\":\"boolean\",\"operands\":[{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source_a\"},\"operator\":\"=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source_b\"},\"operator\":\"=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}}],\"operator\":\"and\"},\"op\":\"conditional_assign\",\"target\":\"target\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},{\"label\":\"Synthetic conjunction\",\"op\":\"set_variable_label\",\"variable\":\"target\"},{\"labels\":[{\"label\":\"No\",\"value\":{\"bits\":\"0000000000000000\",\"type\":\"binary64\"}},{\"label\":\"Yes\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}],\"op\":\"replace_value_labels\",\"variable\":\"target\"},{\"decimals\":0,\"family\":\"F\",\"op\":\"set_format\",\"variable\":\"target\",\"width\":1},{\"level\":\"nominal\",\"op\":\"set_measurement_level\",\"variable\":\"target\"},{\"op\":\"execute\"}]}", "source_hash": "02bede796f1f0c8dcc22eb616d4afa48540ea8f09f637179842368af8a576481", "operation_count": 7, "dolt_branch": "feature/recode", @@ -163,6 +163,7 @@ "physical_table_schema", "physical_table_name", "plan_hash", + "canonical_plan_json", "source_hash", "actor", "status", @@ -218,7 +219,7 @@ } ], "expected_error": null, - "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "7fa8a017ed2476606c20f7b6bc9df09b7bbff208891e03335dfa393c7c260742", "source_hash": "dc936264d68bdf22741a8476d650a4e58fc03a63ac47f86093cbbb732f31594a", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "7fa8a017ed2476606c20f7b6bc9df09b7bbff208891e03335dfa393c7c260742", "canonical_plan_json": "{\"contract\":\"openstatspec-transformation-plan-v0.2\",\"input_alias\":\"parent\",\"operations\":[{\"condition\":{\"expression\":\"boolean\",\"operands\":[{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source_a\"},\"operator\":\"=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source_b\"},\"operator\":\"=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}}],\"operator\":\"or\"},\"op\":\"conditional_assign\",\"target\":\"target\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}}]}", "source_hash": "dc936264d68bdf22741a8476d650a4e58fc03a63ac47f86093cbbb732f31594a", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} }, { "id": "dolt-create-target-fails-before-mutation", @@ -289,7 +290,7 @@ } ], "expected_error": null, - "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "2b52cbd2989c99aac69dbd10e0f3de225ca84f3902193e05931bc3bf9c3e8649", "source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "2b52cbd2989c99aac69dbd10e0f3de225ca84f3902193e05931bc3bf9c3e8649", "canonical_plan_json": "{\"contract\":\"openstatspec-transformation-plan-v0.2\",\"input_alias\":\"parent\",\"operations\":[{\"op\":\"assign\",\"target\":\"target\",\"target_mode\":\"replace\",\"value\":{\"kind\":\"variable\",\"variable\":\"source_a\"}}]}", "source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} }, { "id": "dolt-preprovisioned-target-conditional-variable-missing-propagation", @@ -338,7 +339,7 @@ } ], "expected_error": null, - "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "aa42695ecf8d84d487fe1cdc1b11dd9d9e5d048df670877ed562e9b9fbd23053", "source_hash": "9fb6357cd8b436b69c523d86be0ce1415dc0fded15a425f737e6f3e37bd08b76", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "dolt", "dataset_id": "11111111-1111-4111-8111-111111111111", "physical_table_schema": null, "physical_table_name": "data_synthetic", "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, "plan_hash": "aa42695ecf8d84d487fe1cdc1b11dd9d9e5d048df670877ed562e9b9fbd23053", "canonical_plan_json": "{\"contract\":\"openstatspec-transformation-plan-v0.2\",\"input_alias\":\"parent\",\"operations\":[{\"condition\":{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source_b\"},\"operator\":\"=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},\"op\":\"conditional_assign\",\"target\":\"target\",\"value\":{\"kind\":\"variable\",\"variable\":\"source_a\"}}]}", "source_hash": "9fb6357cd8b436b69c523d86be0ce1415dc0fded15a425f737e6f3e37bd08b76", "dolt_branch": "feature/recode", "dolt_head_before": "provisioning-commit", "dolt_head_after": "provisioning-commit"} }, { "id": "sqlite-inequality-boundary-semantics", @@ -355,7 +356,7 @@ "operations": [{"op": "conditional_assign", "condition": "source < 1", "target": "target_lt", "value": 1}, {"op": "conditional_assign", "condition": "source <= 1", "target": "target_le", "value": 1}, {"op": "conditional_assign", "condition": "source > 1", "target": "target_gt", "value": 1}, {"op": "conditional_assign", "condition": "source >= 1", "target": "target_ge", "value": 1}], "after_rows": [{"__case_ordinal": 1, "source": 0, "target_lt": 1, "target_le": 1, "target_gt": 0, "target_ge": 0}, {"__case_ordinal": 2, "source": 1, "target_lt": 0, "target_le": 1, "target_gt": 0, "target_ge": 1}, {"__case_ordinal": 3, "source": 2, "target_lt": 0, "target_le": 0, "target_gt": 1, "target_ge": 1}], "expected_error": null, - "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "sqlite", "dataset_id": "44444444-4444-4444-8444-444444444444", "physical_table_schema": null, "physical_table_name": "data_inequality", "actor": "conformance-runner", "status": "succeeded", "operation_count": 4, "plan_hash": "cc0bde9b2adbbe76614c51a6122a5c9c17c9e610a63241eb1769956c2e543690", "source_hash": "1208507803e9b69cb523585e83dd60643e61d3082f9b1d76fe4bca33b1db0011"} + "expected_audit": {"contract_id": "openstatspec-in-place-transformation-v0.2", "database_profile": "sqlite", "dataset_id": "44444444-4444-4444-8444-444444444444", "physical_table_schema": null, "physical_table_name": "data_inequality", "actor": "conformance-runner", "status": "succeeded", "operation_count": 4, "plan_hash": "cc0bde9b2adbbe76614c51a6122a5c9c17c9e610a63241eb1769956c2e543690", "canonical_plan_json": "{\"contract\":\"openstatspec-transformation-plan-v0.2\",\"input_alias\":\"parent\",\"operations\":[{\"condition\":{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source\"},\"operator\":\"<\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},\"op\":\"conditional_assign\",\"target\":\"target_lt\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},{\"condition\":{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source\"},\"operator\":\"<=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},\"op\":\"conditional_assign\",\"target\":\"target_le\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},{\"condition\":{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source\"},\"operator\":\">\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},\"op\":\"conditional_assign\",\"target\":\"target_gt\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},{\"condition\":{\"expression\":\"comparison\",\"left\":{\"kind\":\"variable\",\"variable\":\"source\"},\"operator\":\">=\",\"right\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}},\"op\":\"conditional_assign\",\"target\":\"target_ge\",\"value\":{\"kind\":\"literal\",\"value\":{\"bits\":\"3ff0000000000000\",\"type\":\"binary64\"}}}]}", "source_hash": "1208507803e9b69cb523585e83dd60643e61d3082f9b1d76fe4bca33b1db0011"} }, { "id": "dolt-context-changed-after-mutation-rolls-back", @@ -472,7 +473,7 @@ "actor": "conformance-runner", "status": "succeeded", "operation_count": 1, - "plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", + "plan_hash": "25b815bb706bd8692667d251f819c731176c5728205c98b51aa2ae509f39a2c2", "canonical_plan_json": "{\"contract\":\"openstatspec-transformation-plan-v0.2\",\"input_alias\":\"parent\",\"operations\":[{\"op\":\"assign\",\"target\":\"target\",\"target_mode\":\"create\",\"value\":{\"kind\":\"variable\",\"variable\":\"source_a\"}}]}", "source_hash": "febe1c066647e5e50aa6c115a7ef33ade99592c4929a2d8a7411a7dec0ff4c15" } } diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 6a6fac3..1e26b68 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -32,7 +32,7 @@ def validate_transformation_integrity() -> None: } expected_metadata = { "plan": {"manifest_version": "0.2", "profile": "OpenStatSpec Transformation Plan 0.2", "contract": "openstatspec-transformation-plan-v0.2", "canonicalization": "restricted-rfc8785-utf8-sha256"}, - "frontend": {"manifest_version": "0.2", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.2", "contract": "openstatspec-spss-syntax-frontend-v0.2"}, + "frontend": {"manifest_version": "0.2", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.2", "contract": "openstatspec-spss-syntax-frontend-v0.2", "plan_contracts": ["openstatspec-transformation-plan-v0.1", "openstatspec-transformation-plan-v0.2"], "plan_schemas": {"openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json"}}, "binding": {"manifest_version": "0.2", "profile": "OpenStatSpec In-Place Transformation Binding 0.2", "contract": "openstatspec-in-place-transformation-v0.2"}, "plan_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec Transformation Plan 0.1", "contract": "openstatspec-transformation-plan-v0.1"}, "frontend_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.1", "contract": "openstatspec-spss-syntax-frontend-v0.1"}, @@ -77,8 +77,11 @@ def validate_transformation_integrity() -> None: request_validator = schema_validators[(manifest_paths["frontend"].parent / manifests["frontend"]["request_schema"]).resolve()] legacy_request_validator = schema_validators[(manifest_paths["frontend_0_1"].parent / manifests["frontend_0_1"]["request_schema"]).resolve()] + def canonical_json(value: object) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) + def canonical_hash(value: object) -> str: - encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + encoded = canonical_json(value).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def normalized_source_hash(source: str) -> str: @@ -90,6 +93,7 @@ def normalized_source_hash(source: str) -> str: "expected_plan_0_1", "expected_plan_contract", } plan_hashes: dict[str, str] = {} + plan_jsons: dict[str, str] = {} for case in manifests["plan"]["cases"]: require(isinstance(case, dict), "0.2 plan case must be an object.") identifier = require_string(case.get("id"), "0.2 plan case id") @@ -99,6 +103,7 @@ def normalized_source_hash(source: str) -> str: digest = canonical_hash(case["plan"]) require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical plan hash differs.") plan_hashes[identifier] = digest + plan_jsons[identifier] = canonical_json(case["plan"]) else: require(case.get("expected_plan_hash") is None, f"{identifier}: rejected plan hash must be null.") @@ -155,9 +160,11 @@ def normalized_source_hash(source: str) -> str: require(reference in plan_hashes and case.get("expected_plan_hash") == plan_hashes[reference], f"{identifier}: referenced 0.2 plan hash differs.") elif "expected_plan_case_0_1" in case: reference = case["expected_plan_case_0_1"] + require(case.get("expected_plan_contract") == "openstatspec-transformation-plan-v0.1", f"{identifier}: referenced 0.1 plan contract differs.") require(reference in legacy_frontend_hashes and case.get("expected_plan_hash") == legacy_frontend_hashes[reference][0], f"{identifier}: referenced 0.1 plan hash differs.") else: require("expected_plan_0_1" in case, f"{identifier}: embedded plan is missing.") + require(case.get("expected_plan_contract") == "openstatspec-transformation-plan-v0.1", f"{identifier}: embedded 0.1 plan contract differs.") require(legacy_plan_validator.is_valid(case["expected_plan_0_1"]), f"{identifier}: embedded plan violates its declared schema.") require(case.get("expected_plan_hash") == canonical_hash(case["expected_plan_0_1"]), f"{identifier}: embedded plan hash differs.") frontend_hashes[identifier] = (case["expected_plan_hash"], case["expected_source_hash"]) @@ -168,7 +175,9 @@ def normalized_source_hash(source: str) -> str: plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") require(plan_id in plan_hashes and frontend_id in frontend_hashes, f"{case.get('id')}: applied fixture reference is missing.") audit = case.get("expected_audit") - require(frontend_hashes[frontend_id][0] == plan_hashes[plan_id] and isinstance(audit, dict) and audit.get("plan_hash") == plan_hashes[plan_id] and audit.get("source_hash") == frontend_hashes[frontend_id][1], f"{case.get('id')}: applied fixture hashes differ.") + require(frontend_hashes[frontend_id][0] == plan_hashes[plan_id] and isinstance(audit, dict) and audit.get("plan_hash") == plan_hashes[plan_id] and audit.get("canonical_plan_json") == plan_jsons[plan_id] and audit.get("source_hash") == frontend_hashes[frontend_id][1], f"{case.get('id')}: applied fixture hashes or canonical plan differ.") + if "required_audit_fields" in case: + require("canonical_plan_json" in case["required_audit_fields"], f"{case.get('id')}: required audit fields omit canonical plan JSON.") def require_well_formed_create_table_blocks( From 75f39944af8d0fc259e99316188c016e0ceeac27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 12:09:29 +0300 Subject: [PATCH 77/78] Tighten transformation reference declarations --- tools/validate_repository.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 1e26b68..4da74d0 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -33,7 +33,7 @@ def validate_transformation_integrity() -> None: expected_metadata = { "plan": {"manifest_version": "0.2", "profile": "OpenStatSpec Transformation Plan 0.2", "contract": "openstatspec-transformation-plan-v0.2", "canonicalization": "restricted-rfc8785-utf8-sha256"}, "frontend": {"manifest_version": "0.2", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.2", "contract": "openstatspec-spss-syntax-frontend-v0.2", "plan_contracts": ["openstatspec-transformation-plan-v0.1", "openstatspec-transformation-plan-v0.2"], "plan_schemas": {"openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json"}}, - "binding": {"manifest_version": "0.2", "profile": "OpenStatSpec In-Place Transformation Binding 0.2", "contract": "openstatspec-in-place-transformation-v0.2"}, + "binding": {"manifest_version": "0.2", "profile": "OpenStatSpec In-Place Transformation Binding 0.2", "contract": "openstatspec-in-place-transformation-v0.2", "audit_schema": "../sql/transformation-plan-profile-schema.sql"}, "plan_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec Transformation Plan 0.1", "contract": "openstatspec-transformation-plan-v0.1"}, "frontend_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.1", "contract": "openstatspec-spss-syntax-frontend-v0.1"}, "binding_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec In-Place Transformation Binding 0.1", "contract": "openstatspec-in-place-transformation-v0.1"}, @@ -92,6 +92,7 @@ def normalized_source_hash(source: str) -> str: "expected_plan_case", "expected_plan_case_0_1", "expected_plan", "expected_plan_0_1", "expected_plan_contract", } + plan_output_variants = frontend_plan_fields - {"expected_plan_contract"} plan_hashes: dict[str, str] = {} plan_jsons: dict[str, str] = {} for case in manifests["plan"]["cases"]: @@ -129,6 +130,8 @@ def normalized_source_hash(source: str) -> str: require(frontend_plan_fields.isdisjoint(case), f"{identifier}: failed frontend case contains plan output fields.") require(case.get("expected_plan_hash") is None, f"{identifier}: failed frontend plan hash must be absent or null.") continue + require(sum(field in case for field in plan_output_variants) == 1, f"{identifier}: successful frontend case must declare exactly one plan output variant.") + require("expected_plan_contract" not in case, f"{identifier}: 0.1 frontend case has an unexpected plan contract marker.") if "expected_plan_case" in case: reference = case["expected_plan_case"] require(reference in legacy_plan_hashes, f"{identifier}: referenced 0.1 plan is missing.") @@ -155,9 +158,11 @@ def normalized_source_hash(source: str) -> str: require(frontend_plan_fields.isdisjoint(case), f"{identifier}: failed frontend case contains plan output fields.") require(case.get("expected_plan_hash") is None, f"{identifier}: failed frontend plan hash must be absent or null.") continue + require(sum(field in case for field in plan_output_variants) == 1, f"{identifier}: successful frontend case must declare exactly one plan output variant.") if "expected_plan_case" in case: reference = case["expected_plan_case"] require(reference in plan_hashes and case.get("expected_plan_hash") == plan_hashes[reference], f"{identifier}: referenced 0.2 plan hash differs.") + require("expected_plan_contract" not in case, f"{identifier}: direct 0.2 plan reference has an unexpected contract marker.") elif "expected_plan_case_0_1" in case: reference = case["expected_plan_case_0_1"] require(case.get("expected_plan_contract") == "openstatspec-transformation-plan-v0.1", f"{identifier}: referenced 0.1 plan contract differs.") @@ -170,7 +175,12 @@ def normalized_source_hash(source: str) -> str: frontend_hashes[identifier] = (case["expected_plan_hash"], case["expected_source_hash"]) for case in manifests["binding"]["cases"]: - if not isinstance(case, dict) or "applied_plan_case" not in case: + has_plan_reference = "applied_plan_case" in case + has_frontend_reference = "applied_frontend_case" in case + require(has_plan_reference == has_frontend_reference, f"{case.get('id')}: applied fixture references must be paired.") + if "expected_audit" in case: + require(has_plan_reference, f"{case.get('id')}: expected audit is missing applied fixture references.") + if not has_plan_reference: continue plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") require(plan_id in plan_hashes and frontend_id in frontend_hashes, f"{case.get('id')}: applied fixture reference is missing.") From 774f6d70e73961222752189e02ce798d67b7bc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 12:25:12 +0300 Subject: [PATCH 78/78] Remove-transformation-conformance-tooling --- .github/workflows/ci.yml | 1 - README.md | 2 +- requirements-validation.txt | 1 - tools/validate_repository.py | 172 ----------------------------------- 4 files changed, 1 insertion(+), 175 deletions(-) delete mode 100644 requirements-validation.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e6feee..8cd1ff1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,5 +12,4 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.13" - - run: python -m pip install --requirement requirements-validation.txt - run: python tools/validate_repository.py diff --git a/README.md b/README.md index 22e3345..1db09e7 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ to be provisioned separately before an in-place transformation apply. Implementations must preflight target capabilities before import. If the target cannot faithfully create one wide table because of column, identifier, string, or row limits, import must fail atomically with a machine-readable capability diagnostic. It must never silently truncate, drop, split, transpose, pivot, or transform source data. -Validation of Transformation Plan and SPSS syntax frontend artifacts in this specification repository is limited to structural and integrity checks of normative schemas and declarative fixtures. Their executable behavioral conformance, including parser, SQL-engine, transaction, and adapter behavior, belongs in each implementation or adapter repository. +This specification repository publishes only normative Transformation Plan and SPSS syntax frontend schemas, documentation, and declarative fixtures. Executable validation or behavioral conformance code, including parser, SQL-engine, transaction, and adapter behavior, belongs in each implementation or adapter repository. ## Who it is for diff --git a/requirements-validation.txt b/requirements-validation.txt deleted file mode 100644 index d9bf673..0000000 --- a/requirements-validation.txt +++ /dev/null @@ -1 +0,0 @@ -jsonschema==4.10.3 diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 4da74d0..2d0f0bc 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -10,8 +10,6 @@ from pathlib import Path from uuid import UUID -from jsonschema import Draft202012Validator - ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / "conformance/spss-sav-zsav-1.0.json" @@ -21,174 +19,6 @@ def require(condition: bool, message: str) -> None: if not condition: raise SystemExit(message) -def validate_transformation_integrity() -> None: - manifest_paths = { - "plan": ROOT / "conformance/transformation-plan-0.2.json", - "frontend": ROOT / "conformance/spss-syntax-frontend-0.2.json", - "binding": ROOT / "conformance/in-place-transformation-0.2.json", - "plan_0_1": ROOT / "conformance/transformation-plan-0.1.json", - "frontend_0_1": ROOT / "conformance/spss-syntax-frontend-0.1.json", - "binding_0_1": ROOT / "conformance/in-place-transformation-0.1.json", - } - expected_metadata = { - "plan": {"manifest_version": "0.2", "profile": "OpenStatSpec Transformation Plan 0.2", "contract": "openstatspec-transformation-plan-v0.2", "canonicalization": "restricted-rfc8785-utf8-sha256"}, - "frontend": {"manifest_version": "0.2", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.2", "contract": "openstatspec-spss-syntax-frontend-v0.2", "plan_contracts": ["openstatspec-transformation-plan-v0.1", "openstatspec-transformation-plan-v0.2"], "plan_schemas": {"openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json"}}, - "binding": {"manifest_version": "0.2", "profile": "OpenStatSpec In-Place Transformation Binding 0.2", "contract": "openstatspec-in-place-transformation-v0.2", "audit_schema": "../sql/transformation-plan-profile-schema.sql"}, - "plan_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec Transformation Plan 0.1", "contract": "openstatspec-transformation-plan-v0.1"}, - "frontend_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.1", "contract": "openstatspec-spss-syntax-frontend-v0.1"}, - "binding_0_1": {"manifest_version": "0.1", "profile": "OpenStatSpec In-Place Transformation Binding 0.1", "contract": "openstatspec-in-place-transformation-v0.1"}, - } - manifests: dict[str, dict[str, object]] = {} - for name, path in manifest_paths.items(): - document = json.loads(path.read_text(encoding="utf-8")) - require(isinstance(document, dict), f"{path.name}: manifest must be an object.") - require(isinstance(document.get("cases"), list), f"{path.name}: cases must be an array.") - for field, expected in expected_metadata[name].items(): - require(document.get(field) == expected, f"{path.name}: {field} declaration differs.") - identifiers: set[str] = set() - for case in document["cases"]: - require(isinstance(case, dict), f"{path.name}: case must be an object.") - identifier = require_string(case.get("id"), f"{path.name}: case id") - require(identifier not in identifiers, f"{path.name}: duplicate case id: {identifier}") - identifiers.add(identifier) - manifests[name] = document - - links = [ - (manifest_paths["plan"], manifests["plan"]["schema"]), - (manifest_paths["frontend"], manifests["frontend"]["request_schema"]), - *[(manifest_paths["frontend"], link) for link in manifests["frontend"]["plan_schemas"].values()], - (manifest_paths["binding"], manifests["binding"]["audit_schema"]), - (manifest_paths["plan_0_1"], manifests["plan_0_1"]["schema"]), - (manifest_paths["frontend_0_1"], manifests["frontend_0_1"]["request_schema"]), - (manifest_paths["frontend_0_1"], manifests["frontend_0_1"]["plan_schema"]), - ] - schema_validators: dict[Path, Draft202012Validator] = {} - for manifest_path, link in links: - require(isinstance(link, str) and link and not Path(link).is_absolute(), f"{manifest_path.name}: schema link must be relative.") - schema_path = (manifest_path.parent / link).resolve() - require(schema_path.is_file(), f"{manifest_path.name}: schema link does not exist: {link}") - if schema_path.suffix == ".json": - schema = json.loads(schema_path.read_text(encoding="utf-8")) - require(isinstance(schema, dict), f"{schema_path.name}: schema must be an object.") - Draft202012Validator.check_schema(schema) - schema_validators[schema_path] = Draft202012Validator(schema) - plan_validator = schema_validators[(manifest_paths["plan"].parent / manifests["plan"]["schema"]).resolve()] - legacy_plan_validator = schema_validators[(manifest_paths["plan_0_1"].parent / manifests["plan_0_1"]["schema"]).resolve()] - request_validator = schema_validators[(manifest_paths["frontend"].parent / manifests["frontend"]["request_schema"]).resolve()] - legacy_request_validator = schema_validators[(manifest_paths["frontend_0_1"].parent / manifests["frontend_0_1"]["request_schema"]).resolve()] - - def canonical_json(value: object) -> str: - return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) - - def canonical_hash(value: object) -> str: - encoded = canonical_json(value).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - def normalized_source_hash(source: str) -> str: - normalized = source.replace("\r\n", "\n").replace("\r", "\n") - return hashlib.sha256(normalized.encode("utf-8")).hexdigest() - - frontend_plan_fields = { - "expected_plan_case", "expected_plan_case_0_1", "expected_plan", - "expected_plan_0_1", "expected_plan_contract", - } - plan_output_variants = frontend_plan_fields - {"expected_plan_contract"} - plan_hashes: dict[str, str] = {} - plan_jsons: dict[str, str] = {} - for case in manifests["plan"]["cases"]: - require(isinstance(case, dict), "0.2 plan case must be an object.") - identifier = require_string(case.get("id"), "0.2 plan case id") - require(plan_validator.is_valid(case.get("plan")), f"{identifier}: plan violates its declared schema.") - if case.get("expected_error") is None: - require(isinstance(case.get("plan"), dict), f"{identifier}: successful plan is missing.") - digest = canonical_hash(case["plan"]) - require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical plan hash differs.") - plan_hashes[identifier] = digest - plan_jsons[identifier] = canonical_json(case["plan"]) - else: - require(case.get("expected_plan_hash") is None, f"{identifier}: rejected plan hash must be null.") - - legacy_plan_hashes: dict[str, str] = {} - for case in manifests["plan_0_1"]["cases"]: - identifier = require_string(case.get("id"), "0.1 plan case id") - require(legacy_plan_validator.is_valid(case.get("plan")), f"{identifier}: 0.1 plan violates its declared schema.") - if case.get("expected_error") is None: - digest = canonical_hash(case.get("plan")) - require(case.get("expected_plan_hash") == digest, f"{identifier}: canonical 0.1 plan hash differs.") - legacy_plan_hashes[identifier] = digest - else: - require(case.get("expected_plan_hash") is None, f"{identifier}: rejected 0.1 plan hash must be null.") - - for case in manifests["frontend_0_1"]["cases"]: - identifier = require_string(case.get("id"), "0.1 frontend case id") - request = case.get("request") - require(legacy_request_validator.is_valid(request), f"{identifier}: 0.1 request violates its declared schema.") - source = request.get("source_text") - if isinstance(source, str): - require(case.get("expected_source_hash") == normalized_source_hash(source), f"{identifier}: 0.1 source hash differs.") - if case.get("expected_error") is not None: - require(frontend_plan_fields.isdisjoint(case), f"{identifier}: failed frontend case contains plan output fields.") - require(case.get("expected_plan_hash") is None, f"{identifier}: failed frontend plan hash must be absent or null.") - continue - require(sum(field in case for field in plan_output_variants) == 1, f"{identifier}: successful frontend case must declare exactly one plan output variant.") - require("expected_plan_contract" not in case, f"{identifier}: 0.1 frontend case has an unexpected plan contract marker.") - if "expected_plan_case" in case: - reference = case["expected_plan_case"] - require(reference in legacy_plan_hashes, f"{identifier}: referenced 0.1 plan is missing.") - else: - require("expected_plan" in case, f"{identifier}: embedded 0.1 plan is missing.") - require(legacy_plan_validator.is_valid(case["expected_plan"]), f"{identifier}: embedded 0.1 plan violates its declared schema.") - digest = canonical_hash(case["expected_plan"]) - require(case.get("expected_plan_hash") == digest, f"{identifier}: embedded 0.1 plan hash differs.") - legacy_frontend_hashes = { - case["id"]: (case["expected_plan_hash"], case["expected_source_hash"]) - for case in manifests["frontend_0_1"]["cases"] - if case.get("expected_error") is None and "expected_plan_hash" in case - } - frontend_hashes: dict[str, tuple[str, str]] = {} - for case in manifests["frontend"]["cases"]: - require(isinstance(case, dict), "0.2 frontend case must be an object.") - identifier = require_string(case.get("id"), "0.2 frontend case id") - request = case.get("request") - require(request_validator.is_valid(request), f"{identifier}: request violates its declared schema.") - source = request.get("source_text") - if isinstance(source, str): - require(case.get("expected_source_hash") == normalized_source_hash(source), f"{identifier}: source hash differs.") - if case.get("expected_error") is not None: - require(frontend_plan_fields.isdisjoint(case), f"{identifier}: failed frontend case contains plan output fields.") - require(case.get("expected_plan_hash") is None, f"{identifier}: failed frontend plan hash must be absent or null.") - continue - require(sum(field in case for field in plan_output_variants) == 1, f"{identifier}: successful frontend case must declare exactly one plan output variant.") - if "expected_plan_case" in case: - reference = case["expected_plan_case"] - require(reference in plan_hashes and case.get("expected_plan_hash") == plan_hashes[reference], f"{identifier}: referenced 0.2 plan hash differs.") - require("expected_plan_contract" not in case, f"{identifier}: direct 0.2 plan reference has an unexpected contract marker.") - elif "expected_plan_case_0_1" in case: - reference = case["expected_plan_case_0_1"] - require(case.get("expected_plan_contract") == "openstatspec-transformation-plan-v0.1", f"{identifier}: referenced 0.1 plan contract differs.") - require(reference in legacy_frontend_hashes and case.get("expected_plan_hash") == legacy_frontend_hashes[reference][0], f"{identifier}: referenced 0.1 plan hash differs.") - else: - require("expected_plan_0_1" in case, f"{identifier}: embedded plan is missing.") - require(case.get("expected_plan_contract") == "openstatspec-transformation-plan-v0.1", f"{identifier}: embedded 0.1 plan contract differs.") - require(legacy_plan_validator.is_valid(case["expected_plan_0_1"]), f"{identifier}: embedded plan violates its declared schema.") - require(case.get("expected_plan_hash") == canonical_hash(case["expected_plan_0_1"]), f"{identifier}: embedded plan hash differs.") - frontend_hashes[identifier] = (case["expected_plan_hash"], case["expected_source_hash"]) - - for case in manifests["binding"]["cases"]: - has_plan_reference = "applied_plan_case" in case - has_frontend_reference = "applied_frontend_case" in case - require(has_plan_reference == has_frontend_reference, f"{case.get('id')}: applied fixture references must be paired.") - if "expected_audit" in case: - require(has_plan_reference, f"{case.get('id')}: expected audit is missing applied fixture references.") - if not has_plan_reference: - continue - plan_id, frontend_id = case["applied_plan_case"], case.get("applied_frontend_case") - require(plan_id in plan_hashes and frontend_id in frontend_hashes, f"{case.get('id')}: applied fixture reference is missing.") - audit = case.get("expected_audit") - require(frontend_hashes[frontend_id][0] == plan_hashes[plan_id] and isinstance(audit, dict) and audit.get("plan_hash") == plan_hashes[plan_id] and audit.get("canonical_plan_json") == plan_jsons[plan_id] and audit.get("source_hash") == frontend_hashes[frontend_id][1], f"{case.get('id')}: applied fixture hashes or canonical plan differ.") - if "required_audit_fields" in case: - require("canonical_plan_json" in case["required_audit_fields"], f"{case.get('id')}: required audit fields omit canonical plan JSON.") - def require_well_formed_create_table_blocks( schema: str, @@ -1094,8 +924,6 @@ def main() -> None: require(f"CREATE TABLE {table} (" in schema, f"Schema table is missing: {table}") validate_transformation_profile() - validate_transformation_integrity() - validate_dialect_baseline() json.loads(MANIFEST.read_text(encoding="utf-8"))