From e8e19bf9e6c8836080b1c32dd167a5990f7768ae Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 27 Aug 2026 10:03:43 +0200 Subject: [PATCH 01/19] two bug fixes --- README.md | 19 +- rules/README.md | 10 +- rules/default_rules.ttl | 6 +- src/ensure_hdt_index.sh | 25 +- test/test_vcf_as_tsv_unit.py | 32 +++ test/test_vcf_rdfizer_unit.py | 108 ++++++++- vcf_rdfizer.py | 419 ++++++++++++++++++++++++++++++---- 7 files changed, 561 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 810376f..dc25fdf 100644 --- a/README.md +++ b/README.md @@ -438,6 +438,11 @@ HDT Java 3.0.10 does not provide a standalone `hdtGenerateIndex` executable. VCF-RDFizer sends an `exit` command to the supported `hdtSearch.sh` launcher; this opens the HDT through `mapIndexedHDT()` without executing a data query and creates the versioned `.hdt.index.v1-1` sidecar before the run is marked successful. +The helper explicitly selects HDT Java's external-sort disk indexer rather than +the launcher's heap-based default (whose launcher heap is only 1 GiB). Temporary +sort runs and disk-backed sequences are kept under `/work` and removed when the +index command finishes. `HDT_INDEX_WORK_ROOT` can override that scratch root +when invoking the helper directly. For the pinned HDT Java 3.0.10 distribution, this is the HDT v1-1 sidecar `.hdt.index.v1-1`; VCF-RDFizer reports the actual path in its metrics. @@ -445,6 +450,15 @@ The record-safe chunk plan and per-stage timings are retained in the raw partitioned-compression metrics JSON for diagnostics. The temporary chunk files and guide are not retained as host files. +For the default mapping, multi-sample VCF columns remain compact in +`records.tsv`. Canonical `SampleCall` and `FormatFieldValue` triples are streamed +directly into the final `.nt` or `.nt.gz` aggregate instead of first writing +`variants × samples` and `variants × samples × FORMAT fields` helper rows. +The compatibility helper TSVs therefore contain only headers. Custom mappings +that add consumers of those helper sources continue to use expanded tables. +This removes the large temporary-disk multiplier, although the final RDF still +scales with the number of emitted sample and FORMAT triples. + The implementation keeps COTTAS conversion scratch state inside the Docker container and removes temporary unpacked package files when decompression finishes. @@ -461,7 +475,10 @@ If Docker permission issues occur, rerun with a Docker-allowed user (or configur If HDT compression fails on very large RDF files, use `--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with `--hdt-strategy partitioned`, then lower `--chunk-target-bytes` and -`--chunk-max-bytes` to reduce each converter's working set. +`--chunk-max-bytes` to reduce each converter's working set. Final HDT index +creation is disk-backed, so ensure the Docker data volume has enough temporary +space for the external sort; free space in the output filesystem alone does +not increase the JVM heap. Safe termination: diff --git a/rules/README.md b/rules/README.md index 75ce2a6..1925b0f 100644 --- a/rules/README.md +++ b/rules/README.md @@ -13,9 +13,13 @@ This directory contains RML mappings used by the conversion pipeline. - `/data/tsv/records.tsv` - `/data/tsv/sample_calls.tsv` - `/data/tsv/sample_format_values.tsv` - - `sample_calls.tsv` and `sample_format_values.tsv` are derived by the Python wrapper - from `records.tsv` at runtime so FORMAT fields (e.g., `GT:DP:AD`) can be - mapped to per-sample values consistently. + - For the built-in sample maps, `sample_calls.tsv` and + `sample_format_values.tsv` are header-only compatibility sources. The Python + wrapper streams their equivalent `SampleCall` and `FormatFieldValue` triples + directly from `records.tsv` into the RDF aggregate, avoiding helper-table + expansion proportional to variants × samples × FORMAT fields. + - Custom mappings with additional consumers of either helper source retain + expanded TSV generation for compatibility. - The Python wrapper rewrites these template paths per input VCF to: - `/data/tsv/.file_metadata.tsv` - `/data/tsv/.header_lines.tsv` diff --git a/rules/default_rules.ttl b/rules/default_rules.ttl index ba357f4..02bf24e 100644 --- a/rules/default_rules.ttl +++ b/rules/default_rules.ttl @@ -11,9 +11,11 @@ # - /data/tsv/.file_metadata.tsv # - /data/tsv/.header_lines.tsv # - /data/tsv/.records.tsv -# - /data/tsv/.sample_calls.tsv (derived by wrapper) -# - /data/tsv/.sample_format_values.tsv (derived by wrapper) +# - /data/tsv/.sample_calls.tsv (header-only compatibility source) +# - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. +# Canonical sample/FORMAT triples are streamed directly into the RDF aggregate +# so chromosome-scale multi-sample VCFs do not require expanded helper tables. # # Output format: # - The default conversion now targets triples (N-Triples) without named graph terms. diff --git a/src/ensure_hdt_index.sh b/src/ensure_hdt_index.sh index 93a875a..06beffc 100644 --- a/src/ensure_hdt_index.sh +++ b/src/ensure_hdt_index.sh @@ -19,10 +19,33 @@ if [[ ! -x "$HDT_SEARCH_BIN" ]]; then exit 127 fi +# The HDT Java launcher defaults to a 1 GiB heap. Its default "recommended" +# indexer still builds and sorts object lists in that heap, so a large but +# otherwise valid HDT can fail here after the merge has already succeeded. +# HDT Java 3.0.10 provides an external-sort indexer for this case. Keep its +# sequences, bitmap sub-indexes, and sort runs in a disposable directory so +# peak heap usage is bounded by the indexer's chunk budget. +HDT_INDEX_WORK_ROOT=${HDT_INDEX_WORK_ROOT:-/work} +if [[ ! -d "$HDT_INDEX_WORK_ROOT" || ! -w "$HDT_INDEX_WORK_ROOT" ]]; then + echo "HDT index work directory is not writable: $HDT_INDEX_WORK_ROOT" >&2 + exit 2 +fi + +INDEX_WORK_DIR=$(mktemp -d "$HDT_INDEX_WORK_ROOT/vcf-rdfizer-hdt-index.XXXXXXXX") +cleanup() { + rm -rf -- "$INDEX_WORK_DIR" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +HDT_INDEX_OPTIONS="bitmaptriples.indexmethod=disk;bitmaptriples.sequence.disk=true;bitmaptriples.sequence.disk.subindex=true;bitmaptriples.sequence.disk.location=$INDEX_WORK_DIR" + # hdtSearch uses HDT Java's mapIndexedHDT(), which creates the sibling index # lazily. Sending only `exit` initializes the index without running a query or # materializing an unbounded result set. -printf 'exit\n' | "$HDT_SEARCH_BIN" "$HDT_PATH" >/dev/null +printf 'exit\n' | "$HDT_SEARCH_BIN" -quiet -options "$HDT_INDEX_OPTIONS" "$HDT_PATH" >/dev/null shopt -s nullglob # HDT Java 3.0.10 writes the v1-1 index as ``.hdt.index.v1-1``. diff --git a/test/test_vcf_as_tsv_unit.py b/test/test_vcf_as_tsv_unit.py index f293f28..8a3c4a0 100644 --- a/test/test_vcf_as_tsv_unit.py +++ b/test/test_vcf_as_tsv_unit.py @@ -1,3 +1,4 @@ +import csv import gzip import subprocess import tempfile @@ -113,6 +114,37 @@ def test_vcf_as_tsv_collapses_multi_sample_header_and_values_to_single_column(se "multi.vcf\t1\t1\t10\trs1\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:42 0/0:18", ) + def test_vcf_as_tsv_preserves_thousands_of_sample_columns(self): + """A 1000 Genomes-sized sample header and payload remain aligned.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_file = tmp_path / "many-samples.vcf" + output_dir = tmp_path / "tsv" + sample_ids = [f"S{i:04d}" for i in range(2504)] + payloads = [f"{i % 2}/{(i + 1) % 2}:{10 + (i % 90)}" for i in range(2504)] + input_file.write_text( + "##fileformat=VCFv4.2\n" + + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + + "\t".join(sample_ids) + + "\n20\t10\trs1\tA\tG\t50\tPASS\t.\tGT:DP\t" + + "\t".join(payloads) + + "\n", + encoding="utf-8", + ) + + result = subprocess.run( + ["bash", str(SCRIPT), str(input_file), str(output_dir)], + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + with (output_dir / "many-samples.records.tsv").open(newline="", encoding="utf-8") as handle: + rows = list(csv.reader(handle, delimiter="\t")) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0][-1].split(), sample_ids) + self.assertEqual(rows[1][-1].split(), payloads) + def test_vcf_as_tsv_single_gz_file_mode(self): """Single .vcf.gz input: decompresses and writes per-VCF split TSV outputs.""" with tempfile.TemporaryDirectory() as td: diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 9fd09fc..1b3689d 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -1261,6 +1261,90 @@ def test_build_sample_support_tsvs_sanitizes_sample_id_for_uri_paths(self): self.assertIn("sample.vcf\t2\t1\tSAMPLE-A\tSAMPLE-A\t0/1:42", sample_calls_rows) self.assertIn("sample.vcf\t2\t2\tSAMPLE/B\tSAMPLE_B\t0/0:18", sample_calls_rows) + def test_default_sample_maps_stream_without_expanded_helper_tables(self): + """Canonical sample maps emit RDF directly while compatibility TSVs stay empty.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "sample.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tSAMPLE-A SAMPLE/B\n" + "sample.vcf\t2\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:42 ./.:.\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text(" .\n", encoding="utf-8") + sample_calls_tsv = tmp_path / "sample.sample_calls.tsv" + sample_format_tsv = tmp_path / "sample.sample_format_values.tsv" + + vcf_rdfizer.write_sample_support_headers(sample_calls_tsv, sample_format_tsv) + stats = vcf_rdfizer.append_canonical_sample_rdf( + records_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(sample_calls_tsv.read_text().splitlines(), ["\t".join(vcf_rdfizer.SAMPLE_CALLS_HEADER)]) + self.assertEqual(sample_format_tsv.read_text().splitlines(), ["\t".join(vcf_rdfizer.SAMPLE_FORMAT_HEADER)]) + self.assertEqual(stats["records"], 1) + self.assertEqual(stats["sample_calls"], 2) + self.assertEqual(stats["format_values"], 4) + self.assertEqual(stats["triples"], 18) + rdf_lines = rdf_path.read_text(encoding="utf-8").splitlines() + self.assertEqual(len(rdf_lines), 19) + self.assertIn( + " " + " .", + rdf_lines, + ) + self.assertIn( + '"."^^ .', + rdf_lines[-1], + ) + + def test_sample_streaming_accepts_a_csv_field_larger_than_python_default(self): + """Thousands of long sample payloads do not hit csv.field_size_limit.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + sample_ids = [f"S{i:04d}" for i in range(2504)] + payload = "A" * 60 + records_tsv = tmp_path / "large.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + + " ".join(sample_ids) + + "\nlarge.vcf\t1\t20\t1\t.\tA\tG\t.\tPASS\t.\tGT\t" + + " ".join([payload] * len(sample_ids)) + + "\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "large.nt.gz" + with gzip.open(rdf_path, "wt", encoding="utf-8") as handle: + handle.write(" .\n") + + stats = vcf_rdfizer.append_canonical_sample_rdf( + records_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(stats["sample_calls"], 2504) + self.assertEqual(stats["format_values"], 2504) + self.assertEqual(stats["triples"], 2504 * 6) + with gzip.open(rdf_path, "rt", encoding="utf-8") as handle: + self.assertEqual(sum(1 for _line in handle), 1 + (2504 * 6)) + + def test_sample_support_strategy_preserves_custom_helper_consumers(self): + """Only the exact canonical helper maps use direct RDF streaming.""" + default_rules = Path(__file__).parents[1] / "rules" / "default_rules.ttl" + self.assertEqual(vcf_rdfizer.sample_support_strategy(default_rules), "stream") + with tempfile.TemporaryDirectory() as td: + custom_rules = Path(td) / "custom.ttl" + custom_rules.write_text( + default_rules.read_text(encoding="utf-8") + + '\n<#Extra> csvw:url "/data/tsv/sample_calls.tsv" .\n', + encoding="utf-8", + ) + self.assertEqual(vcf_rdfizer.sample_support_strategy(custom_rules), "expanded") + def test_render_rules_for_triplet_rewrites_helper_tsv_placeholders(self): """Rule rendering rewrites records/header/metadata and helper TSV placeholders.""" with tempfile.TemporaryDirectory() as td: @@ -2225,18 +2309,23 @@ def fake_run(cmd, cwd=None, env=None): ) def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): - """The Docker-side helper initializes the index without issuing a query.""" + """The helper uses HDT Java's bounded-memory disk indexer.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) java_home = tmp_path / "java-home" bin_dir = java_home / "bin" bin_dir.mkdir(parents=True) + work_root = tmp_path / "index-work" + work_root.mkdir() + invocation_path = tmp_path / "invocation.txt" search = bin_dir / "hdtSearch.sh" search.write_text( "#!/usr/bin/env bash\n" "read -r command\n" "[[ \"$command\" == \"exit\" ]] || exit 3\n" - "printf 'index\\n' > \"${1}.index.v1-1\"\n", + "printf '%s\\n' \"$@\" > \"$HDT_TEST_INVOCATION\"\n" + "hdt_path=${!#}\n" + "printf 'index\\n' > \"${hdt_path}.index.v1-1\"\n", encoding="utf-8", ) search.chmod(0o755) @@ -2246,7 +2335,12 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): result = subprocess.run( ["bash", str(helper), str(hdt_path)], - env={**os.environ, "HDT_JAVA_HOME": str(java_home)}, + env={ + **os.environ, + "HDT_JAVA_HOME": str(java_home), + "HDT_INDEX_WORK_ROOT": str(work_root), + "HDT_TEST_INVOCATION": str(invocation_path), + }, capture_output=True, text=True, ) @@ -2254,6 +2348,14 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): self.assertEqual(result.returncode, 0, msg=result.stderr) self.assertTrue(Path(str(hdt_path) + ".index.v1-1").exists()) self.assertIn(f"HDT index ready: {hdt_path}.index.v1-1", result.stdout) + invocation = invocation_path.read_text(encoding="utf-8").splitlines() + self.assertEqual(invocation[0:2], ["-quiet", "-options"]) + self.assertIn("bitmaptriples.indexmethod=disk", invocation[2]) + self.assertIn("bitmaptriples.sequence.disk=true", invocation[2]) + self.assertIn("bitmaptriples.sequence.disk.subindex=true", invocation[2]) + self.assertIn("bitmaptriples.sequence.disk.location=", invocation[2]) + self.assertEqual(invocation[3], str(hdt_path)) + self.assertEqual(list(work_root.iterdir()), []) def test_main_decompress_mode_rejects_unknown_extension(self): """Decompression mode rejects unsupported compressed RDF extensions.""" diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 9a1600a..e7bc1c5 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -27,6 +27,7 @@ import time from datetime import datetime from pathlib import Path +from urllib.parse import quote_plus RMLSTREAMER_JAR_CONTAINER = "/opt/rmlstreamer/RMLStreamer-v2.5.0-standalone.jar" @@ -185,6 +186,46 @@ DEFAULT_CHUNK_MAX_BYTES = 1024 * 1024 * 1024 HDT_INDEX_HELPER_CONTAINER = "/opt/vcf-rdfizer/ensure_hdt_index.sh" PARTITIONED_COMPRESSION_RUNNER_CONTAINER = "/opt/vcf-rdfizer/partitioned_compression.py" +SAMPLE_CALLS_HEADER = [ + "SOURCE_FILE", + "ROW_ID", + "SAMPLE_INDEX", + "SAMPLE_ID", + "SAMPLE_URI_ID", + "SAMPLE_PAYLOAD", +] +SAMPLE_FORMAT_HEADER = [ + "SOURCE_FILE", + "ROW_ID", + "SAMPLE_INDEX", + "SAMPLE_ID", + "SAMPLE_URI_ID", + "FORMAT_INDEX", + "FORMAT_KEY", + "FORMAT_VALUE", +] +CANONICAL_SAMPLE_RULE_MARKERS = ( + "<#VariantCallToSampleLinkMap>", + "<#SampleCallMap>", + "<#SampleCallToFormatValueLinkMap>", + "<#FormatFieldValueMap>", +) +CANONICAL_SAMPLE_RULE_FRAGMENTS = ( + 'rr:template "file://{SOURCE_FILE}#call/{ROW_ID}"', + "rr:predicate vcfr:hasSampleCall", + 'rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}"', + "rr:class vcfr:SampleCall", + "rr:predicate vcfr:sampleId", + 'rml:reference "SAMPLE_ID"', + "rr:predicate vcfr:hasFormatValue", + 'rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}"', + "rr:class vcfr:FormatFieldValue", + "rr:predicate vcfr:fieldValue", + 'rml:reference "FORMAT_VALUE"', +) +VCFR_NAMESPACE = "https://w3id.org/vcf-rdfizer/vocab#" +RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +SAMPLE_RDF_BUFFER_BYTES = 8 * 1024 * 1024 # --------------------------------------------------------------------------- @@ -1483,6 +1524,279 @@ def discover_tsv_triplets(tsv_dir: Path): ) +def write_sample_support_headers(sample_calls_tsv: Path, sample_format_tsv: Path): + """Create empty sample helper tables with their canonical TSV headers.""" + sample_calls_tsv.parent.mkdir(parents=True, exist_ok=True) + sample_format_tsv.parent.mkdir(parents=True, exist_ok=True) + with sample_calls_tsv.open("w", newline="", encoding="utf-8") as sample_calls_handle, \ + sample_format_tsv.open("w", newline="", encoding="utf-8") as sample_format_handle: + csv.writer(sample_calls_handle, delimiter="\t").writerow(SAMPLE_CALLS_HEADER) + csv.writer(sample_format_handle, delimiter="\t").writerow(SAMPLE_FORMAT_HEADER) + + +def sample_support_strategy(rules_path: Path) -> str: + """Choose no, streamed, or expanded sample handling for one mapping file. + + The built-in four sample maps can be emitted directly as N-Triples without + writing their enormous Cartesian helper TSVs. A custom mapping with extra + helper-table consumers retains the expanded TSV behavior. + """ + text = rules_path.read_text(encoding="utf-8") + calls_refs = text.count('/data/tsv/sample_calls.tsv') + format_refs = text.count('/data/tsv/sample_format_values.tsv') + if calls_refs == 0 and format_refs == 0: + return "none" + if ( + calls_refs == 2 + and format_refs == 2 + and all(marker in text for marker in CANONICAL_SAMPLE_RULE_MARKERS) + and all(fragment in text for fragment in CANONICAL_SAMPLE_RULE_FRAGMENTS) + ): + return "stream" + return "expanded" + + +def _set_max_csv_field_size(): + """Allow chromosome-scale multi-sample payload columns in Python's CSV reader.""" + limit = sys.maxsize + while True: + try: + csv.field_size_limit(limit) + return + except OverflowError: + limit //= 10 + + +def _sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: + candidate = re.sub(r"[^A-Za-z0-9._~-]+", "_", sample_id).strip("_") + return candidate or f"sample_{fallback_index}" + + +def _rml_uri_component(value: str) -> str: + """Match RMLStreamer's Java URLEncoder-based template substitution.""" + encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") + # urllib follows current RFC rules and always leaves '~' unescaped, whereas + # java.net.URLEncoder (used by RMLStreamer 2.5.0) encodes it. + return encoded.replace("+", "%20").replace("~", "%7E") + + +def _ntriples_literal(value: str) -> str: + escaped = ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + literal = f'"{escaped}"' + if value == ".": + literal += f"^^<{VCFR_NAMESPACE}Null>" + return literal + + +def append_canonical_sample_rdf( + records_tsv: Path, + rdf_path: Path, + *, + progress_interval_records: int = 10_000, +) -> dict: + """Stream the built-in sample mappings directly into an RDF aggregate. + + This produces the same canonical SampleCall and FormatFieldValue triples as + the default RML maps without first materializing V*S and V*S*F TSV rows. + The aggregate is rolled back to its original byte length if streaming fails. + """ + stats = { + "records": 0, + "sample_calls": 0, + "format_values": 0, + "triples": 0, + "appended_bytes": 0, + } + if not records_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for sample streaming: {rdf_path}") + + _set_max_csv_field_size() + original_size = rdf_path.stat().st_size + opener = gzip.open if rdf_path.name.endswith(".gz") else Path.open + mode = "ab" + output_handle = None + buffer = bytearray() + + def emit(line: str): + nonlocal buffer + buffer.extend(line.encode("utf-8")) + stats["triples"] += 1 + if len(buffer) >= SAMPLE_RDF_BUFFER_BYTES: + output_handle.write(buffer) + buffer = bytearray() + + try: + output_handle = opener(rdf_path, mode) + with records_tsv.open(newline="", encoding="utf-8") as records_handle: + reader = csv.reader(records_handle, delimiter="\t") + header = next(reader, None) + if not header: + output_handle.close() + output_handle = None + stats["appended_bytes"] = rdf_path.stat().st_size - original_size + return stats + + sample_header = header[-1].strip() if len(header) >= 12 else "" + declared_sample_ids = ( + [] + if sample_header == "SAMPLES" + else [token for token in sample_header.split() if token] + ) + + for row in reader: + if not row: + continue + if len(row) < len(header): + row += [""] * (len(header) - len(row)) + + source_file = row[0] if len(row) > 0 else "" + row_id = row[1] if len(row) > 1 else "" + format_raw = row[10] if len(row) > 10 else "" + samples_raw = row[-1] if len(row) >= 12 else "" + format_keys = format_raw.split(":") if format_raw else [] + sample_payloads = samples_raw.split() if samples_raw else [] + total_samples = max(len(declared_sample_ids), len(sample_payloads)) + if total_samples == 0: + continue + + source_component = _rml_uri_component(source_file) + row_component = _rml_uri_component(row_id) + call_uri = f"file://{source_component}#call/{row_component}" + sample_uri_seen: dict[str, int] = {} + + for sample_idx in range(total_samples): + sample_id = ( + declared_sample_ids[sample_idx] + if sample_idx < len(declared_sample_ids) + else f"SAMPLE_{sample_idx + 1}" + ) + sample_payload = ( + sample_payloads[sample_idx] + if sample_idx < len(sample_payloads) + else "" + ) + sample_uri_id_base = _sample_id_to_uri_id(sample_id, sample_idx + 1) + sample_uri_seen[sample_uri_id_base] = sample_uri_seen.get(sample_uri_id_base, 0) + 1 + duplicate_index = sample_uri_seen[sample_uri_id_base] + sample_uri_id = ( + f"{sample_uri_id_base}_{duplicate_index}" + if duplicate_index > 1 + else sample_uri_id_base + ) + sample_component = _rml_uri_component(sample_uri_id) + sample_uri = f"file://{source_component}#sample/{row_component}/{sample_component}" + + emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasSampleCall> <{sample_uri}> .\n") + emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleCall> .\n") + emit(f"<{sample_uri}> <{VCFR_NAMESPACE}sampleId> {_ntriples_literal(sample_id)} .\n") + stats["sample_calls"] += 1 + + value_tokens = sample_payload.split(":") if sample_payload else [] + total_fields = max(len(format_keys), len(value_tokens)) + for format_idx in range(total_fields): + format_key = ( + format_keys[format_idx] + if format_idx < len(format_keys) and format_keys[format_idx] + else f"FIELD_{format_idx + 1}" + ) + format_value = value_tokens[format_idx] if format_idx < len(value_tokens) else "" + format_component = _rml_uri_component(format_key) + format_uri = f"{sample_uri}/fmt/{format_component}" + emit(f"<{sample_uri}> <{VCFR_NAMESPACE}hasFormatValue> <{format_uri}> .\n") + emit(f"<{format_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatFieldValue> .\n") + if format_value: + emit( + f"<{format_uri}> <{VCFR_NAMESPACE}fieldValue> " + f"{_ntriples_literal(format_value)} .\n" + ) + stats["format_values"] += 1 + + stats["records"] += 1 + if progress_interval_records > 0 and stats["records"] % progress_interval_records == 0: + print( + " * Sample RDF streaming: " + f"{stats['records']:,} variants, {stats['sample_calls']:,} calls", + flush=True, + ) + + if buffer: + output_handle.write(buffer) + output_handle.close() + output_handle = None + stats["appended_bytes"] = rdf_path.stat().st_size - original_size + return stats + except BaseException: + if output_handle is not None: + try: + output_handle.close() + except OSError: + pass + with rdf_path.open("r+b") as rollback_handle: + rollback_handle.truncate(original_size) + raise + + +def update_conversion_metrics_after_sample_stream( + *, + metrics_dir: Path, + output_name: str, + run_id: str, + rdf_path: Path, + total_triples: int, + sample_stats: dict, +): + """Bring conversion JSON/CSV metrics in sync after direct sample emission.""" + safe_name = safe_metrics_name(output_name) + metrics_json = metrics_dir / "conversion_metrics" / safe_name / f"{run_id}.json" + output_size = int(rdf_path.stat().st_size) + if metrics_json.is_file(): + try: + payload = json.loads(metrics_json.read_text(encoding="utf-8")) + artifacts = payload.setdefault("artifacts", {}) + prior_triples = artifacts.get("output_triples") + if isinstance(prior_triples, dict): + for key in list(prior_triples): + prior_triples[key] = int(total_triples) + prior_triples["TOTAL"] = int(total_triples) + else: + artifacts["output_triples"] = int(total_triples) + artifacts["output_size_bytes"] = output_size + payload["sample_streaming"] = sample_stats + metrics_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except (OSError, json.JSONDecodeError): + pass + + metrics_csv = metrics_dir / "metrics.csv" + if not metrics_csv.is_file(): + return + try: + with metrics_csv.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + fieldnames = list(reader.fieldnames or []) + rows = list(reader) + changed = False + for row in rows: + if row.get("run_id") == run_id and row.get("output_name") == output_name: + row["output_triples"] = str(int(total_triples)) + row["output_dir_size_bytes"] = str(output_size) + changed = True + if changed: + with metrics_csv.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + except OSError: + pass + + def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_format_tsv: Path): """Materialize per-sample helper TSVs from records.tsv. @@ -1491,46 +1805,16 @@ def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_ - one `SampleCall` per sample/record - one `FormatFieldValue` per sample/record/FORMAT key """ - sample_calls_tsv.parent.mkdir(parents=True, exist_ok=True) - sample_format_tsv.parent.mkdir(parents=True, exist_ok=True) - - def sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: - candidate = re.sub(r"[^A-Za-z0-9._~-]+", "_", sample_id).strip("_") - if not candidate: - candidate = f"sample_{fallback_index}" - return candidate - - with sample_calls_tsv.open("w", newline="", encoding="utf-8") as sample_calls_handle, \ - sample_format_tsv.open("w", newline="", encoding="utf-8") as sample_format_handle: + write_sample_support_headers(sample_calls_tsv, sample_format_tsv) + with sample_calls_tsv.open("a", newline="", encoding="utf-8") as sample_calls_handle, \ + sample_format_tsv.open("a", newline="", encoding="utf-8") as sample_format_handle: sample_calls_writer = csv.writer(sample_calls_handle, delimiter="\t") sample_format_writer = csv.writer(sample_format_handle, delimiter="\t") - sample_calls_writer.writerow( - [ - "SOURCE_FILE", - "ROW_ID", - "SAMPLE_INDEX", - "SAMPLE_ID", - "SAMPLE_URI_ID", - "SAMPLE_PAYLOAD", - ] - ) - sample_format_writer.writerow( - [ - "SOURCE_FILE", - "ROW_ID", - "SAMPLE_INDEX", - "SAMPLE_ID", - "SAMPLE_URI_ID", - "FORMAT_INDEX", - "FORMAT_KEY", - "FORMAT_VALUE", - ] - ) - if not records_tsv.exists(): return + _set_max_csv_field_size() with records_tsv.open(newline="", encoding="utf-8") as records_handle: reader = csv.reader(records_handle, delimiter="\t") header = next(reader, None) @@ -1538,7 +1822,11 @@ def sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: return sample_header = header[-1].strip() if len(header) >= 12 else "" - declared_sample_ids = [token for token in sample_header.split() if token] + declared_sample_ids = ( + [] + if sample_header == "SAMPLES" + else [token for token in sample_header.split() if token] + ) for row in reader: if not row: @@ -1571,7 +1859,7 @@ def sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: else "" ) sample_index_value = str(sample_idx + 1) - sample_uri_id_base = sample_id_to_uri_id(sample_id, sample_idx + 1) + sample_uri_id_base = _sample_id_to_uri_id(sample_id, sample_idx + 1) sample_uri_seen[sample_uri_id_base] = sample_uri_seen.get(sample_uri_id_base, 0) + 1 if sample_uri_seen[sample_uri_id_base] > 1: sample_uri_id = f"{sample_uri_id_base}_{sample_uri_seen[sample_uri_id_base]}" @@ -3340,6 +3628,7 @@ def run_full_mode( ensure_dir(metrics_dir) selected_methods = list(methods) + sample_strategy = sample_support_strategy(rules_path) use_partitioned_hdt = should_use_partitioned_hdt( mode="full", methods=selected_methods, @@ -3409,13 +3698,10 @@ def fail_current(stage: str, message: str): # Pre-flight write checks for expected TSV outputs to fail fast on # permission/mount problems before starting container work. - for suffix in ( - "records.tsv", - "header_lines.tsv", - "file_metadata.tsv", - "sample_calls.tsv", - "sample_format_values.tsv", - ): + expected_tsv_suffixes = ["records.tsv", "header_lines.tsv", "file_metadata.tsv"] + if sample_strategy != "none": + expected_tsv_suffixes.extend(["sample_calls.tsv", "sample_format_values.tsv"]) + for suffix in expected_tsv_suffixes: expected_tsv_output = tsv_dir / f"{expected_prefix}.{suffix}" if not ensure_writable_path_or_fix( target_path=expected_tsv_output, @@ -3471,11 +3757,16 @@ def fail_current(stage: str, message: str): sample_calls_tsv = tsv_dir / f"{prefix}.sample_calls.tsv" sample_format_tsv = tsv_dir / f"{prefix}.sample_format_values.tsv" try: - build_sample_support_tsvs( - records_tsv=triplet["records"], - sample_calls_tsv=sample_calls_tsv, - sample_format_tsv=sample_format_tsv, - ) + if sample_strategy == "expanded": + build_sample_support_tsvs( + records_tsv=triplet["records"], + sample_calls_tsv=sample_calls_tsv, + sample_format_tsv=sample_format_tsv, + ) + elif sample_strategy == "stream": + # RMLStreamer sees valid, empty canonical sources. Their + # equivalent triples are appended directly after base mapping. + write_sample_support_headers(sample_calls_tsv, sample_format_tsv) except Exception as exc: fail_current( "tsv-derivation", @@ -3632,8 +3923,40 @@ def fail_current(stage: str, message: str): ) continue + sample_stats = None + if sample_strategy == "stream": + print(" * Streaming canonical multi-sample RDF (no expanded helper TSVs)") + try: + sample_stats = append_canonical_sample_rdf( + records_tsv=triplet["records"], + rdf_path=raw_rdf_files[0], + ) + except Exception as exc: + fail_current( + "sample-rdf-streaming", + f"failed streaming sample RDF for '{prefix}': {exc}. " + f"See log: {wrapper_log_path}", + ) + continue + if triples_produced is not None: + triples_produced += int(sample_stats["triples"]) + print( + " * Sample calls streamed: " + f"{sample_stats['sample_calls']:,}; FORMAT values: " + f"{sample_stats['format_values']:,}" + ) + if triples_produced is None: triples_produced = count_triples_in_nt_files(raw_rdf_files) + if sample_stats is not None and triples_produced is not None: + update_conversion_metrics_after_sample_stream( + metrics_dir=metrics_dir, + output_name=output_name, + run_id=run_id, + rdf_path=raw_rdf_files[0], + total_triples=triples_produced, + sample_stats=sample_stats, + ) if triples_produced is not None: saw_triple_counts = True total_triples_produced += triples_produced From ef12512f36eca620ce95f33aa3ca38f9deede87f Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 27 Aug 2026 10:34:17 +0200 Subject: [PATCH 02/19] added hdt/cottas index-only functionality --- README.md | 56 ++++++++++-- src/cottas_tool.py | 42 +++++++++ src/ensure_hdt_index.sh | 20 +++++ test/test_cottas_tool.py | 65 ++++++++++++++ test/test_vcf_rdfizer_unit.py | 157 ++++++++++++++++++++++++++++++++++ vcf_rdfizer.py | 155 ++++++++++++++++++++++++--------- 6 files changed, 445 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index dc25fdf..9bb638f 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ inside this directory. - `tsv`: VCF -> TSV only (benchmarking) - `compress`: compress an existing `.nt` or `.nt.gz` - `decompress`: decompress `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` -- `index`: eagerly initialize HDT Java's versioned `.hdt.index.*` sidecar for an existing `.hdt` +- `index`: only generate or regenerate the query index for an existing `.hdt` or `.cottas` In `full` mode with multiple VCF inputs, failures are isolated per input: - the run continues with remaining files @@ -165,10 +165,18 @@ host filesystem. - `-C, --compressed-input` required `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` - `-d, --decompress-out` optional explicit output `.nt` path (must be inside `--out`) -## HDT Index Mode Flags - -- `-H, --hdt` required existing `.hdt` file -- HDT Java 3.0.10 generates an HDT v1-1 index named `.hdt.index.v1-1`. +## Index-Only Mode Flags + +- `-H, --hdt` existing `.hdt` file; creates or regenerates its sibling sidecar +- `--cottas` existing `.cottas` file; rebuilds its embedded query index in place +- Exactly one of `--hdt` or `--cottas` is required. +- HDT Java 3.0.10 generates an HDT v1-1 sidecar named `.hdt.index.v1-1`. +- COTTAS indexes are stored inside the Parquet-based `.cottas` file, so the + file is rewritten atomically; no separate COTTAS index file is expected. +- Existing indexes are intentionally replaced. Use this mode when an HDT + sidecar is missing/stale or when a COTTAS file needs its query ordering and + zone-map metadata rebuilt. No VCF conversion, RDF conversion, packaging, or + decompression output is produced. - The operation is also run automatically after each partitioned HDT merge. ## Quick Start @@ -336,6 +344,23 @@ vcf-rdfizer \ --out ./results ``` +Regenerate the embedded index for an existing COTTAS file: + +```bash +vcf-rdfizer \ + --mode index \ + --cottas ./results/sample/sample.cottas \ + --out ./results +``` + +`--mode index` is deliberately an in-place maintenance operation. It mounts +only the directory containing the selected artifact and writes metrics under +`/run_metrics//index_metrics.json`. HDT indexing creates a +versioned sidecar beside the input. COTTAS indexing rewrites the existing +`.cottas` file through `pycottas.cat` with one input file, keeping the data in +the same artifact while rebuilding its embedded index. If the operation fails, +the original COTTAS file is left in place; HDT's previous sidecars are restored. + ## Output Layout Given `--out ./results`: @@ -359,8 +384,10 @@ For an existing RDF input named `test-larger.nt` or `test-larger.nt.gz`, The same basename rule applies in full mode. VCF-RDFizer performs an output collision check before Docker or conversion starts and never overwrites a -planned artifact. Choose a new `--out` directory, or rename/remove the -conflicting output before rerunning. +planned pipeline artifact. The exception is the deliberate `--mode index` +maintenance operation, which regenerates the selected artifact's index in +place. Choose a new `--out` directory, or rename/remove a conflicting pipeline +artifact before rerunning. Intermediates are hidden by default. Raw RDF files are removed after successful compression by default. Use @@ -376,7 +403,9 @@ For each run, VCF-RDFizer writes: - `run_metrics//metrics.csv` - `run_metrics//wrapper_execution_times.csv` - `run_metrics//progress.log` -- `run_metrics//hdt_index_metrics.json` for standalone HDT index mode +- `run_metrics//index_metrics.json` for standalone HDT/COTTAS index mode +- `run_metrics//_index_metrics.json` is also written for the + selected format (`hdt` or `cottas`) for compatibility/discovery Compression metrics now include per-method: @@ -438,6 +467,9 @@ HDT Java 3.0.10 does not provide a standalone `hdtGenerateIndex` executable. VCF-RDFizer sends an `exit` command to the supported `hdtSearch.sh` launcher; this opens the HDT through `mapIndexedHDT()` without executing a data query and creates the versioned `.hdt.index.v1-1` sidecar before the run is marked successful. +For standalone index mode, any existing versioned sidecars are moved aside +while regeneration runs and restored if indexing fails, so rerunning the mode +really rebuilds the index without leaving a partially written replacement. The helper explicitly selects HDT Java's external-sort disk indexer rather than the launcher's heap-based default (whose launcher heap is only 1 GiB). Temporary sort runs and disk-backed sequences are kept under `/work` and removed when the @@ -446,6 +478,14 @@ when invoking the helper directly. For the pinned HDT Java 3.0.10 distribution, this is the HDT v1-1 sidecar `.hdt.index.v1-1`; VCF-RDFizer reports the actual path in its metrics. +COTTAS does not expose a separate index sidecar. Its index is part of the +Parquet artifact and is selected when the artifact is written. Standalone +COTTAS index mode uses `pycottas.cat` to write a new temporary COTTAS file from +the existing one with the default `spo` index, then atomically replaces the +original. This is still index-only from the pipeline's point of view: it does +not rerun VCF-to-RDF conversion or create an RDF output, but it may require +temporary disk space and time comparable to rewriting the COTTAS file. + The record-safe chunk plan and per-stage timings are retained in the raw partitioned-compression metrics JSON for diagnostics. The temporary chunk files and guide are not retained as host files. diff --git a/src/cottas_tool.py b/src/cottas_tool.py index c8ddc23..97ddf90 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -42,6 +42,13 @@ def main() -> int: merge.add_argument("cottas_path") merge.add_argument("index", nargs="?", default="spo") + reindex = subparsers.add_parser( + "reindex", + help="rebuild the embedded COTTAS query index in place", + ) + reindex.add_argument("cottas_path") + reindex.add_argument("index", nargs="?", default="spo") + decompress = subparsers.add_parser("decompress", help="convert COTTAS to RDF") decompress.add_argument("cottas_path") decompress.add_argument("rdf_path") @@ -76,6 +83,41 @@ def main() -> int: pycottas.cottas2rdf(cottas_path, rdf_path) return 0 + if args.command == "reindex": + cottas_path = Path(args.cottas_path).resolve() + if not cottas_path.is_file(): + print(f"COTTAS file not found: {cottas_path}", file=sys.stderr) + return 2 + + # COTTAS indexes are part of the Parquet artifact rather than sibling + # files. Rebuild into a temporary file in the same directory, then + # replace the original only after pycottas has completed successfully. + temporary_path = None + try: + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{cottas_path.name}.reindex-", + suffix=".cottas", + dir=str(cottas_path.parent), + ) + os.close(file_descriptor) + temporary_path = Path(temporary_name) + temporary_path.unlink() + with cottas_scratch_workspace(): + pycottas.cat( + [str(cottas_path)], + str(temporary_path), + index=args.index, + remove_input_files=False, + ) + if not temporary_path.is_file() or temporary_path.stat().st_size == 0: + raise RuntimeError("pycottas did not create a non-empty reindexed file") + os.replace(temporary_path, cottas_path) + temporary_path = None + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + return 0 + left_path = str(Path(args.left_path).resolve()) right_path = str(Path(args.right_path).resolve()) cottas_path = str(Path(args.cottas_path).resolve()) diff --git a/src/ensure_hdt_index.sh b/src/ensure_hdt_index.sh index 06beffc..eff979a 100644 --- a/src/ensure_hdt_index.sh +++ b/src/ensure_hdt_index.sh @@ -32,7 +32,16 @@ if [[ ! -d "$HDT_INDEX_WORK_ROOT" || ! -w "$HDT_INDEX_WORK_ROOT" ]]; then fi INDEX_WORK_DIR=$(mktemp -d "$HDT_INDEX_WORK_ROOT/vcf-rdfizer-hdt-index.XXXXXXXX") +INDEX_BACKUP_DIR="$INDEX_WORK_DIR/existing-indexes" +mkdir -p "$INDEX_BACKUP_DIR" +INDEX_REGEN_COMPLETE=0 cleanup() { + if [[ "$INDEX_REGEN_COMPLETE" -eq 0 ]]; then + shopt -s nullglob + for backup in "$INDEX_BACKUP_DIR"/*; do + mv -- "$backup" "$(dirname "$HDT_PATH")/$(basename "$backup")" + done + fi rm -rf -- "$INDEX_WORK_DIR" } trap cleanup EXIT @@ -40,6 +49,16 @@ trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM +# `mapIndexedHDT()` reuses an existing sidecar when one is present. Move all +# versioned sidecars out of the way so this command really regenerates the +# index. If indexing fails, the EXIT trap restores the previous sidecars. +shopt -s nullglob +for existing_index in "${HDT_PATH}".index.*; do + if [[ -e "$existing_index" ]]; then + mv -- "$existing_index" "$INDEX_BACKUP_DIR/$(basename "$existing_index")" + fi +done + HDT_INDEX_OPTIONS="bitmaptriples.indexmethod=disk;bitmaptriples.sequence.disk=true;bitmaptriples.sequence.disk.subindex=true;bitmaptriples.sequence.disk.location=$INDEX_WORK_DIR" # hdtSearch uses HDT Java's mapIndexedHDT(), which creates the sibling index @@ -63,4 +82,5 @@ if [[ -z "$INDEX_PATH" ]]; then exit 1 fi +INDEX_REGEN_COMPLETE=1 echo "HDT index ready: $INDEX_PATH" diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index 293ab0d..bc78145 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -139,3 +139,68 @@ def fake_cottas2rdf(cottas_path, rdf_path): self.assertEqual(output.read_text(), "

.\n") self.assertEqual(len(observed_workspaces), 1) self.assertFalse(any(scratch_root.iterdir())) + + def test_reindex_rewrites_atomically_without_removing_input(self): + """Reindex uses one-file cat and replaces the source only on success.""" + module = load_cottas_tool() + calls = [] + + def fake_cat(paths, cottas_path, *, index, remove_input_files): + calls.append((paths, cottas_path, index, remove_input_files)) + self.assertNotEqual(Path(cottas_path), source) + self.assertTrue(Path(cottas_path).name.startswith(f".{source.name}.reindex-")) + Path(cottas_path).write_text("reindexed COTTAS\n") + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = root / "scratch" + source = root / "input.cottas" + source.write_text("original COTTAS\n") + + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace(cat=fake_cat)}, + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "reindex", str(source)], + ): + self.assertEqual(module.main(), 0) + + self.assertEqual(source.read_text(), "reindexed COTTAS\n") + self.assertEqual( + calls, + [([str(source.resolve())], mock.ANY, "spo", False)], + ) + self.assertEqual(list(root.glob(".input.cottas.reindex-*.cottas")), []) + + def test_reindex_keeps_original_when_pycottas_fails(self): + """A failed COTTAS rebuild does not replace the existing artifact.""" + module = load_cottas_tool() + + def failing_cat(*args, **kwargs): + raise RuntimeError("simulated reindex failure") + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = root / "scratch" + source = root / "input.cottas" + source.write_text("original COTTAS\n") + + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace(cat=failing_cat)}, + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "reindex", str(source)], + ): + with self.assertRaisesRegex(RuntimeError, "simulated reindex failure"): + module.main() + + self.assertEqual(source.read_text(), "original COTTAS\n") + self.assertEqual(list(root.glob(".input.cottas.reindex-*.cottas")), []) diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 1b3689d..adb56f2 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -1423,6 +1423,7 @@ def test_help_flag_prints_usage_guide(self): self.assertIn("--rdf-compression", text) self.assertIn("--representations", text) self.assertIn("--artifact-compression", text) + self.assertIn("--cottas", text) self.assertNotIn("--keep-rdf", text) self.assertNotIn("--compression", text) @@ -2308,6 +2309,120 @@ def fake_run(cmd, cwd=None, env=None): (tmp_path / "sample.hdt.index.v1-1").resolve(), ) + def test_main_index_mode_regenerates_existing_hdt_sidecar(self): + """Index mode permits an existing HDT sidecar and invokes regeneration.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + hdt_path = tmp_path / "sample.hdt" + hdt_path.write_bytes(b"fake-hdt") + existing_index = tmp_path / "sample.hdt.index.v1-1" + existing_index.write_text("old-index\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + rendered = str(cmd[-1]) + if "ensure_hdt_index.sh" in rendered: + existing_index.write_text("new-index\n") + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ): + rc = invoke_main( + [ + "--mode", + "index", + "--hdt", + str(hdt_path), + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(existing_index.read_text(), "new-index\n") + payload = json.loads( + (latest_metrics_run_dir(out_dir / "run_metrics") / "index_metrics.json").read_text() + ) + self.assertEqual(payload["index_status"], "regenerated") + + def test_main_index_mode_reindexes_existing_cottas(self): + """Index mode invokes the COTTAS adapter and records embedded-index metadata.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + cottas_path = tmp_path / "sample.cottas" + cottas_path.write_text("old-cottas\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + rendered = str(cmd[-1]) + if "cottas_tool.py" in rendered and " reindex " in rendered: + cottas_path.write_text("reindexed-cottas\n") + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ): + rc = invoke_main( + [ + "--mode", + "index", + "--cottas", + str(cottas_path), + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(cottas_path.read_text(), "reindexed-cottas\n") + self.assertEqual(len(commands), 1) + self.assertIn("cottas_tool.py reindex", commands[0][-1]) + self.assertTrue(any(arg.endswith(":/data/cottas") for arg in commands[0])) + payload = json.loads( + (latest_metrics_run_dir(out_dir / "run_metrics") / "index_metrics.json").read_text() + ) + self.assertEqual(payload["index_format"], "cottas") + self.assertEqual(payload["index_location"], "embedded") + self.assertEqual(payload["index_status"], "regenerated") + self.assertTrue( + (latest_metrics_run_dir(out_dir / "run_metrics") / "cottas_index_metrics.json").exists() + ) + + def test_main_index_mode_requires_exactly_one_index_input(self): + """Index mode rejects missing or ambiguous HDT/COTTAS inputs.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + hdt_path = tmp_path / "sample.hdt" + cottas_path = tmp_path / "sample.cottas" + hdt_path.write_bytes(b"hdt") + cottas_path.write_bytes(b"cottas") + + for options in ( + [], + ["--hdt", str(hdt_path), "--cottas", str(cottas_path)], + ): + rc = invoke_main(["--mode", "index", *options, "--out", str(tmp_path / "out")]) + self.assertEqual(rc, 2) + def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): """The helper uses HDT Java's bounded-memory disk indexer.""" with tempfile.TemporaryDirectory() as td: @@ -2331,6 +2446,8 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): search.chmod(0o755) hdt_path = tmp_path / "sample.hdt" hdt_path.write_bytes(b"fake-hdt") + existing_index = Path(str(hdt_path) + ".index.v1-1") + existing_index.write_text("old-index\n", encoding="utf-8") helper = Path(__file__).parents[1] / "src" / "ensure_hdt_index.sh" result = subprocess.run( @@ -2347,6 +2464,7 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): self.assertEqual(result.returncode, 0, msg=result.stderr) self.assertTrue(Path(str(hdt_path) + ".index.v1-1").exists()) + self.assertEqual(existing_index.read_text(encoding="utf-8"), "index\n") self.assertIn(f"HDT index ready: {hdt_path}.index.v1-1", result.stdout) invocation = invocation_path.read_text(encoding="utf-8").splitlines() self.assertEqual(invocation[0:2], ["-quiet", "-options"]) @@ -2357,6 +2475,45 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): self.assertEqual(invocation[3], str(hdt_path)) self.assertEqual(list(work_root.iterdir()), []) + def test_hdt_index_helper_restores_existing_sidecar_on_failure(self): + """Failed HDT regeneration restores the sidecar that was moved aside.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + java_home = tmp_path / "java-home" + bin_dir = java_home / "bin" + bin_dir.mkdir(parents=True) + work_root = tmp_path / "index-work" + work_root.mkdir() + search = bin_dir / "hdtSearch.sh" + search.write_text( + "#!/usr/bin/env bash\n" + "read -r command\n" + "[[ \"$command\" == \"exit\" ]] || exit 3\n" + "exit 7\n", + encoding="utf-8", + ) + search.chmod(0o755) + hdt_path = tmp_path / "sample.hdt" + hdt_path.write_bytes(b"fake-hdt") + existing_index = Path(str(hdt_path) + ".index.v1-1") + existing_index.write_text("old-index\n", encoding="utf-8") + helper = Path(__file__).parents[1] / "src" / "ensure_hdt_index.sh" + + result = subprocess.run( + ["bash", str(helper), str(hdt_path)], + env={ + **os.environ, + "HDT_JAVA_HOME": str(java_home), + "HDT_INDEX_WORK_ROOT": str(work_root), + }, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 7, msg=result.stderr) + self.assertEqual(existing_index.read_text(encoding="utf-8"), "old-index\n") + self.assertEqual(list(work_root.iterdir()), []) + def test_main_decompress_mode_rejects_unknown_extension(self): """Decompression mode rejects unsupported compressed RDF extensions.""" with tempfile.TemporaryDirectory() as td: diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index e7bc1c5..a72ba84 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -6,7 +6,8 @@ 2) convert VCF -> TSV 3) run RMLStreamer conversion 4) run selected compression/decompression operations -5) persist run and compression metrics +5) optionally regenerate the query index for an existing HDT or COTTAS file +6) persist run and compression metrics The implementation is intentionally split into small helpers so failures can be diagnosed at a specific stage and future workflow changes stay localized. @@ -185,6 +186,7 @@ DEFAULT_CHUNK_MIN_BYTES = 128 * 1024 * 1024 DEFAULT_CHUNK_MAX_BYTES = 1024 * 1024 * 1024 HDT_INDEX_HELPER_CONTAINER = "/opt/vcf-rdfizer/ensure_hdt_index.sh" +COTTAS_TOOL_CONTAINER = "/opt/vcf-rdfizer/cottas_tool.py" PARTITIONED_COMPRESSION_RUNNER_CONTAINER = "/opt/vcf-rdfizer/partitioned_compression.py" SAMPLE_CALLS_HEADER = [ "SOURCE_FILE", @@ -4449,27 +4451,47 @@ def default_decompressed_name(path: Path, fmt: str): return f"{path.stem}.nt" -def run_hdt_index_mode( +def run_index_mode( *, - hdt_path: Path, + index_path: Path, + index_format: str, metrics_dir: Path, image_ref: str, wrapper_log_path: Path, ): - """Eagerly create the HDT Java index beside an existing HDT file.""" - print("Step 3/3: Initializing HDT index") + """Generate or regenerate the query index for one existing artifact. + + HDT writes its index as a versioned sibling sidecar. COTTAS stores its + query index in the Parquet artifact itself, so the Docker-side adapter + rewrites that file atomically with the requested index order. + """ + format_label = index_format.upper() + print(f"Step 3/3: Regenerating {format_label} index") ensure_dir(metrics_dir) - existing_index_path = find_hdt_index_sidecar(hdt_path) - index_existed = existing_index_path is not None - source_container = f"/data/hdt/{hdt_path.name}" - command = ( - "set -euo pipefail; " - f"{shlex.quote(HDT_INDEX_HELPER_CONTAINER)} {shlex.quote(source_container)}" + + existing_index_path = ( + find_hdt_index_sidecar(index_path) if index_format == "hdt" else None ) + mount_name = "hdt" if index_format == "hdt" else "cottas" + source_container = f"/data/{mount_name}/{index_path.name}" + if index_format == "hdt": + command = ( + "set -euo pipefail; " + f"{shlex.quote(HDT_INDEX_HELPER_CONTAINER)} {shlex.quote(source_container)}" + ) + else: + command = ( + "set -euo pipefail; " + 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}"; ' + 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ' + 'echo "Missing pycottas Python executable in container" >&2; exit 127; fi; ' + f'"$PYTHON_BIN" {shlex.quote(COTTAS_TOOL_CONTAINER)} reindex ' + f"{shlex.quote(source_container)} spo" + ) cmd = [ *docker_run_base(), "-v", - f"{str(hdt_path.parent)}:/data/hdt", + f"{str(index_path.parent)}:/data/{mount_name}", image_ref, "bash", "-lc", @@ -4479,33 +4501,71 @@ def run_hdt_index_mode( started = time.perf_counter() exit_code = run(cmd) elapsed = time.perf_counter() - started - index_path = find_hdt_index_sidecar(hdt_path) - index_ready = index_path is not None + index_path_after = ( + find_hdt_index_sidecar(index_path) + if index_format == "hdt" + else (index_path if file_size_bytes(index_path) else None) + ) + index_ready = index_path_after is not None final_code = int(exit_code) if int(exit_code) != 0 else (0 if index_ready else 1) + index_was_present = existing_index_path is not None or index_format == "cottas" payload = { - "hdt_path": str(hdt_path), - "index_path": str(index_path) if index_path else "", + "index_format": index_format, + "input_path": str(index_path), + "index_path": str(index_path_after) if index_path_after else "", + "index_location": "sidecar" if index_format == "hdt" else "embedded", "exit_code": final_code, "wall_seconds": elapsed, "index_status": ( - "existing" if index_existed else "generated" + "regenerated" if index_was_present else "generated" ) if index_ready else "failed", - "index_size_bytes": file_size_bytes(index_path) if index_path else 0, + "index_size_bytes": file_size_bytes(index_path_after) if index_path_after else 0, } - (metrics_dir / "hdt_index_metrics.json").write_text( + if index_format == "hdt": + # Preserve the field used by the original HDT-only metrics payload. + payload["hdt_path"] = str(index_path) + else: + payload["cottas_path"] = str(index_path) + metrics_path = metrics_dir / "index_metrics.json" + metrics_path.write_text( json.dumps(payload, indent=2) + "\n", encoding="utf-8", ) + # Keep a format-specific metrics filename for discovery/compatibility, + # while the generic filename is used for both supported formats. + legacy_metrics_path = metrics_dir / f"{index_format}_index_metrics.json" + if legacy_metrics_path != metrics_path: + legacy_metrics_path.write_text( + json.dumps(payload, indent=2) + "\n", + encoding="utf-8", + ) if final_code != 0: - eprint(f"Error: HDT index initialization failed. See log: {wrapper_log_path}") + eprint(f"Error: {format_label} index regeneration failed. See log: {wrapper_log_path}") return 1 - print(f"HDT index ready: {index_path}") - print(f"HDT index metrics: {metrics_dir / 'hdt_index_metrics.json'}") + print(f"{format_label} index ready: {index_path_after}") + print(f"Index metrics: {metrics_path}") return 0 +def run_hdt_index_mode( + *, + hdt_path: Path, + metrics_dir: Path, + image_ref: str, + wrapper_log_path: Path, +): + """Backward-compatible wrapper for the HDT-only index helper.""" + return run_index_mode( + index_path=hdt_path, + index_format="hdt", + metrics_dir=metrics_dir, + image_ref=image_ref, + wrapper_log_path=wrapper_log_path, + ) + + def run_decompress_mode( *, compressed_path: Path, @@ -4634,8 +4694,10 @@ def main(): "--rdf-compression gzip --representations hdt --artifact-compression gzip -o ./results\n" " Decompression-only:\n" " vcf_rdfizer.py -m decompress -C ./results/out/sample/sample.nt.gz -o ./results\n" - " Initialize an existing HDT index:\n" + " Generate or regenerate an index for an existing HDT:\n" " vcf_rdfizer.py -m index -H ./results/sample/sample.hdt -o ./results\n" + " Generate or regenerate an index for an existing COTTAS file:\n" + " vcf_rdfizer.py -m index --cottas ./results/sample/sample.cottas -o ./results\n" ), ) parser.add_argument( @@ -4643,7 +4705,7 @@ def main(): "--mode", choices=["full", "compress", "decompress", "tsv", "index"], default="full", - help="Run mode: full pipeline, TSV benchmark, compression, decompression, or HDT index initialization", + help="Run mode: full pipeline, TSV benchmark, compression, decompression, or index-only regeneration", ) parser.add_argument( "-i", @@ -4666,7 +4728,12 @@ def main(): "-H", "--hdt", default=None, - help="Existing HDT file for --mode index; HDT Java creates a versioned .hdt.index.* sidecar beside it", + help="Existing .hdt file for --mode index; creates or regenerates its versioned .hdt.index.* sidecar", + ) + parser.add_argument( + "--cottas", + default=None, + help="Existing .cottas file for --mode index; rebuilds its embedded query index in place", ) parser.add_argument( "-d", @@ -4987,22 +5054,25 @@ def main(): elif mode == "index": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") - if not args.hdt: - raise ValueError("--hdt is required in --mode index") - hdt_path = Path(args.hdt).expanduser().resolve() - if not hdt_path.exists() or not hdt_path.is_file(): - raise ValueError(f"HDT input file not found: {hdt_path}") - if hdt_path.suffix != ".hdt": - raise ValueError("HDT index input must end with .hdt") - validate_mode_dirs([out_root, out_dir, metrics_root]) - existing_indexes = sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")) - if existing_indexes: + if bool(args.hdt) == bool(args.cottas): raise ValueError( - "Refusing to overwrite existing output file(s): " - + ", ".join(str(path) for path in existing_indexes) - + ". VCF-RDFizer does not overwrite outputs; rename/remove the " - "existing index and try again." + "provide exactly one of --hdt or --cottas in --mode index" ) + if args.hdt: + index_format = "hdt" + index_path = Path(args.hdt).expanduser().resolve() + if index_path.suffix != ".hdt": + raise ValueError("HDT index input must end with .hdt") + else: + index_format = "cottas" + index_path = Path(args.cottas).expanduser().resolve() + if index_path.suffix != ".cottas": + raise ValueError("COTTAS index input must end with .cottas") + if not index_path.exists() or not index_path.is_file(): + raise ValueError( + f"{index_format.upper()} input file not found: {index_path}" + ) + validate_mode_dirs([out_root, out_dir, metrics_root]) else: if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -5134,7 +5204,7 @@ def execute_mode(): elif mode == "index": metrics_write_target = metrics_dir if metrics_dir.exists() else metrics_dir.parent writable_targets = [ - (hdt_path.parent, True), + (index_path.parent, True), (metrics_write_target, True), ] else: @@ -5225,8 +5295,9 @@ def execute_mode(): wrapper_log_path=wrapper_log_path, ) if mode == "index": - return run_hdt_index_mode( - hdt_path=hdt_path, + return run_index_mode( + index_path=index_path, + index_format=index_format, metrics_dir=metrics_dir, image_ref=image_ref, wrapper_log_path=wrapper_log_path, From 950614114b5e7df2179c7f65c2ce3ae948006e26 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 27 Aug 2026 11:07:04 +0200 Subject: [PATCH 03/19] fix full mode fail on hdt-index failure --- README.md | 19 ++ src/partitioned_compression.py | 158 +++++++++++++--- test/test_vcf_rdfizer_unit.py | 193 +++++++++++++++++++ vcf_rdfizer.py | 326 +++++++++++++++++++++++++++------ 4 files changed, 616 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 9bb638f..bf68ffc 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,15 @@ versioned sidecar beside the input. COTTAS indexing rewrites the existing the same artifact while rebuilding its embedded index. If the operation fails, the original COTTAS file is left in place; HDT's previous sidecars are restored. +In `full` mode, an HDT sidecar-index failure is non-fatal when the HDT data +itself remains readable; the run continues and the HDT can be repaired later +with the standalone command above. If COTTAS generation/indexing cannot +produce a usable artifact, COTTAS-specific outputs are skipped while the rest +of the full pipeline continues. These warnings are printed in the run output +and written to `run_metrics//index_warnings.json`. The raw RDF is +retained when a representation-dependent output was unavailable so the +standalone index command or a later rerun has a recoverable source. + ## Output Layout Given `--out ./results`: @@ -403,6 +412,8 @@ For each run, VCF-RDFizer writes: - `run_metrics//metrics.csv` - `run_metrics//wrapper_execution_times.csv` - `run_metrics//progress.log` +- `run_metrics//index_warnings.json` when full-run HDT/COTTAS index + generation was unsuccessful but the pipeline continued - `run_metrics//index_metrics.json` for standalone HDT/COTTAS index mode - `run_metrics//_index_metrics.json` is also written for the selected format (`hdt` or `cottas`) for compatibility/discovery @@ -424,6 +435,14 @@ Compression fails closed if the artifact cannot be decoded or the counts do not match. In compression-only mode, the source count is obtained by a streaming fallback when no upstream conversion metrics are available. +For full runs, a readable HDT whose sidecar index could not be created is +validated with the index check skipped, marked with `index_status: "failed"`, +and reported in `index_warnings.json`; this allows packaging and later stages +to continue. COTTAS failures are reported the same way, but dependent COTTAS +artifacts are marked as not generated because the COTTAS file itself is not +usable. Explicit standalone `--mode index` runs remain strict and return a +failure status when regeneration fails. + For partitioned HDT/COTTAS runs, the final method metric reports one sample-level result, while raw metrics also include a sample-scoped `__partitioned_compression__` artifact describing chunk conversion, merge diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index ff7079a..896e4c2 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -48,6 +48,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--max-chunk-bytes", required=True, type=int) parser.add_argument("--expected-triples", type=int) parser.add_argument("--result-path", required=True) + parser.add_argument( + "--allow-index-failures", + action="store_true", + help="continue with other representations when HDT/COTTAS indexing fails", + ) return parser.parse_args() @@ -303,9 +308,48 @@ def main() -> int: chunk_dir = work_dir / "rdf_chunks" runner = StageRunner(work_dir) results: dict[str, dict] = {} + index_warnings: list[dict] = [] hdt_total = {"exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False} cottas_total = {"exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False} + output_hdt = output_dir / f"{args.output_name}.hdt" + output_cottas = output_dir / f"{args.output_name}.cottas" + cottas_failed = False + cottas_warning = None + + def record_index_warning(index_format: str, stage: str, artifact: Path, message: str) -> dict: + warning = { + "format": index_format, + "stage": stage, + "status": "index_unavailable", + "artifact_path": str(artifact), + "message": " ".join(str(message).split()), + } + index_warnings.append(warning) + print( + f"Warning: {index_format.upper()} index generation failed for '{artifact}'; " + f"continuing with the remaining pipeline. {warning['message']}", + file=sys.stderr, + ) + return warning + + def skipped_cottas_result(method: str) -> dict: + artifact = { + "cottas": output_cottas, + "cottas_gzip": output_dir / f"{args.output_name}.cottas.gz", + "cottas_brotli": output_dir / f"{args.output_name}.cottas.br", + }[method] + return { + "exit_code": 1, + "wall_seconds": 0.0, + "user_seconds": 0.0, + "sys_seconds": 0.0, + "max_rss_kb": 0, + "output_path": str(artifact), + "output_size_bytes": 0, + "source": "index_unavailable", + "details": {"index_status": "failed", "index_warning": cottas_warning}, + } try: if not source.is_file(): @@ -416,6 +460,9 @@ def validate_artifact( raise RuntimeError("partitioned HDT chunk conversion failed") hdt_paths.append(chunk_hdt) if any(method in COTTAS_METHODS for method in methods): + if cottas_failed: + chunk.unlink(missing_ok=True) + continue chunk_cottas = work_dir / f"chunk-{index:05d}.cottas" stage = runner.run( f"cottas-build-{index:05d}", @@ -424,11 +471,22 @@ def validate_artifact( ) add_totals(cottas_total, stage) if stage["exit_code"] != 0: - raise RuntimeError("partitioned COTTAS chunk conversion failed") + if not args.allow_index_failures: + raise RuntimeError("partitioned COTTAS chunk conversion failed") + cottas_failed = True + cottas_total["exit_code"] = 0 + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + "partitioned COTTAS conversion/index creation failed for a chunk", + ) + chunk_cottas.unlink(missing_ok=True) + chunk.unlink(missing_ok=True) + continue cottas_paths.append(chunk_cottas) chunk.unlink(missing_ok=True) - output_hdt = output_dir / f"{args.output_name}.hdt" if hdt_paths: final_hdt, hdt_rounds = merge_pairwise( hdt_paths, @@ -454,7 +512,17 @@ def validate_artifact( index_stage["output_size_bytes"] = output_index.stat().st_size runner.stages[-1].update(index_stage) if index_stage["exit_code"] != 0 or output_index is None: - raise RuntimeError("final HDT index initialization failed") + if not args.allow_index_failures: + raise RuntimeError("final HDT index initialization failed") + hdt_total["exit_code"] = 0 + hdt_index_warning = record_index_warning( + "hdt", + "hdt-index", + output_hdt, + "the HDT artifact remains available, but its query index was not created", + ) + else: + hdt_index_warning = None hdt_validation = validate_artifact( name="hdt-validate", artifact=output_hdt, @@ -471,13 +539,14 @@ def validate_artifact( **plan, "merge_rounds": hdt_rounds, "index_path": str(output_index), - "index_size_bytes": output_index.stat().st_size, + "index_size_bytes": output_index.stat().st_size if output_index else 0, + "index_status": "failed" if hdt_index_warning else "ready", + "index_warning": hdt_index_warning, "validation": hdt_validation, }, } - output_cottas = output_dir / f"{args.output_name}.cottas" - if cottas_paths: + if cottas_paths and not cottas_failed: final_cottas, cottas_rounds = merge_pairwise( cottas_paths, prefix="cottas", @@ -486,27 +555,54 @@ def validate_artifact( total=cottas_total, ) if final_cottas is None: - raise RuntimeError("COTTAS merge failed") - shutil.copyfile(final_cottas, output_cottas) - final_cottas.unlink(missing_ok=True) - cottas_validation = validate_artifact( - name="cottas-validate", - artifact=output_cottas, - artifact_format="cottas", - python_bin=cottas_python, - ) - results["cottas"] = { - **finalize_totals(cottas_total), - "output_path": str(output_cottas), - "output_size_bytes": output_cottas.stat().st_size, - "source": "partitioned_generated", - "details": { - **plan, - "merge_rounds": cottas_rounds, - "index": "spo", - "validation": cottas_validation, - }, - } + if not args.allow_index_failures: + raise RuntimeError("COTTAS merge failed") + cottas_failed = True + cottas_total["exit_code"] = 0 + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + "COTTAS merge/index creation failed", + ) + else: + shutil.copyfile(final_cottas, output_cottas) + final_cottas.unlink(missing_ok=True) + try: + cottas_validation = validate_artifact( + name="cottas-validate", + artifact=output_cottas, + artifact_format="cottas", + python_bin=cottas_python, + ) + except RuntimeError as exc: + if not args.allow_index_failures: + raise + cottas_failed = True + cottas_total["exit_code"] = 0 + output_cottas.unlink(missing_ok=True) + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + str(exc), + ) + if not cottas_failed: + results["cottas"] = { + **finalize_totals(cottas_total), + "output_path": str(output_cottas), + "output_size_bytes": output_cottas.stat().st_size, + "source": "partitioned_generated", + "details": { + **plan, + "merge_rounds": cottas_rounds, + "index": "spo", + "validation": cottas_validation, + }, + } + + if any(method in COTTAS_METHODS for method in methods) and cottas_failed: + results["cottas"] = skipped_cottas_result("cottas") for method in methods: if method == "hdt_gzip": @@ -516,9 +612,15 @@ def validate_artifact( artifact = output_dir / f"{args.output_name}.hdt.br" stage = runner.run("hdt-brotli", ["brotli", "-q", "7", "-c", str(output_hdt)], artifact, artifact) elif method == "cottas_gzip": + if cottas_failed: + results[method] = skipped_cottas_result(method) + continue artifact = output_dir / f"{args.output_name}.cottas.gz" stage = runner.run("cottas-gzip", ["gzip", "-c", str(output_cottas)], artifact, artifact) elif method == "cottas_brotli": + if cottas_failed: + results[method] = skipped_cottas_result(method) + continue artifact = output_dir / f"{args.output_name}.cottas.br" stage = runner.run("cottas-brotli", ["brotli", "-q", "7", "-c", str(output_cottas)], artifact, artifact) else: @@ -528,7 +630,7 @@ def validate_artifact( results[method] = stage result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text(json.dumps({"exit_code": 0, "methods": results, "stages": runner.stages}, indent=2) + "\n", encoding="utf-8") + result_path.write_text(json.dumps({"exit_code": 0, "methods": results, "stages": runner.stages, "index_warnings": index_warnings}, indent=2) + "\n", encoding="utf-8") return 0 except Exception as exc: result_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index adb56f2..ea3c803 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -851,6 +851,117 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("cottas_gzip", payload["methods"]) self.assertIn("cottas_brotli", payload["methods"]) + def test_full_compression_recovers_from_hdt_index_failure(self): + """A readable HDT remains packageable when only index creation fails.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir = tmp_path / "input" + out_dir = tmp_path / "out" + target_dir = out_dir / "sample" + input_dir.mkdir() + target_dir.mkdir(parents=True) + rdf_path = input_dir / "sample.nt" + rdf_path.write_text("

.\n") + hdt_path = target_dir / "sample.hdt" + hdt_path.write_text("readable-hdt\n") + warnings = [] + validation_calls = [] + + def fake_run(cmd, cwd=None, env=None): + rendered = str(cmd[-1]) if cmd else "" + if "validate_compression.py" not in rendered: + if "gzip -c /data/out/sample/sample.hdt" in rendered: + (target_dir / "sample.hdt.gz").write_text("packaged-hdt\n") + return 0 + validation_calls.append(rendered) + result_match = re.search( + r"--result-path\s+['\"]?(/data/out/[^\s'\";]+)", + rendered, + ) + result_path = out_dir / result_match.group(1).replace("/data/out/", "", 1) + result_path.parent.mkdir(parents=True, exist_ok=True) + if "--skip-index-check" in rendered: + result_path.write_text( + json.dumps( + { + "valid": True, + "source_triples": 1, + "decoded_triples": 1, + "count_match": True, + } + ) + ) + return 0 + result_path.write_text( + json.dumps( + { + "valid": False, + "source_triples": 1, + "decoded_triples": 1, + "count_match": False, + "error": "HDT index/readability check failed", + } + ) + ) + return 1 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + target_out_dir=target_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["hdt", "hdt_gzip"], + wrapper_log_path=tmp_path / "wrapper.log", + status_indent=None, + index_warnings=warnings, + ) + + self.assertTrue(ok) + self.assertEqual(len(validation_calls), 2) + self.assertEqual(method_results["hdt"]["index_status"], "failed") + self.assertEqual(method_results["hdt_gzip"]["exit_code"], 0) + self.assertEqual(warnings[0]["format"], "hdt") + + def test_full_compression_continues_after_cottas_index_failure(self): + """COTTAS failure is recorded while independent raw RDF compression continues.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir = tmp_path / "input" + out_dir = tmp_path / "out" + target_dir = out_dir / "sample" + input_dir.mkdir() + target_dir.mkdir(parents=True) + rdf_path = input_dir / "sample.nt" + rdf_path.write_text("

.\n") + warnings = [] + + def fake_run(cmd, cwd=None, env=None): + rendered = str(cmd[-1]) if cmd else "" + if "cottas_tool.py convert" in rendered: + return 1 + if "gzip -c /data/in/sample.nt" in rendered: + (target_dir / "sample.nt.gz").write_text("gzip\n") + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + target_out_dir=target_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["cottas", "gzip"], + wrapper_log_path=tmp_path / "wrapper.log", + status_indent=None, + index_warnings=warnings, + ) + + self.assertTrue(ok) + self.assertEqual(method_results["cottas"]["exit_code"], 1) + self.assertEqual(method_results["gzip"]["exit_code"], 0) + self.assertEqual(warnings[0]["format"], "cottas") + self.assertTrue((target_dir / "sample.nt.gz").exists()) + def test_compression_validation_mismatch_fails_before_success(self): """A decoded triple-count mismatch fails compression and preserves RDF input.""" with tempfile.TemporaryDirectory() as td: @@ -2695,9 +2806,91 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(len(runner_commands), 1) self.assertIn("target=/work", " ".join(runner_commands[0])) self.assertIn("--methods hdt", " ".join(runner_commands[0])) + self.assertIn("--allow-index-failures", runner_commands[0]) self.assertFalse((out_dir / "sample" / ".compression_partitioned").exists()) self.assertTrue((out_dir / "sample" / "sample.hdt").exists()) + def test_main_full_mode_records_cottas_index_warning_and_continues(self): + """Full mode succeeds and writes an index-warning report for skipped COTTAS output.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + stdout = StringIO() + stderr = StringIO() + + def fake_run(cmd, cwd=None, env=None): + if "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + sample_dir = out_dir / "sample" + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "sample.nt").write_text("

.\n") + return 0 + + def fake_partitioned(**kwargs): + warning = { + "format": "cottas", + "stage": "cottas-index", + "status": "index_unavailable", + "artifact_path": str(out_dir / "sample" / "sample.cottas"), + "message": "COTTAS index creation failed", + } + kwargs["index_warnings"].append(warning) + return True, { + "cottas": { + "exit_code": 1, + "output_path": str(out_dir / "sample" / "sample.cottas"), + "output_size_bytes": 0, + "index_status": "failed", + "index_warning": warning, + } + } + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with ( + mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), + mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), + mock.patch.object(vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets()), + mock.patch.object( + vcf_rdfizer, + "run_partitioned_representation_methods_for_rdf_files", + side_effect=fake_partitioned, + ), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + rc = invoke_main( + [ + "--input", + str(input_dir), + "--rules", + str(rules_path), + "--rdf-storage-mode", + "plain", + "--rdf-compression", + "none", + "--representations", + "cottas", + "--artifact-compression", + "none", + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + warning_path = run_metrics_dir / "index_warnings.json" + self.assertTrue(warning_path.exists()) + self.assertEqual(json.loads(warning_path.read_text())["warning_count"], 1) + self.assertIn("Conversion process finished with index warnings.", stdout.getvalue()) + self.assertIn("index_warnings.json", stdout.getvalue()) + self.assertTrue((out_dir / "sample" / "sample.nt").exists()) + def test_main_full_mode_plain_storage_sets_storage_mode(self): """Plain storage passes the canonical storage mode to conversion.""" with tempfile.TemporaryDirectory() as td: diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index a72ba84..4acc2b5 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -1105,6 +1105,19 @@ def write_failed_inputs_report(*, metrics_dir: Path, failures: list[dict]): return report_path +def write_index_warnings_report(*, metrics_dir: Path, run_id: str, warnings: list[dict]): + """Write non-fatal full-run HDT/COTTAS index warnings as JSON.""" + ensure_dir(metrics_dir) + report_path = metrics_dir / "index_warnings.json" + payload = { + "run_id": run_id, + "warning_count": len(warnings), + "warnings": warnings, + } + report_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return report_path + + def print_nt_hdt_summary( *, output_root: Path, @@ -2387,6 +2400,7 @@ def write_compression_metrics_artifacts( combined_size_bytes: int, selected_methods: list[str], method_results: dict[str, dict], + index_warnings: list[dict] | None = None, ): """Write per-output compression artifacts (time files + structured JSON).""" metrics_dir.mkdir(parents=True, exist_ok=True) @@ -2443,6 +2457,7 @@ def timing_payload(result: dict): "compression_methods": ",".join(selected_methods) if selected_methods else "none", "combined_rdf_path": str(source_rdf_path), "combined_rdf_size_bytes": int(combined_size_bytes), + "index_warnings": list(index_warnings or []), "hdt_source": str(hdt_result.get("source") or "not_used"), "gzip_raw_rdf": { "output_gz_path": gzip_result.get("output_path", ""), @@ -2514,6 +2529,7 @@ def write_raw_compression_metrics_artifact( source_rdf_path: Path, selected_methods: list[str], method_results: dict[str, dict], + index_warnings: list[dict] | None = None, ): """Persist per-RDF-file compression metrics under `raw_metrics/`.""" safe_output = safe_metrics_name(output_name) @@ -2528,6 +2544,7 @@ def write_raw_compression_metrics_artifact( "rdf_name": rdf_name, "source_rdf_path": str(source_rdf_path), "compression_methods": ",".join(selected_methods) if selected_methods else "none", + "index_warnings": list(index_warnings or []), "methods": {}, } @@ -2925,6 +2942,7 @@ def run_compression_methods_for_rdf( timestamp: str | None = None, output_name: str | None = None, expected_triples: int | None = None, + index_warnings: list[dict] | None = None, ): """Run selected compression stages for a single RDF file. @@ -2959,6 +2977,7 @@ def run_compression_methods_for_rdf( target_out_container = f"/data/out/{relative_out.as_posix()}" method_results: dict[str, dict] = {} + allow_index_failure = index_warnings is not None hdt_name = f"{input_stem}.hdt" hdt_path = target_out_dir / hdt_name hdt_container = f"{target_out_container}/{hdt_name}" @@ -2968,6 +2987,7 @@ def run_compression_methods_for_rdf( cottas_path = target_out_dir / cottas_name cottas_container = f"{target_out_container}/{cottas_name}" cottas_is_ready = False + cottas_failure_warning: dict | None = None metrics_output_name = output_name or target_out_dir.name safe_output_name = safe_metrics_name(metrics_output_name) safe_rdf_name = safe_metrics_name(rdf_path.name) @@ -2978,6 +2998,7 @@ def run_container_command( artifact_name: str, command: str, record_method: bool = True, + quiet_failure: bool = False, ): """Execute one compression command in Docker and capture timing/size.""" timing_name = f".{input_stem}.{method}.time" @@ -3043,11 +3064,36 @@ def run_container_command( pass if record_method and method == "hdt": method_results[method]["source"] = "generated" - if exit_code != 0 and record_method: - eprint(f"Error: {method} compression failed. See log: {wrapper_log_path}") + if exit_code != 0: + if record_method and not quiet_failure: + eprint(f"Error: {method} compression failed. See log: {wrapper_log_path}") return False return True + def record_index_warning( + *, + index_format: str, + artifact_path: Path, + message: str, + stage: str, + ) -> dict: + """Record one recoverable representation/index problem for a full run.""" + compact = " ".join(str(message).split()) + warning = { + "format": index_format, + "stage": stage, + "status": "index_unavailable", + "artifact_path": str(artifact_path), + "message": compact, + } + if warning not in index_warnings: + index_warnings.append(warning) + eprint( + f"Warning: {index_format.upper()} index generation failed for " + f"'{artifact_path}'; continuing with the remaining pipeline. {compact}" + ) + return warning + def validate_container_artifact( *, method: str, @@ -3056,56 +3102,81 @@ def validate_container_artifact( artifact_container: str, ) -> tuple[bool, dict]: """Validate a base representation before any packaging or cleanup.""" - report_name = f".{input_stem}.{artifact_format}.validation.json" - report_path = target_out_dir / report_name - report_container = f"{target_out_container}/{report_name}" - command_parts = [ - "set -euo pipefail;", - f"rm -f {shlex.quote(report_container)};", - 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}";', - 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ', - 'echo "Missing Python executable in container" >&2; exit 127; fi;', - 'VALIDATOR="/opt/vcf-rdfizer/validate_compression.py";', - 'if [[ ! -f "$VALIDATOR" ]]; then ', - 'echo "Missing compression validator in container" >&2; exit 127; fi;', - '"$PYTHON_BIN" "$VALIDATOR"', - f"--source {shlex.quote(input_container)}", - f"--artifact {shlex.quote(artifact_container)}", - f"--format {shlex.quote(artifact_format)}", - f"--result-path {shlex.quote(report_container)}", - ] - if expected_triples is not None: - command_parts.append(f"--expected-triples {int(expected_triples)}") - command = " ".join(command_parts) - if not run_container_command( - method=f"{method}-validation", - artifact_name=report_name, - command=command, - record_method=False, - ): - report = { - "valid": False, - "count_match": False, - "error": f"{artifact_format.upper()} validation command failed", - } - else: + def perform_validation(*, skip_index_check: bool) -> tuple[bool, dict]: + report_name = f".{input_stem}.{artifact_format}.validation.json" + report_path = target_out_dir / report_name + report_container = f"{target_out_container}/{report_name}" + command_parts = [ + "set -euo pipefail;", + f"rm -f {shlex.quote(report_container)};", + 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}";', + 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ', + 'echo "Missing Python executable in container" >&2; exit 127; fi;', + 'VALIDATOR="/opt/vcf-rdfizer/validate_compression.py";', + 'if [[ ! -f "$VALIDATOR" ]]; then ', + 'echo "Missing compression validator in container" >&2; exit 127; fi;', + '"$PYTHON_BIN" "$VALIDATOR"', + f"--source {shlex.quote(input_container)}", + f"--artifact {shlex.quote(artifact_container)}", + f"--format {shlex.quote(artifact_format)}", + f"--result-path {shlex.quote(report_container)}", + ] + if expected_triples is not None: + command_parts.append(f"--expected-triples {int(expected_triples)}") + if skip_index_check: + command_parts.append("--skip-index-check") + command = " ".join(command_parts) + command_ok = run_container_command( + method=f"{method}-validation", + artifact_name=report_name, + command=command, + record_method=False, + quiet_failure=skip_index_check, + ) try: report = json.loads(report_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: report = { "valid": False, "count_match": False, - "error": f"validator did not produce a valid report: {exc}", + "error": ( + f"{artifact_format.upper()} validation command failed" + if not command_ok + else f"validator did not produce a valid report: {exc}" + ), } - report_path.unlink(missing_ok=True) - valid = bool(report.get("valid")) and bool(report.get("count_match")) - if not valid: - eprint( - f"Error: {artifact_format.upper()} validation failed for {artifact_name}: " - f"{report.get('error', 'decoded triple count mismatch')}. " - f"See log: {wrapper_log_path}" + report_path.unlink(missing_ok=True) + return ( + bool(report.get("valid")) and bool(report.get("count_match")), + report, ) - return valid, report + + valid, report = perform_validation(skip_index_check=False) + if valid: + return True, report + + if allow_index_failure and artifact_format == "hdt": + readable, readable_report = perform_validation(skip_index_check=True) + if readable: + warning = record_index_warning( + index_format="hdt", + artifact_path=target_out_dir / artifact_name, + stage="hdt-index", + message=report.get( + "error", + "HDT validation succeeded when the index check was skipped", + ), + ) + readable_report["index_status"] = "failed" + readable_report["index_warning"] = warning + return True, readable_report + + eprint( + f"Error: {artifact_format.upper()} validation failed for {artifact_name}: " + f"{report.get('error', 'decoded triple count mismatch')}. " + f"See log: {wrapper_log_path}" + ) + return False, report def ensure_hdt_available(): """Ensure `.hdt` exists for HDT-based compound methods.""" @@ -3133,8 +3204,21 @@ def ensure_hdt_available(): artifact_container=hdt_container, ) if not valid: + if allow_index_failure: + cottas_failure_warning = record_index_warning( + index_format="cottas", + artifact_path=cottas_path, + stage="cottas-index", + message=report.get( + "error", + "existing COTTAS validation/index check failed", + ), + ) return False method_results["hdt"]["validation"] = report + if report.get("index_status"): + method_results["hdt"]["index_status"] = report["index_status"] + method_results["hdt"]["index_warning"] = report.get("index_warning") hdt_is_ready = True return True hdt_command = ( @@ -3163,15 +3247,20 @@ def ensure_hdt_available(): if not valid: return False method_results["hdt"]["validation"] = report + if report.get("index_status"): + method_results["hdt"]["index_status"] = report["index_status"] + method_results["hdt"]["index_warning"] = report.get("index_warning") hdt_is_ready = True hdt_source = "generated" return True def ensure_cottas_available(): """Ensure `.cottas` exists for COTTAS packaging stages.""" - nonlocal cottas_is_ready + nonlocal cottas_is_ready, cottas_failure_warning if cottas_is_ready: return True + if cottas_failure_warning is not None: + return False if cottas_path.exists(): method_results.setdefault( "cottas", @@ -3210,7 +3299,15 @@ def ensure_cottas_available(): method="cottas", artifact_name=cottas_name, command=cottas_command, + quiet_failure=allow_index_failure, ): + if allow_index_failure: + cottas_failure_warning = record_index_warning( + index_format="cottas", + artifact_path=cottas_path, + stage="cottas-index", + message="COTTAS conversion/index creation did not produce a usable artifact", + ) return False valid, report = validate_container_artifact( method="cottas", @@ -3219,11 +3316,48 @@ def ensure_cottas_available(): artifact_container=cottas_container, ) if not valid: + if allow_index_failure: + cottas_failure_warning = record_index_warning( + index_format="cottas", + artifact_path=cottas_path, + stage="cottas-index", + message=report.get( + "error", + "COTTAS validation failed after conversion/index creation", + ), + ) return False method_results["cottas"]["validation"] = report cottas_is_ready = True return True + def mark_cottas_method_unavailable(method: str): + """Mark COTTAS and dependent packaging methods as skipped after a failure.""" + warning = cottas_failure_warning + if warning is None: + return + result = method_results.setdefault( + method, + { + "exit_code": 1, + "wall_seconds": 0.0, + "user_seconds": 0.0, + "sys_seconds": 0.0, + "max_rss_kb": 0, + "output_path": str( + { + "cottas": cottas_path, + "cottas_gzip": target_out_dir / f"{input_stem}.cottas.gz", + "cottas_brotli": target_out_dir / f"{input_stem}.cottas.br", + }.get(method, cottas_path) + ), + "output_size_bytes": 0, + }, + ) + result["exit_code"] = 1 + result["index_status"] = "failed" + result["index_warning"] = warning + for method in methods: if method == "gzip": artifact_name = f"{input_stem}.{input_ext}.gz" @@ -3279,7 +3413,10 @@ def ensure_cottas_available(): if method == "cottas": if not ensure_cottas_available(): - return False, method_results + if not allow_index_failure: + return False, method_results + mark_cottas_method_unavailable(method) + continue if status_indent is not None: suffix = " (reused existing COTTAS)" if method_results[method].get("source") == "existing" else "" print(f"{status_indent}- {method}: {cottas_name} {success_symbol()}{suffix}") @@ -3329,7 +3466,10 @@ def ensure_cottas_available(): if method == "cottas_gzip": if not ensure_cottas_available(): - return False, method_results + if not allow_index_failure: + return False, method_results + mark_cottas_method_unavailable(method) + continue artifact_name = f"{input_stem}.cottas.gz" out_container = f"{target_out_container}/{artifact_name}" command = ( @@ -3345,7 +3485,10 @@ def ensure_cottas_available(): if method == "cottas_brotli": if not ensure_cottas_available(): - return False, method_results + if not allow_index_failure: + return False, method_results + mark_cottas_method_unavailable(method) + continue artifact_name = f"{input_stem}.cottas.br" out_container = f"{target_out_container}/{artifact_name}" command = ( @@ -3369,6 +3512,7 @@ def ensure_cottas_available(): source_rdf_path=rdf_path, selected_methods=methods, method_results=method_results, + index_warnings=index_warnings, ) return True, method_results @@ -3389,6 +3533,7 @@ def run_containerized_partitioned_representation_methods( min_chunk_bytes: int, max_chunk_bytes: int, expected_triples: int | None = None, + index_warnings: list[dict] | None = None, ): """Run partitioned compression in an ephemeral Docker-managed volume. @@ -3472,6 +3617,8 @@ def run_containerized_partitioned_representation_methods( f"/data/out/{result_path.name}", ] ) + if index_warnings is not None: + command.append("--allow-index-failures") run_exit_code = run(command) payload = None if result_path.is_file(): @@ -3481,6 +3628,22 @@ def run_containerized_partitioned_representation_methods( eprint(f"Error: invalid partitioned compression result: {exc}. See log: {wrapper_log_path}") if payload is not None: + for warning in payload.get("index_warnings", []): + artifact_path = str(warning.get("artifact_path", "")) + if artifact_path.startswith("/data/out/"): + warning["artifact_path"] = str( + out_dir / artifact_path.removeprefix("/data/out/") + ) + if warning not in (index_warnings or []): + if index_warnings is not None: + index_warnings.append(warning) + eprint( + "Warning: " + f"{str(warning.get('format', 'representation')).upper()} index generation " + f"failed for '{warning.get('artifact_path', output_name)}'; " + "continuing with the remaining pipeline. " + f"{warning.get('message', '')}" + ) method_results = payload.get("methods", {}) for method, result in method_results.items(): artifact_path = out_dir / compression_artifact_name_for_method( @@ -3525,6 +3688,7 @@ def run_containerized_partitioned_representation_methods( source_rdf_path=out_dir, selected_methods=methods, method_results=method_results, + index_warnings=index_warnings, ) return True, method_results finally: @@ -3559,6 +3723,7 @@ def run_partitioned_representation_methods_for_rdf_files( min_chunk_bytes: int, max_chunk_bytes: int, expected_triples: int | None = None, + index_warnings: list[dict] | None = None, ): """Dispatch aggregate RDF to the ephemeral container pipeline. @@ -3590,6 +3755,7 @@ def run_partitioned_representation_methods_for_rdf_files( min_chunk_bytes=min_chunk_bytes, max_chunk_bytes=max_chunk_bytes, expected_triples=expected_triples, + index_warnings=index_warnings, ) @@ -3661,6 +3827,7 @@ def run_full_mode( total_triples_produced = 0 saw_triple_counts = False input_failures: list[dict] = [] + index_warnings: list[dict] = [] total_inputs = len(container_inputs) for idx, (container_input, expected_prefix) in enumerate( @@ -3674,6 +3841,7 @@ def run_full_mode( except ValueError: input_vcf = container_input input_failed = False + input_index_warnings: list[dict] = [] def fail_current(stage: str, message: str): nonlocal input_failed @@ -3994,6 +4162,7 @@ def fail_current(stage: str, message: str): timestamp=timestamp, output_name=output_name, expected_triples=triples_produced, + index_warnings=input_index_warnings, ) if not ok: fail_current( @@ -4019,12 +4188,19 @@ def fail_current(stage: str, message: str): min_chunk_bytes=chunk_min_bytes, max_chunk_bytes=chunk_max_bytes, expected_triples=triples_produced, + index_warnings=input_index_warnings, ) if not ok: fail_current( "compression", f"partitioned compression failed for '{output_name}'. See log: {wrapper_log_path}", ) + for warning in input_index_warnings: + warning.setdefault("input_index", idx) + warning.setdefault("input_vcf", input_vcf) + warning.setdefault("expected_prefix", expected_prefix) + if warning not in index_warnings: + index_warnings.append(warning) if input_failed: continue print(f" * Compression {success_symbol()}") @@ -4050,6 +4226,7 @@ def fail_current(stage: str, message: str): combined_size_bytes=combined_size_before_cleanup, selected_methods=selected_methods, method_results=aggregated_results, + index_warnings=input_index_warnings, ) update_metrics_csv_with_compression( metrics_csv=metrics_dir / "metrics.csv", @@ -4079,11 +4256,27 @@ def fail_current(stage: str, message: str): # Cleanup raw RDF only after every selected compression method has # completed successfully for that specific RDF artifact. cleanup_failed = False + warning_formats = { + warning.get("format") + for warning in input_index_warnings + if warning.get("format") in {"hdt", "cottas"} + } + optional_failed_methods = { + method + for method in selected_methods + if ( + (method in HDT_COMPRESSION_METHODS and "hdt" in warning_formats) + or (method in COTTAS_COMPRESSION_METHODS and "cottas" in warning_formats) + ) + } if use_partitioned_compression: missing_or_failed_hdt = [] for method in partitioned_methods: result = partitioned_representation_results.get(method) - if result is None or int(result.get("exit_code", 1)) != 0: + if ( + method not in optional_failed_methods + and (result is None or int(result.get("exit_code", 1)) != 0) + ): missing_or_failed_hdt.append(method) if missing_or_failed_hdt: fail_current( @@ -4103,7 +4296,10 @@ def fail_current(stage: str, message: str): missing_or_failed = [] for method in methods_to_validate: result = method_results.get(method) - if result is None or int(result.get("exit_code", 1)) != 0: + if ( + method not in optional_failed_methods + and (result is None or int(result.get("exit_code", 1)) != 0) + ): missing_or_failed.append(method) if missing_or_failed: fail_current( @@ -4116,6 +4312,13 @@ def fail_current(stage: str, message: str): cleanup_failed = True break + if optional_failed_methods: + eprint( + f"Warning: retaining raw RDF '{raw_rdf_path.name}' because " + "one or more COTTAS/HDT index-dependent outputs were unavailable." + ) + break + # In space-optimized mode the aggregate `.nt.gz` is itself # the gzip artifact. Do not remove it when gzip was selected. if ( @@ -4281,6 +4484,19 @@ def fail_current(stage: str, message: str): elif not selected_methods: print("Total triples produced (full run): unavailable") + index_warning_report = None + if index_warnings: + index_warning_report = write_index_warnings_report( + metrics_dir=metrics_dir, + run_id=run_id, + warnings=index_warnings, + ) + eprint( + f"Index generation warnings were recorded for {len(index_warnings)} item(s): " + f"{index_warning_report}" + ) + print(f"Index warnings: {index_warning_report}") + if input_failures: report_path = write_failed_inputs_report(metrics_dir=metrics_dir, failures=input_failures) eprint( @@ -4295,9 +4511,15 @@ def fail_current(stage: str, message: str): ) return 1 - print("Conversion process finished.") + if index_warning_report is not None: + print("Conversion process finished with index warnings.") + else: + print("Conversion process finished.") if run_tracker is not None: - run_tracker.mark("Full pipeline finished successfully") + run_tracker.mark( + "Full pipeline finished successfully" + + (f" with index warnings; report: {index_warning_report}" if index_warning_report else "") + ) return 0 From 8c11f83323276620297282d17aedd53be0fb80de Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 27 Aug 2026 11:40:29 +0200 Subject: [PATCH 04/19] hdt indexing feature fix --- Dockerfile | 33 +++++++++++++++++++- README.md | 50 ++++++++++++++++++++---------- THIRD_PARTY_NOTICES.md | 8 +++++ src/ensure_hdt_index.sh | 48 ++++++++++++++++------------- src/partitioned_compression.py | 4 +-- src/validate_compression.py | 4 +-- test/test_vcf_rdfizer_unit.py | 56 ++++++++++++++++++---------------- vcf_rdfizer.py | 15 +++++++-- 8 files changed, 147 insertions(+), 71 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8b3eb98..0562118 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ ARG RMLSTREAMER_VERSION=2.5.0 ARG HDT_JAVA_PACKAGE_VERSION=3.0.10 +ARG HDTC_VERSION=1.1.0 FROM eclipse-temurin:11-jre AS build-hdt-cpp @@ -35,6 +36,29 @@ RUN mkdir -p /opt/third_party_licenses \ && cp /opt/RMLStreamer/LICENSE /opt/third_party_licenses/RMLStreamer.LICENSE +FROM rust:1.93-slim AS build-hdtc + +ARG HDTC_VERSION + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + libbz2-dev \ + liblzma-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt +RUN git clone --branch "v${HDTC_VERSION}" --depth 1 \ + https://github.com/frink-okn/hdtc.git + +WORKDIR /opt/hdtc +RUN cargo build --locked --release \ + && mkdir -p /opt/third_party_licenses \ + && cp LICENSE /opt/third_party_licenses/HDTC.LICENSE + + FROM eclipse-temurin:11-jre ARG RMLSTREAMER_VERSION @@ -50,6 +74,8 @@ RUN apt-get update \ findutils \ gawk \ gzip \ + libbz2-1.0 \ + liblzma5 \ libserd-0-0 \ nodejs \ python3 \ @@ -77,6 +103,8 @@ COPY --from=build-hdt-cpp /usr/local/bin/hdt2rdf /usr/local/bin/hdt2rdf COPY --from=build-hdt-cpp /usr/local/lib/libcds* /usr/local/lib/ COPY --from=build-hdt-cpp /usr/local/lib/libhdt* /usr/local/lib/ COPY --from=build-hdt-cpp /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ +COPY --from=build-hdtc /opt/hdtc/target/release/hdtc /usr/local/bin/hdtc +COPY --from=build-hdtc /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES.md COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ @@ -84,11 +112,14 @@ COPY src/*.py /opt/vcf-rdfizer/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ && find /opt/hdt-java/bin -type f -exec chmod +x {} \; \ && chmod +x /usr/local/bin/rdf2hdt \ - && chmod +x /usr/local/bin/hdt2rdf + && chmod +x /usr/local/bin/hdt2rdf \ + && chmod +x /usr/local/bin/hdtc ENV RMLSTREAMER_JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV HDT_JAVA_HOME=/opt/hdt-java +ENV HDTC_BIN=/usr/local/bin/hdtc +ENV HDT_INDEX_MEMORY_LIMIT=512M ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python diff --git a/README.md b/README.md index bf68ffc..9077374 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,8 @@ host filesystem. - `-H, --hdt` existing `.hdt` file; creates or regenerates its sibling sidecar - `--cottas` existing `.cottas` file; rebuilds its embedded query index in place - Exactly one of `--hdt` or `--cottas` is required. -- HDT Java 3.0.10 generates an HDT v1-1 sidecar named `.hdt.index.v1-1`. +- HDT indexing is Java-free: the image uses `hdtc` 1.1.0 to generate the + canonical v1-1 sidecar named `.hdt.index.v1-1`. - COTTAS indexes are stored inside the Parquet-based `.cottas` file, so the file is rewritten atomically; no separate COTTAS index file is expected. - Existing indexes are intentionally replaced. Use this mode when an HDT @@ -361,6 +362,22 @@ versioned sidecar beside the input. COTTAS indexing rewrites the existing the same artifact while rebuilding its embedded index. If the operation fails, the original COTTAS file is left in place; HDT's previous sidecars are restored. +HDT indexing does not start Java. It uses `hdtc index`, which streams the HDT +through disk-backed external sorters. The default soft memory budget is 512 MiB; +override it for any wrapper mode with an environment variable such as: + +```bash +HDT_INDEX_MEMORY_LIMIT=2G vcf-rdfizer \ + --mode index \ + --hdt ./results/sample/sample.hdt \ + --out ./results +``` + +Accepted values use an `M` or `G` suffix. Lower values reduce in-memory sort +buffers and may increase temporary I/O; higher values can improve indexing +speed when memory is available. Temporary files live in the container's +`/work` area and are removed after the attempt. + In `full` mode, an HDT sidecar-index failure is non-fatal when the HDT data itself remains readable; the run continues and the HDT can be repaired later with the standalone command above. If COTTAS generation/indexing cannot @@ -463,7 +480,7 @@ aggregate. When HDT or COTTAS is selected, the aggregate is read sequentially and split into complete N-Triples records. The same temporary chunks are consumed by both converters before cleanup. HDT chunks are merged with `HDTCat`, the final -HDT index is generated after merging through HDT Java's indexed search loader, +HDT index is generated after merging with the Java-free `hdtc index` command, and COTTAS chunks are merged with `pycottas.cat`, which rebuilds the query indexes for the merged representation. @@ -482,20 +499,21 @@ Each COTTAS conversion and merge also receives a fresh container-local DuckDB workspace, which is removed as soon as that operation completes. This prevents state from one chunk being reused by another and requires no user configuration. -HDT Java 3.0.10 does not provide a standalone `hdtGenerateIndex` executable. -VCF-RDFizer sends an `exit` command to the supported `hdtSearch.sh` launcher; -this opens the HDT through `mapIndexedHDT()` without executing a data query and -creates the versioned `.hdt.index.v1-1` sidecar before the run is marked successful. -For standalone index mode, any existing versioned sidecars are moved aside -while regeneration runs and restored if indexing fails, so rerunning the mode -really rebuilds the index without leaving a partially written replacement. -The helper explicitly selects HDT Java's external-sort disk indexer rather than -the launcher's heap-based default (whose launcher heap is only 1 GiB). Temporary -sort runs and disk-backed sequences are kept under `/work` and removed when the -index command finishes. `HDT_INDEX_WORK_ROOT` can override that scratch root -when invoking the helper directly. -For the pinned HDT Java 3.0.10 distribution, this is the HDT v1-1 sidecar -`.hdt.index.v1-1`; VCF-RDFizer reports the actual path in its metrics. +HDT index generation uses the pinned Rust `hdtc` 1.1.0 executable, not +`hdtSearch.sh` or another Java process. `hdtc index` reads BitmapTriples as a +stream and builds the object/predicate orderings with disk-backed external +sorts. This avoids the JVM heap path that can fail with +`java.lang.OutOfMemoryError` while producing the same canonical HDT v1-1 +sidecar, `.hdt.index.v1-1`, used by hdt-java and hdt-cpp. + +For standalone index mode, existing versioned sidecars are moved aside while +regeneration runs and restored if indexing fails. Incomplete replacements are +removed before restoration, so a failed or interrupted attempt does not leave +a partial index. Sort runs use `/work` and are removed when the command exits. +The image defaults `HDT_INDEX_MEMORY_LIMIT` to `512M`; the wrapper forwards a +host value of that variable into standalone, partitioned, and full-run Docker +commands. `HDT_INDEX_WORK_ROOT` can override the scratch root when invoking +`/opt/vcf-rdfizer/ensure_hdt_index.sh` directly inside the container. COTTAS does not expose a separate index sidecar. Its index is part of the Parquet artifact and is selected when the artifact is written. Standalone diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index f8bffb5..ab06f77 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -27,6 +27,14 @@ their respective authors and apply to those components. - Upstream license: Apache License 2.0 - Installed in the image's `/opt/pycottas-venv` environment +4. `hdtc` +- Repository: +- Usage in this project: Java-free, disk-backed index generation for existing + HDT files +- Upstream license: MIT +- License files copied into image: + - `/usr/share/licenses/vcf-rdfizer/HDTC.LICENSE` + ## Notes for package users - The `pip` and `conda` packages install the Python wrapper CLI only. diff --git a/src/ensure_hdt_index.sh b/src/ensure_hdt_index.sh index eff979a..5bdebde 100644 --- a/src/ensure_hdt_index.sh +++ b/src/ensure_hdt_index.sh @@ -12,20 +12,18 @@ if [[ ! -f "$HDT_PATH" ]]; then exit 2 fi -HDT_JAVA_HOME=${HDT_JAVA_HOME:-/opt/hdt-java} -HDT_SEARCH_BIN=${HDT_SEARCH_BIN:-$HDT_JAVA_HOME/bin/hdtSearch.sh} -if [[ ! -x "$HDT_SEARCH_BIN" ]]; then - echo "Missing HDT Java search launcher: $HDT_SEARCH_BIN" >&2 +HDTC_BIN=${HDTC_BIN:-/usr/local/bin/hdtc} +if [[ ! -x "$HDTC_BIN" ]]; then + echo "Missing Java-free HDT indexer: $HDTC_BIN" >&2 exit 127 fi -# The HDT Java launcher defaults to a 1 GiB heap. Its default "recommended" -# indexer still builds and sorts object lists in that heap, so a large but -# otherwise valid HDT can fail here after the merge has already succeeded. -# HDT Java 3.0.10 provides an external-sort indexer for this case. Keep its -# sequences, bitmap sub-indexes, and sort runs in a disposable directory so -# peak heap usage is bounded by the indexer's chunk budget. +# hdtc implements the canonical HDT v1-1 index format without a JVM. It streams +# BitmapTriples through disk-backed external sorters, so the memory setting is +# a soft buffer budget rather than a Java heap requirement. Keep sort runs in a +# disposable directory; callers may point the work root at a larger/faster disk. HDT_INDEX_WORK_ROOT=${HDT_INDEX_WORK_ROOT:-/work} +HDT_INDEX_MEMORY_LIMIT=${HDT_INDEX_MEMORY_LIMIT:-512M} if [[ ! -d "$HDT_INDEX_WORK_ROOT" || ! -w "$HDT_INDEX_WORK_ROOT" ]]; then echo "HDT index work directory is not writable: $HDT_INDEX_WORK_ROOT" >&2 exit 2 @@ -36,22 +34,31 @@ INDEX_BACKUP_DIR="$INDEX_WORK_DIR/existing-indexes" mkdir -p "$INDEX_BACKUP_DIR" INDEX_REGEN_COMPLETE=0 cleanup() { + cleanup_status=$? + set +e if [[ "$INDEX_REGEN_COMPLETE" -eq 0 ]]; then shopt -s nullglob + # Remove every incomplete replacement before restoring the known-good + # sidecars. hdtc writes the final sibling directly, so interruption can + # otherwise leave a partial file behind. + for generated in "${HDT_PATH}".index.*; do + rm -f -- "$generated" + done for backup in "$INDEX_BACKUP_DIR"/*; do mv -- "$backup" "$(dirname "$HDT_PATH")/$(basename "$backup")" done fi rm -rf -- "$INDEX_WORK_DIR" + trap - EXIT + exit "$cleanup_status" } trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM -# `mapIndexedHDT()` reuses an existing sidecar when one is present. Move all -# versioned sidecars out of the way so this command really regenerates the -# index. If indexing fails, the EXIT trap restores the previous sidecars. +# Move all versioned sidecars out of the way so this command really regenerates +# the index. If indexing fails, the EXIT trap restores the previous sidecars. shopt -s nullglob for existing_index in "${HDT_PATH}".index.*; do if [[ -e "$existing_index" ]]; then @@ -59,15 +66,14 @@ for existing_index in "${HDT_PATH}".index.*; do fi done -HDT_INDEX_OPTIONS="bitmaptriples.indexmethod=disk;bitmaptriples.sequence.disk=true;bitmaptriples.sequence.disk.subindex=true;bitmaptriples.sequence.disk.location=$INDEX_WORK_DIR" - -# hdtSearch uses HDT Java's mapIndexedHDT(), which creates the sibling index -# lazily. Sending only `exit` initializes the index without running a query or -# materializing an unbounded result set. -printf 'exit\n' | "$HDT_SEARCH_BIN" -quiet -options "$HDT_INDEX_OPTIONS" "$HDT_PATH" >/dev/null +# This is a dedicated index command: it does not query, rewrite, or decode the +# source HDT. The output is the hdt-java/hdt-cpp-compatible sibling sidecar. +"$HDTC_BIN" --quiet index "$HDT_PATH" \ + --memory-limit "$HDT_INDEX_MEMORY_LIMIT" \ + --temp-dir "$INDEX_WORK_DIR" shopt -s nullglob -# HDT Java 3.0.10 writes the v1-1 index as ``.hdt.index.v1-1``. +# hdtc writes the canonical v1-1 index as ``.hdt.index.v1-1``. INDEX_CANDIDATES=("${HDT_PATH}".index.*) INDEX_PATH="" for candidate in "${INDEX_CANDIDATES[@]}"; do @@ -78,7 +84,7 @@ for candidate in "${INDEX_CANDIDATES[@]}"; do done if [[ -z "$INDEX_PATH" ]]; then - echo "HDT Java did not create a non-empty index beside: $HDT_PATH" >&2 + echo "hdtc did not create a non-empty index beside: $HDT_PATH" >&2 exit 1 fi diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 896e4c2..46a33de 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -163,7 +163,7 @@ def resolve_executable(candidates: tuple[str, ...], label: str) -> str: def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: - """Locate HDT Java's non-empty versioned index sidecar.""" + """Locate the non-empty canonical HDT versioned index sidecar.""" for candidate in sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")): if candidate.is_file() and candidate.stat().st_size > 0: return candidate @@ -506,7 +506,7 @@ def validate_artifact( add_totals(hdt_total, index_stage) output_index = find_hdt_index_sidecar(output_hdt) if output_index is not None: - # The filename is versioned by HDT Java, so record the actual + # The canonical filename is versioned, so record the actual # sidecar after the helper completes instead of assuming .index. index_stage["output_path"] = str(output_index) index_stage["output_size_bytes"] = output_index.stat().st_size diff --git a/src/validate_compression.py b/src/validate_compression.py index e86b002..dd46b6d 100644 --- a/src/validate_compression.py +++ b/src/validate_compression.py @@ -82,8 +82,8 @@ def validate(args: argparse.Namespace) -> dict: } if args.format == "hdt": - # Loading through the bundled Java launcher also verifies that the HDT - # structure is readable and eagerly creates the query index. + # The bundled Java-free helper streams the HDT and eagerly creates the + # query index. hdt2rdf below independently checks decoded readability. if not args.skip_index_check: index_helper = Path("/opt/vcf-rdfizer/ensure_hdt_index.sh") if not index_helper.is_file(): diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index ea3c803..4eeb58c 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -257,6 +257,14 @@ def latest_metrics_run_dir(metrics_root: Path) -> Path: class WrapperUnitTests(VerboseTestCase): + def test_hdt_index_memory_limit_is_forwarded_to_docker(self): + """A host hdtc memory override is passed to indexing containers.""" + with mock.patch.dict(os.environ, {"HDT_INDEX_MEMORY_LIMIT": "2G"}): + self.assertEqual( + vcf_rdfizer.docker_hdt_index_env_args(), + ["-e", "HDT_INDEX_MEMORY_LIMIT=2G"], + ) + def test_validator_counts_plain_and_gzip_ntriples(self): """The Docker validator's fallback source count handles .nt and .nt.gz.""" validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" @@ -2534,27 +2542,23 @@ def test_main_index_mode_requires_exactly_one_index_input(self): rc = invoke_main(["--mode", "index", *options, "--out", str(tmp_path / "out")]) self.assertEqual(rc, 2) - def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): - """The helper uses HDT Java's bounded-memory disk indexer.""" + def test_hdt_index_helper_uses_hdtc_and_verifies_sidecar(self): + """The helper uses hdtc's Java-free bounded-memory index command.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) - java_home = tmp_path / "java-home" - bin_dir = java_home / "bin" - bin_dir.mkdir(parents=True) work_root = tmp_path / "index-work" work_root.mkdir() invocation_path = tmp_path / "invocation.txt" - search = bin_dir / "hdtSearch.sh" - search.write_text( + hdtc = tmp_path / "hdtc" + hdtc.write_text( "#!/usr/bin/env bash\n" - "read -r command\n" - "[[ \"$command\" == \"exit\" ]] || exit 3\n" "printf '%s\\n' \"$@\" > \"$HDT_TEST_INVOCATION\"\n" - "hdt_path=${!#}\n" + "[[ \"$1\" == \"--quiet\" && \"$2\" == \"index\" ]] || exit 3\n" + "hdt_path=$3\n" "printf 'index\\n' > \"${hdt_path}.index.v1-1\"\n", encoding="utf-8", ) - search.chmod(0o755) + hdtc.chmod(0o755) hdt_path = tmp_path / "sample.hdt" hdt_path.write_bytes(b"fake-hdt") existing_index = Path(str(hdt_path) + ".index.v1-1") @@ -2565,8 +2569,9 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): ["bash", str(helper), str(hdt_path)], env={ **os.environ, - "HDT_JAVA_HOME": str(java_home), + "HDTC_BIN": str(hdtc), "HDT_INDEX_WORK_ROOT": str(work_root), + "HDT_INDEX_MEMORY_LIMIT": "768M", "HDT_TEST_INVOCATION": str(invocation_path), }, capture_output=True, @@ -2578,32 +2583,29 @@ def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): self.assertEqual(existing_index.read_text(encoding="utf-8"), "index\n") self.assertIn(f"HDT index ready: {hdt_path}.index.v1-1", result.stdout) invocation = invocation_path.read_text(encoding="utf-8").splitlines() - self.assertEqual(invocation[0:2], ["-quiet", "-options"]) - self.assertIn("bitmaptriples.indexmethod=disk", invocation[2]) - self.assertIn("bitmaptriples.sequence.disk=true", invocation[2]) - self.assertIn("bitmaptriples.sequence.disk.subindex=true", invocation[2]) - self.assertIn("bitmaptriples.sequence.disk.location=", invocation[2]) - self.assertEqual(invocation[3], str(hdt_path)) + self.assertEqual( + invocation[0:5], + ["--quiet", "index", str(hdt_path), "--memory-limit", "768M"], + ) + self.assertEqual(invocation[5], "--temp-dir") + self.assertTrue(Path(invocation[6]).parent.samefile(work_root)) self.assertEqual(list(work_root.iterdir()), []) def test_hdt_index_helper_restores_existing_sidecar_on_failure(self): """Failed HDT regeneration restores the sidecar that was moved aside.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) - java_home = tmp_path / "java-home" - bin_dir = java_home / "bin" - bin_dir.mkdir(parents=True) work_root = tmp_path / "index-work" work_root.mkdir() - search = bin_dir / "hdtSearch.sh" - search.write_text( + hdtc = tmp_path / "hdtc" + hdtc.write_text( "#!/usr/bin/env bash\n" - "read -r command\n" - "[[ \"$command\" == \"exit\" ]] || exit 3\n" + "hdt_path=$3\n" + "printf 'partial-index\\n' > \"${hdt_path}.index.v1-1\"\n" "exit 7\n", encoding="utf-8", ) - search.chmod(0o755) + hdtc.chmod(0o755) hdt_path = tmp_path / "sample.hdt" hdt_path.write_bytes(b"fake-hdt") existing_index = Path(str(hdt_path) + ".index.v1-1") @@ -2614,7 +2616,7 @@ def test_hdt_index_helper_restores_existing_sidecar_on_failure(self): ["bash", str(helper), str(hdt_path)], env={ **os.environ, - "HDT_JAVA_HOME": str(java_home), + "HDTC_BIN": str(hdtc), "HDT_INDEX_WORK_ROOT": str(work_root), }, capture_output=True, diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 4acc2b5..30c183b 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -422,6 +422,14 @@ def docker_run_base(*, as_user: bool = True): return base +def docker_hdt_index_env_args() -> list[str]: + """Forward an optional host-side hdtc memory budget into Docker.""" + memory_limit = os.environ.get("HDT_INDEX_MEMORY_LIMIT", "").strip() + if not memory_limit: + return [] + return ["-e", f"HDT_INDEX_MEMORY_LIMIT={memory_limit}"] + + def _can_write_dir(path: Path) -> bool: """Best-effort write probe for directories.""" try: @@ -678,7 +686,7 @@ def file_size_bytes(path: Path): def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: - """Return HDT Java's non-empty versioned index sidecar.""" + """Return the non-empty canonical HDT versioned index sidecar.""" for candidate in sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")): size = file_size_bytes(candidate) if size is not None and size > 0: @@ -1245,7 +1253,7 @@ def planned_output_paths( ) if any(method in HDT_COMPRESSION_METHODS for method in methods): planned.add(target_dir / f"{output_name}.hdt") - # The pinned HDT Java package generates this versioned sidecar. + # The pinned Java-free indexer generates this canonical sidecar. planned.add(target_dir / f"{output_name}.hdt.index.v1-1") if any(method in COTTAS_COMPRESSION_METHODS for method in methods): planned.add(target_dir / f"{output_name}.cottas") @@ -3015,6 +3023,7 @@ def run_container_command( ) cmd = [ *docker_run_base(), + *docker_hdt_index_env_args(), "-v", f"{str(in_dir)}:/data/in:ro", "-v", @@ -3586,6 +3595,7 @@ def run_containerized_partitioned_representation_methods( command = [ *docker_run_base(), + *docker_hdt_index_env_args(), "--mount", f"type=volume,source={volume_name},target=/work", ] @@ -4712,6 +4722,7 @@ def run_index_mode( ) cmd = [ *docker_run_base(), + *docker_hdt_index_env_args(), "-v", f"{str(index_path.parent)}:/data/{mount_name}", image_ref, From 0d4f7cea8f574a532f4a62a92a7492281410d4fb Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Sat, 29 Aug 2026 12:24:42 +0200 Subject: [PATCH 05/19] fix partition bug with big compressed intermediates --- README.md | 12 +- src/partitioned_compression.py | 327 ++++++++++++++-------- test/test_partitioned_compression_unit.py | 60 ++++ 3 files changed, 272 insertions(+), 127 deletions(-) create mode 100644 test/test_partitioned_compression_unit.py diff --git a/README.md b/README.md index 9077374..0dd74e6 100644 --- a/README.md +++ b/README.md @@ -478,11 +478,13 @@ one, so it avoids retaining both the part files and a full uncompressed aggregate. When HDT or COTTAS is selected, the aggregate is read sequentially and split -into complete N-Triples records. The same temporary chunks are consumed by -both converters before cleanup. HDT chunks are merged with `HDTCat`, the final -HDT index is generated after merging with the Java-free `hdtc index` command, -and COTTAS chunks are merged with `pycottas.cat`, which rebuilds the query -indexes for the merged representation. +into complete N-Triples records. Only one uncompressed chunk is present at a +time: it is consumed by both converters and removed before the next chunk is +read. This is especially important for `space-optimized` `.nt.gz` aggregates, +which must not be expanded into a second full raw-RDF copy. HDT chunks are +merged with `HDTCat`, the final HDT index is generated after merging with the +Java-free `hdtc index` command, and COTTAS chunks are merged with +`pycottas.cat`, which rebuilds the query indexes for the merged representation. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a streaming decode/count check. This verifies both readability and that the diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 46a33de..aaf1d29 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse +import errno import gzip import json import os @@ -63,41 +64,67 @@ def iter_rdf_lines(path: Path): yield from handle -def plan_chunks( +def stream_chunks( source: Path, chunk_dir: Path, *, target_bytes: int, min_bytes: int, max_bytes: int, -) -> tuple[list[Path], dict]: - """Create chunks on complete N-Triples records in one sequential pass.""" +) -> tuple[object, dict]: + """Yield complete-record chunks while building a mutable chunk plan. + + The old implementation returned only after expanding the *entire* source + into raw ``.nt`` chunks. A space-optimized aggregate is normally gzip + compressed, so that made the Docker workspace hold another full, + uncompressed copy of the aggregate before either converter could reclaim a + single byte. The caller now converts and unlinks each yielded chunk before + asking for the next one. + """ if target_bytes <= 0 or min_bytes <= 0 or max_bytes <= 0: raise ValueError("RDF chunk sizes must be positive") if min_bytes > target_bytes or target_bytes > max_bytes: raise ValueError("RDF chunk sizes must satisfy min <= target <= max") chunk_dir.mkdir(parents=True, exist_ok=True) - chunk_paths: list[Path] = [] - chunk_metadata: list[dict] = [] - handle = None - chunk_path = None - chunk_size = 0 - chunk_start_offset = 0 - chunk_start_record = 0 - logical_offset = 0 - record_count = 0 - chunk_index = 0 - - def close_chunk(): - nonlocal handle, chunk_path, chunk_size - if handle is None or chunk_path is None: - return - handle.close() - chunk_paths.append(chunk_path) - chunk_metadata.append( - { - "chunk_id": len(chunk_metadata), + plan = { + "source_file_count": 1, + "source_paths": [str(source)], + "chunk_count": 0, + "chunk_input_bytes": 0, + "record_count": 0, + "target_chunk_bytes": target_bytes, + "min_chunk_bytes": min_bytes, + "max_chunk_bytes": max_bytes, + "chunks": [], + } + + def generate(): + handle = None + chunk_path = None + chunk_size = 0 + chunk_start_offset = 0 + chunk_start_record = 0 + logical_offset = 0 + record_count = 0 + chunk_index = 0 + + def open_chunk(): + nonlocal handle, chunk_path, chunk_size, chunk_start_offset, chunk_start_record, chunk_index + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_index += 1 + handle = chunk_path.open("wb") + chunk_size = 0 + chunk_start_offset = logical_offset + chunk_start_record = record_count + + def close_chunk(): + nonlocal handle, chunk_path, chunk_size + if handle is None or chunk_path is None: + return None + handle.close() + metadata = { + "chunk_id": len(plan["chunks"]), "path": str(chunk_path), "start_record": chunk_start_record, "end_record": record_count, @@ -106,52 +133,78 @@ def close_chunk(): "record_count": record_count - chunk_start_record, "payload_bytes": chunk_size, } - ) - handle = None - chunk_path = None - chunk_size = 0 + plan["chunks"].append(metadata) + plan["chunk_count"] = len(plan["chunks"]) + completed_path = chunk_path + handle = None + chunk_path = None + chunk_size = 0 + return completed_path, metadata - try: - for line in iter_rdf_lines(source): - if not line.endswith(b"\n"): - raise ValueError(f"RDF source contains a non-line-terminated record: {source}") - line_size = len(line) - if handle is None: - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - elif chunk_size > 0 and ( - (chunk_size >= target_bytes and chunk_size >= min_bytes) - or chunk_size + line_size > max_bytes - ): - close_chunk() - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - - handle.write(line) - chunk_size += line_size - logical_offset += line_size - if is_triple_line(line): - record_count += 1 - finally: - close_chunk() - - return chunk_paths, { - "source_file_count": 1, - "source_paths": [str(source)], - "chunk_count": len(chunk_paths), - "chunk_input_bytes": logical_offset, - "record_count": record_count, - "target_chunk_bytes": target_bytes, - "min_chunk_bytes": min_bytes, - "max_chunk_bytes": max_bytes, - "chunks": chunk_metadata, - } + try: + for line in iter_rdf_lines(source): + if not line.endswith(b"\n"): + raise ValueError(f"RDF source contains a non-line-terminated record: {source}") + line_size = len(line) + if handle is None: + open_chunk() + elif chunk_size > 0 and ( + (chunk_size >= target_bytes and chunk_size >= min_bytes) + or chunk_size + line_size > max_bytes + ): + completed_chunk = close_chunk() + if completed_chunk is not None: + yield completed_chunk + open_chunk() + + handle.write(line) + chunk_size += line_size + logical_offset += line_size + if is_triple_line(line): + record_count += 1 + plan["chunk_input_bytes"] = logical_offset + plan["record_count"] = record_count + + completed_chunk = close_chunk() + if completed_chunk is not None: + yield completed_chunk + finally: + # A write/decompression error can leave one unyielded, partial + # chunk. It has no consumer, so remove it before the volume is + # released. + if handle is not None: + try: + handle.close() + finally: + if chunk_path is not None: + chunk_path.unlink(missing_ok=True) + + return generate(), plan + + +def plan_chunks( + source: Path, + chunk_dir: Path, + *, + target_bytes: int, + min_bytes: int, + max_bytes: int, +) -> tuple[list[Path], dict]: + """Materialize chunks for callers that explicitly need every path. + + The production compressor uses :func:`stream_chunks` so it never retains a + full uncompressed copy of a gzip aggregate. This compatibility helper is + deliberately kept for diagnostics and standalone callers. + """ + stream, plan = stream_chunks( + source, + chunk_dir, + target_bytes=target_bytes, + min_bytes=min_bytes, + max_bytes=max_bytes, + ) + chunk_paths = [path for path, _metadata in stream] + return chunk_paths, plan def resolve_executable(candidates: tuple[str, ...], label: str) -> str: @@ -317,6 +370,17 @@ def main() -> int: cottas_failed = False cottas_warning = None + def write_result(payload: dict): + """Best-effort result handoff that never hides the original failure.""" + try: + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except OSError as write_error: + print( + f"Warning: unable to write partitioned-compression result to {result_path}: {write_error}", + file=sys.stderr, + ) + def record_index_warning(index_format: str, stage: str, artifact: Path, message: str) -> dict: warning = { "format": index_format, @@ -357,21 +421,6 @@ def skipped_cottas_result(method: str) -> dict: if not methods or not all(method in HDT_METHODS | COTTAS_METHODS for method in methods): raise ValueError(f"Unsupported partitioned method list: {methods}") - chunks, plan = plan_chunks( - source, - chunk_dir, - target_bytes=args.target_chunk_bytes, - min_bytes=args.min_chunk_bytes, - max_bytes=args.max_chunk_bytes, - ) - if not chunks: - raise ValueError("RDF source contains no complete records") - source_triples = int(plan["record_count"]) - if args.expected_triples is not None and source_triples != args.expected_triples: - raise ValueError( - "source triple count does not match the upstream conversion count: " - f"source={source_triples}, expected={args.expected_triples}" - ) hdt_paths: list[Path] = [] cottas_paths: list[Path] = [] needs_hdt = any(method in HDT_METHODS for method in methods) @@ -391,6 +440,13 @@ def skipped_cottas_result(method: str) -> dict: cottas_python = os.environ.get("COTTAS_PYTHON_BIN") or shutil.which("python3") if any(method in COTTAS_METHODS for method in methods) and not cottas_python: raise RuntimeError("Missing Python runtime for COTTAS") + chunk_stream, plan = stream_chunks( + source, + chunk_dir, + target_bytes=args.target_chunk_bytes, + min_bytes=args.min_chunk_bytes, + max_bytes=args.max_chunk_bytes, + ) def validate_artifact( *, @@ -451,41 +507,55 @@ def validate_artifact( ) return report - for index, chunk in enumerate(chunks): - if needs_hdt: - chunk_hdt = work_dir / f"chunk-{index:05d}.hdt" - stage = runner.run(f"hdt-build-{index:05d}", [hdt_bin, str(chunk), str(chunk_hdt)], chunk_hdt) - add_totals(hdt_total, stage) - if stage["exit_code"] != 0: - raise RuntimeError("partitioned HDT chunk conversion failed") - hdt_paths.append(chunk_hdt) - if any(method in COTTAS_METHODS for method in methods): - if cottas_failed: - chunk.unlink(missing_ok=True) - continue - chunk_cottas = work_dir / f"chunk-{index:05d}.cottas" - stage = runner.run( - f"cottas-build-{index:05d}", - [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "convert", str(chunk), str(chunk_cottas), "spo"], - chunk_cottas, - ) - add_totals(cottas_total, stage) - if stage["exit_code"] != 0: - if not args.allow_index_failures: - raise RuntimeError("partitioned COTTAS chunk conversion failed") - cottas_failed = True - cottas_total["exit_code"] = 0 - cottas_warning = record_index_warning( - "cottas", - "cottas-index", - output_cottas, - "partitioned COTTAS conversion/index creation failed for a chunk", + # Convert each uncompressed chunk before requesting the next one from + # the gzip reader. This bounds raw-RDF workspace use to one chunk, + # rather than the entire decompressed aggregate. + for index, (chunk, _chunk_metadata) in enumerate(chunk_stream): + try: + if needs_hdt: + chunk_hdt = work_dir / f"chunk-{index:05d}.hdt" + stage = runner.run( + f"hdt-build-{index:05d}", + [hdt_bin, str(chunk), str(chunk_hdt)], + chunk_hdt, ) - chunk_cottas.unlink(missing_ok=True) - chunk.unlink(missing_ok=True) - continue - cottas_paths.append(chunk_cottas) - chunk.unlink(missing_ok=True) + add_totals(hdt_total, stage) + if stage["exit_code"] != 0: + raise RuntimeError("partitioned HDT chunk conversion failed") + hdt_paths.append(chunk_hdt) + if any(method in COTTAS_METHODS for method in methods) and not cottas_failed: + chunk_cottas = work_dir / f"chunk-{index:05d}.cottas" + stage = runner.run( + f"cottas-build-{index:05d}", + [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "convert", str(chunk), str(chunk_cottas), "spo"], + chunk_cottas, + ) + add_totals(cottas_total, stage) + if stage["exit_code"] != 0: + if not args.allow_index_failures: + raise RuntimeError("partitioned COTTAS chunk conversion failed") + cottas_failed = True + cottas_total["exit_code"] = 0 + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + "partitioned COTTAS conversion/index creation failed for a chunk", + ) + chunk_cottas.unlink(missing_ok=True) + else: + cottas_paths.append(chunk_cottas) + finally: + chunk.unlink(missing_ok=True) + + if not plan["chunks"]: + raise ValueError("RDF source contains no complete records") + source_triples = int(plan["record_count"]) + if args.expected_triples is not None and source_triples != args.expected_triples: + raise ValueError( + "source triple count does not match the upstream conversion count: " + f"source={source_triples}, expected={args.expected_triples}" + ) if hdt_paths: final_hdt, hdt_rounds = merge_pairwise( @@ -629,13 +699,26 @@ def validate_artifact( raise RuntimeError(f"{method} packaging failed") results[method] = stage - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text(json.dumps({"exit_code": 0, "methods": results, "stages": runner.stages, "index_warnings": index_warnings}, indent=2) + "\n", encoding="utf-8") + write_result( + { + "exit_code": 0, + "methods": results, + "stages": runner.stages, + "index_warnings": index_warnings, + } + ) return 0 except Exception as exc: - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text(json.dumps({"exit_code": 1, "methods": results, "stages": runner.stages, "error": str(exc)}, indent=2) + "\n", encoding="utf-8") - print(f"partitioned compression failed: {exc}", file=sys.stderr) + error = str(exc) + if isinstance(exc, OSError) and exc.errno == errno.ENOSPC: + error = ( + "temporary partitioned-compression workspace (/work) ran out of storage. " + "Chunks are streamed one at a time, but the workspace must still hold " + "one raw chunk plus the in-progress HDT/COTTAS artifacts. Reduce " + "--chunk-target-bytes and --chunk-max-bytes, or increase Docker's disk limit." + ) + write_result({"exit_code": 1, "methods": results, "stages": runner.stages, "error": error}) + print(f"partitioned compression failed: {error}", file=sys.stderr) return 1 diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py new file mode 100644 index 0000000..8d79582 --- /dev/null +++ b/test/test_partitioned_compression_unit.py @@ -0,0 +1,60 @@ +import gzip +import importlib.util +import tempfile +import unittest +from pathlib import Path + +from test.helpers import VerboseTestCase + + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "src" / "partitioned_compression.py" + + +def load_runner_module(): + spec = importlib.util.spec_from_file_location("partitioned_compression_test", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class PartitionedCompressionUnitTests(VerboseTestCase): + def test_stream_chunks_only_retains_the_chunk_being_consumed(self): + """Gzip chunking does not stage a second full uncompressed aggregate.""" + runner = load_runner_module() + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "source.nt.gz" + source_bytes = b"".join( + f"

.\n".encode("utf-8") + for index in range(12) + ) + with gzip.open(source, "wb") as handle: + handle.write(source_bytes) + + chunk_dir = tmp_path / "chunks" + stream, plan = runner.stream_chunks( + source, + chunk_dir, + target_bytes=40, + min_bytes=20, + max_bytes=60, + ) + emitted = [] + for chunk, metadata in stream: + self.assertEqual(list(chunk_dir.glob("*.nt")), [chunk]) + self.assertTrue(chunk.read_bytes().endswith(b"\n")) + self.assertEqual(metadata["payload_bytes"], chunk.stat().st_size) + emitted.append(chunk.read_bytes()) + chunk.unlink() + + self.assertEqual(b"".join(emitted), source_bytes) + self.assertEqual(plan["record_count"], 12) + self.assertEqual(plan["chunk_count"], len(plan["chunks"])) + self.assertGreater(plan["chunk_count"], 1) + self.assertEqual(list(chunk_dir.glob("*.nt")), []) + + +if __name__ == "__main__": + unittest.main() From acc079bc1cbb8c607aa80882319c78577c3abdb9 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Tue, 1 Sep 2026 14:32:48 +0200 Subject: [PATCH 06/19] add condensed mode for large multi-sample VCF knowledge representation --- README.md | 81 ++- rules/README.md | 21 +- rules/default_rules.ttl | 5 +- RELEASING.md => scripts/RELEASING.md | 0 test/test_vcf_rdfizer_unit.py | 194 +++++- vcf_rdfizer.py | 761 +++++++++++++++++------ vcf_rdfizer_data/rules/default_rules.ttl | 7 +- 7 files changed, 855 insertions(+), 214 deletions(-) rename RELEASING.md => scripts/RELEASING.md (100%) diff --git a/README.md b/README.md index 0dd74e6..5e1dfc5 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: - `--artifact-compression` packaging codecs for selected representations: `gzip`, `brotli`, or `none` - `--hdt-strategy {auto,partitioned,single}` HDT generation policy - `--chunk-target-bytes`, `--chunk-min-bytes`, `--chunk-max-bytes` shared record-safe chunk sizing +- `--sample-representation {dense,condensed}` genotype graph shape (`dense` by default) - `-I, --image` Docker image repo (default `ecrum19/vcf-rdfizer`) - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build @@ -129,6 +130,9 @@ host filesystem. - `-i, --input` required VCF file or directory - `-r, --rules` mapping rules file (`.ttl`) - default: `rules/default_rules.ttl` +- `--sample-representation {dense,condensed}` sample genotype representation + - `dense` (default): one `SampleCall` per record/sample and one `FormatFieldValue` per FORMAT key + - `condensed`: reusable file-level samples plus one ordered value vector per record/FORMAT key - `--rdf-storage-mode {plain,space-optimized}` required full-mode aggregate storage policy - `plain`: merge RMLStreamer parts into one uncompressed `.nt` - `space-optimized`: gzip each part into one `.nt.gz` aggregate and delete the source part immediately @@ -150,6 +154,71 @@ host filesystem. - `--remove-rdf-storage-output` explicitly remove the aggregate `.nt`/`.nt.gz` after successful compression - `-e, --estimate-size` preflight size estimate +## Sample Representation Modes + +Full mode has exactly two explicit sample workflows. There is no automatic +sample-count threshold, so the same command always produces the same graph +shape and downstream consumers can select the contract they support. + +### Dense (default) + +Use `--sample-representation dense` for single-sample and low-sample VCFs. It +preserves the original vocabulary model: + +- every record/sample pair is a `vcfr:SampleCall`; +- every represented FORMAT slot is a `vcfr:FormatFieldValue`; +- the VCF file declares `vcfr:representationProfile vcfr:DenseRepresentation`. + +With the default rules, these triples are appended directly from `records.tsv`; +the large expanded helper TSVs are not materialized. The final graph is still +dense and grows approximately with `variants × samples × FORMAT fields`. + +```bash +vcf-rdfizer --mode full \ + --input ./small.vcf \ + --sample-representation dense \ + --rdf-storage-mode plain \ + --out ./results +``` + +### Condensed + +Use `--sample-representation condensed` for large multi-sample cohorts. It +uses the vocabulary introduced in VCF-RDFizer Vocabulary 1.1.0: + +- sample columns are declared once as an ordered `vcfr:SampleSet` of reusable + `vcfr:VCFSample` resources; +- each genotype-bearing call has one `vcfr:CohortCallMatrix`; +- each FORMAT key has one `vcfr:FormatValueVector`, rather than one RDF value + resource per sample; +- the VCF file declares + `vcfr:representationProfile vcfr:CondensedRepresentation`. + +`vcfr:encodedValues` uses `vcfr:VCFTextVector`: one tab-separated lexical item +per sample in `vcfr:sampleIndex` order. Commas inside a FORMAT value remain part +of that item, and absent values are emitted as `.` so all vectors stay aligned. +Consumers reconstruct sample `i`'s value for a FORMAT key by selecting position +`i` from its vector. This changes genotype graph growth to approximately +`samples + variants × FORMAT fields`; the literal payload still contains all +source values, but they no longer cause per-value RDF structural triples. + +```bash +vcf-rdfizer --mode full \ + --input ./large-cohort.vcf.gz \ + --sample-representation condensed \ + --rdf-storage-mode space-optimized \ + --representations hdt \ + --out ./results +``` + +The workflow resolver runs only one sample emitter. Condensed mode rejects +custom mappings that consume expanded `sample_calls.tsv` or +`sample_format_values.tsv`, because running those dense maps alongside the +condensed emitter would create both representations and restore the semantic +inflation this mode is designed to avoid. Remove those helper-table consumers +or select dense mode. Custom rules with no helper-table consumers remain +compatible with condensed emission. + ## TSV Mode Flags - `-i, --input` required VCF file or directory @@ -530,13 +599,11 @@ partitioned-compression metrics JSON for diagnostics. The temporary chunk files and guide are not retained as host files. For the default mapping, multi-sample VCF columns remain compact in -`records.tsv`. Canonical `SampleCall` and `FormatFieldValue` triples are streamed -directly into the final `.nt` or `.nt.gz` aggregate instead of first writing -`variants × samples` and `variants × samples × FORMAT fields` helper rows. -The compatibility helper TSVs therefore contain only headers. Custom mappings -that add consumers of those helper sources continue to use expanded tables. -This removes the large temporary-disk multiplier, although the final RDF still -scales with the number of emitted sample and FORMAT triples. +`records.tsv`. In dense mode, canonical `SampleCall` and `FormatFieldValue` +triples are streamed directly into the final `.nt` or `.nt.gz` aggregate rather +than first writing expanded helper rows. In condensed mode, the same input pass +emits shared samples, call matrices, and FORMAT vectors, avoiding both the +helper-table multiplier and the per-sample RDF structural multiplier. The implementation keeps COTTAS conversion scratch state inside the Docker container and removes temporary unpacked package files when decompression diff --git a/rules/README.md b/rules/README.md index 1925b0f..4295e34 100644 --- a/rules/README.md +++ b/rules/README.md @@ -14,12 +14,14 @@ This directory contains RML mappings used by the conversion pipeline. - `/data/tsv/sample_calls.tsv` - `/data/tsv/sample_format_values.tsv` - For the built-in sample maps, `sample_calls.tsv` and - `sample_format_values.tsv` are header-only compatibility sources. The Python - wrapper streams their equivalent `SampleCall` and `FormatFieldValue` triples - directly from `records.tsv` into the RDF aggregate, avoiding helper-table - expansion proportional to variants × samples × FORMAT fields. + `sample_format_values.tsv` are header-only compatibility sources. In + `--sample-representation dense`, the Python wrapper streams their equivalent + `SampleCall` and `FormatFieldValue` triples directly from `records.tsv`. In + `--sample-representation condensed`, it instead streams `SampleSet`, + `CohortCallMatrix`, and `FormatValueVector` resources. Only one emitter runs. - Custom mappings with additional consumers of either helper source retain - expanded TSV generation for compatibility. + expanded TSV generation in dense mode. They are rejected in condensed mode + to prevent simultaneous dense and condensed output. - The Python wrapper rewrites these template paths per input VCF to: - `/data/tsv/.file_metadata.tsv` - `/data/tsv/.header_lines.tsv` @@ -36,9 +38,14 @@ This directory contains RML mappings used by the conversion pipeline. 5. Run the wrapper with your custom mapping: ```bash -python3 vcf_rdfizer.py --input --rules rules/my_rules.ttl +python3 vcf_rdfizer.py --input --rules rules/my_rules.ttl \ + --rdf-storage-mode plain --out ./results ``` +Custom rules that do not consume either sample helper table can be used with +`--sample-representation condensed`; the wrapper adds the condensed sample graph +after RMLStreamer emits the custom record-level graph. + ## SHACL Notes The related SHACL constraints are maintained in the vocabulary repository: @@ -51,3 +58,5 @@ The default mapping is structured to align with those classes/properties, especi - `vcfr:VCFHeader` + `vcfr:hasHeaderLine` - `vcfr:VCFRecord` core fields (`chrom`, `pos`, `ref`, `alt`) - `vcfr:VariantCall` with raw call attributes +- dense `vcfr:SampleCall` / `vcfr:FormatFieldValue` resources, or condensed + `vcfr:CohortCallMatrix` / `vcfr:FormatValueVector` resources diff --git a/rules/default_rules.ttl b/rules/default_rules.ttl index 02bf24e..7a0ee2c 100644 --- a/rules/default_rules.ttl +++ b/rules/default_rules.ttl @@ -14,8 +14,9 @@ # - /data/tsv/.sample_calls.tsv (header-only compatibility source) # - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. -# Canonical sample/FORMAT triples are streamed directly into the RDF aggregate -# so chromosome-scale multi-sample VCFs do not require expanded helper tables. +# The wrapper selects exactly one direct sample emitter: dense SampleCall / +# FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / +# FormatValueVector triples. The helper tables stay header-only in either mode. # # Output format: # - The default conversion now targets triples (N-Triples) without named graph terms. diff --git a/RELEASING.md b/scripts/RELEASING.md similarity index 100% rename from RELEASING.md rename to scripts/RELEASING.md diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 4eeb58c..df66171 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -1407,9 +1407,14 @@ def test_default_sample_maps_stream_without_expanded_helper_tables(self): self.assertEqual(stats["records"], 1) self.assertEqual(stats["sample_calls"], 2) self.assertEqual(stats["format_values"], 4) - self.assertEqual(stats["triples"], 18) + self.assertEqual(stats["triples"], 19) rdf_lines = rdf_path.read_text(encoding="utf-8").splitlines() - self.assertEqual(len(rdf_lines), 19) + self.assertEqual(len(rdf_lines), 20) + self.assertIn( + " " + " .", + rdf_lines, + ) self.assertIn( " " " .", @@ -1447,9 +1452,9 @@ def test_sample_streaming_accepts_a_csv_field_larger_than_python_default(self): self.assertEqual(stats["sample_calls"], 2504) self.assertEqual(stats["format_values"], 2504) - self.assertEqual(stats["triples"], 2504 * 6) + self.assertEqual(stats["triples"], 1 + (2504 * 6)) with gzip.open(rdf_path, "rt", encoding="utf-8") as handle: - self.assertEqual(sum(1 for _line in handle), 1 + (2504 * 6)) + self.assertEqual(sum(1 for _line in handle), 2 + (2504 * 6)) def test_sample_support_strategy_preserves_custom_helper_consumers(self): """Only the exact canonical helper maps use direct RDF streaming.""" @@ -1464,6 +1469,187 @@ def test_sample_support_strategy_preserves_custom_helper_consumers(self): ) self.assertEqual(vcf_rdfizer.sample_support_strategy(custom_rules), "expanded") + def test_sample_workflow_resolver_selects_one_compatible_branch(self): + """Dense and condensed plans are mutually exclusive and rules-aware.""" + default_rules = Path(__file__).parents[1] / "rules" / "default_rules.ttl" + + dense = vcf_rdfizer.resolve_sample_workflow("dense", default_rules) + condensed = vcf_rdfizer.resolve_sample_workflow("condensed", default_rules) + + self.assertEqual(dense.helper_strategy, "header-only") + self.assertEqual(dense.emitter, "dense") + self.assertEqual(condensed.helper_strategy, "header-only") + self.assertEqual(condensed.emitter, "condensed") + + with tempfile.TemporaryDirectory() as td: + custom_rules = Path(td) / "custom.ttl" + custom_rules.write_text( + default_rules.read_text(encoding="utf-8") + + '\n<#Extra> csvw:url "/data/tsv/sample_calls.tsv" .\n', + encoding="utf-8", + ) + self.assertEqual( + vcf_rdfizer.resolve_sample_workflow("dense", custom_rules).helper_strategy, + "expanded", + ) + with self.assertRaisesRegex(ValueError, "cannot be combined"): + vcf_rdfizer.resolve_sample_workflow("condensed", custom_rules) + + def test_condensed_sample_emitter_uses_shared_samples_and_ordered_vectors(self): + """Condensed mode emits one SampleSet and FORMAT vector per key, not dense calls.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "cohort.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1 S2 S3\n" + "cohort.vcf\t7\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT:DP:AD\t" + "0/1:42:30,12 ./.:.:. 1/1:9\n", + encoding="utf-8", + ) + headers_tsv = tmp_path / "cohort.header_lines.tsv" + headers_tsv.write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "cohort.vcf\t1\tFORMAT\t\tx\n" + "cohort.vcf\t2\tFORMAT\t\tx\n" + "cohort.vcf\t3\tFORMAT\t\tx\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "cohort.nt" + rdf_path.write_text(" .\n", encoding="utf-8") + + stats = vcf_rdfizer.append_condensed_sample_rdf( + records_tsv, + headers_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(stats["representation"], "condensed") + self.assertEqual(stats["samples"], 3) + self.assertEqual(stats["matrices"], 1) + self.assertEqual(stats["format_vectors"], 3) + self.assertEqual(stats["format_definitions"], 3) + self.assertEqual(stats["triples"], 45) + rdf_text = rdf_path.read_text(encoding="utf-8") + self.assertIn("vocab#CondensedRepresentation", rdf_text) + self.assertIn("vocab#SampleSet", rdf_text) + self.assertIn("#samples/S3> ", rdf_text) + self.assertIn("#call/7/matrix/fmt/GT", rdf_text) + self.assertIn('"0/1\\t./.\\t1/1"', rdf_text) + self.assertIn('"30,12\\t.\\t."', rdf_text) + self.assertIn("#header/line/1> .", rdf_text) + self.assertIn("vocab#fieldNumber> \"1\"", rdf_text) + self.assertIn("vocab#fieldDescription> \"Genotype\"", rdf_text) + self.assertNotIn("vocab#hasSampleCall", rdf_text) + self.assertNotIn("vocab#FormatFieldValue", rdf_text) + + def test_structured_format_header_parser_preserves_quoted_commas(self): + """FORMAT descriptions with commas and escaped quotes remain one attribute.""" + fields = vcf_rdfizer._parse_structured_header_fields( + '' + ) + + self.assertEqual(fields["ID"], "GT") + self.assertEqual(fields["Number"], "1") + self.assertEqual(fields["Description"], 'Genotype, with "quoted" text') + + def test_condensed_sample_emitter_rolls_back_on_sample_count_mismatch(self): + """Malformed sample alignment fails atomically without a partial condensed graph.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "bad.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1\n" + "bad.vcf\t1\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1 0/0\n", + encoding="utf-8", + ) + headers_tsv = tmp_path / "bad.header_lines.tsv" + headers_tsv.write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "bad.nt" + original = " .\n" + rdf_path.write_text(original, encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "2 sample payloads"): + vcf_rdfizer.append_condensed_sample_rdf( + records_tsv, + headers_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(rdf_path.read_text(encoding="utf-8"), original) + + def test_main_condensed_mode_runs_only_the_condensed_emitter(self): + """The full CLI routes default rules to condensed output without dense sample triples.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_file = tmp_path / "cohort.vcf" + input_file.write_text( + "##fileformat=VCFv4.2\n" + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1\tS2\n" + "1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\t0/0\n", + encoding="utf-8", + ) + out_dir = tmp_path / "out" + + def fake_tsv_conversion(**kwargs): + tsv_dir = kwargs["tsv_dir"] + prefix = kwargs["prefix"] + tsv_dir.mkdir(parents=True, exist_ok=True) + (tsv_dir / f"{prefix}.records.tsv").write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1 S2\n" + "cohort.vcf\t1\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1 0/0\n", + encoding="utf-8", + ) + (tsv_dir / f"{prefix}.header_lines.tsv").write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "cohort.vcf\t1\tFORMAT\t\tx\n", + encoding="utf-8", + ) + (tsv_dir / f"{prefix}.file_metadata.tsv").write_text( + "SOURCE_FILE\tFILE_FORMAT\tFILE_DATE\tSOURCE_SOFTWARE\tREFERENCE_GENOME\tHEADER_COUNT\tRECORD_COUNT\n" + "cohort.vcf\tVCFv4.2\t\t\t\t1\t1\n", + encoding="utf-8", + ) + return {"exit_code": 0, "output_size_bytes": 1} + + with mock.patch.object(vcf_rdfizer, "run", return_value=0), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ), mock.patch.object( + vcf_rdfizer, + "run_tsv_conversion_with_metrics", + side_effect=fake_tsv_conversion, + ): + rc = invoke_main( + [ + "--input", + str(input_file), + "--out", + str(out_dir), + "--sample-representation", + "condensed", + "--compression", + "none", + "--keep-tsv", + ] + ) + + self.assertEqual(rc, 0) + rdf_text = (out_dir / "cohort" / "cohort.nt").read_text(encoding="utf-8") + self.assertIn("vocab#CondensedRepresentation", rdf_text) + self.assertIn("vocab#CohortCallMatrix", rdf_text) + self.assertNotIn("vocab#SampleCall", rdf_text) + self.assertNotIn("vocab#FormatFieldValue", rdf_text) + helper_rows = ( + out_dir / ".intermediate" / "tsv" / "cohort.sample_calls.tsv" + ).read_text(encoding="utf-8").splitlines() + self.assertEqual(helper_rows, ["\t".join(vcf_rdfizer.SAMPLE_CALLS_HEADER)]) + def test_render_rules_for_triplet_rewrites_helper_tsv_placeholders(self): """Rule rendering rewrites records/header/metadata and helper TSV placeholders.""" with tempfile.TemporaryDirectory() as td: diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 30c183b..3b40d22 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -26,6 +26,7 @@ import subprocess import sys import time +from dataclasses import dataclass from datetime import datetime from pathlib import Path from urllib.parse import quote_plus @@ -227,7 +228,9 @@ ) VCFR_NAMESPACE = "https://w3id.org/vcf-rdfizer/vocab#" RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +XSD_POSITIVE_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#positiveInteger" SAMPLE_RDF_BUFFER_BYTES = 8 * 1024 * 1024 +SAMPLE_REPRESENTATION_CHOICES = {"dense", "condensed"} # --------------------------------------------------------------------------- @@ -1579,6 +1582,47 @@ def sample_support_strategy(rules_path: Path) -> str: return "expanded" +@dataclass(frozen=True) +class SampleWorkflow: + """One mutually exclusive sample-representation execution plan.""" + + representation: str + helper_strategy: str + emitter: str | None + + +def resolve_sample_workflow(representation: str, rules_path: Path) -> SampleWorkflow: + """Resolve rules compatibility into exactly one sample workflow. + + Dense mode preserves custom helper-table mappings. Condensed mode emits its + RDF directly from records.tsv; it cannot safely coexist with custom rules + that consume expanded dense helper tables because that would execute both + representations and reintroduce semantic inflation. + """ + if representation not in SAMPLE_REPRESENTATION_CHOICES: + choices = ", ".join(sorted(SAMPLE_REPRESENTATION_CHOICES)) + raise ValueError( + f"unsupported sample representation '{representation}'; choose {choices}" + ) + + rules_strategy = sample_support_strategy(rules_path) + if representation == "dense": + if rules_strategy == "stream": + return SampleWorkflow("dense", "header-only", "dense") + if rules_strategy == "expanded": + return SampleWorkflow("dense", "expanded", None) + return SampleWorkflow("dense", "none", None) + + if rules_strategy == "expanded": + raise ValueError( + "--sample-representation condensed cannot be combined with custom rules " + "that consume expanded sample_calls.tsv or sample_format_values.tsv tables. " + "Remove those dense helper-table consumers or use dense mode." + ) + helper_strategy = "header-only" if rules_strategy == "stream" else "none" + return SampleWorkflow("condensed", helper_strategy, "condensed") + + def _set_max_csv_field_size(): """Allow chromosome-scale multi-sample payload columns in Python's CSV reader.""" limit = sys.maxsize @@ -1603,7 +1647,8 @@ def _rml_uri_component(value: str) -> str: return encoded.replace("+", "%20").replace("~", "%7E") -def _ntriples_literal(value: str) -> str: +def _ntriples_string_literal(value: str) -> str: + """Serialize an RDF 1.1 plain/xsd:string literal for N-Triples.""" escaped = ( value.replace("\\", "\\\\") .replace('"', '\\"') @@ -1611,25 +1656,206 @@ def _ntriples_literal(value: str) -> str: .replace("\r", "\\r") .replace("\t", "\\t") ) - literal = f'"{escaped}"' + return f'"{escaped}"' + + +def _ntriples_literal(value: str) -> str: + literal = _ntriples_string_literal(value) if value == ".": literal += f"^^<{VCFR_NAMESPACE}Null>" return literal -def append_canonical_sample_rdf( +@dataclass(frozen=True) +class SampleColumn: + """One reusable VCF sample column.""" + + index: int + sample_id: str + uri_id: str + + +@dataclass(frozen=True) +class ParsedSampleRecord: + """One VCF record with FORMAT keys aligned to all sample columns.""" + + source_file: str + row_id: str + format_keys: tuple[str, ...] + sample_payloads: tuple[str, ...] + sample_values: tuple[tuple[str, ...], ...] + + +class SampleRecordStream: + """Read a records.tsv sample block once and expose a stable sample schema.""" + + def __init__(self, records_tsv: Path): + self.records_tsv = records_tsv + self.columns: tuple[SampleColumn, ...] = () + self.source_file = "" + self._handle = None + self._reader = None + self._header: list[str] = [] + self._pending_row: list[str] | None = None + + def __enter__(self): + _set_max_csv_field_size() + self._handle = self.records_tsv.open(newline="", encoding="utf-8") + self._reader = csv.reader(self._handle, delimiter="\t") + self._header = next(self._reader, None) or [] + self._pending_row = self._next_nonempty_row() + if self._pending_row: + self.source_file = self._pending_row[0] if self._pending_row else "" + + sample_header = self._header[-1].strip() if len(self._header) >= 12 else "" + declared_ids = ( + [] + if sample_header == "SAMPLES" + else [token for token in sample_header.split() if token] + ) + if not declared_ids and self._pending_row is not None and len(self._header) >= 12: + samples_raw = self._pending_row[-1] if self._pending_row else "" + payload_count = len(samples_raw.split()) if samples_raw else 0 + declared_ids = [f"SAMPLE_{index}" for index in range(1, payload_count + 1)] + + uri_id_counts: dict[str, int] = {} + columns: list[SampleColumn] = [] + for index, sample_id in enumerate(declared_ids, start=1): + uri_id_base = _sample_id_to_uri_id(sample_id, index) + uri_id_counts[uri_id_base] = uri_id_counts.get(uri_id_base, 0) + 1 + occurrence = uri_id_counts[uri_id_base] + uri_id = f"{uri_id_base}_{occurrence}" if occurrence > 1 else uri_id_base + columns.append(SampleColumn(index, sample_id, uri_id)) + self.columns = tuple(columns) + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self._handle is not None: + self._handle.close() + self._handle = None + self._reader = None + + def _next_nonempty_row(self) -> list[str] | None: + if self._reader is None: + return None + for row in self._reader: + if row: + return row + return None + + def __iter__(self): + if self._reader is None: + raise RuntimeError("SampleRecordStream must be used as a context manager") + pending = self._pending_row + self._pending_row = None + if pending is not None: + yield self._parse_row(pending) + for row in self._reader: + if row: + yield self._parse_row(row) + + def _parse_row(self, row: list[str]) -> ParsedSampleRecord: + if len(row) < len(self._header): + row = row + [""] * (len(self._header) - len(row)) + + source_file = row[0] if len(row) > 0 else "" + if self.source_file and source_file != self.source_file: + raise ValueError( + f"records TSV mixes SOURCE_FILE values '{self.source_file}' and " + f"'{source_file}'" + ) + row_id = row[1] if len(row) > 1 else "" + format_raw = row[10] if len(row) > 10 else "" + samples_raw = row[-1] if len(row) >= 12 else "" + declared_format_keys = format_raw.split(":") if format_raw else [] + sample_payloads = samples_raw.split() if samples_raw else [] + if len(sample_payloads) > len(self.columns): + raise ValueError( + f"record {row_id or '(unknown)'} contains {len(sample_payloads)} sample " + f"payloads but the TSV header declares {len(self.columns)} sample columns" + ) + sample_payloads.extend([""] * (len(self.columns) - len(sample_payloads))) + + raw_sample_values = [ + payload.split(":") if payload else [] for payload in sample_payloads + ] + total_fields = max( + [len(declared_format_keys), *(len(values) for values in raw_sample_values)], + default=0, + ) + format_keys = tuple( + declared_format_keys[index] + if index < len(declared_format_keys) and declared_format_keys[index] + else f"FIELD_{index + 1}" + for index in range(total_fields) + ) + if len(set(format_keys)) != len(format_keys): + raise ValueError( + f"record {row_id or '(unknown)'} contains duplicate FORMAT keys: " + + ":".join(format_keys) + ) + + sample_values = tuple( + tuple(values[index] if index < len(values) else "" for index in range(total_fields)) + for values in raw_sample_values + ) + return ParsedSampleRecord( + source_file=source_file, + row_id=row_id, + format_keys=format_keys, + sample_payloads=tuple(sample_payloads), + sample_values=sample_values, + ) + + +def _append_rdf_atomically(rdf_path: Path, stats: dict, producer): + """Append generated N-Triples and restore the original artifact on failure.""" + original_size = rdf_path.stat().st_size + opener = gzip.open if rdf_path.name.endswith(".gz") else Path.open + output_handle = None + buffer = bytearray() + + def emit(line: str): + nonlocal buffer + buffer.extend(line.encode("utf-8")) + stats["triples"] += 1 + if len(buffer) >= SAMPLE_RDF_BUFFER_BYTES: + output_handle.write(buffer) + buffer = bytearray() + + try: + output_handle = opener(rdf_path, "ab") + producer(emit) + if buffer: + output_handle.write(buffer) + output_handle.close() + output_handle = None + stats["appended_bytes"] = rdf_path.stat().st_size - original_size + return stats + except BaseException: + if output_handle is not None: + try: + output_handle.close() + except OSError: + pass + with rdf_path.open("r+b") as rollback_handle: + rollback_handle.truncate(original_size) + raise + + +def append_dense_sample_rdf( records_tsv: Path, rdf_path: Path, *, progress_interval_records: int = 10_000, ) -> dict: - """Stream the built-in sample mappings directly into an RDF aggregate. + """Append the dense SampleCall/FormatFieldValue representation. This produces the same canonical SampleCall and FormatFieldValue triples as - the default RML maps without first materializing V*S and V*S*F TSV rows. - The aggregate is rolled back to its original byte length if streaming fails. + the default RML maps without materializing V*S and V*S*F helper TSV rows. """ stats = { + "representation": "dense", "records": 0, "sample_calls": 0, "format_values": 0, @@ -1641,96 +1867,35 @@ def append_canonical_sample_rdf( if not rdf_path.is_file(): raise FileNotFoundError(f"RDF aggregate not found for sample streaming: {rdf_path}") - _set_max_csv_field_size() - original_size = rdf_path.stat().st_size - opener = gzip.open if rdf_path.name.endswith(".gz") else Path.open - mode = "ab" - output_handle = None - buffer = bytearray() + with SampleRecordStream(records_tsv) as record_stream: + if not record_stream.columns or not record_stream.source_file: + return stats - def emit(line: str): - nonlocal buffer - buffer.extend(line.encode("utf-8")) - stats["triples"] += 1 - if len(buffer) >= SAMPLE_RDF_BUFFER_BYTES: - output_handle.write(buffer) - buffer = bytearray() - - try: - output_handle = opener(rdf_path, mode) - with records_tsv.open(newline="", encoding="utf-8") as records_handle: - reader = csv.reader(records_handle, delimiter="\t") - header = next(reader, None) - if not header: - output_handle.close() - output_handle = None - stats["appended_bytes"] = rdf_path.stat().st_size - original_size - return stats - - sample_header = header[-1].strip() if len(header) >= 12 else "" - declared_sample_ids = ( - [] - if sample_header == "SAMPLES" - else [token for token in sample_header.split() if token] + def produce(emit): + source_component = _rml_uri_component(record_stream.source_file) + file_uri = f"file://{source_component}" + emit( + f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " + f"<{VCFR_NAMESPACE}DenseRepresentation> .\n" ) + for record in record_stream: + row_component = _rml_uri_component(record.row_id) + call_uri = f"{file_uri}#call/{row_component}" - for row in reader: - if not row: - continue - if len(row) < len(header): - row += [""] * (len(header) - len(row)) - - source_file = row[0] if len(row) > 0 else "" - row_id = row[1] if len(row) > 1 else "" - format_raw = row[10] if len(row) > 10 else "" - samples_raw = row[-1] if len(row) >= 12 else "" - format_keys = format_raw.split(":") if format_raw else [] - sample_payloads = samples_raw.split() if samples_raw else [] - total_samples = max(len(declared_sample_ids), len(sample_payloads)) - if total_samples == 0: - continue - - source_component = _rml_uri_component(source_file) - row_component = _rml_uri_component(row_id) - call_uri = f"file://{source_component}#call/{row_component}" - sample_uri_seen: dict[str, int] = {} - - for sample_idx in range(total_samples): - sample_id = ( - declared_sample_ids[sample_idx] - if sample_idx < len(declared_sample_ids) - else f"SAMPLE_{sample_idx + 1}" - ) - sample_payload = ( - sample_payloads[sample_idx] - if sample_idx < len(sample_payloads) - else "" - ) - sample_uri_id_base = _sample_id_to_uri_id(sample_id, sample_idx + 1) - sample_uri_seen[sample_uri_id_base] = sample_uri_seen.get(sample_uri_id_base, 0) + 1 - duplicate_index = sample_uri_seen[sample_uri_id_base] - sample_uri_id = ( - f"{sample_uri_id_base}_{duplicate_index}" - if duplicate_index > 1 - else sample_uri_id_base - ) - sample_component = _rml_uri_component(sample_uri_id) + for sample_index, sample_column in enumerate(record_stream.columns): + sample_component = _rml_uri_component(sample_column.uri_id) sample_uri = f"file://{source_component}#sample/{row_component}/{sample_component}" emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasSampleCall> <{sample_uri}> .\n") emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleCall> .\n") - emit(f"<{sample_uri}> <{VCFR_NAMESPACE}sampleId> {_ntriples_literal(sample_id)} .\n") + emit( + f"<{sample_uri}> <{VCFR_NAMESPACE}sampleId> " + f"{_ntriples_literal(sample_column.sample_id)} .\n" + ) stats["sample_calls"] += 1 - value_tokens = sample_payload.split(":") if sample_payload else [] - total_fields = max(len(format_keys), len(value_tokens)) - for format_idx in range(total_fields): - format_key = ( - format_keys[format_idx] - if format_idx < len(format_keys) and format_keys[format_idx] - else f"FIELD_{format_idx + 1}" - ) - format_value = value_tokens[format_idx] if format_idx < len(value_tokens) else "" + for format_index, format_key in enumerate(record.format_keys): + format_value = record.sample_values[sample_index][format_index] format_component = _rml_uri_component(format_key) format_uri = f"{sample_uri}/fmt/{format_component}" emit(f"<{sample_uri}> <{VCFR_NAMESPACE}hasFormatValue> <{format_uri}> .\n") @@ -1750,21 +1915,253 @@ def emit(line: str): flush=True, ) - if buffer: - output_handle.write(buffer) - output_handle.close() - output_handle = None - stats["appended_bytes"] = rdf_path.stat().st_size - original_size + return _append_rdf_atomically(rdf_path, stats, produce) + + +def append_canonical_sample_rdf( + records_tsv: Path, + rdf_path: Path, + *, + progress_interval_records: int = 10_000, +) -> dict: + """Backward-compatible name for the dense sample RDF emitter.""" + return append_dense_sample_rdf( + records_tsv, + rdf_path, + progress_interval_records=progress_interval_records, + ) + + +@dataclass(frozen=True) +class FormatDefinition: + """Structured attributes and RDF identity for one FORMAT declaration.""" + + uri: str + field_number: str + description: str + + +def _parse_structured_header_fields(value: str) -> dict[str, str]: + """Parse comma-delimited VCF header attributes while respecting quotes.""" + inner = value.strip() + if inner.startswith("<") and inner.endswith(">"): + inner = inner[1:-1] + + tokens: list[str] = [] + token: list[str] = [] + in_quotes = False + escaped = False + for character in inner: + if escaped: + token.append(character) + escaped = False + elif character == "\\" and in_quotes: + token.append(character) + escaped = True + elif character == '"': + token.append(character) + in_quotes = not in_quotes + elif character == "," and not in_quotes: + tokens.append("".join(token)) + token = [] + else: + token.append(character) + tokens.append("".join(token)) + + fields: dict[str, str] = {} + for item in tokens: + if "=" not in item: + continue + key, raw_value = item.split("=", 1) + parsed_value = raw_value.strip() + if len(parsed_value) >= 2 and parsed_value[0] == parsed_value[-1] == '"': + parsed_value = parsed_value[1:-1] + parsed_value = parsed_value.replace('\\"', '"').replace("\\\\", "\\") + fields[key.strip()] = parsed_value + return fields + + +def _load_format_definitions(header_lines_tsv: Path) -> dict[str, FormatDefinition]: + """Map FORMAT IDs to structured definitions backed by emitted HeaderLine IRIs.""" + definitions: dict[str, FormatDefinition] = {} + if not header_lines_tsv.is_file(): + return definitions + _set_max_csv_field_size() + with header_lines_tsv.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + if (row.get("HEADER_KEY") or "").upper() != "FORMAT": + continue + fields = _parse_structured_header_fields(row.get("HEADER_VALUE") or "") + format_id = fields.get("ID", "").strip() + if not format_id: + continue + source_component = _rml_uri_component(row.get("SOURCE_FILE") or "") + index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") + definitions.setdefault( + format_id, + FormatDefinition( + uri=f"file://{source_component}#header/line/{index_component}", + field_number=fields.get("Number") or ".", + description=( + fields.get("Description") + or f"FORMAT field {format_id} (source declaration has no Description)" + ), + ), + ) + return definitions + + +def append_condensed_sample_rdf( + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, + *, + progress_interval_records: int = 10_000, +) -> dict: + """Append sample-ordered cohort matrices and FORMAT value vectors.""" + stats = { + "representation": "condensed", + "records": 0, + "samples": 0, + "matrices": 0, + "format_vectors": 0, + "format_definitions": 0, + "triples": 0, + "appended_bytes": 0, + } + if not records_tsv.is_file(): return stats - except BaseException: - if output_handle is not None: - try: - output_handle.close() - except OSError: - pass - with rdf_path.open("r+b") as rollback_handle: - rollback_handle.truncate(original_size) - raise + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for sample streaming: {rdf_path}") + + definitions = _load_format_definitions(header_lines_tsv) + with SampleRecordStream(records_tsv) as record_stream: + if not record_stream.columns or not record_stream.source_file: + return stats + + def produce(emit): + source_component = _rml_uri_component(record_stream.source_file) + file_uri = f"file://{source_component}" + sample_set_uri = f"{file_uri}#samples" + emitted_definitions: set[str] = set() + + emit( + f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " + f"<{VCFR_NAMESPACE}CondensedRepresentation> .\n" + ) + emit(f"<{file_uri}> <{VCFR_NAMESPACE}hasSampleSet> <{sample_set_uri}> .\n") + emit(f"<{sample_set_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleSet> .\n") + + for sample_column in record_stream.columns: + sample_component = _rml_uri_component(sample_column.uri_id) + sample_uri = f"{sample_set_uri}/{sample_component}" + emit(f"<{sample_set_uri}> <{VCFR_NAMESPACE}hasSample> <{sample_uri}> .\n") + emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}VCFSample> .\n") + emit( + f"<{sample_uri}> <{VCFR_NAMESPACE}sampleName> " + f"{_ntriples_string_literal(sample_column.sample_id)} .\n" + ) + emit( + f"<{sample_uri}> <{VCFR_NAMESPACE}sampleIndex> " + f'"{sample_column.index}"^^<{XSD_POSITIVE_INTEGER_URI}> .\n' + ) + stats["samples"] += 1 + + for record in record_stream: + if not record.format_keys: + continue + row_component = _rml_uri_component(record.row_id) + call_uri = f"{file_uri}#call/{row_component}" + matrix_uri = f"{call_uri}/matrix" + emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasCallMatrix> <{matrix_uri}> .\n") + emit(f"<{matrix_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}CohortCallMatrix> .\n") + emit( + f"<{matrix_uri}> <{VCFR_NAMESPACE}appliesToSampleSet> " + f"<{sample_set_uri}> .\n" + ) + stats["matrices"] += 1 + + for format_index, format_key in enumerate(record.format_keys): + format_component = _rml_uri_component(format_key) + vector_uri = f"{matrix_uri}/fmt/{format_component}" + definition = definitions.get(format_key) + if definition is None: + definition = FormatDefinition( + uri=f"{file_uri}#header/format/{format_component}", + field_number=".", + description=( + f"Synthesized definition for undeclared FORMAT key {format_key}" + ), + ) + definition_uri = definition.uri + if definition_uri not in emitted_definitions: + emit( + f"<{definition_uri}> <{RDF_TYPE_URI}> " + f"<{VCFR_NAMESPACE}FormatFieldDefinition> .\n" + ) + emit( + f"<{definition_uri}> <{VCFR_NAMESPACE}fieldId> " + f"{_ntriples_string_literal(format_key)} .\n" + ) + emit( + f"<{definition_uri}> <{VCFR_NAMESPACE}fieldNumber> " + f"{_ntriples_string_literal(definition.field_number)} .\n" + ) + emit( + f"<{definition_uri}> <{VCFR_NAMESPACE}fieldDescription> " + f"{_ntriples_string_literal(definition.description)} .\n" + ) + emitted_definitions.add(definition_uri) + stats["format_definitions"] += 1 + + encoded_values = "\t".join( + values[format_index] or "." for values in record.sample_values + ) + emit( + f"<{matrix_uri}> <{VCFR_NAMESPACE}hasFormatValueVector> " + f"<{vector_uri}> .\n" + ) + emit(f"<{vector_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatValueVector> .\n") + emit( + f"<{vector_uri}> <{VCFR_NAMESPACE}declaredBy> " + f"<{definition_uri}> .\n" + ) + emit( + f"<{vector_uri}> <{VCFR_NAMESPACE}valueEncoding> " + f"<{VCFR_NAMESPACE}VCFTextVector> .\n" + ) + emit( + f"<{vector_uri}> <{VCFR_NAMESPACE}encodedValues> " + f"{_ntriples_string_literal(encoded_values)} .\n" + ) + stats["format_vectors"] += 1 + + stats["records"] += 1 + if progress_interval_records > 0 and stats["records"] % progress_interval_records == 0: + print( + " * Condensed sample RDF streaming: " + f"{stats['records']:,} variants, {stats['format_vectors']:,} vectors", + flush=True, + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def emit_sample_representation( + workflow: SampleWorkflow, + *, + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, +) -> dict | None: + """Execute the workflow's sole direct RDF emitter, if it has one.""" + if workflow.emitter is None: + return None + if workflow.emitter == "dense": + return append_dense_sample_rdf(records_tsv, rdf_path) + if workflow.emitter == "condensed": + return append_condensed_sample_rdf(records_tsv, header_lines_tsv, rdf_path) + raise RuntimeError(f"unknown sample RDF emitter: {workflow.emitter}") def update_conversion_metrics_after_sample_stream( @@ -1792,6 +2189,8 @@ def update_conversion_metrics_after_sample_stream( else: artifacts["output_triples"] = int(total_triples) artifacts["output_size_bytes"] = output_size + payload["sample_representation"] = sample_stats + # Retained for consumers of pre-condensed conversion metrics. payload["sample_streaming"] = sample_stats metrics_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") except (OSError, json.JSONDecodeError): @@ -1837,92 +2236,31 @@ def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_ if not records_tsv.exists(): return - _set_max_csv_field_size() - with records_tsv.open(newline="", encoding="utf-8") as records_handle: - reader = csv.reader(records_handle, delimiter="\t") - header = next(reader, None) - if not header: - return - - sample_header = header[-1].strip() if len(header) >= 12 else "" - declared_sample_ids = ( - [] - if sample_header == "SAMPLES" - else [token for token in sample_header.split() if token] - ) - - for row in reader: - if not row: - continue - if len(row) < len(header): - row = row + [""] * (len(header) - len(row)) - - source_file = row[0] if len(row) > 0 else "" - row_id = row[1] if len(row) > 1 else "" - format_raw = row[10] if len(row) > 10 else "" - samples_raw = row[-1] if len(row) >= 12 else "" - - format_keys = [token for token in format_raw.split(":")] if format_raw else [] - sample_payloads = [token for token in samples_raw.split()] if samples_raw else [] - - total_samples = max(len(declared_sample_ids), len(sample_payloads)) - if total_samples == 0: - continue - - sample_uri_seen: dict[str, int] = {} - for sample_idx in range(total_samples): - sample_id = ( - declared_sample_ids[sample_idx] - if sample_idx < len(declared_sample_ids) - else f"SAMPLE_{sample_idx + 1}" - ) - sample_payload = ( - sample_payloads[sample_idx] - if sample_idx < len(sample_payloads) - else "" - ) - sample_index_value = str(sample_idx + 1) - sample_uri_id_base = _sample_id_to_uri_id(sample_id, sample_idx + 1) - sample_uri_seen[sample_uri_id_base] = sample_uri_seen.get(sample_uri_id_base, 0) + 1 - if sample_uri_seen[sample_uri_id_base] > 1: - sample_uri_id = f"{sample_uri_id_base}_{sample_uri_seen[sample_uri_id_base]}" - else: - sample_uri_id = sample_uri_id_base - + with SampleRecordStream(records_tsv) as record_stream: + for record in record_stream: + for sample_offset, sample_column in enumerate(record_stream.columns): sample_calls_writer.writerow( [ - source_file, - row_id, - sample_index_value, - sample_id, - sample_uri_id, - sample_payload, + record.source_file, + record.row_id, + str(sample_column.index), + sample_column.sample_id, + sample_column.uri_id, + record.sample_payloads[sample_offset], ] ) - value_tokens = sample_payload.split(":") if sample_payload else [] - total_fields = max(len(format_keys), len(value_tokens)) - for format_idx in range(total_fields): - format_key = ( - format_keys[format_idx] - if format_idx < len(format_keys) and format_keys[format_idx] - else f"FIELD_{format_idx + 1}" - ) - format_value = ( - value_tokens[format_idx] - if format_idx < len(value_tokens) - else "" - ) + for format_offset, format_key in enumerate(record.format_keys): sample_format_writer.writerow( [ - source_file, - row_id, - sample_index_value, - sample_id, - sample_uri_id, - str(format_idx + 1), + record.source_file, + record.row_id, + str(sample_column.index), + sample_column.sample_id, + sample_column.uri_id, + str(format_offset + 1), format_key, - format_value, + record.sample_values[sample_offset][format_offset], ] ) @@ -3781,6 +4119,7 @@ def run_full_mode( metrics_dir: Path, image_ref: str, out_name: str, + sample_workflow: SampleWorkflow, rdf_storage_mode: str, methods: list[str], hdt_strategy: str, @@ -3800,13 +4139,13 @@ def run_full_mode( print("Step 3/5: Processing per-input pipeline (TSV -> RDF -> compression)") if spark_partitions is not None: print(f" Spark partition hint: {spark_partitions}") + print(f" Sample representation: {sample_workflow.representation}") intermediate_dir = tsv_dir.parent ensure_dir(tsv_dir) ensure_dir(out_dir) ensure_dir(metrics_dir) selected_methods = list(methods) - sample_strategy = sample_support_strategy(rules_path) use_partitioned_hdt = should_use_partitioned_hdt( mode="full", methods=selected_methods, @@ -3879,7 +4218,7 @@ def fail_current(stage: str, message: str): # Pre-flight write checks for expected TSV outputs to fail fast on # permission/mount problems before starting container work. expected_tsv_suffixes = ["records.tsv", "header_lines.tsv", "file_metadata.tsv"] - if sample_strategy != "none": + if sample_workflow.helper_strategy != "none": expected_tsv_suffixes.extend(["sample_calls.tsv", "sample_format_values.tsv"]) for suffix in expected_tsv_suffixes: expected_tsv_output = tsv_dir / f"{expected_prefix}.{suffix}" @@ -3937,16 +4276,20 @@ def fail_current(stage: str, message: str): sample_calls_tsv = tsv_dir / f"{prefix}.sample_calls.tsv" sample_format_tsv = tsv_dir / f"{prefix}.sample_format_values.tsv" try: - if sample_strategy == "expanded": + if sample_workflow.helper_strategy == "expanded": build_sample_support_tsvs( records_tsv=triplet["records"], sample_calls_tsv=sample_calls_tsv, sample_format_tsv=sample_format_tsv, ) - elif sample_strategy == "stream": + elif sample_workflow.helper_strategy == "header-only": # RMLStreamer sees valid, empty canonical sources. Their # equivalent triples are appended directly after base mapping. write_sample_support_headers(sample_calls_tsv, sample_format_tsv) + elif sample_workflow.helper_strategy != "none": + raise RuntimeError( + f"unknown sample helper strategy: {sample_workflow.helper_strategy}" + ) except Exception as exc: fail_current( "tsv-derivation", @@ -4104,27 +4447,41 @@ def fail_current(stage: str, message: str): continue sample_stats = None - if sample_strategy == "stream": - print(" * Streaming canonical multi-sample RDF (no expanded helper TSVs)") + if sample_workflow.emitter is not None: + print( + f" * Streaming {sample_workflow.representation} multi-sample RDF " + "(no expanded helper TSVs)" + ) try: - sample_stats = append_canonical_sample_rdf( + sample_stats = emit_sample_representation( + sample_workflow, records_tsv=triplet["records"], + header_lines_tsv=triplet["headers"], rdf_path=raw_rdf_files[0], ) except Exception as exc: fail_current( - "sample-rdf-streaming", - f"failed streaming sample RDF for '{prefix}': {exc}. " + f"{sample_workflow.representation}-sample-rdf-streaming", + f"failed streaming {sample_workflow.representation} sample RDF " + f"for '{prefix}': {exc}. " f"See log: {wrapper_log_path}", ) continue if triples_produced is not None: triples_produced += int(sample_stats["triples"]) - print( - " * Sample calls streamed: " - f"{sample_stats['sample_calls']:,}; FORMAT values: " - f"{sample_stats['format_values']:,}" - ) + if sample_workflow.representation == "dense": + print( + " * Sample calls streamed: " + f"{sample_stats['sample_calls']:,}; FORMAT values: " + f"{sample_stats['format_values']:,}" + ) + else: + print( + " * Condensed matrices streamed: " + f"{sample_stats['matrices']:,}; FORMAT vectors: " + f"{sample_stats['format_vectors']:,}; reusable samples: " + f"{sample_stats['samples']:,}" + ) if triples_produced is None: triples_produced = count_triples_in_nt_files(raw_rdf_files) @@ -4909,6 +5266,9 @@ def main(): " Full pipeline (space-optimized aggregate):\n" " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " "--representations hdt,cottas --rdf-compression none -o ./results\n" + " Condensed multi-sample representation:\n" + " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --sample-representation condensed " + "--rdf-storage-mode space-optimized --representations hdt -o ./results\n" " Space-optimized full pipeline with shared HDT/COTTAS chunks:\n" " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " "--representations hdt,cottas --rdf-compression none " @@ -4980,6 +5340,16 @@ def main(): default=None, help="RML mapping rules .ttl (default: /rules/default_rules.ttl)", ) + parser.add_argument( + "--sample-representation", + choices=sorted(SAMPLE_REPRESENTATION_CHOICES), + default="dense", + help=( + "Genotype representation for full mode: dense emits one SampleCall and " + "FORMAT value resource per sample; condensed emits a shared SampleSet and " + "one sample-ordered value vector per FORMAT key (default: dense)" + ), + ) parser.add_argument( "--rdf-storage-mode", choices=sorted(RDF_STORAGE_MODES), @@ -5160,6 +5530,10 @@ def main(): rules_path = Path(args.rules).expanduser().resolve() if not rules_path.exists() or not rules_path.is_file(): raise ValueError(f"rules file not found: {rules_path}") + sample_workflow = resolve_sample_workflow( + args.sample_representation, + rules_path, + ) validate_mode_dirs([out_root, out_dir, tsv_dir, metrics_root]) if args.legacy_compression is not None: if ( @@ -5482,6 +5856,7 @@ def execute_mode(): metrics_dir=metrics_dir, image_ref=image_ref, out_name=args.out_name, + sample_workflow=sample_workflow, rdf_storage_mode=args.rdf_storage_mode, methods=full_methods, hdt_strategy=args.hdt_strategy, diff --git a/vcf_rdfizer_data/rules/default_rules.ttl b/vcf_rdfizer_data/rules/default_rules.ttl index ba357f4..7a0ee2c 100644 --- a/vcf_rdfizer_data/rules/default_rules.ttl +++ b/vcf_rdfizer_data/rules/default_rules.ttl @@ -11,9 +11,12 @@ # - /data/tsv/.file_metadata.tsv # - /data/tsv/.header_lines.tsv # - /data/tsv/.records.tsv -# - /data/tsv/.sample_calls.tsv (derived by wrapper) -# - /data/tsv/.sample_format_values.tsv (derived by wrapper) +# - /data/tsv/.sample_calls.tsv (header-only compatibility source) +# - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. +# The wrapper selects exactly one direct sample emitter: dense SampleCall / +# FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / +# FormatValueVector triples. The helper tables stay header-only in either mode. # # Output format: # - The default conversion now targets triples (N-Triples) without named graph terms. From 5caef5db2dcef98e022ea2765a0185f5791af8e4 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Wed, 2 Sep 2026 09:26:06 +0200 Subject: [PATCH 07/19] remove legacy java hdtcat invocation --- Dockerfile | 12 +--- README.md | 55 ++++++++------ THIRD_PARTY_NOTICES.md | 4 +- src/partitioned_compression.py | 87 +++++++++++++++++++---- test/test_partitioned_compression_unit.py | 36 ++++++++++ test/test_vcf_rdfizer_unit.py | 8 +++ vcf_rdfizer.py | 13 +++- 7 files changed, 163 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0562118..d0177f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ ARG RMLSTREAMER_VERSION=2.5.0 -ARG HDT_JAVA_PACKAGE_VERSION=3.0.10 ARG HDTC_VERSION=1.1.0 FROM eclipse-temurin:11-jre AS build-hdt-cpp @@ -62,7 +61,6 @@ RUN cargo build --locked --release \ FROM eclipse-temurin:11-jre ARG RMLSTREAMER_VERSION -ARG HDT_JAVA_PACKAGE_VERSION RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -91,13 +89,6 @@ RUN mkdir -p /opt/rmlstreamer \ -o /opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar \ https://github.com/RMLio/RMLStreamer/releases/download/v${RMLSTREAMER_VERSION}/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar -RUN mkdir -p /opt/hdt-java \ - && curl -fsSL \ - -o /tmp/hdt-java-package.tar.gz \ - https://repo1.maven.org/maven2/org/rdfhdt/hdt-java-package/${HDT_JAVA_PACKAGE_VERSION}/hdt-java-package-${HDT_JAVA_PACKAGE_VERSION}-distribution.tar.gz \ - && tar -xzf /tmp/hdt-java-package.tar.gz -C /opt/hdt-java --strip-components=1 \ - && rm -f /tmp/hdt-java-package.tar.gz - COPY --from=build-hdt-cpp /usr/local/bin/rdf2hdt /usr/local/bin/rdf2hdt COPY --from=build-hdt-cpp /usr/local/bin/hdt2rdf /usr/local/bin/hdt2rdf COPY --from=build-hdt-cpp /usr/local/lib/libcds* /usr/local/lib/ @@ -110,16 +101,15 @@ COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ - && find /opt/hdt-java/bin -type f -exec chmod +x {} \; \ && chmod +x /usr/local/bin/rdf2hdt \ && chmod +x /usr/local/bin/hdt2rdf \ && chmod +x /usr/local/bin/hdtc ENV RMLSTREAMER_JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar -ENV HDT_JAVA_HOME=/opt/hdt-java ENV HDTC_BIN=/usr/local/bin/hdtc ENV HDT_INDEX_MEMORY_LIMIT=512M +ENV HDT_MERGE_MEMORY_LIMIT=512M ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python diff --git a/README.md b/README.md index 5e1dfc5..0772b1d 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ host filesystem. - `--representations {hdt,cottas,none}` queryable primary representations - `--artifact-compression {gzip,brotli,none}` optional packaging applied to each selected representation - `--hdt-strategy {auto,partitioned,single}` - - `auto`: in full mode, build smaller HDT chunks and merge them with `HDTCat` + - `auto`: in full mode, build smaller HDT chunks and merge them with native `hdtc` - `partitioned`: always use chunked HDT generation for HDT-based methods - `single`: always use one `rdf2hdt` run per RDF input - with `space-optimized`, use `auto` or `partitioned`; `single` cannot consume the gzip stream without expanding it @@ -269,7 +269,7 @@ vcf-rdfizer \ --out ./results ``` -Full pipeline (plain aggregate, chunked HDT + HDTCat merge): +Full pipeline (plain aggregate, chunked HDT + native `hdtc` merge): ```bash vcf-rdfizer \ @@ -431,9 +431,11 @@ versioned sidecar beside the input. COTTAS indexing rewrites the existing the same artifact while rebuilding its embedded index. If the operation fails, the original COTTAS file is left in place; HDT's previous sidecars are restored. -HDT indexing does not start Java. It uses `hdtc index`, which streams the HDT -through disk-backed external sorters. The default soft memory budget is 512 MiB; -override it for any wrapper mode with an environment variable such as: +HDT merging and indexing do not start a Java HDT tool. They use native `hdtc`: +`hdtc create` merges partitioned HDTs and `hdtc index` streams an HDT through +disk-backed external sorters. Both default to a 512 MiB soft memory budget. +Override index creation for any wrapper mode with an environment variable such +as: ```bash HDT_INDEX_MEMORY_LIMIT=2G vcf-rdfizer \ @@ -442,10 +444,11 @@ HDT_INDEX_MEMORY_LIMIT=2G vcf-rdfizer \ --out ./results ``` +Use `HDT_MERGE_MEMORY_LIMIT` in the same way to tune a partitioned merge. Accepted values use an `M` or `G` suffix. Lower values reduce in-memory sort -buffers and may increase temporary I/O; higher values can improve indexing -speed when memory is available. Temporary files live in the container's -`/work` area and are removed after the attempt. +buffers and may increase temporary I/O; higher values can improve performance +when memory is available. Temporary files live in the container's `/work` area +and are removed after the attempt. In `full` mode, an HDT sidecar-index failure is non-fatal when the HDT data itself remains readable; the run continues and the HDT can be repaired later @@ -551,8 +554,9 @@ into complete N-Triples records. Only one uncompressed chunk is present at a time: it is consumed by both converters and removed before the next chunk is read. This is especially important for `space-optimized` `.nt.gz` aggregates, which must not be expanded into a second full raw-RDF copy. HDT chunks are -merged with `HDTCat`, the final HDT index is generated after merging with the -Java-free `hdtc index` command, and COTTAS chunks are merged with +merged with the Java-free `hdtc create` command, which accepts existing HDT +inputs; the final HDT index is generated after merging with `hdtc index`. +COTTAS chunks are merged with `pycottas.cat`, which rebuilds the query indexes for the merged representation. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a @@ -570,20 +574,22 @@ Each COTTAS conversion and merge also receives a fresh container-local DuckDB workspace, which is removed as soon as that operation completes. This prevents state from one chunk being reused by another and requires no user configuration. -HDT index generation uses the pinned Rust `hdtc` 1.1.0 executable, not -`hdtSearch.sh` or another Java process. `hdtc index` reads BitmapTriples as a -stream and builds the object/predicate orderings with disk-backed external -sorts. This avoids the JVM heap path that can fail with -`java.lang.OutOfMemoryError` while producing the same canonical HDT v1-1 -sidecar, `.hdt.index.v1-1`, used by hdt-java and hdt-cpp. +HDT merging and index generation use the pinned Rust `hdtc` 1.1.0 executable, +not `hdtCat`, `hdtSearch.sh`, or another Java HDT process. `hdtc create` +merges the chunk HDTs with disk-backed external sorts, and `hdtc index` reads +BitmapTriples as a stream to build the object/predicate orderings. This avoids +the JVM heap path that can fail with `java.lang.OutOfMemoryError` while +producing the same canonical HDT v1-1 sidecar, +`.hdt.index.v1-1`, used by hdt-java and hdt-cpp. For standalone index mode, existing versioned sidecars are moved aside while regeneration runs and restored if indexing fails. Incomplete replacements are removed before restoration, so a failed or interrupted attempt does not leave a partial index. Sort runs use `/work` and are removed when the command exits. -The image defaults `HDT_INDEX_MEMORY_LIMIT` to `512M`; the wrapper forwards a -host value of that variable into standalone, partitioned, and full-run Docker -commands. `HDT_INDEX_WORK_ROOT` can override the scratch root when invoking +The image defaults both `HDT_INDEX_MEMORY_LIMIT` and +`HDT_MERGE_MEMORY_LIMIT` to `512M`. The wrapper forwards an explicitly set +host value of either variable into the relevant Docker command. +`HDT_INDEX_WORK_ROOT` can override the scratch root when invoking `/opt/vcf-rdfizer/ensure_hdt_index.sh` directly inside the container. COTTAS does not expose a separate index sidecar. Its index is part of the @@ -621,10 +627,13 @@ If Docker permission issues occur, rerun with a Docker-allowed user (or configur If HDT compression fails on very large RDF files, use `--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with `--hdt-strategy partitioned`, then lower `--chunk-target-bytes` and -`--chunk-max-bytes` to reduce each converter's working set. Final HDT index -creation is disk-backed, so ensure the Docker data volume has enough temporary -space for the external sort; free space in the output filesystem alone does -not increase the JVM heap. +`--chunk-max-bytes` to reduce each converter's working set. Both final HDT +merge and index creation are disk-backed, so ensure the Docker data volume has +enough temporary space for their external sorts. To reduce their bounded +in-memory buffers further, set `HDT_MERGE_MEMORY_LIMIT` and/or +`HDT_INDEX_MEMORY_LIMIT` (for example, `512M`); lower limits can require more +temporary I/O. Free space in the output filesystem alone does not increase the +Docker volume capacity. Safe termination: diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index ab06f77..2dfbbef 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -29,8 +29,8 @@ their respective authors and apply to those components. 4. `hdtc` - Repository: -- Usage in this project: Java-free, disk-backed index generation for existing - HDT files +- Usage in this project: Java-free, disk-backed merging of HDT chunks and + index generation for existing HDT files - Upstream license: MIT - License files copied into image: - `/usr/share/licenses/vcf-rdfizer/HDTC.LICENSE` diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index aaf1d29..1ba57a9 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -24,12 +24,6 @@ HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} -HDT_CAT_CANDIDATES = ( - "hdtCat", - "hdtCat.sh", - "/opt/hdt-java/bin/hdtCat.sh", - "/opt/hdt-java/bin/hdtCat", -) def is_triple_line(line: bytes) -> bool: @@ -223,6 +217,42 @@ def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: return None +def hdtc_merge_temp_dir(work_dir: Path, merged_path: Path) -> Path: + """Return the isolated disk workspace for one hdtc merge stage.""" + return work_dir / f".{merged_path.stem}.hdtc-work" + + +def hdtc_merge_command( + hdtc_bin: str, + left: Path, + right: Path, + merged: Path, + *, + work_dir: Path, + memory_limit: str, +) -> list[str]: + """Build a bounded-memory native merge command for two HDT chunks. + + ``hdtc create`` accepts existing ``.hdt`` inputs, so it performs the same + logical merge as hdt-java's ``hdtCat`` without constructing Java HashMaps. + Its temporary files are kept in a per-stage directory so they can be + removed immediately after each pairwise merge. + """ + return [ + hdtc_bin, + "--quiet", + "create", + str(left), + str(right), + "--output", + str(merged), + "--memory-limit", + memory_limit, + "--temp-dir", + str(hdtc_merge_temp_dir(work_dir, merged)), + ] + + def parse_time_log(path: Path) -> dict: if not path.exists(): return {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None} @@ -323,6 +353,7 @@ def merge_pairwise( runner: StageRunner, merge_command, total: dict, + cleanup_merged_workspace=None, ) -> tuple[Path | None, int]: rounds = 0 current = list(paths) @@ -336,11 +367,15 @@ def merge_pairwise( continue right = current[pair_index + 1] merged = left.parent / f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}{left.suffix}" - stage = runner.run( - f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}", - merge_command(left, right, merged), - merged, - ) + try: + stage = runner.run( + f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}", + merge_command(left, right, merged), + merged, + ) + finally: + if cleanup_merged_workspace is not None: + cleanup_merged_workspace(merged) add_totals(total, stage) if stage["exit_code"] != 0: return None, rounds @@ -436,7 +471,21 @@ def skipped_cottas_result(method: str) -> dict: if needs_hdt else None ) - hdt_cat = resolve_executable(HDT_CAT_CANDIDATES, "HDTCat") if needs_hdt else None + hdtc_bin = ( + resolve_executable( + ( + os.environ.get("HDTC_BIN", ""), + "/usr/local/bin/hdtc", + "hdtc", + ), + "hdtc HDT merger", + ) + if needs_hdt + else None + ) + hdt_merge_memory_limit = os.environ.get("HDT_MERGE_MEMORY_LIMIT", "512M").strip() + if needs_hdt and not hdt_merge_memory_limit: + raise ValueError("HDT_MERGE_MEMORY_LIMIT must be a non-empty hdtc memory size") cottas_python = os.environ.get("COTTAS_PYTHON_BIN") or shutil.which("python3") if any(method in COTTAS_METHODS for method in methods) and not cottas_python: raise RuntimeError("Missing Python runtime for COTTAS") @@ -562,11 +611,21 @@ def validate_artifact( hdt_paths, prefix="hdt", runner=runner, - merge_command=lambda left, right, merged: [hdt_cat, str(left), str(right), str(merged)], + merge_command=lambda left, right, merged: hdtc_merge_command( + hdtc_bin, + left, + right, + merged, + work_dir=work_dir, + memory_limit=hdt_merge_memory_limit, + ), total=hdt_total, + cleanup_merged_workspace=lambda merged: shutil.rmtree( + hdtc_merge_temp_dir(work_dir, merged), ignore_errors=True + ), ) if final_hdt is None: - raise RuntimeError("HDTCat merge failed") + raise RuntimeError("hdtc HDT merge failed") shutil.copyfile(final_hdt, output_hdt) final_hdt.unlink(missing_ok=True) index_stage = runner.run( diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index 8d79582..7d7fe75 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -20,6 +20,42 @@ def load_runner_module(): class PartitionedCompressionUnitTests(VerboseTestCase): + def test_hdtc_merge_command_uses_bounded_native_merge(self): + """Partitioned HDT merges must not route through hdt-java's hdtCat.""" + runner = load_runner_module() + with tempfile.TemporaryDirectory() as td: + work_dir = Path(td) / "work" + left = work_dir / "chunk-00000.hdt" + right = work_dir / "chunk-00001.hdt" + merged = work_dir / "hdt-merge-r01-00000.hdt" + command = runner.hdtc_merge_command( + "/usr/local/bin/hdtc", + left, + right, + merged, + work_dir=work_dir, + memory_limit="512M", + ) + + self.assertEqual( + command, + [ + "/usr/local/bin/hdtc", + "--quiet", + "create", + str(left), + str(right), + "--output", + str(merged), + "--memory-limit", + "512M", + "--temp-dir", + str(work_dir / ".hdt-merge-r01-00000.hdtc-work"), + ], + ) + self.assertNotIn("java", " ".join(command).lower()) + self.assertNotIn("hdtcat", " ".join(command).lower()) + def test_stream_chunks_only_retains_the_chunk_being_consumed(self): """Gzip chunking does not stage a second full uncompressed aggregate.""" runner = load_runner_module() diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index df66171..2cb8941 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -265,6 +265,14 @@ def test_hdt_index_memory_limit_is_forwarded_to_docker(self): ["-e", "HDT_INDEX_MEMORY_LIMIT=2G"], ) + def test_hdt_merge_memory_limit_is_forwarded_to_docker(self): + """A host hdtc merge-memory override is passed to partitioned containers.""" + with mock.patch.dict(os.environ, {"HDT_MERGE_MEMORY_LIMIT": "768M"}): + self.assertEqual( + vcf_rdfizer.docker_hdt_merge_env_args(), + ["-e", "HDT_MERGE_MEMORY_LIMIT=768M"], + ) + def test_validator_counts_plain_and_gzip_ntriples(self): """The Docker validator's fallback source count handles .nt and .nt.gz.""" validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 3b40d22..aed78b4 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -433,6 +433,14 @@ def docker_hdt_index_env_args() -> list[str]: return ["-e", f"HDT_INDEX_MEMORY_LIMIT={memory_limit}"] +def docker_hdt_merge_env_args() -> list[str]: + """Forward an optional host-side hdtc merge memory budget into Docker.""" + memory_limit = os.environ.get("HDT_MERGE_MEMORY_LIMIT", "").strip() + if not memory_limit: + return [] + return ["-e", f"HDT_MERGE_MEMORY_LIMIT={memory_limit}"] + + def _can_write_dir(path: Path) -> bool: """Best-effort write probe for directories.""" try: @@ -2457,7 +2465,7 @@ def should_use_partitioned_hdt( hdt_strategy: str, rdf_storage_mode: str | None = None, ) -> bool: - """Resolve whether the HDT pipeline should use chunked generation + HDTCat.""" + """Resolve whether the HDT pipeline should use chunked generation + hdtc merge.""" if not compression_uses_hdt(methods): return False if hdt_strategy == "single": @@ -3934,6 +3942,7 @@ def run_containerized_partitioned_representation_methods( command = [ *docker_run_base(), *docker_hdt_index_env_args(), + *docker_hdt_merge_env_args(), "--mount", f"type=volume,source={volume_name},target=/work", ] @@ -5435,7 +5444,7 @@ def main(): choices=sorted(HDT_STRATEGY_CHOICES), default=DEFAULT_HDT_STRATEGY, help=( - "HDT generation strategy: auto uses partitioned HDT+HDTCat for full-mode aggregate storage, " + "HDT generation strategy: auto uses partitioned HDT+hdtc merge for full-mode aggregate storage, " "single uses one rdf2hdt run, partitioned forces chunked HDT generation" ), ) From 87ad26d8b1b00b4fa645547a67a08c69fb7c6aab Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Wed, 2 Sep 2026 18:22:21 +0200 Subject: [PATCH 08/19] improved condensed COTTAS index generation workflow --- Dockerfile | 2 +- README.md | 19 ++- THIRD_PARTY_NOTICES.md | 1 + changelog.md | 30 ++++ src/cottas_tool.py | 31 ++++ src/partitioned_compression.py | 169 ++++++++++++++++++++-- test/test_cottas_tool.py | 46 ++++++ test/test_partitioned_compression_unit.py | 39 +++++ 8 files changed, 322 insertions(+), 15 deletions(-) create mode 100644 changelog.md diff --git a/Dockerfile b/Dockerfile index d0177f7..188cea7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN python3 -m venv /opt/pycottas-venv \ - && /opt/pycottas-venv/bin/pip install --no-cache-dir pycottas + && /opt/pycottas-venv/bin/pip install --no-cache-dir pycottas==1.1.0 RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ diff --git a/README.md b/README.md index 0772b1d..c981709 100644 --- a/README.md +++ b/README.md @@ -556,8 +556,14 @@ read. This is especially important for `space-optimized` `.nt.gz` aggregates, which must not be expanded into a second full raw-RDF copy. HDT chunks are merged with the Java-free `hdtc create` command, which accepts existing HDT inputs; the final HDT index is generated after merging with `hdtc index`. -COTTAS chunks are merged with -`pycottas.cat`, which rebuilds the query indexes for the merged representation. +COTTAS chunks are merged with one multi-input `pycottas.cat` pass, which +rebuilds the query index once for the complete representation. This avoids the +repeated re-indexing and temporary-file amplification of a pairwise merge tree +on large multi-sample graphs. If that stage fails in full mode, the warning +contains the failing exit code and, when available, a `stderr_tail` from the +COTTAS/DuckDB command. The same warning records the Docker workspace free-space +sample before and after the stage; +the raw RDF remains available for a retry. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a streaming decode/count check. This verifies both readability and that the @@ -624,6 +630,15 @@ finishes. If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). +If COTTAS indexing/merging fails on a very large RDF file, first inspect the +`cottas-merge-all` stage in the raw partitioned-compression metrics JSON and the +run wrapper log. The warning's `stderr_tail` distinguishes a DuckDB/COTTAS +error from an operating-system resource failure. The common resource remedy is +to reduce `--chunk-target-bytes` (and `--chunk-max-bytes`) and ensure the +Docker data volume has enough free space for the temporary DuckDB files and +the final Parquet rewrite. Rebuild the image after upgrading so the pinned +`pycottas==1.1.0` dependency and the multi-input merge adapter are installed. + If HDT compression fails on very large RDF files, use `--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with `--hdt-strategy partitioned`, then lower `--chunk-target-bytes` and diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 2dfbbef..c93ec8b 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -25,6 +25,7 @@ their respective authors and apply to those components. - Repository: - Usage in this project: RDF chunk compression and indexed COTTAS merging - Upstream license: Apache License 2.0 +- Version bundled by the Docker image: `1.1.0` - Installed in the image's `/opt/pycottas-venv` environment 4. `hdtc` diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..cca6e82 --- /dev/null +++ b/changelog.md @@ -0,0 +1,30 @@ +# Changelog + +## 2026-09-02 — COTTAS large-input merge reliability + +- Diagnosed the previous generic `COTTAS merge/index creation failed` warning: + the warning is emitted only after per-chunk COTTAS conversion succeeds and + the final indexed merge returns a non-zero status. HDT generation/indexing + is independent and is not implicated by that warning. +- Replaced the partitioned COTTAS pairwise merge tree with one multi-input + `pycottas.cat` pass. The query index is now built once, avoiding repeated + Parquet rewrites and reducing peak temporary-file use for large VCF-derived + graphs. +- Added cleanup of all COTTAS chunks and merge outputs after a failed COTTAS + attempt so stale intermediates cannot consume the Docker workspace. +- Preserved a bounded stderr tail and exit code for every partitioned stage. + Full-run `index_warnings.json` entries now identify resource/OOM signals, + include before/after Docker-workspace free-space samples, and retain the + underlying COTTAS/DuckDB diagnostic when available. +- Pinned the Docker image to `pycottas==1.1.0` for reproducible behavior. +- Added regression tests for the multi-input merge command, stage diagnostics, + and isolated COTTAS scratch cleanup. + +### Rerun guidance + +Rebuild the image and rerun the same full command. If the COTTAS stage still +fails, inspect `raw_metrics/compression_metrics/.../__partitioned_compression__` +and the run wrapper log for `cottas-merge-all` and its `stderr_tail`. For a +resource error, lower `--chunk-target-bytes` and `--chunk-max-bytes`, or enlarge +the Docker data-volume allocation. The raw `.nt.gz` retained by the failed run +is a valid recovery source. diff --git a/src/cottas_tool.py b/src/cottas_tool.py index 97ddf90..dca57b8 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -42,6 +42,19 @@ def main() -> int: merge.add_argument("cottas_path") merge.add_argument("index", nargs="?", default="spo") + merge_many = subparsers.add_parser( + "merge-many", + help="merge multiple COTTAS files in one indexed pass", + ) + merge_many.add_argument( + "--input-cottas-files", + nargs="+", + required=True, + help="COTTAS inputs to merge", + ) + merge_many.add_argument("--output-cottas-file", required=True) + merge_many.add_argument("--index", default="spo") + reindex = subparsers.add_parser( "reindex", help="rebuild the embedded COTTAS query index in place", @@ -118,6 +131,24 @@ def main() -> int: temporary_path.unlink(missing_ok=True) return 0 + if args.command == "merge-many": + input_paths = [str(Path(path).resolve()) for path in args.input_cottas_files] + cottas_path = str(Path(args.output_cottas_file).resolve()) + if len(input_paths) < 2: + print("merge-many requires at least two input COTTAS files", file=sys.stderr) + return 2 + with cottas_scratch_workspace(): + # pycottas.cat accepts a list of inputs and computes the requested + # index once. This is materially cheaper for large partitioned + # graphs than repeatedly re-indexing pairwise intermediate files. + pycottas.cat( + input_paths, + cottas_path, + index=args.index, + remove_input_files=True, + ) + return 0 + left_path = str(Path(args.left_path).resolve()) right_path = str(Path(args.right_path).resolve()) cottas_path = str(Path(args.cottas_path).resolve()) diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 1ba57a9..5335987 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -24,6 +24,7 @@ HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} +STDERR_TAIL_BYTES = 16 * 1024 def is_triple_line(line: bytes) -> bool: @@ -253,6 +254,31 @@ def hdtc_merge_command( ] +def cottas_merge_many_command( + python_bin: str, + inputs: list[Path], + merged: Path, +) -> list[str]: + """Build one multi-input COTTAS merge command. + + ``pycottas.cat`` accepts a list of COTTAS paths and builds the requested + zonemap index once. Calling it once for the complete chunk set avoids the + repeated re-indexing and temporary-file amplification caused by a + pairwise merge tree on large graphs. + """ + return [ + python_bin, + "/opt/vcf-rdfizer/cottas_tool.py", + "merge-many", + "--input-cottas-files", + *(str(path) for path in inputs), + "--output-cottas-file", + str(merged), + "--index", + "spo", + ] + + def parse_time_log(path: Path) -> dict: if not path.exists(): return {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None} @@ -291,24 +317,65 @@ def run( stdout_path: Path | None = None, ) -> dict: time_path = self.work_dir / f".{name}.time" + stderr_path = self.work_dir / f".{name}.stderr" if time_path.exists(): time_path.unlink() + if stderr_path.exists(): + stderr_path.unlink() started = time.perf_counter() + workspace_free_before = None + workspace_total = None + try: + workspace_usage = shutil.disk_usage(self.work_dir) + workspace_free_before = workspace_usage.free + workspace_total = workspace_usage.total + except OSError: + pass time_bin = "/usr/bin/time" if Path("/usr/bin/time").exists() else None + if time_bin: + # macOS ships a BSD ``time`` at this path; only GNU time supports + # the ``-v`` metrics format used by ``parse_time_log``. + probe = subprocess.run( + [time_bin, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if probe.returncode != 0 or b"GNU time" not in probe.stdout + probe.stderr: + time_bin = None if time_bin: timed_command = [time_bin, "-v", "-o", str(time_path), *command] else: timed_command = command stdout_handle = None + stderr_handle = None try: if stdout_path is not None: stdout_path.parent.mkdir(parents=True, exist_ok=True) stdout_handle = stdout_path.open("wb") - completed = subprocess.run(timed_command, stdout=stdout_handle, check=False) + stderr_handle = stderr_path.open("wb") + completed = subprocess.run( + timed_command, + stdout=stdout_handle, + stderr=stderr_handle, + check=False, + ) finally: if stdout_handle is not None: stdout_handle.close() + if stderr_handle is not None: + stderr_handle.close() + stderr_tail = "" + if stderr_path.exists(): + try: + with stderr_path.open("rb") as handle: + handle.seek(0, os.SEEK_END) + handle.seek(max(0, handle.tell() - STDERR_TAIL_BYTES)) + stderr_tail = handle.read().decode("utf-8", errors="replace").strip() + finally: + stderr_path.unlink(missing_ok=True) result = { + "stage_name": name, "exit_code": completed.returncode, "wall_seconds": time.perf_counter() - started, "output_path": "" if output_path is None else str(output_path), @@ -316,6 +383,18 @@ def run( if output_path is not None and output_path.is_file() else 0, } + try: + workspace_usage = shutil.disk_usage(self.work_dir) + result["workspace_free_bytes_after"] = workspace_usage.free + result["workspace_total_bytes"] = workspace_usage.total + except OSError: + pass + if workspace_free_before is not None: + result["workspace_free_bytes_before"] = workspace_free_before + if workspace_total is not None: + result.setdefault("workspace_total_bytes", workspace_total) + if stderr_tail: + result["stderr_tail"] = stderr_tail result.update(parse_time_log(time_path)) self.stages.append({"name": name, **result}) time_path.unlink(missing_ok=True) @@ -416,7 +495,13 @@ def write_result(payload: dict): file=sys.stderr, ) - def record_index_warning(index_format: str, stage: str, artifact: Path, message: str) -> dict: + def record_index_warning( + index_format: str, + stage: str, + artifact: Path, + message: str, + stage_result: dict | None = None, + ) -> dict: warning = { "format": index_format, "stage": stage, @@ -424,6 +509,19 @@ def record_index_warning(index_format: str, stage: str, artifact: Path, message: "artifact_path": str(artifact), "message": " ".join(str(message).split()), } + if stage_result: + warning["stage_name"] = stage_result.get("stage_name", stage) + warning["exit_code"] = stage_result.get("exit_code") + for key in ( + "workspace_free_bytes_before", + "workspace_free_bytes_after", + "workspace_total_bytes", + ): + if key in stage_result: + warning[key] = stage_result[key] + stderr_tail = str(stage_result.get("stderr_tail") or "").strip() + if stderr_tail: + warning["stderr_tail"] = stderr_tail index_warnings.append(warning) print( f"Warning: {index_format.upper()} index generation failed for '{artifact}'; " @@ -432,6 +530,20 @@ def record_index_warning(index_format: str, stage: str, artifact: Path, message: ) return warning + def failure_message(stage_result: dict | None, fallback: str) -> str: + """Turn a failed subprocess result into an actionable warning.""" + if not stage_result: + return fallback + exit_code = stage_result.get("exit_code") + diagnostics = [] + if exit_code is not None: + diagnostics.append(f"exit_code={exit_code}") + if int(exit_code) == 137: + diagnostics.append("the process was killed (often Docker memory/OOM pressure)") + elif int(exit_code) == 143: + diagnostics.append("the process was terminated (SIGTERM)") + return f"{fallback} ({'; '.join(diagnostics)})" if diagnostics else fallback + def skipped_cottas_result(method: str) -> dict: artifact = { "cottas": output_cottas, @@ -458,6 +570,15 @@ def skipped_cottas_result(method: str) -> dict: hdt_paths: list[Path] = [] cottas_paths: list[Path] = [] + + def cleanup_cottas_intermediates() -> None: + """Release COTTAS chunks/merge outputs after a failed attempt.""" + for path in list(cottas_paths): + path.unlink(missing_ok=True) + cottas_paths.clear() + for path in work_dir.glob("cottas-merge-*.cottas"): + path.unlink(missing_ok=True) + needs_hdt = any(method in HDT_METHODS for method in methods) hdt_bin = ( resolve_executable( @@ -589,9 +710,14 @@ def validate_artifact( "cottas", "cottas-index", output_cottas, - "partitioned COTTAS conversion/index creation failed for a chunk", + failure_message( + stage, + "partitioned COTTAS conversion/index creation failed for a chunk", + ), + stage_result=stage, ) chunk_cottas.unlink(missing_ok=True) + cleanup_cottas_intermediates() else: cottas_paths.append(chunk_cottas) finally: @@ -676,27 +802,46 @@ def validate_artifact( } if cottas_paths and not cottas_failed: - final_cottas, cottas_rounds = merge_pairwise( - cottas_paths, - prefix="cottas", - runner=runner, - merge_command=lambda left, right, merged: [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "merge", str(left), str(right), str(merged), "spo"], - total=cottas_total, - ) + # pycottas.cat supports a list of inputs. Use one indexed pass so + # a large multi-sample run does not repeatedly materialize and + # re-index the growing COTTAS graph at every pairwise round. + cottas_stage = None + cottas_rounds = 0 + if len(cottas_paths) == 1: + final_cottas = cottas_paths[0] + else: + cottas_merged_path = work_dir / "cottas-merge-final.cottas" + cottas_stage = runner.run( + "cottas-merge-all", + cottas_merge_many_command(cottas_python, cottas_paths, cottas_merged_path), + cottas_merged_path, + ) + add_totals(cottas_total, cottas_stage) + cottas_rounds = 1 + final_cottas = ( + cottas_merged_path + if cottas_stage["exit_code"] == 0 and cottas_merged_path.is_file() + else None + ) if final_cottas is None: if not args.allow_index_failures: - raise RuntimeError("COTTAS merge failed") + raise RuntimeError( + failure_message(cottas_stage, "COTTAS merge/index creation failed") + ) cottas_failed = True cottas_total["exit_code"] = 0 + cleanup_cottas_intermediates() cottas_warning = record_index_warning( "cottas", "cottas-index", output_cottas, - "COTTAS merge/index creation failed", + failure_message(cottas_stage, "COTTAS merge/index creation failed"), + stage_result=cottas_stage, ) else: shutil.copyfile(final_cottas, output_cottas) final_cottas.unlink(missing_ok=True) + cleanup_cottas_intermediates() try: cottas_validation = validate_artifact( name="cottas-validate", diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index bc78145..b689344 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -103,6 +103,52 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): self.assertEqual(len(observed_workspaces), 1) self.assertFalse(any(scratch_root.iterdir())) + def test_merge_many_passes_all_inputs_to_pycottas_cat(self): + """The large-graph merge path indexes a complete chunk set once.""" + module = load_cottas_tool() + calls = [] + + def fake_cat(paths, cottas_path, *, index, remove_input_files): + calls.append((paths, cottas_path, index, remove_input_files)) + self.assertEqual(index, "spo") + self.assertTrue(remove_input_files) + Path(cottas_path).write_text("merged COTTAS output\n") + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = (root / "scratch").resolve() + inputs = [] + for number in range(3): + path = root / f"chunk-{number}.cottas" + path.write_text(f"chunk {number}\n") + inputs.append(path) + output = root / "merged.cottas" + + with mock.patch.dict( + sys.modules, {"pycottas": types.SimpleNamespace(cat=fake_cat)} + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + sys, + "argv", + [ + "cottas_tool.py", + "merge-many", + "--input-cottas-files", + *(str(path) for path in inputs), + "--output-cottas-file", + str(output), + ], + ): + self.assertEqual(module.main(), 0) + + self.assertTrue(output.is_file()) + self.assertEqual( + calls, + [([str(path.resolve()) for path in inputs], str(output.resolve()), "spo", True)], + ) + self.assertFalse(any(scratch_root.iterdir())) + def test_decompress_uses_pycottas_and_isolated_scratch(self): """COTTAS decompression writes RDF while cleaning container-local state.""" module = load_cottas_tool() diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index 7d7fe75..a6b2ed0 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -56,6 +56,45 @@ def test_hdtc_merge_command_uses_bounded_native_merge(self): self.assertNotIn("java", " ".join(command).lower()) self.assertNotIn("hdtcat", " ".join(command).lower()) + def test_cottas_merge_many_command_uses_one_indexed_pass(self): + """Large COTTAS partitions are merged through pycottas.cat once.""" + runner = load_runner_module() + command = runner.cottas_merge_many_command( + "/opt/pycottas-venv/bin/python", + [Path("/work/chunk-00000.cottas"), Path("/work/chunk-00001.cottas")], + Path("/work/cottas-merge-final.cottas"), + ) + self.assertEqual( + command, + [ + "/opt/pycottas-venv/bin/python", + "/opt/vcf-rdfizer/cottas_tool.py", + "merge-many", + "--input-cottas-files", + "/work/chunk-00000.cottas", + "/work/chunk-00001.cottas", + "--output-cottas-file", + "/work/cottas-merge-final.cottas", + "--index", + "spo", + ], + ) + + def test_stage_runner_keeps_failed_stderr_tail(self): + """Index warnings retain the subprocess diagnostic instead of hiding it.""" + runner_module = load_runner_module() + with tempfile.TemporaryDirectory() as td: + work_dir = Path(td) / "work" + work_dir.mkdir() + stage_runner = runner_module.StageRunner(work_dir) + result = stage_runner.run( + "cottas-merge-all", + ["sh", "-c", "echo 'No space left on device' >&2; exit 1"], + ) + self.assertEqual(result["exit_code"], 1) + self.assertIn("No space left on device", result["stderr_tail"]) + self.assertFalse((work_dir / ".cottas-merge-all.stderr").exists()) + def test_stream_chunks_only_retains_the_chunk_being_consumed(self): """Gzip chunking does not stage a second full uncompressed aggregate.""" runner = load_runner_module() From 7f46113604b1046041aff2d1516db9a24c9e617a Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 3 Sep 2026 10:10:54 +0200 Subject: [PATCH 09/19] improved condensed COTTAS index generation workflow 2 --- README.md | 44 ++++++++---- changelog.md | 42 +++++++++--- src/cottas_tool.py | 7 +- src/partitioned_compression.py | 84 ++++++++++++++++------- test/test_partitioned_compression_unit.py | 30 ++++++-- 5 files changed, 154 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index c981709..4b41615 100644 --- a/README.md +++ b/README.md @@ -556,14 +556,17 @@ read. This is especially important for `space-optimized` `.nt.gz` aggregates, which must not be expanded into a second full raw-RDF copy. HDT chunks are merged with the Java-free `hdtc create` command, which accepts existing HDT inputs; the final HDT index is generated after merging with `hdtc index`. -COTTAS chunks are merged with one multi-input `pycottas.cat` pass, which -rebuilds the query index once for the complete representation. This avoids the -repeated re-indexing and temporary-file amplification of a pairwise merge tree -on large multi-sample graphs. If that stage fails in full mode, the warning -contains the failing exit code and, when available, a `stderr_tail` from the -COTTAS/DuckDB command. The same warning records the Docker workspace free-space -sample before and after the stage; -the raw RDF remains available for a retry. +COTTAS chunks are merged pairwise (two inputs per `pycottas.cat` pass). The +merge tree bounds the number of Parquet inputs and index buffers held by any +one process; this is important because the COTTAS merge API does not expose +the disk-backed memory budget available to the RDF conversion path. A +one-shot merge of every chunk can be terminated by the kernel/Docker OOM +killer. Pairwise merging takes more passes, but produces the same indexed +COTTAS semantics with a much lower peak working set. If a merge stage fails in +full mode, the warning contains the failing exit code and, when available, a +`stderr_tail` from the COTTAS/DuckDB command, maximum resident set size, and +Docker-workspace free-space samples before and after the stage; the raw RDF +remains available for a retry. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a streaming decode/count check. This verifies both readability and that the @@ -631,13 +634,24 @@ finishes. If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). If COTTAS indexing/merging fails on a very large RDF file, first inspect the -`cottas-merge-all` stage in the raw partitioned-compression metrics JSON and the -run wrapper log. The warning's `stderr_tail` distinguishes a DuckDB/COTTAS -error from an operating-system resource failure. The common resource remedy is -to reduce `--chunk-target-bytes` (and `--chunk-max-bytes`) and ensure the -Docker data volume has enough free space for the temporary DuckDB files and -the final Parquet rewrite. Rebuild the image after upgrading so the pinned -`pycottas==1.1.0` dependency and the multi-input merge adapter are installed. +`cottas-merge-r*` stage in the raw partitioned-compression metrics JSON and the +run wrapper log. An `exit_code=-9` means the child was killed by `SIGKILL`, +which is normally the kernel/Docker OOM killer; a shell wrapper may report the +same event as `137`. The warning's `stderr_tail`, `max_rss_kb`, and workspace +samples distinguish memory pressure from a DuckDB/COTTAS or disk-space error. +The common resource remedy is to reduce `--chunk-target-bytes` (and +`--chunk-max-bytes`) so each pairwise merge is smaller, and ensure the Docker +data volume has enough free space for temporary DuckDB files and the final +Parquet rewrites. Rebuild the image after upgrading so the pinned +`pycottas==1.1.0` dependency and the bounded pairwise merge workflow are +installed. + +If COTTAS is optional for the experiment, rerun with +`--representations hdt`; the HDT path is independent and can remain the +queryable artifact even when COTTAS cannot fit the available memory. If COTTAS +is required and pairwise merges still receive `-9`, the container needs more +RAM (or a smaller RDF chunk target); this is a resource limit, not a vocabulary +or RDF-validity problem. If HDT compression fails on very large RDF files, use `--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with diff --git a/changelog.md b/changelog.md index cca6e82..14d584e 100644 --- a/changelog.md +++ b/changelog.md @@ -1,15 +1,41 @@ # Changelog +## 2026-09-03 — COTTAS SIGKILL/OOM handling + +- Classified `exit_code=-9` in partitioned index warnings as a child process + killed by `SIGKILL` (normally the Linux kernel/Docker OOM killer). Shell + wrappers can surface the same condition as exit code `137`. +- Changed the production COTTAS merge back to a bounded pairwise merge tree: + each `pycottas.cat` invocation receives only two COTTAS inputs. This keeps + peak Parquet/index memory bounded; the previous one-shot multi-input merge + could exceed the container memory limit even though all per-chunk converts + succeeded. +- Added the failing stage's `max_rss_kb` to `index_warnings.json`, alongside + exit code, stderr tail, and workspace free-space samples, so OOM diagnosis + can be compared directly with the container memory limit. +- The `merge-many` adapter remains available for explicit experiments, but is + no longer selected by the default large-file workflow. + +### Rerun guidance + +Rebuild the image and rerun the full conversion. If a pairwise COTTAS merge is +still killed, lower `--chunk-target-bytes` and `--chunk-max-bytes` (for example, +to 256 MiB or 128 MiB) and/or increase the Docker/container memory limit. The +raw `.nt.gz` retained by the failed run is a valid recovery source; no +RMLStreamer reconversion is required. + +When COTTAS is optional, `--representations hdt` avoids the COTTAS merge +resource requirement while retaining a queryable representation. + ## 2026-09-02 — COTTAS large-input merge reliability - Diagnosed the previous generic `COTTAS merge/index creation failed` warning: the warning is emitted only after per-chunk COTTAS conversion succeeds and the final indexed merge returns a non-zero status. HDT generation/indexing is independent and is not implicated by that warning. -- Replaced the partitioned COTTAS pairwise merge tree with one multi-input - `pycottas.cat` pass. The query index is now built once, avoiding repeated - Parquet rewrites and reducing peak temporary-file use for large VCF-derived - graphs. +- Added a multi-input `pycottas.cat` adapter for explicit use. The production + workflow now uses the bounded pairwise strategy documented in the 2026-09-03 + entry because the merge API can require substantial in-memory buffers. - Added cleanup of all COTTAS chunks and merge outputs after a failed COTTAS attempt so stale intermediates cannot consume the Docker workspace. - Preserved a bounded stderr tail and exit code for every partitioned stage. @@ -24,7 +50,7 @@ Rebuild the image and rerun the same full command. If the COTTAS stage still fails, inspect `raw_metrics/compression_metrics/.../__partitioned_compression__` -and the run wrapper log for `cottas-merge-all` and its `stderr_tail`. For a -resource error, lower `--chunk-target-bytes` and `--chunk-max-bytes`, or enlarge -the Docker data-volume allocation. The raw `.nt.gz` retained by the failed run -is a valid recovery source. +and the run wrapper log for the failing `cottas-merge-r*` stage and its +`stderr_tail`. For a resource error, lower `--chunk-target-bytes` and +`--chunk-max-bytes`, or enlarge the Docker data-volume allocation. The raw +`.nt.gz` retained by the failed run is a valid recovery source. diff --git a/src/cottas_tool.py b/src/cottas_tool.py index dca57b8..a66ea05 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -44,7 +44,7 @@ def main() -> int: merge_many = subparsers.add_parser( "merge-many", - help="merge multiple COTTAS files in one indexed pass", + help="merge multiple COTTAS files in one indexed pass (explicit use)", ) merge_many.add_argument( "--input-cottas-files", @@ -139,8 +139,9 @@ def main() -> int: return 2 with cottas_scratch_workspace(): # pycottas.cat accepts a list of inputs and computes the requested - # index once. This is materially cheaper for large partitioned - # graphs than repeatedly re-indexing pairwise intermediate files. + # index once. This adapter is retained for explicit callers; the + # production partitioned workflow uses the two-input ``merge`` + # command because a very large input list can exceed memory. pycottas.cat( input_paths, cottas_path, diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 5335987..d9949ac 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -279,6 +279,32 @@ def cottas_merge_many_command( ] +def cottas_merge_command( + python_bin: str, + left: Path, + right: Path, + merged: Path, +) -> list[str]: + """Build a memory-bounded two-input COTTAS merge command. + + ``pycottas.cat`` has no disk-budget argument for its merge operation. A + single call over every partition can therefore hold too many Parquet + inputs and index buffers at once. The production workflow deliberately + invokes the adapter with two inputs per stage instead; the merge tree + bounds the peak working set while preserving the same indexed COTTAS + semantics. + """ + return [ + python_bin, + "/opt/vcf-rdfizer/cottas_tool.py", + "merge", + str(left), + str(right), + str(merged), + "spo", + ] + + def parse_time_log(path: Path) -> dict: if not path.exists(): return {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None} @@ -516,6 +542,7 @@ def record_index_warning( "workspace_free_bytes_before", "workspace_free_bytes_after", "workspace_total_bytes", + "max_rss_kb", ): if key in stage_result: warning[key] = stage_result[key] @@ -538,9 +565,17 @@ def failure_message(stage_result: dict | None, fallback: str) -> str: diagnostics = [] if exit_code is not None: diagnostics.append(f"exit_code={exit_code}") - if int(exit_code) == 137: + try: + numeric_exit_code = int(exit_code) + except (TypeError, ValueError): + numeric_exit_code = None + if numeric_exit_code == -9: + diagnostics.append( + "the process was killed by SIGKILL (usually the kernel/Docker OOM killer)" + ) + elif numeric_exit_code == 137: diagnostics.append("the process was killed (often Docker memory/OOM pressure)") - elif int(exit_code) == 143: + elif numeric_exit_code == 143: diagnostics.append("the process was terminated (SIGTERM)") return f"{fallback} ({'; '.join(diagnostics)})" if diagnostics else fallback @@ -802,27 +837,30 @@ def validate_artifact( } if cottas_paths and not cottas_failed: - # pycottas.cat supports a list of inputs. Use one indexed pass so - # a large multi-sample run does not repeatedly materialize and - # re-index the growing COTTAS graph at every pairwise round. - cottas_stage = None - cottas_rounds = 0 - if len(cottas_paths) == 1: - final_cottas = cottas_paths[0] - else: - cottas_merged_path = work_dir / "cottas-merge-final.cottas" - cottas_stage = runner.run( - "cottas-merge-all", - cottas_merge_many_command(cottas_python, cottas_paths, cottas_merged_path), - cottas_merged_path, - ) - add_totals(cottas_total, cottas_stage) - cottas_rounds = 1 - final_cottas = ( - cottas_merged_path - if cottas_stage["exit_code"] == 0 and cottas_merged_path.is_file() - else None - ) + # Keep each pycottas.cat invocation to two inputs. The adapter's + # conversion path supports disk-backed parsing, but its merge API + # has no equivalent memory-budget argument. A one-shot merge of + # every partition can consequently be killed by the kernel/Docker + # OOM killer (reported by subprocess as exit_code=-9). Pairwise + # merging bounds the number of simultaneously indexed inputs and + # still produces one semantically equivalent indexed COTTAS file. + final_cottas, cottas_rounds = merge_pairwise( + cottas_paths, + prefix="cottas", + runner=runner, + merge_command=lambda left, right, merged: cottas_merge_command( + cottas_python, + left, + right, + merged, + ), + total=cottas_total, + ) + cottas_stage = ( + runner.stages[-1] + if final_cottas is None and runner.stages + else None + ) if final_cottas is None: if not args.allow_index_failures: raise RuntimeError( diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index a6b2ed0..ab7a016 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -56,8 +56,8 @@ def test_hdtc_merge_command_uses_bounded_native_merge(self): self.assertNotIn("java", " ".join(command).lower()) self.assertNotIn("hdtcat", " ".join(command).lower()) - def test_cottas_merge_many_command_uses_one_indexed_pass(self): - """Large COTTAS partitions are merged through pycottas.cat once.""" + def test_cottas_merge_many_command_is_available_for_explicit_batching(self): + """The adapter retains an explicit multi-input COTTAS operation.""" runner = load_runner_module() command = runner.cottas_merge_many_command( "/opt/pycottas-venv/bin/python", @@ -80,6 +80,28 @@ def test_cottas_merge_many_command_uses_one_indexed_pass(self): ], ) + def test_cottas_merge_command_is_bounded_to_two_inputs(self): + """The production merge stage invokes pycottas.cat pairwise.""" + runner = load_runner_module() + command = runner.cottas_merge_command( + "/opt/pycottas-venv/bin/python", + Path("/work/chunk-00000.cottas"), + Path("/work/chunk-00001.cottas"), + Path("/work/cottas-merge-r01-00000.cottas"), + ) + self.assertEqual( + command, + [ + "/opt/pycottas-venv/bin/python", + "/opt/vcf-rdfizer/cottas_tool.py", + "merge", + "/work/chunk-00000.cottas", + "/work/chunk-00001.cottas", + "/work/cottas-merge-r01-00000.cottas", + "spo", + ], + ) + def test_stage_runner_keeps_failed_stderr_tail(self): """Index warnings retain the subprocess diagnostic instead of hiding it.""" runner_module = load_runner_module() @@ -88,12 +110,12 @@ def test_stage_runner_keeps_failed_stderr_tail(self): work_dir.mkdir() stage_runner = runner_module.StageRunner(work_dir) result = stage_runner.run( - "cottas-merge-all", + "cottas-merge-r01-00000", ["sh", "-c", "echo 'No space left on device' >&2; exit 1"], ) self.assertEqual(result["exit_code"], 1) self.assertIn("No space left on device", result["stderr_tail"]) - self.assertFalse((work_dir / ".cottas-merge-all.stderr").exists()) + self.assertFalse((work_dir / ".cottas-merge-r01-00000.stderr").exists()) def test_stream_chunks_only_retains_the_chunk_being_consumed(self): """Gzip chunking does not stage a second full uncompressed aggregate.""" From bbae36ce314a1df1ba57eba4c20b55959add9fbc Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 3 Sep 2026 15:29:11 +0200 Subject: [PATCH 10/19] cottas condensed index fix --- Dockerfile | 2 + README.md | 71 +++++++----- changelog.md | 42 +++++-- src/cottas_tool.py | 135 ++++++++++++++++++++-- src/partitioned_compression.py | 95 +++++++++------ test/test_cottas_tool.py | 121 +++++++++++++------ test/test_partitioned_compression_unit.py | 8 +- test/test_vcf_rdfizer_unit.py | 19 +++ vcf_rdfizer.py | 14 +++ 9 files changed, 382 insertions(+), 125 deletions(-) diff --git a/Dockerfile b/Dockerfile index 188cea7..56b05f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,6 +110,8 @@ ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV HDTC_BIN=/usr/local/bin/hdtc ENV HDT_INDEX_MEMORY_LIMIT=512M ENV HDT_MERGE_MEMORY_LIMIT=512M +ENV COTTAS_MERGE_MEMORY_LIMIT=512M +ENV COTTAS_MERGE_THREADS=1 ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python diff --git a/README.md b/README.md index 4b41615..b1d2e1f 100644 --- a/README.md +++ b/README.md @@ -427,9 +427,9 @@ vcf-rdfizer \ only the directory containing the selected artifact and writes metrics under `/run_metrics//index_metrics.json`. HDT indexing creates a versioned sidecar beside the input. COTTAS indexing rewrites the existing -`.cottas` file through `pycottas.cat` with one input file, keeping the data in -the same artifact while rebuilding its embedded index. If the operation fails, -the original COTTAS file is left in place; HDT's previous sidecars are restored. +`.cottas` file through a disk-backed DuckDB rewrite, keeping the data in the +same artifact while rebuilding its embedded index. If the operation fails, the +original COTTAS file is left in place; HDT's previous sidecars are restored. HDT merging and indexing do not start a Java HDT tool. They use native `hdtc`: `hdtc create` merges partitioned HDTs and `hdtc index` streams an HDT through @@ -450,6 +450,21 @@ buffers and may increase temporary I/O; higher values can improve performance when memory is available. Temporary files live in the container's `/work` area and are removed after the attempt. +COTTAS uses the same out-of-core principle for its global `DISTINCT` and +`ORDER BY` merge. The image defaults `COTTAS_MERGE_MEMORY_LIMIT` to `512M` and +`COTTAS_MERGE_THREADS` to `1`; DuckDB spills merge state to `/work` instead of +allowing one large condensed graph to consume all container memory. Override +them from the host only when appropriate for the available RAM, for example: + +```bash +COTTAS_MERGE_MEMORY_LIMIT=1G COTTAS_MERGE_THREADS=2 vcf-rdfizer \ + --mode compress \ + --rdf ./results/cohort/cohort.nt.gz \ + --rdf-compression none \ + --representations cottas \ + --out ./results +``` + In `full` mode, an HDT sidecar-index failure is non-fatal when the HDT data itself remains readable; the run continues and the HDT can be repaired later with the standalone command above. If COTTAS generation/indexing cannot @@ -556,17 +571,17 @@ read. This is especially important for `space-optimized` `.nt.gz` aggregates, which must not be expanded into a second full raw-RDF copy. HDT chunks are merged with the Java-free `hdtc create` command, which accepts existing HDT inputs; the final HDT index is generated after merging with `hdtc index`. -COTTAS chunks are merged pairwise (two inputs per `pycottas.cat` pass). The -merge tree bounds the number of Parquet inputs and index buffers held by any -one process; this is important because the COTTAS merge API does not expose -the disk-backed memory budget available to the RDF conversion path. A -one-shot merge of every chunk can be terminated by the kernel/Docker OOM -killer. Pairwise merging takes more passes, but produces the same indexed -COTTAS semantics with a much lower peak working set. If a merge stage fails in +COTTAS chunk conversion uses `pycottas.rdf2cottas(..., disk=True)`. The final +COTTAS merge deliberately does **not** call `pycottas.cat`: version 1.1.0 runs +its global `DISTINCT` plus `ORDER BY` through an unbounded in-memory DuckDB +connection, which can be killed on large condensed graphs. VCF-RDFizer instead +executes the equivalent global merge in a dedicated on-disk DuckDB database, +with a 512 MiB memory budget, one worker, and `/work` as the external spill +directory. This preserves global RDF set semantics and `spo` ordering while +making the memory requirement bounded by configuration. If the stage fails in full mode, the warning contains the failing exit code and, when available, a -`stderr_tail` from the COTTAS/DuckDB command, maximum resident set size, and -Docker-workspace free-space samples before and after the stage; the raw RDF -remains available for a retry. +`stderr_tail`, maximum resident set size, and Docker-workspace free-space +samples; the raw RDF remains available for a retry. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a streaming decode/count check. This verifies both readability and that the @@ -603,11 +618,12 @@ host value of either variable into the relevant Docker command. COTTAS does not expose a separate index sidecar. Its index is part of the Parquet artifact and is selected when the artifact is written. Standalone -COTTAS index mode uses `pycottas.cat` to write a new temporary COTTAS file from -the existing one with the default `spo` index, then atomically replaces the -original. This is still index-only from the pipeline's point of view: it does -not rerun VCF-to-RDF conversion or create an RDF output, but it may require -temporary disk space and time comparable to rewriting the COTTAS file. +COTTAS index mode writes a new temporary COTTAS file through the same +memory-bounded, disk-backed DuckDB operation with the default `spo` index, +then atomically replaces the original. This is still index-only from the +pipeline's point of view: it does not rerun VCF-to-RDF conversion or create an +RDF output, but it may require temporary disk space and time comparable to +rewriting the COTTAS file. The record-safe chunk plan and per-stage timings are retained in the raw partitioned-compression metrics JSON for diagnostics. The temporary chunk @@ -634,24 +650,25 @@ finishes. If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). If COTTAS indexing/merging fails on a very large RDF file, first inspect the -`cottas-merge-r*` stage in the raw partitioned-compression metrics JSON and the +`cottas-merge-disk` stage in the raw partitioned-compression metrics JSON and the run wrapper log. An `exit_code=-9` means the child was killed by `SIGKILL`, which is normally the kernel/Docker OOM killer; a shell wrapper may report the same event as `137`. The warning's `stderr_tail`, `max_rss_kb`, and workspace samples distinguish memory pressure from a DuckDB/COTTAS or disk-space error. -The common resource remedy is to reduce `--chunk-target-bytes` (and -`--chunk-max-bytes`) so each pairwise merge is smaller, and ensure the Docker -data volume has enough free space for temporary DuckDB files and the final -Parquet rewrites. Rebuild the image after upgrading so the pinned -`pycottas==1.1.0` dependency and the bounded pairwise merge workflow are +The disk-backed merge defaults to 512 MiB and one worker. Lower +`COTTAS_MERGE_MEMORY_LIMIT` if the container has a stricter memory cap, or +raise it only when RAM is available; in all cases ensure Docker's data volume +has enough free space for DuckDB spill files and the final Parquet rewrite. +Rebuild the image after upgrading so the bounded disk-backed merge workflow is installed. If COTTAS is optional for the experiment, rerun with `--representations hdt`; the HDT path is independent and can remain the queryable artifact even when COTTAS cannot fit the available memory. If COTTAS -is required and pairwise merges still receive `-9`, the container needs more -RAM (or a smaller RDF chunk target); this is a resource limit, not a vocabulary -or RDF-validity problem. +is required and the disk-backed merge still receives `-9`, lower +`COTTAS_MERGE_MEMORY_LIMIT`, verify that the Docker memory cap exceeds it, and +check the host kernel log for an external kill; this is a resource limit, not a +vocabulary or RDF-validity problem. If HDT compression fails on very large RDF files, use `--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with diff --git a/changelog.md b/changelog.md index 14d584e..a96a1c5 100644 --- a/changelog.md +++ b/changelog.md @@ -1,15 +1,39 @@ # Changelog +## 2026-09-03 — Disk-backed COTTAS merge for condensed cohorts + +- Replaced the final COTTAS merge/reindex implementation with a dedicated + disk-backed DuckDB connection. `pycottas.cat` in version 1.1.0 performs its + global `DISTINCT` and `ORDER BY` through the process-global in-memory + connection, so both all-input and final pairwise merges could receive + `SIGKILL` on the 36-million-triple condensed cohort. +- The new merge preserves the same global RDF set semantics and `spo` Parquet + ordering, but configures DuckDB with a dedicated `.duckdb` database, + `COTTAS_MERGE_MEMORY_LIMIT=512M`, `COTTAS_MERGE_THREADS=1`, and a disposable + `/work` spill directory. The final merge can therefore externalize sort and + distinct state instead of exhausting container memory. +- Forwarded explicit host-side `COTTAS_MERGE_MEMORY_LIMIT` and + `COTTAS_MERGE_THREADS` values into partitioned full/compress runs and + standalone COTTAS index mode. +- Updated diagnostics, README troubleshooting, and regression tests to target + `cottas-merge-disk` rather than the obsolete pairwise merge path. + +### Rerun guidance + +Build an image from this revision before retrying. Start with the default +512 MiB/one-worker merge budget; if the Docker cgroup cap is below that, +lower it (for example `COTTAS_MERGE_MEMORY_LIMIT=256M`). Ensure Docker has +substantial free disk space for temporary DuckDB spill files. The preserved +raw `.nt.gz` can be retried with `--mode compress`, avoiding another RDF run. + ## 2026-09-03 — COTTAS SIGKILL/OOM handling - Classified `exit_code=-9` in partitioned index warnings as a child process killed by `SIGKILL` (normally the Linux kernel/Docker OOM killer). Shell wrappers can surface the same condition as exit code `137`. -- Changed the production COTTAS merge back to a bounded pairwise merge tree: - each `pycottas.cat` invocation receives only two COTTAS inputs. This keeps - peak Parquet/index memory bounded; the previous one-shot multi-input merge - could exceed the container memory limit even though all per-chunk converts - succeeded. +- Initially changed the production COTTAS merge to a bounded pairwise tree. + This reduced input fan-in but did not bound the final graph-wide + `DISTINCT`/`ORDER BY`; the disk-backed replacement above supersedes it. - Added the failing stage's `max_rss_kb` to `index_warnings.json`, alongside exit code, stderr tail, and workspace free-space samples, so OOM diagnosis can be compared directly with the container memory limit. @@ -18,11 +42,9 @@ ### Rerun guidance -Rebuild the image and rerun the full conversion. If a pairwise COTTAS merge is -still killed, lower `--chunk-target-bytes` and `--chunk-max-bytes` (for example, -to 256 MiB or 128 MiB) and/or increase the Docker/container memory limit. The -raw `.nt.gz` retained by the failed run is a valid recovery source; no -RMLStreamer reconversion is required. +Rebuild the image and rerun the full conversion (or use `--mode compress` with +the retained raw `.nt.gz`). The disk-backed merge workflow supersedes this +earlier pairwise guidance. When COTTAS is optional, `--representations hdt` avoids the COTTAS merge resource requirement while retaining a queryable representation. diff --git a/src/cottas_tool.py b/src/cottas_tool.py index a66ea05..3e92f42 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -1,17 +1,26 @@ #!/usr/bin/env python3 -"""Small Docker-side adapter for the pycottas conversion and merge API.""" +"""Docker-side adapter for disk-backed COTTAS conversion and merge operations.""" import argparse import os +import re import sys import tempfile from contextlib import contextmanager from pathlib import Path +DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "512M" +DEFAULT_COTTAS_MERGE_THREADS = 1 +MEMORY_LIMIT_PATTERN = re.compile( + r"^\d+(?:\.\d+)?\s*(?:B|K|M|G|T|KB|MB|GB|TB|KIB|MIB|GIB|TIB)$", + re.IGNORECASE, +) + + @contextmanager def cottas_scratch_workspace(): - """Run one pycottas operation with an isolated DuckDB working directory.""" + """Run one COTTAS operation with an isolated DuckDB working directory.""" scratch_root = Path(os.environ.get("COTTAS_SCRATCH_DIR", "/work")).resolve() scratch_root.mkdir(parents=True, exist_ok=True) original_working_directory = Path.cwd() @@ -27,6 +36,115 @@ def cottas_scratch_workspace(): os.chdir(original_working_directory) +def sql_literal(value: str | Path) -> str: + """Quote one filesystem value for the DuckDB SQL emitted by this adapter.""" + return "'" + str(value).replace("'", "''") + "'" + + +def cottas_merge_memory_limit() -> str: + """Return a validated DuckDB budget for COTTAS merge/reindex operations.""" + memory_limit = os.environ.get( + "COTTAS_MERGE_MEMORY_LIMIT", DEFAULT_COTTAS_MERGE_MEMORY_LIMIT + ).strip() + if not MEMORY_LIMIT_PATTERN.fullmatch(memory_limit): + raise ValueError( + "COTTAS_MERGE_MEMORY_LIMIT must be a positive DuckDB byte value " + "such as 512M or 1G" + ) + return memory_limit + + +def cottas_merge_threads() -> int: + """Return a bounded DuckDB worker count for deterministic merge memory use.""" + raw_threads = os.environ.get( + "COTTAS_MERGE_THREADS", str(DEFAULT_COTTAS_MERGE_THREADS) + ).strip() + try: + threads = int(raw_threads) + except ValueError as exc: + raise ValueError("COTTAS_MERGE_THREADS must be a positive integer") from exc + if threads <= 0: + raise ValueError("COTTAS_MERGE_THREADS must be a positive integer") + return threads + + +def disk_backed_cottas_merge( + input_paths: list[str], + output_path: str, + *, + index: str, + remove_input_files: bool, +) -> None: + """Merge COTTAS Parquet inputs with DuckDB spill files rather than pycottas.cat. + + ``pycottas.cat`` uses DuckDB's process-global in-memory connection. Its + global ``DISTINCT`` plus ``ORDER BY`` can therefore be SIGKILLed on a + large condensed VCF even when every disk-backed chunk conversion succeeds. + This adapter opens a dedicated on-disk database, caps its memory, restricts + merge parallelism, and directs external sort/hash spill files to the + disposable COTTAS scratch directory. The query retains COTTAS's global RDF + set semantics and the requested Parquet sort/index order. + """ + if not input_paths: + raise ValueError("at least one COTTAS input is required for a merge") + if not index or set(index.lower()) != {"s", "p", "o"} or len(index) != 3: + raise ValueError("COTTAS merge index must be a permutation of spo") + + try: + import duckdb + except ImportError as exc: + raise RuntimeError(f"DuckDB dependency is unavailable: {exc}") from exc + + scratch_dir = Path.cwd() + temporary_directory = scratch_dir / "duckdb-merge-tmp" + temporary_directory.mkdir(parents=True, exist_ok=True) + database_path = scratch_dir / "pycottas-merge.duckdb" + memory_limit = cottas_merge_memory_limit() + threads = cottas_merge_threads() + quoted_inputs = ", ".join(sql_literal(path) for path in input_paths) + parquet_scan = f"PARQUET_SCAN([{quoted_inputs}], union_by_name = true)" + + connection = duckdb.connect(str(database_path)) + try: + # Apply limits before DuckDB plans the DISTINCT/ORDER BY operation. + # One worker makes the memory budget predictable on hosts with many + # CPUs and still permits DuckDB's external sort/hash operators to + # spill to the named Docker volume. + connection.execute("SET preserve_insertion_order = false") + connection.execute("SET enable_progress_bar = false") + connection.execute(f"SET temp_directory = {sql_literal(temporary_directory)}") + connection.execute(f"SET memory_limit = {sql_literal(memory_limit)}") + connection.execute(f"SET threads = {threads}") + + columns = { + str(row[0]) + for row in connection.execute( + f"DESCRIBE SELECT * FROM {parquet_scan} LIMIT 1" + ).fetchall() + } + if not {"s", "p", "o"}.issubset(columns): + raise RuntimeError("COTTAS inputs do not contain the required s, p, o columns") + selected_columns = ["s", "p", "o"] + if "g" in columns: + selected_columns.append("g") + selected_columns_sql = ", ".join(selected_columns) + order_columns_sql = ", ".join(index.lower()) + copy_query = ( + f"COPY (SELECT DISTINCT {selected_columns_sql} FROM {parquet_scan} " + f"ORDER BY {order_columns_sql}) TO {sql_literal(output_path)} " + "(FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 22, " + "PARQUET_VERSION v2, " + f"KV_METADATA {{index: {sql_literal(index.lower())}}})" + ) + connection.execute(copy_query) + finally: + connection.close() + + if remove_input_files: + for input_path in input_paths: + Path(input_path).unlink(missing_ok=True) + + def main() -> int: parser = argparse.ArgumentParser(description="VCF-RDFizer COTTAS adapter") subparsers = parser.add_subparsers(dest="command", required=True) @@ -104,7 +222,8 @@ def main() -> int: # COTTAS indexes are part of the Parquet artifact rather than sibling # files. Rebuild into a temporary file in the same directory, then - # replace the original only after pycottas has completed successfully. + # replace the original only after the disk-backed DuckDB rewrite + # completes successfully. temporary_path = None try: file_descriptor, temporary_name = tempfile.mkstemp( @@ -116,7 +235,7 @@ def main() -> int: temporary_path = Path(temporary_name) temporary_path.unlink() with cottas_scratch_workspace(): - pycottas.cat( + disk_backed_cottas_merge( [str(cottas_path)], str(temporary_path), index=args.index, @@ -138,11 +257,7 @@ def main() -> int: print("merge-many requires at least two input COTTAS files", file=sys.stderr) return 2 with cottas_scratch_workspace(): - # pycottas.cat accepts a list of inputs and computes the requested - # index once. This adapter is retained for explicit callers; the - # production partitioned workflow uses the two-input ``merge`` - # command because a very large input list can exceed memory. - pycottas.cat( + disk_backed_cottas_merge( input_paths, cottas_path, index=args.index, @@ -154,7 +269,7 @@ def main() -> int: right_path = str(Path(args.right_path).resolve()) cottas_path = str(Path(args.cottas_path).resolve()) with cottas_scratch_workspace(): - pycottas.cat( + disk_backed_cottas_merge( [left_path, right_path], cottas_path, index=args.index, diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index d9949ac..3f7696d 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -25,6 +25,8 @@ HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} STDERR_TAIL_BYTES = 16 * 1024 +DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "512M" +DEFAULT_COTTAS_MERGE_THREADS = "1" def is_triple_line(line: bytes) -> bool: @@ -259,12 +261,12 @@ def cottas_merge_many_command( inputs: list[Path], merged: Path, ) -> list[str]: - """Build one multi-input COTTAS merge command. + """Build one disk-backed, multi-input COTTAS merge command. - ``pycottas.cat`` accepts a list of COTTAS paths and builds the requested - zonemap index once. Calling it once for the complete chunk set avoids the - repeated re-indexing and temporary-file amplification caused by a - pairwise merge tree on large graphs. + The adapter performs the global distinct/order operation through a + dedicated DuckDB database with a bounded memory budget and `/work` spill + directory. Unlike ``pycottas.cat``, it never routes the large merge through + DuckDB's process-global in-memory connection. """ return [ python_bin, @@ -285,14 +287,11 @@ def cottas_merge_command( right: Path, merged: Path, ) -> list[str]: - """Build a memory-bounded two-input COTTAS merge command. - - ``pycottas.cat`` has no disk-budget argument for its merge operation. A - single call over every partition can therefore hold too many Parquet - inputs and index buffers at once. The production workflow deliberately - invokes the adapter with two inputs per stage instead; the merge tree - bounds the peak working set while preserving the same indexed COTTAS - semantics. + """Build a compatible two-input disk-backed COTTAS merge command. + + The normal partitioned workflow uses ``merge-many`` because its DuckDB + implementation is spill-capable. This command remains available for + explicit callers that need a two-input merge. """ return [ python_bin, @@ -543,6 +542,8 @@ def record_index_warning( "workspace_free_bytes_after", "workspace_total_bytes", "max_rss_kb", + "cottas_merge_memory_limit", + "cottas_merge_threads", ): if key in stage_result: warning[key] = stage_result[key] @@ -837,30 +838,42 @@ def validate_artifact( } if cottas_paths and not cottas_failed: - # Keep each pycottas.cat invocation to two inputs. The adapter's - # conversion path supports disk-backed parsing, but its merge API - # has no equivalent memory-budget argument. A one-shot merge of - # every partition can consequently be killed by the kernel/Docker - # OOM killer (reported by subprocess as exit_code=-9). Pairwise - # merging bounds the number of simultaneously indexed inputs and - # still produces one semantically equivalent indexed COTTAS file. - final_cottas, cottas_rounds = merge_pairwise( - cottas_paths, - prefix="cottas", - runner=runner, - merge_command=lambda left, right, merged: cottas_merge_command( - cottas_python, - left, - right, - merged, - ), - total=cottas_total, - ) - cottas_stage = ( - runner.stages[-1] - if final_cottas is None and runner.stages - else None - ) + # pycottas.cat performs its global DISTINCT/ORDER BY through an + # unbounded process-global in-memory DuckDB connection. That is + # what made both the one-shot and the final pairwise COTTAS merge + # susceptible to SIGKILL on large condensed cohorts. The adapter's + # merge-many command instead opens a dedicated disk-backed DuckDB + # database with a bounded memory limit and /work spill files, so a + # single global indexed merge is safe and avoids duplicate work. + cottas_stage = None + cottas_rounds = 0 + if len(cottas_paths) == 1: + final_cottas = cottas_paths[0] + else: + cottas_merged_path = work_dir / "cottas-merge-final.cottas" + cottas_stage = runner.run( + "cottas-merge-disk", + cottas_merge_many_command( + cottas_python, + cottas_paths, + cottas_merged_path, + ), + cottas_merged_path, + ) + cottas_stage["cottas_merge_memory_limit"] = os.environ.get( + "COTTAS_MERGE_MEMORY_LIMIT", DEFAULT_COTTAS_MERGE_MEMORY_LIMIT + ) + cottas_stage["cottas_merge_threads"] = os.environ.get( + "COTTAS_MERGE_THREADS", DEFAULT_COTTAS_MERGE_THREADS + ) + runner.stages[-1].update(cottas_stage) + add_totals(cottas_total, cottas_stage) + cottas_rounds = 1 + final_cottas = ( + cottas_merged_path + if cottas_stage["exit_code"] == 0 and cottas_merged_path.is_file() + else None + ) if final_cottas is None: if not args.allow_index_failures: raise RuntimeError( @@ -908,6 +921,14 @@ def validate_artifact( "details": { **plan, "merge_rounds": cottas_rounds, + "merge_strategy": "duckdb_disk_backed", + "merge_memory_limit": os.environ.get( + "COTTAS_MERGE_MEMORY_LIMIT", + DEFAULT_COTTAS_MERGE_MEMORY_LIMIT, + ), + "merge_threads": os.environ.get( + "COTTAS_MERGE_THREADS", DEFAULT_COTTAS_MERGE_THREADS + ), "index": "spo", "validation": cottas_validation, }, diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index b689344..177a266 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -1,5 +1,6 @@ import importlib.util import os +import re import sys import tempfile import types @@ -20,6 +21,40 @@ def load_cottas_tool(): return module +class _FakeDuckDBCursor: + def __init__(self, rows=None): + self.rows = rows or [] + + def fetchall(self): + return self.rows + + +class _RecordingDuckDB: + """Minimal DuckDB stand-in for asserting the adapter's SQL contract.""" + + def __init__(self): + self.database_paths = [] + self.queries = [] + self.closed = False + + def connect(self, database_path): + self.database_paths.append(Path(database_path)) + return self + + def execute(self, query): + self.queries.append(query) + if query.startswith("DESCRIBE"): + return _FakeDuckDBCursor([("s",), ("p",), ("o",)]) + if query.startswith("COPY"): + match = re.search(r"\bTO '((?:''|[^'])*)'", query) + assert match is not None + Path(match.group(1).replace("''", "'")).write_text("merged COTTAS output\n") + return _FakeDuckDBCursor() + + def close(self): + self.closed = True + + class CottasToolTests(unittest.TestCase): def test_convert_uses_a_fresh_duckdb_workspace_for_each_invocation(self): """Sequential chunk builds cannot reuse pycottas.duckdb or its quads table.""" @@ -64,22 +99,10 @@ def fake_rdf2cottas(rdf_path, cottas_path, *, index, disk): self.assertTrue(all(path.parent == scratch_root for path in observed_workspaces)) self.assertFalse(any(scratch_root.iterdir())) - def test_merge_uses_the_same_isolated_scratch_policy(self): - """Pairwise merges do not inherit a DuckDB database from chunk conversion.""" + def test_merge_uses_a_bounded_disk_backed_duckdb_connection(self): + """Merge SQL is spill-capable instead of using pycottas.cat in memory.""" module = load_cottas_tool() - observed_workspaces = [] - - def fake_cat(paths, cottas_path, *, index, remove_input_files): - self.assertTrue(all(Path(path).is_absolute() for path in paths)) - self.assertTrue(Path(cottas_path).is_absolute()) - self.assertEqual(index, "spo") - self.assertTrue(remove_input_files) - database_path = Path.cwd() / "pycottas.duckdb" - if database_path.exists(): - raise RuntimeError("Table with name quads already exists") - database_path.write_text("temporary DuckDB state\n") - Path(cottas_path).write_text("merged COTTAS output\n") - observed_workspaces.append(Path.cwd()) + fake_duckdb = _RecordingDuckDB() with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -90,7 +113,13 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): left.write_text("left\n") right.write_text("right\n") - with mock.patch.dict(sys.modules, {"pycottas": types.SimpleNamespace(cat=fake_cat)}), mock.patch.dict( + with mock.patch.dict( + sys.modules, + { + "pycottas": types.SimpleNamespace(), + "duckdb": types.SimpleNamespace(connect=fake_duckdb.connect), + }, + ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False ), mock.patch.object( sys, @@ -100,19 +129,26 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): self.assertEqual(module.main(), 0) self.assertTrue(output.is_file()) - self.assertEqual(len(observed_workspaces), 1) + self.assertTrue(fake_duckdb.closed) + self.assertEqual(len(fake_duckdb.database_paths), 1) + self.assertEqual(fake_duckdb.database_paths[0].parent.parent, scratch_root) + self.assertIn("SET memory_limit = '512M'", fake_duckdb.queries) + self.assertIn("SET threads = 1", fake_duckdb.queries) + self.assertTrue( + any( + "SELECT DISTINCT s, p, o" in query + and "ORDER BY s, p, o" in query + for query in fake_duckdb.queries + ) + ) + self.assertFalse(left.exists()) + self.assertFalse(right.exists()) self.assertFalse(any(scratch_root.iterdir())) - def test_merge_many_passes_all_inputs_to_pycottas_cat(self): - """The large-graph merge path indexes a complete chunk set once.""" + def test_merge_many_passes_all_inputs_to_disk_backed_duckdb(self): + """The production merge scans every chunk through one spill-capable query.""" module = load_cottas_tool() - calls = [] - - def fake_cat(paths, cottas_path, *, index, remove_input_files): - calls.append((paths, cottas_path, index, remove_input_files)) - self.assertEqual(index, "spo") - self.assertTrue(remove_input_files) - Path(cottas_path).write_text("merged COTTAS output\n") + fake_duckdb = _RecordingDuckDB() with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -125,7 +161,11 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): output = root / "merged.cottas" with mock.patch.dict( - sys.modules, {"pycottas": types.SimpleNamespace(cat=fake_cat)} + sys.modules, + { + "pycottas": types.SimpleNamespace(), + "duckdb": types.SimpleNamespace(connect=fake_duckdb.connect), + }, ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False ), mock.patch.object( @@ -143,10 +183,13 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): self.assertEqual(module.main(), 0) self.assertTrue(output.is_file()) - self.assertEqual( - calls, - [([str(path.resolve()) for path in inputs], str(output.resolve()), "spo", True)], - ) + copy_query = next(query for query in fake_duckdb.queries if query.startswith("COPY")) + for path in inputs: + self.assertIn(str(path.resolve()), copy_query) + self.assertFalse(path.exists()) + self.assertIn("SET memory_limit = '512M'", fake_duckdb.queries) + self.assertIn("SET threads = 1", fake_duckdb.queries) + self.assertTrue(fake_duckdb.closed) self.assertFalse(any(scratch_root.iterdir())) def test_decompress_uses_pycottas_and_isolated_scratch(self): @@ -187,11 +230,11 @@ def fake_cottas2rdf(cottas_path, rdf_path): self.assertFalse(any(scratch_root.iterdir())) def test_reindex_rewrites_atomically_without_removing_input(self): - """Reindex uses one-file cat and replaces the source only on success.""" + """Reindex uses a disk-backed rewrite and replaces only on success.""" module = load_cottas_tool() calls = [] - def fake_cat(paths, cottas_path, *, index, remove_input_files): + def fake_disk_merge(paths, cottas_path, *, index, remove_input_files): calls.append((paths, cottas_path, index, remove_input_files)) self.assertNotEqual(Path(cottas_path), source) self.assertTrue(Path(cottas_path).name.startswith(f".{source.name}.reindex-")) @@ -205,9 +248,11 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): with mock.patch.dict( sys.modules, - {"pycottas": types.SimpleNamespace(cat=fake_cat)}, + {"pycottas": types.SimpleNamespace()}, ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "disk_backed_cottas_merge", side_effect=fake_disk_merge ), mock.patch.object( sys, "argv", @@ -222,11 +267,11 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): ) self.assertEqual(list(root.glob(".input.cottas.reindex-*.cottas")), []) - def test_reindex_keeps_original_when_pycottas_fails(self): + def test_reindex_keeps_original_when_disk_backed_merge_fails(self): """A failed COTTAS rebuild does not replace the existing artifact.""" module = load_cottas_tool() - def failing_cat(*args, **kwargs): + def failing_disk_merge(*args, **kwargs): raise RuntimeError("simulated reindex failure") with tempfile.TemporaryDirectory() as temporary_directory: @@ -237,9 +282,11 @@ def failing_cat(*args, **kwargs): with mock.patch.dict( sys.modules, - {"pycottas": types.SimpleNamespace(cat=failing_cat)}, + {"pycottas": types.SimpleNamespace()}, ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "disk_backed_cottas_merge", side_effect=failing_disk_merge ), mock.patch.object( sys, "argv", diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index ab7a016..7eeed0d 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -56,8 +56,8 @@ def test_hdtc_merge_command_uses_bounded_native_merge(self): self.assertNotIn("java", " ".join(command).lower()) self.assertNotIn("hdtcat", " ".join(command).lower()) - def test_cottas_merge_many_command_is_available_for_explicit_batching(self): - """The adapter retains an explicit multi-input COTTAS operation.""" + def test_cottas_merge_many_command_uses_the_disk_backed_merge_adapter(self): + """Production COTTAS merging scans chunks through one spill-capable stage.""" runner = load_runner_module() command = runner.cottas_merge_many_command( "/opt/pycottas-venv/bin/python", @@ -80,8 +80,8 @@ def test_cottas_merge_many_command_is_available_for_explicit_batching(self): ], ) - def test_cottas_merge_command_is_bounded_to_two_inputs(self): - """The production merge stage invokes pycottas.cat pairwise.""" + def test_cottas_merge_command_remains_compatible_for_two_inputs(self): + """The adapter continues to expose a two-input disk-backed merge command.""" runner = load_runner_module() command = runner.cottas_merge_command( "/opt/pycottas-venv/bin/python", diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 2cb8941..94205f9 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -273,6 +273,25 @@ def test_hdt_merge_memory_limit_is_forwarded_to_docker(self): ["-e", "HDT_MERGE_MEMORY_LIMIT=768M"], ) + def test_cottas_merge_limits_are_forwarded_to_docker(self): + """COTTAS merge memory and worker overrides reach Docker commands.""" + with mock.patch.dict( + os.environ, + { + "COTTAS_MERGE_MEMORY_LIMIT": "384M", + "COTTAS_MERGE_THREADS": "1", + }, + ): + self.assertEqual( + vcf_rdfizer.docker_cottas_merge_env_args(), + [ + "-e", + "COTTAS_MERGE_MEMORY_LIMIT=384M", + "-e", + "COTTAS_MERGE_THREADS=1", + ], + ) + def test_validator_counts_plain_and_gzip_ntriples(self): """The Docker validator's fallback source count handles .nt and .nt.gz.""" validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index aed78b4..b0b8aee 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -441,6 +441,18 @@ def docker_hdt_merge_env_args() -> list[str]: return ["-e", f"HDT_MERGE_MEMORY_LIMIT={memory_limit}"] +def docker_cottas_merge_env_args() -> list[str]: + """Forward optional bounded-memory COTTAS merge settings into Docker.""" + options: list[str] = [] + memory_limit = os.environ.get("COTTAS_MERGE_MEMORY_LIMIT", "").strip() + if memory_limit: + options.extend(["-e", f"COTTAS_MERGE_MEMORY_LIMIT={memory_limit}"]) + threads = os.environ.get("COTTAS_MERGE_THREADS", "").strip() + if threads: + options.extend(["-e", f"COTTAS_MERGE_THREADS={threads}"]) + return options + + def _can_write_dir(path: Path) -> bool: """Best-effort write probe for directories.""" try: @@ -3943,6 +3955,7 @@ def run_containerized_partitioned_representation_methods( *docker_run_base(), *docker_hdt_index_env_args(), *docker_hdt_merge_env_args(), + *docker_cottas_merge_env_args(), "--mount", f"type=volume,source={volume_name},target=/work", ] @@ -5089,6 +5102,7 @@ def run_index_mode( cmd = [ *docker_run_base(), *docker_hdt_index_env_args(), + *docker_cottas_merge_env_args(), "-v", f"{str(index_path.parent)}:/data/{mount_name}", image_ref, From b6d17c06d5ef2b200306dc88dde678dc6d26acd6 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 3 Sep 2026 17:41:50 +0200 Subject: [PATCH 11/19] cottas condensed index fix 2 --- Dockerfile | 8 +++- README.md | 3 +- changelog.md | 7 +++ src/cottas_tool.py | 19 +++++++- src/partitioned_compression.py | 57 ++++++++++++++--------- test/test_partitioned_compression_unit.py | 15 ++++++ 6 files changed, 83 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 56b05f5..bf230d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,8 +81,14 @@ RUN apt-get update \ time \ && rm -rf /var/lib/apt/lists/* +# Keep the DuckDB SQL dialect used by the disk-backed COTTAS merge stable. +# pycottas 1.1.0 accepts DuckDB >=1.2.2,<2, but leaving it unpinned makes a +# rebuild (or a cached layer) silently select a different Parquet COPY +# implementation. The adapter is exercised against 1.5.5. RUN python3 -m venv /opt/pycottas-venv \ - && /opt/pycottas-venv/bin/pip install --no-cache-dir pycottas==1.1.0 + && /opt/pycottas-venv/bin/pip install --no-cache-dir \ + pycottas==1.1.0 \ + duckdb==1.5.5 RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ diff --git a/README.md b/README.md index b1d2e1f..1371562 100644 --- a/README.md +++ b/README.md @@ -451,7 +451,8 @@ when memory is available. Temporary files live in the container's `/work` area and are removed after the attempt. COTTAS uses the same out-of-core principle for its global `DISTINCT` and -`ORDER BY` merge. The image defaults `COTTAS_MERGE_MEMORY_LIMIT` to `512M` and +`ORDER BY` merge. The image pins DuckDB to 1.5.5, the version tested with this +merge SQL, and defaults `COTTAS_MERGE_MEMORY_LIMIT` to `512M` and `COTTAS_MERGE_THREADS` to `1`; DuckDB spills merge state to `/work` instead of allowing one large condensed graph to consume all container memory. Override them from the host only when appropriate for the available RAM, for example: diff --git a/changelog.md b/changelog.md index a96a1c5..dbb9313 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,13 @@ ## 2026-09-03 — Disk-backed COTTAS merge for condensed cohorts +- Pinned the image to DuckDB 1.5.5 so COTTAS merge behavior cannot vary with + Docker build-cache state or the newest dependency accepted by pycottas. +- Included the bounded DuckDB stderr tail in code-1 merge failures. The error + now identifies the real cause (such as unavailable spill storage, permission + problems, a malformed chunk, or a DuckDB exception) instead of reporting + only `exit_code=1`. + - Replaced the final COTTAS merge/reindex implementation with a dedicated disk-backed DuckDB connection. `pycottas.cat` in version 1.1.0 performs its global `DISTINCT` and `ORDER BY` through the process-global in-memory diff --git a/src/cottas_tool.py b/src/cottas_tool.py index 3e92f42..636772e 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -104,8 +104,9 @@ def disk_backed_cottas_merge( quoted_inputs = ", ".join(sql_literal(path) for path in input_paths) parquet_scan = f"PARQUET_SCAN([{quoted_inputs}], union_by_name = true)" - connection = duckdb.connect(str(database_path)) + connection = None try: + connection = duckdb.connect(str(database_path)) # Apply limits before DuckDB plans the DISTINCT/ORDER BY operation. # One worker makes the memory budget predictable on hosts with many # CPUs and still permits DuckDB's external sort/hash operators to @@ -137,8 +138,22 @@ def disk_backed_cottas_merge( f"KV_METADATA {{index: {sql_literal(index.lower())}}})" ) connection.execute(copy_query) + except Exception as exc: + # This context is intentionally included in stderr. The host wrapper + # records it in the result JSON and surfaces it in the final error, + # so storage, permissions, Parquet, and DuckDB SQL errors are not + # collapsed into an unhelpful generic non-zero exit code. + duckdb_version = getattr(duckdb, "__version__", "unknown") + raise RuntimeError( + "disk-backed COTTAS merge failed " + f"(duckdb={duckdb_version}; inputs={len(input_paths)}; " + f"memory_limit={memory_limit}; threads={threads}; " + f"scratch={scratch_dir}; temp_directory={temporary_directory}; " + f"output={output_path}): {exc}" + ) from exc finally: - connection.close() + if connection is not None: + connection.close() if remove_input_files: for input_path in input_paths: diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 3f7696d..86b1521 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -450,6 +450,41 @@ def finalize_totals(total: dict) -> dict: } +def failure_message(stage_result: dict | None, fallback: str) -> str: + """Turn a failed subprocess result into an actionable pipeline error. + + ``StageRunner`` intentionally stores a bounded stderr tail in the result + payload. The outer CLI has no access to the ephemeral Docker volume once + the container exits, so omitting that text here turns a concrete DuckDB + error (for example an unwritable spill directory) into a useless + ``exit_code=1`` report. + """ + if not stage_result: + return fallback + exit_code = stage_result.get("exit_code") + diagnostics = [] + if exit_code is not None: + diagnostics.append(f"exit_code={exit_code}") + try: + numeric_exit_code = int(exit_code) + except (TypeError, ValueError): + numeric_exit_code = None + if numeric_exit_code == -9: + diagnostics.append( + "the process was killed by SIGKILL (usually the kernel/Docker OOM killer)" + ) + elif numeric_exit_code == 137: + diagnostics.append("the process was killed (often Docker memory/OOM pressure)") + elif numeric_exit_code == 143: + diagnostics.append("the process was terminated (SIGTERM)") + stderr_tail = " ".join(str(stage_result.get("stderr_tail") or "").split()) + if stderr_tail: + # Preserve the end of a traceback, which contains the concrete + # exception, without making top-level CLI output unbounded. + diagnostics.append(f"stderr={stderr_tail[-2048:]}") + return f"{fallback} ({'; '.join(diagnostics)})" if diagnostics else fallback + + def merge_pairwise( paths: list[Path], *, @@ -558,28 +593,6 @@ def record_index_warning( ) return warning - def failure_message(stage_result: dict | None, fallback: str) -> str: - """Turn a failed subprocess result into an actionable warning.""" - if not stage_result: - return fallback - exit_code = stage_result.get("exit_code") - diagnostics = [] - if exit_code is not None: - diagnostics.append(f"exit_code={exit_code}") - try: - numeric_exit_code = int(exit_code) - except (TypeError, ValueError): - numeric_exit_code = None - if numeric_exit_code == -9: - diagnostics.append( - "the process was killed by SIGKILL (usually the kernel/Docker OOM killer)" - ) - elif numeric_exit_code == 137: - diagnostics.append("the process was killed (often Docker memory/OOM pressure)") - elif numeric_exit_code == 143: - diagnostics.append("the process was terminated (SIGTERM)") - return f"{fallback} ({'; '.join(diagnostics)})" if diagnostics else fallback - def skipped_cottas_result(method: str) -> dict: artifact = { "cottas": output_cottas, diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index 7eeed0d..c4eba83 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -117,6 +117,21 @@ def test_stage_runner_keeps_failed_stderr_tail(self): self.assertIn("No space left on device", result["stderr_tail"]) self.assertFalse((work_dir / ".cottas-merge-r01-00000.stderr").exists()) + def test_failure_message_includes_a_bounded_stderr_tail(self): + """A code-1 COTTAS failure reports DuckDB's actual error to the user.""" + runner = load_runner_module() + message = runner.failure_message( + { + "exit_code": 1, + "stderr_tail": "Traceback\n" + "RuntimeError: disk-backed COTTAS merge failed: " + "IO Error: No space left on device", + }, + "COTTAS merge/index creation failed", + ) + self.assertIn("exit_code=1", message) + self.assertIn("No space left on device", message) + def test_stream_chunks_only_retains_the_chunk_being_consumed(self): """Gzip chunking does not stage a second full uncompressed aggregate.""" runner = load_runner_module() From a3aec0f0a2c54e027282db88a06f8360cf80c92a Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 3 Sep 2026 19:01:01 +0200 Subject: [PATCH 12/19] cottas condensed index fix and loading bar addition --- README.md | 14 +- THIRD_PARTY_NOTICES.md | 7 + changelog.md | 4 + conda-recipe/meta.yaml | 1 + pyproject.toml | 4 +- src/cottas_tool.py | 34 ++- src/partitioned_compression.py | 161 +++++++++- src/run_conversion.sh | 104 ++++++- test/test_cottas_tool.py | 12 +- test/test_partitioned_compression_unit.py | 30 ++ test/test_run_conversion_unit.py | 10 + vcf_rdfizer.py | 346 ++++++++++++++++++++-- 12 files changed, 668 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 1371562..595c532 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,14 @@ The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#] - Python 3.10+ - Docker (installed and running) +When VCF-RDFizer is connected to an interactive terminal, it shows a +lightweight Rich spinner/progress display. The display is disabled +automatically for redirected/CI output and can be disabled explicitly with +`--no-progress`. RMLStreamer progress reports the bytes and output parts +already written; partitioned HDT/COTTAS runs report source triples, chunks, +and the currently active merge/index stage. These updates are best-effort and +do not scan RDF content a second time or retain progress history in memory. + Install options: ```bash @@ -87,6 +95,7 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build - `-B, --no-build` fail if image not found +- `--no-progress` disable interactive spinners and progress bars - `-h, --help` show full usage ## Compression Plan @@ -451,8 +460,9 @@ when memory is available. Temporary files live in the container's `/work` area and are removed after the attempt. COTTAS uses the same out-of-core principle for its global `DISTINCT` and -`ORDER BY` merge. The image pins DuckDB to 1.5.5, the version tested with this -merge SQL, and defaults `COTTAS_MERGE_MEMORY_LIMIT` to `512M` and +external-sort, adjacent-row deduplication merge. The image pins DuckDB to +1.5.5, the version tested with this merge SQL, and defaults +`COTTAS_MERGE_MEMORY_LIMIT` to `512M` and `COTTAS_MERGE_THREADS` to `1`; DuckDB spills merge state to `/work` instead of allowing one large condensed graph to consume all container memory. Override them from the host only when appropriate for the available RAM, for example: diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index c93ec8b..eb7d66d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -5,6 +5,13 @@ VCF-RDFizer (this repository) is released under the MIT License (see `LICENSE`). The Docker image also includes third-party software. Their licenses remain with their respective authors and apply to those components. +## Python package dependency + +1. `Rich` +- Repository: +- Usage in this project: terminal spinners and progress bars +- License: MIT + ## Included in Docker image 1. `HDT-cpp` diff --git a/changelog.md b/changelog.md index dbb9313..e4a2eb1 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,10 @@ now identifies the real cause (such as unavailable spill storage, permission problems, a malformed chunk, or a DuckDB exception) instead of reporting only `exit_code=1`. +- Replaced the final merge's hash-backed `SELECT DISTINCT` with an external + lexical sort and `LAG`-based adjacent-row deduplication. This produces the + same RDF triple set and requested COTTAS ordering without requiring a global + in-memory hash table for every distinct triple. - Replaced the final COTTAS merge/reindex implementation with a dedicated disk-backed DuckDB connection. `pycottas.cat` in version 1.1.0 performs its diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index dce0364..5cd1663 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -24,6 +24,7 @@ requirements: - wheel run: - python >={{ python_min }} + - rich >=13.7.0 test: requires: diff --git a/pyproject.toml b/pyproject.toml index c39ec03..1500ab7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,9 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Bio-Informatics", ] -dependencies = [] +dependencies = [ + "rich>=13.7.0", +] [project.urls] Homepage = "https://github.com/ecrum19/VCF-RDFizer" diff --git a/src/cottas_tool.py b/src/cottas_tool.py index 636772e..b770752 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -78,12 +78,14 @@ def disk_backed_cottas_merge( """Merge COTTAS Parquet inputs with DuckDB spill files rather than pycottas.cat. ``pycottas.cat`` uses DuckDB's process-global in-memory connection. Its - global ``DISTINCT`` plus ``ORDER BY`` can therefore be SIGKILLed on a - large condensed VCF even when every disk-backed chunk conversion succeeds. - This adapter opens a dedicated on-disk database, caps its memory, restricts - merge parallelism, and directs external sort/hash spill files to the - disposable COTTAS scratch directory. The query retains COTTAS's global RDF - set semantics and the requested Parquet sort/index order. + hash-based global ``DISTINCT`` plus ``ORDER BY`` can therefore be SIGKILLed + on a large condensed VCF even when every disk-backed chunk conversion + succeeds. This adapter opens a dedicated on-disk database, caps its + memory, restricts merge parallelism, and directs external-sort spill files + to the disposable COTTAS scratch directory. It sorts all triples, removes + only adjacent equal triples with a streaming ``LAG`` window, then writes + the requested COTTAS index order. That retains RDF set semantics without + materializing one global hash table of every triple. """ if not input_paths: raise ValueError("at least one COTTAS input is required for a merge") @@ -125,14 +127,22 @@ def disk_backed_cottas_merge( } if not {"s", "p", "o"}.issubset(columns): raise RuntimeError("COTTAS inputs do not contain the required s, p, o columns") - selected_columns = ["s", "p", "o"] - if "g" in columns: - selected_columns.append("g") - selected_columns_sql = ", ".join(selected_columns) + # COTTAS's own cat operation writes RDF triples, rather than retaining + # an optional Parquet graph column. Match that contract when deciding + # whether two input records are semantically equal. + selected_columns_sql = "s, p, o" order_columns_sql = ", ".join(index.lower()) copy_query = ( - f"COPY (SELECT DISTINCT {selected_columns_sql} FROM {parquet_scan} " - f"ORDER BY {order_columns_sql}) TO {sql_literal(output_path)} " + "COPY (WITH ordered_rows AS (" + f"SELECT {selected_columns_sql}, " + "LAG(s) OVER (ORDER BY s, p, o) AS prior_s, " + "LAG(p) OVER (ORDER BY s, p, o) AS prior_p, " + "LAG(o) OVER (ORDER BY s, p, o) AS prior_o " + f"FROM {parquet_scan}" + ") SELECT s, p, o FROM ordered_rows " + "WHERE s IS DISTINCT FROM prior_s OR p IS DISTINCT FROM prior_p " + f"OR o IS DISTINCT FROM prior_o ORDER BY {order_columns_sql}) " + f"TO {sql_literal(output_path)} " "(FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 22, " "PARQUET_VERSION v2, " f"KV_METADATA {{index: {sql_literal(index.lower())}}})" diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 86b1521..42164ad 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -27,6 +27,69 @@ STDERR_TAIL_BYTES = 16 * 1024 DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "512M" DEFAULT_COTTAS_MERGE_THREADS = "1" +PROGRESS_HEARTBEAT_BYTES = 64 * 1024 * 1024 + + +def prepare_progress_path(path: Path | None) -> None: + """Best-effort setup for the optional host-mounted progress sidecar.""" + if path is None: + return + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError: + pass + + +def emit_progress( + path: Path | None, + stage: str, + phase: str, + *, + completed: int | float | None = None, + total: int | float | None = None, + unit: str | None = None, + detail: str | None = None, +) -> None: + """Append one small JSONL event; progress must never break conversion.""" + if path is None: + return + payload = {"stage": stage, "phase": phase} + for key, value in ( + ("completed", completed), + ("total", total), + ("unit", unit), + ("detail", detail), + ): + if value is not None: + payload[key] = value + try: + with path.open("a", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + except OSError: + pass + + +def progress_descriptor(name: str) -> tuple[str, str, int | None]: + """Map internal stage names to a small set of terminal progress tasks.""" + for prefix, stage in ( + ("hdt-build-", "hdt-chunks"), + ("cottas-build-", "cottas-chunks"), + ): + if name.startswith(prefix): + try: + return stage, "chunks", int(name[len(prefix) :]) + except ValueError: + break + if name.startswith("hdt-merge-"): + return "hdt-merge", "stage", None + if name.startswith("cottas-merge"): + return "cottas-merge", "stage", None + if name.startswith("hdt-"): + return "hdt", "stage", None + if name.startswith("cottas-"): + return "cottas", "stage", None + return name, "stage", None def is_triple_line(line: bytes) -> bool: @@ -46,6 +109,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--max-chunk-bytes", required=True, type=int) parser.add_argument("--expected-triples", type=int) parser.add_argument("--result-path", required=True) + parser.add_argument( + "--progress-path", + help="optional JSONL sidecar for host-side terminal progress", + ) parser.add_argument( "--allow-index-failures", action="store_true", @@ -68,6 +135,8 @@ def stream_chunks( target_bytes: int, min_bytes: int, max_bytes: int, + progress_path: Path | None = None, + progress_total: int | None = None, ) -> tuple[object, dict]: """Yield complete-record chunks while building a mutable chunk plan. @@ -84,6 +153,7 @@ def stream_chunks( raise ValueError("RDF chunk sizes must satisfy min <= target <= max") chunk_dir.mkdir(parents=True, exist_ok=True) + prepare_progress_path(progress_path) plan = { "source_file_count": 1, "source_paths": [str(source)], @@ -105,6 +175,7 @@ def generate(): logical_offset = 0 record_count = 0 chunk_index = 0 + last_progress_offset = 0 def open_chunk(): nonlocal handle, chunk_path, chunk_size, chunk_start_offset, chunk_start_record, chunk_index @@ -132,6 +203,18 @@ def close_chunk(): } plan["chunks"].append(metadata) plan["chunk_count"] = len(plan["chunks"]) + emit_progress( + progress_path, + "rdf-scan", + "chunk", + completed=record_count, + total=progress_total, + unit="triples", + detail=( + f"{plan['chunk_count']:,} chunks · " + f"{logical_offset:,} bytes read" + ), + ) completed_path = chunk_path handle = None chunk_path = None @@ -161,6 +244,19 @@ def close_chunk(): record_count += 1 plan["chunk_input_bytes"] = logical_offset plan["record_count"] = record_count + if ( + logical_offset - last_progress_offset >= PROGRESS_HEARTBEAT_BYTES + ): + emit_progress( + progress_path, + "rdf-scan", + "heartbeat", + completed=record_count, + total=progress_total, + unit="triples", + detail=f"{logical_offset:,} bytes read", + ) + last_progress_offset = logical_offset completed_chunk = close_chunk() if completed_chunk is not None: @@ -330,8 +426,10 @@ def number(pattern: str, integer: bool = False): class StageRunner: """Execute stages and accumulate both detailed and method-level metrics.""" - def __init__(self, work_dir: Path): + def __init__(self, work_dir: Path, progress_path: Path | None = None): self.work_dir = work_dir + self.progress_path = progress_path + prepare_progress_path(progress_path) self.stages: list[dict] = [] def run( @@ -343,6 +441,15 @@ def run( ) -> dict: time_path = self.work_dir / f".{name}.time" stderr_path = self.work_dir / f".{name}.stderr" + progress_stage, progress_unit, progress_ordinal = progress_descriptor(name) + emit_progress( + self.progress_path, + progress_stage, + "started", + completed=progress_ordinal, + unit=progress_unit, + detail=name, + ) if time_path.exists(): time_path.unlink() if stderr_path.exists(): @@ -390,6 +497,18 @@ def run( stdout_handle.close() if stderr_handle is not None: stderr_handle.close() + emit_progress( + self.progress_path, + progress_stage, + "complete" if completed.returncode == 0 else "failed", + completed=( + progress_ordinal + 1 + if progress_ordinal is not None and completed.returncode == 0 + else progress_ordinal + ), + unit=progress_unit, + detail=name, + ) stderr_tail = "" if stderr_path.exists(): try: @@ -533,7 +652,8 @@ def main() -> int: methods = [method.strip() for method in args.methods.split(",") if method.strip()] work_dir = Path("/work") chunk_dir = work_dir / "rdf_chunks" - runner = StageRunner(work_dir) + progress_path = Path(args.progress_path) if args.progress_path else None + runner = StageRunner(work_dir, progress_path=progress_path) results: dict[str, dict] = {} index_warnings: list[dict] = [] @@ -659,12 +779,22 @@ def cleanup_cottas_intermediates() -> None: cottas_python = os.environ.get("COTTAS_PYTHON_BIN") or shutil.which("python3") if any(method in COTTAS_METHODS for method in methods) and not cottas_python: raise RuntimeError("Missing Python runtime for COTTAS") + emit_progress( + progress_path, + "rdf-scan", + "started", + completed=0, + total=args.expected_triples, + unit="triples", + ) chunk_stream, plan = stream_chunks( source, chunk_dir, target_bytes=args.target_chunk_bytes, min_bytes=args.min_chunk_bytes, max_bytes=args.max_chunk_bytes, + progress_path=progress_path, + progress_total=args.expected_triples, ) def validate_artifact( @@ -772,6 +902,33 @@ def validate_artifact( finally: chunk.unlink(missing_ok=True) + emit_progress( + progress_path, + "rdf-scan", + "complete", + completed=plan["record_count"], + total=args.expected_triples, + unit="triples", + detail=f"{plan['chunk_count']:,} chunks", + ) + if needs_hdt: + emit_progress( + progress_path, + "hdt-chunks", + "complete", + completed=len(hdt_paths), + total=plan["chunk_count"], + unit="chunks", + ) + if any(method in COTTAS_METHODS for method in methods): + emit_progress( + progress_path, + "cottas-chunks", + "failed" if cottas_failed else "complete", + completed=len(cottas_paths), + total=plan["chunk_count"], + unit="chunks", + ) if not plan["chunks"]: raise ValueError("RDF source contains no complete records") source_triples = int(plan["record_count"]) diff --git a/src/run_conversion.sh b/src/run_conversion.sh index 0fdb0ff..c14cdae 100644 --- a/src/run_conversion.sh +++ b/src/run_conversion.sh @@ -57,6 +57,8 @@ cleanup_parts_dir() { rm -rf "$PARTS_DIR" } trap cleanup_parts_dir EXIT +PROGRESS_FILE=${PROGRESS_FILE:-} +PROGRESS_READY=0 TIME_LOG_DIR="$LOGDIR/conversion_time/${SAFE_OUT_NAME}" METRICS_JSON_DIR="$LOGDIR/conversion_metrics/${SAFE_OUT_NAME}" mkdir -p "$TIME_LOG_DIR" "$METRICS_JSON_DIR" @@ -111,6 +113,58 @@ stat_size() { echo 0 } +# Write sparse machine-readable progress events for the host-side Rich display. +# Progress is deliberately best-effort: a UI sidecar must never make a data +# conversion fail. +progress_emit() { + local stage="$1" + local phase="$2" + local completed="${3:-null}" + local total="${4:-null}" + local unit="${5:-}" + local parts="${6:-null}" + if [[ -z "$PROGRESS_FILE" ]]; then + return 0 + fi + if (( PROGRESS_READY == 0 )); then + mkdir -p "$(dirname "$PROGRESS_FILE")" >/dev/null 2>&1 || return 0 + PROGRESS_READY=1 + fi + printf '{"stage":"%s","phase":"%s","completed":%s,"total":%s,"unit":"%s","parts":%s}\n' \ + "$stage" "$phase" "$completed" "$total" "$unit" "$parts" \ + >> "$PROGRESS_FILE" 2>/dev/null || true +} + +# RMLStreamer writes part files while it runs. Summing their metadata once per +# second gives useful throughput feedback without reading or counting RDF. +part_bytes() { + local total=0 + local file size + shopt -s nullglob + for file in "$PARTS_DIR"/*; do + if [[ ! -f "$file" || "$(basename "$file")" == .* ]]; then + continue + fi + size=$(stat_size "$file") + total=$((total + size)) + done + shopt -u nullglob + echo "$total" +} + +part_count() { + local total=0 + local file + shopt -s nullglob + for file in "$PARTS_DIR"/*; do + if [[ -f "$file" && "$(basename "$file")" != .* ]]; then + total=$((total + 1)) + fi + done + shopt -u nullglob + echo "$total" +} + # Report comparable input VCF bytes. # - .vcf -> on-disk bytes # - .vcf.gz -> decompressed bytes @@ -254,10 +308,30 @@ VCF_SIZE=$(normalized_vcf_size "$IN_VCF") # ---------- Run RMLStreamer with timing ---------- EXIT_CODE=0 -if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG" -- "${JAVA_CMD[@]}" || EXIT_CODE=$? +run_rmlstreamer() { + if have_gnu_time; then + /usr/bin/time -v -o "$TIME_LOG" -- "${JAVA_CMD[@]}" + else + { time -p "${JAVA_CMD[@]}"; } >"$TIME_LOG" 2>&1 + fi +} + +if [[ -n "$PROGRESS_FILE" ]]; then + progress_emit "rmlstreamer" "started" 0 null bytes 0 + run_rmlstreamer & + RMLSTREAMER_PID=$! + while kill -0 "$RMLSTREAMER_PID" >/dev/null 2>&1; do + progress_emit "rmlstreamer" "heartbeat" "$(part_bytes)" null bytes "$(part_count)" + sleep 1 + done + wait "$RMLSTREAMER_PID" || EXIT_CODE=$? + if (( EXIT_CODE == 0 )); then + progress_emit "rmlstreamer" "complete" "$(part_bytes)" null bytes "$(part_count)" + else + progress_emit "rmlstreamer" "failed" "$(part_bytes)" null bytes "$(part_count)" + fi else - { time -p "${JAVA_CMD[@]}"; } >"$TIME_LOG" 2>&1 || EXIT_CODE=$? + run_rmlstreamer || EXIT_CODE=$? fi # Normalize output files to .nt for downstream line-oriented processing. The @@ -278,6 +352,11 @@ done # disk spikes. shopt -s nullglob PART_FILES=("$PARTS_DIR"/*.nt) +PART_TOTAL=${#PART_FILES[@]} +PART_INDEX=0 +if (( PART_TOTAL > 0 )); then + progress_emit "rdf-aggregate" "started" 0 "$PART_TOTAL" parts 0 +fi if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then MERGED_RDF="${MERGED_NT}.gz" MERGED_TMP="${MERGED_RDF}.partial.$$" @@ -296,6 +375,8 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then FIRST_SEEN=$(awk -F'\t' -v hash="$PART_HASH" '$1 == hash { print $2; exit }' "$SEEN_MAP_FILE") echo "WARNING: skipping duplicate RDF part '$PART_NT' (same content as '$FIRST_SEEN')." >&2 rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" continue fi printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" @@ -305,6 +386,8 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then # allowing each completed RMLStreamer part to be deleted immediately. gzip -c "$PART_NT" >> "$MERGED_TMP" rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" done rm -f "$SEEN_HASH_FILE" "$SEEN_MAP_FILE" mv "$MERGED_TMP" "$MERGED_RDF" @@ -327,22 +410,29 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then FIRST_SEEN=$(awk -F'\t' -v hash="$PART_HASH" '$1 == hash { print $2; exit }' "$SEEN_MAP_FILE") echo "WARNING: skipping duplicate RDF part '$PART_NT' (same content as '$FIRST_SEEN')." >&2 rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" continue fi printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" printf "%s\t%s\n" "$PART_HASH" "$PART_NT" >> "$SEEN_MAP_FILE" cat "$PART_NT" >> "$MERGED_NT" rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" done rm -f "$SEEN_HASH_FILE" "$SEEN_MAP_FILE" else : > "$MERGED_NT" OUTPUT_PATH="$MERGED_NT" -fi - shopt -u nullglob - if [[ "$RDF_STORAGE_MODE" != "space-optimized" ]]; then - OUTPUT_PATH="$MERGED_NT" fi +shopt -u nullglob +if [[ "$RDF_STORAGE_MODE" != "space-optimized" ]]; then + OUTPUT_PATH="$MERGED_NT" +fi +if (( PART_TOTAL > 0 )); then + progress_emit "rdf-aggregate" "complete" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" +fi rm -rf "$PARTS_DIR" trap - EXIT diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index 177a266..7bcb5e6 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -134,13 +134,11 @@ def test_merge_uses_a_bounded_disk_backed_duckdb_connection(self): self.assertEqual(fake_duckdb.database_paths[0].parent.parent, scratch_root) self.assertIn("SET memory_limit = '512M'", fake_duckdb.queries) self.assertIn("SET threads = 1", fake_duckdb.queries) - self.assertTrue( - any( - "SELECT DISTINCT s, p, o" in query - and "ORDER BY s, p, o" in query - for query in fake_duckdb.queries - ) - ) + copy_query = next(query for query in fake_duckdb.queries if query.startswith("COPY")) + self.assertIn("LAG(s) OVER (ORDER BY s, p, o)", copy_query) + self.assertIn("s IS DISTINCT FROM prior_s", copy_query) + self.assertIn("ORDER BY s, p, o", copy_query) + self.assertNotIn("SELECT DISTINCT", copy_query) self.assertFalse(left.exists()) self.assertFalse(right.exists()) self.assertFalse(any(scratch_root.iterdir())) diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index c4eba83..862c625 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -1,5 +1,6 @@ import gzip import importlib.util +import json import tempfile import unittest from pathlib import Path @@ -117,6 +118,26 @@ def test_stage_runner_keeps_failed_stderr_tail(self): self.assertIn("No space left on device", result["stderr_tail"]) self.assertFalse((work_dir / ".cottas-merge-r01-00000.stderr").exists()) + def test_stage_runner_emits_compact_progress_events(self): + """Stage progress reuses one task for chunk builds.""" + runner_module = load_runner_module() + with tempfile.TemporaryDirectory() as td: + work_dir = Path(td) / "work" + work_dir.mkdir() + progress_path = work_dir / ".progress" / "partitioned.jsonl" + stage_runner = runner_module.StageRunner(work_dir, progress_path=progress_path) + result = stage_runner.run( + "hdt-build-00000", + ["sh", "-c", "exit 0"], + ) + events = [json.loads(line) for line in progress_path.read_text().splitlines()] + self.assertEqual(result["exit_code"], 0) + self.assertEqual(events[0]["stage"], "hdt-chunks") + self.assertEqual(events[0]["phase"], "started") + self.assertEqual(events[-1]["phase"], "complete") + self.assertEqual(events[-1]["completed"], 1) + self.assertEqual(events[-1]["unit"], "chunks") + def test_failure_message_includes_a_bounded_stderr_tail(self): """A code-1 COTTAS failure reports DuckDB's actual error to the user.""" runner = load_runner_module() @@ -146,12 +167,15 @@ def test_stream_chunks_only_retains_the_chunk_being_consumed(self): handle.write(source_bytes) chunk_dir = tmp_path / "chunks" + progress_path = tmp_path / ".progress" / "partitioned.jsonl" stream, plan = runner.stream_chunks( source, chunk_dir, target_bytes=40, min_bytes=20, max_bytes=60, + progress_path=progress_path, + progress_total=12, ) emitted = [] for chunk, metadata in stream: @@ -166,6 +190,12 @@ def test_stream_chunks_only_retains_the_chunk_being_consumed(self): self.assertEqual(plan["chunk_count"], len(plan["chunks"])) self.assertGreater(plan["chunk_count"], 1) self.assertEqual(list(chunk_dir.glob("*.nt")), []) + progress_events = [ + json.loads(line) for line in progress_path.read_text().splitlines() + ] + self.assertTrue(all(event["stage"] == "rdf-scan" for event in progress_events)) + self.assertEqual(progress_events[-1]["completed"], 12) + self.assertEqual(progress_events[-1]["total"], 12) if __name__ == "__main__": diff --git a/test/test_run_conversion_unit.py b/test/test_run_conversion_unit.py index 79b1ab3..643fb84 100644 --- a/test/test_run_conversion_unit.py +++ b/test/test_run_conversion_unit.py @@ -60,6 +60,7 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): "LOGDIR": str(metrics_dir), "RUN_ID": "run123", "TIMESTAMP": "2026-01-01T00:00:00", + "PROGRESS_FILE": str(metrics_dir / ".progress" / "rml.jsonl"), } ) @@ -83,6 +84,15 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): self.assertEqual(row["output_name"], "rdf") self.assertEqual(row["exit_code_java"], "0") + progress_path = metrics_dir / ".progress" / "rml.jsonl" + progress_events = [ + json.loads(line) for line in progress_path.read_text().splitlines() + ] + self.assertEqual(progress_events[0]["stage"], "rmlstreamer") + self.assertEqual(progress_events[0]["phase"], "started") + self.assertEqual(progress_events[-1]["stage"], "rdf-aggregate") + self.assertEqual(progress_events[-1]["phase"], "complete") + def test_run_conversion_preserves_unrelated_files_in_existing_output_directory(self): """RMLStreamer parts do not require deleting the sample output directory.""" with tempfile.TemporaryDirectory() as td: diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index b0b8aee..1d667ef 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -31,10 +31,34 @@ from pathlib import Path from urllib.parse import quote_plus +try: + from rich.console import Console + from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, + ) +except ImportError: # pragma: no cover - package metadata installs Rich + Console = None + Progress = None + BarColumn = None + SpinnerColumn = None + TaskProgressColumn = None + TextColumn = None + TimeElapsedColumn = None + TimeRemainingColumn = None + RMLSTREAMER_JAR_CONTAINER = "/opt/rmlstreamer/RMLStreamer-v2.5.0-standalone.jar" _COMMAND_LOGGER = None _DOCKER_USE_SUDO = False +_ACTIVE_PROGRESS = None +_PROGRESS_ALLOWED = True +PROGRESS_POLL_INTERVAL_SECONDS = 0.25 COMPRESSED_VCF_EXPANSION_FACTOR = 5.0 TSV_OVERHEAD_FACTOR = 1.10 @@ -252,17 +276,29 @@ def run(self, cmd, cwd=None, env=None): self._handle.write(f"cwd={cwd}\n") self._handle.flush() - result = subprocess.run( - cmd, - cwd=cwd, - env=env, - stdout=self._handle, - stderr=self._handle, - text=True, - ) - self._handle.write(f"[exit {result.returncode}]\n") + if _ACTIVE_PROGRESS is not None and _ACTIVE_PROGRESS.enabled: + process = subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=self._handle, + stderr=self._handle, + text=True, + ) + exit_code = _ACTIVE_PROGRESS.wait_for_process(process) + else: + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + stdout=self._handle, + stderr=self._handle, + text=True, + ) + exit_code = result.returncode + self._handle.write(f"[exit {exit_code}]\n") self._handle.flush() - return result.returncode + return exit_code def close(self): if not self._handle.closed: @@ -290,6 +326,189 @@ def success_symbol() -> str: return ui_symbol("✅", "[ok]") +def progress_ui_enabled() -> bool: + """Return whether transient Rich progress output should be displayed.""" + if not _PROGRESS_ALLOWED or Progress is None or Console is None: + return False + if os.environ.get("VCF_RDFIZER_NO_PROGRESS"): + return False + if os.environ.get("CI"): + return False + stream = getattr(sys, "stderr", None) + isatty = getattr(stream, "isatty", None) + return bool(callable(isatty) and isatty()) + + +class ProgressSession: + """Render low-volume progress events from one Docker operation with Rich. + + The container writes newline-delimited JSON to ``path``. The host polls the + small sidecar while the Docker process runs, keeping command logs and + binary subprocess stdout separate from terminal UI output. + """ + + def __init__(self, path: Path | None, label: str): + self.path = path + self.label = label + self.enabled = progress_ui_enabled() + self._offset = 0 + self._progress = None + self._starter_task = None + self._tasks: dict[str, int] = {} + self._previous = None + + def __enter__(self): + global _ACTIVE_PROGRESS + self._previous = _ACTIVE_PROGRESS + _ACTIVE_PROGRESS = self + if not self.enabled: + return self + + if self.path is not None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.unlink(missing_ok=True) + + console = Console(stderr=True, highlight=False) + self._progress = Progress( + SpinnerColumn(), + TextColumn("{task.description}", style="progress.description", markup=False), + BarColumn(), + TaskProgressColumn(), + TextColumn("{task.fields[detail]}", markup=False), + TimeElapsedColumn(), + TimeRemainingColumn(), + console=console, + refresh_per_second=8, + transient=True, + auto_refresh=False, + ) + self._starter_task = self._progress.add_task( + self.label, + total=None, + detail="starting", + ) + self._progress.start() + return self + + @staticmethod + def _number(value): + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return value + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None + + @staticmethod + def _detail(event: dict) -> str: + completed = ProgressSession._number(event.get("completed")) + unit = event.get("unit") + if completed is None: + detail = "" + elif unit == "bytes": + detail = f"{format_bytes(int(completed))}" + elif unit == "triples": + detail = f"{int(completed):,} triples" + elif unit == "chunks": + detail = f"{int(completed):,} chunks" + elif unit == "parts": + detail = f"{int(completed):,} parts" + else: + detail = f"{int(completed):,}" + + parts = ProgressSession._number(event.get("parts")) + if parts is not None and unit != "parts": + detail = f"{detail} · {int(parts):,} parts" if detail else f"{int(parts):,} parts" + extra = event.get("detail") + if extra: + detail = f"{detail} · {extra}" if detail else str(extra) + return detail + + def _update_event(self, event: dict): + if self._progress is None: + return + stage = str(event.get("stage") or "work") + phase = str(event.get("phase") or "working") + task_id = self._tasks.get(stage) + if task_id is None: + if self._starter_task is not None: + self._progress.remove_task(self._starter_task) + self._starter_task = None + total = self._number(event.get("total")) + task_id = self._progress.add_task( + f"{stage}: {phase}", + total=total if total is not None else None, + detail=self._detail(event), + ) + self._tasks[stage] = task_id + return + + update = { + "description": f"{stage}: {phase}", + "detail": self._detail(event), + } + if "total" in event: + total = self._number(event.get("total")) + update["total"] = total if total is not None else None + completed = self._number(event.get("completed")) + if completed is not None: + update["completed"] = completed + self._progress.update(task_id, **update) + + def poll_events(self): + """Consume complete JSONL events without retaining the event stream.""" + if not self.enabled or self.path is None or not self.path.exists(): + return + try: + with self.path.open("rb") as handle: + handle.seek(self._offset) + data = handle.read() + except OSError: + return + + consumed = 0 + for raw_line in data.splitlines(keepends=True): + if not raw_line.endswith(b"\n"): + break + consumed += len(raw_line) + try: + event = json.loads(raw_line.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + if isinstance(event, dict): + self._update_event(event) + self._offset += consumed + + def wait_for_process(self, process): + """Wait while polling progress, without a second monitor thread.""" + while process.poll() is None: + self.poll_events() + if self._progress is not None: + self._progress.refresh() + time.sleep(PROGRESS_POLL_INTERVAL_SECONDS) + self.poll_events() + if self._progress is not None: + self._progress.refresh() + return process.returncode + + def __exit__(self, exc_type, exc_value, traceback): + global _ACTIVE_PROGRESS + self.poll_events() + if self._progress is not None: + self._progress.stop() + if self.path is not None: + try: + self.path.unlink() + except OSError: + pass + _ACTIVE_PROGRESS = self._previous + return False + + class RunTracker: """Track run progress and intermediate artifacts for safe interruption cleanup.""" @@ -400,6 +619,15 @@ def run(cmd, cwd=None, env=None): """ if _COMMAND_LOGGER is not None: return _COMMAND_LOGGER.run(cmd, cwd=cwd, env=env) + if _ACTIVE_PROGRESS is not None and _ACTIVE_PROGRESS.enabled: + process = subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return _ACTIVE_PROGRESS.wait_for_process(process) return subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True).returncode @@ -2495,6 +2723,26 @@ def safe_metrics_name(value: str) -> str: return safe or "rdf" +def progress_event_path(metrics_dir: Path | None, *components: str) -> Path | None: + """Return a hidden, per-operation progress sidecar path.""" + if metrics_dir is None: + return None + safe_components = [safe_metrics_name(component) for component in components] + filename = "-".join(safe_components or ["run"]) + ".jsonl" + return metrics_dir / ".progress" / filename + + +def container_progress_path(path: Path | None, metrics_dir: Path | None) -> str | None: + """Translate a host progress sidecar into its mounted container path.""" + if path is None or metrics_dir is None: + return None + try: + relative = path.resolve().relative_to(metrics_dir.resolve()) + except ValueError: + return None + return f"/data/metrics/{relative.as_posix()}" + + def metrics_header_for_methods(selected_methods: list[str]) -> list[str]: """Build a run-specific metrics.csv header with only relevant columns.""" methods = list(selected_methods or []) @@ -3037,7 +3285,8 @@ def run_tsv_conversion_with_metrics( ] started = time.perf_counter() - exit_code = run(cmd) + with ProgressSession(None, f"TSV conversion: {prefix}"): + exit_code = run(cmd) elapsed = time.perf_counter() - started timing = parse_time_log_metrics(time_log_host) @@ -3256,7 +3505,9 @@ def ensure_image_available( return 2 print(f"{step_label}: Ensuring Docker image is available") print(" - Building Docker image") - if docker_build_image(image_ref, repo_root) != 0: + with ProgressSession(None, "Building Docker image"): + image_exit_code = docker_build_image(image_ref, repo_root) + if image_exit_code != 0: eprint(f"Error: docker build failed. See log: {wrapper_log_path}") return 1 print(f"{step_label}: Ensuring Docker image is available {success_symbol()}") @@ -3269,7 +3520,9 @@ def ensure_image_available( if version_requested: print(f"{step_label}: Ensuring Docker image is available") print(f" - Pulling image: {image_ref}") - if docker_pull_image(image_ref) != 0: + with ProgressSession(None, "Pulling Docker image"): + image_exit_code = docker_pull_image(image_ref) + if image_exit_code != 0: eprint(f"Error: image version '{image_ref}' not found. See log: {wrapper_log_path}") return 2 print(f"{step_label}: Ensuring Docker image is available {success_symbol()}") @@ -3282,12 +3535,16 @@ def ensure_image_available( print(f"{step_label}: Ensuring Docker image is available") if has_local_dockerfile: print(" - Image missing locally, building") - if docker_build_image(image_ref, repo_root) != 0: + with ProgressSession(None, "Building Docker image"): + image_exit_code = docker_build_image(image_ref, repo_root) + if image_exit_code != 0: eprint(f"Error: docker build failed. See log: {wrapper_log_path}") return 1 else: print(f" - Image missing locally, pulling: {image_ref}") - if docker_pull_image(image_ref) != 0: + with ProgressSession(None, "Pulling Docker image"): + image_exit_code = docker_pull_image(image_ref) + if image_exit_code != 0: eprint(f"Error: image '{image_ref}' could not be pulled. See log: {wrapper_log_path}") return 2 print(f"{step_label}: Ensuring Docker image is available {success_symbol()}") @@ -3386,13 +3643,12 @@ def run_container_command( f"{str(in_dir)}:/data/in:ro", "-v", f"{str(out_dir)}:/data/out", - image_ref, - "bash", - "-lc", - wrapped_command, ] + cmd.extend([image_ref, "bash", "-lc", wrapped_command]) started = time.perf_counter() - exit_code = run(cmd) + progress_label = f"{method.replace('_', ' ').title()}: {input_stem}" + with ProgressSession(None, progress_label): + exit_code = run(cmd) elapsed = time.perf_counter() - started timing = parse_time_log_metrics(timing_host) output_path = target_out_dir / artifact_name @@ -3904,9 +4160,10 @@ def run_containerized_partitioned_representation_methods( ): """Run partitioned compression in an ephemeral Docker-managed volume. - The source and final outputs are the only bind mounts. Chunks, DuckDB - scratch data, HDT/COTTAS merge intermediates, and stage timing files stay - in a named volume that is removed in ``finally`` on both success and + The source and final outputs are the primary bind mounts. When interactive + progress is enabled, a small metrics sidecar is mounted as well. Chunks, + DuckDB scratch data, HDT/COTTAS merge intermediates, and stage timing files + stay in a named volume that is removed in ``finally`` on both success and failure. A short JSON handoff carries the container-side metrics back to the host without exposing the temporary workspace. """ @@ -3920,6 +4177,12 @@ def run_containerized_partitioned_representation_methods( f"vcf-rdfizer-{safe_output_name[:40]}-{os.getpid()}-{time.time_ns()}" ) result_path = out_dir / f".{safe_output_name}.partitioned-results.json" + progress_host_path = ( + progress_event_path(metrics_dir, "partitioned", safe_output_name) + if progress_ui_enabled() + else None + ) + progress_container_ref = container_progress_path(progress_host_path, metrics_dir) method_results: dict[str, dict] = {} volume_created = False @@ -3965,6 +4228,11 @@ def run_containerized_partitioned_representation_methods( [ "-v", f"{out_resolved}:/data/out", + *( + ["-v", f"{metrics_dir.resolve()}:/data/metrics"] + if metrics_dir is not None and progress_container_ref is not None + else [] + ), image_ref, "python3", PARTITIONED_COMPRESSION_RUNNER_CONTAINER, @@ -3985,11 +4253,17 @@ def run_containerized_partitioned_representation_methods( *(["--expected-triples", str(expected_triples)] if expected_triples is not None else []), "--result-path", f"/data/out/{result_path.name}", + *( + ["--progress-path", progress_container_ref] + if progress_container_ref is not None + else [] + ), ] ) if index_warnings is not None: command.append("--allow-index-failures") - run_exit_code = run(command) + with ProgressSession(progress_host_path, f"Partitioned compression: {output_name}"): + run_exit_code = run(command) payload = None if result_path.is_file(): try: @@ -4348,6 +4622,12 @@ def fail_current(stage: str, message: str): ) continue container_generated_rules = f"/data/rules/{generated_rules.name}" + progress_host_path = ( + progress_event_path(metrics_dir, "rmlstreamer", safe_prefix) + if progress_ui_enabled() + else None + ) + progress_container_ref = container_progress_path(progress_host_path, metrics_dir) run_cmd = [ *docker_run_base(), @@ -4434,11 +4714,13 @@ def fail_current(stage: str, message: str): f"IN_VCF={container_input}", "-e", "LOGDIR=/data/metrics", - image_ref, - "bash", - "/opt/vcf-rdfizer/run_conversion.sh", ] - if run(run_cmd) != 0: + if progress_container_ref is not None: + run_cmd.extend(["-e", f"PROGRESS_FILE={progress_container_ref}"]) + run_cmd.extend([image_ref, "bash", "/opt/vcf-rdfizer/run_conversion.sh"]) + with ProgressSession(progress_host_path, f"RMLStreamer: {prefix}"): + rdf_exit_code = run(run_cmd) + if rdf_exit_code != 0: fail_current( "rdf-conversion", f"RMLStreamer step failed for '{prefix}'. See log: {wrapper_log_path}", @@ -5484,6 +5766,11 @@ def main(): action="store_true", help="Print a rough storage estimate before running conversion", ) + parser.add_argument( + "--no-progress", + action="store_true", + help="Disable interactive Rich spinners and progress bars", + ) rdf_output_group = parser.add_mutually_exclusive_group() rdf_output_group.add_argument( "-R", @@ -5499,6 +5786,9 @@ def main(): ) args = parser.parse_args() + global _PROGRESS_ALLOWED + _PROGRESS_ALLOWED = not args.no_progress + if args.build and args.no_build: eprint("Error: --build and --no-build are mutually exclusive.") return 2 From ed8c9efb39edb273e004c4eda5c40c8b7142886f Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 3 Sep 2026 19:58:56 +0200 Subject: [PATCH 13/19] increased upper limit of memory allocation for cottas compression --- Dockerfile | 2 +- README.md | 25 ++++++++++------- changelog.md | 13 +++++++-- src/cottas_tool.py | 4 +-- src/partitioned_compression.py | 2 +- test/test_cottas_tool.py | 4 +-- test/test_vcf_rdfizer_unit.py | 37 ++++++++++++++++++++++++ vcf_rdfizer.py | 51 ++++++++++++++++++++++++++++------ 8 files changed, 111 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf230d9..9bffd84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -116,7 +116,7 @@ ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV HDTC_BIN=/usr/local/bin/hdtc ENV HDT_INDEX_MEMORY_LIMIT=512M ENV HDT_MERGE_MEMORY_LIMIT=512M -ENV COTTAS_MERGE_MEMORY_LIMIT=512M +ENV COTTAS_MERGE_MEMORY_LIMIT=4G ENV COTTAS_MERGE_THREADS=1 ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf diff --git a/README.md b/README.md index 595c532..8c180bb 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,13 @@ The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#] - Docker (installed and running) When VCF-RDFizer is connected to an interactive terminal, it shows a -lightweight Rich spinner/progress display. The display is disabled -automatically for redirected/CI output and can be disabled explicitly with -`--no-progress`. RMLStreamer progress reports the bytes and output parts -already written; partitioned HDT/COTTAS runs report source triples, chunks, -and the currently active merge/index stage. These updates are best-effort and -do not scan RDF content a second time or retain progress history in memory. +lightweight Rich spinner/progress display. With redirected output or without +Rich installed, it instead prints compact status lines; CI output remains +quiet. Either display can be disabled explicitly with `--no-progress`. +RMLStreamer progress reports the bytes and output parts already written; +partitioned HDT/COTTAS runs report source triples, chunks, and the currently +active merge/index stage. These updates are best-effort and do not scan RDF +content a second time or retain progress history in memory. Install options: @@ -95,7 +96,7 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build - `-B, --no-build` fail if image not found -- `--no-progress` disable interactive spinners and progress bars +- `--no-progress` disable terminal progress updates - `-h, --help` show full usage ## Compression Plan @@ -462,13 +463,13 @@ and are removed after the attempt. COTTAS uses the same out-of-core principle for its global `DISTINCT` and external-sort, adjacent-row deduplication merge. The image pins DuckDB to 1.5.5, the version tested with this merge SQL, and defaults -`COTTAS_MERGE_MEMORY_LIMIT` to `512M` and +`COTTAS_MERGE_MEMORY_LIMIT` to `4G` and `COTTAS_MERGE_THREADS` to `1`; DuckDB spills merge state to `/work` instead of allowing one large condensed graph to consume all container memory. Override them from the host only when appropriate for the available RAM, for example: ```bash -COTTAS_MERGE_MEMORY_LIMIT=1G COTTAS_MERGE_THREADS=2 vcf-rdfizer \ +COTTAS_MERGE_MEMORY_LIMIT=4G COTTAS_MERGE_THREADS=1 vcf-rdfizer \ --mode compress \ --rdf ./results/cohort/cohort.nt.gz \ --rdf-compression none \ @@ -667,7 +668,11 @@ which is normally the kernel/Docker OOM killer; a shell wrapper may report the same event as `137`. The warning's `stderr_tail`, `max_rss_kb`, and workspace samples distinguish memory pressure from a DuckDB/COTTAS or disk-space error. The disk-backed merge defaults to 512 MiB and one worker. Lower -`COTTAS_MERGE_MEMORY_LIMIT` if the container has a stricter memory cap, or +`COTTAS_MERGE_MEMORY_LIMIT` if the container has a stricter memory cap. The +external-sort merge still needs enough headroom for DuckDB's sort blocks; use +at least `1G` for large multi-sample inputs, and the default `4G` when the +Docker host can provide it. If you lower this value and see an allocation error, +raise it rather than lowering it further. Also verify raise it only when RAM is available; in all cases ensure Docker's data volume has enough free space for DuckDB spill files and the final Parquet rewrite. Rebuild the image after upgrading so the bounded disk-backed merge workflow is diff --git a/changelog.md b/changelog.md index e4a2eb1..470b14d 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,14 @@ ## 2026-09-03 — Disk-backed COTTAS merge for condensed cohorts +- Raised the default COTTAS DuckDB merge budget from `512M` to `4G`. DuckDB's + external sort spills data to disk, but it still requires an in-memory sort + block; a 52-chunk cohort merge requested a 128 MiB block after reaching the + old 512 MiB cap. The `4G` value is a cap, not an eager allocation. +- Made compression progress visible even when Rich cannot redraw a terminal + spinner (for example, with redirected stderr or a scheduler). Those runs now + emit compact source-scan, chunk-build, merge, and validation status lines. + - Pinned the image to DuckDB 1.5.5 so COTTAS merge behavior cannot vary with Docker build-cache state or the newest dependency accepted by pycottas. - Included the bounded DuckDB stderr tail in code-1 merge failures. The error @@ -20,7 +28,7 @@ `SIGKILL` on the 36-million-triple condensed cohort. - The new merge preserves the same global RDF set semantics and `spo` Parquet ordering, but configures DuckDB with a dedicated `.duckdb` database, - `COTTAS_MERGE_MEMORY_LIMIT=512M`, `COTTAS_MERGE_THREADS=1`, and a disposable + `COTTAS_MERGE_MEMORY_LIMIT=4G`, `COTTAS_MERGE_THREADS=1`, and a disposable `/work` spill directory. The final merge can therefore externalize sort and distinct state instead of exhausting container memory. - Forwarded explicit host-side `COTTAS_MERGE_MEMORY_LIMIT` and @@ -33,7 +41,8 @@ Build an image from this revision before retrying. Start with the default 512 MiB/one-worker merge budget; if the Docker cgroup cap is below that, -lower it (for example `COTTAS_MERGE_MEMORY_LIMIT=256M`). Ensure Docker has +lower it only when necessary (but keep it at least `1G` for a large cohort). +Ensure Docker has substantial free disk space for temporary DuckDB spill files. The preserved raw `.nt.gz` can be retried with `--mode compress`, avoiding another RDF run. diff --git a/src/cottas_tool.py b/src/cottas_tool.py index b770752..3d3a207 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -10,7 +10,7 @@ from pathlib import Path -DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "512M" +DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "4G" DEFAULT_COTTAS_MERGE_THREADS = 1 MEMORY_LIMIT_PATTERN = re.compile( r"^\d+(?:\.\d+)?\s*(?:B|K|M|G|T|KB|MB|GB|TB|KIB|MIB|GIB|TIB)$", @@ -49,7 +49,7 @@ def cottas_merge_memory_limit() -> str: if not MEMORY_LIMIT_PATTERN.fullmatch(memory_limit): raise ValueError( "COTTAS_MERGE_MEMORY_LIMIT must be a positive DuckDB byte value " - "such as 512M or 1G" + "such as 1G or 4G" ) return memory_limit diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 42164ad..5654118 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -25,7 +25,7 @@ HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} STDERR_TAIL_BYTES = 16 * 1024 -DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "512M" +DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "4G" DEFAULT_COTTAS_MERGE_THREADS = "1" PROGRESS_HEARTBEAT_BYTES = 64 * 1024 * 1024 diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index 7bcb5e6..570c99b 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -132,7 +132,7 @@ def test_merge_uses_a_bounded_disk_backed_duckdb_connection(self): self.assertTrue(fake_duckdb.closed) self.assertEqual(len(fake_duckdb.database_paths), 1) self.assertEqual(fake_duckdb.database_paths[0].parent.parent, scratch_root) - self.assertIn("SET memory_limit = '512M'", fake_duckdb.queries) + self.assertIn("SET memory_limit = '4G'", fake_duckdb.queries) self.assertIn("SET threads = 1", fake_duckdb.queries) copy_query = next(query for query in fake_duckdb.queries if query.startswith("COPY")) self.assertIn("LAG(s) OVER (ORDER BY s, p, o)", copy_query) @@ -185,7 +185,7 @@ def test_merge_many_passes_all_inputs_to_disk_backed_duckdb(self): for path in inputs: self.assertIn(str(path.resolve()), copy_query) self.assertFalse(path.exists()) - self.assertIn("SET memory_limit = '512M'", fake_duckdb.queries) + self.assertIn("SET memory_limit = '4G'", fake_duckdb.queries) self.assertIn("SET threads = 1", fake_duckdb.queries) self.assertTrue(fake_duckdb.closed) self.assertFalse(any(scratch_root.iterdir())) diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 94205f9..d0e3256 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -292,6 +292,43 @@ def test_cottas_merge_limits_are_forwarded_to_docker(self): ], ) + def test_plain_progress_is_shown_when_rich_or_a_tty_is_unavailable(self): + """Long compression work remains visible through redirected terminals.""" + previous_progress_setting = vcf_rdfizer._PROGRESS_ALLOWED + try: + vcf_rdfizer._PROGRESS_ALLOWED = True + with tempfile.TemporaryDirectory() as td, mock.patch.dict( + os.environ, + {"VCF_RDFIZER_NO_PROGRESS": "", "CI": ""}, + clear=False, + ), mock.patch.object(vcf_rdfizer, "Progress", None), mock.patch.object( + vcf_rdfizer, "Console", None + ): + progress_path = Path(td) / "partitioned.jsonl" + terminal = StringIO() + with redirect_stderr(terminal), vcf_rdfizer.ProgressSession( + progress_path, "Partitioned compression: cohort" + ) as session: + progress_path.write_text( + json.dumps( + { + "stage": "cottas-merge", + "phase": "started", + "unit": "stage", + "detail": "cottas-merge-disk", + } + ) + + "\n" + ) + session.poll_events() + + displayed = terminal.getvalue() + self.assertIn("Partitioned compression: cohort: started", displayed) + self.assertIn("cottas-merge started", displayed) + self.assertIn("cottas-merge-disk", displayed) + finally: + vcf_rdfizer._PROGRESS_ALLOWED = previous_progress_setting + def test_validator_counts_plain_and_gzip_ntriples(self): """The Docker validator's fallback source count handles .nt and .nt.gz.""" validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 1d667ef..a57670a 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -339,6 +339,21 @@ def progress_ui_enabled() -> bool: return bool(callable(isatty) and isatty()) +def progress_events_enabled() -> bool: + """Return whether progress sidecars should be consumed at all. + + Rich needs a TTY to redraw a spinner, but compression is often launched + through ``tee``, a scheduler, or a remote shell with redirected stderr. + Keep collecting the same low-volume events in those cases so + ``ProgressSession`` can render readable line-based status instead. + """ + if not _PROGRESS_ALLOWED: + return False + if os.environ.get("VCF_RDFIZER_NO_PROGRESS"): + return False + return not os.environ.get("CI") + + class ProgressSession: """Render low-volume progress events from one Docker operation with Rich. @@ -350,24 +365,29 @@ class ProgressSession: def __init__(self, path: Path | None, label: str): self.path = path self.label = label - self.enabled = progress_ui_enabled() + self.enabled = progress_events_enabled() + self.rich_enabled = self.enabled and progress_ui_enabled() self._offset = 0 self._progress = None self._starter_task = None self._tasks: dict[str, int] = {} self._previous = None + self._plain_event_states: dict[str, tuple[str, object, object]] = {} def __enter__(self): global _ACTIVE_PROGRESS self._previous = _ACTIVE_PROGRESS _ACTIVE_PROGRESS = self - if not self.enabled: - return self - if self.path is not None: self.path.parent.mkdir(parents=True, exist_ok=True) self.path.unlink(missing_ok=True) + if not self.enabled: + return self + if not self.rich_enabled: + eprint(f"{self.label}: started") + return self + console = Console(stderr=True, highlight=False) self._progress = Progress( SpinnerColumn(), @@ -429,10 +449,21 @@ def _detail(event: dict) -> str: return detail def _update_event(self, event: dict): - if self._progress is None: - return stage = str(event.get("stage") or "work") phase = str(event.get("phase") or "working") + if self._progress is None: + # A redirected terminal cannot host a redrawable Rich spinner. + # Emit compact, state-changing lines instead so lengthy + # compression still visibly advances in a terminal or log. + completed = event.get("completed") + total = event.get("total") + state = (phase, completed, total) + if self._plain_event_states.get(stage) != state: + self._plain_event_states[stage] = state + detail = self._detail(event) + suffix = f" — {detail}" if detail else "" + eprint(f" {self.label}: {stage} {phase}{suffix}") + return task_id = self._tasks.get(stage) if task_id is None: if self._starter_task is not None: @@ -500,6 +531,8 @@ def __exit__(self, exc_type, exc_value, traceback): self.poll_events() if self._progress is not None: self._progress.stop() + elif self.enabled: + eprint(f"{self.label}: finished") if self.path is not None: try: self.path.unlink() @@ -4179,7 +4212,7 @@ def run_containerized_partitioned_representation_methods( result_path = out_dir / f".{safe_output_name}.partitioned-results.json" progress_host_path = ( progress_event_path(metrics_dir, "partitioned", safe_output_name) - if progress_ui_enabled() + if progress_events_enabled() else None ) progress_container_ref = container_progress_path(progress_host_path, metrics_dir) @@ -4624,7 +4657,7 @@ def fail_current(stage: str, message: str): container_generated_rules = f"/data/rules/{generated_rules.name}" progress_host_path = ( progress_event_path(metrics_dir, "rmlstreamer", safe_prefix) - if progress_ui_enabled() + if progress_events_enabled() else None ) progress_container_ref = container_progress_path(progress_host_path, metrics_dir) @@ -5769,7 +5802,7 @@ def main(): parser.add_argument( "--no-progress", action="store_true", - help="Disable interactive Rich spinners and progress bars", + help="Disable terminal compression/conversion progress updates", ) rdf_output_group = parser.add_mutually_exclusive_group() rdf_output_group.add_argument( From 6a5b5492391737c352403b0e2f7db0b5a1155f54 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 3 Sep 2026 22:09:43 +0200 Subject: [PATCH 14/19] increased upper limit of memory allocation for cottas compression --- Dockerfile | 13 +- README.md | 80 +++-- THIRD_PARTY_NOTICES.md | 10 +- changelog.md | 65 ++-- src/cottas_tool.py | 386 +++++++++++++++------- src/partitioned_compression.py | 60 ++-- test/test_cottas_tool.py | 113 +++---- test/test_partitioned_compression_unit.py | 18 +- test/test_vcf_rdfizer_unit.py | 11 +- vcf_rdfizer.py | 12 +- 10 files changed, 441 insertions(+), 327 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9bffd84..51be464 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,14 +81,14 @@ RUN apt-get update \ time \ && rm -rf /var/lib/apt/lists/* -# Keep the DuckDB SQL dialect used by the disk-backed COTTAS merge stable. -# pycottas 1.1.0 accepts DuckDB >=1.2.2,<2, but leaving it unpinned makes a -# rebuild (or a cached layer) silently select a different Parquet COPY -# implementation. The adapter is exercised against 1.5.5. +# Keep the COTTAS conversion runtime and Parquet streaming-writer dialect +# stable. pycottas performs the per-chunk conversion; PyArrow performs the +# bounded-memory k-way merge of those already sorted chunks. RUN python3 -m venv /opt/pycottas-venv \ && /opt/pycottas-venv/bin/pip install --no-cache-dir \ pycottas==1.1.0 \ - duckdb==1.5.5 + duckdb==1.5.5 \ + pyarrow==22.0.0 RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ @@ -116,8 +116,7 @@ ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV HDTC_BIN=/usr/local/bin/hdtc ENV HDT_INDEX_MEMORY_LIMIT=512M ENV HDT_MERGE_MEMORY_LIMIT=512M -ENV COTTAS_MERGE_MEMORY_LIMIT=4G -ENV COTTAS_MERGE_THREADS=1 +ENV COTTAS_MERGE_BATCH_ROWS=2048 ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python diff --git a/README.md b/README.md index 8c180bb..e6e4ee7 100644 --- a/README.md +++ b/README.md @@ -437,9 +437,10 @@ vcf-rdfizer \ only the directory containing the selected artifact and writes metrics under `/run_metrics//index_metrics.json`. HDT indexing creates a versioned sidecar beside the input. COTTAS indexing rewrites the existing -`.cottas` file through a disk-backed DuckDB rewrite, keeping the data in the -same artifact while rebuilding its embedded index. If the operation fails, the -original COTTAS file is left in place; HDT's previous sidecars are restored. +`.cottas` file through a bounded streaming Parquet rewrite, keeping the data +in the same artifact while rebuilding its embedded index. If the operation +fails, the original COTTAS file is left in place; HDT's previous sidecars are +restored. HDT merging and indexing do not start a Java HDT tool. They use native `hdtc`: `hdtc create` merges partitioned HDTs and `hdtc index` streams an HDT through @@ -460,16 +461,16 @@ buffers and may increase temporary I/O; higher values can improve performance when memory is available. Temporary files live in the container's `/work` area and are removed after the attempt. -COTTAS uses the same out-of-core principle for its global `DISTINCT` and -external-sort, adjacent-row deduplication merge. The image pins DuckDB to -1.5.5, the version tested with this merge SQL, and defaults -`COTTAS_MERGE_MEMORY_LIMIT` to `4G` and -`COTTAS_MERGE_THREADS` to `1`; DuckDB spills merge state to `/work` instead of -allowing one large condensed graph to consume all container memory. Override -them from the host only when appropriate for the available RAM, for example: +COTTAS avoids both a global in-memory `DISTINCT` and a global external sort. +Each chunk is already written in `spo` order, so the final stage performs a +bounded k-way Parquet merge: it holds one small batch from each chunk, writes +one copy of each adjacent equal triple, and preserves the `spo` index. Its +memory use is controlled by `COTTAS_MERGE_BATCH_ROWS` (default `2048`), not by +the total RDF graph size or a temporary DuckDB sort area. Override it only to +tune the memory/throughput tradeoff, for example: ```bash -COTTAS_MERGE_MEMORY_LIMIT=4G COTTAS_MERGE_THREADS=1 vcf-rdfizer \ +COTTAS_MERGE_BATCH_ROWS=4096 vcf-rdfizer \ --mode compress \ --rdf ./results/cohort/cohort.nt.gz \ --rdf-compression none \ @@ -587,13 +588,14 @@ COTTAS chunk conversion uses `pycottas.rdf2cottas(..., disk=True)`. The final COTTAS merge deliberately does **not** call `pycottas.cat`: version 1.1.0 runs its global `DISTINCT` plus `ORDER BY` through an unbounded in-memory DuckDB connection, which can be killed on large condensed graphs. VCF-RDFizer instead -executes the equivalent global merge in a dedicated on-disk DuckDB database, -with a 512 MiB memory budget, one worker, and `/work` as the external spill -directory. This preserves global RDF set semantics and `spo` ordering while -making the memory requirement bounded by configuration. If the stage fails in -full mode, the warning contains the failing exit code and, when available, a -`stderr_tail`, maximum resident set size, and Docker-workspace free-space -samples; the raw RDF remains available for a retry. +uses a PyArrow k-way merge of the already `spo`-sorted Parquet chunks. It keeps +only a configurable batch from each input, drops adjacent duplicate triples, +and writes the final COTTAS file incrementally—no graph-wide DuckDB hash table +or external-sort spill directory is created. The merge emits processed-source +and distinct-written triple counts in the terminal progress display. If the +stage fails in full mode, the warning contains the failing exit code and, when +available, a `stderr_tail`, maximum resident set size, and Docker-workspace +free-space samples; the raw RDF remains available for a retry. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a streaming decode/count check. This verifies both readability and that the @@ -602,13 +604,13 @@ performed before `.hdt.gz`, `.hdt.br`, `.cottas.gz`, or `.cottas.br` packaging, and before raw RDF cleanup. Partitioned HDT/COTTAS compression runs in an ephemeral Docker-managed -workspace. Temporary RDF chunks, COTTAS/DuckDB scratch data, intermediate +workspace. Temporary RDF chunks, COTTAS conversion scratch data, intermediate representations, and merge files are not written to the output directory. After a successful or failed run, the temporary workspace is removed; only the selected final artifacts and normal run metrics remain on the host. -Each COTTAS conversion and merge also receives a fresh container-local DuckDB -workspace, which is removed as soon as that operation completes. This prevents -state from one chunk being reused by another and requires no user configuration. +Each COTTAS conversion receives a fresh container-local DuckDB workspace, +which is removed as soon as that operation completes. The final streaming +merge does not need a DuckDB workspace or a full-data temporary sort file. HDT merging and index generation use the pinned Rust `hdtc` 1.1.0 executable, not `hdtCat`, `hdtSearch.sh`, or another Java HDT process. `hdtc create` @@ -631,11 +633,10 @@ host value of either variable into the relevant Docker command. COTTAS does not expose a separate index sidecar. Its index is part of the Parquet artifact and is selected when the artifact is written. Standalone COTTAS index mode writes a new temporary COTTAS file through the same -memory-bounded, disk-backed DuckDB operation with the default `spo` index, -then atomically replaces the original. This is still index-only from the -pipeline's point of view: it does not rerun VCF-to-RDF conversion or create an -RDF output, but it may require temporary disk space and time comparable to -rewriting the COTTAS file. +bounded streaming rewrite with the default `spo` index, then atomically +replaces the original. This is still index-only from the pipeline's point of +view: it does not rerun VCF-to-RDF conversion or create an RDF output, but it +rewrites the artifact once. The record-safe chunk plan and per-stage timings are retained in the raw partitioned-compression metrics JSON for diagnostics. The temporary chunk @@ -662,27 +663,24 @@ finishes. If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). If COTTAS indexing/merging fails on a very large RDF file, first inspect the -`cottas-merge-disk` stage in the raw partitioned-compression metrics JSON and the +`cottas-merge-stream` stage in the raw partitioned-compression metrics JSON and the run wrapper log. An `exit_code=-9` means the child was killed by `SIGKILL`, which is normally the kernel/Docker OOM killer; a shell wrapper may report the same event as `137`. The warning's `stderr_tail`, `max_rss_kb`, and workspace -samples distinguish memory pressure from a DuckDB/COTTAS or disk-space error. -The disk-backed merge defaults to 512 MiB and one worker. Lower -`COTTAS_MERGE_MEMORY_LIMIT` if the container has a stricter memory cap. The -external-sort merge still needs enough headroom for DuckDB's sort blocks; use -at least `1G` for large multi-sample inputs, and the default `4G` when the -Docker host can provide it. If you lower this value and see an allocation error, -raise it rather than lowering it further. Also verify -raise it only when RAM is available; in all cases ensure Docker's data volume -has enough free space for DuckDB spill files and the final Parquet rewrite. -Rebuild the image after upgrading so the bounded disk-backed merge workflow is -installed. +samples distinguish COTTAS data/schema problems from Docker disk-space errors. +The streaming merge has no graph-wide DuckDB spill area: it reads at most +`COTTAS_MERGE_BATCH_ROWS` rows per input at a time (default `2048`) and writes +the result incrementally. If the host is especially memory-constrained, lower +that value; if the merge is CPU-bound and RAM is available, raise it gradually. +The final COTTAS artifact still needs ordinary output disk space. Rebuild the +image after upgrading so the streaming merge workflow is installed. If COTTAS is optional for the experiment, rerun with `--representations hdt`; the HDT path is independent and can remain the queryable artifact even when COTTAS cannot fit the available memory. If COTTAS -is required and the disk-backed merge still receives `-9`, lower -`COTTAS_MERGE_MEMORY_LIMIT`, verify that the Docker memory cap exceeds it, and +is required and the streaming merge still receives `-9`, lower +`COTTAS_MERGE_BATCH_ROWS`, verify that Docker has enough RAM for the selected +batch size, and check the host kernel log for an external kill; this is a resource limit, not a vocabulary or RDF-validity problem. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index eb7d66d..88fbce2 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -35,7 +35,15 @@ their respective authors and apply to those components. - Version bundled by the Docker image: `1.1.0` - Installed in the image's `/opt/pycottas-venv` environment -4. `hdtc` +4. `PyArrow` +- Repository: +- Usage in this project: bounded-memory Parquet reads and writes for the final + COTTAS k-way merge +- Upstream license: Apache License 2.0 +- Version bundled by the Docker image: `22.0.0` +- Installed in the image's `/opt/pycottas-venv` environment + +5. `hdtc` - Repository: - Usage in this project: Java-free, disk-backed merging of HDT chunks and index generation for existing HDT files diff --git a/changelog.md b/changelog.md index 470b14d..3e7a88b 100644 --- a/changelog.md +++ b/changelog.md @@ -1,50 +1,31 @@ # Changelog -## 2026-09-03 — Disk-backed COTTAS merge for condensed cohorts +## 2026-09-03 — Streaming COTTAS merge for condensed cohorts -- Raised the default COTTAS DuckDB merge budget from `512M` to `4G`. DuckDB's - external sort spills data to disk, but it still requires an in-memory sort - block; a 52-chunk cohort merge requested a 128 MiB block after reaching the - old 512 MiB cap. The `4G` value is a cap, not an eager allocation. -- Made compression progress visible even when Rich cannot redraw a terminal - spinner (for example, with redirected stderr or a scheduler). Those runs now - emit compact source-scan, chunk-build, merge, and validation status lines. - -- Pinned the image to DuckDB 1.5.5 so COTTAS merge behavior cannot vary with - Docker build-cache state or the newest dependency accepted by pycottas. -- Included the bounded DuckDB stderr tail in code-1 merge failures. The error - now identifies the real cause (such as unavailable spill storage, permission - problems, a malformed chunk, or a DuckDB exception) instead of reporting - only `exit_code=1`. -- Replaced the final merge's hash-backed `SELECT DISTINCT` with an external - lexical sort and `LAG`-based adjacent-row deduplication. This produces the - same RDF triple set and requested COTTAS ordering without requiring a global - in-memory hash table for every distinct triple. - -- Replaced the final COTTAS merge/reindex implementation with a dedicated - disk-backed DuckDB connection. `pycottas.cat` in version 1.1.0 performs its - global `DISTINCT` and `ORDER BY` through the process-global in-memory - connection, so both all-input and final pairwise merges could receive - `SIGKILL` on the 36-million-triple condensed cohort. -- The new merge preserves the same global RDF set semantics and `spo` Parquet - ordering, but configures DuckDB with a dedicated `.duckdb` database, - `COTTAS_MERGE_MEMORY_LIMIT=4G`, `COTTAS_MERGE_THREADS=1`, and a disposable - `/work` spill directory. The final merge can therefore externalize sort and - distinct state instead of exhausting container memory. -- Forwarded explicit host-side `COTTAS_MERGE_MEMORY_LIMIT` and - `COTTAS_MERGE_THREADS` values into partitioned full/compress runs and - standalone COTTAS index mode. -- Updated diagnostics, README troubleshooting, and regression tests to target - `cottas-merge-disk` rather than the obsolete pairwise merge path. +- Replaced the final COTTAS merge/reindex implementation with a PyArrow k-way + merge of the already `spo`-sorted Parquet chunks. It keeps one configurable + batch per input (`COTTAS_MERGE_BATCH_ROWS`, default `2048`), deduplicates + adjacent equal triples, and writes the final COTTAS artifact incrementally. + It preserves RDF set semantics and the embedded COTTAS index without a + graph-wide DuckDB hash table or external-sort spill area. +- This supersedes the prior DuckDB merge variants. A 52-chunk, 36-million-triple + cohort exhausted 41.4 GiB of Docker workspace while DuckDB attempted an + external sort; the streaming merge has no proportional temporary-sort file. +- Added PyArrow 22.0.0 to the image and third-party notices for deterministic + Parquet reader/writer support. +- COTTAS merge progress now reports source triples processed and distinct + triples written. Rich interactive displays remain available, and redirected + terminals receive compact line-based progress updates. +- Retained the bounded stderr tail in failures so malformed COTTAS input, + incompatible index metadata, output disk errors, and other code-1 failures + remain actionable after the ephemeral workspace is removed. ### Rerun guidance -Build an image from this revision before retrying. Start with the default -512 MiB/one-worker merge budget; if the Docker cgroup cap is below that, -lower it only when necessary (but keep it at least `1G` for a large cohort). -Ensure Docker has -substantial free disk space for temporary DuckDB spill files. The preserved -raw `.nt.gz` can be retried with `--mode compress`, avoiding another RDF run. +Build an image from this revision before retrying. The preserved raw `.nt.gz` +can be retried with `--mode compress`, avoiding another RDF run. The merge +requires normal space for the COTTAS chunks and final artifact, but no longer +requires a large DuckDB spill directory. ## 2026-09-03 — COTTAS SIGKILL/OOM handling @@ -63,7 +44,7 @@ raw `.nt.gz` can be retried with `--mode compress`, avoiding another RDF run. ### Rerun guidance Rebuild the image and rerun the full conversion (or use `--mode compress` with -the retained raw `.nt.gz`). The disk-backed merge workflow supersedes this +the retained raw `.nt.gz`). The current streaming k-way merge supersedes this earlier pairwise guidance. When COTTAS is optional, `--representations hdt` avoids the COTTAS merge diff --git a/src/cottas_tool.py b/src/cottas_tool.py index 3d3a207..a119227 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -1,21 +1,19 @@ #!/usr/bin/env python3 -"""Docker-side adapter for disk-backed COTTAS conversion and merge operations.""" +"""Docker-side adapter for bounded-memory COTTAS conversion and merging.""" import argparse +import heapq +import json import os -import re import sys import tempfile from contextlib import contextmanager from pathlib import Path -DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "4G" -DEFAULT_COTTAS_MERGE_THREADS = 1 -MEMORY_LIMIT_PATTERN = re.compile( - r"^\d+(?:\.\d+)?\s*(?:B|K|M|G|T|KB|MB|GB|TB|KIB|MIB|GIB|TIB)$", - re.IGNORECASE, -) +DEFAULT_COTTAS_MERGE_BATCH_ROWS = 2048 +COTTAS_OUTPUT_BATCH_ROWS = 16 * 1024 +COTTAS_MERGE_PROGRESS_ROWS = 250_000 @contextmanager @@ -36,56 +34,151 @@ def cottas_scratch_workspace(): os.chdir(original_working_directory) -def sql_literal(value: str | Path) -> str: - """Quote one filesystem value for the DuckDB SQL emitted by this adapter.""" - return "'" + str(value).replace("'", "''") + "'" - - -def cottas_merge_memory_limit() -> str: - """Return a validated DuckDB budget for COTTAS merge/reindex operations.""" - memory_limit = os.environ.get( - "COTTAS_MERGE_MEMORY_LIMIT", DEFAULT_COTTAS_MERGE_MEMORY_LIMIT +def cottas_merge_batch_rows() -> int: + """Return a small, bounded Parquet batch size for the streaming merge.""" + raw_rows = os.environ.get( + "COTTAS_MERGE_BATCH_ROWS", str(DEFAULT_COTTAS_MERGE_BATCH_ROWS) ).strip() - if not MEMORY_LIMIT_PATTERN.fullmatch(memory_limit): - raise ValueError( - "COTTAS_MERGE_MEMORY_LIMIT must be a positive DuckDB byte value " - "such as 1G or 4G" - ) - return memory_limit + try: + batch_rows = int(raw_rows) + except ValueError as exc: + raise ValueError("COTTAS_MERGE_BATCH_ROWS must be a positive integer") from exc + if batch_rows <= 0: + raise ValueError("COTTAS_MERGE_BATCH_ROWS must be a positive integer") + return batch_rows -def cottas_merge_threads() -> int: - """Return a bounded DuckDB worker count for deterministic merge memory use.""" - raw_threads = os.environ.get( - "COTTAS_MERGE_THREADS", str(DEFAULT_COTTAS_MERGE_THREADS) - ).strip() +def emit_merge_progress( + progress_path: Path | None, + phase: str, + *, + completed: int, + total: int, + detail: str, +) -> None: + """Append a best-effort COTTAS merge heartbeat for the host progress UI.""" + if progress_path is None: + return + payload = { + "stage": "cottas-merge", + "phase": phase, + "completed": completed, + "total": total, + "unit": "triples", + "detail": detail, + } try: - threads = int(raw_threads) - except ValueError as exc: - raise ValueError("COTTAS_MERGE_THREADS must be a positive integer") from exc - if threads <= 0: - raise ValueError("COTTAS_MERGE_THREADS must be a positive integer") - return threads + with progress_path.open("a", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + except OSError: + # A missing/unwritable optional sidecar must not invalidate an index. + pass + + +def cottas_file_index(parquet_file) -> str | None: + """Read COTTAS's embedded Parquet sort-index metadata when available.""" + metadata = getattr(parquet_file.metadata, "metadata", None) or {} + value = metadata.get(b"index") or metadata.get("index") + if value is None: + return None + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + return str(value).lower() + + +class CottasTripleStream: + """Read one sorted COTTAS Parquet file in a bounded number of rows.""" + + def __init__(self, parquet_module, path: Path, index: str, batch_rows: int): + self.path = path + self.index = index + self.parquet_file = parquet_module.ParquetFile(path) + self.schema = self.parquet_file.schema_arrow + missing_columns = {"s", "p", "o"} - set(self.schema.names) + if missing_columns: + raise RuntimeError( + f"COTTAS input {path.name} is missing columns: {', '.join(sorted(missing_columns))}" + ) + source_index = cottas_file_index(self.parquet_file) + if source_index != index: + found = source_index or "missing" + raise RuntimeError( + f"COTTAS input {path.name} is indexed as {found!r}, not {index!r}; " + "a streaming merge requires every input to use the requested index" + ) + self.fields = tuple(self.schema.field(name) for name in ("s", "p", "o")) + positions = {"s": 0, "p": 1, "o": 2} + self._sort_positions = tuple(positions[column] for column in index) + self._batches = self.parquet_file.iter_batches( + batch_size=batch_rows, + columns=["s", "p", "o"], + use_threads=False, + ) + self._values: tuple[list, list, list] | None = None + self._row = 0 + self.current: tuple | None = None + self.sort_key: tuple | None = None + self._previous_sort_key: tuple | None = None + self.exhausted = False + self.advance() + + @property + def row_count(self) -> int: + return int(self.parquet_file.metadata.num_rows) + + def advance(self) -> None: + """Move to one next triple, validating the COTTAS sort-order contract.""" + while self._values is None or self._row >= len(self._values[0]): + try: + batch = next(self._batches) + except StopIteration: + self.current = None + self.sort_key = None + self.exhausted = True + return + values = tuple(column.to_pylist() for column in batch.columns) + if not values[0]: + continue + self._values = values + self._row = 0 + + triple = (self._values[0][self._row], self._values[1][self._row], self._values[2][self._row]) + self._row += 1 + if any(value is None for value in triple): + raise RuntimeError(f"COTTAS input {self.path.name} contains a null RDF term") + sort_key = tuple(triple[position] for position in self._sort_positions) + if self._previous_sort_key is not None and sort_key < self._previous_sort_key: + raise RuntimeError( + f"COTTAS input {self.path.name} is not sorted by its declared {self.index!r} index" + ) + self._previous_sort_key = sort_key + self.current = triple + self.sort_key = sort_key + + def close(self) -> None: + """Release a Parquet file handle before the surrounding workspace exits.""" + close = getattr(self.parquet_file, "close", None) + if callable(close): + close() -def disk_backed_cottas_merge( +def streaming_cottas_merge( input_paths: list[str], output_path: str, *, index: str, remove_input_files: bool, + progress_path: Path | None = None, ) -> None: - """Merge COTTAS Parquet inputs with DuckDB spill files rather than pycottas.cat. - - ``pycottas.cat`` uses DuckDB's process-global in-memory connection. Its - hash-based global ``DISTINCT`` plus ``ORDER BY`` can therefore be SIGKILLed - on a large condensed VCF even when every disk-backed chunk conversion - succeeds. This adapter opens a dedicated on-disk database, caps its - memory, restricts merge parallelism, and directs external-sort spill files - to the disposable COTTAS scratch directory. It sorts all triples, removes - only adjacent equal triples with a streaming ``LAG`` window, then writes - the requested COTTAS index order. That retains RDF set semantics without - materializing one global hash table of every triple. + """Merge already-indexed COTTAS files without a global sort or spill area. + + Chunk conversion writes every COTTAS input in the requested lexical index + order. A k-way heap therefore needs only one small Parquet batch from each + input; equal triples meet at the heap head and are written once. This + preserves the RDF set and COTTAS index semantics while avoiding DuckDB's + full-data external sort, whose temporary files can exceed the original RDF + size for large multi-sample VCFs. """ if not input_paths: raise ValueError("at least one COTTAS input is required for a merge") @@ -93,77 +186,139 @@ def disk_backed_cottas_merge( raise ValueError("COTTAS merge index must be a permutation of spo") try: - import duckdb + import pyarrow as pa + import pyarrow.parquet as pq except ImportError as exc: - raise RuntimeError(f"DuckDB dependency is unavailable: {exc}") from exc - - scratch_dir = Path.cwd() - temporary_directory = scratch_dir / "duckdb-merge-tmp" - temporary_directory.mkdir(parents=True, exist_ok=True) - database_path = scratch_dir / "pycottas-merge.duckdb" - memory_limit = cottas_merge_memory_limit() - threads = cottas_merge_threads() - quoted_inputs = ", ".join(sql_literal(path) for path in input_paths) - parquet_scan = f"PARQUET_SCAN([{quoted_inputs}], union_by_name = true)" - - connection = None + raise RuntimeError( + "PyArrow is required for bounded-memory COTTAS merging: " + f"{exc}" + ) from exc + + normalized_index = index.lower() + batch_rows = cottas_merge_batch_rows() + streams: list[CottasTripleStream] = [] + temporary_path: Path | None = None + writer = None try: - connection = duckdb.connect(str(database_path)) - # Apply limits before DuckDB plans the DISTINCT/ORDER BY operation. - # One worker makes the memory budget predictable on hosts with many - # CPUs and still permits DuckDB's external sort/hash operators to - # spill to the named Docker volume. - connection.execute("SET preserve_insertion_order = false") - connection.execute("SET enable_progress_bar = false") - connection.execute(f"SET temp_directory = {sql_literal(temporary_directory)}") - connection.execute(f"SET memory_limit = {sql_literal(memory_limit)}") - connection.execute(f"SET threads = {threads}") - - columns = { - str(row[0]) - for row in connection.execute( - f"DESCRIBE SELECT * FROM {parquet_scan} LIMIT 1" - ).fetchall() - } - if not {"s", "p", "o"}.issubset(columns): - raise RuntimeError("COTTAS inputs do not contain the required s, p, o columns") - # COTTAS's own cat operation writes RDF triples, rather than retaining - # an optional Parquet graph column. Match that contract when deciding - # whether two input records are semantically equal. - selected_columns_sql = "s, p, o" - order_columns_sql = ", ".join(index.lower()) - copy_query = ( - "COPY (WITH ordered_rows AS (" - f"SELECT {selected_columns_sql}, " - "LAG(s) OVER (ORDER BY s, p, o) AS prior_s, " - "LAG(p) OVER (ORDER BY s, p, o) AS prior_p, " - "LAG(o) OVER (ORDER BY s, p, o) AS prior_o " - f"FROM {parquet_scan}" - ") SELECT s, p, o FROM ordered_rows " - "WHERE s IS DISTINCT FROM prior_s OR p IS DISTINCT FROM prior_p " - f"OR o IS DISTINCT FROM prior_o ORDER BY {order_columns_sql}) " - f"TO {sql_literal(output_path)} " - "(FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 22, " - "PARQUET_VERSION v2, " - f"KV_METADATA {{index: {sql_literal(index.lower())}}})" + streams = [ + CottasTripleStream(pq, Path(path), normalized_index, batch_rows) + for path in input_paths + ] + fields = streams[0].fields + expected_types = tuple(field.type for field in fields) + for stream in streams[1:]: + if tuple(field.type for field in stream.fields) != expected_types: + raise RuntimeError( + "COTTAS inputs do not share the same RDF term column types" + ) + + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output.name}.merge-", + suffix=".cottas", + dir=str(output.parent), + ) + os.close(descriptor) + temporary_path = Path(temporary_name) + temporary_path.unlink() + output_schema = pa.schema( + list(fields), metadata={b"index": normalized_index.encode("utf-8")} + ) + writer = pq.ParquetWriter( + temporary_path, + output_schema, + compression="zstd", + compression_level=22, + version="2.6", + ) + total_rows = sum(stream.row_count for stream in streams) + emit_merge_progress( + progress_path, + "started", + completed=0, + total=total_rows, + detail=f"{len(streams):,} sorted COTTAS chunks", + ) + heap = [ + (stream.sort_key, stream_number, stream.current) + for stream_number, stream in enumerate(streams) + if not stream.exhausted + ] + heapq.heapify(heap) + output_rows: list[tuple] = [] + previous_triple = None + processed_rows = 0 + written_rows = 0 + last_progress_rows = 0 + while heap: + _, stream_number, triple = heapq.heappop(heap) + processed_rows += 1 + if triple != previous_triple: + output_rows.append(triple) + previous_triple = triple + # Keep Parquet row groups substantially larger than the per-input + # read batch. That avoids creating tens of thousands of tiny row + # groups for a cohort-sized graph without changing input memory. + if len(output_rows) >= COTTAS_OUTPUT_BATCH_ROWS: + arrays = [ + pa.array([row[column] for row in output_rows], type=fields[column].type) + for column in range(3) + ] + writer.write_batch(pa.RecordBatch.from_arrays(arrays, schema=output_schema)) + written_rows += len(output_rows) + output_rows.clear() + + stream = streams[stream_number] + stream.advance() + if not stream.exhausted: + heapq.heappush(heap, (stream.sort_key, stream_number, stream.current)) + if processed_rows - last_progress_rows >= COTTAS_MERGE_PROGRESS_ROWS: + emit_merge_progress( + progress_path, + "merging", + completed=processed_rows, + total=total_rows, + detail=f"{written_rows + len(output_rows):,} distinct triples written", + ) + last_progress_rows = processed_rows + + if output_rows: + arrays = [ + pa.array([row[column] for row in output_rows], type=fields[column].type) + for column in range(3) + ] + writer.write_batch(pa.RecordBatch.from_arrays(arrays, schema=output_schema)) + written_rows += len(output_rows) + writer.close() + writer = None + os.replace(temporary_path, output) + temporary_path = None + emit_merge_progress( + progress_path, + "complete", + completed=processed_rows, + total=total_rows, + detail=f"{written_rows:,} distinct triples written", ) - connection.execute(copy_query) except Exception as exc: - # This context is intentionally included in stderr. The host wrapper - # records it in the result JSON and surfaces it in the final error, - # so storage, permissions, Parquet, and DuckDB SQL errors are not - # collapsed into an unhelpful generic non-zero exit code. - duckdb_version = getattr(duckdb, "__version__", "unknown") + # This context is surfaced by the host wrapper after its ephemeral + # Docker volume is removed, so a malformed or unexpectedly indexed + # input does not become a generic non-zero exit code. + pyarrow_version = getattr(pa, "__version__", "unknown") raise RuntimeError( - "disk-backed COTTAS merge failed " - f"(duckdb={duckdb_version}; inputs={len(input_paths)}; " - f"memory_limit={memory_limit}; threads={threads}; " - f"scratch={scratch_dir}; temp_directory={temporary_directory}; " + "streaming COTTAS merge failed " + f"(pyarrow={pyarrow_version}; inputs={len(input_paths)}; " + f"index={normalized_index}; batch_rows={batch_rows}; " f"output={output_path}): {exc}" ) from exc finally: - if connection is not None: - connection.close() + if writer is not None: + writer.close() + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + for stream in streams: + stream.close() if remove_input_files: for input_path in input_paths: @@ -197,6 +352,10 @@ def main() -> int: ) merge_many.add_argument("--output-cottas-file", required=True) merge_many.add_argument("--index", default="spo") + merge_many.add_argument( + "--progress-path", + help="optional JSONL sidecar for bounded streaming-merge progress", + ) reindex = subparsers.add_parser( "reindex", @@ -247,7 +406,7 @@ def main() -> int: # COTTAS indexes are part of the Parquet artifact rather than sibling # files. Rebuild into a temporary file in the same directory, then - # replace the original only after the disk-backed DuckDB rewrite + # replace the original only after the streaming Parquet rewrite # completes successfully. temporary_path = None try: @@ -260,7 +419,7 @@ def main() -> int: temporary_path = Path(temporary_name) temporary_path.unlink() with cottas_scratch_workspace(): - disk_backed_cottas_merge( + streaming_cottas_merge( [str(cottas_path)], str(temporary_path), index=args.index, @@ -282,11 +441,12 @@ def main() -> int: print("merge-many requires at least two input COTTAS files", file=sys.stderr) return 2 with cottas_scratch_workspace(): - disk_backed_cottas_merge( + streaming_cottas_merge( input_paths, cottas_path, index=args.index, remove_input_files=True, + progress_path=(Path(args.progress_path) if args.progress_path else None), ) return 0 @@ -294,7 +454,7 @@ def main() -> int: right_path = str(Path(args.right_path).resolve()) cottas_path = str(Path(args.cottas_path).resolve()) with cottas_scratch_workspace(): - disk_backed_cottas_merge( + streaming_cottas_merge( [left_path, right_path], cottas_path, index=args.index, diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index 5654118..d015f4e 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -25,8 +25,7 @@ HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} STDERR_TAIL_BYTES = 16 * 1024 -DEFAULT_COTTAS_MERGE_MEMORY_LIMIT = "4G" -DEFAULT_COTTAS_MERGE_THREADS = "1" +DEFAULT_COTTAS_MERGE_BATCH_ROWS = "2048" PROGRESS_HEARTBEAT_BYTES = 64 * 1024 * 1024 @@ -356,13 +355,15 @@ def cottas_merge_many_command( python_bin: str, inputs: list[Path], merged: Path, + *, + progress_path: Path | None = None, ) -> list[str]: - """Build one disk-backed, multi-input COTTAS merge command. + """Build one bounded-memory, multi-input COTTAS merge command. - The adapter performs the global distinct/order operation through a - dedicated DuckDB database with a bounded memory budget and `/work` spill - directory. Unlike ``pycottas.cat``, it never routes the large merge through - DuckDB's process-global in-memory connection. + The adapter performs a streaming k-way merge of the already indexed chunk + files. Unlike ``pycottas.cat`` and an external-sort query, it does not + construct a global in-memory hash table or a full-data temporary spill + area. """ return [ python_bin, @@ -374,6 +375,7 @@ def cottas_merge_many_command( str(merged), "--index", "spo", + *(["--progress-path", str(progress_path)] if progress_path else []), ] @@ -383,11 +385,11 @@ def cottas_merge_command( right: Path, merged: Path, ) -> list[str]: - """Build a compatible two-input disk-backed COTTAS merge command. + """Build a compatible two-input bounded-memory COTTAS merge command. - The normal partitioned workflow uses ``merge-many`` because its DuckDB - implementation is spill-capable. This command remains available for - explicit callers that need a two-input merge. + The normal partitioned workflow uses ``merge-many`` to merge all sorted + chunks in one pass. This command remains available for explicit callers + that need a two-input merge. """ return [ python_bin, @@ -697,8 +699,7 @@ def record_index_warning( "workspace_free_bytes_after", "workspace_total_bytes", "max_rss_kb", - "cottas_merge_memory_limit", - "cottas_merge_threads", + "cottas_merge_batch_rows", ): if key in stage_result: warning[key] = stage_result[key] @@ -1008,13 +1009,11 @@ def validate_artifact( } if cottas_paths and not cottas_failed: - # pycottas.cat performs its global DISTINCT/ORDER BY through an - # unbounded process-global in-memory DuckDB connection. That is - # what made both the one-shot and the final pairwise COTTAS merge - # susceptible to SIGKILL on large condensed cohorts. The adapter's - # merge-many command instead opens a dedicated disk-backed DuckDB - # database with a bounded memory limit and /work spill files, so a - # single global indexed merge is safe and avoids duplicate work. + # Each COTTAS chunk is already sorted by the `spo` index. The + # adapter k-way merges those ordered Parquet streams with one small + # batch per input, avoiding both pycottas.cat's global hash table + # and DuckDB external-sort spill files that exceeded 41 GiB for a + # 52-chunk condensed cohort. cottas_stage = None cottas_rounds = 0 if len(cottas_paths) == 1: @@ -1022,19 +1021,17 @@ def validate_artifact( else: cottas_merged_path = work_dir / "cottas-merge-final.cottas" cottas_stage = runner.run( - "cottas-merge-disk", + "cottas-merge-stream", cottas_merge_many_command( cottas_python, cottas_paths, cottas_merged_path, + progress_path=progress_path, ), cottas_merged_path, ) - cottas_stage["cottas_merge_memory_limit"] = os.environ.get( - "COTTAS_MERGE_MEMORY_LIMIT", DEFAULT_COTTAS_MERGE_MEMORY_LIMIT - ) - cottas_stage["cottas_merge_threads"] = os.environ.get( - "COTTAS_MERGE_THREADS", DEFAULT_COTTAS_MERGE_THREADS + cottas_stage["cottas_merge_batch_rows"] = os.environ.get( + "COTTAS_MERGE_BATCH_ROWS", DEFAULT_COTTAS_MERGE_BATCH_ROWS ) runner.stages[-1].update(cottas_stage) add_totals(cottas_total, cottas_stage) @@ -1091,13 +1088,10 @@ def validate_artifact( "details": { **plan, "merge_rounds": cottas_rounds, - "merge_strategy": "duckdb_disk_backed", - "merge_memory_limit": os.environ.get( - "COTTAS_MERGE_MEMORY_LIMIT", - DEFAULT_COTTAS_MERGE_MEMORY_LIMIT, - ), - "merge_threads": os.environ.get( - "COTTAS_MERGE_THREADS", DEFAULT_COTTAS_MERGE_THREADS + "merge_strategy": "pyarrow_streaming_k_way", + "merge_batch_rows": os.environ.get( + "COTTAS_MERGE_BATCH_ROWS", + DEFAULT_COTTAS_MERGE_BATCH_ROWS, ), "index": "spo", "validation": cottas_validation, diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index 570c99b..81f3c82 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -1,6 +1,5 @@ import importlib.util import os -import re import sys import tempfile import types @@ -21,40 +20,6 @@ def load_cottas_tool(): return module -class _FakeDuckDBCursor: - def __init__(self, rows=None): - self.rows = rows or [] - - def fetchall(self): - return self.rows - - -class _RecordingDuckDB: - """Minimal DuckDB stand-in for asserting the adapter's SQL contract.""" - - def __init__(self): - self.database_paths = [] - self.queries = [] - self.closed = False - - def connect(self, database_path): - self.database_paths.append(Path(database_path)) - return self - - def execute(self, query): - self.queries.append(query) - if query.startswith("DESCRIBE"): - return _FakeDuckDBCursor([("s",), ("p",), ("o",)]) - if query.startswith("COPY"): - match = re.search(r"\bTO '((?:''|[^'])*)'", query) - assert match is not None - Path(match.group(1).replace("''", "'")).write_text("merged COTTAS output\n") - return _FakeDuckDBCursor() - - def close(self): - self.closed = True - - class CottasToolTests(unittest.TestCase): def test_convert_uses_a_fresh_duckdb_workspace_for_each_invocation(self): """Sequential chunk builds cannot reuse pycottas.duckdb or its quads table.""" @@ -99,10 +64,17 @@ def fake_rdf2cottas(rdf_path, cottas_path, *, index, disk): self.assertTrue(all(path.parent == scratch_root for path in observed_workspaces)) self.assertFalse(any(scratch_root.iterdir())) - def test_merge_uses_a_bounded_disk_backed_duckdb_connection(self): - """Merge SQL is spill-capable instead of using pycottas.cat in memory.""" + def test_merge_uses_the_bounded_streaming_adapter(self): + """Two input merges delegate to the non-sorting streaming implementation.""" module = load_cottas_tool() - fake_duckdb = _RecordingDuckDB() + calls = [] + + def fake_streaming_merge(paths, output_path, *, index, remove_input_files, progress_path=None): + calls.append((paths, output_path, index, remove_input_files, progress_path)) + Path(output_path).write_text("merged COTTAS output\n") + if remove_input_files: + for path in paths: + Path(path).unlink() with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -115,12 +87,11 @@ def test_merge_uses_a_bounded_disk_backed_duckdb_connection(self): with mock.patch.dict( sys.modules, - { - "pycottas": types.SimpleNamespace(), - "duckdb": types.SimpleNamespace(connect=fake_duckdb.connect), - }, + {"pycottas": types.SimpleNamespace()}, ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "streaming_cottas_merge", side_effect=fake_streaming_merge ), mock.patch.object( sys, "argv", @@ -129,24 +100,25 @@ def test_merge_uses_a_bounded_disk_backed_duckdb_connection(self): self.assertEqual(module.main(), 0) self.assertTrue(output.is_file()) - self.assertTrue(fake_duckdb.closed) - self.assertEqual(len(fake_duckdb.database_paths), 1) - self.assertEqual(fake_duckdb.database_paths[0].parent.parent, scratch_root) - self.assertIn("SET memory_limit = '4G'", fake_duckdb.queries) - self.assertIn("SET threads = 1", fake_duckdb.queries) - copy_query = next(query for query in fake_duckdb.queries if query.startswith("COPY")) - self.assertIn("LAG(s) OVER (ORDER BY s, p, o)", copy_query) - self.assertIn("s IS DISTINCT FROM prior_s", copy_query) - self.assertIn("ORDER BY s, p, o", copy_query) - self.assertNotIn("SELECT DISTINCT", copy_query) + self.assertEqual( + calls, + [([str(left.resolve()), str(right.resolve())], str(output.resolve()), "spo", True, None)], + ) self.assertFalse(left.exists()) self.assertFalse(right.exists()) self.assertFalse(any(scratch_root.iterdir())) - def test_merge_many_passes_all_inputs_to_disk_backed_duckdb(self): - """The production merge scans every chunk through one spill-capable query.""" + def test_merge_many_passes_all_inputs_to_streaming_merge(self): + """The production merge passes every sorted chunk through one bounded pass.""" module = load_cottas_tool() - fake_duckdb = _RecordingDuckDB() + calls = [] + + def fake_streaming_merge(paths, output_path, *, index, remove_input_files, progress_path=None): + calls.append((paths, output_path, index, remove_input_files, progress_path)) + Path(output_path).write_text("merged COTTAS output\n") + if remove_input_files: + for path in paths: + Path(path).unlink() with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -160,12 +132,11 @@ def test_merge_many_passes_all_inputs_to_disk_backed_duckdb(self): with mock.patch.dict( sys.modules, - { - "pycottas": types.SimpleNamespace(), - "duckdb": types.SimpleNamespace(connect=fake_duckdb.connect), - }, + {"pycottas": types.SimpleNamespace()}, ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "streaming_cottas_merge", side_effect=fake_streaming_merge ), mock.patch.object( sys, "argv", @@ -181,13 +152,11 @@ def test_merge_many_passes_all_inputs_to_disk_backed_duckdb(self): self.assertEqual(module.main(), 0) self.assertTrue(output.is_file()) - copy_query = next(query for query in fake_duckdb.queries if query.startswith("COPY")) - for path in inputs: - self.assertIn(str(path.resolve()), copy_query) - self.assertFalse(path.exists()) - self.assertIn("SET memory_limit = '4G'", fake_duckdb.queries) - self.assertIn("SET threads = 1", fake_duckdb.queries) - self.assertTrue(fake_duckdb.closed) + self.assertEqual( + calls, + [([str(path.resolve()) for path in inputs], str(output.resolve()), "spo", True, None)], + ) + self.assertTrue(all(not path.exists() for path in inputs)) self.assertFalse(any(scratch_root.iterdir())) def test_decompress_uses_pycottas_and_isolated_scratch(self): @@ -228,11 +197,11 @@ def fake_cottas2rdf(cottas_path, rdf_path): self.assertFalse(any(scratch_root.iterdir())) def test_reindex_rewrites_atomically_without_removing_input(self): - """Reindex uses a disk-backed rewrite and replaces only on success.""" + """Reindex uses a streaming rewrite and replaces only on success.""" module = load_cottas_tool() calls = [] - def fake_disk_merge(paths, cottas_path, *, index, remove_input_files): + def fake_streaming_merge(paths, cottas_path, *, index, remove_input_files, progress_path=None): calls.append((paths, cottas_path, index, remove_input_files)) self.assertNotEqual(Path(cottas_path), source) self.assertTrue(Path(cottas_path).name.startswith(f".{source.name}.reindex-")) @@ -250,7 +219,7 @@ def fake_disk_merge(paths, cottas_path, *, index, remove_input_files): ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False ), mock.patch.object( - module, "disk_backed_cottas_merge", side_effect=fake_disk_merge + module, "streaming_cottas_merge", side_effect=fake_streaming_merge ), mock.patch.object( sys, "argv", @@ -265,11 +234,11 @@ def fake_disk_merge(paths, cottas_path, *, index, remove_input_files): ) self.assertEqual(list(root.glob(".input.cottas.reindex-*.cottas")), []) - def test_reindex_keeps_original_when_disk_backed_merge_fails(self): + def test_reindex_keeps_original_when_streaming_merge_fails(self): """A failed COTTAS rebuild does not replace the existing artifact.""" module = load_cottas_tool() - def failing_disk_merge(*args, **kwargs): + def failing_streaming_merge(*args, **kwargs): raise RuntimeError("simulated reindex failure") with tempfile.TemporaryDirectory() as temporary_directory: @@ -284,7 +253,7 @@ def failing_disk_merge(*args, **kwargs): ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False ), mock.patch.object( - module, "disk_backed_cottas_merge", side_effect=failing_disk_merge + module, "streaming_cottas_merge", side_effect=failing_streaming_merge ), mock.patch.object( sys, "argv", diff --git a/test/test_partitioned_compression_unit.py b/test/test_partitioned_compression_unit.py index 862c625..91773f2 100644 --- a/test/test_partitioned_compression_unit.py +++ b/test/test_partitioned_compression_unit.py @@ -57,8 +57,8 @@ def test_hdtc_merge_command_uses_bounded_native_merge(self): self.assertNotIn("java", " ".join(command).lower()) self.assertNotIn("hdtcat", " ".join(command).lower()) - def test_cottas_merge_many_command_uses_the_disk_backed_merge_adapter(self): - """Production COTTAS merging scans chunks through one spill-capable stage.""" + def test_cottas_merge_many_command_uses_the_streaming_merge_adapter(self): + """Production COTTAS merging scans sorted chunks in one bounded pass.""" runner = load_runner_module() command = runner.cottas_merge_many_command( "/opt/pycottas-venv/bin/python", @@ -81,6 +81,20 @@ def test_cottas_merge_many_command_uses_the_disk_backed_merge_adapter(self): ], ) + def test_cottas_merge_many_command_forwards_the_progress_sidecar(self): + """The long-running streaming merge can update the terminal display.""" + runner = load_runner_module() + command = runner.cottas_merge_many_command( + "/opt/pycottas-venv/bin/python", + [Path("/work/chunk-00000.cottas"), Path("/work/chunk-00001.cottas")], + Path("/work/cottas-merge-final.cottas"), + progress_path=Path("/data/metrics/.progress/partitioned.jsonl"), + ) + self.assertEqual( + command[-2:], + ["--progress-path", "/data/metrics/.progress/partitioned.jsonl"], + ) + def test_cottas_merge_command_remains_compatible_for_two_inputs(self): """The adapter continues to expose a two-input disk-backed merge command.""" runner = load_runner_module() diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index d0e3256..b95c568 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -273,22 +273,19 @@ def test_hdt_merge_memory_limit_is_forwarded_to_docker(self): ["-e", "HDT_MERGE_MEMORY_LIMIT=768M"], ) - def test_cottas_merge_limits_are_forwarded_to_docker(self): - """COTTAS merge memory and worker overrides reach Docker commands.""" + def test_cottas_merge_batch_rows_are_forwarded_to_docker(self): + """The bounded COTTAS streaming-merge batch size reaches Docker.""" with mock.patch.dict( os.environ, { - "COTTAS_MERGE_MEMORY_LIMIT": "384M", - "COTTAS_MERGE_THREADS": "1", + "COTTAS_MERGE_BATCH_ROWS": "1024", }, ): self.assertEqual( vcf_rdfizer.docker_cottas_merge_env_args(), [ "-e", - "COTTAS_MERGE_MEMORY_LIMIT=384M", - "-e", - "COTTAS_MERGE_THREADS=1", + "COTTAS_MERGE_BATCH_ROWS=1024", ], ) diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index a57670a..54666a8 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -703,15 +703,9 @@ def docker_hdt_merge_env_args() -> list[str]: def docker_cottas_merge_env_args() -> list[str]: - """Forward optional bounded-memory COTTAS merge settings into Docker.""" - options: list[str] = [] - memory_limit = os.environ.get("COTTAS_MERGE_MEMORY_LIMIT", "").strip() - if memory_limit: - options.extend(["-e", f"COTTAS_MERGE_MEMORY_LIMIT={memory_limit}"]) - threads = os.environ.get("COTTAS_MERGE_THREADS", "").strip() - if threads: - options.extend(["-e", f"COTTAS_MERGE_THREADS={threads}"]) - return options + """Forward an optional bounded COTTAS streaming-merge batch size.""" + batch_rows = os.environ.get("COTTAS_MERGE_BATCH_ROWS", "").strip() + return ["-e", f"COTTAS_MERGE_BATCH_ROWS={batch_rows}"] if batch_rows else [] def _can_write_dir(path: Path) -> bool: From fd829647b99110d33e05ba1c99d0506153fa5fd7 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Fri, 4 Sep 2026 12:49:57 +0200 Subject: [PATCH 15/19] added validation mode with test queries --- Dockerfile | 11 +- README.md | 119 ++- changelog.md | 36 +- docs/sample-representation-guide.md | 537 +++++++++++++ docs/validation.md | 91 +++ rules/README.md | 8 +- rules/default_rules.ttl | 2 +- src/compression.sh | 16 +- src/run_conversion.sh | 10 +- .../preflight_missing_token_conformance.rq | 9 + .../common/preflight_position_datatype.rq | 9 + .../common/preflight_record_cardinality.rq | 19 + .../queries/common/q01_record_density_1mb.rq | 10 + .../common/q02_variant_shape_counts.rq | 34 + src/validation/queries/common/q03_titv.rq | 12 + .../queries/common/q04_filter_distribution.rq | 11 + .../preflight_representation_profile.rq | 9 + .../preflight_sample_gt_inventory.rq | 16 + .../condensed/q05_sample_genotype_counts.rq | 27 + .../condensed/q06_ac_an_distribution.rq | 33 + .../preflight_representation_profile.rq | 9 + .../expanded/preflight_sample_gt_inventory.rq | 16 + .../expanded/q05_sample_genotype_counts.rq | 20 + .../expanded/q06_ac_an_distribution.rq | 28 + src/validation/validation_runner.py | 612 +++++++++++++++ test/test_compression_unit.py | 8 +- test/test_run_conversion_unit.py | 6 +- test/test_vcf_rdfizer_unit.py | 233 ++++-- vcf_rdfizer.py | 704 ++++++++++++++++-- vcf_rdfizer_data/rules/default_rules.ttl | 2 +- 30 files changed, 2453 insertions(+), 204 deletions(-) create mode 100644 docs/sample-representation-guide.md create mode 100644 docs/validation.md create mode 100644 src/validation/queries/common/preflight_missing_token_conformance.rq create mode 100644 src/validation/queries/common/preflight_position_datatype.rq create mode 100644 src/validation/queries/common/preflight_record_cardinality.rq create mode 100644 src/validation/queries/common/q01_record_density_1mb.rq create mode 100644 src/validation/queries/common/q02_variant_shape_counts.rq create mode 100644 src/validation/queries/common/q03_titv.rq create mode 100644 src/validation/queries/common/q04_filter_distribution.rq create mode 100644 src/validation/queries/condensed/preflight_representation_profile.rq create mode 100644 src/validation/queries/condensed/preflight_sample_gt_inventory.rq create mode 100644 src/validation/queries/condensed/q05_sample_genotype_counts.rq create mode 100644 src/validation/queries/condensed/q06_ac_an_distribution.rq create mode 100644 src/validation/queries/expanded/preflight_representation_profile.rq create mode 100644 src/validation/queries/expanded/preflight_sample_gt_inventory.rq create mode 100644 src/validation/queries/expanded/q05_sample_genotype_counts.rq create mode 100644 src/validation/queries/expanded/q06_ac_an_distribution.rq create mode 100644 src/validation/validation_runner.py diff --git a/Dockerfile b/Dockerfile index 51be464..2520435 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ ARG RMLSTREAMER_VERSION=2.5.0 ARG HDTC_VERSION=1.1.0 +ARG COMUNICA_VERSION=5.3.0 FROM eclipse-temurin:11-jre AS build-hdt-cpp @@ -65,6 +66,7 @@ ARG RMLSTREAMER_VERSION RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ + bcftools \ brotli \ ca-certificates \ coreutils \ @@ -76,8 +78,10 @@ RUN apt-get update \ liblzma5 \ libserd-0-0 \ nodejs \ + npm \ python3 \ python3-venv \ + raptor2-utils \ time \ && rm -rf /var/lib/apt/lists/* @@ -88,7 +92,11 @@ RUN python3 -m venv /opt/pycottas-venv \ && /opt/pycottas-venv/bin/pip install --no-cache-dir \ pycottas==1.1.0 \ duckdb==1.5.5 \ - pyarrow==22.0.0 + pyarrow==22.0.0 \ + numpy==2.4.6 \ + cyvcf2==0.34.0 + +RUN npm install --global "@comunica/query-sparql-file@${COMUNICA_VERSION}" RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ @@ -105,6 +113,7 @@ COPY --from=build-hdtc /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfize COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES.md COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ +COPY src/validation/ /opt/vcf-rdfizer/validation/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ && chmod +x /usr/local/bin/rdf2hdt \ diff --git a/README.md b/README.md index e6e4ee7..5baeb9f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ VCF-RDFizer is a Docker-first CLI wrapper for: 1. VCF -> RDF (N-Triples) with RMLStreamer 2. Optional RDF compression/decompression +3. Semantic validation of a compressed RDF graph against its source VCF The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#](https://w3id.org/vcf-rdfizer/vocab#). @@ -76,22 +77,23 @@ inside this directory. - `tsv`: VCF -> TSV only (benchmarking) - `compress`: compress an existing `.nt` or `.nt.gz` - `decompress`: decompress `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` +- `validation`: compare a source VCF with its `.nt.gz` RDF using six semantic SPARQL queries - `index`: only generate or regenerate the query index for an existing `.hdt` or `.cottas` In `full` mode with multiple VCF inputs, failures are isolated per input: - the run continues with remaining files -- failed inputs are summarized in `run_metrics//failed_inputs.csv` +- failed inputs are summarized in `run_metrics/__/reports/failed_inputs.csv` ## Main Flags (Most Used) -- `-m, --mode {full,compress,decompress,tsv,index}` +- `-m, --mode {full,compress,decompress,tsv,validation,index}` - `-o, --out` required output root directory - `--rdf-compression` final raw RDF codecs: `gzip`, `brotli`, or `none` - `--representations` queryable RDF outputs: `hdt`, `cottas`, or `none` - `--artifact-compression` packaging codecs for selected representations: `gzip`, `brotli`, or `none` - `--hdt-strategy {auto,partitioned,single}` HDT generation policy - `--chunk-target-bytes`, `--chunk-min-bytes`, `--chunk-max-bytes` shared record-safe chunk sizing -- `--sample-representation {dense,condensed}` genotype graph shape (`dense` by default) +- `--sample-representation {expanded,condensed}` genotype graph shape (`expanded` by default) - `-I, --image` Docker image repo (default `ecrum19/vcf-rdfizer`) - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build @@ -99,6 +101,25 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: - `--no-progress` disable terminal progress updates - `-h, --help` show full usage +## Validation Mode + +Validate one source VCF against the `.nt.gz` aggregate from the same +conversion. The aggregate is decompressed, parsed, and queried only inside the +Docker container; raw N-Triples are removed before the container exits. + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz \ + --rdf ./results/cohort/cohort.nt.gz \ + --sample-representation condensed \ + --out ./validation-results +``` + +Use `expanded` for the default graph shape and `condensed` for the vector-based +cohort graph. Reports are written to `/validation//`. See +[Semantic VCF/RDF validation](docs/validation.md) for query definitions, +preflight checks, result statuses, and cleanup evidence. + ## Compression Plan Compression is configured as three independent decisions: @@ -140,8 +161,8 @@ host filesystem. - `-i, --input` required VCF file or directory - `-r, --rules` mapping rules file (`.ttl`) - default: `rules/default_rules.ttl` -- `--sample-representation {dense,condensed}` sample genotype representation - - `dense` (default): one `SampleCall` per record/sample and one `FormatFieldValue` per FORMAT key +- `--sample-representation {expanded,condensed}` sample genotype representation + - `expanded` (default): one `SampleCall` per record/sample and one `FormatFieldValue` per FORMAT key - `condensed`: reusable file-level samples plus one ordered value vector per record/FORMAT key - `--rdf-storage-mode {plain,space-optimized}` required full-mode aggregate storage policy - `plain`: merge RMLStreamer parts into one uncompressed `.nt` @@ -170,23 +191,27 @@ Full mode has exactly two explicit sample workflows. There is no automatic sample-count threshold, so the same command always produces the same graph shape and downstream consumers can select the contract they support. -### Dense (default) +See [Expanded and Condensed Knowledge Representations](docs/sample-representation-guide.md) +for a detailed, worked explanation of the graph shapes, scaling behavior, and +selection trade-offs. + +### Expanded (default) -Use `--sample-representation dense` for single-sample and low-sample VCFs. It +Use `--sample-representation expanded` for single-sample and low-sample VCFs. It preserves the original vocabulary model: - every record/sample pair is a `vcfr:SampleCall`; - every represented FORMAT slot is a `vcfr:FormatFieldValue`; -- the VCF file declares `vcfr:representationProfile vcfr:DenseRepresentation`. +- the VCF file declares `vcfr:representationProfile vcfr:ExpandedRepresentation`. With the default rules, these triples are appended directly from `records.tsv`; -the large expanded helper TSVs are not materialized. The final graph is still -dense and grows approximately with `variants × samples × FORMAT fields`. +the large materialized helper TSVs are not created. The final graph is still +expanded and grows approximately with `variants × samples × FORMAT fields`. ```bash vcf-rdfizer --mode full \ --input ./small.vcf \ - --sample-representation dense \ + --sample-representation expanded \ --rdf-storage-mode plain \ --out ./results ``` @@ -222,18 +247,19 @@ vcf-rdfizer --mode full \ ``` The workflow resolver runs only one sample emitter. Condensed mode rejects -custom mappings that consume expanded `sample_calls.tsv` or -`sample_format_values.tsv`, because running those dense maps alongside the -condensed emitter would create both representations and restore the semantic +custom mappings that consume materialized `sample_calls.tsv` or +`sample_format_values.tsv`, because running those helper-table mappings alongside +the condensed emitter would create both representations and restore the semantic inflation this mode is designed to avoid. Remove those helper-table consumers -or select dense mode. Custom rules with no helper-table consumers remain +or select expanded mode. Custom rules with no helper-table consumers remain compatible with condensed emission. ## TSV Mode Flags - `-i, --input` required VCF file or directory -- Outputs per-run benchmark summary in `run_metrics//tsv_metrics.csv` -- Raw TSV timing + artifact JSON per input in `run_metrics//raw_metrics/tsv_*` +- Outputs per-run benchmark summary in `run_metrics/__/tsv_metrics.csv` +- Writes container timing and structured TSV metrics under `timings/tsv/` and + `stages/tsv/` in that run directory ## Compression Mode Flags @@ -435,7 +461,7 @@ vcf-rdfizer \ `--mode index` is deliberately an in-place maintenance operation. It mounts only the directory containing the selected artifact and writes metrics under -`/run_metrics//index_metrics.json`. HDT indexing creates a +`/run_metrics/__/stages/index/`. HDT indexing creates a versioned sidecar beside the input. COTTAS indexing rewrites the existing `.cottas` file through a bounded streaming Parquet rewrite, keeping the data in the same artifact while rebuilding its embedded index. If the operation @@ -483,7 +509,7 @@ itself remains readable; the run continues and the HDT can be repaired later with the standalone command above. If COTTAS generation/indexing cannot produce a usable artifact, COTTAS-specific outputs are skipped while the rest of the full pipeline continues. These warnings are printed in the run output -and written to `run_metrics//index_warnings.json`. The raw RDF is +and written to `run_metrics/__/reports/index_warnings.json`. The raw RDF is retained when a representation-dependent output was unavailable so the standalone index command or a later rerun has a recoverable source. @@ -494,7 +520,7 @@ Given `--out ./results`: - final outputs: - `./results//...` - per-run metrics/logs: - - `./results/run_metrics//...` + - `./results/run_metrics/__/...` - hidden intermediates: - `./results/.intermediate/tsv/` @@ -524,16 +550,37 @@ because that file is the gzip artifact itself. ## Metrics -For each run, VCF-RDFizer writes: +Each invocation receives a descriptive metrics directory: + +```text +run_metrics/__/ +``` -- `run_metrics//metrics.csv` -- `run_metrics//wrapper_execution_times.csv` -- `run_metrics//progress.log` -- `run_metrics//index_warnings.json` when full-run HDT/COTTAS index - generation was unsuccessful but the pipeline continued -- `run_metrics//index_metrics.json` for standalone HDT/COTTAS index mode -- `run_metrics//_index_metrics.json` is also written for the - selected format (`hdt` or `cottas`) for compatibility/discovery +`` is the source filename without its recognized VCF/RDF or +representation suffix (for example, `1000G_phase3_chr20`). A multi-file input +directory uses a batch label such as `batch-vcf_data-4-inputs`. This makes a +metrics directory recognizable without opening a timestamp-named folder. + +Within each run directory, VCF-RDFizer writes: + +- `run.json`: source identity, resolved input paths, requested workflow + configuration, and image selection +- `summary.json`: final status, wrapper wall time, summary table rows, and an + index of every stage report and log +- `metrics.csv`, `tsv_metrics.csv`, and `wrapper_execution_times.csv`: compact + analysis-ready tables when applicable +- `logs/wrapper.log` and `logs/progress.log` +- `timings//...`: raw GNU `time -v` output from inside the relevant + Docker container +- `stages/tsv/`, `stages/conversion/`, `stages/compression/`, + `stages/compression_operations/`, `stages/decompression/`, and + `stages/index/`: structured stage results. `compression_operations/` + preserves the underlying per-RDF operation and validation reports, while + `compression/` provides the final output-level summary. +- `stages/partitioned/`: the full result handoff from the temporary + partitioned-compression container, including every chunk build, merge, + validation, workspace free-space sample, exit code, CPU time, and peak RSS +- `reports/index_warnings.json` and `reports/failed_inputs.csv` when applicable Compression metrics now include per-method: @@ -561,9 +608,11 @@ usable. Explicit standalone `--mode index` runs remain strict and return a failure status when regeneration fails. For partitioned HDT/COTTAS runs, the final method metric reports one -sample-level result, while raw metrics also include a sample-scoped -`__partitioned_compression__` artifact describing chunk conversion, merge -rounds, and the generated chunk guide. +sample-level result while `stages/partitioned/.json` retains the full +container-stage history. It includes chunk conversion, merge strategy and +rounds, validation, generated chunk plan, workspace free-space samples, CPU +time, peak RSS, exit codes, and bounded stderr diagnostics. This report is +preserved even when the temporary Docker volume is deleted after a failure. Metrics may use internal stage names such as `hdt_gzip` and `cottas_brotli`. These correspond to the public combination of `--representations` and @@ -643,9 +692,9 @@ partitioned-compression metrics JSON for diagnostics. The temporary chunk files and guide are not retained as host files. For the default mapping, multi-sample VCF columns remain compact in -`records.tsv`. In dense mode, canonical `SampleCall` and `FormatFieldValue` +`records.tsv`. In expanded mode, canonical `SampleCall` and `FormatFieldValue` triples are streamed directly into the final `.nt` or `.nt.gz` aggregate rather -than first writing expanded helper rows. In condensed mode, the same input pass +than first writing materialized helper rows. In condensed mode, the same input pass emits shared samples, call matrices, and FORMAT vectors, avoiding both the helper-table multiplier and the per-sample RDF structural multiplier. @@ -698,7 +747,7 @@ Docker volume capacity. Safe termination: - Press `Ctrl+C` to interrupt a run. -- The wrapper exits with code `130`, writes progress to `run_metrics//progress.log`, and performs best-effort cleanup of tracked intermediates. +- The wrapper exits with code `130`, writes progress to `run_metrics/__/logs/progress.log`, and performs best-effort cleanup of tracked intermediates. - Raw RDF cleanup on interrupt follows `--keep-rmlstreamer-rdf-output`: - with `--keep-rmlstreamer-rdf-output`, raw RDF files are preserved - without it, tracked raw RDF files are removed during interrupt cleanup diff --git a/changelog.md b/changelog.md index 3e7a88b..2e8c9dd 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,37 @@ # Changelog +## 2026-09-04 — Expanded sample representation naming + +- Renamed the default per-sample genotype graph strategy to `expanded`, while + retaining `condensed` as the alternative strategy. +- Updated the CLI default and accepted values, internal workflow/emitter names, + tests, RML comments, and user documentation. +- The emitted profile IRI is now + `vcfr:representationProfile vcfr:ExpandedRepresentation`. + +## 2026-09-04 — Input-labelled, container-complete metrics layout + +- Replaced timestamp-only run-metrics directories with + `run_metrics/__/`. A single VCF/RDF/representation input + uses its source stem (for example, `1000G_phase3_chr20`); a directory input + with multiple VCFs receives a deterministic batch label. +- Added `run.json` and `summary.json` as the stable entry points for every + mode. The manifest records inputs, requested configuration, output root, and + resolved image; the summary records final status, wrapper runtime, tabular + rows, and an index of stage reports and logs. +- Organized metrics by purpose: `logs/`, `timings/`, `stages/`, and `reports/`. + TSV, RML conversion, compression, decompression, indexing, and warnings now + have predictable locations instead of unrelated `raw_metrics`, + `*_metrics`, and timestamp-nested directories. +- Compression-only, decompression, and standalone index modes now persist + container wall/CPU/system time, peak RSS, exit code, input/output sizes, and + structured stage reports. Full and TSV modes retain the same detail. +- Preserved the complete partitioned-compression runner handoff under + `stages/partitioned/`, including every chunk build, merge, validation, + temporary-workspace free-space sample, timing, peak RSS, and bounded stderr + diagnostic. The report is retained on both successful and failed runs before + the temporary Docker volume is removed. + ## 2026-09-03 — Streaming COTTAS merge for condensed cohorts - Replaced the final COTTAS merge/reindex implementation with a PyArrow k-way @@ -72,8 +104,8 @@ resource requirement while retaining a queryable representation. ### Rerun guidance Rebuild the image and rerun the same full command. If the COTTAS stage still -fails, inspect `raw_metrics/compression_metrics/.../__partitioned_compression__` -and the run wrapper log for the failing `cottas-merge-r*` stage and its +fails, inspect `stages/partitioned/.json` and the wrapper log for the +failing `cottas-merge-r*` stage and its `stderr_tail`. For a resource error, lower `--chunk-target-bytes` and `--chunk-max-bytes`, or enlarge the Docker data-volume allocation. The raw `.nt.gz` retained by the failed run is a valid recovery source. diff --git a/docs/sample-representation-guide.md b/docs/sample-representation-guide.md new file mode 100644 index 0000000..1488cbc --- /dev/null +++ b/docs/sample-representation-guide.md @@ -0,0 +1,537 @@ +# Expanded and Condensed Knowledge Representations + +This guide explains the two ways VCF-RDFizer can represent sample genotype +information in RDF: + +- **expanded**, the default representation; +- **condensed**, selected with `--sample-representation condensed`. + +The word *representation* here means the shape of the RDF graph. It does not +mean that the input VCF is changed or that genotype values are discarded. + +## 1. Why sample representation matters + +A VCF is naturally a wide table. A small part of a multi-sample VCF might look +like this: + +```text +#CHROM POS REF ALT FORMAT SAMPLE_A SAMPLE_B SAMPLE_C +20 100 A G GT:DP:AD 0/1:42:30,12 0/0:18:18,0 1/1:9:9,0 +``` + +There is one variant row, followed by one sample column for every individual. +The `FORMAT` column says that each sample value has three fields: + +- `GT`: genotype; +- `DP`: read depth; +- `AD`: allele depths. + +For a real cohort, the numbers can be much larger. For example, a file with +10,000 variants, 2,500 samples, and three FORMAT fields contains: + +```text +10,000 × 2,500 × 3 = 75,000,000 individual FORMAT values +``` + +RDF can describe each of those values separately, but doing so creates a very +large number of RDF resources and relationships. The two modes let users +choose between a graph that is easy to query one value at a time and a graph +that is much more economical for large cohorts. + +## 2. Expanded representation: one RDF object per sample value + +Expanded mode preserves the original, explicit vocabulary model. For every +variant/sample pair, VCF-RDFizer creates a `vcfr:SampleCall`. For every FORMAT +field in that pair, it creates a `vcfr:FormatFieldValue`. + +For the example above, the conceptual graph includes resources like these +(the full output has one line per RDF statement): + +```text + + vcfr:hasSampleCall . + + + a vcfr:SampleCall ; + vcfr:sampleId "SAMPLE_A" ; + vcfr:hasFormatValue . + + + a vcfr:FormatFieldValue ; + vcfr:fieldValue "0/1" . +``` + +The same structure is repeated for `DP` and `AD`, and then repeated again for +`SAMPLE_B` and `SAMPLE_C`. + +### What expanded mode provides + +Expanded mode makes each value easy to address directly: + +- a consumer can find one sample call without decoding a vector; +- a consumer can ask for one FORMAT field as one RDF resource; +- existing consumers that expect `SampleCall` and `FormatFieldValue` can use + the graph directly; +- the sample identifier is attached to every sample-call resource. + +The VCF file is marked with +`vcfr:representationProfile vcfr:ExpandedRepresentation`, so a consumer can +identify the graph shape explicitly. + +### Expanded growth + +Let: + +- `V` = number of variant records; +- `S` = number of samples; +- `F` = average number of FORMAT fields per record. + +Expanded mode creates approximately: + +```text +V × S SampleCall resources +V × S × F FormatFieldValue resources +``` + +In the simplified case where each sample call has three structural statements +(link, type, and sample ID), and each FORMAT value has three statements (link, +type, and value), the sample portion of the graph contains approximately: + +```text +3 × (V × S) + 3 × (V × S × F) +``` + +statements, before counting the rest of the VCF graph. + +For the worked example (`V=1`, `S=3`, `F=3`): + +| Expanded item | Count | +|---|---:| +| `SampleCall` resources | 3 | +| `FormatFieldValue` resources | 9 | +| Approximate sample-related statements | 36 | +| Representation-profile statement | 1 | + +The exact number can vary when values are missing or when custom mappings add +properties, but the important pattern is the multiplication by both samples +and FORMAT fields. + +## 3. Condensed representation: shared samples plus ordered vectors + +Condensed mode avoids creating a separate RDF resource for every scalar sample +value. It describes the sample columns once, then stores the values for each +variant and FORMAT key in one ordered vector. + +For the same example, the graph first declares a reusable sample set: + +```text + + a vcfr:SampleSet ; + vcfr:hasSample ; + vcfr:hasSample ; + vcfr:hasSample . + + + a vcfr:VCFSample ; + vcfr:sampleName "SAMPLE_A" ; + vcfr:sampleIndex "1" . +``` + +The variant then has one cohort matrix. The matrix points back to the sample +set and has one vector for each FORMAT key: + +```text + + a vcfr:CohortCallMatrix ; + vcfr:appliesToSampleSet ; + vcfr:hasFormatValueVector . + + + a vcfr:FormatValueVector ; + vcfr:valueEncoding vcfr:VCFTextVector ; + vcfr:encodedValues "0/1\t0/0\t1/1" . +``` + +The three tab-separated items in `encodedValues` are in `sampleIndex` order: + +| `sampleIndex` | Sample | `GT` value in the vector | +|---:|---|---| +| 1 | `SAMPLE_A` | `0/1` | +| 2 | `SAMPLE_B` | `0/0` | +| 3 | `SAMPLE_C` | `1/1` | + +The `DP` vector is `"42\t18\t9"`, and the `AD` vector is +`"30,12\t18,0\t9,0"`. A comma inside one VCF value remains inside that item; +it is not treated as a sample separator. If a value is absent, the vector +uses `.` to keep every sample position aligned. + +The file is marked with +`vcfr:representationProfile vcfr:CondensedRepresentation`. + +### What condensed mode provides + +- Sample resources are declared once per file, not once per variant. +- Each variant has one `CohortCallMatrix`, rather than one sample-call + resource for every sample. +- Each variant/FORMAT-key pair has one `FormatValueVector`, rather than one + `FormatFieldValue` resource per sample. +- The original sample order and all lexical values remain available. +- A vector-aware consumer can reconstruct sample `i` by looking up the sample + with `sampleIndex=i` and selecting item `i` from the corresponding vector. + +Condensed mode therefore moves repetition from RDF structure into the literal +payload of a vector. It is a storage and graph-shape optimization, not a loss +of genotype information. + +### Condensed growth + +Using the same variables, condensed mode creates approximately: + +```text +S reusable sample descriptions +V cohort matrices +V × F format-value vectors +F shared FORMAT definitions (per file/header) +``` + +The RDF structure grows roughly with `S + V + (V × F)`, while each vector +literal still contains the `S` values for that variant and key. + +For `V=1`, `S=3`, and `F=3`, the graph has approximately: + +| Condensed item | Count | +|---|---:| +| Reusable `VCFSample` resources | 3 | +| `CohortCallMatrix` resources | 1 | +| `FormatValueVector` resources | 3 | +| Shared FORMAT definitions | 3 | +| Vector literals containing sample values | 3 | + +Because a condensed graph has fixed metadata for the sample set, matrix, and +FORMAT definitions, it can be similar in size to—or even slightly larger than— +an expanded graph for a tiny file. Its advantage appears when many variants reuse +the same sample columns. + +## 4. The scaling difference in numbers + +Assume a cohort has: + +```text +V = 1,000 variants +S = 2,500 samples +F = 3 FORMAT fields +``` + +### Expanded counts + +```text +SampleCall resources: 1,000 × 2,500 = 2,500,000 +FormatFieldValue resources: 1,000 × 2,500 × 3 = 7,500,000 +Total per-value resources: 10,000,000 +``` + +Using the simplified three-statements-per-resource estimate, this is about +30,000,000 sample-related RDF statements. + +### Condensed counts + +```text +Reusable samples: 2,500 +Cohort matrices: 1,000 +Format-value vectors: 1,000 × 3 = 3,000 +FORMAT definitions: 3 +``` + +That is roughly 6,500 resources in the sample representation, plus vector +literals containing the same 7,500,000 scalar values. Counting the explicit +statements emitted by the built-in condensed writer gives about 28,000 +sample-related statements for this simplified case (the exact total depends on +header metadata and other record properties). + +So the RDF *structure* is reduced by roughly: + +```text +30,000,000 ÷ 28,000 ≈ 1,070× +``` + +This does not mean the compressed file will always be 1,070 times smaller. The +75 million values still exist inside vector literals, and their text length, +escaping, dictionary compression, and the rest of the VCF graph affect the +final `.nt`, `.hdt`, or `.cottas` size. The large saving is the removal of +millions of repeated RDF nodes, predicates, and type statements. + +## 5. Why VCF-RDFizer keeps both strategies + +The modes serve different downstream needs. Combining them into one universal +graph would either make small files unnecessarily complicated or make large +cohorts unnecessarily expensive. + +### Expanded is useful when direct RDF querying is the priority + +Expanded mode is a good fit when: + +- the VCF has one sample or a small number of samples; +- downstream software already expects `SampleCall` and + `FormatFieldValue` resources; +- queries need to match one sample value directly, without splitting a vector; +- maximum vocabulary-level transparency is more important than graph size. + +For example, a consumer can navigate directly from a call to the sample whose +`sampleId` is `SAMPLE_A`, then to its `GT` `FormatFieldValue`. Every value is a +normal RDF resource and literal. + +### Condensed is useful when cohort scale is the priority + +Condensed mode is a good fit when: + +- there are hundreds or thousands of samples; +- the same sample columns repeat across many variant rows; +- materializing one RDF object per sample/variant/FORMAT combination causes + excessive memory, disk, or indexing work; +- downstream software can use `sampleIndex` and decode tab-separated vectors. + +For a 2,500-sample cohort, expanded mode creates 2,500 sample-call resources for +*every* variant. Condensed mode creates the 2,500 sample descriptions once and +reuses them for all variants. + +### The semantic trade-off + +| Question | Expanded | Condensed | +|---|---|---| +| Where is each sample value? | Its own RDF resource | An item in an ordered vector literal | +| How often are samples declared? | Repeated per variant/sample call | Once per file/sample set | +| Simple one-value RDF query | Easier | Requires vector lookup/decoding | +| Large-cohort graph size | Grows with `V × S × F` | Structure grows roughly with `S + V × F` | +| All source values retained? | Yes | Yes | +| Vocabulary profile | `ExpandedRepresentation` | `CondensedRepresentation` | + +Neither mode is universally “more correct.” They expose the same underlying +VCF information through different graph contracts. + +## 6. How this is implemented in VCF-RDFizer + +The command-line choice is explicit: + +```bash +# Expanded is the default +vcf-rdfizer --mode full --input cohort.vcf.gz \ + --sample-representation expanded \ + --rdf-storage-mode plain --out results + +# Condensed is intended for large multi-sample cohorts +vcf-rdfizer --mode full --input cohort.vcf.gz \ + --sample-representation condensed \ + --rdf-storage-mode space-optimized \ + --representations hdt --out results +``` + +There is no automatic “switch at N samples” threshold. This is deliberate: +the same command should produce the same graph contract every time, and a +downstream consumer should not unexpectedly receive a different RDF shape +just because a file happened to contain more samples. + +For the built-in rules, both modes read the wide sample payload in +`records.tsv` and stream the selected RDF representation directly. The current +implementation does **not** first materialize the enormous helper +tables for the built-in maps. That implementation optimization is separate +from the semantic choice: + +- expanded still emits the expanded `SampleCall`/`FormatFieldValue` graph; +- condensed still emits the condensed `SampleSet`/`CohortCallMatrix`/ + `FormatValueVector` graph. + +Custom rules that consume `sample_calls.tsv` or +`sample_format_values.tsv` retain the materialized helper-table behavior. Such +custom helper-table mappings are rejected in condensed mode, because running them +alongside the condensed emitter would produce both graph shapes and recreate +the expansion that condensed mode is intended to avoid. + +## 7. Assessment of GeoSPARQL and GraphDB SPARQL extensions + +GraphDB's [SPARQL extensions reference](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html) +contains several different kinds of functionality. They should not all be +treated as ways to read a condensed VCF vector. + +### 7.1 GeoSPARQL geometry functions are not a direct match + +The standard `geof:` functions in the GraphDB reference operate on a +`geomLiteral`, such as a literal with the datatype `geo:wktLiteral` or +`geo:gmlLiteral`. They perform geometry operations such as distance, buffer, +intersection, union, envelope, and spatial relationship tests. GraphDB's +additional `geoext:` functions provide operations such as area, geometry +validity, simplification, and Hausdorff distance. These functions are intended +for points, lines, polygons, and other spatial objects, not arbitrary ordered +text values. See the [GeoSPARQL function table](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html#geosparql-functions) +and [GraphDB GeoSPARQL extensions](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html#geosparql-extension-functions). + +Our condensed value is instead a normal RDF string associated with +`vcfr:VCFTextVector`, for example: + +```text +"0/1\t0/0\t1/1" +``` + +It is not a WKT geometry. + +The analogy with WKT remains useful at the design level: both pack a sequence +into one literal. The GeoSPARQL operations themselves, however, are not +reusable for genotype-vector extraction. + +### 7.2 The useful GraphDB feature is string splitting + +The reference also documents the GraphDB/SPIN magic predicate `spif:split`. +It takes a string and a regular expression and produces one result row for +each split item. The implementation is described as using Java's +`String.split()` method. A prototype query for all tokens in condensed vectors +could look like this: + +```sparql +PREFIX vcfr: +PREFIX spif: + +SELECT ?vector ?token WHERE { + ?vector a vcfr:FormatValueVector ; + vcfr:valueEncoding vcfr:VCFTextVector ; + vcfr:encodedValues ?encoded . + ?token spif:split (STR(?encoded) "\t") . +} +``` + +For a `GT` vector, this would expose `0/1`, `0/0`, and `1/1` as separate query +bindings. It is useful for experimentation, validation, or exporting values +to an application. + +There is an important limitation: splitting gives the values, but the query +also needs the ordinal position of each value in order to identify the sample. +The condensed model says that the first token belongs to `sampleIndex=1`, the +second to `sampleIndex=2`, and so on. The documented `spif:split` pattern does +not itself return that ordinal. GraphDB also documents `spif:for`, which can +generate a sequence of integers, but the reference does not provide a direct +“zip these generated indexes with the split results” operation. This means +that `spif:split` alone is not a complete, reliable sample lookup mechanism. + +The page also lists RDF-list functions such as `list:index` and +`list:length`. Those operate on RDF Collections. `vcfr:encodedValues` is a +single string literal, not an RDF Collection, so these functions do not apply +unless the data model is changed to materialize every vector item as list +structure. That would reintroduce much of the per-item RDF overhead that +condensed mode was designed to avoid. GraphDB's `helper:tuple` and +`helper:iterate` functions similarly operate on internal query-time lists; +they do not automatically parse a persisted VCF vector literal. + +### 7.3 What is useful now and what is not + +| GraphDB feature | Usefulness for `VCFTextVector` | Assessment | +|---|---|---| +| `geof:*` GeoSPARQL functions | Geometry calculations | Not applicable to genotype text vectors | +| `geoext:*` geometry extensions | Geometry validity and transformations | Not applicable unless a future dataset contains real sample geometries | +| `spif:split` | Split tab-separated vector text | Useful prototype, but loses the token ordinal | +| `spif:for` | Generate expected sample positions | Helpful support function, but does not pair positions with split tokens | +| `list:index` / `list:length` | Index RDF Collections | Not applicable to the current string encoding | +| `helper:tuple` / `helper:iterate` | Work with internal query lists | Not a persisted-vector parser | +| A custom `vcfr:` vector function | Return value at a sample index | Most direct future SPARQL integration | +| Application-side parsing | Decode one selected vector after retrieval | Best portable near-term approach | + +GraphDB explicitly labels these as extensions beyond the W3C SPARQL +specification, so a query using `spif:*` or a custom function would be tied to +GraphDB (or to another engine that implements the same extension). That can be +acceptable for a GraphDB deployment, but it should not be presented as a +portable SPARQL solution. These are query-time GraphDB operations; using them +would not change VCF-RDFizer's native HDT creation or indexing path, but their +memory and latency behavior would still need to be benchmarked inside the +GraphDB server. + +### 7.4 Recommended future extraction design + +If query-time sample lookup becomes an important use case, the most useful +addition would be an extractor that understands the VCF-RDFizer metadata and +returns both the position and the value. Conceptually, it could have a +function or magic-predicate contract like: + +```text +vcfr:vectorValue(?vector, ?sampleIndex) → ?value +``` + +or: + +```text +?vector vcfr:valueAt (?sampleIndex ?value) +``` + +For a sample-aware form, the function could accept the `vcfr:VCFSample` IRI +instead of an integer, resolve that resource's `sampleIndex`, and then extract +the matching token. Returning the index as well as the value would make it +possible to validate that a vector has the expected number of positions. + +A production implementation should also consider these safeguards: + +1. Verify that the vector belongs to the expected `SampleSet` through its + `CohortCallMatrix`. +2. Check that the requested index is within the sample-set size. +3. Preserve `.` as the VCF missing-value marker rather than silently turning + it into an unbound result. +4. Treat tab as the vector separator and keep commas inside a value such as + `AD=30,12`. +5. Validate that the vector contains exactly one item per declared sample. +6. Expose the function through a GraphDB plugin only when GraphDB-specific + deployment is acceptable; otherwise provide a small application-side + decoder or a materialized per-value projection. + +There is also a performance question. With 1,000 variants, 2,500 samples, and +three FORMAT keys, condensed mode stores 3,000 vectors but each vector contains +2,500 values. Expanding every vector in a SPARQL query would produce up to + +```text +1,000 × 3 × 2,500 = 7,500,000 query result rows +``` + +before filtering to a particular sample. For a targeted lookup, a function that +extracts one position is preferable to `spif:split` over the entire vector. For +large cohort-wide analyses, an external decoder or a precomputed analytical +table may be more appropriate than forcing a graph database to emit millions +of token bindings. + +### 7.5 Overall conclusion + +The GraphDB work is useful as a source of implementation ideas, but not as a +drop-in GeoSPARQL solution: + +- **GeoSPARQL geometry functions:** no, they should not be used for genotype + vectors. +- **`spif:split`:** yes, as a prototype tokenizer and validation aid. +- **`spif:split` plus `spif:for`:** potentially useful building blocks, but not + sufficient for a trustworthy sample/value join without an ordinal pairing + mechanism. +- **Custom vector extraction function or application decoder:** yes, this is + the most promising future direction. + +The condensed model should therefore remain as it is for storage efficiency, +while future query support should be added as a separate decoding/projection +layer rather than changing genotype vectors into geometries or RDF Lists. + +## 8. Practical choice + +Use this short rule of thumb: + +```text +Small or single-sample VCF + simple RDF queries → expanded +Large multi-sample cohort + storage/scale focus → condensed +``` + +If a consumer is unsure which graph it received, inspect the file’s +`vcfr:representationProfile` value before querying. That profile is the +explicit signal that tells the consumer whether sample values are represented +as individual resources or as ordered vectors. + +## Glossary + +- **Variant record**: one row describing a genomic position and its alleles. +- **Sample**: one individual or biological sample represented by a VCF column. +- **FORMAT key**: a field name such as `GT`, `DP`, or `AD` describing one part + of a sample’s value. +- **RDF resource**: a named graph object that can have properties, such as one + `SampleCall` or one `FormatFieldValue`. +- **Vector**: an ordered list of values. In condensed mode, values are stored + as one tab-separated `VCFTextVector` literal in `sampleIndex` order. +- **Graph shape**: which resources and relationships are present, independent + of the underlying biological values. diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..a4c4384 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,91 @@ +# Semantic VCF/RDF validation + +`vcf-rdfizer --mode validation` checks that a converted RDF graph reproduces +six deterministic VCF summaries. It computes one result from the source VCF +using `cyvcf2` (and `bcftools` for exact FILTER strings when available), runs +the equivalent SPARQL queries with Comunica, then compares canonical integer +results exactly. + +The mode consumes a single N-Triples gzip aggregate (`.nt.gz`). It mounts that +file read-only, expands it under `/work` **inside the Docker container**, uses +the temporary `.nt` for Raptor syntax validation and Comunica queries, then +removes it before the container exits. No decompressed RDF is written beneath +`--out`; only reports are retained. + +## Run it + +Use the representation selected when the RDF was created. + +```bash +# Expanded (the default VCF-RDFizer graph shape) +vcf-rdfizer --mode validation \ + --input ./vcf_data/HG004_GRCh38.vcf.gz \ + --rdf ./results/HG004_GRCh38/HG004_GRCh38.nt.gz \ + --sample-representation expanded \ + --out ./validation-results + +# Condensed multi-sample graph +vcf-rdfizer --mode validation \ + --input ./vcf_data/1000G_phase3_chr20.vcf.gz \ + --rdf ./results/1000G_phase3_chr20/1000G_phase3_chr20.nt.gz \ + --sample-representation condensed \ + --out ./validation-results +``` + +The input VCF and `.nt.gz` must originate from the same conversion. `--rdf` +must name an existing `.nt.gz` file; validation deliberately does not accept an +uncompressed `.nt`, HDT, or COTTAS artifact. Use full mode with +`--rdf-storage-mode space-optimized` to produce the required gzip aggregate. + +`--validation-id NAME` changes the report directory name. The default is the +source VCF basename without `.vcf` or `.vcf.gz`. Existing result directories +are never overwritten. `--filter-oracle {auto,bcftools,cyvcf2}` controls the +FILTER-field oracle; `auto` uses `bcftools` when it is available in the image. + +## What is tested + +The common record-level queries are used for both graph shapes: + +| Query | Exact comparison | +|---|---| +| Q1 | record counts per source contig and zero-based 1 Mb window | +| Q2 | record-level ALT/REF shape classes | +| Q3 | biallelic A/C/G/T SNV transition and transversion counts | +| Q4 | FILTER broad status and exact lexical value | +| Q5 | per-sample genotype class counts | +| Q6 | genotype-derived single-ALT `(AN, AC, siteCount)` distribution | + +Q5/Q6 are only required when the VCF has both samples and a `GT` FORMAT field. +The suite additionally checks N-Triples syntax, record cardinality, POS +datatype, representation profile, and representation-specific sample/GT +inventory before interpreting the six result sets. + +For the expanded representation, Q5/Q6 traverse `SampleCall` and +`FormatFieldValue` resources. For the condensed representation, their matching +queries traverse `SampleSet`, `CohortCallMatrix`, and `FormatValueVector`, then +extract each tab-delimited GT value by its `sampleIndex`. This makes the +condensed tests semantically equivalent without inflating the persisted RDF +back into per-sample value resources. + +## Results and cleanup evidence + +Results are written to: + +```text +/validation// +``` + +Important files include `summary.json`, `manifest.json`, `parser.json`, +`rdf-validation.json`, `preflight.json`, `sparql.json`, and `comparison.json`. +Raw Comunica JSON, stderr, and query resource logs are in `raw/`; normalized +results are in `normalized/`. + +The parent VCF-RDFizer run metrics include +`run_metrics/__/stages/validation/.json`. +Both that report and `summary.json` record that the `.nt.gz` was decompressed +inside the container and that no raw RDF was retained on the host. + +`PASS` means every required query and invariant matched. `MISMATCH` means that +both paths ran but differ. `BLOCKED_BY_PREFLIGHT` means RDF syntax or core graph +structure failed. `EXECUTION_FAILED` means a parser or query engine could not +complete. diff --git a/rules/README.md b/rules/README.md index 4295e34..afb6b1c 100644 --- a/rules/README.md +++ b/rules/README.md @@ -15,13 +15,13 @@ This directory contains RML mappings used by the conversion pipeline. - `/data/tsv/sample_format_values.tsv` - For the built-in sample maps, `sample_calls.tsv` and `sample_format_values.tsv` are header-only compatibility sources. In - `--sample-representation dense`, the Python wrapper streams their equivalent + `--sample-representation expanded`, the Python wrapper streams their equivalent `SampleCall` and `FormatFieldValue` triples directly from `records.tsv`. In `--sample-representation condensed`, it instead streams `SampleSet`, `CohortCallMatrix`, and `FormatValueVector` resources. Only one emitter runs. - Custom mappings with additional consumers of either helper source retain - expanded TSV generation in dense mode. They are rejected in condensed mode - to prevent simultaneous dense and condensed output. + materialized TSV generation in expanded mode. They are rejected in condensed mode + to prevent simultaneous expanded and condensed output. - The Python wrapper rewrites these template paths per input VCF to: - `/data/tsv/.file_metadata.tsv` - `/data/tsv/.header_lines.tsv` @@ -58,5 +58,5 @@ The default mapping is structured to align with those classes/properties, especi - `vcfr:VCFHeader` + `vcfr:hasHeaderLine` - `vcfr:VCFRecord` core fields (`chrom`, `pos`, `ref`, `alt`) - `vcfr:VariantCall` with raw call attributes -- dense `vcfr:SampleCall` / `vcfr:FormatFieldValue` resources, or condensed +- expanded `vcfr:SampleCall` / `vcfr:FormatFieldValue` resources, or condensed `vcfr:CohortCallMatrix` / `vcfr:FormatValueVector` resources diff --git a/rules/default_rules.ttl b/rules/default_rules.ttl index 7a0ee2c..7332230 100644 --- a/rules/default_rules.ttl +++ b/rules/default_rules.ttl @@ -14,7 +14,7 @@ # - /data/tsv/.sample_calls.tsv (header-only compatibility source) # - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. -# The wrapper selects exactly one direct sample emitter: dense SampleCall / +# The wrapper selects exactly one direct sample emitter: expanded SampleCall / # FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / # FormatValueVector triples. The helper tables stay header-only in either mode. # diff --git a/src/compression.sh b/src/compression.sh index a1bf33b..e426165 100644 --- a/src/compression.sh +++ b/src/compression.sh @@ -209,15 +209,15 @@ for OUT in "${OUTPUT_DIRS[@]}"; do SAFE_BASENAME="rdf" fi - TIME_LOG_GZIP_DIR="$LOGDIR/compression_time/gzip/${SAFE_BASENAME}" - TIME_LOG_BROTLI_DIR="$LOGDIR/compression_time/brotli/${SAFE_BASENAME}" - TIME_LOG_HDT_DIR="$LOGDIR/compression_time/hdt/${SAFE_BASENAME}" - METRICS_JSON_DIR="$LOGDIR/compression_metrics/${SAFE_BASENAME}" + TIME_LOG_GZIP_DIR="$LOGDIR/timings/compression/${SAFE_BASENAME}" + TIME_LOG_BROTLI_DIR="$LOGDIR/timings/compression/${SAFE_BASENAME}" + TIME_LOG_HDT_DIR="$LOGDIR/timings/compression/${SAFE_BASENAME}" + METRICS_JSON_DIR="$LOGDIR/stages/compression" mkdir -p "$TIME_LOG_GZIP_DIR" "$TIME_LOG_BROTLI_DIR" "$TIME_LOG_HDT_DIR" "$METRICS_JSON_DIR" - TIME_LOG_GZIP="$TIME_LOG_GZIP_DIR/${RUN_ID}.txt" - TIME_LOG_BROTLI="$TIME_LOG_BROTLI_DIR/${RUN_ID}.txt" - TIME_LOG_HDT="$TIME_LOG_HDT_DIR/${RUN_ID}.txt" - METRICS_JSON="$METRICS_JSON_DIR/${RUN_ID}.json" + TIME_LOG_GZIP="$TIME_LOG_GZIP_DIR/gzip.txt" + TIME_LOG_BROTLI="$TIME_LOG_BROTLI_DIR/brotli.txt" + TIME_LOG_HDT="$TIME_LOG_HDT_DIR/hdt.txt" + METRICS_JSON="$METRICS_JSON_DIR/${SAFE_BASENAME}.json" HDT_SOURCE="not_used" GZIP_ON_HDT_SIZE=0 diff --git a/src/run_conversion.sh b/src/run_conversion.sh index c14cdae..776a9e5 100644 --- a/src/run_conversion.sh +++ b/src/run_conversion.sh @@ -59,11 +59,13 @@ cleanup_parts_dir() { trap cleanup_parts_dir EXIT PROGRESS_FILE=${PROGRESS_FILE:-} PROGRESS_READY=0 -TIME_LOG_DIR="$LOGDIR/conversion_time/${SAFE_OUT_NAME}" -METRICS_JSON_DIR="$LOGDIR/conversion_metrics/${SAFE_OUT_NAME}" +# One invocation owns its metrics directory, so output names—not timestamps— +# make the stage files both stable and immediately discoverable. +TIME_LOG_DIR="$LOGDIR/timings/conversion" +METRICS_JSON_DIR="$LOGDIR/stages/conversion" mkdir -p "$TIME_LOG_DIR" "$METRICS_JSON_DIR" -TIME_LOG="$TIME_LOG_DIR/${RUN_ID}.txt" -METRICS_JSON="$METRICS_JSON_DIR/${RUN_ID}.json" +TIME_LOG="$TIME_LOG_DIR/${SAFE_OUT_NAME}.txt" +METRICS_JSON="$METRICS_JSON_DIR/${SAFE_OUT_NAME}.json" METRICS_CSV="$LOGDIR/metrics.csv" TSV_EXIT_CODE=${TSV_EXIT_CODE:-0} TSV_WALL_SECONDS=${TSV_WALL_SECONDS:-null} diff --git a/src/validation/queries/common/preflight_missing_token_conformance.rq b/src/validation/queries/common/preflight_missing_token_conformance.rq new file mode 100644 index 0000000..bfbec42 --- /dev/null +++ b/src/validation/queries/common/preflight_missing_token_conformance.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +SELECT ?s ?p ?o (DATATYPE(?o) AS ?datatype) +WHERE { + ?s ?p ?o . + FILTER(ISLITERAL(?o) && STR(?o) = ".") + FILTER(DATATYPE(?o) != vcfr:Null) +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_position_datatype.rq b/src/validation/queries/common/preflight_position_datatype.rq new file mode 100644 index 0000000..5932226 --- /dev/null +++ b/src/validation/queries/common/preflight_position_datatype.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: +PREFIX xsd: + +SELECT ?record ?pos +WHERE { + ?record a vcfr:VCFRecord ; vcfr:pos ?pos . + FILTER(DATATYPE(?pos) != xsd:integer) +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_record_cardinality.rq b/src/validation/queries/common/preflight_record_cardinality.rq new file mode 100644 index 0000000..5b683cb --- /dev/null +++ b/src/validation/queries/common/preflight_record_cardinality.rq @@ -0,0 +1,19 @@ +PREFIX vcfr: + +SELECT ?record + (COUNT(DISTINCT ?chrom) AS ?chromCount) + (COUNT(DISTINCT ?pos) AS ?posCount) + (COUNT(DISTINCT ?ref) AS ?refCount) + (COUNT(DISTINCT ?alt) AS ?altCount) + (COUNT(DISTINCT ?call) AS ?callCount) +WHERE { + ?record a vcfr:VCFRecord . + OPTIONAL { ?record vcfr:chrom ?chrom } + OPTIONAL { ?record vcfr:pos ?pos } + OPTIONAL { ?record vcfr:ref ?ref } + OPTIONAL { ?record vcfr:alt ?alt } + OPTIONAL { ?record vcfr:hasCall ?call } +} +GROUP BY ?record +HAVING(COUNT(DISTINCT ?chrom) != 1 || COUNT(DISTINCT ?pos) != 1 || COUNT(DISTINCT ?ref) != 1 || COUNT(DISTINCT ?alt) != 1 || COUNT(DISTINCT ?call) != 1) +LIMIT 100 diff --git a/src/validation/queries/common/q01_record_density_1mb.rq b/src/validation/queries/common/q01_record_density_1mb.rq new file mode 100644 index 0000000..e37b999 --- /dev/null +++ b/src/validation/queries/common/q01_record_density_1mb.rq @@ -0,0 +1,10 @@ +PREFIX vcfr: +PREFIX xsd: + +SELECT ?chrom ?windowIndex (COUNT(DISTINCT ?record) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:chrom ?chrom ; vcfr:pos ?pos . + BIND(FLOOR((xsd:integer(?pos) - 1) / 1000000) AS ?windowIndex) +} +GROUP BY ?chrom ?windowIndex +ORDER BY ?chrom ?windowIndex diff --git a/src/validation/queries/common/q02_variant_shape_counts.rq b/src/validation/queries/common/q02_variant_shape_counts.rq new file mode 100644 index 0000000..6f29eb0 --- /dev/null +++ b/src/validation/queries/common/q02_variant_shape_counts.rq @@ -0,0 +1,34 @@ +PREFIX vcfr: + +SELECT ?variantClass (COUNT(DISTINCT ?record) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:ref ?refLiteral ; vcfr:alt ?altLiteral . + BIND(UCASE(STR(?refLiteral)) AS ?ref) + BIND(UCASE(STR(?altLiteral)) AS ?alt) + BIND( + IF(?alt = ".", "NO_ALT", + IF(CONTAINS(?alt, ","), "MULTIALLELIC", + IF( + ?alt = "*" || CONTAINS(?alt, "[") || CONTAINS(?alt, "]") || + (STRSTARTS(?alt, "<") && STRENDS(?alt, ">")), + "SYMBOLIC_OR_BREAKEND", + IF( + !REGEX(?ref, "^[ACGTN]+$") || !REGEX(?alt, "^[ACGTN]+$"), + "OTHER", + IF( + STRLEN(?ref) = 1 && STRLEN(?alt) = 1, + "SNV", + IF( + STRLEN(?ref) = STRLEN(?alt), + "MNV_OR_EQUAL_LENGTH_SUBSTITUTION", + IF(STRLEN(?ref) < STRLEN(?alt), "INSERTION_SHAPE", "DELETION_SHAPE") + ) + ) + ) + ) + ) + ) AS ?variantClass + ) +} +GROUP BY ?variantClass +ORDER BY ?variantClass diff --git a/src/validation/queries/common/q03_titv.rq b/src/validation/queries/common/q03_titv.rq new file mode 100644 index 0000000..a768a91 --- /dev/null +++ b/src/validation/queries/common/q03_titv.rq @@ -0,0 +1,12 @@ +PREFIX vcfr: + +SELECT + (COUNT(DISTINCT ?record) AS ?biallelicSnvCount) + (SUM(IF((?ref = "A" && ?alt = "G") || (?ref = "G" && ?alt = "A") || (?ref = "C" && ?alt = "T") || (?ref = "T" && ?alt = "C"), 1, 0)) AS ?transitionCount) + (SUM(IF((?ref = "A" && ?alt = "G") || (?ref = "G" && ?alt = "A") || (?ref = "C" && ?alt = "T") || (?ref = "T" && ?alt = "C"), 0, 1)) AS ?transversionCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:ref ?refLiteral ; vcfr:alt ?altLiteral . + BIND(UCASE(STR(?refLiteral)) AS ?ref) + BIND(UCASE(STR(?altLiteral)) AS ?alt) + FILTER(REGEX(?ref, "^[ACGT]$") && REGEX(?alt, "^[ACGT]$") && ?ref != ?alt) +} diff --git a/src/validation/queries/common/q04_filter_distribution.rq b/src/validation/queries/common/q04_filter_distribution.rq new file mode 100644 index 0000000..9545410 --- /dev/null +++ b/src/validation/queries/common/q04_filter_distribution.rq @@ -0,0 +1,11 @@ +PREFIX vcfr: + +SELECT ?filterStatus ?filterLexical (COUNT(DISTINCT ?record) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:hasCall ?call . + ?call vcfr:filter ?filterLiteral . + BIND(STR(?filterLiteral) AS ?filterLexical) + BIND(IF(?filterLexical = "PASS", "PASS", IF(?filterLexical = ".", "NOT_APPLIED", "FAILED")) AS ?filterStatus) +} +GROUP BY ?filterStatus ?filterLexical +ORDER BY ?filterStatus ?filterLexical diff --git a/src/validation/queries/condensed/preflight_representation_profile.rq b/src/validation/queries/condensed/preflight_representation_profile.rq new file mode 100644 index 0000000..362062c --- /dev/null +++ b/src/validation/queries/condensed/preflight_representation_profile.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +SELECT ?file ?profile +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:CondensedRepresentation) +} +LIMIT 100 diff --git a/src/validation/queries/condensed/preflight_sample_gt_inventory.rq b/src/validation/queries/condensed/preflight_sample_gt_inventory.rq new file mode 100644 index 0000000..9fbde64 --- /dev/null +++ b/src/validation/queries/condensed/preflight_sample_gt_inventory.rq @@ -0,0 +1,16 @@ +PREFIX vcfr: + +SELECT + (COUNT(DISTINCT ?sample) AS ?sampleCount) + (COUNT(DISTINCT ?sampleId) AS ?sampleIdCount) + (COUNT(DISTINCT ?gtVector) AS ?gtVectorCount) +WHERE { + OPTIONAL { + ?sampleSet a vcfr:SampleSet ; vcfr:hasSample ?sample . + ?sample vcfr:sampleName ?sampleId . + } + OPTIONAL { + ?gtVector a vcfr:FormatValueVector ; vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . + ?definition vcfr:fieldId "GT" . + } +} diff --git a/src/validation/queries/condensed/q05_sample_genotype_counts.rq b/src/validation/queries/condensed/q05_sample_genotype_counts.rq new file mode 100644 index 0000000..32bf44a --- /dev/null +++ b/src/validation/queries/condensed/q05_sample_genotype_counts.rq @@ -0,0 +1,27 @@ +PREFIX vcfr: + +SELECT ?sampleId ?genotypeClass (COUNT(DISTINCT ?record) AS ?callCount) +WHERE { + ?file a vcfr:VCFFile ; vcfr:hasRecord ?record ; vcfr:hasSampleSet ?sampleSet . + ?record a vcfr:VCFRecord ; vcfr:hasCall ?call . + ?sampleSet vcfr:hasSample ?sample . + ?sample vcfr:sampleName ?sampleId ; vcfr:sampleIndex ?sampleIndex . + OPTIONAL { + ?sample vcfr:sampleIndex ?sampleIndexForVector . + ?call vcfr:hasCallMatrix ?matrix . + ?matrix vcfr:appliesToSampleSet ?sampleSet ; vcfr:hasFormatValueVector ?gtVector . + ?gtVector vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . + ?definition vcfr:fieldId "GT" . + BIND(?sampleIndexForVector - 1 AS ?zeroIndex) + BIND(REPLACE(STR(?encoded), CONCAT("^(?:[^\\t]*\\t){", STR(?zeroIndex), "}([^\\t]*).*$"), "$1") AS ?gtLiteral) + } + BIND(IF(BOUND(?gtLiteral), REPLACE(STR(?gtLiteral), "[|]", "/"), "") AS ?gt) + BIND(IF(!BOUND(?gtLiteral), "NO_GT_FIELD", + IF(CONTAINS(?gt, "."), "MISSING", + IF(REGEX(?gt, "^[0-9]+$"), IF(?gt = "0", "HAPLOID_REF", "HAPLOID_ALT"), + IF(REGEX(?gt, "^[0-9]+/[0-9]+$"), + IF(STRBEFORE(?gt, "/") = STRAFTER(?gt, "/"), IF(STRBEFORE(?gt, "/") = "0", "HOM_REF", "HOM_ALT"), "HET"), + "OTHER_PLOIDY")))) AS ?genotypeClass) +} +GROUP BY ?sampleId ?genotypeClass +ORDER BY ?sampleId ?genotypeClass diff --git a/src/validation/queries/condensed/q06_ac_an_distribution.rq b/src/validation/queries/condensed/q06_ac_an_distribution.rq new file mode 100644 index 0000000..5c26b70 --- /dev/null +++ b/src/validation/queries/condensed/q06_ac_an_distribution.rq @@ -0,0 +1,33 @@ +PREFIX vcfr: + +SELECT ?an ?ac (COUNT(DISTINCT ?record) AS ?siteCount) +WHERE { + { + SELECT ?record (SUM(?anContribution) AS ?an) (SUM(?acContribution) AS ?ac) + WHERE { + { + SELECT DISTINCT ?record ?sample ?gt + WHERE { + ?file a vcfr:VCFFile ; vcfr:hasRecord ?record ; vcfr:hasSampleSet ?sampleSet . + ?record a vcfr:VCFRecord ; vcfr:alt ?altLiteral ; vcfr:hasCall ?call . + FILTER(STR(?altLiteral) != "." && !CONTAINS(STR(?altLiteral), ",")) + ?sampleSet vcfr:hasSample ?sample . + ?sample vcfr:sampleIndex ?sampleIndex . + ?call vcfr:hasCallMatrix ?matrix . + ?matrix vcfr:appliesToSampleSet ?sampleSet ; vcfr:hasFormatValueVector ?gtVector . + ?gtVector vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . + ?definition vcfr:fieldId "GT" . + BIND(?sampleIndex - 1 AS ?zeroIndex) + BIND(REPLACE(STR(?encoded), CONCAT("^(?:[^\\t]*\\t){", STR(?zeroIndex), "}([^\\t]*).*$"), "$1") AS ?gtLiteral) + BIND(REPLACE(STR(?gtLiteral), "[|]", "/") AS ?gt) + } + } + BIND(IF(REGEX(?gt, "^[01]$"), 1, IF(REGEX(?gt, "^[01]/[01]$"), 2, 0)) AS ?anContribution) + BIND(IF(?gt = "1", 1, IF(?gt = "0/1" || ?gt = "1/0", 1, IF(?gt = "1/1", 2, 0))) AS ?acContribution) + } + GROUP BY ?record + } + FILTER(?an > 0) +} +GROUP BY ?an ?ac +ORDER BY ?an ?ac diff --git a/src/validation/queries/expanded/preflight_representation_profile.rq b/src/validation/queries/expanded/preflight_representation_profile.rq new file mode 100644 index 0000000..3adeb40 --- /dev/null +++ b/src/validation/queries/expanded/preflight_representation_profile.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +SELECT ?file ?profile +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:ExpandedRepresentation) +} +LIMIT 100 diff --git a/src/validation/queries/expanded/preflight_sample_gt_inventory.rq b/src/validation/queries/expanded/preflight_sample_gt_inventory.rq new file mode 100644 index 0000000..aa59ccb --- /dev/null +++ b/src/validation/queries/expanded/preflight_sample_gt_inventory.rq @@ -0,0 +1,16 @@ +PREFIX vcfr: + +SELECT + (COUNT(DISTINCT ?sampleCall) AS ?sampleCallCount) + (COUNT(DISTINCT ?sampleId) AS ?sampleIdCount) + (COUNT(DISTINCT ?gtValueNode) AS ?gtValueNodeCount) +WHERE { + OPTIONAL { + ?sampleCall a vcfr:SampleCall ; vcfr:sampleId ?sampleId . + OPTIONAL { + ?sampleCall vcfr:hasFormatValue ?gtValueNode . + FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) + ?gtValueNode vcfr:fieldValue ?gtLiteral . + } + } +} diff --git a/src/validation/queries/expanded/q05_sample_genotype_counts.rq b/src/validation/queries/expanded/q05_sample_genotype_counts.rq new file mode 100644 index 0000000..bed78a2 --- /dev/null +++ b/src/validation/queries/expanded/q05_sample_genotype_counts.rq @@ -0,0 +1,20 @@ +PREFIX vcfr: + +SELECT ?sampleId ?genotypeClass (COUNT(DISTINCT ?sampleCall) AS ?callCount) +WHERE { + ?sampleCall a vcfr:SampleCall ; vcfr:sampleId ?sampleId . + OPTIONAL { + ?sampleCall vcfr:hasFormatValue ?gtValueNode . + FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) + ?gtValueNode vcfr:fieldValue ?gtLiteral . + } + BIND(IF(BOUND(?gtLiteral), REPLACE(STR(?gtLiteral), "[|]", "/"), "") AS ?gt) + BIND(IF(!BOUND(?gtLiteral), "NO_GT_FIELD", + IF(CONTAINS(?gt, "."), "MISSING", + IF(REGEX(?gt, "^[0-9]+$"), IF(?gt = "0", "HAPLOID_REF", "HAPLOID_ALT"), + IF(REGEX(?gt, "^[0-9]+/[0-9]+$"), + IF(STRBEFORE(?gt, "/") = STRAFTER(?gt, "/"), IF(STRBEFORE(?gt, "/") = "0", "HOM_REF", "HOM_ALT"), "HET"), + "OTHER_PLOIDY")))) AS ?genotypeClass) +} +GROUP BY ?sampleId ?genotypeClass +ORDER BY ?sampleId ?genotypeClass diff --git a/src/validation/queries/expanded/q06_ac_an_distribution.rq b/src/validation/queries/expanded/q06_ac_an_distribution.rq new file mode 100644 index 0000000..1a0fc93 --- /dev/null +++ b/src/validation/queries/expanded/q06_ac_an_distribution.rq @@ -0,0 +1,28 @@ +PREFIX vcfr: + +SELECT ?an ?ac (COUNT(DISTINCT ?record) AS ?siteCount) +WHERE { + { + SELECT ?record (SUM(?anContribution) AS ?an) (SUM(?acContribution) AS ?ac) + WHERE { + { + SELECT DISTINCT ?record ?sampleCall ?gt + WHERE { + ?record a vcfr:VCFRecord ; vcfr:alt ?altLiteral ; vcfr:hasCall ?call . + FILTER(STR(?altLiteral) != "." && !CONTAINS(STR(?altLiteral), ",")) + ?call vcfr:hasSampleCall ?sampleCall . + ?sampleCall vcfr:hasFormatValue ?gtValueNode . + FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) + ?gtValueNode vcfr:fieldValue ?gtLiteral . + BIND(REPLACE(STR(?gtLiteral), "[|]", "/") AS ?gt) + } + } + BIND(IF(REGEX(?gt, "^[01]$"), 1, IF(REGEX(?gt, "^[01]/[01]$"), 2, 0)) AS ?anContribution) + BIND(IF(?gt = "1", 1, IF(?gt = "0/1" || ?gt = "1/0", 1, IF(?gt = "1/1", 2, 0))) AS ?acContribution) + } + GROUP BY ?record + } + FILTER(?an > 0) +} +GROUP BY ?an ?ac +ORDER BY ?an ?ac diff --git a/src/validation/validation_runner.py b/src/validation/validation_runner.py new file mode 100644 index 0000000..9b82e69 --- /dev/null +++ b/src/validation/validation_runner.py @@ -0,0 +1,612 @@ +#!/usr/bin/env python3 +"""Validate a VCF-RDFizer .nt.gz graph against its source VCF. + +The compressed RDF input is expanded only to a container-local temporary file. +It is parsed with Raptor, queried with Comunica, and removed in a finally-safe +temporary directory before this process exits. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import time +from collections import Counter +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + +from cyvcf2 import VCF +import cyvcf2 + + +SCRIPT_DIR = Path(__file__).resolve().parent +QUERY_ROOT = SCRIPT_DIR / "queries" +CORE_QUERIES = ( + "q01_record_density_1mb", + "q02_variant_shape_counts", + "q03_titv", + "q04_filter_distribution", + "q05_sample_genotype_counts", + "q06_ac_an_distribution", +) +PREFLIGHT_QUERIES = ( + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_missing_token_conformance", + "preflight_representation_profile", + "preflight_sample_gt_inventory", +) +TRANSITIONS = {("A", "G"), ("G", "A"), ("C", "T"), ("T", "C")} + +QUERY_SPECS = { + "q01_record_density_1mb": (("chrom", "windowIndex"), ("recordCount",)), + "q02_variant_shape_counts": (("variantClass",), ("recordCount",)), + "q03_titv": ((), ("biallelicSnvCount", "transitionCount", "transversionCount")), + "q04_filter_distribution": (("filterStatus", "filterLexical"), ("recordCount",)), + "q05_sample_genotype_counts": (("sampleId", "genotypeClass"), ("callCount",)), + "q06_ac_an_distribution": (("an", "ac"), ("siteCount",)), +} +QUERY_SCHEMAS = { + "q01_record_density_1mb": (("chrom", "windowIndex", "recordCount"), {"windowIndex", "recordCount"}, ("chrom", "windowIndex")), + "q02_variant_shape_counts": (("variantClass", "recordCount"), {"recordCount"}, ("variantClass",)), + "q03_titv": (("biallelicSnvCount", "transitionCount", "transversionCount"), {"biallelicSnvCount", "transitionCount", "transversionCount"}, ()), + "q04_filter_distribution": (("filterStatus", "filterLexical", "recordCount"), {"recordCount"}, ("filterStatus", "filterLexical")), + "q05_sample_genotype_counts": (("sampleId", "genotypeClass", "callCount"), {"callCount"}, ("sampleId", "genotypeClass")), + "q06_ac_an_distribution": (("an", "ac", "siteCount"), {"an", "ac", "siteCount"}, ("an", "ac")), +} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def tool_version(command: list[str], *, table_label: str | None = None) -> str | None: + try: + result = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + timeout=20, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + output = result.stdout.strip() + if table_label: + for line in output.splitlines(): + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if len(cells) >= 2 and cells[0] == table_label: + return cells[1] + return output.splitlines()[0] if output else None + + +def alt_lexical(variant: Any) -> str: + return ",".join("." if item is None else str(item) for item in (variant.ALT or [None])) + + +def classify_variant_shape(ref_value: str, alt_value: str) -> str: + ref, alt = str(ref_value).upper(), str(alt_value).upper() + if alt == ".": + return "NO_ALT" + if "," in alt: + return "MULTIALLELIC" + if alt == "*" or "[" in alt or "]" in alt or (alt.startswith("<") and alt.endswith(">")): + return "SYMBOLIC_OR_BREAKEND" + if not re.fullmatch(r"[ACGTN]+", ref) or not re.fullmatch(r"[ACGTN]+", alt): + return "OTHER" + if len(ref) == len(alt) == 1: + return "SNV" + if len(ref) == len(alt): + return "MNV_OR_EQUAL_LENGTH_SUBSTITUTION" + return "INSERTION_SHAPE" if len(ref) < len(alt) else "DELETION_SHAPE" + + +def filter_status(value: str) -> str: + return "PASS" if value == "PASS" else "NOT_APPLIED" if value == "." else "FAILED" + + +def exact_filter_lexical(variant: Any) -> str: + values = list(variant.FILTERS or []) + if values: + return ";".join(str(value) for value in values) + fields = str(variant).rstrip("\r\n").split("\t", 8) + if len(fields) < 7: + raise ValueError("Could not recover FILTER from serialized VCF record") + return fields[6] + + +def filters_with_bcftools(vcf_path: Path) -> Counter[tuple[str, str]]: + process = subprocess.run( + ["bcftools", "query", "-f", "%FILTER\\n", str(vcf_path)], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + if process.returncode: + raise RuntimeError(f"bcftools FILTER extraction failed: {process.stderr.strip()}") + values = [line.rstrip("\r") for line in process.stdout.splitlines()] + if any(value == "" for value in values): + raise ValueError("bcftools returned an empty FILTER value") + return Counter((filter_status(value), value) for value in values) + + +def genotype_alleles(raw_gt: Any) -> tuple[int | None, ...] | None: + if raw_gt is None: + return None + return tuple(None if value is None or int(value) < 0 else int(value) for value in raw_gt[:-1]) + + +def classify_genotype(alleles: tuple[int | None, ...] | None, *, has_gt: bool) -> str: + if not has_gt: + return "NO_GT_FIELD" + if alleles is None or not alleles or any(value is None for value in alleles): + return "MISSING" + complete = tuple(int(value) for value in alleles if value is not None) + if len(complete) == 1: + return "HAPLOID_REF" if complete[0] == 0 else "HAPLOID_ALT" + if len(complete) == 2: + if complete[0] == complete[1]: + return "HOM_REF" if complete[0] == 0 else "HOM_ALT" + return "HET" + return "OTHER_PLOIDY" + + +def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: + use_bcftools = filter_oracle == "bcftools" or ( + filter_oracle == "auto" and shutil.which("bcftools") is not None + ) + if filter_oracle == "bcftools" and not shutil.which("bcftools"): + raise RuntimeError("--filter-oracle=bcftools requested but bcftools is unavailable") + filters = filters_with_bcftools(vcf_path) if use_bcftools else Counter() + reader = VCF(str(vcf_path), strict_gt=True) + samples = list(reader.samples) + density: Counter[tuple[str, int]] = Counter() + shapes: Counter[str] = Counter() + genotypes: Counter[tuple[str, str]] = Counter() + ac_an: Counter[tuple[int, int]] = Counter() + total_records = gt_records = single_alt_records = q06_eligible = 0 + transition_count = transversion_count = biallelic_snv_count = 0 + try: + for variant in reader: + total_records += 1 + density[(str(variant.CHROM), (int(variant.POS) - 1) // 1_000_000)] += 1 + alt = alt_lexical(variant) + shapes[classify_variant_shape(variant.REF, alt)] += 1 + ref_upper, alt_upper = str(variant.REF).upper(), alt.upper() + if re.fullmatch(r"[ACGT]", ref_upper) and re.fullmatch(r"[ACGT]", alt_upper) and ref_upper != alt_upper: + biallelic_snv_count += 1 + if (ref_upper, alt_upper) in TRANSITIONS: + transition_count += 1 + else: + transversion_count += 1 + if not use_bcftools: + raw_filter = exact_filter_lexical(variant) + filters[(filter_status(raw_filter), raw_filter)] += 1 + raw_format = variant.FORMAT + if not raw_format: + format_keys: list[str] = [] + elif isinstance(raw_format, str): + format_keys = raw_format.split(":") + else: + format_keys = [str(item) for item in raw_format] + has_gt = "GT" in format_keys + if has_gt: + gt_records += 1 + alleles = [None] * len(samples) + if samples and has_gt: + raw_genotypes = list(variant.genotypes) + if len(raw_genotypes) != len(samples): + raise ValueError(f"Sample/genotype length mismatch at {variant.CHROM}:{variant.POS}") + alleles = [genotype_alleles(raw) for raw in raw_genotypes] + for sample, call in zip(samples, alleles, strict=True): + genotypes[(sample, classify_genotype(call, has_gt=has_gt))] += 1 + if alt != "." and "," not in alt: + single_alt_records += 1 + if has_gt: + an = ac = 0 + for call in alleles: + if call is None or any(value is None for value in call): + continue + complete = tuple(int(value) for value in call if value is not None) + if len(complete) not in (1, 2) or any(value not in (0, 1) for value in complete): + continue + an += len(complete) + ac += sum(complete) + if an: + ac_an[(an, ac)] += 1 + q06_eligible += 1 + finally: + reader.close() + if sum(filters.values()) != total_records: + raise ValueError("FILTER oracle record count differs from the VCF record count") + q05 = [ + {"sampleId": sample, "genotypeClass": genotype_class, "callCount": int(count)} + for (sample, genotype_class), count in sorted(genotypes.items()) + ] + return { + "source": str(vcf_path), + "sourceSha256": sha256_file(vcf_path), + "filterOracle": "bcftools" if use_bcftools else "cyvcf2-serialization", + "sampleCount": len(samples), + "samples": samples, + "totalRecords": total_records, + "gtRecordCount": gt_records, + "singleAltRecordCount": single_alt_records, + "q06EligibleSiteCount": q06_eligible, + "q01_record_density_1mb": [ + {"chrom": chrom, "windowIndex": window, "recordCount": int(count)} + for (chrom, window), count in sorted(density.items()) + ], + "q02_variant_shape_counts": [ + {"variantClass": kind, "recordCount": int(count)} for kind, count in sorted(shapes.items()) + ], + "q03_titv": { + "biallelicSnvCount": biallelic_snv_count, + "transitionCount": transition_count, + "transversionCount": transversion_count, + "tiTvRatio": transition_count / transversion_count if transversion_count else None, + }, + "q04_filter_distribution": [ + {"filterStatus": status, "filterLexical": lexical, "recordCount": int(count)} + for (status, lexical), count in sorted(filters.items()) + ], + "q05_sample_genotype_counts": q05, + "q06_ac_an_distribution": [ + {"an": an, "ac": ac, "siteCount": int(count), "af": ac / an} + for (an, ac), count in sorted(ac_an.items()) + ], + } + + +def validate_ntriples(source: Path, results_dir: Path) -> dict[str, Any]: + rapper = shutil.which("rapper") + if not rapper: + return {"status": "EXECUTION_FAILED", "error": "rapper is not installed"} + result = subprocess.run( + [rapper, "-i", "ntriples", "-c", str(source)], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + log = results_dir / "rdf-validation" / "rapper.txt" + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(result.stdout + result.stderr, encoding="utf-8") + return {"status": "PASS" if result.returncode == 0 else "FAIL", "log": str(log), "exitCode": result.returncode} + + +def execute_query(query_id: str, source: Path, query_path: Path, raw_dir: Path) -> dict[str, Any]: + executable = shutil.which("comunica-sparql-file") + if not executable: + return {"status": "EXECUTION_FAILED", "error": "comunica-sparql-file is not installed"} + raw_path = raw_dir / f"{query_id}.sparql.json" + stderr_path = raw_dir / f"{query_id}.stderr.txt" + time_path = raw_dir / f"{query_id}.time.txt" + command = [executable, str(source), "-f", str(query_path), "-t", "application/sparql-results+json"] + timed = ["/usr/bin/time", "-v", "-o", str(time_path), *command] if tool_version(["/usr/bin/time", "--version"]) else command + started = time.monotonic() + with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + result = subprocess.run(timed, check=False, stdout=stdout, stderr=stderr) + return { + "status": "PASS" if result.returncode == 0 else "EXECUTION_FAILED", + "exitCode": result.returncode, + "wallSeconds": time.monotonic() - started, + "query": str(query_path), + "rawResult": str(raw_path), + "stderr": str(stderr_path), + "resourceMetrics": str(time_path) if time_path.exists() else None, + } + + +def bindings(path: Path) -> list[dict[str, Any]]: + try: + value = read_json(path)["results"]["bindings"] + except (KeyError, TypeError) as error: + raise ValueError(f"Invalid SPARQL Results JSON: {path}") from error + if not isinstance(value, list): + raise ValueError(f"SPARQL bindings are not a list: {path}") + return value + + +def binding_int(binding: dict[str, Any], field: str) -> int: + return int(binding[field]["value"]) + + +def normalize(query_id: str, path: Path) -> Any: + fields, integer_fields, sort_fields = QUERY_SCHEMAS[query_id] + rows: list[dict[str, Any]] = [] + for number, binding in enumerate(bindings(path), start=1): + row: dict[str, Any] = {} + for field in fields: + try: + lexical = binding[field]["value"] + except (KeyError, TypeError) as error: + raise ValueError(f"Row {number} has no {field!r}") from error + if field in integer_fields: + try: + numeric = Decimal(lexical) + except InvalidOperation as error: + raise ValueError(f"{field} is not numeric: {lexical!r}") from error + if not numeric.is_finite() or numeric != numeric.to_integral_value(): + raise ValueError(f"{field} is not an integer: {lexical!r}") + row[field] = int(numeric) + else: + row[field] = str(lexical) + rows.append(row) + if sort_fields: + rows.sort(key=lambda row: tuple(row[field] for field in sort_fields)) + keys = [tuple(row[field] for field in sort_fields) for row in rows] + if len(keys) != len(set(keys)): + raise ValueError(f"{query_id} returned duplicate canonical keys") + if query_id == "q03_titv": + if len(rows) != 1: + raise ValueError(f"q03_titv must return exactly one row, got {len(rows)}") + row = rows[0] + row["tiTvRatio"] = row["transitionCount"] / row["transversionCount"] if row["transversionCount"] else None + return row + if query_id == "q06_ac_an_distribution": + for row in rows: + row["af"] = row["ac"] / row["an"] + return rows + + +def preflight(executions: dict[str, dict[str, Any]], parser: dict[str, Any], representation: str) -> dict[str, Any]: + report: dict[str, Any] = {} + for query_id in PREFLIGHT_QUERIES: + execution = executions[query_id] + if execution["status"] != "PASS": + report[query_id] = {"status": "EXECUTION_FAILED", "execution": execution} + continue + returned = bindings(Path(execution["rawResult"])) + if query_id in {"preflight_record_cardinality", "preflight_position_datatype"}: + report[query_id] = {"status": "PASS" if not returned else "FAIL", "anomalyCountReturned": len(returned), "limitedTo": 100} + elif query_id == "preflight_representation_profile": + # VCF-RDFizer intentionally omits a sample representation profile + # when a VCF has no sample columns, because no sample graph is + # emitted. That is not a representation mismatch. + profile_missing_is_expected = parser["sampleCount"] == 0 + report[query_id] = { + "status": "PASS" if not returned or profile_missing_is_expected else "FAIL", + "anomalyCountReturned": len(returned), + "limitedTo": 100, + "note": "No sample representation is emitted for a sample-free VCF." if returned and profile_missing_is_expected else None, + } + elif query_id == "preflight_missing_token_conformance": + report[query_id] = { + "status": "PASS" if not returned else "EXPECTED_CONFORMANCE_FAILURE", + "plainDotCountReturned": len(returned), + "limitedTo": 100, + } + elif len(returned) != 1: + report[query_id] = {"status": "FAIL", "error": f"Expected one aggregate row, got {len(returned)}"} + elif representation == "expanded": + actual = {field: binding_int(returned[0], field) for field in ("sampleCallCount", "sampleIdCount", "gtValueNodeCount")} + expected = { + "sampleCallCount": parser["sampleCount"] * parser["totalRecords"], + "sampleIdCount": parser["sampleCount"], + "gtValueNodeCount": parser["sampleCount"] * parser["gtRecordCount"], + } + report[query_id] = {"status": "PASS" if actual == expected else "FAIL", "expected": expected, "actual": actual} + else: + actual = {field: binding_int(returned[0], field) for field in ("sampleCount", "sampleIdCount", "gtVectorCount")} + expected = {"sampleCount": parser["sampleCount"], "sampleIdCount": parser["sampleCount"], "gtVectorCount": parser["gtRecordCount"]} + report[query_id] = {"status": "PASS" if actual == expected else "FAIL", "expected": expected, "actual": actual} + return report + + +def compare_rows(query_id: str, expected: Any, actual: Any) -> dict[str, Any]: + keys, values = QUERY_SPECS[query_id] + selected = keys + values + if not keys: + expected_value = {field: expected[field] for field in values} + actual_value = {field: actual[field] for field in values} + return { + "status": "PASS" if expected_value == actual_value else "MISMATCH", + "expected": expected_value, + "actual": actual_value, + } + def index(rows: list[dict[str, Any]]) -> dict[tuple[Any, ...], dict[str, Any]]: + indexed: dict[tuple[Any, ...], dict[str, Any]] = {} + for row in rows: + key = tuple(row[field] for field in keys) + if key in indexed: + raise ValueError(f"Duplicate {query_id} key: {key!r}") + indexed[key] = {field: row[field] for field in selected} + return indexed + expected_index, actual_index = index(expected), index(actual) + missing_keys, extra_keys = set(expected_index) - set(actual_index), set(actual_index) - set(expected_index) + differing = [ + {"expected": expected_index[key], "actual": actual_index[key]} + for key in sorted(set(expected_index) & set(actual_index)) + if any(expected_index[key][field] != actual_index[key][field] for field in values) + ] + return { + "status": "PASS" if not missing_keys and not extra_keys and not differing else "MISMATCH", + "missingRows": [expected_index[key] for key in sorted(missing_keys)], + "extraRows": [actual_index[key] for key in sorted(extra_keys)], + "differingRows": differing, + } + + +def invariant_checks(payload: dict[str, Any], parser: dict[str, Any], *, check_q06_exact: bool) -> list[dict[str, Any]]: + checks: list[tuple[str, bool, str]] = [] + total = parser["totalRecords"] + for query_id, field in (("q01_record_density_1mb", "recordCount"), ("q02_variant_shape_counts", "recordCount"), ("q04_filter_distribution", "recordCount")): + count = sum(int(row[field]) for row in payload[query_id]) + checks.append((f"{query_id}_total", count == total, f"{count} == {total}")) + q3 = payload["q03_titv"] + checks.append(("q03_partition", q3["transitionCount"] + q3["transversionCount"] == q3["biallelicSnvCount"], "transition + transversion == biallelic SNV")) + if parser["sampleCount"] and parser["gtRecordCount"]: + per_sample: Counter[str] = Counter() + for row in payload["q05_sample_genotype_counts"]: + per_sample[row["sampleId"]] += int(row["callCount"]) + for sample, count in sorted(per_sample.items()): + checks.append((f"q05_total_{sample}", count == total, f"{count} == {total}")) + q06_total = sum(int(row["siteCount"]) for row in payload["q06_ac_an_distribution"]) + checks.append(("q06_subset", q06_total <= parser["singleAltRecordCount"], f"{q06_total} <= {parser['singleAltRecordCount']}")) + if check_q06_exact: + checks.append(("q06_eligible_total", q06_total == parser["q06EligibleSiteCount"], f"{q06_total} == {parser['q06EligibleSiteCount']}")) + for row in payload["q06_ac_an_distribution"]: + checks.append((f"q06_bounds_{row['an']}_{row['ac']}", row["an"] > 0 and 0 <= row["ac"] <= row["an"] and row["siteCount"] > 0, "AN/AC bounds")) + return [{"name": name, "status": "PASS" if passed else "FAIL", "detail": detail} for name, passed, detail in checks] + + +def compare(parser: dict[str, Any], sparql: dict[str, Any]) -> dict[str, Any]: + required = bool(parser["sampleCount"] and parser["gtRecordCount"]) + query_results = {query_id: compare_rows(query_id, parser[query_id], sparql[query_id]) for query_id in CORE_QUERIES} + if not required: + for query_id in ("q05_sample_genotype_counts", "q06_ac_an_distribution"): + query_results[query_id] = {"status": "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT", "diagnosticComparison": query_results[query_id]} + parser_invariants = invariant_checks(parser, parser, check_q06_exact=True) + sparql_invariants = invariant_checks(sparql, parser, check_q06_exact=False) + allowed = {"PASS", "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT"} + passed = all(value["status"] in allowed for value in query_results.values()) and all(item["status"] == "PASS" for item in parser_invariants + sparql_invariants) + return {"status": "PASS" if passed else "MISMATCH", "queries": query_results, "invariants": {"parser": parser_invariants, "sparql": sparql_invariants}} + + +def build_manifest(args: argparse.Namespace, query_dir: Path, parser: dict[str, Any]) -> dict[str, Any]: + query_paths = sorted({query_path(query_dir, query_id) for query_id in PREFLIGHT_QUERIES + CORE_QUERIES}) + return { + "datasetId": args.dataset_id, + "representation": args.representation, + "commandLine": sys.argv, + "sourceVcf": {"path": str(args.vcf), "sha256": parser["sourceSha256"]}, + "sourceRdfGzip": {"path": str(args.rdf_gz), "sha256": sha256_file(args.rdf_gz)}, + "temporaryRdf": {"decompressedInsideContainer": True, "persisted": False, "cleanupConfirmed": True}, + "tools": { + "python": platform.python_version(), "cyvcf2": cyvcf2.__version__, + "bcftools": tool_version(["bcftools", "--version"]), "node": tool_version(["node", "--version"]), + "comunicaQuerySparqlFile": tool_version(["comunica-sparql-file", "--version"], table_label="Comunica Engine"), + "rapper": tool_version(["rapper", "--version"]), + }, + "queries": {path.stem: {"path": str(path), "sha256": sha256_file(path)} for path in query_paths}, + } + + +def query_path(representation_dir: Path, query_id: str) -> Path: + """Resolve a representation-specific query, falling back to common RDF queries.""" + candidate = representation_dir / f"{query_id}.rq" + return candidate if candidate.is_file() else QUERY_ROOT / "common" / f"{query_id}.rq" + + +def run_validation(args: argparse.Namespace) -> int: + results_dir = args.results_dir.resolve() + raw_dir, normalized_dir = results_dir / "raw", results_dir / "normalized" + raw_dir.mkdir(parents=True, exist_ok=True) + normalized_dir.mkdir(parents=True, exist_ok=True) + query_dir = QUERY_ROOT / args.representation + missing = [name for name in PREFLIGHT_QUERIES + CORE_QUERIES if not query_path(query_dir, name).is_file()] + if missing: + raise RuntimeError(f"Missing {args.representation} validation query files: {', '.join(missing)}") + + summary: dict[str, Any] | None = None + try: + with tempfile.TemporaryDirectory(prefix="vcf-rdfizer-validation-", dir=args.scratch_dir) as scratch: + decoded = Path(scratch) / "input.nt" + with gzip.open(args.rdf_gz, "rb") as source, decoded.open("wb") as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + parser = parse_vcf(args.vcf, filter_oracle=args.filter_oracle) + write_json(results_dir / "parser.json", parser) + rdf_validation = validate_ntriples(decoded, results_dir) + write_json(results_dir / "rdf-validation.json", rdf_validation) + manifest = build_manifest(args, query_dir, parser) + write_json(results_dir / "manifest.json", manifest) + if rdf_validation["status"] != "PASS": + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "BLOCKED_BY_PREFLIGHT", "rdfValidation": rdf_validation} + return 1 + executions: dict[str, dict[str, Any]] = {} + for query_id in PREFLIGHT_QUERIES + CORE_QUERIES: + print(f"[{args.dataset_id}] running {args.representation}/{query_id}", flush=True) + executions[query_id] = execute_query(query_id, decoded, query_path(query_dir, query_id), raw_dir) + write_json(results_dir / "query-executions.json", executions) + if any(item["status"] != "PASS" for item in executions.values()): + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "queryExecutions": executions} + return 1 + report = preflight(executions, parser, args.representation) + write_json(results_dir / "preflight.json", report) + sparql: dict[str, Any] = {} + failures: dict[str, str] = {} + for query_id in CORE_QUERIES: + try: + sparql[query_id] = normalize(query_id, Path(executions[query_id]["rawResult"])) + write_json(normalized_dir / f"{query_id}.json", sparql[query_id]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: + failures[query_id] = str(error) + if failures: + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "normalizationFailures": failures, "preflight": report} + return 1 + write_json(results_dir / "sparql.json", sparql) + comparison = compare(parser, sparql) + write_json(results_dir / "comparison.json", comparison) + blocking = any(report[name]["status"] != "PASS" for name in ("preflight_record_cardinality", "preflight_position_datatype", "preflight_representation_profile")) + inventory_failed = report["preflight_sample_gt_inventory"]["status"] != "PASS" + status = "BLOCKED_BY_PREFLIGHT" if blocking else "MISMATCH" if inventory_failed or comparison["status"] != "PASS" else "PASS" + summary = { + "datasetId": args.dataset_id, "representation": args.representation, "status": status, + "recordCount": parser["totalRecords"], "sampleCount": parser["sampleCount"], "gtRecordCount": parser["gtRecordCount"], + "preflight": report, "comparisonStatus": comparison["status"], + "results": {"manifest": str(results_dir / "manifest.json"), "parser": str(results_dir / "parser.json"), "sparql": str(results_dir / "sparql.json"), "comparison": str(results_dir / "comparison.json")}, + } + return 0 if status == "PASS" else 1 + except Exception as error: + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": str(error)} + return 1 + finally: + if summary is None: + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": "validation ended without a result"} + summary["temporaryRdf"] = {"decompressedInsideContainer": True, "persisted": False, "cleanupConfirmed": True} + write_json(results_dir / "summary.json", summary) + print(json.dumps(summary, indent=2), flush=True) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vcf", type=Path, required=True) + parser.add_argument("--rdf-gz", type=Path, required=True, help="N-Triples gzip input (.nt.gz)") + parser.add_argument("--representation", choices=("expanded", "condensed"), required=True) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--dataset-id", required=True) + parser.add_argument("--filter-oracle", choices=("auto", "bcftools", "cyvcf2"), default="auto") + parser.add_argument("--scratch-dir", type=Path, default=Path("/work")) + args = parser.parse_args() + args.vcf = args.vcf.resolve() + args.rdf_gz = args.rdf_gz.resolve() + if not args.vcf.is_file(): + parser.error(f"VCF does not exist: {args.vcf}") + if not args.rdf_gz.is_file() or not args.rdf_gz.name.endswith(".nt.gz"): + parser.error("--rdf-gz must be an existing .nt.gz file") + if not re.fullmatch(r"[A-Za-z0-9._-]+", args.dataset_id): + parser.error("--dataset-id may contain only letters, digits, dot, underscore, and hyphen") + if not args.scratch_dir.is_dir(): + parser.error(f"Scratch directory does not exist: {args.scratch_dir}") + return args + + +if __name__ == "__main__": + raise SystemExit(run_validation(parse_args())) diff --git a/test/test_compression_unit.py b/test/test_compression_unit.py index 30b0d65..897be27 100644 --- a/test/test_compression_unit.py +++ b/test/test_compression_unit.py @@ -154,10 +154,10 @@ def test_compression_updates_existing_metrics_row_with_mocked_tools(self): self.assertTrue((output / "rdf.nt.gz").exists()) self.assertTrue((output / "rdf.nt.br").exists()) self.assertTrue((output / "rdf.hdt").exists()) - self.assertTrue((logdir / "compression_time" / "gzip" / "rdf" / "run-compress-1.txt").exists()) - self.assertTrue((logdir / "compression_time" / "brotli" / "rdf" / "run-compress-1.txt").exists()) - self.assertTrue((logdir / "compression_time" / "hdt" / "rdf" / "run-compress-1.txt").exists()) - self.assertTrue((logdir / "compression_metrics" / "rdf" / "run-compress-1.json").exists()) + self.assertTrue((logdir / "timings" / "compression" / "rdf" / "gzip.txt").exists()) + self.assertTrue((logdir / "timings" / "compression" / "rdf" / "brotli.txt").exists()) + self.assertTrue((logdir / "timings" / "compression" / "rdf" / "hdt.txt").exists()) + self.assertTrue((logdir / "stages" / "compression" / "rdf.json").exists()) row = read_metrics_row(metrics_csv, run_id, "rdf") self.assertEqual(row["run_id"], run_id) diff --git a/test/test_run_conversion_unit.py b/test/test_run_conversion_unit.py index 643fb84..d02a90a 100644 --- a/test/test_run_conversion_unit.py +++ b/test/test_run_conversion_unit.py @@ -70,8 +70,8 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): merged_nt = out_dir / "rdf" / "rdf.nt" self.assertTrue(merged_nt.exists()) self.assertIn("

.", merged_nt.read_text()) - self.assertTrue((metrics_dir / "conversion_time" / "rdf" / "run123.txt").exists()) - self.assertTrue((metrics_dir / "conversion_metrics" / "rdf" / "run123.json").exists()) + self.assertTrue((metrics_dir / "timings" / "conversion" / "rdf.txt").exists()) + self.assertTrue((metrics_dir / "stages" / "conversion" / "rdf.json").exists()) metrics_csv = metrics_dir / "metrics.csv" self.assertTrue(metrics_csv.exists()) @@ -602,7 +602,7 @@ def test_run_conversion_space_optimized_streams_gzip_members_and_deletes_parts(s self.assertIn("

.", aggregate_text) self.assertIn("

.", aggregate_text) self.assertEqual(list((out_dir / "rdf").glob("part-*.nt")), []) - metrics = metrics_dir / "conversion_metrics" / "rdf" / "run-space.json" + metrics = metrics_dir / "stages" / "conversion" / "rdf.json" payload = json.loads(metrics.read_text()) self.assertEqual(payload["rdf_storage"]["mode"], "space-optimized") self.assertTrue(payload["rdf_storage"]["compressed"]) diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index b95c568..d561695 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -243,20 +243,34 @@ def output_name_from_command(cmd): def latest_metrics_run_dir(metrics_root: Path) -> Path: """Return the single/latest per-run metrics directory.""" - run_dirs = sorted( - ( - path - for path in metrics_root.iterdir() - if path.is_dir() and re.match(r"^\d{8}T\d{6}$", path.name) - ), - key=lambda path: path.name, - ) + run_dirs = [] + for path in metrics_root.iterdir(): + if not path.is_dir(): + continue + match = re.match(r"^.+__(\d{8}T\d{6})$", path.name) + if match: + run_dirs.append((match.group(1), path)) + run_dirs.sort(key=lambda item: item[0]) if not run_dirs: raise AssertionError(f"No per-run metrics directories found under {metrics_root}") - return run_dirs[-1] + return run_dirs[-1][1] class WrapperUnitTests(VerboseTestCase): + def test_metrics_directory_uses_recognizable_input_label(self): + """Metrics runs keep the full input stem alongside their timestamp.""" + metrics_root = Path("/tmp/metrics") + label = vcf_rdfizer.metrics_run_label( + [Path("/data/1000G_phase3_chr20.vcf.gz")] + ) + self.assertEqual(label, "1000G_phase3_chr20") + self.assertEqual( + vcf_rdfizer.metrics_run_directory( + metrics_root, label, "20260904T123456" + ), + metrics_root / "1000G_phase3_chr20__20260904T123456", + ) + def test_hdt_index_memory_limit_is_forwarded_to_docker(self): """A host hdtc memory override is passed to indexing containers.""" with mock.patch.dict(os.environ, {"HDT_INDEX_MEMORY_LIMIT": "2G"}): @@ -609,7 +623,7 @@ def fake_run(cmd, cwd=None, env=None): if emulate_validation_command(cmd): return 0 script = str(cmd[-1]) if cmd else "" - time_match = re.search(r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", script) + time_match = re.search(r"-o\s+(/data/metrics/timings/tsv/[^\s;]+)", script) if metrics_mount and time_match: time_log = Path(metrics_mount) / time_match.group(1).replace("/data/metrics/", "", 1) time_log.parent.mkdir(parents=True, exist_ok=True) @@ -645,12 +659,12 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(metrics["sys_seconds"], 1.5) self.assertEqual(metrics["max_rss_kb"], 2048) - raw_json = metrics_dir / "raw_metrics" / "tsv_metrics" / "sample" / "run-tsv-elapsed.json" + raw_json = metrics_dir / "stages" / "tsv" / "sample.json" payload = json.loads(raw_json.read_text()) self.assertEqual(payload["timing"]["wall_seconds"], 360.0) - def test_run_compression_methods_persists_raw_metrics_and_time_logs(self): - """Per-file compression timing/metrics are retained under raw_metrics.""" + def test_run_compression_methods_persists_stage_metrics_and_time_logs(self): + """Per-file compression timing/metrics use the canonical stages/timings tree.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) out_dir = tmp_path / "out" / "sample" @@ -703,29 +717,24 @@ def fake_run(cmd, cwd=None, env=None): safe_rdf = vcf_rdfizer.safe_metrics_name("sample.nt") hdt_time = ( metrics_dir - / "raw_metrics" - / "compression_time" + / "timings" + / "compression" / safe_output - / safe_rdf - / "hdt" - / "run-1.txt" + / "hdt.txt" ) gzip_time = ( metrics_dir - / "raw_metrics" - / "compression_time" + / "timings" + / "compression" / safe_output - / safe_rdf - / "gzip" - / "run-1.txt" + / "gzip.txt" ) raw_json = ( metrics_dir - / "raw_metrics" - / "compression_metrics" + / "stages" + / "compression_operations" / safe_output - / safe_rdf - / "run-1.json" + / f"{safe_rdf}.json" ) self.assertTrue(hdt_time.exists()) @@ -782,8 +791,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(method_results["hdt"]["sys_seconds"], 4.5) self.assertEqual(method_results["hdt"]["max_rss_kb"], 8192) - def test_run_compression_methods_records_implicit_hdt_in_raw_metrics(self): - """Compound HDT methods include the implicit HDT stage in raw metrics JSON.""" + def test_run_compression_methods_records_implicit_hdt_in_stage_metrics(self): + """Compound HDT methods include the implicit HDT stage in the operation report.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) out_dir = tmp_path / "out" / "sample" @@ -836,11 +845,10 @@ def fake_run(cmd, cwd=None, env=None): safe_rdf = vcf_rdfizer.safe_metrics_name("sample.nt") raw_json = ( metrics_dir - / "raw_metrics" - / "compression_metrics" + / "stages" + / "compression_operations" / safe_output - / safe_rdf - / "run-2.json" + / f"{safe_rdf}.json" ) self.assertTrue(raw_json.exists()) @@ -909,11 +917,10 @@ def fake_run(cmd, cwd=None, env=None): raw_json = ( metrics_dir - / "raw_metrics" - / "compression_metrics" + / "stages" + / "compression_operations" / "sample" - / "sample.nt" - / "run-cottas-package.json" + / "sample.nt.json" ) self.assertTrue(raw_json.exists()) payload = json.loads(raw_json.read_text()) @@ -1210,6 +1217,7 @@ def test_containerized_partitioned_pipeline_cleans_ephemeral_volume(self): source = tmp_path / "input.nt" source.write_text("

.\n") out_dir = tmp_path / "out" / "sample" + metrics_dir = tmp_path / "metrics" commands = [] def fake_run(cmd, cwd=None, env=None): @@ -1226,6 +1234,9 @@ def fake_run(cmd, cwd=None, env=None): image_ref="example/vcf-rdfizer:latest", methods=["hdt", "cottas"], wrapper_log_path=tmp_path / "wrapper.log", + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", output_name="sample", target_chunk_bytes=40, min_chunk_bytes=10, @@ -1244,6 +1255,15 @@ def fake_run(cmd, cwd=None, env=None): cmd for cmd in commands if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd ) self.assertIn(f"{source.parent.resolve()}:/data/in:ro", runner_command) + report = json.loads( + (metrics_dir / "stages" / "partitioned" / "sample.json").read_text() + ) + self.assertEqual(report["runtime_environment"], "docker-volume") + self.assertEqual(report["container_result"]["exit_code"], 0) + self.assertEqual( + report["container_result"]["methods"]["hdt"]["details"]["workspace"], + "docker-volume", + ) def test_containerized_partitioned_pipeline_removes_volume_after_failure(self): """A failed container run still removes its named workspace volume.""" @@ -1252,6 +1272,7 @@ def test_containerized_partitioned_pipeline_removes_volume_after_failure(self): source = tmp_path / "input.nt" source.write_text("

.\n") out_dir = tmp_path / "out" + metrics_dir = tmp_path / "metrics" commands = [] def fake_run(cmd, cwd=None, env=None): @@ -1277,6 +1298,9 @@ def fake_run(cmd, cwd=None, env=None): image_ref="example/vcf-rdfizer:latest", methods=["cottas"], wrapper_log_path=tmp_path / "wrapper.log", + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", output_name="sample", target_chunk_bytes=40, min_chunk_bytes=10, @@ -1287,6 +1311,10 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(method_results, {}) self.assertTrue(any("volume" in cmd and "rm" in cmd for cmd in commands)) self.assertFalse(any(path.name.startswith(".") for path in out_dir.glob("*"))) + report = json.loads( + (metrics_dir / "stages" / "partitioned" / "sample.json").read_text() + ) + self.assertEqual(report["container_result"]["exit_code"], 1) def test_main_full_mode_records_tsv_metrics_and_raw_artifacts(self): """Full mode stores TSV timing/metrics artifacts and writes TSV fields into metrics.csv.""" @@ -1311,7 +1339,7 @@ def fake_run(cmd, cwd=None, env=None): ) script = str(cmd[-1]) if cmd else "" time_match = re.search( - r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", + r"-o\s+(/data/metrics/timings/tsv/[^\s;]+)", script, ) if metrics_mount and time_match: @@ -1364,10 +1392,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(rc, 0) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - run_id = run_metrics_dir.name - - tsv_time = run_metrics_dir / "raw_metrics" / "tsv_time" / "sample" / f"{run_id}.txt" - tsv_json = run_metrics_dir / "raw_metrics" / "tsv_metrics" / "sample" / f"{run_id}.json" + tsv_time = run_metrics_dir / "timings" / "tsv" / "sample.txt" + tsv_json = run_metrics_dir / "stages" / "tsv" / "sample.json" self.assertTrue(tsv_time.exists()) self.assertTrue(tsv_json.exists()) @@ -1473,7 +1499,7 @@ def test_default_sample_maps_stream_without_expanded_helper_tables(self): self.assertEqual(len(rdf_lines), 20) self.assertIn( " " - " .", + " .", rdf_lines, ) self.assertIn( @@ -1531,14 +1557,14 @@ def test_sample_support_strategy_preserves_custom_helper_consumers(self): self.assertEqual(vcf_rdfizer.sample_support_strategy(custom_rules), "expanded") def test_sample_workflow_resolver_selects_one_compatible_branch(self): - """Dense and condensed plans are mutually exclusive and rules-aware.""" + """Expanded and condensed plans are mutually exclusive and rules-aware.""" default_rules = Path(__file__).parents[1] / "rules" / "default_rules.ttl" - dense = vcf_rdfizer.resolve_sample_workflow("dense", default_rules) + expanded = vcf_rdfizer.resolve_sample_workflow("expanded", default_rules) condensed = vcf_rdfizer.resolve_sample_workflow("condensed", default_rules) - self.assertEqual(dense.helper_strategy, "header-only") - self.assertEqual(dense.emitter, "dense") + self.assertEqual(expanded.helper_strategy, "header-only") + self.assertEqual(expanded.emitter, "expanded") self.assertEqual(condensed.helper_strategy, "header-only") self.assertEqual(condensed.emitter, "condensed") @@ -1550,14 +1576,14 @@ def test_sample_workflow_resolver_selects_one_compatible_branch(self): encoding="utf-8", ) self.assertEqual( - vcf_rdfizer.resolve_sample_workflow("dense", custom_rules).helper_strategy, + vcf_rdfizer.resolve_sample_workflow("expanded", custom_rules).helper_strategy, "expanded", ) with self.assertRaisesRegex(ValueError, "cannot be combined"): vcf_rdfizer.resolve_sample_workflow("condensed", custom_rules) def test_condensed_sample_emitter_uses_shared_samples_and_ordered_vectors(self): - """Condensed mode emits one SampleSet and FORMAT vector per key, not dense calls.""" + """Condensed mode emits one SampleSet and FORMAT vector per key, not expanded calls.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) records_tsv = tmp_path / "cohort.records.tsv" @@ -1644,7 +1670,7 @@ def test_condensed_sample_emitter_rolls_back_on_sample_count_mismatch(self): self.assertEqual(rdf_path.read_text(encoding="utf-8"), original) def test_main_condensed_mode_runs_only_the_condensed_emitter(self): - """The full CLI routes default rules to condensed output without dense sample triples.""" + """The full CLI routes default rules to condensed output without expanded sample triples.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_file = tmp_path / "cohort.vcf" @@ -1782,7 +1808,7 @@ def test_help_flag_prints_usage_guide(self): self.assertEqual(exc.exception.code, 0) text = out_buf.getvalue() self.assertIn("Examples:", text) - self.assertIn("-m {full,compress,decompress,tsv,index}", text) + self.assertIn("-m {full,compress,decompress,tsv,index,validation}", text) self.assertIn("-i INPUT", text) self.assertIn("--keep-rmlstreamer-rdf-output", text) self.assertIn("--remove-rdf-storage-output", text) @@ -1925,7 +1951,7 @@ def fake_run(cmd, cwd=None, env=None): ) script = str(cmd[-1]) if cmd else "" time_match = re.search( - r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", + r"-o\s+(/data/metrics/timings/tsv/[^\s;]+)", script, ) if metrics_mount and time_match: @@ -1968,9 +1994,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertNotIn("/opt/vcf-rdfizer/run_conversion.sh", " ".join(map(str, commands[0]))) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - run_id = run_metrics_dir.name - tsv_time = run_metrics_dir / "raw_metrics" / "tsv_time" / "sample" / f"{run_id}.txt" - tsv_json = run_metrics_dir / "raw_metrics" / "tsv_metrics" / "sample" / f"{run_id}.json" + tsv_time = run_metrics_dir / "timings" / "tsv" / "sample.txt" + tsv_json = run_metrics_dir / "stages" / "tsv" / "sample.json" self.assertTrue(tsv_time.exists()) self.assertTrue(tsv_json.exists()) @@ -2159,6 +2184,13 @@ def fake_run(cmd, cwd=None, env=None): rows = list(csv.DictReader(handle)) self.assertEqual(rows[-1]["mode"], "compress") self.assertEqual(rows[-1]["status"], "success") + self.assertRegex(run_metrics_dir.name, r"^sample__\d{8}T\d{6}$") + manifest = json.loads((run_metrics_dir / "run.json").read_text()) + summary = json.loads((run_metrics_dir / "summary.json").read_text()) + self.assertEqual(manifest["source_label"], "sample") + self.assertEqual(manifest["inputs"][0]["path"], str(rdf_path.resolve())) + self.assertEqual(summary["status"], "success") + self.assertIn("logs/wrapper.log", summary["logs"]) def test_main_full_mode_prints_triplets_and_logs_total(self): """Full mode prints produced triples and records them in runtime timing log.""" @@ -2182,11 +2214,14 @@ def fake_run(cmd, cwd=None, env=None): sample_dir.mkdir(parents=True, exist_ok=True) (sample_dir / f"{out_name}.nt").write_text("

.\n") payload = {"artifacts": {"output_triples": {"TOTAL": 17}}} - run_metrics_dir = metrics_dir / run_id - run_metrics_dir.mkdir(parents=True, exist_ok=True) - conversion_metrics_dir = run_metrics_dir / "conversion_metrics" / out_name + metrics_mount = next( + part.split(":", 1)[0] + for part in cmd + if isinstance(part, str) and part.endswith(":/data/metrics") + ) + conversion_metrics_dir = Path(metrics_mount) / "stages" / "conversion" conversion_metrics_dir.mkdir(parents=True, exist_ok=True) - (conversion_metrics_dir / f"{run_id}.json").write_text( + (conversion_metrics_dir / f"{out_name}.json").write_text( json.dumps(payload), encoding="utf-8", ) @@ -2346,6 +2381,53 @@ def test_main_compress_mode_requires_rdf_argument(self): rc = invoke_main(["--mode", "compress"]) self.assertEqual(rc, 2) + def test_main_validation_mode_runs_container_local_nt_gzip_validation(self): + """Validation mounts only .nt.gz input and invokes the internal runner.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt.gz" + with gzip.open(rdf_path, "wt", encoding="utf-8") as handle: + handle.write("

.\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True): + rc = invoke_main( + [ + "--mode", + "validation", + "--input", + str(vcf_path), + "--rdf", + str(rdf_path), + "--sample-representation", + "condensed", + "--out", + str(out_dir), + ] + ) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + command = commands[0] + self.assertIn("/opt/vcf-rdfizer/validation/validation_runner.py", command) + self.assertIn("/data/rdf/sample.nt.gz", command) + self.assertIn("condensed", command) + results_dir = out_dir / "validation" / "sample" + self.assertTrue(results_dir.is_dir()) + stage = next((out_dir / "run_metrics").glob("*/stages/validation/sample.json")) + payload = json.loads(stage.read_text()) + self.assertTrue(payload["temporary_rdf"]["decompressed_inside_container"]) + self.assertFalse(payload["temporary_rdf"]["persisted_on_host"]) + def test_main_rejects_spark_partitions_outside_full_mode(self): """--spark-partitions is rejected for non-full modes.""" rc = invoke_main( @@ -2453,7 +2535,7 @@ def test_main_full_mode_keyboard_interrupt_returns_130_and_writes_progress_log(s self.assertEqual(rc, 130) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - progress_log = run_metrics_dir / "progress.log" + progress_log = run_metrics_dir / "logs" / "progress.log" self.assertTrue(progress_log.exists()) self.assertIn("Run interrupted by user signal", progress_log.read_text()) @@ -2667,7 +2749,12 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(len(commands), 1) self.assertIn("ensure_hdt_index.sh", commands[0][-1]) self.assertTrue(any(arg.endswith(":/data/hdt") for arg in commands[0])) - metrics = latest_metrics_run_dir(out_dir / "run_metrics") / "hdt_index_metrics.json" + metrics = ( + latest_metrics_run_dir(out_dir / "run_metrics") + / "stages" + / "index" + / "hdt-sample.hdt.json" + ) payload = json.loads(metrics.read_text()) self.assertEqual(payload["index_status"], "generated") self.assertEqual( @@ -2717,7 +2804,12 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(rc, 0) self.assertEqual(existing_index.read_text(), "new-index\n") payload = json.loads( - (latest_metrics_run_dir(out_dir / "run_metrics") / "index_metrics.json").read_text() + ( + latest_metrics_run_dir(out_dir / "run_metrics") + / "stages" + / "index" + / "hdt-sample.hdt.json" + ).read_text() ) self.assertEqual(payload["index_status"], "regenerated") @@ -2764,13 +2856,18 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("cottas_tool.py reindex", commands[0][-1]) self.assertTrue(any(arg.endswith(":/data/cottas") for arg in commands[0])) payload = json.loads( - (latest_metrics_run_dir(out_dir / "run_metrics") / "index_metrics.json").read_text() + ( + latest_metrics_run_dir(out_dir / "run_metrics") + / "stages" + / "index" + / "cottas-sample.cottas.json" + ).read_text() ) self.assertEqual(payload["index_format"], "cottas") self.assertEqual(payload["index_location"], "embedded") self.assertEqual(payload["index_status"], "regenerated") self.assertTrue( - (latest_metrics_run_dir(out_dir / "run_metrics") / "cottas_index_metrics.json").exists() + (latest_metrics_run_dir(out_dir / "run_metrics") / "summary.json").exists() ) def test_main_index_mode_requires_exactly_one_index_input(self): @@ -3133,7 +3230,7 @@ def fake_partitioned(**kwargs): self.assertEqual(rc, 0) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - warning_path = run_metrics_dir / "index_warnings.json" + warning_path = run_metrics_dir / "reports" / "index_warnings.json" self.assertTrue(warning_path.exists()) self.assertEqual(json.loads(warning_path.read_text())["warning_count"], 1) self.assertIn("Conversion process finished with index warnings.", stdout.getvalue()) @@ -3369,7 +3466,7 @@ def fake_run(cmd, cwd=None, env=None): self.assertTrue((out_dir / "sample_b" / "sample_b.hdt").exists()) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - failed_report = run_metrics_dir / "failed_inputs.csv" + failed_report = run_metrics_dir / "reports" / "failed_inputs.csv" self.assertTrue(failed_report.exists()) report_text = failed_report.read_text() self.assertIn("sample_a", report_text) @@ -3599,8 +3696,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("sample", csv_text) self.assertIn("hdt", csv_text) - json_file = run_metrics_dir / "compression_metrics" / "sample" / f"{run_metrics_dir.name}.json" - time_file = run_metrics_dir / "compression_time" / "hdt" / "sample" / f"{run_metrics_dir.name}.txt" + json_file = run_metrics_dir / "stages" / "compression" / "sample.json" + time_file = run_metrics_dir / "timings" / "compression" / "sample" / "hdt.txt" self.assertTrue(json_file.exists()) self.assertTrue(time_file.exists()) diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 54666a8..1b44bdc 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -254,7 +254,11 @@ RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" XSD_POSITIVE_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#positiveInteger" SAMPLE_RDF_BUFFER_BYTES = 8 * 1024 * 1024 -SAMPLE_REPRESENTATION_CHOICES = {"dense", "condensed"} +SAMPLE_REPRESENTATION_CHOICES = {"expanded", "condensed"} +# This is an internal rules-compatibility value, not a third public +# representation. It means that custom helper TSV rows must be materialized. +SAMPLE_HELPER_STRATEGY_MATERIALIZED = "expanded" +METRICS_LAYOUT_VERSION = 2 # --------------------------------------------------------------------------- @@ -1255,6 +1259,7 @@ def read_conversion_total_triples(metrics_dir: Path, output_name: str, run_id: s """Read TOTAL triple count for one conversion output from conversion metrics JSON.""" safe_name = safe_metrics_name(output_name) candidates = [ + metrics_dir / "stages" / "conversion" / f"{safe_name}.json", metrics_dir / "conversion_metrics" / safe_name / f"{run_id}.json", metrics_dir / "conversion_metrics" / safe_name / run_id, # Backward compatibility with older artifact names: @@ -1294,12 +1299,13 @@ def collect_full_mode_total_triples(metrics_dir: Path, run_id: str): total = 0 found = False candidate_files = [] + candidate_files.extend(sorted((metrics_dir / "stages" / "conversion").glob("*.json"))) candidate_files.extend(sorted(metrics_dir.glob("conversion_metrics/*/*"))) # Backward compatibility with older artifact names: candidate_files.extend(sorted(metrics_dir.glob(f"conversion-metrics-*-{run_id}.json"))) for metrics_json in candidate_files: - if ( + if metrics_json.parent.name != "conversion" and ( metrics_json.name != run_id and metrics_json.name != f"{run_id}.json" and not metrics_json.name.endswith(f"-{run_id}.json") @@ -1366,8 +1372,9 @@ def append_wrapper_timing_log( def write_failed_inputs_report(*, metrics_dir: Path, failures: list[dict]): """Write per-input failure summary for multi-input modes.""" - ensure_dir(metrics_dir) - report_path = metrics_dir / "failed_inputs.csv" + report_dir = metrics_dir / "reports" + ensure_dir(report_dir) + report_path = report_dir / "failed_inputs.csv" header = [ "input_index", "input_vcf", @@ -1393,8 +1400,9 @@ def write_failed_inputs_report(*, metrics_dir: Path, failures: list[dict]): def write_index_warnings_report(*, metrics_dir: Path, run_id: str, warnings: list[dict]): """Write non-fatal full-run HDT/COTTAS index warnings as JSON.""" - ensure_dir(metrics_dir) - report_path = metrics_dir / "index_warnings.json" + report_dir = metrics_dir / "reports" + ensure_dir(report_dir) + report_path = report_dir / "index_warnings.json" payload = { "run_id": run_id, "warning_count": len(warnings), @@ -1836,11 +1844,11 @@ def write_sample_support_headers(sample_calls_tsv: Path, sample_format_tsv: Path def sample_support_strategy(rules_path: Path) -> str: - """Choose no, streamed, or expanded sample handling for one mapping file. + """Choose no, streamed, or materialized sample handling for one mapping file. The built-in four sample maps can be emitted directly as N-Triples without writing their enormous Cartesian helper TSVs. A custom mapping with extra - helper-table consumers retains the expanded TSV behavior. + helper-table consumers retains the materialized TSV behavior. """ text = rules_path.read_text(encoding="utf-8") calls_refs = text.count('/data/tsv/sample_calls.tsv') @@ -1854,7 +1862,7 @@ def sample_support_strategy(rules_path: Path) -> str: and all(fragment in text for fragment in CANONICAL_SAMPLE_RULE_FRAGMENTS) ): return "stream" - return "expanded" + return SAMPLE_HELPER_STRATEGY_MATERIALIZED @dataclass(frozen=True) @@ -1869,9 +1877,9 @@ class SampleWorkflow: def resolve_sample_workflow(representation: str, rules_path: Path) -> SampleWorkflow: """Resolve rules compatibility into exactly one sample workflow. - Dense mode preserves custom helper-table mappings. Condensed mode emits its + Expanded mode preserves custom helper-table mappings. Condensed mode emits its RDF directly from records.tsv; it cannot safely coexist with custom rules - that consume expanded dense helper tables because that would execute both + that consume materialized helper tables because that would execute both representations and reintroduce semantic inflation. """ if representation not in SAMPLE_REPRESENTATION_CHOICES: @@ -1881,18 +1889,18 @@ def resolve_sample_workflow(representation: str, rules_path: Path) -> SampleWork ) rules_strategy = sample_support_strategy(rules_path) - if representation == "dense": + if representation == "expanded": if rules_strategy == "stream": - return SampleWorkflow("dense", "header-only", "dense") - if rules_strategy == "expanded": - return SampleWorkflow("dense", "expanded", None) - return SampleWorkflow("dense", "none", None) + return SampleWorkflow("expanded", "header-only", "expanded") + if rules_strategy == SAMPLE_HELPER_STRATEGY_MATERIALIZED: + return SampleWorkflow("expanded", SAMPLE_HELPER_STRATEGY_MATERIALIZED, None) + return SampleWorkflow("expanded", "none", None) - if rules_strategy == "expanded": + if rules_strategy == SAMPLE_HELPER_STRATEGY_MATERIALIZED: raise ValueError( "--sample-representation condensed cannot be combined with custom rules " - "that consume expanded sample_calls.tsv or sample_format_values.tsv tables. " - "Remove those dense helper-table consumers or use dense mode." + "that consume materialized sample_calls.tsv or sample_format_values.tsv tables. " + "Remove those materialized helper-table consumers or use expanded mode." ) helper_strategy = "header-only" if rules_strategy == "stream" else "none" return SampleWorkflow("condensed", helper_strategy, "condensed") @@ -2118,19 +2126,19 @@ def emit(line: str): raise -def append_dense_sample_rdf( +def append_expanded_sample_rdf( records_tsv: Path, rdf_path: Path, *, progress_interval_records: int = 10_000, ) -> dict: - """Append the dense SampleCall/FormatFieldValue representation. + """Append the expanded SampleCall/FormatFieldValue representation. This produces the same canonical SampleCall and FormatFieldValue triples as the default RML maps without materializing V*S and V*S*F helper TSV rows. """ stats = { - "representation": "dense", + "representation": "expanded", "records": 0, "sample_calls": 0, "format_values": 0, @@ -2151,7 +2159,7 @@ def produce(emit): file_uri = f"file://{source_component}" emit( f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " - f"<{VCFR_NAMESPACE}DenseRepresentation> .\n" + f"<{VCFR_NAMESPACE}ExpandedRepresentation> .\n" ) for record in record_stream: row_component = _rml_uri_component(record.row_id) @@ -2199,8 +2207,8 @@ def append_canonical_sample_rdf( *, progress_interval_records: int = 10_000, ) -> dict: - """Backward-compatible name for the dense sample RDF emitter.""" - return append_dense_sample_rdf( + """Backward-compatible name for the expanded sample RDF emitter.""" + return append_expanded_sample_rdf( records_tsv, rdf_path, progress_interval_records=progress_interval_records, @@ -2432,8 +2440,8 @@ def emit_sample_representation( """Execute the workflow's sole direct RDF emitter, if it has one.""" if workflow.emitter is None: return None - if workflow.emitter == "dense": - return append_dense_sample_rdf(records_tsv, rdf_path) + if workflow.emitter == "expanded": + return append_expanded_sample_rdf(records_tsv, rdf_path) if workflow.emitter == "condensed": return append_condensed_sample_rdf(records_tsv, header_lines_tsv, rdf_path) raise RuntimeError(f"unknown sample RDF emitter: {workflow.emitter}") @@ -2450,7 +2458,7 @@ def update_conversion_metrics_after_sample_stream( ): """Bring conversion JSON/CSV metrics in sync after direct sample emission.""" safe_name = safe_metrics_name(output_name) - metrics_json = metrics_dir / "conversion_metrics" / safe_name / f"{run_id}.json" + metrics_json = metrics_dir / "stages" / "conversion" / f"{safe_name}.json" output_size = int(rdf_path.stat().st_size) if metrics_json.is_file(): try: @@ -2750,6 +2758,163 @@ def safe_metrics_name(value: str) -> str: return safe or "rdf" +def input_artifact_stem(path: Path) -> str: + """Return a stable, human-readable label for a source artifact. + + Metric directories are intended for people first, so remove only the + recognized VCF/RDF/representation suffixes rather than repeatedly applying + :attr:`Path.stem` (which turns ``cohort.vcf.gz`` into ``cohort.vcf``). + """ + name = path.name + for suffix in ( + ".vcf.gz", + ".vcf", + ".nt.gz", + ".nt.br", + ".nt", + ".cottas.gz", + ".cottas.br", + ".cottas", + ".hdt", + ".gz", + ".br", + ): + if name.endswith(suffix): + return name[: -len(suffix)] or "input" + return path.stem or "input" + + +def metrics_run_label(source_paths: list[Path], source_root: Path | None = None) -> str: + """Return the input-identifying label for one metrics-run directory.""" + if len(source_paths) == 1: + return safe_metrics_name(input_artifact_stem(source_paths[0])) + + root_label = input_artifact_stem(source_root) if source_root is not None else "inputs" + return safe_metrics_name(f"batch-{root_label}-{len(source_paths)}-inputs") + + +def metrics_run_directory(metrics_root: Path, source_label: str, run_id: str) -> Path: + """Build the canonical input-labelled directory for one invocation.""" + return metrics_root / f"{safe_metrics_name(source_label)}__{run_id}" + + +def read_metrics_csv_rows(path: Path) -> list[dict]: + """Read a metrics CSV without allowing a damaged optional report to fail a run.""" + if not path.is_file(): + return [] + try: + with path.open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + except OSError: + return [] + + +def write_run_manifest( + *, + metrics_dir: Path, + run_id: str, + timestamp: str, + mode: str, + source_label: str, + source_paths: list[Path], + out_root: Path, + options: dict, +): + """Write the static, human-readable identity and configuration of a run.""" + ensure_dir(metrics_dir) + payload = { + "metrics_layout_version": METRICS_LAYOUT_VERSION, + "run_id": run_id, + "timestamp": timestamp, + "mode": mode, + "source_label": source_label, + "output_root": str(out_root), + "metrics_directory": str(metrics_dir), + "inputs": [ + { + "path": str(path), + "file_name": path.name, + "size_bytes": file_size_bytes(path), + } + for path in source_paths + ], + "options": options, + } + path = metrics_dir / "run.json" + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return path + + +def update_run_manifest(metrics_dir: Path, **updates) -> None: + """Merge late-bound runtime details (such as the resolved image) into run.json.""" + path = metrics_dir / "run.json" + try: + payload = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + payload.update(updates) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except (OSError, json.JSONDecodeError): + # Metadata must improve diagnosability, never invalidate a completed run. + pass + + +def write_run_summary( + *, + metrics_dir: Path, + run_id: str, + timestamp: str, + mode: str, + exit_code: int, + elapsed_seconds: float, + total_triples: int | None, +): + """Write one discoverable end-of-run summary for all workflow modes. + + Individual stage reports retain their native detail; this file provides the + compact landing page that links their location and repeats the tabular rows + most often used for analysis. + """ + stage_dir = metrics_dir / "stages" + stage_reports = [ + path.relative_to(metrics_dir).as_posix() + for path in sorted(stage_dir.rglob("*.json")) + ] if stage_dir.is_dir() else [] + report_dir = metrics_dir / "reports" + reports = [ + path.relative_to(metrics_dir).as_posix() + for path in sorted(report_dir.iterdir()) + if path.is_file() + ] if report_dir.is_dir() else [] + payload = { + "metrics_layout_version": METRICS_LAYOUT_VERSION, + "run_id": run_id, + "timestamp": timestamp, + "completed_at": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + "mode": mode, + "status": "success" if int(exit_code) == 0 else "failure", + "exit_code": int(exit_code), + "execution": { + "wall_seconds": round(float(elapsed_seconds), 6), + "wall_human": format_duration(elapsed_seconds), + "total_triples": total_triples, + }, + "summary_tables": { + "conversion_and_compression": read_metrics_csv_rows(metrics_dir / "metrics.csv"), + "tsv": read_metrics_csv_rows(metrics_dir / "tsv_metrics.csv"), + "wrapper": read_metrics_csv_rows(metrics_dir / "wrapper_execution_times.csv"), + }, + "stage_reports": stage_reports, + "reports": reports, + "logs": [ + path.relative_to(metrics_dir).as_posix() + for path in sorted((metrics_dir / "logs").rglob("*")) + if path.is_file() + ] if (metrics_dir / "logs").is_dir() else [], + } + path = metrics_dir / "summary.json" + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return path + + def progress_event_path(metrics_dir: Path | None, *components: str) -> Path | None: """Return a hidden, per-operation progress sidecar path.""" if metrics_dir is None: @@ -3043,14 +3208,14 @@ def write_compression_metrics_artifacts( method_results: dict[str, dict], index_warnings: list[dict] | None = None, ): - """Write per-output compression artifacts (time files + structured JSON).""" + """Write the final per-output compression summary and method timing files.""" metrics_dir.mkdir(parents=True, exist_ok=True) safe_name = safe_metrics_name(output_name) for method, result in method_results.items(): - time_log_dir = metrics_dir / "compression_time" / method / safe_name + time_log_dir = metrics_dir / "timings" / "compression" / safe_name time_log_dir.mkdir(parents=True, exist_ok=True) - time_log = time_log_dir / f"{run_id}.txt" + time_log = time_log_dir / f"{safe_metrics_name(method)}.txt" lines = [ f"method={method}", f"exit_code={result.get('exit_code', 1)}", @@ -3099,6 +3264,10 @@ def timing_payload(result: dict): "combined_rdf_path": str(source_rdf_path), "combined_rdf_size_bytes": int(combined_size_bytes), "index_warnings": list(index_warnings or []), + # This preserves every method-specific detail returned by a container + # (validation, chunk plan, index information, and workspace metrics), + # rather than reducing it to the CSV's scalar columns. + "methods": method_results, "hdt_source": str(hdt_result.get("source") or "not_used"), "gzip_raw_rdf": { "output_gz_path": gzip_result.get("output_path", ""), @@ -3154,9 +3323,9 @@ def timing_payload(result: dict): }, } - metrics_json_dir = metrics_dir / "compression_metrics" / safe_name + metrics_json_dir = metrics_dir / "stages" / "compression" metrics_json_dir.mkdir(parents=True, exist_ok=True) - metrics_json = metrics_json_dir / f"{run_id}.json" + metrics_json = metrics_json_dir / f"{safe_name}.json" metrics_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") @@ -3171,11 +3340,12 @@ def write_raw_compression_metrics_artifact( selected_methods: list[str], method_results: dict[str, dict], index_warnings: list[dict] | None = None, + auxiliary_stages: dict[str, dict] | None = None, ): - """Persist per-RDF-file compression metrics under `raw_metrics/`.""" + """Persist the operation-level compression detail for one RDF source.""" safe_output = safe_metrics_name(output_name) safe_rdf = safe_metrics_name(rdf_name) - raw_json_dir = metrics_dir / "raw_metrics" / "compression_metrics" / safe_output / safe_rdf + raw_json_dir = metrics_dir / "stages" / "compression_operations" / safe_output raw_json_dir.mkdir(parents=True, exist_ok=True) payload = { @@ -3186,6 +3356,7 @@ def write_raw_compression_metrics_artifact( "source_rdf_path": str(source_rdf_path), "compression_methods": ",".join(selected_methods) if selected_methods else "none", "index_warnings": list(index_warnings or []), + "auxiliary_stages": dict(auxiliary_stages or {}), "methods": {}, } @@ -3204,10 +3375,37 @@ def write_raw_compression_metrics_artifact( "validation": result.get("validation") or details.get("validation"), } - raw_json = raw_json_dir / f"{run_id}.json" + raw_json = raw_json_dir / f"{safe_rdf}.json" raw_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") +def write_partitioned_container_stage_report( + *, + metrics_dir: Path, + output_name: str, + source_rdf_path: Path, + payload: dict, +): + """Preserve the detailed stage handoff from the ephemeral Docker volume. + + The partitioned runner records every chunk build, merge, validation, disk + watermark, and GNU-time resource measurement. Its workspace is deleted at + the end of the operation, so this report is the durable location for those + deeper container metrics on both success and failure. + """ + report_dir = metrics_dir / "stages" / "partitioned" + ensure_dir(report_dir) + report = { + "runtime_environment": "docker-volume", + "source_rdf_path": str(source_rdf_path), + "output_name": output_name, + "container_result": payload, + } + report_path = report_dir / f"{safe_metrics_name(output_name)}.json" + report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + return report_path + + def validate_mode_dirs(paths): """Validate that expected directory arguments are not file paths.""" for p in paths: @@ -3244,9 +3442,9 @@ def write_tsv_metrics_artifacts( output_paths: list[Path], output_size_bytes: int, ): - """Persist raw TSV-step metrics under `raw_metrics/`.""" + """Persist one TSV container stage report in the canonical stage tree.""" safe_prefix = safe_metrics_name(prefix) - json_dir = metrics_dir / "raw_metrics" / "tsv_metrics" / safe_prefix + json_dir = metrics_dir / "stages" / "tsv" json_dir.mkdir(parents=True, exist_ok=True) payload = { "run_id": run_id, @@ -3265,7 +3463,7 @@ def write_tsv_metrics_artifacts( "output_size_bytes": int(output_size_bytes), }, } - (json_dir / f"{run_id}.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + (json_dir / f"{safe_prefix}.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") def run_tsv_conversion_with_metrics( @@ -3281,10 +3479,10 @@ def run_tsv_conversion_with_metrics( ): """Run VCF->TSV conversion and collect per-input timing/resource metrics.""" safe_prefix = safe_metrics_name(prefix) - raw_time_dir = metrics_dir / "raw_metrics" / "tsv_time" / safe_prefix + raw_time_dir = metrics_dir / "timings" / "tsv" raw_time_dir.mkdir(parents=True, exist_ok=True) - time_log_host = raw_time_dir / f"{run_id}.txt" - time_log_container = f"/data/metrics/raw_metrics/tsv_time/{safe_prefix}/{run_id}.txt" + time_log_host = raw_time_dir / f"{safe_prefix}.txt" + time_log_container = f"/data/metrics/timings/tsv/{safe_prefix}.txt" wrapped_command = ( "set -euo pipefail; " @@ -3641,6 +3839,7 @@ def run_compression_methods_for_rdf( metrics_output_name = output_name or target_out_dir.name safe_output_name = safe_metrics_name(metrics_output_name) safe_rdf_name = safe_metrics_name(rdf_path.name) + auxiliary_stage_results: dict[str, dict] = {} def run_container_command( *, @@ -3692,17 +3891,20 @@ def run_container_command( } if record_method: method_results[method] = result - if record_method and metrics_dir is not None and run_id is not None and timing_host.exists(): + else: + # Validation/index checks consume container resources too. Retain + # their measurements in the operation report rather than dropping + # them because they do not produce a final representation. + auxiliary_stage_results[method] = result + if metrics_dir is not None and timing_host.exists(): raw_time_dir = ( metrics_dir - / "raw_metrics" - / "compression_time" + / "timings" + / "compression" / safe_output_name - / safe_rdf_name - / method ) raw_time_dir.mkdir(parents=True, exist_ok=True) - raw_time_path = raw_time_dir / f"{run_id}.txt" + raw_time_path = raw_time_dir / f"{safe_metrics_name(method)}.txt" try: shutil.copyfile(timing_host, raw_time_path) except OSError: @@ -3796,6 +3998,9 @@ def perform_validation(*, skip_index_check: bool) -> tuple[bool, dict]: ), } report_path.unlink(missing_ok=True) + execution = auxiliary_stage_results.get(f"{method}-validation") + if execution is not None: + report["execution"] = execution return ( bool(report.get("valid")) and bool(report.get("count_match")), report, @@ -4163,6 +4368,7 @@ def mark_cottas_method_unavailable(method: str): selected_methods=methods, method_results=method_results, index_warnings=index_warnings, + auxiliary_stages=auxiliary_stage_results, ) return True, method_results @@ -4334,6 +4540,19 @@ def run_containerized_partitioned_representation_methods( chunk["path"] = Path(str(chunk["path"])).name details["workspace"] = "docker-volume" details["workspace_cleanup"] = "removed" + if metrics_dir is not None: + try: + write_partitioned_container_stage_report( + metrics_dir=metrics_dir, + output_name=output_name, + source_rdf_path=source_rdf_path, + payload=payload, + ) + except OSError as exc: + eprint( + "Warning: failed to preserve detailed partitioned container metrics: " + f"{exc}" + ) missing_methods = set(methods) - set(method_results) if missing_methods and payload is not None and int(payload.get("exit_code", 1)) == 0: @@ -4599,7 +4818,7 @@ def fail_current(stage: str, message: str): sample_calls_tsv = tsv_dir / f"{prefix}.sample_calls.tsv" sample_format_tsv = tsv_dir / f"{prefix}.sample_format_values.tsv" try: - if sample_workflow.helper_strategy == "expanded": + if sample_workflow.helper_strategy == SAMPLE_HELPER_STRATEGY_MATERIALIZED: build_sample_support_tsvs( records_tsv=triplet["records"], sample_calls_tsv=sample_calls_tsv, @@ -4800,7 +5019,7 @@ def fail_current(stage: str, message: str): continue if triples_produced is not None: triples_produced += int(sample_stats["triples"]) - if sample_workflow.representation == "dense": + if sample_workflow.representation == "expanded": print( " * Sample calls streamed: " f"{sample_stats['sample_calls']:,}; FORMAT values: " @@ -5322,6 +5541,31 @@ def run_compress_mode( return 1 target_out_dir = out_dir / input_stem + source_size_bytes = int(file_size_bytes(rdf_path) or 0) + try: + write_compression_metrics_artifacts( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + source_rdf_path=rdf_path, + combined_size_bytes=source_size_bytes, + selected_methods=methods, + method_results=method_results, + ) + update_metrics_csv_with_compression( + metrics_csv=metrics_dir / "metrics.csv", + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + output_dir=target_out_dir, + combined_size_bytes=source_size_bytes, + selected_methods=methods, + method_results=method_results, + ) + except OSError as exc: + eprint(f"Error: unable to write compression metrics: {exc}") + return 1 hdt_path = target_out_dir / f"{input_stem}.hdt" print_nt_hdt_summary( output_root=target_out_dir, @@ -5378,6 +5622,8 @@ def run_index_mode( metrics_dir: Path, image_ref: str, wrapper_log_path: Path, + run_id: str | None = None, + timestamp: str | None = None, ): """Generate or regenerate the query index for one existing artifact. @@ -5408,21 +5654,38 @@ def run_index_mode( f'"$PYTHON_BIN" {shlex.quote(COTTAS_TOOL_CONTAINER)} reindex ' f"{shlex.quote(source_container)} spo" ) + safe_input = safe_metrics_name(index_path.name) + timing_host = metrics_dir / "timings" / "index" / f"{safe_input}.txt" + timing_host.parent.mkdir(parents=True, exist_ok=True) + timing_container = f"/data/metrics/timings/index/{safe_input}.txt" + timed_command = ( + "set -euo pipefail; " + f"rm -f {shlex.quote(timing_container)}; " + 'if [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; then ' + f"/usr/bin/time -v -o {shlex.quote(timing_container)} -- bash -lc {shlex.quote(command)}; " + "else " + f"{{ time -p bash -lc {shlex.quote(command)}; }} > {shlex.quote(timing_container)} 2>&1; " + "fi" + ) + input_size_bytes = int(file_size_bytes(index_path) or 0) cmd = [ *docker_run_base(), *docker_hdt_index_env_args(), *docker_cottas_merge_env_args(), "-v", f"{str(index_path.parent)}:/data/{mount_name}", + "-v", + f"{str(metrics_dir.resolve())}:/data/metrics", image_ref, "bash", "-lc", - command, + timed_command, ] started = time.perf_counter() exit_code = run(cmd) elapsed = time.perf_counter() - started + timing = parse_time_log_metrics(timing_host) index_path_after = ( find_hdt_index_sidecar(index_path) if index_format == "hdt" @@ -5432,12 +5695,21 @@ def run_index_mode( final_code = int(exit_code) if int(exit_code) != 0 else (0 if index_ready else 1) index_was_present = existing_index_path is not None or index_format == "cottas" payload = { + "run_id": run_id, + "timestamp": timestamp, "index_format": index_format, "input_path": str(index_path), + "input_size_bytes": input_size_bytes, "index_path": str(index_path_after) if index_path_after else "", "index_location": "sidecar" if index_format == "hdt" else "embedded", "exit_code": final_code, "wall_seconds": elapsed, + "timing": { + "wall_seconds": elapsed, + "user_seconds": timing.get("user_seconds"), + "sys_seconds": timing.get("sys_seconds"), + "max_rss_kb": timing.get("max_rss_kb"), + }, "index_status": ( "regenerated" if index_was_present else "generated" ) if index_ready else "failed", @@ -5448,20 +5720,12 @@ def run_index_mode( payload["hdt_path"] = str(index_path) else: payload["cottas_path"] = str(index_path) - metrics_path = metrics_dir / "index_metrics.json" + metrics_path = metrics_dir / "stages" / "index" / f"{index_format}-{safe_input}.json" + metrics_path.parent.mkdir(parents=True, exist_ok=True) metrics_path.write_text( json.dumps(payload, indent=2) + "\n", encoding="utf-8", ) - # Keep a format-specific metrics filename for discovery/compatibility, - # while the generic filename is used for both supported formats. - legacy_metrics_path = metrics_dir / f"{index_format}_index_metrics.json" - if legacy_metrics_path != metrics_path: - legacy_metrics_path.write_text( - json.dumps(payload, indent=2) + "\n", - encoding="utf-8", - ) - if final_code != 0: eprint(f"Error: {format_label} index regeneration failed. See log: {wrapper_log_path}") return 1 @@ -5492,6 +5756,9 @@ def run_decompress_mode( *, compressed_path: Path, decompressed_out: Path, + metrics_dir: Path, + run_id: str, + timestamp: str, image_ref: str, wrapper_log_path: Path, ): @@ -5562,22 +5829,165 @@ def run_decompress_mode( f"{shlex.quote(cottas_input)} {shlex.quote(output_container)}" ) + safe_input = safe_metrics_name(compressed_path.name) + timing_host = metrics_dir / "timings" / "decompression" / f"{safe_input}.txt" + timing_host.parent.mkdir(parents=True, exist_ok=True) + timing_container = f"/data/metrics/timings/decompression/{safe_input}.txt" + timed_command = ( + "set -euo pipefail; " + f"rm -f {shlex.quote(timing_container)}; " + 'if [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; then ' + f"/usr/bin/time -v -o {shlex.quote(timing_container)} -- bash -lc {shlex.quote(command)}; " + "else " + f"{{ time -p bash -lc {shlex.quote(command)}; }} > {shlex.quote(timing_container)} 2>&1; " + "fi" + ) + input_size_bytes = int(file_size_bytes(compressed_path) or 0) cmd = [ *docker_run_base(), "-v", f"{str(compressed_path.parent)}:/data/in:ro", "-v", f"{str(decompressed_out.parent)}:/data/out", + "-v", + f"{str(metrics_dir.resolve())}:/data/metrics", image_ref, "bash", "-lc", - command, + timed_command, ] - if run(cmd) != 0: + started = time.perf_counter() + exit_code = run(cmd) + elapsed = time.perf_counter() - started + timing = parse_time_log_metrics(timing_host) + stage_payload = { + "run_id": run_id, + "timestamp": timestamp, + "format": fmt, + "input_path": str(compressed_path), + "input_size_bytes": input_size_bytes, + "output_path": str(decompressed_out), + "output_size_bytes": int(file_size_bytes(decompressed_out) or 0), + # Decompression is deliberately a single pass. Counting N-Triples + # here would reread a cohort-scale output solely for observability. + "output_triples": None, + "exit_code": int(exit_code), + "timing": { + "wall_seconds": elapsed, + "user_seconds": timing.get("user_seconds"), + "sys_seconds": timing.get("sys_seconds"), + "max_rss_kb": timing.get("max_rss_kb"), + }, + } + report_path = metrics_dir / "stages" / "decompression" / f"{safe_input}.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(stage_payload, indent=2) + "\n", encoding="utf-8") + if exit_code != 0: eprint(f"Error: decompression failed. See log: {wrapper_log_path}") return 1 print(f"Done. Decompressed file: {decompressed_out}") + print(f"Decompression metrics: {report_path}") + return 0 + + +def run_validation_mode( + *, + vcf_path: Path, + rdf_gzip_path: Path, + representation: str, + validation_id: str, + results_dir: Path, + metrics_dir: Path, + run_id: str, + timestamp: str, + image_ref: str, + filter_oracle: str, + wrapper_log_path: Path, +): + """Run VCF/RDF semantic queries with ephemeral in-container RDF expansion. + + The only mounted RDF source is the input ``.nt.gz`` file. The validation + runner inflates it under the container's ``/work`` temporary filesystem and + removes that source before the container exits; no raw N-Triples are + materialized in the user-selected output directory. + """ + # An empty directory may be left behind if Docker itself fails before the + # runner starts. Reuse only that empty shell; never overwrite reports. + results_dir.mkdir(parents=True, exist_ok=True) + cmd = [ + *docker_run_base(), + "--init", + "-v", + f"{str(vcf_path.parent)}:/data/vcf:ro", + "-v", + f"{str(rdf_gzip_path.parent)}:/data/rdf:ro", + "-v", + f"{str(results_dir)}:/data/validation", + image_ref, + "/opt/pycottas-venv/bin/python", + "/opt/vcf-rdfizer/validation/validation_runner.py", + "--vcf", + f"/data/vcf/{vcf_path.name}", + "--rdf-gz", + f"/data/rdf/{rdf_gzip_path.name}", + "--representation", + representation, + "--results-dir", + "/data/validation", + "--dataset-id", + validation_id, + "--filter-oracle", + filter_oracle, + "--scratch-dir", + "/work", + ] + started = time.perf_counter() + exit_code = run(cmd) + elapsed = time.perf_counter() - started + summary_path = results_dir / "summary.json" + summary = None + if summary_path.is_file(): + try: + summary = json.loads(summary_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + summary = None + report_path = metrics_dir / "stages" / "validation" / f"{safe_metrics_name(validation_id)}.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text( + json.dumps( + { + "run_id": run_id, + "timestamp": timestamp, + "vcf_path": str(vcf_path), + "rdf_gzip_path": str(rdf_gzip_path), + "representation": representation, + "results_dir": str(results_dir), + "input_rdf_size_bytes": int(file_size_bytes(rdf_gzip_path) or 0), + "exit_code": int(exit_code), + "status": summary.get("status") if isinstance(summary, dict) else None, + "timing": {"wall_seconds": elapsed}, + "temporary_rdf": { + "decompressed_inside_container": True, + "persisted_on_host": False, + "cleanup_confirmed_by_runner": ( + summary.get("temporaryRdf", {}).get("cleanupConfirmed") + if isinstance(summary, dict) + else False + ), + }, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + if exit_code != 0: + eprint(f"Error: validation failed. See results: {results_dir}") + eprint(f"See log for details: {wrapper_log_path}") + return 1 + print(f"Validation results: {results_dir}") + print(f"Validation metrics: {report_path}") return 0 @@ -5619,6 +6029,12 @@ def main(): "--rdf-compression gzip --representations hdt --artifact-compression gzip -o ./results\n" " Decompression-only:\n" " vcf_rdfizer.py -m decompress -C ./results/out/sample/sample.nt.gz -o ./results\n" + " Semantic VCF/RDF validation (expanded graph):\n" + " vcf_rdfizer.py -m validation -i ./sample.vcf.gz --rdf ./results/sample/sample.nt.gz " + "--sample-representation expanded -o ./validation-results\n" + " Semantic VCF/RDF validation (condensed graph):\n" + " vcf_rdfizer.py -m validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz " + "--sample-representation condensed -o ./validation-results\n" " Generate or regenerate an index for an existing HDT:\n" " vcf_rdfizer.py -m index -H ./results/sample/sample.hdt -o ./results\n" " Generate or regenerate an index for an existing COTTAS file:\n" @@ -5628,20 +6044,20 @@ def main(): parser.add_argument( "-m", "--mode", - choices=["full", "compress", "decompress", "tsv", "index"], + choices=["full", "compress", "decompress", "tsv", "index", "validation"], default="full", - help="Run mode: full pipeline, TSV benchmark, compression, decompression, or index-only regeneration", + help="Run mode: full pipeline, TSV benchmark, compression, decompression, validation, or index-only regeneration", ) parser.add_argument( "-i", "--input", default=None, - help="VCF file or directory (required for --mode full and --mode tsv)", + help="VCF file or directory (required for --mode full/tsv; file required for --mode validation)", ) parser.add_argument( "--rdf", default=None, - help="Input RDF file (.nt or .nt.gz) for --mode compress", + help="Input RDF file (.nt or .nt.gz) for --mode compress; .nt.gz required for --mode validation", ) parser.add_argument( "-C", @@ -5675,11 +6091,11 @@ def main(): parser.add_argument( "--sample-representation", choices=sorted(SAMPLE_REPRESENTATION_CHOICES), - default="dense", + default="expanded", help=( - "Genotype representation for full mode: dense emits one SampleCall and " + "Genotype representation for full/validation mode: expanded emits one SampleCall and " "FORMAT value resource per sample; condensed emits a shared SampleSet and " - "one sample-ordered value vector per FORMAT key (default: dense)" + "one sample-ordered value vector per FORMAT key (default: expanded)" ), ) parser.add_argument( @@ -5798,6 +6214,17 @@ def main(): action="store_true", help="Disable terminal compression/conversion progress updates", ) + parser.add_argument( + "--validation-id", + default=None, + help="Validation result identifier (default: source VCF basename)", + ) + parser.add_argument( + "--filter-oracle", + choices=("auto", "bcftools", "cyvcf2"), + default="auto", + help="FILTER oracle for validation mode (default: auto)", + ) rdf_output_group = parser.add_mutually_exclusive_group() rdf_output_group.add_argument( "-R", @@ -5827,7 +6254,7 @@ def main(): metrics_root = out_root / "run_metrics" run_id = datetime.now().strftime("%Y%m%dT%H%M%S") timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - metrics_dir = metrics_root / run_id + metrics_dir = None mode = args.mode spark_partitions = None chunk_target_bytes = DEFAULT_CHUNK_TARGET_BYTES @@ -5941,6 +6368,31 @@ def main(): expected_prefixes, ) = resolve_input_snapshot(input_path) validate_mode_dirs([out_root, out_dir, tsv_dir, metrics_root]) + elif mode == "validation": + if args.spark_partitions is not None: + raise ValueError("--spark-partitions is only valid in --mode full") + if args.input is None: + raise ValueError("--input is required in --mode validation") + validation_vcf_path = Path(args.input).expanduser().resolve() + if not validation_vcf_path.is_file() or not is_vcf_file(validation_vcf_path): + raise ValueError("Validation input must be an existing .vcf or .vcf.gz file") + if not args.rdf: + raise ValueError("--rdf is required in --mode validation") + validation_rdf_gzip_path = Path(args.rdf).expanduser().resolve() + if not validation_rdf_gzip_path.is_file() or not validation_rdf_gzip_path.name.endswith(".nt.gz"): + raise ValueError("Validation RDF input must be an existing .nt.gz file") + validation_id = args.validation_id or vcf_output_prefix(validation_vcf_path) + if not re.fullmatch(r"[A-Za-z0-9._-]+", validation_id): + raise ValueError("--validation-id may contain only letters, digits, dot, underscore, and hyphen") + validation_results_dir = out_dir / "validation" / validation_id + if validation_results_dir.exists() and ( + not validation_results_dir.is_dir() or any(validation_results_dir.iterdir()) + ): + raise ValueError( + f"Refusing to overwrite existing validation results: {validation_results_dir}. " + "Choose --validation-id or --out with a new destination." + ) + validate_mode_dirs([out_root, out_dir, metrics_root]) elif mode == "compress": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -6054,7 +6506,65 @@ def main(): eprint(f"Error: {exc}") return 2 + if mode in {"full", "tsv"}: + metrics_source_paths = [] + for container_input in container_inputs: + try: + relative_input = Path(container_input).relative_to("/data/in") + except ValueError: + relative_input = Path(container_input).name + metrics_source_paths.append((input_mount_dir / relative_input).resolve()) + metrics_source_root = input_path + elif mode == "compress": + metrics_source_paths = [rdf_path] + metrics_source_root = rdf_path + elif mode == "validation": + metrics_source_paths = [validation_vcf_path, validation_rdf_gzip_path] + metrics_source_root = validation_vcf_path + elif mode == "index": + metrics_source_paths = [index_path] + metrics_source_root = index_path + else: + metrics_source_paths = [compressed_path] + metrics_source_root = compressed_path + + source_label = metrics_run_label(metrics_source_paths, metrics_source_root) + metrics_dir = metrics_run_directory(metrics_root, source_label, run_id) + manifest_options = { + "requested_image": args.image, + "requested_image_version": args.image_version, + "sample_representation": args.sample_representation if mode == "full" else None, + "rdf_storage_mode": args.rdf_storage_mode if mode == "full" else None, + "compression_methods": ( + full_methods if mode == "full" else methods if mode == "compress" else [] + ), + "hdt_strategy": args.hdt_strategy if mode in {"full", "compress"} else None, + "chunk_target_bytes": chunk_target_bytes if mode in {"full", "compress"} else None, + "chunk_min_bytes": chunk_min_bytes if mode in {"full", "compress"} else None, + "chunk_max_bytes": chunk_max_bytes if mode in {"full", "compress"} else None, + "spark_partitions": spark_partitions if mode == "full" else None, + "index_format": index_format if mode == "index" else None, + "decompression_format": fmt if mode == "decompress" else None, + "validation_representation": args.sample_representation if mode == "validation" else None, + "validation_rdf_gzip": str(validation_rdf_gzip_path) if mode == "validation" else None, + } + try: + manifest_path = write_run_manifest( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + mode=mode, + source_label=source_label, + source_paths=metrics_source_paths, + out_root=out_root, + options=manifest_options, + ) + except OSError as exc: + eprint(f"Error: unable to create metrics directory '{metrics_dir}': {exc}") + return 1 + print(f"{step1_label}: Validating inputs {success_symbol()}") + print(f" Metrics: {metrics_dir}") if mode == "full" and args.estimate_size: # Optional coarse sizing estimate for disk-risk visibility. @@ -6076,8 +6586,8 @@ def main(): "You may run out of space." ) - wrapper_log_path = metrics_dir / "wrapper_logs" / f"{run_id}.log" - progress_log_path = metrics_dir / "progress.log" + wrapper_log_path = metrics_dir / "logs" / "wrapper.log" + progress_log_path = metrics_dir / "logs" / "progress.log" execution_started = time.perf_counter() global _COMMAND_LOGGER _COMMAND_LOGGER = CommandLogger(wrapper_log_path) @@ -6103,6 +6613,11 @@ def execute_mode(): try: image_ref, version_requested = resolve_image_ref(args.image, args.image_version) resolved_image_ref = image_ref + update_run_manifest( + metrics_dir, + resolved_image=image_ref, + image_version_requested=version_requested, + ) except ValueError as exc: run_tracker.mark(f"Image resolution failed: {exc}") eprint(f"Error: {exc}") @@ -6148,6 +6663,17 @@ def execute_mode(): (out_write_target, True), (metrics_write_target, True), ] + elif mode == "validation": + validation_write_target = ( + validation_results_dir.parent + if validation_results_dir.parent.exists() + else validation_results_dir.parent.parent + ) + metrics_write_target = metrics_dir if metrics_dir.exists() else metrics_dir.parent + writable_targets = [ + (validation_write_target, True), + (metrics_write_target, True), + ] elif mode == "index": metrics_write_target = metrics_dir if metrics_dir.exists() else metrics_dir.parent writable_targets = [ @@ -6242,6 +6768,20 @@ def execute_mode(): chunk_max_bytes=chunk_max_bytes, wrapper_log_path=wrapper_log_path, ) + if mode == "validation": + return run_validation_mode( + vcf_path=validation_vcf_path, + rdf_gzip_path=validation_rdf_gzip_path, + representation=args.sample_representation, + validation_id=validation_id, + results_dir=validation_results_dir, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + image_ref=image_ref, + filter_oracle=args.filter_oracle, + wrapper_log_path=wrapper_log_path, + ) if mode == "index": return run_index_mode( index_path=index_path, @@ -6249,11 +6789,16 @@ def execute_mode(): metrics_dir=metrics_dir, image_ref=image_ref, wrapper_log_path=wrapper_log_path, + run_id=run_id, + timestamp=timestamp, ) # Decompression-only mode. return run_decompress_mode( compressed_path=compressed_path, decompressed_out=decompressed_out, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, image_ref=image_ref, wrapper_log_path=wrapper_log_path, ) @@ -6317,6 +6862,19 @@ def _interrupt_handler(_signum, _frame): if run_tracker is not None: run_tracker.mark(f"Run finished (exit_code={result_code})") run_tracker.close() + try: + summary_path = write_run_summary( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + mode=mode, + exit_code=result_code, + elapsed_seconds=elapsed_seconds, + total_triples=total_triples, + ) + print(f"Metrics summary: {summary_path}") + except OSError as exc: + eprint(f"Warning: failed to write metrics summary: {exc}") return result_code diff --git a/vcf_rdfizer_data/rules/default_rules.ttl b/vcf_rdfizer_data/rules/default_rules.ttl index 7a0ee2c..7332230 100644 --- a/vcf_rdfizer_data/rules/default_rules.ttl +++ b/vcf_rdfizer_data/rules/default_rules.ttl @@ -14,7 +14,7 @@ # - /data/tsv/.sample_calls.tsv (header-only compatibility source) # - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. -# The wrapper selects exactly one direct sample emitter: dense SampleCall / +# The wrapper selects exactly one direct sample emitter: expanded SampleCall / # FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / # FormatValueVector triples. The helper tables stay header-only in either mode. # From 6cbbe6d1f1facad7557a62ffc30bd6248598bc3a Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Fri, 4 Sep 2026 13:45:22 +0200 Subject: [PATCH 16/19] validation mode improvements and addition of quiet run mode --- README.md | 50 +++- changelog.md | 21 ++ docs/validation.md | 63 +++- src/validation/validation_runner.py | 187 ++++++++++-- test/test_vcf_rdfizer_unit.py | 306 ++++++++++++++++++- vcf_rdfizer.py | 448 ++++++++++++++++++++++------ 6 files changed, 939 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 5baeb9f..bad02e3 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,11 @@ The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#] When VCF-RDFizer is connected to an interactive terminal, it shows a lightweight Rich spinner/progress display. With redirected output or without Rich installed, it instead prints compact status lines; CI output remains -quiet. Either display can be disabled explicitly with `--no-progress`. +quiet. Validation uses the same display and reports each preflight/SPARQL +query as it starts and completes. Use `--quiet` to suppress these terminal +progress displays (and validation's per-query/summary chatter) while keeping +the progress sidecar, command log, and metrics collection active. Use +`--no-progress` when the sidecar and terminal progress should both be disabled. RMLStreamer progress reports the bytes and output parts already written; partitioned HDT/COTTAS runs report source triples, chunks, and the currently active merge/index stage. These updates are best-effort and do not scan RDF @@ -73,11 +77,11 @@ inside this directory. ## Modes -- `full`: VCF -> TSV -> RDF -> compression +- `full`: VCF -> TSV -> RDF -> compression (and optional semantic validation with `--validate`) - `tsv`: VCF -> TSV only (benchmarking) - `compress`: compress an existing `.nt` or `.nt.gz` - `decompress`: decompress `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` -- `validation`: compare a source VCF with its `.nt.gz` RDF using six semantic SPARQL queries +- `validation`: compare a source VCF with its `.nt` or `.nt.gz` RDF using six semantic SPARQL queries - `index`: only generate or regenerate the query index for an existing `.hdt` or `.cottas` In `full` mode with multiple VCF inputs, failures are isolated per input: @@ -94,18 +98,25 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: - `--hdt-strategy {auto,partitioned,single}` HDT generation policy - `--chunk-target-bytes`, `--chunk-min-bytes`, `--chunk-max-bytes` shared record-safe chunk sizing - `--sample-representation {expanded,condensed}` genotype graph shape (`expanded` by default) +- `--validate` (or `--run-validation`) run semantic VCF/RDF validation for every input in a full run +- `--filter-oracle {auto,bcftools,cyvcf2}` FILTER oracle used by validation (`auto` by default) +- `--quiet` suppress terminal progress displays while retaining sidecar/log/metrics tracking +- `--no-progress` disable terminal progress and progress sidecar creation - `-I, --image` Docker image repo (default `ecrum19/vcf-rdfizer`) - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build - `-B, --no-build` fail if image not found -- `--no-progress` disable terminal progress updates - `-h, --help` show full usage ## Validation Mode -Validate one source VCF against the `.nt.gz` aggregate from the same -conversion. The aggregate is decompressed, parsed, and queried only inside the -Docker container; raw N-Triples are removed before the container exits. +Validate one source VCF against the `.nt` or `.nt.gz` aggregate from the same +conversion. Gzip input is decompressed, parsed, and queried only inside the +Docker container; any temporary raw N-Triples are removed before the container +exits. To run +the same checks as part of a full conversion, add `--validate`; validation then +runs once per input after RDF/compression and accepts either the generated +`.nt` or `.nt.gz` aggregate. ```bash vcf-rdfizer --mode validation \ @@ -116,9 +127,12 @@ vcf-rdfizer --mode validation \ ``` Use `expanded` for the default graph shape and `condensed` for the vector-based -cohort graph. Reports are written to `/validation//`. See -[Semantic VCF/RDF validation](docs/validation.md) for query definitions, -preflight checks, result statuses, and cleanup evidence. +cohort graph. Reports from standalone and full-run validation use the canonical +metrics tree: +`run_metrics/__/reports/validation//`, with a +stage summary at `stages/validation/.json`. See [Semantic VCF/RDF +validation](docs/validation.md) for query definitions, preflight checks, result +statuses, and cleanup evidence. ## Compression Plan @@ -164,6 +178,11 @@ host filesystem. - `--sample-representation {expanded,condensed}` sample genotype representation - `expanded` (default): one `SampleCall` per record/sample and one `FormatFieldValue` per FORMAT key - `condensed`: reusable file-level samples plus one ordered value vector per record/FORMAT key +- `--validate` run semantic VCF/RDF validation once per input after RDF/compression; + detailed results are stored beneath `run_metrics/.../reports/validation/` +- `--filter-oracle {auto,bcftools,cyvcf2}` FILTER oracle for `--validate` +- `--quiet` suppress terminal progress and validation query chatter while retaining logs/metrics +- `--no-progress` disable progress sidecars and terminal progress displays - `--rdf-storage-mode {plain,space-optimized}` required full-mode aggregate storage policy - `plain`: merge RMLStreamer parts into one uncompressed `.nt` - `space-optimized`: gzip each part into one `.nt.gz` aggregate and delete the source part immediately @@ -574,13 +593,17 @@ Within each run directory, VCF-RDFizer writes: Docker container - `stages/tsv/`, `stages/conversion/`, `stages/compression/`, `stages/compression_operations/`, `stages/decompression/`, and - `stages/index/`: structured stage results. `compression_operations/` + `stages/index/`, and `stages/validation/`: structured stage results. + `compression_operations/` preserves the underlying per-RDF operation and validation reports, while `compression/` provides the final output-level summary. - `stages/partitioned/`: the full result handoff from the temporary partitioned-compression container, including every chunk build, merge, validation, workspace free-space sample, exit code, CPU time, and peak RSS - `reports/index_warnings.json` and `reports/failed_inputs.csv` when applicable +- `reports/validation//`: detailed semantic-validation reports + (`summary.json`, query results, preflight checks, and cleanup evidence) when + validation is requested Compression metrics now include per-method: @@ -618,6 +641,11 @@ Metrics may use internal stage names such as `hdt_gzip` and `cottas_brotli`. These correspond to the public combination of `--representations` and `--artifact-compression`; users do not need to pass those compound names. +When `--validate` is used in full mode, the same `metrics.csv` row also carries +`validation_status`, `validation_exit_code`, validation wall/CPU/RSS timings, +the detailed report path, and the RDF path that was validated. The validation +stage JSON retains the full status and temporary-RDF cleanup metadata. + ## Chunked Compression Full mode always uses one of the two aggregate storage modes. Both modes create diff --git a/changelog.md b/changelog.md index 2e8c9dd..b73ff45 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,26 @@ # Changelog +## 2026-09-04 — Validation progress and quiet mode + +- Integrated semantic validation with the existing JSONL progress sidecar and + host `ProgressSession`; validation now reports its preflight/core query + progress using the same terminal UI as conversion and compression. +- Added `--quiet` to suppress terminal progress displays and validation query + chatter while retaining progress bookkeeping, command logs, and metrics. +- Kept `--no-progress` as the stronger opt-out that disables sidecar creation + and progress rendering entirely. + +## 2026-09-04 — Full-run semantic validation + +- Added `--validate`/`--run-validation` to run the semantic VCF/RDF validator + once per input as the final stage of a `--mode full` run. +- Full-mode validation accepts the generated plain `.nt` or gzip `.nt.gz` + aggregate, and retains the validator's container-local temporary-file + guarantees. +- Validation timing, status, exit code, input RDF, and report paths are now + included in the run's `metrics.csv`, compression JSON, stage reports, and + recursive `summary.json` report index. + ## 2026-09-04 — Expanded sample representation naming - Renamed the default per-sample genotype graph strategy to `expanded`, while diff --git a/docs/validation.md b/docs/validation.md index a4c4384..9786b70 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -1,16 +1,20 @@ # Semantic VCF/RDF validation `vcf-rdfizer --mode validation` checks that a converted RDF graph reproduces -six deterministic VCF summaries. It computes one result from the source VCF +six deterministic VCF summaries. The same validator can be added to a full run +with `--validate`; in that case it runs once for each input after RDF creation +and compression. It computes one result from the source VCF using `cyvcf2` (and `bcftools` for exact FILTER strings when available), runs the equivalent SPARQL queries with Comunica, then compares canonical integer results exactly. -The mode consumes a single N-Triples gzip aggregate (`.nt.gz`). It mounts that -file read-only, expands it under `/work` **inside the Docker container**, uses -the temporary `.nt` for Raptor syntax validation and Comunica queries, then -removes it before the container exits. No decompressed RDF is written beneath -`--out`; only reports are retained. +Standalone validation consumes a single N-Triples aggregate (`.nt` or +`.nt.gz`). It mounts the source read-only; gzip input is expanded under +`/work` **inside the Docker container**, while plain `.nt` input is read in +place. The validator uses the resulting stream for Raptor syntax validation +and Comunica queries, and removes any temporary expansion before the container +exits. No decompressed RDF is written beneath `--out`; only reports are +retained. ## Run it @@ -32,16 +36,40 @@ vcf-rdfizer --mode validation \ --out ./validation-results ``` -The input VCF and `.nt.gz` must originate from the same conversion. `--rdf` -must name an existing `.nt.gz` file; validation deliberately does not accept an -uncompressed `.nt`, HDT, or COTTAS artifact. Use full mode with -`--rdf-storage-mode space-optimized` to produce the required gzip aggregate. +To include validation in the conversion itself, use the full mode's opt-in +stage. It runs after the aggregate and selected compression artifacts are +available, once per VCF input: + +```bash +vcf-rdfizer --mode full \ + --input ./cohort.vcf.gz \ + --sample-representation condensed \ + --rdf-storage-mode space-optimized \ + --rdf-compression none \ + --representations hdt \ + --validate \ + --out ./results +``` + +For standalone validation, the input VCF and `.nt`/`.nt.gz` must originate from +the same conversion. `--rdf` must name an existing `.nt` or `.nt.gz` file; +standalone validation does not accept HDT or COTTAS artifacts. Use full mode +with `--rdf-storage-mode space-optimized` for a gzip aggregate, or `plain` for +an uncompressed aggregate. `--validation-id NAME` changes the report directory name. The default is the source VCF basename without `.vcf` or `.vcf.gz`. Existing result directories are never overwritten. `--filter-oracle {auto,bcftools,cyvcf2}` controls the FILTER-field oracle; `auto` uses `bcftools` when it is available in the image. +Validation progress uses the same JSONL sidecar protocol as conversion and +partitioned compression. It emits a `validation` task with a total of eleven +preflight/core queries, then records each query start and completion under the +run's temporary `.progress/` area while the host displays it through the +normal Rich/plain progress session. `--quiet` suppresses that terminal display +and the validator's per-query/summary stdout, but still writes command logs, +stage reports, and metrics. `--no-progress` disables sidecar creation as well. + ## What is tested The common record-level queries are used for both graph shapes: @@ -69,12 +97,18 @@ back into per-sample value resources. ## Results and cleanup evidence -Results are written to: +Results are written to the run's canonical metrics tree: ```text -/validation// +/run_metrics/__/reports/validation// ``` +Standalone validation consumes either `.nt` or `.nt.gz`. Full-mode validation +uses the aggregate produced by that run and can read either `.nt` (plain +storage) or `.nt.gz` (space-optimized storage). In both cases the detailed +results live beneath `reports/validation/`, so they are indexed by the same +`summary.json` used for conversion and compression metrics. + Important files include `summary.json`, `manifest.json`, `parser.json`, `rdf-validation.json`, `preflight.json`, `sparql.json`, and `comparison.json`. Raw Comunica JSON, stderr, and query resource logs are in `raw/`; normalized @@ -82,8 +116,9 @@ results are in `normalized/`. The parent VCF-RDFizer run metrics include `run_metrics/__/stages/validation/.json`. -Both that report and `summary.json` record that the `.nt.gz` was decompressed -inside the container and that no raw RDF was retained on the host. +That stage report and the detailed `reports/validation/.../summary.json` record +whether a gzip aggregate was decompressed inside the container and that no +validation scratch RDF was retained on the host. `PASS` means every required query and invariant matched. `MISMATCH` means that both paths ran but differ. `BLOCKED_BY_PREFLIGHT` means RDF syntax or core graph diff --git a/src/validation/validation_runner.py b/src/validation/validation_runner.py index 9b82e69..4287240 100644 --- a/src/validation/validation_runner.py +++ b/src/validation/validation_runner.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -"""Validate a VCF-RDFizer .nt.gz graph against its source VCF. +"""Validate a VCF-RDFizer N-Triples graph against its source VCF. -The compressed RDF input is expanded only to a container-local temporary file. -It is parsed with Raptor, queried with Comunica, and removed in a finally-safe -temporary directory before this process exits. +When the input is compressed, it is expanded only to a container-local +temporary file. It is parsed with Raptor, queried with Comunica, and removed in +a finally-safe temporary directory before this process exits. Plain ``.nt`` +inputs are read directly and are never copied by the validator. """ from __future__ import annotations @@ -79,6 +80,56 @@ def write_json(path: Path, value: Any) -> None: path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") +class ValidationProgress: + """Best-effort JSONL progress writer shared with the host progress UI. + + Validation can spend most of its time inside a single SPARQL query. A + small event before and after each query gives the existing host-side + ``ProgressSession`` a useful total without retaining query output or RDF + data in memory. Failures to write the optional sidecar are deliberately + ignored so observability can never change validation semantics. + """ + + def __init__(self, path: Path | None, total: int): + self.path = path + self.total = total + if path is not None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.unlink(missing_ok=True) + except OSError: + pass + + def emit( + self, + phase: str, + *, + completed: int | None = None, + query_id: str | None = None, + detail: str | None = None, + ) -> None: + if self.path is None: + return + payload: dict[str, Any] = { + "stage": "validation", + "phase": phase, + "total": self.total, + "unit": "queries", + } + if completed is not None: + payload["completed"] = completed + if query_id is not None: + payload["query"] = query_id + if detail is not None: + payload["detail"] = detail + try: + with self.path.open("a", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + except OSError: + pass + + def read_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) @@ -492,13 +543,24 @@ def compare(parser: dict[str, Any], sparql: dict[str, Any]) -> dict[str, Any]: def build_manifest(args: argparse.Namespace, query_dir: Path, parser: dict[str, Any]) -> dict[str, Any]: query_paths = sorted({query_path(query_dir, query_id) for query_id in PREFLIGHT_QUERIES + CORE_QUERIES}) - return { + source_rdf = args.rdf_gz or args.rdf_nt + source_rdf_format = "nt.gz" if args.rdf_gz else "nt" + source_rdf_entry = { + "path": str(source_rdf), + "sha256": sha256_file(source_rdf), + "format": source_rdf_format, + } + manifest = { "datasetId": args.dataset_id, "representation": args.representation, "commandLine": sys.argv, "sourceVcf": {"path": str(args.vcf), "sha256": parser["sourceSha256"]}, - "sourceRdfGzip": {"path": str(args.rdf_gz), "sha256": sha256_file(args.rdf_gz)}, - "temporaryRdf": {"decompressedInsideContainer": True, "persisted": False, "cleanupConfirmed": True}, + "sourceRdf": source_rdf_entry, + "temporaryRdf": { + "decompressedInsideContainer": bool(args.rdf_gz), + "persisted": False, + "cleanupConfirmed": True, + }, "tools": { "python": platform.python_version(), "cyvcf2": cyvcf2.__version__, "bcftools": tool_version(["bcftools", "--version"]), "node": tool_version(["node", "--version"]), @@ -507,6 +569,14 @@ def build_manifest(args: argparse.Namespace, query_dir: Path, parser: dict[str, }, "queries": {path.stem: {"path": str(path), "sha256": sha256_file(path)} for path in query_paths}, } + # Preserve the original key for consumers of standalone gzip-validation + # manifests while exposing the format-neutral ``sourceRdf`` entry. + if args.rdf_gz: + manifest["sourceRdfGzip"] = { + "path": str(args.rdf_gz), + "sha256": sha256_file(args.rdf_gz), + } + return manifest def query_path(representation_dir: Path, query_id: str) -> Path: @@ -521,18 +591,44 @@ def run_validation(args: argparse.Namespace) -> int: raw_dir.mkdir(parents=True, exist_ok=True) normalized_dir.mkdir(parents=True, exist_ok=True) query_dir = QUERY_ROOT / args.representation - missing = [name for name in PREFLIGHT_QUERIES + CORE_QUERIES if not query_path(query_dir, name).is_file()] + query_ids = PREFLIGHT_QUERIES + CORE_QUERIES + missing = [name for name in query_ids if not query_path(query_dir, name).is_file()] if missing: raise RuntimeError(f"Missing {args.representation} validation query files: {', '.join(missing)}") - + progress = ValidationProgress(getattr(args, "progress_path", None), len(query_ids)) + quiet = bool(getattr(args, "quiet", False)) + progress.emit( + "started", + completed=0, + detail=f"{args.representation} validation started", + ) summary: dict[str, Any] | None = None try: with tempfile.TemporaryDirectory(prefix="vcf-rdfizer-validation-", dir=args.scratch_dir) as scratch: - decoded = Path(scratch) / "input.nt" - with gzip.open(args.rdf_gz, "rb") as source, decoded.open("wb") as target: - shutil.copyfileobj(source, target, length=1024 * 1024) + temporary_rdf = bool(args.rdf_gz) + if temporary_rdf: + decoded = Path(scratch) / "input.nt" + progress.emit( + "progress", + completed=0, + detail="decompressing RDF inside container", + ) + with gzip.open(args.rdf_gz, "rb") as source, decoded.open("wb") as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + else: + decoded = args.rdf_nt + progress.emit( + "progress", + completed=0, + detail="parsing source VCF", + ) parser = parse_vcf(args.vcf, filter_oracle=args.filter_oracle) write_json(results_dir / "parser.json", parser) + progress.emit( + "progress", + completed=0, + detail="validating RDF syntax and cardinality", + ) rdf_validation = validate_ntriples(decoded, results_dir) write_json(results_dir / "rdf-validation.json", rdf_validation) manifest = build_manifest(args, query_dir, parser) @@ -541,9 +637,27 @@ def run_validation(args: argparse.Namespace) -> int: summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "BLOCKED_BY_PREFLIGHT", "rdfValidation": rdf_validation} return 1 executions: dict[str, dict[str, Any]] = {} - for query_id in PREFLIGHT_QUERIES + CORE_QUERIES: - print(f"[{args.dataset_id}] running {args.representation}/{query_id}", flush=True) - executions[query_id] = execute_query(query_id, decoded, query_path(query_dir, query_id), raw_dir) + for completed, query_id in enumerate(query_ids, start=1): + progress.emit( + "progress", + completed=completed - 1, + query_id=query_id, + detail=f"running {args.representation}/{query_id}", + ) + if not quiet: + print( + f"[{args.dataset_id}] running {args.representation}/{query_id}", + flush=True, + ) + executions[query_id] = execute_query( + query_id, decoded, query_path(query_dir, query_id), raw_dir + ) + progress.emit( + "progress", + completed=completed, + query_id=query_id, + detail=f"completed {args.representation}/{query_id}", + ) write_json(results_dir / "query-executions.json", executions) if any(item["status"] != "PASS" for item in executions.values()): summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "queryExecutions": executions} @@ -580,31 +694,64 @@ def run_validation(args: argparse.Namespace) -> int: finally: if summary is None: summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": "validation ended without a result"} - summary["temporaryRdf"] = {"decompressedInsideContainer": True, "persisted": False, "cleanupConfirmed": True} + summary["temporaryRdf"] = { + "decompressedInsideContainer": bool(args.rdf_gz), + "persisted": False, + "cleanupConfirmed": True, + } write_json(results_dir / "summary.json", summary) - print(json.dumps(summary, indent=2), flush=True) + status = str(summary.get("status", "EXECUTION_FAILED")) + if status == "EXECUTION_FAILED": + progress.emit("failed", detail="validation execution failed") + else: + progress.emit( + "complete", + completed=progress.total if status in {"PASS", "MISMATCH"} else 0, + detail=f"validation {status.lower()}", + ) + if not quiet: + print(json.dumps(summary, indent=2), flush=True) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--vcf", type=Path, required=True) - parser.add_argument("--rdf-gz", type=Path, required=True, help="N-Triples gzip input (.nt.gz)") + rdf_group = parser.add_mutually_exclusive_group(required=True) + rdf_group.add_argument("--rdf-gz", type=Path, help="N-Triples gzip input (.nt.gz)") + rdf_group.add_argument("--rdf-nt", type=Path, help="Uncompressed N-Triples input (.nt)") parser.add_argument("--representation", choices=("expanded", "condensed"), required=True) parser.add_argument("--results-dir", type=Path, required=True) parser.add_argument("--dataset-id", required=True) parser.add_argument("--filter-oracle", choices=("auto", "bcftools", "cyvcf2"), default="auto") parser.add_argument("--scratch-dir", type=Path, default=Path("/work")) + parser.add_argument( + "--progress-path", + type=Path, + help="optional JSONL sidecar consumed by the host progress display", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="suppress per-query and summary output on stdout", + ) args = parser.parse_args() args.vcf = args.vcf.resolve() - args.rdf_gz = args.rdf_gz.resolve() + if args.rdf_gz is not None: + args.rdf_gz = args.rdf_gz.resolve() + if args.rdf_nt is not None: + args.rdf_nt = args.rdf_nt.resolve() if not args.vcf.is_file(): parser.error(f"VCF does not exist: {args.vcf}") - if not args.rdf_gz.is_file() or not args.rdf_gz.name.endswith(".nt.gz"): + if args.rdf_gz is not None and (not args.rdf_gz.is_file() or not args.rdf_gz.name.endswith(".nt.gz")): parser.error("--rdf-gz must be an existing .nt.gz file") + if args.rdf_nt is not None and (not args.rdf_nt.is_file() or not args.rdf_nt.name.endswith(".nt")): + parser.error("--rdf-nt must be an existing .nt file") if not re.fullmatch(r"[A-Za-z0-9._-]+", args.dataset_id): parser.error("--dataset-id may contain only letters, digits, dot, underscore, and hyphen") if not args.scratch_dir.is_dir(): parser.error(f"Scratch directory does not exist: {args.scratch_dir}") + if args.progress_path is not None: + args.progress_path = args.progress_path.resolve() return args diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index d561695..67ff539 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -9,6 +9,7 @@ import subprocess import sys import tempfile +import types import unittest from contextlib import redirect_stderr, redirect_stdout from io import StringIO @@ -218,6 +219,20 @@ def prepare_inputs(base: Path): return input_dir, rules_path +def load_validation_runner_module(): + """Load the container validator without requiring cyvcf2 on the host.""" + validator_path = Path(__file__).parents[1] / "src" / "validation" / "validation_runner.py" + fake_cyvcf2 = types.ModuleType("cyvcf2") + fake_cyvcf2.VCF = object + fake_cyvcf2.__version__ = "test" + spec = importlib.util.spec_from_file_location("validation_runner_test", validator_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + with mock.patch.dict(sys.modules, {"cyvcf2": fake_cyvcf2}): + spec.loader.exec_module(module) + return module + + def mocked_triplets(): return [ { @@ -340,6 +355,151 @@ def test_plain_progress_is_shown_when_rich_or_a_tty_is_unavailable(self): finally: vcf_rdfizer._PROGRESS_ALLOWED = previous_progress_setting + def test_quiet_progress_consumes_sidecar_without_terminal_output(self): + """Quiet mode keeps event handling active but renders no progress lines.""" + previous_allowed = vcf_rdfizer._PROGRESS_ALLOWED + previous_events = vcf_rdfizer._PROGRESS_EVENTS_ALLOWED + previous_quiet = vcf_rdfizer._QUIET + try: + vcf_rdfizer._PROGRESS_ALLOWED = False + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = True + vcf_rdfizer._QUIET = True + with tempfile.TemporaryDirectory() as td, mock.patch.dict( + os.environ, + {"VCF_RDFIZER_NO_PROGRESS": "", "CI": ""}, + clear=False, + ), mock.patch.object(vcf_rdfizer, "Progress", None), mock.patch.object( + vcf_rdfizer, "Console", None + ): + progress_path = Path(td) / "validation.jsonl" + terminal = StringIO() + with redirect_stderr(terminal), vcf_rdfizer.ProgressSession( + progress_path, "Validation: cohort" + ) as session: + progress_path.write_text( + json.dumps( + { + "stage": "validation", + "phase": "started", + "completed": 0, + "total": 2, + } + ) + + "\n" + ) + session.poll_events() + + self.assertEqual(terminal.getvalue(), "") + finally: + vcf_rdfizer._PROGRESS_ALLOWED = previous_allowed + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = previous_events + vcf_rdfizer._QUIET = previous_quiet + + def test_validation_forwards_progress_sidecar_and_quiet_flag(self): + """Validation uses the shared sidecar protocol and propagates --quiet.""" + previous_allowed = vcf_rdfizer._PROGRESS_ALLOWED + previous_events = vcf_rdfizer._PROGRESS_EVENTS_ALLOWED + previous_quiet = vcf_rdfizer._QUIET + try: + vcf_rdfizer._PROGRESS_ALLOWED = False + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = True + vcf_rdfizer._QUIET = True + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text("

.\n") + metrics_dir = tmp_path / "metrics" + results_dir = metrics_dir / "reports" / "validation" / "sample" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), redirect_stdout( + StringIO() + ): + rc = vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=rdf_path, + representation="expanded", + validation_id="sample", + results_dir=results_dir, + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + wrapper_log_path=tmp_path / "wrapper.log", + ) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + command = commands[0] + self.assertIn("--progress-path", command) + self.assertIn( + "/data/metrics/.progress/validation-sample.jsonl", command + ) + self.assertIn("--quiet", command) + finally: + vcf_rdfizer._PROGRESS_ALLOWED = previous_allowed + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = previous_events + vcf_rdfizer._QUIET = previous_quiet + + def test_validation_runner_emits_query_progress_in_quiet_mode(self): + """The container validator still writes lifecycle events when stdout is quiet.""" + validator = load_validation_runner_module() + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text("

.\n") + scratch_dir = tmp_path / "scratch" + scratch_dir.mkdir() + results_dir = tmp_path / "results" + progress_path = tmp_path / ".progress" / "validation.jsonl" + args = argparse.Namespace( + results_dir=results_dir, + representation="expanded", + progress_path=progress_path, + quiet=True, + scratch_dir=scratch_dir, + rdf_gz=None, + rdf_nt=rdf_path, + vcf=vcf_path, + filter_oracle="cyvcf2", + dataset_id="sample", + ) + query_file = tmp_path / "query.rq" + query_file.write_text("SELECT * WHERE { ?s ?p ?o }\n") + parser = { + "totalRecords": 1, + "sampleCount": 0, + "gtRecordCount": 0, + } + with mock.patch.object(validator, "query_path", return_value=query_file), mock.patch.object( + validator, "parse_vcf", return_value=parser + ), mock.patch.object( + validator, "validate_ntriples", return_value={"status": "PASS"} + ), mock.patch.object(validator, "build_manifest", return_value={}), mock.patch.object( + validator, "execute_query", return_value={"status": "FAILED"} + ), redirect_stdout(StringIO()) as output: + rc = validator.run_validation(args) + + self.assertEqual(rc, 1) + self.assertEqual(output.getvalue(), "") + events = [ + json.loads(line) for line in progress_path.read_text().splitlines() + ] + self.assertEqual(events[0]["phase"], "started") + self.assertEqual(events[0]["unit"], "queries") + self.assertEqual(events[1]["phase"], "progress") + self.assertEqual(events[1]["completed"], 0) + self.assertEqual(events[-1]["phase"], "failed") + def test_validator_counts_plain_and_gzip_ntriples(self): """The Docker validator's fallback source count handles .nt and .nt.gz.""" validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" @@ -1406,6 +1566,103 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("wall_seconds_tsv", row) self.assertNotEqual(row.get("tsv_output_size_bytes", ""), "") + def test_main_full_mode_runs_validation_and_harmonizes_reports_and_metrics(self): + """--validate adds one canonical validation stage and CSV row to a full run.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + validation_calls = [] + + def fake_run(cmd, cwd=None, env=None): + if "/opt/vcf-rdfizer/vcf_as_tsv.sh" in cmd: + tsv_dir = out_dir / ".intermediate" / "tsv" + tsv_dir.mkdir(parents=True, exist_ok=True) + (tsv_dir / "sample.records.tsv").write_text("SOURCE_FILE\tROW_ID\nsample.vcf\t1\n") + (tsv_dir / "sample.header_lines.tsv").write_text("SOURCE_FILE\tLINE\nsample.vcf\t##x\n") + (tsv_dir / "sample.file_metadata.tsv").write_text("SOURCE_FILE\tKEY\tVALUE\nsample.vcf\tk\tv\n") + elif "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + sample_dir = out_dir / "sample" + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "sample.nt").write_text("

.\n") + return 0 + + def fake_validation(**kwargs): + validation_calls.append(kwargs) + results_dir = kwargs["results_dir"] + results_dir.mkdir(parents=True, exist_ok=True) + summary_path = results_dir / "summary.json" + summary_path.write_text(json.dumps({"status": "PASS", "temporaryRdf": {"persisted": False}})) + payload = { + "run_id": kwargs["run_id"], + "timestamp": kwargs["timestamp"], + "stage": "validation", + "vcf_path": str(kwargs["vcf_path"]), + "rdf_path": str(kwargs["rdf_path"]), + "rdf_format": "nt", + "representation": kwargs["representation"], + "results_dir": str(results_dir), + "summary_path": str(summary_path), + "exit_code": 0, + "status": "PASS", + "timing": {"wall_seconds": 0.25, "user_seconds": 0.1, "sys_seconds": 0.02, "max_rss_kb": 512}, + "temporary_rdf": {"decompressed_inside_container": False, "persisted_on_host": False, "cleanup_confirmed_by_runner": True}, + } + kwargs["stage_result"].update(payload) + stage_path = kwargs["metrics_dir"] / "stages" / "validation" / "sample.json" + stage_path.parent.mkdir(parents=True, exist_ok=True) + stage_path.write_text(json.dumps(payload)) + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ), mock.patch.object( + vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets() + ), mock.patch.object( + vcf_rdfizer, "run_validation_mode", side_effect=fake_validation + ): + rc = invoke_main( + [ + "--input", + str(input_dir), + "--rules", + str(rules_path), + "--rdf-storage-mode", + "plain", + "--compression", + "none", + "--validate", + "--out", + str(out_dir), + "--keep-tsv", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(len(validation_calls), 1) + self.assertEqual(validation_calls[0]["rdf_path"].name, "sample.nt") + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + report_dir = run_metrics_dir / "reports" / "validation" / "sample" + self.assertTrue((report_dir / "summary.json").exists()) + self.assertTrue((run_metrics_dir / "stages" / "validation" / "sample.json").exists()) + with (run_metrics_dir / "metrics.csv").open(newline="", encoding="utf-8") as handle: + row = next(csv.DictReader(handle)) + self.assertEqual(row["validation_status"], "PASS") + self.assertEqual(row["validation_exit_code"], "0") + self.assertEqual( + Path(row["validation_rdf_path"]).resolve(), + (out_dir / "sample" / "sample.nt").resolve(), + ) + summary = json.loads((run_metrics_dir / "summary.json").read_text()) + self.assertIn("reports/validation/sample/summary.json", summary["reports"]) + def test_build_sample_support_tsvs_expands_per_sample_and_per_format_rows(self): """records.tsv is expanded into helper tables for sample calls and format key/value pairs.""" with tempfile.TemporaryDirectory() as td: @@ -1816,6 +2073,7 @@ def test_help_flag_prints_usage_guide(self): self.assertIn("--representations", text) self.assertIn("--artifact-compression", text) self.assertIn("--cottas", text) + self.assertIn("--quiet", text) self.assertNotIn("--keep-rdf", text) self.assertNotIn("--compression", text) @@ -2421,12 +2679,58 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("/opt/vcf-rdfizer/validation/validation_runner.py", command) self.assertIn("/data/rdf/sample.nt.gz", command) self.assertIn("condensed", command) - results_dir = out_dir / "validation" / "sample" + results_dir = next((out_dir / "run_metrics").glob("*/reports/validation/sample")) self.assertTrue(results_dir.is_dir()) stage = next((out_dir / "run_metrics").glob("*/stages/validation/sample.json")) payload = json.loads(stage.read_text()) self.assertTrue(payload["temporary_rdf"]["decompressed_inside_container"]) self.assertFalse(payload["temporary_rdf"]["persisted_on_host"]) + with stage.parent.parent.parent.joinpath("metrics.csv").open( + newline="", encoding="utf-8" + ) as handle: + row = next(csv.DictReader(handle)) + self.assertEqual(row["validation_status"], "PASS") + self.assertEqual(row["validation_exit_code"], "0") + + def test_run_validation_mode_accepts_plain_nt_for_full_runs(self): + """The shared validation runner selects --rdf-nt for a plain aggregate.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text("

.\n") + metrics_dir = tmp_path / "metrics" + results_dir = metrics_dir / "reports" / "validation" / "sample" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + stage_result = {} + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + rc = vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=rdf_path, + representation="expanded", + validation_id="sample", + results_dir=results_dir, + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + wrapper_log_path=tmp_path / "wrapper.log", + stage_result=stage_result, + ) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + self.assertIn("--rdf-nt", commands[0]) + self.assertIn("/data/rdf/sample.nt", commands[0]) + self.assertEqual(stage_result["rdf_format"], "nt") + self.assertFalse(stage_result["temporary_rdf"]["decompressed_inside_container"]) def test_main_rejects_spark_partitions_outside_full_mode(self): """--spark-partitions is rejected for non-full modes.""" diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 1b44bdc..d4b08b1 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -58,6 +58,8 @@ _DOCKER_USE_SUDO = False _ACTIVE_PROGRESS = None _PROGRESS_ALLOWED = True +_PROGRESS_EVENTS_ALLOWED = True +_QUIET = False PROGRESS_POLL_INTERVAL_SECONDS = 0.25 COMPRESSED_VCF_EXPANSION_FACTOR = 5.0 @@ -111,6 +113,17 @@ COMPRESSION_COMMON_COLUMNS = ["combined_rdf_size_bytes", "compression_methods"] +VALIDATION_METRICS_COLUMNS = [ + "validation_status", + "validation_exit_code", + "validation_wall_seconds", + "validation_user_seconds", + "validation_sys_seconds", + "validation_max_rss_kb", + "validation_results_path", + "validation_rdf_path", +] + COMPRESSION_METHOD_COLUMNS = { "gzip": [ "gzip_size_bytes", @@ -351,7 +364,7 @@ def progress_events_enabled() -> bool: Keep collecting the same low-volume events in those cases so ``ProgressSession`` can render readable line-based status instead. """ - if not _PROGRESS_ALLOWED: + if not _PROGRESS_EVENTS_ALLOWED or (not _PROGRESS_ALLOWED and not _QUIET): return False if os.environ.get("VCF_RDFIZER_NO_PROGRESS"): return False @@ -370,7 +383,11 @@ def __init__(self, path: Path | None, label: str): self.path = path self.label = label self.enabled = progress_events_enabled() - self.rich_enabled = self.enabled and progress_ui_enabled() + # ``--quiet`` keeps producing/consuming sidecar events so the normal + # progress bookkeeping remains available to logs and orchestration, + # but disables every terminal rendering path. + self.render_enabled = self.enabled and _PROGRESS_ALLOWED + self.rich_enabled = self.render_enabled and progress_ui_enabled() self._offset = 0 self._progress = None self._starter_task = None @@ -386,7 +403,7 @@ def __enter__(self): self.path.parent.mkdir(parents=True, exist_ok=True) self.path.unlink(missing_ok=True) - if not self.enabled: + if not self.render_enabled: return self if not self.rich_enabled: eprint(f"{self.label}: started") @@ -441,6 +458,8 @@ def _detail(event: dict) -> str: detail = f"{int(completed):,} chunks" elif unit == "parts": detail = f"{int(completed):,} parts" + elif unit == "queries": + detail = f"{int(completed):,} queries" else: detail = f"{int(completed):,}" @@ -453,6 +472,8 @@ def _detail(event: dict) -> str: return detail def _update_event(self, event: dict): + if not self.render_enabled: + return stage = str(event.get("stage") or "work") phase = str(event.get("phase") or "working") if self._progress is None: @@ -535,7 +556,7 @@ def __exit__(self, exc_type, exc_value, traceback): self.poll_events() if self._progress is not None: self._progress.stop() - elif self.enabled: + elif self.render_enabled: eprint(f"{self.label}: finished") if self.path is not None: try: @@ -2881,7 +2902,7 @@ def write_run_summary( report_dir = metrics_dir / "reports" reports = [ path.relative_to(metrics_dir).as_posix() - for path in sorted(report_dir.iterdir()) + for path in sorted(report_dir.rglob("*")) if path.is_file() ] if report_dir.is_dir() else [] payload = { @@ -2935,36 +2956,40 @@ def container_progress_path(path: Path | None, metrics_dir: Path | None) -> str return f"/data/metrics/{relative.as_posix()}" -def metrics_header_for_methods(selected_methods: list[str]) -> list[str]: +def metrics_header_for_methods( + selected_methods: list[str], *, include_validation: bool = False +) -> list[str]: """Build a run-specific metrics.csv header with only relevant columns.""" methods = list(selected_methods or []) header = list(CONVERSION_METRICS_HEADER) - if not methods: + if methods: + header.extend(COMPRESSION_COMMON_COLUMNS) + if "gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["gzip"]) + if "brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["brotli"]) + + uses_cottas = any(method in COTTAS_COMPRESSION_METHODS for method in methods) + if uses_cottas: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas"]) + + uses_hdt = any(method in HDT_COMPRESSION_METHODS for method in methods) + if uses_hdt: + header.extend(COMPRESSION_METHOD_COLUMNS["hdt"]) + header.append(HDT_SOURCE_COLUMN) + if "hdt_gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["hdt_gzip"]) + if "hdt_brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["hdt_brotli"]) + if "cottas_gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas_gzip"]) + if "cottas_brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas_brotli"]) + + if include_validation: + header.extend(VALIDATION_METRICS_COLUMNS) + if not methods and not include_validation: return header - - header.extend(COMPRESSION_COMMON_COLUMNS) - if "gzip" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["gzip"]) - if "brotli" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["brotli"]) - - uses_cottas = any(method in COTTAS_COMPRESSION_METHODS for method in methods) - if uses_cottas: - header.extend(COMPRESSION_METHOD_COLUMNS["cottas"]) - - uses_hdt = any(method in HDT_COMPRESSION_METHODS for method in methods) - if uses_hdt: - header.extend(COMPRESSION_METHOD_COLUMNS["hdt"]) - header.append(HDT_SOURCE_COLUMN) - if "hdt_gzip" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["hdt_gzip"]) - if "hdt_brotli" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["hdt_brotli"]) - if "cottas_gzip" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["cottas_gzip"]) - if "cottas_brotli" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["cottas_brotli"]) - return unique_in_order(header) @@ -2982,6 +3007,7 @@ def update_metrics_csv_with_compression( selected_methods: list[str], method_results: dict[str, dict], tsv_metrics: dict | None = None, + validation_result: dict | None = None, ): """Upsert compression-related columns in `metrics.csv` for one output artifact. @@ -2989,7 +3015,9 @@ def update_metrics_csv_with_compression( HDT-first metrics (gzip_on_hdt / brotli_on_hdt) to avoid ambiguity. """ metrics_csv.parent.mkdir(parents=True, exist_ok=True) - target_header = metrics_header_for_methods(selected_methods) + target_header = metrics_header_for_methods( + selected_methods, include_validation=validation_result is not None + ) rows = [] existing_header = [] @@ -3027,7 +3055,6 @@ def update_metrics_csv_with_compression( row["combined_rdf_size_bytes"] = str(int(combined_size_bytes)) if "compression_methods" in row: row["compression_methods"] = "|".join(selected_methods) if selected_methods else "none" - defaults = { "exit_code_tsv": "0", "wall_seconds_tsv": "null", @@ -3080,6 +3107,19 @@ def update_metrics_csv_with_compression( "sys_seconds_brotli_on_hdt": "null", "max_rss_kb_brotli_on_hdt": "null", } + if validation_result is not None: + defaults.update( + { + "validation_status": "NOT_RUN", + "validation_exit_code": "null", + "validation_wall_seconds": "null", + "validation_user_seconds": "null", + "validation_sys_seconds": "null", + "validation_max_rss_kb": "null", + "validation_results_path": "", + "validation_rdf_path": "", + } + ) for key, value in defaults.items(): if key in row: row[key] = value @@ -3190,6 +3230,35 @@ def assign_validation(method: str, result: dict): row["exit_code_brotli_on_cottas"] = str(int(cottas_brotli_result.get("exit_code") or 0)) assign_timing("brotli_on_cottas", cottas_brotli_result) + if validation_result is not None: + validation_timing = validation_result.get("timing") or {} + if "validation_status" in row: + row["validation_status"] = str( + validation_result.get("status") or "EXECUTION_FAILED" + ) + if "validation_exit_code" in row: + exit_code = validation_result.get("exit_code") + row["validation_exit_code"] = "null" if exit_code is None else str(int(exit_code)) + for key, metric_key in ( + ("validation_wall_seconds", "wall_seconds"), + ("validation_user_seconds", "user_seconds"), + ("validation_sys_seconds", "sys_seconds"), + ): + if key in row: + value = validation_timing.get(metric_key) + row[key] = "null" if value is None else f"{float(value):.6f}" + if "validation_max_rss_kb" in row: + rss = validation_timing.get("max_rss_kb") + row["validation_max_rss_kb"] = "null" if rss is None else str(int(rss)) + if "validation_results_path" in row: + row["validation_results_path"] = str(validation_result.get("results_dir") or "") + if "validation_rdf_path" in row: + row["validation_rdf_path"] = str( + validation_result.get("rdf_path") + or validation_result.get("rdf_gzip_path") + or "" + ) + with metrics_csv.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=target_header) writer.writeheader() @@ -3207,6 +3276,7 @@ def write_compression_metrics_artifacts( selected_methods: list[str], method_results: dict[str, dict], index_warnings: list[dict] | None = None, + validation_result: dict | None = None, ): """Write the final per-output compression summary and method timing files.""" metrics_dir.mkdir(parents=True, exist_ok=True) @@ -3322,6 +3392,8 @@ def timing_payload(result: dict): "timing": timing_payload(cottas_brotli_result), }, } + if validation_result is not None: + payload["semantic_validation"] = validation_result metrics_json_dir = metrics_dir / "stages" / "compression" metrics_json_dir.mkdir(parents=True, exist_ok=True) @@ -4662,6 +4734,8 @@ def run_full_mode( image_ref: str, out_name: str, sample_workflow: SampleWorkflow, + run_validation: bool = False, + filter_oracle: str = "auto", rdf_storage_mode: str, methods: list[str], hdt_strategy: str, @@ -4677,8 +4751,11 @@ def run_full_mode( wrapper_log_path: Path, run_tracker: RunTracker | None = None, ): - """Execute full pipeline: per-input TSV -> RDF -> compression -> metrics.""" - print("Step 3/5: Processing per-input pipeline (TSV -> RDF -> compression)") + """Execute full pipeline: per-input TSV -> RDF -> compression -> validation.""" + pipeline_label = "TSV -> RDF -> compression" + if run_validation: + pipeline_label += " -> validation" + print(f"Step 3/5: Processing per-input pipeline ({pipeline_label})") if spark_partitions is not None: print(f" Spark partition hint: {spark_partitions}") print(f" Sample representation: {sample_workflow.representation}") @@ -4733,6 +4810,8 @@ def run_full_mode( input_vcf = container_input input_failed = False input_index_warnings: list[dict] = [] + validation_failed = False + validation_result: dict | None = None def fail_current(stage: str, message: str): nonlocal input_failed @@ -5124,6 +5203,69 @@ def fail_current(stage: str, message: str): if run_tracker is not None: run_tracker.mark(f"Input {idx}: compression completed for {output_name}") + if run_validation: + validation_rdf_path = raw_rdf_files[0] + validation_results_dir = ( + metrics_dir / "reports" / "validation" / safe_metrics_name(output_name) + ) + print(f" * Semantic validation: {output_name}") + validation_result = { + "run_id": run_id, + "timestamp": timestamp, + "stage": "validation", + "validation_id": output_name, + "vcf_path": str(input_vcf), + "rdf_path": str(validation_rdf_path), + "rdf_format": "nt.gz" if validation_rdf_path.name.endswith(".nt.gz") else "nt", + "representation": sample_workflow.representation, + "results_dir": str(validation_results_dir), + "summary_path": str(validation_results_dir / "summary.json"), + "input_rdf_size_bytes": int(file_size_bytes(validation_rdf_path) or 0), + "exit_code": 1, + "status": "EXECUTION_FAILED", + "timing": { + "wall_seconds": None, + "user_seconds": None, + "sys_seconds": None, + "max_rss_kb": None, + }, + "temporary_rdf": { + "decompressed_inside_container": validation_rdf_path.name.endswith(".nt.gz"), + "persisted_on_host": False, + "cleanup_confirmed_by_runner": False, + }, + } + try: + validation_exit_code = run_validation_mode( + vcf_path=Path(input_vcf), + rdf_path=validation_rdf_path, + representation=sample_workflow.representation, + validation_id=output_name, + results_dir=validation_results_dir, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + image_ref=image_ref, + filter_oracle=filter_oracle, + wrapper_log_path=wrapper_log_path, + run_tracker=run_tracker, + stage_result=validation_result, + ) + except Exception as exc: + validation_result["error"] = str(exc) + try: + stage_path = metrics_dir / "stages" / "validation" / f"{safe_metrics_name(output_name)}.json" + stage_path.parent.mkdir(parents=True, exist_ok=True) + stage_path.write_text(json.dumps(validation_result, indent=2) + "\n", encoding="utf-8") + except OSError: + pass + eprint( + f"Error: semantic validation could not be completed for '{output_name}': " + f"{exc}. See log: {wrapper_log_path}" + ) + validation_exit_code = 1 + validation_failed = int(validation_exit_code) != 0 + raw_size_before_cleanup_by_file = { raw_rdf_path.name: int(file_size_bytes(raw_rdf_path) or 0) for raw_rdf_path in raw_rdf_files } @@ -5144,6 +5286,7 @@ def fail_current(stage: str, message: str): selected_methods=selected_methods, method_results=aggregated_results, index_warnings=input_index_warnings, + validation_result=validation_result, ) update_metrics_csv_with_compression( metrics_csv=metrics_dir / "metrics.csv", @@ -5155,6 +5298,7 @@ def fail_current(stage: str, message: str): selected_methods=selected_methods, method_results=aggregated_results, tsv_metrics=tsv_metrics, + validation_result=validation_result, ) except PermissionError as exc: blocked_path = exc.filename or str(metrics_dir) @@ -5166,8 +5310,15 @@ def fail_current(stage: str, message: str): ) return 1 + if validation_failed: + fail_current( + "validation", + f"semantic VCF/RDF validation failed for '{output_name}'. " + f"See results: {validation_result.get('results_dir') if validation_result else metrics_dir / 'reports' / 'validation' / safe_metrics_name(output_name)}", + ) + rdf_storage_removed = False - if selected_methods and ( + if not validation_failed and selected_methods and ( remove_rdf_storage_output or not keep_rmlstreamer_rdf_output ): # Cleanup raw RDF only after every selected compression method has @@ -5265,7 +5416,9 @@ def fail_current(stage: str, message: str): output_root = out_dir / output_name raw_total_size = sum(raw_size_before_cleanup_by_file.values()) - if keep_rmlstreamer_rdf_output: + if validation_failed: + raw_note = "retained after validation failure" + elif keep_rmlstreamer_rdf_output: raw_note = "retained via --keep-rmlstreamer-rdf-output" elif rdf_storage_removed and remove_rdf_storage_output: raw_note = "removed via --remove-rdf-storage-output" @@ -5379,7 +5532,10 @@ def fail_current(stage: str, message: str): continue if run_tracker is not None: - run_tracker.mark(f"Input {idx}/{total_inputs} completed: {output_name}") + if input_failed: + run_tracker.mark(f"Input {idx}/{total_inputs} finished with failures: {output_name}") + else: + run_tracker.mark(f"Input {idx}/{total_inputs} completed: {output_name}") if not keep_tsv and intermediate_dir.exists(): if not remove_path_with_docker_fallback( @@ -5894,7 +6050,7 @@ def run_decompress_mode( def run_validation_mode( *, vcf_path: Path, - rdf_gzip_path: Path, + rdf_path: Path, representation: str, validation_id: str, results_dir: Path, @@ -5904,33 +6060,53 @@ def run_validation_mode( image_ref: str, filter_oracle: str, wrapper_log_path: Path, + run_tracker: RunTracker | None = None, + stage_result: dict | None = None, + write_metrics_csv: bool = False, ): - """Run VCF/RDF semantic queries with ephemeral in-container RDF expansion. + """Run VCF/RDF semantic queries with container-local temporary state. - The only mounted RDF source is the input ``.nt.gz`` file. The validation - runner inflates it under the container's ``/work`` temporary filesystem and - removes that source before the container exits; no raw N-Triples are - materialized in the user-selected output directory. + Gzip RDF inputs are inflated under the container's ``/work`` temporary + filesystem. Plain N-Triples inputs are read directly from their read-only + mount. In both cases, no validation scratch RDF is persisted on the host. """ # An empty directory may be left behind if Docker itself fails before the # runner starts. Reuse only that empty shell; never overwrite reports. results_dir.mkdir(parents=True, exist_ok=True) + safe_validation_id = safe_metrics_name(validation_id) + timing_host = metrics_dir / "timings" / "validation" / f"{safe_validation_id}.txt" + timing_host.parent.mkdir(parents=True, exist_ok=True) + timing_container = f"/data/metrics/timings/validation/{safe_validation_id}.txt" + progress_host_path = ( + progress_event_path(metrics_dir, "validation", safe_validation_id) + if progress_events_enabled() + else None + ) + progress_container_ref = container_progress_path(progress_host_path, metrics_dir) + rdf_flag = "--rdf-gz" if rdf_path.name.endswith(".nt.gz") else "--rdf-nt" cmd = [ *docker_run_base(), "--init", "-v", f"{str(vcf_path.parent)}:/data/vcf:ro", "-v", - f"{str(rdf_gzip_path.parent)}:/data/rdf:ro", + f"{str(rdf_path.parent)}:/data/rdf:ro", "-v", f"{str(results_dir)}:/data/validation", + "-v", + f"{str(metrics_dir.resolve())}:/data/metrics", image_ref, + "/usr/bin/time", + "-v", + "-o", + timing_container, + "--", "/opt/pycottas-venv/bin/python", "/opt/vcf-rdfizer/validation/validation_runner.py", "--vcf", f"/data/vcf/{vcf_path.name}", - "--rdf-gz", - f"/data/rdf/{rdf_gzip_path.name}", + rdf_flag, + f"/data/rdf/{rdf_path.name}", "--representation", representation, "--results-dir", @@ -5941,10 +6117,21 @@ def run_validation_mode( filter_oracle, "--scratch-dir", "/work", + *( + ["--progress-path", progress_container_ref] + if progress_container_ref is not None + else [] + ), + *(["--quiet"] if _QUIET else []), ] + if run_tracker is not None: + run_tracker.mark(f"Validation started: {validation_id}") started = time.perf_counter() - exit_code = run(cmd) + with ProgressSession(progress_host_path, f"Validation: {validation_id}"): + exit_code = run(cmd) elapsed = time.perf_counter() - started + timing = parse_time_log_metrics(timing_host) + timing["wall_seconds"] = elapsed summary_path = results_dir / "summary.json" summary = None if summary_path.is_file(): @@ -5952,40 +6139,80 @@ def run_validation_mode( summary = json.loads(summary_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): summary = None - report_path = metrics_dir / "stages" / "validation" / f"{safe_metrics_name(validation_id)}.json" + report_path = metrics_dir / "stages" / "validation" / f"{safe_validation_id}.json" report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text( - json.dumps( - { - "run_id": run_id, - "timestamp": timestamp, - "vcf_path": str(vcf_path), - "rdf_gzip_path": str(rdf_gzip_path), - "representation": representation, - "results_dir": str(results_dir), - "input_rdf_size_bytes": int(file_size_bytes(rdf_gzip_path) or 0), - "exit_code": int(exit_code), - "status": summary.get("status") if isinstance(summary, dict) else None, - "timing": {"wall_seconds": elapsed}, - "temporary_rdf": { - "decompressed_inside_container": True, - "persisted_on_host": False, - "cleanup_confirmed_by_runner": ( - summary.get("temporaryRdf", {}).get("cleanupConfirmed") - if isinstance(summary, dict) - else False - ), - }, - }, - indent=2, - ) - + "\n", - encoding="utf-8", - ) + status = summary.get("status") if isinstance(summary, dict) else None + if not status: + status = "PASS" if int(exit_code) == 0 else "EXECUTION_FAILED" + if int(exit_code) != 0 and status == "PASS": + status = "EXECUTION_FAILED" + summary_temporary_rdf = summary.get("temporaryRdf", {}) if isinstance(summary, dict) else {} + payload = { + "run_id": run_id, + "timestamp": timestamp, + "stage": "validation", + "validation_id": validation_id, + "vcf_path": str(vcf_path), + "rdf_path": str(rdf_path), + "rdf_format": "nt.gz" if rdf_path.name.endswith(".nt.gz") else "nt", + "representation": representation, + "results_dir": str(results_dir), + "summary_path": str(summary_path), + "input_rdf_size_bytes": int(file_size_bytes(rdf_path) or 0), + "exit_code": int(exit_code), + "status": status, + "timing": { + "wall_seconds": timing.get("wall_seconds"), + "user_seconds": timing.get("user_seconds"), + "sys_seconds": timing.get("sys_seconds"), + "max_rss_kb": timing.get("max_rss_kb"), + }, + "temporary_rdf": { + "decompressed_inside_container": bool( + summary_temporary_rdf.get( + "decompressedInsideContainer", rdf_path.name.endswith(".nt.gz") + ) + ), + "persisted_on_host": bool(summary_temporary_rdf.get("persisted", False)), + "cleanup_confirmed_by_runner": bool( + summary_temporary_rdf.get("cleanupConfirmed", False) + ), + }, + } + if rdf_path.name.endswith(".nt.gz"): + # Compatibility alias for consumers of the original standalone report + # schema; ``rdf_path`` is the canonical format-neutral field. + payload["rdf_gzip_path"] = str(rdf_path) + report_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + if stage_result is not None: + stage_result.update(payload) + if write_metrics_csv: + # Standalone validation has no preceding conversion/compression stage, + # but it still exposes the same analysis-ready metrics schema used by a + # full run. Full mode writes its row after compression so its selected + # compression columns are preserved. + try: + update_metrics_csv_with_compression( + metrics_csv=metrics_dir / "metrics.csv", + run_id=run_id, + timestamp=timestamp, + output_name=validation_id, + output_dir=results_dir, + combined_size_bytes=int(payload["input_rdf_size_bytes"]), + selected_methods=[], + method_results={}, + validation_result=payload, + ) + except OSError as exc: + eprint(f"Warning: failed to write validation metrics CSV: {exc}") if exit_code != 0: eprint(f"Error: validation failed. See results: {results_dir}") eprint(f"See log for details: {wrapper_log_path}") + if run_tracker is not None: + run_tracker.mark(f"Validation failed: {validation_id} (exit_code={exit_code})") return 1 + if run_tracker is not None: + run_tracker.mark(f"Validation completed: {validation_id} ({status})") print(f"Validation results: {results_dir}") print(f"Validation metrics: {report_path}") return 0 @@ -6008,6 +6235,9 @@ def main(): " Full pipeline (space-optimized aggregate):\n" " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " "--representations hdt,cottas --rdf-compression none -o ./results\n" + " Full pipeline with semantic validation:\n" + " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --rdf-storage-mode plain " + "--representations none --rdf-compression none --validate -o ./results\n" " Condensed multi-sample representation:\n" " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --sample-representation condensed " "--rdf-storage-mode space-optimized --representations hdt -o ./results\n" @@ -6098,6 +6328,16 @@ def main(): "one sample-ordered value vector per FORMAT key (default: expanded)" ), ) + parser.add_argument( + "--validate", + "--run-validation", + dest="run_validation", + action="store_true", + help=( + "Run semantic VCF/RDF validation for each input during full mode; " + "reports are stored under the run metrics directory" + ), + ) parser.add_argument( "--rdf-storage-mode", choices=sorted(RDF_STORAGE_MODES), @@ -6212,7 +6452,15 @@ def main(): parser.add_argument( "--no-progress", action="store_true", - help="Disable terminal compression/conversion progress updates", + help="Disable progress sidecars and terminal progress updates", + ) + parser.add_argument( + "--quiet", + action="store_true", + help=( + "Suppress terminal progress displays (including validation query updates) " + "while retaining progress logging and metrics" + ), ) parser.add_argument( "--validation-id", @@ -6223,7 +6471,7 @@ def main(): "--filter-oracle", choices=("auto", "bcftools", "cyvcf2"), default="auto", - help="FILTER oracle for validation mode (default: auto)", + help="FILTER oracle for full-mode validation and standalone validation (default: auto)", ) rdf_output_group = parser.add_mutually_exclusive_group() rdf_output_group.add_argument( @@ -6240,8 +6488,13 @@ def main(): ) args = parser.parse_args() - global _PROGRESS_ALLOWED - _PROGRESS_ALLOWED = not args.no_progress + global _PROGRESS_ALLOWED, _PROGRESS_EVENTS_ALLOWED, _QUIET + _QUIET = bool(args.quiet) + # ``--no-progress`` is the opt-out for the sidecar itself. ``--quiet`` + # only turns off terminal rendering, leaving the existing event stream + # available to the wrapper log/metrics machinery. + _PROGRESS_ALLOWED = not (args.no_progress or args.quiet) + _PROGRESS_EVENTS_ALLOWED = not args.no_progress if args.build and args.no_build: eprint("Error: --build and --no-build are mutually exclusive.") @@ -6355,6 +6608,8 @@ def main(): partitioned=full_uses_partitioning, ) validate_no_output_collisions(output_plans) + elif args.run_validation: + raise ValueError("--validate/--run-validation is only valid in --mode full") elif mode == "tsv": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -6384,14 +6639,6 @@ def main(): validation_id = args.validation_id or vcf_output_prefix(validation_vcf_path) if not re.fullmatch(r"[A-Za-z0-9._-]+", validation_id): raise ValueError("--validation-id may contain only letters, digits, dot, underscore, and hyphen") - validation_results_dir = out_dir / "validation" / validation_id - if validation_results_dir.exists() and ( - not validation_results_dir.is_dir() or any(validation_results_dir.iterdir()) - ): - raise ValueError( - f"Refusing to overwrite existing validation results: {validation_results_dir}. " - "Choose --validation-id or --out with a new destination." - ) validate_mode_dirs([out_root, out_dir, metrics_root]) elif mode == "compress": if args.spark_partitions is not None: @@ -6530,6 +6777,20 @@ def main(): source_label = metrics_run_label(metrics_source_paths, metrics_source_root) metrics_dir = metrics_run_directory(metrics_root, source_label, run_id) + if mode == "validation": + # Keep standalone validation reports in the same discoverable tree as + # full-mode stage artifacts. The validation id remains the leaf name + # so existing stage/report consumers can use one layout for both modes. + validation_results_dir = ( + metrics_dir / "reports" / "validation" / safe_metrics_name(validation_id) + ) + if validation_results_dir.exists() and ( + not validation_results_dir.is_dir() or any(validation_results_dir.iterdir()) + ): + raise ValueError( + f"Refusing to overwrite existing validation results: {validation_results_dir}. " + "Choose --validation-id or --out with a new destination." + ) manifest_options = { "requested_image": args.image, "requested_image_version": args.image_version, @@ -6543,6 +6804,10 @@ def main(): "chunk_min_bytes": chunk_min_bytes if mode in {"full", "compress"} else None, "chunk_max_bytes": chunk_max_bytes if mode in {"full", "compress"} else None, "spark_partitions": spark_partitions if mode == "full" else None, + "run_validation": bool(args.run_validation) if mode == "full" else False, + "filter_oracle": args.filter_oracle if mode in {"full", "validation"} else None, + "quiet": bool(args.quiet), + "no_progress": bool(args.no_progress), "index_format": index_format if mode == "index" else None, "decompression_format": fmt if mode == "decompress" else None, "validation_representation": args.sample_representation if mode == "validation" else None, @@ -6723,6 +6988,8 @@ def execute_mode(): image_ref=image_ref, out_name=args.out_name, sample_workflow=sample_workflow, + run_validation=args.run_validation, + filter_oracle=args.filter_oracle, rdf_storage_mode=args.rdf_storage_mode, methods=full_methods, hdt_strategy=args.hdt_strategy, @@ -6771,7 +7038,7 @@ def execute_mode(): if mode == "validation": return run_validation_mode( vcf_path=validation_vcf_path, - rdf_gzip_path=validation_rdf_gzip_path, + rdf_path=validation_rdf_gzip_path, representation=args.sample_representation, validation_id=validation_id, results_dir=validation_results_dir, @@ -6781,6 +7048,7 @@ def execute_mode(): image_ref=image_ref, filter_oracle=args.filter_oracle, wrapper_log_path=wrapper_log_path, + write_metrics_csv=True, ) if mode == "index": return run_index_mode( From 309e67f65e3ce5ac7ef8f981b19d504f7c22227b Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Fri, 4 Sep 2026 14:32:57 +0200 Subject: [PATCH 17/19] repo cleaning and small bug fixes --- ACKNOWLEDGEMENTS.md | 29 ++ Dockerfile | 3 + README.md | 127 +++++- changelog.md | 169 ++++++++ pyproject.toml | 3 +- rules/README.md | 12 + src/compression.sh | 643 ------------------------------ src/run_conversion.sh | 176 ++++++--- src/update_rules.sh | 48 --- test/README.md | 18 +- test/test_compression_unit.py | 377 ------------------ test/test_gzip_size_unit.py | 241 ++++++++++++ test/test_rules_helper_unit.py | 241 ++++++++++++ test/test_update_rules_unit.py | 81 ---- test/test_vcf_rdfizer_unit.py | 98 ++--- vcf_rdfizer.py | 695 ++++++++++++++------------------- vcf_rdfizer_gzip.py | 356 +++++++++++++++++ vcf_rdfizer_rules.py | 452 +++++++++++++++++++++ 18 files changed, 2091 insertions(+), 1678 deletions(-) create mode 100644 ACKNOWLEDGEMENTS.md delete mode 100644 src/compression.sh delete mode 100644 src/update_rules.sh delete mode 100644 test/test_compression_unit.py create mode 100644 test/test_gzip_size_unit.py create mode 100644 test/test_rules_helper_unit.py delete mode 100644 test/test_update_rules_unit.py create mode 100644 vcf_rdfizer_gzip.py create mode 100644 vcf_rdfizer_rules.py diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md new file mode 100644 index 0000000..b8c67dc --- /dev/null +++ b/ACKNOWLEDGEMENTS.md @@ -0,0 +1,29 @@ +# Acknowledgements + +VCF-RDFizer is developed at [KNoWS](https://knows.idlab.ugent.be/), IDLab, +Ghent University - imec, as part of the doctoral research of +Elias Crum (). + +Please cite the software as described in [`README.md`](README.md#citation) and +[`CITATION.cff`](CITATION.cff). + +## Acknowledgement sentence for papers + +When VCF-RDFizer is used in a publication, the following template can be +adapted for the Acknowledgements section: + +> This work was carried out at KNoWS, IDLab, Ghent University - imec, as part +> of the doctoral research of Elias Crum. The authors thank the maintainers of +> the open-source components on which VCF-RDFizer builds - RMLStreamer, +> hdt-cpp, hdtc, pycottas, Comunica, bcftools, and cyvcf2. + +If the work is co-funded by a specific project or grant, add that project name +and grant number to the sentence above and record it in the table at the top of +this file. + +## Third-party components + +VCF-RDFizer bundles and orchestrates third-party software inside its Docker +image. Their licenses and notices are collected in +[`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md) and, in the built image, +under `/usr/share/licenses/vcf-rdfizer/`. diff --git a/Dockerfile b/Dockerfile index 2520435..e0cc32e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,6 +114,9 @@ COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES. COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ COPY src/validation/ /opt/vcf-rdfizer/validation/ +# Shared, dependency-free helper used by both the host CLI and the in-container +# conversion runner, so it lives at the repository root rather than in src/. +COPY vcf_rdfizer_gzip.py /opt/vcf-rdfizer/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ && chmod +x /usr/local/bin/rdf2hdt \ diff --git a/README.md b/README.md index bad02e3..089ec5f 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ or pull the prebuilt Docker image directly: docker pull ecrum19/vcf-rdfizer:latest ``` -Release maintainers: see [`RELEASING.md`](RELEASING.md) for the PyPI, +Release maintainers: see [`scripts/RELEASING.md`](scripts/RELEASING.md) for the PyPI, Docker Hub, and conda-forge release procedure. ## Important CLI Rule @@ -605,6 +605,25 @@ Within each run directory, VCF-RDFizer writes: (`summary.json`, query results, preflight checks, and cleanup evidence) when validation is requested +`input_vcf_size_bytes` in `stages/conversion/*.json` and `metrics.csv` is the +*uncompressed* size of the source VCF, so that ratios are comparable between +plain and compressed inputs. The accompanying `input_vcf_size_method` records +how it was obtained: + +| Method | Meaning | +| --- | --- | +| `stat` | Input was not compressed; the on-disk size is the answer | +| `bgzf` | Exact, summed from a `bgzip`/BGZF file's block headers with no decompression | +| `gzip-sample` | Exact; the whole single-member stream fitted in the sampling budget | +| `gzip-trailer` | Exact; the 32-bit `ISIZE` trailer resolved against the file's measured compression ratio | +| `inflate` / `inflate-shell` | Fallback full decompression pass, used when the file's structure cannot settle the answer (for example concatenated non-BGZF members) | + +Only the fallback costs a full pass over the input. Because indexed `.vcf.gz` +files from `bcftools`/`tabix`/`htslib` are BGZF, the usual case is measured in +milliseconds rather than minutes. The same machinery makes `--estimate-size` +report a real uncompressed input size instead of an assumed expansion factor; +it says so when it had to fall back to the assumption. + Compression metrics now include per-method: - `wall_seconds_*` @@ -735,6 +754,112 @@ finishes. - default rules file: `rules/default_rules.ttl` - rules guide: `rules/README.md` +### Custom RML Mappings + +`--rules` accepts any RML mapping, so you can change what RDF the pipeline +produces without touching the wrapper. A custom mapping has to honour a small +contract, and `vcf-rdfizer-rules` (installed alongside `vcf-rdfizer`) makes it +discoverable and checkable: + +```bash +vcf-rdfizer-rules columns +``` + +Lists the five TSV sources the pipeline generates and every column each one +provides, so you know what a mapping can reference. + +```bash +vcf-rdfizer-rules init -o my_rules.ttl +``` + +Writes an annotated copy of the shipped default mapping to start from. + +```bash +vcf-rdfizer-rules check my_rules.ttl +``` + +Validates the mapping *before* you spend hours on a run. It reports: + +- logical-source paths the wrapper cannot rewrite per input, +- referenced columns no generated TSV provides (typos such as `CHROMOSOME`), +- which `--sample-representation` values remain usable, +- whether the mapping forces the large sample helper tables to be materialized. + +Exit code is `0` when the mapping is usable and `1` when it is not; add +`--json` for scripted use. Then run it: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz --rules my_rules.ttl --rdf-storage-mode plain -o ./results +``` + +#### The contract + +1. **Keep the five `csvw:url` values exactly as they are.** Full mode processes + one VCF at a time and rewrites those literal strings to the per-input file + names (`/data/tsv/records.tsv` becomes `/data/tsv/.records.tsv`, and + so on). Any other path is left untouched and will not resolve. + + | Logical source | Contents | + | --- | --- | + | `/data/tsv/records.tsv` | One row per VCF data line | + | `/data/tsv/header_lines.tsv` | One row per `##` header line | + | `/data/tsv/file_metadata.tsv` | One row summarising the source VCF | + | `/data/tsv/sample_calls.tsv` | Helper: one row per variant x sample | + | `/data/tsv/sample_format_values.tsv` | Helper: one row per variant x sample x FORMAT key | + +2. **Only reference columns the pipeline writes.** `vcf-rdfizer-rules columns` + is authoritative; a unit test pins those lists to what `src/vcf_as_tsv.sh` + actually emits, so they cannot drift. + +3. **Think before consuming the two helper tables.** The four built-in sample + maps are recognised by the wrapper, which then keeps those tables + header-only and streams the genotype RDF itself. A mapping that consumes + them in any other way forces them to be materialized in full - the largest + intermediate the pipeline can produce - and is rejected in + `--sample-representation condensed`, which would otherwise emit both + genotype representations at once. `check` warns about this explicitly. + +The last column of `records.tsv` is the whitespace-joined sample ids from the +`#CHROM` line (or `SAMPLES` when the VCF declares none), so its *name* varies +per input and it cannot be referenced by a fixed name. Genotype RDF is emitted +by the wrapper from that column instead; see +[`docs/sample-representation-guide.md`](docs/sample-representation-guide.md). + +## Repository Layout + +VCF-RDFizer is deliberately split into a thin host-side CLI and a set of +container-side stages. Nothing on the host walks the RDF itself; it plans work, +launches Docker, and reads back the JSON/CSV reports each stage writes. + +| Path | Role | +| --- | --- | +| `vcf_rdfizer.py` | Host CLI: argument validation, output-collision planning, Docker orchestration, metrics assembly, mode dispatch. Also emits the multi-sample genotype RDF (see below). | +| `vcf_rdfizer_rules.py` | `vcf-rdfizer-rules` CLI: scaffold, document, and validate custom RML mappings. | +| `vcf_rdfizer_gzip.py` | Uncompressed size of a gzip/BGZF VCF without decompressing it. Used by the host preflight estimate and, inside the image, by `run_conversion.sh`. | +| `src/vcf_as_tsv.sh` | VCF -> per-input `records`/`header_lines`/`file_metadata` TSV, in one `awk` pass. | +| `src/run_conversion.sh` | Runs RMLStreamer, normalizes Spark part files, merges them into one `.nt`/`.nt.gz` aggregate, records conversion metrics. | +| `src/partitioned_compression.py` | Record-safe RDF chunking plus chunked HDT/COTTAS generation and pairwise merge, inside an ephemeral Docker volume. | +| `src/cottas_tool.py` | COTTAS `convert` / `merge` / `reindex` / `decompress` adapter over `pycottas`, with a bounded-memory streaming merge. | +| `src/ensure_hdt_index.sh` | Java-free canonical `.hdt.index.v1-1` sidecar generation via `hdtc`, with restore-on-failure. | +| `src/validate_compression.py` | Round-trip check: decode a `.hdt`/`.cottas` artifact and compare its triple count against the source. | +| `src/validation/` | Semantic VCF-vs-RDF validation: `cyvcf2`/`bcftools` oracle, SPARQL queries per representation, comparison report. | +| `rules/default_rules.ttl` | Default RML mapping (also shipped as package data in `vcf_rdfizer_data/`). | +| `test/` | `unittest` suite; the shell/pipeline tests stub `java`, `docker`, and friends so no real external tool is needed. | +| `scripts/release.py` | Version bump + release metadata automation (see `scripts/RELEASING.md`). | + +Genotype RDF is the one deliberate exception to "all data processing happens in +the container": `append_expanded_sample_rdf` and `append_condensed_sample_rdf` +in `vcf_rdfizer.py` append it directly to the aggregate, because the equivalent +RML maps would first have to materialize variants x samples (x FORMAT keys) +helper TSV rows. See [`docs/sample-representation-guide.md`](docs/sample-representation-guide.md). + +Further reading: + +- [`docs/validation.md`](docs/validation.md) - semantic validation design and query set +- [`docs/sample-representation-guide.md`](docs/sample-representation-guide.md) - emitted genotype shapes +- [`changelog.md`](changelog.md) - dated change history +- [`ACKNOWLEDGEMENTS.md`](ACKNOWLEDGEMENTS.md) - funding and attribution + ## Troubleshooting If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). diff --git a/changelog.md b/changelog.md index b73ff45..267bce0 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,174 @@ # Changelog +## 2026-09-04 — Custom-mapping tooling, cheap gzip sizing, dead-script removal + +### Added + +- `vcf-rdfizer-rules`, a second console script (`vcf_rdfizer_rules.py`) for + authoring custom RML mappings. It replaces the `update_rules.sh` sed script + with something that serves the actual need: + - `columns` documents the five generated TSV sources and every column each + one provides, including why `records.tsv`'s final column has no fixed name. + - `init` writes an annotated copy of the default mapping to start from. + - `check` validates a mapping against the wrapper contract *before* a long + run: logical-source paths the wrapper cannot rewrite per input, referenced + columns no TSV provides, which `--sample-representation` values remain + usable, and whether the mapping forces the large sample helper tables to be + materialized. Exits 1 on a contract violation; `--json` for scripting. + The old script rewrote a rules file *in place* to point at fixed TSV paths, + which the pipeline has not needed since `render_rules_for_triplet` began + rendering a per-input copy - and which silently broke the per-input rewrite + it was supposed to help with. +- `vcf_rdfizer_gzip.py`: uncompressed size of a gzip/BGZF file without + decompressing it. Three tiers, and the method used is recorded next to the + size so the metric stays auditable: + - `bgzf` - sums the per-block `ISIZE` trailers of a `bgzip` file by walking + block headers. Exact, no inflate, and immune to the 32-bit wrap. This is + what `bcftools`/`tabix`/`htslib` produce, so it covers the usual case. + - `gzip-sample` / `gzip-trailer` - single-member gzip: measured outright when + it fits the sampling budget, otherwise the 32-bit `ISIZE` trailer resolved + against a compression ratio sampled from the file's own prefix. + - `inflate` - full pass, used whenever the structure cannot settle the + answer. Concatenated non-BGZF members are detected and routed here rather + than trusting a trailer that describes only the final member. + +### Changed + +- `run_conversion.sh` uses that helper for `input_vcf_size_bytes`, which + previously always cost a full `gzip -dc | wc -c` pass over the source VCF. + On the repository's 52 MB test fixture this is 1.115s -> 0.066s (17x); on a + 634 MB gzip whose uncompressed size exceeds 4 GiB, 13.6s -> 0.029s (470x), + where a naive trailer read would have been 93% wrong. The shell keeps its + `gzip -dc` fallback if the helper is unavailable. +- `stages/conversion/*.json` gains `input_vcf_size_method` recording how the + size was obtained. +- `--estimate-size` now uses the real uncompressed input size when the file's + structure can supply it, instead of always assuming a 5x expansion, and says + so when it had to fall back to the assumption. + +### Removed + +- `src/compression.sh` and `test/test_compression_unit.py`. The wrapper issues + its own gzip/brotli/rdf2hdt commands and delegates chunked work to + `partitioned_compression.py`; nothing had called this script for some time, + and it wrote a stale flat `metrics.csv` schema. +- `src/update_rules.sh` and `test/test_update_rules_unit.py`, superseded by + `vcf-rdfizer-rules` above. + +### Documentation + +- README: new "Custom RML Mappings" section with the three-point contract, the + logical-source table, and the `vcf-rdfizer-rules` workflow; a metrics table + explaining each `input_vcf_size_method`; repository-layout and test-suite + entries for the two new modules. +- `rules/README.md` points at the new CLI as the way to author a mapping. + +### Tests + +- `test/test_rules_helper_unit.py` (12 tests), including a drift guard that + runs `src/vcf_as_tsv.sh` and asserts the documented column lists still match + the headers it writes. +- `test/test_gzip_size_unit.py` (13 tests), each checking a measured size + against a full-inflate ground truth across BGZF, plain, concatenated, and + optional-header-field gzip layouts. + +## 2026-09-04 — Code review: cleanup, hot-path performance, and bug fixes + +Behaviour-preserving unless noted. Emitted RDF was verified byte-identical +(SHA-256) against the previous implementation for both sample representations +and for both `run_conversion.sh` storage modes. + +### Fixed + +- `run_compression_methods_for_rdf`: an existing `.hdt` that failed validation + during a full run recorded a **COTTAS** index warning (wrong format, wrong + artifact path, wrong message) via a copy-pasted block whose assignment also + never propagated, because `cottas_failure_warning` was not `nonlocal` there. + The block is removed; `validate_container_artifact` already downgrades a + recoverable HDT index problem to a warning, so reaching that point means the + artifact is genuinely unusable. +- `--mode validation` raised an uncaught `ValueError` traceback when the + results directory already existed and was non-empty, because that check sits + after the argument-validation `try/except`. It now prints `Error: ...` and + exits 2 like every other input error. Covered by a new regression test. +- `count_triples_in_nt_files` (the host-side fallback triple count) used a + slightly different rule than the container-side counters in + `validate_compression.py` / `partitioned_compression.py`, so a fallback count + could disagree with the authoritative one. The predicate is now shared as + `is_triple_line` with identical semantics. + +### Performance + +- Sample RDF emission (`append_expanded_sample_rdf`, + `append_condensed_sample_rdf`) no longer percent-encodes the same values on + every record: sample-column URI components and `sampleId` literals are + computed once per file, and FORMAT-key components are memoized. Previously + these ran `variants x samples` and `variants x samples x FORMAT keys` times. + ~1.6x faster at 50 samples, with the gain growing with cohort width. +- `_rml_uri_component` and `_ntriples_string_literal` short-circuit values that + need no encoding/escaping (~2.7x and ~1.3x on typical VCF tokens). +- `SampleRecordStream` caches the derived FORMAT-key tuple and its + duplicate-key check per distinct `FORMAT` string instead of rebuilding both + for every record. +- `count_triples_in_nt_files` scans bytes instead of decoding to `str`, and + reads gzip aggregates through a `BufferedReader` rather than `GzipFile` + line-by-line. +- `run()` discards subprocess output at the file-descriptor level instead of + buffering a whole container's output in memory only to drop it. +- `run_conversion.sh`: the `"."` -> `"."^^vcfr:Null` rewrite is now applied to + each RMLStreamer part while it is streamed into the aggregate, instead of as + a separate pass over the finished aggregate. This removes one full read and + one full write of the complete RDF output, and removes the transient + full-size temporary copy (which previously required 2x the aggregate size in + free disk at the end of a plain-storage run). +- `run_conversion.sh`: triple counting uses one `LC_ALL=C grep -c` instead of + `awk` / `grep | wc -l`. On a 228 MB gzip aggregate this is ~5x faster + (6.5s -> 1.2s); it is the last unavoidable full pass over the RDF output. +- `run_conversion.sh`: the `stat` dialect is detected once instead of probed on + every call, and the per-second progress heartbeat does one directory walk + with no `basename` subprocess per part. + +### Changed + +- Space-optimized aggregates are ~14 bytes per RMLStreamer part smaller, + because gzip members are now produced from a pipe and therefore no longer + embed the temporary part filename and mtime. Decompressed content is + unchanged and the output is more reproducible. + +### Removed + +- Dead host-side code: `resolve_input` (superseded by + `resolve_input_snapshot`), and the host chunk planners + `plan_record_safe_rdf_chunks`, `plan_partitioned_hdt_chunks`, + `split_nt_file_for_hdt`, `write_nt_chunk`, `iter_rdf_binary_lines`. Chunking + has been the container runner's job (`src/partitioned_compression.py`) since + partitioned compression moved into an ephemeral Docker volume; keeping a + second host-side implementation was exactly what + `run_partitioned_representation_methods_for_rdf_files` documents against. + Their two tests were removed with them. +- An unreachable `else:` arm in `run_full_mode`'s output summary (it iterated a + list the guard had just proved empty), the unused `input_metrics_target` + parameter of `run_full_mode`, an always-true `is not None` guard, and two + unused local assignments. + +### Documentation + +- `vcf_rdfizer.py` gained a module-level division-of-labour note and section + map, plus section banners for the previously unmarked regions (triple + counting, artifact naming, destructive filesystem operations, preflight + estimation, genotype representations, compression-plan parsing, metrics + layout). +- `README.md`: new "Repository Layout" section explaining the host/container + split and what each file does; fixed the broken `RELEASING.md` link + (`scripts/RELEASING.md`); added links to `docs/`, `changelog.md`, and + `ACKNOWLEDGEMENTS.md`. +- `src/compression.sh` and `src/update_rules.sh` headers now state that they + are standalone utilities the pipeline does not call, and what supersedes + each. `compression.sh` previously claimed the Python wrapper was its primary + caller, which has not been true since compression moved inline. +- Added `ACKNOWLEDGEMENTS.md` with funding/attribution and a paper + acknowledgement template. + ## 2026-09-04 — Validation progress and quiet mode - Integrated semantic validation with the existing JSONL progress sidecar and diff --git a/pyproject.toml b/pyproject.toml index 1500ab7..b42218e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ Issues = "https://github.com/ecrum19/VCF-RDFizer/issues" [project.scripts] vcf-rdfizer = "vcf_rdfizer:main" +vcf-rdfizer-rules = "vcf_rdfizer_rules:main" [project.optional-dependencies] dev = [ @@ -42,7 +43,7 @@ dev = [ ] [tool.setuptools] -py-modules = ["vcf_rdfizer"] +py-modules = ["vcf_rdfizer", "vcf_rdfizer_gzip", "vcf_rdfizer_rules"] packages = ["vcf_rdfizer_data", "vcf_rdfizer_data.rules"] include-package-data = true diff --git a/rules/README.md b/rules/README.md index afb6b1c..32edbe1 100644 --- a/rules/README.md +++ b/rules/README.md @@ -2,6 +2,18 @@ This directory contains RML mappings used by the conversion pipeline. +Writing your own mapping? Use the `vcf-rdfizer-rules` CLI, which documents the +available TSV columns, scaffolds a starting point, and validates a mapping +against the wrapper's contract before you spend a run on it: + +```bash +vcf-rdfizer-rules columns # what a mapping can reference +vcf-rdfizer-rules init -o my.ttl # annotated copy of default_rules.ttl +vcf-rdfizer-rules check my.ttl # validate before running the pipeline +``` + +See "Custom RML Mappings" in the top-level `README.md` for the full contract. + ## Files - `default_rules.ttl` diff --git a/src/compression.sh b/src/compression.sh deleted file mode 100644 index e426165..0000000 --- a/src/compression.sh +++ /dev/null @@ -1,643 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# ------------------------------------------------------------------------------ -# RDF compression runner -# ------------------------------------------------------------------------------ -# This script can be used standalone, but the Python wrapper is the primary -# caller. It performs selected compression methods over RDF outputs and updates -# run_metrics/metrics.csv with method-level timing/size/exit-code fields. -# -# NOTE: Compound HDT-first compression is orchestrated by the Python wrapper. -# Here we maintain explicit metric columns so CSV schema stays aligned. -# ------------------------------------------------------------------------------ - -# ---------- Config ---------- -# Root output directory; contains one or more output subdirs -OUT_ROOT_DIR=${OUT_ROOT_DIR:-run_output} -# Optional single output name (compress only this subdir) -OUT_NAME=${OUT_NAME:-} - -# Metrics directory -LOGDIR=${LOGDIR:-run_metrics} - -# rdf2hdt binary (HDT-cpp) -HDT=${RDF2HDT:-${RDF2HDT_BIN:-/usr/local/bin/rdf2hdt}} - -# Base URI for rdf2hdt (reserved for future use) -BASE_URI=${BASE_URI:-http://example.org/base} - -RUN_ID=${RUN_ID:-$(date +%Y%m%dT%H%M%S)} -TIMESTAMP=${TIMESTAMP:-$(date +"%Y-%m-%dT%H:%M:%S")} - -mkdir -p "$LOGDIR" "$OUT_ROOT_DIR" - -METRICS_CSV="$LOGDIR/metrics.csv" -METRICS_HEADER="run_id,timestamp,output_name,output_dir,exit_code_java,wall_seconds_java,user_seconds_java,sys_seconds_java,max_rss_kb_java,input_mapping_size_bytes,input_vcf_size_bytes,output_dir_size_bytes,output_triples,jar,mapping_file,output_path,combined_rdf_size_bytes,gzip_size_bytes,brotli_size_bytes,hdt_size_bytes,exit_code_gzip,exit_code_brotli,exit_code_hdt,wall_seconds_gzip,user_seconds_gzip,sys_seconds_gzip,max_rss_kb_gzip,wall_seconds_brotli,user_seconds_brotli,sys_seconds_brotli,max_rss_kb_brotli,wall_seconds_hdt,user_seconds_hdt,sys_seconds_hdt,max_rss_kb_hdt,compression_methods,hdt_source,gzip_on_hdt_size_bytes,brotli_on_hdt_size_bytes,exit_code_gzip_on_hdt,exit_code_brotli_on_hdt,wall_seconds_gzip_on_hdt,user_seconds_gzip_on_hdt,sys_seconds_gzip_on_hdt,max_rss_kb_gzip_on_hdt,wall_seconds_brotli_on_hdt,user_seconds_brotli_on_hdt,sys_seconds_brotli_on_hdt,max_rss_kb_brotli_on_hdt" - -# ---------- Compression selection ---------- -# Usage: compression.sh [-m gzip,brotli,hdt|none] -# Or set COMPRESSION_METHODS env var (default: gzip,brotli,hdt) -COMPRESSION_METHODS=${COMPRESSION_METHODS:-gzip,brotli,hdt} - -while getopts ":m:h" opt; do - case "$opt" in - m) COMPRESSION_METHODS="$OPTARG" ;; - h) - echo "Usage: $0 [-m gzip,brotli,hdt|none]" - exit 0 - ;; - \?) - echo "Error: invalid option -$OPTARG" >&2 - exit 2 - ;; - :) - echo "Error: option -$OPTARG requires an argument" >&2 - exit 2 - ;; - esac -done - -COMPRESSION_METHODS_CSV=${COMPRESSION_METHODS//,/|} - -DO_GZIP=0 -DO_BROTLI=0 -DO_HDT=0 - -if [[ -n "${COMPRESSION_METHODS// }" && "$COMPRESSION_METHODS" != "none" ]]; then - IFS=',' read -r -a METHODS_ARR <<< "$COMPRESSION_METHODS" - for method in "${METHODS_ARR[@]}"; do - m="${method// /}" - case "$m" in - gzip) DO_GZIP=1 ;; - brotli) DO_BROTLI=1 ;; - hdt) DO_HDT=1 ;; - "" ) ;; - *) - echo "Error: unsupported compression method '$m'. Use gzip,brotli,hdt, or none." >&2 - exit 2 - ;; - esac - done -fi - -ANY_COMPRESS=$((DO_GZIP + DO_BROTLI + DO_HDT)) - -if (( DO_HDT == 1 )); then - if [[ ! -x "$HDT" ]]; then - echo "Error: rdf2hdt binary not found or not executable at '$HDT'." >&2 - exit 2 - fi -fi - -# Resolve output directories to compress -OUTPUT_DIRS=() -if [[ -n "$OUT_NAME" ]]; then - OUTPUT_DIRS=("$OUT_ROOT_DIR/$OUT_NAME") -else - while IFS= read -r discovered_dir; do - OUTPUT_DIRS+=("$discovered_dir") - done < <(find "$OUT_ROOT_DIR" -maxdepth 1 -type d ! -path "$OUT_ROOT_DIR" | sort) - if (( ${#OUTPUT_DIRS[@]} == 0 )); then - OUTPUT_DIRS=("$OUT_ROOT_DIR") - fi -fi - -if (( ${#OUTPUT_DIRS[@]} == 0 )); then - echo "Error: no output directories found in '$OUT_ROOT_DIR'." >&2 - exit 2 -fi - -# ---------- Helper functions ---------- -# Return byte size for file or directory (GNU + BSD compatible). -stat_size() { - local path="$1" - - # --- CASE 1: Regular file --- - if [[ -f "$path" ]]; then - # Linux (GNU coreutils) - if stat -c%s "$path" >/dev/null 2>&1; then - stat -c%s "$path" - # macOS/BSD - elif stat -f%z "$path" >/dev/null 2>&1; then - stat -f%z "$path" - else - wc -c < "$path" | tr -d ' ' - fi - return - fi - - # --- CASE 2: Directory --- - if [[ -d "$path" ]]; then - # Linux (GNU du) - if du -sb "$path" >/dev/null 2>&1; then - du -sb "$path" | awk '{print $1}' - # macOS/BSD (no -b) - elif du -sk "$path" >/dev/null 2>&1; then - local kb - kb=$(du -sk "$path" | awk '{print $1}') - echo $((kb * 1024)) - else - echo 0 - fi - return - fi - - echo 0 -} - -have_gnu_time() { [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; } - -# Count triples via number of non-comment lines ending in ".". -count_triples_json() { - local path="$1" - local total=0 - - echo "{" - shopt -s nullglob - - for f in "$path"/*; do - if [[ -f "$f" ]]; then - local count - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$f" || true) | wc -l | tr -d ' ' ) - total=$((total + count)) - printf " \"%s\": %s,\n" "$f" "$count" - fi - done - - shopt -u nullglob - printf " \"TOTAL\": %s\n" "$total" - echo "}" -} - -# Convert elapsed clock text from `time` output to numeric seconds. -elapsed_to_seconds() { - awk -F':' '{ - if (NF==3) { h=$1+0; m=$2+0; s=$3+0; printf("%.3f", h*3600 + m*60 + s) } - else if (NF==2) { m=$1+0; s=$2+0; printf("%.3f", m*60 + s) } - else { s=$1+0; printf("%.3f", s) } - }' -} - -# CSV header -if [[ ! -f "$METRICS_CSV" ]]; then - echo "$METRICS_HEADER" > "$METRICS_CSV" -else - EXISTING_HEADER=$(head -n 1 "$METRICS_CSV") - if [[ "$EXISTING_HEADER" != "$METRICS_HEADER" ]]; then - BACKUP="$LOGDIR/metrics_csv_bak_${RUN_ID}.csv" - cp "$METRICS_CSV" "$BACKUP" - echo "WARNING: metrics header mismatch; backed up to $BACKUP and creating new metrics file." >&2 - echo "$METRICS_HEADER" > "$METRICS_CSV" - fi -fi - -# ---------- Main loop over output dirs ---------- -OVERALL_EXIT=0 - -for OUT in "${OUTPUT_DIRS[@]}"; do - if [[ ! -d "$OUT" ]]; then - echo "WARNING: output directory '$OUT' not found, skipping." >&2 - OVERALL_EXIT=1 - continue - fi - mkdir -p "$OUT" - - BASENAME=$(basename "$OUT") - SAFE_BASENAME=$(printf "%s" "$BASENAME" | tr -cs 'A-Za-z0-9._-' '_') - if [[ -z "$SAFE_BASENAME" ]]; then - SAFE_BASENAME="rdf" - fi - - TIME_LOG_GZIP_DIR="$LOGDIR/timings/compression/${SAFE_BASENAME}" - TIME_LOG_BROTLI_DIR="$LOGDIR/timings/compression/${SAFE_BASENAME}" - TIME_LOG_HDT_DIR="$LOGDIR/timings/compression/${SAFE_BASENAME}" - METRICS_JSON_DIR="$LOGDIR/stages/compression" - mkdir -p "$TIME_LOG_GZIP_DIR" "$TIME_LOG_BROTLI_DIR" "$TIME_LOG_HDT_DIR" "$METRICS_JSON_DIR" - TIME_LOG_GZIP="$TIME_LOG_GZIP_DIR/gzip.txt" - TIME_LOG_BROTLI="$TIME_LOG_BROTLI_DIR/brotli.txt" - TIME_LOG_HDT="$TIME_LOG_HDT_DIR/hdt.txt" - METRICS_JSON="$METRICS_JSON_DIR/${SAFE_BASENAME}.json" - - HDT_SOURCE="not_used" - GZIP_ON_HDT_SIZE=0 - BROTLI_ON_HDT_SIZE=0 - EXIT_CODE_GZIP_ON_HDT=0 - EXIT_CODE_BROTLI_ON_HDT=0 - WALL_SEC_GZIP_ON_HDT="null" - USER_SEC_GZIP_ON_HDT="null" - SYS_SEC_GZIP_ON_HDT="null" - MAX_RSS_KB_GZIP_ON_HDT="null" - WALL_SEC_BROTLI_ON_HDT="null" - USER_SEC_BROTLI_ON_HDT="null" - SYS_SEC_BROTLI_ON_HDT="null" - MAX_RSS_KB_BROTLI_ON_HDT="null" - - # Collect current output footprint/triples before compression. - OUT_SIZE=$(stat_size "$OUT") - TRIPLES_JSON=$(count_triples_json "$OUT") - TOTAL_TRIPLES=$(echo "$TRIPLES_JSON" | grep '"TOTAL"' | awk -F': ' '{print $2}' | tr -d '", ') - - shopt -s nullglob - NT_FILES=("$OUT"/*.nt) - shopt -u nullglob - PRIMARY_NT="$OUT/${BASENAME}.nt" - - if (( ${#NT_FILES[@]} == 0 )); then - echo "WARNING: no .nt files found in '$OUT'; skipping compression for this output." >&2 - SOURCE_RDF="" - SOURCE_EXT="" - RDF_SIZE=0 - GZ_PATH="" - GZ_SIZE=0 - BROTLI_PATH="" - BROTLI_SIZE=0 - HDT_PATH="" - HDT_SIZE=0 - EXIT_CODE_GZIP=$(( DO_GZIP == 1 ? 1 : 0 )) - EXIT_CODE_BROTLI=$(( DO_BROTLI == 1 ? 1 : 0 )) - EXIT_CODE_HDT=$(( DO_HDT == 1 ? 1 : 0 )) - WALL_SEC_GZIP="null" - USER_SEC_GZIP="null" - SYS_SEC_GZIP="null" - MAX_RSS_KB_GZIP="null" - WALL_SEC_BROTLI="null" - USER_SEC_BROTLI="null" - SYS_SEC_BROTLI="null" - MAX_RSS_KB_BROTLI="null" - WALL_SEC_HDT="null" - USER_SEC_HDT="null" - SYS_SEC_HDT="null" - MAX_RSS_KB_HDT="null" - if (( ANY_COMPRESS > 0 )); then - OVERALL_EXIT=1 - fi - else - if (( ANY_COMPRESS > 0 )); then - if [[ -f "$PRIMARY_NT" ]]; then - SOURCE_RDF="$PRIMARY_NT" - SOURCE_EXT="nt" - elif (( ${#NT_FILES[@]} == 1 )); then - SOURCE_RDF="${NT_FILES[0]}" - SOURCE_EXT="nt" - else - echo "WARNING: unable to determine a unique primary RDF file in '$OUT'." >&2 - SOURCE_RDF="" - SOURCE_EXT="" - fi - - if [[ -z "$SOURCE_RDF" ]]; then - RDF_SIZE=0 - EXIT_CODE_GZIP=$(( DO_GZIP == 1 ? 1 : 0 )) - EXIT_CODE_BROTLI=$(( DO_BROTLI == 1 ? 1 : 0 )) - EXIT_CODE_HDT=$(( DO_HDT == 1 ? 1 : 0 )) - WALL_SEC_GZIP="null" - USER_SEC_GZIP="null" - SYS_SEC_GZIP="null" - MAX_RSS_KB_GZIP="null" - WALL_SEC_BROTLI="null" - USER_SEC_BROTLI="null" - SYS_SEC_BROTLI="null" - MAX_RSS_KB_BROTLI="null" - WALL_SEC_HDT="null" - USER_SEC_HDT="null" - SYS_SEC_HDT="null" - MAX_RSS_KB_HDT="null" - OVERALL_EXIT=1 - else - RDF_SIZE=$(stat_size "$SOURCE_RDF") - fi - else - SOURCE_RDF="" - SOURCE_EXT="" - RDF_SIZE=0 - fi - - # ----- gzip raw RDF with timing ----- - if (( DO_GZIP == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then - GZ_PATH="$OUT/${BASENAME}.${SOURCE_EXT}.gz" - EXIT_CODE_GZIP=0 - - if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG_GZIP" -- gzip -c "$SOURCE_RDF" > "$GZ_PATH" || EXIT_CODE_GZIP=$? - else - { time -p gzip -c "$SOURCE_RDF" > "$GZ_PATH"; } >"$TIME_LOG_GZIP" 2>&1 || EXIT_CODE_GZIP=$? - fi - - GZ_SIZE=$(stat_size "$GZ_PATH") - - WALL_SEC_GZIP="" - USER_SEC_GZIP="" - SYS_SEC_GZIP="" - MAX_RSS_KB_GZIP="" - - if have_gnu_time; then - ELAPSED=$(awk -F': ' '/Elapsed \(wall clock\) time/ {print $2}' "$TIME_LOG_GZIP") - WALL_SEC_GZIP=$(printf "%s" "$ELAPSED" | elapsed_to_seconds) - - USER_SEC_GZIP=$(awk -F': ' '/User time \(seconds\)/ {print $2}' "$TIME_LOG_GZIP") - SYS_SEC_GZIP=$(awk -F': ' '/System time \(seconds\)/ {print $2}' "$TIME_LOG_GZIP") - MAX_RSS_KB_GZIP=$(awk -F': ' '/Maximum resident set size/ {print $2}' "$TIME_LOG_GZIP") - else - WALL_SEC_GZIP=$(awk '/^real/ {print $2}' "$TIME_LOG_GZIP") - USER_SEC_GZIP=$(awk '/^user/ {print $2}' "$TIME_LOG_GZIP") - SYS_SEC_GZIP=$(awk '/^sys/ {print $2}' "$TIME_LOG_GZIP") - MAX_RSS_KB_GZIP="" - fi - - [[ -z "$MAX_RSS_KB_GZIP" ]] && MAX_RSS_KB_GZIP="null" - if [[ "$EXIT_CODE_GZIP" -ne 0 ]]; then - OVERALL_EXIT=1 - fi - else - GZ_PATH="" - GZ_SIZE=0 - EXIT_CODE_GZIP=0 - WALL_SEC_GZIP="null" - USER_SEC_GZIP="null" - SYS_SEC_GZIP="null" - MAX_RSS_KB_GZIP="null" - fi - - # ----- brotli raw RDF with timing ----- - if (( DO_BROTLI == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then - BROTLI_PATH="$OUT/${BASENAME}.${SOURCE_EXT}.br" - EXIT_CODE_BROTLI=0 - - if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG_BROTLI" -- brotli -q 7 -c "$SOURCE_RDF" > "$BROTLI_PATH" || EXIT_CODE_BROTLI=$? - else - { time -p brotli -q 7 -c "$SOURCE_RDF" > "$BROTLI_PATH"; } >"$TIME_LOG_BROTLI" 2>&1 || EXIT_CODE_BROTLI=$? - fi - - BROTLI_SIZE=$(stat_size "$BROTLI_PATH") - - WALL_SEC_BROTLI="" - USER_SEC_BROTLI="" - SYS_SEC_BROTLI="" - MAX_RSS_KB_BROTLI="" - - if have_gnu_time; then - ELAPSED=$(awk -F': ' '/Elapsed \(wall clock\) time/ {print $2}' "$TIME_LOG_BROTLI") - WALL_SEC_BROTLI=$(printf "%s" "$ELAPSED" | elapsed_to_seconds) - USER_SEC_BROTLI=$(awk -F': ' '/User time \(seconds\)/ {print $2}' "$TIME_LOG_BROTLI") - SYS_SEC_BROTLI=$(awk -F': ' '/System time \(seconds\)/ {print $2}' "$TIME_LOG_BROTLI") - MAX_RSS_KB_BROTLI=$(awk -F': ' '/Maximum resident set size/ {print $2}' "$TIME_LOG_BROTLI") - else - WALL_SEC_BROTLI=$(awk '/^real/ {print $2}' "$TIME_LOG_BROTLI") - USER_SEC_BROTLI=$(awk '/^user/ {print $2}' "$TIME_LOG_BROTLI") - SYS_SEC_BROTLI=$(awk '/^sys/ {print $2}' "$TIME_LOG_BROTLI") - MAX_RSS_KB_BROTLI="" - fi - - [[ -z "$MAX_RSS_KB_BROTLI" ]] && MAX_RSS_KB_BROTLI="null" - if [[ "$EXIT_CODE_BROTLI" -ne 0 ]]; then - OVERALL_EXIT=1 - fi - else - BROTLI_PATH="" - BROTLI_SIZE=0 - EXIT_CODE_BROTLI=0 - WALL_SEC_BROTLI="null" - USER_SEC_BROTLI="null" - SYS_SEC_BROTLI="null" - MAX_RSS_KB_BROTLI="null" - fi - - # ----- Convert raw RDF to HDT with timing ----- - if (( DO_HDT == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then - HDT_PATH="$OUT/$BASENAME.hdt" - HDT_SOURCE="generated" - EXIT_CODE_HDT=0 - - HDT_CMD="\"$HDT\" \"$SOURCE_RDF\" \"$HDT_PATH\"" - - if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG_HDT" -- bash -lc "$HDT_CMD" || EXIT_CODE_HDT=$? - else - { time -p bash -lc "$HDT_CMD"; } >"$TIME_LOG_HDT" 2>&1 || EXIT_CODE_HDT=$? - fi - - HDT_SIZE=$(stat_size "$HDT_PATH") - - WALL_SEC_HDT="" - USER_SEC_HDT="" - SYS_SEC_HDT="" - MAX_RSS_KB_HDT="" - - if have_gnu_time; then - ELAPSED=$(awk -F': ' '/Elapsed \(wall clock\) time/ {print $2}' "$TIME_LOG_HDT") - WALL_SEC_HDT=$(printf "%s" "$ELAPSED" | elapsed_to_seconds) - - USER_SEC_HDT=$(awk -F': ' '/User time \(seconds\)/ {print $2}' "$TIME_LOG_HDT") - SYS_SEC_HDT=$(awk -F': ' '/System time \(seconds\)/ {print $2}' "$TIME_LOG_HDT") - MAX_RSS_KB_HDT=$(awk -F': ' '/Maximum resident set size/ {print $2}' "$TIME_LOG_HDT") - else - WALL_SEC_HDT=$(awk '/^real/ {print $2}' "$TIME_LOG_HDT") - USER_SEC_HDT=$(awk '/^user/ {print $2}' "$TIME_LOG_HDT") - SYS_SEC_HDT=$(awk '/^sys/ {print $2}' "$TIME_LOG_HDT") - MAX_RSS_KB_HDT="" - fi - - [[ -z "$MAX_RSS_KB_HDT" ]] && MAX_RSS_KB_HDT="null" - if [[ "$EXIT_CODE_HDT" -ne 0 ]]; then - if [[ "$EXIT_CODE_HDT" -eq 127 ]]; then - echo "ERROR: HDT conversion failed with exit code 127 (command not found)." >&2 - echo "ERROR: Check dependencies used by '$HDT' and the timing log '$TIME_LOG_HDT'." >&2 - if [[ -f "$TIME_LOG_HDT" ]]; then - tail -n 20 "$TIME_LOG_HDT" >&2 || true - fi - fi - OVERALL_EXIT=1 - fi - else - HDT_PATH="" - HDT_SIZE=0 - EXIT_CODE_HDT=0 - WALL_SEC_HDT="null" - USER_SEC_HDT="null" - SYS_SEC_HDT="null" - MAX_RSS_KB_HDT="null" - fi - fi - - # Persist per-output compression metrics as JSON. - cat > "$METRICS_JSON" < "$tmp_csv" - mv "$tmp_csv" "$METRICS_CSV" - - echo "Done for $OUT." - echo " JSON metrics: $METRICS_JSON" - echo -done - -echo "Compression finished." -echo "CSV summary: $METRICS_CSV" - -if [[ "$OVERALL_EXIT" -ne 0 ]]; then - echo "Compression completed with one or more errors." >&2 -fi -exit "$OVERALL_EXIT" diff --git a/src/run_conversion.sh b/src/run_conversion.sh index 776a9e5..3affc27 100644 --- a/src/run_conversion.sh +++ b/src/run_conversion.sh @@ -77,21 +77,35 @@ TSV_OUTPUT_PATH=${TSV_OUTPUT_PATH:-} METRICS_HEADER="run_id,timestamp,output_name,output_dir,exit_code_java,wall_seconds_java,user_seconds_java,sys_seconds_java,max_rss_kb_java,input_mapping_size_bytes,input_vcf_size_bytes,output_dir_size_bytes,output_triples,jar,mapping_file,output_path,exit_code_tsv,wall_seconds_tsv,user_seconds_tsv,sys_seconds_tsv,max_rss_kb_tsv,tsv_output_size_bytes,tsv_output_path" +# Detect the local `stat` dialect once. `stat_size` is called for every +# RMLStreamer part on every progress tick, so probing the dialect per call +# doubles the number of processes this script spawns for no benefit. +STAT_FILE_FLAVOR="" +detect_stat_file_flavor() { + if [[ -n "$STAT_FILE_FLAVOR" ]]; then + return + fi + if stat -c%s . >/dev/null 2>&1; then + STAT_FILE_FLAVOR="gnu" + elif stat -f%z . >/dev/null 2>&1; then + STAT_FILE_FLAVOR="bsd" + else + STAT_FILE_FLAVOR="wc" + fi +} + # Return byte size for file or directory (GNU + BSD compatible). stat_size() { local path="$1" # --- CASE 1: Regular file --- if [[ -f "$path" ]]; then - # Linux (GNU coreutils) - if stat -c%s "$path" >/dev/null 2>&1; then - stat -c%s "$path" - # macOS/BSD - elif stat -f%z "$path" >/dev/null 2>&1; then - stat -f%z "$path" - else - wc -c < "$path" | tr -d ' ' - fi + detect_stat_file_flavor + case "$STAT_FILE_FLAVOR" in + gnu) stat -c%s "$path" ;; + bsd) stat -f%z "$path" ;; + *) wc -c < "$path" | tr -d ' ' ;; + esac return fi @@ -139,48 +153,57 @@ progress_emit() { # RMLStreamer writes part files while it runs. Summing their metadata once per # second gives useful throughput feedback without reading or counting RDF. -part_bytes() { - local total=0 - local file size +# One directory walk fills both the byte total and the part count, and the +# leading-dot test uses parameter expansion rather than a `basename` process. +PART_BYTES=0 +PART_COUNT=0 +scan_parts() { + local file name size + PART_BYTES=0 + PART_COUNT=0 shopt -s nullglob for file in "$PARTS_DIR"/*; do - if [[ ! -f "$file" || "$(basename "$file")" == .* ]]; then + name="${file##*/}" + if [[ ! -f "$file" || "$name" == .* ]]; then continue fi size=$(stat_size "$file") - total=$((total + size)) + PART_BYTES=$((PART_BYTES + size)) + PART_COUNT=$((PART_COUNT + 1)) done shopt -u nullglob - echo "$total" -} - -part_count() { - local total=0 - local file - shopt -s nullglob - for file in "$PARTS_DIR"/*; do - if [[ -f "$file" && "$(basename "$file")" != .* ]]; then - total=$((total + 1)) - fi - done - shopt -u nullglob - echo "$total" } # Report comparable input VCF bytes. # - .vcf -> on-disk bytes # - .vcf.gz -> decompressed bytes # - dir -> sum of normalized sizes for contained .vcf/.vcf.gz files -normalized_vcf_size() { +# +# `vcf_rdfizer_gzip.py` answers this from the file's own structure for BGZF +# (what bgzip/bcftools/tabix emit) and for ordinary single-member gzip, so the +# common case costs milliseconds instead of a full decompression pass. It falls +# back to inflating internally when the structure cannot settle the answer, and +# the shell falls back to `gzip -dc` if the helper is unavailable. The method +# used is recorded alongside the size so the metric stays auditable. +VCF_SIZE_HELPER=${VCF_SIZE_HELPER:-/opt/vcf-rdfizer/vcf_rdfizer_gzip.py} + +# Prints " ". Callers run this in a command substitution, so the +# method has to travel out with the value rather than through a global. +vcf_size_with_method() { local path="$1" local total=0 + local method="" entry entry_size entry_method measured if [[ -f "$path" ]]; then if [[ "$path" == *.vcf.gz ]]; then - gzip -dc "$path" | wc -c | tr -d ' ' + if [[ -f "$VCF_SIZE_HELPER" ]] && measured=$(python3 "$VCF_SIZE_HELPER" --print-method "$path" 2>/dev/null); then + printf '%s\n' "$measured" + return + fi + printf '%s inflate-shell\n' "$(gzip -dc "$path" | wc -c | tr -d ' ')" return fi - stat_size "$path" + printf '%s stat\n' "$(stat_size "$path")" return fi @@ -190,15 +213,29 @@ normalized_vcf_size() { if [[ ! -f "$file" ]]; then continue fi - size=$(normalized_vcf_size "$file") - total=$((total + size)) + entry=$(vcf_size_with_method "$file") + entry_size="${entry%% *}" + entry_method="${entry#* }" + total=$((total + entry_size)) + if [[ -z "$method" ]]; then + method="$entry_method" + elif [[ "$method" != "$entry_method" ]]; then + method="mixed" + fi done shopt -u nullglob - echo "$total" + printf '%s %s\n' "$total" "${method:-none}" return fi - echo 0 + printf '0 none\n' +} + +# Backwards-compatible size-only accessor for direct callers of this script. +normalized_vcf_size() { + local result + result=$(vcf_size_with_method "$1") + printf '%s\n' "${result%% *}" } have_gnu_time() { [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; } @@ -219,17 +256,31 @@ hash_file_sha256() { } # Count triples via non-comment RDF lines ending in '.'. +# +# `grep -c` counts in one process and, in the C locale, works byte-wise instead +# of validating UTF-8 for every line. On a cohort-scale aggregate this is +# several times faster than piping through `awk` or `wc -l`, and it is the last +# unavoidable full pass over the RDF output. `grep` exits 1 when nothing +# matched, so the `|| true` sits inside the pipeline's last stage to keep a +# genuine `gzip` failure visible under `set -o pipefail`. +TRIPLE_LINE_REGEX='^[[:space:]]*[^#].*\.[[:space:]]*$' + +count_triple_lines() { + local path="$1" + if [[ "$path" == *.gz ]]; then + gzip -dc "$path" | { LC_ALL=C grep -cE "$TRIPLE_LINE_REGEX" || true; } + return + fi + { LC_ALL=C grep -cE "$TRIPLE_LINE_REGEX" "$path" || true; } +} + count_triples_json() { local path="$1" local total=0 if [[ -f "$path" ]]; then local count - if [[ "$path" == *.gz ]]; then - count=$(gzip -dc "$path" | awk '/^[[:space:]]*[^#].*\.[[:space:]]*$/ { count++ } END { print count + 0 }') - else - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$path" || true) | wc -l | tr -d ' ' ) - fi + count=$(count_triple_lines "$path") echo "{" printf " \"%s\": %s,\n" "$path" "$count" printf " \"TOTAL\": %s\n" "$count" @@ -243,7 +294,7 @@ count_triples_json() { for f in "$path"/*; do if [[ -f "$f" ]]; then local count - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$f" || true) | wc -l | tr -d ' ' ) + count=$(count_triple_lines "$f") total=$((total + count)) printf " \"%s\": %s,\n" "$f" "$count" fi @@ -257,12 +308,14 @@ count_triples_json() { # Replace plain VCF null marker literals (`"."`) with typed null literals. # Ontology alignment: # "." -> "."^^vcfr:Null -annotate_null_literals_nt() { - local nt_path="$1" - local tmp_path - tmp_path="$(mktemp "${nt_path}.nullfix.XXXXXX")" - sed -E 's/"\."([[:space:]]+\.)/"."^^\1/g' "$nt_path" > "$tmp_path" - mv "$tmp_path" "$nt_path" +# +# The rewrite is per-line and has no cross-line context, so it is applied to +# each RMLStreamer part as it is streamed into the aggregate. Running it as a +# separate pass over the finished aggregate instead would read and rewrite the +# complete RDF output a second time. +NULL_LITERAL_SED='s/"\."([[:space:]]+\.)/"."^^\1/g' +annotate_null_literals_stream() { + sed -E "$NULL_LITERAL_SED" "$1" } @@ -306,7 +359,9 @@ fi # ---------- Pre-run ---------- IN_SIZE=$(stat_size "$IN") -VCF_SIZE=$(normalized_vcf_size "$IN_VCF") +VCF_SIZE_INFO=$(vcf_size_with_method "$IN_VCF") +VCF_SIZE="${VCF_SIZE_INFO%% *}" +VCF_SIZE_METHOD="${VCF_SIZE_INFO#* }" # ---------- Run RMLStreamer with timing ---------- EXIT_CODE=0 @@ -323,14 +378,16 @@ if [[ -n "$PROGRESS_FILE" ]]; then run_rmlstreamer & RMLSTREAMER_PID=$! while kill -0 "$RMLSTREAMER_PID" >/dev/null 2>&1; do - progress_emit "rmlstreamer" "heartbeat" "$(part_bytes)" null bytes "$(part_count)" + scan_parts + progress_emit "rmlstreamer" "heartbeat" "$PART_BYTES" null bytes "$PART_COUNT" sleep 1 done wait "$RMLSTREAMER_PID" || EXIT_CODE=$? + scan_parts if (( EXIT_CODE == 0 )); then - progress_emit "rmlstreamer" "complete" "$(part_bytes)" null bytes "$(part_count)" + progress_emit "rmlstreamer" "complete" "$PART_BYTES" null bytes "$PART_COUNT" else - progress_emit "rmlstreamer" "failed" "$(part_bytes)" null bytes "$(part_count)" + progress_emit "rmlstreamer" "failed" "$PART_BYTES" null bytes "$PART_COUNT" fi else run_rmlstreamer || EXIT_CODE=$? @@ -383,10 +440,9 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then fi printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" printf "%s\t%s\n" "$PART_HASH" "$PART_NT" >> "$SEEN_MAP_FILE" - annotate_null_literals_nt "$PART_NT" # Concatenated gzip members form one valid sequential gzip stream while # allowing each completed RMLStreamer part to be deleted immediately. - gzip -c "$PART_NT" >> "$MERGED_TMP" + annotate_null_literals_stream "$PART_NT" | gzip -c >> "$MERGED_TMP" rm -f "$PART_NT" PART_INDEX=$((PART_INDEX + 1)) progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" @@ -418,7 +474,7 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then fi printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" printf "%s\t%s\n" "$PART_HASH" "$PART_NT" >> "$SEEN_MAP_FILE" - cat "$PART_NT" >> "$MERGED_NT" + annotate_null_literals_stream "$PART_NT" >> "$MERGED_NT" rm -f "$PART_NT" PART_INDEX=$((PART_INDEX + 1)) progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" @@ -439,17 +495,6 @@ fi rm -rf "$PARTS_DIR" trap - EXIT -# Apply ontology-compliant null datatype annotation to produced RDF files. -if [[ -f "$OUTPUT_PATH" && "$OUTPUT_PATH" != *.gz ]]; then - annotate_null_literals_nt "$OUTPUT_PATH" -elif [[ -d "$OUTPUT_PATH" ]]; then - shopt -s nullglob - for RDF_NT in "$OUTPUT_PATH"/*.nt; do - annotate_null_literals_nt "$RDF_NT" - done - shopt -u nullglob -fi - OUT_SIZE=$(stat_size "$OUTPUT_PATH") TRIPLES_JSON=$(count_triples_json "$OUTPUT_PATH") @@ -490,6 +535,7 @@ cat > "$METRICS_JSON" <&2 - exit 1 -fi - -RECORDS_TSV="$1" -HEADERS_TSV="${2:-/data/tsv/header_lines.tsv}" -METADATA_TSV="${3:-/data/tsv/file_metadata.tsv}" -if [ "$#" -eq 4 ]; then - RULES_FILE="$4" -fi - -# Make sure rules file exists -if [ ! -f "$RULES_FILE" ]; then - echo "Error: $RULES_FILE not found in current directory." >&2 - exit 1 -fi - -escape_replacement() { - # Escape characters meaningful to sed replacement text. - printf '%s' "$1" | sed -e 's/[&#]/\\&/g' -} - -ESCAPED_RECORDS_TSV=$(escape_replacement "$RECORDS_TSV") -ESCAPED_HEADERS_TSV=$(escape_replacement "$HEADERS_TSV") -ESCAPED_METADATA_TSV=$(escape_replacement "$METADATA_TSV") - -# Replace default TSV locations in the mapping. -sed -i.bak -E \ - -e "s#(csvw:url \")/data/tsv/records\\.tsv(\";)#\1${ESCAPED_RECORDS_TSV}\2#g" \ - -e "s#(csvw:url \")/data/tsv/header_lines\\.tsv(\";)#\1${ESCAPED_HEADERS_TSV}\2#g" \ - -e "s#(csvw:url \")/data/tsv/file_metadata\\.tsv(\";)#\1${ESCAPED_METADATA_TSV}\2#g" \ - "$RULES_FILE" - -echo "Updated TSV source paths in $RULES_FILE" -echo " records: $RECORDS_TSV" -echo " headers: $HEADERS_TSV" -echo " metadata: $METADATA_TSV" -echo "Backup created as ${RULES_FILE}.bak" diff --git a/test/README.md b/test/README.md index 108d4e2..267c740 100644 --- a/test/README.md +++ b/test/README.md @@ -24,10 +24,20 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - Verifies output normalization to `.nt`. - Verifies unified metrics CSV row creation and schema consistency. -- `test/test_compression_unit.py` - - Replaces `gzip`, `brotli`, and `rdf2hdt` with fake executables for the standalone helper. - - Verifies helper compression artifact generation and metrics row update. - - Verifies no-op behavior (no compression outputs, metrics still updated). +- `test/test_partitioned_compression_unit.py` and `test/test_cottas_tool.py` + - Exercise the container-side chunking, merge, and COTTAS adapter logic. + +- `test/test_rules_helper_unit.py` + - Verifies the `vcf-rdfizer-rules` contract checks (source paths, column + references, sample-representation compatibility, helper-table warnings). + - Pins the documented TSV column lists to the headers `src/vcf_as_tsv.sh` + actually writes, so the two cannot drift apart. + +- `test/test_gzip_size_unit.py` + - Verifies uncompressed-size measurement for BGZF, single-member gzip, and + concatenated members, each against a full-inflate ground truth. + - Verifies that an unresolvable file falls back rather than reporting a wrong + size, including the 32-bit `ISIZE` wrap and multi-member trailers. ## CI matrix behavior diff --git a/test/test_compression_unit.py b/test/test_compression_unit.py deleted file mode 100644 index 897be27..0000000 --- a/test/test_compression_unit.py +++ /dev/null @@ -1,377 +0,0 @@ -import csv -import os -import subprocess -import tempfile -import unittest -from pathlib import Path - -from test.helpers import VerboseTestCase, env_with_path, make_executable, seed_conversion_metrics_row - - -REPO_ROOT = Path(__file__).resolve().parents[1] -SCRIPT = REPO_ROOT / "src" / "compression.sh" - - -def prepare_fake_tools(bin_dir: Path, fail_gzip: bool = False, fail_brotli: bool = False, fail_hdt: bool = False): - gzip_fail = "exit 9\n" if fail_gzip else "" - brotli_fail = "exit 8\n" if fail_brotli else "" - hdt_fail = "exit 7\n" if fail_hdt else "" - make_executable( - bin_dir / "gzip", - """#!/usr/bin/env bash -set -euo pipefail -{gzip_fail}file="" -for arg in "$@"; do - if [[ "$arg" != -* ]]; then - file="$arg" - fi -done -cat "$file" -""".format(gzip_fail=gzip_fail), - ) - make_executable( - bin_dir / "brotli", - """#!/usr/bin/env bash -set -euo pipefail -{brotli_fail}file="" -for arg in "$@"; do - if [[ "$arg" != -* ]]; then - file="$arg" - fi -done -cat "$file" -""".format(brotli_fail=brotli_fail), - ) - hdt = bin_dir / "rdf2hdt.sh" - make_executable( - hdt, - """#!/usr/bin/env bash -set -euo pipefail -{hdt_fail}cp "$1" "$2" -""".format(hdt_fail=hdt_fail), - ) - return hdt - - -def read_metrics_row(metrics_csv: Path, run_id: str, output_name: str): - with metrics_csv.open() as f: - rows = list(csv.DictReader(f)) - for row in rows: - if row["run_id"] == run_id and row["output_name"] == output_name: - return row - raise AssertionError(f"Metrics row not found for run_id={run_id}, output_name={output_name}") - - -class CompressionUnitTests(VerboseTestCase): - def test_compression_errors_for_invalid_cli_option(self): - """Invalid CLI option: compression script exits non-zero with an option error.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - result = subprocess.run( - ["bash", str(SCRIPT), "-z"], - cwd=tmp_path, - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("invalid option", result.stderr) - - def test_compression_errors_for_unsupported_method(self): - """Unsupported compression method: script exits non-zero with clear guidance.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - (out_root / "rdf").mkdir(parents=True) - env = {"OUT_ROOT_DIR": str(out_root), "LOGDIR": str(tmp_path / "metrics")} - env.update({"PATH": os.environ["PATH"]}) - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "snappy"], - env=env, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 2) - self.assertIn("unsupported compression method", result.stderr) - - def test_compression_errors_when_hdt_requested_but_binary_missing(self): - """HDT requested with missing binary: script exits non-zero before processing.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - env = {"OUT_ROOT_DIR": str(out_root), "OUT_NAME": "rdf", "LOGDIR": str(tmp_path / "metrics")} - env.update({"PATH": os.environ["PATH"], "RDF2HDT": str(tmp_path / "missing.sh")}) - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "hdt"], - env=env, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 2) - self.assertIn("rdf2hdt binary not found", result.stderr) - - def test_compression_updates_existing_metrics_row_with_mocked_tools(self): - """Compression mode gzip|brotli|hdt updates existing metrics row and writes artifacts.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n .\n") - - logdir = tmp_path / "metrics" - run_id = "run-compress-1" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "gzip,brotli,hdt"], - env=env, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((output / "rdf.nt.gz").exists()) - self.assertTrue((output / "rdf.nt.br").exists()) - self.assertTrue((output / "rdf.hdt").exists()) - self.assertTrue((logdir / "timings" / "compression" / "rdf" / "gzip.txt").exists()) - self.assertTrue((logdir / "timings" / "compression" / "rdf" / "brotli.txt").exists()) - self.assertTrue((logdir / "timings" / "compression" / "rdf" / "hdt.txt").exists()) - self.assertTrue((logdir / "stages" / "compression" / "rdf.json").exists()) - - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertEqual(row["run_id"], run_id) - self.assertEqual(row["output_name"], "rdf") - self.assertEqual(row["exit_code_java"], "0") - self.assertEqual(row["compression_methods"], "gzip|brotli|hdt") - self.assertEqual(row["exit_code_gzip"], "0") - self.assertEqual(row["exit_code_brotli"], "0") - self.assertEqual(row["exit_code_hdt"], "0") - self.assertGreater(int(row["combined_rdf_size_bytes"]), 0) - - def test_compression_prefers_nt_when_present(self): - """Compression mode uses primary .nt output when multiple .nt files are present.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - (output / "other.nt").write_text("

.\n") - - logdir = tmp_path / "metrics" - run_id = "run-prefers-nt" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "gzip,brotli,hdt"], - env=env, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((output / "rdf.nt.gz").exists()) - self.assertTrue((output / "rdf.nt.br").exists()) - self.assertTrue((output / "rdf.hdt").exists()) - - def test_compression_none_updates_metrics_without_generating_outputs(self): - """Compression mode none leaves no compressed artifacts and records zero sizes.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - - logdir = tmp_path / "metrics" - run_id = "run-compress-2" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "none"], - env=env, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertFalse((output / "rdf.nt.gz").exists()) - self.assertFalse((output / "rdf.nt.br").exists()) - self.assertFalse((output / "rdf.hdt").exists()) - - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertEqual(row["compression_methods"], "none") - self.assertEqual(row["combined_rdf_size_bytes"], "0") - self.assertEqual(row["gzip_size_bytes"], "0") - self.assertEqual(row["brotli_size_bytes"], "0") - self.assertEqual(row["hdt_size_bytes"], "0") - - def test_compression_fails_when_no_nt_files_are_available_for_requested_methods(self): - """Requested compression with no .nt files returns non-zero and records method failures.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - logdir = tmp_path / "metrics" - run_id = "run-no-nt" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "gzip,brotli,hdt"], - env=env, - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertEqual(row["exit_code_gzip"], "1") - self.assertEqual(row["exit_code_brotli"], "1") - self.assertEqual(row["exit_code_hdt"], "1") - - def test_compression_backs_up_metrics_file_on_header_mismatch(self): - """Header mismatch in metrics.csv creates a backup and rewrites a compatible header.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - logdir = tmp_path / "metrics" - logdir.mkdir(parents=True, exist_ok=True) - (logdir / "metrics.csv").write_text("bad,header\nx,y\n") - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": "run-hdr", - "TIMESTAMP": "2026-01-01T00:00:00", - "RDF2HDT": str(hdt_path), - } - ) - result = subprocess.run(["bash", str(SCRIPT), "-m", "gzip"], env=env, capture_output=True, text=True) - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((logdir / "metrics_csv_bak_run-hdr.csv").exists()) - - def test_compression_reports_failure_when_gzip_fails(self): - """gzip failure path returns non-zero and records gzip exit code.""" - self._assert_method_failure(methods="gzip", fail_gzip=True, expected_field="exit_code_gzip") - - def test_compression_reports_failure_when_brotli_fails(self): - """brotli failure path returns non-zero and records brotli exit code.""" - self._assert_method_failure(methods="brotli", fail_brotli=True, expected_field="exit_code_brotli") - - def test_compression_reports_failure_when_hdt_fails(self): - """hdt failure path returns non-zero and records hdt exit code.""" - self._assert_method_failure(methods="hdt", fail_hdt=True, expected_field="exit_code_hdt") - - def _assert_method_failure(self, methods: str, expected_field: str, fail_gzip: bool = False, fail_brotli: bool = False, fail_hdt: bool = False): - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - logdir = tmp_path / "metrics" - run_id = f"run-{methods}-fail" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin, fail_gzip=fail_gzip, fail_brotli=fail_brotli, fail_hdt=fail_hdt) - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - result = subprocess.run(["bash", str(SCRIPT), "-m", methods], env=env, capture_output=True, text=True) - self.assertNotEqual(result.returncode, 0) - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertNotEqual(row[expected_field], "0") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_gzip_size_unit.py b/test/test_gzip_size_unit.py new file mode 100644 index 0000000..00663cc --- /dev/null +++ b/test/test_gzip_size_unit.py @@ -0,0 +1,241 @@ +import gzip +import struct +import tempfile +import unittest +import zlib +from pathlib import Path + +import vcf_rdfizer +import vcf_rdfizer_gzip +from test.helpers import VerboseTestCase + +VCF_LINE = "20\t%d\trs%d\tA\tG\t50\tPASS\tAC=1;AN=2;DP=%d\tGT:AD:DP:GQ:PL\t0|1:35,0:35:99:0,99,1013\n" + + +def vcf_body(count: int, start: int = 0) -> bytes: + return "".join( + VCF_LINE % (i, i, i % 90) for i in range(start, start + count) + ).encode("utf-8") + + +def write_bgzf(path: Path, payload: bytes, block_size: int = 65280) -> None: + """Write a real BGZF (bgzip) file: gzip members carrying a BC extra subfield.""" + with path.open("wb") as out: + for offset in range(0, len(payload), block_size): + chunk = payload[offset : offset + block_size] + compressor = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS) + body = compressor.compress(chunk) + compressor.flush() + total = len(body) + 26 + header = ( + bytes([31, 139, 8, 4, 0, 0, 0, 0, 0, 255]) + + struct.pack(" int: + total = 0 + with gzip.open(path, "rb") as handle: + while True: + chunk = handle.read(1 << 20) + if not chunk: + return total + total += len(chunk) + + +class GzipSizeUnitTests(VerboseTestCase): + def assert_matches_truth(self, path: Path, expected_method: str | None = None): + size, method = vcf_rdfizer_gzip.uncompressed_size(path) + self.assertEqual(size, inflate_truth(path), f"wrong size via {method}") + if expected_method is not None: + self.assertEqual(method, expected_method) + return size, method + + def test_bgzf_size_is_exact_without_inflating(self): + """A bgzip file is measured from its block headers alone.""" + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "cohort.vcf.gz" + payload = vcf_body(40_000) + write_bgzf(path, payload) + size, _ = self.assert_matches_truth(path, "bgzf") + self.assertEqual(size, len(payload)) + + def test_bgzf_single_block_and_eof_marker_only(self): + """A one-block file and a bare EOF marker both measure correctly.""" + with tempfile.TemporaryDirectory() as td: + small = Path(td) / "small.vcf.gz" + write_bgzf(small, b"tiny\n") + self.assert_matches_truth(small, "bgzf") + + empty = Path(td) / "empty.vcf.gz" + write_bgzf(empty, b"") + size, method = vcf_rdfizer_gzip.uncompressed_size(empty) + self.assertEqual((size, method), (0, "bgzf")) + + def test_single_member_gzip_within_the_sample_budget_is_measured(self): + """A small plain gzip is measured outright rather than extrapolated.""" + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "small.vcf.gz" + path.write_bytes(gzip.compress(vcf_body(2_000))) + self.assert_matches_truth(path, "gzip-sample") + + def test_large_single_member_gzip_resolves_via_the_trailer(self): + """Past the sample budget the ISIZE trailer is resolved by sampled ratio.""" + with tempfile.TemporaryDirectory() as td: + # Comfortably past the sample budget so the member cannot finish + # inside it, which is what forces the trailer-resolution path. + payload = vcf_body(400_000) + self.assertGreater( + len(payload), 2 * vcf_rdfizer_gzip.RATIO_SAMPLE_BYTES + ) + for level in (1, 6, 9): + path = Path(td) / f"big-{level}.vcf.gz" + path.write_bytes(gzip.compress(payload, level)) + self.assert_matches_truth(path, "gzip-trailer") + + def test_trailer_resolution_recovers_a_size_past_the_32_bit_wrap(self): + """ISIZE is modulo 2**32; the resolved size adds back the missing wraps.""" + compressed_size = 634_493_967 + true_size = 4_600_694_700 + wrapped = true_size % vcf_rdfizer_gzip.ISIZE_MODULUS + self.assertNotEqual(wrapped, true_size) + + # Reproduce the resolution step against a file whose true size cannot be + # expressed by the trailer, without materialising 4.3 GiB on disk. + ratio = true_size / compressed_size + projected = ratio * compressed_size + upper_bound = compressed_size * vcf_rdfizer_gzip.MAX_DEFLATE_RATIO + accepted = [ + candidate + for candidate in range( + wrapped, + int(upper_bound) + vcf_rdfizer_gzip.ISIZE_MODULUS, + vcf_rdfizer_gzip.ISIZE_MODULUS, + ) + if candidate <= upper_bound + and abs(candidate - projected) + <= vcf_rdfizer_gzip.RATIO_TOLERANCE * projected + ] + self.assertEqual(accepted, [true_size]) + + def test_concatenated_members_fall_back_instead_of_trusting_the_trailer(self): + """A trailer covering only the last member must never be used as-is.""" + with tempfile.TemporaryDirectory() as td: + first = vcf_body(120_000) + second = vcf_body(120_000, start=120_000) + + equal = Path(td) / "equal.vcf.gz" + equal.write_bytes(gzip.compress(first) + gzip.compress(second)) + self.assert_matches_truth(equal, "inflate") + + lopsided = Path(td) / "lopsided.vcf.gz" + lopsided.write_bytes(gzip.compress(first) + gzip.compress(b"tail\n")) + self.assert_matches_truth(lopsided, "inflate") + + many = Path(td) / "many.vcf.gz" + many.write_bytes( + b"".join( + gzip.compress(first[i : i + 50_000]) + for i in range(0, len(first), 50_000) + ) + ) + self.assert_matches_truth(many, "inflate") + + def test_optional_gzip_header_fields_are_skipped(self): + """FEXTRA, FNAME, and FCOMMENT headers do not confuse the sampler.""" + with tempfile.TemporaryDirectory() as td: + payload = vcf_body(2_000) + deflate_and_trailer = gzip.compress(payload)[10:] + extra = struct.pack("<2sH", b"ZZ", 3) + b"abc" + header = ( + bytes([31, 139, 8, 0x04 | 0x08 | 0x10, 0, 0, 0, 0, 0, 255]) + + struct.pack(" +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHG001\tHG002 +20\t100\trs1\tA\tG\t50\tPASS\tAC=1\tGT\t0|1\t1/1 +""" + + +def invoke(argv): + """Run the rules CLI, returning (exit_code, stdout, stderr).""" + out, err = StringIO(), StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = vcf_rdfizer_rules.main(argv) + return code, out.getvalue(), err.getvalue() + + +class RulesHelperUnitTests(VerboseTestCase): + def test_documented_columns_match_what_vcf_as_tsv_actually_writes(self): + """The documented TSV schemas cannot drift from the generator script.""" + if sys.platform.startswith("win"): + self.skipTest("vcf_as_tsv.sh requires a POSIX shell") + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text(SAMPLE_VCF, encoding="utf-8") + out_dir = tmp_path / "tsv" + result = subprocess.run( + ["bash", str(VCF_AS_TSV), str(vcf_path), str(out_dir)], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + emitted = {} + for name in ("records", "header_lines", "file_metadata"): + path = out_dir / f"sample.{name}.tsv" + with path.open(newline="", encoding="utf-8") as handle: + emitted[name] = next(csv.reader(handle, delimiter="\t")) + + # The records header's final column is the whitespace-joined sample + # ids, which is exactly why it is modelled as a dynamic tail. + records = vcf_rdfizer_rules.TSV_SOURCES["records"] + self.assertEqual(emitted["records"][:-1], list(records.columns)) + self.assertEqual(emitted["records"][-1], "HG001 HG002") + + for name in ("header_lines", "file_metadata"): + self.assertEqual( + emitted[name], + list(vcf_rdfizer_rules.TSV_SOURCES[name].columns), + f"{name}.tsv header drifted from the documented columns", + ) + + def test_helper_table_columns_come_from_the_wrapper_constants(self): + """The two helper tables reuse the wrapper's own header definitions.""" + self.assertEqual( + list(vcf_rdfizer_rules.TSV_SOURCES["sample_calls"].columns), + list(vcf_rdfizer.SAMPLE_CALLS_HEADER), + ) + self.assertEqual( + list(vcf_rdfizer_rules.TSV_SOURCES["sample_format_values"].columns), + list(vcf_rdfizer.SAMPLE_FORMAT_HEADER), + ) + + def test_documented_source_paths_match_the_wrapper_rewrite_targets(self): + """Every advertised source path is one render_rules_for_triplet rewrites.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + rendered = tmp_path / "rendered.ttl" + vcf_rdfizer.render_rules_for_triplet( + DEFAULT_RULES, + rendered, + "s.records.tsv", + "s.header_lines.tsv", + "s.file_metadata.tsv", + "s.sample_calls.tsv", + "s.sample_format_values.tsv", + ) + text = rendered.read_text(encoding="utf-8") + for path in vcf_rdfizer_rules.CANONICAL_SOURCE_PATHS: + self.assertNotIn( + f'csvw:url "{path}"', + text, + f"{path} survived rendering, so the wrapper does not rewrite it", + ) + + def test_check_accepts_the_shipped_default_mapping(self): + """The default mapping satisfies the contract the checker enforces.""" + code, stdout, _ = invoke(["check", str(DEFAULT_RULES)]) + self.assertEqual(code, 0) + self.assertIn("Result: OK", stdout) + self.assertIn("--sample-representation condensed: usable", stdout) + + def test_check_rejects_non_canonical_source_paths(self): + """A hard-coded TSV path the wrapper cannot rewrite is an error.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8").replace( + "/data/tsv/records.tsv", "/data/tsv/my_records.tsv" + ), + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("Result: FAILED", stdout) + self.assertIn("/data/tsv/my_records.tsv", stdout) + + def test_check_rejects_columns_no_tsv_provides(self): + """A misspelled column reference is caught before the pipeline runs.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8").replace( + 'rml:reference "CHROM"', 'rml:reference "CHROMOSOME"' + ), + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("CHROMOSOME", stdout) + + def test_check_reports_template_variables_as_referenced_columns(self): + """Columns used only inside rr:template are checked too.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8").replace( + "file://{SOURCE_FILE}#call/{ROW_ID}", + "file://{SOURCE_FILE}#call/{NO_SUCH_COLUMN}", + ), + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("NO_SUCH_COLUMN", stdout) + + def test_check_warns_about_materialized_helper_tables(self): + """A custom helper-table consumer is flagged, and blocks condensed mode.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8") + + """ +<#ExtraSampleMap> a rr:TriplesMap; + rml:logicalSource [ + rml:source [ a csvw:Table; csvw:url "/data/tsv/sample_calls.tsv" ]; + rml:referenceFormulation ql:CSV + ]; + rr:subjectMap [ rr:template "file://{SOURCE_FILE}#extra/{ROW_ID}" ]; + rr:predicateObjectMap [ + rr:predicate vcfr:rawPayload; + rr:objectMap [ rml:reference "SAMPLE_PAYLOAD" ] + ]. +""", + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + # Materialization is a cost, not a contract violation. + self.assertEqual(code, 0) + self.assertIn("materialized", stdout) + self.assertIn("--sample-representation condensed: NOT usable", stdout) + + def test_check_rejects_a_file_that_is_not_an_rml_mapping(self): + """A Turtle file with no logical source fails rather than silently passing.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "plain.ttl" + rules.write_text( + "@prefix ex: .\nex:a ex:b ex:c .\n", encoding="utf-8" + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("does not look like an RML mapping", stdout) + + def test_check_emits_machine_readable_json(self): + """--json exposes the same findings for scripted use.""" + code, stdout, _ = invoke(["check", str(DEFAULT_RULES), "--json"]) + payload = json.loads(stdout) + self.assertEqual(code, 0) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["errors"], []) + self.assertEqual( + sorted(payload["sources"]), sorted(vcf_rdfizer_rules.CANONICAL_SOURCE_PATHS) + ) + + def test_init_scaffolds_a_checkable_mapping_and_refuses_to_clobber(self): + """init produces a file that passes check, and never overwrites silently.""" + with tempfile.TemporaryDirectory() as td: + output = Path(td) / "mine.ttl" + code, _, _ = invoke(["init", "-o", str(output)]) + self.assertEqual(code, 0) + self.assertTrue(output.is_file()) + self.assertIn("Custom VCF-RDFizer RML mapping", output.read_text(encoding="utf-8")) + + check_code, stdout, _ = invoke(["check", str(output)]) + self.assertEqual(check_code, 0, stdout) + + clobber_code, _, stderr = invoke(["init", "-o", str(output)]) + self.assertEqual(clobber_code, 2) + self.assertIn("already exists", stderr) + + force_code, _, _ = invoke(["init", "-o", str(output), "--force"]) + self.assertEqual(force_code, 0) + + def test_columns_lists_every_source_and_supports_json(self): + """columns documents all five sources the pipeline generates.""" + code, stdout, _ = invoke(["columns"]) + self.assertEqual(code, 0) + for name, source in vcf_rdfizer_rules.TSV_SOURCES.items(): + self.assertIn(name, stdout) + self.assertIn(source.container_path, stdout) + + code, stdout, _ = invoke(["columns", "--json"]) + payload = json.loads(stdout) + self.assertEqual(code, 0) + self.assertEqual(sorted(payload), sorted(vcf_rdfizer_rules.TSV_SOURCES)) + self.assertEqual(payload["records"]["dynamic_tail_column"], "SAMPLES") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_update_rules_unit.py b/test/test_update_rules_unit.py deleted file mode 100644 index 6dc741c..0000000 --- a/test/test_update_rules_unit.py +++ /dev/null @@ -1,81 +0,0 @@ -import subprocess -import tempfile -import unittest -from pathlib import Path - -from test.helpers import VerboseTestCase - - -REPO_ROOT = Path(__file__).resolve().parents[1] -SCRIPT = REPO_ROOT / "src" / "update_rules.sh" - - -class UpdateRulesUnitTests(VerboseTestCase): - def test_update_rules_errors_with_wrong_argument_count(self): - """No argument: script exits non-zero and prints usage.""" - with tempfile.TemporaryDirectory() as td: - result = subprocess.run(["bash", str(SCRIPT)], cwd=td, capture_output=True, text=True) - self.assertNotEqual(result.returncode, 0) - self.assertIn("Usage:", result.stderr) - - def test_update_rules_errors_when_rules_file_missing(self): - """Missing default rules file: script exits non-zero with a clear error.""" - with tempfile.TemporaryDirectory() as td: - result = subprocess.run( - ["bash", str(SCRIPT), "/data/tsv/records.tsv"], - cwd=td, - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("rules/default_rules.ttl not found", result.stderr) - - def test_update_rules_updates_csvw_url_and_creates_backup(self): - """Valid input updates records/header/metadata csvw:url paths and writes backup.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - rules_dir = tmp_path / "rules" - rules_dir.mkdir() - rules = rules_dir / "default_rules.ttl" - rules.write_text( - 'ex:a csvw:url "/data/tsv/records.tsv";\n' - 'ex:b csvw:url "/data/tsv/header_lines.tsv";\n' - 'ex:c csvw:url "/data/tsv/file_metadata.tsv";\n' - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "/tmp/new_records.tsv", "/tmp/new_headers.tsv", "/tmp/new_metadata.tsv"], - cwd=tmp_path, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - content = rules.read_text() - self.assertIn('csvw:url "/tmp/new_records.tsv";', content) - self.assertIn('csvw:url "/tmp/new_headers.tsv";', content) - self.assertIn('csvw:url "/tmp/new_metadata.tsv";', content) - self.assertTrue((rules_dir / "default_rules.ttl.bak").exists()) - - def test_update_rules_escapes_ampersand_in_filename(self): - """Ampersands in replacement paths are preserved literally.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - rules_dir = tmp_path / "rules" - rules_dir.mkdir() - rules = rules_dir / "default_rules.ttl" - rules.write_text('ex:a csvw:url "/data/tsv/records.tsv";\n') - - result = subprocess.run( - ["bash", str(SCRIPT), "/tmp/A&B.tsv"], - cwd=tmp_path, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertIn('csvw:url "/tmp/A&B.tsv";', rules.read_text()) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 67ff539..75cacf0 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -1250,70 +1250,6 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(method_results["hdt"]["exit_code"], 0) self.assertTrue(rdf_path.exists()) - def test_plan_partitioned_hdt_chunks_groups_small_inputs_and_splits_large_ones(self): - """Partition planning coalesces small RDF parts and line-splits oversized ones.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - rdf_dir = tmp_path / "rdf" - chunk_dir = tmp_path / "chunks" - rdf_dir.mkdir() - - small_a = rdf_dir / "part-a.nt" - small_b = rdf_dir / "part-b.nt" - large = rdf_dir / "part-c.nt" - small_a.write_text("

.\n") - small_b.write_text("

.\n") - large.write_text(("

.\n".lstrip()) * 8) - - chunk_inputs, plan = vcf_rdfizer.plan_partitioned_hdt_chunks( - [small_a, small_b, large], - chunk_dir, - target_bytes=40, - min_bytes=10, - max_bytes=60, - ) - - self.assertGreaterEqual(len(chunk_inputs), 2) - self.assertEqual(plan["source_file_count"], 3) - self.assertEqual(plan["chunk_count"], len(chunk_inputs)) - self.assertTrue(all(path.exists() for path in chunk_inputs)) - first_chunk_text = chunk_inputs[0].read_text() - self.assertIn("

.", first_chunk_text) - self.assertIn("

.", first_chunk_text) - - def test_record_safe_chunk_planner_reads_gzip_and_writes_boundary_guide(self): - """Gzip-backed chunking preserves complete records and records exact ranges.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - source = tmp_path / "aggregate.nt.gz" - source_payload = ( - b"

.\n" - b"

.\n" - b"

.\n" - ) - import gzip - - with gzip.open(source, "wb") as handle: - handle.write(source_payload) - - chunk_dir = tmp_path / "chunks" - guide = tmp_path / "chunks.json" - chunk_paths, plan = vcf_rdfizer.plan_record_safe_rdf_chunks( - [source], - chunk_dir, - target_bytes=20, - min_bytes=10, - max_bytes=30, - guide_path=guide, - ) - - self.assertTrue(guide.exists()) - self.assertEqual(plan["record_count"], 3) - self.assertEqual(plan["chunk_count"], len(chunk_paths)) - self.assertEqual(b"".join(path.read_bytes() for path in chunk_paths), source_payload) - self.assertTrue(all(path.read_bytes().endswith(b"\n") for path in chunk_paths)) - self.assertEqual(json.loads(guide.read_text())["chunks"], plan["chunks"]) - def test_run_partitioned_hdt_methods_merges_chunks_and_generates_index(self): """Partitioned HDT/COTTAS pipelines share chunks in one container.""" with tempfile.TemporaryDirectory() as td: @@ -2692,6 +2628,40 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(row["validation_status"], "PASS") self.assertEqual(row["validation_exit_code"], "0") + def test_main_validation_mode_reports_existing_results_without_traceback(self): + """A populated results directory exits 2 with a message, not an exception.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt.gz" + with gzip.open(rdf_path, "wt", encoding="utf-8") as handle: + handle.write("

.\n") + metrics_dir = tmp_path / "metrics" + existing = metrics_dir / "reports" / "validation" / "sample" + existing.mkdir(parents=True) + (existing / "summary.json").write_text("{}", encoding="utf-8") + + stderr = StringIO() + with mock.patch.object( + vcf_rdfizer, "metrics_run_directory", return_value=metrics_dir + ), redirect_stderr(stderr): + rc = invoke_main( + [ + "--mode", + "validation", + "--input", + str(vcf_path), + "--rdf", + str(rdf_path), + "--out", + str(tmp_path / "out"), + ] + ) + + self.assertEqual(rc, 2) + self.assertIn("Refusing to overwrite existing validation results", stderr.getvalue()) + def test_run_validation_mode_accepts_plain_nt_for_full_runs(self): """The shared validation runner selects --rdf-nt for a plain aggregate.""" with tempfile.TemporaryDirectory() as td: diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index d4b08b1..bb0b2cb 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -11,12 +11,51 @@ The implementation is intentionally split into small helpers so failures can be diagnosed at a specific stage and future workflow changes stay localized. + +Division of labour +------------------ +This file is the *host* side and stays free of heavy data processing: it plans +work, launches containers, and reads their reports back. Everything that walks +the RDF itself lives in the image (``src/``): + +- ``src/vcf_as_tsv.sh`` VCF -> per-input records/header/metadata TSV +- ``src/run_conversion.sh`` RMLStreamer run + part merge into one aggregate +- ``src/partitioned_compression.py`` record-safe chunking, HDT/COTTAS merge +- ``src/cottas_tool.py`` COTTAS convert/merge/reindex/decompress +- ``src/validate_compression.py`` round-trip triple-count check per artifact +- ``src/validation/`` semantic VCF-vs-RDF SPARQL validation + +The one deliberate exception is genotype RDF emission (see the "Multi-sample +(genotype) representations" section): doing it through RML would require +materializing variants x samples helper tables first. + +Section map (search for the banner comments) +-------------------------------------------- +Command execution and Docker environment helpers .. `run`, `CommandLogger`, + `ProgressSession`, `check_docker` +Input discovery and naming helpers ................ `resolve_input_snapshot` +General formatting and file-system utility helpers `format_bytes`, `ensure_dir` +Triple counting and run-level metrics reporting ... `count_triples_in_nt_files` +Console summaries, artifact naming, ... ........... `planned_output_paths` +Destructive filesystem operations ................. `remove_*_with_docker_fallback` +Preflight input collection and disk estimation .... `estimate_pipeline_sizes` +Per-input TSV discovery ........................... `discover_tsv_triplets` +Multi-sample (genotype) representations ........... `append_expanded_sample_rdf`, + `append_condensed_sample_rdf`, `SampleRecordStream` +RML mapping rendering and Docker image resolution . `render_rules_for_triplet` +Compression plan parsing and strategy selection ... `build_compression_methods` +Run metrics layout: naming, manifest, and summary . `write_run_manifest` +Metrics serialization helpers ..................... `update_metrics_csv_*` +Mode runners ...................................... `run_full_mode`, + `run_tsv_mode`, `run_compress_mode`, `run_decompress_mode`, + `run_index_mode`, `run_validation_mode`, `main` """ import argparse import csv import gzip import importlib.resources as importlib_resources +import io import json import os import re @@ -31,6 +70,11 @@ from pathlib import Path from urllib.parse import quote_plus +try: + import vcf_rdfizer_gzip +except ImportError: # pragma: no cover - shipped alongside this module + vcf_rdfizer_gzip = None + try: from rich.console import Console from rich.progress import ( @@ -267,6 +311,9 @@ RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" XSD_POSITIVE_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#positiveInteger" SAMPLE_RDF_BUFFER_BYTES = 8 * 1024 * 1024 +# An N-Triples subject is always an IRI reference or a blank node label, so a +# line starting with one of these bytes and ending in " ." is a statement. +_NTRIPLES_SUBJECT_STARTS = (b"<", b"_") SAMPLE_REPRESENTATION_CHOICES = {"expanded", "condensed"} # This is an internal rules-compatibility value, not a third public # representation. It means that custom helper TSV rows must be materialized. @@ -686,7 +733,16 @@ def run(cmd, cwd=None, env=None): stderr=subprocess.DEVNULL, ) return _ACTIVE_PROGRESS.wait_for_process(process) - return subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True).returncode + # Only the exit code is returned, so discard the streams at the fd level. + # `capture_output=True` would buffer an entire container's output in memory + # before throwing it away. + return subprocess.run( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode def docker_cmd_prefix(*, use_sudo: bool | None = None): @@ -873,28 +929,6 @@ def list_vcfs_in_dir(path: Path): return files -def resolve_input(input_path: Path): - """Legacy input resolver (single mount + container input path).""" - if not input_path.exists(): - raise ValueError(f"Input path not found: {input_path}") - - if input_path.is_file(): - if not is_vcf_file(input_path): - raise ValueError("Input file must end with .vcf or .vcf.gz") - input_dir = input_path.parent - container_input = f"/data/in/{input_path.name}" - return input_dir, container_input - - if input_path.is_dir(): - vcfs = list_vcfs_in_dir(input_path) - if not vcfs: - raise ValueError("No .vcf or .vcf.gz files found in the input directory") - container_input = "/data/in" - return input_path, container_input - - raise ValueError("Input path must be a file or a directory") - - def vcf_output_prefix(path: Path) -> str: """Derive stable sample prefix from VCF filename.""" name = path.name @@ -997,261 +1031,54 @@ def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: return None -def write_nt_chunk(chunk_path: Path, source_paths: list[Path]) -> int: - """Concatenate one or more RDF files into a chunk-local `.nt` input.""" - ensure_dir(chunk_path.parent) - total_bytes = 0 - with chunk_path.open("w", encoding="utf-8") as out_handle: - for source_path in source_paths: - with source_path.open("r", encoding="utf-8", errors="replace") as in_handle: - for line in in_handle: - out_handle.write(line) - total_bytes += len(line.encode("utf-8")) - return total_bytes - - -def split_nt_file_for_hdt( - source_path: Path, - chunk_dir: Path, - *, - target_bytes: int, - max_bytes: int, -) -> list[Path]: - """Split an oversized RDF file into line-preserving chunk files for HDT conversion.""" - ensure_dir(chunk_dir) - chunk_paths: list[Path] = [] - chunk_handle = None - chunk_path = None - chunk_size = 0 - - def open_chunk(index: int): - path = chunk_dir / f"{source_path.stem}.split-{index:05d}.nt" - return path, path.open("w", encoding="utf-8") - - try: - with source_path.open("r", encoding="utf-8", errors="replace") as in_handle: - chunk_index = 0 - for line in in_handle: - line_size = len(line.encode("utf-8")) - if chunk_handle is None: - chunk_path, chunk_handle = open_chunk(chunk_index) - chunk_paths.append(chunk_path) - chunk_size = 0 - chunk_index += 1 - elif chunk_size > 0 and ( - chunk_size >= target_bytes or chunk_size + line_size > max_bytes - ): - chunk_handle.close() - chunk_path, chunk_handle = open_chunk(chunk_index) - chunk_paths.append(chunk_path) - chunk_size = 0 - chunk_index += 1 - - chunk_handle.write(line) - chunk_size += line_size - finally: - if chunk_handle is not None and not chunk_handle.closed: - chunk_handle.close() - - return chunk_paths or [source_path] - - -def iter_rdf_binary_lines(path: Path): - """Yield RDF records from plain or gzip-compressed line-oriented RDF.""" - opener = gzip.open if path.name.endswith(".gz") else Path.open - with opener(path, "rb") as handle: - for line in handle: - yield line +def open_rdf_binary(path: Path): + """Open plain or gzip line-oriented RDF for buffered binary line iteration. - -def plan_record_safe_rdf_chunks( - source_paths: list[Path], - chunk_dir: Path, - *, - target_bytes: int, - min_bytes: int, - max_bytes: int, - guide_path: Path | None = None, -) -> tuple[list[Path], dict]: - """Create bounded RDF chunks without splitting a line-level statement. - - The guide is written as boundaries are discovered during this single - sequential pass. A separate pre-scan would read/decompress the complete - aggregate twice, so the guide and chunk files are produced together. - Logical offsets are uncompressed offsets and therefore work for both plain - and gzip-backed aggregate sources. + ``GzipFile.readline`` is markedly slower than a ``BufferedReader`` wrapped + around the same stream, which matters when the caller walks every record of + a cohort-scale aggregate. """ - if not source_paths: - return [], {"source_file_count": 0, "chunk_count": 0, "chunk_input_bytes": 0} - if target_bytes <= 0 or min_bytes <= 0 or max_bytes <= 0: - raise ValueError("RDF chunk sizes must be positive.") - if min_bytes > target_bytes or target_bytes > max_bytes: - raise ValueError("RDF chunk sizes must satisfy min <= target <= max.") - - ensure_dir(chunk_dir) - chunk_paths: list[Path] = [] - guide_chunks: list[dict] = [] - chunk_handle = None - chunk_path = None - chunk_size = 0 - chunk_start_offset = 0 - chunk_start_record = 0 - logical_offset = 0 - record_count = 0 - total_bytes = 0 - chunk_index = 0 - - def close_chunk(): - nonlocal chunk_handle, chunk_path, chunk_size - if chunk_handle is None or chunk_path is None: - return - chunk_handle.close() - chunk_paths.append(chunk_path) - guide_chunks.append( - { - "chunk_id": len(guide_chunks), - "path": str(chunk_path), - "start_record": chunk_start_record, - "end_record": record_count, - "start_uncompressed_byte": chunk_start_offset, - "end_uncompressed_byte": logical_offset, - "record_count": record_count - chunk_start_record, - "payload_bytes": chunk_size, - } - ) - chunk_handle = None - chunk_path = None - chunk_size = 0 + if path.name.endswith(".gz"): + return io.BufferedReader(gzip.open(path, "rb")) + return path.open("rb") - try: - for source_path in source_paths: - for line in iter_rdf_binary_lines(source_path): - if not line.endswith(b"\n"): - raise ValueError( - f"RDF source contains a non-line-terminated record: {source_path}" - ) - line_size = len(line) - if chunk_handle is None: - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - chunk_handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - elif chunk_size > 0 and ( - (chunk_size >= target_bytes and chunk_size >= min_bytes) - or chunk_size + line_size > max_bytes - ): - close_chunk() - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - chunk_handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - - chunk_handle.write(line) - chunk_size += line_size - logical_offset += line_size - total_bytes += line_size - record_count += 1 - finally: - close_chunk() - - plan = { - "source_file_count": len(source_paths), - "source_paths": [str(path) for path in source_paths], - "chunk_count": len(chunk_paths), - "chunk_input_bytes": total_bytes, - "record_count": record_count, - "target_chunk_bytes": target_bytes, - "min_chunk_bytes": min_bytes, - "max_chunk_bytes": max_bytes, - "chunks": guide_chunks, - } - if guide_path is not None: - ensure_dir(guide_path.parent) - plan["guide_path"] = str(guide_path) - guide_path.write_text(json.dumps(plan, indent=2) + "\n", encoding="utf-8") - return chunk_paths, plan +def is_triple_line(line: bytes) -> bool: + """Return whether a serialized line represents an N-Triples statement. -def plan_partitioned_hdt_chunks( - rdf_paths: list[Path], - chunk_dir: Path, - *, - target_bytes: int, - min_bytes: int, - max_bytes: int, -) -> tuple[list[Path], dict]: - """Plan chunk-local `.nt` inputs for partitioned HDT generation. - - The goal is to keep HDT conversion work units small enough to be fast, - while also avoiding the pathological "many tiny HDTs" case. Existing RDF - part files are treated as the first split boundary, and only oversized - parts are re-split on line boundaries. + This mirrors the container-side predicate in ``validate_compression.py`` and + ``partitioned_compression.py`` so a host-side fallback count can never + disagree with the authoritative count produced inside Docker. The leading + check short-circuits the ordinary `` ... .`` line without + allocating a stripped copy of it. """ - ensure_dir(chunk_dir) - - prepared_inputs: list[tuple[Path, int]] = [] - for rdf_path in rdf_paths: - size = int(file_size_bytes(rdf_path) or 0) - if size <= max_bytes: - prepared_inputs.append((rdf_path, size)) - continue - for split_path in split_nt_file_for_hdt( - rdf_path, - chunk_dir / "_split_inputs", - target_bytes=target_bytes, - max_bytes=max_bytes, - ): - prepared_inputs.append((split_path, int(file_size_bytes(split_path) or 0))) - - chunk_groups: list[list[tuple[Path, int]]] = [] - current_group: list[tuple[Path, int]] = [] - current_size = 0 - for path, size in prepared_inputs: - if not current_group: - current_group = [(path, size)] - current_size = size - continue - if current_size < min_bytes or current_size + size <= target_bytes: - current_group.append((path, size)) - current_size += size - continue - chunk_groups.append(current_group) - current_group = [(path, size)] - current_size = size - if current_group: - chunk_groups.append(current_group) - - chunk_inputs: list[Path] = [] - chunk_input_bytes = 0 - for chunk_index, group in enumerate(chunk_groups): - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_input_bytes += write_nt_chunk(chunk_path, [path for path, _size in group]) - chunk_inputs.append(chunk_path) - - plan = { - "source_file_count": len(rdf_paths), - "prepared_input_count": len(prepared_inputs), - "chunk_count": len(chunk_inputs), - "chunk_input_bytes": chunk_input_bytes, - } - return chunk_inputs, plan + if line.endswith(b".\n") and line[:1] in _NTRIPLES_SUBJECT_STARTS: + return True + stripped = line.strip() + return bool(stripped) and not stripped.startswith(b"#") and stripped.endswith(b".") +# --------------------------------------------------------------------------- +# Triple counting and run-level metrics reporting +# The container stages own the authoritative counts; the helpers here read +# them back and provide host-side fallbacks. +# --------------------------------------------------------------------------- def count_triples_in_nt_files(paths: list[Path]) -> int | None: - """Count triples in RDF line-oriented files as a fallback when metrics are missing.""" + """Count triples in RDF line-oriented files as a fallback when metrics are missing. + + The scan stays at the byte level: decoding a cohort-scale aggregate to + ``str`` only to run a regex per line costs several times more than the + read itself. + """ total = 0 matched_any = False - pattern = re.compile(r"^\s*[^#].*\.\s*$") for path in paths: if not path.exists() or not path.is_file(): continue try: - opener = gzip.open if path.name.endswith(".gz") else Path.open - with opener(path, "rt", encoding="utf-8", errors="replace") as handle: + with open_rdf_binary(path) as handle: for line in handle: - if pattern.match(line): + if is_triple_line(line): total += 1 matched_any = True except OSError: @@ -1433,6 +1260,9 @@ def write_index_warnings_report(*, metrics_dir: Path, run_id: str, warnings: lis return report_path +# --------------------------------------------------------------------------- +# Console summaries, artifact naming, and output-collision planning +# --------------------------------------------------------------------------- def print_nt_hdt_summary( *, output_root: Path, @@ -1615,6 +1445,11 @@ def compression_method_label_for_path(path: Path, method: str) -> str: return labels.get(method, method) +# --------------------------------------------------------------------------- +# Destructive filesystem operations +# Container stages can leave artifacts owned by another uid, so every +# deletion retries inside the image before it is reported as a failure. +# --------------------------------------------------------------------------- def remove_file_with_docker_fallback( *, path: Path, @@ -1755,6 +1590,9 @@ def cleanup_interrupted_full_run( return removed, failed +# --------------------------------------------------------------------------- +# Preflight input collection and disk-footprint estimation +# --------------------------------------------------------------------------- def existing_parent(path: Path) -> Path: """Return the closest existing parent path (used for disk free-space anchor).""" cur = path @@ -1783,20 +1621,42 @@ def collect_input_vcfs(input_path: Path): return [] +def uncompressed_vcf_bytes(vcf: Path) -> tuple[float, str]: + """Return ``(uncompressed_bytes, method)`` for one VCF input. + + A gzip input's real uncompressed size is read from its structure when that + is possible without inflating it (see :mod:`vcf_rdfizer_gzip`); otherwise + this falls back to the coarse expansion factor rather than spending a full + decompression pass on a preflight estimate. + """ + size = vcf.stat().st_size + if not vcf.name.endswith(".vcf.gz"): + return float(size), "stat" + if vcf_rdfizer_gzip is not None: + try: + measured, method = vcf_rdfizer_gzip.uncompressed_size_without_inflating(vcf) + except OSError: + measured, method = None, "unknown" + # A `.vcf.gz` whose content is not really gzip reports method "stat"; + # that is not a measurement of anything, so keep the conservative + # expansion factor rather than under-stating a disk-space warning. + if measured is not None and method != "stat": + return float(measured), method + return size * COMPRESSED_VCF_EXPANSION_FACTOR, "estimated" + + def estimate_pipeline_sizes(vcf_files, out_dir: Path): """Estimate rough TSV/RDF footprint for preflight disk-space warnings.""" input_bytes = 0 est_tsv_bytes = 0 est_rdf_low_bytes = 0 est_rdf_high_bytes = 0 + methods: set[str] = set() for vcf in vcf_files: - size = vcf.stat().st_size - input_bytes += size - if vcf.name.endswith(".vcf.gz"): - expanded_vcf = size * COMPRESSED_VCF_EXPANSION_FACTOR - else: - expanded_vcf = float(size) + input_bytes += vcf.stat().st_size + expanded_vcf, method = uncompressed_vcf_bytes(vcf) + methods.add(method) est_tsv_bytes += expanded_vcf * TSV_OVERHEAD_FACTOR est_rdf_low_bytes += expanded_vcf * RDF_EXPANSION_LOW_FACTOR @@ -1812,11 +1672,15 @@ def estimate_pipeline_sizes(vcf_files, out_dir: Path): "rdf_high_bytes": int(est_rdf_high_bytes), "free_disk_bytes": int(free_disk_bytes), "disk_anchor": out_anchor, + # "estimated" means at least one gzip input fell back to the coarse + # expansion factor; anything else means every input size was measured. + "input_size_methods": sorted(methods), + "uncompressed_input_estimated": "estimated" in methods, } # --------------------------------------------------------------------------- -# Mapping/rules and Docker image management helpers +# Per-input TSV discovery # --------------------------------------------------------------------------- def slugify(value: str) -> str: """Normalize a value for safe filesystem naming.""" @@ -1854,6 +1718,13 @@ def discover_tsv_triplets(tsv_dir: Path): ) +# --------------------------------------------------------------------------- +# Multi-sample (genotype) representations +# Expanded and condensed genotype RDF is emitted here rather than by +# RMLStreamer: the equivalent RML maps would first have to materialize +# variants x samples (x FORMAT keys) helper TSV rows. +# See docs/sample-representation-guide.md for the emitted shapes. +# --------------------------------------------------------------------------- def write_sample_support_headers(sample_calls_tsv: Path, sample_format_tsv: Path): """Create empty sample helper tables with their canonical TSV headers.""" sample_calls_tsv.parent.mkdir(parents=True, exist_ok=True) @@ -1943,8 +1814,20 @@ def _sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: return candidate or f"sample_{fallback_index}" +# Characters that ``quote_plus(..., safe="*-._")`` leaves untouched. Sample +# ids, FORMAT keys, and numeric row ids are almost always drawn from this set, +# so the fast path below skips percent-encoding entirely for them. +_URI_COMPONENT_PASSTHROUGH_RE = re.compile(r"[A-Za-z0-9*\-._]*\Z") +# Characters that require N-Triples escaping. VCF genotype/FORMAT payloads +# essentially never contain them, so the common case avoids five str.replace +# scans of the value. +_NTRIPLES_ESCAPE_RE = re.compile(r'[\\"\n\r\t]') + + def _rml_uri_component(value: str) -> str: """Match RMLStreamer's Java URLEncoder-based template substitution.""" + if _URI_COMPONENT_PASSTHROUGH_RE.match(value) is not None: + return value encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") # urllib follows current RFC rules and always leaves '~' unescaped, whereas # java.net.URLEncoder (used by RMLStreamer 2.5.0) encodes it. @@ -1953,6 +1836,8 @@ def _rml_uri_component(value: str) -> str: def _ntriples_string_literal(value: str) -> str: """Serialize an RDF 1.1 plain/xsd:string literal for N-Triples.""" + if _NTRIPLES_ESCAPE_RE.search(value) is None: + return f'"{value}"' escaped = ( value.replace("\\", "\\\\") .replace('"', '\\"') @@ -2001,6 +1886,10 @@ def __init__(self, records_tsv: Path): self._reader = None self._header: list[str] = [] self._pending_row: list[str] | None = None + # A cohort VCF repeats the same FORMAT string on nearly every record. + # Cache the derived key tuple (and its duplicate check) per distinct + # FORMAT/width combination instead of rebuilding it per record. + self._format_key_cache: dict[tuple[str, int], tuple[str, ...]] = {} def __enter__(self): _set_max_csv_field_size() @@ -2087,17 +1976,21 @@ def _parse_row(self, row: list[str]) -> ParsedSampleRecord: [len(declared_format_keys), *(len(values) for values in raw_sample_values)], default=0, ) - format_keys = tuple( - declared_format_keys[index] - if index < len(declared_format_keys) and declared_format_keys[index] - else f"FIELD_{index + 1}" - for index in range(total_fields) - ) - if len(set(format_keys)) != len(format_keys): - raise ValueError( - f"record {row_id or '(unknown)'} contains duplicate FORMAT keys: " - + ":".join(format_keys) + cache_key = (format_raw, total_fields) + format_keys = self._format_key_cache.get(cache_key) + if format_keys is None: + format_keys = tuple( + declared_format_keys[index] + if index < len(declared_format_keys) and declared_format_keys[index] + else f"FIELD_{index + 1}" + for index in range(total_fields) ) + if len(set(format_keys)) != len(format_keys): + raise ValueError( + f"record {row_id or '(unknown)'} contains duplicate FORMAT keys: " + + ":".join(format_keys) + ) + self._format_key_cache[cache_key] = format_keys sample_values = tuple( tuple(values[index] if index < len(values) else "" for index in range(total_fields)) @@ -2118,11 +2011,15 @@ def _append_rdf_atomically(rdf_path: Path, stats: dict, producer): opener = gzip.open if rdf_path.name.endswith(".gz") else Path.open output_handle = None buffer = bytearray() + # ``emit`` runs once per emitted triple (billions of times for a cohort + # aggregate), so the counter is a local int and only reaches ``stats`` + # once the producer has finished. + emitted = 0 def emit(line: str): - nonlocal buffer + nonlocal buffer, emitted buffer.extend(line.encode("utf-8")) - stats["triples"] += 1 + emitted += 1 if len(buffer) >= SAMPLE_RDF_BUFFER_BYTES: output_handle.write(buffer) buffer = bytearray() @@ -2130,6 +2027,7 @@ def emit(line: str): try: output_handle = opener(rdf_path, "ab") producer(emit) + stats["triples"] += emitted if buffer: output_handle.write(buffer) output_handle.close() @@ -2182,25 +2080,47 @@ def produce(emit): f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " f"<{VCFR_NAMESPACE}ExpandedRepresentation> .\n" ) + # Sample columns and their serialized sampleId literal are fixed for + # the whole file; FORMAT keys are drawn from a handful of distinct + # values. Encoding them once per file instead of once per + # record/sample removes the dominant cost of this loop. + sample_prefixes = [ + ( + _rml_uri_component(column.uri_id), + _ntriples_literal(column.sample_id), + ) + for column in record_stream.columns + ] + format_components: dict[str, str] = {} + sample_call_count = 0 + format_value_count = 0 + for record in record_stream: row_component = _rml_uri_component(record.row_id) call_uri = f"{file_uri}#call/{row_component}" + format_keys = record.format_keys + record_sample_values = record.sample_values - for sample_index, sample_column in enumerate(record_stream.columns): - sample_component = _rml_uri_component(sample_column.uri_id) - sample_uri = f"file://{source_component}#sample/{row_component}/{sample_component}" + for sample_index, (sample_component, sample_id_literal) in enumerate( + sample_prefixes + ): + sample_uri = f"{file_uri}#sample/{row_component}/{sample_component}" + sample_values = record_sample_values[sample_index] emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasSampleCall> <{sample_uri}> .\n") emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleCall> .\n") emit( f"<{sample_uri}> <{VCFR_NAMESPACE}sampleId> " - f"{_ntriples_literal(sample_column.sample_id)} .\n" + f"{sample_id_literal} .\n" ) - stats["sample_calls"] += 1 - - for format_index, format_key in enumerate(record.format_keys): - format_value = record.sample_values[sample_index][format_index] - format_component = _rml_uri_component(format_key) + sample_call_count += 1 + + for format_index, format_key in enumerate(format_keys): + format_value = sample_values[format_index] + format_component = format_components.get(format_key) + if format_component is None: + format_component = _rml_uri_component(format_key) + format_components[format_key] = format_component format_uri = f"{sample_uri}/fmt/{format_component}" emit(f"<{sample_uri}> <{VCFR_NAMESPACE}hasFormatValue> <{format_uri}> .\n") emit(f"<{format_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatFieldValue> .\n") @@ -2209,13 +2129,15 @@ def produce(emit): f"<{format_uri}> <{VCFR_NAMESPACE}fieldValue> " f"{_ntriples_literal(format_value)} .\n" ) - stats["format_values"] += 1 + format_value_count += 1 stats["records"] += 1 + stats["sample_calls"] = sample_call_count + stats["format_values"] = format_value_count if progress_interval_records > 0 and stats["records"] % progress_interval_records == 0: print( " * Sample RDF streaming: " - f"{stats['records']:,} variants, {stats['sample_calls']:,} calls", + f"{stats['records']:,} variants, {sample_call_count:,} calls", flush=True, ) @@ -2348,6 +2270,8 @@ def produce(emit): file_uri = f"file://{source_component}" sample_set_uri = f"{file_uri}#samples" emitted_definitions: set[str] = set() + # FORMAT keys repeat on every record; encode each distinct key once. + format_components: dict[str, str] = {} emit( f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " @@ -2386,7 +2310,10 @@ def produce(emit): stats["matrices"] += 1 for format_index, format_key in enumerate(record.format_keys): - format_component = _rml_uri_component(format_key) + format_component = format_components.get(format_key) + if format_component is None: + format_component = _rml_uri_component(format_key) + format_components[format_key] = format_component vector_uri = f"{matrix_uri}/fmt/{format_component}" definition = definitions.get(format_key) if definition is None: @@ -2569,6 +2496,9 @@ def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_ ) +# --------------------------------------------------------------------------- +# RML mapping rendering and Docker image resolution +# --------------------------------------------------------------------------- def render_rules_for_triplet( template_rules: Path, output_rules: Path, @@ -2647,6 +2577,12 @@ def resolve_image_ref(image: str, image_version: str | None): return f"{image}:{image_version}", True +# --------------------------------------------------------------------------- +# Compression plan parsing and strategy selection +# The public CLI takes three orthogonal options (--rdf-compression, +# --representations, --artifact-compression); they are translated here into +# the flat internal stage names used by the execution helpers. +# --------------------------------------------------------------------------- def parse_compression_methods(raw: str): """Parse internal compression stage names used by the execution helpers.""" value = (raw or "").strip() @@ -2773,6 +2709,9 @@ def should_use_partitioned_hdt( return mode == "full" and rdf_storage_mode in RDF_STORAGE_MODES +# --------------------------------------------------------------------------- +# Run metrics layout: naming, manifest, and summary +# --------------------------------------------------------------------------- def safe_metrics_name(value: str) -> str: """Sanitize names used in metrics artifact filenames.""" safe = re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("_") @@ -3910,7 +3849,6 @@ def run_compression_methods_for_rdf( cottas_failure_warning: dict | None = None metrics_output_name = output_name or target_out_dir.name safe_output_name = safe_metrics_name(metrics_output_name) - safe_rdf_name = safe_metrics_name(rdf_path.name) auxiliary_stage_results: dict[str, dict] = {} def run_container_command( @@ -4131,16 +4069,9 @@ def ensure_hdt_available(): artifact_container=hdt_container, ) if not valid: - if allow_index_failure: - cottas_failure_warning = record_index_warning( - index_format="cottas", - artifact_path=cottas_path, - stage="cottas-index", - message=report.get( - "error", - "existing COTTAS validation/index check failed", - ), - ) + # ``validate_container_artifact`` already downgrades a + # recoverable HDT index problem to a warning and returns True, + # so reaching this point means the artifact itself is unusable. return False method_results["hdt"]["validation"] = report if report.get("index_status"): @@ -4527,8 +4458,7 @@ def run_containerized_partitioned_representation_methods( "--mount", f"type=volume,source={volume_name},target=/work", ] - if source_mount is not None: - command.extend(["-v", source_mount]) + command.extend(["-v", source_mount]) command.extend( [ "-v", @@ -4725,7 +4655,6 @@ def run_full_mode( *, input_mount_dir: Path, container_inputs: list[str], - input_metrics_target: str, expected_prefixes: list[str], rules_path: Path, out_dir: Path, @@ -5412,95 +5341,64 @@ def fail_current(stage: str, message: str): continue rdf_storage_removed = not any(path.exists() for path in raw_rdf_files) - if raw_rdf_files: - output_root = out_dir / output_name - raw_total_size = sum(raw_size_before_cleanup_by_file.values()) - - if validation_failed: - raw_note = "retained after validation failure" - elif keep_rmlstreamer_rdf_output: - raw_note = "retained via --keep-rmlstreamer-rdf-output" - elif rdf_storage_removed and remove_rdf_storage_output: - raw_note = "removed via --remove-rdf-storage-output" - elif rdf_storage_removed and selected_methods: - raw_note = "removed after successful compression" - elif selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: - raw_note = "retained because it is also the selected gzip artifact" - elif selected_methods: - raw_note = "retained" - else: - raw_note = "kept (compression methods set to none)" + # Full mode always produces exactly one aggregate per input. + output_root = out_dir / output_name + raw_total_size = sum(raw_size_before_cleanup_by_file.values()) - first_path = raw_rdf_files[0] - print(f" * Output directory: {output_root}") - print(f" - RDF aggregate: {first_path.name}") - raw_text = format_bytes(raw_total_size) - print( - f" - {rdf_label_for_path(first_path)}: {raw_text} " - f"({raw_note})" - ) + if validation_failed: + raw_note = "retained after validation failure" + elif keep_rmlstreamer_rdf_output: + raw_note = "retained via --keep-rmlstreamer-rdf-output" + elif rdf_storage_removed and remove_rdf_storage_output: + raw_note = "removed via --remove-rdf-storage-output" + elif rdf_storage_removed and selected_methods: + raw_note = "removed after successful compression" + elif selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: + raw_note = "retained because it is also the selected gzip artifact" + elif selected_methods: + raw_note = "retained" + else: + raw_note = "kept (compression methods set to none)" - if selected_methods: - for method in selected_methods: - if use_partitioned_compression and method in PARTITIONED_COMPRESSION_METHODS: - result = partitioned_representation_results.get(method) - label = compression_method_label_for_path(first_path, method) - if not result or int(result.get("exit_code", 1)) != 0: - print(f" - {label}: not generated") - else: - print( - f" - {label}: {format_bytes(int(result.get('output_size_bytes') or 0))} " - f"({result.get('output_path', '')})" - ) - continue - method_total = 0 - method_count = 0 - for raw_rdf_path in raw_rdf_files: - result = method_results_by_file.get(raw_rdf_path.name, {}).get(method) - if not result or int(result.get("exit_code", 1)) != 0: - continue - method_total += int(result.get("output_size_bytes") or 0) - method_count += 1 + first_path = raw_rdf_files[0] + print(f" * Output directory: {output_root}") + print(f" - RDF aggregate: {first_path.name}") + raw_text = format_bytes(raw_total_size) + print( + f" - {rdf_label_for_path(first_path)}: {raw_text} " + f"({raw_note})" + ) + if selected_methods: + for method in selected_methods: + if use_partitioned_compression and method in PARTITIONED_COMPRESSION_METHODS: + result = partitioned_representation_results.get(method) label = compression_method_label_for_path(first_path, method) - if method_count == 0: + if not result or int(result.get("exit_code", 1)) != 0: print(f" - {label}: not generated") else: - print(f" - {label}: {format_bytes(method_total)}") - else: - print(" - Compression: none selected") - print(f" - Final RDF size (no compression): {format_bytes(raw_total_size)}") - else: - for raw_rdf_path in raw_rdf_files: - hdt_path = (out_dir / output_name) / f"{raw_rdf_path.stem}.hdt" - rdf_size = file_size_bytes(raw_rdf_path) - nt_note = None - method_results = method_results_by_file.get(raw_rdf_path.name, {}) - if raw_rdf_path.exists() and keep_rmlstreamer_rdf_output: - nt_note = "retained via --keep-rmlstreamer-rdf-output" - elif raw_rdf_path.exists() and selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: - nt_note = "retained because it is also the selected gzip artifact" - elif not raw_rdf_path.exists() and remove_rdf_storage_output: - nt_note = "removed via --remove-rdf-storage-output" - elif not raw_rdf_path.exists() and selected_methods: - nt_note = "removed after successful compression" - elif not raw_rdf_path.exists() and not selected_methods: - nt_note = "kept (compression methods set to none)" + print( + f" - {label}: {format_bytes(int(result.get('output_size_bytes') or 0))} " + f"({result.get('output_path', '')})" + ) + continue + method_total = 0 + method_count = 0 + for raw_rdf_path in raw_rdf_files: + result = method_results_by_file.get(raw_rdf_path.name, {}).get(method) + if not result or int(result.get("exit_code", 1)) != 0: + continue + method_total += int(result.get("output_size_bytes") or 0) + method_count += 1 + + label = compression_method_label_for_path(first_path, method) + if method_count == 0: + print(f" - {label}: not generated") else: - nt_note = "retained" - print_nt_hdt_summary( - output_root=out_dir / output_name, - nt_path=raw_rdf_path, - hdt_path=hdt_path, - indent=" ", - nt_note=nt_note, - nt_size_override=rdf_size, - selected_methods=selected_methods, - method_results=method_results, - ) - if not selected_methods: - total_raw_size = sum(raw_size_before_cleanup_by_file.values()) - print(f" * Final RDF size (no compression): {format_bytes(total_raw_size)}") + print(f" - {label}: {format_bytes(method_total)}") + else: + print(" - Compression: none selected") + print(f" - Final RDF size (no compression): {format_bytes(raw_total_size)}") if not keep_tsv: # Cleanup only the triplet generated for this input iteration. @@ -6541,7 +6439,7 @@ def main(): ( input_mount_dir, container_inputs, - input_metrics_target, + _input_metrics_target, expected_prefixes, ) = resolve_input_snapshot(input_path) if args.rules is None: @@ -6787,10 +6685,15 @@ def main(): if validation_results_dir.exists() and ( not validation_results_dir.is_dir() or any(validation_results_dir.iterdir()) ): - raise ValueError( - f"Refusing to overwrite existing validation results: {validation_results_dir}. " + # This runs after the argument-validation try/except above, so it + # must report the same way that block does instead of surfacing an + # uncaught traceback. + eprint( + f"Error: Refusing to overwrite existing validation results: " + f"{validation_results_dir}. " "Choose --validation-id or --out with a new destination." ) + return 2 manifest_options = { "requested_image": args.image, "requested_image_version": args.image_version, @@ -6814,7 +6717,7 @@ def main(): "validation_rdf_gzip": str(validation_rdf_gzip_path) if mode == "validation" else None, } try: - manifest_path = write_run_manifest( + write_run_manifest( metrics_dir=metrics_dir, run_id=run_id, timestamp=timestamp, @@ -6842,6 +6745,11 @@ def main(): " - Estimated RDF N-Triples size: " f"{format_bytes(estimate['rdf_low_bytes'])} to {format_bytes(estimate['rdf_high_bytes'])}" ) + if estimate["uncompressed_input_estimated"]: + print( + " (a gzip input's uncompressed size could not be read from its " + "structure, so its expansion was assumed)" + ) print( f" - Free disk space at {estimate['disk_anchor']}: {format_bytes(estimate['free_disk_bytes'])}" ) @@ -6979,7 +6887,6 @@ def execute_mode(): return run_full_mode( input_mount_dir=input_mount_dir, container_inputs=container_inputs, - input_metrics_target=input_metrics_target, expected_prefixes=expected_prefixes, rules_path=rules_path, out_dir=out_dir, diff --git a/vcf_rdfizer_gzip.py b/vcf_rdfizer_gzip.py new file mode 100644 index 0000000..a8bca50 --- /dev/null +++ b/vcf_rdfizer_gzip.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Uncompressed size of a gzip/BGZF file, without decompressing it when possible. + +VCF-RDFizer records ``input_vcf_size_bytes`` as the *uncompressed* size of the +source VCF so that compression ratios are comparable across plain and +compressed inputs. Obtaining that number by inflating the whole file costs a +full pass over a cohort-scale VCF purely for a metric. + +Three strategies, in order of preference. Every result carries the method that +produced it so a reported size is always auditable: + +``bgzf`` + BGZF (``bgzip``) files - what ``bcftools``/``tabix``/``htslib`` produce and + what indexed ``.vcf.gz`` files in practice are - are a sequence of gzip + members, each holding at most 64 KiB and declaring its own compressed + length in a ``BC`` extra subfield. Walking those declarations and summing + each member's ``ISIZE`` trailer is exact, needs no inflate at all, and is + immune to the 32-bit ``ISIZE`` wrap because every member is small. + +``gzip-trailer`` + A plain single-member gzip stores the uncompressed size in its last four + bytes, but only modulo 2**32, and a concatenated multi-member file exposes + only its final member's value. Both hazards are resolved by inflating a + small prefix to measure this file's actual compression ratio and requiring + exactly one ``ISIZE + k * 2**32`` candidate to agree with it. + +``inflate`` + Full streaming inflate. Always correct, always available, and the fallback + whenever the checks above cannot settle the answer. + +The module is dependency-free so it can run both on the host (for +``--estimate-size``) and inside the image (from ``run_conversion.sh``). +""" + +from __future__ import annotations + +import argparse +import struct +import sys +import zlib +from pathlib import Path + +GZIP_MAGIC = b"\x1f\x8b" +GZIP_DEFLATE = 0x08 +FEXTRA = 0x04 +FNAME = 0x08 +FCOMMENT = 0x10 +FHCRC = 0x02 + +ISIZE_MODULUS = 1 << 32 +# A BGZF member never holds more than 64 KiB, so its ISIZE can never wrap. +BGZF_MAX_BLOCK_ISIZE = 1 << 16 + +# DEFLATE's proven upper bound is 1032:1 (a 258-byte match encoded in ~2 bits). +# Nothing can decompress by more than this, so it bounds the candidate search. +MAX_DEFLATE_RATIO = 1032.0 +# Inflate at most this much before extrapolating a ratio. A VCF body is highly +# homogeneous, so a prefix this size characterises the whole file closely. +RATIO_SAMPLE_BYTES = 16 * 1024 * 1024 +# A candidate must land this close to the sampled projection to be accepted. +# Candidates are 4 GiB apart, so this is a wide margin for a correct answer and +# still rejects a multi-member file whose trailer describes one member only. +RATIO_TOLERANCE = 0.25 + +READ_CHUNK_BYTES = 1024 * 1024 + + +class GzipStructureError(ValueError): + """The file is not shaped the way the chosen fast path requires.""" + + +def looks_like_gzip(path: Path) -> bool: + """Return whether the file starts with the gzip magic bytes.""" + try: + with path.open("rb") as handle: + return handle.read(2) == GZIP_MAGIC + except OSError: + return False + + +def _read_exactly(handle, count: int) -> bytes: + data = handle.read(count) + if len(data) != count: + raise GzipStructureError("unexpected end of gzip stream") + return data + + +def _bgzf_block_size(header: bytes, handle) -> int: + """Return the total on-disk length of one BGZF member, or raise.""" + if header[:2] != GZIP_MAGIC or header[2] != GZIP_DEFLATE: + raise GzipStructureError("not a gzip member") + flags = header[3] + if not flags & FEXTRA: + raise GzipStructureError("gzip member has no FEXTRA field") + (xlen,) = struct.unpack(" int: + """Sum every BGZF member's ISIZE by walking block headers only. + + Raises :class:`GzipStructureError` when the file is not valid BGZF, which + includes an ordinary single-member gzip. + """ + total = 0 + file_size = path.stat().st_size + with path.open("rb") as handle: + offset = 0 + while offset < file_size: + handle.seek(offset) + header = _read_exactly(handle, 12) + block_size = _bgzf_block_size(header, handle) + if block_size < 28 or offset + block_size > file_size: + raise GzipStructureError("BGZF block size runs past end of file") + handle.seek(offset + block_size - 4) + (isize,) = struct.unpack("= BGZF_MAX_BLOCK_ISIZE: + raise GzipStructureError("BGZF block declares an oversized payload") + total += isize + offset += block_size + return total + + +def _skip_gzip_header(handle) -> None: + """Advance past one gzip member header, leaving the deflate stream next.""" + header = _read_exactly(handle, 10) + if header[:2] != GZIP_MAGIC or header[2] != GZIP_DEFLATE: + raise GzipStructureError("not a gzip stream") + flags = header[3] + if flags & FEXTRA: + (xlen,) = struct.unpack(" float: + return self.produced / self.consumed if self.consumed else 0.0 + + @property + def single_member(self) -> bool: + return self.member_complete and self.trailing_bytes == 8 + + +def sample_first_member(path: Path, sample_bytes: int = RATIO_SAMPLE_BYTES) -> MemberSample: + """Inflate a bounded prefix of the first member to characterise the file. + + The cost is capped by ``sample_bytes`` regardless of how large the file is, + which is what keeps the fast path constant-time. + """ + decompressor = zlib.decompressobj(-zlib.MAX_WBITS) + produced = 0 + consumed = 0 + with path.open("rb") as handle: + _skip_gzip_header(handle) + header_bytes = handle.tell() + while produced < sample_bytes: + chunk = handle.read(READ_CHUNK_BYTES) + if not chunk: + break + produced += len(decompressor.decompress(chunk)) + consumed += len(chunk) + if decompressor.eof: + # `unused_data` holds the member's 8-byte trailer plus anything + # that follows it, so its length identifies a concatenation. + trailing = len(decompressor.unused_data) + deflate_bytes = consumed - trailing + return MemberSample( + produced, header_bytes + deflate_bytes, True, trailing + ) + if consumed <= 0 or produced <= 0: + raise GzipStructureError("could not sample the deflate stream") + return MemberSample(produced, header_bytes + consumed, False, 0) + + +def trailer_isize(path: Path) -> int: + """Read the final member's ISIZE field (uncompressed size modulo 2**32).""" + with path.open("rb") as handle: + handle.seek(-4, 2) + return struct.unpack(" tuple[int, str]: + """Resolve the 32-bit ISIZE trailer into a full uncompressed size. + + Raises :class:`GzipStructureError` unless the file is provably a single + member measured in full, or exactly one ``ISIZE + k * 2**32`` candidate + agrees with the ratio measured from the file's own prefix. That single + requirement rules out both a >4 GiB wrap the trailer cannot express and a + concatenated file whose trailer describes only its final member. + """ + compressed_size = path.stat().st_size + if compressed_size <= 18: + raise GzipStructureError("file is too small to carry a gzip trailer") + + sample = sample_first_member(path) + if sample.member_complete: + if sample.single_member: + # The whole file fitted in the sample budget: this is not an + # estimate resolved against a trailer, it is the measured size. + return sample.produced, "gzip-sample" + raise GzipStructureError( + "concatenated gzip members; the trailer covers only the last one" + ) + + projected = sample.ratio * compressed_size + if projected <= 0: + raise GzipStructureError("could not project an uncompressed size") + + isize = trailer_isize(path) + upper_bound = compressed_size * MAX_DEFLATE_RATIO + accepted = [ + candidate + for candidate in range(isize, int(upper_bound) + ISIZE_MODULUS, ISIZE_MODULUS) + if candidate <= upper_bound + and abs(candidate - projected) <= RATIO_TOLERANCE * projected + ] + if len(accepted) != 1: + raise GzipStructureError( + f"gzip trailer is ambiguous ({len(accepted)} plausible sizes)" + ) + return accepted[0], "gzip-trailer" + + +def inflate_uncompressed_size(path: Path) -> int: + """Stream the whole file through zlib and count the output bytes. + + Raises :class:`zlib.error` on a truncated stream so a damaged input is + reported rather than silently measured as short. + """ + total = 0 + decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) + with path.open("rb") as handle: + while True: + chunk = handle.read(READ_CHUNK_BYTES) + if not chunk: + break + total += len(decompressor.decompress(chunk)) + while decompressor.eof and decompressor.unused_data: + # Concatenated members: continue with the next one. + tail = decompressor.unused_data + decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) + total += len(decompressor.decompress(tail)) + total += len(decompressor.flush()) + if not decompressor.eof: + raise zlib.error(f"truncated or corrupt gzip stream: {path}") + return total + + +def uncompressed_size(path: Path, *, exact: bool = False) -> tuple[int, str]: + """Return ``(uncompressed_bytes, method)`` for a plain or gzip file. + + Set ``exact`` to skip the structural fast paths and always inflate. + """ + path = Path(path) + if not looks_like_gzip(path): + return path.stat().st_size, "stat" + + if not exact: + try: + return bgzf_uncompressed_size(path), "bgzf" + except (GzipStructureError, OSError, struct.error): + pass + try: + return resolve_trailer_size(path) + except (GzipStructureError, OSError, zlib.error, struct.error): + pass + + return inflate_uncompressed_size(path), "inflate" + + +def uncompressed_size_without_inflating(path: Path) -> tuple[int | None, str]: + """Structural size only: return ``(None, "unknown")`` rather than inflating. + + Callers that merely want a preflight estimate would gain nothing from a + full decompression pass, so they use this instead of + :func:`uncompressed_size`. + """ + path = Path(path) + if not looks_like_gzip(path): + return path.stat().st_size, "stat" + try: + return bgzf_uncompressed_size(path), "bgzf" + except (GzipStructureError, OSError, struct.error): + pass + try: + return resolve_trailer_size(path) + except (GzipStructureError, OSError, zlib.error, struct.error): + pass + return None, "unknown" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Print the uncompressed size in bytes of a gzip/BGZF file, " + "avoiding a full decompression pass where the file's structure " + "makes that possible." + ) + ) + parser.add_argument("path", help="File to measure") + parser.add_argument( + "--exact", + action="store_true", + help="Always inflate instead of using a structural fast path", + ) + parser.add_argument( + "--print-method", + action="store_true", + help="Print ' ' instead of just the byte count", + ) + args = parser.parse_args(argv) + + try: + size, method = uncompressed_size(Path(args.path), exact=args.exact) + except (OSError, zlib.error) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(f"{size} {method}" if args.print_method else size) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vcf_rdfizer_rules.py b/vcf_rdfizer_rules.py new file mode 100644 index 0000000..ea7b5ef --- /dev/null +++ b/vcf_rdfizer_rules.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +"""Authoring helper for custom VCF-RDFizer RML mappings. + +VCF-RDFizer accepts any RML mapping through ``--rules``, but a custom mapping +has to honour a small contract with the wrapper. This tool makes that contract +discoverable and checkable instead of something you find out on the next +multi-hour run: + +``vcf-rdfizer-rules columns`` + Show the TSV sources the pipeline generates and every column each one has, + so you know what is available to reference. + +``vcf-rdfizer-rules init -o my_rules.ttl`` + Copy the shipped default mapping as an annotated starting point. + +``vcf-rdfizer-rules check my_rules.ttl`` + Validate a mapping against the contract before running the pipeline: the + logical-source paths the wrapper can rewrite, the column names it can + supply, and which ``--sample-representation`` values remain usable. + +The contract itself +------------------- +1. Logical sources must use the five canonical container paths verbatim + (``/data/tsv/records.tsv`` and friends). Full mode processes one VCF at a + time and rewrites exactly those five strings to the per-input file names, so + a mapping that hard-codes anything else silently reads the wrong file. +2. Referenced columns must exist in the TSV that ``vcf_as_tsv.sh`` writes. +3. Consuming ``sample_calls.tsv`` or ``sample_format_values.tsv`` beyond the + four built-in sample maps forces the wrapper to materialize those helper + tables, which costs one row per variant x sample (x FORMAT key), and is + incompatible with ``--sample-representation condensed``. + +The checks are deliberately lexical rather than a full Turtle parse: the tool +stays dependency-free, and the mistakes worth catching before a long run are +wrong paths and misspelled column names. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +try: + from vcf_rdfizer import ( + SAMPLE_CALLS_HEADER, + SAMPLE_FORMAT_HEADER, + resolve_default_rules_path, + resolve_sample_workflow, + sample_support_strategy, + ) +except ImportError as exc: # pragma: no cover - installed together + raise SystemExit(f"vcf-rdfizer-rules requires the vcf_rdfizer module: {exc}") + + +@dataclass(frozen=True) +class TsvSource: + """One TSV table the pipeline generates for RMLStreamer to read.""" + + container_path: str + description: str + columns: tuple[str, ...] + #: A trailing column whose *name* varies per input (the VCF sample header). + dynamic_tail_column: str | None = None + notes: tuple[str, ...] = field(default_factory=tuple) + + +# The first three column lists mirror the headers written by `src/vcf_as_tsv.sh`; +# `test_rules_helper_unit.py` runs that script and asserts they still match, so +# the two cannot drift apart silently. The last two reuse the wrapper's own +# constants directly. +TSV_SOURCES: dict[str, TsvSource] = { + "records": TsvSource( + container_path="/data/tsv/records.tsv", + description="One row per VCF data line.", + columns=( + "SOURCE_FILE", + "ROW_ID", + "CHROM", + "POS", + "ID", + "REF", + "ALT", + "QUAL", + "FILTER", + "INFO", + "FORMAT", + ), + dynamic_tail_column="SAMPLES", + notes=( + "ROW_ID is a 1-based counter over data lines, stable within one file.", + "The final column holds every sample's payload separated by spaces. Its" + " header is the whitespace-joined sample ids from #CHROM, or SAMPLES when" + " the VCF declares none, so it cannot be referenced by a fixed name.", + "Genotypes are emitted by the wrapper from this table directly; see" + " --sample-representation.", + ), + ), + "header_lines": TsvSource( + container_path="/data/tsv/header_lines.tsv", + description="One row per '##' meta-information header line.", + columns=( + "SOURCE_FILE", + "HEADER_INDEX", + "HEADER_KEY", + "HEADER_VALUE", + "RAW_LINE", + ), + notes=( + "HEADER_KEY/HEADER_VALUE split on the first '='; RAW_LINE keeps the" + " original text without the leading '##'.", + ), + ), + "file_metadata": TsvSource( + container_path="/data/tsv/file_metadata.tsv", + description="Exactly one row summarising the source VCF.", + columns=( + "SOURCE_FILE", + "FILE_FORMAT", + "FILE_DATE", + "SOURCE_SOFTWARE", + "REFERENCE_GENOME", + "HEADER_COUNT", + "RECORD_COUNT", + ), + ), + "sample_calls": TsvSource( + container_path="/data/tsv/sample_calls.tsv", + description="Helper table: one row per variant x sample.", + columns=tuple(SAMPLE_CALLS_HEADER), + notes=( + "Header-only for the four built-in sample maps. Referencing it from a" + " custom map makes the wrapper materialize every row.", + ), + ), + "sample_format_values": TsvSource( + container_path="/data/tsv/sample_format_values.tsv", + description="Helper table: one row per variant x sample x FORMAT key.", + columns=tuple(SAMPLE_FORMAT_HEADER), + notes=( + "Header-only for the four built-in sample maps. Referencing it from a" + " custom map makes the wrapper materialize every row, which is the" + " largest table the pipeline can produce.", + ), + ), +} + +CANONICAL_SOURCE_PATHS = {source.container_path for source in TSV_SOURCES.values()} + +CSVW_URL_RE = re.compile(r'csvw:url\s+"([^"]*)"') +REFERENCE_RE = re.compile(r'rml:reference\s+"([^"]*)"') +TEMPLATE_RE = re.compile(r'rr:template\s+"((?:[^"\\]|\\.)*)"') +TEMPLATE_VAR_RE = re.compile(r"(?.records.tsv` +# and so on). A different path is not rewritten and will not be found. +# +# 2. Only reference columns the pipeline actually writes. Run +# `vcf-rdfizer-rules columns` to see them all. +# +# Validate this file before a long run: +# +# vcf-rdfizer-rules check +# +# Then use it with: +# +# vcf-rdfizer --mode full --rules ... +# +""" + + +def all_known_columns() -> dict[str, list[str]]: + """Map each column name to the sources that provide it.""" + owners: dict[str, list[str]] = {} + for name, source in TSV_SOURCES.items(): + for column in source.columns: + owners.setdefault(column, []).append(name) + if source.dynamic_tail_column: + owners.setdefault(source.dynamic_tail_column, []).append(name) + return owners + + +def referenced_columns(text: str) -> set[str]: + """Collect every column name the mapping reads, from references and templates.""" + names = set(REFERENCE_RE.findall(text)) + for template in TEMPLATE_RE.findall(text): + names.update( + variable.strip() for variable in TEMPLATE_VAR_RE.findall(template) if variable.strip() + ) + return names + + +def check_rules(rules_path: Path) -> dict: + """Validate one mapping against the wrapper contract. + + Returns a report with ``errors`` (the run will not work), ``warnings`` + (it will work but at a cost worth knowing about), and ``info``. + """ + report: dict = { + "rules_path": str(rules_path), + "errors": [], + "warnings": [], + "info": [], + "sources": [], + "columns": [], + "sample_representations": {}, + } + text = rules_path.read_text(encoding="utf-8") + + if not TRIPLES_MAP_RE.search(text): + report["errors"].append( + "No rml:logicalSource or rr:subjectMap found; this does not look like an " + "RML mapping." + ) + + urls = CSVW_URL_RE.findall(text) + report["sources"] = sorted(set(urls)) + if not urls: + report["errors"].append( + 'No csvw:url found. Each logical source needs one of: ' + + ", ".join(sorted(CANONICAL_SOURCE_PATHS)) + ) + for url in sorted(set(urls)): + if url not in CANONICAL_SOURCE_PATHS: + report["errors"].append( + f"Logical source '{url}' is not one of the five paths the wrapper " + "rewrites per input, so it will not point at the generated TSV. Use " + "one of: " + ", ".join(sorted(CANONICAL_SOURCE_PATHS)) + ) + + owners = all_known_columns() + used = referenced_columns(text) + report["columns"] = sorted(used) + for column in sorted(used): + if column not in owners: + report["errors"].append( + f"Column '{column}' is not written by any pipeline TSV. Run " + "'vcf-rdfizer-rules columns' for the full list." + ) + + unused_sources = CANONICAL_SOURCE_PATHS - set(urls) + if unused_sources: + report["info"].append( + "Sources not used by this mapping: " + ", ".join(sorted(unused_sources)) + ) + + strategy = sample_support_strategy(rules_path) + if strategy == "none": + report["info"].append( + "No sample helper tables are consumed; the wrapper emits genotype RDF " + "directly and no helper rows are materialized." + ) + elif strategy == "stream": + report["info"].append( + "The four built-in sample maps are present unchanged, so the helper " + "tables stay header-only and genotype RDF is streamed directly." + ) + else: + report["warnings"].append( + "This mapping consumes sample_calls.tsv or sample_format_values.tsv in a " + "way the wrapper cannot stream, so those helper tables will be " + "materialized: one row per variant x sample, and one per variant x " + "sample x FORMAT key. On a cohort VCF that is the largest intermediate " + "the pipeline can produce." + ) + + for representation in ("expanded", "condensed"): + try: + workflow = resolve_sample_workflow(representation, rules_path) + except ValueError as exc: + report["sample_representations"][representation] = { + "usable": False, + "reason": str(exc), + } + continue + report["sample_representations"][representation] = { + "usable": True, + "helper_strategy": workflow.helper_strategy, + "emitter": workflow.emitter or "none (RML emits genotypes)", + } + + report["ok"] = not report["errors"] + return report + + +def print_report(report: dict) -> None: + print(f"Checked: {report['rules_path']}") + print() + print("Logical sources:") + for url in report["sources"]: + mark = "ok " if url in CANONICAL_SOURCE_PATHS else "BAD" + print(f" [{mark}] {url}") + print() + print(f"Referenced columns ({len(report['columns'])}): {', '.join(report['columns']) or '(none)'}") + print() + print("Sample representation compatibility:") + for representation, detail in report["sample_representations"].items(): + if detail["usable"]: + print( + f" --sample-representation {representation}: usable " + f"(helper tables: {detail['helper_strategy']}, emitter: {detail['emitter']})" + ) + else: + print(f" --sample-representation {representation}: NOT usable") + print(f" {detail['reason']}") + + for label, items in (("Errors", report["errors"]), ("Warnings", report["warnings"]), ("Notes", report["info"])): + if not items: + continue + print() + print(f"{label}:") + for item in items: + print(f" - {item}") + + print() + print("Result: OK" if report["ok"] else "Result: FAILED") + + +def command_columns(args: argparse.Namespace) -> int: + selected = TSV_SOURCES if args.source is None else {args.source: TSV_SOURCES[args.source]} + if args.json: + payload = { + name: { + "container_path": source.container_path, + "description": source.description, + "columns": list(source.columns), + "dynamic_tail_column": source.dynamic_tail_column, + "notes": list(source.notes), + } + for name, source in selected.items() + } + print(json.dumps(payload, indent=2)) + return 0 + + for name, source in selected.items(): + print(f"{name} ({source.container_path})") + print(f" {source.description}") + print(" Columns:") + for column in source.columns: + print(f" - {column}") + if source.dynamic_tail_column: + print(f" - or {source.dynamic_tail_column} (name varies per input)") + for note in source.notes: + print(f" Note: {note}") + print() + return 0 + + +def command_init(args: argparse.Namespace) -> int: + output = Path(args.output).expanduser() + if output.exists() and not args.force: + print(f"error: {output} already exists (use --force to overwrite)", file=sys.stderr) + return 2 + try: + default_rules = resolve_default_rules_path(Path(__file__).resolve().parent) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + INIT_HEADER + default_rules.read_text(encoding="utf-8"), encoding="utf-8" + ) + print(f"Wrote {output} (from {default_rules})") + print("Next: edit it, then run") + print(f" vcf-rdfizer-rules check {output}") + return 0 + + +def command_check(args: argparse.Namespace) -> int: + rules_path = Path(args.rules).expanduser() + if not rules_path.is_file(): + print(f"error: rules file not found: {rules_path}", file=sys.stderr) + return 2 + try: + report = check_rules(rules_path) + except OSError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if args.json: + print(json.dumps(report, indent=2)) + else: + print_report(report) + return 0 if report["ok"] else 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="vcf-rdfizer-rules", + description=( + "Inspect, scaffold, and validate custom RML mappings for VCF-RDFizer." + ), + formatter_class=argparse.RawTextHelpFormatter, + epilog=( + "Examples:\n" + " See what columns a mapping can reference:\n" + " vcf-rdfizer-rules columns\n" + " Start a custom mapping from the shipped default:\n" + " vcf-rdfizer-rules init -o my_rules.ttl\n" + " Validate it before a long run:\n" + " vcf-rdfizer-rules check my_rules.ttl\n" + " Then use it:\n" + " vcf-rdfizer --mode full -i cohort.vcf.gz --rules my_rules.ttl \\\n" + " --rdf-storage-mode plain -o ./results\n" + ), + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + columns = subparsers.add_parser( + "columns", help="List the TSV sources and columns a mapping can reference" + ) + columns.add_argument( + "--source", choices=sorted(TSV_SOURCES), default=None, help="Show one source only" + ) + columns.add_argument("--json", action="store_true", help="Emit machine-readable JSON") + columns.set_defaults(func=command_columns) + + init = subparsers.add_parser( + "init", help="Write an annotated copy of the default mapping to start from" + ) + init.add_argument( + "-o", "--output", default="custom_rules.ttl", help="Destination path (default: custom_rules.ttl)" + ) + init.add_argument("--force", action="store_true", help="Overwrite an existing file") + init.set_defaults(func=command_init) + + check = subparsers.add_parser( + "check", help="Validate a mapping against the wrapper contract" + ) + check.add_argument("rules", help="Path to the .ttl mapping to check") + check.add_argument("--json", action="store_true", help="Emit machine-readable JSON") + check.set_defaults(func=command_check) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) From ed50c838bcfcc9bfb83d625dad96e05b7faedfc1 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Fri, 4 Sep 2026 20:05:50 +0200 Subject: [PATCH 18/19] extensive improvement of validation suite and general cleaning --- .github/workflows/validation-mutation.yml | 70 + ACKNOWLEDGEMENTS.md | 2 +- Dockerfile | 58 +- README.md | 97 +- changelog.md | 421 ++++ docs/README.md | 124 ++ docs/architecture.md | 195 ++ docs/cli-reference.md | 162 ++ docs/conversion.md | 237 +++ docs/datalinking-design.md | 300 +++ docs/limitations.md | 201 ++ docs/output-and-metrics.md | 187 ++ docs/representations.md | 187 ++ docs/rml-mappings.md | 170 ++ docs/roadmap.md | 130 ++ docs/sample-representation-guide.md | 14 + docs/validation-methodology.md | 198 ++ docs/validation.md | 322 ++- docs/vcf-coverage.md | 150 ++ pyproject.toml | 3 + rules/default_rules.ttl | 6 + .../queries/common/preflight_blank_nodes.rq | 18 + .../common/preflight_blank_nodes_count.rq | 8 + .../common/preflight_distinct_triple_count.rq | 17 + .../queries/common/preflight_empty_values.rq | 23 + .../common/preflight_empty_values_count.rq | 11 + ...eflight_missing_token_conformance_count.rq | 9 + .../common/preflight_position_datatype.rq | 16 +- .../preflight_position_datatype_count.rq | 13 + .../preflight_record_cardinality_count.rq | 19 + .../queries/common/q07_file_metadata.rq | 17 + .../queries/common/q08_header_line_census.rq | 12 + .../queries/common/q09_predicate_census.rq | 15 + .../queries/common/q10_class_census.rq | 11 + .../queries/common/q11_record_digest.rq | 43 + .../queries/common/q12_info_value_digest.rq | 17 + .../preflight_representation_profile_count.rq | 9 + .../condensed/q13_format_value_digest.rq | 15 + .../preflight_representation_profile_count.rq | 9 + .../expanded/q13_format_value_digest.rq | 15 + src/validation/validation_runner.py | 1819 +++++++++++++++-- test/README.md | 28 + test/cross_engine_agreement.py | 143 ++ test/test_validation_engines_unit.py | 534 +++++ test/test_validation_logic_unit.py | 467 +++++ test/test_validation_mutation_unit.py | 217 ++ test/test_vcf_rdfizer_unit.py | 20 +- test/validation_fixtures.py | 611 ++++++ test/validation_mutations.py | 474 +++++ vcf_rdfizer.py | 991 ++++++++- 50 files changed, 8605 insertions(+), 230 deletions(-) create mode 100644 .github/workflows/validation-mutation.yml create mode 100644 docs/README.md create mode 100644 docs/architecture.md create mode 100644 docs/cli-reference.md create mode 100644 docs/conversion.md create mode 100644 docs/datalinking-design.md create mode 100644 docs/limitations.md create mode 100644 docs/output-and-metrics.md create mode 100644 docs/representations.md create mode 100644 docs/rml-mappings.md create mode 100644 docs/roadmap.md create mode 100644 docs/validation-methodology.md create mode 100644 docs/vcf-coverage.md create mode 100644 src/validation/queries/common/preflight_blank_nodes.rq create mode 100644 src/validation/queries/common/preflight_blank_nodes_count.rq create mode 100644 src/validation/queries/common/preflight_distinct_triple_count.rq create mode 100644 src/validation/queries/common/preflight_empty_values.rq create mode 100644 src/validation/queries/common/preflight_empty_values_count.rq create mode 100644 src/validation/queries/common/preflight_missing_token_conformance_count.rq create mode 100644 src/validation/queries/common/preflight_position_datatype_count.rq create mode 100644 src/validation/queries/common/preflight_record_cardinality_count.rq create mode 100644 src/validation/queries/common/q07_file_metadata.rq create mode 100644 src/validation/queries/common/q08_header_line_census.rq create mode 100644 src/validation/queries/common/q09_predicate_census.rq create mode 100644 src/validation/queries/common/q10_class_census.rq create mode 100644 src/validation/queries/common/q11_record_digest.rq create mode 100644 src/validation/queries/common/q12_info_value_digest.rq create mode 100644 src/validation/queries/condensed/preflight_representation_profile_count.rq create mode 100644 src/validation/queries/condensed/q13_format_value_digest.rq create mode 100644 src/validation/queries/expanded/preflight_representation_profile_count.rq create mode 100644 src/validation/queries/expanded/q13_format_value_digest.rq create mode 100644 test/cross_engine_agreement.py create mode 100644 test/test_validation_engines_unit.py create mode 100644 test/test_validation_logic_unit.py create mode 100644 test/test_validation_mutation_unit.py create mode 100644 test/validation_fixtures.py create mode 100644 test/validation_mutations.py diff --git a/.github/workflows/validation-mutation.yml b/.github/workflows/validation-mutation.yml new file mode 100644 index 0000000..e8fe117 --- /dev/null +++ b/.github/workflows/validation-mutation.yml @@ -0,0 +1,70 @@ +name: validation-mutation + +# The mutation score is the measured coverage of the semantic validation suite. +# The host job runs on every push because it needs no Docker; the container job +# is the authority, replaying the same catalogue under the engines that ship in +# the image. + +on: + push: + branches: + - "**" + pull_request: + workflow_dispatch: + +jobs: + mutation-score: + name: mutation score (rdflib) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install test dependencies + run: python -m pip install --upgrade pip rdflib + + - name: Run the mutation harness + env: + VCF_RDFIZER_MUTATION_REPORT: mutation-score.json + run: python -m unittest test.test_validation_mutation_unit -v + + - name: Report the score + run: | + python - <<'PY' + import json + report = json.load(open("mutation-score.json")) + print(f"mutation score: {report['detected']}/{report['total']} " + f"({report['score']:.0%})") + for item in report["mutations"]: + if not item["detected"]: + print(f" undetected: {item['id']} ({item['representation']}) " + f"- {item['knownUndetected']}") + PY + + - name: Upload the score + uses: actions/upload-artifact@v4 + with: + name: mutation-score + path: mutation-score.json + + engine-agreement: + name: cross-engine query agreement + runs-on: ubuntu-latest + # The image build is slow, so this gates merges rather than every push. + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build the image + run: docker build -t vcf-rdfizer:mutation-ci . + + - name: Every validation query must agree across Comunica and QLever + run: | + docker run --rm -v "$PWD:/repo:ro" vcf-rdfizer:mutation-ci \ + /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index b8c67dc..e741ae6 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -15,7 +15,7 @@ adapted for the Acknowledgements section: > This work was carried out at KNoWS, IDLab, Ghent University - imec, as part > of the doctoral research of Elias Crum. The authors thank the maintainers of > the open-source components on which VCF-RDFizer builds - RMLStreamer, -> hdt-cpp, hdtc, pycottas, Comunica, bcftools, and cyvcf2. +> hdt-cpp, hdtc, pycottas, Comunica, QLever, pySHACL, bcftools, and cyvcf2. If the work is co-funded by a specific project or grant, add that project name and grant number to the sentence above and record it in the table at the top of diff --git a/Dockerfile b/Dockerfile index e0cc32e..25422ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,11 @@ ARG RMLSTREAMER_VERSION=2.5.0 ARG HDTC_VERSION=1.1.0 ARG COMUNICA_VERSION=5.3.0 +# QLever is an optional second SPARQL engine for validation. Its binaries are +# copied from the upstream published image rather than built here: compiling +# QLever needs a large C++ toolchain and would dominate this image's build. +# Pin a digest or release tag here to make validation runs reproducible. +ARG QLEVER_IMAGE=adfreiburg/qlever:latest FROM eclipse-temurin:11-jre AS build-hdt-cpp @@ -59,6 +64,10 @@ RUN cargo build --locked --release \ && cp LICENSE /opt/third_party_licenses/HDTC.LICENSE +# Named stage so the runtime image can COPY QLever's binaries out of it. +FROM ${QLEVER_IMAGE} AS qlever + + FROM eclipse-temurin:11-jre ARG RMLSTREAMER_VERSION @@ -94,7 +103,8 @@ RUN python3 -m venv /opt/pycottas-venv \ duckdb==1.5.5 \ pyarrow==22.0.0 \ numpy==2.4.6 \ - cyvcf2==0.34.0 + cyvcf2==0.34.0 \ + pyshacl==0.30.1 RUN npm install --global "@comunica/query-sparql-file@${COMUNICA_VERSION}" @@ -110,6 +120,33 @@ COPY --from=build-hdt-cpp /usr/local/lib/libhdt* /usr/local/lib/ COPY --from=build-hdt-cpp /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ COPY --from=build-hdtc /opt/hdtc/target/release/hdtc /usr/local/bin/hdtc COPY --from=build-hdtc /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ + +# Optional QLever SPARQL engine (--validation-engine qlever). Comunica remains +# the default, so an image whose QLever binaries turn out to be unusable is +# still fully functional; the validator reports the reason instead of failing +# obscurely. Pin a different tag or digest with +# --build-arg QLEVER_IMAGE=adfreiburg/qlever:. +# +# QLever's image is built on a different Ubuntu release than this one, so the +# binaries alone are not enough: their Boost, ICU, jemalloc and io_uring +# sonames are release-specific and absent here. Those libraries travel with the +# binaries into a private directory that only QLever's own processes are +# pointed at, so they cannot shadow anything the rest of the image links +# against. (glibc itself is not copied - it is backward compatible, and this +# base is newer than QLever's.) +COPY --from=qlever /qlever/qlever-index /opt/qlever/bin/qlever-index +COPY --from=qlever /qlever/qlever-server /opt/qlever/bin/qlever-server +COPY --from=qlever \ + /lib/x86_64-linux-gnu/libboost_iostreams.so.1.83.0 \ + /lib/x86_64-linux-gnu/libboost_program_options.so.1.83.0 \ + /lib/x86_64-linux-gnu/libboost_url.so.1.83.0 \ + /lib/x86_64-linux-gnu/libgomp.so.1 \ + /lib/x86_64-linux-gnu/libicudata.so.74 \ + /lib/x86_64-linux-gnu/libicui18n.so.74 \ + /lib/x86_64-linux-gnu/libicuuc.so.74 \ + /lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /lib/x86_64-linux-gnu/liburing.so.2 \ + /opt/qlever/lib/ COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES.md COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ @@ -121,7 +158,22 @@ COPY vcf_rdfizer_gzip.py /opt/vcf-rdfizer/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ && chmod +x /usr/local/bin/rdf2hdt \ && chmod +x /usr/local/bin/hdt2rdf \ - && chmod +x /usr/local/bin/hdtc + && chmod +x /usr/local/bin/hdtc \ + && chmod +x /opt/qlever/bin/qlever-index /opt/qlever/bin/qlever-server + +# QLever's binaries come from a different base image, so record at build time +# whether they actually link here. The validator reads this marker to explain +# an unavailable engine instead of surfacing a bare "not found". +RUN set -eu; \ + status="ok"; \ + for binary in qlever-index qlever-server; do \ + missing="$(LD_LIBRARY_PATH=/opt/qlever/lib ldd "/opt/qlever/bin/$binary" 2>&1 | grep 'not found' || true)"; \ + if [ -n "$missing" ]; then \ + status="QLever binary $binary has unresolved shared libraries in this base image: $missing"; \ + echo "WARNING: $status" >&2; \ + fi; \ + done; \ + printf '%s\n' "$status" > /opt/vcf-rdfizer/qlever-status.txt ENV RMLSTREAMER_JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar @@ -132,6 +184,8 @@ ENV COTTAS_MERGE_BATCH_ROWS=2048 ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python +ENV QLEVER_INDEX_BUILDER_BIN=/opt/qlever/bin/qlever-index +ENV QLEVER_SERVER_BIN=/opt/qlever/bin/qlever-server ENV LD_LIBRARY_PATH=/usr/local/lib # COTTAS creates a temporary DuckDB database in the container working diff --git a/README.md b/README.md index 089ec5f..b4a6ef3 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,12 @@ VCF-RDFizer is a Docker-first CLI wrapper for: The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#](https://w3id.org/vcf-rdfizer/vocab#). +This README is the task-oriented reference. For how the tool works, why it is +built that way, and where it stops working, see the documentation set in +**[`docs/`](docs/README.md)** - starting with +[Architecture](docs/architecture.md) and, before you rely on the output, +[Limitations](docs/limitations.md). + ## Requirements - Python 3.10+ @@ -134,6 +140,59 @@ stage summary at `stages/validation/.json`. See [Semantic VCF/RDF validation](docs/validation.md) for query definitions, preflight checks, result statuses, and cleanup evidence. + +### Artifacts and engines + +`--rdf` accepts any artifact the pipeline produces - `.nt`, `.nt.gz`, `.nt.br`, +`.hdt`, `.cottas`, `.cottas.gz`, `.cottas.br`. A compressed or indexed artifact +is decoded back to N-Triples **inside the container** and then put through the +full semantic suite, which proves it decodes to a graph that still reproduces +every VCF summary - stronger than the triple-count round-trip that runs during +compression. + +In full mode, `--validate-artifacts {aggregate,hdt,cottas,all}` chooses which +produced artifacts to check; each is validated independently with its own +report: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz \ + --rdf-storage-mode space-optimized --representations hdt,cottas \ + --validate --validate-artifacts all -o ./results +``` + +`--validation-engine {comunica,qlever}` selects the SPARQL backend. Comunica +(default) queries the file in memory; [QLever](https://github.com/ad-freiburg/qlever) +builds an on-disk index inside the container and serves it, which is what makes +cohort-scale graphs queryable. Both answer identical queries, so the choice is +never semantic, and every report records which engine ran. + +```bash +vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.hdt \ + --validation-engine qlever --qlever-memory-gb 32 -o ./validation-results +``` + +Tuning: `--qlever-memory-gb`, `--qlever-port`, `--qlever-startup-timeout`, +`--validation-query-timeout`, and repeatable `--qlever-index-arg` / +`--qlever-server-arg` escape hatches. + +Validation runs three independent layers: exact aggregate comparison against +the VCF, a predicate/class census plus per-record and per-value identity +digests, and — with `--shacl-shapes` — SHACL conformance against the +vocabulary's published shapes. + +```bash +vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --shacl-shapes ./vocabulary/shacl/vcf-rdfizer-vocabulary.shacl.ttl \ + --strict-conformance -o ./validation-results +``` + +> **What a PASS means.** Coverage is measured, not asserted: a mutation harness +> corrupts a correct graph in 36 named ways and records which are detected +> (currently **64/66**). See [`docs/vcf-coverage.md`](docs/vcf-coverage.md) for +> the element-by-element matrix and the remaining gaps, and +> [`docs/validation-methodology.md`](docs/validation-methodology.md) for how the +> number is produced. + ## Compression Plan Compression is configured as three independent decisions: @@ -204,6 +263,23 @@ host filesystem. - `--remove-rdf-storage-output` explicitly remove the aggregate `.nt`/`.nt.gz` after successful compression - `-e, --estimate-size` preflight size estimate +## VCF Coverage + +Full-mode conversion covers every VCF column. Three options control how much +structure is emitted; all default to the richer form. + +| Option | Effect | +| --- | --- | +| `--sample-representation {expanded,condensed}` | Genotype shape (see below) | +| `--info-representation {structured,raw}` | `structured` adds one `vcfr:InfoFieldValue` per record and key, with typed values, alongside `vcfr:infoRaw` | +| `--header-representation {structured,basic}` | `structured` types each `##` line with its vocabulary subclass and lifts FILTER/ALT/contig attributes into their own properties | + +QUAL is always emitted, typed `xsd:decimal` or `vcfr:Null`, because the +published SHACL shape requires a datatype that depends on the value. + +[`docs/vcf-coverage.md`](docs/vcf-coverage.md) maps every VCF element to its RDF +terms and to the validation check that covers it. + ## Sample Representation Modes Full mode has exactly two explicit sample workflows. There is no automatic @@ -853,10 +929,25 @@ in `vcf_rdfizer.py` append it directly to the aggregate, because the equivalent RML maps would first have to materialize variants x samples (x FORMAT keys) helper TSV rows. See [`docs/sample-representation-guide.md`](docs/sample-representation-guide.md). -Further reading: +Further reading: **[`docs/`](docs/README.md) is the in-depth documentation set** - +how each part of the tool works, why, and where it stops working. + +| Document | Covers | +| --- | --- | +| [Architecture](docs/architecture.md) | Host/container split, failure policy, pinned toolchain | +| [Conversion](docs/conversion.md) | VCF -> TSV -> RDF, stage by stage | +| [Representations](docs/representations.md) | Compression, HDT/COTTAS, chunking, round-trip checks | +| [Output and metrics](docs/output-and-metrics.md) | Run layout, reports, progress, exit codes | +| [Custom RML mappings](docs/rml-mappings.md) | The `--rules` contract in full | +| [Sample representations](docs/sample-representation-guide.md) | Expanded vs condensed genotype shapes | +| [Validation](docs/validation.md) | The semantic suite, and what it does not test | +| [Validation methodology](docs/validation-methodology.md) | How coverage is measured, not asserted | +| [VCF coverage matrix](docs/vcf-coverage.md) | Element by element, with the mutation that proves each row | +| [CLI reference](docs/cli-reference.md) | Every flag, with constraints and interactions | +| [Limitations](docs/limitations.md) | Everything the tool cannot do, in one place | +| [Roadmap](docs/roadmap.md) | Planned work, known defects, and rejected options | +| [Data linking design](docs/datalinking-design.md) | Proposal: a plug-in system for external links | -- [`docs/validation.md`](docs/validation.md) - semantic validation design and query set -- [`docs/sample-representation-guide.md`](docs/sample-representation-guide.md) - emitted genotype shapes - [`changelog.md`](changelog.md) - dated change history - [`ACKNOWLEDGEMENTS.md`](ACKNOWLEDGEMENTS.md) - funding and attribution diff --git a/changelog.md b/changelog.md index 267bce0..a0e682b 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,426 @@ # Changelog +## 2026-09-04 — Documentation set, and a data-linking design proposal + +`docs/` becomes an in-depth explanation of the whole tool rather than four +validation-focused documents. Every part of the pipeline is now described, with +its limitations stated next to its capabilities rather than in a footnote. + +### Added + +- [`docs/README.md`](docs/README.md) — index, reading paths per audience, and + the conventions the documentation set follows. +- [`docs/architecture.md`](docs/architecture.md) — the host/container split, + what runs where, the **three places that split deliberately leaks** (genotype, + header and QUAL/INFO emission happen on the host because RML cannot choose a + datatype or class per row), the three-way failure policy, and the pinned + toolchain versions. +- [`docs/conversion.md`](docs/conversion.md) — VCF -> TSV -> RDF stage by stage, + including what the single-`awk`-pass parser can and cannot see, the complete + IRI template table, and the datatype and missing-value decisions. +- [`docs/representations.md`](docs/representations.md) — the compression plan's + three independent decisions, record-safe chunking, the `hdtc` and PyArrow + rationales, and what the round-trip check does *not* prove. +- [`docs/output-and-metrics.md`](docs/output-and-metrics.md) — output layout, + the `run_metrics/` tree, input size accounting, progress, interrupts, exit + codes. +- [`docs/rml-mappings.md`](docs/rml-mappings.md) — the `--rules` contract in + full, plus what a custom mapping does *not* control and what it costs in + validation. +- [`docs/cli-reference.md`](docs/cli-reference.md) — every flag, grouped, with + enforced constraints, environment variables, and exit codes. +- [`docs/limitations.md`](docs/limitations.md) — one consolidated, honest + account: operational, input handling, modelling, extension points, + compression, validation, vocabulary, and scope. +- [`docs/roadmap.md`](docs/roadmap.md) — planned work, known defects, and + options that were assessed and deliberately rejected. +- [`docs/datalinking-design.md`](docs/datalinking-design.md) — **design + proposal, not implemented.** A plug-in architecture for linking the graph to + external resources, built on the observation that rsID, gene and clinical + linking are all one of three joins (`token`, `interval`, `allele`). Three + plugin tiers (declarative template, local reference bundle, live service), + enforced network safeguards, side-graph output with provenance, and a build + order that freezes the extension contract before capability is added. + +### Fixed + +- **Documented behaviour that did not match the code.** `README.md` and + `docs/validation.md` both stated that the wrapper switches `q09`-`q13` to + report-only under a custom `--rules`. It does not: `--mapping-policy` exists + in `validation_runner.py` but `vcf_rdfizer.py` never forwards it, so the + runner always executes `strict` and a *correct* custom-mapping conversion is + reported as `MISMATCH`. The claim is corrected in both places, recorded in + [`docs/limitations.md`](docs/limitations.md), and tracked as a defect in + [`docs/roadmap.md`](docs/roadmap.md). No code change yet. + +### Changed + +- Existing documents gained consistent navigation headers and "See also" + footers, and `README.md` now points at `docs/README.md` as the documentation + entry point rather than listing two files. + +## 2026-09-04 — Graph integrity: blank nodes, empty terms, duplicate statements + +Three checks that verify the N-Triples graph itself, independently of what the +VCF contains. Mutation score **64/66 -> 76/78**, catalogue 36 -> 42 mutations. + +### Added + +- `preflight_blank_nodes` — any blank node, in subject or object position. Every + class in the vocabulary declares an `vcfr:iriTemplate`, so a blank node means + a term map produced no IRI; the record and value digests could not address + such a node, and neither could anything that later merges the graph. A + predicate cannot be blank in RDF, so it is not examined. +- `preflight_empty_values` — empty or whitespace-only literals and IRIs, each + labelled `EMPTY_LITERAL` or `EMPTY_IRI` in the sample. An empty literal means + a value was lost rather than marked missing (the pipeline writes + `"."^^vcfr:Null` for a genuine missing token); an empty IRI means a template + substitution collapsed. Whitespace-only counts as empty: it carries no more + information and is just as certainly a defect. +- `preflight_duplicate_triples` — the same statement emitted more than once. + +Both anomaly checks have exact-count companions, so severity is measured rather +than saturating at the 100-row diagnostic sample. + +### How duplicates are detected, and why it could not be a query + +A SPARQL store holds a **set**. A repeated line is collapsed on load, so it is +invisible to every other check here — `COUNT(*)` over `?s ?p ?o` returns the +distinct total on Comunica, QLever and rdflib alike. Verified before building +anything: a three-line file with one repeat reports 2. + +The detector compares the statements the parser read against the distinct +triples the store holds. Raptor supplies the first (`rapper -c` counts parsed +statements including repeats — confirmed in the image: it reports 3 where the +store reports 2), `preflight_distinct_triple_count` the second, and the +difference is exactly the number of redundant statements. Duplicating the whole +fixture graph reports `parsed=624, distinct=312, duplicates=312`. + +This is worth having because duplicated RDF parts are a known failure mode of +the conversion — `run_conversion.sh` already carries a defensive dedupe for it — +and a duplicated aggregate costs twice the storage for no added information. + +### Behaviour + +- All three are blocking: a graph that fails one is not worth comparing. +- When an input a check needs is unavailable, it reports `NOT_EVALUATED` rather + than `PASS`, and `NOT_EVALUATED` never fails a run. A check that could not run + must not be mistaken for a clean result in either direction. + +### Verified + +- All 22 queries (13 core, 9 preflight/count) return identical values under + Comunica and QLever for both representations, and both engines produce `PASS` + against the Python oracle — including the new `ISBLANK`, `ISIRI` and + `REGEX`-based checks. +- 317 tests pass. Six new mutations (`introduce_blank_node`, + `blank_node_object`, `empty_literal`, `whitespace_only_literal`, + `duplicate_triple`, `duplicate_whole_graph`) are all detected. + +### Documentation + +- `docs/validation.md` gained a "Graph integrity" section explaining why + duplicates cannot be found by a query. +- `docs/vcf-coverage.md` lists the three under graph-level properties. +- `docs/validation-methodology.md` now describes four independent layers rather + than three. + +## 2026-09-04 — Full VCF coverage: census, identity digests, INFO, headers, SHACL + +Phases 2-6 of the validation-coverage plan. Mutation score **29/45 -> 64/66** +(64% -> 97%), across a catalogue that grew from 21 to 36 distinct mutations. + +### Fixed — silent data loss + +- **QUAL was extracted from every VCF and never mapped into RDF.** It is now + emitted, typed `xsd:decimal` or `"."^^vcfr:Null` as the published SHACL shape + requires. RML cannot choose a datatype per row, so it comes from a new + record-detail emitter rather than `default_rules.ttl`. +- **`##fileDate` was never mapped.** Now emitted as `vcfr:fileDate`, typed + `xsd:date` when the value's form allows and lexically otherwise. + +### Added — completeness (Phase 2) + +- `q09_predicate_census` and `q10_class_census` compare the graph's predicate + and class inventory against counts derived from the VCF. One comparison + catches three things: a predicate missing, one with the wrong cardinality, and + one that should not be in the graph at all. Expectations are derived from the + VCF and the emitters' documented shapes, never from `default_rules.ttl` - + deriving them from the mapping is how QUAL stayed invisible. +- `--mapping-policy {strict,report-only}`: a custom `--rules` changes the + inventory and IRI templates by design, so these checks become report-only. + +### Added — record and value identity (Phases 3-4) + +- `q11_record_digest`, `q12_info_value_digest`, `q13_format_value_digest` hash + each record, INFO value and FORMAT value **together with its own IRI** and + bucket the result. Binding identity into the hash is what catches a + permutation; bucketing keeps the result at most 256 rows for any graph size + and needs no ordering guarantee, which `GROUP_CONCAT` could not have given + portably. Fields are separated by U+001F, which cannot occur in a VCF field. +- These close the two largest gaps: values permuted between records, and non-GT + FORMAT values (a mangled DP, GQ or AD previously passed). + +### Added — structured INFO (Phase 4) + +- `--info-representation {structured,raw}`, default `structured`: one + `vcfr:InfoFieldValue` per record and key at the vocabulary's declared IRI + template, linked by `vcfr:declaredBy` to an `InfoFieldDefinition` carrying + `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription`. Single-valued + Integer/Float fields also get `fieldValueInteger`/`fieldValueDecimal`; a Flag + gets `fieldValueBoolean true`. `vcfr:infoRaw` is retained. +- The model was already fully defined in the vocabulary and simply unused. + +### Added — header section (Phase 5) + +- `--header-representation {structured,basic}`, default `structured`: each `##` + line is typed with its vocabulary subclass, and FILTER, ALT and contig + declarations get `filterId`, `altId`, `contigId`, `contigLength`, + `contigMd5`, `contigAssembly` and `contigCount`. An unrecognized key keeps + only the base `HeaderLine` type rather than inventing an undefined term. + +### Added — SHACL (Phase 6) + +- `--shacl-shapes PATH` validates the graph against a SHACL shapes file with + `pyshacl`, inside the container, as a layer independent of the VCF entirely. + Off by default because `pyshacl` loads the graph into memory. A violation + blocks the run; a missing `pyshacl` is reported as `EXECUTION_FAILED`, never + as a conformance failure. +- It found three violations the other layers could not see. Two are fixed above. + The third is a **contradiction inside the published vocabulary**: + `vcfr:missingValuePolicy` says a missing token SHOULD be `"."^^vcfr:Null`, + while `VCFRecordShape` constrains `vcfr:alt` to `xsd:string`, so a record with + `ALT=.` cannot satisfy both. Recorded in `docs/vcf-coverage.md` for a decision + in the vocabulary repository. + +### Changed + +- `evaluate_validation()` is the single place the PASS/MISMATCH/BLOCKED decision + is made, used by real runs and the mutation harness alike. +- `test_validation_logic_unit.py` now shares the canonical fixture instead of + maintaining a second description of the same data. + +### Verified + +- All 13 core queries plus 4 exact-count preflights return **identical values + under Comunica and QLever**, for both representations, and both engines + produce `PASS` against the Python oracle end to end. The agreement script now + checks that last part too: engines agreeing with each other is not enough when + the digests are compared against values Python computes. +- 297 tests pass; the suite still passes without `rdflib` (those tests skip). + +### Documentation + +- `docs/vcf-coverage.md` rewritten: every VCF element with separate + "represented" and "validated" columns, each validated claim backed by a named + mutation, the remaining gaps, and the vocabulary alignment problem. +- `docs/validation-methodology.md`: the three independent layers, why identity + digests are histograms, and how to reproduce the score. +- README gained a "VCF Coverage" section for the three representation options. + +## 2026-09-04 — Validation mutation harness, and the first coverage gaps closed + +### Added — measurement (Phase 0) + +- A mutation-testing harness for the semantic validation suite. A correct graph + is corrupted in ~25 named ways and the validator is asked for a verdict; the + proportion detected is a reproducible **mutation score**. Before this, the + suite's coverage was prose, and that prose was wrong: `QUAL` is extracted from + every VCF and then never mapped into RDF, and no check could have noticed. + - `test/validation_fixtures.py` derives the VCF, the RDF graph and the parser + oracle from one declarative specification, so they cannot drift apart. The + genotype triples come from the project's own emitters. + - `test/validation_mutations.py` is the catalogue: each entry names the VCF + element it targets, the check expected to catch it, and — for a recorded + gap — why it is not caught. + - `test/test_validation_mutation_unit.py` evaluates queries in-process with + rdflib (test-only dependency, tests skip without it) and routes every + verdict through the shipped `evaluate_validation()`, so the harness measures + real code rather than a reimplementation. + - **Known gaps are assertions.** A mutation marked `known_undetected` is + asserted to still be undetected, so closing a gap fails a test and forces + the catalogue and `docs/vcf-coverage.md` to be updated. +- `test/cross_engine_agreement.py` plus a CI job asserting every validation + query returns identical values under Comunica and QLever. Verified: all 34 + executions across both representations agree. +- `.github/workflows/validation-mutation.yml` publishes `mutation-score.json` + as a build artifact. + +### Added — coverage (Phase 1) + +- **Exact anomaly counts.** Each structural anomaly preflight now has a + companion aggregate returning the true count alongside the `LIMIT 100` + diagnostic sample. Reports carry `anomalyCount`, `anomalyCountReturned` and + `sampleTruncated`, so ten million anomalies is no longer indistinguishable + from a hundred. Verified against a graph with 250 anomalies: exact count 250, + sample 100, truncated true. +- **`--strict-conformance`** (runner and wrapper) promotes a missing-token + conformance failure from a report-only observation to a validation failure. +- **File metadata and header coverage.** New `q07_file_metadata` compares the + `##fileformat` / `##reference` / `##source` declarations, and + `q08_header_line_census` compares how many `##` lines carry each header key. + `parse_vcf` gained `parse_header_metadata` to compute the oracle side. These + close the two entirely uncovered triples maps. + +Mutation score: **23/45 → 29/45 (51% → 64%)**. + +### Changed + +- The PASS / MISMATCH / BLOCKED_BY_PREFLIGHT decision is extracted into + `evaluate_validation()`, the single place real runs and the mutation harness + both use. +- `normalize()` generalises its single-row handling beyond `q03_titv` via + `SINGLE_ROW_QUERIES`. +- `rdflib>=7.0.0` added to the `dev` extra. It is not a runtime dependency and + the test suite passes without it. + +### Documentation + +- **`docs/vcf-coverage.md`** — the coverage matrix: one row per VCF element, + with separate "represented" and "validated" columns, each validated claim + backed by a named mutation. Includes the vocabulary alignment gap (17 emitted + terms the vocabulary does not define, all condensed-representation) and the + open gaps in priority order. This is the table intended for publication. +- **`docs/validation-methodology.md`** — the mutation-testing method, why + self-reported coverage failed, the two-engine two-layer design, and how to + reproduce the score. +- `docs/validation.md` — Q7/Q8 documented, the exact-count behaviour and + `--strict-conformance` described, and the "What is *not* tested" section + narrowed to what is still true. + +## 2026-09-04 — QLever integration verified against a live container + +Everything below came out of actually running the integration, and each item +was wrong or missing beforehand. + +### Fixed + +- **`preflight_position_datatype` would have failed every QLever run.** QLever + canonicalises numeric literals at index time, so `"100"^^xsd:integer` is + reported by `DATATYPE()` as `xsd:int`. The query required exactly + `xsd:integer`, so on QLever it flagged every record and the run ended as + `BLOCKED_BY_PREFLIGHT`, while passing on Comunica. It now accepts the XSD + integer family, which is what the check means; a plain-string or + `xsd:decimal` POS is still reported as an anomaly on both engines (verified). +- **QLever's binaries are not called `IndexBuilderMain`/`ServerMain`, and are + not in `/usr/bin`.** They are `/qlever/qlever-index` and + `/qlever/qlever-server`. The Dockerfile COPY paths and the default argv were + both wrong and would have failed the build. +- **Copying the binaries alone does not work.** The upstream image is Ubuntu + 24.04 and this base is Ubuntu 26.04; the binaries need Boost 1.83, ICU 74, + jemalloc and io_uring sonames that do not exist here and cannot be + apt-installed (the sonames are release-pinned). Those nine libraries are now + copied into `/opt/qlever/lib`, and `LD_LIBRARY_PATH` is set for QLever's own + processes only, so they cannot shadow anything else in the image. glibc is + deliberately not copied - it is backward compatible and this base is newer. +- **`qlever-server` defaults to a 30-second query timeout**, far below what the + aggregate queries need. The server is now started with `-s` matching + `--validation-query-timeout`. +- Index/server flags corrected against the real CLI: `-m` is `--stxxl-memory` + for the indexer and `--memory-max-size` for the server (`--stxxl-memory` is + not a server flag). + +### Verified + +- Both binaries link and run in the target image (`qlever-index --version` + reports build `bfd5741`). +- A real index build, server start, readiness poll, HTTP query, SPARQL Results + JSON parse and `normalize()` round-trip, driven through the shipped + `QleverEngine` code rather than a reimplementation. +- **Engine equivalence**: all eleven validation queries run under both Comunica + and QLever against the same expanded and condensed graphs — graphs built with + the project's own sample emitters — produce identical normalized results, and + the values are independently correct. + +### Tests + +- `QleverEnvironmentTests`: the private library path is prepended for QLever + only, and the POS datatype preflight accepts the integer family while still + rejecting string/decimal/double. +- The QLever lifecycle test now asserts the verified binary names and flags, + including the explicit server-side query timeout. + +### Still unverified + +- The HDT and COTTAS decode paths, and a full `docker build .` of the real + image. Those need hdt-cpp and the pycottas venv; the QLever work above was + done against a minimal probe image that isolates the integration. + +## 2026-09-04 — Validation: QLever engine, HDT/COTTAS artifacts, coverage audit + +### Added + +- `--validation-engine {comunica,qlever}`. Comunica remains the default and + queries the graph in memory; [QLever](https://github.com/ad-freiburg/qlever) + builds an on-disk index inside the container, serves it on a container-local + port, answers the queries over HTTP, and tears both down. Both engines answer + identical queries and feed the same comparison layer, so the choice is a scale + decision, never a semantic one, and every report records which engine ran. + Tunable with `--qlever-memory-gb`, `--qlever-port`, + `--qlever-startup-timeout`, and `--validation-query-timeout`. +- QLever binaries are copied into the image from the upstream + `adfreiburg/qlever` image; pin a version with + `--build-arg QLEVER_IMAGE=adfreiburg/qlever:`. A build-time linkage + check records whether they resolve against this base, and the validator + surfaces that note if the engine cannot start. +- QLever's CLI has changed across releases, so both command lines are + overridable without code changes: repeatable `--qlever-index-arg` / + `--qlever-server-arg`, or full replacement via `QLEVER_INDEX_COMMAND` / + `QLEVER_SERVER_COMMAND` with `{index}`, `{input}`, `{memory}`, `{port}` + placeholders. The exact argv that ran is recorded in `manifest.json`. +- Validation of compressed and indexed artifacts. `--rdf` now accepts `.nt`, + `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, and `.cottas.br`; + `--rdf-format` overrides filename detection. Anything that is not already + plain N-Triples is decoded in container scratch (`hdt2rdf`, + `cottas_tool.py decompress`, gzip/brotli) and then put through the full + semantic suite. That is strictly stronger than the triple-count round-trip + `validate_compression.py` performs during compression: it proves the artifact + decodes to a graph that still reproduces every VCF summary. Nothing decoded + is written beneath `--out`. +- `--validate-artifacts {aggregate,hdt,cottas,all}` for full mode. Each + requested artifact is validated independently with its own report directory + (``, `__hdt`, `__cottas`), so one run can prove the aggregate and + both representations agree with the VCF. A representation that was not + selected, or whose artifact is missing after a recoverable index warning, is + skipped rather than reported as a failure. Default is `aggregate`, which + preserves the previous behaviour. +- `materialization.json` per validation run, recording how the artifact was + decoded and how many triples that yielded; `manifest.json` gained `engine` + and a format-tagged `sourceRdf`. Rapper's parsed triple count is now captured + rather than discarded. + +### Changed + +- The validator's cyvcf2 import is lazy, so the pure normalization/comparison + layer is importable and testable on the host without the container. +- `--mode validation` no longer requires `.nt.gz`; any supported artifact works. + `--rdf-nt` / `--rdf-gz` remain accepted as aliases. + +### Tests + +- `test/test_validation_logic_unit.py` (16 tests): mutation tests that drive the + comparison layer directly and record what it detects — dropped records, + misclassified shapes, flipped genotypes, corrupted FILTER lexicals, Ti/Tv + swaps, allele-count errors, spurious records, impossible AC/AN — **and** what + it does not: record permutation, and the entirely unvalidated `ID`, `QUAL`, + `INFO`, non-`GT` `FORMAT`, and header/metadata columns. The blind-spot tests + are deliberate, so closing a gap fails a test instead of passing unnoticed. +- `test/test_validation_engines_unit.py` (25 tests): artifact format detection + and host/container agreement, each decode path, decode-failure reporting, + engine registry, Comunica command construction, QLever index/serve/teardown, + crashed-server diagnostics, command-line overrides, CLI argument validation, + and the wrapper's target resolution and docker argv. + +### Documentation + +- `docs/validation.md`: new "Which artifact is validated", "Which SPARQL engine + runs the queries", and — importantly — "What is *not* tested", which states + the coverage limits plainly: aggregate-only comparison with no per-record + identity, unvalidated VCF columns, no completeness bound, saturating preflight + anomaly counts. +- README: validation artifacts/engines subsection with a short note on what a + `PASS` does and does not mean. + ## 2026-09-04 — Custom-mapping tooling, cheap gzip sizing, dead-script removal ### Added diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..e6b8398 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,124 @@ +# VCF-RDFizer documentation + +VCF-RDFizer converts VCF files into RDF, optionally into compressed and +queryable representations (HDT, COTTAS), and can prove that the result still +reproduces the source VCF's semantics. + +The [top-level README](../README.md) is the task-oriented quick start: install, +flags, worked commands. **These documents are the explanation** — how each part +works, why it was built that way, and where it stops working. + +Every document here states its own limits. If you only read one page before +deciding whether the tool fits your problem, read +[Limitations](limitations.md). + +--- + +## Start here + +| If you want to… | Read | +| --- | --- | +| Understand how the tool is put together | [Architecture](architecture.md) | +| Know exactly what happens to a VCF | [Conversion](conversion.md) | +| Decide what to convert *into* | [Representations](representations.md) | +| Trust the output | [Validation](validation.md) | +| Know what the tool cannot do | [Limitations](limitations.md) | + +## The whole set + +### How it works + +- **[Architecture](architecture.md)** — the host/container split, what runs + where, the three places the split deliberately leaks, the failure policy, and + the pinned toolchain. +- **[Conversion](conversion.md)** — VCF → TSV → RDF stage by stage: the `awk` + parser and its blind spots, the RML mapping, the wrapper's own emitters, the + IRI templates, datatype and missing-value decisions. +- **[Representations](representations.md)** — the compression plan's three + independent decisions, record-safe chunking, HDT via native `hdtc`, the + bounded COTTAS merge, round-trip verification, and index maintenance. +- **[Output and metrics](output-and-metrics.md)** — the output layout, the + `run_metrics/` tree, input size accounting, progress, interrupts, exit codes. + +### How to extend it + +- **[Custom RML mappings](rml-mappings.md)** — the `--rules` contract, the + `vcf-rdfizer-rules` CLI, and an honest account of what a custom mapping does + *not* control and what it costs in validation. +- **[Data linking design](datalinking-design.md)** — *proposal, not + implemented.* A plug-in architecture for connecting the graph to external + resources (rsIDs, genes, clinical assertions) with declarative linkers, + reference bundles, live-API safeguards, and provenance. + +### What the graph looks like + +- **[Sample representations](sample-representation-guide.md)** — expanded versus + condensed genotype shapes, worked examples, scaling arithmetic, the trade-off, + and an assessment of query-time decoding options. +- **[VCF coverage matrix](vcf-coverage.md)** — element by element: is it + represented, and would a corruption of it be detected. Includes the current + mutation score and the vocabulary alignment gaps. + +### Whether to trust it + +- **[Validation](validation.md)** — how to run the semantic suite, which + artifact and engine to choose, what the thirteen queries and the preflights + check, and a candid "what is *not* tested". +- **[Validation methodology](validation-methodology.md)** — how coverage is + *measured* rather than asserted: the mutation-testing harness, why known gaps + are assertions rather than comments, and how to reproduce the score. + +### Where it is going + +- **[Limitations](limitations.md)** — everything the tool cannot do, does badly, + or does surprisingly, in one place. +- **[Roadmap](roadmap.md)** — what is planned, what is known-broken, and what + has been assessed and deliberately rejected. + +--- + +## Reading paths + +**"I have a VCF and I want RDF."** +[README](../README.md) → [Conversion](conversion.md) → +[Sample representations](sample-representation-guide.md) → +[Representations](representations.md) + +**"I need to defend this output in a paper."** +[Validation](validation.md) → [Validation methodology](validation-methodology.md) +→ [VCF coverage matrix](vcf-coverage.md) → [Limitations](limitations.md) + +**"I want to change what RDF comes out."** +[Architecture](architecture.md) → [Conversion](conversion.md) → +[Custom RML mappings](rml-mappings.md) + +**"I want to add my own domain's links."** +[Data linking design](datalinking-design.md) → +[Custom RML mappings](rml-mappings.md) → +[Validation methodology](validation-methodology.md) + +**"A cohort-scale run just failed."** +[Representations §10](representations.md#10-limitations) → +[Output and metrics §2](output-and-metrics.md#2-the-run-metrics-directory) → +[Limitations §1](limitations.md#1-operational) + +--- + +## A note on how these documents are written + +Three conventions, kept deliberately: + +1. **Claims are measured, not asserted.** Where a document says a defect would + be caught, there is a named mutation in + [`test/validation_mutations.py`](../test/validation_mutations.py) that proves + it — and where one would *not* be caught, that gap is also an assertion, so + closing it fails a test rather than passing silently. +2. **Limitations sit next to capabilities**, not in a footnote. A section that + describes what something does also describes where it stops. +3. **Design rationale is recorded**, including for options that were assessed + and rejected, so the same ground is not re-covered later. + +Related files outside `docs/`: [`rules/README.md`](../rules/README.md) (the +mapping directory), [`test/README.md`](../test/README.md) (the test suite), +[`changelog.md`](../changelog.md), [`scripts/RELEASING.md`](../scripts/RELEASING.md), +and [`ACKNOWLEDGEMENTS.md`](../ACKNOWLEDGEMENTS.md). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..1932b1f --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,195 @@ +# Architecture + +How VCF-RDFizer is put together, why it is split the way it is, and where the +split leaks. Read this before [`conversion.md`](conversion.md) or +[`representations.md`](representations.md), which go stage by stage. + +--- + +## 1. The shape of the tool + +VCF-RDFizer is a **thin host-side Python CLI wrapping a fat Docker image**. The +host process never installs Java, Flink, HDT, QLever, `bcftools` or `cyvcf2`; it +plans work, launches containers, and reads back the JSON and CSV each stage +writes. + +```text + host container (ecrum19/vcf-rdfizer) + ───────────────────────────────── ────────────────────────────────── + vcf_rdfizer.py + ├─ parse + validate arguments + ├─ snapshot inputs + ├─ plan output paths + ├─ collision check + ├─ docker run ────────────────────▶ src/vcf_as_tsv.sh (awk) + │ src/run_conversion.sh (RMLStreamer/Flink) + ├─ append direct RDF (see §4) + ├─ docker run ────────────────────▶ src/partitioned_compression.py + │ src/cottas_tool.py + │ src/ensure_hdt_index.sh + │ src/validate_compression.py + ├─ docker run ────────────────────▶ src/validation/validation_runner.py + └─ assemble run_metrics/ +``` + +The reason for this shape is reproducibility. The toolchain is a pinned set of +awkward dependencies — RMLStreamer 2.5.0 on Flink, a Rust `hdtc` 1.1.0 build, +`pycottas`, Comunica 5.3.0, QLever, `pyshacl`, `cyvcf2`, `bcftools`. Asking a +user to assemble that on their own machine is asking for irreproducible results. +Pinning it in one image means a run on a laptop and a run on a cluster execute +the same binaries. The price is that **Docker is a hard requirement** and the +image is large; see [`limitations.md`](limitations.md). + +## 2. Host responsibilities + +| Responsibility | Why on the host | +| --- | --- | +| Argument validation and mode dispatch | Fail before the first container starts | +| Input snapshotting | A directory input is enumerated once, so files appearing mid-run cannot change the work set | +| Output path planning and collision check | The pipeline never overwrites a planned artifact; the check runs before Docker | +| Uncompressed input sizing | `vcf_rdfizer_gzip.py` reads BGZF block headers / the gzip trailer rather than decompressing | +| Docker orchestration | Mounts, user mapping, environment forwarding, permission auto-fix | +| Progress rendering | Containers write a JSONL sidecar; the host polls and renders it | +| Metrics assembly | Every stage's JSON is collected into one `run_metrics/` tree | +| Interrupt handling | `Ctrl+C` triggers best-effort cleanup of tracked intermediates | + +## 3. Container responsibilities + +| Component | Role | +| --- | --- | +| `src/vcf_as_tsv.sh` | One `awk` pass per VCF producing `records`, `header_lines`, `file_metadata` TSVs | +| `src/run_conversion.sh` | Runs RMLStreamer, normalizes Spark/Flink part files, merges them into one aggregate, records conversion metrics | +| `src/partitioned_compression.py` | Record-safe RDF chunking, chunked HDT/COTTAS generation, merge, all inside an ephemeral Docker volume | +| `src/cottas_tool.py` | `convert` / `merge` / `reindex` / `decompress` over `pycottas`, with a bounded-memory streaming merge | +| `src/ensure_hdt_index.sh` | Java-free `.hdt.index.v1-1` sidecar generation via `hdtc`, restoring the previous sidecar on failure | +| `src/validate_compression.py` | Round-trip check: decode an artifact, compare its triple count against the source | +| `src/validation/validation_runner.py` | Semantic VCF-vs-RDF validation: parser oracle, SPARQL queries, comparison report | + +## 4. Where the split leaks, and why + +The design intent is "all data processing happens in the container". There are +**three deliberate exceptions**, all in `vcf_rdfizer.py`, all appending directly +to the RDF aggregate between the conversion container and the compression +container: + +| Emitter | Emits | Why not RML | +| --- | --- | --- | +| `emit_sample_representation` | `SampleCall`/`FormatFieldValue`, or `SampleSet`/`CohortCallMatrix`/`FormatValueVector` | RML would first have to materialize a helper table of variants × samples (× FORMAT keys) — the largest intermediate the pipeline can produce | +| `append_header_representation_rdf` | Header-line subclasses, FILTER/ALT/contig attributes, INFO/FORMAT declarations | RML cannot choose a class per row from a parsed attribute string | +| `emit_record_detail` | `QUAL` (typed per value) and structured `InfoFieldValue` nodes | RML cannot switch a literal's datatype based on a declared `Type`, and structured INFO would need a variants × INFO-keys helper table | + +This is worth stating plainly because it has consequences: + +- A **custom `--rules` mapping does not control these triples.** They are emitted + by the wrapper regardless, unless the corresponding + `--sample-representation` / `--header-representation` / `--info-representation` + option turns them off. A mapping author who expects `--rules` to be the single + source of truth for the graph will be surprised. +- The host process **does** read `records.tsv` and write N-Triples, so the claim + that nothing on the host touches the data is not literally true. What is true + is that no *third-party* toolchain runs on the host. +- These emitters are streaming and append-only, guarded by + `_append_rdf_atomically`, so an interrupted append does not leave a + half-written aggregate. + +## 5. Data flow in full mode + +```text + input.vcf(.gz) + │ src/vcf_as_tsv.sh (awk, one pass) + ▼ + .records.tsv, .header_lines.tsv, .file_metadata.tsv + │ RMLStreamer + rules/default_rules.ttl + ▼ + Flink part files ──▶ merged aggregate .nt or .nt.gz + │ host-side direct emitters (§4) append in place + ▼ + complete aggregate + │ optional: --rdf-compression gzip,brotli + ├──────────────▶ .nt.gz / .nt.br + │ optional: --representations hdt,cottas (chunked) + ├──────────────▶ .hdt + .hdt.index.v1-1 + ├──────────────▶ .cottas + │ round-trip triple-count check on each base artifact + │ optional: --artifact-compression gzip,brotli + ├──────────────▶ .hdt.gz / .cottas.br / ... + │ optional: --validate + └──────────────▶ run_metrics/.../reports/validation/... +``` + +Multiple VCF inputs are processed **one at a time**, and a failure is isolated +to that input: the run continues and the failure is recorded in +`reports/failed_inputs.csv`. + +## 6. Failure policy + +The tool distinguishes three kinds of failure, and treats them differently: + +| Kind | Example | Behaviour | +| --- | --- | --- | +| **Fail closed** | An HDT decodes to the wrong triple count | The stage fails; no artifact is published | +| **Degrade and record** | An HDT is readable but its sidecar index could not be built | The run continues, the artifact is marked `index_status: "failed"`, and the raw RDF is retained so it can be repaired later; recorded in `reports/index_warnings.json` | +| **Isolate** | One VCF out of twelve fails conversion | That input is abandoned, the rest proceed, and it appears in `reports/failed_inputs.csv` | + +A COTTAS failure degrades differently from HDT: because the `.cottas` file +itself is unusable, every dependent COTTAS artifact is marked not generated +rather than published with a warning. + +Standalone `--mode index` is deliberately **strict** — it is a maintenance +operation whose whole purpose is to produce a valid index, so a partial result +is a failure, and the previous sidecar is restored. + +## 7. Isolation and cleanup guarantees + +These are properties the tool tries hard to hold, and states in its reports so +they can be checked rather than trusted: + +- Partitioned compression runs in an **ephemeral Docker-managed volume**. + Chunks, COTTAS scratch, and merge files never touch the output directory, and + the volume is removed on success *and* on failure. +- Validation mounts the source artifact **read-only** and decodes compressed + input under the container's `/work`. Nothing decoded is written beneath + `--out`; the stage report records that it was cleaned up. +- Each COTTAS conversion gets a fresh container-local DuckDB workspace, removed + when that operation completes. +- Output collisions are checked *before* Docker starts, and no planned artifact + is ever overwritten. `--mode index` is the single deliberate exception. + +## 8. Source map + +| Path | Role | +| --- | --- | +| [`vcf_rdfizer.py`](../vcf_rdfizer.py) | Host CLI: validation, planning, Docker orchestration, metrics, and the three direct emitters of §4 | +| [`vcf_rdfizer_rules.py`](../vcf_rdfizer_rules.py) | `vcf-rdfizer-rules`: scaffold, document, and check custom RML mappings | +| [`vcf_rdfizer_gzip.py`](../vcf_rdfizer_gzip.py) | Uncompressed size of a gzip/BGZF VCF without decompressing it | +| [`src/`](../src) | Container-side stages (table in §3) | +| [`rules/default_rules.ttl`](../rules/default_rules.ttl) | Default RML mapping, also shipped as package data in `vcf_rdfizer_data/` | +| [`test/`](../test) | `unittest` suite; pipeline tests stub `java`, `docker` and friends so no external tool is required | +| [`scripts/release.py`](../scripts/release.py) | Version bump and release metadata automation | + +## 9. Pinned toolchain + +Set as build arguments in the [`Dockerfile`](../Dockerfile): + +| Component | Version | Used for | +| --- | --- | --- | +| RMLStreamer | 2.5.0 | RML mapping execution on Flink | +| `hdtc` | 1.1.0 (Rust) | HDT create/merge/index, Java-free | +| Comunica | 5.3.0 | Default validation SPARQL engine | +| QLever | from `adfreiburg/qlever` (verified against build `bfd5741`) | Scale validation SPARQL engine | +| `pycottas` | image-pinned | COTTAS conversion and decode | + +QLever is copied from a differently based upstream image, so its +release-specific Boost/ICU/jemalloc/io_uring libraries are copied alongside into +`/opt/qlever/lib` and only QLever's own processes are pointed there. A +build-time `ldd` check records whether they resolve; Comunica remains the +default, so an image whose QLever binaries do not link still validates. + +--- + +## See also + +- [Conversion](conversion.md) — VCF to RDF, stage by stage +- [Representations](representations.md) — compression and queryable artifacts +- [Output and metrics](output-and-metrics.md) — what a run writes and where +- [Limitations](limitations.md) — what this architecture cannot do diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 0000000..f04ae93 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,162 @@ +# CLI reference + +Every flag `vcf-rdfizer` accepts, grouped by where it applies, with defaults and +the constraints that are enforced. `vcf-rdfizer --help` is the authority; this +page adds the *why* and the interactions. + +Two companion CLIs are installed alongside: `vcf-rdfizer-rules` (see +[`rml-mappings.md`](rml-mappings.md)) and, once implemented, +`vcf-rdfizer-link` (see [`datalinking-design.md`](datalinking-design.md)). + +--- + +## The one universal rule + +`-o, --out` is **required in every mode**. It is the run output root: final +artifacts, run metrics and logs, and hidden intermediates all live beneath it. + +## Modes + +| `-m, --mode` | Purpose | Required input | +| --- | --- | --- | +| `full` *(default)* | VCF → TSV → RDF → compression, optionally validated | `-i/--input` | +| `tsv` | VCF → TSV only, for benchmarking | `-i/--input` | +| `compress` | Compress an existing `.nt` / `.nt.gz` | `--rdf` | +| `decompress` | Decode a compressed or indexed artifact back to N-Triples | `-C/--compressed-input` | +| `validation` | Compare a source VCF against its RDF | `-i/--input` **and** `--rdf` | +| `index` | Regenerate an artifact's query index in place | exactly one of `-H/--hdt` or `--cottas` | + +## Inputs and outputs + +| Flag | Meaning | +| --- | --- | +| `-i, --input` | VCF file or directory. Only `*.vcf` and `*.vcf.gz` are recognised; a directory is enumerated one level deep and snapshotted at run start | +| `--rdf` | RDF input for `compress`, or the artifact to check in `validation` | +| `-C, --compressed-input` | `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, `.cottas.br` | +| `-H, --hdt` / `--cottas` | Existing artifact for `--mode index` | +| `-d, --decompress-out` | Explicit output `.nt` path; must be inside `--out` | +| `-o, --out` | **Required.** Run output root | +| `-n, --out-name` | Fallback basename when one cannot be inferred (default `rdf`) | + +## Graph shape + +| Flag | Values | Default | Effect | +| --- | --- | --- | --- | +| `-r, --rules` | path to `.ttl` | shipped `default_rules.ttl` | RML mapping; see the contract in [`rml-mappings.md`](rml-mappings.md) | +| `--sample-representation` | `expanded`, `condensed` | `expanded` | Genotype graph shape | +| `--info-representation` | `structured`, `raw` | `structured` | `structured` adds typed `InfoFieldValue` nodes alongside `infoRaw` | +| `--header-representation` | `structured`, `basic` | `structured` | `structured` types each `##` line and lifts its attributes | + +There is **no automatic sample-count threshold**: the same command always +produces the same graph shape, so a downstream consumer can rely on the contract +it selected. QUAL is always emitted regardless of these options. + +`condensed` rejects a custom mapping that consumes the materialized sample +helper tables, because that would emit both genotype representations at once. + +## Compression plan + +| Flag | Values | Default | +| --- | --- | --- | +| `--rdf-storage-mode` | `plain`, `space-optimized` | **required in full mode** | +| `--rdf-compression` | `gzip`, `brotli`, `none` | `gzip,brotli` | +| `--representations` | `hdt`, `cottas`, `none` | `hdt` | +| `--artifact-compression` | `gzip`, `brotli`, `none` | `none` | +| `--hdt-strategy` | `auto`, `partitioned`, `single` | `auto` | +| `--chunk-target-bytes` | bytes | 512 MiB | +| `--chunk-min-bytes` | bytes | 128 MiB | +| `--chunk-max-bytes` | bytes | 1 GiB | + +Constraints that are enforced rather than documented-and-hoped: + +- Each selector takes a comma-separated list; `none` must appear alone. +- `--artifact-compression` requires at least one selected representation. +- `--hdt-strategy single` cannot consume a `space-optimized` gzip stream. + +`-c, --compression` is a hidden legacy alias retained for backward +compatibility. Use the three explicit selectors. + +## Intermediates and cleanup + +| Flag | Effect | +| --- | --- | +| `-k, --keep-tsv` | Keep the hidden TSV intermediates | +| `-R, --keep-rmlstreamer-rdf-output` | Keep the RDF aggregate after compression | +| `--remove-rdf-storage-output` | Explicitly remove the aggregate after successful compression | +| `-e, --estimate-size` | Preflight size estimate; no conversion | + +## Validation + +| Flag | Values | Default | Meaning | +| --- | --- | --- | --- | +| `--validate` / `--run-validation` | — | off | Run validation once per input in full mode | +| `--validate-artifacts` | `aggregate`, `hdt`, `cottas`, `all` | `aggregate` | Which produced artifacts to check, each in its own report directory | +| `--validation-id` | name | source basename | Report directory name; existing directories are never overwritten | +| `--validation-engine` | `comunica`, `qlever` | `comunica` | SPARQL backend; a scale decision, never a semantic one | +| `--filter-oracle` | `auto`, `bcftools`, `cyvcf2` | `auto` | FILTER-field oracle | +| `--shacl-shapes` | path | off | Independent structural layer via `pyshacl`; in-memory, so not for cohort scale | +| `--strict-conformance` | — | off | Promote a missing-token conformance anomaly from report to failure | +| `--validation-query-timeout` | seconds | 3600 | Per-query timeout, both engines | +| `--qlever-memory-gb` | N | 4 | QLever index and server memory budget | +| `--qlever-port` | N | 7019 | Container-local only; never published | +| `--qlever-startup-timeout` | seconds | 900 | Wait for the server after indexing | +| `--qlever-index-arg` / `--qlever-server-arg` | string, repeatable | — | Extra arguments for the QLever binaries | + +A validation failure in full mode marks that input as failed, retains its raw +RDF for inspection, and makes the run exit non-zero — so a pipeline can be gated +on semantic correctness rather than on Docker having returned. + +`--mapping-policy` exists in `validation_runner.py` but is **not exposed by the +wrapper**; see the known gap in +[`rml-mappings.md`](rml-mappings.md#4-what-a-custom-mapping-costs-in-validation). + +## Docker + +| Flag | Default | Meaning | +| --- | --- | --- | +| `-I, --image` | `ecrum19/vcf-rdfizer` | Image repository | +| `-v, --image-version` | latest resolved | Image tag | +| `-b, --build` | off | Force a local build | +| `-B, --no-build` | off | Fail if the image is not present rather than building | + +## Output control + +| Flag | Effect | +| --- | --- | +| `--quiet` | Suppress terminal progress and the validator's per-query chatter; sidecar, command log and metrics are still written | +| `--no-progress` | Also disable progress-sidecar creation | +| `-P, --spark-partitions` | RMLStreamer parallelism hint; does not change the output | + +## Environment variables + +These are read by the container and, where noted, forwarded by the wrapper when +set on the host. + +| Variable | Default | Effect | +| --- | --- | --- | +| `HDT_MERGE_MEMORY_LIMIT` | `512M` | Soft memory budget for `hdtc create` | +| `HDT_INDEX_MEMORY_LIMIT` | `512M` | Soft memory budget for `hdtc index` | +| `COTTAS_MERGE_BATCH_ROWS` | `2048` | Rows held per input in the streaming COTTAS merge | +| `HDT_INDEX_WORK_ROOT` | `/work` | Scratch root when calling `ensure_hdt_index.sh` directly | +| `QLEVER_INDEX_COMMAND` | built-in | Full argv template with `{index}` `{input}` `{memory}` placeholders | +| `QLEVER_SERVER_COMMAND` | built-in | Full argv template with `{index}` `{port}` `{memory}` placeholders | + +Memory limits accept an `M` or `G` suffix. Lower values reduce in-memory sort +buffers and increase temporary I/O. Their scratch lives on the **Docker data +volume**, not the output filesystem. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Success (possibly with recorded index warnings) | +| `1` | One or more inputs failed, including a semantic validation failure | +| `130` | Interrupted with `Ctrl+C`; progress written and tracked intermediates cleaned up | + +--- + +## See also + +- [Representations](representations.md) — what the compression flags build +- [Validation](validation.md) — what the validation flags check +- [Output and metrics](output-and-metrics.md) — where the results land diff --git a/docs/conversion.md b/docs/conversion.md new file mode 100644 index 0000000..31d9d4c --- /dev/null +++ b/docs/conversion.md @@ -0,0 +1,237 @@ +# Conversion: VCF to RDF + +What the conversion actually does to a VCF, in order, and what it decides on +your behalf. The companion documents are +[`architecture.md`](architecture.md) (how the pieces fit), +[`rml-mappings.md`](rml-mappings.md) (how to change the mapping), and +[`vcf-coverage.md`](vcf-coverage.md) (which VCF element ends up where). + +--- + +## 1. Input acceptance + +`--input` takes a single file or a directory. + +- A file must be named `*.vcf` or `*.vcf.gz`. **Nothing else is accepted** — not + `.bcf`, not `.vcf.bgz`, not a bare `.gz`. Extension, not content, decides. +- A directory is enumerated **one level deep**, sorted, at the moment the run + starts. Files that appear later are not picked up. Subdirectories are ignored. +- Each input is processed independently and end to end before the next one + begins. A failure is isolated to its input. + +The output basename is the filename with `.vcf` / `.vcf.gz` removed. Two inputs +that reduce to the same basename would collide, and the pre-flight collision +check rejects the run before Docker starts. + +## 2. Stage one — VCF to TSV + +[`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) reads the VCF (through `gzip -dc` +when compressed) in **one `awk` pass** and writes three tab-separated tables. + +| Output | One row per | Columns | +| --- | --- | --- | +| `.file_metadata.tsv` | file (exactly one row) | `SOURCE_FILE`, `FILE_FORMAT`, `FILE_DATE`, `SOURCE_SOFTWARE`, `REFERENCE_GENOME`, `HEADER_COUNT`, `RECORD_COUNT` | +| `.header_lines.tsv` | `##` meta-information line | `SOURCE_FILE`, `HEADER_INDEX`, `HEADER_KEY`, `HEADER_VALUE`, `RAW_LINE` | +| `.records.tsv` | VCF data line | `SOURCE_FILE`, `ROW_ID`, `CHROM`, `POS`, `ID`, `REF`, `ALT`, `QUAL`, `FILTER`, `INFO`, `FORMAT`, *(sample payload)* | + +Two more paths exist for compatibility, `sample_calls.tsv` and +`sample_format_values.tsv`. Under the shipped mapping they are written +**header-only** — see §5. + +What this stage does, precisely: + +- `##` lines are split on the **first** `=`. Everything after it is + `HEADER_VALUE`; `RAW_LINE` keeps the original text minus the leading `##`. + No structural parsing happens here. +- Four keys are lifted into `file_metadata.tsv` by case-insensitive match: + `fileformat`, `filedate`, `source`, `reference`. +- `HEADER_INDEX` counts `##` lines *and* the `#CHROM` line, 1-based. +- `ROW_ID` is a 1-based counter over data lines, stable within one file. It is + the join key for every record-level IRI. +- Trailing carriage returns are stripped from every field. +- All sample columns are joined into **one** trailing field separated by single + spaces, and runs of whitespace are collapsed. The column's *name* is the + whitespace-joined sample IDs from `#CHROM`, or the literal `SAMPLES` when the + VCF declares none — which is why a mapping cannot reference it by a fixed name. + +**Limitations of this stage, stated plainly.** There is no `htslib` here. The +parser is regex-and-field-index `awk`, which means: + +- The `#CHROM` line is recognised only when it is **tab-delimited**. A + space-delimited header line is matched by no rule, so sample column names are + lost and the records header falls back to `SAMPLES` — silently. +- Nothing validates VCF spec conformance. A malformed file produces a malformed + graph rather than an error, and the first thing that notices is the validation + suite. +- A data line with fewer than 8 columns yields empty strings for the missing + fields rather than an error. +- Because sample fields are whitespace-normalized, a value containing a literal + space would be corrupted. The VCF specification forbids spaces in data fields, + so this is only reachable with an already-invalid file — but it is not detected. + +## 3. Stage two — TSV to RDF with RMLStreamer + +[`src/run_conversion.sh`](../src/run_conversion.sh) runs RMLStreamer 2.5.0 over +the TSVs using the mapping at `--rules` (default +[`rules/default_rules.ttl`](../rules/default_rules.ttl)), then normalizes the +Flink part files and merges them into one aggregate. + +The mapping's five `csvw:url` values are **literal container paths** +(`/data/tsv/records.tsv` and friends) that the wrapper rewrites per input to +`/data/tsv/.records.tsv`. That rewrite is why the contract in +[`rml-mappings.md`](rml-mappings.md) insists those strings stay verbatim. + +`--rdf-storage-mode` decides how the aggregate is assembled: + +| Mode | Behaviour | +| --- | --- | +| `plain` | Parts are merged into one uncompressed `.nt` | +| `space-optimized` | Each part is streamed through gzip into one `.nt.gz` and the source part is deleted immediately, so a full uncompressed copy never exists | + +The output is **N-Triples**. No named graphs are produced, and no blank nodes: +every class in the vocabulary declares an IRI template, and +`preflight_blank_nodes` treats a blank node as a validation failure precisely +because it means a term map produced no IRI. + +`--spark-partitions` is a parallelism hint passed through to RMLStreamer. It +does not change the output, only how many parts are produced before the merge. + +## 4. Stage three — the wrapper's own emitters + +Three classes of triple cannot come from RML, and are appended to the aggregate +by the host process before compression. See +[`architecture.md`](architecture.md#4-where-the-split-leaks-and-why) for why. + +### `emit_record_detail` — QUAL and structured INFO + +`QUAL` is **always** emitted. The published SHACL shape is +`sh:or([sh:datatype xsd:decimal] [sh:datatype vcfr:Null])`, and RML cannot pick +a datatype per row: + +| QUAL value | Emitted as | +| --- | --- | +| a number | `""^^xsd:decimal` — the source lexical form, so no precision is gained or lost | +| `.` or empty | `"."^^vcfr:Null` | +| anything else | a plain string literal, deliberately kept rather than dropped, and reported by the SHACL layer | + +`--info-representation structured` (the default) additionally emits one +`vcfr:InfoFieldValue` per record and key at +`…#call/{ROW_ID}/info/{KEY}`, linked to the `##INFO` declaration through +`vcfr:declaredBy`. A single-valued field (`Number=1`) whose declared `Type` is +`Integer` or `Float` also gets a typed `fieldValueInteger` / `fieldValueDecimal`; +a `Flag` gets `vcfr:fieldValueBoolean true`. A multi-valued field +(`Number=A/R/G/.`) keeps only the lexical value, because the vocabulary's IRI +template gives one node per key, not per value — a real modelling gap, recorded +in [`limitations.md`](limitations.md). + +`--info-representation raw` emits only the opaque `vcfr:infoRaw` string. + +### `append_header_representation_rdf` — structured headers + +`--header-representation structured` (the default) types each `##` line with its +vocabulary subclass (`INFOHeaderLine`, `ContigHeaderLine`, `FilterDefinition`, +`AltDefinition`, …) and lifts the `` +attributes into their own properties. The attribute parser respects quoting and +backslash escapes, so a `Description` containing a comma is handled correctly. + +An **unrecognised** `##` key keeps only the base `vcfr:HeaderLine` type. +Inventing a subclass would put a term in the graph that the vocabulary does not +define. + +`##fileDate` is emitted as `xsd:date` when its form is recognisable, and +verbatim as a plain literal otherwise — again preserved rather than dropped, and +reported by SHACL. + +`--header-representation basic` keeps only the base header-line triples. + +### `emit_sample_representation` — genotypes + +Exactly one sample emitter runs per conversion, chosen by +`--sample-representation`. Full treatment in +[`sample-representation-guide.md`](sample-representation-guide.md); the short +version: + +| Mode | Shape | Growth | +| --- | --- | --- | +| `expanded` (default) | one `vcfr:SampleCall` per record × sample, one `vcfr:FormatFieldValue` per FORMAT key | ≈ variants × samples × FORMAT fields | +| `condensed` | one reusable `vcfr:SampleSet`, one `vcfr:CohortCallMatrix` per call, one `vcfr:FormatValueVector` per FORMAT key holding a tab-separated `vcfr:VCFTextVector` | ≈ samples + variants × FORMAT fields | + +The file declares which shape it carries via `vcfr:representationProfile`, so a +consumer can branch on it before querying. + +## 5. The two helper tables + +`sample_calls.tsv` and `sample_format_values.tsv` exist so that a mapping *can* +express genotypes in RML if it wants to. Under the shipped mapping they stay +header-only and the wrapper streams genotypes directly, because materializing +them means one row per variant × sample (× FORMAT key) — the largest +intermediate the pipeline can produce. + +A custom mapping that consumes them in any other way forces full +materialization, and is **rejected** in `--sample-representation condensed`, +which would otherwise emit both genotype representations into one graph. + +## 6. IRI templates + +Every IRI is derived from the source filename and the row counter, so a +conversion is deterministic and re-running it produces byte-comparable subjects. + +| Resource | Template | +| --- | --- | +| `VCFFile` | `file://{SOURCE_FILE}` | +| `VCFHeader` | `file://{SOURCE_FILE}#header` | +| `HeaderLine` | `file://{SOURCE_FILE}#header/line/{HEADER_INDEX}` | +| `VCFRecord` | `file://{SOURCE_FILE}#record/{ROW_ID}` | +| `VariantCall` | `file://{SOURCE_FILE}#call/{ROW_ID}` | +| `InfoFieldValue` | `file://{SOURCE_FILE}#call/{ROW_ID}/info/{KEY}` | +| `SampleCall` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}` | +| `FormatFieldValue` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}/fmt/{KEY}` | +| `SampleSet` (condensed) | `file://{SOURCE_FILE}#samples` | +| `VCFSample` (condensed) | `file://{SOURCE_FILE}#samples/{SAMPLE}` | +| `CohortCallMatrix` (condensed) | `file://{SOURCE_FILE}#call/{ROW_ID}/matrix` | +| `FormatValueVector` (condensed) | `file://{SOURCE_FILE}#call/{ROW_ID}/matrix/fmt/{KEY}` | + +`{SOURCE_FILE}` is the **basename**, not a path, so a graph does not encode +where the VCF happened to live. The consequence is that two different VCFs with +the same filename mint the same IRIs; if you convert `chr1/data.vcf` and +`chr2/data.vcf`, their graphs will collide when merged. Rename before +converting, or keep the graphs separate. + +## 7. Missing values + +The vocabulary's `vcfr:missingValuePolicy` says a missing token should be +`"."^^vcfr:Null`, and the conversion follows it. `preflight_missing_token_conformance` +reports a plain `"."` literal as an anomaly, and `--strict-conformance` promotes +that from a report to a failure. + +This policy currently **conflicts with the published SHACL shapes** for +`ref`/`alt`/`chrom`, which constrain those to `sh:datatype xsd:string`. A record +with `ALT=.` therefore cannot satisfy both. The conflict is documented in +[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary) and +needs a decision in the vocabulary repository, not here. + +## 8. What conversion does *not* do + +- **No normalization.** No left-alignment, no trimming, no multi-allelic + splitting. `ALT=A,T` stays one record with one `alt` literal. This is + deliberate — the graph is a faithful transcription of the file — but it means + the graph is not directly joinable with normalized external resources. This is + the central problem the [data-linking design](datalinking-design.md) has to + solve. +- **No reference checking.** `REF` is not verified against any genome. +- **No structural-variant modelling.** Symbolic ALTs (``), breakends + (`N[chr2:321[`) and `*` are carried as literals. The validation suite + classifies them (`SYMBOLIC_OR_BREAKEND`) but the vocabulary gives them no + structure, and `INFO` keys such as `END` and `SVTYPE` get no special meaning. +- **No genotype interpretation.** A `GT` value is a lexical token. Phasing + (`|` versus `/`) is preserved in the literal but not modelled. +- **No cross-file merging.** Each VCF produces its own graph. + +--- + +## See also + +- [VCF coverage matrix](vcf-coverage.md) — element-by-element, with the validation check that covers each +- [Sample representations](sample-representation-guide.md) — the genotype shapes in depth +- [Custom RML mappings](rml-mappings.md) — changing what is emitted +- [Limitations](limitations.md) — consolidated diff --git a/docs/datalinking-design.md b/docs/datalinking-design.md new file mode 100644 index 0000000..4cc04db --- /dev/null +++ b/docs/datalinking-design.md @@ -0,0 +1,300 @@ +# Data linking: a plug-in architecture + +*Status: **design proposal**. Nothing described here is implemented yet. This +document exists to fix the extension contract before code is written, because +third-party linkers are the point and a contract is much harder to change once +people depend on it.* + +VCF-RDFizer converts a VCF into RDF that is faithful to the VCF and to nothing +else. Every IRI it mints is derived from the source file, so the graph is +self-contained, reproducible, and — deliberately — an island. It says +`vcfr:recordId "rs334"` and stops there. + +Data linking is the step that connects that island to the rest of the +linked-data web: the rsID becomes a dbSNP resource, the coordinate becomes an +Ensembl gene, the allele becomes a ClinVar assertion. The requirement is that +users other than the maintainers can add those connections for their own +domain without patching the tool. + +--- + +## 1. The observation the design rests on + +Linking a variant to an external resource is always a **join**, and in this +domain there are only three join keys: + +| Strategy | Key | Typical targets | +| --- | --- | --- | +| `token` | a value in `ID`, or the value of a named `INFO` key | dbSNP/Ensembl rsIDs, COSMIC identifiers, `GENEINFO` gene symbols | +| `interval` | `CHROM` + `POS` (+ `REF` length) overlap | genes, transcripts, exons, regulatory regions, cytobands, panel membership | +| `allele` | normalized `CHROM`-`POS`-`REF`-`ALT` | ClinVar assertions, gnomAD frequencies, CADD scores, variant-annotation APIs | + +Everything on the near-term wish list — rsID linking, medically relevant gene +linking, clinical significance, population frequency, drug-target association — +is one of those three. + +This matters because it determines where the seam goes. If the framework owns +the three join strategies, a plugin author never writes VCF parsing, never +mints an IRI, never escapes an N-Triples literal, and never thinks about +streaming. They declare *which key* and *what to emit on a match*. If the +framework instead offered a generic "here is a record, do what you like" hook, +every plugin would reimplement the same four things, each slightly wrong. + +--- + +## 2. Three tiers of plugin + +A linker is a directory. It is resolved both from a plugin search path and from +Python entry points (`vcf_rdfizer.linkers`), so a linker can be `pip install`ed +*or* dropped into a directory and mounted into the container. + +```text +linkers/rsid-dbsnp/ + linker.ttl # manifest: identity, join, source, emission, budget + resolver.py # OPTIONAL - only when resolution is not templatable + queries/ # OPTIONAL - .rq validation queries for the produced links + mutations.py # OPTIONAL - mutation catalogue entries for those links +``` + +### Tier 1 — declarative, no code + +The rsID case needs no Python and no network. It is a token join feeding an IRI +template: + +```turtle +@prefix vcfl: . + +<#rsIDdbSNP> a vcfl:Linker ; + vcfl:id "rsid-dbsnp" ; + vcfl:version "1.0.0" ; + vcfl:title "rsID to dbSNP" ; + vcfl:join [ a vcfl:TokenJoin ; + vcfl:field "ID" ; + vcfl:splitOn ";" ; + vcfl:accept "^rs[0-9]+$" ] ; + vcfl:emit [ vcfl:subject vcfl:VariantCall ; + vcfl:predicate vcfl:sameVariantAs ; + vcfl:objectTemplate "https://identifiers.org/dbsnp:{TOKEN}" ] . +``` + +Roughly two thirds of genuinely useful linkers are expressible at this tier. +Making Tier 1 real is the highest-leverage decision in the whole design: it +turns "write a plugin" from a Python project into a fifteen-line file, which is +the difference between an extension point that gets used and one that does not. + +### Tier 2 — local reference bundle + +Gene linking is an interval join against a reference the plugin *declares* +rather than ships: + +```turtle + vcfl:reference [ vcfl:url "https://ftp.ensembl.org/.../Homo_sapiens.GRCh38.113.gff3.gz" ; + vcfl:sha256 "e3b0c442..." ; + vcfl:assembly "GRCh38" ; + vcfl:format vcfl:GFF3 ] ; +``` + +The runner fetches once, verifies the digest, caches under +`~/.cache/vcf-rdfizer/linkers/`, and builds the interval index. Because the +assembly is declared, the runner can compare it against the VCF's `##reference` +and **refuse to run on a mismatch**. That strictness is deliberate: a +GRCh37-coordinate variant linked against GRCh38 gene intervals produces +confident, plausible, wrong annotations, and nothing downstream would catch it. + +### Tier 3 — live service + +A `resolver.py` implementing one narrow protocol: + +```python +def resolve(batch: Sequence[LinkKey], ctx: LinkerContext) -> Iterable[Link]: + """Map a batch of join keys to links. Called with deduplicated keys.""" +``` + +Batch in, links out. The plugin never opens a socket itself — it calls +`ctx.session.get(...)`. That indirection is what makes the safeguards in §4 +enforceable rather than advisory. + +--- + +## 3. Where linking happens in the pipeline + +Two entry points, sharing one implementation: + +**In-run.** `--link rsid-dbsnp,gene-ensembl` adds a linking stage to full mode. +It runs beside the existing direct emitters (`emit_record_detail`, +`emit_sample_representation` in `vcf_rdfizer.py`), reusing the +`_append_rdf_atomically(rdf_path, stats, producer)` contract so links stream +with the same atomicity, the same stats shape, and the same progress sidecar +events as everything else. + +**Post-hoc.** `--mode link --rdf .nt.gz --link ` enriches a graph +from a conversion that has already happened. This is what makes the feature +adoptable by anyone with existing outputs, and it is also how a linkset is +re-generated when a reference bundle is updated. + +--- + +## 4. Network safeguards + +Live APIs and local bundles are both supported, but every request a Tier 3 +plugin makes goes through `ctx.session`, so the runner — not the plugin — +enforces policy: + +- **Declared, enforced budget.** `vcfl:maxRequestsPerSecond`, + `vcfl:maxRequestsPerRun`, `vcfl:batchSize`, and `vcfl:contactEmail` are + manifest fields. Exceeding the per-run ceiling aborts that linker with a clear + error rather than degrading into an unattended crawl. +- **Per-host token bucket, shared across plugins.** Two linkers that both target + `rest.ensembl.org` share one rate budget; a service sees one client, not two. +- **Mandatory on-disk response cache**, keyed by + `(linker id, linker version, request hash)`. On a cohort VCF the hit rate is + the entire performance story. +- **Deduplication before dispatch.** The full run's key set is deduplicated + before a single request is issued. Five million variants routinely reduce to a + few hundred thousand distinct rsIDs. +- **Backoff that respects the service.** `Retry-After` honoured, exponential + backoff on 429/5xx, a hard concurrency cap, and a `User-Agent` carrying the + tool version and the declared contact address. +- **`--offline` and `--links-cache-only`.** Reproducing a published run must not + require the service to still be up, and must not silently pick up an answer + that has since changed. +- **Full accounting in the run manifest.** Requests issued, cache hits, bytes + transferred, wall time, and final service status, per linker — using the + existing `write_run_manifest` and `RunTracker` plumbing. + +**An honest scale caveat.** Tier 3 linking is a different order of operation +from the rest of the pipeline: the conversion is bounded by local I/O, a live +API is bounded by someone else's rate limit. Tier 3 is appropriate for filtered +or prioritized variant sets. Anything genome-wide should use a Tier 2 bundle, +and the documentation should say so rather than let users discover it after +eight hours. + +--- + +## 5. Output and provenance + +Links are written to a **side-graph**, `.links.nt`, next to the main +aggregate — not merged into it by default. Three reasons: + +1. The validation suite's predicate census, class census and identity digests + (`q09`–`q13`) are defined against what the VCF implies. Injecting external + triples into the aggregate would make every one of them report a false + positive. +2. Users can adopt links without changing the graph they already query. +3. A linkset can be regenerated, versioned, or discarded independently of an + expensive conversion. + +Each linkset carries its own provenance node: + +```turtle + + a vcfl:Linkset ; + vcfl:producedBy ; + vcfl:referenceDigest "sha256:e3b0c442..." ; + vcfl:assembly "GRCh38" ; + vcfl:linkCount 41233 ; + prov:generatedAtTime "2026-09-04T11:04:22Z"^^xsd:dateTime . +``` + +`--merge-links` folds the side-graph into the aggregate for consumers who want +one file, at which point the mapping-policy fallback already used for custom +`--rules` applies and `q09`–`q13` become report-only. + +**On predicate choice.** `owl:sameAs` between a VCF record and a dbSNP resource +is a much stronger claim than most users intend — it licenses inferring every +dbSNP property onto the VCF record and vice versa. The default vocabulary should +therefore be explicit and weak: `vcfl:sameVariantAs`, `vcfl:overlapsGene`, +`vcfl:hasAnnotation`. A manifest can override it, deliberately. + +--- + +## 6. Authoring tooling + +`vcf-rdfizer-link`, mirroring the existing `vcf-rdfizer-rules` CLI: + +| Command | Purpose | +| --- | --- | +| `vcf-rdfizer-link list` | Installed linkers, versions, tiers, and declared references | +| `vcf-rdfizer-link keys` | The join keys a manifest may reference — the analogue of `rules columns` | +| `vcf-rdfizer-link init -o my-linker/` | Scaffold a manifest with annotated examples | +| `vcf-rdfizer-link check my-linker/` | Validate the manifest, reference digest, assembly, and budget before a long run | +| `vcf-rdfizer-link dry-run my-linker/ -i sample.vcf` | Run over the first N records, write nothing, report the links it would emit and the requests it would issue | + +`dry-run` is the one that earns its keep: it is what stops an author from +discovering a two-million-request mistake at hour three of a run. + +--- + +## 7. Validation of linksets + +A linker that ships `queries/*.rq` and `mutations.py` is picked up by the same +discovery already used for `src/validation/queries/`, so its links are +mutation-scored exactly like the core graph — see +[`validation-methodology.md`](validation-methodology.md). + +Linkset-specific checks worth making standard: + +- every link subject resolves to a subject that exists in the base graph; +- every link object is a syntactically valid absolute IRI; +- no blank nodes and no empty terms (the existing preflights apply unchanged); +- the declared `vcfl:linkCount` matches the emitted triple count; +- the reference digest recorded in the linkset matches the bundle actually used. + +Holding third-party linkers to the same mutation-testing bar as the core +conversion is a genuinely defensible claim, and it is only available because +the validation harness was built to be extended. + +--- + +## 8. Build order + +| Step | Delivers | Unlocks | +| --- | --- | --- | +| 1 | `vcfl:` vocabulary, manifest parsing, `token` join, Tier 1 templates | rsID linking with no network and no plugin code | +| 2 | Discovery (path + entry points), `vcf-rdfizer-link` CLI, side-graph + provenance, manifest metrics | The third-party extension contract | +| 3 | `interval` join, reference fetch/cache/digest, assembly guard | Gene and region linking | +| 4 | `allele` join and allele normalization | ClinVar, gnomAD, CADD from bundles | +| 5 | `ctx.session` with the full safeguard stack | Tier 3 live services | + +Steps 1–2 are the contract; everything after is capability. The manifest +vocabulary should be frozen and versioned at the end of step 2. + +--- + +## 9. Known hard problems + +Stated up front rather than discovered later. + +**Allele normalization is the real correctness risk.** Left-alignment, +trimming, and multi-allelic splitting must match whatever the reference bundle +did, or the join silently under-matches. Under-matching looks like "this variant +has no ClinVar entry", which is indistinguishable from a true negative. Step 4 +should not ship without a normalization test set with known-answer cases. + +**"Medically relevant" is a curation claim, not a data fact.** A gene-relevance +linker should link to a *named, versioned panel* — and say which — rather than +asserting clinical relevance in VCF-RDFizer's own voice. The tool is not a +clinical authority and its output should not imply that it is. + +**Assembly mismatch is silent.** Hence the declared assembly and the hard +refusal in §2. There is no way to detect it after the fact from the links alone. + +**External identifiers drift.** rsIDs are merged and retired; ClinVar +significance is revised. A link is true as of the reference version recorded in +the linkset, which is exactly why the provenance node in §5 is mandatory rather +than optional. + +**Licensing is the plugin author's problem, and must be visible.** Some +reference resources are not redistributable and some APIs forbid bulk use. The +manifest should carry a `vcfl:license` and `vcfl:termsOfUse` that +`vcf-rdfizer-link list` prints, so a user can see what they are agreeing to +before a run rather than after a publication. + +--- + +## See also + +- [Architecture](architecture.md) — where a linking stage sits in the pipeline +- [Custom RML mappings](rml-mappings.md) — the existing extension point this design is modelled on +- [Validation methodology](validation-methodology.md) — the harness linksets would plug into +- [Roadmap](roadmap.md) — how this relates to the other planned work diff --git a/docs/limitations.md b/docs/limitations.md new file mode 100644 index 0000000..1d0e6a7 --- /dev/null +++ b/docs/limitations.md @@ -0,0 +1,201 @@ +# Limitations + +A single place that says what VCF-RDFizer cannot do, does badly, or does in a +way that will surprise you. Nothing here is hidden elsewhere in the +documentation — this page collects it so a prospective user can decide against +the tool without reading everything first. + +Each item says what it is, why it is that way, and whether it is fixable. + +--- + +## 1. Operational + +**Docker is mandatory.** There is no pure-Python path. The toolchain — Flink, +RMLStreamer, a Rust `hdtc`, `pycottas`, Comunica, QLever, `pyshacl`, `cyvcf2`, +`bcftools` — is pinned in one image, which is what makes results reproducible +across machines. The cost is a large image, a Docker daemon requirement, and no +usable story on a cluster that only offers Singularity/Apptainer. +*Fixable in principle; not planned.* + +**The Docker data volume, not `--out`, is the binding disk constraint.** +Partitioned merges and HDT indexing perform disk-backed external sorts under the +container's `/work`. Free space in the output filesystem does not help, and the +resulting failure surfaces as `exit_code=-9` / `137` — an OOM kill — rather than +as a disk-space message. This is the single most common cause of a failed +cohort-scale run. + +**Single machine, single process per input.** Inputs are processed one at a +time. `--spark-partitions` tunes RMLStreamer's internal parallelism but there is +no distributed execution and no work queue. + +**No incremental update.** Adding variants to a converted dataset means +reconverting the VCF and rebuilding every representation from scratch. + +**Interrupt cleanup is best-effort.** `Ctrl+C` exits 130 and removes tracked +intermediates, but a `SIGKILL` or a host crash can leave a partial output +directory that the collision check will then refuse to write into. + +## 2. Input handling + +**Only `*.vcf` and `*.vcf.gz`.** Extension decides, not content. `.bcf` is not +supported; neither is `.vcf.bgz` or a differently named gzip stream. A +directory input is enumerated one level deep, sorted, once, at run start. + +**The VCF parser is `awk`, not `htslib`.** [`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) +makes one pass and splits on tabs. Consequences: + +- The `#CHROM` line is only recognised when tab-delimited. A space-delimited + header line matches no rule, so sample column names are lost and the records + header silently falls back to `SAMPLES`. +- Nothing validates VCF spec conformance. A malformed file yields a malformed + graph rather than an error; the validation suite is the first thing that + notices. +- A data line with fewer than eight columns produces empty fields, not an error. +- Sample fields are whitespace-normalized, so a value containing a literal space + would be corrupted. The specification forbids that, so it is only reachable + with an already-invalid file — but it is undetected. + +**IRIs are minted from the filename, not the path.** Two different VCFs named +`data.vcf` produce identical subject IRIs and their graphs collide when merged. +Rename before converting, or keep the graphs apart. + +## 3. What the RDF does and does not model + +**No variant normalization.** No left-alignment, no trimming, no multi-allelic +splitting. `ALT=A,T` stays one record with one `alt` literal. This is deliberate +— the graph is a faithful transcription — but it means the graph is not directly +joinable with normalized external resources, which is the central problem the +[data-linking design](datalinking-design.md) has to solve. + +**No reference checking.** `REF` is never verified against a genome, and the +declared assembly is recorded but not used for anything. + +**Structural variants are literals.** Symbolic ALTs (``), breakends and `*` +are carried as strings. The validation suite classifies them +(`SYMBOLIC_OR_BREAKEND`) but the vocabulary gives them no structure, and `END` / +`SVTYPE` carry no special meaning. + +**Multi-valued INFO fields keep only their lexical value.** A field declared +`Number=A/R/G/.` gets no typed per-value nodes, because the vocabulary's IRI +template gives one node per key rather than per value. This is a modelling gap +in the vocabulary, not a bug in the conversion. + +**Genotypes are lexical.** Phasing is preserved inside the literal but not +modelled; ploidy is not interpreted. + +**Triples only.** No named graphs anywhere in the pipeline, and no blank nodes — +a blank node is treated as a validation failure, because every class in the +vocabulary declares an IRI template. + +**No cross-file merging.** Each VCF produces its own graph. + +## 4. The custom-mapping extension point + +**`--rules` does not control the whole graph.** Three families of triple — +genotypes, structured headers, and QUAL/structured INFO — are emitted by the +wrapper rather than by RML, because RML cannot choose a datatype or class per +row and the alternatives require materializing enormous helper tables. See +[`architecture.md`](architecture.md#4-where-the-split-leaks-and-why). They can be +narrowed with `--sample-representation` / `--header-representation` / +`--info-representation`, but not replaced by a mapping. + +**A custom mapping is validated less thoroughly.** Queries `q09`–`q13` assume +the shipped mapping's predicate inventory and IRI templates. + +**And, currently, it is validated incorrectly.** `validation_runner.py` supports +`--mapping-policy report-only` for exactly this case, but the wrapper never +forwards it, so a custom mapping run through `vcf-rdfizer --validate` reports +`MISMATCH` on those five queries even when the conversion is correct. +*Fixable; tracked in [`roadmap.md`](roadmap.md).* + +**`vcf-rdfizer-rules check` is lexical, not semantic.** It catches wrong logical +-source paths and misspelled columns — the two mistakes that waste the most time +— and nothing subtler. A mapping that passes `check` can still be wrong. + +## 5. Compression and representations + +**Packaged artifacts are not queryable.** `.hdt.gz`, `.hdt.br`, `.cottas.gz` and +`.cottas.br` are archives. This is easy to forget when +`--remove-rdf-storage-output` has already removed the alternative. + +**COTTAS is the more fragile path.** Its upstream `cat` cannot handle large +condensed graphs, which is why VCF-RDFizer implements its own bounded k-way +merge. Even so, a memory-constrained host may need a reduced +`COTTAS_MERGE_BATCH_ROWS`, and `--representations hdt` remains the independent +fallback. + +**The round-trip check counts, it does not compare.** Matching triple counts +prove an artifact decodes and holds the right *number* of statements — not that +they are the right statements. The stronger claim requires +`--validate-artifacts hdt,cottas`. + +**A degraded HDT index is a success, not a failure.** In full mode an HDT whose +data is readable but whose sidecar could not be built is published with +`index_status: "failed"` and a warning. That is intentional, but it means a +successful run can leave a non-indexed artifact. + +## 6. Validation + +The validation suite has its own detailed limits in +[`validation.md`](validation.md#what-is-not-tested) and +[`vcf-coverage.md`](vcf-coverage.md#remaining-gaps). The headline items: + +**A `PASS` is a regression gate, not a correctness proof.** It reliably catches +dropped records, misclassified variants, flipped genotypes, corrupted FILTER +strings, allele-count errors, missing header lines and altered file metadata. It +is not proof of a faithful record-by-record round-trip. + +**Coverage is relative to the mutation catalogue.** The score says "almost every +corruption we thought to write down is caught". It is a lower bound on +blindness, not a measure of correctness, and a score that rises without the +catalogue growing means nothing. + +**`vcfr:contigCount` is counted, not read**, so a wrong derived contig total is +undetected. **Header line values are not compared** — only how many lines carry +each key, their types, and the structured attributes lifted out of them. + +**SHACL is opt-in and does not scale.** `pyshacl` loads the whole graph into +memory, so it is for a single-sample graph or a sample of a cohort. + +**QLever's argv is a moving target.** Its CLI has changed across releases; the +`QLEVER_*_COMMAND` environment overrides exist because of that, and the exact +argv is recorded in every report so a future divergence is diagnosable. + +## 7. Vocabulary + +**Condensed graphs are not ontology-backed.** The tool emits 17 terms that +`https://w3id.org/vcf-rdfizer/vocab#` does not define, all belonging to the +condensed representation. Dereferencing any of them returns nothing. This is the +one remaining publication blocker, and the work is in the vocabulary repository. + +**The published SHACL shapes contradict the missing-value policy.** +`vcfr:missingValuePolicy` says a missing token should be `"."^^vcfr:Null`, while +`VCFRecordShape` constrains `vcfr:alt` to `sh:datatype xsd:string`. A record with +`ALT=.` cannot satisfy both. The conversion follows the missing-value policy. +The vocabulary needs a decision; see +[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). + +**Condensed mode has no query-time decoder.** Reconstructing sample *i*'s value +means splitting a tab-separated literal, which SPARQL cannot do portably. The +options are assessed in +[`sample-representation-guide.md`](sample-representation-guide.md#7-assessment-of-geosparql-and-graphdb-sparql-extensions); +none is currently implemented. + +## 8. Scope + +**No data linking yet.** The graph is self-contained and connects to nothing +external. This is by design so far, and the plan to change it is +[`datalinking-design.md`](datalinking-design.md). + +**No clinical claims.** The tool transcribes a VCF. It does not interpret, +annotate, prioritize, or assess pathogenicity, and its output should not be +presented as if it did. + +--- + +## See also + +- [Roadmap](roadmap.md) — which of these are being addressed +- [VCF coverage matrix](vcf-coverage.md) — the element-by-element measurement +- [Validation](validation.md) — the detailed "what is not tested" diff --git a/docs/output-and-metrics.md b/docs/output-and-metrics.md new file mode 100644 index 0000000..0fcccb4 --- /dev/null +++ b/docs/output-and-metrics.md @@ -0,0 +1,187 @@ +# Output layout, metrics, and run reports + +Every VCF-RDFizer run writes three kinds of thing: the artifacts you asked for, +a complete record of how they were produced, and evidence about what was +cleaned up. This document is the map. + +--- + +## 1. Output layout + +`--out` is required for every mode. It is the run output **root**, and +everything lives beneath it: + +```text +/ + / final artifacts + .nt | .nt.gz | .nt.br + .hdt + .hdt.index.v1-1 + .cottas + .hdt.gz | .cottas.br | ... + run_metrics/__/ per-run reports and logs + .intermediate/tsv/ hidden intermediates + decompressed/ --mode decompress default target +``` + +The directory name and every artifact basename come from the source filename +with its recognized VCF/RDF/representation suffix removed. `--mode compress` on +`test-larger.nt` and on `test-larger.nt.gz` therefore both write into +`/test-larger/`. + +**Collision policy.** Before Docker starts, the wrapper computes every artifact +the run intends to write and fails if any already exists. It never overwrites a +planned pipeline artifact. Choose a new `--out`, or move the conflicting file. +`--mode index` is the single deliberate exception, because regenerating an index +in place is its entire purpose. + +**Intermediate cleanup.** TSV intermediates are hidden and removed unless +`--keep-tsv`. Raw RDF is removed after successful compression by default; +`--remove-rdf-storage-output` makes that explicit and +`--keep-rmlstreamer-rdf-output` prevents it. Under `space-optimized`, the +`.nt.gz` aggregate is retained when `gzip` is a selected RDF codec, because that +file *is* the gzip artifact. + +When `--validate` fails for an input, raw RDF cleanup is **skipped** for that +input and the aggregate is kept with the note `retained after validation +failure`, so the graph that failed is still available to inspect. + +## 2. The run metrics directory + +```text +run_metrics/__/ +``` + +`` is the source filename without its suffix (for example +`1000G_phase3_chr20`); a multi-file directory input uses a batch label such as +`batch-vcf_data-4-inputs`. The point is that a metrics directory is +recognisable without opening a timestamp-named folder. + +| Path | Contents | +| --- | --- | +| `run.json` | Source identity, resolved input paths, requested configuration, image selection | +| `summary.json` | Final status, wrapper wall time, summary rows, and an index of every stage report and log | +| `metrics.csv` | Analysis-ready per-output table | +| `tsv_metrics.csv` | TSV-mode benchmark table | +| `wrapper_execution_times.csv` | Host-side stage timings | +| `logs/wrapper.log`, `logs/progress.log` | Command log and progress history | +| `timings//…` | Raw GNU `time -v` output from inside the relevant container | +| `stages/tsv/`, `stages/conversion/`, `stages/compression/`, `stages/compression_operations/`, `stages/decompression/`, `stages/index/`, `stages/validation/` | Structured per-stage results | +| `stages/partitioned/.json` | Full handoff from the partitioned-compression container | +| `reports/index_warnings.json` | Degraded-index events | +| `reports/failed_inputs.csv` | Inputs abandoned during a multi-file run | +| `reports/validation//` | Detailed semantic-validation reports | + +`compression_operations/` preserves the per-RDF operation and round-trip +validation reports; `compression/` is the final output-level summary over them. + +`stages/partitioned/.json` is the one to open when a large run fails. +It retains every chunk build, the merge strategy and round count, validation +results, the generated chunk plan, workspace free-space samples, CPU time, peak +RSS, exit codes, and bounded `stderr` diagnostics — **and it survives the +deletion of the temporary Docker volume**, which is what makes a failed +cohort-scale run diagnosable at all. + +## 3. Input size accounting + +`input_vcf_size_bytes` in `stages/conversion/*.json` and `metrics.csv` is the +**uncompressed** size of the source VCF, so ratios are comparable between plain +and compressed inputs. `input_vcf_size_method` records how it was obtained: + +| Method | Meaning | +| --- | --- | +| `stat` | Input was not compressed; the on-disk size is the answer | +| `bgzf` | Exact, summed from BGZF block headers with no decompression | +| `gzip-sample` | Exact; the whole single-member stream fitted in the sampling budget | +| `gzip-trailer` | Exact; the 32-bit `ISIZE` trailer resolved against the measured compression ratio | +| `inflate` / `inflate-shell` | Fallback full decompression pass | + +Only the fallback costs a full pass. Indexed `.vcf.gz` files from +`bcftools`/`tabix`/`htslib` are BGZF, so the usual case is measured in +milliseconds. The same machinery makes `--estimate-size` report a real +uncompressed size rather than an assumed expansion factor — and it says so when +it had to fall back to the assumption. + +## 4. Compression metrics + +Per method, `metrics.csv` carries `wall_seconds_*`, `user_seconds_*`, +`sys_seconds_*`, and `max_rss_kb_*`, plus `source_triples` / `decoded_triples` +from the round-trip check described in +[`representations.md`](representations.md#6-round-trip-verification). + +Metrics may use internal stage names such as `hdt_gzip` and `cottas_brotli`. +These correspond to the public combination of `--representations` and +`--artifact-compression`; nothing on the command line uses those compound names. + +For partitioned runs, the method-level metric reports one sample-level result +while `stages/partitioned/` retains the full history. + +## 5. Validation reports + +Detailed results live at: + +```text +/run_metrics/__/reports/validation// +``` + +with a stage-level summary at `stages/validation/.json`. When +several artifacts are validated in one run, the aggregate keeps +`/` and the representations use `__hdt/` and +`__cottas/`. + +Key files: `summary.json`, `manifest.json`, `parser.json`, +`rdf-validation.json`, `materialization.json`, `preflight.json`, `sparql.json`, +`comparison.json`. Raw SPARQL Results JSON, stderr and query resource logs are +in `raw/`; normalized results in `normalized/`. + +`manifest.json` records the engine that ran (including QLever's exact argv), the +source artifact's format and checksum, and every decode step. +`materialization.json` records how a compressed artifact was turned into +N-Triples and how many triples that yielded. Both the stage report and +`summary.json` record whether a gzip aggregate was decompressed inside the +container and that no validation scratch RDF was retained on the host. + +Statuses: `PASS`, `MISMATCH` (both paths ran and differ), +`BLOCKED_BY_PREFLIGHT` (syntax or core graph structure failed), +`EXECUTION_FAILED` (a parser or engine could not complete). + +Full treatment in [`validation.md`](validation.md). + +## 6. Progress + +Containers write newline-delimited JSON to a small sidecar under the run's +temporary `.progress/` area; the host polls it while the container runs and +renders it with Rich when attached to an interactive terminal, or as compact +status lines otherwise. Command logs and binary subprocess output stay separate +from the terminal UI. + +| Flag | Effect | +| --- | --- | +| *(none)* | Sidecar written, terminal progress rendered | +| `--quiet` | Sidecar, command log and metrics retained; **all** terminal rendering and the validator's per-query chatter suppressed | +| `--no-progress` | Sidecar creation disabled as well | + +Progress is best-effort: RMLStreamer reports bytes and output parts already +written, partitioned runs report source triples, chunks, and the active +merge/index stage. Nothing rescans RDF content a second time to produce a +number, and no progress history is retained in memory. + +## 7. Interrupts and exit codes + +`Ctrl+C` exits with **130**, writes progress to `logs/progress.log`, and +performs best-effort cleanup of tracked intermediates. Raw RDF cleanup on +interrupt follows `--keep-rmlstreamer-rdf-output`: with it, raw RDF is +preserved; without it, tracked raw RDF files are removed. + +Otherwise the wrapper exits `0` on success and non-zero on failure. In full mode +with `--validate`, a validation failure sets a non-zero exit even when the +conversion itself succeeded, so a run can be gated in CI on semantic +correctness rather than on Docker having returned. + +--- + +## See also + +- [Architecture](architecture.md) — which process writes which report +- [Representations](representations.md) — what the compression metrics describe +- [Validation](validation.md) — reading a validation report +- [CLI reference](cli-reference.md) — the flags referenced above diff --git a/docs/representations.md b/docs/representations.md new file mode 100644 index 0000000..9938280 --- /dev/null +++ b/docs/representations.md @@ -0,0 +1,187 @@ +# Compression and queryable representations + +What VCF-RDFizer can produce from an RDF aggregate, how each artifact is built, +what is verified about it, and where each one runs out of road. + +--- + +## 1. Three independent decisions + +Compression is deliberately not one option. It is three: + +| Decision | Flag | Values | Default | +| --- | --- | --- | --- | +| How the aggregate is staged | `--rdf-storage-mode` | `plain`, `space-optimized` | *(required in full mode)* | +| Which raw RDF artifacts to keep | `--rdf-compression` | `gzip`, `brotli`, `none` | `gzip,brotli` | +| Which queryable representations to build | `--representations` | `hdt`, `cottas`, `none` | `hdt` | +| How to package those representations | `--artifact-compression` | `gzip`, `brotli`, `none` | `none` | + +Each selector takes a comma-separated list. `none` must appear alone. +`--artifact-compression` requires at least one selected representation. + +The gzip used by `--rdf-storage-mode space-optimized` is **staging**, not +automatically a final artifact. It is retained as the gzip artifact only when +`gzip` is also selected in `--rdf-compression`. + +For the smallest output: `--rdf-compression none`, one representation, and +`--remove-rdf-storage-output`. + +## 2. The artifact matrix + +| Artifact | Queryable | Built by | Verified by | +| --- | --- | --- | --- | +| `.nt` | with any RDF store | the merge step | Raptor syntax check during validation | +| `.nt.gz`, `.nt.br` | after decompression | `gzip` / `brotli` | — | +| `.hdt` + `.hdt.index.v1-1` | yes, directly | `hdtc create` (+ `hdtc index`) | streaming decode + triple-count equality | +| `.cottas` | yes, directly | `pycottas` + PyArrow merge | streaming decode + triple-count equality | +| `.hdt.gz`, `.hdt.br`, `.cottas.gz`, `.cottas.br` | **no** | packaging | — | + +The packaged forms are archives. Keep the unwrapped `.hdt` / `.cottas` if +queries must run without a decompression step. + +## 3. Record-safe chunking + +When HDT or COTTAS is selected, the aggregate is read sequentially and split +into chunks on **complete N-Triples line boundaries**. Only one uncompressed +chunk exists at a time: it is consumed by both converters and removed before the +next is read. That property is what makes a `space-optimized` `.nt.gz` aggregate +usable without ever expanding a second full raw copy. + +| Flag | Meaning | Default | +| --- | --- | --- | +| `--chunk-target-bytes` | target uncompressed bytes per chunk | 512 MiB | +| `--chunk-min-bytes` | minimum before a chunk group is flushed | 128 MiB | +| `--chunk-max-bytes` | hard ceiling; boundaries stay on complete lines | 1 GiB | + +`--hdt-strategy` chooses the policy: `auto` (build chunks and merge with native +`hdtc`), `partitioned` (always chunk), `single` (one `rdf2hdt` run). `single` +cannot consume a gzip stream without expanding it, so it is incompatible with +`space-optimized`. + +The whole partitioned stage runs in an **ephemeral Docker-managed volume**. +Chunks, scratch, and merge files never reach the output directory, and the +volume is removed on success and on failure alike. + +## 4. HDT + +HDT generation, merging, and indexing use the pinned Rust **`hdtc` 1.1.0**, not +`hdtCat`, `hdtSearch.sh`, or any Java HDT process. This is not a stylistic +preference: the JVM path fails with `java.lang.OutOfMemoryError` on +cohort-scale graphs, while `hdtc create` merges chunk HDTs through disk-backed +external sorts and `hdtc index` streams BitmapTriples to build the +object/predicate orderings. The sidecar it produces is the canonical HDT v1-1 +file, `.hdt.index.v1-1`, readable by hdt-java and hdt-cpp. + +Memory is bounded by two environment variables, both defaulting to `512M` in +the image and forwarded by the wrapper when set on the host: + +```bash +HDT_MERGE_MEMORY_LIMIT=2G HDT_INDEX_MEMORY_LIMIT=2G vcf-rdfizer ... +``` + +Lower values reduce in-memory sort buffers and increase temporary I/O. Both +stages need scratch space in the container's `/work`, which lives on the +**Docker data volume** — free space in the output filesystem does not help. + +An HDT whose data is readable but whose sidecar could not be built is a +*degraded success* in full mode: it is validated with the index check skipped, +marked `index_status: "failed"`, reported in `reports/index_warnings.json`, and +the raw RDF is retained so it can be repaired later with `--mode index`. + +## 5. COTTAS + +COTTAS is a Parquet-based representation with its index **inside** the artifact; +there is no sidecar. Chunk conversion uses +`pycottas.rdf2cottas(..., disk=True)` with a fresh container-local DuckDB +workspace per operation. + +The final merge deliberately does **not** call `pycottas.cat`. In version 1.1.0 +that runs a global `DISTINCT` plus `ORDER BY` through an unbounded in-memory +DuckDB connection, which gets OOM-killed on large condensed graphs. VCF-RDFizer +instead performs a **k-way PyArrow merge** of the already `spo`-sorted Parquet +chunks: it holds at most `COTTAS_MERGE_BATCH_ROWS` (default 2048) rows per +input, drops adjacent duplicate triples, and writes the result incrementally. +There is no graph-wide hash table and no external-sort spill directory; memory +is a function of the batch size and the chunk count, not of the graph size. + +A COTTAS failure degrades differently from HDT: because the `.cottas` file +itself is unusable, dependent COTTAS artifacts are marked *not generated* rather +than published with a warning. + +## 6. Round-trip verification + +Every HDT and COTTAS base artifact is verified **before** packaging and before +raw RDF cleanup: [`src/validate_compression.py`](../src/validate_compression.py) +reads the source triple count, streams the artifact back through its native +decoder, and requires equality. Compression **fails closed** if the artifact +cannot be decoded or the counts differ. Results and the +`source_triples`/`decoded_triples` pair land in the per-run compression JSON and +in the HDT/COTTAS columns of `metrics.csv`. + +Be clear about what this proves and what it does not. A matching triple count +proves the artifact decodes and contains the right *number* of statements. It +does not prove the statements are the right ones. The stronger claim comes from +running the semantic suite against the decoded artifact — `--validate-artifacts +hdt,cottas` — which re-derives every VCF summary from it. See +[`validation.md`](validation.md#which-artifact-is-validated). + +## 7. Index maintenance + +`--mode index` regenerates an existing artifact's index in place. It is the one +deliberate exception to "never overwrite a planned artifact". + +| Input | Behaviour | +| --- | --- | +| `--hdt file.hdt` | Existing versioned sidecars are moved aside, regenerated, and restored if indexing fails; incomplete replacements are removed first | +| `--cottas file.cottas` | The artifact is rewritten atomically through the same bounded streaming Parquet rewrite with the default `spo` index; the original stays in place if it fails | + +No conversion, packaging, or decompression output is produced. Standalone index +mode is **strict** — unlike the in-run degradation above, a failure is a failure. +The same operation runs automatically after each partitioned HDT merge. + +## 8. Decompression + +`--mode decompress` decodes `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz` +and `.cottas.br` back to N-Triples. A packaged COTTAS is unwrapped **inside the +container** before `pycottas` writes the decoded output, so the intermediate +unwrapped file never appears on the host. + +## 9. Choosing + +| Situation | Selection | +| --- | --- | +| Load into an existing triple store | `--representations none --rdf-compression gzip` | +| Queryable, single artifact, smallest footprint | `--representations hdt --rdf-compression none --remove-rdf-storage-output` | +| Comparing HDT against COTTAS | `--representations hdt,cottas` | +| Archival transfer | `--artifact-compression brotli` on top of the chosen representation | +| Memory-constrained host | `space-optimized` + `partitioned` + lower `--chunk-*-bytes` | + +## 10. Limitations + +- **Docker volume space, not output space, is the binding constraint** for + partitioned merges and HDT indexing. This is the single most common cause of + a failed large run, and the error surfaces as an OOM/SIGKILL rather than as a + disk message. +- **`exit_code=-9` (or `137`) means the kernel or Docker OOM-killer intervened**, + not that the RDF is invalid. Check `stderr_tail`, `max_rss_kb`, and the + workspace free-space samples in `stages/partitioned/.json` before + suspecting the data. +- **COTTAS is the more fragile path.** If it cannot fit the available memory + even at a reduced `COTTAS_MERGE_BATCH_ROWS`, `--representations hdt` is + independent and remains queryable. +- **Packaged representations are not queryable**, which is easy to forget when + `--artifact-compression` is set and `--remove-rdf-storage-output` has removed + the alternative. +- **Neither HDT nor COTTAS carries named graphs**, matching the conversion's + triples-only output. +- **No incremental update.** Adding variants means reconverting and rebuilding + the representation from scratch. + +--- + +## See also + +- [Architecture](architecture.md) — where these stages run +- [Output and metrics](output-and-metrics.md) — what each stage records +- [Validation](validation.md) — proving a representation still means what the VCF meant +- [CLI reference](cli-reference.md) — every flag, with constraints diff --git a/docs/rml-mappings.md b/docs/rml-mappings.md new file mode 100644 index 0000000..5f43668 --- /dev/null +++ b/docs/rml-mappings.md @@ -0,0 +1,170 @@ +# Custom RML mappings + +`--rules` accepts any RML mapping, so you can change what RDF the pipeline +produces without touching the wrapper. This document is the full contract, the +tooling that checks it, and an honest account of what a custom mapping costs you. + +--- + +## 1. The tooling first + +`vcf-rdfizer-rules` is installed alongside `vcf-rdfizer` and exists so that the +contract is discoverable and checkable, rather than something you find out three +hours into a run. + +```bash +vcf-rdfizer-rules columns # what a mapping can reference +vcf-rdfizer-rules init -o my.ttl # annotated copy of the shipped default +vcf-rdfizer-rules check my.ttl # validate before spending a run on it +``` + +`check` reports: + +- logical-source paths the wrapper cannot rewrite per input; +- referenced columns no generated TSV provides (typos such as `CHROMOSOME`); +- which `--sample-representation` values remain usable; +- whether the mapping forces the large sample helper tables to be materialized. + +Exit code is `0` when the mapping is usable and `1` when it is not; `--json` +gives the same report machine-readably. + +The checks are deliberately **lexical**, not a full Turtle parse. The tool stays +dependency-free, and the mistakes worth catching before a long run are wrong +paths and misspelled column names — not subtle RML semantics. A mapping that +passes `check` can still be wrong; it just will not be wrong in the two ways +that waste the most time. + +## 2. The contract + +### Rule 1 — keep the five `csvw:url` values verbatim + +Full mode processes one VCF at a time and rewrites those exact literal strings +to the per-input file names. Any other path is left untouched and will not +resolve. + +| Logical source | Contents | +| --- | --- | +| `/data/tsv/records.tsv` | One row per VCF data line | +| `/data/tsv/header_lines.tsv` | One row per `##` header line | +| `/data/tsv/file_metadata.tsv` | One row summarising the source VCF | +| `/data/tsv/sample_calls.tsv` | Helper: one row per variant × sample | +| `/data/tsv/sample_format_values.tsv` | Helper: one row per variant × sample × FORMAT key | + +### Rule 2 — only reference columns the pipeline writes + +`vcf-rdfizer-rules columns` is authoritative. A unit test runs +[`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) and asserts those lists still match +what it emits, so the documentation cannot drift from the implementation. + +The last column of `records.tsv` is the whitespace-joined sample IDs from the +`#CHROM` line (or `SAMPLES` when the VCF declares none). Its *name* varies per +input, so it **cannot be referenced by a fixed name** — which is one of the +reasons genotype RDF is emitted by the wrapper instead. + +### Rule 3 — think before consuming the helper tables + +The four built-in sample maps are recognised by the wrapper, which then keeps +`sample_calls.tsv` and `sample_format_values.tsv` header-only and streams +genotype RDF itself. A mapping that consumes them in any other way forces both +to be materialized in full — one row per variant × sample, and one per +variant × sample × FORMAT key. On a cohort VCF that is the largest intermediate +the pipeline can produce. + +Such a mapping is **rejected** in `--sample-representation condensed`, which +would otherwise emit the expanded and condensed genotype graphs simultaneously +and restore exactly the inflation condensed mode exists to avoid. Remove the +helper-table consumers, or use expanded mode. A custom mapping with no +helper-table consumers works in both modes. + +## 3. What a custom mapping does *not* control + +This is the part most easily missed. Three families of triple are emitted by the +wrapper, not by RML, and they appear in your graph regardless of your mapping: + +| Triples | Turned off with | +| --- | --- | +| Genotypes (`SampleCall`/`FormatFieldValue`, or the condensed equivalents) | nothing — one emitter always runs; choose which with `--sample-representation` | +| `QUAL`, and structured `InfoFieldValue` nodes | `--info-representation raw` still emits QUAL | +| Header-line subclasses and lifted FILTER/ALT/contig/INFO/FORMAT attributes | `--header-representation basic` | + +The reasons are in [`architecture.md`](architecture.md#4-where-the-split-leaks-and-why): +RML cannot choose a datatype or a class per row, and the alternatives would +require materializing helper tables. It is a real constraint on the extension +point, not an oversight, but a mapping author who assumes `--rules` is the sole +source of truth for the graph will be surprised. + +## 4. What a custom mapping costs in validation + +The semantic validation suite has three layers. A custom mapping affects one of +them. + +Queries `q09`–`q13` — the predicate census, the class census, and the three +identity digests — assume the shipped mapping's predicate inventory and IRI +templates. A custom mapping changes both **by design**. + +`validation_runner.py` has a `--mapping-policy report-only` setting for exactly +this case: it records those five queries without failing on them, leaving the +run to rest on the aggregate comparisons (`q01`–`q08`). + +> **Known gap.** The `vcf-rdfizer` wrapper does **not** currently forward +> `--mapping-policy`, so the runner always executes with the default `strict`. +> A custom mapping validated through `vcf-rdfizer --validate` or +> `--mode validation` will therefore report `MISMATCH` on `q09`–`q13` even when +> the conversion is correct. Until the flag is exposed, either read past those +> five rows in `comparison.json`, or invoke the runner directly inside the +> container with `--mapping-policy report-only`. Tracked in +> [`roadmap.md`](roadmap.md). + +Once that gap is closed, the trade-off is the intended one: with a custom +mapping you keep distributional checking and lose the completeness and +permutation checks. That is correct behaviour — the alternative is a false +failure on every custom run — but it means a custom mapping is validated less +thoroughly than the default one, and you should know that before treating a +`PASS` as equivalent. + +Graph-integrity preflights (blank nodes, empty terms, duplicate statements) and +SHACL are mapping-independent and still apply in full. + +## 5. Worked start + +```bash +vcf-rdfizer-rules init -o my_rules.ttl +$EDITOR my_rules.ttl +vcf-rdfizer-rules check my_rules.ttl + +vcf-rdfizer --mode full \ + -i ./cohort.vcf.gz \ + --rules my_rules.ttl \ + --rdf-storage-mode plain \ + -o ./results +``` + +Recommendations that come from experience rather than from the checker: + +1. **Preserve the stable subject templates** (`file://{SOURCE_FILE}`, + `file://{SOURCE_FILE}#record/{ROW_ID}`, `file://{SOURCE_FILE}#call/{ROW_ID}`) + if you want joins with the wrapper-emitted triples to keep working. The + genotype, QUAL, INFO and header emitters all mint IRIs against those + templates and will not follow a mapping that changes them. +2. **Add, don't replace.** A mapping that extends the default with additional + `rr:TriplesMap` blocks keeps the full validation suite meaningful for the + parts it did not change; one that rebuilds the graph from scratch does not. +3. **Test on a small VCF first.** `test/test_vcf_files/test-100.vcf` converts in + seconds and exercises the same code path as a cohort file. + +## 6. SHACL + +The SHACL constraints are maintained in the vocabulary repository, not here: +[`vcf-rdfizer-vocabulary.shacl.ttl`](https://github.com/ecrum19/VCF-RDFizer-vocabulary/blob/main/shacl/vcf-rdfizer-vocabulary.shacl.ttl). +Point `--shacl-shapes` at them to check a graph structurally, independently of +the VCF. Note the known contradiction inside the published shapes, recorded in +[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). + +--- + +## See also + +- [Conversion](conversion.md) — what the default mapping produces +- [Architecture](architecture.md) — why some triples bypass RML +- [Validation](validation.md) — what changes under a custom mapping +- [`rules/README.md`](../rules/README.md) — the mapping directory itself diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..c44d74b --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,130 @@ +# Roadmap and future development + +What is planned, what is merely known-to-be-wrong, and what has been assessed +and deliberately rejected. Every item links to the document where it is +described in full. + +This is a working document, not a commitment. Items are ordered by how much they +block something else, not by effort. + +--- + +## Blocking publication + +### 1. Vocabulary coverage for the condensed representation + +The conversion emits 17 terms that `https://w3id.org/vcf-rdfizer/vocab#` does +not define — `CohortCallMatrix`, `FormatValueVector`, `SampleSet`, `VCFSample`, +`VCFTextVector`, `representationProfile`, `sampleIndex` and the rest. They do +not dereference, so **condensed graphs are not ontology-backed**. + +The work is in the vocabulary repository, not here. Until it lands, the +condensed representation is usable but not citable as linked data. +Full list: [`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). + +### 2. The SHACL / missing-value contradiction + +`vcfr:missingValuePolicy` requires `"."^^vcfr:Null` for a missing token; +`VCFRecordShape` constrains `vcfr:alt` to `sh:datatype xsd:string`. A record with +`ALT=.` cannot satisfy both, so it is impossible for the tool to be conformant. + +The fix is a decision in the vocabulary: either relax the `alt`/`ref`/`chrom` +shapes to `sh:or([xsd:string] [vcfr:Null])` — as the `qual` shape already does — +or drop the missing-value policy for those fields. The conversion currently +follows the policy. +[Detail](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). + +## Known defects + +### 3. Validation mapping policy is not forwarded + +`validation_runner.py` supports `--mapping-policy report-only`, which is exactly +what a custom `--rules` mapping needs: it records the mapping-dependent queries +(`q09`–`q13`) without failing on them. The `vcf-rdfizer` wrapper never passes +it, so the runner always executes `strict` and a correct custom-mapping +conversion is reported as `MISMATCH`. + +Fix: expose a `--mapping-policy` flag on the wrapper, and default it to +`report-only` automatically whenever `--rules` is not the shipped default. +Small change; it makes the custom-mapping extension point actually usable. +[Detail](rml-mappings.md#4-what-a-custom-mapping-costs-in-validation). + +### 4. Validation coverage gaps with named fixes + +From [`vcf-coverage.md`](vcf-coverage.md#remaining-gaps): + +- **`vcfr:contigCount` is counted, not read.** A wrong derived contig total is + undetected. Needs the value in a comparison, not just the predicate in the + census. +- **Header line values are not compared.** `q08` compares how many lines carry + each key and `q10` their types; `vcfr:headerValue` itself is only counted. The + structured attributes that matter are already covered. + +Both are query-set additions with matching mutation-catalogue entries, so +closing either one will *fail* the mutation harness until +[`vcf-coverage.md`](vcf-coverage.md) is updated — by design. + +## New capability + +### 5. Data linking as a plug-in system + +The largest planned addition: let users connect the graph to external resources +(rsIDs, genes, clinical assertions, frequencies) through declarative, +distributable linker plugins rather than by patching the tool. + +Full design, including the three join strategies, the three plugin tiers, the +network safeguards, the provenance model and the build order, is in +[`datalinking-design.md`](datalinking-design.md). Nothing is implemented yet; +the design exists to fix the extension contract before code depends on it. + +The hardest part is not the plugin system — it is allele normalization +(§9 of that document), because an under-matching join looks exactly like a true +negative. + +### 6. Query-time decoding for condensed graphs + +Condensed mode stores each FORMAT key as one tab-separated vector, which is what +makes cohort scale tractable. Reconstructing sample *i*'s value at query time +needs a split operation SPARQL does not portably provide. + +The options have been assessed in detail in +[`sample-representation-guide.md`](sample-representation-guide.md#7-assessment-of-geosparql-and-graphdb-sparql-extensions): + +| Option | Verdict | +| --- | --- | +| GeoSPARQL geometry functions | **No.** Genotype vectors are not geometries; the analogy does not survive contact | +| GraphDB `spif:split` | Useful as a prototype tokenizer and validation aid, not as a production join | +| `spif:split` + `spif:for` | Building blocks, but no ordinal pairing mechanism, so no trustworthy sample↔value join | +| A custom vector-extraction function, or an application-side decoder | **The promising direction** | + +The recommended shape is a function that takes a vector and a sample index (or a +`VCFSample` IRI) and returns both the position and the value, with the six +safeguards listed in that section. Crucially, this belongs in a **separate +decoding/projection layer** — the condensed model itself should not change. + +There is also a scale caveat that any implementation has to respect: expanding +every vector in a cohort-wide query produces millions of bindings before +filtering. A targeted single-position extractor is the right primitive; a +whole-graph expansion is not. + +## Deliberately not planned + +Stated so the absence reads as a decision rather than an oversight. + +| Not planned | Why | +| --- | --- | +| A Docker-free installation path | The pinned image is what makes results reproducible across machines | +| Variant normalization inside the conversion | The graph is a faithful transcription of the file; normalization is a linking-time concern | +| Distributed or multi-node execution | Out of scope; chunking already bounds memory on one machine | +| Incremental graph update | Would require identity and provenance machinery the current model does not have | +| Clinical interpretation or pathogenicity assertion | The tool transcribes; it is not a clinical authority | +| `.bcf` input | Would pull `htslib` into the parsing path; convert with `bcftools` first | + +--- + +## See also + +- [Limitations](limitations.md) — the current state, honestly +- [Data linking design](datalinking-design.md) — the largest planned addition +- [Validation methodology](validation-methodology.md) — how coverage is measured, so gaps stay falsifiable +- [`changelog.md`](../changelog.md) — what has actually shipped diff --git a/docs/sample-representation-guide.md b/docs/sample-representation-guide.md index 1488cbc..287c112 100644 --- a/docs/sample-representation-guide.md +++ b/docs/sample-representation-guide.md @@ -1,5 +1,9 @@ # Expanded and Condensed Knowledge Representations +*Part of the [VCF-RDFizer documentation](README.md). How these triples are emitted: +[`conversion.md`](conversion.md). Where section 7's assessment leads: +[`roadmap.md`](roadmap.md#6-query-time-decoding-for-condensed-graphs).* + This guide explains the two ways VCF-RDFizer can represent sample genotype information in RDF: @@ -535,3 +539,13 @@ as individual resources or as ordered vectors. as one tab-separated `VCFTextVector` literal in `sampleIndex` order. - **Graph shape**: which resources and relationships are present, independent of the underlying biological values. + + +--- + +## See also + +- [Conversion](conversion.md) — how these triples are emitted, and why not through RML +- [VCF coverage matrix](vcf-coverage.md) — which genotype elements are validated +- [Representations](representations.md) — why condensed matters at cohort scale +- [Roadmap](roadmap.md#6-query-time-decoding-for-condensed-graphs) — where section 7 leads diff --git a/docs/validation-methodology.md b/docs/validation-methodology.md new file mode 100644 index 0000000..b73d9f5 --- /dev/null +++ b/docs/validation-methodology.md @@ -0,0 +1,198 @@ +# How validation coverage is measured + +*Part of the [VCF-RDFizer documentation](README.md). How to run the validator: +[`validation.md`](validation.md). Current results: +[`vcf-coverage.md`](vcf-coverage.md).* + +The semantic validation suite compares a converted RDF graph against summaries +computed independently from the source VCF. That answers "do these agree?" but +not "what would we have missed?" — and a validator nobody has tried to fool is +an assumption, not evidence. + +This document describes the mutation-testing method used to answer the second +question, and how to reproduce the number. + +## The problem with self-reported coverage + +Before this harness existed, the suite's coverage was described in prose. That +description was wrong in a way nobody had noticed: `QUAL` is extracted from +every VCF into `records.tsv` and then never mapped into RDF, and no validation +check could have detected it, because none of them look at QUAL. A test suite +that passes on a graph missing an entire VCF column is not measuring what its +documentation claims. + +## Method + +Mutation testing treats the validator as the system under test. A *correct* +graph is deliberately corrupted in a specific, named way, and the validator is +asked for its verdict. If the verdict stops being `PASS`, the mutation is +**detected**. The proportion detected is the **mutation score**. + +Three properties make the result trustworthy: + +**The fixture cannot lie to itself.** The VCF, the RDF graph, and the parser +oracle are all derived from one declarative specification in +[`test/validation_fixtures.py`](../test/validation_fixtures.py). The graph's +genotype triples come from the project's own emitters rather than being written +by hand, so the fixture tracks the real implementation. A container test closes +the loop by running the real `parse_vcf` over the fixture VCF and asserting it +equals the derived oracle. + +**The harness measures shipped code.** Mutations are evaluated through +`evaluate_validation()` in `validation_runner.py` — the single function that +also decides real runs. There is no second implementation of the decision logic +for the tests to agree with. + +**Known gaps are assertions, not comments.** A mutation the suite cannot catch +carries a `known_undetected` reason, and the harness asserts it is *still* not +detected. Closing a gap therefore **fails a test**, forcing both the catalogue +and [`vcf-coverage.md`](vcf-coverage.md) to be updated. Coverage cannot silently +drift in either direction. + +## The catalogue + +[`test/validation_mutations.py`](../test/validation_mutations.py) holds one +entry per named corruption: an id, the VCF element it targets, the graph edit, +the check expected to catch it, and — where applicable — why it is not caught. + +Mutations cover dropped and duplicated records, corrupted CHROM/POS/ALT values, +retyped literals, permuted values between records, dropped and corrupted FILTER +strings, flipped genotypes, dropped sample calls, non-GT FORMAT values, QUAL, +INFO, header lines, file metadata, missing-token policy, representation +profiles, and spurious predicates. + +A mutation may target a triple the shipped mapping does not emit yet: the +fixture opts that shape in, so the harness answers "if we emitted this, would +we notice it breaking?" ahead of the fix. That is how the QUAL gap was measured +before QUAL was mapped. + +## Three independent layers + +The suite is not one check but three, deliberately independent so that a defect +has to evade all of them: + +1. **Aggregate comparison** — six deterministic VCF summaries recomputed from + the graph and compared exactly. Catches distributional error. +2. **Census and digest** — the graph's predicate and class inventory compared + against what the VCF implies (`q09`, `q10`), and per-record and per-value + identity digests (`q11`–`q13`). Catches missing, extraneous, permuted and + altered data that leaves distributions intact. +3. **Graph integrity** — blank nodes, empty or whitespace-only terms, and + duplicated statements. Catches malformed emission with no reference to the + VCF's contents. +4. **SHACL** — the shapes the vocabulary publishes, checked with `pyshacl` via + `--shacl-shapes`. Catches structural and datatype error, also independent of + the VCF. + +Duplicate detection is the one check that cannot be a query. A SPARQL store +holds a set, so a repeated line is collapsed on load and is invisible to every +other check here. It is found by comparing the statements the parser read +against the distinct triples the store holds; the difference is the number of +redundant statements. When either number is unavailable the check reports +`NOT_EVALUATED`, never `PASS` - a check that could not run must not look like a +clean result. + +The SHACL layer earned its place immediately: it found three conformance +violations the other two could not see, two of which were fixed (QUAL typed as +a plain literal instead of `xsd:decimal`/`vcfr:Null`; `##fileDate` untyped +instead of `xsd:date`) and one of which is a contradiction inside the +vocabulary itself, recorded in [`vcf-coverage.md`](vcf-coverage.md). + +SHACL is opt-in because `pyshacl` loads the graph into memory and does not +scale to a cohort-sized aggregate. + +### Identity digests, and why they are histograms + +`q11`–`q13` hash each record, INFO value and FORMAT value **together with its +own IRI**, then bucket on the first byte of the hash. Two properties matter: + +- **Binding identity into the hash is the whole point.** Digesting values alone + would give a permutation the same multiset and change nothing. +- **A histogram needs no ordering guarantee.** `GROUP_CONCAT` would have been + the obvious alternative and is not portable, because SPARQL does not define + its order. Bucketing is order-independent by construction, and keeps the + result at most 256 rows for a graph of any size. A mismatch is localized by + re-querying only the differing buckets. + +Fields are separated by U+001F, which cannot occur in a VCF field, so no shift +of a field boundary can forge a match. + +## Two engines, two layers + +The host layer runs queries under **rdflib**, in process, needing no Docker, so +the whole catalogue runs in the normal test loop. rdflib is also a third +independent SPARQL implementation, which incidentally guards against queries +that only work on one engine. + +The container layer is the authority: +[`test/cross_engine_agreement.py`](../test/cross_engine_agreement.py) runs every +validation query under **Comunica and QLever** inside the image and asserts they +return identical values. + +That second layer is not ceremony. QLever canonicalises numeric literals at +index time, reporting `"100"^^xsd:integer` as `xsd:int`. The POS datatype +preflight originally required exactly `xsd:integer`, so it flagged every record +under QLever while passing under Comunica — every QLever run would have ended +`BLOCKED_BY_PREFLIGHT`. Only cross-engine execution surfaces that class of bug. + +**When adding a query**, prefer datatype-*family* checks and lexical +comparisons over anything that assumes a store's internal representation, and +run the agreement script before trusting it. + +## Reproducing the score + +```bash +pip install rdflib + +VCF_RDFIZER_MUTATION_REPORT=mutation-score.json \ + python -m unittest test.test_validation_mutation_unit -v +``` + +`mutation-score.json` records the total, the detected count, the score, and a +per-mutation row with its status and any recorded gap reason. + +Cross-engine agreement, inside the image: + +```bash +docker build -t vcf-rdfizer:local . +docker run --rm -v "$PWD:/repo:ro" vcf-rdfizer:local \ + /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py +``` + +Both run in CI via +[`.github/workflows/validation-mutation.yml`](../.github/workflows/validation-mutation.yml). + +## Interpreting the number + +The mutation score is relative to the catalogue, not to the space of all +possible conversion bugs. A high score means "almost every corruption we +thought to write down is caught" — it is a lower bound on blindness, not a +proof of correctness. Growing the catalogue is as valuable as raising the +score, and a score that rises without the catalogue growing means nothing. Its value is that it is falsifiable, reproducible, and +moves in a direction you can point at. + +Current score and the full gap list: [`vcf-coverage.md`](vcf-coverage.md). + +## What a validation PASS means + +A `PASS` now means more than it did: with the census and the identity digests, +the graph must contain exactly the predicates and classes the VCF implies, and +every record, INFO value and FORMAT value must hash to the same bucket as its +counterpart in the source. It reliably catches dropped +records, misclassified variants, flipped genotypes, corrupted FILTER strings, +allele-count errors, missing header lines, and altered file metadata. + +It is **not** proof of a faithful record-by-record round-trip. Treat it as a +regression gate. The precise limits are enumerated in +[`vcf-coverage.md`](vcf-coverage.md) and in the "What is *not* tested" section +of [`validation.md`](validation.md). + + +--- + +## See also + +- [Validation](validation.md) — running the suite, and what it does and does not test +- [VCF coverage matrix](vcf-coverage.md) — the current score and the full gap list +- [Data linking design](datalinking-design.md) — how third-party linksets would plug into this harness +- [Roadmap](roadmap.md) — the coverage gaps with named fixes diff --git a/docs/validation.md b/docs/validation.md index 9786b70..4d8c178 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -1,20 +1,26 @@ # Semantic VCF/RDF validation +*Part of the [VCF-RDFizer documentation](README.md). How coverage is measured: +[`validation-methodology.md`](validation-methodology.md). Element-by-element +results: [`vcf-coverage.md`](vcf-coverage.md).* + `vcf-rdfizer --mode validation` checks that a converted RDF graph reproduces six deterministic VCF summaries. The same validator can be added to a full run with `--validate`; in that case it runs once for each input after RDF creation and compression. It computes one result from the source VCF using `cyvcf2` (and `bcftools` for exact FILTER strings when available), runs -the equivalent SPARQL queries with Comunica, then compares canonical integer -results exactly. +the equivalent SPARQL queries against the graph, then compares canonical +integer results exactly. Two axes are configurable and independent: which +artifact is validated (N-Triples, HDT, or COTTAS) and which SPARQL engine runs +the queries (Comunica or QLever). Read "What is *not* tested" below before +treating a `PASS` as a correctness proof. -Standalone validation consumes a single N-Triples aggregate (`.nt` or -`.nt.gz`). It mounts the source read-only; gzip input is expanded under -`/work` **inside the Docker container**, while plain `.nt` input is read in -place. The validator uses the resulting stream for Raptor syntax validation -and Comunica queries, and removes any temporary expansion before the container -exits. No decompressed RDF is written beneath `--out`; only reports are -retained. +The source artifact is mounted read-only. Anything that is not already plain +N-Triples is decoded under `/work` **inside the Docker container**; a plain +`.nt` input is read in place. The validator uses the resulting stream for +Raptor syntax validation and for the SPARQL queries, and removes any temporary +decode before the container exits. No decoded RDF is written beneath `--out`; +only reports are retained. ## Run it @@ -51,11 +57,131 @@ vcf-rdfizer --mode full \ --out ./results ``` -For standalone validation, the input VCF and `.nt`/`.nt.gz` must originate from -the same conversion. `--rdf` must name an existing `.nt` or `.nt.gz` file; -standalone validation does not accept HDT or COTTAS artifacts. Use full mode -with `--rdf-storage-mode space-optimized` for a gzip aggregate, or `plain` for -an uncompressed aggregate. +For standalone validation, the input VCF and the RDF artifact must originate +from the same conversion. + +## Which artifact is validated + +`--rdf` accepts any artifact the pipeline produces: + +| Extension | How it is read | +|---|---| +| `.nt` | Read in place; nothing is copied | +| `.nt.gz` | Expanded into the container scratch directory | +| `.nt.br` | Expanded with `brotli -d` | +| `.hdt` | Decoded with `hdt2rdf` | +| `.cottas` | Decoded with `cottas_tool.py decompress` (pycottas) | +| `.cottas.gz`, `.cottas.br` | Unwrapped, then decoded as above | + +Format is inferred from the filename; `--rdf-format` overrides that for an +artifact with an unusual name. + +Validating an `.hdt` or `.cottas` decodes it back to N-Triples and then runs the +full semantic suite over the result. That is a strictly stronger statement than +the triple-count round-trip `validate_compression.py` performs during +compression: it proves the artifact decodes to a graph that still reproduces +every VCF summary, not merely to the right number of triples. The decode needs +scratch space of roughly the uncompressed graph size under the container's +`/work`, and nothing decoded is ever written beneath `--out`. + +```bash +# Validate a compressed representation directly +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz \ + --rdf ./results/cohort/cohort.hdt \ + --sample-representation condensed \ + --out ./validation-results +``` + +In full mode, `--validate-artifacts` chooses which produced artifacts to check. +Each one is validated independently and gets its own report directory, so a run +can prove the aggregate, the HDT, and the COTTAS file all agree with the VCF: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz \ + --rdf-storage-mode space-optimized \ + --representations hdt,cottas \ + --validate --validate-artifacts all \ + --out ./results +``` + +Accepted values are `aggregate` (the default), `hdt`, `cottas`, `all`, or a +comma-separated subset. A representation that was not selected, or whose +artifact is missing after a recoverable index warning, is skipped rather than +reported as a failure - the run already records why it is absent. + +## Which SPARQL engine runs the queries + +`--validation-engine` selects the backend. Both answer identical queries and +feed the same comparison layer, so the choice is a scale decision, never a +semantic one; every report records which engine produced it. + +| Engine | Behaviour | Use it when | +|---|---|---| +| `comunica` (default) | Queries the N-Triples file with no setup, holding the graph in the Node heap | The graph fits comfortably in RAM | +| `qlever` | Builds an on-disk [QLever](https://github.com/ad-freiburg/qlever) index in container scratch, serves it on a container-local port, then tears both down | The graph no longer fits in memory, or the aggregate queries are too slow | + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --validation-engine qlever --qlever-memory-gb 32 \ + --out ./validation-results +``` + +QLever tuning, all optional: + +| Option | Meaning | +|---|---| +| `--qlever-memory-gb N` | Index and server memory budget (default 4) | +| `--qlever-port N` | Container-local port (default 7019; never published) | +| `--qlever-startup-timeout N` | Seconds to wait for the server after indexing (default 900) | +| `--validation-query-timeout N` | Per-query timeout, both engines (default 3600) | +| `--qlever-index-arg ARG` | Extra argument for the index builder (repeatable) | +| `--qlever-server-arg ARG` | Extra argument for the server (repeatable) | + +QLever's command-line interface has changed across releases. If a future +QLever image disagrees with the defaults, the whole command line can be +replaced without changing code, using `{index}`, `{input}`, `{memory}`, and +`{port}` placeholders: + +```bash +docker run -e QLEVER_INDEX_COMMAND='qlever-index -i {index} -f {input} -F nt -m {memory}' ... +docker run -e QLEVER_SERVER_COMMAND='qlever-server -i {index} -p {port} -m {memory}' ... +``` + +The exact argv that ran is recorded in each report's `manifest.json` under +`engine.commands`. The QLever index lives only in container scratch and is +removed as soon as the queries finish. + +QLever is copied into the image from the upstream `adfreiburg/qlever` image at +build time (`qlever-index` and `qlever-server`, verified against build +`bfd5741`); pin a version with +`--build-arg QLEVER_IMAGE=adfreiburg/qlever:`. That image is built on a +different Ubuntu release, so its release-specific Boost, ICU, jemalloc and +io_uring libraries are copied alongside the binaries into `/opt/qlever/lib`, +and only QLever's own processes are pointed there - they cannot shadow +anything the rest of the image links against. A build-time `ldd` check records +whether they resolve, and the validator reports that note if the engine cannot +start. Comunica remains the default, so an image whose QLever binaries do not +link still validates normally. + +### Engine equivalence, and one place they differed + +Both engines are held to producing identical results. That is verified, not +assumed: all eleven queries were run under both engines against the same +expanded and condensed graphs, and every normalized result matched. + +One difference had to be fixed to make that true. QLever canonicalises numeric +literals at index time, so `"100"^^xsd:integer` is reported by `DATATYPE()` as +`xsd:int`. `preflight_position_datatype` originally required exactly +`xsd:integer`, which meant it flagged every record on QLever while passing on +Comunica - every run would have been `BLOCKED_BY_PREFLIGHT`. The query now +accepts the XSD integer family, which is what the check actually means. It +still reports a plain-string or `xsd:decimal` POS as an anomaly on both +engines. + +If you add a query, prefer datatype-family checks and lexical comparisons over +assertions about a store's internal numeric representation. `--validation-id NAME` changes the report directory name. The default is the source VCF basename without `.vcf` or `.vcf.gz`. Existing result directories @@ -63,8 +189,8 @@ are never overwritten. `--filter-oracle {auto,bcftools,cyvcf2}` controls the FILTER-field oracle; `auto` uses `bcftools` when it is available in the image. Validation progress uses the same JSONL sidecar protocol as conversion and -partitioned compression. It emits a `validation` task with a total of eleven -preflight/core queries, then records each query start and completion under the +partitioned compression. It emits a `validation` task covering every +preflight, exact-count and core query, then records each query start and completion under the run's temporary `.progress/` area while the host displays it through the normal Rich/plain progress session. `--quiet` suppresses that terminal display and the validator's per-query/summary stdout, but still writes command logs, @@ -83,10 +209,90 @@ The common record-level queries are used for both graph shapes: | Q5 | per-sample genotype class counts | | Q6 | genotype-derived single-ALT `(AN, AC, siteCount)` distribution | +| Q7 | file-level `##fileformat`, `##reference` and `##source` declarations | +| Q8 | how many `##` meta-information lines carry each header key | +| Q9 | every predicate in the graph and its triple count | +| Q10 | every asserted class and its resource count | +| Q11 | per-record identity digest, bucketed | +| Q12 | per-INFO-value identity digest, bucketed | +| Q13 | per-FORMAT-value identity digest, bucketed | + +Q9 and Q10 are the completeness check: comparing the graph's inventory against +what the VCF implies catches a predicate that is missing, one with the wrong +cardinality, and one that should not be there at all. Q11-Q13 close the +permutation gap - see +[`validation-methodology.md`](validation-methodology.md#identity-digests-and-why-they-are-histograms). + +Q9-Q13 assume the shipped RML mapping's predicate inventory and IRI templates. +A custom `--rules` changes both by design. `validation_runner.py` has a +`--mapping-policy report-only` setting for exactly that case, which records +those five queries without failing on them and lets the aggregate comparisons +carry the run. + +> **Known gap.** The `vcf-rdfizer` wrapper does not currently forward +> `--mapping-policy`, so the runner always runs `strict` and a *correct* custom +> mapping is reported as `MISMATCH` on Q9-Q13. See +> [`rml-mappings.md`](rml-mappings.md#4-what-a-custom-mapping-costs-in-validation) +> and [`roadmap.md`](roadmap.md#3-validation-mapping-policy-is-not-forwarded). + Q5/Q6 are only required when the VCF has both samples and a `GT` FORMAT field. The suite additionally checks N-Triples syntax, record cardinality, POS datatype, representation profile, and representation-specific sample/GT -inventory before interpreting the six result sets. +inventory before interpreting the result sets. + +### Graph integrity + +Three checks run before any comparison, because a graph that fails one is not +worth comparing: + +| Check | What it catches | +|---|---| +| `preflight_blank_nodes` | Any blank node. Every class in the vocabulary declares an IRI template, so a blank node means a term map produced no IRI - and the record and value digests could not address such a node. | +| `preflight_empty_values` | Empty or whitespace-only literals and IRIs. An empty literal means a value was lost rather than marked missing; an empty IRI means a template substitution collapsed. | +| `preflight_duplicate_triples` | The same statement emitted more than once. | + +Duplicates need explaining, because no SPARQL query can see them: a store holds +a **set**, so a repeated line is collapsed on load and every other check here +counts it once. The only way to detect it is to compare the statements the +parser read against the distinct triples the store holds. Raptor reports the +first, `preflight_distinct_triple_count` the second, and the difference is +exactly the number of redundant statements. This is worth having because +duplicated RDF parts are a known failure mode of the conversion - +`run_conversion.sh` already carries a defensive dedupe for it - and a duplicated +aggregate costs twice the storage for no added information. + +All three are blocking. When an input a check needs is unavailable it reports +`NOT_EVALUATED` rather than `PASS`, so a check that could not run is never +mistaken for a clean result, and never fails the run either. + +Each structural anomaly preflight runs twice: a `LIMIT 100` query returning +example rows for diagnosis, and a companion aggregate returning the **exact** +anomaly count. Reports carry both, plus `sampleTruncated`, so a graph with ten +million anomalies is never confused with one that has a hundred. + +`--strict-conformance` promotes a missing-token conformance failure (a plain +`"."` literal not typed as `vcfr:Null`) from a report-only observation to a +validation failure. + +### SHACL + +`--shacl-shapes PATH` adds an independent structural layer: the graph is checked +against a SHACL shapes file (for example the vocabulary's published +`shacl/vcf-rdfizer-vocabulary.shacl.ttl`) with `pyshacl`, inside the container. +A violation blocks the run, because a structurally wrong graph makes the +aggregate comparisons uninterpretable. + +It is **off by default**: `pyshacl` loads the whole graph into memory, so it is +suitable for a single-sample graph or a sample of a cohort, not for a +cohort-scale aggregate. The report records the conformance verdict, the exact +violation count, the distinct property paths involved, and the first 50 +violations; the full text is written alongside it. + +```bash +vcf-rdfizer --mode validation -i ./sample.vcf.gz --rdf ./results/sample/sample.nt.gz \ + --shacl-shapes ./vocabulary/shacl/vcf-rdfizer-vocabulary.shacl.ttl \ + -o ./validation-results +``` For the expanded representation, Q5/Q6 traverse `SampleCall` and `FormatFieldValue` resources. For the condensed representation, their matching @@ -95,6 +301,56 @@ extract each tab-delimited GT value by its `sampleIndex`. This makes the condensed tests semantically equivalent without inflating the persisted RDF back into per-sample value resources. +## What is *not* tested + +The suite is deliberately a set of exact aggregate comparisons. That makes it +cheap, engine-independent, and free of a second RDF parser to disagree with - +but it bounds what it can detect, and those bounds are worth stating plainly. +`test/test_validation_logic_unit.py` encodes both the detections and the gaps +below, so a change that closes one will fail a test rather than pass silently. + +**No per-record identity.** Every core query is a `GROUP BY` count. A graph that +swapped `POS` between two records on the same contig and in the same 1 Mb +window - or `REF`/`ALT` between two records of the same shape class - produces +byte-identical results and passes. The suite proves the *distributions* match, +not that record *i* carries record *i*'s values. Detecting a permutation needs a +per-record identity check, which the current query set does not have. + +**Whole VCF columns are unchecked.** Nothing compares `ID`, `QUAL`, or `INFO`, +and no `FORMAT` field other than `GT` is validated. A conversion bug that +mangled every `DP` value, dropped every rsID, or corrupted `INFO` would pass. +The `header_lines` and `file_metadata` triples maps - the `HeaderLine` +resources and the `VCFFile` attributes - are likewise never queried. + +**No completeness bound.** Nothing asserts a total triple count or the absence +of extraneous triples. `preflight_record_cardinality` catches duplicated or +missing per-record properties, but only for `VCFRecord` subjects. + +**Header line values are compared only through their structured form.** The +attributes that matter - `filterId`, `altId`, `contigId`, contig length/md5/ +assembly, and the INFO/FORMAT declarations - are lifted into their own +properties and counted. The raw `vcfr:headerValue` literal itself is counted but +never read, and `vcfr:contigCount` is likewise checked for presence rather than +value. + +**The census assumes the shipped mapping.** Under a custom `--rules`, Q9-Q13 +are no longer meaningful and the run should rest on the aggregate comparisons +alone - which is what `--mapping-policy report-only` is for, and which the +wrapper does not yet enable (see the note above). + +In short: a `PASS` is strong evidence that the conversion preserved the VCF's +*summary statistics* over the elements it covers, and it reliably catches +dropped records, misclassified variants, flipped genotypes, corrupted FILTER +strings, allele-count errors, missing header lines, and altered file metadata. +It is not proof of a faithful record-by-record round-trip. Treat it as a +regression gate, not a correctness proof. + +These limits are measured rather than asserted. Every claim above corresponds +to a named mutation in the catalogue described in +[`validation-methodology.md`](validation-methodology.md); the current score and +the full element-by-element breakdown are in +[`vcf-coverage.md`](vcf-coverage.md). + ## Results and cleanup evidence Results are written to the run's canonical metrics tree: @@ -103,16 +359,21 @@ Results are written to the run's canonical metrics tree: /run_metrics/__/reports/validation// ``` -Standalone validation consumes either `.nt` or `.nt.gz`. Full-mode validation -uses the aggregate produced by that run and can read either `.nt` (plain -storage) or `.nt.gz` (space-optimized storage). In both cases the detailed -results live beneath `reports/validation/`, so they are indexed by the same -`summary.json` used for conversion and compression metrics. +Detailed results live beneath `reports/validation/`, so they are indexed by +the same `summary.json` used for conversion and compression metrics. Important files include `summary.json`, `manifest.json`, `parser.json`, -`rdf-validation.json`, `preflight.json`, `sparql.json`, and `comparison.json`. -Raw Comunica JSON, stderr, and query resource logs are in `raw/`; normalized -results are in `normalized/`. +`rdf-validation.json`, `materialization.json`, `preflight.json`, `sparql.json`, +and `comparison.json`. Raw SPARQL Results JSON, stderr, and query resource logs +are in `raw/`; normalized results are in `normalized/`. `manifest.json` records +the engine that ran (including QLever's exact argv), the source artifact format +and checksum, and every decode step; `materialization.json` records how a +compressed or indexed artifact was turned into N-Triples and how many triples +that yielded. + +When several artifacts are validated in one full run, each gets its own +directory: the aggregate keeps `/`, and the representations use +`__hdt/` and `__cottas/`. The parent VCF-RDFizer run metrics include `run_metrics/__/stages/validation/.json`. @@ -124,3 +385,14 @@ validation scratch RDF was retained on the host. both paths ran but differ. `BLOCKED_BY_PREFLIGHT` means RDF syntax or core graph structure failed. `EXECUTION_FAILED` means a parser or query engine could not complete. + + +--- + +## See also + +- [Validation methodology](validation-methodology.md) — how this suite's coverage is measured +- [VCF coverage matrix](vcf-coverage.md) — element by element, with the mutation that proves each row +- [Representations](representations.md) — the round-trip check that runs during compression +- [Output and metrics](output-and-metrics.md) — where reports land in the run tree +- [Limitations](limitations.md) — consolidated, across the whole tool diff --git a/docs/vcf-coverage.md b/docs/vcf-coverage.md new file mode 100644 index 0000000..37d1a6d --- /dev/null +++ b/docs/vcf-coverage.md @@ -0,0 +1,150 @@ +# VCF coverage matrix + +*Part of the [VCF-RDFizer documentation](README.md). What the conversion emits: +[`conversion.md`](conversion.md). How this table is measured: +[`validation-methodology.md`](validation-methodology.md).* + +What of a VCF file VCF-RDFizer represents in RDF, and what the semantic +validation suite actually verifies. One row per VCF element. This is the +tracking artifact for the coverage work and the source table for publication. + +Two independent questions per row, deliberately kept apart: + +- **Represented** — does the conversion emit RDF for this element at all? +- **Validated** — would a corruption of it be *detected*? Backed by a named + mutation in [`test/validation_mutations.py`](../test/validation_mutations.py), + not by inspection. + +Last measured mutation score: **76/78 (97%)** across 42 distinct mutations. +Regenerate with: + +```bash +VCF_RDFIZER_MUTATION_REPORT=mutation-score.json \ + python -m unittest test.test_validation_mutation_unit +``` + +--- + +## Fixed fields (the eight mandatory columns) + +| VCF element | `records.tsv` | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | --- | +| CHROM | `CHROM` | `vcfr:chrom` | yes | `q01`, `q11` | `corrupt_chrom` | +| POS | `POS` | `vcfr:pos` (`xsd:integer`) | yes | `q01`, `q11`, `preflight_position_datatype` | `corrupt_pos`, `drop_pos`, `retype_pos_as_string` | +| POS ↔ record binding | — | — | yes | `q11_record_digest` | `permute_pos` | +| ID | `ID` | `vcfr:recordId` | yes | `q11_record_digest` | (covered by digest) | +| REF | `REF` | `vcfr:ref` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | +| ALT | `ALT` | `vcfr:alt` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | +| REF/ALT ↔ record binding | — | — | yes | `q11_record_digest` | `permute_ref_alt` | +| QUAL | `QUAL` | `vcfr:qual` (`xsd:decimal` / `vcfr:Null`) | yes | `q09`, `q11` | `drop_qual`, `drop_all_qual`, `corrupt_qual` | +| FILTER | `FILTER` | `vcfr:filter` | yes | `q04` (exact lexical), `q11` | `drop_filter`, `corrupt_filter_lexical` | +| INFO (raw) | `INFO` | `vcfr:infoRaw` | yes | `q11_record_digest` | `corrupt_info_raw` | +| INFO (structured) | `INFO` | `vcfr:hasInfoValue` → `vcfr:InfoFieldValue` → `vcfr:declaredBy` | yes | `q09`, `q12_info_value_digest` | `drop_info_value`, `corrupt_info_value`, `retype_info_value` | + +INFO values carry `vcfr:fieldValue` plus a typed `fieldValueInteger` / +`fieldValueDecimal` when the declaration says `Number=1` and the value parses; +a Flag entry carries `vcfr:fieldValueBoolean true`. A multi-valued field +(`Number=A/R/G/.`) keeps only the lexical value, because the vocabulary's IRI +template gives one node per key. + +## Genotype fields + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| FORMAT declaration | `vcfr:formatRaw` | yes | `q09_predicate_census` | — | +| Sample identity | `vcfr:sampleId` / `vcfr:sampleName` | yes | `preflight_sample_gt_inventory`, `q09` | `drop_sample_call` | +| GT values | `vcfr:hasFormatValue` → `vcfr:fieldValue` | yes | `q05`, `q06`, `q13` | `flip_genotype` | +| Non-GT FORMAT (DP, GQ, AD, PL…) | `vcfr:FormatFieldValue` / `vcfr:FormatValueVector` | yes | `q13_format_value_digest` | `drop_format_value_dp`, `corrupt_format_value_dp`, `corrupt_format_vector` | + +## Header section + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| `##` line count and keys | `vcfr:HeaderLine` + `vcfr:headerKey` | yes | `q08_header_line_census` | `drop_header_line` | +| Line type | `FileFormatHeaderLine`, `INFOHeaderLine`, `ContigHeaderLine`, … | yes | `q10_class_census` | `untype_header_line` | +| `##fileformat` | `vcfr:fileFormat` | yes | `q07_file_metadata` | `corrupt_file_metadata` | +| `##reference` | `vcfr:referenceGenome` | yes | `q07_file_metadata` | `drop_reference_genome` | +| `##source` | `vcfr:sourceSoftware` | yes | `q07_file_metadata` | — | +| `##fileDate` | `vcfr:fileDate` (`xsd:date` when the form allows) | yes | `q09_predicate_census` | `drop_file_date` | +| `##FILTER` | `vcfr:FilterDefinition` + `vcfr:filterId` | yes | `q09`, `q10` | `drop_filter_definition` | +| `##ALT` | `vcfr:AltDefinition` + `vcfr:altId` | yes | `q09`, `q10` | `drop_alt_definition` | +| `##contig` | `vcfr:ContigHeaderLine` + `contigId`/`contigLength`/`contigMd5`/`contigAssembly` | yes | `q09`, `q10` | `drop_contig_attribute` | +| `##INFO` / `##FORMAT` definitions | `vcfr:InfoFieldDefinition` / `vcfr:FormatFieldDefinition` with `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription` | yes | `q09`, `q10` | `drop_info_definition` | +| Header line *values* | `vcfr:headerValue` | yes | `q09` (count only) | — | +| `vcfr:contigCount` | derived scalar | yes | **count only, not value** | `corrupt_contig_count` | + +An unrecognized `##` key keeps only the base `vcfr:HeaderLine` type: inventing +a subclass would put a term in the graph that the vocabulary does not define. + +## Graph-level properties + +| Property | Validated by | Mutation | +| --- | --- | --- | +| N-Triples syntax | Raptor (`rapper -c`) | — | +| SHACL shape conformance | `--shacl-shapes` (opt-in, independent layer) | — | +| Per-record property cardinality | `preflight_record_cardinality` + exact count | `drop_pos` | +| Missing-token policy | `preflight_missing_token_conformance`, fatal under `--strict-conformance` | `plain_dot_literal` | +| Representation profile | `preflight_representation_profile` | `wrong_representation_profile` | +| Record count | q01/q02/q04 totals | `drop_record`, `duplicate_record` | +| **No extraneous triples** | `q09_predicate_census` (extra rows) | `spurious_predicate` | +| **No blank nodes** | `preflight_blank_nodes` | `introduce_blank_node`, `blank_node_object` | +| **No empty or whitespace-only terms** | `preflight_empty_values` | `empty_literal`, `whitespace_only_literal` | +| **No duplicate statements** | `preflight_duplicate_triples` (parsed vs distinct) | `duplicate_triple`, `duplicate_whole_graph` | +| Class inventory | `q10_class_census` | `untype_header_line` | +| Per-record identity | `q11_record_digest` | `permute_pos`, `permute_ref_alt` | + +## Remaining gaps + +1. **`vcfr:contigCount` is counted, not read.** A wrong derived contig total is + not detected. Closing it needs the value in a comparison, not just the + predicate in the census. +2. **Header line values are not compared.** `q08` compares how many lines carry + each key and `q10` their types, but `vcfr:headerValue` itself is only + counted. The structured attributes that matter (`filterId`, `contigId`, …) + *are* covered. +3. **The census assumes the shipped mapping.** A custom `--rules` changes the + inventory and IRI templates by design, so `q09`–`q13` fall back to + report-only. That is correct, but it means a custom mapping is validated + only by the aggregate comparisons. + +## Vocabulary alignment + +The tool emits 17 terms that `https://w3id.org/vcf-rdfizer/vocab#` does **not** +define — all belonging to the condensed representation: + +`CohortCallMatrix`, `CondensedRepresentation`, `ExpandedRepresentation`, +`FormatValueVector`, `SampleSet`, `VCFSample`, `VCFTextVector`, +`appliesToSampleSet`, `encodedValues`, `hasCallMatrix`, `hasFormatValueVector`, +`hasSample`, `hasSampleSet`, `representationProfile`, `sampleIndex`, +`sampleName`, `valueEncoding` + +Dereferencing any of these returns nothing, so **condensed graphs are not +ontology-backed**. This is the one remaining publication blocker and is work in +the vocabulary repository, not here. + +### An open conflict inside the vocabulary + +Running SHACL surfaced a contradiction between two parts of the published +vocabulary that the tool cannot satisfy simultaneously: + +- `vcfr:missingValuePolicy` says a missing token SHOULD be `"."^^vcfr:Null`. +- `VCFRecordShape` constrains `vcfr:alt` to `sh:datatype xsd:string`. + +A record with `ALT=.` therefore violates the shape no matter which rule the +conversion follows. The tool currently follows the missing-value policy. This +needs a decision in the vocabulary: either relax the `alt`/`ref`/`chrom` shapes +to `sh:or([xsd:string] [vcfr:Null])`, as the `qual` shape already does, or drop +the missing-value policy for those fields. + +Two shape violations found the same way have already been fixed here: QUAL is +now `xsd:decimal`/`vcfr:Null` rather than a plain literal, and `##fileDate` is +typed `xsd:date` when its form allows. + +--- + +## See also + +- [Validation methodology](validation-methodology.md) — how the coverage in this table is measured +- [Validation](validation.md) — running the validator +- [Conversion](conversion.md) — how each represented element is emitted +- [Roadmap](roadmap.md) — the vocabulary and coverage gaps above, with their planned fixes diff --git a/pyproject.toml b/pyproject.toml index b42218e..695dcc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,9 @@ vcf-rdfizer-rules = "vcf_rdfizer_rules:main" dev = [ "build>=1.2.2", "twine>=5.1.1", + # Test-only: an in-process SPARQL engine for the validation mutation harness, + # so it needs no Docker. The suite skips those tests when it is absent. + "rdflib>=7.0.0", ] [tool.setuptools] diff --git a/rules/default_rules.ttl b/rules/default_rules.ttl index 7332230..d229b3c 100644 --- a/rules/default_rules.ttl +++ b/rules/default_rules.ttl @@ -47,6 +47,9 @@ rr:predicate vcfr:fileFormat ; rr:objectMap [ rml:reference "FILE_FORMAT" ] ] ; + # fileDate is emitted by the wrapper: the published shape requires xsd:date, + # but ##fileDate has no mandated format in the VCF specification, so the + # value has to be inspected before it can be typed. rr:predicateObjectMap [ rr:predicate vcfr:sourceSoftware ; rr:objectMap [ rml:reference "SOURCE_SOFTWARE" ] @@ -215,6 +218,9 @@ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ; rr:class vcfr:VariantCall ] ; + # QUAL is emitted by the wrapper, not here: the published SHACL shape + # requires xsd:decimal or vcfr:Null depending on whether the value is + # present, and RML cannot choose a datatype per row. rr:predicateObjectMap [ rr:predicate vcfr:filter ; rr:objectMap [ rml:reference "FILTER" ] diff --git a/src/validation/queries/common/preflight_blank_nodes.rq b/src/validation/queries/common/preflight_blank_nodes.rq new file mode 100644 index 0000000..bc217a3 --- /dev/null +++ b/src/validation/queries/common/preflight_blank_nodes.rq @@ -0,0 +1,18 @@ +PREFIX vcfr: + +# Blank nodes anywhere in the graph. +# +# Every class in the VCF-RDFizer vocabulary declares an vcfr:iriTemplate, so a +# blank node means a term map produced no IRI. That is always a defect here: +# blank-node identity is scoped to the file, so the record and value digests +# could not address such a node, and neither could anything downstream that +# merges or re-serializes the graph. +# +# A predicate cannot be a blank node in RDF, so only subject and object are +# examined. +SELECT ?s ?p ?o +WHERE { + ?s ?p ?o . + FILTER(ISBLANK(?s) || ISBLANK(?o)) +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_blank_nodes_count.rq b/src/validation/queries/common/preflight_blank_nodes_count.rq new file mode 100644 index 0000000..8cfd553 --- /dev/null +++ b/src/validation/queries/common/preflight_blank_nodes_count.rq @@ -0,0 +1,8 @@ +PREFIX vcfr: + +# Exact number of triples involving a blank node. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?s ?p ?o . + FILTER(ISBLANK(?s) || ISBLANK(?o)) +} diff --git a/src/validation/queries/common/preflight_distinct_triple_count.rq b/src/validation/queries/common/preflight_distinct_triple_count.rq new file mode 100644 index 0000000..6b23075 --- /dev/null +++ b/src/validation/queries/common/preflight_distinct_triple_count.rq @@ -0,0 +1,17 @@ +PREFIX vcfr: + +# How many distinct triples the graph contains. +# +# A SPARQL store holds a set, so a duplicated line in the N-Triples file is +# invisible to every other query here: the store collapses it on load. The only +# way to see duplication is to compare this against the number of statements the +# parser actually read, which the Raptor step records. The difference is exactly +# the number of redundant lines. +# +# This matters because duplicate RDF parts are a known failure mode of the +# conversion - run_conversion.sh already carries a defensive dedupe for it - and +# a duplicated aggregate is twice the size for no added information. +SELECT (COUNT(*) AS ?distinctTripleCount) +WHERE { + ?s ?p ?o . +} diff --git a/src/validation/queries/common/preflight_empty_values.rq b/src/validation/queries/common/preflight_empty_values.rq new file mode 100644 index 0000000..2d0effb --- /dev/null +++ b/src/validation/queries/common/preflight_empty_values.rq @@ -0,0 +1,23 @@ +PREFIX vcfr: + +# Empty or whitespace-only terms. +# +# The pipeline should never materialize one. RML emits no triple for an empty +# reference, and every direct emitter either guards on a non-empty value or +# substitutes the VCF missing token, which becomes "."^^vcfr:Null. An empty +# literal therefore means a value was lost rather than marked missing, and an +# empty IRI means a template substitution collapsed. +# +# Whitespace-only counts as empty: a literal of spaces or tabs carries no more +# information than "" and is just as certainly a defect. +SELECT ?s ?p ?o ?issue +WHERE { + ?s ?p ?o . + BIND( + IF(ISLITERAL(?o) && REGEX(STR(?o), "^\\s*$"), "EMPTY_LITERAL", + IF(STR(?s) = "" || STR(?p) = "" || (ISIRI(?o) && STR(?o) = ""), "EMPTY_IRI", "") + ) AS ?issue + ) + FILTER(?issue != "") +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_empty_values_count.rq b/src/validation/queries/common/preflight_empty_values_count.rq new file mode 100644 index 0000000..022c095 --- /dev/null +++ b/src/validation/queries/common/preflight_empty_values_count.rq @@ -0,0 +1,11 @@ +PREFIX vcfr: + +# Exact number of triples carrying an empty or whitespace-only term. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?s ?p ?o . + FILTER( + (ISLITERAL(?o) && REGEX(STR(?o), "^\\s*$")) + || STR(?s) = "" || STR(?p) = "" || (ISIRI(?o) && STR(?o) = "") + ) +} diff --git a/src/validation/queries/common/preflight_missing_token_conformance_count.rq b/src/validation/queries/common/preflight_missing_token_conformance_count.rq new file mode 100644 index 0000000..a909af3 --- /dev/null +++ b/src/validation/queries/common/preflight_missing_token_conformance_count.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +# Exact number of missing tokens serialized as a plain "." literal. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?s ?p ?o . + FILTER(ISLITERAL(?o) && STR(?o) = ".") + FILTER(DATATYPE(?o) != vcfr:Null) +} diff --git a/src/validation/queries/common/preflight_position_datatype.rq b/src/validation/queries/common/preflight_position_datatype.rq index 5932226..132a55e 100644 --- a/src/validation/queries/common/preflight_position_datatype.rq +++ b/src/validation/queries/common/preflight_position_datatype.rq @@ -1,9 +1,21 @@ PREFIX vcfr: PREFIX xsd: -SELECT ?record ?pos +# POS must carry an integer datatype, never a plain string or a fractional type. +# +# The accepted set is the XSD integer family rather than xsd:integer alone, +# because a store may canonicalise an integer literal to a narrower derived +# type at index time. QLever, for instance, reports "100"^^xsd:integer as +# xsd:int, which would otherwise make this preflight fail every run on that +# engine while passing on Comunica. Anything outside the family - xsd:string, +# xsd:decimal, xsd:double, a plain literal - is still reported. +SELECT ?record ?pos (DATATYPE(?pos) AS ?datatype) WHERE { ?record a vcfr:VCFRecord ; vcfr:pos ?pos . - FILTER(DATATYPE(?pos) != xsd:integer) + FILTER(DATATYPE(?pos) NOT IN ( + xsd:integer, xsd:int, xsd:long, xsd:short, xsd:byte, + xsd:nonNegativeInteger, xsd:positiveInteger, + xsd:unsignedLong, xsd:unsignedInt, xsd:unsignedShort, xsd:unsignedByte + )) } LIMIT 100 diff --git a/src/validation/queries/common/preflight_position_datatype_count.rq b/src/validation/queries/common/preflight_position_datatype_count.rq new file mode 100644 index 0000000..e773e8e --- /dev/null +++ b/src/validation/queries/common/preflight_position_datatype_count.rq @@ -0,0 +1,13 @@ +PREFIX vcfr: +PREFIX xsd: + +# Exact number of POS values outside the XSD integer family. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:pos ?pos . + FILTER(DATATYPE(?pos) NOT IN ( + xsd:integer, xsd:int, xsd:long, xsd:short, xsd:byte, + xsd:nonNegativeInteger, xsd:positiveInteger, + xsd:unsignedLong, xsd:unsignedInt, xsd:unsignedShort, xsd:unsignedByte + )) +} diff --git a/src/validation/queries/common/preflight_record_cardinality_count.rq b/src/validation/queries/common/preflight_record_cardinality_count.rq new file mode 100644 index 0000000..1f25ced --- /dev/null +++ b/src/validation/queries/common/preflight_record_cardinality_count.rq @@ -0,0 +1,19 @@ +PREFIX vcfr: + +# Exact number of records with anomalous property cardinality. +# The companion query returns at most 100 example rows for diagnosis; this one +# reports the true severity, which a LIMITed sample cannot. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + SELECT ?record + WHERE { + ?record a vcfr:VCFRecord . + OPTIONAL { ?record vcfr:chrom ?chrom } + OPTIONAL { ?record vcfr:pos ?pos } + OPTIONAL { ?record vcfr:ref ?ref } + OPTIONAL { ?record vcfr:alt ?alt } + OPTIONAL { ?record vcfr:hasCall ?call } + } + GROUP BY ?record + HAVING(COUNT(DISTINCT ?chrom) != 1 || COUNT(DISTINCT ?pos) != 1 || COUNT(DISTINCT ?ref) != 1 || COUNT(DISTINCT ?alt) != 1 || COUNT(DISTINCT ?call) != 1) +} diff --git a/src/validation/queries/common/q07_file_metadata.rq b/src/validation/queries/common/q07_file_metadata.rq new file mode 100644 index 0000000..26155d3 --- /dev/null +++ b/src/validation/queries/common/q07_file_metadata.rq @@ -0,0 +1,17 @@ +PREFIX vcfr: + +# The file-level declarations carried over from the VCF meta-information lines. +# +# COALESCE binds an explicit sentinel rather than leaving the variable unbound, +# so a missing declaration is reported as a value mismatch against the parser +# instead of failing result normalization with a less useful error. +SELECT + (COALESCE(?ff, "(absent)") AS ?fileFormat) + (COALESCE(?rg, "(absent)") AS ?referenceGenome) + (COALESCE(?ss, "(absent)") AS ?sourceSoftware) +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:fileFormat ?ffLiteral BIND(STR(?ffLiteral) AS ?ff) } + OPTIONAL { ?file vcfr:referenceGenome ?rgLiteral BIND(STR(?rgLiteral) AS ?rg) } + OPTIONAL { ?file vcfr:sourceSoftware ?ssLiteral BIND(STR(?ssLiteral) AS ?ss) } +} diff --git a/src/validation/queries/common/q08_header_line_census.rq b/src/validation/queries/common/q08_header_line_census.rq new file mode 100644 index 0000000..4bfa223 --- /dev/null +++ b/src/validation/queries/common/q08_header_line_census.rq @@ -0,0 +1,12 @@ +PREFIX vcfr: + +# One row per '##' meta-information key, with how many lines carry it. +# Counting per key rather than in total means a dropped INFO or FILTER +# declaration is localized instead of only shifting a single number. +SELECT ?headerKey (COUNT(DISTINCT ?line) AS ?lineCount) +WHERE { + ?line a vcfr:HeaderLine ; vcfr:headerKey ?headerKeyLiteral . + BIND(STR(?headerKeyLiteral) AS ?headerKey) +} +GROUP BY ?headerKey +ORDER BY ?headerKey diff --git a/src/validation/queries/common/q09_predicate_census.rq b/src/validation/queries/common/q09_predicate_census.rq new file mode 100644 index 0000000..cd62500 --- /dev/null +++ b/src/validation/queries/common/q09_predicate_census.rq @@ -0,0 +1,15 @@ +PREFIX vcfr: + +# Every predicate in the graph and how many triples use it. +# +# This is the completeness check. Comparing it against counts derived from the +# VCF catches three things at once: a predicate that is missing entirely, one +# with the wrong cardinality, and - because an unexpected predicate has no +# expected count - one that should not be in the graph at all. Result size is +# the number of distinct predicates, so it stays small on any input. +SELECT ?predicate (COUNT(*) AS ?tripleCount) +WHERE { + ?s ?predicate ?o . +} +GROUP BY ?predicate +ORDER BY ?predicate diff --git a/src/validation/queries/common/q10_class_census.rq b/src/validation/queries/common/q10_class_census.rq new file mode 100644 index 0000000..872a642 --- /dev/null +++ b/src/validation/queries/common/q10_class_census.rq @@ -0,0 +1,11 @@ +PREFIX vcfr: + +# Every asserted rdf:type and how many resources carry it. Complements the +# predicate census: together they bound both the properties and the resources +# a conversion is allowed to produce. +SELECT ?class (COUNT(DISTINCT ?resource) AS ?resourceCount) +WHERE { + ?resource a ?class . +} +GROUP BY ?class +ORDER BY ?class diff --git a/src/validation/queries/common/q11_record_digest.rq b/src/validation/queries/common/q11_record_digest.rq new file mode 100644 index 0000000..69d686b --- /dev/null +++ b/src/validation/queries/common/q11_record_digest.rq @@ -0,0 +1,43 @@ +PREFIX vcfr: + +# Per-record identity, aggregated into a fixed-size histogram. +# +# Every other core query is a GROUP BY count, so values permuted between +# records are invisible to all of them: swap POS between two records in the +# same 1 Mb window and nothing changes. This closes that gap. +# +# The record IRI is hashed together with its values. That binding is the whole +# point: digesting the values alone would give a permutation the same multiset +# and change nothing. Bucketing on the first byte of the hash keeps the result +# at most 256 rows for any graph size, and a mismatch is localized by +# re-querying only the differing buckets. +# +# Fields are separated by U+001F (unit separator), written as a SPARQL UCHAR +# escape. It cannot appear in a VCF field, so no combination of values can be +# made to collide by shifting a field boundary. +SELECT ?bucket (COUNT(*) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; + vcfr:chrom ?chrom ; + vcfr:pos ?pos ; + vcfr:recordId ?recordIdLiteral ; + vcfr:ref ?ref ; + vcfr:alt ?alt ; + vcfr:hasCall ?call . + ?call vcfr:qual ?qual ; + vcfr:filter ?filter ; + vcfr:infoRaw ?info . + BIND(SUBSTR(SHA256(CONCAT( + STR(?record), "\u001F", + STR(?chrom), "\u001F", + STR(?pos), "\u001F", + STR(?recordIdLiteral), "\u001F", + STR(?ref), "\u001F", + STR(?alt), "\u001F", + STR(?qual), "\u001F", + STR(?filter), "\u001F", + STR(?info) + )), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/queries/common/q12_info_value_digest.rq b/src/validation/queries/common/q12_info_value_digest.rq new file mode 100644 index 0000000..48b2a4b --- /dev/null +++ b/src/validation/queries/common/q12_info_value_digest.rq @@ -0,0 +1,17 @@ +PREFIX vcfr: + +# Structured INFO values, bound to the node that identifies record and key. +# +# The census counts INFO value nodes and the record digest covers the raw INFO +# string, but neither notices a structured value that disagrees with the key it +# was parsed from. Hashing the value together with its own IRI closes that, and +# because the result is a histogram it needs no ordering guarantee from the +# engine - GROUP_CONCAT would not be portable here. +SELECT ?bucket (COUNT(*) AS ?valueCount) +WHERE { + ?value a vcfr:InfoFieldValue ; vcfr:fieldValue ?lexLiteral . + BIND(STR(?lexLiteral) AS ?lex) + BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/queries/condensed/preflight_representation_profile_count.rq b/src/validation/queries/condensed/preflight_representation_profile_count.rq new file mode 100644 index 0000000..2c90126 --- /dev/null +++ b/src/validation/queries/condensed/preflight_representation_profile_count.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +# Exact number of VCFFile resources not declaring the condensed profile. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:CondensedRepresentation) +} diff --git a/src/validation/queries/condensed/q13_format_value_digest.rq b/src/validation/queries/condensed/q13_format_value_digest.rq new file mode 100644 index 0000000..c8e4ff6 --- /dev/null +++ b/src/validation/queries/condensed/q13_format_value_digest.rq @@ -0,0 +1,15 @@ +PREFIX vcfr: + +# The condensed equivalent of the expanded per-cell FORMAT digest. +# +# Condensed graphs store one sample-ordered vector per record and FORMAT key, +# so the vector IRI plus its encoded values carries the same information as the +# per-cell nodes of the expanded shape. +SELECT ?bucket (COUNT(*) AS ?valueCount) +WHERE { + ?value a vcfr:FormatValueVector ; vcfr:encodedValues ?lexLiteral . + BIND(STR(?lexLiteral) AS ?lex) + BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/queries/expanded/preflight_representation_profile_count.rq b/src/validation/queries/expanded/preflight_representation_profile_count.rq new file mode 100644 index 0000000..9174a06 --- /dev/null +++ b/src/validation/queries/expanded/preflight_representation_profile_count.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +# Exact number of VCFFile resources not declaring the expanded profile. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:ExpandedRepresentation) +} diff --git a/src/validation/queries/expanded/q13_format_value_digest.rq b/src/validation/queries/expanded/q13_format_value_digest.rq new file mode 100644 index 0000000..647abed --- /dev/null +++ b/src/validation/queries/expanded/q13_format_value_digest.rq @@ -0,0 +1,15 @@ +PREFIX vcfr: + +# Every FORMAT value, bound to the node that identifies record, sample and key. +# +# Only GT is compared by q05/q06, so a mangled DP, GQ or AD would otherwise +# pass. The FormatFieldValue IRI already encodes record, sample and key, so +# hashing it with the value makes each cell individually accountable. +SELECT ?bucket (COUNT(*) AS ?valueCount) +WHERE { + ?value a vcfr:FormatFieldValue ; vcfr:fieldValue ?lexLiteral . + BIND(STR(?lexLiteral) AS ?lex) + BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/validation_runner.py b/src/validation/validation_runner.py index 4287240..427f591 100644 --- a/src/validation/validation_runner.py +++ b/src/validation/validation_runner.py @@ -1,10 +1,24 @@ #!/usr/bin/env python3 -"""Validate a VCF-RDFizer N-Triples graph against its source VCF. - -When the input is compressed, it is expanded only to a container-local -temporary file. It is parsed with Raptor, queried with Comunica, and removed in -a finally-safe temporary directory before this process exits. Plain ``.nt`` -inputs are read directly and are never copied by the validator. +"""Validate a VCF-RDFizer RDF graph against its source VCF. + +The validator computes six deterministic summaries from the VCF with cyvcf2 +(and bcftools for exact FILTER strings), runs the equivalent SPARQL queries +against the graph, and compares canonical integer results exactly. + +Two axes are independent, and both are recorded in every report: + +**Artifact format** - ``.nt``, ``.nt.gz``, ``.nt.br``, ``.hdt``, ``.cottas``, +``.cottas.gz``, ``.cottas.br``. Anything that is not already plain N-Triples is +decoded into the container scratch directory first, so validating an ``.hdt`` +proves it decodes to a graph that still satisfies every semantic check - a +strictly stronger statement than the triple-count round-trip that +``validate_compression.py`` performs during compression. A plain ``.nt`` source +is read in place and never copied. Nothing decoded is written beneath ``--out``. + +**SPARQL engine** - ``comunica`` (default) queries the file with no setup but +holds the graph in memory; ``qlever`` builds an on-disk index and answers over +a container-local HTTP server, which is what makes cohort-scale graphs +queryable. Both answer identical queries, so the choice is never semantic. """ from __future__ import annotations @@ -16,18 +30,28 @@ import os import platform import re +import shlex import shutil import subprocess import sys import tempfile import time +import urllib.parse from collections import Counter from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any +from urllib.parse import quote_plus -from cyvcf2 import VCF -import cyvcf2 +# cyvcf2 only exists inside the image. Importing it lazily keeps the pure +# normalization/comparison layer below importable on the host, which is what +# the mutation tests in test/test_validation_logic_unit.py exercise. +try: # pragma: no cover - present in the container, absent on the host + import cyvcf2 + from cyvcf2 import VCF +except ImportError: # pragma: no cover + cyvcf2 = None + VCF = None SCRIPT_DIR = Path(__file__).resolve().parent @@ -39,14 +63,41 @@ "q04_filter_distribution", "q05_sample_genotype_counts", "q06_ac_an_distribution", + "q07_file_metadata", + "q08_header_line_census", + "q09_predicate_census", + "q10_class_census", + "q11_record_digest", + "q12_info_value_digest", + "q13_format_value_digest", ) +#: Queries that must return exactly one row, normalized to a dict rather than +#: a list so the comparison reads as a field-by-field check. +SINGLE_ROW_QUERIES = frozenset({"q03_titv", "q07_file_metadata"}) PREFLIGHT_QUERIES = ( "preflight_record_cardinality", "preflight_position_datatype", "preflight_missing_token_conformance", "preflight_representation_profile", "preflight_sample_gt_inventory", + "preflight_blank_nodes", + "preflight_empty_values", + "preflight_distinct_triple_count", +) +#: Anomaly-style preflights return at most 100 example rows so a broken graph +#: cannot produce an unbounded report. That makes the sample useless as a +#: severity measure, so each one is paired with an aggregate that returns the +#: exact count. Both are reported: the count says how bad, the sample says how. +ANOMALY_PREFLIGHT_QUERIES = ( + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_missing_token_conformance", + "preflight_representation_profile", + "preflight_blank_nodes", + "preflight_empty_values", ) +PREFLIGHT_COUNT_QUERIES = tuple(f"{name}_count" for name in ANOMALY_PREFLIGHT_QUERIES) +ANOMALY_SAMPLE_LIMIT = 100 TRANSITIONS = {("A", "G"), ("G", "A"), ("C", "T"), ("T", "C")} QUERY_SPECS = { @@ -56,6 +107,13 @@ "q04_filter_distribution": (("filterStatus", "filterLexical"), ("recordCount",)), "q05_sample_genotype_counts": (("sampleId", "genotypeClass"), ("callCount",)), "q06_ac_an_distribution": (("an", "ac"), ("siteCount",)), + "q07_file_metadata": ((), ("fileFormat", "referenceGenome", "sourceSoftware")), + "q08_header_line_census": (("headerKey",), ("lineCount",)), + "q09_predicate_census": (("predicate",), ("tripleCount",)), + "q10_class_census": (("class",), ("resourceCount",)), + "q11_record_digest": (("bucket",), ("recordCount",)), + "q12_info_value_digest": (("bucket",), ("valueCount",)), + "q13_format_value_digest": (("bucket",), ("valueCount",)), } QUERY_SCHEMAS = { "q01_record_density_1mb": (("chrom", "windowIndex", "recordCount"), {"windowIndex", "recordCount"}, ("chrom", "windowIndex")), @@ -64,7 +122,25 @@ "q04_filter_distribution": (("filterStatus", "filterLexical", "recordCount"), {"recordCount"}, ("filterStatus", "filterLexical")), "q05_sample_genotype_counts": (("sampleId", "genotypeClass", "callCount"), {"callCount"}, ("sampleId", "genotypeClass")), "q06_ac_an_distribution": (("an", "ac", "siteCount"), {"an", "ac", "siteCount"}, ("an", "ac")), + "q07_file_metadata": (("fileFormat", "referenceGenome", "sourceSoftware"), set(), ()), + "q08_header_line_census": (("headerKey", "lineCount"), {"lineCount"}, ("headerKey",)), + "q09_predicate_census": (("predicate", "tripleCount"), {"tripleCount"}, ("predicate",)), + "q10_class_census": (("class", "resourceCount"), {"resourceCount"}, ("class",)), + "q11_record_digest": (("bucket", "recordCount"), {"recordCount"}, ("bucket",)), + "q12_info_value_digest": (("bucket", "valueCount"), {"valueCount"}, ("bucket",)), + "q13_format_value_digest": (("bucket", "valueCount"), {"valueCount"}, ("bucket",)), } +#: Checks that are only meaningful for the shipped RML mapping: they assume its +#: predicate inventory and its IRI templates. A custom mapping changes both by +#: design, so the wrapper switches these to report-only in that case. +DEFAULT_MAPPING_QUERIES = ( + "q09_predicate_census", + "q10_class_census", + "q11_record_digest", + "q12_info_value_digest", + "q13_format_value_digest", +) +MAPPING_POLICIES = ("strict", "report-only") def sha256_file(path: Path) -> str: @@ -228,7 +304,403 @@ def classify_genotype(alleles: tuple[int | None, ...] | None, *, has_gt: bool) - return "OTHER_PLOIDY" +#: VCF meta-information keys the mapping lifts onto the VCFFile resource, in +#: the case-insensitive form `vcf_as_tsv.sh` matches them. +FILE_METADATA_KEYS = { + "fileformat": "fileFormat", + "reference": "referenceGenome", + "source": "sourceSoftware", + "filedate": "fileDate", +} +METADATA_ABSENT = "(absent)" + + +# --------------------------------------------------------------------------- +# Graph census expectations +# --------------------------------------------------------------------------- +# The census compares the graph's predicate and class inventory against counts +# derived from the VCF. It is the completeness check: a predicate missing, one +# with the wrong cardinality, and one that should not exist at all are all the +# same comparison. +# +# Expectations are derived from the VCF and the emitters' documented shapes - +# never from default_rules.ttl. Deriving them from the mapping would make the +# mapping test itself, which is exactly how QUAL stayed invisible for so long. +# A custom mapping therefore invalidates them, and the wrapper switches the +# policy to report-only in that case. + +VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" + + +def _nonzero(counts: dict[str, int]) -> dict[str, int]: + return {key: value for key, value in counts.items() if value} + + +#: Field separator for the record digest. U+001F cannot appear in a VCF field, +#: so no shift of a field boundary can make two different records collide. The +#: query in q11_record_digest.rq writes it as a SPARQL \\u001F escape. +def rml_uri_component(value: str) -> str: + """Encode a template value the way RMLStreamer 2.5.0 does. + + Mirrors ``_rml_uri_component`` in ``vcf_rdfizer.py``; that module is not on + the container's import path, and ``test_validation_logic_unit.py`` asserts + the two agree so they cannot drift. + """ + encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") + # urllib leaves '~' unescaped; java.net.URLEncoder does not. + return encoded.replace("+", "%20").replace("~", "%7E") + + +DIGEST_SEPARATOR = "\u001f" +#: Hash prefix length in hex characters. Two gives 256 buckets: enough to make +#: an accidental collision of a *changed* record with its own bucket unlikely +#: to hide anything, while keeping the result fixed-size for any graph. +DIGEST_BUCKET_CHARS = 2 + + +def record_digest_bucket(fields: list[str]) -> str: + """Bucket one record exactly as q11_record_digest.rq does.""" + joined = DIGEST_SEPARATOR.join(fields) + return hashlib.sha256(joined.encode("utf-8")).hexdigest()[:DIGEST_BUCKET_CHARS] + + +def expected_census( + parser: dict[str, Any], representation: str, *, info_representation: str = "structured", + header_representation: str = "structured", +) -> dict[str, list[dict[str, Any]]]: + """Predicate and class counts the graph must contain, and nothing else.""" + records = parser["totalRecords"] + samples = parser["sampleCount"] + header_lines = parser["headerLineCount"] + + classes: dict[str, int] = { + f"{VCFR}VCFFile": 1, + f"{VCFR}VCFHeader": 1, + f"{VCFR}HeaderLine": header_lines, + f"{VCFR}VCFRecord": records, + f"{VCFR}VariantCall": records, + } + predicates: dict[str, int] = { + f"{VCFR}hasHeader": 1, + f"{VCFR}hasHeaderLine": header_lines, + f"{VCFR}headerKey": header_lines, + f"{VCFR}headerValue": parser["headerValueCount"], + f"{VCFR}hasRecord": records, + f"{VCFR}chrom": records, + f"{VCFR}pos": records, + f"{VCFR}recordId": records, + f"{VCFR}ref": records, + f"{VCFR}alt": records, + f"{VCFR}hasCall": records, + f"{VCFR}qual": records, + f"{VCFR}filter": records, + f"{VCFR}infoRaw": records, + f"{VCFR}formatRaw": parser["recordsWithFormatColumn"], + } + # RMLStreamer emits nothing for an absent value, so an undeclared + # meta-information line contributes no triple rather than an empty one. + for field, predicate in ( + ("fileFormat", f"{VCFR}fileFormat"), + ("referenceGenome", f"{VCFR}referenceGenome"), + ("sourceSoftware", f"{VCFR}sourceSoftware"), + ): + predicates[predicate] = 0 if parser[field] == METADATA_ABSENT else 1 + + if samples: + predicates[f"{VCFR}representationProfile"] = 1 + if representation == "expanded": + classes[f"{VCFR}SampleCall"] = records * samples + classes[f"{VCFR}FormatFieldValue"] = parser["formatValueSlots"] + predicates[f"{VCFR}hasSampleCall"] = records * samples + predicates[f"{VCFR}sampleId"] = records * samples + predicates[f"{VCFR}hasFormatValue"] = parser["formatValueSlots"] + predicates[f"{VCFR}fieldValue"] = parser["nonEmptyFormatValues"] + else: + definitions = parser["distinctFormatKeyCount"] + classes[f"{VCFR}SampleSet"] = 1 + classes[f"{VCFR}VCFSample"] = samples + classes[f"{VCFR}CohortCallMatrix"] = parser["recordsWithFormatKeys"] + classes[f"{VCFR}FormatValueVector"] = parser["formatKeyOccurrences"] + classes[f"{VCFR}FormatFieldDefinition"] = definitions + predicates[f"{VCFR}hasSampleSet"] = 1 + predicates[f"{VCFR}hasSample"] = samples + predicates[f"{VCFR}sampleName"] = samples + predicates[f"{VCFR}sampleIndex"] = samples + predicates[f"{VCFR}hasCallMatrix"] = parser["recordsWithFormatKeys"] + predicates[f"{VCFR}appliesToSampleSet"] = parser["recordsWithFormatKeys"] + predicates[f"{VCFR}hasFormatValueVector"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}declaredBy"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}valueEncoding"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}encodedValues"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}fieldId"] = definitions + predicates[f"{VCFR}fieldNumber"] = definitions + predicates[f"{VCFR}fieldDescription"] = definitions + + predicates[f"{VCFR}fileDate"] = 0 if parser["fileDate"] == METADATA_ABSENT else 1 + + if header_representation == "structured": + for class_name, count in parser["headerLineClassCounts"].items(): + classes[f"{VCFR}{class_name}"] = classes.get(f"{VCFR}{class_name}", 0) + count + # A FILTER/ALT line carries both its header-line subclass and its + # definition class, so it contributes two rdf:type triples. + classes[f"{VCFR}FilterDefinition"] = parser["filterDefinitionCount"] + classes[f"{VCFR}AltDefinition"] = parser["altDefinitionCount"] + predicates[f"{VCFR}filterId"] = parser["filterDefinitionCount"] + predicates[f"{VCFR}altId"] = parser["altDefinitionCount"] + predicates[f"{VCFR}contigId"] = parser["contigCount"] + predicates[f"{VCFR}contigCount"] = 1 if parser["contigCount"] else 0 + for attribute, predicate in ( + ("length", "contigLength"), ("md5", "contigMd5"), ("assembly", "contigAssembly"), + ): + predicates[f"{VCFR}{predicate}"] = parser["contigAttributeCounts"].get(attribute, 0) + predicates[f"{VCFR}fieldDescription"] = ( + predicates.get(f"{VCFR}fieldDescription", 0) + + parser["describedDefinitionCount"] + ) + + if info_representation == "structured": + values = parser["infoValueCount"] + definitions = parser["infoDefinitionCount"] + classes[f"{VCFR}InfoFieldValue"] = values + classes[f"{VCFR}InfoFieldDefinition"] = definitions + predicates[f"{VCFR}hasInfoValue"] = values + predicates[f"{VCFR}fieldValueBoolean"] = parser["infoFlagCount"] + predicates[f"{VCFR}fieldValueInteger"] = parser["infoTypedIntegerCount"] + predicates[f"{VCFR}fieldValueDecimal"] = parser["infoTypedDecimalCount"] + predicates[f"{VCFR}fieldType"] = definitions + # declaredBy and the field* descriptors are shared with the condensed + # FORMAT definitions, so accumulate rather than overwrite. + for predicate, count in ( + (f"{VCFR}declaredBy", values), + (f"{VCFR}fieldId", definitions), + (f"{VCFR}fieldNumber", definitions), + (f"{VCFR}fieldDescription", definitions), + ): + predicates[predicate] = predicates.get(predicate, 0) + count + predicates[f"{VCFR}fieldValue"] = ( + predicates.get(f"{VCFR}fieldValue", 0) + values - parser["infoFlagCount"] + ) + + classes = _nonzero(classes) + predicates = _nonzero(predicates) + # Every typed resource contributes exactly one rdf:type triple here. + predicates[RDF_TYPE] = sum(classes.values()) + + return { + "q09_predicate_census": [ + {"predicate": iri, "tripleCount": count} + for iri, count in sorted(predicates.items()) + ], + "q10_class_census": [ + {"class": iri, "resourceCount": count} for iri, count in sorted(classes.items()) + ], + } + + +def sample_uri_ids(sample_ids: list[str]) -> list[str]: + """Derive each sample's IRI component the way the emitters do. + + Mirrors ``_sample_id_to_uri_id`` in ``vcf_rdfizer.py``, including the + occurrence suffix that keeps two samples whose ids sanitize to the same + string distinguishable. + """ + counts: dict[str, int] = {} + out: list[str] = [] + for index, sample_id in enumerate(sample_ids, start=1): + base = re.sub(r"[^A-Za-z0-9._~-]+", "_", sample_id).strip("_") or f"sample_{index}" + counts[base] = counts.get(base, 0) + 1 + out.append(f"{base}_{counts[base]}" if counts[base] > 1 else base) + return out + + +#: '##' key (lower-cased) -> vocabulary subclass. Mirrors HEADER_LINE_CLASSES +#: in vcf_rdfizer.py; the unit tests assert the two stay identical. +HEADER_LINE_CLASSES = { + "fileformat": "FileFormatHeaderLine", + "filedate": "FileDateHeaderLine", + "source": "SourceHeaderLine", + "reference": "ReferenceHeaderLine", + "info": "INFOHeaderLine", + "format": "FORMATHeaderLine", + "filter": "FILTERHeaderLine", + "alt": "ALTHeaderLine", + "contig": "ContigHeaderLine", +} + + +def parse_structured_header_fields(value: str) -> dict[str, str]: + """Parse '' attributes, respecting quoting. + + Mirrors ``_parse_structured_header_fields`` in ``vcf_rdfizer.py``; the unit + tests assert the two agree. + """ + inner = value.strip() + if inner.startswith("<") and inner.endswith(">"): + inner = inner[1:-1] + tokens: list[str] = [] + token: list[str] = [] + in_quotes = escaped = False + for character in inner: + if escaped: + token.append(character) + escaped = False + elif character == "\\" and in_quotes: + token.append(character) + escaped = True + elif character == '"': + token.append(character) + in_quotes = not in_quotes + elif character == "," and not in_quotes: + tokens.append("".join(token)) + token = [] + else: + token.append(character) + tokens.append("".join(token)) + fields: dict[str, str] = {} + for item in tokens: + if "=" not in item: + continue + key, raw_value = item.split("=", 1) + parsed = raw_value.strip() + if len(parsed) >= 2 and parsed[0] == parsed[-1] == '"': + parsed = parsed[1:-1].replace('\\"', '"').replace("\\\\", "\\") + fields[key.strip()] = parsed + return fields + + +def parse_info_entries(info: str) -> list[tuple[str, str | None]]: + """Split an INFO column into ``(key, value)`` pairs; a bare key is a Flag. + + Mirrors ``parse_info_entries`` in ``vcf_rdfizer.py``. The two are asserted + to agree in ``test_validation_logic_unit.py``. + """ + if info in ("", "."): + return [] + entries: list[tuple[str, str | None]] = [] + for item in info.split(";"): + if not item: + continue + key, separator, value = item.partition("=") + entries.append((key, value if separator else None)) + return entries + + +def info_declared_types(raw_header: str) -> dict[str, str]: + """Map each declared INFO key to its VCF Type, for typed-value counting.""" + types: dict[str, str] = {} + for line in raw_header.splitlines(): + if not line.startswith("##INFO="): + continue + body = line[len("##INFO=") :].strip() + if body.startswith("<") and body.endswith(">"): + body = body[1:-1] + fields: dict[str, str] = {} + for match in re.finditer(r'(\w+)=("(?:[^"\\]|\\.)*"|[^,]*)', body): + value = match.group(2) + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + fields[match.group(1)] = value + key = fields.get("ID", "").strip() + if key: + types.setdefault(key, fields.get("Type") or "String") + return types + + +def parse_header_metadata(raw_header: str) -> dict[str, Any]: + """Summarise the '##' meta-information block of a VCF. + + The `#CHROM` line is deliberately excluded: it is not a meta-information + line and the mapping emits no HeaderLine resource for it, so counting it + here would guarantee a spurious mismatch. + """ + metadata: dict[str, Any] = {value: METADATA_ABSENT for value in FILE_METADATA_KEYS.values()} + keys: Counter[str] = Counter() + for line in raw_header.splitlines(): + if not line.startswith("##"): + continue + body = line[2:] + key, _, value = body.partition("=") + keys[key] += 1 + field = FILE_METADATA_KEYS.get(key.lower()) + # A repeated declaration keeps the first, matching the awk parser. + if field is not None and metadata[field] == METADATA_ABSENT: + metadata[field] = value + # Header-representation counters. The class map mirrors HEADER_LINE_CLASSES + # in vcf_rdfizer.py; the two are asserted to agree in the unit tests. + header_classes: Counter[str] = Counter() + filter_definitions = alt_definitions = contigs = described = 0 + contig_attributes: Counter[str] = Counter() + for line in raw_header.splitlines(): + if not line.startswith("##"): + continue + key, _, value = line[2:].partition("=") + class_name = HEADER_LINE_CLASSES.get(key.lower()) + if class_name is None: + continue + header_classes[class_name] += 1 + if key.lower() not in {"filter", "alt", "contig"}: + continue + fields = parse_structured_header_fields(value) + if not (fields.get("ID") or "").strip(): + continue + if key.lower() == "contig": + contigs += 1 + for attribute in ("length", "md5", "assembly"): + if fields.get(attribute): + contig_attributes[attribute] += 1 + continue + if key.lower() == "filter": + filter_definitions += 1 + else: + alt_definitions += 1 + if fields.get("Description"): + described += 1 + metadata["headerLineClassCounts"] = dict(header_classes) + metadata["filterDefinitionCount"] = filter_definitions + metadata["altDefinitionCount"] = alt_definitions + metadata["contigCount"] = contigs + metadata["contigAttributeCounts"] = dict(contig_attributes) + metadata["describedDefinitionCount"] = described + metadata["headerLineCount"] = sum(keys.values()) + metadata["headerValueCount"] = sum( + 1 for line in raw_header.splitlines() + if line.startswith("##") and line[2:].partition("=")[2] != "" + ) + metadata["q08_header_line_census"] = [ + {"headerKey": key, "lineCount": int(count)} for key, count in sorted(keys.items()) + ] + return metadata + + +def attach_census_expectations( + parser: dict[str, Any], representation: str, *, info_representation: str = "structured", + header_representation: str = "structured", +) -> dict[str, Any]: + """Add the expected predicate/class inventory to a parser summary. + + Kept separate from ``parse_vcf`` because the expectation depends on the + representation, which is a validation-run choice rather than a property of + the VCF. + """ + parser.update(expected_census( + parser, representation, info_representation=info_representation, + header_representation=header_representation, + )) + key = ( + "_expandedFormatValueDigest" if representation == "expanded" + else "_condensedFormatValueDigest" + ) + parser["q13_format_value_digest"] = parser.get(key, []) + return parser + + def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: + if VCF is None: + raise RuntimeError( + "cyvcf2 is unavailable; the validator must run inside the " + "VCF-RDFizer container image" + ) use_bcftools = filter_oracle == "bcftools" or ( filter_oracle == "auto" and shutil.which("bcftools") is not None ) @@ -237,12 +709,27 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: filters = filters_with_bcftools(vcf_path) if use_bcftools else Counter() reader = VCF(str(vcf_path), strict_gt=True) samples = list(reader.samples) + header_metadata = parse_header_metadata(reader.raw_header) density: Counter[tuple[str, int]] = Counter() shapes: Counter[str] = Counter() genotypes: Counter[tuple[str, str]] = Counter() ac_an: Counter[tuple[int, int]] = Counter() total_records = gt_records = single_alt_records = q06_eligible = 0 transition_count = transversion_count = biallelic_snv_count = 0 + # Census inputs. These mirror SampleRecordStream._parse_row, which widens + # FORMAT to the longest sample payload when a record drops trailing fields. + digest_buckets: Counter[str] = Counter() + info_value_digest: Counter[str] = Counter() + expanded_format_digest: Counter[str] = Counter() + condensed_format_digest: Counter[str] = Counter() + sample_components = [rml_uri_component(uri_id) for uri_id in sample_uri_ids(samples)] + declared_info_types = info_declared_types(reader.raw_header) + info_definitions: set[str] = set() + info_values = info_flags = info_typed_integers = info_typed_decimals = 0 + source_component = rml_uri_component(vcf_path.name) + records_with_format_column = records_with_format_keys = 0 + format_key_occurrences = format_value_slots = non_empty_format_values = 0 + distinct_format_keys: set[str] = set() try: for variant in reader: total_records += 1 @@ -269,6 +756,98 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: has_gt = "GT" in format_keys if has_gt: gt_records += 1 + + # `vcfr:formatRaw` comes from the FORMAT column itself, so it is + # present whenever that column is non-empty - samples or not. + if format_keys: + records_with_format_column += 1 + + columns = str(variant).rstrip("\r\n").split("\t") + # The record digest is computed from the raw line so it matches the + # lexical values the mapping puts in the graph, character for + # character, with no round-trip through cyvcf2's typed accessors. + record_iri = ( + f"file://{source_component}#record/" + f"{rml_uri_component(str(total_records))}" + ) + digest_buckets[record_digest_bucket([record_iri, *columns[:8]])] += 1 + + # Structured INFO counts. The typed-value rules mirror + # `_typed_info_object` in vcf_rdfizer.py: only a single-valued + # Integer/Float that actually parses gains a typed predicate. + row_component = rml_uri_component(str(total_records)) + for key, value in parse_info_entries(columns[7] if len(columns) > 7 else ""): + info_values += 1 + info_definitions.add(key) + if value is None: + info_flags += 1 + continue + info_iri = ( + f"file://{source_component}#call/{row_component}" + f"/info/{rml_uri_component(key)}" + ) + info_value_digest[record_digest_bucket([info_iri, value])] += 1 + declared = declared_info_types.get(key, "String") + if "," in value or value == ".": + continue + try: + if declared == "Integer": + int(value) + info_typed_integers += 1 + elif declared == "Float": + float(value) + info_typed_decimals += 1 + except ValueError: + pass + payload_fields = [ + payload.split(":") if payload else [] for payload in columns[9:] + ] + # SampleRecordStream widens FORMAT to the longest sample payload and + # names any surplus field FIELD_, so the emitted key set can be + # wider than the declared one. Mirror that exactly. + width = max([len(format_keys), *(len(fields) for fields in payload_fields)], default=0) + widened_keys = [ + format_keys[index] if index < len(format_keys) and format_keys[index] + else f"FIELD_{index + 1}" + for index in range(width) + ] + if samples and widened_keys: + records_with_format_keys += 1 + distinct_format_keys.update(widened_keys) + format_key_occurrences += width + format_value_slots += width * len(samples) + non_empty_format_values += sum( + 1 + for fields in payload_fields + for index in range(width) + if index < len(fields) and fields[index] != "" + ) + # Both representations are accumulated: only one is compared, + # chosen by the run's representation, but computing both keeps + # this loop single-pass. + call_iri = f"file://{source_component}#call/{row_component}" + for key_index, key in enumerate(widened_keys): + key_component = rml_uri_component(key) + for sample_index, fields in enumerate(payload_fields): + cell = fields[key_index] if key_index < len(fields) else "" + if not cell: + continue + value_iri = ( + f"file://{source_component}#sample/{row_component}" + f"/{sample_components[sample_index]}/fmt/{key_component}" + ) + expanded_format_digest[ + record_digest_bucket([value_iri, cell]) + ] += 1 + encoded = "\t".join( + (fields[key_index] if key_index < len(fields) and fields[key_index] + else ".") + for fields in payload_fields + ) + vector_iri = f"{call_iri}/matrix/fmt/{key_component}" + condensed_format_digest[ + record_digest_bucket([vector_iri, encoded]) + ] += 1 alleles = [None] * len(samples) if samples and has_gt: raw_genotypes = list(variant.genotypes) @@ -304,6 +883,51 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: "source": str(vcf_path), "sourceSha256": sha256_file(vcf_path), "filterOracle": "bcftools" if use_bcftools else "cyvcf2-serialization", + "headerLineCount": header_metadata["headerLineCount"], + "fileDate": header_metadata["fileDate"], + "headerLineClassCounts": header_metadata["headerLineClassCounts"], + "filterDefinitionCount": header_metadata["filterDefinitionCount"], + "altDefinitionCount": header_metadata["altDefinitionCount"], + "contigCount": header_metadata["contigCount"], + "contigAttributeCounts": header_metadata["contigAttributeCounts"], + "describedDefinitionCount": header_metadata["describedDefinitionCount"], + "headerValueCount": header_metadata["headerValueCount"], + "recordsWithFormatColumn": records_with_format_column, + "recordsWithFormatKeys": records_with_format_keys, + "formatKeyOccurrences": format_key_occurrences, + "formatValueSlots": format_value_slots, + "nonEmptyFormatValues": non_empty_format_values, + "distinctFormatKeyCount": len(distinct_format_keys), + "q12_info_value_digest": [ + {"bucket": bucket, "valueCount": int(count)} + for bucket, count in sorted(info_value_digest.items()) + ], + "_expandedFormatValueDigest": [ + {"bucket": bucket, "valueCount": int(count)} + for bucket, count in sorted(expanded_format_digest.items()) + ], + "_condensedFormatValueDigest": [ + {"bucket": bucket, "valueCount": int(count)} + for bucket, count in sorted(condensed_format_digest.items()) + ], + "infoValueCount": info_values, + "infoDefinitionCount": len(info_definitions), + "infoFlagCount": info_flags, + "infoTypedIntegerCount": info_typed_integers, + "infoTypedDecimalCount": info_typed_decimals, + "q11_record_digest": [ + {"bucket": bucket, "recordCount": int(count)} + for bucket, count in sorted(digest_buckets.items()) + ], + "fileFormat": header_metadata["fileFormat"], + "referenceGenome": header_metadata["referenceGenome"], + "sourceSoftware": header_metadata["sourceSoftware"], + "q07_file_metadata": { + "fileFormat": header_metadata["fileFormat"], + "referenceGenome": header_metadata["referenceGenome"], + "sourceSoftware": header_metadata["sourceSoftware"], + }, + "q08_header_line_census": header_metadata["q08_header_line_census"], "sampleCount": len(samples), "samples": samples, "totalRecords": total_records, @@ -335,6 +959,217 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# Source artifact materialization +# --------------------------------------------------------------------------- +# Semantic validation always runs against N-Triples. A compressed or indexed +# artifact is therefore decoded into the container-local scratch directory +# first, which is what makes "validate the HDT" mean "prove the HDT decodes to +# a graph that still satisfies every check", rather than only that it decodes +# to the right number of triples (which validate_compression.py already does). +# +# Decoding needs scratch space of roughly the uncompressed graph size. The +# scratch directory is removed before the process exits in all cases. + +RDF_FORMATS = ("nt", "nt.gz", "nt.br", "hdt", "cottas", "cottas.gz", "cottas.br") +#: Formats that are read in place, with nothing written to scratch. +DIRECT_FORMATS = frozenset({"nt"}) +FORMAT_SUFFIXES = ( + (".nt.gz", "nt.gz"), + (".nt.br", "nt.br"), + (".nt", "nt"), + (".cottas.gz", "cottas.gz"), + (".cottas.br", "cottas.br"), + (".cottas", "cottas"), + (".hdt", "hdt"), +) +COTTAS_TOOL = Path("/opt/vcf-rdfizer/cottas_tool.py") + + +def detect_rdf_format(path: Path) -> str | None: + """Infer the artifact format from a filename, or None when unrecognised.""" + name = path.name + for suffix, fmt in FORMAT_SUFFIXES: + if name.endswith(suffix): + return fmt + return None + + +def _resolve_binary(env_var: str, *candidates: str) -> str: + """Find a container binary the same way the compression stages do.""" + override = os.environ.get(env_var, "").strip() + if override and (Path(override).is_file() or shutil.which(override)): + return override + for candidate in candidates: + found = shutil.which(candidate) or (candidate if Path(candidate).is_file() else None) + if found: + return found + raise RuntimeError(f"Required binary not found in container: {candidates[0]}") + + +def _run_step( + command: list[str], *, label: str, log_dir: Path, env: dict[str, str] | None = None +) -> None: + """Run one decode step, preserving its output for diagnosis on failure.""" + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / f"{label}.log" + with log_path.open("wb") as log: + result = subprocess.run( + command, check=False, stdout=log, stderr=subprocess.STDOUT, env=env + ) + if result.returncode != 0: + tail = "" + try: + tail = log_path.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + raise RuntimeError( + f"{label} failed with exit code {result.returncode}. Output tail: {tail}" + ) + + +def materialize_ntriples( + source: Path, + rdf_format: str, + scratch: Path, + *, + log_dir: Path, +) -> tuple[Path, dict[str, Any]]: + """Return ``(ntriples_path, provenance)`` for any supported artifact. + + A plain ``.nt`` source is used in place and never copied. + """ + provenance: dict[str, Any] = { + "sourceFormat": rdf_format, + "materialized": rdf_format not in DIRECT_FORMATS, + "steps": [], + } + if rdf_format == "nt": + return source, provenance + + target = scratch / "input.nt" + if rdf_format == "nt.gz": + with gzip.open(source, "rb") as handle, target.open("wb") as out: + shutil.copyfileobj(handle, out, length=1024 * 1024) + provenance["steps"].append({"tool": "python-gzip", "output": str(target)}) + return target, provenance + + if rdf_format == "nt.br": + brotli = _resolve_binary("BROTLI_BIN", "brotli") + _run_step([brotli, "-d", "-c", "-o", str(target), str(source)], + label="brotli-decompress", log_dir=log_dir) + provenance["steps"].append({"tool": brotli, "output": str(target)}) + return target, provenance + + if rdf_format == "hdt": + hdt2rdf = _resolve_binary("HDT2RDF_BIN", "hdt2rdf", "/usr/local/bin/hdt2rdf") + _run_step([hdt2rdf, str(source), str(target)], label="hdt2rdf", log_dir=log_dir) + provenance["steps"].append({"tool": hdt2rdf, "output": str(target)}) + return target, provenance + + if rdf_format in {"cottas", "cottas.gz", "cottas.br"}: + cottas_input = source + if rdf_format != "cottas": + # pycottas needs a seekable Parquet file, so a packaged artifact is + # unwrapped into scratch before it is decoded. + cottas_input = scratch / "input.cottas" + if rdf_format == "cottas.gz": + with gzip.open(source, "rb") as handle, cottas_input.open("wb") as out: + shutil.copyfileobj(handle, out, length=1024 * 1024) + provenance["steps"].append({"tool": "python-gzip", "output": str(cottas_input)}) + else: + brotli = _resolve_binary("BROTLI_BIN", "brotli") + _run_step([brotli, "-d", "-c", "-o", str(cottas_input), str(source)], + label="brotli-unwrap", log_dir=log_dir) + provenance["steps"].append({"tool": brotli, "output": str(cottas_input)}) + python_bin = os.environ.get("COTTAS_PYTHON_BIN") or sys.executable + _run_step( + [python_bin, str(COTTAS_TOOL), "decompress", str(cottas_input), str(target)], + label="cottas-decompress", + log_dir=log_dir, + ) + provenance["steps"].append({"tool": f"{python_bin} cottas_tool.py decompress", + "output": str(target)}) + if cottas_input != source: + cottas_input.unlink(missing_ok=True) + return target, provenance + + raise ValueError(f"Unsupported RDF artifact format: {rdf_format}") + + +#: Violations retained in the report. A badly broken graph can produce one per +#: record, so the full text is never inlined into a JSON report. +SHACL_SAMPLE_LIMIT = 50 + + +def validate_shacl(source: Path, shapes: Path, results_dir: Path) -> dict[str, Any]: + """Validate the graph against SHACL shapes, if pyshacl is available. + + This is an independent structural layer: it checks the shapes the + vocabulary publishes, rather than comparing counts against the VCF, so it + catches a different class of defect from everything else here. + + pyshacl loads the graph into memory, so this is opt-in and unsuitable for a + cohort-scale aggregate. It is reported as EXECUTION_FAILED rather than a + conformance failure when the tool is missing, so an absent optional + dependency can never look like a bad graph. + """ + report_path = results_dir / "shacl.json" + try: + from pyshacl import validate as pyshacl_validate + except ImportError as error: + result = { + "status": "EXECUTION_FAILED", + "error": f"pyshacl is not installed in this image: {error}", + "shapes": str(shapes), + } + write_json(report_path, result) + return result + + started = time.monotonic() + try: + conforms, _graph, text = pyshacl_validate( + str(source), + shacl_graph=str(shapes), + data_graph_format="nt", + shacl_graph_format="turtle", + inference="none", + advanced=False, + ) + except Exception as error: # noqa: BLE001 - reported, never fatal here + result = { + "status": "EXECUTION_FAILED", + "error": f"SHACL validation could not run: {error}", + "shapes": str(shapes), + } + write_json(report_path, result) + return result + + violations = [ + line.strip() for line in text.splitlines() + if line.strip().startswith("Constraint Violation") + ] + paths = sorted({ + line.split("Result Path:", 1)[1].strip() + for line in text.splitlines() if "Result Path:" in line + }) + log_path = results_dir / "shacl-report.txt" + log_path.write_text(text, encoding="utf-8") + result = { + "status": "PASS" if conforms else "FAIL", + "conforms": bool(conforms), + "shapes": str(shapes), + "violationCount": len(violations), + "violationPaths": paths, + "report": str(log_path), + "wallSeconds": time.monotonic() - started, + "sampleLimitedTo": SHACL_SAMPLE_LIMIT, + "sample": violations[:SHACL_SAMPLE_LIMIT], + } + write_json(report_path, result) + return result + + def validate_ntriples(source: Path, results_dir: Path) -> dict[str, Any]: rapper = shutil.which("rapper") if not rapper: @@ -348,33 +1183,401 @@ def validate_ntriples(source: Path, results_dir: Path) -> dict[str, Any]: ) log = results_dir / "rdf-validation" / "rapper.txt" log.parent.mkdir(parents=True, exist_ok=True) - log.write_text(result.stdout + result.stderr, encoding="utf-8") - return {"status": "PASS" if result.returncode == 0 else "FAIL", "log": str(log), "exitCode": result.returncode} - - -def execute_query(query_id: str, source: Path, query_path: Path, raw_dir: Path) -> dict[str, Any]: - executable = shutil.which("comunica-sparql-file") - if not executable: - return {"status": "EXECUTION_FAILED", "error": "comunica-sparql-file is not installed"} - raw_path = raw_dir / f"{query_id}.sparql.json" - stderr_path = raw_dir / f"{query_id}.stderr.txt" - time_path = raw_dir / f"{query_id}.time.txt" - command = [executable, str(source), "-f", str(query_path), "-t", "application/sparql-results+json"] - timed = ["/usr/bin/time", "-v", "-o", str(time_path), *command] if tool_version(["/usr/bin/time", "--version"]) else command - started = time.monotonic() - with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: - result = subprocess.run(timed, check=False, stdout=stdout, stderr=stderr) + output = result.stdout + result.stderr + log.write_text(output, encoding="utf-8") + # Rapper reports the parsed triple count; keeping it makes a decoded + # HDT/COTTAS artifact directly comparable against its source graph. + match = re.search(r"Parsing returned (\d+) triples", output) return { - "status": "PASS" if result.returncode == 0 else "EXECUTION_FAILED", + "status": "PASS" if result.returncode == 0 else "FAIL", + "log": str(log), "exitCode": result.returncode, - "wallSeconds": time.monotonic() - started, - "query": str(query_path), - "rawResult": str(raw_path), - "stderr": str(stderr_path), - "resourceMetrics": str(time_path) if time_path.exists() else None, + "tripleCount": int(match.group(1)) if match else None, } +# --------------------------------------------------------------------------- +# SPARQL engines +# --------------------------------------------------------------------------- +# Every engine answers the same queries and writes SPARQL Results JSON to the +# same place, so the comparison layer is engine-agnostic and a run's engine +# choice is only ever a performance/scale decision, never a semantic one. +# +# comunica: zero setup, loads the whole graph into the Node heap. Fine for the +# scale most single-sample runs reach, and the default. +# qlever: builds an on-disk index, then answers over HTTP. Slower to start, +# but the only option once a graph stops fitting in memory. + +SPARQL_ENGINES = ("comunica", "qlever") +# QLever's binaries are copied out of the upstream image, which is built on a +# different Ubuntu release, so their Boost/ICU/jemalloc sonames come with them +# in a private directory. Pointing only QLever's own processes at it keeps +# those libraries from shadowing anything the rest of the image uses. +QLEVER_LIB_DIR = "/opt/qlever/lib" +QLEVER_BIN_DIR = "/opt/qlever/bin" +DEFAULT_QLEVER_PORT = 7019 +DEFAULT_QLEVER_MEMORY_GB = 4 +DEFAULT_QLEVER_STARTUP_TIMEOUT = 900 +DEFAULT_QUERY_TIMEOUT = 3600 + + +QLEVER_STATUS_FILE = Path("/opt/vcf-rdfizer/qlever-status.txt") + + +def qlever_build_status() -> str: + """Read the image's build-time note about whether QLever links here.""" + try: + status = QLEVER_STATUS_FILE.read_text(encoding="utf-8").strip() + except OSError: + return "QLever build status unknown (marker file absent)." + return "QLever binaries linked cleanly at image build time." if status == "ok" else status + + +def _expand_template(template: str, **fields: str) -> list[str]: + """Split a shell-style override and substitute ``{name}`` placeholders.""" + return [part.format(**fields) for part in shlex.split(template)] + + +class QueryEngine: + """Common lifecycle and result envelope for a SPARQL backend.""" + + name = "abstract" + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + self.source = source + self.raw_dir = raw_dir + self.scratch = scratch + self.options = options + self.query_timeout = int(options.get("query_timeout") or DEFAULT_QUERY_TIMEOUT) + + def __enter__(self) -> "QueryEngine": + self.start() + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + self.stop() + return False + + def start(self) -> None: + """Prepare the engine. Raise RuntimeError when it cannot be used.""" + + def stop(self) -> None: + """Release anything ``start`` acquired. Must be safe to call twice.""" + + def describe(self) -> dict[str, Any]: + return {"engine": self.name} + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raise NotImplementedError + + def _envelope( + self, + query_id: str, + query_path: Path, + *, + returncode: int, + started: float, + raw_path: Path, + stderr_path: Path, + time_path: Path | None = None, + error: str | None = None, + ) -> dict[str, Any]: + envelope = { + "status": "PASS" if returncode == 0 else "EXECUTION_FAILED", + "engine": self.name, + "exitCode": returncode, + "wallSeconds": time.monotonic() - started, + "query": str(query_path), + "rawResult": str(raw_path), + "stderr": str(stderr_path), + "resourceMetrics": str(time_path) if time_path and time_path.exists() else None, + } + if error is not None: + envelope["error"] = error + return envelope + + +class ComunicaEngine(QueryEngine): + """Query the N-Triples file directly with comunica-sparql-file.""" + + name = "comunica" + + def start(self) -> None: + self.executable = shutil.which("comunica-sparql-file") + if not self.executable: + raise RuntimeError( + "comunica-sparql-file is not installed in this image; rebuild it or " + "select --engine qlever" + ) + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "version": tool_version( + ["comunica-sparql-file", "--version"], table_label="Comunica Engine" + ), + "mode": "in-memory file query", + } + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + time_path = self.raw_dir / f"{query_id}.time.txt" + command = [ + self.executable, + str(self.source), + "-f", + str(query_path), + "-t", + "application/sparql-results+json", + ] + timed = ( + ["/usr/bin/time", "-v", "-o", str(time_path), *command] + if tool_version(["/usr/bin/time", "--version"]) + else command + ) + started = time.monotonic() + try: + with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + result = subprocess.run( + timed, check=False, stdout=stdout, stderr=stderr, + timeout=self.query_timeout, + ) + returncode = result.returncode + error = None + except subprocess.TimeoutExpired: + returncode = 124 + error = f"query exceeded {self.query_timeout}s" + return self._envelope( + query_id, query_path, returncode=returncode, started=started, + raw_path=raw_path, stderr_path=stderr_path, time_path=time_path, error=error, + ) + + +class QleverEngine(QueryEngine): + """Build a QLever index over the graph, then answer queries over HTTP. + + QLever has no one-shot "query this file" mode, so the index build and the + short-lived server are both owned by this object and torn down in ``stop``. + The index lives in the container scratch directory and never reaches the + host. + """ + + name = "qlever" + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + super().__init__(source, raw_dir=raw_dir, scratch=scratch, options=options) + self.port = int(options.get("port") or DEFAULT_QLEVER_PORT) + self.memory_gb = int(options.get("memory_gb") or DEFAULT_QLEVER_MEMORY_GB) + self.startup_timeout = int( + options.get("startup_timeout") or DEFAULT_QLEVER_STARTUP_TIMEOUT + ) + self.index_dir = self.scratch / "qlever-index" + self.index_base = self.index_dir / "vcf-rdfizer" + self.log_dir = raw_dir / "engine" + self.server: subprocess.Popen | None = None + self.index_seconds: float | None = None + self.commands: dict[str, list[str]] = {} + self.server_binary: str | None = None + self.extra_index_args = list(options.get("extra_index_args") or []) + shlex.split( + os.environ.get("QLEVER_EXTRA_INDEX_ARGS", "") + ) + self.extra_server_args = list(options.get("extra_server_args") or []) + shlex.split( + os.environ.get("QLEVER_EXTRA_SERVER_ARGS", "") + ) + + def _binary(self, env_var: str, name: str) -> str: + try: + return _resolve_binary( + env_var, f"{QLEVER_BIN_DIR}/{name}", name, f"/usr/local/bin/{name}" + ) + except RuntimeError as error: + raise RuntimeError(f"{error}. {qlever_build_status()}") from error + + def _environment(self) -> dict[str, str]: + """Prepend QLever's private library directory for its processes only.""" + environment = dict(os.environ) + existing = environment.get("LD_LIBRARY_PATH", "") + environment["LD_LIBRARY_PATH"] = ( + f"{QLEVER_LIB_DIR}:{existing}" if existing else QLEVER_LIB_DIR + ) + return environment + + def index_command(self, index_builder: str) -> list[str]: + """QLever's CLI has changed across releases, so the default argv is a + starting point rather than a contract: ``--qlever-index-arg`` (or + ``QLEVER_EXTRA_INDEX_ARGS``) appends to it, and + ``QLEVER_INDEX_COMMAND`` replaces it entirely. ``{index}``, ``{input}`` + and ``{memory}`` are substituted in a replacement template. + """ + template = os.environ.get("QLEVER_INDEX_COMMAND", "").strip() + if template: + return _expand_template( + template, index=str(self.index_base), input=str(self.source), + memory=f"{self.memory_gb}G", + ) + return [ + index_builder, + "-i", str(self.index_base), + "-f", str(self.source), + "-F", "nt", + "-m", f"{self.memory_gb}G", + *self.extra_index_args, + ] + + def server_command(self, server_main: str) -> list[str]: + """See :meth:`index_command`; ``QLEVER_SERVER_COMMAND`` replaces this.""" + template = os.environ.get("QLEVER_SERVER_COMMAND", "").strip() + if template: + return _expand_template( + template, index=str(self.index_base), port=str(self.port), + memory=f"{self.memory_gb}G", + ) + return [ + server_main, + "-i", str(self.index_base), + "-p", str(self.port), + "-m", f"{self.memory_gb}G", + # qlever-server's own default query timeout is 30s, which the + # cohort-scale aggregate queries here routinely exceed. Keep the + # server-side limit aligned with the client-side one. + "-s", f"{self.query_timeout}s", + *self.extra_server_args, + ] + + def start(self) -> None: + index_builder = self._binary("QLEVER_INDEX_BUILDER_BIN", "qlever-index") + server_main = self._binary("QLEVER_SERVER_BIN", "qlever-server") + self.index_dir.mkdir(parents=True, exist_ok=True) + self.log_dir.mkdir(parents=True, exist_ok=True) + + self.server_binary = server_main + self.commands["index"] = self.index_command(index_builder) + self.commands["server"] = self.server_command(server_main) + + environment = self._environment() + started = time.monotonic() + _run_step( + self.commands["index"], label="qlever-index", + log_dir=self.log_dir, env=environment, + ) + self.index_seconds = time.monotonic() - started + + server_log = self.log_dir / "qlever-server.log" + self.server = subprocess.Popen( + self.commands["server"], + stdout=server_log.open("wb"), + stderr=subprocess.STDOUT, + env=environment, + ) + self._await_ready(server_log) + + def _await_ready(self, server_log: Path) -> None: + """Poll the server until it answers a trivial query, or give up.""" + deadline = time.monotonic() + self.startup_timeout + last_error = "server did not become ready" + while time.monotonic() < deadline: + if self.server is not None and self.server.poll() is not None: + tail = "" + try: + tail = server_log.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + raise RuntimeError( + f"QLever server exited with code {self.server.returncode}. Log tail: {tail}" + ) + try: + self._post("SELECT * WHERE { ?s ?p ?o } LIMIT 1", timeout=10) + return + except Exception as error: # noqa: BLE001 - readiness probe + last_error = str(error) + time.sleep(0.5) + raise RuntimeError( + f"QLever server was not ready within {self.startup_timeout}s: {last_error}" + ) + + def _post(self, query: str, *, timeout: int) -> bytes: + import urllib.error + import urllib.request + + request = urllib.request.Request( + f"http://127.0.0.1:{self.port}/", + data=urllib.parse.urlencode({"query": query}).encode("utf-8"), + headers={ + "Accept": "application/sparql-results+json", + "Content-Type": "application/x-www-form-urlencoded", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "version": tool_version([self.server_binary, "--help"]) if self.server_binary else None, + "mode": "on-disk index served over HTTP", + "indexDirectory": str(self.index_dir), + "indexBuildSeconds": self.index_seconds, + "memoryLimitGb": self.memory_gb, + "port": self.port, + # The exact argv is recorded because QLever's CLI is overridable and + # varies by release; a report should say what actually ran. + "commands": {name: list(argv) for name, argv in self.commands.items()}, + "buildStatus": qlever_build_status(), + } + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + started = time.monotonic() + try: + payload = self._post( + query_path.read_text(encoding="utf-8"), timeout=self.query_timeout + ) + except Exception as error: # noqa: BLE001 - reported as EXECUTION_FAILED + stderr_path.write_text(str(error), encoding="utf-8") + raw_path.write_bytes(b"") + return self._envelope( + query_id, query_path, returncode=1, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=str(error), + ) + raw_path.write_bytes(payload) + stderr_path.write_bytes(b"") + return self._envelope( + query_id, query_path, returncode=0, started=started, + raw_path=raw_path, stderr_path=stderr_path, + ) + + def stop(self) -> None: + if self.server is not None and self.server.poll() is None: + self.server.terminate() + try: + self.server.wait(timeout=30) + except subprocess.TimeoutExpired: + self.server.kill() + self.server.wait(timeout=30) + self.server = None + # The index is large and container-local; remove it as soon as the + # queries are done rather than waiting for the scratch teardown. + shutil.rmtree(self.index_dir, ignore_errors=True) + + +ENGINE_CLASSES = {"comunica": ComunicaEngine, "qlever": QleverEngine} + + +def build_engine( + name: str, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any] +) -> QueryEngine: + try: + engine_class = ENGINE_CLASSES[name] + except KeyError as error: + raise ValueError( + f"Unknown SPARQL engine {name!r}; choose one of {', '.join(SPARQL_ENGINES)}" + ) from error + return engine_class(source, raw_dir=raw_dir, scratch=scratch, options=options) + + def bindings(path: Path) -> list[dict[str, Any]]: try: value = read_json(path)["results"]["bindings"] @@ -415,11 +1618,15 @@ def normalize(query_id: str, path: Path) -> Any: keys = [tuple(row[field] for field in sort_fields) for row in rows] if len(keys) != len(set(keys)): raise ValueError(f"{query_id} returned duplicate canonical keys") - if query_id == "q03_titv": + if query_id in SINGLE_ROW_QUERIES: if len(rows) != 1: - raise ValueError(f"q03_titv must return exactly one row, got {len(rows)}") + raise ValueError(f"{query_id} must return exactly one row, got {len(rows)}") row = rows[0] - row["tiTvRatio"] = row["transitionCount"] / row["transversionCount"] if row["transversionCount"] else None + if query_id == "q03_titv": + row["tiTvRatio"] = ( + row["transitionCount"] / row["transversionCount"] + if row["transversionCount"] else None + ) return row if query_id == "q06_ac_an_distribution": for row in rows: @@ -427,16 +1634,99 @@ def normalize(query_id: str, path: Path) -> Any: return rows -def preflight(executions: dict[str, dict[str, Any]], parser: dict[str, Any], representation: str) -> dict[str, Any]: +def anomaly_count(executions: dict[str, dict[str, Any]], query_id: str) -> Any: + """Exact anomaly total from the companion aggregate, or None if unavailable. + + Returns ``None`` rather than the sample size, so a report never presents a + truncated count as if it were exact. + """ + execution = executions.get(f"{query_id}_count") + if execution is None or execution.get("status") != "PASS": + return None + try: + rows = bindings(Path(execution["rawResult"])) + except (OSError, ValueError, json.JSONDecodeError): + return None + if len(rows) != 1: + return None + try: + return binding_int(rows[0], "anomalyCount") + except (KeyError, TypeError, ValueError): + return None + + +def duplicate_triple_report( + executions: dict[str, dict[str, Any]], parsed_triple_count: int | None +) -> dict[str, Any]: + """Compare statements parsed against distinct triples stored. + + A SPARQL store deduplicates on load, so no query can see a repeated line. + The parser counted every statement it read; the store counted the distinct + ones. Their difference is exactly the number of redundant statements. + + Reports NOT_EVALUATED rather than PASS when either number is unavailable, so + a missing input can never be mistaken for a clean result. + """ + execution = executions.get("preflight_distinct_triple_count") + if parsed_triple_count is None: + return { + "status": "NOT_EVALUATED", + "reason": "the parser did not report a statement count", + } + if execution is None or execution.get("status") != "PASS": + return { + "status": "NOT_EVALUATED", + "reason": "the distinct-triple query did not run", + } + try: + rows = bindings(Path(execution["rawResult"])) + distinct = binding_int(rows[0], "distinctTripleCount") + except (OSError, ValueError, KeyError, IndexError, TypeError, json.JSONDecodeError) as error: + return {"status": "NOT_EVALUATED", "reason": f"unreadable result: {error}"} + + duplicates = int(parsed_triple_count) - distinct + return { + "status": "PASS" if duplicates == 0 else "FAIL", + "parsedTripleCount": int(parsed_triple_count), + "distinctTripleCount": distinct, + "duplicateTripleCount": duplicates, + } + + +def preflight( + executions: dict[str, dict[str, Any]], + parser: dict[str, Any], + representation: str, + *, + parsed_triple_count: int | None = None, +) -> dict[str, Any]: report: dict[str, Any] = {} + report["preflight_duplicate_triples"] = duplicate_triple_report( + executions, parsed_triple_count + ) for query_id in PREFLIGHT_QUERIES: execution = executions[query_id] if execution["status"] != "PASS": report[query_id] = {"status": "EXECUTION_FAILED", "execution": execution} continue returned = bindings(Path(execution["rawResult"])) - if query_id in {"preflight_record_cardinality", "preflight_position_datatype"}: - report[query_id] = {"status": "PASS" if not returned else "FAIL", "anomalyCountReturned": len(returned), "limitedTo": 100} + exact = anomaly_count(executions, query_id) + if query_id == "preflight_distinct_triple_count": + # Consumed by duplicate_triple_report above; nothing to judge alone. + report[query_id] = {"status": "PASS", "rows": len(returned)} + elif query_id in { + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_blank_nodes", + "preflight_empty_values", + }: + report[query_id] = { + "status": "PASS" if not returned else "FAIL", + "anomalyCount": exact, + "anomalyCountReturned": len(returned), + "limitedTo": ANOMALY_SAMPLE_LIMIT, + "sampleTruncated": len(returned) >= ANOMALY_SAMPLE_LIMIT, + } elif query_id == "preflight_representation_profile": # VCF-RDFizer intentionally omits a sample representation profile # when a VCF has no sample columns, because no sample graph is @@ -444,15 +1734,19 @@ def preflight(executions: dict[str, dict[str, Any]], parser: dict[str, Any], rep profile_missing_is_expected = parser["sampleCount"] == 0 report[query_id] = { "status": "PASS" if not returned or profile_missing_is_expected else "FAIL", + "anomalyCount": exact, "anomalyCountReturned": len(returned), - "limitedTo": 100, + "limitedTo": ANOMALY_SAMPLE_LIMIT, + "sampleTruncated": len(returned) >= ANOMALY_SAMPLE_LIMIT, "note": "No sample representation is emitted for a sample-free VCF." if returned and profile_missing_is_expected else None, } elif query_id == "preflight_missing_token_conformance": report[query_id] = { "status": "PASS" if not returned else "EXPECTED_CONFORMANCE_FAILURE", + "anomalyCount": exact, "plainDotCountReturned": len(returned), - "limitedTo": 100, + "limitedTo": ANOMALY_SAMPLE_LIMIT, + "sampleTruncated": len(returned) >= ANOMALY_SAMPLE_LIMIT, } elif len(returned) != 1: report[query_id] = {"status": "FAIL", "error": f"Expected one aggregate row, got {len(returned)}"} @@ -528,27 +1822,125 @@ def invariant_checks(payload: dict[str, Any], parser: dict[str, Any], *, check_q return [{"name": name, "status": "PASS" if passed else "FAIL", "detail": detail} for name, passed, detail in checks] -def compare(parser: dict[str, Any], sparql: dict[str, Any]) -> dict[str, Any]: +def compare( + parser: dict[str, Any], sparql: dict[str, Any], *, mapping_policy: str = "strict" +) -> dict[str, Any]: required = bool(parser["sampleCount"] and parser["gtRecordCount"]) query_results = {query_id: compare_rows(query_id, parser[query_id], sparql[query_id]) for query_id in CORE_QUERIES} if not required: for query_id in ("q05_sample_genotype_counts", "q06_ac_an_distribution"): query_results[query_id] = {"status": "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT", "diagnosticComparison": query_results[query_id]} + if mapping_policy == "report-only": + # A custom mapping emits a different inventory and different IRIs by + # design, so these are reported for inspection but cannot fail a run. + for query_id in DEFAULT_MAPPING_QUERIES: + query_results[query_id] = { + "status": "NOT_APPLICABLE_CUSTOM_MAPPING", + "diagnosticComparison": query_results[query_id], + } parser_invariants = invariant_checks(parser, parser, check_q06_exact=True) sparql_invariants = invariant_checks(sparql, parser, check_q06_exact=False) - allowed = {"PASS", "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT"} + allowed = { + "PASS", + "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT", + "NOT_APPLICABLE_CUSTOM_MAPPING", + } passed = all(value["status"] in allowed for value in query_results.values()) and all(item["status"] == "PASS" for item in parser_invariants + sparql_invariants) return {"status": "PASS" if passed else "MISMATCH", "queries": query_results, "invariants": {"parser": parser_invariants, "sparql": sparql_invariants}} -def build_manifest(args: argparse.Namespace, query_dir: Path, parser: dict[str, Any]) -> dict[str, Any]: +# Queries whose failure means the graph's core structure is wrong, so the +# aggregate comparison below them cannot be interpreted. +BLOCKING_PREFLIGHT_QUERIES = ( + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_representation_profile", + # Graph-integrity checks. A blank node breaks the IRI-template identity the + # digests rely on; an empty term means a value was lost rather than marked + # missing; a duplicated statement means the conversion emitted the same + # data twice. None of these leave a comparison worth interpreting. + "preflight_blank_nodes", + "preflight_empty_values", + "preflight_duplicate_triples", +) + + +def evaluate_validation( + executions: dict[str, dict[str, Any]], + parser: dict[str, Any], + representation: str, + *, + strict_conformance: bool = False, + mapping_policy: str = "strict", + parsed_triple_count: int | None = None, +) -> dict[str, Any]: + """Turn raw query executions into a validation verdict. + + This is the single place the PASS / MISMATCH / BLOCKED_BY_PREFLIGHT + decision is made. ``run_validation`` uses it for real runs and the mutation + harness uses it directly, so the harness measures the shipped decision + logic rather than a reimplementation of it. + """ + report = preflight( + executions, parser, representation, parsed_triple_count=parsed_triple_count + ) + sparql: dict[str, Any] = {} + failures: dict[str, str] = {} + for query_id in CORE_QUERIES: + try: + sparql[query_id] = normalize(query_id, Path(executions[query_id]["rawResult"])) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: + failures[query_id] = str(error) + if failures: + return { + "status": "EXECUTION_FAILED", + "preflight": report, + "normalizationFailures": failures, + "sparql": sparql, + "comparison": None, + } + + comparison = compare(parser, sparql, mapping_policy=mapping_policy) + # NOT_EVALUATED is not a failure: it means an input the check needs was + # unavailable, which must not be reported as a bad graph. + blocking = any( + report[name]["status"] not in {"PASS", "NOT_EVALUATED"} + for name in BLOCKING_PREFLIGHT_QUERIES + ) + inventory_failed = report["preflight_sample_gt_inventory"]["status"] != "PASS" + conformance_failed = strict_conformance and ( + report["preflight_missing_token_conformance"]["status"] != "PASS" + ) + if blocking: + status = "BLOCKED_BY_PREFLIGHT" + elif inventory_failed or conformance_failed or comparison["status"] != "PASS": + status = "MISMATCH" + else: + status = "PASS" + return { + "status": status, + "preflight": report, + "normalizationFailures": {}, + "sparql": sparql, + "comparison": comparison, + "strictConformanceApplied": bool(strict_conformance), + "mappingPolicy": mapping_policy, + } + + +def build_manifest( + args: argparse.Namespace, + query_dir: Path, + parser: dict[str, Any], + *, + engine_description: dict[str, Any], + materialization: dict[str, Any], +) -> dict[str, Any]: query_paths = sorted({query_path(query_dir, query_id) for query_id in PREFLIGHT_QUERIES + CORE_QUERIES}) - source_rdf = args.rdf_gz or args.rdf_nt - source_rdf_format = "nt.gz" if args.rdf_gz else "nt" source_rdf_entry = { - "path": str(source_rdf), - "sha256": sha256_file(source_rdf), - "format": source_rdf_format, + "path": str(args.rdf), + "sha256": sha256_file(args.rdf), + "format": args.rdf_format, } manifest = { "datasetId": args.dataset_id, @@ -556,13 +1948,16 @@ def build_manifest(args: argparse.Namespace, query_dir: Path, parser: dict[str, "commandLine": sys.argv, "sourceVcf": {"path": str(args.vcf), "sha256": parser["sourceSha256"]}, "sourceRdf": source_rdf_entry, + "engine": engine_description, + "materialization": materialization, "temporaryRdf": { - "decompressedInsideContainer": bool(args.rdf_gz), + "decompressedInsideContainer": bool(materialization.get("materialized")), "persisted": False, "cleanupConfirmed": True, }, "tools": { - "python": platform.python_version(), "cyvcf2": cyvcf2.__version__, + "python": platform.python_version(), + "cyvcf2": getattr(cyvcf2, "__version__", None), "bcftools": tool_version(["bcftools", "--version"]), "node": tool_version(["node", "--version"]), "comunicaQuerySparqlFile": tool_version(["comunica-sparql-file", "--version"], table_label="Comunica Engine"), "rapper": tool_version(["rapper", "--version"]), @@ -571,11 +1966,8 @@ def build_manifest(args: argparse.Namespace, query_dir: Path, parser: dict[str, } # Preserve the original key for consumers of standalone gzip-validation # manifests while exposing the format-neutral ``sourceRdf`` entry. - if args.rdf_gz: - manifest["sourceRdfGzip"] = { - "path": str(args.rdf_gz), - "sha256": sha256_file(args.rdf_gz), - } + if args.rdf_format == "nt.gz": + manifest["sourceRdfGzip"] = dict(source_rdf_entry) return manifest @@ -591,7 +1983,7 @@ def run_validation(args: argparse.Namespace) -> int: raw_dir.mkdir(parents=True, exist_ok=True) normalized_dir.mkdir(parents=True, exist_ok=True) query_dir = QUERY_ROOT / args.representation - query_ids = PREFLIGHT_QUERIES + CORE_QUERIES + query_ids = PREFLIGHT_QUERIES + PREFLIGHT_COUNT_QUERIES + CORE_QUERIES missing = [name for name in query_ids if not query_path(query_dir, name).is_file()] if missing: raise RuntimeError(f"Missing {args.representation} validation query files: {', '.join(missing)}") @@ -605,24 +1997,27 @@ def run_validation(args: argparse.Namespace) -> int: summary: dict[str, Any] | None = None try: with tempfile.TemporaryDirectory(prefix="vcf-rdfizer-validation-", dir=args.scratch_dir) as scratch: - temporary_rdf = bool(args.rdf_gz) - if temporary_rdf: - decoded = Path(scratch) / "input.nt" - progress.emit( - "progress", - completed=0, - detail="decompressing RDF inside container", - ) - with gzip.open(args.rdf_gz, "rb") as source, decoded.open("wb") as target: - shutil.copyfileobj(source, target, length=1024 * 1024) - else: - decoded = args.rdf_nt + scratch_path = Path(scratch) + progress.emit( + "progress", + completed=0, + detail=f"materializing {args.rdf_format} artifact inside container", + ) + decoded, materialization = materialize_ntriples( + args.rdf, + args.rdf_format, + scratch_path, + log_dir=raw_dir / "materialization", + ) + materialization["ntriplesPath"] = str(decoded) progress.emit( "progress", completed=0, detail="parsing source VCF", ) - parser = parse_vcf(args.vcf, filter_oracle=args.filter_oracle) + parser = attach_census_expectations( + parse_vcf(args.vcf, filter_oracle=args.filter_oracle), args.representation + ) write_json(results_dir / "parser.json", parser) progress.emit( "progress", @@ -630,59 +2025,116 @@ def run_validation(args: argparse.Namespace) -> int: detail="validating RDF syntax and cardinality", ) rdf_validation = validate_ntriples(decoded, results_dir) + shacl_result = None + if args.shacl_shapes is not None: + progress.emit("progress", completed=0, detail="validating SHACL shapes") + shacl_result = validate_shacl(decoded, args.shacl_shapes, results_dir) write_json(results_dir / "rdf-validation.json", rdf_validation) - manifest = build_manifest(args, query_dir, parser) + materialization["decodedTripleCount"] = rdf_validation.get("tripleCount") + write_json(results_dir / "materialization.json", materialization) + engine_options = { + "port": args.qlever_port, + "memory_gb": args.qlever_memory_gb, + "startup_timeout": args.qlever_startup_timeout, + "query_timeout": args.query_timeout, + "extra_index_args": list(args.qlever_index_arg), + "extra_server_args": list(args.qlever_server_arg), + } + engine = build_engine( + args.engine, decoded, raw_dir=raw_dir, scratch=scratch_path, + options=engine_options, + ) + manifest = build_manifest( + args, query_dir, parser, + engine_description={"engine": args.engine, "options": engine_options}, + materialization=materialization, + ) write_json(results_dir / "manifest.json", manifest) if rdf_validation["status"] != "PASS": summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "BLOCKED_BY_PREFLIGHT", "rdfValidation": rdf_validation} return 1 + if shacl_result is not None and shacl_result["status"] == "FAIL": + # A shape violation means the graph is structurally wrong, so + # the aggregate comparisons below it cannot be interpreted. + summary = { + "datasetId": args.dataset_id, "representation": args.representation, + "status": "BLOCKED_BY_PREFLIGHT", "shacl": shacl_result, + } + return 1 executions: dict[str, dict[str, Any]] = {} - for completed, query_id in enumerate(query_ids, start=1): - progress.emit( - "progress", - completed=completed - 1, - query_id=query_id, - detail=f"running {args.representation}/{query_id}", - ) - if not quiet: - print( - f"[{args.dataset_id}] running {args.representation}/{query_id}", - flush=True, + progress.emit("progress", completed=0, detail=f"preparing {args.engine} engine") + if not quiet: + print(f"[{args.dataset_id}] preparing {args.engine} engine", flush=True) + try: + engine.start() + except (RuntimeError, OSError) as error: + summary = { + "datasetId": args.dataset_id, "representation": args.representation, + "status": "EXECUTION_FAILED", + "error": f"{args.engine} engine could not be prepared: {error}", + } + return 1 + try: + engine_description = engine.describe() + manifest["engine"] = engine_description + write_json(results_dir / "manifest.json", manifest) + for completed, query_id in enumerate(query_ids, start=1): + progress.emit( + "progress", + completed=completed - 1, + query_id=query_id, + detail=f"running {args.representation}/{query_id}", ) - executions[query_id] = execute_query( - query_id, decoded, query_path(query_dir, query_id), raw_dir - ) - progress.emit( - "progress", - completed=completed, - query_id=query_id, - detail=f"completed {args.representation}/{query_id}", - ) + if not quiet: + print( + f"[{args.dataset_id}] running {args.representation}/{query_id}" + f" ({args.engine})", + flush=True, + ) + executions[query_id] = engine.execute( + query_id, query_path(query_dir, query_id) + ) + progress.emit( + "progress", + completed=completed, + query_id=query_id, + detail=f"completed {args.representation}/{query_id}", + ) + finally: + engine.stop() write_json(results_dir / "query-executions.json", executions) if any(item["status"] != "PASS" for item in executions.values()): summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "queryExecutions": executions} return 1 - report = preflight(executions, parser, args.representation) + verdict = evaluate_validation( + executions, parser, args.representation, + strict_conformance=args.strict_conformance, + mapping_policy=args.mapping_policy, + parsed_triple_count=rdf_validation.get("tripleCount"), + ) + report = verdict["preflight"] write_json(results_dir / "preflight.json", report) - sparql: dict[str, Any] = {} - failures: dict[str, str] = {} - for query_id in CORE_QUERIES: - try: - sparql[query_id] = normalize(query_id, Path(executions[query_id]["rawResult"])) - write_json(normalized_dir / f"{query_id}.json", sparql[query_id]) - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: - failures[query_id] = str(error) - if failures: - summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "normalizationFailures": failures, "preflight": report} + if verdict["status"] == "EXECUTION_FAILED": + summary = { + "datasetId": args.dataset_id, "representation": args.representation, + "status": "EXECUTION_FAILED", + "normalizationFailures": verdict["normalizationFailures"], + "preflight": report, + } return 1 + sparql = verdict["sparql"] + for query_id, rows in sparql.items(): + write_json(normalized_dir / f"{query_id}.json", rows) write_json(results_dir / "sparql.json", sparql) - comparison = compare(parser, sparql) + comparison = verdict["comparison"] write_json(results_dir / "comparison.json", comparison) - blocking = any(report[name]["status"] != "PASS" for name in ("preflight_record_cardinality", "preflight_position_datatype", "preflight_representation_profile")) - inventory_failed = report["preflight_sample_gt_inventory"]["status"] != "PASS" - status = "BLOCKED_BY_PREFLIGHT" if blocking else "MISMATCH" if inventory_failed or comparison["status"] != "PASS" else "PASS" + status = verdict["status"] summary = { "datasetId": args.dataset_id, "representation": args.representation, "status": status, + "engine": args.engine, "sourceFormat": args.rdf_format, + "strictConformance": bool(args.strict_conformance), + "shacl": shacl_result, + "mappingPolicy": args.mapping_policy, "recordCount": parser["totalRecords"], "sampleCount": parser["sampleCount"], "gtRecordCount": parser["gtRecordCount"], "preflight": report, "comparisonStatus": comparison["status"], "results": {"manifest": str(results_dir / "manifest.json"), "parser": str(results_dir / "parser.json"), "sparql": str(results_dir / "sparql.json"), "comparison": str(results_dir / "comparison.json")}, @@ -694,8 +2146,10 @@ def run_validation(args: argparse.Namespace) -> int: finally: if summary is None: summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": "validation ended without a result"} + summary.setdefault("engine", args.engine) + summary.setdefault("sourceFormat", args.rdf_format) summary["temporaryRdf"] = { - "decompressedInsideContainer": bool(args.rdf_gz), + "decompressedInsideContainer": args.rdf_format not in DIRECT_FORMATS, "persisted": False, "cleanupConfirmed": True, } @@ -713,17 +2167,118 @@ def run_validation(args: argparse.Namespace) -> int: print(json.dumps(summary, indent=2), flush=True) -def parse_args() -> argparse.Namespace: +def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--vcf", type=Path, required=True) rdf_group = parser.add_mutually_exclusive_group(required=True) - rdf_group.add_argument("--rdf-gz", type=Path, help="N-Triples gzip input (.nt.gz)") - rdf_group.add_argument("--rdf-nt", type=Path, help="Uncompressed N-Triples input (.nt)") + rdf_group.add_argument( + "--rdf", + type=Path, + help=( + "RDF artifact to validate: .nt, .nt.gz, .nt.br, .hdt, .cottas, " + ".cottas.gz, or .cottas.br" + ), + ) + # Retained so existing callers and reports keep working. + rdf_group.add_argument("--rdf-gz", type=Path, help="Deprecated alias for --rdf (.nt.gz)") + rdf_group.add_argument("--rdf-nt", type=Path, help="Deprecated alias for --rdf (.nt)") + parser.add_argument( + "--rdf-format", + choices=("auto", *RDF_FORMATS), + default="auto", + help="Override artifact format detection (default: infer from the filename)", + ) parser.add_argument("--representation", choices=("expanded", "condensed"), required=True) parser.add_argument("--results-dir", type=Path, required=True) parser.add_argument("--dataset-id", required=True) parser.add_argument("--filter-oracle", choices=("auto", "bcftools", "cyvcf2"), default="auto") parser.add_argument("--scratch-dir", type=Path, default=Path("/work")) + parser.add_argument( + "--engine", + choices=SPARQL_ENGINES, + default="comunica", + help=( + "SPARQL engine: comunica queries the file in memory; qlever builds " + "an on-disk index and serves it (default: comunica)" + ), + ) + parser.add_argument( + "--query-timeout", + type=int, + default=DEFAULT_QUERY_TIMEOUT, + help=f"Per-query timeout in seconds (default: {DEFAULT_QUERY_TIMEOUT})", + ) + parser.add_argument( + "--qlever-memory-gb", + type=int, + default=DEFAULT_QLEVER_MEMORY_GB, + help=f"QLever index/server memory budget in GiB (default: {DEFAULT_QLEVER_MEMORY_GB})", + ) + parser.add_argument( + "--qlever-port", + type=int, + default=DEFAULT_QLEVER_PORT, + help=f"Container-local QLever port (default: {DEFAULT_QLEVER_PORT})", + ) + parser.add_argument( + "--qlever-startup-timeout", + type=int, + default=DEFAULT_QLEVER_STARTUP_TIMEOUT, + help=( + "Seconds to wait for the QLever server to answer after indexing " + f"(default: {DEFAULT_QLEVER_STARTUP_TIMEOUT})" + ), + ) + parser.add_argument( + "--shacl-shapes", + type=Path, + default=None, + help=( + "Validate the graph against a SHACL shapes file as an independent " + "structural layer. Off by default: pyshacl loads the whole graph " + "into memory, so it does not scale to a cohort-sized aggregate" + ), + ) + parser.add_argument( + "--mapping-policy", + choices=MAPPING_POLICIES, + default="strict", + help=( + "How to treat checks that assume the shipped RML mapping (the " + "predicate/class census and the record-identity digest): strict " + "requires an exact match; report-only records them without failing, " + "which is what a custom mapping needs (default: strict)" + ), + ) + parser.add_argument( + "--strict-conformance", + action="store_true", + help=( + "Treat a missing-token conformance failure (a plain '.' literal not " + "typed as vcfr:Null) as a validation failure instead of a report-only " + "observation" + ), + ) + parser.add_argument( + "--qlever-index-arg", + action="append", + default=[], + metavar="ARG", + help=( + "Extra argument for QLever's IndexBuilderMain (repeatable). " + "QLEVER_INDEX_COMMAND replaces the whole command line instead." + ), + ) + parser.add_argument( + "--qlever-server-arg", + action="append", + default=[], + metavar="ARG", + help=( + "Extra argument for QLever's ServerMain (repeatable). " + "QLEVER_SERVER_COMMAND replaces the whole command line instead." + ), + ) parser.add_argument( "--progress-path", type=Path, @@ -734,26 +2289,58 @@ def parse_args() -> argparse.Namespace: action="store_true", help="suppress per-query and summary output on stdout", ) - args = parser.parse_args() + return parser + + +def resolve_args(parser: argparse.ArgumentParser, argv: list[str] | None = None) -> argparse.Namespace: + """Parse and normalise arguments, collapsing the RDF aliases into --rdf.""" + args = parser.parse_args(argv) args.vcf = args.vcf.resolve() - if args.rdf_gz is not None: - args.rdf_gz = args.rdf_gz.resolve() - if args.rdf_nt is not None: - args.rdf_nt = args.rdf_nt.resolve() + + supplied = args.rdf or args.rdf_gz or args.rdf_nt + args.rdf = supplied.resolve() + if args.rdf_gz is not None and args.rdf_format == "auto": + args.rdf_format = "nt.gz" + if args.rdf_nt is not None and args.rdf_format == "auto": + args.rdf_format = "nt" + if args.rdf_format == "auto": + detected = detect_rdf_format(args.rdf) + if detected is None: + parser.error( + f"Could not infer an RDF format from {args.rdf.name!r}; pass " + f"--rdf-format with one of: {', '.join(RDF_FORMATS)}" + ) + args.rdf_format = detected + if not args.vcf.is_file(): parser.error(f"VCF does not exist: {args.vcf}") - if args.rdf_gz is not None and (not args.rdf_gz.is_file() or not args.rdf_gz.name.endswith(".nt.gz")): - parser.error("--rdf-gz must be an existing .nt.gz file") - if args.rdf_nt is not None and (not args.rdf_nt.is_file() or not args.rdf_nt.name.endswith(".nt")): - parser.error("--rdf-nt must be an existing .nt file") + if not args.rdf.is_file(): + parser.error(f"RDF artifact does not exist: {args.rdf}") if not re.fullmatch(r"[A-Za-z0-9._-]+", args.dataset_id): parser.error("--dataset-id may contain only letters, digits, dot, underscore, and hyphen") if not args.scratch_dir.is_dir(): parser.error(f"Scratch directory does not exist: {args.scratch_dir}") + for name, value in ( + ("--query-timeout", args.query_timeout), + ("--qlever-memory-gb", args.qlever_memory_gb), + ("--qlever-startup-timeout", args.qlever_startup_timeout), + ): + if value <= 0: + parser.error(f"{name} must be a positive integer") + if not 1 <= args.qlever_port <= 65535: + parser.error("--qlever-port must be between 1 and 65535") + if args.shacl_shapes is not None: + args.shacl_shapes = args.shacl_shapes.resolve() + if not args.shacl_shapes.is_file(): + parser.error(f"SHACL shapes file does not exist: {args.shacl_shapes}") if args.progress_path is not None: args.progress_path = args.progress_path.resolve() return args +def parse_args() -> argparse.Namespace: + return resolve_args(build_arg_parser()) + + if __name__ == "__main__": raise SystemExit(run_validation(parse_args())) diff --git a/test/README.md b/test/README.md index 267c740..0fb3fd2 100644 --- a/test/README.md +++ b/test/README.md @@ -33,6 +33,34 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - Pins the documented TSV column lists to the headers `src/vcf_as_tsv.sh` actually writes, so the two cannot drift apart. +- `test/test_validation_mutation_unit.py` (+ `validation_fixtures.py`, + `validation_mutations.py`) + - Mutation testing for the semantic validation suite: corrupts a correct + graph in ~25 named ways and asserts which corruptions the validator + detects, producing a reproducible mutation score. + - Requires `rdflib` (test-only, in the `dev` extra); the tests skip cleanly + without it. See `docs/validation-methodology.md`. + - The fixture derives the VCF, the RDF graph and the parser oracle from one + declarative spec, and builds its graph with the project's own emitters, so + the two halves cannot drift apart. + +- `test/cross_engine_agreement.py` + - Not a unittest module: run inside the image to assert every validation + query returns identical values under Comunica and QLever. + +- `test/test_validation_logic_unit.py` + - Mutation tests over the validator's pure comparison layer, run on the host + without cyvcf2 or Docker. + - Records both what a validation `PASS` detects and the coverage gaps it does + not, so closing a gap fails a test rather than passing unnoticed. + +- `test/test_validation_engines_unit.py` + - Verifies artifact format detection and decode paths (`.nt`, `.nt.gz`, + `.nt.br`, `.hdt`, `.cottas[.gz|.br]`) with the container tools faked. + - Verifies Comunica and QLever engine construction, QLever's + index/serve/teardown lifecycle and overridable command lines, and the + wrapper's validation-target resolution. + - `test/test_gzip_size_unit.py` - Verifies uncompressed-size measurement for BGZF, single-member gzip, and concatenated members, each against a full-inflate ground truth. diff --git a/test/cross_engine_agreement.py b/test/cross_engine_agreement.py new file mode 100644 index 0000000..f5d2343 --- /dev/null +++ b/test/cross_engine_agreement.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Assert every validation query returns the same values on every engine. + +Run inside the VCF-RDFizer image, where Comunica and QLever both exist: + + docker run --rm -v "$PWD:/repo:ro" \\ + /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py + +This exists because an engine disagreement is silent and severe: QLever +canonicalises numeric literals at index time, which once made a POS datatype +preflight fail every QLever run while passing on Comunica. Only the values the +validator actually consumes are compared - a store is free to report its own +datatype IRI for a count, and the normalization layer is datatype-agnostic by +design. + +Exits non-zero on the first disagreement, printing both sides. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) + + +def load_runner(): + path = REPO_ROOT / "src" / "validation" / "validation_runner.py" + spec = importlib.util.spec_from_file_location("validation_runner_agreement", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() +ENGINES = ("comunica", "qlever") + + +def evaluate(engine_name: str, representation: str, source: Path, scratch: Path) -> dict: + results: dict[str, object] = {} + raw_dir = scratch / f"raw-{engine_name}-{representation}" + raw_dir.mkdir(parents=True, exist_ok=True) + engine = V.build_engine( + engine_name, source, raw_dir=raw_dir, scratch=scratch, + options={"memory_gb": 2, "startup_timeout": 300, "query_timeout": 300}, + ) + with engine: + for query_id in V.PREFLIGHT_QUERIES + V.PREFLIGHT_COUNT_QUERIES + V.CORE_QUERIES: + execution = engine.execute(query_id, V.query_path(V.QUERY_ROOT / representation, query_id)) + if execution["status"] != "PASS": + results[query_id] = { + "__executionFailed__": Path(execution["stderr"]).read_text( + encoding="utf-8", errors="replace")[:400] + } + continue + raw = Path(execution["rawResult"]) + if query_id in V.QUERY_SCHEMAS: + results[query_id] = V.normalize(query_id, raw) + else: + results[query_id] = [ + {name: binding["value"] for name, binding in row.items()} + for row in V.bindings(raw) + ] + return results + + +def verdict_for(engine_name: str, representation: str, source: Path, scratch: Path) -> dict: + """Run the shipped validation decision for one engine against the fixture.""" + from test import validation_fixtures as fixtures + + raw_dir = scratch / f"verdict-{engine_name}-{representation}" + raw_dir.mkdir(parents=True, exist_ok=True) + engine = V.build_engine( + engine_name, source, raw_dir=raw_dir, scratch=scratch, + options={"memory_gb": 2, "startup_timeout": 300, "query_timeout": 300}, + ) + with engine: + executions = { + query_id: engine.execute(query_id, V.query_path(V.QUERY_ROOT / representation, query_id)) + for query_id in V.PREFLIGHT_QUERIES + V.PREFLIGHT_COUNT_QUERIES + V.CORE_QUERIES + } + # Supply the parsed statement count so the duplicate check is exercised + # here too; for well-formed N-Triples it equals the non-empty line count, + # which is what rapper reports on the real path. + parsed = sum( + 1 for line in source.read_text(encoding="utf-8").splitlines() if line.strip() + ) + return V.evaluate_validation( + executions, fixtures.parser_summary(representation), representation, + parsed_triple_count=parsed, + ) + + +def main() -> int: + from test import validation_fixtures as fixtures + + disagreements = 0 + with tempfile.TemporaryDirectory(dir="/work") as td: + scratch = Path(td) + for representation in ("expanded", "condensed"): + source = scratch / f"{representation}.nt" + source.write_text(fixtures.build_graph(representation), encoding="utf-8") + per_engine = { + engine: evaluate(engine, representation, source, scratch) for engine in ENGINES + } + print(f"\n=== {representation} ===") + for query_id in per_engine[ENGINES[0]]: + values = [per_engine[engine][query_id] for engine in ENGINES] + if all(value == values[0] for value in values): + print(f" {query_id:44s} agree") + continue + disagreements += 1 + print(f" {query_id:44s} *** DIFFER ***") + for engine, value in zip(ENGINES, values): + print(f" {engine:9s} {json.dumps(value)[:300]}") + + # Engines agreeing with each other is not enough: the digests and + # censuses are compared against values Python computes, so each + # engine must also agree with that oracle. Running the real + # decision proves it end to end. + for engine_name in ENGINES: + verdict = verdict_for(engine_name, representation, source, scratch) + status = verdict["status"] + print(f" {'full validation verdict: ' + engine_name:44s} {status}") + if status != "PASS": + disagreements += 1 + print(f" {json.dumps(verdict.get('comparison'))[:500]}") + + if disagreements: + print(f"\n{disagreements} disagreement(s) found.") + return 1 + print("\nAll validation queries agree across " + ", ".join(ENGINES) + ",") + print("and every engine agrees with the Python oracle.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test_validation_engines_unit.py b/test/test_validation_engines_unit.py new file mode 100644 index 0000000..45afa60 --- /dev/null +++ b/test/test_validation_engines_unit.py @@ -0,0 +1,534 @@ +"""Engine selection and artifact materialization for semantic validation. + +The container tools (hdt2rdf, pycottas, comunica, QLever) are replaced with +fakes, so these tests verify the command lines and control flow the validator +builds rather than the third-party tools themselves. +""" + +import gzip +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import vcf_rdfizer +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_engines", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() +TRIPLES = b"

.\n

.\n" + + +class ArtifactFormatTests(VerboseTestCase): + def test_every_pipeline_artifact_suffix_is_recognised(self): + """Format detection covers each artifact the pipeline can produce.""" + expected = { + "cohort.nt": "nt", + "cohort.nt.gz": "nt.gz", + "cohort.nt.br": "nt.br", + "cohort.hdt": "hdt", + "cohort.cottas": "cottas", + "cohort.cottas.gz": "cottas.gz", + "cohort.cottas.br": "cottas.br", + } + for name, fmt in expected.items(): + self.assertEqual(V.detect_rdf_format(Path(name)), fmt, name) + self.assertIsNone(V.detect_rdf_format(Path("cohort.ttl"))) + + def test_wrapper_and_runner_agree_on_formats(self): + """The host and container must not disagree about what is supported.""" + wrapper = {fmt for _suffix, fmt in vcf_rdfizer.VALIDATION_RDF_SUFFIXES} + self.assertEqual(wrapper, set(V.RDF_FORMATS)) + for name in ("a.nt", "a.nt.gz", "a.nt.br", "a.hdt", "a.cottas", "a.cottas.gz", "a.cottas.br"): + self.assertEqual( + vcf_rdfizer.detect_validation_rdf_format(Path(name)), + V.detect_rdf_format(Path(name)), + name, + ) + + def test_plain_ntriples_is_read_in_place(self): + """A .nt source is never copied into scratch.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + scratch = tmp_path / "scratch" + scratch.mkdir() + decoded, provenance = V.materialize_ntriples( + source, "nt", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(decoded, source) + self.assertFalse(provenance["materialized"]) + self.assertEqual(list(scratch.iterdir()), []) + + def test_gzip_is_expanded_into_scratch(self): + """A .nt.gz source is decoded to scratch and reports its provenance.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt.gz" + with gzip.open(source, "wb") as handle: + handle.write(TRIPLES) + scratch = tmp_path / "scratch" + scratch.mkdir() + decoded, provenance = V.materialize_ntriples( + source, "nt.gz", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(decoded.read_bytes(), TRIPLES) + self.assertTrue(provenance["materialized"]) + self.assertEqual(provenance["sourceFormat"], "nt.gz") + + def test_hdt_is_decoded_with_hdt2rdf(self): + """An .hdt source is decoded by the container's hdt2rdf binary.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.hdt" + source.write_bytes(b"fake-hdt") + scratch = tmp_path / "scratch" + scratch.mkdir() + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + Path(command[2]).write_bytes(TRIPLES) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(V, "_resolve_binary", return_value="/usr/local/bin/hdt2rdf"), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + decoded, provenance = V.materialize_ntriples( + source, "hdt", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(calls[0][0], "/usr/local/bin/hdt2rdf") + self.assertEqual(calls[0][1], str(source)) + self.assertEqual(decoded.read_bytes(), TRIPLES) + self.assertTrue(provenance["materialized"]) + + def test_cottas_is_decoded_with_the_cottas_tool(self): + """A .cottas source is decoded through cottas_tool.py decompress.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.cottas" + source.write_bytes(b"fake-cottas") + scratch = tmp_path / "scratch" + scratch.mkdir() + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + Path(command[-1]).write_bytes(TRIPLES) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.dict(os.environ, {"COTTAS_PYTHON_BIN": "/opt/py/bin/python"}), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + decoded, _ = V.materialize_ntriples( + source, "cottas", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(calls[0][0], "/opt/py/bin/python") + self.assertIn("decompress", calls[0]) + self.assertEqual(decoded.read_bytes(), TRIPLES) + + def test_packaged_cottas_is_unwrapped_then_decoded(self): + """A .cottas.gz is gunzipped before pycottas, and the unwrap is cleaned up.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.cottas.gz" + with gzip.open(source, "wb") as handle: + handle.write(b"fake-cottas") + scratch = tmp_path / "scratch" + scratch.mkdir() + + def fake_run(command, **kwargs): + self.assertTrue(Path(command[-2]).is_file(), "unwrapped input must exist") + Path(command[-1]).write_bytes(TRIPLES) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(V.subprocess, "run", side_effect=fake_run): + decoded, provenance = V.materialize_ntriples( + source, "cottas.gz", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(decoded.read_bytes(), TRIPLES) + self.assertFalse((scratch / "input.cottas").exists()) + self.assertEqual(len(provenance["steps"]), 2) + + def test_a_failed_decode_reports_the_tool_output(self): + """A decode failure raises with the tool's own output, not a bare code.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.hdt" + source.write_bytes(b"broken") + scratch = tmp_path / "scratch" + scratch.mkdir() + + def fake_run(command, stdout=None, **kwargs): + stdout.write(b"hdt2rdf: corrupt header\n") + return subprocess.CompletedProcess(command, 3) + + with mock.patch.object(V, "_resolve_binary", return_value="hdt2rdf"), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + with self.assertRaises(RuntimeError) as caught: + V.materialize_ntriples(source, "hdt", scratch, log_dir=tmp_path / "log") + self.assertIn("corrupt header", str(caught.exception)) + + +class EngineTests(VerboseTestCase): + def test_engine_registry_matches_the_wrapper_choices(self): + """The host CLI cannot offer an engine the container does not implement.""" + self.assertEqual( + set(vcf_rdfizer.VALIDATION_ENGINE_CHOICES), set(V.ENGINE_CLASSES) + ) + self.assertEqual(set(V.SPARQL_ENGINES), set(V.ENGINE_CLASSES)) + + def test_unknown_engine_is_rejected(self): + with tempfile.TemporaryDirectory() as td: + with self.assertRaises(ValueError): + V.build_engine( + "sparqlmagic", Path(td) / "a.nt", + raw_dir=Path(td), scratch=Path(td), options={}, + ) + + def test_comunica_engine_builds_the_expected_command(self): + """The default engine still queries the file directly.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }\n", encoding="utf-8") + engine = V.build_engine( + "comunica", source, raw_dir=tmp_path, scratch=tmp_path, options={} + ) + captured = [] + + def fake_run(command, **kwargs): + captured.append(command) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(V.shutil, "which", return_value="/usr/bin/comunica-sparql-file"), \ + mock.patch.object(V, "tool_version", return_value=None), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + engine.start() + result = engine.execute("q02_variant_shape_counts", query) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["engine"], "comunica") + self.assertIn(str(source), captured[0]) + self.assertIn("application/sparql-results+json", captured[0]) + + def test_comunica_reports_a_missing_binary_clearly(self): + """An image without Comunica must say so, not fail obscurely.""" + with tempfile.TemporaryDirectory() as td: + engine = V.build_engine( + "comunica", Path(td) / "a.nt", raw_dir=Path(td), scratch=Path(td), options={} + ) + with mock.patch.object(V.shutil, "which", return_value=None): + with self.assertRaises(RuntimeError) as caught: + engine.start() + self.assertIn("qlever", str(caught.exception)) + + def test_qlever_builds_an_index_then_serves_it(self): + """QLever indexes into scratch, waits for readiness, and answers over HTTP.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }\n", encoding="utf-8") + engine = V.build_engine( + "qlever", source, raw_dir=tmp_path, scratch=tmp_path, + options={"memory_gb": 8, "port": 7031}, + ) + steps = [] + server = mock.MagicMock() + server.poll.return_value = None + + with mock.patch.object(V, "_resolve_binary", side_effect=lambda env, *names: names[0]), \ + mock.patch.object(V, "_run_step", side_effect=lambda cmd, **kw: steps.append(cmd)), \ + mock.patch.object(V.subprocess, "Popen", return_value=server), \ + mock.patch.object(V, "tool_version", return_value="QLever 1.0"), \ + mock.patch.object(V.QleverEngine, "_post", return_value=b'{"results":{"bindings":[]}}'): + engine.start() + result = engine.execute("q03_titv", query) + description = engine.describe() + engine.stop() + + # Flags verified against qlever-index/qlever-server bfd5741. + self.assertEqual(steps[0][0], f"{V.QLEVER_BIN_DIR}/qlever-index") + self.assertIn("-f", steps[0]) + self.assertIn(str(source), steps[0]) + self.assertEqual(steps[0][steps[0].index("-F") + 1], "nt") + self.assertEqual(steps[0][steps[0].index("-m") + 1], "8G") + server_argv = engine.commands["server"] + self.assertEqual(server_argv[0], f"{V.QLEVER_BIN_DIR}/qlever-server") + # qlever-server defaults to a 30s query timeout, far below what the + # aggregate queries need, so it must be set explicitly. + self.assertIn("-s", server_argv) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["engine"], "qlever") + self.assertEqual(json.loads(Path(result["rawResult"]).read_bytes())["results"]["bindings"], []) + self.assertEqual(description["port"], 7031) + self.assertIn("index", description["commands"]) + server.terminate.assert_called_once() + self.assertFalse(engine.index_dir.exists(), "index must not outlive the engine") + + def test_qlever_surfaces_a_server_that_died_during_startup(self): + """A crashed server is reported with its log tail, not a silent timeout.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + engine = V.build_engine( + "qlever", source, raw_dir=tmp_path, scratch=tmp_path, + options={"startup_timeout": 5}, + ) + server = mock.MagicMock() + server.poll.return_value = 1 + server.returncode = 1 + + def fake_popen(command, stdout=None, **kwargs): + stdout.write(b"ServerMain: index not found\n") + stdout.flush() + return server + + with mock.patch.object(V, "_resolve_binary", side_effect=lambda env, *names: names[0]), \ + mock.patch.object(V, "_run_step"), \ + mock.patch.object(V.subprocess, "Popen", side_effect=fake_popen): + with self.assertRaises(RuntimeError) as caught: + engine.start() + self.assertIn("index not found", str(caught.exception)) + + def test_qlever_command_lines_are_overridable(self): + """QLever's CLI varies by release, so both argv are user-overridable.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + engine = V.build_engine( + "qlever", tmp_path / "cohort.nt", raw_dir=tmp_path, scratch=tmp_path, + options={"memory_gb": 2, "extra_index_args": ["--parser-buffer-size", "20"]}, + ) + self.assertIn("--parser-buffer-size", engine.index_command("qlever-index")) + with mock.patch.dict( + os.environ, + {"QLEVER_INDEX_COMMAND": "qlever index -b {index} -i {input} -m {memory}"}, + ): + argv = engine.index_command("qlever-index") + self.assertEqual(argv[:3], ["qlever", "index", "-b"]) + self.assertIn(str(tmp_path / "cohort.nt"), argv) + self.assertIn("2G", argv) + + +class RunnerArgumentTests(VerboseTestCase): + def _resolve(self, argv): + return V.resolve_args(V.build_arg_parser(), argv) + + def _base_args(self, tmp_path: Path, rdf_name: str) -> list[str]: + vcf = tmp_path / "cohort.vcf" + vcf.write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + rdf = tmp_path / rdf_name + rdf.write_bytes(TRIPLES) + return [ + "--vcf", str(vcf), + "--rdf", str(rdf), + "--representation", "expanded", + "--results-dir", str(tmp_path / "results"), + "--dataset-id", "cohort", + "--scratch-dir", str(tmp_path), + ] + + def test_format_is_inferred_from_the_filename(self): + with tempfile.TemporaryDirectory() as td: + args = self._resolve(self._base_args(Path(td), "cohort.hdt")) + self.assertEqual(args.rdf_format, "hdt") + self.assertEqual(args.engine, "comunica") + + def test_explicit_format_overrides_detection(self): + with tempfile.TemporaryDirectory() as td: + argv = self._base_args(Path(td), "cohort.bin") + ["--rdf-format", "nt"] + self.assertEqual(self._resolve(argv).rdf_format, "nt") + + def test_unrecognised_extension_is_rejected_with_guidance(self): + with tempfile.TemporaryDirectory() as td: + with self.assertRaises(SystemExit): + self._resolve(self._base_args(Path(td), "cohort.bin")) + + def test_deprecated_aliases_still_work(self): + """Existing callers that pass --rdf-nt/--rdf-gz keep working.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf = tmp_path / "cohort.vcf" + vcf.write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + rdf = tmp_path / "cohort.nt.gz" + with gzip.open(rdf, "wb") as handle: + handle.write(TRIPLES) + args = self._resolve([ + "--vcf", str(vcf), + "--rdf-gz", str(rdf), + "--representation", "condensed", + "--results-dir", str(tmp_path / "results"), + "--dataset-id", "cohort", + "--scratch-dir", str(tmp_path), + ]) + self.assertEqual(args.rdf, rdf.resolve()) + self.assertEqual(args.rdf_format, "nt.gz") + + def test_engine_tuning_is_validated(self): + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + argv = self._base_args(tmp_path, "cohort.nt") + [ + "--engine", "qlever", "--qlever-memory-gb", "16", "--qlever-port", "7040", + ] + args = self._resolve(argv) + self.assertEqual((args.engine, args.qlever_memory_gb, args.qlever_port), + ("qlever", 16, 7040)) + with self.assertRaises(SystemExit): + self._resolve(self._base_args(tmp_path, "cohort.nt") + ["--qlever-port", "99999"]) + with self.assertRaises(SystemExit): + self._resolve(self._base_args(tmp_path, "cohort.nt") + ["--query-timeout", "0"]) + + +if __name__ == "__main__": + unittest.main() + + +class WrapperValidationTargetTests(VerboseTestCase): + def test_targets_resolve_only_to_artifacts_that_exist(self): + """A representation that was not produced is skipped, not failed.""" + with tempfile.TemporaryDirectory() as td: + out_dir = Path(td) / "cohort" + out_dir.mkdir() + aggregate = out_dir / "cohort.nt.gz" + with gzip.open(aggregate, "wb") as handle: + handle.write(TRIPLES) + (out_dir / "cohort.hdt").write_bytes(b"fake-hdt") + + targets = vcf_rdfizer.resolve_validation_targets( + requested=["aggregate", "hdt", "cottas"], + output_dir=out_dir, + output_name="cohort", + aggregate_path=aggregate, + selected_methods=["hdt", "cottas"], + ) + self.assertEqual([t["name"] for t in targets], ["aggregate", "hdt"]) + self.assertEqual(targets[0]["format"], "nt.gz") + self.assertEqual(targets[1]["format"], "hdt") + + def test_targets_skip_representations_that_were_not_selected(self): + """Asking for hdt without selecting it produces no target.""" + with tempfile.TemporaryDirectory() as td: + out_dir = Path(td) / "cohort" + out_dir.mkdir() + aggregate = out_dir / "cohort.nt" + aggregate.write_bytes(TRIPLES) + (out_dir / "cohort.hdt").write_bytes(b"fake-hdt") + targets = vcf_rdfizer.resolve_validation_targets( + requested=["hdt"], + output_dir=out_dir, + output_name="cohort", + aggregate_path=aggregate, + selected_methods=["gzip"], + ) + self.assertEqual(targets, []) + + def test_target_option_parsing(self): + self.assertEqual(vcf_rdfizer.parse_validation_targets("all"), ["aggregate", "hdt", "cottas"]) + self.assertEqual(vcf_rdfizer.parse_validation_targets("hdt,cottas,hdt"), ["hdt", "cottas"]) + self.assertEqual(vcf_rdfizer.parse_validation_targets("none"), []) + with self.assertRaises(ValueError): + vcf_rdfizer.parse_validation_targets("hdt,bogus") + + def test_validation_command_carries_engine_and_format(self): + """The wrapper passes the engine and artifact format to the container.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "cohort.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n", encoding="utf-8") + hdt_path = tmp_path / "cohort.hdt" + hdt_path.write_bytes(b"fake-hdt") + commands = [] + + with mock.patch.object(vcf_rdfizer, "run", side_effect=lambda cmd, **kw: commands.append(cmd) or 0): + vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=hdt_path, + representation="condensed", + validation_id="cohort", + results_dir=tmp_path / "results", + metrics_dir=tmp_path / "metrics", + run_id="RID", + timestamp="TS", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + engine="qlever", + engine_options={"qlever_memory_gb": 12, "qlever_index_args": ["--x", "1"]}, + wrapper_log_path=tmp_path / "wrapper.log", + ) + command = commands[0] + self.assertIn("--rdf-format", command) + self.assertEqual(command[command.index("--rdf-format") + 1], "hdt") + self.assertEqual(command[command.index("--engine") + 1], "qlever") + self.assertEqual(command[command.index("--qlever-memory-gb") + 1], "12") + self.assertIn("--qlever-index-arg", command) + + def test_unsupported_artifact_is_rejected_by_the_wrapper(self): + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "cohort.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + bogus = tmp_path / "cohort.ttl" + bogus.write_text("", encoding="utf-8") + with self.assertRaises(ValueError): + vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=bogus, + representation="expanded", + validation_id="cohort", + results_dir=tmp_path / "results", + metrics_dir=tmp_path / "metrics", + run_id="RID", + timestamp="TS", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + wrapper_log_path=tmp_path / "wrapper.log", + ) + + +class QleverEnvironmentTests(VerboseTestCase): + def test_qlever_processes_get_their_private_library_path(self): + """QLever's copied Boost/ICU/jemalloc must not shadow the rest of the image.""" + with tempfile.TemporaryDirectory() as td: + engine = V.build_engine( + "qlever", Path(td) / "a.nt", raw_dir=Path(td), scratch=Path(td), options={} + ) + with mock.patch.dict(os.environ, {"LD_LIBRARY_PATH": "/usr/local/lib"}): + environment = engine._environment() + self.assertEqual( + environment["LD_LIBRARY_PATH"], f"{V.QLEVER_LIB_DIR}:/usr/local/lib" + ) + self.assertTrue(V.QLEVER_LIB_DIR.startswith("/opt/qlever")) + + def test_position_datatype_preflight_accepts_the_xsd_integer_family(self): + """QLever canonicalises xsd:integer to xsd:int, so both must pass. + + Verified against qlever-index/qlever-server bfd5741: without the wider + accepted set this preflight failed every run on QLever while passing on + Comunica. xsd:string and xsd:decimal are still reported as anomalies. + """ + text = (V.QUERY_ROOT / "common" / "preflight_position_datatype.rq").read_text( + encoding="utf-8" + ) + self.assertIn("NOT IN", text) + accepted_list = text.split("NOT IN", 1)[1].split("))", 1)[0] + for accepted in ("xsd:integer", "xsd:int", "xsd:long", "xsd:nonNegativeInteger"): + self.assertIn(accepted, accepted_list) + for rejected in ("xsd:string", "xsd:decimal", "xsd:double"): + self.assertNotIn(rejected, accepted_list) diff --git a/test/test_validation_logic_unit.py b/test/test_validation_logic_unit.py new file mode 100644 index 0000000..6f172aa --- /dev/null +++ b/test/test_validation_logic_unit.py @@ -0,0 +1,467 @@ +"""What the semantic validator's comparison layer does and does not detect. + +These tests drive the pure normalization/comparison functions directly, so they +run on the host without cyvcf2, Comunica, or Docker. Each one mutates a +graph-derived result and asserts the outcome, which turns the validator's +coverage into something recorded rather than assumed. + +The ``test_blind_spot_*`` cases are deliberate: they document real gaps, so a +future change that closes one will fail here and prompt an update instead of +passing unnoticed. +""" + +import copy +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +from test import validation_fixtures as fixtures +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_logic", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +def parser_fixture() -> dict: + """The canonical fixture's oracle. + + Shared with the graph-level mutation harness rather than hand-written here, + so there is exactly one description of what the fixture VCF contains. + """ + return copy.deepcopy(fixtures.parser_summary("expanded")) + + +def sparql_fixture() -> dict: + """A graph-derived result that agrees with the parser exactly.""" + source = parser_fixture() + return {key: copy.deepcopy(source[key]) for key in V.CORE_QUERIES} + + +class ValidationDetectionTests(VerboseTestCase): + def test_matching_results_pass(self): + """An agreeing graph and parser produce a PASS with no failed invariant.""" + report = V.compare(parser_fixture(), sparql_fixture()) + self.assertEqual(report["status"], "PASS") + self.assertTrue(all(q["status"] == "PASS" for q in report["queries"].values())) + + def test_detects_a_dropped_record(self): + """Losing one record breaks both the row comparison and the totals.""" + sparql = sparql_fixture() + sparql["q01_record_density_1mb"][0]["recordCount"] -= 1 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["status"], "MISMATCH") + self.assertEqual(report["queries"]["q01_record_density_1mb"]["status"], "MISMATCH") + self.assertIn( + "FAIL", + [item["status"] for item in report["invariants"]["sparql"]], + ) + + def test_detects_a_misclassified_variant_shape(self): + """Moving a record between shape classes is caught even though the total holds.""" + sparql = sparql_fixture() + rows = sparql["q02_variant_shape_counts"] + self.assertGreaterEqual(len(rows), 2, "fixture needs two shape classes") + rows[0]["recordCount"] += 1 + rows[1]["recordCount"] -= 1 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q02_variant_shape_counts"]["status"], "MISMATCH") + # The record total is unchanged, so only the distribution catches this. + totals = {item["name"]: item["status"] for item in report["invariants"]["sparql"]} + self.assertEqual(totals["q02_variant_shape_counts_total"], "PASS") + + def test_detects_a_flipped_genotype(self): + """A genotype decoded into the wrong class changes the q05 distribution.""" + sparql = sparql_fixture() + # A class the fixture does not already use, so this is a flip rather + # than a collision with an existing row. + sparql["q05_sample_genotype_counts"][0]["genotypeClass"] = "OTHER_PLOIDY" + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q05_sample_genotype_counts"]["status"], "MISMATCH") + + def test_detects_a_corrupted_filter_string(self): + """FILTER is compared by exact lexical value, not only by status class.""" + sparql = sparql_fixture() + sparql["q04_filter_distribution"][0]["filterLexical"] = "q20" + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q04_filter_distribution"]["status"], "MISMATCH") + + def test_detects_a_transition_transversion_swap(self): + """Ti/Tv is compared field-by-field, so a swap is not hidden by the sum.""" + sparql = sparql_fixture() + sparql["q03_titv"]["transitionCount"] = 1 + sparql["q03_titv"]["transversionCount"] = 2 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q03_titv"]["status"], "MISMATCH") + + def test_detects_an_allele_count_error(self): + """A wrong AC/AN pairing shows up as missing and extra q06 rows.""" + sparql = sparql_fixture() + sparql["q06_ac_an_distribution"][0]["ac"] = 2 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q06_ac_an_distribution"]["status"], "MISMATCH") + + def test_detects_an_extra_spurious_record(self): + """A record the VCF does not contain appears as an extra row.""" + sparql = sparql_fixture() + sparql["q01_record_density_1mb"].append( + {"chrom": "chrUnplaced", "windowIndex": 0, "recordCount": 1} + ) + report = V.compare(parser_fixture(), sparql) + comparison = report["queries"]["q01_record_density_1mb"] + self.assertEqual(comparison["status"], "MISMATCH") + self.assertTrue(comparison["extraRows"]) + + def test_impossible_allele_counts_fail_the_bounds_invariant(self): + """AC > AN is rejected by an invariant even without a parser disagreement.""" + payload = parser_fixture() + payload["q06_ac_an_distribution"] = [{"an": 2, "ac": 5, "siteCount": 1, "af": 2.5}] + checks = V.invariant_checks(payload, parser_fixture(), check_q06_exact=False) + self.assertIn("FAIL", [item["status"] for item in checks]) + + # -- Recorded blind spots ------------------------------------------------- + + def test_permuted_records_are_caught_only_by_the_digest(self): + """The aggregates still cannot see a permutation; q11 is what catches it. + + Every other core query is a GROUP BY count, so moving a value between + two records in the same bucket leaves them all identical. Removing the + digest from the comparison must therefore restore a PASS. + """ + parser = parser_fixture() + sparql = sparql_fixture() + # A permutation changes which digests exist, but no aggregate. + sparql["q11_record_digest"] = [ + {"bucket": "00", "recordCount": parser["totalRecords"]} + ] + report = V.compare(parser, sparql) + self.assertEqual(report["status"], "MISMATCH") + self.assertEqual(report["queries"]["q11_record_digest"]["status"], "MISMATCH") + aggregates_only = { + name: result for name, result in report["queries"].items() + if name != "q11_record_digest" + } + self.assertTrue( + all(result["status"] != "MISMATCH" for result in aggregates_only.values()), + "an aggregate query should not have noticed a permutation", + ) + + def test_core_query_set_is_pinned(self): + """The query set is part of the contract; changing it changes coverage.""" + self.assertEqual( + set(V.CORE_QUERIES), + { + "q01_record_density_1mb", + "q02_variant_shape_counts", + "q03_titv", + "q04_filter_distribution", + "q05_sample_genotype_counts", + "q06_ac_an_distribution", + "q07_file_metadata", + "q08_header_line_census", + "q09_predicate_census", + "q10_class_census", + "q11_record_digest", + "q12_info_value_digest", + "q13_format_value_digest", + }, + ) + + def test_digest_covers_every_fixed_field(self): + """ID, QUAL and INFO are only covered through the record digest.""" + query = (V.QUERY_ROOT / "common" / "q11_record_digest.rq").read_text(encoding="utf-8") + for term in ("vcfr:chrom", "vcfr:pos", "vcfr:recordId", "vcfr:ref", + "vcfr:alt", "vcfr:qual", "vcfr:filter", "vcfr:infoRaw"): + self.assertIn(term, query, f"{term} missing from the record digest") + + def test_runner_and_wrapper_agree_on_template_encoding(self): + """The digest rebuilds record IRIs, so both encoders must match.""" + import vcf_rdfizer + + for value in ("plain", "a b", "x~y", "\u00e9", "cohort.vcf.gz", "1"): + self.assertEqual( + V.rml_uri_component(value), + vcf_rdfizer._rml_uri_component(value), + value, + ) + + def test_anomaly_severity_is_measured_by_a_companion_aggregate(self): + """The LIMIT 100 sample is for diagnosis; an aggregate gives exact severity.""" + for query_id in V.ANOMALY_PREFLIGHT_QUERIES: + sample = V.query_path(V.QUERY_ROOT / "expanded", query_id).read_text(encoding="utf-8") + self.assertIn("LIMIT 100", sample) + exact = V.query_path(V.QUERY_ROOT / "expanded", f"{query_id}_count") + self.assertTrue(exact.is_file(), f"missing exact-count companion for {query_id}") + body = "\n".join( + line for line in exact.read_text(encoding="utf-8").splitlines() + if not line.lstrip().startswith("#") + ) + self.assertIn("anomalyCount", body) + # An exact count must not be capped, or it measures nothing. + self.assertNotIn("LIMIT", body) + + +class NormalizationTests(VerboseTestCase): + def _write_bindings(self, tmp_path: Path, rows: list[dict]) -> Path: + path = tmp_path / "result.json" + path.write_text(json.dumps({"results": {"bindings": rows}}), encoding="utf-8") + return path + + def test_non_integer_counts_are_rejected(self): + """A count that is not an integer fails rather than being coerced.""" + with tempfile.TemporaryDirectory() as td: + path = self._write_bindings( + Path(td), + [{"variantClass": {"value": "SNV"}, "recordCount": {"value": "3.5"}}], + ) + with self.assertRaises(ValueError): + V.normalize("q02_variant_shape_counts", path) + + def test_duplicate_canonical_keys_are_rejected(self): + """Two rows for the same key would silently hide one; that is an error.""" + with tempfile.TemporaryDirectory() as td: + path = self._write_bindings( + Path(td), + [ + {"variantClass": {"value": "SNV"}, "recordCount": {"value": "1"}}, + {"variantClass": {"value": "SNV"}, "recordCount": {"value": "2"}}, + ], + ) + with self.assertRaises(ValueError): + V.normalize("q02_variant_shape_counts", path) + + def test_missing_projection_variable_is_rejected(self): + """An engine that omits a selected variable fails loudly.""" + with tempfile.TemporaryDirectory() as td: + path = self._write_bindings(Path(td), [{"variantClass": {"value": "SNV"}}]) + with self.assertRaises(ValueError): + V.normalize("q02_variant_shape_counts", path) + + def test_engine_independent_normalization(self): + """Both engines' SPARQL Results JSON normalizes to the same rows.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + comunica_style = self._write_bindings( + tmp_path, + [{"variantClass": {"type": "literal", "value": "SNV"}, + "recordCount": {"type": "literal", "value": "3"}}], + ) + rows = V.normalize("q02_variant_shape_counts", comunica_style) + qlever_style = tmp_path / "qlever.json" + qlever_style.write_text( + json.dumps( + { + "head": {"vars": ["variantClass", "recordCount"]}, + "results": { + "bindings": [ + { + "variantClass": {"type": "literal", "value": "SNV"}, + "recordCount": { + "type": "literal", + "value": "3", + "datatype": "http://www.w3.org/2001/XMLSchema#int", + }, + } + ] + }, + } + ), + encoding="utf-8", + ) + self.assertEqual(rows, V.normalize("q02_variant_shape_counts", qlever_style)) + + +if __name__ == "__main__": + unittest.main() + + +class CensusAndDigestTests(VerboseTestCase): + """The completeness and identity checks added in Phases 2-5.""" + + def test_census_rejects_an_unexpected_predicate(self): + """A predicate the VCF does not imply is an extra row, not a silent pass.""" + parser = parser_fixture() + sparql = sparql_fixture() + sparql["q09_predicate_census"].append( + {"predicate": "https://example.org/invented", "tripleCount": 1} + ) + report = V.compare(parser, sparql) + self.assertEqual(report["queries"]["q09_predicate_census"]["status"], "MISMATCH") + self.assertTrue(report["queries"]["q09_predicate_census"]["extraRows"]) + + def test_census_is_report_only_for_a_custom_mapping(self): + """A custom mapping changes the inventory by design, so it cannot fail.""" + parser = parser_fixture() + sparql = sparql_fixture() + sparql["q09_predicate_census"].append( + {"predicate": "https://example.org/invented", "tripleCount": 1} + ) + report = V.compare(parser, sparql, mapping_policy="report-only") + self.assertEqual(report["status"], "PASS") + self.assertEqual( + report["queries"]["q09_predicate_census"]["status"], + "NOT_APPLICABLE_CUSTOM_MAPPING", + ) + + def test_census_covers_qual_and_structured_info(self): + """The elements added in Phases 2, 4 and 5 are in the expected inventory.""" + predicates = {row["predicate"] for row in parser_fixture()["q09_predicate_census"]} + for term in ("qual", "hasInfoValue", "fileDate", "contigId", "filterId", "altId"): + self.assertIn(f"{V.VCFR}{term}", predicates, term) + + def test_digest_separator_prevents_field_boundary_collisions(self): + """Shifting a field boundary must not be able to forge a matching digest.""" + self.assertEqual(V.DIGEST_SEPARATOR, chr(0x1F)) + joined_left = V.DIGEST_SEPARATOR.join(["ab", "c"]) + joined_right = V.DIGEST_SEPARATOR.join(["a", "bc"]) + self.assertNotEqual(joined_left, joined_right) + + def test_header_class_map_matches_the_wrapper(self): + """The oracle and the emitter must agree on every '##' line's subclass.""" + import vcf_rdfizer + + self.assertEqual(V.HEADER_LINE_CLASSES, vcf_rdfizer.HEADER_LINE_CLASSES) + + def test_info_entry_parsers_agree(self): + """Flag entries and '=' splitting must be identical on both sides.""" + import vcf_rdfizer + + for info in ("AC=1;DB", ".", "", "A=1;B=2,3;C", "KEY=a=b"): + self.assertEqual( + V.parse_info_entries(info), vcf_rdfizer.parse_info_entries(info), info + ) + + def test_qual_is_typed_as_the_published_shape_requires(self): + """The shape demands xsd:decimal or vcfr:Null, never a plain literal.""" + import vcf_rdfizer + + self.assertIn("XMLSchema#decimal", vcf_rdfizer._qual_object("12.5")) + self.assertIn("vocab#Null", vcf_rdfizer._qual_object(".")) + # A non-numeric QUAL is preserved rather than dropped, and the SHACL + # layer reports it. + self.assertEqual(vcf_rdfizer._qual_object("bogus"), '"bogus"') + + def test_file_date_is_typed_when_its_form_allows(self): + """##fileDate has no mandated format, so typing is conditional.""" + import vcf_rdfizer + + self.assertIn("XMLSchema#date", vcf_rdfizer.file_date_object("20260101")) + self.assertIn("2026-01-01", vcf_rdfizer.file_date_object("20260101")) + self.assertEqual(vcf_rdfizer.file_date_object("Jan 2026"), '"Jan 2026"') + self.assertIsNone(vcf_rdfizer.file_date_object("")) + + +class ShaclLayerTests(VerboseTestCase): + """SHACL is an independent structural layer, and an optional one.""" + + def test_missing_pyshacl_is_an_execution_failure_not_a_conformance_failure(self): + """An absent optional dependency must never look like a bad graph.""" + import builtins + import tempfile + import unittest.mock + + real_import = builtins.__import__ + + def blocked(name, *args, **kwargs): + if name == "pyshacl": + raise ImportError("blocked for test") + return real_import(name, *args, **kwargs) + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "g.nt" + source.write_text("

.\n", encoding="utf-8") + shapes = tmp_path / "s.ttl" + shapes.write_text("", encoding="utf-8") + with unittest.mock.patch.object(builtins, "__import__", blocked): + result = V.validate_shacl(source, shapes, tmp_path) + self.assertEqual(result["status"], "EXECUTION_FAILED") + self.assertIn("pyshacl", result["error"]) + self.assertNotIn("conforms", result) + + +class GraphIntegrityTests(VerboseTestCase): + """Blank nodes, empty terms, and duplicated statements.""" + + def _execution(self, tmp_path: Path, rows: list) -> dict: + path = tmp_path / "distinct.json" + path.write_text(json.dumps({"results": {"bindings": rows}}), encoding="utf-8") + return {"preflight_distinct_triple_count": {"status": "PASS", "rawResult": str(path)}} + + def test_duplicates_are_the_gap_between_parsed_and_distinct(self): + """A store deduplicates on load, so only the parser sees a repeat.""" + with tempfile.TemporaryDirectory() as td: + executions = self._execution( + Path(td), [{"distinctTripleCount": {"value": "100"}}] + ) + clean = V.duplicate_triple_report(executions, 100) + self.assertEqual(clean["status"], "PASS") + self.assertEqual(clean["duplicateTripleCount"], 0) + + duplicated = V.duplicate_triple_report(executions, 137) + self.assertEqual(duplicated["status"], "FAIL") + self.assertEqual(duplicated["duplicateTripleCount"], 37) + self.assertEqual(duplicated["parsedTripleCount"], 137) + self.assertEqual(duplicated["distinctTripleCount"], 100) + + def test_a_missing_parsed_count_is_not_reported_as_clean(self): + """An unavailable input must never look like an absence of duplicates.""" + with tempfile.TemporaryDirectory() as td: + executions = self._execution( + Path(td), [{"distinctTripleCount": {"value": "100"}}] + ) + result = V.duplicate_triple_report(executions, None) + self.assertEqual(result["status"], "NOT_EVALUATED") + self.assertNotIn("duplicateTripleCount", result) + + def test_a_missing_distinct_query_is_not_reported_as_clean(self): + result = V.duplicate_triple_report({}, 100) + self.assertEqual(result["status"], "NOT_EVALUATED") + + def test_not_evaluated_does_not_block_a_run(self): + """A check that could not run must not be treated as a failure.""" + self.assertIn("preflight_duplicate_triples", V.BLOCKING_PREFLIGHT_QUERIES) + parser = parser_fixture() + sparql = sparql_fixture() + report = V.compare(parser, sparql) + self.assertEqual(report["status"], "PASS") + + def test_integrity_checks_are_blocking(self): + """Each of the three is treated as a structural defect, not a mismatch.""" + for name in ("preflight_blank_nodes", "preflight_empty_values", + "preflight_duplicate_triples"): + self.assertIn(name, V.BLOCKING_PREFLIGHT_QUERIES, name) + + def test_blank_node_query_examines_subject_and_object_only(self): + """A predicate cannot be a blank node in RDF, so it is not tested.""" + query = (V.QUERY_ROOT / "common" / "preflight_blank_nodes.rq").read_text(encoding="utf-8") + self.assertIn("ISBLANK(?s)", query) + self.assertIn("ISBLANK(?o)", query) + self.assertNotIn("ISBLANK(?p)", query) + + def test_empty_value_query_covers_literals_and_iris(self): + """Both a lost value and a collapsed template substitution are caught.""" + query = (V.QUERY_ROOT / "common" / "preflight_empty_values.rq").read_text(encoding="utf-8") + self.assertIn("EMPTY_LITERAL", query) + self.assertIn("EMPTY_IRI", query) + # Whitespace-only carries no more information than empty. + self.assertIn("REGEX", query) + + def test_every_anomaly_preflight_has_an_exact_count_companion(self): + """Severity must be measurable for each new check too.""" + for query_id in V.ANOMALY_PREFLIGHT_QUERIES: + companion = V.query_path(V.QUERY_ROOT / "expanded", f"{query_id}_count") + self.assertTrue(companion.is_file(), f"missing companion for {query_id}") + self.assertIn("preflight_blank_nodes", V.ANOMALY_PREFLIGHT_QUERIES) + self.assertIn("preflight_empty_values", V.ANOMALY_PREFLIGHT_QUERIES) diff --git a/test/test_validation_mutation_unit.py b/test/test_validation_mutation_unit.py new file mode 100644 index 0000000..302a6a7 --- /dev/null +++ b/test/test_validation_mutation_unit.py @@ -0,0 +1,217 @@ +"""Mutation testing for the semantic validation suite. + +Each catalogued mutation is applied to a correct graph, the real validation +queries are evaluated over the result, and the shipped decision logic +(``evaluate_validation``) is asked for a verdict. A mutation is "detected" when +that verdict stops being PASS. + +Queries run under rdflib here so the harness needs no Docker and stays fast +enough for the normal test loop. rdflib is a third independent engine, which +also cross-checks that the queries are not accidentally engine-specific; the +container job in ``.github/workflows/validation-mutation.yml`` replays the same +catalogue under Comunica and QLever as the authority. + +The run writes ``mutation-score.json`` next to the repository root when +``VCF_RDFIZER_MUTATION_REPORT`` is set, which is the artifact the coverage +documentation quotes. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import tempfile +import unittest +from pathlib import Path + +from test import validation_fixtures as fixtures +from test import validation_mutations as mutations +from test.helpers import VerboseTestCase + +try: + import rdflib +except ImportError: # pragma: no cover - optional test-only dependency + rdflib = None + +REPO_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = REPO_ROOT / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_mutation", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +class RdflibEngine: + """Evaluate the validation queries in-process and emit SPARQL Results JSON. + + The output is written in exactly the shape the real engines produce, so the + normalization and comparison layers below are the shipped ones, unmodified. + """ + + def __init__(self, graph_text: str, raw_dir: Path): + self.graph = rdflib.Graph() + self.graph.parse(data=graph_text, format="nt") + self.raw_dir = raw_dir + self.raw_dir.mkdir(parents=True, exist_ok=True) + + def execute(self, query_id: str, query_path: Path) -> dict: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + try: + result = self.graph.query(query_path.read_text(encoding="utf-8")) + raw_path.write_bytes(result.serialize(format="json")) + return {"status": "PASS", "exitCode": 0, "rawResult": str(raw_path), + "engine": "rdflib"} + except Exception as error: # noqa: BLE001 - surfaced as EXECUTION_FAILED + raw_path.write_text("{}", encoding="utf-8") + return {"status": "EXECUTION_FAILED", "exitCode": 1, + "rawResult": str(raw_path), "engine": "rdflib", "error": str(error)} + + +def validate_graph( + graph_text: str, representation: str, *, strict_conformance: bool = False, + include_qual: bool = True, mapping_policy: str = "strict", +) -> dict: + """Run the whole validation decision over one graph.""" + parser = fixtures.parser_summary(representation, include_qual=include_qual) + query_dir = V.QUERY_ROOT / representation + with tempfile.TemporaryDirectory() as td: + engine = RdflibEngine(graph_text, Path(td) / "raw") + executions = { + query_id: engine.execute(query_id, V.query_path(query_dir, query_id)) + for query_id in V.PREFLIGHT_QUERIES + V.PREFLIGHT_COUNT_QUERIES + V.CORE_QUERIES + } + if any(item["status"] != "PASS" for item in executions.values()): + failed = {k: v.get("error", "") for k, v in executions.items() if v["status"] != "PASS"} + return {"status": "EXECUTION_FAILED", "queryErrors": failed} + # rapper is a container tool, so the harness supplies the parsed + # statement count itself. For well-formed N-Triples that is exactly the + # number of non-empty lines, which is what rapper would report. + parsed = sum(1 for line in graph_text.splitlines() if line.strip()) + return V.evaluate_validation( + executions, parser, representation, strict_conformance=strict_conformance, + mapping_policy=mapping_policy, parsed_triple_count=parsed, + ) + + +@unittest.skipIf(rdflib is None, "rdflib is required for the host mutation harness") +class MutationHarnessTests(VerboseTestCase): + """The harness itself must agree with a correct graph before it can judge.""" + + def test_unmutated_expanded_graph_passes(self): + """A correct expanded graph validates cleanly against the fixture oracle.""" + verdict = validate_graph(fixtures.build_graph("expanded"), "expanded") + self.assertEqual(verdict["status"], "PASS", json.dumps(verdict.get("comparison"), indent=2)[:2000]) + + def test_unmutated_condensed_graph_passes(self): + """The same holds for the condensed representation.""" + verdict = validate_graph(fixtures.build_graph("condensed"), "condensed") + self.assertEqual(verdict["status"], "PASS", json.dumps(verdict.get("comparison"), indent=2)[:2000]) + + def test_fixture_graph_and_oracle_describe_the_same_data(self): + """A guard that the fixture's two halves cannot drift apart silently.""" + oracle = fixtures.parser_summary("expanded") + self.assertEqual(oracle["totalRecords"], len(fixtures.RECORDS)) + self.assertEqual(oracle["headerLineCount"], len(fixtures.HEADER_LINES)) + self.assertEqual( + sum(row["recordCount"] for row in oracle["q01_record_density_1mb"]), + oracle["totalRecords"], + ) + + +@unittest.skipIf(rdflib is None, "rdflib is required for the host mutation harness") +class MutationDetectionTests(VerboseTestCase): + """Every catalogued mutation, and whether the suite notices it.""" + + @classmethod + def setUpClass(cls): + cls.graphs: dict[tuple, str] = {} + cls.results: list[dict] = [] + + @classmethod + def graph_for(cls, representation: str, options: tuple) -> str: + """Build (and cache) the fixture graph a mutation needs.""" + key = (representation, options) + if key not in cls.graphs: + cls.graphs[key] = fixtures.build_graph(representation, **dict(options)) + return cls.graphs[key] + + @classmethod + def tearDownClass(cls): + detected = [r for r in cls.results if r["detected"]] + report = { + "total": len(cls.results), + "detected": len(detected), + "score": round(len(detected) / len(cls.results), 4) if cls.results else 0.0, + "knownUndetected": [r["id"] for r in cls.results if not r["detected"]], + "mutations": cls.results, + } + destination = os.environ.get("VCF_RDFIZER_MUTATION_REPORT") + if destination: + Path(destination).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print( + f"\n mutation score: {report['detected']}/{report['total']} " + f"({report['score']:.0%} detected)" + ) + + def _check(self, mutation: mutations.Mutation, representation: str): + options = dict(mutation.graph_options) + original = self.graph_for(representation, mutation.graph_options) + mutated = mutation.apply(original) + self.assertNotEqual(mutated, original, f"{mutation.id} did not change the graph") + # The conformance mutation is only a failure under the strict policy. + strict = mutation.id == "plain_dot_literal" + verdict = validate_graph( + mutated, representation, strict_conformance=strict, + include_qual=options.get("include_qual", True), + mapping_policy=mutation.mapping_policy, + ) + detected = verdict["status"] != "PASS" + self.results.append({ + "id": mutation.id, + "representation": representation, + "vcfElement": mutation.vcf_element, + "expectedDetectedBy": mutation.expected_detected_by, + "detected": detected, + "status": verdict["status"], + "knownUndetected": mutation.known_undetected, + "graphOptions": dict(mutation.graph_options), + }) + if mutation.known_undetected: + self.assertFalse( + detected, + f"{mutation.id} is now DETECTED. This gap has been closed - remove " + f"`known_undetected` from the catalogue and update " + f"docs/vcf-coverage.md. Recorded reason was: {mutation.known_undetected}", + ) + else: + self.assertTrue( + detected, + f"{mutation.id} was NOT detected (status={verdict['status']}). " + f"Expected {mutation.expected_detected_by} to catch it.", + ) + + +def _attach_mutation_tests(): + """Generate one test per (mutation, representation) so failures are named.""" + for representation in ("expanded", "condensed"): + for mutation in mutations.for_representation(representation): + def test(self, _m=mutation, _r=representation): + self._check(_m, _r) + + test.__doc__ = f"[{representation}] {mutation.description}" + setattr(MutationDetectionTests, f"test_{representation}_{mutation.id}", test) + + +_attach_mutation_tests() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 75cacf0..78e1ecc 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -467,8 +467,15 @@ def test_validation_runner_emits_query_progress_in_quiet_mode(self): progress_path=progress_path, quiet=True, scratch_dir=scratch_dir, - rdf_gz=None, - rdf_nt=rdf_path, + rdf=rdf_path, + rdf_format="nt", + engine="comunica", + query_timeout=60, + qlever_memory_gb=4, + qlever_port=7019, + qlever_startup_timeout=60, + qlever_index_arg=[], + qlever_server_arg=[], vcf=vcf_path, filter_oracle="cyvcf2", dataset_id="sample", @@ -480,12 +487,15 @@ def test_validation_runner_emits_query_progress_in_quiet_mode(self): "sampleCount": 0, "gtRecordCount": 0, } + failing_engine = mock.MagicMock() + failing_engine.describe.return_value = {"engine": "comunica"} + failing_engine.execute.return_value = {"status": "FAILED"} with mock.patch.object(validator, "query_path", return_value=query_file), mock.patch.object( validator, "parse_vcf", return_value=parser ), mock.patch.object( validator, "validate_ntriples", return_value={"status": "PASS"} ), mock.patch.object(validator, "build_manifest", return_value={}), mock.patch.object( - validator, "execute_query", return_value={"status": "FAILED"} + validator, "build_engine", return_value=failing_engine ), redirect_stdout(StringIO()) as output: rc = validator.run_validation(args) @@ -2697,7 +2707,9 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(rc, 0) self.assertEqual(len(commands), 1) - self.assertIn("--rdf-nt", commands[0]) + self.assertIn("--rdf", commands[0]) + self.assertIn("--rdf-format", commands[0]) + self.assertIn("nt", commands[0]) self.assertIn("/data/rdf/sample.nt", commands[0]) self.assertEqual(stage_result["rdf_format"], "nt") self.assertFalse(stage_result["temporary_rdf"]["decompressed_inside_container"]) diff --git a/test/validation_fixtures.py b/test/validation_fixtures.py new file mode 100644 index 0000000..458ffc7 --- /dev/null +++ b/test/validation_fixtures.py @@ -0,0 +1,611 @@ +"""One declarative fixture, three derived artifacts. + +The mutation harness needs a VCF, the RDF graph that VCF should convert to, and +the parser summary the validator's oracle should compute from it. Deriving all +three from a single specification below is what keeps them honest: a fixture +change cannot make the graph and the oracle disagree by accident. + +``test_validation_container_unit.py`` closes the loop in CI by running the real +``parse_vcf`` over :func:`write_vcf` output and asserting it equals +:func:`parser_summary`, so the hand-derived oracle is pinned to the real one. + +The records are chosen to exercise every branch the validation queries have: +a transition SNV, a transversion SNV, a deletion shape, a multi-allelic site, +a no-ALT site, PASS and non-PASS FILTER values, a missing genotype, a +non-diploid genotype, and a non-GT FORMAT field that nothing currently checks. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer" +XSD_POSITIVE_INTEGER = "http://www.w3.org/2001/XMLSchema#positiveInteger" + +def _runner(): + """Load the shipped validation runner (it lives outside the package path).""" + import importlib.util + + path = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + spec = importlib.util.spec_from_file_location("validation_runner_fixture", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +SOURCE_FILE = "fixture.vcf" +SAMPLES = ("HG001", "HG002") +FILE_FORMAT = "VCFv4.2" +FILE_DATE = "20260101" +SOURCE_SOFTWARE = "vcf-rdfizer-fixture" +REFERENCE_GENOME = "GRCh38" + +#: ``##`` meta-information lines, in file order. The ``#CHROM`` line is not one +#: of these; ``header_count`` below counts only the ``##`` lines. +HEADER_LINES: tuple[tuple[str, str], ...] = ( + ("fileformat", FILE_FORMAT), + ("fileDate", FILE_DATE), + ("source", SOURCE_SOFTWARE), + ("reference", REFERENCE_GENOME), + ("FILTER", ''), + ("INFO", ''), + ("INFO", ''), + ("FORMAT", ''), + ("FORMAT", ''), + # Exercise the structured header paths: a contig with every optional + # attribute, one with none, and a symbolic ALT declaration. + ("contig", ""), + ("contig", ""), + ("ALT", ''), + # An unrecognized key: it must keep only the base HeaderLine type. + ("phasing", "partial"), +) + + +@dataclass(frozen=True) +class FixtureRecord: + """One VCF data line and everything derived from it.""" + + row_id: str + chrom: str + pos: int + record_id: str + ref: str + alt: str + qual: str + filter_value: str + info: str + format_keys: tuple[str, ...] + #: One ``:``-joined payload per sample, aligned to ``format_keys``. + sample_payloads: tuple[str, ...] = field(default_factory=tuple) + + +RECORDS: tuple[FixtureRecord, ...] = ( + # Transition SNV, PASS, both samples called. + FixtureRecord("1", "20", 100, "rs100", "A", "G", "50", "PASS", "AC=1;DB", + ("GT", "DP"), ("0|1:30", "0/0:28")), + # Transversion SNV in the same 1 Mb window as record 1: a POS swap between + # these two is invisible to every aggregate query. + FixtureRecord("2", "20", 300, ".", "A", "C", "99", "PASS", "AC=2", + ("GT", "DP"), ("1/1:41", "0/1:35")), + # Deletion shape in a different window, failing FILTER, missing genotype. + FixtureRecord("3", "20", 2000200, "rs200", "AAT", "A", "12.5", "q10", "AC=1", + ("GT", "DP"), ("0/1:15", "./.:0")), + # Multi-allelic: excluded from q06, classified MULTIALLELIC by q02. + FixtureRecord("4", "21", 500, ".", "C", "T,G", ".", "PASS", "AC=1,1", + ("GT", "DP"), ("1/2:22", "0/1:19")), + # No ALT, haploid genotype: exercises NO_ALT and HAPLOID_* branches. + FixtureRecord("5", "21", 900, ".", "G", ".", "7", "q10", "DB", + ("GT", "DP"), ("0:11", "1:9")), + # A second transition SNV sharing record 1's contig and 1 Mb window. A + # REF/ALT permutation between records 1 and 6 leaves every aggregate + # identical, which is the blind spot the record digest is meant to close. + FixtureRecord("6", "20", 800, ".", "C", "T", "60", "PASS", "AC=1", + ("GT", "DP"), ("0/1:33", "0/1:31")), +) + +TRANSITIONS = {("A", "G"), ("G", "A"), ("C", "T"), ("T", "C")} + + +# --------------------------------------------------------------------------- +# VCF +# --------------------------------------------------------------------------- +def vcf_text() -> str: + lines = [f"##{key}={value}" for key, value in HEADER_LINES] + lines.append( + "#" + "\t".join( + ["CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", *SAMPLES] + ) + ) + for record in RECORDS: + lines.append("\t".join([ + record.chrom, str(record.pos), record.record_id, record.ref, record.alt, + record.qual, record.filter_value, record.info, + ":".join(record.format_keys), *record.sample_payloads, + ])) + return "\n".join(lines) + "\n" + + +def write_vcf(path: Path) -> Path: + path.write_text(vcf_text(), encoding="utf-8") + return path + + +def records_tsv_text() -> str: + """The `vcf_as_tsv.sh` output for this fixture, used by the RDF emitters.""" + header = ["SOURCE_FILE", "ROW_ID", "CHROM", "POS", "ID", "REF", "ALT", "QUAL", + "FILTER", "INFO", "FORMAT", " ".join(SAMPLES)] + rows = ["\t".join(header)] + for record in RECORDS: + rows.append("\t".join([ + SOURCE_FILE, record.row_id, record.chrom, str(record.pos), record.record_id, + record.ref, record.alt, record.qual, record.filter_value, record.info, + ":".join(record.format_keys), " ".join(record.sample_payloads), + ])) + return "\n".join(rows) + "\n" + + +def header_lines_tsv_text() -> str: + rows = ["\t".join(["SOURCE_FILE", "HEADER_INDEX", "HEADER_KEY", "HEADER_VALUE", "RAW_LINE"])] + for index, (key, value) in enumerate(HEADER_LINES, start=1): + rows.append("\t".join([SOURCE_FILE, str(index), key, value, f"{key}={value}"])) + return "\n".join(rows) + "\n" + + +# --------------------------------------------------------------------------- +# RDF graph +# --------------------------------------------------------------------------- +def _literal(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + if value == ".": + return f'"{escaped}"^^<{VCFR}Null>' + return f'"{escaped}"' + + +def _info_entries(record: "FixtureRecord") -> list[tuple[str, str | None]]: + """Split an INFO column into (key, value) pairs; a bare key is a Flag.""" + entries: list[tuple[str, str | None]] = [] + if record.info in ("", "."): + return entries + for item in record.info.split(";"): + if "=" in item: + key, value = item.split("=", 1) + entries.append((key, value)) + else: + entries.append((item, None)) + return entries + + +def base_triples(*, include_qual: bool = True, include_info: bool = True) -> list[str]: + """The triples RMLStreamer produces from `default_rules.ttl`. + + ``include_qual`` is on because the shipped mapping now emits ``vcfr:qual``; + turning it off models the pre-fix mapping, which is how the harness shows + that dropping QUAL is now detected. ``include_info`` models the structured + INFO representation. + """ + file_uri = f"file://{SOURCE_FILE}" + header_uri = f"{file_uri}#header" + out = [ + f"<{file_uri}> <{RDF_TYPE}> <{VCFR}VCFFile> .", + f"<{file_uri}> <{VCFR}fileFormat> {_literal(FILE_FORMAT)} .", + f"<{file_uri}> <{VCFR}sourceSoftware> {_literal(SOURCE_SOFTWARE)} .", + f"<{file_uri}> <{VCFR}referenceGenome> {_literal(REFERENCE_GENOME)} .", + + f"<{file_uri}> <{VCFR}hasHeader> <{header_uri}> .", + f"<{header_uri}> <{RDF_TYPE}> <{VCFR}VCFHeader> .", + ] + for index, (key, value) in enumerate(HEADER_LINES, start=1): + line_uri = f"{file_uri}#header/line/{index}" + out += [ + f"<{header_uri}> <{VCFR}hasHeaderLine> <{line_uri}> .", + f"<{line_uri}> <{RDF_TYPE}> <{VCFR}HeaderLine> .", + f"<{line_uri}> <{VCFR}headerKey> {_literal(key)} .", + f"<{line_uri}> <{VCFR}headerValue> {_literal(value)} .", + ] + for record in RECORDS: + record_uri = f"{file_uri}#record/{record.row_id}" + call_uri = f"{file_uri}#call/{record.row_id}" + out += [ + f"<{file_uri}> <{VCFR}hasRecord> <{record_uri}> .", + f"<{record_uri}> <{RDF_TYPE}> <{VCFR}VCFRecord> .", + f"<{record_uri}> <{VCFR}chrom> {_literal(record.chrom)} .", + f'<{record_uri}> <{VCFR}pos> "{record.pos}"^^<{XSD_INTEGER}> .', + f"<{record_uri}> <{VCFR}recordId> {_literal(record.record_id)} .", + f"<{record_uri}> <{VCFR}ref> {_literal(record.ref)} .", + f"<{record_uri}> <{VCFR}alt> {_literal(record.alt)} .", + f"<{record_uri}> <{VCFR}hasCall> <{call_uri}> .", + f"<{call_uri}> <{RDF_TYPE}> <{VCFR}VariantCall> .", + f"<{call_uri}> <{VCFR}filter> {_literal(record.filter_value)} .", + f"<{call_uri}> <{VCFR}infoRaw> {_literal(record.info)} .", + f"<{call_uri}> <{VCFR}formatRaw> {_literal(':'.join(record.format_keys))} .", + ] + return out + + +def build_graph( + representation: str = "expanded", *, include_qual: bool = True, + include_info: bool = True, include_headers: bool = True, +) -> str: + """Return the N-Triples graph for this fixture, sample triples included. + + The sample triples come from the project's own emitters rather than being + written out here, so the fixture tracks the real implementation. + """ + import tempfile + + import vcf_rdfizer + + if representation not in {"expanded", "condensed"}: + raise ValueError(f"unknown representation: {representation}") + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / f"{SOURCE_FILE}.records.tsv" + records_tsv.write_text(records_tsv_text(), encoding="utf-8") + headers_tsv = tmp_path / f"{SOURCE_FILE}.header_lines.tsv" + headers_tsv.write_text(header_lines_tsv_text(), encoding="utf-8") + + graph = tmp_path / "graph.nt" + graph.write_text( + "\n".join(base_triples(include_qual=include_qual, include_info=include_info)) + "\n", + encoding="utf-8", + ) + if include_qual or include_info: + vcf_rdfizer.append_record_detail_rdf( + records_tsv, headers_tsv, graph, + emit_qual=include_qual, emit_info=include_info, + progress_interval_records=0, + ) + if representation == "expanded": + vcf_rdfizer.append_expanded_sample_rdf(records_tsv, graph, progress_interval_records=0) + else: + vcf_rdfizer.append_condensed_sample_rdf( + records_tsv, headers_tsv, graph, progress_interval_records=0 + ) + if include_headers: + vcf_rdfizer.append_header_representation_rdf(headers_tsv, graph) + return graph.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Parser oracle +# --------------------------------------------------------------------------- +def _classify_shape(ref: str, alt: str) -> str: + import re + + ref, alt = ref.upper(), alt.upper() + if alt == ".": + return "NO_ALT" + if "," in alt: + return "MULTIALLELIC" + if alt == "*" or "[" in alt or "]" in alt or (alt.startswith("<") and alt.endswith(">")): + return "SYMBOLIC_OR_BREAKEND" + if not re.fullmatch(r"[ACGTN]+", ref) or not re.fullmatch(r"[ACGTN]+", alt): + return "OTHER" + if len(ref) == len(alt) == 1: + return "SNV" + if len(ref) == len(alt): + return "MNV_OR_EQUAL_LENGTH_SUBSTITUTION" + return "INSERTION_SHAPE" if len(ref) < len(alt) else "DELETION_SHAPE" + + +def _classify_genotype(raw: str) -> str: + normalized = raw.replace("|", "/") + if "." in normalized: + return "MISSING" + alleles = normalized.split("/") + if len(alleles) == 1: + return "HAPLOID_REF" if alleles[0] == "0" else "HAPLOID_ALT" + if len(alleles) == 2: + if alleles[0] == alleles[1]: + return "HOM_REF" if alleles[0] == "0" else "HOM_ALT" + return "HET" + return "OTHER_PLOIDY" + + +def _genotype_of(record: FixtureRecord, sample_index: int) -> str: + payload = record.sample_payloads[sample_index].split(":") + return payload[record.format_keys.index("GT")] + + +def _format_shape() -> dict: + """FORMAT counters the census needs, mirroring SampleRecordStream widening.""" + records_with_column = records_with_keys = 0 + occurrences = slots = non_empty = 0 + distinct: set[str] = set() + for record in RECORDS: + if record.format_keys: + records_with_column += 1 + payload_fields = [p.split(":") if p else [] for p in record.sample_payloads] + width = max([len(record.format_keys), *(len(f) for f in payload_fields)], default=0) + keys = [ + record.format_keys[i] if i < len(record.format_keys) and record.format_keys[i] + else f"FIELD_{i + 1}" + for i in range(width) + ] + if SAMPLES and keys: + records_with_keys += 1 + distinct.update(keys) + occurrences += width + slots += width * len(SAMPLES) + non_empty += sum( + 1 for f in payload_fields for i in range(width) + if i < len(f) and f[i] != "" + ) + return { + "recordsWithFormatColumn": records_with_column, + "recordsWithFormatKeys": records_with_keys, + "formatKeyOccurrences": occurrences, + "formatValueSlots": slots, + "nonEmptyFormatValues": non_empty, + "distinctFormatKeyCount": len(distinct), + } + + +def _header_shape() -> dict: + """Header counters the census needs, derived with the shipped parser.""" + runner = _runner() + classes: Counter[str] = Counter() + filters = alts = contigs = described = 0 + attributes: Counter[str] = Counter() + for key, value in HEADER_LINES: + class_name = runner.HEADER_LINE_CLASSES.get(key.lower()) + if class_name is None: + continue + classes[class_name] += 1 + if key.lower() not in {"filter", "alt", "contig"}: + continue + fields = runner.parse_structured_header_fields(value) + if not (fields.get("ID") or "").strip(): + continue + if key.lower() == "contig": + contigs += 1 + for attribute in ("length", "md5", "assembly"): + if fields.get(attribute): + attributes[attribute] += 1 + continue + if key.lower() == "filter": + filters += 1 + else: + alts += 1 + if fields.get("Description"): + described += 1 + return { + "headerLineClassCounts": dict(classes), + "filterDefinitionCount": filters, + "altDefinitionCount": alts, + "contigCount": contigs, + "contigAttributeCounts": dict(attributes), + "describedDefinitionCount": described, + } + + +def _info_shape() -> dict: + """INFO counters the census needs, mirroring the emitter's typing rules.""" + declared = {} + for key, value in HEADER_LINES: + if key != "INFO": + continue + import re as _re + + fields = dict(_re.findall(r'(\w+)=("[^"]*"|[^,>]*)', value)) + ident = fields.get("ID", "").strip() + if ident: + declared[ident] = fields.get("Type", "String").strip('"') + values = flags = typed_int = typed_dec = 0 + keys: set[str] = set() + for record in RECORDS: + for key, value in _info_entries(record): + values += 1 + keys.add(key) + if value is None: + flags += 1 + continue + if "," in value or value == ".": + continue + kind = declared.get(key, "String") + try: + if kind == "Integer": + int(value); typed_int += 1 + elif kind == "Float": + float(value); typed_dec += 1 + except ValueError: + pass + return { + "infoValueCount": values, + "infoDefinitionCount": len(keys), + "infoFlagCount": flags, + "infoTypedIntegerCount": typed_int, + "infoTypedDecimalCount": typed_dec, + } + + +def parser_summary( + representation: str = "expanded", *, include_qual: bool = True, + include_info: bool = True, include_headers: bool = True, +) -> dict: + """The summary ``parse_vcf`` must produce for this fixture. + + ``representation`` selects the census expectation, which depends on which + genotype emitter ran. ``include_qual`` mirrors ``build_graph`` so a graph + built without QUAL is compared against an expectation that also omits it. + """ + import re + + density: Counter[tuple[str, int]] = Counter() + shapes: Counter[str] = Counter() + filters: Counter[tuple[str, str]] = Counter() + genotypes: Counter[tuple[str, str]] = Counter() + ac_an: Counter[tuple[int, int]] = Counter() + transitions = transversions = biallelic_snvs = single_alt = q06_eligible = 0 + + for record in RECORDS: + density[(record.chrom, (record.pos - 1) // 1_000_000)] += 1 + shapes[_classify_shape(record.ref, record.alt)] += 1 + ref_upper, alt_upper = record.ref.upper(), record.alt.upper() + if (re.fullmatch(r"[ACGT]", ref_upper) and re.fullmatch(r"[ACGT]", alt_upper) + and ref_upper != alt_upper): + biallelic_snvs += 1 + if (ref_upper, alt_upper) in TRANSITIONS: + transitions += 1 + else: + transversions += 1 + status = ("PASS" if record.filter_value == "PASS" + else "NOT_APPLIED" if record.filter_value == "." else "FAILED") + filters[(status, record.filter_value)] += 1 + + for index, sample in enumerate(SAMPLES): + genotypes[(sample, _classify_genotype(_genotype_of(record, index)))] += 1 + + if record.alt != "." and "," not in record.alt: + single_alt += 1 + an = ac = 0 + for index in range(len(SAMPLES)): + alleles = _genotype_of(record, index).replace("|", "/").split("/") + if any(a == "." for a in alleles): + continue + if len(alleles) not in (1, 2) or any(a not in ("0", "1") for a in alleles): + continue + an += len(alleles) + ac += sum(int(a) for a in alleles) + if an: + ac_an[(an, ac)] += 1 + q06_eligible += 1 + + summary = { + "sampleCount": len(SAMPLES), + "samples": list(SAMPLES), + "totalRecords": len(RECORDS), + "gtRecordCount": sum(1 for record in RECORDS if "GT" in record.format_keys), + "singleAltRecordCount": single_alt, + "q06EligibleSiteCount": q06_eligible, + "headerLineCount": len(HEADER_LINES), + "headerValueCount": sum(1 for _key, value in HEADER_LINES if value != ""), + **_format_shape(), + **_info_shape(), + "fileFormat": FILE_FORMAT, + "referenceGenome": REFERENCE_GENOME, + "sourceSoftware": SOURCE_SOFTWARE, + "fileDate": FILE_DATE, + **_header_shape(), + "q07_file_metadata": { + "fileFormat": FILE_FORMAT, + "referenceGenome": REFERENCE_GENOME, + "sourceSoftware": SOURCE_SOFTWARE, + }, + "q08_header_line_census": [ + {"headerKey": key, "lineCount": count} + for key, count in sorted(Counter(key for key, _ in HEADER_LINES).items()) + ], + "q01_record_density_1mb": [ + {"chrom": chrom, "windowIndex": window, "recordCount": count} + for (chrom, window), count in sorted(density.items()) + ], + "q02_variant_shape_counts": [ + {"variantClass": name, "recordCount": count} for name, count in sorted(shapes.items()) + ], + "q03_titv": { + "biallelicSnvCount": biallelic_snvs, + "transitionCount": transitions, + "transversionCount": transversions, + "tiTvRatio": transitions / transversions if transversions else None, + }, + "q04_filter_distribution": [ + {"filterStatus": status, "filterLexical": lexical, "recordCount": count} + for (status, lexical), count in sorted(filters.items()) + ], + "q05_sample_genotype_counts": [ + {"sampleId": sample, "genotypeClass": genotype_class, "callCount": count} + for (sample, genotype_class), count in sorted(genotypes.items()) + ], + "q06_ac_an_distribution": [ + {"an": an, "ac": ac, "siteCount": count, "af": ac / an} + for (an, ac), count in sorted(ac_an.items()) + ], + } + # The census and digest expectations come from the shipped derivations + # rather than hand-written copies, so the fixture cannot drift from them. + runner = _runner() + summary.update(runner.expected_census( + summary, representation, + info_representation="structured" if include_info else "raw", + header_representation="structured" if include_headers else "basic", + )) + source_component = runner.rml_uri_component(SOURCE_FILE) + digest: Counter[str] = Counter() + for record in RECORDS: + record_iri = ( + f"file://{source_component}#record/" + f"{runner.rml_uri_component(record.row_id)}" + ) + digest[runner.record_digest_bucket([ + record_iri, record.chrom, str(record.pos), record.record_id, + record.ref, record.alt, + record.qual if include_qual else "", + record.filter_value, record.info, + ])] += 1 + summary["q11_record_digest"] = [ + {"bucket": bucket, "recordCount": count} for bucket, count in sorted(digest.items()) + ] + + # Value-level digests, derived the same way the runner derives them. + sample_components = [ + runner.rml_uri_component(uri) + for uri in runner.sample_uri_ids(list(SAMPLES)) + ] + info_digest: Counter[str] = Counter() + format_digest: Counter[str] = Counter() + for record in RECORDS: + row_component = runner.rml_uri_component(record.row_id) + call_iri = f"file://{source_component}#call/{row_component}" + if include_info: + for key, value in _info_entries(record): + if value is None: + continue + info_iri = f"{call_iri}/info/{runner.rml_uri_component(key)}" + info_digest[runner.record_digest_bucket([info_iri, value])] += 1 + payload_fields = [p.split(":") if p else [] for p in record.sample_payloads] + width = max([len(record.format_keys), *(len(f) for f in payload_fields)], default=0) + keys = [ + record.format_keys[i] if i < len(record.format_keys) and record.format_keys[i] + else f"FIELD_{i + 1}" + for i in range(width) + ] + for key_index, key in enumerate(keys): + key_component = runner.rml_uri_component(key) + if representation == "expanded": + for sample_index, fields in enumerate(payload_fields): + cell = fields[key_index] if key_index < len(fields) else "" + if not cell: + continue + value_iri = ( + f"file://{source_component}#sample/{row_component}" + f"/{sample_components[sample_index]}/fmt/{key_component}" + ) + format_digest[runner.record_digest_bucket([value_iri, cell])] += 1 + else: + encoded = "\t".join( + (fields[key_index] if key_index < len(fields) and fields[key_index] + else ".") + for fields in payload_fields + ) + vector_iri = f"{call_iri}/matrix/fmt/{key_component}" + format_digest[runner.record_digest_bucket([vector_iri, encoded])] += 1 + summary["q12_info_value_digest"] = [ + {"bucket": b, "valueCount": c} for b, c in sorted(info_digest.items()) + ] + summary["q13_format_value_digest"] = [ + {"bucket": b, "valueCount": c} for b, c in sorted(format_digest.items()) + ] + if not include_qual: + summary["q09_predicate_census"] = [ + row for row in summary["q09_predicate_census"] + if not row["predicate"].endswith("#qual") + ] + return summary diff --git a/test/validation_mutations.py b/test/validation_mutations.py new file mode 100644 index 0000000..21d8285 --- /dev/null +++ b/test/validation_mutations.py @@ -0,0 +1,474 @@ +"""A catalogue of graph mutations, and what the validator should make of each. + +Every entry names a specific way a conversion could be wrong, and declares +whether the validation suite catches it. Entries with ``known_undetected`` set +are recorded gaps: the harness asserts they are still *not* detected, so +closing a gap makes a test fail and forces both the catalogue and the coverage +documentation to be updated. That is what turns "coverage" into a number +instead of an opinion. + +Mutations operate on N-Triples text so they are engine-independent and can be +replayed under rdflib on the host or Comunica/QLever in the container. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable + +VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +FILE = "file://fixture.vcf" + + +# --------------------------------------------------------------------------- +# N-Triples editing helpers +# --------------------------------------------------------------------------- +def _lines(text: str) -> list[str]: + return [line for line in text.splitlines() if line.strip()] + + +def _join(lines: list[str]) -> str: + return "\n".join(lines) + "\n" + + +def drop_matching(text: str, *, subject: str | None = None, predicate: str | None = None, + limit: int | None = 1) -> str: + """Remove triples matching a subject and/or predicate.""" + out, removed = [], 0 + for line in _lines(text): + matches = ( + (subject is None or line.startswith(f"<{subject}> ")) + and (predicate is None or f"<{predicate}>" in line) + ) + if matches and (limit is None or removed < limit): + removed += 1 + continue + out.append(line) + if removed == 0: + raise AssertionError(f"mutation matched nothing (subject={subject}, predicate={predicate})") + return _join(out) + + +def replace_object(text: str, *, subject: str, predicate: str, new_object: str) -> str: + """Rewrite the object of the first triple matching subject+predicate.""" + out, done = [], False + for line in _lines(text): + if not done and line.startswith(f"<{subject}> ") and f"<{predicate}>" in line: + out.append(f"<{subject}> <{predicate}> {new_object} .") + done = True + continue + out.append(line) + if not done: + raise AssertionError(f"mutation matched nothing ({subject} {predicate})") + return _join(out) + + +def swap_objects(text: str, *, subject_a: str, subject_b: str, predicate: str) -> str: + """Exchange the objects of the same predicate between two subjects.""" + objects: dict[str, str] = {} + for line in _lines(text): + for subject in (subject_a, subject_b): + if line.startswith(f"<{subject}> ") and f"<{predicate}>" in line: + objects[subject] = line.split(f"<{predicate}>", 1)[1].rsplit(" .", 1)[0].strip() + if len(objects) != 2: + raise AssertionError(f"swap needs both subjects to have {predicate}") + text = replace_object(text, subject=subject_a, predicate=predicate, + new_object=objects[subject_b]) + return replace_object(text, subject=subject_b, predicate=predicate, + new_object=objects[subject_a]) + + +def append_lines(text: str, *new: str) -> str: + return _join(_lines(text) + list(new)) + + +def duplicate_subject(text: str, *, subject: str, new_subject: str) -> str: + """Copy every triple of a subject under a new IRI (a spurious extra record).""" + copies = [ + line.replace(f"<{subject}>", f"<{new_subject}>", 1) + for line in _lines(text) + if line.startswith(f"<{subject}> ") + ] + if not copies: + raise AssertionError(f"nothing to duplicate for {subject}") + return append_lines(text, *copies) + + +# --------------------------------------------------------------------------- +# Catalogue +# --------------------------------------------------------------------------- +@dataclass(frozen=True) +class Mutation: + """One way a converted graph can be wrong.""" + + id: str + description: str + #: Which part of the VCF this corrupts, for the coverage matrix. + vcf_element: str + apply: Callable[[str], str] + #: The check expected to catch it, for documentation. + expected_detected_by: str + #: Set when the suite provably cannot catch this yet, with the reason. + known_undetected: str | None = None + #: Representations this mutation is meaningful for. + representations: tuple[str, ...] = ("expanded", "condensed") + #: Fixture options the mutation needs present in the graph. A mutation that + #: targets a triple the shipped mapping does not yet emit still measures + #: something real - "if we emitted this, would we notice it breaking?" - so + #: the fixture is asked to include it rather than the mutation being skipped. + graph_options: tuple[tuple[str, bool], ...] = () + #: Census policy to validate under. A graph carrying triples the census does + #: not model yet would otherwise fail for that reason alone, masking whether + #: the mutation itself is detectable. + mapping_policy: str = "strict" + + +MUTATIONS: tuple[Mutation, ...] = ( + Mutation( + id="drop_record", + description="Remove every triple of one VCFRecord.", + vcf_element="record", + apply=lambda t: drop_matching(t, subject=f"{FILE}#record/1", limit=None), + expected_detected_by="q01/q02 record totals", + ), + Mutation( + id="drop_pos", + description="Remove one record's POS triple.", + vcf_element="POS", + apply=lambda t: drop_matching(t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos"), + expected_detected_by="preflight_record_cardinality", + ), + Mutation( + id="corrupt_pos", + description="Move a record into a different 1 Mb window.", + vcf_element="POS", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos", + new_object='"9100100"^^'), + expected_detected_by="q01_record_density_1mb", + ), + Mutation( + id="permute_pos", + description="Swap POS between two records in the same contig and 1 Mb window.", + vcf_element="record identity", + apply=lambda t: swap_objects( + t, subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/2", + predicate=f"{VCFR}pos"), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="permute_ref_alt", + description="Swap REF and ALT between two transition SNVs in the same window.", + vcf_element="record identity", + apply=lambda t: swap_objects( + swap_objects(t, subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/6", + predicate=f"{VCFR}ref"), + subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/6", + predicate=f"{VCFR}alt"), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="corrupt_chrom", + description="Change one record's CHROM.", + vcf_element="CHROM", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}chrom", new_object='"X"'), + expected_detected_by="q01_record_density_1mb", + ), + Mutation( + id="corrupt_alt", + description="Change one record's ALT so its shape class changes.", + vcf_element="ALT", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}alt", new_object='"GGG"'), + expected_detected_by="q02_variant_shape_counts", + ), + Mutation( + id="retype_pos_as_string", + description="Emit POS as a plain string instead of an integer.", + vcf_element="POS datatype", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos", new_object='"100"'), + expected_detected_by="preflight_position_datatype", + ), + Mutation( + id="duplicate_record", + description="Emit a second copy of a record under a new IRI.", + vcf_element="record", + apply=lambda t: duplicate_subject( + t, subject=f"{FILE}#record/1", new_subject=f"{FILE}#record/1-copy"), + expected_detected_by="q01/q02 record totals", + ), + Mutation( + id="introduce_blank_node", + description="Emit a record as a blank node instead of an IRI.", + vcf_element="graph integrity", + apply=lambda t: append_lines( + t, f'_:orphan <{VCFR}chrom> "20" .'), + expected_detected_by="preflight_blank_nodes", + ), + Mutation( + id="blank_node_object", + description="Point a record at a blank node instead of its call resource.", + vcf_element="graph integrity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}hasCall", + new_object="_:call1"), + expected_detected_by="preflight_blank_nodes", + ), + Mutation( + id="empty_literal", + description="Emit an empty literal where a value was expected.", + vcf_element="graph integrity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}chrom", new_object='""'), + expected_detected_by="preflight_empty_values", + ), + Mutation( + id="whitespace_only_literal", + description="Emit a literal containing only whitespace.", + vcf_element="graph integrity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/2", predicate=f"{VCFR}chrom", new_object='" "'), + expected_detected_by="preflight_empty_values", + ), + Mutation( + id="duplicate_triple", + description="Emit the same statement twice, as a duplicated RDF part would.", + vcf_element="graph integrity", + apply=lambda t: append_lines( + t, f'<{FILE}#record/1> <{VCFR}chrom> "20" .'), + expected_detected_by="preflight_duplicate_triples", + ), + Mutation( + id="duplicate_whole_graph", + description="Concatenate the graph with itself, as a duplicated part file would.", + vcf_element="graph integrity", + apply=lambda t: t + t, + expected_detected_by="preflight_duplicate_triples", + ), + Mutation( + id="spurious_predicate", + description="Add a triple using a predicate the vocabulary does not define.", + vcf_element="graph completeness", + apply=lambda t: append_lines( + t, f'<{FILE}#record/1> <{VCFR}notARealProperty> "x" .'), + expected_detected_by="q09_predicate_census (extra row)", + ), + Mutation( + id="drop_filter", + description="Remove one FILTER triple.", + vcf_element="FILTER", + apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFR}filter"), + expected_detected_by="q04_filter_distribution", + ), + Mutation( + id="corrupt_filter_lexical", + description="Change a FILTER value while keeping its broad status class.", + vcf_element="FILTER", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/3", predicate=f"{VCFR}filter", new_object='"q20"'), + expected_detected_by="q04_filter_distribution", + ), + Mutation( + id="plain_dot_literal", + description="Emit a missing token as a plain '.' instead of '.'^^vcfr:Null.", + vcf_element="missing-value policy", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/2", predicate=f"{VCFR}recordId", new_object='"."'), + expected_detected_by="preflight_missing_token_conformance (--strict-conformance)", + ), + Mutation( + id="flip_genotype", + description="Change one sample's GT value.", + vcf_element="FORMAT/GT", + apply=lambda t: replace_object( + t, subject=f"{FILE}#sample/1/HG001/fmt/GT", predicate=f"{VCFR}fieldValue", + new_object='"1/1"'), + expected_detected_by="q05_sample_genotype_counts", + representations=("expanded",), + ), + Mutation( + id="drop_sample_call", + description="Remove one SampleCall entirely.", + vcf_element="sample call", + apply=lambda t: drop_matching(t, subject=f"{FILE}#sample/1/HG001", limit=None), + expected_detected_by="q05 per-sample totals", + representations=("expanded",), + ), + Mutation( + id="drop_format_value_dp", + description="Remove a non-GT FORMAT value node (DP).", + vcf_element="FORMAT/DP", + apply=lambda t: drop_matching(t, subject=f"{FILE}#sample/1/HG001/fmt/DP", limit=None), + expected_detected_by="q09_predicate_census", + representations=("expanded",), + ), + Mutation( + id="corrupt_format_value_dp", + description="Change a DP value.", + vcf_element="FORMAT/DP", + apply=lambda t: replace_object( + t, subject=f"{FILE}#sample/1/HG001/fmt/DP", predicate=f"{VCFR}fieldValue", + new_object='"999"'), + expected_detected_by="q13_format_value_digest", + representations=("expanded",), + ), + Mutation( + id="drop_qual", + description="Remove a QUAL triple.", + vcf_element="QUAL", + apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFR}qual"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_all_qual", + description="Emit no QUAL at all, as the mapping did before it was fixed.", + vcf_element="QUAL", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}qual", limit=None), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_qual", + description="Change a QUAL value.", + vcf_element="QUAL", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1", predicate=f"{VCFR}qual", new_object='"0"'), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="drop_info_value", + description="Remove a structured INFO value node.", + vcf_element="INFO", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}hasInfoValue"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_info_value", + description="Change a structured INFO value.", + vcf_element="INFO", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1/info/AC", predicate=f"{VCFR}fieldValue", + new_object='"99"'), + expected_detected_by="q12_info_value_digest", + ), + Mutation( + id="drop_info_definition", + description="Remove an INFO field declaration resource.", + vcf_element="INFO declaration", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#header/line/6", predicate=f"{VCFR}fieldType"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="retype_info_value", + description="Drop the typed integer form of an INFO value.", + vcf_element="INFO typing", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}fieldValueInteger"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_info_raw", + description="Change a record's raw INFO string.", + vcf_element="INFO", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1", predicate=f"{VCFR}infoRaw", new_object='"AC=99"'), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="corrupt_format_vector", + description="Change one sample's value inside a condensed FORMAT vector.", + vcf_element="FORMAT/DP", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1/matrix/fmt/DP", predicate=f"{VCFR}encodedValues", + new_object='"999\t28"'), + expected_detected_by="q13_format_value_digest", + representations=("condensed",), + ), + Mutation( + id="drop_header_line", + description="Remove one HeaderLine resource.", + vcf_element="header line", + apply=lambda t: drop_matching(t, subject=f"{FILE}#header/line/5", limit=None), + expected_detected_by="q08_header_line_census (Phase 1c)", + ), + Mutation( + id="untype_header_line", + description="Strip a header line's vocabulary subclass.", + vcf_element="header line typing", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#header/line/10", + predicate="http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + expected_detected_by="q10_class_census", + ), + Mutation( + id="drop_contig_attribute", + description="Remove a contig's declared length.", + vcf_element="contig declaration", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}contigLength"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_filter_definition", + description="Remove a FILTER declaration's id.", + vcf_element="FILTER declaration", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}filterId"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_alt_definition", + description="Remove a symbolic ALT declaration's id.", + vcf_element="ALT declaration", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}altId"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_file_date", + description="Remove the declared file date.", + vcf_element="file metadata", + apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFR}fileDate"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_contig_count", + description="Declare the wrong number of contigs.", + vcf_element="contig declaration", + apply=lambda t: replace_object( + t, subject=FILE, predicate=f"{VCFR}contigCount", + new_object='"99"^^'), + expected_detected_by="header census", + known_undetected=( + "contigCount is a derived scalar: the census counts that one triple " + "exists but never reads its value." + ), + ), + Mutation( + id="corrupt_file_metadata", + description="Change the declared fileformat.", + vcf_element="file metadata", + apply=lambda t: replace_object( + t, subject=FILE, predicate=f"{VCFR}fileFormat", new_object='"VCFv9.9"'), + expected_detected_by="q07_file_metadata (Phase 1c)", + ), + Mutation( + id="drop_reference_genome", + description="Remove the declared reference genome.", + vcf_element="file metadata", + apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFR}referenceGenome"), + expected_detected_by="q07_file_metadata (Phase 1c)", + ), + Mutation( + id="wrong_representation_profile", + description="Declare the wrong sample representation profile.", + vcf_element="representation profile", + apply=lambda t: replace_object( + t, subject=FILE, predicate=f"{VCFR}representationProfile", + new_object=f"<{VCFR}ExpandedRepresentation>"), + expected_detected_by="preflight_representation_profile", + representations=("condensed",), + ), +) + + +def for_representation(representation: str) -> tuple[Mutation, ...]: + return tuple(m for m in MUTATIONS if representation in m.representations) diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index bb0b2cb..8571b7b 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -315,6 +315,29 @@ # line starting with one of these bytes and ending in " ." is a statement. _NTRIPLES_SUBJECT_STARTS = (b"<", b"_") SAMPLE_REPRESENTATION_CHOICES = {"expanded", "condensed"} +# How the INFO column is represented. "raw" is the historical behaviour (an +# opaque vcfr:infoRaw string). "structured" additionally emits one +# vcfr:InfoFieldValue per record and key, which is what makes INFO queryable. +INFO_REPRESENTATION_CHOICES = ("raw", "structured") +DEFAULT_INFO_REPRESENTATION = "structured" +# Semantic validation accepts any artifact the pipeline can produce. Anything +# that is not already plain N-Triples is decoded inside the container first. +VALIDATION_RDF_SUFFIXES = ( + (".nt.gz", "nt.gz"), + (".nt.br", "nt.br"), + (".nt", "nt"), + (".cottas.gz", "cottas.gz"), + (".cottas.br", "cottas.br"), + (".cottas", "cottas"), + (".hdt", "hdt"), +) +VALIDATION_ENGINE_CHOICES = ("comunica", "qlever") +DEFAULT_VALIDATION_ENGINE = "comunica" +# Which produced artifacts a full run should semantically validate. "aggregate" +# is the .nt/.nt.gz RMLStreamer output; the others are the selected +# representations, each decoded back to N-Triples before it is checked. +VALIDATION_TARGET_CHOICES = ("aggregate", "hdt", "cottas") +DEFAULT_VALIDATION_TARGETS = "aggregate" # This is an internal rules-compatibility value, not a third public # representation. It means that custom helper TSV rows must be materialized. SAMPLE_HELPER_STRATEGY_MATERIALIZED = "expanded" @@ -1870,6 +1893,9 @@ class ParsedSampleRecord: source_file: str row_id: str + #: Raw QUAL and INFO columns, needed by the record-detail emitter. + qual: str + info: str format_keys: tuple[str, ...] sample_payloads: tuple[str, ...] sample_values: tuple[tuple[str, ...], ...] @@ -1958,6 +1984,8 @@ def _parse_row(self, row: list[str]) -> ParsedSampleRecord: f"'{source_file}'" ) row_id = row[1] if len(row) > 1 else "" + qual_raw = row[7] if len(row) > 7 else "" + info_raw = row[9] if len(row) > 9 else "" format_raw = row[10] if len(row) > 10 else "" samples_raw = row[-1] if len(row) >= 12 else "" declared_format_keys = format_raw.split(":") if format_raw else [] @@ -1999,6 +2027,8 @@ def _parse_row(self, row: list[str]) -> ParsedSampleRecord: return ParsedSampleRecord( source_file=source_file, row_id=row_id, + qual=qual_raw, + info=info_raw, format_keys=format_keys, sample_payloads=tuple(sample_payloads), sample_values=sample_values, @@ -2165,6 +2195,9 @@ class FormatDefinition: uri: str field_number: str description: str + #: The declared VCF Type (Integer/Float/Flag/Character/String). Defaults to + #: String so an undeclared field still has a usable value type. + value_type: str = "String" def _parse_structured_header_fields(value: str) -> dict[str, str]: @@ -2207,36 +2240,446 @@ def _parse_structured_header_fields(value: str) -> dict[str, str]: return fields -def _load_format_definitions(header_lines_tsv: Path) -> dict[str, FormatDefinition]: - """Map FORMAT IDs to structured definitions backed by emitted HeaderLine IRIs.""" +def _load_field_definitions( + header_lines_tsv: Path, header_key: str +) -> dict[str, FormatDefinition]: + """Map declared field IDs to structured definitions backed by HeaderLine IRIs. + + Serves both ``##FORMAT`` and ``##INFO`` declarations, which share the + ```` syntax and the same vocabulary shape + (``vcfr:FieldDefinition`` with ``fieldId``/``fieldNumber``/``fieldType``). + """ definitions: dict[str, FormatDefinition] = {} if not header_lines_tsv.is_file(): return definitions + wanted = header_key.upper() _set_max_csv_field_size() with header_lines_tsv.open(newline="", encoding="utf-8") as handle: for row in csv.DictReader(handle, delimiter="\t"): - if (row.get("HEADER_KEY") or "").upper() != "FORMAT": + if (row.get("HEADER_KEY") or "").upper() != wanted: continue fields = _parse_structured_header_fields(row.get("HEADER_VALUE") or "") - format_id = fields.get("ID", "").strip() - if not format_id: + field_id = fields.get("ID", "").strip() + if not field_id: continue source_component = _rml_uri_component(row.get("SOURCE_FILE") or "") index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") definitions.setdefault( - format_id, + field_id, FormatDefinition( uri=f"file://{source_component}#header/line/{index_component}", field_number=fields.get("Number") or ".", description=( fields.get("Description") - or f"FORMAT field {format_id} (source declaration has no Description)" + or f"{wanted} field {field_id} (source declaration has no Description)" ), + value_type=fields.get("Type") or "String", ), ) return definitions +def _load_format_definitions(header_lines_tsv: Path) -> dict[str, FormatDefinition]: + """Backward-compatible accessor for the FORMAT declarations.""" + return _load_field_definitions(header_lines_tsv, "FORMAT") + + +#: Maps a declared VCF INFO/FORMAT Type to its vocabulary value-type class. +VCF_VALUE_TYPE_CLASSES = { + "Integer": "IntegerType", + "Float": "FloatType", + "Flag": "FlagType", + "Character": "CharacterType", + "String": "StringType", +} +#: Typed value predicates used when a single-valued field declares a numeric or +#: flag type. Multi-valued fields (Number=A/R/G/.) keep only the lexical value, +#: because the vocabulary gives one InfoFieldValue node per key. +XSD_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#integer" +XSD_DECIMAL_URI = "http://www.w3.org/2001/XMLSchema#decimal" +XSD_BOOLEAN_URI = "http://www.w3.org/2001/XMLSchema#boolean" + + +def parse_info_entries(info: str) -> list[tuple[str, str | None]]: + """Split an INFO column into ``(key, value)`` pairs. + + An entry without ``=`` is a Flag: a presence assertion with no value, which + the vocabulary models as ``vcfr:fieldValueBoolean true``. + """ + if info in ("", "."): + return [] + entries: list[tuple[str, str | None]] = [] + for item in info.split(";"): + if not item: + continue + key, separator, value = item.partition("=") + entries.append((key, value if separator else None)) + return entries + + +def _typed_info_object(value: str, declared_type: str) -> tuple[str, str] | None: + """Return ``(predicate_local_name, literal)`` for a typed single value.""" + if "," in value or value == ".": + return None + try: + if declared_type == "Integer": + return "fieldValueInteger", f'"{int(value)}"^^<{XSD_INTEGER_URI}>' + if declared_type == "Float": + # Serialize the source lexical form rather than a reparsed float, so + # the graph never gains or loses precision relative to the VCF. + float(value) + return "fieldValueDecimal", f'"{value}"^^<{XSD_DECIMAL_URI}>' + except ValueError: + # A value that contradicts its declared type is kept as a plain literal + # rather than dropped; the conversion must not silently lose data. + return None + return None + + +# How the VCF meta-information block is represented. "basic" is the historical +# behaviour: every '##' line becomes an untyped vcfr:HeaderLine carrying its raw +# key and value. "structured" additionally types each line with the vocabulary's +# subclass and lifts the attributes of FILTER, ALT and contig declarations into +# their own properties, so the header becomes queryable rather than just present. +HEADER_REPRESENTATION_CHOICES = ("basic", "structured") +DEFAULT_HEADER_REPRESENTATION = "structured" + +#: '##' key (lower-cased) -> the vocabulary subclass for that line. +HEADER_LINE_CLASSES = { + "fileformat": "FileFormatHeaderLine", + "filedate": "FileDateHeaderLine", + "source": "SourceHeaderLine", + "reference": "ReferenceHeaderLine", + "info": "INFOHeaderLine", + "format": "FORMATHeaderLine", + "filter": "FILTERHeaderLine", + "alt": "ALTHeaderLine", + "contig": "ContigHeaderLine", +} +#: contig attribute -> vocabulary predicate. +CONTIG_ATTRIBUTES = { + "length": "contigLength", + "md5": "contigMd5", + "assembly": "contigAssembly", +} + + +XSD_DATE_URI = "http://www.w3.org/2001/XMLSchema#date" +#: ##fileDate has no mandated format. These are the two forms seen in practice +#: that map unambiguously onto xsd:date, which the SHACL shape requires. +FILE_DATE_PATTERNS = ( + (re.compile(r"^(\d{4})(\d{2})(\d{2})$"), "{0}-{1}-{2}"), + (re.compile(r"^(\d{4})-(\d{2})-(\d{2})$"), "{0}-{1}-{2}"), +) + + +def file_date_object(value: str) -> str | None: + """Serialize ##fileDate as xsd:date when its form allows, else lexically. + + Returns None for an absent value so no triple is emitted, matching RML's + behaviour for an empty reference. + """ + value = (value or "").strip() + if not value or value == ".": + return None + for pattern, template in FILE_DATE_PATTERNS: + match = pattern.match(value) + if match: + return f'"{template.format(*match.groups())}"^^<{XSD_DATE_URI}>' + # An unrecognized form is preserved verbatim rather than dropped; the SHACL + # layer reports it as non-conformant. + return _ntriples_string_literal(value) + + +def append_header_representation_rdf( + header_lines_tsv: Path, + rdf_path: Path, +) -> dict: + """Append typed header lines and structured FILTER/ALT/contig declarations. + + The default mapping emits every '##' line as an untyped ``vcfr:HeaderLine`` + with a raw key and value. The vocabulary already defines a subclass per line + type and dedicated properties for the FILTER, ALT and contig attributes; + this emits them, which is what makes the meta-information block queryable. + + Emitted directly rather than through RML because the attributes live inside + a single ```` value that RML cannot decompose. + """ + stats = { + "representation": "header", + "header_lines": 0, + "typed_lines": 0, + "filter_definitions": 0, + "alt_definitions": 0, + "contigs": 0, + "file_dates": 0, + "triples": 0, + "appended_bytes": 0, + } + if not header_lines_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for header streaming: {rdf_path}") + + _set_max_csv_field_size() + with header_lines_tsv.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle, delimiter="\t")) + + def produce(emit): + source_file = rows[0].get("SOURCE_FILE", "") if rows else "" + file_uri = f"file://{_rml_uri_component(source_file)}" + contig_count = 0 + for row in rows: + key = (row.get("HEADER_KEY") or "").strip() + value = row.get("HEADER_VALUE") or "" + index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") + line_uri = f"file://{_rml_uri_component(row.get('SOURCE_FILE') or '')}" \ + f"#header/line/{index_component}" + stats["header_lines"] += 1 + + line_class = HEADER_LINE_CLASSES.get(key.lower()) + if line_class is None: + # An unrecognized '##' key keeps only the base HeaderLine type + # the mapping already emitted; inventing a subclass for it would + # put a term in the graph that the vocabulary does not define. + continue + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}{line_class}> .\n") + stats["typed_lines"] += 1 + + if key.lower() not in {"filter", "alt", "contig"}: + continue + fields = _parse_structured_header_fields(value) + identifier = (fields.get("ID") or "").strip() + if not identifier: + continue + description = fields.get("Description") + + if key.lower() == "filter": + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FilterDefinition> .\n") + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}filterId> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + stats["filter_definitions"] += 1 + elif key.lower() == "alt": + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}AltDefinition> .\n") + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}altId> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + stats["alt_definitions"] += 1 + else: + contig_count += 1 + stats["contigs"] += 1 + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}contigId> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + for attribute, predicate in CONTIG_ATTRIBUTES.items(): + attribute_value = fields.get(attribute) + if attribute_value: + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}{predicate}> " + f"{_ntriples_string_literal(attribute_value)} .\n" + ) + continue + + if description: + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}fieldDescription> " + f"{_ntriples_string_literal(description)} .\n" + ) + + for row in rows: + if (row.get("HEADER_KEY") or "").strip().lower() != "filedate": + continue + date_object = file_date_object(row.get("HEADER_VALUE") or "") + if date_object is not None: + emit(f"<{file_uri}> <{VCFR_NAMESPACE}fileDate> {date_object} .\n") + stats["file_dates"] += 1 + break + + if contig_count: + emit( + f"<{file_uri}> <{VCFR_NAMESPACE}contigCount> " + f'"{contig_count}"^^<{XSD_INTEGER_URI}> .\n' + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def _qual_object(value: str) -> str: + """Serialize QUAL as the SHACL shape requires: xsd:decimal, or vcfr:Null. + + The shape is ``sh:or([sh:datatype xsd:decimal] [sh:datatype vcfr:Null])``, + which RML cannot satisfy because the datatype depends on the row. A value + that is neither numeric nor the missing token is kept as a plain literal + rather than dropped: a non-conformant graph is more useful than a lossy one, + and the SHACL layer reports it. + """ + if value in ("", "."): + return f'"."^^<{VCFR_NAMESPACE}Null>' + try: + float(value) + except ValueError: + return _ntriples_string_literal(value) + # Serialize the source lexical form so no precision is gained or lost. + return f'"{value}"^^<{XSD_DECIMAL_URI}>' + + +def append_record_detail_rdf( + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, + *, + emit_qual: bool = True, + emit_info: bool = True, + progress_interval_records: int = 10_000, +) -> dict: + """Append per-record detail the RML mapping cannot express. + + Covers QUAL (whose datatype depends on the value) and the structured INFO + representation (which would otherwise need a materialized helper table of + variants x INFO keys). Both are per-record, so they share one pass over + ``records.tsv``. + + The default mapping emits INFO only as an opaque ``vcfr:infoRaw`` string. + This adds the structured form the vocabulary already defines - one + ``vcfr:InfoFieldValue`` per record and key, at the IRI template the + vocabulary declares - so INFO becomes queryable rather than just present. + + Emitted directly rather than through RML for the same reason the genotype + representations are: RML would need a materialized helper table of + variants x INFO keys, and it cannot switch value datatype on a declared + ``Type``. + """ + stats = { + "representation": "record-detail", + "records": 0, + "qual_values": 0, + "info_values": 0, + "info_definitions": 0, + "triples": 0, + "appended_bytes": 0, + } + if not records_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for INFO streaming: {rdf_path}") + + definitions = _load_field_definitions(header_lines_tsv, "INFO") + + with SampleRecordStream(records_tsv) as record_stream: + if not record_stream.source_file: + return stats + + def produce(emit): + source_component = _rml_uri_component(record_stream.source_file) + file_uri = f"file://{source_component}" + emitted_definitions: set[str] = set() + key_components: dict[str, str] = {} + value_count = 0 + definition_count = 0 + + qual_count = 0 + for record in record_stream: + row_component = _rml_uri_component(record.row_id) + call_uri = f"{file_uri}#call/{row_component}" + if emit_qual: + emit( + f"<{call_uri}> <{VCFR_NAMESPACE}qual> " + f"{_qual_object(record.qual)} .\n" + ) + qual_count += 1 + for key, value in (parse_info_entries(record.info) if emit_info else ()): + key_component = key_components.get(key) + if key_component is None: + key_component = _rml_uri_component(key) + key_components[key] = key_component + info_uri = f"{call_uri}/info/{key_component}" + + definition = definitions.get(key) + if definition is None: + definition = FormatDefinition( + uri=f"{file_uri}#header/info/{key_component}", + field_number=".", + description=( + f"Synthesized definition for undeclared INFO key {key}" + ), + value_type="Flag" if value is None else "String", + ) + if definition.uri not in emitted_definitions: + emitted_definitions.add(definition.uri) + definition_count += 1 + type_class = VCF_VALUE_TYPE_CLASSES.get( + definition.value_type, "StringType" + ) + emit( + f"<{definition.uri}> <{RDF_TYPE_URI}> " + f"<{VCFR_NAMESPACE}InfoFieldDefinition> .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldId> " + f"{_ntriples_string_literal(key)} .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldNumber> " + f"{_ntriples_string_literal(definition.field_number)} .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldDescription> " + f"{_ntriples_string_literal(definition.description)} .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldType> " + f"<{VCFR_NAMESPACE}{type_class}> .\n" + ) + + emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasInfoValue> <{info_uri}> .\n") + emit( + f"<{info_uri}> <{RDF_TYPE_URI}> " + f"<{VCFR_NAMESPACE}InfoFieldValue> .\n" + ) + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}declaredBy> " + f"<{definition.uri}> .\n" + ) + if value is None: + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}fieldValueBoolean> " + f'"true"^^<{XSD_BOOLEAN_URI}> .\n' + ) + else: + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}fieldValue> " + f"{_ntriples_literal(value)} .\n" + ) + typed = _typed_info_object(value, definition.value_type) + if typed is not None: + predicate, literal = typed + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}{predicate}> " + f"{literal} .\n" + ) + value_count += 1 + + stats["records"] += 1 + stats["qual_values"] = qual_count + stats["info_values"] = value_count + stats["info_definitions"] = definition_count + if ( + progress_interval_records > 0 + and stats["records"] % progress_interval_records == 0 + ): + print( + f" * INFO RDF streaming: {stats['records']:,} variants, " + f"{value_count:,} values", + flush=True, + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + def append_condensed_sample_rdf( records_tsv: Path, header_lines_tsv: Path, @@ -2378,6 +2821,26 @@ def produce(emit): return _append_rdf_atomically(rdf_path, stats, produce) +def emit_record_detail( + info_representation: str, + *, + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, +) -> dict | None: + """Append QUAL and, when selected, the structured INFO representation. + + QUAL is always emitted: the RML mapping cannot type it per row, so this is + the only place it can come from. + """ + if info_representation not in INFO_REPRESENTATION_CHOICES: + raise ValueError(f"unknown INFO representation: {info_representation}") + return append_record_detail_rdf( + records_tsv, header_lines_tsv, rdf_path, + emit_qual=True, emit_info=info_representation == "structured", + ) + + def emit_sample_representation( workflow: SampleWorkflow, *, @@ -4663,7 +5126,14 @@ def run_full_mode( image_ref: str, out_name: str, sample_workflow: SampleWorkflow, + info_representation: str = DEFAULT_INFO_REPRESENTATION, + header_representation: str = DEFAULT_HEADER_REPRESENTATION, run_validation: bool = False, + validation_artifacts: list[str] | None = None, + validation_engine: str = DEFAULT_VALIDATION_ENGINE, + validation_engine_options: dict | None = None, + validation_strict_conformance: bool = False, + validation_shacl_shapes: Path | None = None, filter_oracle: str = "auto", rdf_storage_mode: str, methods: list[str], @@ -4688,6 +5158,13 @@ def run_full_mode( if spark_partitions is not None: print(f" Spark partition hint: {spark_partitions}") print(f" Sample representation: {sample_workflow.representation}") + print(f" INFO representation: {info_representation}") + print(f" Header representation: {header_representation}") + validation_artifacts = list(validation_artifacts or ["aggregate"]) + if run_validation: + print( + f" Validation: {', '.join(validation_artifacts)} via {validation_engine}" + ) intermediate_dir = tsv_dir.parent ensure_dir(tsv_dir) ensure_dir(out_dir) @@ -5041,8 +5518,59 @@ def fail_current(stage: str, message: str): f"{sample_stats['samples']:,}" ) + header_stats = None + if header_representation != "basic": + print(f" * Streaming {header_representation} header RDF") + try: + header_stats = append_header_representation_rdf( + triplet["headers"], raw_rdf_files[0] + ) + except Exception as exc: + fail_current( + "header-rdf-streaming", + f"failed streaming {header_representation} header RDF for " + f"'{prefix}': {exc}. See log: {wrapper_log_path}", + ) + continue + if triples_produced is not None: + triples_produced += int(header_stats["triples"]) + print( + f" * Header lines typed: {header_stats['typed_lines']:,}; " + f"contigs: {header_stats['contigs']:,}" + ) + + info_stats = None + if True: + print(f" * Streaming QUAL and {info_representation} INFO RDF") + try: + info_stats = emit_record_detail( + info_representation, + records_tsv=triplet["records"], + header_lines_tsv=triplet["headers"], + rdf_path=raw_rdf_files[0], + ) + except Exception as exc: + fail_current( + "record-detail-rdf-streaming", + f"failed streaming QUAL/{info_representation} INFO RDF for " + f"'{prefix}': {exc}. See log: {wrapper_log_path}", + ) + continue + if info_stats is not None: + if triples_produced is not None: + triples_produced += int(info_stats["triples"]) + print( + f" * QUAL values streamed: {info_stats['qual_values']:,}; " + f"INFO values: {info_stats['info_values']:,}; " + f"declarations: {info_stats['info_definitions']:,}" + ) + if triples_produced is None: triples_produced = count_triples_in_nt_files(raw_rdf_files) + if header_stats is not None: + sample_stats = {**(sample_stats or {}), "header": header_stats} + if info_stats is not None: + sample_stats = {**(sample_stats or {}), "info": info_stats} if sample_stats is not None and triples_produced is not None: update_conversion_metrics_after_sample_stream( metrics_dir=metrics_dir, @@ -5133,67 +5661,108 @@ def fail_current(stage: str, message: str): run_tracker.mark(f"Input {idx}: compression completed for {output_name}") if run_validation: - validation_rdf_path = raw_rdf_files[0] - validation_results_dir = ( - metrics_dir / "reports" / "validation" / safe_metrics_name(output_name) + # Each requested target is validated independently, so a run can + # prove that the aggregate, the HDT, and the COTTAS artifact all + # reproduce the same VCF summaries. `validation_result` keeps the + # aggregate's report in the metrics row for backward compatibility; + # every target also gets its own stage report and results tree. + validation_targets = resolve_validation_targets( + requested=validation_artifacts, + output_dir=out_dir / output_name, + output_name=output_name, + aggregate_path=raw_rdf_files[0], + selected_methods=selected_methods, ) - print(f" * Semantic validation: {output_name}") - validation_result = { - "run_id": run_id, - "timestamp": timestamp, - "stage": "validation", - "validation_id": output_name, - "vcf_path": str(input_vcf), - "rdf_path": str(validation_rdf_path), - "rdf_format": "nt.gz" if validation_rdf_path.name.endswith(".nt.gz") else "nt", - "representation": sample_workflow.representation, - "results_dir": str(validation_results_dir), - "summary_path": str(validation_results_dir / "summary.json"), - "input_rdf_size_bytes": int(file_size_bytes(validation_rdf_path) or 0), - "exit_code": 1, - "status": "EXECUTION_FAILED", - "timing": { - "wall_seconds": None, - "user_seconds": None, - "sys_seconds": None, - "max_rss_kb": None, - }, - "temporary_rdf": { - "decompressed_inside_container": validation_rdf_path.name.endswith(".nt.gz"), - "persisted_on_host": False, - "cleanup_confirmed_by_runner": False, - }, - } - try: - validation_exit_code = run_validation_mode( - vcf_path=Path(input_vcf), - rdf_path=validation_rdf_path, - representation=sample_workflow.representation, - validation_id=output_name, - results_dir=validation_results_dir, - metrics_dir=metrics_dir, - run_id=run_id, - timestamp=timestamp, - image_ref=image_ref, - filter_oracle=filter_oracle, - wrapper_log_path=wrapper_log_path, - run_tracker=run_tracker, - stage_result=validation_result, + validation_failures: list[str] = [] + for target in validation_targets: + target_id = ( + output_name + if target["name"] == "aggregate" + else f"{output_name}__{target['name']}" ) - except Exception as exc: - validation_result["error"] = str(exc) - try: - stage_path = metrics_dir / "stages" / "validation" / f"{safe_metrics_name(output_name)}.json" - stage_path.parent.mkdir(parents=True, exist_ok=True) - stage_path.write_text(json.dumps(validation_result, indent=2) + "\n", encoding="utf-8") - except OSError: - pass - eprint( - f"Error: semantic validation could not be completed for '{output_name}': " - f"{exc}. See log: {wrapper_log_path}" + safe_target_id = safe_metrics_name(target_id) + target_results_dir = ( + metrics_dir / "reports" / "validation" / safe_target_id ) - validation_exit_code = 1 - validation_failed = int(validation_exit_code) != 0 + print( + f" * Semantic validation ({target['name']}, {validation_engine}): " + f"{target['path'].name}" + ) + target_result = { + "run_id": run_id, + "timestamp": timestamp, + "stage": "validation", + "validation_id": target_id, + "validation_target": target["name"], + "vcf_path": str(input_vcf), + "rdf_path": str(target["path"]), + "rdf_format": target["format"], + "engine": validation_engine, + "representation": sample_workflow.representation, + "results_dir": str(target_results_dir), + "summary_path": str(target_results_dir / "summary.json"), + "input_rdf_size_bytes": int(file_size_bytes(target["path"]) or 0), + "exit_code": 1, + "status": "EXECUTION_FAILED", + "timing": { + "wall_seconds": None, + "user_seconds": None, + "sys_seconds": None, + "max_rss_kb": None, + }, + "temporary_rdf": { + "decompressed_inside_container": target["format"] != "nt", + "persisted_on_host": False, + "cleanup_confirmed_by_runner": False, + }, + } + try: + target_exit_code = run_validation_mode( + vcf_path=Path(input_vcf), + rdf_path=target["path"], + rdf_format=target["format"], + representation=sample_workflow.representation, + validation_id=target_id, + results_dir=target_results_dir, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + image_ref=image_ref, + filter_oracle=filter_oracle, + engine=validation_engine, + engine_options=validation_engine_options, + strict_conformance=validation_strict_conformance, + shacl_shapes=validation_shacl_shapes, + wrapper_log_path=wrapper_log_path, + run_tracker=run_tracker, + stage_result=target_result, + ) + except Exception as exc: + target_result["error"] = str(exc) + try: + stage_path = ( + metrics_dir / "stages" / "validation" / f"{safe_target_id}.json" + ) + stage_path.parent.mkdir(parents=True, exist_ok=True) + stage_path.write_text( + json.dumps(target_result, indent=2) + "\n", encoding="utf-8" + ) + except OSError: + pass + eprint( + f"Error: semantic validation could not be completed for " + f"'{target_id}': {exc}. See log: {wrapper_log_path}" + ) + target_exit_code = 1 + if target["name"] == "aggregate": + validation_result = target_result + if int(target_exit_code) != 0: + validation_failures.append(target["name"]) + if validation_result is None and validation_targets: + # Only non-aggregate targets ran; keep the first for the row. + validation_result = target_result + validation_failed = bool(validation_failures) + validation_exit_code = 1 if validation_failed else 0 raw_size_before_cleanup_by_file = { raw_rdf_path.name: int(file_size_bytes(raw_rdf_path) or 0) for raw_rdf_path in raw_rdf_files @@ -5242,8 +5811,9 @@ def fail_current(stage: str, message: str): if validation_failed: fail_current( "validation", - f"semantic VCF/RDF validation failed for '{output_name}'. " - f"See results: {validation_result.get('results_dir') if validation_result else metrics_dir / 'reports' / 'validation' / safe_metrics_name(output_name)}", + "semantic VCF/RDF validation failed for " + f"'{output_name}' ({', '.join(validation_failures)}). " + f"See results: {metrics_dir / 'reports' / 'validation'}", ) rdf_storage_removed = False @@ -5633,6 +6203,69 @@ def run_compress_mode( return 0 +def detect_validation_rdf_format(path: Path) -> str | None: + """Infer the validator's artifact format from a filename, or None.""" + for suffix, fmt in VALIDATION_RDF_SUFFIXES: + if path.name.endswith(suffix): + return fmt + return None + + +def resolve_validation_targets( + *, + requested: list[str], + output_dir: Path, + output_name: str, + aggregate_path: Path, + selected_methods: list[str], +) -> list[dict]: + """Map requested validation targets onto artifacts that actually exist. + + A representation that was not selected, or whose artifact is missing after + a recoverable index warning, is skipped rather than reported as a failure: + the run already recorded why it is absent. + """ + targets: list[dict] = [] + for name in requested: + if name == "aggregate": + fmt = detect_validation_rdf_format(aggregate_path) + if aggregate_path.is_file() and fmt is not None: + targets.append({"name": name, "path": aggregate_path, "format": fmt}) + continue + method_group = ( + HDT_COMPRESSION_METHODS if name == "hdt" else COTTAS_COMPRESSION_METHODS + ) + if not any(method in method_group for method in selected_methods): + continue + candidate = output_dir / f"{output_name}.{name}" + if candidate.is_file(): + targets.append({"name": name, "path": candidate, "format": name}) + return targets + + +def parse_validation_targets(raw: str) -> list[str]: + """Parse --validate-artifacts into an ordered, de-duplicated target list.""" + value = (raw or "").strip() + if value == "" or value == "none": + return [] + if value == "all": + return list(VALIDATION_TARGET_CHOICES) + targets: list[str] = [] + for token in value.split(","): + target = token.strip() + if not target: + continue + if target not in VALIDATION_TARGET_CHOICES: + allowed = ",".join(VALIDATION_TARGET_CHOICES) + raise ValueError( + f"Unsupported value '{target}' for --validate-artifacts. " + f"Use {allowed}, all, or none." + ) + if target not in targets: + targets.append(target) + return targets + + def detect_compressed_format(path: Path): """Infer compressed RDF format from filename/extension.""" if ( @@ -5958,15 +6591,25 @@ def run_validation_mode( image_ref: str, filter_oracle: str, wrapper_log_path: Path, + engine: str = DEFAULT_VALIDATION_ENGINE, + engine_options: dict | None = None, + rdf_format: str | None = None, + strict_conformance: bool = False, + shacl_shapes: Path | None = None, run_tracker: RunTracker | None = None, stage_result: dict | None = None, write_metrics_csv: bool = False, ): """Run VCF/RDF semantic queries with container-local temporary state. - Gzip RDF inputs are inflated under the container's ``/work`` temporary - filesystem. Plain N-Triples inputs are read directly from their read-only - mount. In both cases, no validation scratch RDF is persisted on the host. + Accepts any artifact the pipeline produces (``.nt``, ``.nt.gz``, ``.nt.br``, + ``.hdt``, ``.cottas``, ``.cottas[.gz|.br]``). Anything other than plain + N-Triples is decoded under the container's ``/work`` filesystem, so + validating an ``.hdt`` proves it decodes to a graph that still satisfies + every semantic check. No validation scratch RDF is persisted on the host. + + ``engine`` selects the SPARQL backend (``comunica`` or ``qlever``); both + answer the same queries, so it is a scale decision, not a semantic one. """ # An empty directory may be left behind if Docker itself fails before the # runner starts. Reuse only that empty shell; never overwrite reports. @@ -5981,7 +6624,38 @@ def run_validation_mode( else None ) progress_container_ref = container_progress_path(progress_host_path, metrics_dir) - rdf_flag = "--rdf-gz" if rdf_path.name.endswith(".nt.gz") else "--rdf-nt" + resolved_format = rdf_format or detect_validation_rdf_format(rdf_path) + if resolved_format is None: + raise ValueError( + f"Unsupported RDF artifact for validation: {rdf_path.name}. " + "Expected one of .nt, .nt.gz, .nt.br, .hdt, .cottas, .cottas.gz, .cottas.br" + ) + options = dict(engine_options or {}) + engine_args: list[str] = ["--engine", engine] + for flag, key in ( + ("--query-timeout", "query_timeout"), + ("--qlever-memory-gb", "qlever_memory_gb"), + ("--qlever-port", "qlever_port"), + ("--qlever-startup-timeout", "qlever_startup_timeout"), + ): + value = options.get(key) + if value is not None: + engine_args.extend([flag, str(value)]) + for flag, key in ( + ("--qlever-index-arg", "qlever_index_args"), + ("--qlever-server-arg", "qlever_server_args"), + ): + for value in options.get(key) or []: + engine_args.extend([flag, str(value)]) + if strict_conformance: + engine_args.append("--strict-conformance") + shacl_mount: list[str] = [] + if shacl_shapes is not None: + # Mounted read-only in its own directory so the shapes file can live + # anywhere on the host without exposing its parent tree for writing. + shacl_mount = ["-v", f"{shacl_shapes.parent.resolve()}:/data/shacl:ro"] + engine_args.extend(["--shacl-shapes", f"/data/shacl/{shacl_shapes.name}"]) + cmd = [ *docker_run_base(), "--init", @@ -5993,6 +6667,7 @@ def run_validation_mode( f"{str(results_dir)}:/data/validation", "-v", f"{str(metrics_dir.resolve())}:/data/metrics", + *shacl_mount, image_ref, "/usr/bin/time", "-v", @@ -6003,8 +6678,11 @@ def run_validation_mode( "/opt/vcf-rdfizer/validation/validation_runner.py", "--vcf", f"/data/vcf/{vcf_path.name}", - rdf_flag, + "--rdf", f"/data/rdf/{rdf_path.name}", + "--rdf-format", + resolved_format, + *engine_args, "--representation", representation, "--results-dir", @@ -6052,7 +6730,9 @@ def run_validation_mode( "validation_id": validation_id, "vcf_path": str(vcf_path), "rdf_path": str(rdf_path), - "rdf_format": "nt.gz" if rdf_path.name.endswith(".nt.gz") else "nt", + "rdf_format": resolved_format, + "engine": engine, + "strict_conformance": bool(strict_conformance), "representation": representation, "results_dir": str(results_dir), "summary_path": str(summary_path), @@ -6068,7 +6748,7 @@ def run_validation_mode( "temporary_rdf": { "decompressed_inside_container": bool( summary_temporary_rdf.get( - "decompressedInsideContainer", rdf_path.name.endswith(".nt.gz") + "decompressedInsideContainer", resolved_format != "nt" ) ), "persisted_on_host": bool(summary_temporary_rdf.get("persisted", False)), @@ -6077,7 +6757,7 @@ def run_validation_mode( ), }, } - if rdf_path.name.endswith(".nt.gz"): + if resolved_format == "nt.gz": # Compatibility alias for consumers of the original standalone report # schema; ``rdf_path`` is the canonical format-neutral field. payload["rdf_gzip_path"] = str(rdf_path) @@ -6226,6 +6906,28 @@ def main(): "one sample-ordered value vector per FORMAT key (default: expanded)" ), ) + parser.add_argument( + "--header-representation", + choices=HEADER_REPRESENTATION_CHOICES, + default=DEFAULT_HEADER_REPRESENTATION, + help=( + "VCF meta-information representation: structured types each '##' line " + "with its vocabulary subclass and lifts FILTER/ALT/contig attributes " + "into their own properties; basic keeps only untyped header lines " + f"(default: {DEFAULT_HEADER_REPRESENTATION})" + ), + ) + parser.add_argument( + "--info-representation", + choices=INFO_REPRESENTATION_CHOICES, + default=DEFAULT_INFO_REPRESENTATION, + help=( + "INFO column representation: structured also emits one " + "vcfr:InfoFieldValue per record and key alongside the raw string, " + "making INFO queryable; raw keeps only vcfr:infoRaw " + f"(default: {DEFAULT_INFO_REPRESENTATION})" + ), + ) parser.add_argument( "--validate", "--run-validation", @@ -6371,6 +7073,78 @@ def main(): default="auto", help="FILTER oracle for full-mode validation and standalone validation (default: auto)", ) + parser.add_argument( + "--shacl-shapes", + default=None, + help=( + "Validate each graph against a SHACL shapes file as an independent " + "structural layer (for example the vocabulary's published shapes). " + "Off by default: it loads the whole graph into memory, so it does " + "not scale to a cohort-sized aggregate" + ), + ) + parser.add_argument( + "--strict-conformance", + action="store_true", + help=( + "Fail validation when a missing token is serialized as a plain '.' " + "literal instead of '.'^^vcfr:Null (reported but non-fatal by default)" + ), + ) + parser.add_argument( + "--validation-engine", + choices=VALIDATION_ENGINE_CHOICES, + default=DEFAULT_VALIDATION_ENGINE, + help=( + "SPARQL engine used for validation: comunica queries the graph in " + "memory; qlever builds an on-disk index inside the container and " + f"serves it (default: {DEFAULT_VALIDATION_ENGINE})" + ), + ) + parser.add_argument( + "--validate-artifacts", + default=DEFAULT_VALIDATION_TARGETS, + help=( + "Which produced artifacts full-mode --validate should check " + "(comma-separated): aggregate,hdt,cottas, or all " + f"(default: {DEFAULT_VALIDATION_TARGETS}). A representation that was " + "not produced is skipped." + ), + ) + parser.add_argument( + "--validation-query-timeout", + default=None, + help="Per-query timeout in seconds for validation (default: engine default)", + ) + parser.add_argument( + "--qlever-memory-gb", + default=None, + help="QLever index/server memory budget in GiB (default: 4)", + ) + parser.add_argument( + "--qlever-port", + default=None, + help="Container-local port for the QLever server (default: 7019)", + ) + parser.add_argument( + "--qlever-startup-timeout", + default=None, + help="Seconds to wait for the QLever server to answer after indexing (default: 900)", + ) + parser.add_argument( + "--qlever-index-arg", + action="append", + default=[], + metavar="ARG", + help="Extra argument for QLever's index builder (repeatable)", + ) + parser.add_argument( + "--qlever-server-arg", + action="append", + default=[], + metavar="ARG", + help="Extra argument for QLever's server (repeatable)", + ) rdf_output_group = parser.add_mutually_exclusive_group() rdf_output_group.add_argument( "-R", @@ -6414,7 +7188,33 @@ def main(): step1_label = "Step 1/5" if mode == "full" else "Step 1/3" + validation_artifacts: list[str] = [] + validation_engine_options: dict = {} + shacl_shapes_path: Path | None = None try: + for option_name, key in ( + ("--validation-query-timeout", "query_timeout"), + ("--qlever-memory-gb", "qlever_memory_gb"), + ("--qlever-port", "qlever_port"), + ("--qlever-startup-timeout", "qlever_startup_timeout"), + ): + raw_value = getattr(args, option_name.lstrip("-").replace("-", "_")) + if raw_value is not None: + validation_engine_options[key] = parse_positive_int( + raw_value, name=option_name + ) + if args.qlever_index_arg: + validation_engine_options["qlever_index_args"] = list(args.qlever_index_arg) + if args.qlever_server_arg: + validation_engine_options["qlever_server_args"] = list(args.qlever_server_arg) + if validation_engine_options.get("qlever_port", 1) > 65535: + raise ValueError("--qlever-port must be between 1 and 65535") + validation_artifacts = parse_validation_targets(args.validate_artifacts) + if args.shacl_shapes is not None: + shacl_shapes_path = Path(args.shacl_shapes).expanduser().resolve() + if not shacl_shapes_path.is_file(): + raise ValueError(f"SHACL shapes file not found: {shacl_shapes_path}") + chunk_target_bytes = parse_positive_int( args.chunk_target_bytes, name="--chunk-target-bytes" ) @@ -6452,6 +7252,13 @@ def main(): args.sample_representation, rules_path, ) + if args.validate_artifacts != DEFAULT_VALIDATION_TARGETS and not args.run_validation: + raise ValueError("--validate-artifacts requires --validate") + if not validation_artifacts and args.run_validation: + raise ValueError( + "--validate-artifacts resolved to no targets; choose at least one of " + + ", ".join(VALIDATION_TARGET_CHOICES) + ) validate_mode_dirs([out_root, out_dir, tsv_dir, metrics_root]) if args.legacy_compression is not None: if ( @@ -6532,8 +7339,19 @@ def main(): if not args.rdf: raise ValueError("--rdf is required in --mode validation") validation_rdf_gzip_path = Path(args.rdf).expanduser().resolve() - if not validation_rdf_gzip_path.is_file() or not validation_rdf_gzip_path.name.endswith(".nt.gz"): - raise ValueError("Validation RDF input must be an existing .nt.gz file") + if not validation_rdf_gzip_path.is_file(): + raise ValueError(f"Validation RDF input not found: {validation_rdf_gzip_path}") + validation_rdf_format = detect_validation_rdf_format(validation_rdf_gzip_path) + if validation_rdf_format is None: + supported = ", ".join(fmt for _suffix, fmt in VALIDATION_RDF_SUFFIXES) + raise ValueError( + "Validation RDF input must be one of: " + supported + ) + if args.validate_artifacts != DEFAULT_VALIDATION_TARGETS: + raise ValueError( + "--validate-artifacts is only valid in --mode full; in validation " + "mode pass the artifact directly with --rdf" + ) validation_id = args.validation_id or vcf_output_prefix(validation_vcf_path) if not re.fullmatch(r"[A-Za-z0-9._-]+", validation_id): raise ValueError("--validation-id may contain only letters, digits, dot, underscore, and hyphen") @@ -6698,6 +7516,8 @@ def main(): "requested_image": args.image, "requested_image_version": args.image_version, "sample_representation": args.sample_representation if mode == "full" else None, + "info_representation": args.info_representation if mode == "full" else None, + "header_representation": args.header_representation if mode == "full" else None, "rdf_storage_mode": args.rdf_storage_mode if mode == "full" else None, "compression_methods": ( full_methods if mode == "full" else methods if mode == "compress" else [] @@ -6708,6 +7528,11 @@ def main(): "chunk_max_bytes": chunk_max_bytes if mode in {"full", "compress"} else None, "spark_partitions": spark_partitions if mode == "full" else None, "run_validation": bool(args.run_validation) if mode == "full" else False, + "validation_artifacts": validation_artifacts if mode == "full" and args.run_validation else None, + "validation_engine": args.validation_engine if mode in {"full", "validation"} else None, + "validation_strict_conformance": bool(args.strict_conformance) if mode in {"full", "validation"} else None, + "validation_shacl_shapes": str(shacl_shapes_path) if shacl_shapes_path else None, + "validation_engine_options": validation_engine_options or None, "filter_oracle": args.filter_oracle if mode in {"full", "validation"} else None, "quiet": bool(args.quiet), "no_progress": bool(args.no_progress), @@ -6895,7 +7720,14 @@ def execute_mode(): image_ref=image_ref, out_name=args.out_name, sample_workflow=sample_workflow, + info_representation=args.info_representation, + header_representation=args.header_representation, run_validation=args.run_validation, + validation_artifacts=validation_artifacts, + validation_engine=args.validation_engine, + validation_engine_options=validation_engine_options, + validation_strict_conformance=args.strict_conformance, + validation_shacl_shapes=shacl_shapes_path, filter_oracle=args.filter_oracle, rdf_storage_mode=args.rdf_storage_mode, methods=full_methods, @@ -6946,6 +7778,7 @@ def execute_mode(): return run_validation_mode( vcf_path=validation_vcf_path, rdf_path=validation_rdf_gzip_path, + rdf_format=validation_rdf_format, representation=args.sample_representation, validation_id=validation_id, results_dir=validation_results_dir, @@ -6954,6 +7787,10 @@ def execute_mode(): timestamp=timestamp, image_ref=image_ref, filter_oracle=args.filter_oracle, + engine=args.validation_engine, + engine_options=validation_engine_options, + strict_conformance=args.strict_conformance, + shacl_shapes=shacl_shapes_path, wrapper_log_path=wrapper_log_path, write_metrics_csv=True, ) From d81cddf4b5275c9ddf7a30c502fc508c4b7a1e34 Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Fri, 4 Sep 2026 21:00:12 +0200 Subject: [PATCH 19/19] add multi-engine validation for benchmarking --- Dockerfile | 17 +- README.md | 26 +- changelog.md | 146 ++++++ docs/README.md | 11 + docs/cli-reference.md | 4 +- docs/limitations.md | 19 + docs/privacy-policy-design.md | 595 ++++++++++++++++++++++++ docs/roadmap.md | 31 ++ docs/validation-methodology.md | 39 +- docs/validation.md | 240 +++++++--- src/validation/validation_runner.py | 617 ++++++++++++++++++++++--- test/README.md | 20 +- test/cross_engine_agreement.py | 50 +- test/test_validation_benchmark_unit.py | 331 +++++++++++++ test/test_validation_logic_unit.py | 50 ++ test/test_vcf_rdfizer_unit.py | 4 + vcf_rdfizer.py | 109 ++++- 17 files changed, 2129 insertions(+), 180 deletions(-) create mode 100644 docs/privacy-policy-design.md create mode 100644 test/test_validation_benchmark_unit.py diff --git a/Dockerfile b/Dockerfile index 25422ca..59d042d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ ARG RMLSTREAMER_VERSION=2.5.0 ARG HDTC_VERSION=1.1.0 ARG COMUNICA_VERSION=5.3.0 +# Comunica's HDT engine is versioned separately from its file engine and lags +# behind it. It provides native SPARQL over a .hdt artifact, so validation can +# query the compressed representation directly instead of decoding it first. +ARG COMUNICA_HDT_VERSION=5.0.1 # QLever is an optional second SPARQL engine for validation. Its binaries are # copied from the upstream published image rather than built here: compiling # QLever needs a large C++ toolchain and would dominate this image's build. @@ -106,7 +110,18 @@ RUN python3 -m venv /opt/pycottas-venv \ cyvcf2==0.34.0 \ pyshacl==0.30.1 -RUN npm install --global "@comunica/query-sparql-file@${COMUNICA_VERSION}" +ARG COMUNICA_HDT_VERSION + +# The HDT engine compiles native bindings, so a toolchain is needed at install +# time but not afterwards; it is purged in the same layer to keep it out of the +# image. `python3` is already present and is what node-gyp needs. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && npm install --global \ + "@comunica/query-sparql-file@${COMUNICA_VERSION}" \ + "@comunica/query-sparql-hdt@${COMUNICA_HDT_VERSION}" \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ diff --git a/README.md b/README.md index b4a6ef3..744ee26 100644 --- a/README.md +++ b/README.md @@ -160,10 +160,12 @@ vcf-rdfizer --mode full -i ./cohort.vcf.gz \ --validate --validate-artifacts all -o ./results ``` -`--validation-engine {comunica,qlever}` selects the SPARQL backend. Comunica -(default) queries the file in memory; [QLever](https://github.com/ad-freiburg/qlever) -builds an on-disk index inside the container and serves it, which is what makes -cohort-scale graphs queryable. Both answer identical queries, so the choice is +`--validation-engine` selects the SPARQL backend. Comunica (default) queries the +file in memory; [QLever](https://github.com/ad-freiburg/qlever) builds an +on-disk index inside the container and serves it, which is what makes +cohort-scale graphs queryable; `hdt` and `cottas` query the **compressed +artifact in place**, without decoding it, through `comunica-sparql-hdt` and +`pycottas`'s rdflib store. All four answer identical queries, so the choice is never semantic, and every report records which engine ran. ```bash @@ -171,6 +173,17 @@ vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.h --validation-engine qlever --qlever-memory-gb 32 -o ./validation-results ``` +Several engines can run in one pass - `--validation-engine comunica,qlever` or +`all`. Each answers the whole query set, so the run cross-checks them against +each other (`engine-agreement.json`) and times them against each other and +against the cyvcf2 oracle computing the same answers, in `benchmark.json` and a +long-format `benchmark.csv`. + +```bash +vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --validation-engine all -o ./validation-results +``` + Tuning: `--qlever-memory-gb`, `--qlever-port`, `--qlever-startup-timeout`, `--validation-query-timeout`, and repeatable `--qlever-index-arg` / `--qlever-server-arg` escape hatches. @@ -187,8 +200,8 @@ vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.n ``` > **What a PASS means.** Coverage is measured, not asserted: a mutation harness -> corrupts a correct graph in 36 named ways and records which are detected -> (currently **64/66**). See [`docs/vcf-coverage.md`](docs/vcf-coverage.md) for +> corrupts a correct graph in 42 named ways and records which are detected +> (currently **76/78**). See [`docs/vcf-coverage.md`](docs/vcf-coverage.md) for > the element-by-element matrix and the remaining gaps, and > [`docs/validation-methodology.md`](docs/validation-methodology.md) for how the > number is produced. @@ -947,6 +960,7 @@ how each part of the tool works, why, and where it stops working. | [Limitations](docs/limitations.md) | Everything the tool cannot do, in one place | | [Roadmap](docs/roadmap.md) | Planned work, known defects, and rejected options | | [Data linking design](docs/datalinking-design.md) | Proposal: a plug-in system for external links | +| [Privacy policy design](docs/privacy-policy-design.md) | Proposal: ODRL-based granular disclosure control over the graph | - [`changelog.md`](changelog.md) - dated change history - [`ACKNOWLEDGEMENTS.md`](ACKNOWLEDGEMENTS.md) - funding and attribution diff --git a/changelog.md b/changelog.md index a0e682b..a2621a0 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,151 @@ # Changelog +## 2026-09-04 — Multi-engine validation, native HDT/COTTAS querying, and benchmarking + +Validation was one engine against one N-Triples file. It is now up to four +engines against the artifact each of them reads best, in a single run, timed +and cross-checked. + +### Added + +- **Several SPARQL engines in one run.** `--validation-engine` accepts a + comma-separated list or `all`. Every requested engine answers the whole query + set; the first is the primary and keeps the existing single-engine report + layout, while each engine additionally gets `engines//` with its own + preflight, SPARQL, comparison and execution reports. The run's `status` is + `PASS` only if every engine passed, and `summary.json` carries + `engineStatuses`. +- **Cross-engine agreement on the real graph.** Every engine's normalized + results are compared against each other and written to + `engine-agreement.json`, naming the queries that differ. This is not + theoretical: QLever's literal canonicalisation had already been caught this + way in the test harness. +- **Native HDT and COTTAS querying** (`--validation-engine hdt` / `cottas`). + `--validation-target hdt,cottas` validates those artifacts by *decoding* them + first, which proves the decode is faithful and says nothing about querying + them — and measures the wrong thing entirely in a performance comparison. The + new engines query the artifact in place: HDT through `comunica-sparql-hdt`, + COTTAS in-process through `pycottas.COTTASStore`, an rdflib `Store` over the + Parquet artifact. When the run's own artifact is already in the engine's + format it is used directly; otherwise one is built in container scratch and + that build is timed as **setup**, kept out of the query total. Each report + records which happened, in `artifactOrigin`. +- **`@comunica/query-sparql-hdt` in the image**, installed alongside + `build-essential` (needed for its native bindings) which is purged in the same + layer. +- **Timings for everything, and a comparison against the parser.** Every run + writes `benchmark.json` and a long-format `benchmark.csv` — one row per engine + and query, ready to plot without reshaping. `benchmark.json` adds per-engine + setup and query totals, the slowest query per engine, materialization and + SHACL time, and the **oracle**: what it costs to compute the same answers + directly from the VCF with cyvcf2, split into parse and census. The suite + already computes every expected value twice, once by parsing and once by + querying, so those two costs are for identical work on identical input — the + performance comparison the reports were missing. +- **Benchmark columns in the run's `metrics.csv`**: `validation_engines`, + `validation_oracle_seconds`, `validation_engine_query_seconds` and + `validation_engine_setup_seconds`, the last two as `engine=seconds` pairs. + +### Fixed + +- **Comunica could not query an HDT file by path.** A bare path is treated as a + link to dereference and failed with "Could not dereference". The source is now + addressed as `hdt@`. Found by the first real four-engine run, not by + inspection. +- **The parser oracle was reading htslib's header, not the file's.** cyvcf2's + `raw_header` is normalised: htslib injects + `##FILTER=` into files that never + declared it. The oracle therefore expected a `FilterDefinition` resource the + graph could not contain, and every header check failed. `read_vcf_header_text` + now reads the header block from the file itself — the same text the conversion + reads — for both plain and gzipped VCFs. + +### Verified + +- A four-engine run over the fixture: all four `PASS`, all four agree, and the + benchmark records per-query timings for each. +- `test/cross_engine_agreement.py` extended from two engines to all four, + across both representations, with per-run scratch so the native engines cannot + read each other's artifacts. All queries agree and every engine agrees with + the Python oracle. +- 347 host tests pass (24 new); mutation score unchanged at **76/78**. + +### Documentation + +- [`docs/validation.md`](docs/validation.md) — rewritten engine section covering + all four engines, multi-engine runs, native artifact querying, and the + benchmark report. Its "What is *not* tested" section was stale — it predated + the census, the identity digests, and the QUAL/INFO coverage work, and + contradicted [`docs/vcf-coverage.md`](docs/vcf-coverage.md) — and now states + the gaps that actually remain. +- [`docs/validation-methodology.md`](docs/validation-methodology.md) — "Two + engines" became "Four engines", with the two new cross-engine findings and a + section on why a multi-engine run is a controlled benchmark rather than an + incidental one. +- [`README.md`](README.md), [`docs/cli-reference.md`](docs/cli-reference.md), + [`test/README.md`](test/README.md) updated; the README's stale mutation score + (36 mutations, 64/66) corrected to 42 and 76/78. + +## 2026-09-04 — Design proposal: granular privacy policies over the VCF graph + +[`docs/privacy-policy-design.md`](docs/privacy-policy-design.md) — **design +proposal, not implemented.** A system for governed, partial release of a VCF +graph: which participants, which regions, which fields, at what resolution, to +whom, for what purpose — declared in ODRL and enforced by the pipeline. + +### Added + +- **An ODRL profile whose assets are graph selectors.** ODRL's `odrl:target` + addresses an Asset IRI and has no notion of "these triples", so the profile + supplies `ClassSelector`, `PredicateSelector`, `SampleSelector`, + `RegionSelector`, `FieldSelector`, `HeaderSelector` and a SPARQL + `PatternSelector` escape hatch. It also pins the conflict semantics ODRL + leaves informative: deny-overrides, default-deny, most-restrictive-first + composition. +- **Effects beyond permit/prohibit**, because the useful answer for genomic data + is usually less resolution rather than nothing: `drop`, `pseudonymize`, + `generalize`, `threshold`, `aggregateOnly`, and `maskVectorPositions`. +- **Three enforcement tiers derived from the existing pipeline** — TSV + pre-filtering before RMLStreamer (cheapest, covers RML-produced triples), + emitter-time filtering inside `_append_rdf_atomically` (full record context, + no second pass), and a post-hoc `--mode redact` (needs two passes for region + rules, since an N-Triples stream has no ordering guarantee). The compiler + assigns each rule to the cheapest tier that can express it, and **aborts** on + any rule no tier can enforce rather than warning. +- **GA4GH DUO consent codes as `odrl:purpose` right operands**, so a policy is + reviewable by the data access committees that already speak DUO. +- **Release manifests and policy conformance verification**: prohibitions + compile to `ASK` preflights that must return zero, with mutation-catalogue + entries that break the redactor and assert the checks catch it — the same + "measured, not asserted" standard as + [`validation-methodology.md`](docs/validation-methodology.md). + +### Findings that affect work outside the proposal + +- **`ParsedSampleRecord` carries no `CHROM` or `POS`.** `_parse_row` reads + columns 0, 1, 7, 9, 10 and −1, skipping 2 and 3. Any region-scoped feature + evaluated at emission time needs both; adding them is two dataclass fields and + two index reads. +- **Condensed mode cannot express per-sample protection by triple filtering.** + One `FormatValueVector` literal holds every participant's value for a FORMAT + key, so the unit of protection is finer than the unit of storage. Redaction + must rewrite the literal in place (replacing a position with `.` to preserve + `sampleIndex` alignment), and a single masked position is itself disclosive. +- **IRIs and header lines leak independently of genotypes.** Sample names, the + source filename and a monotonic row counter are all embedded in IRIs, and + `##source` / `##SAMPLE` / `##PEDIGREE` / free-text `Description` fields are + transcribed verbatim. Recorded in + [`docs/limitations.md`](docs/limitations.md). + +### Framing kept throughout + +The document is explicit that this is **governed release, not anonymization**. +Genotypes are identifiers — a few dozen independent common variants single out +an individual — so an access-control layer governs who receives what and creates +an audit trail; it does not make released data non-identifying. Differential +privacy is discussed and explicitly **not** proposed, because without a +persistent per-recipient budget ledger it provides no protection. + ## 2026-09-04 — Documentation set, and a data-linking design proposal `docs/` becomes an in-depth explanation of the whole tool rather than four diff --git a/docs/README.md b/docs/README.md index e6b8398..6cabb44 100644 --- a/docs/README.md +++ b/docs/README.md @@ -49,6 +49,11 @@ deciding whether the tool fits your problem, read implemented.* A plug-in architecture for connecting the graph to external resources (rsIDs, genes, clinical assertions) with declarative linkers, reference bundles, live-API safeguards, and provenance. +- **[Privacy policy design](privacy-policy-design.md)** — *proposal, not + implemented.* Granular, machine-readable disclosure control over parts of the + graph: an ODRL profile with graph selectors, three enforcement tiers, and + verification — plus a candid account of why access control is not + anonymization when the genotypes are themselves identifiers. ### What the graph looks like @@ -97,6 +102,12 @@ deciding whether the tool fits your problem, read [Custom RML mappings](rml-mappings.md) → [Validation methodology](validation-methodology.md) +**"I need to release only part of this cohort."** +[Privacy policy design](privacy-policy-design.md) → +[Sample representations](sample-representation-guide.md) → +[Conversion §6](conversion.md#6-iri-templates) → +[Validation methodology](validation-methodology.md) + **"A cohort-scale run just failed."** [Representations §10](representations.md#10-limitations) → [Output and metrics §2](output-and-metrics.md#2-the-run-metrics-directory) → diff --git a/docs/cli-reference.md b/docs/cli-reference.md index f04ae93..9e3c716 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -92,11 +92,11 @@ compatibility. Use the three explicit selectors. | `--validate` / `--run-validation` | — | off | Run validation once per input in full mode | | `--validate-artifacts` | `aggregate`, `hdt`, `cottas`, `all` | `aggregate` | Which produced artifacts to check, each in its own report directory | | `--validation-id` | name | source basename | Report directory name; existing directories are never overwritten | -| `--validation-engine` | `comunica`, `qlever` | `comunica` | SPARQL backend; a scale decision, never a semantic one | +| `--validation-engine` | `comunica`, `qlever`, `hdt`, `cottas`, `all`, or a comma-separated list | `comunica` | SPARQL backend(s); a scale and performance decision, never a semantic one. `hdt`/`cottas` query the compressed artifact in place. Several engines answer the whole query set, are cross-checked against each other, and are timed in `benchmark.csv` | | `--filter-oracle` | `auto`, `bcftools`, `cyvcf2` | `auto` | FILTER-field oracle | | `--shacl-shapes` | path | off | Independent structural layer via `pyshacl`; in-memory, so not for cohort scale | | `--strict-conformance` | — | off | Promote a missing-token conformance anomaly from report to failure | -| `--validation-query-timeout` | seconds | 3600 | Per-query timeout, both engines | +| `--validation-query-timeout` | seconds | 3600 | Per-query timeout, every engine | | `--qlever-memory-gb` | N | 4 | QLever index and server memory budget | | `--qlever-port` | N | 7019 | Container-local only; never published | | `--qlever-startup-timeout` | seconds | 900 | Wait for the server after indexing | diff --git a/docs/limitations.md b/docs/limitations.md index 1d0e6a7..cf0dcb3 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -188,6 +188,24 @@ none is currently implemented. external. This is by design so far, and the plan to change it is [`datalinking-design.md`](datalinking-design.md). +**No disclosure control.** Conversion is all-or-nothing: every sample, every +genotype, every header line and every free-text `Description` goes into the +graph, and there is no way to withhold a participant, degrade a region, or +record what an artifact was permitted to contain. Three consequences today: + +- **IRIs carry identifiers.** `file://cohort.vcf#sample/1/NA12878` embeds the + sample name, `{SOURCE_FILE}` embeds the VCF's basename, and `#record/{ROW_ID}` + is a monotonic counter that discloses source ordering. +- **Header lines are a leak surface.** `##source`, `##SAMPLE`, `##PEDIGREE` and + free-text `Description` fields are transcribed verbatim into + `vcfr:headerValue`. +- **Even if that were fixed, genotypes identify people.** A few dozen + independent common variants are enough to single out an individual, so no + amount of label removal makes a released genotype graph non-identifying. + +The plan is [`privacy-policy-design.md`](privacy-policy-design.md), which is +explicit that what it offers is *governed release*, not anonymization. + **No clinical claims.** The tool transcribes a VCF. It does not interpret, annotate, prioritize, or assess pathogenicity, and its output should not be presented as if it did. @@ -197,5 +215,6 @@ presented as if it did. ## See also - [Roadmap](roadmap.md) — which of these are being addressed +- [Privacy policy design](privacy-policy-design.md) — the disclosure-control gap, and the proposal to close it - [VCF coverage matrix](vcf-coverage.md) — the element-by-element measurement - [Validation](validation.md) — the detailed "what is not tested" diff --git a/docs/privacy-policy-design.md b/docs/privacy-policy-design.md new file mode 100644 index 0000000..7fe6128 --- /dev/null +++ b/docs/privacy-policy-design.md @@ -0,0 +1,595 @@ +# Granular privacy policies over a VCF graph + +*Status: **design proposal**. Nothing described here is implemented yet. Like +[`datalinking-design.md`](datalinking-design.md), this document exists to fix a +contract before code depends on it — and, in this case, to be explicit about +what a policy layer can and cannot achieve, because the failure mode of a +privacy feature is a false sense of safety.* + +VCF-RDFizer currently makes one decision about disclosure: it converts +everything. Every sample, every genotype, every header line, every free-text +`Description` goes into the graph, and every artifact is all-or-nothing. The +only granularity available today is "convert a different VCF". + +What is wanted is the ability to say, in a machine-readable and auditable way: +*these samples, under this purpose, may see these regions at this resolution* — +and to have the tool produce artifacts that actually honour it. + +--- + +## 1. The uncomfortable premise + +**Genotypes are identifiers.** A few dozen independent common variants are +enough to pick one individual out of a population (Lin, Owen & Altman, +*Science* 2004), aggregate allele frequencies alone can reveal whether a known +individual was in a cohort (Homer et al., *PLoS Genetics* 2008), and Y-chromosome +haplotypes have been used to recover surnames from public genealogy databases +(Gymrek et al., *Science* 2013). + +Three consequences follow, and they shape everything below: + +1. **Removing `vcfr:sampleId` does not anonymize anything.** The genotype vector + that remains is a stronger identifier than the label you deleted. +2. **Access control is not anonymization.** A policy layer governs *who is + permitted to receive what*, and creates an audit trail. It does not make the + released data non-identifying, and nothing in this design should be described + as if it did. +3. **The artifact is the enforcement boundary.** Once someone holds a `.hdt` + file, no policy engine is in the loop. Query-time enforcement only exists + while you control the endpoint. + +This document therefore proposes a system for **governed release**, not for +anonymity. Where a mechanism reduces re-identification risk (thresholding, +generalization), it says by how much and under what assumption. + +--- + +## 2. What "a portion of the VCF graph" means + +The graph has a small number of natural cut planes, and they map cleanly onto +the shapes the conversion already emits (see +[`conversion.md`](conversion.md#6-iri-templates)): + +| Cut | Addresses | Example | +| --- | --- | --- | +| **By class** | all resources of a type | every `vcfr:SampleCall` | +| **By predicate** | all triples with a predicate | every `vcfr:sampleName` | +| **By sample** | one participant's contribution | everything hanging off `#samples/NA12878` | +| **By region** | a genomic interval | `chr19:44,905,791-44,909,393` (*APOE*) | +| **By declared field** | one INFO or FORMAT key | `DP` yes, `GT` no | +| **By record predicate** | an arbitrary graph pattern | records where `FILTER != PASS` | +| **By granularity** | not a subgraph at all | counts permitted, individual calls not | + +The last row is the one that access control alone cannot express, and it is the +one that matters most for the attacks in §1. It is handled in §9. + +**A note on scope.** Two families of triple leak more than people expect and are +easy to forget when thinking in terms of genotypes: + +- **IRIs themselves.** `file://cohort.vcf#sample/1/NA12878` contains the sample + name. `#record/{ROW_ID}` is a monotonic counter, so row identifiers disclose + the source ordering — and therefore approximate genomic position — even if + `vcfr:pos` is dropped. And `{SOURCE_FILE}` is the VCF's basename, which in + practice is often `patient_12345.vcf`. Filtering triples while leaving IRIs + intact leaks membership. See §7. +- **Header lines.** `vcfr:headerValue` and the raw line text carry `##source` + (pipeline and centre identifiers), `##SAMPLE` and `##PEDIGREE` (family + structure, by design), free-text `Description` fields, and `##fileDate`. + A policy that covers genotypes and ignores the header section has not covered + the graph. + +--- + +## 3. Is ODRL the right choice? + +Yes for the policy layer, no as the whole answer. Being precise about the gap is +what makes the rest of the design tractable. + +**What ODRL gives you, that is genuinely hard to get elsewhere:** + +- A W3C Recommendation, so a policy is a citable, interoperable artifact rather + than a bespoke config file. +- RDF-native: the policy lives in the same graph model as the data, can be + published, dereferenced, versioned and signed. +- The vocabulary the problem actually needs — `odrl:permission` / + `odrl:prohibition` / `odrl:duty`, `odrl:assignee`, `odrl:constraint` with + `odrl:purpose` and `odrl:recipient` left operands. +- A **profile mechanism** designed for exactly this situation: extend the core + with domain terms without forking it. + +**What ODRL does not give you, and must be supplied:** + +| Gap | Consequence | +| --- | --- | +| `odrl:target` points at an *Asset* (an IRI). There is no notion of "these triples". | A selector vocabulary is required. This is the core technical work. | +| Evaluation semantics in the ODRL Information Model are **informative**, not normative. | Conflict resolution and rule ordering must be pinned by the profile, or two implementations will disagree. | +| Rules permit or prohibit. They cannot say *how* to degrade. | An effect/transform vocabulary is required (drop, generalize, threshold, pseudonymize). | +| No enforcement engine exists. | Enforcement is ours to build; ODRL is the declaration, not the mechanism. | + +**Alternatives considered:** + +| Option | Verdict | +| --- | --- | +| SHACL shapes as policy | Excellent at *describing* a permitted shape, and worth reusing to **verify** a release (§11) — but it is a validation language with no notion of party, purpose or duty. Not a policy language. | +| Solid WAC / ACP | Resource-granular. A VCF graph is one resource; the whole problem is sub-resource granularity. | +| XACML | Mature and battle-tested, but not RDF-native, heavyweight, and it would put the policy outside the data ecosystem the rest of the tool lives in. | +| Bare SPARQL views | Simplest possible enforcement, and in fact what the compiler emits — but as a *policy* it records no party, purpose, obligation or provenance, so it cannot be audited or presented to a data access committee. | + +**Recommended shape:** ODRL as the front end, a VCF-RDFizer profile supplying +selectors and effects, compiled down to a plain internal **release plan**. Keep +the enforcement layer independent of ODRL so a second front end (a DUO-only +consent file, say) can be added without touching the engine. + +--- + +## 4. The profile + +Namespace `vcfp:` = `https://w3id.org/vcf-rdfizer/policy#`, declared through +`odrl:profile`. + +### 4.1 Assets are graph selections + +```turtle +@prefix odrl: . +@prefix vcfp: . +@prefix vcfr: . +@prefix duo: . + +<#apoe-locus> a odrl:Asset , vcfp:GraphSelection ; + vcfp:selector [ a vcfp:RegionSelector ; + vcfp:assembly "GRCh38" ; + vcfp:chrom "chr19" ; + vcfp:start 44905791 ; + vcfp:end 44909393 ] . + +<#direct-identifiers> a odrl:Asset , vcfp:GraphSelection ; + vcfp:selector [ a vcfp:PredicateSelector ; + vcfp:predicate vcfr:sampleName , vcfr:sampleId ] . + +<#withdrawn-participants> a odrl:Asset , vcfp:GraphSelection ; + vcfp:selector [ a vcfp:SampleSelector ; + vcfp:sampleId "NA12878" , "NA12891" ] . +``` + +| Selector | Keys | Notes | +| --- | --- | --- | +| `vcfp:ClassSelector` | `vcfp:class` | Every resource asserting that type, and its outbound triples | +| `vcfp:PredicateSelector` | `vcfp:predicate` | Triple-level; the cheapest to enforce | +| `vcfp:SampleSelector` | `vcfp:sampleId`, `vcfp:sampleIndex` | Resolves through `SampleSet` membership | +| `vcfp:RegionSelector` | `vcfp:assembly`, `vcfp:chrom`, `vcfp:start`, `vcfp:end` | **`vcfp:assembly` is mandatory** and checked against `##reference`; a mismatch aborts, for the same reason it does in the linking design | +| `vcfp:FieldSelector` | `vcfp:formatKey`, `vcfp:infoKey` | By declared field id | +| `vcfp:HeaderSelector` | `vcfp:headerKey` | `##SAMPLE`, `##PEDIGREE`, `##source`, … | +| `vcfp:PatternSelector` | `vcfp:ask` | A SPARQL graph pattern. The escape hatch; expensive, and excluded from the cheap enforcement tiers (§5) | + +Selectors compose with `vcfp:allOf` / `vcfp:anyOf` / `vcfp:not`. ODRL's own +`odrl:AssetCollection` with `odrl:refinement` is deliberately **not** reused +here: its refinement semantics are about collection membership, not about +sub-graph extents, and overloading it would produce policies that look standard +while meaning something non-standard. + +### 4.2 Effects: what a rule actually does + +A prohibition that can only mean "drop everything" is too blunt for genomic +data, where the useful answer is usually *less resolution*, not *nothing*. The +profile therefore refines `odrl:duty` with a transform: + +| `vcfp:transform` | Effect | +| --- | --- | +| `vcfp:drop` | Omit matched triples entirely (default for a prohibition) | +| `vcfp:pseudonymize` | Replace a term with a keyed, per-release token (§7) | +| `vcfp:generalize` | Reduce resolution: `POS` → 1 Mb window, `GT` → carrier / non-carrier, `DP` → banded | +| `vcfp:threshold` | Suppress a value below a count, e.g. allele count < 5 | +| `vcfp:aggregateOnly` | The subgraph is reachable only through a counting query (§9) | +| `vcfp:maskVectorPositions` | Condensed mode only: rewrite the tab-separated literal, replacing masked sample positions with `.` (§8) | + +```turtle +<#cohort-release-1.2> a odrl:Set ; + odrl:uid ; + odrl:profile ; + odrl:conflict odrl:prohibit ; + + # Hard exclusion: withdrawn consent. No purpose overrides this. + odrl:prohibition [ + odrl:target <#withdrawn-participants> ; + odrl:action odrl:read ; + odrl:assignee odrl:All ] ; + + # Degrade rather than deny: APOE is readable only as carrier status. + odrl:permission [ + odrl:target <#apoe-locus> ; + odrl:action odrl:read ; + odrl:duty [ odrl:action odrl:anonymize ; + vcfp:transform vcfp:generalize ; + vcfp:genotypeResolution vcfp:carrierStatus ] ] ; + + # Purpose-bound permission, expressed with GA4GH DUO codes. + odrl:permission [ + odrl:target <#genotypes> ; + odrl:action odrl:read ; + odrl:assignee ; + odrl:constraint [ odrl:leftOperand odrl:purpose ; + odrl:operator odrl:isAnyOf ; + odrl:rightOperand duo:DUO_0000007 ] ; + odrl:duty [ odrl:action odrl:inform ; + vcfp:auditSink ] ] . +``` + +### 4.3 Consent codes as purposes + +`odrl:purpose` right operands should be **GA4GH Data Use Ontology** terms rather +than free strings. DUO is what data access committees and repositories already +speak, so this makes a policy reviewable by the people who actually grant +access, and it means the tool is not inventing a consent vocabulary. + +Pin the DUO release in the policy (`vcfp:duoVersion`) and resolve term IRIs +against that release — DUO evolves, and a policy that silently re-interprets a +consent code is worse than one that fails to load. + +### 4.4 Conflict resolution, pinned + +Because ODRL leaves this informative, the profile **normatively** fixes it: + +1. `odrl:conflict odrl:prohibit` is the default and the only value recommended + for a release policy. Deny wins. +2. A rule with no applicable transform defaults to `vcfp:drop`. +3. Effects on the same selection compose most-restrictive-first + (`drop` > `aggregateOnly` > `threshold` > `generalize` > `pseudonymize`). +4. `odrl:conflict odrl:invalid` is supported for strict deployments: any + conflict voids the policy and the run aborts rather than guessing. +5. **Anything not matched by a permission is denied.** Default-deny is the only + defensible posture; a default-allow policy language for genomic release is a + trap. + +--- + +## 5. Where enforcement happens + +Three tiers, in increasing cost, falling directly out of the existing pipeline +([`architecture.md`](architecture.md#5-data-flow-in-full-mode)). The compiler +assigns each rule to the cheapest tier that can express it. + +### Tier 1 — pre-RML, on the TSV + +Row and column removal applied to `.records.tsv` and +`.header_lines.tsv` **before RMLStreamer runs**. + +This is by far the cheapest option and the only one that covers RML-produced +triples as well as wrapper-produced ones, because nothing downstream ever sees +the excluded data. It handles `SampleSelector` (drop a sample column), +`RegionSelector` (drop rows — CHROM and POS are columns 3 and 4, right there), +and `HeaderSelector` (drop header rows). + +Its limit is granularity: it removes whole rows and columns, so it cannot +express "keep this record but drop its `DP`". + +### Tier 2 — emission time, in the wrapper's emitters + +The three direct emitters +([`architecture.md`](architecture.md#4-where-the-split-leaks-and-why)) hold the +full parsed record while they emit, so a predicate-, field- or value-level rule +costs one branch per emitted triple and needs no second pass. + +Concrete hook points in [`vcf_rdfizer.py`](../vcf_rdfizer.py): + +| Where | Change | +| --- | --- | +| `_append_rdf_atomically(rdf_path, stats, producer)` | Wrap `emit` in a policy filter; the atomic-append and rollback behaviour is unchanged | +| `append_expanded_sample_rdf`, `append_condensed_sample_rdf` | Sample- and field-level rules; the condensed one also needs §8 | +| `append_record_detail_rdf` | QUAL and INFO field rules | +| `append_header_representation_rdf` | Header rules | + +**One small prerequisite.** `ParsedSampleRecord` currently carries +`source_file`, `row_id`, `qual`, `info`, `format_keys`, `sample_payloads` and +`sample_values` — `_parse_row` reads columns 0, 1, 7, 9, 10 and −1, and **skips +CHROM and POS** (columns 2 and 3). A `RegionSelector` evaluated at emission time +needs both. That is two extra fields on the dataclass and two extra index reads +in `_parse_row`, with no measurable cost, and it should be the first commit in +this work. + +### Tier 3 — post-hoc, over an existing artifact + +For applying a policy to a graph that has already been converted +(`--mode redact --rdf .nt.gz --policy

.ttl`). + +Triple- and predicate-level rules are a single streaming pass. **Region rules +are not**, because an N-Triples stream has no ordering guarantee: by the time +you see `<…#sample/4711/NA12878> vcfr:fieldValue "0/1"`, the triple that told +you record 4711 is at `chr19:44906000` may be long gone. This needs two passes: +pass one builds a `row_id → (chrom, pos)` map restricted to records that fall in +a policy region, pass two filters. Memory scales with the number of **in-scope** +records, not the graph, which keeps it bounded in practice — but it is +materially more expensive than doing the same work in Tier 1 or 2, and the +documentation should say so rather than let users discover it. + +### Query time, and why it is secondary + +A policy can also be compiled to SPARQL rewriting at an endpoint. It is worth +building, but it is **not** the primary enforcement point for this tool, for a +reason worth stating plainly: VCF-RDFizer's principal output is *files that +people take away*. An HDT or COTTAS artifact handed to a collaborator is outside +any policy engine forever. Query-time enforcement protects an endpoint you +operate; materialization protects a file you ship. Most users of this tool need +the second. + +Where an endpoint *is* operated, note also that rewriting is easy to get subtly +wrong: a prohibition enforced with `FILTER NOT EXISTS` still lets a caller infer +the excluded set from counts and negative results. Query-time enforcement should +be paired with the aggregate controls in §9, not treated as sufficient alone. + +--- + +## 6. The compile step + +```text + ODRL policy (.ttl) + │ parse, validate against the profile, resolve DUO version + ▼ + rule set ──▶ conflict resolution (§4.4) ──▶ release plan + │ + ├── Tier 1 ops: TSV row/column predicates + ├── Tier 2 ops: per-triple predicates, keyed by emitter + ├── Tier 3 ops: post-hoc stream filters (+ any two-pass requirements) + └── residual: rules no tier can enforce ──▶ ABORT +``` + +The **residual set must abort the run, never warn**. A privacy rule that was +parsed, reported, and then not applied is the single worst outcome this design +can produce: the operator believes the release is governed and it is not. If the +compiler cannot enforce a rule, it must refuse to produce an artifact. + +The release plan is a plain data structure with no ODRL dependency, which keeps +the engine testable in isolation and leaves room for a second front end. + +--- + +## 7. Pseudonymization and IRI re-minting + +Because IRIs embed sample names, the source filename and a positional row +counter (§2), filtering triples is not enough. Under `vcfp:pseudonymize` the +release view must re-mint IRIs. + +**Construction.** `token = base32(HMAC-SHA256(release_key, namespace ‖ value))`, +truncated to a documented length, where `namespace` distinguishes sample names +from row ids from filenames so the same string in two roles does not produce the +same token. + +**Properties this gives, and the ones it does not:** + +- Stable *within* a release, so joins inside the released graph still work. +- Unlinkable *across* releases, because `release_key` is fresh per release — + unless linkage is deliberately wanted, in which case a named, reused key is an + explicit policy choice recorded in the manifest. +- The key is **never** written into the artifact or the manifest; only its + identifier and algorithm are. Re-identification remains possible for whoever + holds the key, which is the point — this is pseudonymization, not + anonymization, and the manifest should use that word. +- It does **not** defeat genotype-based re-identification (§1). Nothing here + does. + +Row-id re-minting deserves specific attention: replacing `#record/4711` with a +token removes the ordering leak, but only if the tokens are emitted in an order +that does not reconstruct it. Sort the released graph by token, not by source +order. + +--- + +## 8. The condensed-representation problem + +This one is specific to VCF-RDFizer and easy to miss until it produces a leak. + +In **expanded** mode, one participant's value is its own resource: + +```text +<…#sample/4711/NA12878/fmt/GT> vcfr:fieldValue "0/1" . +``` + +Excluding a sample is triple filtering. Straightforward. + +In **condensed** mode, all participants' values for one FORMAT key live in +**one literal**: + +```text +<…#call/4711/matrix/fmt/GT> vcfr:encodedValues "0/1\t0/0\t1/1\t./."^^vcfr:VCFTextVector . +``` + +There is no triple to remove for one sample. Graph-pattern access control cannot +express "sample 2 only" over this shape at all — the unit of protection is +finer than the unit of storage. + +Two consequences: + +1. **Redaction must rewrite the literal.** `vcfp:maskVectorPositions` replaces + the masked participant's token with `.`, which is the VCF missing marker and + keeps the vector aligned with `vcfr:sampleIndex` — alignment the format + depends on. Dropping a position instead would silently shift every downstream + sample's value, which is a data-corruption bug wearing a privacy feature's + clothing. +2. **Masking is itself visible.** A `.` where neighbours have values discloses + that a value existed and was withheld, and combined with `SampleSet` + membership it discloses *whose*. Where that matters, the policy must mask the + position across **all** samples for that record, or drop the vector entirely. + The compiler should detect a single-position mask and warn — or, under a + strict profile setting, refuse. + +A blunter option is worth offering: `vcfp:requireRepresentation expanded`, which +makes a policy refuse to run against a condensed graph. For policies with +per-sample rules that is often the honest answer, and it trades storage +efficiency for enforceability deliberately rather than by accident. + +--- + +## 9. Beyond access control: disclosure limitation + +Access control does not address §1's attacks, all of which operate on data the +recipient was *permitted* to see. Three mechanisms, in increasing ambition: + +**Count thresholds (`vcfp:threshold`).** Suppress a variant whose allele count +falls below *k*. Cheap, well understood, and directly targets the +rare-variant-as-fingerprint problem, since rare alleles carry most of the +identifying signal. Implementable in Tier 2 with a counting pre-pass. +Recommended default for any cohort release: suppress `AC < 5`, and say so in the +manifest. + +**Aggregate-only exposure (`vcfp:aggregateOnly`).** The subgraph is not +materialized; only counting queries over it are answerable. This is the Beacon +model, and it inherits the Beacon model's known weakness — repeated membership +queries leak — so it must be paired with query budgeting and audit, not offered +as a safe default. + +**Differential privacy.** Noise on aggregate counts is the only mechanism here +with a formal guarantee, and it is also the easiest to implement incorrectly: +a per-query epsilon with no global budget provides no protection at all against +a patient adversary. **Not proposed for implementation.** If it is added later +it needs a persistent budget ledger per recipient, and the design should say +plainly that without one it is decoration. + +--- + +## 10. The release manifest + +Every policy-derived artifact carries provenance, in the same spirit as the +linkset node in [`datalinking-design.md`](datalinking-design.md#5-output-and-provenance): + +```turtle + + a vcfp:ReleaseView ; + vcfp:derivedFrom ; + vcfp:policy ; + vcfp:policyDigest "sha256:9f2c…" ; + vcfp:duoVersion "2024-11-03" ; + vcfp:pseudonymKeyId "release-2026-09-key-3" ; + vcfp:representation vcfr:ExpandedRepresentation ; + vcfp:triplesWithheld 1840221 ; + vcfp:samplesWithheld 2 ; + vcfp:thresholdApplied 5 ; + prov:generatedAtTime "2026-09-04T11:04:22Z"^^xsd:dateTime . +``` + +Three requirements that are easy to get wrong: + +- **The policy is referenced by digest, not just by IRI.** A policy IRI whose + content changed later cannot explain an artifact produced last year. +- **A release view gets its own IRI namespace**, so it can never be silently + mistaken for, or merged with, the full graph. +- **Counts of what was withheld are published.** They are not themselves + sensitive at this granularity, and they are what makes a release auditable + rather than merely asserted. + +--- + +## 11. Verifying a release + +A privacy claim that is not checked is a privacy claim that is wrong. This is +where the existing validation harness earns its keep a second time. + +**Policy conformance checks.** Compile every prohibition into a SPARQL `ASK` (or +a `SELECT COUNT`) that must return zero over the released artifact, and run them +as preflights, exactly as `preflight_blank_nodes` and friends already work. A +release that fails one is not published. This turns "the APOE region was +excluded" from a claim into a measurement — the distinction +[`validation-methodology.md`](validation-methodology.md) is built around. + +**SHACL for shape-level guarantees.** "No `vcfr:sampleName` appears anywhere" is +naturally a shape constraint, and the `--shacl-shapes` layer already exists. + +**Mutation testing the redactor.** Add a mutation class to +[`test/validation_mutations.py`](../test/validation_mutations.py) that +deliberately breaks the redaction — reinstate a withheld sample, un-mask one +vector position, restore a dropped predicate — and assert the conformance checks +catch it. A redactor nobody has tried to defeat is an assumption, not evidence, +which is precisely the argument the validation methodology already makes about +the validator itself. + +**Adversarial checks worth writing early**, because they catch the leaks that +correct-looking implementations produce anyway: + +- Does any IRI in the released graph contain a withheld sample id? +- Do row identifiers reconstruct the source ordering? +- Is any masked vector position identifiable as masked-for-one-sample (§8)? +- Does the header section still name a withheld participant? + +--- + +## 12. Authoring tooling + +`vcf-rdfizer-policy`, mirroring `vcf-rdfizer-rules` and the proposed +`vcf-rdfizer-link`: + +| Command | Purpose | +| --- | --- | +| `policy init -o p.ttl` | Scaffold with annotated examples and a default-deny skeleton | +| `policy check p.ttl` | Validate against the profile: unknown selectors, unresolvable DUO terms, assembly declared, conflicts, and **any rule no tier can enforce** | +| `policy explain p.ttl` | Render the resolved release plan in English: what is dropped, degraded, thresholded, and what remains | +| `policy dry-run p.ttl -i sample.vcf` | Report counts that *would* be withheld, per rule, writing nothing | +| `policy diff p1.ttl p2.ttl` | What changes between two policy versions — the question a data access committee actually asks | + +`explain` is the one that earns its keep. A policy nobody can read is a policy +nobody can approve, and the failure mode is approval-by-exhaustion. + +--- + +## 13. Build order + +| Step | Delivers | Unlocks | +| --- | --- | --- | +| 1 | `vcfp:` profile, policy parsing, `Predicate`/`Class`/`Sample`/`Header` selectors, `vcfp:drop`, default-deny, conflict rules | Coarse governed release with no new machinery | +| 2 | Tier 1 (TSV) + Tier 2 (emitter) enforcement, `CHROM`/`POS` on `ParsedSampleRecord`, `RegionSelector` | Region and field granularity at no runtime cost | +| 3 | Release manifest, policy conformance preflights, mutation entries | Releases become auditable and verified rather than asserted | +| 4 | `vcf-rdfizer-policy` CLI, `explain`, `diff` | Policies become reviewable by non-implementers | +| 5 | `vcfp:pseudonymize` with IRI re-minting, `maskVectorPositions`, `threshold` | The leaks in §7 and §8 closed | +| 6 | `--mode redact` (Tier 3), then optional query-time rewriting | Existing artifacts, and endpoint deployments | + +Steps 1–3 are the contract and the evidence. Note that step 3 comes *before* the +sophisticated transforms in step 5 — deliberately. A crude redaction that is +verified is worth more than a sophisticated one that is not. + +--- + +## 14. Known hard problems + +Stated up front, as in the linking design. + +**This is not anonymization, and the vocabulary must not drift.** Every user- +facing string, manifest field and log line should say *pseudonymized*, +*withheld*, or *governed release*. The moment the tool says "anonymized", it is +making a claim it cannot support (§1), and someone will rely on it. + +**Compliance is not a feature.** GDPR special-category processing, national +genomic-data law, and the scope of a specific consent form are determinations +for a data protection officer and a data access committee. The tool can +*implement* a policy and *evidence* what it did. It cannot decide what the +policy should be, and the documentation must not imply otherwise. + +**Structural leakage survives redaction.** A withheld region still shows as a +gap; a withheld sample still shows as a missing `SampleSet` member unless the +set itself is rewritten; a thresholded variant still shows as absent. Whether +those gaps matter is a policy question, but the tool should surface them — +`policy explain` is the right place. + +**Composition across releases is unmanaged.** Two separately compliant releases +to the same recipient can jointly disclose more than either alone: different +regions, different thresholds, or a fresh pseudonym key that is nonetheless +linkable through the genotypes. Nothing in this design tracks cumulative +disclosure, and the manifest is the only thread that would make it possible +later. Say so rather than implying releases are independent. + +**Policy drift versus artifact.** A policy IRI resolves to whatever it resolves +to *today*; the artifact was produced under what it said *then*. The digest in +§10 is what makes that recoverable, and it is why it is mandatory rather than +nice to have. + +**The escape hatch is a footgun.** `vcfp:PatternSelector` accepts arbitrary +SPARQL, which cannot be pushed into Tier 1 or 2 and whose cost is unbounded. It +should be permitted, reported prominently by `explain`, and excluded from any +profile intended for unattended use. + +--- + +## See also + +- [Data linking design](datalinking-design.md) — the sibling proposal; same plugin, provenance and verification patterns +- [Conversion](conversion.md) — the IRI templates and graph shapes a policy addresses +- [Sample representations](sample-representation-guide.md) — why condensed mode changes the enforceability of per-sample rules +- [Validation methodology](validation-methodology.md) — the harness that makes a privacy claim measurable +- [Limitations](limitations.md) — what the tool does not do today +- [Roadmap](roadmap.md) — how this relates to the other planned work diff --git a/docs/roadmap.md b/docs/roadmap.md index c44d74b..fdf3c14 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -107,6 +107,34 @@ every vector in a cohort-wide query produces millions of bindings before filtering. A targeted single-position extractor is the right primitive; a whole-graph expansion is not. +### 7. Granular privacy policies over the graph + +Today the tool has exactly one disclosure setting: convert everything. There is +no way to release a cohort graph with some participants withheld, some regions +degraded, or some fields suppressed, and no machine-readable record of what a +given artifact was permitted to contain. + +The proposal is an ODRL profile whose assets are **graph selectors** (by class, +predicate, sample, genomic region, declared field or pattern), compiled to a +release plan and enforced at the cheapest available point in the existing +pipeline — TSV pre-filtering, emitter-time filtering, or a post-hoc pass. +Full design in [`privacy-policy-design.md`](privacy-policy-design.md). + +Two findings from that design are worth surfacing here because they affect work +outside it: + +- **`ParsedSampleRecord` does not carry `CHROM` or `POS`.** `_parse_row` reads + columns 0, 1, 7, 9, 10 and −1. Any region-scoped feature evaluated at emission + time needs both, and adding them is two fields and two index reads. +- **Condensed mode is not per-sample enforceable by triple filtering.** One + `FormatValueVector` literal holds every participant's value for a FORMAT key, + so the unit of protection is finer than the unit of storage. Redaction has to + rewrite the literal, and a single masked position is itself disclosive. + +The honest framing, which the design keeps throughout: this is *governed +release*, not anonymization. Genotypes identify people, and no access-control +layer changes that. + ## Deliberately not planned Stated so the absence reads as a decision rather than an oversight. @@ -118,6 +146,8 @@ Stated so the absence reads as a decision rather than an oversight. | Distributed or multi-node execution | Out of scope; chunking already bounds memory on one machine | | Incremental graph update | Would require identity and provenance machinery the current model does not have | | Clinical interpretation or pathogenicity assertion | The tool transcribes; it is not a clinical authority | +| Differential privacy on aggregate queries | Formal guarantees need a persistent per-recipient budget ledger; without one it is decoration. See [`privacy-policy-design.md` §9](privacy-policy-design.md#9-beyond-access-control-disclosure-limitation) | +| Deciding whether a release is legally compliant | A tool can implement and evidence a policy; it cannot make a data-protection determination | | `.bcf` input | Would pull `htslib` into the parsing path; convert with `bcftools` first | --- @@ -126,5 +156,6 @@ Stated so the absence reads as a decision rather than an oversight. - [Limitations](limitations.md) — the current state, honestly - [Data linking design](datalinking-design.md) — the largest planned addition +- [Privacy policy design](privacy-policy-design.md) — governed release over parts of the graph - [Validation methodology](validation-methodology.md) — how coverage is measured, so gaps stay falsifiable - [`changelog.md`](../changelog.md) — what has actually shipped diff --git a/docs/validation-methodology.md b/docs/validation-methodology.md index b73d9f5..47000a6 100644 --- a/docs/validation-methodology.md +++ b/docs/validation-methodology.md @@ -117,28 +117,54 @@ own IRI**, then bucket on the first byte of the hash. Two properties matter: Fields are separated by U+001F, which cannot occur in a VCF field, so no shift of a field boundary can forge a match. -## Two engines, two layers +## Four engines, two layers The host layer runs queries under **rdflib**, in process, needing no Docker, so -the whole catalogue runs in the normal test loop. rdflib is also a third +the whole mutation catalogue runs in the normal test loop. rdflib is also an independent SPARQL implementation, which incidentally guards against queries that only work on one engine. The container layer is the authority: [`test/cross_engine_agreement.py`](../test/cross_engine_agreement.py) runs every -validation query under **Comunica and QLever** inside the image and asserts they -return identical values. +validation query under **Comunica, QLever, native HDT and native COTTAS** inside +the image, across both representations, and asserts they return identical +values. It then runs the shipped validation decision under each engine +separately, because engines agreeing with each other while all being wrong is a +real failure mode that only the Python oracle rules out. That second layer is not ceremony. QLever canonicalises numeric literals at index time, reporting `"100"^^xsd:integer` as `xsd:int`. The POS datatype preflight originally required exactly `xsd:integer`, so it flagged every record -under QLever while passing under Comunica — every QLever run would have ended +under QLever while passing under Comunica - every QLever run would have ended `BLOCKED_BY_PREFLIGHT`. Only cross-engine execution surfaces that class of bug. +It has since caught two more. Comunica's HDT engine treats a bare filesystem +path as a link to dereference, so the source must be addressed as +`hdt@`; and cyvcf2's `raw_header` is htslib's *normalised* header, which +injects `##FILTER=` into files that +never declared it - making the oracle expect a `FilterDefinition` the graph +could not contain. The oracle now reads the header block from the file itself, +which is the same text the conversion reads. + **When adding a query**, prefer datatype-*family* checks and lexical comparisons over anything that assumes a store's internal representation, and run the agreement script before trusting it. +### Measuring, not just comparing + +Because every engine answers the same query set against the same graph in the +same container, a multi-engine run is also a controlled benchmark, and the +suite records it: per-query wall time for each engine, setup time (a QLever +index build, or an HDT/COTTAS artifact build) kept separate from query time, +and the **oracle** - what it costs to compute the same answers directly from +the VCF with cyvcf2. + +That last figure makes the comparison meaningful rather than merely internal. +The suite computes every expected value twice, once by parsing and once by +querying, so the two costs are for identical work on identical input. The +report is written as `benchmark.json` plus a long-format `benchmark.csv`, one +row per engine and query. See [`validation.md`](validation.md#timings-and-comparing-sparql-against-the-parser). + ## Reproducing the score ```bash @@ -159,6 +185,9 @@ docker run --rm -v "$PWD:/repo:ro" vcf-rdfizer:local \ /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py ``` +All four engines are compared by default; pass a comma-separated subset as the +first argument to narrow it. + Both run in CI via [`.github/workflows/validation-mutation.yml`](../.github/workflows/validation-mutation.yml). diff --git a/docs/validation.md b/docs/validation.md index 4d8c178..5b45242 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -12,8 +12,10 @@ using `cyvcf2` (and `bcftools` for exact FILTER strings when available), runs the equivalent SPARQL queries against the graph, then compares canonical integer results exactly. Two axes are configurable and independent: which artifact is validated (N-Triples, HDT, or COTTAS) and which SPARQL engine runs -the queries (Comunica or QLever). Read "What is *not* tested" below before -treating a `PASS` as a correctness proof. +the queries (Comunica, QLever, native HDT, native COTTAS - or several at once, +which cross-checks them and benchmarks them against each other and against the +parser). Read "What is *not* tested" below before treating a `PASS` as a +correctness proof. The source artifact is mounted read-only. Anything that is not already plain N-Triples is decoded under `/work` **inside the Docker container**; a plain @@ -112,14 +114,110 @@ reported as a failure - the run already records why it is absent. ## Which SPARQL engine runs the queries -`--validation-engine` selects the backend. Both answer identical queries and -feed the same comparison layer, so the choice is a scale decision, never a -semantic one; every report records which engine produced it. +`--validation-engine` selects the backend. Every engine answers the same +queries and feeds the same comparison layer, so the choice is a scale and +performance decision, never a semantic one; every report records which engine +produced it. -| Engine | Behaviour | Use it when | -|---|---|---| -| `comunica` (default) | Queries the N-Triples file with no setup, holding the graph in the Node heap | The graph fits comfortably in RAM | -| `qlever` | Builds an on-disk [QLever](https://github.com/ad-freiburg/qlever) index in container scratch, serves it on a container-local port, then tears both down | The graph no longer fits in memory, or the aggregate queries are too slow | +| Engine | Queries | Setup | Use it when | +|---|---|---|---| +| `comunica` (default) | The N-Triples file directly | none | The graph fits comfortably in RAM | +| `qlever` | An on-disk [QLever](https://github.com/ad-freiburg/qlever) index, served on a container-local port | index build | The graph no longer fits in memory, or the aggregate queries are too slow | +| `hdt` | A `.hdt` artifact **in place**, through Comunica's HDT engine | reuses the run's HDT, or builds one | Checking that the compressed artifact is queryable, not just decodable | +| `cottas` | A `.cottas` artifact **in place**, through `pycottas`'s rdflib store | reuses the run's COTTAS, or builds one | Same, for COTTAS | + +### Validating a compressed artifact without decoding it + +`--validation-target hdt,cottas` validates those artifacts by decoding them +back to N-Triples first. That proves the decode is faithful; it says nothing +about querying them, and it measures the wrong thing entirely in a performance +comparison. + +`--validation-engine hdt` and `--validation-engine cottas` instead query the +artifact where it lies. When the run's own artifact is already in the engine's +format it is used directly - the honest measurement. Otherwise the engine +builds one in container scratch from the materialized N-Triples, and that build +is timed as **setup**, kept out of the query total. Each report records which +of the two happened, in `artifactOrigin`. + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz --rdf ./results/cohort/cohort.hdt \ + --validation-engine hdt \ + --out ./validation-results +``` + +HDT is queried with `comunica-sparql-hdt`, installed in the image alongside +Comunica; the source is addressed as `hdt@`, because a bare path would be +treated as a link to dereference. COTTAS is queried in-process through +`pycottas.COTTASStore`, an rdflib `Store` over the Parquet artifact, under the +same interpreter that already owns pycottas. + +### Several engines in one run + +`--validation-engine` accepts a comma-separated list, or `all`: + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --validation-engine all \ + --out ./validation-results +``` + +Every requested engine answers the whole query set. Two things follow: + +- **Cross-checking.** The normalized results of every engine are compared + against each other and written to `engine-agreement.json`. Engines + disagreeing is a finding in its own right - see below. +- **Benchmarking.** Every engine is timed identically, against the same graph, + in the same container, in one run. + +The first engine named is the **primary**. Its reports keep the single-engine +layout at the top of the results directory, so existing consumers are +unaffected; each engine additionally gets `engines//` with its own +`preflight.json`, `sparql.json`, `comparison.json` and `query-executions.json`. +The run's `status` is `PASS` only if every engine passed, and `summary.json` +carries `engineStatuses` with the per-engine verdict. + +### Timings, and comparing SPARQL against the parser + +Every run writes `benchmark.json` and a long-format `benchmark.csv` - one row +per engine and query, ready to plot without reshaping: + +``` +engine,query_id,status,wall_seconds,oracle_wall_seconds,engine_setup_seconds,artifact_origin +``` + +`benchmark.json` adds the breakdown: per-engine setup and total query time, the +slowest query per engine, N-Triples materialization and SHACL time, and the +**oracle** - what it costs to compute the same answers directly from the VCF +with cyvcf2, split into parse and census. That last figure is the point of +comparison: the validation suite computes every expected value twice, once by +parsing and once by querying, so a run measures a SPARQL engine against a +purpose-built parser on identical work. + +An engine that produced no timed query reports `null` rather than `0`, because +zero seconds reads as "instant" rather than "never ran". + +The parent run's `metrics.csv` carries the summary columns +`validation_engines`, `validation_oracle_seconds`, +`validation_engine_query_seconds` and `validation_engine_setup_seconds`, the +last two as `engine=seconds` pairs. + +An indicative single-container run over a small fixture (312 triples, expanded) +gives the shape of the difference rather than a benchmark result: + +| Engine | Setup | 27 queries | Notes | +|---|---|---|---| +| oracle (cyvcf2) | - | 0.003 s | the parser computing the same answers | +| qlever | 0.17 s | 0.31 s | index built once, then served | +| cottas | 0.28 s | 0.97 s | in-process, no subprocess per query | +| hdt | 0.003 s | 23.9 s | one Comunica process per query | +| comunica | - | 25.1 s | one Comunica process per query | + +Comunica's cost here is almost entirely per-process startup, which a +fixture-sized graph cannot amortize; the ordering says nothing about how these +engines behave on a cohort-sized graph. ```bash vcf-rdfizer --mode validation \ @@ -135,7 +233,7 @@ QLever tuning, all optional: | `--qlever-memory-gb N` | Index and server memory budget (default 4) | | `--qlever-port N` | Container-local port (default 7019; never published) | | `--qlever-startup-timeout N` | Seconds to wait for the server after indexing (default 900) | -| `--validation-query-timeout N` | Per-query timeout, both engines (default 3600) | +| `--validation-query-timeout N` | Per-query timeout, every engine (default 3600) | | `--qlever-index-arg ARG` | Extra argument for the index builder (repeatable) | | `--qlever-server-arg ARG` | Extra argument for the server (repeatable) | @@ -167,9 +265,16 @@ link still validates normally. ### Engine equivalence, and one place they differed -Both engines are held to producing identical results. That is verified, not -assumed: all eleven queries were run under both engines against the same -expanded and condensed graphs, and every normalized result matched. +Every engine is held to producing identical results. That is verified, not +assumed: [`test/cross_engine_agreement.py`](../test/cross_engine_agreement.py) +runs the full query set under all four engines, against the same expanded and +condensed graphs, and additionally runs the shipped validation decision under +each - so an engine must agree both with the other engines and with the Python +oracle. Engines agreeing with each other and all being wrong is a real failure +mode; only the oracle rules it out. + +A multi-engine production run performs the first half of that check on the +real graph and records it in `engine-agreement.json`. One difference had to be fixed to make that true. QLever canonicalises numeric literals at index time, so `"100"^^xsd:integer` is reported by `DATATYPE()` as @@ -303,53 +408,60 @@ back into per-sample value resources. ## What is *not* tested -The suite is deliberately a set of exact aggregate comparisons. That makes it -cheap, engine-independent, and free of a second RDF parser to disagree with - -but it bounds what it can detect, and those bounds are worth stating plainly. -`test/test_validation_logic_unit.py` encodes both the detections and the gaps -below, so a change that closes one will fail a test rather than pass silently. - -**No per-record identity.** Every core query is a `GROUP BY` count. A graph that -swapped `POS` between two records on the same contig and in the same 1 Mb -window - or `REF`/`ALT` between two records of the same shape class - produces -byte-identical results and passes. The suite proves the *distributions* match, -not that record *i* carries record *i*'s values. Detecting a permutation needs a -per-record identity check, which the current query set does not have. - -**Whole VCF columns are unchecked.** Nothing compares `ID`, `QUAL`, or `INFO`, -and no `FORMAT` field other than `GT` is validated. A conversion bug that -mangled every `DP` value, dropped every rsID, or corrupted `INFO` would pass. -The `header_lines` and `file_metadata` triples maps - the `HeaderLine` -resources and the `VCFFile` attributes - are likewise never queried. - -**No completeness bound.** Nothing asserts a total triple count or the absence -of extraneous triples. `preflight_record_cardinality` catches duplicated or -missing per-record properties, but only for `VCFRecord` subjects. +The suite is three independent layers - exact aggregate comparisons, a +predicate/class census with per-record and per-value identity digests, and +graph-integrity checks - plus optional SHACL. That is a broad net, but it is +still a net, and its holes are worth stating plainly. Every claim below is +backed by a named mutation in +[`test/validation_mutations.py`](../test/validation_mutations.py), so a change +that closes a gap fails a test rather than passing silently. + +**`vcfr:contigCount` is counted, not read.** The census asserts the predicate is +present the expected number of times; nothing compares its *value*. A derived +contig total that is simply wrong passes. This is the one mutation in the +catalogue that is still undetected. **Header line values are compared only through their structured form.** The attributes that matter - `filterId`, `altId`, `contigId`, contig length/md5/ assembly, and the INFO/FORMAT declarations - are lifted into their own -properties and counted. The raw `vcfr:headerValue` literal itself is counted but -never read, and `vcfr:contigCount` is likewise checked for presence rather than -value. - -**The census assumes the shipped mapping.** Under a custom `--rules`, Q9-Q13 -are no longer meaningful and the run should rest on the aggregate comparisons -alone - which is what `--mapping-policy report-only` is for, and which the -wrapper does not yet enable (see the note above). - -In short: a `PASS` is strong evidence that the conversion preserved the VCF's -*summary statistics* over the elements it covers, and it reliably catches -dropped records, misclassified variants, flipped genotypes, corrupted FILTER -strings, allele-count errors, missing header lines, and altered file metadata. -It is not proof of a faithful record-by-record round-trip. Treat it as a -regression gate, not a correctness proof. - -These limits are measured rather than asserted. Every claim above corresponds -to a named mutation in the catalogue described in -[`validation-methodology.md`](validation-methodology.md); the current score and +properties and compared. The raw `vcfr:headerValue` literal itself is counted +but never read. + +**Multi-valued INFO fields keep only their lexical value.** A `Number=1` INFO +value additionally carries a typed `fieldValueInteger`/`fieldValueDecimal`; a +`Number=A/R/G/.` field does not, because the vocabulary's IRI template gives one +node per key. The full lexical value is still digested, so corruption is caught +- but no per-element typing is checked. + +**The census assumes the shipped mapping.** A custom `--rules` changes the +predicate inventory and the IRI templates by design, so `q09`-`q13` fall back to +report-only and the run rests on the aggregate comparisons alone. That is +correct behaviour, not a bug, but it means a custom mapping is validated more +weakly than the default one. + +**Condensed graphs are not ontology-backed.** The condensed representation uses +17 terms the published vocabulary does not define, so SHACL cannot meaningfully +check it and dereferencing those terms returns nothing. The list is in +[`vcf-coverage.md`](vcf-coverage.md); closing it is work in the vocabulary +repository. + +**A high mutation score is a lower bound on blindness, not a proof.** It says +"almost every corruption we thought to write down is caught". Corruptions nobody +wrote down are, by construction, not measured. + +In short: a `PASS` means the graph contains exactly the predicates and classes +the VCF implies, that every record, INFO value and FORMAT value hashes to the +same bucket as its counterpart in the source, that the summary statistics agree +exactly, and that the graph carries no blank nodes, empty terms or duplicated +statements. It reliably catches dropped records, misclassified variants, flipped +genotypes, corrupted FILTER strings, allele-count errors, permuted record +fields, missing header lines, and altered file metadata. It is still not proof +of a faithful record-by-record round-trip. Treat it as a regression gate. + +These limits are measured rather than asserted. The current mutation score and the full element-by-element breakdown are in -[`vcf-coverage.md`](vcf-coverage.md). +[`vcf-coverage.md`](vcf-coverage.md); the method is described in +[`validation-methodology.md`](validation-methodology.md). ## Results and cleanup evidence @@ -364,12 +476,18 @@ the same `summary.json` used for conversion and compression metrics. Important files include `summary.json`, `manifest.json`, `parser.json`, `rdf-validation.json`, `materialization.json`, `preflight.json`, `sparql.json`, -and `comparison.json`. Raw SPARQL Results JSON, stderr, and query resource logs -are in `raw/`; normalized results are in `normalized/`. `manifest.json` records -the engine that ran (including QLever's exact argv), the source artifact format -and checksum, and every decode step; `materialization.json` records how a -compressed or indexed artifact was turned into N-Triples and how many triples -that yielded. +`comparison.json`, `benchmark.json` and `benchmark.csv`. Raw SPARQL Results +JSON, stderr, and query resource logs are in `raw//`; normalized results +are in `normalized/`. `manifest.json` records the engine that ran (including +QLever's exact argv), the source artifact format and checksum, and every decode +step; `materialization.json` records how a compressed or indexed artifact was +turned into N-Triples and how many triples that yielded. + +A multi-engine run adds `engines//` - one `preflight.json`, +`sparql.json`, `comparison.json` and `query-executions.json` per engine - and +`engine-agreement.json`, which records whether every engine returned the same +normalized results and, if not, which queries differed. The top-level reports +remain those of the primary (first-named) engine. When several artifacts are validated in one full run, each gets its own directory: the aggregate keeps `/`, and the representations use diff --git a/src/validation/validation_runner.py b/src/validation/validation_runner.py index 427f591..ecc2db5 100644 --- a/src/validation/validation_runner.py +++ b/src/validation/validation_runner.py @@ -24,6 +24,7 @@ from __future__ import annotations import argparse +import csv import gzip import hashlib import json @@ -607,6 +608,29 @@ def info_declared_types(raw_header: str) -> dict[str, str]: return types +def read_vcf_header_text(vcf_path: Path) -> str: + """Read the header block straight from the VCF file. + + Returns the leading ``#`` lines verbatim - the ``##`` meta-information + lines and the ``#CHROM`` column line - which is the same span cyvcf2's + ``raw_header`` covers, so callers of either see the same shape. + + Deliberately not cyvcf2's ``raw_header``: htslib normalises the header it + exposes, injecting declarations the file does not contain - notably + ``##FILTER=``. The conversion + reads the file's own text, so the oracle must too, or it expects header + resources the graph could never contain. + """ + opener = gzip.open if vcf_path.name.endswith(".gz") else open + lines: list[str] = [] + with opener(vcf_path, "rt", encoding="utf-8", errors="replace") as handle: + for line in handle: + if not line.startswith("#"): + break + lines.append(line.rstrip("\n")) + return "\n".join(lines) + + def parse_header_metadata(raw_header: str) -> dict[str, Any]: """Summarise the '##' meta-information block of a VCF. @@ -709,7 +733,7 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: filters = filters_with_bcftools(vcf_path) if use_bcftools else Counter() reader = VCF(str(vcf_path), strict_gt=True) samples = list(reader.samples) - header_metadata = parse_header_metadata(reader.raw_header) + header_metadata = parse_header_metadata(read_vcf_header_text(vcf_path)) density: Counter[tuple[str, int]] = Counter() shapes: Counter[str] = Counter() genotypes: Counter[tuple[str, str]] = Counter() @@ -723,7 +747,7 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: expanded_format_digest: Counter[str] = Counter() condensed_format_digest: Counter[str] = Counter() sample_components = [rml_uri_component(uri_id) for uri_id in sample_uri_ids(samples)] - declared_info_types = info_declared_types(reader.raw_header) + declared_info_types = info_declared_types(read_vcf_header_text(vcf_path)) info_definitions: set[str] = set() info_values = info_flags = info_typed_integers = info_typed_decimals = 0 source_component = rml_uri_component(vcf_path.name) @@ -1208,7 +1232,18 @@ def validate_ntriples(source: Path, results_dir: Path) -> dict[str, Any]: # qlever: builds an on-disk index, then answers over HTTP. Slower to start, # but the only option once a graph stops fitting in memory. -SPARQL_ENGINES = ("comunica", "qlever") +#: Every backend that can answer the validation queries. `comunica` and +#: `qlever` read the materialized N-Triples; `hdt` and `cottas` query the +#: compressed representation *directly*, without decoding it first, which is +#: the property those formats exist for. Several may be requested in one run: +#: they all answer the same queries, so the comparison is meaningful and the +#: recorded timings are directly comparable. +SPARQL_ENGINES = ("comunica", "qlever", "hdt", "cottas") +#: Engines that query a compressed artifact natively, and the artifact format +#: each one needs. When the run's source is a different format, the artifact is +#: built in scratch first and the build is timed separately from the queries, +#: so an index build is never mistaken for query cost. +NATIVE_ENGINE_FORMATS = {"hdt": "hdt", "cottas": "cottas"} # QLever's binaries are copied out of the upstream image, which is built on a # different Ubuntu release, so their Boost/ICU/jemalloc sonames come with them # in a private directory. Pointing only QLever's own processes at it keeps @@ -1249,9 +1284,19 @@ def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[ self.scratch = scratch self.options = options self.query_timeout = int(options.get("query_timeout") or DEFAULT_QUERY_TIMEOUT) + #: Seconds spent preparing the backend (index build, format conversion, + #: server startup) before any query ran. Reported separately from query + #: time so a benchmark can attribute cost correctly. + self.setup_seconds: float | None = None - def __enter__(self) -> "QueryEngine": + def prepare(self) -> None: + """Time ``start`` so setup cost is recorded for every engine alike.""" + started = time.monotonic() self.start() + self.setup_seconds = time.monotonic() - started + + def __enter__(self) -> "QueryEngine": + self.prepare() return self def __exit__(self, exc_type, exc, tb) -> bool: @@ -1265,7 +1310,7 @@ def stop(self) -> None: """Release anything ``start`` acquired. Must be safe to call twice.""" def describe(self) -> dict[str, Any]: - return {"engine": self.name} + return {"engine": self.name, "setupSeconds": self.setup_seconds} def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: raise NotImplementedError @@ -1313,6 +1358,7 @@ def start(self) -> None: def describe(self) -> dict[str, Any]: return { "engine": self.name, + "setupSeconds": self.setup_seconds, "version": tool_version( ["comunica-sparql-file", "--version"], table_label="Comunica Engine" ), @@ -1515,6 +1561,7 @@ def _post(self, query: str, *, timeout: int) -> bytes: def describe(self) -> dict[str, Any]: return { "engine": self.name, + "setupSeconds": self.setup_seconds, "version": tool_version([self.server_binary, "--help"]) if self.server_binary else None, "mode": "on-disk index served over HTTP", "indexDirectory": str(self.index_dir), @@ -1563,7 +1610,176 @@ def stop(self) -> None: shutil.rmtree(self.index_dir, ignore_errors=True) -ENGINE_CLASSES = {"comunica": ComunicaEngine, "qlever": QleverEngine} +class NativeArtifactEngine(QueryEngine): + """Base for engines that query a compressed artifact without decoding it. + + The point of HDT and COTTAS is that they are queryable in place. Validating + them by decoding to N-Triples first proves the decode is faithful but says + nothing about querying them, and measures the wrong thing entirely for a + performance comparison. + + When the run's own artifact is already in this engine's format it is used + directly, which is the honest measurement. Otherwise one is built in scratch + from the materialized N-Triples, and that build is timed as setup rather + than as query cost. + """ + + #: Artifact format this engine reads, e.g. "hdt". + artifact_format = "" + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + super().__init__(source, raw_dir=raw_dir, scratch=scratch, options=options) + self.artifact: Path | None = None + self.artifact_origin = "unknown" + self.log_dir = raw_dir / "engine" + + def _resolve_artifact(self) -> Path: + """Use the run's own artifact when it matches, else build one.""" + supplied = self.options.get("artifact_path") + if supplied is not None and self.options.get("artifact_format") == self.artifact_format: + self.artifact_origin = "run artifact" + return Path(supplied) + self.artifact_origin = "built from N-Triples for this engine" + target = self.scratch / f"{self.name}-engine.{self.artifact_format}" + self.log_dir.mkdir(parents=True, exist_ok=True) + self.build_artifact(target) + return target + + def build_artifact(self, target: Path) -> None: + raise NotImplementedError + + def start(self) -> None: + self.artifact = self._resolve_artifact() + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "setupSeconds": self.setup_seconds, + "mode": f"native {self.artifact_format} query, no decode", + "artifact": str(self.artifact) if self.artifact else None, + "artifactOrigin": self.artifact_origin, + "artifactSizeBytes": ( + self.artifact.stat().st_size + if self.artifact and self.artifact.is_file() else None + ), + } + + +class HdtEngine(NativeArtifactEngine): + """Query a .hdt artifact in place with Comunica's HDT engine.""" + + name = "hdt" + artifact_format = "hdt" + + def build_artifact(self, target: Path) -> None: + rdf2hdt = _resolve_binary("RDF2HDT_BIN", "rdf2hdt", "/usr/local/bin/rdf2hdt") + _run_step( + [rdf2hdt, str(self.source), str(target)], + label="hdt-engine-build", log_dir=self.log_dir, + ) + + def start(self) -> None: + self.executable = shutil.which("comunica-sparql-hdt") + if not self.executable: + raise RuntimeError( + "comunica-sparql-hdt is not installed in this image, so HDT cannot " + "be queried natively. Rebuild the image, or validate the HDT " + "artifact by decoding it (--rdf file.hdt --engine comunica)" + ) + super().start() + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + command = [ + self.executable, + # Comunica needs the source type declared: a bare path is treated + # as a link to dereference and fails with "could not dereference". + f"hdt@{self.artifact}", + "-f", str(query_path), + "-t", "application/sparql-results+json", + ] + started = time.monotonic() + try: + with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + result = subprocess.run( + command, check=False, stdout=stdout, stderr=stderr, + timeout=self.query_timeout, + ) + returncode, error = result.returncode, None + except subprocess.TimeoutExpired: + returncode, error = 124, f"query exceeded {self.query_timeout}s" + return self._envelope( + query_id, query_path, returncode=returncode, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=error, + ) + + +class CottasEngine(NativeArtifactEngine): + """Query a .cottas artifact in place through pycottas's rdflib store. + + pycottas exposes ``COTTASStore``, an rdflib Store backed by the Parquet + artifact, so this runs in-process rather than shelling out. The validator + already runs under the interpreter that owns pycottas. + """ + + name = "cottas" + artifact_format = "cottas" + + def build_artifact(self, target: Path) -> None: + import pycottas + + pycottas.rdf2cottas(str(self.source), str(target)) + + def start(self) -> None: + try: + import pycottas # noqa: F401 + import rdflib # noqa: F401 + except ImportError as error: + raise RuntimeError( + f"pycottas and rdflib are required to query COTTAS natively: {error}" + ) from error + super().start() + + import pycottas + import rdflib + + self.graph = rdflib.Graph(store=pycottas.COTTASStore(str(self.artifact))) + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + started = time.monotonic() + try: + result = self.graph.query(query_path.read_text(encoding="utf-8")) + raw_path.write_bytes(result.serialize(format="json")) + stderr_path.write_bytes(b"") + returncode, error = 0, None + except Exception as failure: # noqa: BLE001 - reported as EXECUTION_FAILED + raw_path.write_text("{}", encoding="utf-8") + stderr_path.write_text(str(failure), encoding="utf-8") + returncode, error = 1, str(failure) + return self._envelope( + query_id, query_path, returncode=returncode, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=error, + ) + + def stop(self) -> None: + graph = getattr(self, "graph", None) + if graph is not None: + try: + graph.close() + except Exception: # noqa: BLE001 - teardown must not mask a result + pass + self.graph = None + + +ENGINE_CLASSES = { + "comunica": ComunicaEngine, + "qlever": QleverEngine, + "hdt": HdtEngine, + "cottas": CottasEngine, +} def build_engine( @@ -1928,6 +2144,154 @@ def evaluate_validation( } +# --------------------------------------------------------------------------- +# Benchmarking +# --------------------------------------------------------------------------- +# A validation run already measures everything a performance comparison needs: +# it computes the answers twice, once from the VCF with a conventional parser +# and once from RDF with a SPARQL engine, over the same data and to the same +# result. Recording those timings turns each run into a directly comparable +# oracle-versus-SPARQL measurement, and a multi-engine run into a comparison +# between the engines as well. + +BENCHMARK_CSV_HEADER = [ + "engine", + "query_id", + "status", + "wall_seconds", + "oracle_wall_seconds", + "engine_setup_seconds", + "artifact_origin", +] + + +def build_benchmark( + per_engine: dict[str, dict[str, Any]], + engine_descriptions: dict[str, Any], + *, + oracle_seconds: dict[str, float] | None, + materialization_seconds: float | None, + shacl_seconds: float | None, + query_ids: tuple[str, ...], +) -> dict[str, Any]: + """Assemble per-query and per-engine timings into one report.""" + oracle_seconds = oracle_seconds or {} + engines: dict[str, Any] = {} + for name, verdict in per_engine.items(): + executions = verdict.get("executions") or verdict.get("queryExecutions") or {} + queries = { + query_id: { + "status": execution.get("status"), + "wallSeconds": execution.get("wallSeconds"), + } + for query_id, execution in executions.items() + } + timed = [ + entry["wallSeconds"] for entry in queries.values() + if isinstance(entry["wallSeconds"], (int, float)) + ] + description = engine_descriptions.get(name, {}) + engines[name] = { + "status": verdict.get("status"), + "setupSeconds": description.get("setupSeconds"), + "artifactOrigin": description.get("artifactOrigin"), + "artifactSizeBytes": description.get("artifactSizeBytes"), + "querySeconds": sum(timed) if timed else None, + "slowestQuery": ( + max(queries.items(), key=lambda item: item[1]["wallSeconds"] or 0)[0] + if timed else None + ), + "queries": queries, + } + + return { + "oracle": { + # The parser side of the comparison: what it costs to compute the + # same answers from the VCF directly. + "totalSeconds": oracle_seconds.get("total"), + "vcfParseSeconds": oracle_seconds.get("parse"), + "censusSeconds": oracle_seconds.get("census"), + }, + "preparation": { + "materializationSeconds": materialization_seconds, + "shaclSeconds": shacl_seconds, + }, + "engines": engines, + "totals": { + "oracleSeconds": oracle_seconds.get("total"), + "engineQuerySeconds": { + name: value["querySeconds"] for name, value in engines.items() + }, + "engineSetupSeconds": { + name: value["setupSeconds"] for name, value in engines.items() + }, + }, + "queryIds": list(query_ids), + } + + +def write_benchmark_csv(path: Path, benchmark: dict[str, Any]) -> Path: + """Write one row per engine and query, for direct analysis.""" + path.parent.mkdir(parents=True, exist_ok=True) + oracle_total = benchmark["oracle"]["totalSeconds"] + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=BENCHMARK_CSV_HEADER) + writer.writeheader() + for name, engine in benchmark["engines"].items(): + for query_id, entry in engine["queries"].items(): + writer.writerow({ + "engine": name, + "query_id": query_id, + "status": entry["status"], + "wall_seconds": entry["wallSeconds"], + # Repeated on each row so the CSV is usable without a join. + "oracle_wall_seconds": oracle_total, + "engine_setup_seconds": engine["setupSeconds"], + "artifact_origin": engine["artifactOrigin"] or "", + }) + return path + + +def compare_engines(per_engine: dict[str, dict[str, Any]]) -> dict[str, Any]: + """Check that every engine that ran produced the same normalized results. + + Engines disagreeing is a finding in its own right: the graph may be correct + and one engine wrong, as when QLever's literal canonicalisation made a + datatype preflight fail on it alone. A multi-engine run is the cheapest + place to notice that, so it is checked rather than assumed. + """ + usable = { + name: verdict["sparql"] + for name, verdict in per_engine.items() + if verdict.get("status") != "EXECUTION_FAILED" and verdict.get("sparql") + } + if len(usable) < 2: + return { + "agree": True, + "comparedEngines": sorted(usable), + "note": "fewer than two engines produced results, so there is nothing to compare", + "differences": {}, + } + reference_name = sorted(usable)[0] + reference = usable[reference_name] + differences: dict[str, Any] = {} + for name in sorted(usable): + if name == reference_name: + continue + differing = [ + query_id for query_id in reference + if usable[name].get(query_id) != reference[query_id] + ] + if differing: + differences[name] = differing + return { + "agree": not differences, + "comparedEngines": sorted(usable), + "referenceEngine": reference_name, + "differences": differences, + } + + def build_manifest( args: argparse.Namespace, query_dir: Path, @@ -2003,21 +2367,31 @@ def run_validation(args: argparse.Namespace) -> int: completed=0, detail=f"materializing {args.rdf_format} artifact inside container", ) + materialize_started = time.monotonic() decoded, materialization = materialize_ntriples( args.rdf, args.rdf_format, scratch_path, log_dir=raw_dir / "materialization", ) + materialization["wallSeconds"] = time.monotonic() - materialize_started materialization["ntriplesPath"] = str(decoded) progress.emit( "progress", completed=0, detail="parsing source VCF", ) - parser = attach_census_expectations( - parse_vcf(args.vcf, filter_oracle=args.filter_oracle), args.representation - ) + oracle_started = time.monotonic() + parser = parse_vcf(args.vcf, filter_oracle=args.filter_oracle) + parse_seconds = time.monotonic() - oracle_started + census_started = time.monotonic() + parser = attach_census_expectations(parser, args.representation) + oracle_seconds = { + "parse": parse_seconds, + "census": time.monotonic() - census_started, + "total": time.monotonic() - oracle_started, + } + parser["oracleSeconds"] = oracle_seconds write_json(results_dir / "parser.json", parser) progress.emit( "progress", @@ -2040,13 +2414,11 @@ def run_validation(args: argparse.Namespace) -> int: "extra_index_args": list(args.qlever_index_arg), "extra_server_args": list(args.qlever_server_arg), } - engine = build_engine( - args.engine, decoded, raw_dir=raw_dir, scratch=scratch_path, - options=engine_options, - ) + engine_options["artifact_path"] = str(args.rdf) + engine_options["artifact_format"] = args.rdf_format manifest = build_manifest( args, query_dir, parser, - engine_description={"engine": args.engine, "options": engine_options}, + engine_description={"engines": list(args.engines), "options": engine_options}, materialization=materialization, ) write_json(results_dir / "manifest.json", manifest) @@ -2061,83 +2433,138 @@ def run_validation(args: argparse.Namespace) -> int: "status": "BLOCKED_BY_PREFLIGHT", "shacl": shacl_result, } return 1 - executions: dict[str, dict[str, Any]] = {} - progress.emit("progress", completed=0, detail=f"preparing {args.engine} engine") - if not quiet: - print(f"[{args.dataset_id}] preparing {args.engine} engine", flush=True) - try: - engine.start() - except (RuntimeError, OSError) as error: - summary = { - "datasetId": args.dataset_id, "representation": args.representation, - "status": "EXECUTION_FAILED", - "error": f"{args.engine} engine could not be prepared: {error}", - } - return 1 - try: - engine_description = engine.describe() - manifest["engine"] = engine_description - write_json(results_dir / "manifest.json", manifest) - for completed, query_id in enumerate(query_ids, start=1): - progress.emit( - "progress", - completed=completed - 1, - query_id=query_id, - detail=f"running {args.representation}/{query_id}", - ) - if not quiet: - print( - f"[{args.dataset_id}] running {args.representation}/{query_id}" - f" ({args.engine})", - flush=True, + + # Each requested engine answers the whole query set independently. + # They are compared against the same oracle and against each other, + # and every engine's timings are recorded, which is what makes a + # multi-engine run usable as a benchmark. + per_engine: dict[str, dict[str, Any]] = {} + engine_descriptions: dict[str, Any] = {} + for engine_name in args.engines: + engine_raw_dir = raw_dir / engine_name + engine_raw_dir.mkdir(parents=True, exist_ok=True) + engine = build_engine( + engine_name, decoded, raw_dir=engine_raw_dir, scratch=scratch_path, + options=engine_options, + ) + progress.emit("progress", completed=0, detail=f"preparing {engine_name} engine") + if not quiet: + print(f"[{args.dataset_id}] preparing {engine_name} engine", flush=True) + try: + engine.prepare() + except (RuntimeError, OSError) as error: + per_engine[engine_name] = { + "status": "EXECUTION_FAILED", + "error": f"{engine_name} engine could not be prepared: {error}", + } + engine_descriptions[engine_name] = {"engine": engine_name, "error": str(error)} + continue + + executions: dict[str, dict[str, Any]] = {} + try: + engine_descriptions[engine_name] = engine.describe() + for completed, query_id in enumerate(query_ids, start=1): + progress.emit( + "progress", completed=completed - 1, query_id=query_id, + detail=f"{engine_name}: {args.representation}/{query_id}", ) - executions[query_id] = engine.execute( - query_id, query_path(query_dir, query_id) - ) - progress.emit( - "progress", - completed=completed, - query_id=query_id, - detail=f"completed {args.representation}/{query_id}", - ) - finally: - engine.stop() - write_json(results_dir / "query-executions.json", executions) - if any(item["status"] != "PASS" for item in executions.values()): - summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "queryExecutions": executions} - return 1 - verdict = evaluate_validation( - executions, parser, args.representation, - strict_conformance=args.strict_conformance, - mapping_policy=args.mapping_policy, - parsed_triple_count=rdf_validation.get("tripleCount"), + if not quiet: + print( + f"[{args.dataset_id}] running {args.representation}/{query_id}" + f" ({engine_name})", + flush=True, + ) + executions[query_id] = engine.execute( + query_id, query_path(query_dir, query_id) + ) + progress.emit( + "progress", completed=completed, query_id=query_id, + detail=f"{engine_name}: completed {query_id}", + ) + finally: + engine.stop() + + engine_dir = results_dir / "engines" / engine_name + engine_dir.mkdir(parents=True, exist_ok=True) + write_json(engine_dir / "query-executions.json", executions) + if any(item["status"] != "PASS" for item in executions.values()): + per_engine[engine_name] = { + "status": "EXECUTION_FAILED", "queryExecutions": executions, + } + continue + + verdict = evaluate_validation( + executions, parser, args.representation, + strict_conformance=args.strict_conformance, + mapping_policy=args.mapping_policy, + parsed_triple_count=rdf_validation.get("tripleCount"), + ) + write_json(engine_dir / "preflight.json", verdict["preflight"]) + if verdict["status"] != "EXECUTION_FAILED": + write_json(engine_dir / "sparql.json", verdict["sparql"]) + write_json(engine_dir / "comparison.json", verdict["comparison"]) + verdict["executions"] = executions + per_engine[engine_name] = verdict + + benchmark = build_benchmark( + per_engine, engine_descriptions, + oracle_seconds=oracle_seconds, + materialization_seconds=materialization.get("wallSeconds"), + shacl_seconds=(shacl_result or {}).get("wallSeconds"), + query_ids=query_ids, ) - report = verdict["preflight"] + write_json(results_dir / "benchmark.json", benchmark) + write_benchmark_csv(results_dir / "benchmark.csv", benchmark) + + agreement = compare_engines(per_engine) + write_json(results_dir / "engine-agreement.json", agreement) + + # The primary engine's artifacts keep their historical locations so + # existing single-engine consumers are unaffected. + primary = args.engines[0] + primary_verdict = per_engine[primary] + report = primary_verdict.get("preflight", {}) write_json(results_dir / "preflight.json", report) - if verdict["status"] == "EXECUTION_FAILED": + if primary_verdict["status"] == "EXECUTION_FAILED": summary = { "datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", - "normalizationFailures": verdict["normalizationFailures"], - "preflight": report, + "engine": primary, "engines": list(args.engines), + "normalizationFailures": primary_verdict.get("normalizationFailures", {}), + "queryExecutions": primary_verdict.get("queryExecutions"), + "error": primary_verdict.get("error"), + "preflight": report, "benchmark": benchmark["totals"], } return 1 - sparql = verdict["sparql"] + sparql = primary_verdict["sparql"] for query_id, rows in sparql.items(): write_json(normalized_dir / f"{query_id}.json", rows) write_json(results_dir / "sparql.json", sparql) - comparison = verdict["comparison"] + comparison = primary_verdict["comparison"] write_json(results_dir / "comparison.json", comparison) - status = verdict["status"] + + statuses = {name: value["status"] for name, value in per_engine.items()} + failed_engines = [name for name, value in statuses.items() if value != "PASS"] + status = statuses[primary] + if not failed_engines and not agreement["agree"]: + # Every engine validated, but they disagree with each other. The + # graph may be fine and an engine wrong; either way the run + # cannot be called a pass. + status = "ENGINE_DISAGREEMENT" + elif failed_engines: + status = statuses[primary] if statuses[primary] != "PASS" else "MISMATCH" summary = { "datasetId": args.dataset_id, "representation": args.representation, "status": status, - "engine": args.engine, "sourceFormat": args.rdf_format, + "engine": primary, "engines": list(args.engines), + "engineStatuses": statuses, "engineAgreement": agreement["agree"], + "sourceFormat": args.rdf_format, "strictConformance": bool(args.strict_conformance), "shacl": shacl_result, "mappingPolicy": args.mapping_policy, "recordCount": parser["totalRecords"], "sampleCount": parser["sampleCount"], "gtRecordCount": parser["gtRecordCount"], "preflight": report, "comparisonStatus": comparison["status"], - "results": {"manifest": str(results_dir / "manifest.json"), "parser": str(results_dir / "parser.json"), "sparql": str(results_dir / "sparql.json"), "comparison": str(results_dir / "comparison.json")}, + "benchmark": benchmark["totals"], + "results": {"manifest": str(results_dir / "manifest.json"), "parser": str(results_dir / "parser.json"), "sparql": str(results_dir / "sparql.json"), "comparison": str(results_dir / "comparison.json"), "benchmark": str(results_dir / "benchmark.csv")}, } return 0 if status == "PASS" else 1 except Exception as error: @@ -2146,7 +2573,8 @@ def run_validation(args: argparse.Namespace) -> int: finally: if summary is None: summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": "validation ended without a result"} - summary.setdefault("engine", args.engine) + summary.setdefault("engine", args.engines[0]) + summary.setdefault("engines", list(args.engines)) summary.setdefault("sourceFormat", args.rdf_format) summary["temporaryRdf"] = { "decompressedInsideContainer": args.rdf_format not in DIRECT_FORMATS, @@ -2195,11 +2623,16 @@ def build_arg_parser() -> argparse.ArgumentParser: parser.add_argument("--scratch-dir", type=Path, default=Path("/work")) parser.add_argument( "--engine", - choices=SPARQL_ENGINES, default="comunica", help=( - "SPARQL engine: comunica queries the file in memory; qlever builds " - "an on-disk index and serves it (default: comunica)" + "SPARQL engine(s), comma-separated, or 'all'. comunica queries the " + "N-Triples in memory; qlever builds an on-disk index and serves it; " + "hdt and cottas query those compressed artifacts natively, without " + "decoding them. Several may be given: each answers the whole query " + "set, results are compared across engines, and every engine's " + "timings are recorded for benchmarking. The first is the primary, " + "whose reports keep the single-engine layout " + f"(choices: {', '.join(SPARQL_ENGINES)}; default: comunica)" ), ) parser.add_argument( @@ -2292,9 +2725,41 @@ def build_arg_parser() -> argparse.ArgumentParser: return parser +def parse_engine_list(raw: str) -> list[str]: + """Parse a comma-separated engine list, preserving order and de-duplicating. + + Order matters: the first engine is the primary, whose reports keep the + single-engine layout that existing consumers read. + """ + value = (raw or "").strip() + if value == "all": + return list(SPARQL_ENGINES) + engines: list[str] = [] + for token in value.split(","): + engine = token.strip() + if not engine: + continue + if engine not in SPARQL_ENGINES: + raise ValueError( + f"unknown SPARQL engine '{engine}'; choose from " + f"{', '.join(SPARQL_ENGINES)}, or 'all'" + ) + if engine not in engines: + engines.append(engine) + if not engines: + raise ValueError("--engine requires at least one engine") + return engines + + def resolve_args(parser: argparse.ArgumentParser, argv: list[str] | None = None) -> argparse.Namespace: """Parse and normalise arguments, collapsing the RDF aliases into --rdf.""" args = parser.parse_args(argv) + try: + args.engines = parse_engine_list(args.engine) + except ValueError as error: + parser.error(str(error)) + # Retained as the primary engine's name for reports and callers that read it. + args.engine = args.engines[0] args.vcf = args.vcf.resolve() supplied = args.rdf or args.rdf_gz or args.rdf_nt diff --git a/test/README.md b/test/README.md index 0fb3fd2..c66c5d7 100644 --- a/test/README.md +++ b/test/README.md @@ -36,8 +36,8 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - `test/test_validation_mutation_unit.py` (+ `validation_fixtures.py`, `validation_mutations.py`) - Mutation testing for the semantic validation suite: corrupts a correct - graph in ~25 named ways and asserts which corruptions the validator - detects, producing a reproducible mutation score. + graph in 42 named ways and asserts which corruptions the validator + detects, producing a reproducible mutation score (currently 76/78). - Requires `rdflib` (test-only, in the `dev` extra); the tests skip cleanly without it. See `docs/validation-methodology.md`. - The fixture derives the VCF, the RDF graph and the parser oracle from one @@ -46,7 +46,11 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - `test/cross_engine_agreement.py` - Not a unittest module: run inside the image to assert every validation - query returns identical values under Comunica and QLever. + query returns identical values under all four engines (Comunica, QLever, + native HDT, native COTTAS), across both representations. + - Also runs the shipped validation decision under each engine, so an engine + must agree with the Python oracle and not merely with the other engines. + - Pass a comma-separated subset as the first argument to narrow it. - `test/test_validation_logic_unit.py` - Mutation tests over the validator's pure comparison layer, run on the host @@ -61,6 +65,16 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati index/serve/teardown lifecycle and overridable command lines, and the wrapper's validation-target resolution. +- `test/test_validation_benchmark_unit.py` + - Verifies multi-engine selection (`--validation-engine a,b` and `all`) in + both the host wrapper and the container runner, and that the two layers + cannot drift apart on which engines exist. + - Verifies the native HDT/COTTAS engines: reusing the run's own artifact + versus building one, Comunica's `hdt@` typed-source prefix, and that + a failing query is reported rather than raised. + - Verifies the benchmark report and its long-format CSV, and the + cross-engine agreement comparison. + - `test/test_gzip_size_unit.py` - Verifies uncompressed-size measurement for BGZF, single-member gzip, and concatenated members, each against a full-inflate ground truth. diff --git a/test/cross_engine_agreement.py b/test/cross_engine_agreement.py index f5d2343..4762497 100644 --- a/test/cross_engine_agreement.py +++ b/test/cross_engine_agreement.py @@ -1,11 +1,17 @@ #!/usr/bin/env python3 """Assert every validation query returns the same values on every engine. -Run inside the VCF-RDFizer image, where Comunica and QLever both exist: +Run inside the VCF-RDFizer image, where every engine exists: docker run --rm -v "$PWD:/repo:ro" \\ /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py +All four engines are compared by default. Pass a comma-separated subset as the +first argument (or in VCF_RDFIZER_AGREEMENT_ENGINES) to narrow it, which is +useful when one engine is slow: Comunica answers each query in about a second +against N-Triples or HDT, while QLever and COTTAS answer in milliseconds once +their artifact exists. + This exists because an engine disagreement is silent and severe: QLever canonicalises numeric literals at index time, which once made a POS datatype preflight fail every QLever run while passing on Comunica. Only the values the @@ -20,6 +26,7 @@ import importlib.util import json +import os import sys import tempfile from pathlib import Path @@ -38,15 +45,32 @@ def load_runner(): V = load_runner() -ENGINES = ("comunica", "qlever") +DEFAULT_ENGINES = V.SPARQL_ENGINES + + +def selected_engines(argv: list[str]) -> tuple[str, ...]: + """Engines to compare: the argument, then the environment, then all of them.""" + raw = "" + if len(argv) > 1: + raw = argv[1] + else: + raw = os.environ.get("VCF_RDFIZER_AGREEMENT_ENGINES", "") + if not raw.strip(): + return tuple(DEFAULT_ENGINES) + return tuple(V.parse_engine_list(raw)) def evaluate(engine_name: str, representation: str, source: Path, scratch: Path) -> dict: results: dict[str, object] = {} raw_dir = scratch / f"raw-{engine_name}-{representation}" raw_dir.mkdir(parents=True, exist_ok=True) + # A private scratch per run: the native engines name their built artifact + # after the engine, so a shared directory would have HDT reading the graph + # it built for the other representation. + run_scratch = scratch / f"scratch-{engine_name}-{representation}" + run_scratch.mkdir(parents=True, exist_ok=True) engine = V.build_engine( - engine_name, source, raw_dir=raw_dir, scratch=scratch, + engine_name, source, raw_dir=raw_dir, scratch=run_scratch, options={"memory_gb": 2, "startup_timeout": 300, "query_timeout": 300}, ) with engine: @@ -75,8 +99,10 @@ def verdict_for(engine_name: str, representation: str, source: Path, scratch: Pa raw_dir = scratch / f"verdict-{engine_name}-{representation}" raw_dir.mkdir(parents=True, exist_ok=True) + run_scratch = scratch / f"verdict-scratch-{engine_name}-{representation}" + run_scratch.mkdir(parents=True, exist_ok=True) engine = V.build_engine( - engine_name, source, raw_dir=raw_dir, scratch=scratch, + engine_name, source, raw_dir=raw_dir, scratch=run_scratch, options={"memory_gb": 2, "startup_timeout": 300, "query_timeout": 300}, ) with engine: @@ -96,9 +122,11 @@ def verdict_for(engine_name: str, representation: str, source: Path, scratch: Pa ) -def main() -> int: +def main(argv: list[str] | None = None) -> int: from test import validation_fixtures as fixtures + engines = selected_engines(argv if argv is not None else sys.argv) + print("Comparing: " + ", ".join(engines)) disagreements = 0 with tempfile.TemporaryDirectory(dir="/work") as td: scratch = Path(td) @@ -106,24 +134,24 @@ def main() -> int: source = scratch / f"{representation}.nt" source.write_text(fixtures.build_graph(representation), encoding="utf-8") per_engine = { - engine: evaluate(engine, representation, source, scratch) for engine in ENGINES + engine: evaluate(engine, representation, source, scratch) for engine in engines } print(f"\n=== {representation} ===") - for query_id in per_engine[ENGINES[0]]: - values = [per_engine[engine][query_id] for engine in ENGINES] + for query_id in per_engine[engines[0]]: + values = [per_engine[engine][query_id] for engine in engines] if all(value == values[0] for value in values): print(f" {query_id:44s} agree") continue disagreements += 1 print(f" {query_id:44s} *** DIFFER ***") - for engine, value in zip(ENGINES, values): + for engine, value in zip(engines, values): print(f" {engine:9s} {json.dumps(value)[:300]}") # Engines agreeing with each other is not enough: the digests and # censuses are compared against values Python computes, so each # engine must also agree with that oracle. Running the real # decision proves it end to end. - for engine_name in ENGINES: + for engine_name in engines: verdict = verdict_for(engine_name, representation, source, scratch) status = verdict["status"] print(f" {'full validation verdict: ' + engine_name:44s} {status}") @@ -134,7 +162,7 @@ def main() -> int: if disagreements: print(f"\n{disagreements} disagreement(s) found.") return 1 - print("\nAll validation queries agree across " + ", ".join(ENGINES) + ",") + print("\nAll validation queries agree across " + ", ".join(engines) + ",") print("and every engine agrees with the Python oracle.") return 0 diff --git a/test/test_validation_benchmark_unit.py b/test/test_validation_benchmark_unit.py new file mode 100644 index 0000000..47fe899 --- /dev/null +++ b/test/test_validation_benchmark_unit.py @@ -0,0 +1,331 @@ +"""Multi-engine selection, native artifact querying, and the benchmark report. + +One validation run may answer the whole query set under several engines. These +tests cover the three things that makes possible: parsing the engine list, the +two native-artifact engines that query HDT and COTTAS without decoding them, +and the benchmark that turns the resulting timings into a comparison. +""" + +import csv +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import vcf_rdfizer +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_benchmark", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +class EngineListTests(VerboseTestCase): + def test_a_single_engine_still_parses_to_a_one_element_list(self): + """The single-engine case is the multi-engine case with one entry.""" + self.assertEqual(V.parse_engine_list("comunica"), ["comunica"]) + self.assertEqual(vcf_rdfizer.parse_validation_engines("comunica"), ["comunica"]) + + def test_several_engines_keep_the_order_they_were_requested_in(self): + """The first engine is the primary, so order is not incidental.""" + self.assertEqual( + V.parse_engine_list("qlever,comunica"), ["qlever", "comunica"] + ) + self.assertEqual( + vcf_rdfizer.parse_validation_engines("qlever,comunica"), + ["qlever", "comunica"], + ) + + def test_repeats_and_surrounding_whitespace_are_absorbed(self): + """'comunica, qlever, comunica' asks for two engines, not three.""" + self.assertEqual( + V.parse_engine_list(" comunica , qlever , comunica "), + ["comunica", "qlever"], + ) + self.assertEqual( + vcf_rdfizer.parse_validation_engines(" comunica , qlever , comunica "), + ["comunica", "qlever"], + ) + + def test_all_expands_to_every_supported_engine(self): + """The benchmarking shorthand must cover the full set, in both layers.""" + self.assertEqual(V.parse_engine_list("all"), list(V.SPARQL_ENGINES)) + self.assertEqual( + vcf_rdfizer.parse_validation_engines("all"), + list(vcf_rdfizer.VALIDATION_ENGINE_CHOICES), + ) + + def test_the_host_and_container_agree_on_the_engine_set(self): + """A wrapper that offers an engine the runner rejects is unusable.""" + self.assertEqual( + tuple(vcf_rdfizer.VALIDATION_ENGINE_CHOICES), tuple(V.SPARQL_ENGINES) + ) + self.assertEqual(set(V.ENGINE_CLASSES), set(V.SPARQL_ENGINES)) + + def test_an_unknown_or_empty_engine_is_rejected_by_both_layers(self): + """Failing at parse time beats failing after materialization.""" + for raw in ("virtuoso", "comunica,virtuoso", "", " ", ","): + with self.assertRaises(ValueError, msg=raw): + V.parse_engine_list(raw) + with self.assertRaises(ValueError, msg=raw): + vcf_rdfizer.parse_validation_engines(raw) + + def test_the_cli_resolves_engines_and_keeps_engine_as_the_primary(self): + """Existing consumers read args.engine; it stays the first requested.""" + parser = V.build_arg_parser() + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + (tmp_path / "in.vcf").write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + (tmp_path / "in.nt").write_text("", encoding="utf-8") + args = V.resolve_args(parser, [ + "--vcf", str(tmp_path / "in.vcf"), + "--rdf", str(tmp_path / "in.nt"), + "--results-dir", str(tmp_path / "out"), + "--representation", "expanded", + "--dataset-id", "fixture", + "--scratch-dir", str(tmp_path), + "--engine", "qlever,cottas", + ]) + self.assertEqual(args.engines, ["qlever", "cottas"]) + self.assertEqual(args.engine, "qlever") + + +class NativeArtifactEngineTests(VerboseTestCase): + """HDT and COTTAS are queried in place, not decoded to N-Triples first.""" + + def _engine(self, name, tmp_path, options=None): + return V.build_engine( + name, + tmp_path / "cohort.nt", + raw_dir=tmp_path / "raw", + scratch=tmp_path / "scratch", + options=options or {}, + ) + + def test_the_native_engines_declare_no_decode_in_their_description(self): + """The mode string is what a reader of the report sees; it must be true.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + for name, artifact_format in (("hdt", "hdt"), ("cottas", "cottas")): + engine = self._engine(name, tmp_path) + self.assertIsInstance(engine, V.NativeArtifactEngine) + self.assertEqual(engine.artifact_format, artifact_format) + self.assertIn("no decode", engine.describe()["mode"]) + + def test_a_matching_run_artifact_is_queried_directly(self): + """Building a second copy would measure the build, not the artifact.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + supplied = tmp_path / "cohort.hdt" + supplied.write_bytes(b"fake-hdt") + engine = self._engine("hdt", tmp_path, options={ + "artifact_path": supplied, "artifact_format": "hdt", + }) + with mock.patch.object(V.HdtEngine, "build_artifact") as build: + resolved = engine._resolve_artifact() + build.assert_not_called() + self.assertEqual(resolved, supplied) + self.assertEqual(engine.artifact_origin, "run artifact") + + def test_a_mismatched_run_artifact_makes_the_engine_build_its_own(self): + """Validating a .nt aggregate under --engine hdt still needs an HDT.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + engine = self._engine("hdt", tmp_path, options={ + "artifact_path": tmp_path / "cohort.cottas", "artifact_format": "cottas", + }) + with mock.patch.object(V.HdtEngine, "build_artifact") as build: + resolved = engine._resolve_artifact() + build.assert_called_once() + self.assertEqual(resolved.suffix, ".hdt") + self.assertEqual(resolved.parent, tmp_path / "scratch") + self.assertEqual(engine.artifact_origin, "built from N-Triples for this engine") + + def test_hdt_queries_are_addressed_with_comunicas_typed_source_prefix(self): + """A bare path is treated as a link to dereference and fails. + + Verified against comunica-sparql-hdt 5.0.1: '/tmp/g.hdt' reports + "Could not dereference", while 'hdt@/tmp/g.hdt' answers the query. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + raw_dir = tmp_path / "raw" + raw_dir.mkdir() + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }", encoding="utf-8") + engine = self._engine("hdt", tmp_path) + engine.executable = "/usr/bin/comunica-sparql-hdt" + engine.artifact = tmp_path / "cohort.hdt" + recorded = {} + + def fake_run(command, **kwargs): + recorded["command"] = command + return mock.Mock(returncode=0) + + with mock.patch.object(V.subprocess, "run", side_effect=fake_run): + envelope = engine.execute("q01", query) + + self.assertEqual(recorded["command"][1], f"hdt@{tmp_path / 'cohort.hdt'}") + self.assertIn("application/sparql-results+json", recorded["command"]) + self.assertEqual(envelope["status"], "PASS") + self.assertEqual(envelope["engine"], "hdt") + + def test_hdt_without_the_native_binary_explains_the_decode_alternative(self): + """An unusable engine must say what the user can do instead.""" + with tempfile.TemporaryDirectory() as td: + engine = self._engine("hdt", Path(td)) + with mock.patch.object(V.shutil, "which", return_value=None): + with self.assertRaises(RuntimeError) as raised: + engine.start() + self.assertIn("--engine comunica", str(raised.exception)) + + def test_a_failing_cottas_query_is_reported_not_raised(self): + """One bad query must not abort the other engines in a benchmark run.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + raw_dir = tmp_path / "raw" + raw_dir.mkdir() + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }", encoding="utf-8") + engine = self._engine("cottas", tmp_path) + engine.graph = mock.Mock() + engine.graph.query.side_effect = RuntimeError("parquet is unreadable") + envelope = engine.execute("q01", query) + + self.assertEqual(envelope["status"], "EXECUTION_FAILED") + self.assertIn("parquet is unreadable", envelope["error"]) + self.assertEqual(json.loads(Path(envelope["rawResult"]).read_text()), {}) + self.assertIn( + "parquet is unreadable", + Path(envelope["stderr"]).read_text(encoding="utf-8"), + ) + + def test_an_unknown_engine_name_names_the_supported_ones(self): + with tempfile.TemporaryDirectory() as td: + with self.assertRaises(ValueError) as raised: + self._engine("virtuoso", Path(td)) + for name in V.SPARQL_ENGINES: + self.assertIn(name, str(raised.exception)) + + +def _verdict(status, wall_seconds, sparql=None): + return { + "status": status, + "executions": { + query_id: {"status": "PASS", "wallSeconds": seconds} + for query_id, seconds in wall_seconds.items() + }, + "sparql": sparql if sparql is not None else {"q01": [{"n": 1}]}, + } + + +class BenchmarkReportTests(VerboseTestCase): + def _benchmark(self): + return V.build_benchmark( + { + "comunica": _verdict("PASS", {"q01": 1.0, "q02": 3.0}), + "qlever": _verdict("PASS", {"q01": 0.01, "q02": 0.02}), + }, + { + "comunica": {"setupSeconds": 0.001, "artifactOrigin": None, + "artifactSizeBytes": None}, + "qlever": {"setupSeconds": 2.5, "artifactOrigin": None, + "artifactSizeBytes": None}, + }, + oracle_seconds={"total": 0.4, "parse": 0.3, "census": 0.1}, + materialization_seconds=0.7, + shacl_seconds=None, + query_ids=("q01", "q02"), + ) + + def test_query_time_is_summed_per_engine_and_the_slowest_named(self): + """The point of the report is comparing engines, so both must be right.""" + benchmark = self._benchmark() + self.assertAlmostEqual(benchmark["engines"]["comunica"]["querySeconds"], 4.0) + self.assertAlmostEqual(benchmark["engines"]["qlever"]["querySeconds"], 0.03) + self.assertEqual(benchmark["engines"]["comunica"]["slowestQuery"], "q02") + + def test_the_oracle_is_timed_so_sparql_can_be_compared_against_it(self): + """The comparison the user asked for is engine versus parser.""" + benchmark = self._benchmark() + self.assertEqual(benchmark["oracle"]["totalSeconds"], 0.4) + self.assertEqual(benchmark["oracle"]["vcfParseSeconds"], 0.3) + self.assertEqual(benchmark["oracle"]["censusSeconds"], 0.1) + self.assertEqual(benchmark["totals"]["oracleSeconds"], 0.4) + self.assertEqual(benchmark["preparation"]["materializationSeconds"], 0.7) + + def test_an_engine_with_no_timed_query_reports_none_not_zero(self): + """Zero seconds would read as 'instant' rather than 'never ran'.""" + benchmark = V.build_benchmark( + {"hdt": {"status": "EXECUTION_FAILED", "executions": {}, "sparql": {}}}, + {"hdt": {"setupSeconds": 0.1}}, + oracle_seconds=None, + materialization_seconds=None, + shacl_seconds=None, + query_ids=("q01",), + ) + self.assertIsNone(benchmark["engines"]["hdt"]["querySeconds"]) + self.assertIsNone(benchmark["engines"]["hdt"]["slowestQuery"]) + self.assertIsNone(benchmark["oracle"]["totalSeconds"]) + + def test_the_csv_carries_one_row_per_engine_and_query(self): + """A long-format CSV is directly usable in a plot without reshaping.""" + benchmark = self._benchmark() + with tempfile.TemporaryDirectory() as td: + path = V.write_benchmark_csv(Path(td) / "benchmark.csv", benchmark) + rows = list(csv.DictReader(path.open(encoding="utf-8"))) + self.assertEqual(len(rows), 4) + self.assertEqual(list(rows[0]), V.BENCHMARK_CSV_HEADER) + self.assertEqual( + {(row["engine"], row["query_id"]) for row in rows}, + {("comunica", "q01"), ("comunica", "q02"), + ("qlever", "q01"), ("qlever", "q02")}, + ) + # Repeated on every row so the file needs no join to be usable. + self.assertEqual({row["oracle_wall_seconds"] for row in rows}, {"0.4"}) + + +class EngineAgreementTests(VerboseTestCase): + def test_engines_returning_the_same_rows_agree(self): + result = V.compare_engines({ + "comunica": _verdict("PASS", {"q01": 1.0}, sparql={"q01": [{"n": 2}]}), + "qlever": _verdict("PASS", {"q01": 0.1}, sparql={"q01": [{"n": 2}]}), + }) + self.assertTrue(result["agree"]) + self.assertEqual(result["differences"], {}) + self.assertEqual(result["comparedEngines"], ["comunica", "qlever"]) + + def test_a_disagreement_names_the_engine_and_the_queries(self): + """This is how QLever's literal canonicalisation would have been caught.""" + result = V.compare_engines({ + "comunica": _verdict("PASS", {"q01": 1.0}, sparql={"q01": [{"n": 2}], "q02": []}), + "qlever": _verdict("PASS", {"q01": 0.1}, sparql={"q01": [{"n": 3}], "q02": []}), + }) + self.assertFalse(result["agree"]) + self.assertEqual(result["differences"], {"qlever": ["q01"]}) + self.assertEqual(result["referenceEngine"], "comunica") + + def test_an_engine_that_failed_to_execute_is_left_out_of_the_comparison(self): + """Comparing against an empty result set would be a false disagreement.""" + result = V.compare_engines({ + "comunica": _verdict("PASS", {"q01": 1.0}, sparql={"q01": [{"n": 2}]}), + "hdt": {"status": "EXECUTION_FAILED", "executions": {}, "sparql": {}}, + }) + self.assertEqual(result["comparedEngines"], ["comunica"]) + self.assertTrue(result["agree"]) + self.assertIn("fewer than two", result["note"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validation_logic_unit.py b/test/test_validation_logic_unit.py index 6f172aa..a945cec 100644 --- a/test/test_validation_logic_unit.py +++ b/test/test_validation_logic_unit.py @@ -11,6 +11,7 @@ """ import copy +import gzip import importlib.util import json import tempfile @@ -391,6 +392,55 @@ def blocked(name, *args, **kwargs): self.assertNotIn("conforms", result) +class HeaderSourceTests(VerboseTestCase): + """The oracle reads the VCF's own header text, not htslib's version of it.""" + + def test_the_header_block_is_read_verbatim_and_stops_at_the_first_record(self): + """Every '##' line in order, then '#CHROM', and nothing after it.""" + with tempfile.TemporaryDirectory() as td: + vcf_path = fixtures.write_vcf(Path(td) / "fixture.vcf") + lines = V.read_vcf_header_text(vcf_path).splitlines() + meta = [f"##{key}={value}" for key, value in fixtures.HEADER_LINES] + self.assertEqual(lines[:len(meta)], meta) + self.assertEqual(len(lines), len(meta) + 1) + self.assertTrue(lines[-1].startswith("#CHROM\t")) + + def test_htslibs_injected_declarations_are_not_treated_as_header_lines(self): + """cyvcf2's raw_header is normalised; the conversion's input is not. + + htslib injects '##FILTER=' + into the header it exposes even when the file never declared it. Taking + the oracle's header from there makes it expect a FilterDefinition the + graph could not contain, and every header check fails. Reading the file + keeps the oracle and the conversion looking at the same bytes. + """ + injected = '##FILTER=' + with tempfile.TemporaryDirectory() as td: + vcf_path = fixtures.write_vcf(Path(td) / "fixture.vcf") + file_lines = { + line for line in V.read_vcf_header_text(vcf_path).splitlines() + if line.startswith("##") + } + source_lines = { + line for line in vcf_path.read_text(encoding="utf-8").splitlines() + if line.startswith("##") + } + self.assertEqual(file_lines, source_lines) + self.assertNotIn(injected, file_lines) + + def test_a_gzipped_vcf_header_reads_the_same_as_a_plain_one(self): + """Aggregates arrive compressed, so the oracle must handle both.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = fixtures.write_vcf(tmp_path / "fixture.vcf") + gz_path = tmp_path / "fixture.vcf.gz" + with gzip.open(gz_path, "wt", encoding="utf-8") as handle: + handle.write(vcf_path.read_text(encoding="utf-8")) + self.assertEqual( + V.read_vcf_header_text(gz_path), V.read_vcf_header_text(vcf_path) + ) + + class GraphIntegrityTests(VerboseTestCase): """Blank nodes, empty terms, and duplicated statements.""" diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 78e1ecc..5dea3b7 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -470,6 +470,10 @@ def test_validation_runner_emits_query_progress_in_quiet_mode(self): rdf=rdf_path, rdf_format="nt", engine="comunica", + engines=["comunica"], + mapping_policy="strict", + strict_conformance=False, + shacl_shapes=None, query_timeout=60, qlever_memory_gb=4, qlever_port=7019, diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 8571b7b..3a15403 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -166,6 +166,13 @@ "validation_max_rss_kb", "validation_results_path", "validation_rdf_path", + # Benchmark columns. The oracle and the engine compute the same answers + # from the same data, so these two are directly comparable; the engine + # column is a "|"-joined list when several engines ran. + "validation_engines", + "validation_oracle_seconds", + "validation_engine_query_seconds", + "validation_engine_setup_seconds", ] COMPRESSION_METHOD_COLUMNS = { @@ -331,8 +338,37 @@ (".cottas", "cottas"), (".hdt", "hdt"), ) -VALIDATION_ENGINE_CHOICES = ("comunica", "qlever") +VALIDATION_ENGINE_CHOICES = ("comunica", "qlever", "hdt", "cottas") DEFAULT_VALIDATION_ENGINE = "comunica" + + +def parse_validation_engines(raw: str) -> list[str]: + """Parse --validation-engine into an ordered, de-duplicated engine list. + + Several engines may be requested in one run. Each answers the whole query + set, so their results are cross-checked and their timings are directly + comparable. The first is the primary, whose reports keep the single-engine + layout that existing consumers read. + """ + value = (raw or "").strip() + if value == "all": + return list(VALIDATION_ENGINE_CHOICES) + engines: list[str] = [] + for token in value.split(","): + engine = token.strip() + if not engine: + continue + if engine not in VALIDATION_ENGINE_CHOICES: + allowed = ",".join(VALIDATION_ENGINE_CHOICES) + raise ValueError( + f"Unsupported value '{engine}' for --validation-engine. " + f"Use {allowed}, or all." + ) + if engine not in engines: + engines.append(engine) + if not engines: + raise ValueError("--validation-engine requires at least one engine") + return engines # Which produced artifacts a full run should semantically validate. "aggregate" # is the .nt/.nt.gz RMLStreamer output; the others are the selected # representations, each decoded back to N-Triples before it is checked. @@ -3512,6 +3548,10 @@ def update_metrics_csv_with_compression( if validation_result is not None: defaults.update( { + "validation_engines": "", + "validation_oracle_seconds": "null", + "validation_engine_query_seconds": "", + "validation_engine_setup_seconds": "", "validation_status": "NOT_RUN", "validation_exit_code": "null", "validation_wall_seconds": "null", @@ -3660,6 +3700,26 @@ def assign_validation(method: str, result: dict): or validation_result.get("rdf_gzip_path") or "" ) + benchmark = validation_result.get("benchmark") or {} + if "validation_engines" in row: + row["validation_engines"] = "|".join(validation_result.get("engines") or []) + if "validation_oracle_seconds" in row: + oracle = benchmark.get("oracleSeconds") + row["validation_oracle_seconds"] = ( + "null" if oracle is None else f"{float(oracle):.6f}" + ) + for column, key in ( + ("validation_engine_query_seconds", "engineQuerySeconds"), + ("validation_engine_setup_seconds", "engineSetupSeconds"), + ): + if column in row: + # "engine=seconds" pairs, so one column holds every engine's + # figure without the header depending on which engines ran. + values = benchmark.get(key) or {} + row[column] = "|".join( + f"{name}={float(value):.6f}" + for name, value in values.items() if value is not None + ) with metrics_csv.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=target_header) @@ -5130,7 +5190,7 @@ def run_full_mode( header_representation: str = DEFAULT_HEADER_REPRESENTATION, run_validation: bool = False, validation_artifacts: list[str] | None = None, - validation_engine: str = DEFAULT_VALIDATION_ENGINE, + validation_engine: str | list[str] = DEFAULT_VALIDATION_ENGINE, validation_engine_options: dict | None = None, validation_strict_conformance: bool = False, validation_shacl_shapes: Path | None = None, @@ -5162,8 +5222,12 @@ def run_full_mode( print(f" Header representation: {header_representation}") validation_artifacts = list(validation_artifacts or ["aggregate"]) if run_validation: + engine_label = ( + validation_engine if isinstance(validation_engine, str) + else ", ".join(validation_engine) + ) print( - f" Validation: {', '.join(validation_artifacts)} via {validation_engine}" + f" Validation: {', '.join(validation_artifacts)} via {engine_label}" ) intermediate_dir = tsv_dir.parent ensure_dir(tsv_dir) @@ -5685,7 +5749,7 @@ def fail_current(stage: str, message: str): metrics_dir / "reports" / "validation" / safe_target_id ) print( - f" * Semantic validation ({target['name']}, {validation_engine}): " + f" * Semantic validation ({target['name']}, {engine_label}): " f"{target['path'].name}" ) target_result = { @@ -5697,7 +5761,7 @@ def fail_current(stage: str, message: str): "vcf_path": str(input_vcf), "rdf_path": str(target["path"]), "rdf_format": target["format"], - "engine": validation_engine, + "engine": engine_label, "representation": sample_workflow.representation, "results_dir": str(target_results_dir), "summary_path": str(target_results_dir / "summary.json"), @@ -6591,7 +6655,7 @@ def run_validation_mode( image_ref: str, filter_oracle: str, wrapper_log_path: Path, - engine: str = DEFAULT_VALIDATION_ENGINE, + engine: str | list[str] = DEFAULT_VALIDATION_ENGINE, engine_options: dict | None = None, rdf_format: str | None = None, strict_conformance: bool = False, @@ -6631,7 +6695,8 @@ def run_validation_mode( "Expected one of .nt, .nt.gz, .nt.br, .hdt, .cottas, .cottas.gz, .cottas.br" ) options = dict(engine_options or {}) - engine_args: list[str] = ["--engine", engine] + engine_list = [engine] if isinstance(engine, str) else list(engine) + engine_args: list[str] = ["--engine", ",".join(engine_list)] for flag, key in ( ("--query-timeout", "query_timeout"), ("--qlever-memory-gb", "qlever_memory_gb"), @@ -6731,11 +6796,19 @@ def run_validation_mode( "vcf_path": str(vcf_path), "rdf_path": str(rdf_path), "rdf_format": resolved_format, - "engine": engine, + "engine": engine_list[0], + "engines": engine_list, "strict_conformance": bool(strict_conformance), "representation": representation, "results_dir": str(results_dir), "summary_path": str(summary_path), + # Lift the oracle-versus-SPARQL timings into the stage report so a + # benchmark can be read from the run metrics without opening the + # detailed validation results. + "engine_statuses": summary.get("engineStatuses") if isinstance(summary, dict) else None, + "engine_agreement": summary.get("engineAgreement") if isinstance(summary, dict) else None, + "benchmark": summary.get("benchmark") if isinstance(summary, dict) else None, + "benchmark_csv_path": str(results_dir / "benchmark.csv"), "input_rdf_size_bytes": int(file_size_bytes(rdf_path) or 0), "exit_code": int(exit_code), "status": status, @@ -7093,12 +7166,16 @@ def main(): ) parser.add_argument( "--validation-engine", - choices=VALIDATION_ENGINE_CHOICES, default=DEFAULT_VALIDATION_ENGINE, help=( - "SPARQL engine used for validation: comunica queries the graph in " - "memory; qlever builds an on-disk index inside the container and " - f"serves it (default: {DEFAULT_VALIDATION_ENGINE})" + "SPARQL engine(s) for validation, comma-separated, or 'all'. " + "comunica queries the graph in memory; qlever builds an on-disk " + "index and serves it; hdt and cottas query those compressed " + "artifacts natively without decoding them. Requesting several runs " + "the whole query set on each, cross-checks their answers, and " + "records comparable timings for benchmarking " + f"(choices: {', '.join(VALIDATION_ENGINE_CHOICES)}; " + f"default: {DEFAULT_VALIDATION_ENGINE})" ), ) parser.add_argument( @@ -7189,6 +7266,7 @@ def main(): step1_label = "Step 1/5" if mode == "full" else "Step 1/3" validation_artifacts: list[str] = [] + validation_engines: list[str] = [DEFAULT_VALIDATION_ENGINE] validation_engine_options: dict = {} shacl_shapes_path: Path | None = None try: @@ -7209,6 +7287,7 @@ def main(): validation_engine_options["qlever_server_args"] = list(args.qlever_server_arg) if validation_engine_options.get("qlever_port", 1) > 65535: raise ValueError("--qlever-port must be between 1 and 65535") + validation_engines = parse_validation_engines(args.validation_engine) validation_artifacts = parse_validation_targets(args.validate_artifacts) if args.shacl_shapes is not None: shacl_shapes_path = Path(args.shacl_shapes).expanduser().resolve() @@ -7529,7 +7608,7 @@ def main(): "spark_partitions": spark_partitions if mode == "full" else None, "run_validation": bool(args.run_validation) if mode == "full" else False, "validation_artifacts": validation_artifacts if mode == "full" and args.run_validation else None, - "validation_engine": args.validation_engine if mode in {"full", "validation"} else None, + "validation_engine": validation_engines if mode in {"full", "validation"} else None, "validation_strict_conformance": bool(args.strict_conformance) if mode in {"full", "validation"} else None, "validation_shacl_shapes": str(shacl_shapes_path) if shacl_shapes_path else None, "validation_engine_options": validation_engine_options or None, @@ -7724,7 +7803,7 @@ def execute_mode(): header_representation=args.header_representation, run_validation=args.run_validation, validation_artifacts=validation_artifacts, - validation_engine=args.validation_engine, + validation_engine=validation_engines, validation_engine_options=validation_engine_options, validation_strict_conformance=args.strict_conformance, validation_shacl_shapes=shacl_shapes_path, @@ -7787,7 +7866,7 @@ def execute_mode(): timestamp=timestamp, image_ref=image_ref, filter_oracle=args.filter_oracle, - engine=args.validation_engine, + engine=validation_engines, engine_options=validation_engine_options, strict_conformance=args.strict_conformance, shacl_shapes=shacl_shapes_path,