diff --git a/.github/workflows/validation-mutation.yml b/.github/workflows/validation-mutation.yml new file mode 100644 index 0000000..e8fe117 --- /dev/null +++ b/.github/workflows/validation-mutation.yml @@ -0,0 +1,70 @@ +name: validation-mutation + +# The mutation score is the measured coverage of the semantic validation suite. +# The host job runs on every push because it needs no Docker; the container job +# is the authority, replaying the same catalogue under the engines that ship in +# the image. + +on: + push: + branches: + - "**" + pull_request: + workflow_dispatch: + +jobs: + mutation-score: + name: mutation score (rdflib) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install test dependencies + run: python -m pip install --upgrade pip rdflib + + - name: Run the mutation harness + env: + VCF_RDFIZER_MUTATION_REPORT: mutation-score.json + run: python -m unittest test.test_validation_mutation_unit -v + + - name: Report the score + run: | + python - <<'PY' + import json + report = json.load(open("mutation-score.json")) + print(f"mutation score: {report['detected']}/{report['total']} " + f"({report['score']:.0%})") + for item in report["mutations"]: + if not item["detected"]: + print(f" undetected: {item['id']} ({item['representation']}) " + f"- {item['knownUndetected']}") + PY + + - name: Upload the score + uses: actions/upload-artifact@v4 + with: + name: mutation-score + path: mutation-score.json + + engine-agreement: + name: cross-engine query agreement + runs-on: ubuntu-latest + # The image build is slow, so this gates merges rather than every push. + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build the image + run: docker build -t vcf-rdfizer:mutation-ci . + + - name: Every validation query must agree across Comunica and QLever + run: | + docker run --rm -v "$PWD:/repo:ro" vcf-rdfizer:mutation-ci \ + /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md new file mode 100644 index 0000000..e741ae6 --- /dev/null +++ b/ACKNOWLEDGEMENTS.md @@ -0,0 +1,29 @@ +# Acknowledgements + +VCF-RDFizer is developed at [KNoWS](https://knows.idlab.ugent.be/), IDLab, +Ghent University - imec, as part of the doctoral research of +Elias Crum (). + +Please cite the software as described in [`README.md`](README.md#citation) and +[`CITATION.cff`](CITATION.cff). + +## Acknowledgement sentence for papers + +When VCF-RDFizer is used in a publication, the following template can be +adapted for the Acknowledgements section: + +> This work was carried out at KNoWS, IDLab, Ghent University - imec, as part +> of the doctoral research of Elias Crum. The authors thank the maintainers of +> the open-source components on which VCF-RDFizer builds - RMLStreamer, +> hdt-cpp, hdtc, pycottas, Comunica, QLever, pySHACL, bcftools, and cyvcf2. + +If the work is co-funded by a specific project or grant, add that project name +and grant number to the sentence above and record it in the table at the top of +this file. + +## Third-party components + +VCF-RDFizer bundles and orchestrates third-party software inside its Docker +image. Their licenses and notices are collected in +[`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md) and, in the built image, +under `/usr/share/licenses/vcf-rdfizer/`. diff --git a/Dockerfile b/Dockerfile index 8b3eb98..59d042d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,15 @@ ARG RMLSTREAMER_VERSION=2.5.0 -ARG HDT_JAVA_PACKAGE_VERSION=3.0.10 +ARG HDTC_VERSION=1.1.0 +ARG COMUNICA_VERSION=5.3.0 +# Comunica's HDT engine is versioned separately from its file engine and lags +# behind it. It provides native SPARQL over a .hdt artifact, so validation can +# query the compressed representation directly instead of decoding it first. +ARG COMUNICA_HDT_VERSION=5.0.1 +# QLever is an optional second SPARQL engine for validation. Its binaries are +# copied from the upstream published image rather than built here: compiling +# QLever needs a large C++ toolchain and would dominate this image's build. +# Pin a digest or release tag here to make validation runs reproducible. +ARG QLEVER_IMAGE=adfreiburg/qlever:latest FROM eclipse-temurin:11-jre AS build-hdt-cpp @@ -35,14 +45,41 @@ RUN mkdir -p /opt/third_party_licenses \ && cp /opt/RMLStreamer/LICENSE /opt/third_party_licenses/RMLStreamer.LICENSE +FROM rust:1.93-slim AS build-hdtc + +ARG HDTC_VERSION + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + libbz2-dev \ + liblzma-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt +RUN git clone --branch "v${HDTC_VERSION}" --depth 1 \ + https://github.com/frink-okn/hdtc.git + +WORKDIR /opt/hdtc +RUN cargo build --locked --release \ + && mkdir -p /opt/third_party_licenses \ + && cp LICENSE /opt/third_party_licenses/HDTC.LICENSE + + +# Named stage so the runtime image can COPY QLever's binaries out of it. +FROM ${QLEVER_IMAGE} AS qlever + + FROM eclipse-temurin:11-jre ARG RMLSTREAMER_VERSION -ARG HDT_JAVA_PACKAGE_VERSION RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ + bcftools \ brotli \ ca-certificates \ coreutils \ @@ -50,48 +87,120 @@ RUN apt-get update \ findutils \ gawk \ gzip \ + libbz2-1.0 \ + liblzma5 \ libserd-0-0 \ nodejs \ + npm \ python3 \ python3-venv \ + raptor2-utils \ time \ && rm -rf /var/lib/apt/lists/* +# Keep the COTTAS conversion runtime and Parquet streaming-writer dialect +# stable. pycottas performs the per-chunk conversion; PyArrow performs the +# bounded-memory k-way merge of those already sorted chunks. RUN python3 -m venv /opt/pycottas-venv \ - && /opt/pycottas-venv/bin/pip install --no-cache-dir pycottas + && /opt/pycottas-venv/bin/pip install --no-cache-dir \ + pycottas==1.1.0 \ + duckdb==1.5.5 \ + pyarrow==22.0.0 \ + numpy==2.4.6 \ + cyvcf2==0.34.0 \ + pyshacl==0.30.1 + +ARG COMUNICA_HDT_VERSION + +# The HDT engine compiles native bindings, so a toolchain is needed at install +# time but not afterwards; it is purged in the same layer to keep it out of the +# image. `python3` is already present and is what node-gyp needs. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && npm install --global \ + "@comunica/query-sparql-file@${COMUNICA_VERSION}" \ + "@comunica/query-sparql-hdt@${COMUNICA_HDT_VERSION}" \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ -o /opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar \ https://github.com/RMLio/RMLStreamer/releases/download/v${RMLSTREAMER_VERSION}/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar -RUN mkdir -p /opt/hdt-java \ - && curl -fsSL \ - -o /tmp/hdt-java-package.tar.gz \ - https://repo1.maven.org/maven2/org/rdfhdt/hdt-java-package/${HDT_JAVA_PACKAGE_VERSION}/hdt-java-package-${HDT_JAVA_PACKAGE_VERSION}-distribution.tar.gz \ - && tar -xzf /tmp/hdt-java-package.tar.gz -C /opt/hdt-java --strip-components=1 \ - && rm -f /tmp/hdt-java-package.tar.gz - COPY --from=build-hdt-cpp /usr/local/bin/rdf2hdt /usr/local/bin/rdf2hdt COPY --from=build-hdt-cpp /usr/local/bin/hdt2rdf /usr/local/bin/hdt2rdf COPY --from=build-hdt-cpp /usr/local/lib/libcds* /usr/local/lib/ COPY --from=build-hdt-cpp /usr/local/lib/libhdt* /usr/local/lib/ COPY --from=build-hdt-cpp /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ +COPY --from=build-hdtc /opt/hdtc/target/release/hdtc /usr/local/bin/hdtc +COPY --from=build-hdtc /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ + +# Optional QLever SPARQL engine (--validation-engine qlever). Comunica remains +# the default, so an image whose QLever binaries turn out to be unusable is +# still fully functional; the validator reports the reason instead of failing +# obscurely. Pin a different tag or digest with +# --build-arg QLEVER_IMAGE=adfreiburg/qlever:. +# +# QLever's image is built on a different Ubuntu release than this one, so the +# binaries alone are not enough: their Boost, ICU, jemalloc and io_uring +# sonames are release-specific and absent here. Those libraries travel with the +# binaries into a private directory that only QLever's own processes are +# pointed at, so they cannot shadow anything the rest of the image links +# against. (glibc itself is not copied - it is backward compatible, and this +# base is newer than QLever's.) +COPY --from=qlever /qlever/qlever-index /opt/qlever/bin/qlever-index +COPY --from=qlever /qlever/qlever-server /opt/qlever/bin/qlever-server +COPY --from=qlever \ + /lib/x86_64-linux-gnu/libboost_iostreams.so.1.83.0 \ + /lib/x86_64-linux-gnu/libboost_program_options.so.1.83.0 \ + /lib/x86_64-linux-gnu/libboost_url.so.1.83.0 \ + /lib/x86_64-linux-gnu/libgomp.so.1 \ + /lib/x86_64-linux-gnu/libicudata.so.74 \ + /lib/x86_64-linux-gnu/libicui18n.so.74 \ + /lib/x86_64-linux-gnu/libicuuc.so.74 \ + /lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /lib/x86_64-linux-gnu/liburing.so.2 \ + /opt/qlever/lib/ COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES.md COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ +COPY src/validation/ /opt/vcf-rdfizer/validation/ +# Shared, dependency-free helper used by both the host CLI and the in-container +# conversion runner, so it lives at the repository root rather than in src/. +COPY vcf_rdfizer_gzip.py /opt/vcf-rdfizer/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ - && find /opt/hdt-java/bin -type f -exec chmod +x {} \; \ && chmod +x /usr/local/bin/rdf2hdt \ - && chmod +x /usr/local/bin/hdt2rdf + && chmod +x /usr/local/bin/hdt2rdf \ + && chmod +x /usr/local/bin/hdtc \ + && chmod +x /opt/qlever/bin/qlever-index /opt/qlever/bin/qlever-server + +# QLever's binaries come from a different base image, so record at build time +# whether they actually link here. The validator reads this marker to explain +# an unavailable engine instead of surfacing a bare "not found". +RUN set -eu; \ + status="ok"; \ + for binary in qlever-index qlever-server; do \ + missing="$(LD_LIBRARY_PATH=/opt/qlever/lib ldd "/opt/qlever/bin/$binary" 2>&1 | grep 'not found' || true)"; \ + if [ -n "$missing" ]; then \ + status="QLever binary $binary has unresolved shared libraries in this base image: $missing"; \ + echo "WARNING: $status" >&2; \ + fi; \ + done; \ + printf '%s\n' "$status" > /opt/vcf-rdfizer/qlever-status.txt ENV RMLSTREAMER_JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar -ENV HDT_JAVA_HOME=/opt/hdt-java +ENV HDTC_BIN=/usr/local/bin/hdtc +ENV HDT_INDEX_MEMORY_LIMIT=512M +ENV HDT_MERGE_MEMORY_LIMIT=512M +ENV COTTAS_MERGE_BATCH_ROWS=2048 ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python +ENV QLEVER_INDEX_BUILDER_BIN=/opt/qlever/bin/qlever-index +ENV QLEVER_SERVER_BIN=/opt/qlever/bin/qlever-server ENV LD_LIBRARY_PATH=/usr/local/lib # COTTAS creates a temporary DuckDB database in the container working diff --git a/README.md b/README.md index 810376f..744ee26 100644 --- a/README.md +++ b/README.md @@ -15,14 +15,34 @@ VCF-RDFizer is a Docker-first CLI wrapper for: 1. VCF -> RDF (N-Triples) with RMLStreamer 2. Optional RDF compression/decompression +3. Semantic validation of a compressed RDF graph against its source VCF The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#](https://w3id.org/vcf-rdfizer/vocab#). +This README is the task-oriented reference. For how the tool works, why it is +built that way, and where it stops working, see the documentation set in +**[`docs/`](docs/README.md)** - starting with +[Architecture](docs/architecture.md) and, before you rely on the output, +[Limitations](docs/limitations.md). + ## Requirements - Python 3.10+ - Docker (installed and running) +When VCF-RDFizer is connected to an interactive terminal, it shows a +lightweight Rich spinner/progress display. With redirected output or without +Rich installed, it instead prints compact status lines; CI output remains +quiet. Validation uses the same display and reports each preflight/SPARQL +query as it starts and completes. Use `--quiet` to suppress these terminal +progress displays (and validation's per-query/summary chatter) while keeping +the progress sidecar, command log, and metrics collection active. Use +`--no-progress` when the sidecar and terminal progress should both be disabled. +RMLStreamer progress reports the bytes and output parts already written; +partitioned HDT/COTTAS runs report source triples, chunks, and the currently +active merge/index stage. These updates are best-effort and do not scan RDF +content a second time or retain progress history in memory. + Install options: ```bash @@ -47,7 +67,7 @@ or pull the prebuilt Docker image directly: docker pull ecrum19/vcf-rdfizer:latest ``` -Release maintainers: see [`RELEASING.md`](RELEASING.md) for the PyPI, +Release maintainers: see [`scripts/RELEASING.md`](scripts/RELEASING.md) for the PyPI, Docker Hub, and conda-forge release procedure. ## Important CLI Rule @@ -63,31 +83,129 @@ inside this directory. ## Modes -- `full`: VCF -> TSV -> RDF -> compression +- `full`: VCF -> TSV -> RDF -> compression (and optional semantic validation with `--validate`) - `tsv`: VCF -> TSV only (benchmarking) - `compress`: compress an existing `.nt` or `.nt.gz` - `decompress`: decompress `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` -- `index`: eagerly initialize HDT Java's versioned `.hdt.index.*` sidecar for an existing `.hdt` +- `validation`: compare a source VCF with its `.nt` or `.nt.gz` RDF using six semantic SPARQL queries +- `index`: only generate or regenerate the query index for an existing `.hdt` or `.cottas` In `full` mode with multiple VCF inputs, failures are isolated per input: - the run continues with remaining files -- failed inputs are summarized in `run_metrics//failed_inputs.csv` +- failed inputs are summarized in `run_metrics/__/reports/failed_inputs.csv` ## Main Flags (Most Used) -- `-m, --mode {full,compress,decompress,tsv,index}` +- `-m, --mode {full,compress,decompress,tsv,validation,index}` - `-o, --out` required output root directory - `--rdf-compression` final raw RDF codecs: `gzip`, `brotli`, or `none` - `--representations` queryable RDF outputs: `hdt`, `cottas`, or `none` - `--artifact-compression` packaging codecs for selected representations: `gzip`, `brotli`, or `none` - `--hdt-strategy {auto,partitioned,single}` HDT generation policy - `--chunk-target-bytes`, `--chunk-min-bytes`, `--chunk-max-bytes` shared record-safe chunk sizing +- `--sample-representation {expanded,condensed}` genotype graph shape (`expanded` by default) +- `--validate` (or `--run-validation`) run semantic VCF/RDF validation for every input in a full run +- `--filter-oracle {auto,bcftools,cyvcf2}` FILTER oracle used by validation (`auto` by default) +- `--quiet` suppress terminal progress displays while retaining sidecar/log/metrics tracking +- `--no-progress` disable terminal progress and progress sidecar creation - `-I, --image` Docker image repo (default `ecrum19/vcf-rdfizer`) - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build - `-B, --no-build` fail if image not found - `-h, --help` show full usage +## Validation Mode + +Validate one source VCF against the `.nt` or `.nt.gz` aggregate from the same +conversion. Gzip input is decompressed, parsed, and queried only inside the +Docker container; any temporary raw N-Triples are removed before the container +exits. To run +the same checks as part of a full conversion, add `--validate`; validation then +runs once per input after RDF/compression and accepts either the generated +`.nt` or `.nt.gz` aggregate. + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz \ + --rdf ./results/cohort/cohort.nt.gz \ + --sample-representation condensed \ + --out ./validation-results +``` + +Use `expanded` for the default graph shape and `condensed` for the vector-based +cohort graph. Reports from standalone and full-run validation use the canonical +metrics tree: +`run_metrics/__/reports/validation//`, with a +stage summary at `stages/validation/.json`. See [Semantic VCF/RDF +validation](docs/validation.md) for query definitions, preflight checks, result +statuses, and cleanup evidence. + + +### Artifacts and engines + +`--rdf` accepts any artifact the pipeline produces - `.nt`, `.nt.gz`, `.nt.br`, +`.hdt`, `.cottas`, `.cottas.gz`, `.cottas.br`. A compressed or indexed artifact +is decoded back to N-Triples **inside the container** and then put through the +full semantic suite, which proves it decodes to a graph that still reproduces +every VCF summary - stronger than the triple-count round-trip that runs during +compression. + +In full mode, `--validate-artifacts {aggregate,hdt,cottas,all}` chooses which +produced artifacts to check; each is validated independently with its own +report: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz \ + --rdf-storage-mode space-optimized --representations hdt,cottas \ + --validate --validate-artifacts all -o ./results +``` + +`--validation-engine` selects the SPARQL backend. Comunica (default) queries the +file in memory; [QLever](https://github.com/ad-freiburg/qlever) builds an +on-disk index inside the container and serves it, which is what makes +cohort-scale graphs queryable; `hdt` and `cottas` query the **compressed +artifact in place**, without decoding it, through `comunica-sparql-hdt` and +`pycottas`'s rdflib store. All four answer identical queries, so the choice is +never semantic, and every report records which engine ran. + +```bash +vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.hdt \ + --validation-engine qlever --qlever-memory-gb 32 -o ./validation-results +``` + +Several engines can run in one pass - `--validation-engine comunica,qlever` or +`all`. Each answers the whole query set, so the run cross-checks them against +each other (`engine-agreement.json`) and times them against each other and +against the cyvcf2 oracle computing the same answers, in `benchmark.json` and a +long-format `benchmark.csv`. + +```bash +vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --validation-engine all -o ./validation-results +``` + +Tuning: `--qlever-memory-gb`, `--qlever-port`, `--qlever-startup-timeout`, +`--validation-query-timeout`, and repeatable `--qlever-index-arg` / +`--qlever-server-arg` escape hatches. + +Validation runs three independent layers: exact aggregate comparison against +the VCF, a predicate/class census plus per-record and per-value identity +digests, and — with `--shacl-shapes` — SHACL conformance against the +vocabulary's published shapes. + +```bash +vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --shacl-shapes ./vocabulary/shacl/vcf-rdfizer-vocabulary.shacl.ttl \ + --strict-conformance -o ./validation-results +``` + +> **What a PASS means.** Coverage is measured, not asserted: a mutation harness +> corrupts a correct graph in 42 named ways and records which are detected +> (currently **76/78**). See [`docs/vcf-coverage.md`](docs/vcf-coverage.md) for +> the element-by-element matrix and the remaining gaps, and +> [`docs/validation-methodology.md`](docs/validation-methodology.md) for how the +> number is produced. + ## Compression Plan Compression is configured as three independent decisions: @@ -129,6 +247,14 @@ host filesystem. - `-i, --input` required VCF file or directory - `-r, --rules` mapping rules file (`.ttl`) - default: `rules/default_rules.ttl` +- `--sample-representation {expanded,condensed}` sample genotype representation + - `expanded` (default): one `SampleCall` per record/sample and one `FormatFieldValue` per FORMAT key + - `condensed`: reusable file-level samples plus one ordered value vector per record/FORMAT key +- `--validate` run semantic VCF/RDF validation once per input after RDF/compression; + detailed results are stored beneath `run_metrics/.../reports/validation/` +- `--filter-oracle {auto,bcftools,cyvcf2}` FILTER oracle for `--validate` +- `--quiet` suppress terminal progress and validation query chatter while retaining logs/metrics +- `--no-progress` disable progress sidecars and terminal progress displays - `--rdf-storage-mode {plain,space-optimized}` required full-mode aggregate storage policy - `plain`: merge RMLStreamer parts into one uncompressed `.nt` - `space-optimized`: gzip each part into one `.nt.gz` aggregate and delete the source part immediately @@ -136,7 +262,7 @@ host filesystem. - `--representations {hdt,cottas,none}` queryable primary representations - `--artifact-compression {gzip,brotli,none}` optional packaging applied to each selected representation - `--hdt-strategy {auto,partitioned,single}` - - `auto`: in full mode, build smaller HDT chunks and merge them with `HDTCat` + - `auto`: in full mode, build smaller HDT chunks and merge them with native `hdtc` - `partitioned`: always use chunked HDT generation for HDT-based methods - `single`: always use one `rdf2hdt` run per RDF input - with `space-optimized`, use `auto` or `partitioned`; `single` cannot consume the gzip stream without expanding it @@ -150,11 +276,98 @@ host filesystem. - `--remove-rdf-storage-output` explicitly remove the aggregate `.nt`/`.nt.gz` after successful compression - `-e, --estimate-size` preflight size estimate +## VCF Coverage + +Full-mode conversion covers every VCF column. Three options control how much +structure is emitted; all default to the richer form. + +| Option | Effect | +| --- | --- | +| `--sample-representation {expanded,condensed}` | Genotype shape (see below) | +| `--info-representation {structured,raw}` | `structured` adds one `vcfr:InfoFieldValue` per record and key, with typed values, alongside `vcfr:infoRaw` | +| `--header-representation {structured,basic}` | `structured` types each `##` line with its vocabulary subclass and lifts FILTER/ALT/contig attributes into their own properties | + +QUAL is always emitted, typed `xsd:decimal` or `vcfr:Null`, because the +published SHACL shape requires a datatype that depends on the value. + +[`docs/vcf-coverage.md`](docs/vcf-coverage.md) maps every VCF element to its RDF +terms and to the validation check that covers it. + +## Sample Representation Modes + +Full mode has exactly two explicit sample workflows. There is no automatic +sample-count threshold, so the same command always produces the same graph +shape and downstream consumers can select the contract they support. + +See [Expanded and Condensed Knowledge Representations](docs/sample-representation-guide.md) +for a detailed, worked explanation of the graph shapes, scaling behavior, and +selection trade-offs. + +### Expanded (default) + +Use `--sample-representation expanded` for single-sample and low-sample VCFs. It +preserves the original vocabulary model: + +- every record/sample pair is a `vcfr:SampleCall`; +- every represented FORMAT slot is a `vcfr:FormatFieldValue`; +- the VCF file declares `vcfr:representationProfile vcfr:ExpandedRepresentation`. + +With the default rules, these triples are appended directly from `records.tsv`; +the large materialized helper TSVs are not created. The final graph is still +expanded and grows approximately with `variants × samples × FORMAT fields`. + +```bash +vcf-rdfizer --mode full \ + --input ./small.vcf \ + --sample-representation expanded \ + --rdf-storage-mode plain \ + --out ./results +``` + +### Condensed + +Use `--sample-representation condensed` for large multi-sample cohorts. It +uses the vocabulary introduced in VCF-RDFizer Vocabulary 1.1.0: + +- sample columns are declared once as an ordered `vcfr:SampleSet` of reusable + `vcfr:VCFSample` resources; +- each genotype-bearing call has one `vcfr:CohortCallMatrix`; +- each FORMAT key has one `vcfr:FormatValueVector`, rather than one RDF value + resource per sample; +- the VCF file declares + `vcfr:representationProfile vcfr:CondensedRepresentation`. + +`vcfr:encodedValues` uses `vcfr:VCFTextVector`: one tab-separated lexical item +per sample in `vcfr:sampleIndex` order. Commas inside a FORMAT value remain part +of that item, and absent values are emitted as `.` so all vectors stay aligned. +Consumers reconstruct sample `i`'s value for a FORMAT key by selecting position +`i` from its vector. This changes genotype graph growth to approximately +`samples + variants × FORMAT fields`; the literal payload still contains all +source values, but they no longer cause per-value RDF structural triples. + +```bash +vcf-rdfizer --mode full \ + --input ./large-cohort.vcf.gz \ + --sample-representation condensed \ + --rdf-storage-mode space-optimized \ + --representations hdt \ + --out ./results +``` + +The workflow resolver runs only one sample emitter. Condensed mode rejects +custom mappings that consume materialized `sample_calls.tsv` or +`sample_format_values.tsv`, because running those helper-table mappings alongside +the condensed emitter would create both representations and restore the semantic +inflation this mode is designed to avoid. Remove those helper-table consumers +or select expanded mode. Custom rules with no helper-table consumers remain +compatible with condensed emission. + ## TSV Mode Flags - `-i, --input` required VCF file or directory -- Outputs per-run benchmark summary in `run_metrics//tsv_metrics.csv` -- Raw TSV timing + artifact JSON per input in `run_metrics//raw_metrics/tsv_*` +- Outputs per-run benchmark summary in `run_metrics/__/tsv_metrics.csv` +- Writes container timing and structured TSV metrics under `timings/tsv/` and + `stages/tsv/` in that run directory ## Compression Mode Flags @@ -165,10 +378,19 @@ host filesystem. - `-C, --compressed-input` required `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` - `-d, --decompress-out` optional explicit output `.nt` path (must be inside `--out`) -## HDT Index Mode Flags - -- `-H, --hdt` required existing `.hdt` file -- HDT Java 3.0.10 generates an HDT v1-1 index named `.hdt.index.v1-1`. +## Index-Only Mode Flags + +- `-H, --hdt` existing `.hdt` file; creates or regenerates its sibling sidecar +- `--cottas` existing `.cottas` file; rebuilds its embedded query index in place +- Exactly one of `--hdt` or `--cottas` is required. +- HDT indexing is Java-free: the image uses `hdtc` 1.1.0 to generate the + canonical v1-1 sidecar named `.hdt.index.v1-1`. +- COTTAS indexes are stored inside the Parquet-based `.cottas` file, so the + file is rewritten atomically; no separate COTTAS index file is expected. +- Existing indexes are intentionally replaced. Use this mode when an HDT + sidecar is missing/stale or when a COTTAS file needs its query ordering and + zone-map metadata rebuilt. No VCF conversion, RDF conversion, packaging, or + decompression output is produced. - The operation is also run automatically after each partitioned HDT merge. ## Quick Start @@ -191,7 +413,7 @@ vcf-rdfizer \ --out ./results ``` -Full pipeline (plain aggregate, chunked HDT + HDTCat merge): +Full pipeline (plain aggregate, chunked HDT + native `hdtc` merge): ```bash vcf-rdfizer \ @@ -336,6 +558,69 @@ vcf-rdfizer \ --out ./results ``` +Regenerate the embedded index for an existing COTTAS file: + +```bash +vcf-rdfizer \ + --mode index \ + --cottas ./results/sample/sample.cottas \ + --out ./results +``` + +`--mode index` is deliberately an in-place maintenance operation. It mounts +only the directory containing the selected artifact and writes metrics under +`/run_metrics/__/stages/index/`. HDT indexing creates a +versioned sidecar beside the input. COTTAS indexing rewrites the existing +`.cottas` file through a bounded streaming Parquet rewrite, keeping the data +in the same artifact while rebuilding its embedded index. If the operation +fails, the original COTTAS file is left in place; HDT's previous sidecars are +restored. + +HDT merging and indexing do not start a Java HDT tool. They use native `hdtc`: +`hdtc create` merges partitioned HDTs and `hdtc index` streams an HDT through +disk-backed external sorters. Both default to a 512 MiB soft memory budget. +Override index creation for any wrapper mode with an environment variable such +as: + +```bash +HDT_INDEX_MEMORY_LIMIT=2G vcf-rdfizer \ + --mode index \ + --hdt ./results/sample/sample.hdt \ + --out ./results +``` + +Use `HDT_MERGE_MEMORY_LIMIT` in the same way to tune a partitioned merge. +Accepted values use an `M` or `G` suffix. Lower values reduce in-memory sort +buffers and may increase temporary I/O; higher values can improve performance +when memory is available. Temporary files live in the container's `/work` area +and are removed after the attempt. + +COTTAS avoids both a global in-memory `DISTINCT` and a global external sort. +Each chunk is already written in `spo` order, so the final stage performs a +bounded k-way Parquet merge: it holds one small batch from each chunk, writes +one copy of each adjacent equal triple, and preserves the `spo` index. Its +memory use is controlled by `COTTAS_MERGE_BATCH_ROWS` (default `2048`), not by +the total RDF graph size or a temporary DuckDB sort area. Override it only to +tune the memory/throughput tradeoff, for example: + +```bash +COTTAS_MERGE_BATCH_ROWS=4096 vcf-rdfizer \ + --mode compress \ + --rdf ./results/cohort/cohort.nt.gz \ + --rdf-compression none \ + --representations cottas \ + --out ./results +``` + +In `full` mode, an HDT sidecar-index failure is non-fatal when the HDT data +itself remains readable; the run continues and the HDT can be repaired later +with the standalone command above. If COTTAS generation/indexing cannot +produce a usable artifact, COTTAS-specific outputs are skipped while the rest +of the full pipeline continues. These warnings are printed in the run output +and written to `run_metrics/__/reports/index_warnings.json`. The raw RDF is +retained when a representation-dependent output was unavailable so the +standalone index command or a later rerun has a recoverable source. + ## Output Layout Given `--out ./results`: @@ -343,7 +628,7 @@ Given `--out ./results`: - final outputs: - `./results//...` - per-run metrics/logs: - - `./results/run_metrics//...` + - `./results/run_metrics/__/...` - hidden intermediates: - `./results/.intermediate/tsv/` @@ -359,8 +644,10 @@ For an existing RDF input named `test-larger.nt` or `test-larger.nt.gz`, The same basename rule applies in full mode. VCF-RDFizer performs an output collision check before Docker or conversion starts and never overwrites a -planned artifact. Choose a new `--out` directory, or rename/remove the -conflicting output before rerunning. +planned pipeline artifact. The exception is the deliberate `--mode index` +maintenance operation, which regenerates the selected artifact's index in +place. Choose a new `--out` directory, or rename/remove a conflicting pipeline +artifact before rerunning. Intermediates are hidden by default. Raw RDF files are removed after successful compression by default. Use @@ -371,12 +658,60 @@ because that file is the gzip artifact itself. ## Metrics -For each run, VCF-RDFizer writes: +Each invocation receives a descriptive metrics directory: + +```text +run_metrics/__/ +``` -- `run_metrics//metrics.csv` -- `run_metrics//wrapper_execution_times.csv` -- `run_metrics//progress.log` -- `run_metrics//hdt_index_metrics.json` for standalone HDT index mode +`` is the source filename without its recognized VCF/RDF or +representation suffix (for example, `1000G_phase3_chr20`). A multi-file input +directory uses a batch label such as `batch-vcf_data-4-inputs`. This makes a +metrics directory recognizable without opening a timestamp-named folder. + +Within each run directory, VCF-RDFizer writes: + +- `run.json`: source identity, resolved input paths, requested workflow + configuration, and image selection +- `summary.json`: final status, wrapper wall time, summary table rows, and an + index of every stage report and log +- `metrics.csv`, `tsv_metrics.csv`, and `wrapper_execution_times.csv`: compact + analysis-ready tables when applicable +- `logs/wrapper.log` and `logs/progress.log` +- `timings//...`: raw GNU `time -v` output from inside the relevant + Docker container +- `stages/tsv/`, `stages/conversion/`, `stages/compression/`, + `stages/compression_operations/`, `stages/decompression/`, and + `stages/index/`, and `stages/validation/`: structured stage results. + `compression_operations/` + preserves the underlying per-RDF operation and validation reports, while + `compression/` provides the final output-level summary. +- `stages/partitioned/`: the full result handoff from the temporary + partitioned-compression container, including every chunk build, merge, + validation, workspace free-space sample, exit code, CPU time, and peak RSS +- `reports/index_warnings.json` and `reports/failed_inputs.csv` when applicable +- `reports/validation//`: detailed semantic-validation reports + (`summary.json`, query results, preflight checks, and cleanup evidence) when + validation is requested + +`input_vcf_size_bytes` in `stages/conversion/*.json` and `metrics.csv` is the +*uncompressed* size of the source VCF, so that ratios are comparable between +plain and compressed inputs. The accompanying `input_vcf_size_method` records +how it was obtained: + +| Method | Meaning | +| --- | --- | +| `stat` | Input was not compressed; the on-disk size is the answer | +| `bgzf` | Exact, summed from a `bgzip`/BGZF file's block headers with no decompression | +| `gzip-sample` | Exact; the whole single-member stream fitted in the sampling budget | +| `gzip-trailer` | Exact; the 32-bit `ISIZE` trailer resolved against the file's measured compression ratio | +| `inflate` / `inflate-shell` | Fallback full decompression pass, used when the file's structure cannot settle the answer (for example concatenated non-BGZF members) | + +Only the fallback costs a full pass over the input. Because indexed `.vcf.gz` +files from `bcftools`/`tabix`/`htslib` are BGZF, the usual case is measured in +milliseconds rather than minutes. The same machinery makes `--estimate-size` +report a real uncompressed input size instead of an assumed expansion factor; +it says so when it had to fall back to the assumption. Compression metrics now include per-method: @@ -395,15 +730,30 @@ Compression fails closed if the artifact cannot be decoded or the counts do not match. In compression-only mode, the source count is obtained by a streaming fallback when no upstream conversion metrics are available. +For full runs, a readable HDT whose sidecar index could not be created is +validated with the index check skipped, marked with `index_status: "failed"`, +and reported in `index_warnings.json`; this allows packaging and later stages +to continue. COTTAS failures are reported the same way, but dependent COTTAS +artifacts are marked as not generated because the COTTAS file itself is not +usable. Explicit standalone `--mode index` runs remain strict and return a +failure status when regeneration fails. + For partitioned HDT/COTTAS runs, the final method metric reports one -sample-level result, while raw metrics also include a sample-scoped -`__partitioned_compression__` artifact describing chunk conversion, merge -rounds, and the generated chunk guide. +sample-level result while `stages/partitioned/.json` retains the full +container-stage history. It includes chunk conversion, merge strategy and +rounds, validation, generated chunk plan, workspace free-space samples, CPU +time, peak RSS, exit codes, and bounded stderr diagnostics. This report is +preserved even when the temporary Docker volume is deleted after a failure. Metrics may use internal stage names such as `hdt_gzip` and `cottas_brotli`. These correspond to the public combination of `--representations` and `--artifact-compression`; users do not need to pass those compound names. +When `--validate` is used in full mode, the same `metrics.csv` row also carries +`validation_status`, `validation_exit_code`, validation wall/CPU/RSS timings, +the detailed report path, and the RDF path that was validated. The validation +stage JSON retains the full status and temporary-RDF cleanup metadata. + ## Chunked Compression Full mode always uses one of the two aggregate storage modes. Both modes create @@ -413,11 +763,24 @@ one, so it avoids retaining both the part files and a full uncompressed aggregate. When HDT or COTTAS is selected, the aggregate is read sequentially and split -into complete N-Triples records. The same temporary chunks are consumed by -both converters before cleanup. HDT chunks are merged with `HDTCat`, the final -HDT index is generated after merging through HDT Java's indexed search loader, -and COTTAS chunks are merged with `pycottas.cat`, which rebuilds the query -indexes for the merged representation. +into complete N-Triples records. Only one uncompressed chunk is present at a +time: it is consumed by both converters and removed before the next chunk is +read. This is especially important for `space-optimized` `.nt.gz` aggregates, +which must not be expanded into a second full raw-RDF copy. HDT chunks are +merged with the Java-free `hdtc create` command, which accepts existing HDT +inputs; the final HDT index is generated after merging with `hdtc index`. +COTTAS chunk conversion uses `pycottas.rdf2cottas(..., disk=True)`. The final +COTTAS merge deliberately does **not** call `pycottas.cat`: version 1.1.0 runs +its global `DISTINCT` plus `ORDER BY` through an unbounded in-memory DuckDB +connection, which can be killed on large condensed graphs. VCF-RDFizer instead +uses a PyArrow k-way merge of the already `spo`-sorted Parquet chunks. It keeps +only a configurable batch from each input, drops adjacent duplicate triples, +and writes the final COTTAS file incrementally—no graph-wide DuckDB hash table +or external-sort spill directory is created. The merge emits processed-source +and distinct-written triple counts in the terminal progress display. If the +stage fails in full mode, the warning contains the failing exit code and, when +available, a `stderr_tail`, maximum resident set size, and Docker-workspace +free-space samples; the raw RDF remains available for a retry. After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a streaming decode/count check. This verifies both readability and that the @@ -426,25 +789,51 @@ performed before `.hdt.gz`, `.hdt.br`, `.cottas.gz`, or `.cottas.br` packaging, and before raw RDF cleanup. Partitioned HDT/COTTAS compression runs in an ephemeral Docker-managed -workspace. Temporary RDF chunks, COTTAS/DuckDB scratch data, intermediate +workspace. Temporary RDF chunks, COTTAS conversion scratch data, intermediate representations, and merge files are not written to the output directory. After a successful or failed run, the temporary workspace is removed; only the selected final artifacts and normal run metrics remain on the host. -Each COTTAS conversion and merge also receives a fresh container-local DuckDB -workspace, which is removed as soon as that operation completes. This prevents -state from one chunk being reused by another and requires no user configuration. - -HDT Java 3.0.10 does not provide a standalone `hdtGenerateIndex` executable. -VCF-RDFizer sends an `exit` command to the supported `hdtSearch.sh` launcher; -this opens the HDT through `mapIndexedHDT()` without executing a data query and -creates the versioned `.hdt.index.v1-1` sidecar before the run is marked successful. -For the pinned HDT Java 3.0.10 distribution, this is the HDT v1-1 sidecar -`.hdt.index.v1-1`; VCF-RDFizer reports the actual path in its metrics. +Each COTTAS conversion receives a fresh container-local DuckDB workspace, +which is removed as soon as that operation completes. The final streaming +merge does not need a DuckDB workspace or a full-data temporary sort file. + +HDT merging and index generation use the pinned Rust `hdtc` 1.1.0 executable, +not `hdtCat`, `hdtSearch.sh`, or another Java HDT process. `hdtc create` +merges the chunk HDTs with disk-backed external sorts, and `hdtc index` reads +BitmapTriples as a stream to build the object/predicate orderings. This avoids +the JVM heap path that can fail with `java.lang.OutOfMemoryError` while +producing the same canonical HDT v1-1 sidecar, +`.hdt.index.v1-1`, used by hdt-java and hdt-cpp. + +For standalone index mode, existing versioned sidecars are moved aside while +regeneration runs and restored if indexing fails. Incomplete replacements are +removed before restoration, so a failed or interrupted attempt does not leave +a partial index. Sort runs use `/work` and are removed when the command exits. +The image defaults both `HDT_INDEX_MEMORY_LIMIT` and +`HDT_MERGE_MEMORY_LIMIT` to `512M`. The wrapper forwards an explicitly set +host value of either variable into the relevant Docker command. +`HDT_INDEX_WORK_ROOT` can override the scratch root when invoking +`/opt/vcf-rdfizer/ensure_hdt_index.sh` directly inside the container. + +COTTAS does not expose a separate index sidecar. Its index is part of the +Parquet artifact and is selected when the artifact is written. Standalone +COTTAS index mode writes a new temporary COTTAS file through the same +bounded streaming rewrite with the default `spo` index, then atomically +replaces the original. This is still index-only from the pipeline's point of +view: it does not rerun VCF-to-RDF conversion or create an RDF output, but it +rewrites the artifact once. The record-safe chunk plan and per-stage timings are retained in the raw partitioned-compression metrics JSON for diagnostics. The temporary chunk files and guide are not retained as host files. +For the default mapping, multi-sample VCF columns remain compact in +`records.tsv`. In expanded mode, canonical `SampleCall` and `FormatFieldValue` +triples are streamed directly into the final `.nt` or `.nt.gz` aggregate rather +than first writing materialized helper rows. In condensed mode, the same input pass +emits shared samples, call matrices, and FORMAT vectors, avoiding both the +helper-table multiplier and the per-sample RDF structural multiplier. + The implementation keeps COTTAS conversion scratch state inside the Docker container and removes temporary unpacked package files when decompression finishes. @@ -454,19 +843,169 @@ finishes. - default rules file: `rules/default_rules.ttl` - rules guide: `rules/README.md` +### Custom RML Mappings + +`--rules` accepts any RML mapping, so you can change what RDF the pipeline +produces without touching the wrapper. A custom mapping has to honour a small +contract, and `vcf-rdfizer-rules` (installed alongside `vcf-rdfizer`) makes it +discoverable and checkable: + +```bash +vcf-rdfizer-rules columns +``` + +Lists the five TSV sources the pipeline generates and every column each one +provides, so you know what a mapping can reference. + +```bash +vcf-rdfizer-rules init -o my_rules.ttl +``` + +Writes an annotated copy of the shipped default mapping to start from. + +```bash +vcf-rdfizer-rules check my_rules.ttl +``` + +Validates the mapping *before* you spend hours on a run. It reports: + +- logical-source paths the wrapper cannot rewrite per input, +- referenced columns no generated TSV provides (typos such as `CHROMOSOME`), +- which `--sample-representation` values remain usable, +- whether the mapping forces the large sample helper tables to be materialized. + +Exit code is `0` when the mapping is usable and `1` when it is not; add +`--json` for scripted use. Then run it: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz --rules my_rules.ttl --rdf-storage-mode plain -o ./results +``` + +#### The contract + +1. **Keep the five `csvw:url` values exactly as they are.** Full mode processes + one VCF at a time and rewrites those literal strings to the per-input file + names (`/data/tsv/records.tsv` becomes `/data/tsv/.records.tsv`, and + so on). Any other path is left untouched and will not resolve. + + | Logical source | Contents | + | --- | --- | + | `/data/tsv/records.tsv` | One row per VCF data line | + | `/data/tsv/header_lines.tsv` | One row per `##` header line | + | `/data/tsv/file_metadata.tsv` | One row summarising the source VCF | + | `/data/tsv/sample_calls.tsv` | Helper: one row per variant x sample | + | `/data/tsv/sample_format_values.tsv` | Helper: one row per variant x sample x FORMAT key | + +2. **Only reference columns the pipeline writes.** `vcf-rdfizer-rules columns` + is authoritative; a unit test pins those lists to what `src/vcf_as_tsv.sh` + actually emits, so they cannot drift. + +3. **Think before consuming the two helper tables.** The four built-in sample + maps are recognised by the wrapper, which then keeps those tables + header-only and streams the genotype RDF itself. A mapping that consumes + them in any other way forces them to be materialized in full - the largest + intermediate the pipeline can produce - and is rejected in + `--sample-representation condensed`, which would otherwise emit both + genotype representations at once. `check` warns about this explicitly. + +The last column of `records.tsv` is the whitespace-joined sample ids from the +`#CHROM` line (or `SAMPLES` when the VCF declares none), so its *name* varies +per input and it cannot be referenced by a fixed name. Genotype RDF is emitted +by the wrapper from that column instead; see +[`docs/sample-representation-guide.md`](docs/sample-representation-guide.md). + +## Repository Layout + +VCF-RDFizer is deliberately split into a thin host-side CLI and a set of +container-side stages. Nothing on the host walks the RDF itself; it plans work, +launches Docker, and reads back the JSON/CSV reports each stage writes. + +| Path | Role | +| --- | --- | +| `vcf_rdfizer.py` | Host CLI: argument validation, output-collision planning, Docker orchestration, metrics assembly, mode dispatch. Also emits the multi-sample genotype RDF (see below). | +| `vcf_rdfizer_rules.py` | `vcf-rdfizer-rules` CLI: scaffold, document, and validate custom RML mappings. | +| `vcf_rdfizer_gzip.py` | Uncompressed size of a gzip/BGZF VCF without decompressing it. Used by the host preflight estimate and, inside the image, by `run_conversion.sh`. | +| `src/vcf_as_tsv.sh` | VCF -> per-input `records`/`header_lines`/`file_metadata` TSV, in one `awk` pass. | +| `src/run_conversion.sh` | Runs RMLStreamer, normalizes Spark part files, merges them into one `.nt`/`.nt.gz` aggregate, records conversion metrics. | +| `src/partitioned_compression.py` | Record-safe RDF chunking plus chunked HDT/COTTAS generation and pairwise merge, inside an ephemeral Docker volume. | +| `src/cottas_tool.py` | COTTAS `convert` / `merge` / `reindex` / `decompress` adapter over `pycottas`, with a bounded-memory streaming merge. | +| `src/ensure_hdt_index.sh` | Java-free canonical `.hdt.index.v1-1` sidecar generation via `hdtc`, with restore-on-failure. | +| `src/validate_compression.py` | Round-trip check: decode a `.hdt`/`.cottas` artifact and compare its triple count against the source. | +| `src/validation/` | Semantic VCF-vs-RDF validation: `cyvcf2`/`bcftools` oracle, SPARQL queries per representation, comparison report. | +| `rules/default_rules.ttl` | Default RML mapping (also shipped as package data in `vcf_rdfizer_data/`). | +| `test/` | `unittest` suite; the shell/pipeline tests stub `java`, `docker`, and friends so no real external tool is needed. | +| `scripts/release.py` | Version bump + release metadata automation (see `scripts/RELEASING.md`). | + +Genotype RDF is the one deliberate exception to "all data processing happens in +the container": `append_expanded_sample_rdf` and `append_condensed_sample_rdf` +in `vcf_rdfizer.py` append it directly to the aggregate, because the equivalent +RML maps would first have to materialize variants x samples (x FORMAT keys) +helper TSV rows. See [`docs/sample-representation-guide.md`](docs/sample-representation-guide.md). + +Further reading: **[`docs/`](docs/README.md) is the in-depth documentation set** - +how each part of the tool works, why, and where it stops working. + +| Document | Covers | +| --- | --- | +| [Architecture](docs/architecture.md) | Host/container split, failure policy, pinned toolchain | +| [Conversion](docs/conversion.md) | VCF -> TSV -> RDF, stage by stage | +| [Representations](docs/representations.md) | Compression, HDT/COTTAS, chunking, round-trip checks | +| [Output and metrics](docs/output-and-metrics.md) | Run layout, reports, progress, exit codes | +| [Custom RML mappings](docs/rml-mappings.md) | The `--rules` contract in full | +| [Sample representations](docs/sample-representation-guide.md) | Expanded vs condensed genotype shapes | +| [Validation](docs/validation.md) | The semantic suite, and what it does not test | +| [Validation methodology](docs/validation-methodology.md) | How coverage is measured, not asserted | +| [VCF coverage matrix](docs/vcf-coverage.md) | Element by element, with the mutation that proves each row | +| [CLI reference](docs/cli-reference.md) | Every flag, with constraints and interactions | +| [Limitations](docs/limitations.md) | Everything the tool cannot do, in one place | +| [Roadmap](docs/roadmap.md) | Planned work, known defects, and rejected options | +| [Data linking design](docs/datalinking-design.md) | Proposal: a plug-in system for external links | +| [Privacy policy design](docs/privacy-policy-design.md) | Proposal: ODRL-based granular disclosure control over the graph | + +- [`changelog.md`](changelog.md) - dated change history +- [`ACKNOWLEDGEMENTS.md`](ACKNOWLEDGEMENTS.md) - funding and attribution + ## Troubleshooting If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). +If COTTAS indexing/merging fails on a very large RDF file, first inspect the +`cottas-merge-stream` stage in the raw partitioned-compression metrics JSON and the +run wrapper log. An `exit_code=-9` means the child was killed by `SIGKILL`, +which is normally the kernel/Docker OOM killer; a shell wrapper may report the +same event as `137`. The warning's `stderr_tail`, `max_rss_kb`, and workspace +samples distinguish COTTAS data/schema problems from Docker disk-space errors. +The streaming merge has no graph-wide DuckDB spill area: it reads at most +`COTTAS_MERGE_BATCH_ROWS` rows per input at a time (default `2048`) and writes +the result incrementally. If the host is especially memory-constrained, lower +that value; if the merge is CPU-bound and RAM is available, raise it gradually. +The final COTTAS artifact still needs ordinary output disk space. Rebuild the +image after upgrading so the streaming merge workflow is installed. + +If COTTAS is optional for the experiment, rerun with +`--representations hdt`; the HDT path is independent and can remain the +queryable artifact even when COTTAS cannot fit the available memory. If COTTAS +is required and the streaming merge still receives `-9`, lower +`COTTAS_MERGE_BATCH_ROWS`, verify that Docker has enough RAM for the selected +batch size, and +check the host kernel log for an external kill; this is a resource limit, not a +vocabulary or RDF-validity problem. + If HDT compression fails on very large RDF files, use `--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with `--hdt-strategy partitioned`, then lower `--chunk-target-bytes` and -`--chunk-max-bytes` to reduce each converter's working set. +`--chunk-max-bytes` to reduce each converter's working set. Both final HDT +merge and index creation are disk-backed, so ensure the Docker data volume has +enough temporary space for their external sorts. To reduce their bounded +in-memory buffers further, set `HDT_MERGE_MEMORY_LIMIT` and/or +`HDT_INDEX_MEMORY_LIMIT` (for example, `512M`); lower limits can require more +temporary I/O. Free space in the output filesystem alone does not increase the +Docker volume capacity. Safe termination: - Press `Ctrl+C` to interrupt a run. -- The wrapper exits with code `130`, writes progress to `run_metrics//progress.log`, and performs best-effort cleanup of tracked intermediates. +- The wrapper exits with code `130`, writes progress to `run_metrics/__/logs/progress.log`, and performs best-effort cleanup of tracked intermediates. - Raw RDF cleanup on interrupt follows `--keep-rmlstreamer-rdf-output`: - with `--keep-rmlstreamer-rdf-output`, raw RDF files are preserved - without it, tracked raw RDF files are removed during interrupt cleanup diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index f8bffb5..88fbce2 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -5,6 +5,13 @@ VCF-RDFizer (this repository) is released under the MIT License (see `LICENSE`). The Docker image also includes third-party software. Their licenses remain with their respective authors and apply to those components. +## Python package dependency + +1. `Rich` +- Repository: +- Usage in this project: terminal spinners and progress bars +- License: MIT + ## Included in Docker image 1. `HDT-cpp` @@ -25,8 +32,25 @@ their respective authors and apply to those components. - Repository: - Usage in this project: RDF chunk compression and indexed COTTAS merging - Upstream license: Apache License 2.0 +- Version bundled by the Docker image: `1.1.0` +- Installed in the image's `/opt/pycottas-venv` environment + +4. `PyArrow` +- Repository: +- Usage in this project: bounded-memory Parquet reads and writes for the final + COTTAS k-way merge +- Upstream license: Apache License 2.0 +- Version bundled by the Docker image: `22.0.0` - Installed in the image's `/opt/pycottas-venv` environment +5. `hdtc` +- Repository: +- Usage in this project: Java-free, disk-backed merging of HDT chunks and + index generation for existing HDT files +- Upstream license: MIT +- License files copied into image: + - `/usr/share/licenses/vcf-rdfizer/HDTC.LICENSE` + ## Notes for package users - The `pip` and `conda` packages install the Python wrapper CLI only. diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..a2621a0 --- /dev/null +++ b/changelog.md @@ -0,0 +1,868 @@ +# Changelog + +## 2026-09-04 — Multi-engine validation, native HDT/COTTAS querying, and benchmarking + +Validation was one engine against one N-Triples file. It is now up to four +engines against the artifact each of them reads best, in a single run, timed +and cross-checked. + +### Added + +- **Several SPARQL engines in one run.** `--validation-engine` accepts a + comma-separated list or `all`. Every requested engine answers the whole query + set; the first is the primary and keeps the existing single-engine report + layout, while each engine additionally gets `engines//` with its own + preflight, SPARQL, comparison and execution reports. The run's `status` is + `PASS` only if every engine passed, and `summary.json` carries + `engineStatuses`. +- **Cross-engine agreement on the real graph.** Every engine's normalized + results are compared against each other and written to + `engine-agreement.json`, naming the queries that differ. This is not + theoretical: QLever's literal canonicalisation had already been caught this + way in the test harness. +- **Native HDT and COTTAS querying** (`--validation-engine hdt` / `cottas`). + `--validation-target hdt,cottas` validates those artifacts by *decoding* them + first, which proves the decode is faithful and says nothing about querying + them — and measures the wrong thing entirely in a performance comparison. The + new engines query the artifact in place: HDT through `comunica-sparql-hdt`, + COTTAS in-process through `pycottas.COTTASStore`, an rdflib `Store` over the + Parquet artifact. When the run's own artifact is already in the engine's + format it is used directly; otherwise one is built in container scratch and + that build is timed as **setup**, kept out of the query total. Each report + records which happened, in `artifactOrigin`. +- **`@comunica/query-sparql-hdt` in the image**, installed alongside + `build-essential` (needed for its native bindings) which is purged in the same + layer. +- **Timings for everything, and a comparison against the parser.** Every run + writes `benchmark.json` and a long-format `benchmark.csv` — one row per engine + and query, ready to plot without reshaping. `benchmark.json` adds per-engine + setup and query totals, the slowest query per engine, materialization and + SHACL time, and the **oracle**: what it costs to compute the same answers + directly from the VCF with cyvcf2, split into parse and census. The suite + already computes every expected value twice, once by parsing and once by + querying, so those two costs are for identical work on identical input — the + performance comparison the reports were missing. +- **Benchmark columns in the run's `metrics.csv`**: `validation_engines`, + `validation_oracle_seconds`, `validation_engine_query_seconds` and + `validation_engine_setup_seconds`, the last two as `engine=seconds` pairs. + +### Fixed + +- **Comunica could not query an HDT file by path.** A bare path is treated as a + link to dereference and failed with "Could not dereference". The source is now + addressed as `hdt@`. Found by the first real four-engine run, not by + inspection. +- **The parser oracle was reading htslib's header, not the file's.** cyvcf2's + `raw_header` is normalised: htslib injects + `##FILTER=` into files that never + declared it. The oracle therefore expected a `FilterDefinition` resource the + graph could not contain, and every header check failed. `read_vcf_header_text` + now reads the header block from the file itself — the same text the conversion + reads — for both plain and gzipped VCFs. + +### Verified + +- A four-engine run over the fixture: all four `PASS`, all four agree, and the + benchmark records per-query timings for each. +- `test/cross_engine_agreement.py` extended from two engines to all four, + across both representations, with per-run scratch so the native engines cannot + read each other's artifacts. All queries agree and every engine agrees with + the Python oracle. +- 347 host tests pass (24 new); mutation score unchanged at **76/78**. + +### Documentation + +- [`docs/validation.md`](docs/validation.md) — rewritten engine section covering + all four engines, multi-engine runs, native artifact querying, and the + benchmark report. Its "What is *not* tested" section was stale — it predated + the census, the identity digests, and the QUAL/INFO coverage work, and + contradicted [`docs/vcf-coverage.md`](docs/vcf-coverage.md) — and now states + the gaps that actually remain. +- [`docs/validation-methodology.md`](docs/validation-methodology.md) — "Two + engines" became "Four engines", with the two new cross-engine findings and a + section on why a multi-engine run is a controlled benchmark rather than an + incidental one. +- [`README.md`](README.md), [`docs/cli-reference.md`](docs/cli-reference.md), + [`test/README.md`](test/README.md) updated; the README's stale mutation score + (36 mutations, 64/66) corrected to 42 and 76/78. + +## 2026-09-04 — Design proposal: granular privacy policies over the VCF graph + +[`docs/privacy-policy-design.md`](docs/privacy-policy-design.md) — **design +proposal, not implemented.** A system for governed, partial release of a VCF +graph: which participants, which regions, which fields, at what resolution, to +whom, for what purpose — declared in ODRL and enforced by the pipeline. + +### Added + +- **An ODRL profile whose assets are graph selectors.** ODRL's `odrl:target` + addresses an Asset IRI and has no notion of "these triples", so the profile + supplies `ClassSelector`, `PredicateSelector`, `SampleSelector`, + `RegionSelector`, `FieldSelector`, `HeaderSelector` and a SPARQL + `PatternSelector` escape hatch. It also pins the conflict semantics ODRL + leaves informative: deny-overrides, default-deny, most-restrictive-first + composition. +- **Effects beyond permit/prohibit**, because the useful answer for genomic data + is usually less resolution rather than nothing: `drop`, `pseudonymize`, + `generalize`, `threshold`, `aggregateOnly`, and `maskVectorPositions`. +- **Three enforcement tiers derived from the existing pipeline** — TSV + pre-filtering before RMLStreamer (cheapest, covers RML-produced triples), + emitter-time filtering inside `_append_rdf_atomically` (full record context, + no second pass), and a post-hoc `--mode redact` (needs two passes for region + rules, since an N-Triples stream has no ordering guarantee). The compiler + assigns each rule to the cheapest tier that can express it, and **aborts** on + any rule no tier can enforce rather than warning. +- **GA4GH DUO consent codes as `odrl:purpose` right operands**, so a policy is + reviewable by the data access committees that already speak DUO. +- **Release manifests and policy conformance verification**: prohibitions + compile to `ASK` preflights that must return zero, with mutation-catalogue + entries that break the redactor and assert the checks catch it — the same + "measured, not asserted" standard as + [`validation-methodology.md`](docs/validation-methodology.md). + +### Findings that affect work outside the proposal + +- **`ParsedSampleRecord` carries no `CHROM` or `POS`.** `_parse_row` reads + columns 0, 1, 7, 9, 10 and −1, skipping 2 and 3. Any region-scoped feature + evaluated at emission time needs both; adding them is two dataclass fields and + two index reads. +- **Condensed mode cannot express per-sample protection by triple filtering.** + One `FormatValueVector` literal holds every participant's value for a FORMAT + key, so the unit of protection is finer than the unit of storage. Redaction + must rewrite the literal in place (replacing a position with `.` to preserve + `sampleIndex` alignment), and a single masked position is itself disclosive. +- **IRIs and header lines leak independently of genotypes.** Sample names, the + source filename and a monotonic row counter are all embedded in IRIs, and + `##source` / `##SAMPLE` / `##PEDIGREE` / free-text `Description` fields are + transcribed verbatim. Recorded in + [`docs/limitations.md`](docs/limitations.md). + +### Framing kept throughout + +The document is explicit that this is **governed release, not anonymization**. +Genotypes are identifiers — a few dozen independent common variants single out +an individual — so an access-control layer governs who receives what and creates +an audit trail; it does not make released data non-identifying. Differential +privacy is discussed and explicitly **not** proposed, because without a +persistent per-recipient budget ledger it provides no protection. + +## 2026-09-04 — Documentation set, and a data-linking design proposal + +`docs/` becomes an in-depth explanation of the whole tool rather than four +validation-focused documents. Every part of the pipeline is now described, with +its limitations stated next to its capabilities rather than in a footnote. + +### Added + +- [`docs/README.md`](docs/README.md) — index, reading paths per audience, and + the conventions the documentation set follows. +- [`docs/architecture.md`](docs/architecture.md) — the host/container split, + what runs where, the **three places that split deliberately leaks** (genotype, + header and QUAL/INFO emission happen on the host because RML cannot choose a + datatype or class per row), the three-way failure policy, and the pinned + toolchain versions. +- [`docs/conversion.md`](docs/conversion.md) — VCF -> TSV -> RDF stage by stage, + including what the single-`awk`-pass parser can and cannot see, the complete + IRI template table, and the datatype and missing-value decisions. +- [`docs/representations.md`](docs/representations.md) — the compression plan's + three independent decisions, record-safe chunking, the `hdtc` and PyArrow + rationales, and what the round-trip check does *not* prove. +- [`docs/output-and-metrics.md`](docs/output-and-metrics.md) — output layout, + the `run_metrics/` tree, input size accounting, progress, interrupts, exit + codes. +- [`docs/rml-mappings.md`](docs/rml-mappings.md) — the `--rules` contract in + full, plus what a custom mapping does *not* control and what it costs in + validation. +- [`docs/cli-reference.md`](docs/cli-reference.md) — every flag, grouped, with + enforced constraints, environment variables, and exit codes. +- [`docs/limitations.md`](docs/limitations.md) — one consolidated, honest + account: operational, input handling, modelling, extension points, + compression, validation, vocabulary, and scope. +- [`docs/roadmap.md`](docs/roadmap.md) — planned work, known defects, and + options that were assessed and deliberately rejected. +- [`docs/datalinking-design.md`](docs/datalinking-design.md) — **design + proposal, not implemented.** A plug-in architecture for linking the graph to + external resources, built on the observation that rsID, gene and clinical + linking are all one of three joins (`token`, `interval`, `allele`). Three + plugin tiers (declarative template, local reference bundle, live service), + enforced network safeguards, side-graph output with provenance, and a build + order that freezes the extension contract before capability is added. + +### Fixed + +- **Documented behaviour that did not match the code.** `README.md` and + `docs/validation.md` both stated that the wrapper switches `q09`-`q13` to + report-only under a custom `--rules`. It does not: `--mapping-policy` exists + in `validation_runner.py` but `vcf_rdfizer.py` never forwards it, so the + runner always executes `strict` and a *correct* custom-mapping conversion is + reported as `MISMATCH`. The claim is corrected in both places, recorded in + [`docs/limitations.md`](docs/limitations.md), and tracked as a defect in + [`docs/roadmap.md`](docs/roadmap.md). No code change yet. + +### Changed + +- Existing documents gained consistent navigation headers and "See also" + footers, and `README.md` now points at `docs/README.md` as the documentation + entry point rather than listing two files. + +## 2026-09-04 — Graph integrity: blank nodes, empty terms, duplicate statements + +Three checks that verify the N-Triples graph itself, independently of what the +VCF contains. Mutation score **64/66 -> 76/78**, catalogue 36 -> 42 mutations. + +### Added + +- `preflight_blank_nodes` — any blank node, in subject or object position. Every + class in the vocabulary declares an `vcfr:iriTemplate`, so a blank node means + a term map produced no IRI; the record and value digests could not address + such a node, and neither could anything that later merges the graph. A + predicate cannot be blank in RDF, so it is not examined. +- `preflight_empty_values` — empty or whitespace-only literals and IRIs, each + labelled `EMPTY_LITERAL` or `EMPTY_IRI` in the sample. An empty literal means + a value was lost rather than marked missing (the pipeline writes + `"."^^vcfr:Null` for a genuine missing token); an empty IRI means a template + substitution collapsed. Whitespace-only counts as empty: it carries no more + information and is just as certainly a defect. +- `preflight_duplicate_triples` — the same statement emitted more than once. + +Both anomaly checks have exact-count companions, so severity is measured rather +than saturating at the 100-row diagnostic sample. + +### How duplicates are detected, and why it could not be a query + +A SPARQL store holds a **set**. A repeated line is collapsed on load, so it is +invisible to every other check here — `COUNT(*)` over `?s ?p ?o` returns the +distinct total on Comunica, QLever and rdflib alike. Verified before building +anything: a three-line file with one repeat reports 2. + +The detector compares the statements the parser read against the distinct +triples the store holds. Raptor supplies the first (`rapper -c` counts parsed +statements including repeats — confirmed in the image: it reports 3 where the +store reports 2), `preflight_distinct_triple_count` the second, and the +difference is exactly the number of redundant statements. Duplicating the whole +fixture graph reports `parsed=624, distinct=312, duplicates=312`. + +This is worth having because duplicated RDF parts are a known failure mode of +the conversion — `run_conversion.sh` already carries a defensive dedupe for it — +and a duplicated aggregate costs twice the storage for no added information. + +### Behaviour + +- All three are blocking: a graph that fails one is not worth comparing. +- When an input a check needs is unavailable, it reports `NOT_EVALUATED` rather + than `PASS`, and `NOT_EVALUATED` never fails a run. A check that could not run + must not be mistaken for a clean result in either direction. + +### Verified + +- All 22 queries (13 core, 9 preflight/count) return identical values under + Comunica and QLever for both representations, and both engines produce `PASS` + against the Python oracle — including the new `ISBLANK`, `ISIRI` and + `REGEX`-based checks. +- 317 tests pass. Six new mutations (`introduce_blank_node`, + `blank_node_object`, `empty_literal`, `whitespace_only_literal`, + `duplicate_triple`, `duplicate_whole_graph`) are all detected. + +### Documentation + +- `docs/validation.md` gained a "Graph integrity" section explaining why + duplicates cannot be found by a query. +- `docs/vcf-coverage.md` lists the three under graph-level properties. +- `docs/validation-methodology.md` now describes four independent layers rather + than three. + +## 2026-09-04 — Full VCF coverage: census, identity digests, INFO, headers, SHACL + +Phases 2-6 of the validation-coverage plan. Mutation score **29/45 -> 64/66** +(64% -> 97%), across a catalogue that grew from 21 to 36 distinct mutations. + +### Fixed — silent data loss + +- **QUAL was extracted from every VCF and never mapped into RDF.** It is now + emitted, typed `xsd:decimal` or `"."^^vcfr:Null` as the published SHACL shape + requires. RML cannot choose a datatype per row, so it comes from a new + record-detail emitter rather than `default_rules.ttl`. +- **`##fileDate` was never mapped.** Now emitted as `vcfr:fileDate`, typed + `xsd:date` when the value's form allows and lexically otherwise. + +### Added — completeness (Phase 2) + +- `q09_predicate_census` and `q10_class_census` compare the graph's predicate + and class inventory against counts derived from the VCF. One comparison + catches three things: a predicate missing, one with the wrong cardinality, and + one that should not be in the graph at all. Expectations are derived from the + VCF and the emitters' documented shapes, never from `default_rules.ttl` - + deriving them from the mapping is how QUAL stayed invisible. +- `--mapping-policy {strict,report-only}`: a custom `--rules` changes the + inventory and IRI templates by design, so these checks become report-only. + +### Added — record and value identity (Phases 3-4) + +- `q11_record_digest`, `q12_info_value_digest`, `q13_format_value_digest` hash + each record, INFO value and FORMAT value **together with its own IRI** and + bucket the result. Binding identity into the hash is what catches a + permutation; bucketing keeps the result at most 256 rows for any graph size + and needs no ordering guarantee, which `GROUP_CONCAT` could not have given + portably. Fields are separated by U+001F, which cannot occur in a VCF field. +- These close the two largest gaps: values permuted between records, and non-GT + FORMAT values (a mangled DP, GQ or AD previously passed). + +### Added — structured INFO (Phase 4) + +- `--info-representation {structured,raw}`, default `structured`: one + `vcfr:InfoFieldValue` per record and key at the vocabulary's declared IRI + template, linked by `vcfr:declaredBy` to an `InfoFieldDefinition` carrying + `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription`. Single-valued + Integer/Float fields also get `fieldValueInteger`/`fieldValueDecimal`; a Flag + gets `fieldValueBoolean true`. `vcfr:infoRaw` is retained. +- The model was already fully defined in the vocabulary and simply unused. + +### Added — header section (Phase 5) + +- `--header-representation {structured,basic}`, default `structured`: each `##` + line is typed with its vocabulary subclass, and FILTER, ALT and contig + declarations get `filterId`, `altId`, `contigId`, `contigLength`, + `contigMd5`, `contigAssembly` and `contigCount`. An unrecognized key keeps + only the base `HeaderLine` type rather than inventing an undefined term. + +### Added — SHACL (Phase 6) + +- `--shacl-shapes PATH` validates the graph against a SHACL shapes file with + `pyshacl`, inside the container, as a layer independent of the VCF entirely. + Off by default because `pyshacl` loads the graph into memory. A violation + blocks the run; a missing `pyshacl` is reported as `EXECUTION_FAILED`, never + as a conformance failure. +- It found three violations the other layers could not see. Two are fixed above. + The third is a **contradiction inside the published vocabulary**: + `vcfr:missingValuePolicy` says a missing token SHOULD be `"."^^vcfr:Null`, + while `VCFRecordShape` constrains `vcfr:alt` to `xsd:string`, so a record with + `ALT=.` cannot satisfy both. Recorded in `docs/vcf-coverage.md` for a decision + in the vocabulary repository. + +### Changed + +- `evaluate_validation()` is the single place the PASS/MISMATCH/BLOCKED decision + is made, used by real runs and the mutation harness alike. +- `test_validation_logic_unit.py` now shares the canonical fixture instead of + maintaining a second description of the same data. + +### Verified + +- All 13 core queries plus 4 exact-count preflights return **identical values + under Comunica and QLever**, for both representations, and both engines + produce `PASS` against the Python oracle end to end. The agreement script now + checks that last part too: engines agreeing with each other is not enough when + the digests are compared against values Python computes. +- 297 tests pass; the suite still passes without `rdflib` (those tests skip). + +### Documentation + +- `docs/vcf-coverage.md` rewritten: every VCF element with separate + "represented" and "validated" columns, each validated claim backed by a named + mutation, the remaining gaps, and the vocabulary alignment problem. +- `docs/validation-methodology.md`: the three independent layers, why identity + digests are histograms, and how to reproduce the score. +- README gained a "VCF Coverage" section for the three representation options. + +## 2026-09-04 — Validation mutation harness, and the first coverage gaps closed + +### Added — measurement (Phase 0) + +- A mutation-testing harness for the semantic validation suite. A correct graph + is corrupted in ~25 named ways and the validator is asked for a verdict; the + proportion detected is a reproducible **mutation score**. Before this, the + suite's coverage was prose, and that prose was wrong: `QUAL` is extracted from + every VCF and then never mapped into RDF, and no check could have noticed. + - `test/validation_fixtures.py` derives the VCF, the RDF graph and the parser + oracle from one declarative specification, so they cannot drift apart. The + genotype triples come from the project's own emitters. + - `test/validation_mutations.py` is the catalogue: each entry names the VCF + element it targets, the check expected to catch it, and — for a recorded + gap — why it is not caught. + - `test/test_validation_mutation_unit.py` evaluates queries in-process with + rdflib (test-only dependency, tests skip without it) and routes every + verdict through the shipped `evaluate_validation()`, so the harness measures + real code rather than a reimplementation. + - **Known gaps are assertions.** A mutation marked `known_undetected` is + asserted to still be undetected, so closing a gap fails a test and forces + the catalogue and `docs/vcf-coverage.md` to be updated. +- `test/cross_engine_agreement.py` plus a CI job asserting every validation + query returns identical values under Comunica and QLever. Verified: all 34 + executions across both representations agree. +- `.github/workflows/validation-mutation.yml` publishes `mutation-score.json` + as a build artifact. + +### Added — coverage (Phase 1) + +- **Exact anomaly counts.** Each structural anomaly preflight now has a + companion aggregate returning the true count alongside the `LIMIT 100` + diagnostic sample. Reports carry `anomalyCount`, `anomalyCountReturned` and + `sampleTruncated`, so ten million anomalies is no longer indistinguishable + from a hundred. Verified against a graph with 250 anomalies: exact count 250, + sample 100, truncated true. +- **`--strict-conformance`** (runner and wrapper) promotes a missing-token + conformance failure from a report-only observation to a validation failure. +- **File metadata and header coverage.** New `q07_file_metadata` compares the + `##fileformat` / `##reference` / `##source` declarations, and + `q08_header_line_census` compares how many `##` lines carry each header key. + `parse_vcf` gained `parse_header_metadata` to compute the oracle side. These + close the two entirely uncovered triples maps. + +Mutation score: **23/45 → 29/45 (51% → 64%)**. + +### Changed + +- The PASS / MISMATCH / BLOCKED_BY_PREFLIGHT decision is extracted into + `evaluate_validation()`, the single place real runs and the mutation harness + both use. +- `normalize()` generalises its single-row handling beyond `q03_titv` via + `SINGLE_ROW_QUERIES`. +- `rdflib>=7.0.0` added to the `dev` extra. It is not a runtime dependency and + the test suite passes without it. + +### Documentation + +- **`docs/vcf-coverage.md`** — the coverage matrix: one row per VCF element, + with separate "represented" and "validated" columns, each validated claim + backed by a named mutation. Includes the vocabulary alignment gap (17 emitted + terms the vocabulary does not define, all condensed-representation) and the + open gaps in priority order. This is the table intended for publication. +- **`docs/validation-methodology.md`** — the mutation-testing method, why + self-reported coverage failed, the two-engine two-layer design, and how to + reproduce the score. +- `docs/validation.md` — Q7/Q8 documented, the exact-count behaviour and + `--strict-conformance` described, and the "What is *not* tested" section + narrowed to what is still true. + +## 2026-09-04 — QLever integration verified against a live container + +Everything below came out of actually running the integration, and each item +was wrong or missing beforehand. + +### Fixed + +- **`preflight_position_datatype` would have failed every QLever run.** QLever + canonicalises numeric literals at index time, so `"100"^^xsd:integer` is + reported by `DATATYPE()` as `xsd:int`. The query required exactly + `xsd:integer`, so on QLever it flagged every record and the run ended as + `BLOCKED_BY_PREFLIGHT`, while passing on Comunica. It now accepts the XSD + integer family, which is what the check means; a plain-string or + `xsd:decimal` POS is still reported as an anomaly on both engines (verified). +- **QLever's binaries are not called `IndexBuilderMain`/`ServerMain`, and are + not in `/usr/bin`.** They are `/qlever/qlever-index` and + `/qlever/qlever-server`. The Dockerfile COPY paths and the default argv were + both wrong and would have failed the build. +- **Copying the binaries alone does not work.** The upstream image is Ubuntu + 24.04 and this base is Ubuntu 26.04; the binaries need Boost 1.83, ICU 74, + jemalloc and io_uring sonames that do not exist here and cannot be + apt-installed (the sonames are release-pinned). Those nine libraries are now + copied into `/opt/qlever/lib`, and `LD_LIBRARY_PATH` is set for QLever's own + processes only, so they cannot shadow anything else in the image. glibc is + deliberately not copied - it is backward compatible and this base is newer. +- **`qlever-server` defaults to a 30-second query timeout**, far below what the + aggregate queries need. The server is now started with `-s` matching + `--validation-query-timeout`. +- Index/server flags corrected against the real CLI: `-m` is `--stxxl-memory` + for the indexer and `--memory-max-size` for the server (`--stxxl-memory` is + not a server flag). + +### Verified + +- Both binaries link and run in the target image (`qlever-index --version` + reports build `bfd5741`). +- A real index build, server start, readiness poll, HTTP query, SPARQL Results + JSON parse and `normalize()` round-trip, driven through the shipped + `QleverEngine` code rather than a reimplementation. +- **Engine equivalence**: all eleven validation queries run under both Comunica + and QLever against the same expanded and condensed graphs — graphs built with + the project's own sample emitters — produce identical normalized results, and + the values are independently correct. + +### Tests + +- `QleverEnvironmentTests`: the private library path is prepended for QLever + only, and the POS datatype preflight accepts the integer family while still + rejecting string/decimal/double. +- The QLever lifecycle test now asserts the verified binary names and flags, + including the explicit server-side query timeout. + +### Still unverified + +- The HDT and COTTAS decode paths, and a full `docker build .` of the real + image. Those need hdt-cpp and the pycottas venv; the QLever work above was + done against a minimal probe image that isolates the integration. + +## 2026-09-04 — Validation: QLever engine, HDT/COTTAS artifacts, coverage audit + +### Added + +- `--validation-engine {comunica,qlever}`. Comunica remains the default and + queries the graph in memory; [QLever](https://github.com/ad-freiburg/qlever) + builds an on-disk index inside the container, serves it on a container-local + port, answers the queries over HTTP, and tears both down. Both engines answer + identical queries and feed the same comparison layer, so the choice is a scale + decision, never a semantic one, and every report records which engine ran. + Tunable with `--qlever-memory-gb`, `--qlever-port`, + `--qlever-startup-timeout`, and `--validation-query-timeout`. +- QLever binaries are copied into the image from the upstream + `adfreiburg/qlever` image; pin a version with + `--build-arg QLEVER_IMAGE=adfreiburg/qlever:`. A build-time linkage + check records whether they resolve against this base, and the validator + surfaces that note if the engine cannot start. +- QLever's CLI has changed across releases, so both command lines are + overridable without code changes: repeatable `--qlever-index-arg` / + `--qlever-server-arg`, or full replacement via `QLEVER_INDEX_COMMAND` / + `QLEVER_SERVER_COMMAND` with `{index}`, `{input}`, `{memory}`, `{port}` + placeholders. The exact argv that ran is recorded in `manifest.json`. +- Validation of compressed and indexed artifacts. `--rdf` now accepts `.nt`, + `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, and `.cottas.br`; + `--rdf-format` overrides filename detection. Anything that is not already + plain N-Triples is decoded in container scratch (`hdt2rdf`, + `cottas_tool.py decompress`, gzip/brotli) and then put through the full + semantic suite. That is strictly stronger than the triple-count round-trip + `validate_compression.py` performs during compression: it proves the artifact + decodes to a graph that still reproduces every VCF summary. Nothing decoded + is written beneath `--out`. +- `--validate-artifacts {aggregate,hdt,cottas,all}` for full mode. Each + requested artifact is validated independently with its own report directory + (``, `__hdt`, `__cottas`), so one run can prove the aggregate and + both representations agree with the VCF. A representation that was not + selected, or whose artifact is missing after a recoverable index warning, is + skipped rather than reported as a failure. Default is `aggregate`, which + preserves the previous behaviour. +- `materialization.json` per validation run, recording how the artifact was + decoded and how many triples that yielded; `manifest.json` gained `engine` + and a format-tagged `sourceRdf`. Rapper's parsed triple count is now captured + rather than discarded. + +### Changed + +- The validator's cyvcf2 import is lazy, so the pure normalization/comparison + layer is importable and testable on the host without the container. +- `--mode validation` no longer requires `.nt.gz`; any supported artifact works. + `--rdf-nt` / `--rdf-gz` remain accepted as aliases. + +### Tests + +- `test/test_validation_logic_unit.py` (16 tests): mutation tests that drive the + comparison layer directly and record what it detects — dropped records, + misclassified shapes, flipped genotypes, corrupted FILTER lexicals, Ti/Tv + swaps, allele-count errors, spurious records, impossible AC/AN — **and** what + it does not: record permutation, and the entirely unvalidated `ID`, `QUAL`, + `INFO`, non-`GT` `FORMAT`, and header/metadata columns. The blind-spot tests + are deliberate, so closing a gap fails a test instead of passing unnoticed. +- `test/test_validation_engines_unit.py` (25 tests): artifact format detection + and host/container agreement, each decode path, decode-failure reporting, + engine registry, Comunica command construction, QLever index/serve/teardown, + crashed-server diagnostics, command-line overrides, CLI argument validation, + and the wrapper's target resolution and docker argv. + +### Documentation + +- `docs/validation.md`: new "Which artifact is validated", "Which SPARQL engine + runs the queries", and — importantly — "What is *not* tested", which states + the coverage limits plainly: aggregate-only comparison with no per-record + identity, unvalidated VCF columns, no completeness bound, saturating preflight + anomaly counts. +- README: validation artifacts/engines subsection with a short note on what a + `PASS` does and does not mean. + +## 2026-09-04 — Custom-mapping tooling, cheap gzip sizing, dead-script removal + +### Added + +- `vcf-rdfizer-rules`, a second console script (`vcf_rdfizer_rules.py`) for + authoring custom RML mappings. It replaces the `update_rules.sh` sed script + with something that serves the actual need: + - `columns` documents the five generated TSV sources and every column each + one provides, including why `records.tsv`'s final column has no fixed name. + - `init` writes an annotated copy of the default mapping to start from. + - `check` validates a mapping against the wrapper contract *before* a long + run: logical-source paths the wrapper cannot rewrite per input, referenced + columns no TSV provides, which `--sample-representation` values remain + usable, and whether the mapping forces the large sample helper tables to be + materialized. Exits 1 on a contract violation; `--json` for scripting. + The old script rewrote a rules file *in place* to point at fixed TSV paths, + which the pipeline has not needed since `render_rules_for_triplet` began + rendering a per-input copy - and which silently broke the per-input rewrite + it was supposed to help with. +- `vcf_rdfizer_gzip.py`: uncompressed size of a gzip/BGZF file without + decompressing it. Three tiers, and the method used is recorded next to the + size so the metric stays auditable: + - `bgzf` - sums the per-block `ISIZE` trailers of a `bgzip` file by walking + block headers. Exact, no inflate, and immune to the 32-bit wrap. This is + what `bcftools`/`tabix`/`htslib` produce, so it covers the usual case. + - `gzip-sample` / `gzip-trailer` - single-member gzip: measured outright when + it fits the sampling budget, otherwise the 32-bit `ISIZE` trailer resolved + against a compression ratio sampled from the file's own prefix. + - `inflate` - full pass, used whenever the structure cannot settle the + answer. Concatenated non-BGZF members are detected and routed here rather + than trusting a trailer that describes only the final member. + +### Changed + +- `run_conversion.sh` uses that helper for `input_vcf_size_bytes`, which + previously always cost a full `gzip -dc | wc -c` pass over the source VCF. + On the repository's 52 MB test fixture this is 1.115s -> 0.066s (17x); on a + 634 MB gzip whose uncompressed size exceeds 4 GiB, 13.6s -> 0.029s (470x), + where a naive trailer read would have been 93% wrong. The shell keeps its + `gzip -dc` fallback if the helper is unavailable. +- `stages/conversion/*.json` gains `input_vcf_size_method` recording how the + size was obtained. +- `--estimate-size` now uses the real uncompressed input size when the file's + structure can supply it, instead of always assuming a 5x expansion, and says + so when it had to fall back to the assumption. + +### Removed + +- `src/compression.sh` and `test/test_compression_unit.py`. The wrapper issues + its own gzip/brotli/rdf2hdt commands and delegates chunked work to + `partitioned_compression.py`; nothing had called this script for some time, + and it wrote a stale flat `metrics.csv` schema. +- `src/update_rules.sh` and `test/test_update_rules_unit.py`, superseded by + `vcf-rdfizer-rules` above. + +### Documentation + +- README: new "Custom RML Mappings" section with the three-point contract, the + logical-source table, and the `vcf-rdfizer-rules` workflow; a metrics table + explaining each `input_vcf_size_method`; repository-layout and test-suite + entries for the two new modules. +- `rules/README.md` points at the new CLI as the way to author a mapping. + +### Tests + +- `test/test_rules_helper_unit.py` (12 tests), including a drift guard that + runs `src/vcf_as_tsv.sh` and asserts the documented column lists still match + the headers it writes. +- `test/test_gzip_size_unit.py` (13 tests), each checking a measured size + against a full-inflate ground truth across BGZF, plain, concatenated, and + optional-header-field gzip layouts. + +## 2026-09-04 — Code review: cleanup, hot-path performance, and bug fixes + +Behaviour-preserving unless noted. Emitted RDF was verified byte-identical +(SHA-256) against the previous implementation for both sample representations +and for both `run_conversion.sh` storage modes. + +### Fixed + +- `run_compression_methods_for_rdf`: an existing `.hdt` that failed validation + during a full run recorded a **COTTAS** index warning (wrong format, wrong + artifact path, wrong message) via a copy-pasted block whose assignment also + never propagated, because `cottas_failure_warning` was not `nonlocal` there. + The block is removed; `validate_container_artifact` already downgrades a + recoverable HDT index problem to a warning, so reaching that point means the + artifact is genuinely unusable. +- `--mode validation` raised an uncaught `ValueError` traceback when the + results directory already existed and was non-empty, because that check sits + after the argument-validation `try/except`. It now prints `Error: ...` and + exits 2 like every other input error. Covered by a new regression test. +- `count_triples_in_nt_files` (the host-side fallback triple count) used a + slightly different rule than the container-side counters in + `validate_compression.py` / `partitioned_compression.py`, so a fallback count + could disagree with the authoritative one. The predicate is now shared as + `is_triple_line` with identical semantics. + +### Performance + +- Sample RDF emission (`append_expanded_sample_rdf`, + `append_condensed_sample_rdf`) no longer percent-encodes the same values on + every record: sample-column URI components and `sampleId` literals are + computed once per file, and FORMAT-key components are memoized. Previously + these ran `variants x samples` and `variants x samples x FORMAT keys` times. + ~1.6x faster at 50 samples, with the gain growing with cohort width. +- `_rml_uri_component` and `_ntriples_string_literal` short-circuit values that + need no encoding/escaping (~2.7x and ~1.3x on typical VCF tokens). +- `SampleRecordStream` caches the derived FORMAT-key tuple and its + duplicate-key check per distinct `FORMAT` string instead of rebuilding both + for every record. +- `count_triples_in_nt_files` scans bytes instead of decoding to `str`, and + reads gzip aggregates through a `BufferedReader` rather than `GzipFile` + line-by-line. +- `run()` discards subprocess output at the file-descriptor level instead of + buffering a whole container's output in memory only to drop it. +- `run_conversion.sh`: the `"."` -> `"."^^vcfr:Null` rewrite is now applied to + each RMLStreamer part while it is streamed into the aggregate, instead of as + a separate pass over the finished aggregate. This removes one full read and + one full write of the complete RDF output, and removes the transient + full-size temporary copy (which previously required 2x the aggregate size in + free disk at the end of a plain-storage run). +- `run_conversion.sh`: triple counting uses one `LC_ALL=C grep -c` instead of + `awk` / `grep | wc -l`. On a 228 MB gzip aggregate this is ~5x faster + (6.5s -> 1.2s); it is the last unavoidable full pass over the RDF output. +- `run_conversion.sh`: the `stat` dialect is detected once instead of probed on + every call, and the per-second progress heartbeat does one directory walk + with no `basename` subprocess per part. + +### Changed + +- Space-optimized aggregates are ~14 bytes per RMLStreamer part smaller, + because gzip members are now produced from a pipe and therefore no longer + embed the temporary part filename and mtime. Decompressed content is + unchanged and the output is more reproducible. + +### Removed + +- Dead host-side code: `resolve_input` (superseded by + `resolve_input_snapshot`), and the host chunk planners + `plan_record_safe_rdf_chunks`, `plan_partitioned_hdt_chunks`, + `split_nt_file_for_hdt`, `write_nt_chunk`, `iter_rdf_binary_lines`. Chunking + has been the container runner's job (`src/partitioned_compression.py`) since + partitioned compression moved into an ephemeral Docker volume; keeping a + second host-side implementation was exactly what + `run_partitioned_representation_methods_for_rdf_files` documents against. + Their two tests were removed with them. +- An unreachable `else:` arm in `run_full_mode`'s output summary (it iterated a + list the guard had just proved empty), the unused `input_metrics_target` + parameter of `run_full_mode`, an always-true `is not None` guard, and two + unused local assignments. + +### Documentation + +- `vcf_rdfizer.py` gained a module-level division-of-labour note and section + map, plus section banners for the previously unmarked regions (triple + counting, artifact naming, destructive filesystem operations, preflight + estimation, genotype representations, compression-plan parsing, metrics + layout). +- `README.md`: new "Repository Layout" section explaining the host/container + split and what each file does; fixed the broken `RELEASING.md` link + (`scripts/RELEASING.md`); added links to `docs/`, `changelog.md`, and + `ACKNOWLEDGEMENTS.md`. +- `src/compression.sh` and `src/update_rules.sh` headers now state that they + are standalone utilities the pipeline does not call, and what supersedes + each. `compression.sh` previously claimed the Python wrapper was its primary + caller, which has not been true since compression moved inline. +- Added `ACKNOWLEDGEMENTS.md` with funding/attribution and a paper + acknowledgement template. + +## 2026-09-04 — Validation progress and quiet mode + +- Integrated semantic validation with the existing JSONL progress sidecar and + host `ProgressSession`; validation now reports its preflight/core query + progress using the same terminal UI as conversion and compression. +- Added `--quiet` to suppress terminal progress displays and validation query + chatter while retaining progress bookkeeping, command logs, and metrics. +- Kept `--no-progress` as the stronger opt-out that disables sidecar creation + and progress rendering entirely. + +## 2026-09-04 — Full-run semantic validation + +- Added `--validate`/`--run-validation` to run the semantic VCF/RDF validator + once per input as the final stage of a `--mode full` run. +- Full-mode validation accepts the generated plain `.nt` or gzip `.nt.gz` + aggregate, and retains the validator's container-local temporary-file + guarantees. +- Validation timing, status, exit code, input RDF, and report paths are now + included in the run's `metrics.csv`, compression JSON, stage reports, and + recursive `summary.json` report index. + +## 2026-09-04 — Expanded sample representation naming + +- Renamed the default per-sample genotype graph strategy to `expanded`, while + retaining `condensed` as the alternative strategy. +- Updated the CLI default and accepted values, internal workflow/emitter names, + tests, RML comments, and user documentation. +- The emitted profile IRI is now + `vcfr:representationProfile vcfr:ExpandedRepresentation`. + +## 2026-09-04 — Input-labelled, container-complete metrics layout + +- Replaced timestamp-only run-metrics directories with + `run_metrics/__/`. A single VCF/RDF/representation input + uses its source stem (for example, `1000G_phase3_chr20`); a directory input + with multiple VCFs receives a deterministic batch label. +- Added `run.json` and `summary.json` as the stable entry points for every + mode. The manifest records inputs, requested configuration, output root, and + resolved image; the summary records final status, wrapper runtime, tabular + rows, and an index of stage reports and logs. +- Organized metrics by purpose: `logs/`, `timings/`, `stages/`, and `reports/`. + TSV, RML conversion, compression, decompression, indexing, and warnings now + have predictable locations instead of unrelated `raw_metrics`, + `*_metrics`, and timestamp-nested directories. +- Compression-only, decompression, and standalone index modes now persist + container wall/CPU/system time, peak RSS, exit code, input/output sizes, and + structured stage reports. Full and TSV modes retain the same detail. +- Preserved the complete partitioned-compression runner handoff under + `stages/partitioned/`, including every chunk build, merge, validation, + temporary-workspace free-space sample, timing, peak RSS, and bounded stderr + diagnostic. The report is retained on both successful and failed runs before + the temporary Docker volume is removed. + +## 2026-09-03 — Streaming COTTAS merge for condensed cohorts + +- Replaced the final COTTAS merge/reindex implementation with a PyArrow k-way + merge of the already `spo`-sorted Parquet chunks. It keeps one configurable + batch per input (`COTTAS_MERGE_BATCH_ROWS`, default `2048`), deduplicates + adjacent equal triples, and writes the final COTTAS artifact incrementally. + It preserves RDF set semantics and the embedded COTTAS index without a + graph-wide DuckDB hash table or external-sort spill area. +- This supersedes the prior DuckDB merge variants. A 52-chunk, 36-million-triple + cohort exhausted 41.4 GiB of Docker workspace while DuckDB attempted an + external sort; the streaming merge has no proportional temporary-sort file. +- Added PyArrow 22.0.0 to the image and third-party notices for deterministic + Parquet reader/writer support. +- COTTAS merge progress now reports source triples processed and distinct + triples written. Rich interactive displays remain available, and redirected + terminals receive compact line-based progress updates. +- Retained the bounded stderr tail in failures so malformed COTTAS input, + incompatible index metadata, output disk errors, and other code-1 failures + remain actionable after the ephemeral workspace is removed. + +### Rerun guidance + +Build an image from this revision before retrying. The preserved raw `.nt.gz` +can be retried with `--mode compress`, avoiding another RDF run. The merge +requires normal space for the COTTAS chunks and final artifact, but no longer +requires a large DuckDB spill directory. + +## 2026-09-03 — COTTAS SIGKILL/OOM handling + +- Classified `exit_code=-9` in partitioned index warnings as a child process + killed by `SIGKILL` (normally the Linux kernel/Docker OOM killer). Shell + wrappers can surface the same condition as exit code `137`. +- Initially changed the production COTTAS merge to a bounded pairwise tree. + This reduced input fan-in but did not bound the final graph-wide + `DISTINCT`/`ORDER BY`; the disk-backed replacement above supersedes it. +- Added the failing stage's `max_rss_kb` to `index_warnings.json`, alongside + exit code, stderr tail, and workspace free-space samples, so OOM diagnosis + can be compared directly with the container memory limit. +- The `merge-many` adapter remains available for explicit experiments, but is + no longer selected by the default large-file workflow. + +### Rerun guidance + +Rebuild the image and rerun the full conversion (or use `--mode compress` with +the retained raw `.nt.gz`). The current streaming k-way merge supersedes this +earlier pairwise guidance. + +When COTTAS is optional, `--representations hdt` avoids the COTTAS merge +resource requirement while retaining a queryable representation. + +## 2026-09-02 — COTTAS large-input merge reliability + +- Diagnosed the previous generic `COTTAS merge/index creation failed` warning: + the warning is emitted only after per-chunk COTTAS conversion succeeds and + the final indexed merge returns a non-zero status. HDT generation/indexing + is independent and is not implicated by that warning. +- Added a multi-input `pycottas.cat` adapter for explicit use. The production + workflow now uses the bounded pairwise strategy documented in the 2026-09-03 + entry because the merge API can require substantial in-memory buffers. +- Added cleanup of all COTTAS chunks and merge outputs after a failed COTTAS + attempt so stale intermediates cannot consume the Docker workspace. +- Preserved a bounded stderr tail and exit code for every partitioned stage. + Full-run `index_warnings.json` entries now identify resource/OOM signals, + include before/after Docker-workspace free-space samples, and retain the + underlying COTTAS/DuckDB diagnostic when available. +- Pinned the Docker image to `pycottas==1.1.0` for reproducible behavior. +- Added regression tests for the multi-input merge command, stage diagnostics, + and isolated COTTAS scratch cleanup. + +### Rerun guidance + +Rebuild the image and rerun the same full command. If the COTTAS stage still +fails, inspect `stages/partitioned/.json` and the wrapper log for the +failing `cottas-merge-r*` stage and its +`stderr_tail`. For a resource error, lower `--chunk-target-bytes` and +`--chunk-max-bytes`, or enlarge the Docker data-volume allocation. The raw +`.nt.gz` retained by the failed run is a valid recovery source. diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index dce0364..5cd1663 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -24,6 +24,7 @@ requirements: - wheel run: - python >={{ python_min }} + - rich >=13.7.0 test: requires: diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..6cabb44 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,135 @@ +# VCF-RDFizer documentation + +VCF-RDFizer converts VCF files into RDF, optionally into compressed and +queryable representations (HDT, COTTAS), and can prove that the result still +reproduces the source VCF's semantics. + +The [top-level README](../README.md) is the task-oriented quick start: install, +flags, worked commands. **These documents are the explanation** — how each part +works, why it was built that way, and where it stops working. + +Every document here states its own limits. If you only read one page before +deciding whether the tool fits your problem, read +[Limitations](limitations.md). + +--- + +## Start here + +| If you want to… | Read | +| --- | --- | +| Understand how the tool is put together | [Architecture](architecture.md) | +| Know exactly what happens to a VCF | [Conversion](conversion.md) | +| Decide what to convert *into* | [Representations](representations.md) | +| Trust the output | [Validation](validation.md) | +| Know what the tool cannot do | [Limitations](limitations.md) | + +## The whole set + +### How it works + +- **[Architecture](architecture.md)** — the host/container split, what runs + where, the three places the split deliberately leaks, the failure policy, and + the pinned toolchain. +- **[Conversion](conversion.md)** — VCF → TSV → RDF stage by stage: the `awk` + parser and its blind spots, the RML mapping, the wrapper's own emitters, the + IRI templates, datatype and missing-value decisions. +- **[Representations](representations.md)** — the compression plan's three + independent decisions, record-safe chunking, HDT via native `hdtc`, the + bounded COTTAS merge, round-trip verification, and index maintenance. +- **[Output and metrics](output-and-metrics.md)** — the output layout, the + `run_metrics/` tree, input size accounting, progress, interrupts, exit codes. + +### How to extend it + +- **[Custom RML mappings](rml-mappings.md)** — the `--rules` contract, the + `vcf-rdfizer-rules` CLI, and an honest account of what a custom mapping does + *not* control and what it costs in validation. +- **[Data linking design](datalinking-design.md)** — *proposal, not + implemented.* A plug-in architecture for connecting the graph to external + resources (rsIDs, genes, clinical assertions) with declarative linkers, + reference bundles, live-API safeguards, and provenance. +- **[Privacy policy design](privacy-policy-design.md)** — *proposal, not + implemented.* Granular, machine-readable disclosure control over parts of the + graph: an ODRL profile with graph selectors, three enforcement tiers, and + verification — plus a candid account of why access control is not + anonymization when the genotypes are themselves identifiers. + +### What the graph looks like + +- **[Sample representations](sample-representation-guide.md)** — expanded versus + condensed genotype shapes, worked examples, scaling arithmetic, the trade-off, + and an assessment of query-time decoding options. +- **[VCF coverage matrix](vcf-coverage.md)** — element by element: is it + represented, and would a corruption of it be detected. Includes the current + mutation score and the vocabulary alignment gaps. + +### Whether to trust it + +- **[Validation](validation.md)** — how to run the semantic suite, which + artifact and engine to choose, what the thirteen queries and the preflights + check, and a candid "what is *not* tested". +- **[Validation methodology](validation-methodology.md)** — how coverage is + *measured* rather than asserted: the mutation-testing harness, why known gaps + are assertions rather than comments, and how to reproduce the score. + +### Where it is going + +- **[Limitations](limitations.md)** — everything the tool cannot do, does badly, + or does surprisingly, in one place. +- **[Roadmap](roadmap.md)** — what is planned, what is known-broken, and what + has been assessed and deliberately rejected. + +--- + +## Reading paths + +**"I have a VCF and I want RDF."** +[README](../README.md) → [Conversion](conversion.md) → +[Sample representations](sample-representation-guide.md) → +[Representations](representations.md) + +**"I need to defend this output in a paper."** +[Validation](validation.md) → [Validation methodology](validation-methodology.md) +→ [VCF coverage matrix](vcf-coverage.md) → [Limitations](limitations.md) + +**"I want to change what RDF comes out."** +[Architecture](architecture.md) → [Conversion](conversion.md) → +[Custom RML mappings](rml-mappings.md) + +**"I want to add my own domain's links."** +[Data linking design](datalinking-design.md) → +[Custom RML mappings](rml-mappings.md) → +[Validation methodology](validation-methodology.md) + +**"I need to release only part of this cohort."** +[Privacy policy design](privacy-policy-design.md) → +[Sample representations](sample-representation-guide.md) → +[Conversion §6](conversion.md#6-iri-templates) → +[Validation methodology](validation-methodology.md) + +**"A cohort-scale run just failed."** +[Representations §10](representations.md#10-limitations) → +[Output and metrics §2](output-and-metrics.md#2-the-run-metrics-directory) → +[Limitations §1](limitations.md#1-operational) + +--- + +## A note on how these documents are written + +Three conventions, kept deliberately: + +1. **Claims are measured, not asserted.** Where a document says a defect would + be caught, there is a named mutation in + [`test/validation_mutations.py`](../test/validation_mutations.py) that proves + it — and where one would *not* be caught, that gap is also an assertion, so + closing it fails a test rather than passing silently. +2. **Limitations sit next to capabilities**, not in a footnote. A section that + describes what something does also describes where it stops. +3. **Design rationale is recorded**, including for options that were assessed + and rejected, so the same ground is not re-covered later. + +Related files outside `docs/`: [`rules/README.md`](../rules/README.md) (the +mapping directory), [`test/README.md`](../test/README.md) (the test suite), +[`changelog.md`](../changelog.md), [`scripts/RELEASING.md`](../scripts/RELEASING.md), +and [`ACKNOWLEDGEMENTS.md`](../ACKNOWLEDGEMENTS.md). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..1932b1f --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,195 @@ +# Architecture + +How VCF-RDFizer is put together, why it is split the way it is, and where the +split leaks. Read this before [`conversion.md`](conversion.md) or +[`representations.md`](representations.md), which go stage by stage. + +--- + +## 1. The shape of the tool + +VCF-RDFizer is a **thin host-side Python CLI wrapping a fat Docker image**. The +host process never installs Java, Flink, HDT, QLever, `bcftools` or `cyvcf2`; it +plans work, launches containers, and reads back the JSON and CSV each stage +writes. + +```text + host container (ecrum19/vcf-rdfizer) + ───────────────────────────────── ────────────────────────────────── + vcf_rdfizer.py + ├─ parse + validate arguments + ├─ snapshot inputs + ├─ plan output paths + ├─ collision check + ├─ docker run ────────────────────▶ src/vcf_as_tsv.sh (awk) + │ src/run_conversion.sh (RMLStreamer/Flink) + ├─ append direct RDF (see §4) + ├─ docker run ────────────────────▶ src/partitioned_compression.py + │ src/cottas_tool.py + │ src/ensure_hdt_index.sh + │ src/validate_compression.py + ├─ docker run ────────────────────▶ src/validation/validation_runner.py + └─ assemble run_metrics/ +``` + +The reason for this shape is reproducibility. The toolchain is a pinned set of +awkward dependencies — RMLStreamer 2.5.0 on Flink, a Rust `hdtc` 1.1.0 build, +`pycottas`, Comunica 5.3.0, QLever, `pyshacl`, `cyvcf2`, `bcftools`. Asking a +user to assemble that on their own machine is asking for irreproducible results. +Pinning it in one image means a run on a laptop and a run on a cluster execute +the same binaries. The price is that **Docker is a hard requirement** and the +image is large; see [`limitations.md`](limitations.md). + +## 2. Host responsibilities + +| Responsibility | Why on the host | +| --- | --- | +| Argument validation and mode dispatch | Fail before the first container starts | +| Input snapshotting | A directory input is enumerated once, so files appearing mid-run cannot change the work set | +| Output path planning and collision check | The pipeline never overwrites a planned artifact; the check runs before Docker | +| Uncompressed input sizing | `vcf_rdfizer_gzip.py` reads BGZF block headers / the gzip trailer rather than decompressing | +| Docker orchestration | Mounts, user mapping, environment forwarding, permission auto-fix | +| Progress rendering | Containers write a JSONL sidecar; the host polls and renders it | +| Metrics assembly | Every stage's JSON is collected into one `run_metrics/` tree | +| Interrupt handling | `Ctrl+C` triggers best-effort cleanup of tracked intermediates | + +## 3. Container responsibilities + +| Component | Role | +| --- | --- | +| `src/vcf_as_tsv.sh` | One `awk` pass per VCF producing `records`, `header_lines`, `file_metadata` TSVs | +| `src/run_conversion.sh` | Runs RMLStreamer, normalizes Spark/Flink part files, merges them into one aggregate, records conversion metrics | +| `src/partitioned_compression.py` | Record-safe RDF chunking, chunked HDT/COTTAS generation, merge, all inside an ephemeral Docker volume | +| `src/cottas_tool.py` | `convert` / `merge` / `reindex` / `decompress` over `pycottas`, with a bounded-memory streaming merge | +| `src/ensure_hdt_index.sh` | Java-free `.hdt.index.v1-1` sidecar generation via `hdtc`, restoring the previous sidecar on failure | +| `src/validate_compression.py` | Round-trip check: decode an artifact, compare its triple count against the source | +| `src/validation/validation_runner.py` | Semantic VCF-vs-RDF validation: parser oracle, SPARQL queries, comparison report | + +## 4. Where the split leaks, and why + +The design intent is "all data processing happens in the container". There are +**three deliberate exceptions**, all in `vcf_rdfizer.py`, all appending directly +to the RDF aggregate between the conversion container and the compression +container: + +| Emitter | Emits | Why not RML | +| --- | --- | --- | +| `emit_sample_representation` | `SampleCall`/`FormatFieldValue`, or `SampleSet`/`CohortCallMatrix`/`FormatValueVector` | RML would first have to materialize a helper table of variants × samples (× FORMAT keys) — the largest intermediate the pipeline can produce | +| `append_header_representation_rdf` | Header-line subclasses, FILTER/ALT/contig attributes, INFO/FORMAT declarations | RML cannot choose a class per row from a parsed attribute string | +| `emit_record_detail` | `QUAL` (typed per value) and structured `InfoFieldValue` nodes | RML cannot switch a literal's datatype based on a declared `Type`, and structured INFO would need a variants × INFO-keys helper table | + +This is worth stating plainly because it has consequences: + +- A **custom `--rules` mapping does not control these triples.** They are emitted + by the wrapper regardless, unless the corresponding + `--sample-representation` / `--header-representation` / `--info-representation` + option turns them off. A mapping author who expects `--rules` to be the single + source of truth for the graph will be surprised. +- The host process **does** read `records.tsv` and write N-Triples, so the claim + that nothing on the host touches the data is not literally true. What is true + is that no *third-party* toolchain runs on the host. +- These emitters are streaming and append-only, guarded by + `_append_rdf_atomically`, so an interrupted append does not leave a + half-written aggregate. + +## 5. Data flow in full mode + +```text + input.vcf(.gz) + │ src/vcf_as_tsv.sh (awk, one pass) + ▼ + .records.tsv, .header_lines.tsv, .file_metadata.tsv + │ RMLStreamer + rules/default_rules.ttl + ▼ + Flink part files ──▶ merged aggregate .nt or .nt.gz + │ host-side direct emitters (§4) append in place + ▼ + complete aggregate + │ optional: --rdf-compression gzip,brotli + ├──────────────▶ .nt.gz / .nt.br + │ optional: --representations hdt,cottas (chunked) + ├──────────────▶ .hdt + .hdt.index.v1-1 + ├──────────────▶ .cottas + │ round-trip triple-count check on each base artifact + │ optional: --artifact-compression gzip,brotli + ├──────────────▶ .hdt.gz / .cottas.br / ... + │ optional: --validate + └──────────────▶ run_metrics/.../reports/validation/... +``` + +Multiple VCF inputs are processed **one at a time**, and a failure is isolated +to that input: the run continues and the failure is recorded in +`reports/failed_inputs.csv`. + +## 6. Failure policy + +The tool distinguishes three kinds of failure, and treats them differently: + +| Kind | Example | Behaviour | +| --- | --- | --- | +| **Fail closed** | An HDT decodes to the wrong triple count | The stage fails; no artifact is published | +| **Degrade and record** | An HDT is readable but its sidecar index could not be built | The run continues, the artifact is marked `index_status: "failed"`, and the raw RDF is retained so it can be repaired later; recorded in `reports/index_warnings.json` | +| **Isolate** | One VCF out of twelve fails conversion | That input is abandoned, the rest proceed, and it appears in `reports/failed_inputs.csv` | + +A COTTAS failure degrades differently from HDT: because the `.cottas` file +itself is unusable, every dependent COTTAS artifact is marked not generated +rather than published with a warning. + +Standalone `--mode index` is deliberately **strict** — it is a maintenance +operation whose whole purpose is to produce a valid index, so a partial result +is a failure, and the previous sidecar is restored. + +## 7. Isolation and cleanup guarantees + +These are properties the tool tries hard to hold, and states in its reports so +they can be checked rather than trusted: + +- Partitioned compression runs in an **ephemeral Docker-managed volume**. + Chunks, COTTAS scratch, and merge files never touch the output directory, and + the volume is removed on success *and* on failure. +- Validation mounts the source artifact **read-only** and decodes compressed + input under the container's `/work`. Nothing decoded is written beneath + `--out`; the stage report records that it was cleaned up. +- Each COTTAS conversion gets a fresh container-local DuckDB workspace, removed + when that operation completes. +- Output collisions are checked *before* Docker starts, and no planned artifact + is ever overwritten. `--mode index` is the single deliberate exception. + +## 8. Source map + +| Path | Role | +| --- | --- | +| [`vcf_rdfizer.py`](../vcf_rdfizer.py) | Host CLI: validation, planning, Docker orchestration, metrics, and the three direct emitters of §4 | +| [`vcf_rdfizer_rules.py`](../vcf_rdfizer_rules.py) | `vcf-rdfizer-rules`: scaffold, document, and check custom RML mappings | +| [`vcf_rdfizer_gzip.py`](../vcf_rdfizer_gzip.py) | Uncompressed size of a gzip/BGZF VCF without decompressing it | +| [`src/`](../src) | Container-side stages (table in §3) | +| [`rules/default_rules.ttl`](../rules/default_rules.ttl) | Default RML mapping, also shipped as package data in `vcf_rdfizer_data/` | +| [`test/`](../test) | `unittest` suite; pipeline tests stub `java`, `docker` and friends so no external tool is required | +| [`scripts/release.py`](../scripts/release.py) | Version bump and release metadata automation | + +## 9. Pinned toolchain + +Set as build arguments in the [`Dockerfile`](../Dockerfile): + +| Component | Version | Used for | +| --- | --- | --- | +| RMLStreamer | 2.5.0 | RML mapping execution on Flink | +| `hdtc` | 1.1.0 (Rust) | HDT create/merge/index, Java-free | +| Comunica | 5.3.0 | Default validation SPARQL engine | +| QLever | from `adfreiburg/qlever` (verified against build `bfd5741`) | Scale validation SPARQL engine | +| `pycottas` | image-pinned | COTTAS conversion and decode | + +QLever is copied from a differently based upstream image, so its +release-specific Boost/ICU/jemalloc/io_uring libraries are copied alongside into +`/opt/qlever/lib` and only QLever's own processes are pointed there. A +build-time `ldd` check records whether they resolve; Comunica remains the +default, so an image whose QLever binaries do not link still validates. + +--- + +## See also + +- [Conversion](conversion.md) — VCF to RDF, stage by stage +- [Representations](representations.md) — compression and queryable artifacts +- [Output and metrics](output-and-metrics.md) — what a run writes and where +- [Limitations](limitations.md) — what this architecture cannot do diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 0000000..9e3c716 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,162 @@ +# CLI reference + +Every flag `vcf-rdfizer` accepts, grouped by where it applies, with defaults and +the constraints that are enforced. `vcf-rdfizer --help` is the authority; this +page adds the *why* and the interactions. + +Two companion CLIs are installed alongside: `vcf-rdfizer-rules` (see +[`rml-mappings.md`](rml-mappings.md)) and, once implemented, +`vcf-rdfizer-link` (see [`datalinking-design.md`](datalinking-design.md)). + +--- + +## The one universal rule + +`-o, --out` is **required in every mode**. It is the run output root: final +artifacts, run metrics and logs, and hidden intermediates all live beneath it. + +## Modes + +| `-m, --mode` | Purpose | Required input | +| --- | --- | --- | +| `full` *(default)* | VCF → TSV → RDF → compression, optionally validated | `-i/--input` | +| `tsv` | VCF → TSV only, for benchmarking | `-i/--input` | +| `compress` | Compress an existing `.nt` / `.nt.gz` | `--rdf` | +| `decompress` | Decode a compressed or indexed artifact back to N-Triples | `-C/--compressed-input` | +| `validation` | Compare a source VCF against its RDF | `-i/--input` **and** `--rdf` | +| `index` | Regenerate an artifact's query index in place | exactly one of `-H/--hdt` or `--cottas` | + +## Inputs and outputs + +| Flag | Meaning | +| --- | --- | +| `-i, --input` | VCF file or directory. Only `*.vcf` and `*.vcf.gz` are recognised; a directory is enumerated one level deep and snapshotted at run start | +| `--rdf` | RDF input for `compress`, or the artifact to check in `validation` | +| `-C, --compressed-input` | `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, `.cottas.br` | +| `-H, --hdt` / `--cottas` | Existing artifact for `--mode index` | +| `-d, --decompress-out` | Explicit output `.nt` path; must be inside `--out` | +| `-o, --out` | **Required.** Run output root | +| `-n, --out-name` | Fallback basename when one cannot be inferred (default `rdf`) | + +## Graph shape + +| Flag | Values | Default | Effect | +| --- | --- | --- | --- | +| `-r, --rules` | path to `.ttl` | shipped `default_rules.ttl` | RML mapping; see the contract in [`rml-mappings.md`](rml-mappings.md) | +| `--sample-representation` | `expanded`, `condensed` | `expanded` | Genotype graph shape | +| `--info-representation` | `structured`, `raw` | `structured` | `structured` adds typed `InfoFieldValue` nodes alongside `infoRaw` | +| `--header-representation` | `structured`, `basic` | `structured` | `structured` types each `##` line and lifts its attributes | + +There is **no automatic sample-count threshold**: the same command always +produces the same graph shape, so a downstream consumer can rely on the contract +it selected. QUAL is always emitted regardless of these options. + +`condensed` rejects a custom mapping that consumes the materialized sample +helper tables, because that would emit both genotype representations at once. + +## Compression plan + +| Flag | Values | Default | +| --- | --- | --- | +| `--rdf-storage-mode` | `plain`, `space-optimized` | **required in full mode** | +| `--rdf-compression` | `gzip`, `brotli`, `none` | `gzip,brotli` | +| `--representations` | `hdt`, `cottas`, `none` | `hdt` | +| `--artifact-compression` | `gzip`, `brotli`, `none` | `none` | +| `--hdt-strategy` | `auto`, `partitioned`, `single` | `auto` | +| `--chunk-target-bytes` | bytes | 512 MiB | +| `--chunk-min-bytes` | bytes | 128 MiB | +| `--chunk-max-bytes` | bytes | 1 GiB | + +Constraints that are enforced rather than documented-and-hoped: + +- Each selector takes a comma-separated list; `none` must appear alone. +- `--artifact-compression` requires at least one selected representation. +- `--hdt-strategy single` cannot consume a `space-optimized` gzip stream. + +`-c, --compression` is a hidden legacy alias retained for backward +compatibility. Use the three explicit selectors. + +## Intermediates and cleanup + +| Flag | Effect | +| --- | --- | +| `-k, --keep-tsv` | Keep the hidden TSV intermediates | +| `-R, --keep-rmlstreamer-rdf-output` | Keep the RDF aggregate after compression | +| `--remove-rdf-storage-output` | Explicitly remove the aggregate after successful compression | +| `-e, --estimate-size` | Preflight size estimate; no conversion | + +## Validation + +| Flag | Values | Default | Meaning | +| --- | --- | --- | --- | +| `--validate` / `--run-validation` | — | off | Run validation once per input in full mode | +| `--validate-artifacts` | `aggregate`, `hdt`, `cottas`, `all` | `aggregate` | Which produced artifacts to check, each in its own report directory | +| `--validation-id` | name | source basename | Report directory name; existing directories are never overwritten | +| `--validation-engine` | `comunica`, `qlever`, `hdt`, `cottas`, `all`, or a comma-separated list | `comunica` | SPARQL backend(s); a scale and performance decision, never a semantic one. `hdt`/`cottas` query the compressed artifact in place. Several engines answer the whole query set, are cross-checked against each other, and are timed in `benchmark.csv` | +| `--filter-oracle` | `auto`, `bcftools`, `cyvcf2` | `auto` | FILTER-field oracle | +| `--shacl-shapes` | path | off | Independent structural layer via `pyshacl`; in-memory, so not for cohort scale | +| `--strict-conformance` | — | off | Promote a missing-token conformance anomaly from report to failure | +| `--validation-query-timeout` | seconds | 3600 | Per-query timeout, every engine | +| `--qlever-memory-gb` | N | 4 | QLever index and server memory budget | +| `--qlever-port` | N | 7019 | Container-local only; never published | +| `--qlever-startup-timeout` | seconds | 900 | Wait for the server after indexing | +| `--qlever-index-arg` / `--qlever-server-arg` | string, repeatable | — | Extra arguments for the QLever binaries | + +A validation failure in full mode marks that input as failed, retains its raw +RDF for inspection, and makes the run exit non-zero — so a pipeline can be gated +on semantic correctness rather than on Docker having returned. + +`--mapping-policy` exists in `validation_runner.py` but is **not exposed by the +wrapper**; see the known gap in +[`rml-mappings.md`](rml-mappings.md#4-what-a-custom-mapping-costs-in-validation). + +## Docker + +| Flag | Default | Meaning | +| --- | --- | --- | +| `-I, --image` | `ecrum19/vcf-rdfizer` | Image repository | +| `-v, --image-version` | latest resolved | Image tag | +| `-b, --build` | off | Force a local build | +| `-B, --no-build` | off | Fail if the image is not present rather than building | + +## Output control + +| Flag | Effect | +| --- | --- | +| `--quiet` | Suppress terminal progress and the validator's per-query chatter; sidecar, command log and metrics are still written | +| `--no-progress` | Also disable progress-sidecar creation | +| `-P, --spark-partitions` | RMLStreamer parallelism hint; does not change the output | + +## Environment variables + +These are read by the container and, where noted, forwarded by the wrapper when +set on the host. + +| Variable | Default | Effect | +| --- | --- | --- | +| `HDT_MERGE_MEMORY_LIMIT` | `512M` | Soft memory budget for `hdtc create` | +| `HDT_INDEX_MEMORY_LIMIT` | `512M` | Soft memory budget for `hdtc index` | +| `COTTAS_MERGE_BATCH_ROWS` | `2048` | Rows held per input in the streaming COTTAS merge | +| `HDT_INDEX_WORK_ROOT` | `/work` | Scratch root when calling `ensure_hdt_index.sh` directly | +| `QLEVER_INDEX_COMMAND` | built-in | Full argv template with `{index}` `{input}` `{memory}` placeholders | +| `QLEVER_SERVER_COMMAND` | built-in | Full argv template with `{index}` `{port}` `{memory}` placeholders | + +Memory limits accept an `M` or `G` suffix. Lower values reduce in-memory sort +buffers and increase temporary I/O. Their scratch lives on the **Docker data +volume**, not the output filesystem. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Success (possibly with recorded index warnings) | +| `1` | One or more inputs failed, including a semantic validation failure | +| `130` | Interrupted with `Ctrl+C`; progress written and tracked intermediates cleaned up | + +--- + +## See also + +- [Representations](representations.md) — what the compression flags build +- [Validation](validation.md) — what the validation flags check +- [Output and metrics](output-and-metrics.md) — where the results land diff --git a/docs/conversion.md b/docs/conversion.md new file mode 100644 index 0000000..31d9d4c --- /dev/null +++ b/docs/conversion.md @@ -0,0 +1,237 @@ +# Conversion: VCF to RDF + +What the conversion actually does to a VCF, in order, and what it decides on +your behalf. The companion documents are +[`architecture.md`](architecture.md) (how the pieces fit), +[`rml-mappings.md`](rml-mappings.md) (how to change the mapping), and +[`vcf-coverage.md`](vcf-coverage.md) (which VCF element ends up where). + +--- + +## 1. Input acceptance + +`--input` takes a single file or a directory. + +- A file must be named `*.vcf` or `*.vcf.gz`. **Nothing else is accepted** — not + `.bcf`, not `.vcf.bgz`, not a bare `.gz`. Extension, not content, decides. +- A directory is enumerated **one level deep**, sorted, at the moment the run + starts. Files that appear later are not picked up. Subdirectories are ignored. +- Each input is processed independently and end to end before the next one + begins. A failure is isolated to its input. + +The output basename is the filename with `.vcf` / `.vcf.gz` removed. Two inputs +that reduce to the same basename would collide, and the pre-flight collision +check rejects the run before Docker starts. + +## 2. Stage one — VCF to TSV + +[`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) reads the VCF (through `gzip -dc` +when compressed) in **one `awk` pass** and writes three tab-separated tables. + +| Output | One row per | Columns | +| --- | --- | --- | +| `.file_metadata.tsv` | file (exactly one row) | `SOURCE_FILE`, `FILE_FORMAT`, `FILE_DATE`, `SOURCE_SOFTWARE`, `REFERENCE_GENOME`, `HEADER_COUNT`, `RECORD_COUNT` | +| `.header_lines.tsv` | `##` meta-information line | `SOURCE_FILE`, `HEADER_INDEX`, `HEADER_KEY`, `HEADER_VALUE`, `RAW_LINE` | +| `.records.tsv` | VCF data line | `SOURCE_FILE`, `ROW_ID`, `CHROM`, `POS`, `ID`, `REF`, `ALT`, `QUAL`, `FILTER`, `INFO`, `FORMAT`, *(sample payload)* | + +Two more paths exist for compatibility, `sample_calls.tsv` and +`sample_format_values.tsv`. Under the shipped mapping they are written +**header-only** — see §5. + +What this stage does, precisely: + +- `##` lines are split on the **first** `=`. Everything after it is + `HEADER_VALUE`; `RAW_LINE` keeps the original text minus the leading `##`. + No structural parsing happens here. +- Four keys are lifted into `file_metadata.tsv` by case-insensitive match: + `fileformat`, `filedate`, `source`, `reference`. +- `HEADER_INDEX` counts `##` lines *and* the `#CHROM` line, 1-based. +- `ROW_ID` is a 1-based counter over data lines, stable within one file. It is + the join key for every record-level IRI. +- Trailing carriage returns are stripped from every field. +- All sample columns are joined into **one** trailing field separated by single + spaces, and runs of whitespace are collapsed. The column's *name* is the + whitespace-joined sample IDs from `#CHROM`, or the literal `SAMPLES` when the + VCF declares none — which is why a mapping cannot reference it by a fixed name. + +**Limitations of this stage, stated plainly.** There is no `htslib` here. The +parser is regex-and-field-index `awk`, which means: + +- The `#CHROM` line is recognised only when it is **tab-delimited**. A + space-delimited header line is matched by no rule, so sample column names are + lost and the records header falls back to `SAMPLES` — silently. +- Nothing validates VCF spec conformance. A malformed file produces a malformed + graph rather than an error, and the first thing that notices is the validation + suite. +- A data line with fewer than 8 columns yields empty strings for the missing + fields rather than an error. +- Because sample fields are whitespace-normalized, a value containing a literal + space would be corrupted. The VCF specification forbids spaces in data fields, + so this is only reachable with an already-invalid file — but it is not detected. + +## 3. Stage two — TSV to RDF with RMLStreamer + +[`src/run_conversion.sh`](../src/run_conversion.sh) runs RMLStreamer 2.5.0 over +the TSVs using the mapping at `--rules` (default +[`rules/default_rules.ttl`](../rules/default_rules.ttl)), then normalizes the +Flink part files and merges them into one aggregate. + +The mapping's five `csvw:url` values are **literal container paths** +(`/data/tsv/records.tsv` and friends) that the wrapper rewrites per input to +`/data/tsv/.records.tsv`. That rewrite is why the contract in +[`rml-mappings.md`](rml-mappings.md) insists those strings stay verbatim. + +`--rdf-storage-mode` decides how the aggregate is assembled: + +| Mode | Behaviour | +| --- | --- | +| `plain` | Parts are merged into one uncompressed `.nt` | +| `space-optimized` | Each part is streamed through gzip into one `.nt.gz` and the source part is deleted immediately, so a full uncompressed copy never exists | + +The output is **N-Triples**. No named graphs are produced, and no blank nodes: +every class in the vocabulary declares an IRI template, and +`preflight_blank_nodes` treats a blank node as a validation failure precisely +because it means a term map produced no IRI. + +`--spark-partitions` is a parallelism hint passed through to RMLStreamer. It +does not change the output, only how many parts are produced before the merge. + +## 4. Stage three — the wrapper's own emitters + +Three classes of triple cannot come from RML, and are appended to the aggregate +by the host process before compression. See +[`architecture.md`](architecture.md#4-where-the-split-leaks-and-why) for why. + +### `emit_record_detail` — QUAL and structured INFO + +`QUAL` is **always** emitted. The published SHACL shape is +`sh:or([sh:datatype xsd:decimal] [sh:datatype vcfr:Null])`, and RML cannot pick +a datatype per row: + +| QUAL value | Emitted as | +| --- | --- | +| a number | `""^^xsd:decimal` — the source lexical form, so no precision is gained or lost | +| `.` or empty | `"."^^vcfr:Null` | +| anything else | a plain string literal, deliberately kept rather than dropped, and reported by the SHACL layer | + +`--info-representation structured` (the default) additionally emits one +`vcfr:InfoFieldValue` per record and key at +`…#call/{ROW_ID}/info/{KEY}`, linked to the `##INFO` declaration through +`vcfr:declaredBy`. A single-valued field (`Number=1`) whose declared `Type` is +`Integer` or `Float` also gets a typed `fieldValueInteger` / `fieldValueDecimal`; +a `Flag` gets `vcfr:fieldValueBoolean true`. A multi-valued field +(`Number=A/R/G/.`) keeps only the lexical value, because the vocabulary's IRI +template gives one node per key, not per value — a real modelling gap, recorded +in [`limitations.md`](limitations.md). + +`--info-representation raw` emits only the opaque `vcfr:infoRaw` string. + +### `append_header_representation_rdf` — structured headers + +`--header-representation structured` (the default) types each `##` line with its +vocabulary subclass (`INFOHeaderLine`, `ContigHeaderLine`, `FilterDefinition`, +`AltDefinition`, …) and lifts the `` +attributes into their own properties. The attribute parser respects quoting and +backslash escapes, so a `Description` containing a comma is handled correctly. + +An **unrecognised** `##` key keeps only the base `vcfr:HeaderLine` type. +Inventing a subclass would put a term in the graph that the vocabulary does not +define. + +`##fileDate` is emitted as `xsd:date` when its form is recognisable, and +verbatim as a plain literal otherwise — again preserved rather than dropped, and +reported by SHACL. + +`--header-representation basic` keeps only the base header-line triples. + +### `emit_sample_representation` — genotypes + +Exactly one sample emitter runs per conversion, chosen by +`--sample-representation`. Full treatment in +[`sample-representation-guide.md`](sample-representation-guide.md); the short +version: + +| Mode | Shape | Growth | +| --- | --- | --- | +| `expanded` (default) | one `vcfr:SampleCall` per record × sample, one `vcfr:FormatFieldValue` per FORMAT key | ≈ variants × samples × FORMAT fields | +| `condensed` | one reusable `vcfr:SampleSet`, one `vcfr:CohortCallMatrix` per call, one `vcfr:FormatValueVector` per FORMAT key holding a tab-separated `vcfr:VCFTextVector` | ≈ samples + variants × FORMAT fields | + +The file declares which shape it carries via `vcfr:representationProfile`, so a +consumer can branch on it before querying. + +## 5. The two helper tables + +`sample_calls.tsv` and `sample_format_values.tsv` exist so that a mapping *can* +express genotypes in RML if it wants to. Under the shipped mapping they stay +header-only and the wrapper streams genotypes directly, because materializing +them means one row per variant × sample (× FORMAT key) — the largest +intermediate the pipeline can produce. + +A custom mapping that consumes them in any other way forces full +materialization, and is **rejected** in `--sample-representation condensed`, +which would otherwise emit both genotype representations into one graph. + +## 6. IRI templates + +Every IRI is derived from the source filename and the row counter, so a +conversion is deterministic and re-running it produces byte-comparable subjects. + +| Resource | Template | +| --- | --- | +| `VCFFile` | `file://{SOURCE_FILE}` | +| `VCFHeader` | `file://{SOURCE_FILE}#header` | +| `HeaderLine` | `file://{SOURCE_FILE}#header/line/{HEADER_INDEX}` | +| `VCFRecord` | `file://{SOURCE_FILE}#record/{ROW_ID}` | +| `VariantCall` | `file://{SOURCE_FILE}#call/{ROW_ID}` | +| `InfoFieldValue` | `file://{SOURCE_FILE}#call/{ROW_ID}/info/{KEY}` | +| `SampleCall` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}` | +| `FormatFieldValue` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}/fmt/{KEY}` | +| `SampleSet` (condensed) | `file://{SOURCE_FILE}#samples` | +| `VCFSample` (condensed) | `file://{SOURCE_FILE}#samples/{SAMPLE}` | +| `CohortCallMatrix` (condensed) | `file://{SOURCE_FILE}#call/{ROW_ID}/matrix` | +| `FormatValueVector` (condensed) | `file://{SOURCE_FILE}#call/{ROW_ID}/matrix/fmt/{KEY}` | + +`{SOURCE_FILE}` is the **basename**, not a path, so a graph does not encode +where the VCF happened to live. The consequence is that two different VCFs with +the same filename mint the same IRIs; if you convert `chr1/data.vcf` and +`chr2/data.vcf`, their graphs will collide when merged. Rename before +converting, or keep the graphs separate. + +## 7. Missing values + +The vocabulary's `vcfr:missingValuePolicy` says a missing token should be +`"."^^vcfr:Null`, and the conversion follows it. `preflight_missing_token_conformance` +reports a plain `"."` literal as an anomaly, and `--strict-conformance` promotes +that from a report to a failure. + +This policy currently **conflicts with the published SHACL shapes** for +`ref`/`alt`/`chrom`, which constrain those to `sh:datatype xsd:string`. A record +with `ALT=.` therefore cannot satisfy both. The conflict is documented in +[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary) and +needs a decision in the vocabulary repository, not here. + +## 8. What conversion does *not* do + +- **No normalization.** No left-alignment, no trimming, no multi-allelic + splitting. `ALT=A,T` stays one record with one `alt` literal. This is + deliberate — the graph is a faithful transcription of the file — but it means + the graph is not directly joinable with normalized external resources. This is + the central problem the [data-linking design](datalinking-design.md) has to + solve. +- **No reference checking.** `REF` is not verified against any genome. +- **No structural-variant modelling.** Symbolic ALTs (``), breakends + (`N[chr2:321[`) and `*` are carried as literals. The validation suite + classifies them (`SYMBOLIC_OR_BREAKEND`) but the vocabulary gives them no + structure, and `INFO` keys such as `END` and `SVTYPE` get no special meaning. +- **No genotype interpretation.** A `GT` value is a lexical token. Phasing + (`|` versus `/`) is preserved in the literal but not modelled. +- **No cross-file merging.** Each VCF produces its own graph. + +--- + +## See also + +- [VCF coverage matrix](vcf-coverage.md) — element-by-element, with the validation check that covers each +- [Sample representations](sample-representation-guide.md) — the genotype shapes in depth +- [Custom RML mappings](rml-mappings.md) — changing what is emitted +- [Limitations](limitations.md) — consolidated diff --git a/docs/datalinking-design.md b/docs/datalinking-design.md new file mode 100644 index 0000000..4cc04db --- /dev/null +++ b/docs/datalinking-design.md @@ -0,0 +1,300 @@ +# Data linking: a plug-in architecture + +*Status: **design proposal**. Nothing described here is implemented yet. This +document exists to fix the extension contract before code is written, because +third-party linkers are the point and a contract is much harder to change once +people depend on it.* + +VCF-RDFizer converts a VCF into RDF that is faithful to the VCF and to nothing +else. Every IRI it mints is derived from the source file, so the graph is +self-contained, reproducible, and — deliberately — an island. It says +`vcfr:recordId "rs334"` and stops there. + +Data linking is the step that connects that island to the rest of the +linked-data web: the rsID becomes a dbSNP resource, the coordinate becomes an +Ensembl gene, the allele becomes a ClinVar assertion. The requirement is that +users other than the maintainers can add those connections for their own +domain without patching the tool. + +--- + +## 1. The observation the design rests on + +Linking a variant to an external resource is always a **join**, and in this +domain there are only three join keys: + +| Strategy | Key | Typical targets | +| --- | --- | --- | +| `token` | a value in `ID`, or the value of a named `INFO` key | dbSNP/Ensembl rsIDs, COSMIC identifiers, `GENEINFO` gene symbols | +| `interval` | `CHROM` + `POS` (+ `REF` length) overlap | genes, transcripts, exons, regulatory regions, cytobands, panel membership | +| `allele` | normalized `CHROM`-`POS`-`REF`-`ALT` | ClinVar assertions, gnomAD frequencies, CADD scores, variant-annotation APIs | + +Everything on the near-term wish list — rsID linking, medically relevant gene +linking, clinical significance, population frequency, drug-target association — +is one of those three. + +This matters because it determines where the seam goes. If the framework owns +the three join strategies, a plugin author never writes VCF parsing, never +mints an IRI, never escapes an N-Triples literal, and never thinks about +streaming. They declare *which key* and *what to emit on a match*. If the +framework instead offered a generic "here is a record, do what you like" hook, +every plugin would reimplement the same four things, each slightly wrong. + +--- + +## 2. Three tiers of plugin + +A linker is a directory. It is resolved both from a plugin search path and from +Python entry points (`vcf_rdfizer.linkers`), so a linker can be `pip install`ed +*or* dropped into a directory and mounted into the container. + +```text +linkers/rsid-dbsnp/ + linker.ttl # manifest: identity, join, source, emission, budget + resolver.py # OPTIONAL - only when resolution is not templatable + queries/ # OPTIONAL - .rq validation queries for the produced links + mutations.py # OPTIONAL - mutation catalogue entries for those links +``` + +### Tier 1 — declarative, no code + +The rsID case needs no Python and no network. It is a token join feeding an IRI +template: + +```turtle +@prefix vcfl: . + +<#rsIDdbSNP> a vcfl:Linker ; + vcfl:id "rsid-dbsnp" ; + vcfl:version "1.0.0" ; + vcfl:title "rsID to dbSNP" ; + vcfl:join [ a vcfl:TokenJoin ; + vcfl:field "ID" ; + vcfl:splitOn ";" ; + vcfl:accept "^rs[0-9]+$" ] ; + vcfl:emit [ vcfl:subject vcfl:VariantCall ; + vcfl:predicate vcfl:sameVariantAs ; + vcfl:objectTemplate "https://identifiers.org/dbsnp:{TOKEN}" ] . +``` + +Roughly two thirds of genuinely useful linkers are expressible at this tier. +Making Tier 1 real is the highest-leverage decision in the whole design: it +turns "write a plugin" from a Python project into a fifteen-line file, which is +the difference between an extension point that gets used and one that does not. + +### Tier 2 — local reference bundle + +Gene linking is an interval join against a reference the plugin *declares* +rather than ships: + +```turtle + vcfl:reference [ vcfl:url "https://ftp.ensembl.org/.../Homo_sapiens.GRCh38.113.gff3.gz" ; + vcfl:sha256 "e3b0c442..." ; + vcfl:assembly "GRCh38" ; + vcfl:format vcfl:GFF3 ] ; +``` + +The runner fetches once, verifies the digest, caches under +`~/.cache/vcf-rdfizer/linkers/`, and builds the interval index. Because the +assembly is declared, the runner can compare it against the VCF's `##reference` +and **refuse to run on a mismatch**. That strictness is deliberate: a +GRCh37-coordinate variant linked against GRCh38 gene intervals produces +confident, plausible, wrong annotations, and nothing downstream would catch it. + +### Tier 3 — live service + +A `resolver.py` implementing one narrow protocol: + +```python +def resolve(batch: Sequence[LinkKey], ctx: LinkerContext) -> Iterable[Link]: + """Map a batch of join keys to links. Called with deduplicated keys.""" +``` + +Batch in, links out. The plugin never opens a socket itself — it calls +`ctx.session.get(...)`. That indirection is what makes the safeguards in §4 +enforceable rather than advisory. + +--- + +## 3. Where linking happens in the pipeline + +Two entry points, sharing one implementation: + +**In-run.** `--link rsid-dbsnp,gene-ensembl` adds a linking stage to full mode. +It runs beside the existing direct emitters (`emit_record_detail`, +`emit_sample_representation` in `vcf_rdfizer.py`), reusing the +`_append_rdf_atomically(rdf_path, stats, producer)` contract so links stream +with the same atomicity, the same stats shape, and the same progress sidecar +events as everything else. + +**Post-hoc.** `--mode link --rdf .nt.gz --link ` enriches a graph +from a conversion that has already happened. This is what makes the feature +adoptable by anyone with existing outputs, and it is also how a linkset is +re-generated when a reference bundle is updated. + +--- + +## 4. Network safeguards + +Live APIs and local bundles are both supported, but every request a Tier 3 +plugin makes goes through `ctx.session`, so the runner — not the plugin — +enforces policy: + +- **Declared, enforced budget.** `vcfl:maxRequestsPerSecond`, + `vcfl:maxRequestsPerRun`, `vcfl:batchSize`, and `vcfl:contactEmail` are + manifest fields. Exceeding the per-run ceiling aborts that linker with a clear + error rather than degrading into an unattended crawl. +- **Per-host token bucket, shared across plugins.** Two linkers that both target + `rest.ensembl.org` share one rate budget; a service sees one client, not two. +- **Mandatory on-disk response cache**, keyed by + `(linker id, linker version, request hash)`. On a cohort VCF the hit rate is + the entire performance story. +- **Deduplication before dispatch.** The full run's key set is deduplicated + before a single request is issued. Five million variants routinely reduce to a + few hundred thousand distinct rsIDs. +- **Backoff that respects the service.** `Retry-After` honoured, exponential + backoff on 429/5xx, a hard concurrency cap, and a `User-Agent` carrying the + tool version and the declared contact address. +- **`--offline` and `--links-cache-only`.** Reproducing a published run must not + require the service to still be up, and must not silently pick up an answer + that has since changed. +- **Full accounting in the run manifest.** Requests issued, cache hits, bytes + transferred, wall time, and final service status, per linker — using the + existing `write_run_manifest` and `RunTracker` plumbing. + +**An honest scale caveat.** Tier 3 linking is a different order of operation +from the rest of the pipeline: the conversion is bounded by local I/O, a live +API is bounded by someone else's rate limit. Tier 3 is appropriate for filtered +or prioritized variant sets. Anything genome-wide should use a Tier 2 bundle, +and the documentation should say so rather than let users discover it after +eight hours. + +--- + +## 5. Output and provenance + +Links are written to a **side-graph**, `.links.nt`, next to the main +aggregate — not merged into it by default. Three reasons: + +1. The validation suite's predicate census, class census and identity digests + (`q09`–`q13`) are defined against what the VCF implies. Injecting external + triples into the aggregate would make every one of them report a false + positive. +2. Users can adopt links without changing the graph they already query. +3. A linkset can be regenerated, versioned, or discarded independently of an + expensive conversion. + +Each linkset carries its own provenance node: + +```turtle + + a vcfl:Linkset ; + vcfl:producedBy ; + vcfl:referenceDigest "sha256:e3b0c442..." ; + vcfl:assembly "GRCh38" ; + vcfl:linkCount 41233 ; + prov:generatedAtTime "2026-09-04T11:04:22Z"^^xsd:dateTime . +``` + +`--merge-links` folds the side-graph into the aggregate for consumers who want +one file, at which point the mapping-policy fallback already used for custom +`--rules` applies and `q09`–`q13` become report-only. + +**On predicate choice.** `owl:sameAs` between a VCF record and a dbSNP resource +is a much stronger claim than most users intend — it licenses inferring every +dbSNP property onto the VCF record and vice versa. The default vocabulary should +therefore be explicit and weak: `vcfl:sameVariantAs`, `vcfl:overlapsGene`, +`vcfl:hasAnnotation`. A manifest can override it, deliberately. + +--- + +## 6. Authoring tooling + +`vcf-rdfizer-link`, mirroring the existing `vcf-rdfizer-rules` CLI: + +| Command | Purpose | +| --- | --- | +| `vcf-rdfizer-link list` | Installed linkers, versions, tiers, and declared references | +| `vcf-rdfizer-link keys` | The join keys a manifest may reference — the analogue of `rules columns` | +| `vcf-rdfizer-link init -o my-linker/` | Scaffold a manifest with annotated examples | +| `vcf-rdfizer-link check my-linker/` | Validate the manifest, reference digest, assembly, and budget before a long run | +| `vcf-rdfizer-link dry-run my-linker/ -i sample.vcf` | Run over the first N records, write nothing, report the links it would emit and the requests it would issue | + +`dry-run` is the one that earns its keep: it is what stops an author from +discovering a two-million-request mistake at hour three of a run. + +--- + +## 7. Validation of linksets + +A linker that ships `queries/*.rq` and `mutations.py` is picked up by the same +discovery already used for `src/validation/queries/`, so its links are +mutation-scored exactly like the core graph — see +[`validation-methodology.md`](validation-methodology.md). + +Linkset-specific checks worth making standard: + +- every link subject resolves to a subject that exists in the base graph; +- every link object is a syntactically valid absolute IRI; +- no blank nodes and no empty terms (the existing preflights apply unchanged); +- the declared `vcfl:linkCount` matches the emitted triple count; +- the reference digest recorded in the linkset matches the bundle actually used. + +Holding third-party linkers to the same mutation-testing bar as the core +conversion is a genuinely defensible claim, and it is only available because +the validation harness was built to be extended. + +--- + +## 8. Build order + +| Step | Delivers | Unlocks | +| --- | --- | --- | +| 1 | `vcfl:` vocabulary, manifest parsing, `token` join, Tier 1 templates | rsID linking with no network and no plugin code | +| 2 | Discovery (path + entry points), `vcf-rdfizer-link` CLI, side-graph + provenance, manifest metrics | The third-party extension contract | +| 3 | `interval` join, reference fetch/cache/digest, assembly guard | Gene and region linking | +| 4 | `allele` join and allele normalization | ClinVar, gnomAD, CADD from bundles | +| 5 | `ctx.session` with the full safeguard stack | Tier 3 live services | + +Steps 1–2 are the contract; everything after is capability. The manifest +vocabulary should be frozen and versioned at the end of step 2. + +--- + +## 9. Known hard problems + +Stated up front rather than discovered later. + +**Allele normalization is the real correctness risk.** Left-alignment, +trimming, and multi-allelic splitting must match whatever the reference bundle +did, or the join silently under-matches. Under-matching looks like "this variant +has no ClinVar entry", which is indistinguishable from a true negative. Step 4 +should not ship without a normalization test set with known-answer cases. + +**"Medically relevant" is a curation claim, not a data fact.** A gene-relevance +linker should link to a *named, versioned panel* — and say which — rather than +asserting clinical relevance in VCF-RDFizer's own voice. The tool is not a +clinical authority and its output should not imply that it is. + +**Assembly mismatch is silent.** Hence the declared assembly and the hard +refusal in §2. There is no way to detect it after the fact from the links alone. + +**External identifiers drift.** rsIDs are merged and retired; ClinVar +significance is revised. A link is true as of the reference version recorded in +the linkset, which is exactly why the provenance node in §5 is mandatory rather +than optional. + +**Licensing is the plugin author's problem, and must be visible.** Some +reference resources are not redistributable and some APIs forbid bulk use. The +manifest should carry a `vcfl:license` and `vcfl:termsOfUse` that +`vcf-rdfizer-link list` prints, so a user can see what they are agreeing to +before a run rather than after a publication. + +--- + +## See also + +- [Architecture](architecture.md) — where a linking stage sits in the pipeline +- [Custom RML mappings](rml-mappings.md) — the existing extension point this design is modelled on +- [Validation methodology](validation-methodology.md) — the harness linksets would plug into +- [Roadmap](roadmap.md) — how this relates to the other planned work diff --git a/docs/limitations.md b/docs/limitations.md new file mode 100644 index 0000000..cf0dcb3 --- /dev/null +++ b/docs/limitations.md @@ -0,0 +1,220 @@ +# Limitations + +A single place that says what VCF-RDFizer cannot do, does badly, or does in a +way that will surprise you. Nothing here is hidden elsewhere in the +documentation — this page collects it so a prospective user can decide against +the tool without reading everything first. + +Each item says what it is, why it is that way, and whether it is fixable. + +--- + +## 1. Operational + +**Docker is mandatory.** There is no pure-Python path. The toolchain — Flink, +RMLStreamer, a Rust `hdtc`, `pycottas`, Comunica, QLever, `pyshacl`, `cyvcf2`, +`bcftools` — is pinned in one image, which is what makes results reproducible +across machines. The cost is a large image, a Docker daemon requirement, and no +usable story on a cluster that only offers Singularity/Apptainer. +*Fixable in principle; not planned.* + +**The Docker data volume, not `--out`, is the binding disk constraint.** +Partitioned merges and HDT indexing perform disk-backed external sorts under the +container's `/work`. Free space in the output filesystem does not help, and the +resulting failure surfaces as `exit_code=-9` / `137` — an OOM kill — rather than +as a disk-space message. This is the single most common cause of a failed +cohort-scale run. + +**Single machine, single process per input.** Inputs are processed one at a +time. `--spark-partitions` tunes RMLStreamer's internal parallelism but there is +no distributed execution and no work queue. + +**No incremental update.** Adding variants to a converted dataset means +reconverting the VCF and rebuilding every representation from scratch. + +**Interrupt cleanup is best-effort.** `Ctrl+C` exits 130 and removes tracked +intermediates, but a `SIGKILL` or a host crash can leave a partial output +directory that the collision check will then refuse to write into. + +## 2. Input handling + +**Only `*.vcf` and `*.vcf.gz`.** Extension decides, not content. `.bcf` is not +supported; neither is `.vcf.bgz` or a differently named gzip stream. A +directory input is enumerated one level deep, sorted, once, at run start. + +**The VCF parser is `awk`, not `htslib`.** [`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) +makes one pass and splits on tabs. Consequences: + +- The `#CHROM` line is only recognised when tab-delimited. A space-delimited + header line matches no rule, so sample column names are lost and the records + header silently falls back to `SAMPLES`. +- Nothing validates VCF spec conformance. A malformed file yields a malformed + graph rather than an error; the validation suite is the first thing that + notices. +- A data line with fewer than eight columns produces empty fields, not an error. +- Sample fields are whitespace-normalized, so a value containing a literal space + would be corrupted. The specification forbids that, so it is only reachable + with an already-invalid file — but it is undetected. + +**IRIs are minted from the filename, not the path.** Two different VCFs named +`data.vcf` produce identical subject IRIs and their graphs collide when merged. +Rename before converting, or keep the graphs apart. + +## 3. What the RDF does and does not model + +**No variant normalization.** No left-alignment, no trimming, no multi-allelic +splitting. `ALT=A,T` stays one record with one `alt` literal. This is deliberate +— the graph is a faithful transcription — but it means the graph is not directly +joinable with normalized external resources, which is the central problem the +[data-linking design](datalinking-design.md) has to solve. + +**No reference checking.** `REF` is never verified against a genome, and the +declared assembly is recorded but not used for anything. + +**Structural variants are literals.** Symbolic ALTs (``), breakends and `*` +are carried as strings. The validation suite classifies them +(`SYMBOLIC_OR_BREAKEND`) but the vocabulary gives them no structure, and `END` / +`SVTYPE` carry no special meaning. + +**Multi-valued INFO fields keep only their lexical value.** A field declared +`Number=A/R/G/.` gets no typed per-value nodes, because the vocabulary's IRI +template gives one node per key rather than per value. This is a modelling gap +in the vocabulary, not a bug in the conversion. + +**Genotypes are lexical.** Phasing is preserved inside the literal but not +modelled; ploidy is not interpreted. + +**Triples only.** No named graphs anywhere in the pipeline, and no blank nodes — +a blank node is treated as a validation failure, because every class in the +vocabulary declares an IRI template. + +**No cross-file merging.** Each VCF produces its own graph. + +## 4. The custom-mapping extension point + +**`--rules` does not control the whole graph.** Three families of triple — +genotypes, structured headers, and QUAL/structured INFO — are emitted by the +wrapper rather than by RML, because RML cannot choose a datatype or class per +row and the alternatives require materializing enormous helper tables. See +[`architecture.md`](architecture.md#4-where-the-split-leaks-and-why). They can be +narrowed with `--sample-representation` / `--header-representation` / +`--info-representation`, but not replaced by a mapping. + +**A custom mapping is validated less thoroughly.** Queries `q09`–`q13` assume +the shipped mapping's predicate inventory and IRI templates. + +**And, currently, it is validated incorrectly.** `validation_runner.py` supports +`--mapping-policy report-only` for exactly this case, but the wrapper never +forwards it, so a custom mapping run through `vcf-rdfizer --validate` reports +`MISMATCH` on those five queries even when the conversion is correct. +*Fixable; tracked in [`roadmap.md`](roadmap.md).* + +**`vcf-rdfizer-rules check` is lexical, not semantic.** It catches wrong logical +-source paths and misspelled columns — the two mistakes that waste the most time +— and nothing subtler. A mapping that passes `check` can still be wrong. + +## 5. Compression and representations + +**Packaged artifacts are not queryable.** `.hdt.gz`, `.hdt.br`, `.cottas.gz` and +`.cottas.br` are archives. This is easy to forget when +`--remove-rdf-storage-output` has already removed the alternative. + +**COTTAS is the more fragile path.** Its upstream `cat` cannot handle large +condensed graphs, which is why VCF-RDFizer implements its own bounded k-way +merge. Even so, a memory-constrained host may need a reduced +`COTTAS_MERGE_BATCH_ROWS`, and `--representations hdt` remains the independent +fallback. + +**The round-trip check counts, it does not compare.** Matching triple counts +prove an artifact decodes and holds the right *number* of statements — not that +they are the right statements. The stronger claim requires +`--validate-artifacts hdt,cottas`. + +**A degraded HDT index is a success, not a failure.** In full mode an HDT whose +data is readable but whose sidecar could not be built is published with +`index_status: "failed"` and a warning. That is intentional, but it means a +successful run can leave a non-indexed artifact. + +## 6. Validation + +The validation suite has its own detailed limits in +[`validation.md`](validation.md#what-is-not-tested) and +[`vcf-coverage.md`](vcf-coverage.md#remaining-gaps). The headline items: + +**A `PASS` is a regression gate, not a correctness proof.** It reliably catches +dropped records, misclassified variants, flipped genotypes, corrupted FILTER +strings, allele-count errors, missing header lines and altered file metadata. It +is not proof of a faithful record-by-record round-trip. + +**Coverage is relative to the mutation catalogue.** The score says "almost every +corruption we thought to write down is caught". It is a lower bound on +blindness, not a measure of correctness, and a score that rises without the +catalogue growing means nothing. + +**`vcfr:contigCount` is counted, not read**, so a wrong derived contig total is +undetected. **Header line values are not compared** — only how many lines carry +each key, their types, and the structured attributes lifted out of them. + +**SHACL is opt-in and does not scale.** `pyshacl` loads the whole graph into +memory, so it is for a single-sample graph or a sample of a cohort. + +**QLever's argv is a moving target.** Its CLI has changed across releases; the +`QLEVER_*_COMMAND` environment overrides exist because of that, and the exact +argv is recorded in every report so a future divergence is diagnosable. + +## 7. Vocabulary + +**Condensed graphs are not ontology-backed.** The tool emits 17 terms that +`https://w3id.org/vcf-rdfizer/vocab#` does not define, all belonging to the +condensed representation. Dereferencing any of them returns nothing. This is the +one remaining publication blocker, and the work is in the vocabulary repository. + +**The published SHACL shapes contradict the missing-value policy.** +`vcfr:missingValuePolicy` says a missing token should be `"."^^vcfr:Null`, while +`VCFRecordShape` constrains `vcfr:alt` to `sh:datatype xsd:string`. A record with +`ALT=.` cannot satisfy both. The conversion follows the missing-value policy. +The vocabulary needs a decision; see +[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). + +**Condensed mode has no query-time decoder.** Reconstructing sample *i*'s value +means splitting a tab-separated literal, which SPARQL cannot do portably. The +options are assessed in +[`sample-representation-guide.md`](sample-representation-guide.md#7-assessment-of-geosparql-and-graphdb-sparql-extensions); +none is currently implemented. + +## 8. Scope + +**No data linking yet.** The graph is self-contained and connects to nothing +external. This is by design so far, and the plan to change it is +[`datalinking-design.md`](datalinking-design.md). + +**No disclosure control.** Conversion is all-or-nothing: every sample, every +genotype, every header line and every free-text `Description` goes into the +graph, and there is no way to withhold a participant, degrade a region, or +record what an artifact was permitted to contain. Three consequences today: + +- **IRIs carry identifiers.** `file://cohort.vcf#sample/1/NA12878` embeds the + sample name, `{SOURCE_FILE}` embeds the VCF's basename, and `#record/{ROW_ID}` + is a monotonic counter that discloses source ordering. +- **Header lines are a leak surface.** `##source`, `##SAMPLE`, `##PEDIGREE` and + free-text `Description` fields are transcribed verbatim into + `vcfr:headerValue`. +- **Even if that were fixed, genotypes identify people.** A few dozen + independent common variants are enough to single out an individual, so no + amount of label removal makes a released genotype graph non-identifying. + +The plan is [`privacy-policy-design.md`](privacy-policy-design.md), which is +explicit that what it offers is *governed release*, not anonymization. + +**No clinical claims.** The tool transcribes a VCF. It does not interpret, +annotate, prioritize, or assess pathogenicity, and its output should not be +presented as if it did. + +--- + +## See also + +- [Roadmap](roadmap.md) — which of these are being addressed +- [Privacy policy design](privacy-policy-design.md) — the disclosure-control gap, and the proposal to close it +- [VCF coverage matrix](vcf-coverage.md) — the element-by-element measurement +- [Validation](validation.md) — the detailed "what is not tested" diff --git a/docs/output-and-metrics.md b/docs/output-and-metrics.md new file mode 100644 index 0000000..0fcccb4 --- /dev/null +++ b/docs/output-and-metrics.md @@ -0,0 +1,187 @@ +# Output layout, metrics, and run reports + +Every VCF-RDFizer run writes three kinds of thing: the artifacts you asked for, +a complete record of how they were produced, and evidence about what was +cleaned up. This document is the map. + +--- + +## 1. Output layout + +`--out` is required for every mode. It is the run output **root**, and +everything lives beneath it: + +```text +/ + / final artifacts + .nt | .nt.gz | .nt.br + .hdt + .hdt.index.v1-1 + .cottas + .hdt.gz | .cottas.br | ... + run_metrics/__/ per-run reports and logs + .intermediate/tsv/ hidden intermediates + decompressed/ --mode decompress default target +``` + +The directory name and every artifact basename come from the source filename +with its recognized VCF/RDF/representation suffix removed. `--mode compress` on +`test-larger.nt` and on `test-larger.nt.gz` therefore both write into +`/test-larger/`. + +**Collision policy.** Before Docker starts, the wrapper computes every artifact +the run intends to write and fails if any already exists. It never overwrites a +planned pipeline artifact. Choose a new `--out`, or move the conflicting file. +`--mode index` is the single deliberate exception, because regenerating an index +in place is its entire purpose. + +**Intermediate cleanup.** TSV intermediates are hidden and removed unless +`--keep-tsv`. Raw RDF is removed after successful compression by default; +`--remove-rdf-storage-output` makes that explicit and +`--keep-rmlstreamer-rdf-output` prevents it. Under `space-optimized`, the +`.nt.gz` aggregate is retained when `gzip` is a selected RDF codec, because that +file *is* the gzip artifact. + +When `--validate` fails for an input, raw RDF cleanup is **skipped** for that +input and the aggregate is kept with the note `retained after validation +failure`, so the graph that failed is still available to inspect. + +## 2. The run metrics directory + +```text +run_metrics/__/ +``` + +`` is the source filename without its suffix (for example +`1000G_phase3_chr20`); a multi-file directory input uses a batch label such as +`batch-vcf_data-4-inputs`. The point is that a metrics directory is +recognisable without opening a timestamp-named folder. + +| Path | Contents | +| --- | --- | +| `run.json` | Source identity, resolved input paths, requested configuration, image selection | +| `summary.json` | Final status, wrapper wall time, summary rows, and an index of every stage report and log | +| `metrics.csv` | Analysis-ready per-output table | +| `tsv_metrics.csv` | TSV-mode benchmark table | +| `wrapper_execution_times.csv` | Host-side stage timings | +| `logs/wrapper.log`, `logs/progress.log` | Command log and progress history | +| `timings//…` | Raw GNU `time -v` output from inside the relevant container | +| `stages/tsv/`, `stages/conversion/`, `stages/compression/`, `stages/compression_operations/`, `stages/decompression/`, `stages/index/`, `stages/validation/` | Structured per-stage results | +| `stages/partitioned/.json` | Full handoff from the partitioned-compression container | +| `reports/index_warnings.json` | Degraded-index events | +| `reports/failed_inputs.csv` | Inputs abandoned during a multi-file run | +| `reports/validation//` | Detailed semantic-validation reports | + +`compression_operations/` preserves the per-RDF operation and round-trip +validation reports; `compression/` is the final output-level summary over them. + +`stages/partitioned/.json` is the one to open when a large run fails. +It retains every chunk build, the merge strategy and round count, validation +results, the generated chunk plan, workspace free-space samples, CPU time, peak +RSS, exit codes, and bounded `stderr` diagnostics — **and it survives the +deletion of the temporary Docker volume**, which is what makes a failed +cohort-scale run diagnosable at all. + +## 3. Input size accounting + +`input_vcf_size_bytes` in `stages/conversion/*.json` and `metrics.csv` is the +**uncompressed** size of the source VCF, so ratios are comparable between plain +and compressed inputs. `input_vcf_size_method` records how it was obtained: + +| Method | Meaning | +| --- | --- | +| `stat` | Input was not compressed; the on-disk size is the answer | +| `bgzf` | Exact, summed from BGZF block headers with no decompression | +| `gzip-sample` | Exact; the whole single-member stream fitted in the sampling budget | +| `gzip-trailer` | Exact; the 32-bit `ISIZE` trailer resolved against the measured compression ratio | +| `inflate` / `inflate-shell` | Fallback full decompression pass | + +Only the fallback costs a full pass. Indexed `.vcf.gz` files from +`bcftools`/`tabix`/`htslib` are BGZF, so the usual case is measured in +milliseconds. The same machinery makes `--estimate-size` report a real +uncompressed size rather than an assumed expansion factor — and it says so when +it had to fall back to the assumption. + +## 4. Compression metrics + +Per method, `metrics.csv` carries `wall_seconds_*`, `user_seconds_*`, +`sys_seconds_*`, and `max_rss_kb_*`, plus `source_triples` / `decoded_triples` +from the round-trip check described in +[`representations.md`](representations.md#6-round-trip-verification). + +Metrics may use internal stage names such as `hdt_gzip` and `cottas_brotli`. +These correspond to the public combination of `--representations` and +`--artifact-compression`; nothing on the command line uses those compound names. + +For partitioned runs, the method-level metric reports one sample-level result +while `stages/partitioned/` retains the full history. + +## 5. Validation reports + +Detailed results live at: + +```text +/run_metrics/__/reports/validation// +``` + +with a stage-level summary at `stages/validation/.json`. When +several artifacts are validated in one run, the aggregate keeps +`/` and the representations use `__hdt/` and +`__cottas/`. + +Key files: `summary.json`, `manifest.json`, `parser.json`, +`rdf-validation.json`, `materialization.json`, `preflight.json`, `sparql.json`, +`comparison.json`. Raw SPARQL Results JSON, stderr and query resource logs are +in `raw/`; normalized results in `normalized/`. + +`manifest.json` records the engine that ran (including QLever's exact argv), the +source artifact's format and checksum, and every decode step. +`materialization.json` records how a compressed artifact was turned into +N-Triples and how many triples that yielded. Both the stage report and +`summary.json` record whether a gzip aggregate was decompressed inside the +container and that no validation scratch RDF was retained on the host. + +Statuses: `PASS`, `MISMATCH` (both paths ran and differ), +`BLOCKED_BY_PREFLIGHT` (syntax or core graph structure failed), +`EXECUTION_FAILED` (a parser or engine could not complete). + +Full treatment in [`validation.md`](validation.md). + +## 6. Progress + +Containers write newline-delimited JSON to a small sidecar under the run's +temporary `.progress/` area; the host polls it while the container runs and +renders it with Rich when attached to an interactive terminal, or as compact +status lines otherwise. Command logs and binary subprocess output stay separate +from the terminal UI. + +| Flag | Effect | +| --- | --- | +| *(none)* | Sidecar written, terminal progress rendered | +| `--quiet` | Sidecar, command log and metrics retained; **all** terminal rendering and the validator's per-query chatter suppressed | +| `--no-progress` | Sidecar creation disabled as well | + +Progress is best-effort: RMLStreamer reports bytes and output parts already +written, partitioned runs report source triples, chunks, and the active +merge/index stage. Nothing rescans RDF content a second time to produce a +number, and no progress history is retained in memory. + +## 7. Interrupts and exit codes + +`Ctrl+C` exits with **130**, writes progress to `logs/progress.log`, and +performs best-effort cleanup of tracked intermediates. Raw RDF cleanup on +interrupt follows `--keep-rmlstreamer-rdf-output`: with it, raw RDF is +preserved; without it, tracked raw RDF files are removed. + +Otherwise the wrapper exits `0` on success and non-zero on failure. In full mode +with `--validate`, a validation failure sets a non-zero exit even when the +conversion itself succeeded, so a run can be gated in CI on semantic +correctness rather than on Docker having returned. + +--- + +## See also + +- [Architecture](architecture.md) — which process writes which report +- [Representations](representations.md) — what the compression metrics describe +- [Validation](validation.md) — reading a validation report +- [CLI reference](cli-reference.md) — the flags referenced above diff --git a/docs/privacy-policy-design.md b/docs/privacy-policy-design.md new file mode 100644 index 0000000..7fe6128 --- /dev/null +++ b/docs/privacy-policy-design.md @@ -0,0 +1,595 @@ +# Granular privacy policies over a VCF graph + +*Status: **design proposal**. Nothing described here is implemented yet. Like +[`datalinking-design.md`](datalinking-design.md), this document exists to fix a +contract before code depends on it — and, in this case, to be explicit about +what a policy layer can and cannot achieve, because the failure mode of a +privacy feature is a false sense of safety.* + +VCF-RDFizer currently makes one decision about disclosure: it converts +everything. Every sample, every genotype, every header line, every free-text +`Description` goes into the graph, and every artifact is all-or-nothing. The +only granularity available today is "convert a different VCF". + +What is wanted is the ability to say, in a machine-readable and auditable way: +*these samples, under this purpose, may see these regions at this resolution* — +and to have the tool produce artifacts that actually honour it. + +--- + +## 1. The uncomfortable premise + +**Genotypes are identifiers.** A few dozen independent common variants are +enough to pick one individual out of a population (Lin, Owen & Altman, +*Science* 2004), aggregate allele frequencies alone can reveal whether a known +individual was in a cohort (Homer et al., *PLoS Genetics* 2008), and Y-chromosome +haplotypes have been used to recover surnames from public genealogy databases +(Gymrek et al., *Science* 2013). + +Three consequences follow, and they shape everything below: + +1. **Removing `vcfr:sampleId` does not anonymize anything.** The genotype vector + that remains is a stronger identifier than the label you deleted. +2. **Access control is not anonymization.** A policy layer governs *who is + permitted to receive what*, and creates an audit trail. It does not make the + released data non-identifying, and nothing in this design should be described + as if it did. +3. **The artifact is the enforcement boundary.** Once someone holds a `.hdt` + file, no policy engine is in the loop. Query-time enforcement only exists + while you control the endpoint. + +This document therefore proposes a system for **governed release**, not for +anonymity. Where a mechanism reduces re-identification risk (thresholding, +generalization), it says by how much and under what assumption. + +--- + +## 2. What "a portion of the VCF graph" means + +The graph has a small number of natural cut planes, and they map cleanly onto +the shapes the conversion already emits (see +[`conversion.md`](conversion.md#6-iri-templates)): + +| Cut | Addresses | Example | +| --- | --- | --- | +| **By class** | all resources of a type | every `vcfr:SampleCall` | +| **By predicate** | all triples with a predicate | every `vcfr:sampleName` | +| **By sample** | one participant's contribution | everything hanging off `#samples/NA12878` | +| **By region** | a genomic interval | `chr19:44,905,791-44,909,393` (*APOE*) | +| **By declared field** | one INFO or FORMAT key | `DP` yes, `GT` no | +| **By record predicate** | an arbitrary graph pattern | records where `FILTER != PASS` | +| **By granularity** | not a subgraph at all | counts permitted, individual calls not | + +The last row is the one that access control alone cannot express, and it is the +one that matters most for the attacks in §1. It is handled in §9. + +**A note on scope.** Two families of triple leak more than people expect and are +easy to forget when thinking in terms of genotypes: + +- **IRIs themselves.** `file://cohort.vcf#sample/1/NA12878` contains the sample + name. `#record/{ROW_ID}` is a monotonic counter, so row identifiers disclose + the source ordering — and therefore approximate genomic position — even if + `vcfr:pos` is dropped. And `{SOURCE_FILE}` is the VCF's basename, which in + practice is often `patient_12345.vcf`. Filtering triples while leaving IRIs + intact leaks membership. See §7. +- **Header lines.** `vcfr:headerValue` and the raw line text carry `##source` + (pipeline and centre identifiers), `##SAMPLE` and `##PEDIGREE` (family + structure, by design), free-text `Description` fields, and `##fileDate`. + A policy that covers genotypes and ignores the header section has not covered + the graph. + +--- + +## 3. Is ODRL the right choice? + +Yes for the policy layer, no as the whole answer. Being precise about the gap is +what makes the rest of the design tractable. + +**What ODRL gives you, that is genuinely hard to get elsewhere:** + +- A W3C Recommendation, so a policy is a citable, interoperable artifact rather + than a bespoke config file. +- RDF-native: the policy lives in the same graph model as the data, can be + published, dereferenced, versioned and signed. +- The vocabulary the problem actually needs — `odrl:permission` / + `odrl:prohibition` / `odrl:duty`, `odrl:assignee`, `odrl:constraint` with + `odrl:purpose` and `odrl:recipient` left operands. +- A **profile mechanism** designed for exactly this situation: extend the core + with domain terms without forking it. + +**What ODRL does not give you, and must be supplied:** + +| Gap | Consequence | +| --- | --- | +| `odrl:target` points at an *Asset* (an IRI). There is no notion of "these triples". | A selector vocabulary is required. This is the core technical work. | +| Evaluation semantics in the ODRL Information Model are **informative**, not normative. | Conflict resolution and rule ordering must be pinned by the profile, or two implementations will disagree. | +| Rules permit or prohibit. They cannot say *how* to degrade. | An effect/transform vocabulary is required (drop, generalize, threshold, pseudonymize). | +| No enforcement engine exists. | Enforcement is ours to build; ODRL is the declaration, not the mechanism. | + +**Alternatives considered:** + +| Option | Verdict | +| --- | --- | +| SHACL shapes as policy | Excellent at *describing* a permitted shape, and worth reusing to **verify** a release (§11) — but it is a validation language with no notion of party, purpose or duty. Not a policy language. | +| Solid WAC / ACP | Resource-granular. A VCF graph is one resource; the whole problem is sub-resource granularity. | +| XACML | Mature and battle-tested, but not RDF-native, heavyweight, and it would put the policy outside the data ecosystem the rest of the tool lives in. | +| Bare SPARQL views | Simplest possible enforcement, and in fact what the compiler emits — but as a *policy* it records no party, purpose, obligation or provenance, so it cannot be audited or presented to a data access committee. | + +**Recommended shape:** ODRL as the front end, a VCF-RDFizer profile supplying +selectors and effects, compiled down to a plain internal **release plan**. Keep +the enforcement layer independent of ODRL so a second front end (a DUO-only +consent file, say) can be added without touching the engine. + +--- + +## 4. The profile + +Namespace `vcfp:` = `https://w3id.org/vcf-rdfizer/policy#`, declared through +`odrl:profile`. + +### 4.1 Assets are graph selections + +```turtle +@prefix odrl: . +@prefix vcfp: . +@prefix vcfr: . +@prefix duo: . + +<#apoe-locus> a odrl:Asset , vcfp:GraphSelection ; + vcfp:selector [ a vcfp:RegionSelector ; + vcfp:assembly "GRCh38" ; + vcfp:chrom "chr19" ; + vcfp:start 44905791 ; + vcfp:end 44909393 ] . + +<#direct-identifiers> a odrl:Asset , vcfp:GraphSelection ; + vcfp:selector [ a vcfp:PredicateSelector ; + vcfp:predicate vcfr:sampleName , vcfr:sampleId ] . + +<#withdrawn-participants> a odrl:Asset , vcfp:GraphSelection ; + vcfp:selector [ a vcfp:SampleSelector ; + vcfp:sampleId "NA12878" , "NA12891" ] . +``` + +| Selector | Keys | Notes | +| --- | --- | --- | +| `vcfp:ClassSelector` | `vcfp:class` | Every resource asserting that type, and its outbound triples | +| `vcfp:PredicateSelector` | `vcfp:predicate` | Triple-level; the cheapest to enforce | +| `vcfp:SampleSelector` | `vcfp:sampleId`, `vcfp:sampleIndex` | Resolves through `SampleSet` membership | +| `vcfp:RegionSelector` | `vcfp:assembly`, `vcfp:chrom`, `vcfp:start`, `vcfp:end` | **`vcfp:assembly` is mandatory** and checked against `##reference`; a mismatch aborts, for the same reason it does in the linking design | +| `vcfp:FieldSelector` | `vcfp:formatKey`, `vcfp:infoKey` | By declared field id | +| `vcfp:HeaderSelector` | `vcfp:headerKey` | `##SAMPLE`, `##PEDIGREE`, `##source`, … | +| `vcfp:PatternSelector` | `vcfp:ask` | A SPARQL graph pattern. The escape hatch; expensive, and excluded from the cheap enforcement tiers (§5) | + +Selectors compose with `vcfp:allOf` / `vcfp:anyOf` / `vcfp:not`. ODRL's own +`odrl:AssetCollection` with `odrl:refinement` is deliberately **not** reused +here: its refinement semantics are about collection membership, not about +sub-graph extents, and overloading it would produce policies that look standard +while meaning something non-standard. + +### 4.2 Effects: what a rule actually does + +A prohibition that can only mean "drop everything" is too blunt for genomic +data, where the useful answer is usually *less resolution*, not *nothing*. The +profile therefore refines `odrl:duty` with a transform: + +| `vcfp:transform` | Effect | +| --- | --- | +| `vcfp:drop` | Omit matched triples entirely (default for a prohibition) | +| `vcfp:pseudonymize` | Replace a term with a keyed, per-release token (§7) | +| `vcfp:generalize` | Reduce resolution: `POS` → 1 Mb window, `GT` → carrier / non-carrier, `DP` → banded | +| `vcfp:threshold` | Suppress a value below a count, e.g. allele count < 5 | +| `vcfp:aggregateOnly` | The subgraph is reachable only through a counting query (§9) | +| `vcfp:maskVectorPositions` | Condensed mode only: rewrite the tab-separated literal, replacing masked sample positions with `.` (§8) | + +```turtle +<#cohort-release-1.2> a odrl:Set ; + odrl:uid ; + odrl:profile ; + odrl:conflict odrl:prohibit ; + + # Hard exclusion: withdrawn consent. No purpose overrides this. + odrl:prohibition [ + odrl:target <#withdrawn-participants> ; + odrl:action odrl:read ; + odrl:assignee odrl:All ] ; + + # Degrade rather than deny: APOE is readable only as carrier status. + odrl:permission [ + odrl:target <#apoe-locus> ; + odrl:action odrl:read ; + odrl:duty [ odrl:action odrl:anonymize ; + vcfp:transform vcfp:generalize ; + vcfp:genotypeResolution vcfp:carrierStatus ] ] ; + + # Purpose-bound permission, expressed with GA4GH DUO codes. + odrl:permission [ + odrl:target <#genotypes> ; + odrl:action odrl:read ; + odrl:assignee ; + odrl:constraint [ odrl:leftOperand odrl:purpose ; + odrl:operator odrl:isAnyOf ; + odrl:rightOperand duo:DUO_0000007 ] ; + odrl:duty [ odrl:action odrl:inform ; + vcfp:auditSink ] ] . +``` + +### 4.3 Consent codes as purposes + +`odrl:purpose` right operands should be **GA4GH Data Use Ontology** terms rather +than free strings. DUO is what data access committees and repositories already +speak, so this makes a policy reviewable by the people who actually grant +access, and it means the tool is not inventing a consent vocabulary. + +Pin the DUO release in the policy (`vcfp:duoVersion`) and resolve term IRIs +against that release — DUO evolves, and a policy that silently re-interprets a +consent code is worse than one that fails to load. + +### 4.4 Conflict resolution, pinned + +Because ODRL leaves this informative, the profile **normatively** fixes it: + +1. `odrl:conflict odrl:prohibit` is the default and the only value recommended + for a release policy. Deny wins. +2. A rule with no applicable transform defaults to `vcfp:drop`. +3. Effects on the same selection compose most-restrictive-first + (`drop` > `aggregateOnly` > `threshold` > `generalize` > `pseudonymize`). +4. `odrl:conflict odrl:invalid` is supported for strict deployments: any + conflict voids the policy and the run aborts rather than guessing. +5. **Anything not matched by a permission is denied.** Default-deny is the only + defensible posture; a default-allow policy language for genomic release is a + trap. + +--- + +## 5. Where enforcement happens + +Three tiers, in increasing cost, falling directly out of the existing pipeline +([`architecture.md`](architecture.md#5-data-flow-in-full-mode)). The compiler +assigns each rule to the cheapest tier that can express it. + +### Tier 1 — pre-RML, on the TSV + +Row and column removal applied to `.records.tsv` and +`.header_lines.tsv` **before RMLStreamer runs**. + +This is by far the cheapest option and the only one that covers RML-produced +triples as well as wrapper-produced ones, because nothing downstream ever sees +the excluded data. It handles `SampleSelector` (drop a sample column), +`RegionSelector` (drop rows — CHROM and POS are columns 3 and 4, right there), +and `HeaderSelector` (drop header rows). + +Its limit is granularity: it removes whole rows and columns, so it cannot +express "keep this record but drop its `DP`". + +### Tier 2 — emission time, in the wrapper's emitters + +The three direct emitters +([`architecture.md`](architecture.md#4-where-the-split-leaks-and-why)) hold the +full parsed record while they emit, so a predicate-, field- or value-level rule +costs one branch per emitted triple and needs no second pass. + +Concrete hook points in [`vcf_rdfizer.py`](../vcf_rdfizer.py): + +| Where | Change | +| --- | --- | +| `_append_rdf_atomically(rdf_path, stats, producer)` | Wrap `emit` in a policy filter; the atomic-append and rollback behaviour is unchanged | +| `append_expanded_sample_rdf`, `append_condensed_sample_rdf` | Sample- and field-level rules; the condensed one also needs §8 | +| `append_record_detail_rdf` | QUAL and INFO field rules | +| `append_header_representation_rdf` | Header rules | + +**One small prerequisite.** `ParsedSampleRecord` currently carries +`source_file`, `row_id`, `qual`, `info`, `format_keys`, `sample_payloads` and +`sample_values` — `_parse_row` reads columns 0, 1, 7, 9, 10 and −1, and **skips +CHROM and POS** (columns 2 and 3). A `RegionSelector` evaluated at emission time +needs both. That is two extra fields on the dataclass and two extra index reads +in `_parse_row`, with no measurable cost, and it should be the first commit in +this work. + +### Tier 3 — post-hoc, over an existing artifact + +For applying a policy to a graph that has already been converted +(`--mode redact --rdf .nt.gz --policy

.ttl`). + +Triple- and predicate-level rules are a single streaming pass. **Region rules +are not**, because an N-Triples stream has no ordering guarantee: by the time +you see `<…#sample/4711/NA12878> vcfr:fieldValue "0/1"`, the triple that told +you record 4711 is at `chr19:44906000` may be long gone. This needs two passes: +pass one builds a `row_id → (chrom, pos)` map restricted to records that fall in +a policy region, pass two filters. Memory scales with the number of **in-scope** +records, not the graph, which keeps it bounded in practice — but it is +materially more expensive than doing the same work in Tier 1 or 2, and the +documentation should say so rather than let users discover it. + +### Query time, and why it is secondary + +A policy can also be compiled to SPARQL rewriting at an endpoint. It is worth +building, but it is **not** the primary enforcement point for this tool, for a +reason worth stating plainly: VCF-RDFizer's principal output is *files that +people take away*. An HDT or COTTAS artifact handed to a collaborator is outside +any policy engine forever. Query-time enforcement protects an endpoint you +operate; materialization protects a file you ship. Most users of this tool need +the second. + +Where an endpoint *is* operated, note also that rewriting is easy to get subtly +wrong: a prohibition enforced with `FILTER NOT EXISTS` still lets a caller infer +the excluded set from counts and negative results. Query-time enforcement should +be paired with the aggregate controls in §9, not treated as sufficient alone. + +--- + +## 6. The compile step + +```text + ODRL policy (.ttl) + │ parse, validate against the profile, resolve DUO version + ▼ + rule set ──▶ conflict resolution (§4.4) ──▶ release plan + │ + ├── Tier 1 ops: TSV row/column predicates + ├── Tier 2 ops: per-triple predicates, keyed by emitter + ├── Tier 3 ops: post-hoc stream filters (+ any two-pass requirements) + └── residual: rules no tier can enforce ──▶ ABORT +``` + +The **residual set must abort the run, never warn**. A privacy rule that was +parsed, reported, and then not applied is the single worst outcome this design +can produce: the operator believes the release is governed and it is not. If the +compiler cannot enforce a rule, it must refuse to produce an artifact. + +The release plan is a plain data structure with no ODRL dependency, which keeps +the engine testable in isolation and leaves room for a second front end. + +--- + +## 7. Pseudonymization and IRI re-minting + +Because IRIs embed sample names, the source filename and a positional row +counter (§2), filtering triples is not enough. Under `vcfp:pseudonymize` the +release view must re-mint IRIs. + +**Construction.** `token = base32(HMAC-SHA256(release_key, namespace ‖ value))`, +truncated to a documented length, where `namespace` distinguishes sample names +from row ids from filenames so the same string in two roles does not produce the +same token. + +**Properties this gives, and the ones it does not:** + +- Stable *within* a release, so joins inside the released graph still work. +- Unlinkable *across* releases, because `release_key` is fresh per release — + unless linkage is deliberately wanted, in which case a named, reused key is an + explicit policy choice recorded in the manifest. +- The key is **never** written into the artifact or the manifest; only its + identifier and algorithm are. Re-identification remains possible for whoever + holds the key, which is the point — this is pseudonymization, not + anonymization, and the manifest should use that word. +- It does **not** defeat genotype-based re-identification (§1). Nothing here + does. + +Row-id re-minting deserves specific attention: replacing `#record/4711` with a +token removes the ordering leak, but only if the tokens are emitted in an order +that does not reconstruct it. Sort the released graph by token, not by source +order. + +--- + +## 8. The condensed-representation problem + +This one is specific to VCF-RDFizer and easy to miss until it produces a leak. + +In **expanded** mode, one participant's value is its own resource: + +```text +<…#sample/4711/NA12878/fmt/GT> vcfr:fieldValue "0/1" . +``` + +Excluding a sample is triple filtering. Straightforward. + +In **condensed** mode, all participants' values for one FORMAT key live in +**one literal**: + +```text +<…#call/4711/matrix/fmt/GT> vcfr:encodedValues "0/1\t0/0\t1/1\t./."^^vcfr:VCFTextVector . +``` + +There is no triple to remove for one sample. Graph-pattern access control cannot +express "sample 2 only" over this shape at all — the unit of protection is +finer than the unit of storage. + +Two consequences: + +1. **Redaction must rewrite the literal.** `vcfp:maskVectorPositions` replaces + the masked participant's token with `.`, which is the VCF missing marker and + keeps the vector aligned with `vcfr:sampleIndex` — alignment the format + depends on. Dropping a position instead would silently shift every downstream + sample's value, which is a data-corruption bug wearing a privacy feature's + clothing. +2. **Masking is itself visible.** A `.` where neighbours have values discloses + that a value existed and was withheld, and combined with `SampleSet` + membership it discloses *whose*. Where that matters, the policy must mask the + position across **all** samples for that record, or drop the vector entirely. + The compiler should detect a single-position mask and warn — or, under a + strict profile setting, refuse. + +A blunter option is worth offering: `vcfp:requireRepresentation expanded`, which +makes a policy refuse to run against a condensed graph. For policies with +per-sample rules that is often the honest answer, and it trades storage +efficiency for enforceability deliberately rather than by accident. + +--- + +## 9. Beyond access control: disclosure limitation + +Access control does not address §1's attacks, all of which operate on data the +recipient was *permitted* to see. Three mechanisms, in increasing ambition: + +**Count thresholds (`vcfp:threshold`).** Suppress a variant whose allele count +falls below *k*. Cheap, well understood, and directly targets the +rare-variant-as-fingerprint problem, since rare alleles carry most of the +identifying signal. Implementable in Tier 2 with a counting pre-pass. +Recommended default for any cohort release: suppress `AC < 5`, and say so in the +manifest. + +**Aggregate-only exposure (`vcfp:aggregateOnly`).** The subgraph is not +materialized; only counting queries over it are answerable. This is the Beacon +model, and it inherits the Beacon model's known weakness — repeated membership +queries leak — so it must be paired with query budgeting and audit, not offered +as a safe default. + +**Differential privacy.** Noise on aggregate counts is the only mechanism here +with a formal guarantee, and it is also the easiest to implement incorrectly: +a per-query epsilon with no global budget provides no protection at all against +a patient adversary. **Not proposed for implementation.** If it is added later +it needs a persistent budget ledger per recipient, and the design should say +plainly that without one it is decoration. + +--- + +## 10. The release manifest + +Every policy-derived artifact carries provenance, in the same spirit as the +linkset node in [`datalinking-design.md`](datalinking-design.md#5-output-and-provenance): + +```turtle + + a vcfp:ReleaseView ; + vcfp:derivedFrom ; + vcfp:policy ; + vcfp:policyDigest "sha256:9f2c…" ; + vcfp:duoVersion "2024-11-03" ; + vcfp:pseudonymKeyId "release-2026-09-key-3" ; + vcfp:representation vcfr:ExpandedRepresentation ; + vcfp:triplesWithheld 1840221 ; + vcfp:samplesWithheld 2 ; + vcfp:thresholdApplied 5 ; + prov:generatedAtTime "2026-09-04T11:04:22Z"^^xsd:dateTime . +``` + +Three requirements that are easy to get wrong: + +- **The policy is referenced by digest, not just by IRI.** A policy IRI whose + content changed later cannot explain an artifact produced last year. +- **A release view gets its own IRI namespace**, so it can never be silently + mistaken for, or merged with, the full graph. +- **Counts of what was withheld are published.** They are not themselves + sensitive at this granularity, and they are what makes a release auditable + rather than merely asserted. + +--- + +## 11. Verifying a release + +A privacy claim that is not checked is a privacy claim that is wrong. This is +where the existing validation harness earns its keep a second time. + +**Policy conformance checks.** Compile every prohibition into a SPARQL `ASK` (or +a `SELECT COUNT`) that must return zero over the released artifact, and run them +as preflights, exactly as `preflight_blank_nodes` and friends already work. A +release that fails one is not published. This turns "the APOE region was +excluded" from a claim into a measurement — the distinction +[`validation-methodology.md`](validation-methodology.md) is built around. + +**SHACL for shape-level guarantees.** "No `vcfr:sampleName` appears anywhere" is +naturally a shape constraint, and the `--shacl-shapes` layer already exists. + +**Mutation testing the redactor.** Add a mutation class to +[`test/validation_mutations.py`](../test/validation_mutations.py) that +deliberately breaks the redaction — reinstate a withheld sample, un-mask one +vector position, restore a dropped predicate — and assert the conformance checks +catch it. A redactor nobody has tried to defeat is an assumption, not evidence, +which is precisely the argument the validation methodology already makes about +the validator itself. + +**Adversarial checks worth writing early**, because they catch the leaks that +correct-looking implementations produce anyway: + +- Does any IRI in the released graph contain a withheld sample id? +- Do row identifiers reconstruct the source ordering? +- Is any masked vector position identifiable as masked-for-one-sample (§8)? +- Does the header section still name a withheld participant? + +--- + +## 12. Authoring tooling + +`vcf-rdfizer-policy`, mirroring `vcf-rdfizer-rules` and the proposed +`vcf-rdfizer-link`: + +| Command | Purpose | +| --- | --- | +| `policy init -o p.ttl` | Scaffold with annotated examples and a default-deny skeleton | +| `policy check p.ttl` | Validate against the profile: unknown selectors, unresolvable DUO terms, assembly declared, conflicts, and **any rule no tier can enforce** | +| `policy explain p.ttl` | Render the resolved release plan in English: what is dropped, degraded, thresholded, and what remains | +| `policy dry-run p.ttl -i sample.vcf` | Report counts that *would* be withheld, per rule, writing nothing | +| `policy diff p1.ttl p2.ttl` | What changes between two policy versions — the question a data access committee actually asks | + +`explain` is the one that earns its keep. A policy nobody can read is a policy +nobody can approve, and the failure mode is approval-by-exhaustion. + +--- + +## 13. Build order + +| Step | Delivers | Unlocks | +| --- | --- | --- | +| 1 | `vcfp:` profile, policy parsing, `Predicate`/`Class`/`Sample`/`Header` selectors, `vcfp:drop`, default-deny, conflict rules | Coarse governed release with no new machinery | +| 2 | Tier 1 (TSV) + Tier 2 (emitter) enforcement, `CHROM`/`POS` on `ParsedSampleRecord`, `RegionSelector` | Region and field granularity at no runtime cost | +| 3 | Release manifest, policy conformance preflights, mutation entries | Releases become auditable and verified rather than asserted | +| 4 | `vcf-rdfizer-policy` CLI, `explain`, `diff` | Policies become reviewable by non-implementers | +| 5 | `vcfp:pseudonymize` with IRI re-minting, `maskVectorPositions`, `threshold` | The leaks in §7 and §8 closed | +| 6 | `--mode redact` (Tier 3), then optional query-time rewriting | Existing artifacts, and endpoint deployments | + +Steps 1–3 are the contract and the evidence. Note that step 3 comes *before* the +sophisticated transforms in step 5 — deliberately. A crude redaction that is +verified is worth more than a sophisticated one that is not. + +--- + +## 14. Known hard problems + +Stated up front, as in the linking design. + +**This is not anonymization, and the vocabulary must not drift.** Every user- +facing string, manifest field and log line should say *pseudonymized*, +*withheld*, or *governed release*. The moment the tool says "anonymized", it is +making a claim it cannot support (§1), and someone will rely on it. + +**Compliance is not a feature.** GDPR special-category processing, national +genomic-data law, and the scope of a specific consent form are determinations +for a data protection officer and a data access committee. The tool can +*implement* a policy and *evidence* what it did. It cannot decide what the +policy should be, and the documentation must not imply otherwise. + +**Structural leakage survives redaction.** A withheld region still shows as a +gap; a withheld sample still shows as a missing `SampleSet` member unless the +set itself is rewritten; a thresholded variant still shows as absent. Whether +those gaps matter is a policy question, but the tool should surface them — +`policy explain` is the right place. + +**Composition across releases is unmanaged.** Two separately compliant releases +to the same recipient can jointly disclose more than either alone: different +regions, different thresholds, or a fresh pseudonym key that is nonetheless +linkable through the genotypes. Nothing in this design tracks cumulative +disclosure, and the manifest is the only thread that would make it possible +later. Say so rather than implying releases are independent. + +**Policy drift versus artifact.** A policy IRI resolves to whatever it resolves +to *today*; the artifact was produced under what it said *then*. The digest in +§10 is what makes that recoverable, and it is why it is mandatory rather than +nice to have. + +**The escape hatch is a footgun.** `vcfp:PatternSelector` accepts arbitrary +SPARQL, which cannot be pushed into Tier 1 or 2 and whose cost is unbounded. It +should be permitted, reported prominently by `explain`, and excluded from any +profile intended for unattended use. + +--- + +## See also + +- [Data linking design](datalinking-design.md) — the sibling proposal; same plugin, provenance and verification patterns +- [Conversion](conversion.md) — the IRI templates and graph shapes a policy addresses +- [Sample representations](sample-representation-guide.md) — why condensed mode changes the enforceability of per-sample rules +- [Validation methodology](validation-methodology.md) — the harness that makes a privacy claim measurable +- [Limitations](limitations.md) — what the tool does not do today +- [Roadmap](roadmap.md) — how this relates to the other planned work diff --git a/docs/representations.md b/docs/representations.md new file mode 100644 index 0000000..9938280 --- /dev/null +++ b/docs/representations.md @@ -0,0 +1,187 @@ +# Compression and queryable representations + +What VCF-RDFizer can produce from an RDF aggregate, how each artifact is built, +what is verified about it, and where each one runs out of road. + +--- + +## 1. Three independent decisions + +Compression is deliberately not one option. It is three: + +| Decision | Flag | Values | Default | +| --- | --- | --- | --- | +| How the aggregate is staged | `--rdf-storage-mode` | `plain`, `space-optimized` | *(required in full mode)* | +| Which raw RDF artifacts to keep | `--rdf-compression` | `gzip`, `brotli`, `none` | `gzip,brotli` | +| Which queryable representations to build | `--representations` | `hdt`, `cottas`, `none` | `hdt` | +| How to package those representations | `--artifact-compression` | `gzip`, `brotli`, `none` | `none` | + +Each selector takes a comma-separated list. `none` must appear alone. +`--artifact-compression` requires at least one selected representation. + +The gzip used by `--rdf-storage-mode space-optimized` is **staging**, not +automatically a final artifact. It is retained as the gzip artifact only when +`gzip` is also selected in `--rdf-compression`. + +For the smallest output: `--rdf-compression none`, one representation, and +`--remove-rdf-storage-output`. + +## 2. The artifact matrix + +| Artifact | Queryable | Built by | Verified by | +| --- | --- | --- | --- | +| `.nt` | with any RDF store | the merge step | Raptor syntax check during validation | +| `.nt.gz`, `.nt.br` | after decompression | `gzip` / `brotli` | — | +| `.hdt` + `.hdt.index.v1-1` | yes, directly | `hdtc create` (+ `hdtc index`) | streaming decode + triple-count equality | +| `.cottas` | yes, directly | `pycottas` + PyArrow merge | streaming decode + triple-count equality | +| `.hdt.gz`, `.hdt.br`, `.cottas.gz`, `.cottas.br` | **no** | packaging | — | + +The packaged forms are archives. Keep the unwrapped `.hdt` / `.cottas` if +queries must run without a decompression step. + +## 3. Record-safe chunking + +When HDT or COTTAS is selected, the aggregate is read sequentially and split +into chunks on **complete N-Triples line boundaries**. Only one uncompressed +chunk exists at a time: it is consumed by both converters and removed before the +next is read. That property is what makes a `space-optimized` `.nt.gz` aggregate +usable without ever expanding a second full raw copy. + +| Flag | Meaning | Default | +| --- | --- | --- | +| `--chunk-target-bytes` | target uncompressed bytes per chunk | 512 MiB | +| `--chunk-min-bytes` | minimum before a chunk group is flushed | 128 MiB | +| `--chunk-max-bytes` | hard ceiling; boundaries stay on complete lines | 1 GiB | + +`--hdt-strategy` chooses the policy: `auto` (build chunks and merge with native +`hdtc`), `partitioned` (always chunk), `single` (one `rdf2hdt` run). `single` +cannot consume a gzip stream without expanding it, so it is incompatible with +`space-optimized`. + +The whole partitioned stage runs in an **ephemeral Docker-managed volume**. +Chunks, scratch, and merge files never reach the output directory, and the +volume is removed on success and on failure alike. + +## 4. HDT + +HDT generation, merging, and indexing use the pinned Rust **`hdtc` 1.1.0**, not +`hdtCat`, `hdtSearch.sh`, or any Java HDT process. This is not a stylistic +preference: the JVM path fails with `java.lang.OutOfMemoryError` on +cohort-scale graphs, while `hdtc create` merges chunk HDTs through disk-backed +external sorts and `hdtc index` streams BitmapTriples to build the +object/predicate orderings. The sidecar it produces is the canonical HDT v1-1 +file, `.hdt.index.v1-1`, readable by hdt-java and hdt-cpp. + +Memory is bounded by two environment variables, both defaulting to `512M` in +the image and forwarded by the wrapper when set on the host: + +```bash +HDT_MERGE_MEMORY_LIMIT=2G HDT_INDEX_MEMORY_LIMIT=2G vcf-rdfizer ... +``` + +Lower values reduce in-memory sort buffers and increase temporary I/O. Both +stages need scratch space in the container's `/work`, which lives on the +**Docker data volume** — free space in the output filesystem does not help. + +An HDT whose data is readable but whose sidecar could not be built is a +*degraded success* in full mode: it is validated with the index check skipped, +marked `index_status: "failed"`, reported in `reports/index_warnings.json`, and +the raw RDF is retained so it can be repaired later with `--mode index`. + +## 5. COTTAS + +COTTAS is a Parquet-based representation with its index **inside** the artifact; +there is no sidecar. Chunk conversion uses +`pycottas.rdf2cottas(..., disk=True)` with a fresh container-local DuckDB +workspace per operation. + +The final merge deliberately does **not** call `pycottas.cat`. In version 1.1.0 +that runs a global `DISTINCT` plus `ORDER BY` through an unbounded in-memory +DuckDB connection, which gets OOM-killed on large condensed graphs. VCF-RDFizer +instead performs a **k-way PyArrow merge** of the already `spo`-sorted Parquet +chunks: it holds at most `COTTAS_MERGE_BATCH_ROWS` (default 2048) rows per +input, drops adjacent duplicate triples, and writes the result incrementally. +There is no graph-wide hash table and no external-sort spill directory; memory +is a function of the batch size and the chunk count, not of the graph size. + +A COTTAS failure degrades differently from HDT: because the `.cottas` file +itself is unusable, dependent COTTAS artifacts are marked *not generated* rather +than published with a warning. + +## 6. Round-trip verification + +Every HDT and COTTAS base artifact is verified **before** packaging and before +raw RDF cleanup: [`src/validate_compression.py`](../src/validate_compression.py) +reads the source triple count, streams the artifact back through its native +decoder, and requires equality. Compression **fails closed** if the artifact +cannot be decoded or the counts differ. Results and the +`source_triples`/`decoded_triples` pair land in the per-run compression JSON and +in the HDT/COTTAS columns of `metrics.csv`. + +Be clear about what this proves and what it does not. A matching triple count +proves the artifact decodes and contains the right *number* of statements. It +does not prove the statements are the right ones. The stronger claim comes from +running the semantic suite against the decoded artifact — `--validate-artifacts +hdt,cottas` — which re-derives every VCF summary from it. See +[`validation.md`](validation.md#which-artifact-is-validated). + +## 7. Index maintenance + +`--mode index` regenerates an existing artifact's index in place. It is the one +deliberate exception to "never overwrite a planned artifact". + +| Input | Behaviour | +| --- | --- | +| `--hdt file.hdt` | Existing versioned sidecars are moved aside, regenerated, and restored if indexing fails; incomplete replacements are removed first | +| `--cottas file.cottas` | The artifact is rewritten atomically through the same bounded streaming Parquet rewrite with the default `spo` index; the original stays in place if it fails | + +No conversion, packaging, or decompression output is produced. Standalone index +mode is **strict** — unlike the in-run degradation above, a failure is a failure. +The same operation runs automatically after each partitioned HDT merge. + +## 8. Decompression + +`--mode decompress` decodes `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz` +and `.cottas.br` back to N-Triples. A packaged COTTAS is unwrapped **inside the +container** before `pycottas` writes the decoded output, so the intermediate +unwrapped file never appears on the host. + +## 9. Choosing + +| Situation | Selection | +| --- | --- | +| Load into an existing triple store | `--representations none --rdf-compression gzip` | +| Queryable, single artifact, smallest footprint | `--representations hdt --rdf-compression none --remove-rdf-storage-output` | +| Comparing HDT against COTTAS | `--representations hdt,cottas` | +| Archival transfer | `--artifact-compression brotli` on top of the chosen representation | +| Memory-constrained host | `space-optimized` + `partitioned` + lower `--chunk-*-bytes` | + +## 10. Limitations + +- **Docker volume space, not output space, is the binding constraint** for + partitioned merges and HDT indexing. This is the single most common cause of + a failed large run, and the error surfaces as an OOM/SIGKILL rather than as a + disk message. +- **`exit_code=-9` (or `137`) means the kernel or Docker OOM-killer intervened**, + not that the RDF is invalid. Check `stderr_tail`, `max_rss_kb`, and the + workspace free-space samples in `stages/partitioned/.json` before + suspecting the data. +- **COTTAS is the more fragile path.** If it cannot fit the available memory + even at a reduced `COTTAS_MERGE_BATCH_ROWS`, `--representations hdt` is + independent and remains queryable. +- **Packaged representations are not queryable**, which is easy to forget when + `--artifact-compression` is set and `--remove-rdf-storage-output` has removed + the alternative. +- **Neither HDT nor COTTAS carries named graphs**, matching the conversion's + triples-only output. +- **No incremental update.** Adding variants means reconverting and rebuilding + the representation from scratch. + +--- + +## See also + +- [Architecture](architecture.md) — where these stages run +- [Output and metrics](output-and-metrics.md) — what each stage records +- [Validation](validation.md) — proving a representation still means what the VCF meant +- [CLI reference](cli-reference.md) — every flag, with constraints diff --git a/docs/rml-mappings.md b/docs/rml-mappings.md new file mode 100644 index 0000000..5f43668 --- /dev/null +++ b/docs/rml-mappings.md @@ -0,0 +1,170 @@ +# Custom RML mappings + +`--rules` accepts any RML mapping, so you can change what RDF the pipeline +produces without touching the wrapper. This document is the full contract, the +tooling that checks it, and an honest account of what a custom mapping costs you. + +--- + +## 1. The tooling first + +`vcf-rdfizer-rules` is installed alongside `vcf-rdfizer` and exists so that the +contract is discoverable and checkable, rather than something you find out three +hours into a run. + +```bash +vcf-rdfizer-rules columns # what a mapping can reference +vcf-rdfizer-rules init -o my.ttl # annotated copy of the shipped default +vcf-rdfizer-rules check my.ttl # validate before spending a run on it +``` + +`check` reports: + +- logical-source paths the wrapper cannot rewrite per input; +- referenced columns no generated TSV provides (typos such as `CHROMOSOME`); +- which `--sample-representation` values remain usable; +- whether the mapping forces the large sample helper tables to be materialized. + +Exit code is `0` when the mapping is usable and `1` when it is not; `--json` +gives the same report machine-readably. + +The checks are deliberately **lexical**, not a full Turtle parse. The tool stays +dependency-free, and the mistakes worth catching before a long run are wrong +paths and misspelled column names — not subtle RML semantics. A mapping that +passes `check` can still be wrong; it just will not be wrong in the two ways +that waste the most time. + +## 2. The contract + +### Rule 1 — keep the five `csvw:url` values verbatim + +Full mode processes one VCF at a time and rewrites those exact literal strings +to the per-input file names. Any other path is left untouched and will not +resolve. + +| Logical source | Contents | +| --- | --- | +| `/data/tsv/records.tsv` | One row per VCF data line | +| `/data/tsv/header_lines.tsv` | One row per `##` header line | +| `/data/tsv/file_metadata.tsv` | One row summarising the source VCF | +| `/data/tsv/sample_calls.tsv` | Helper: one row per variant × sample | +| `/data/tsv/sample_format_values.tsv` | Helper: one row per variant × sample × FORMAT key | + +### Rule 2 — only reference columns the pipeline writes + +`vcf-rdfizer-rules columns` is authoritative. A unit test runs +[`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) and asserts those lists still match +what it emits, so the documentation cannot drift from the implementation. + +The last column of `records.tsv` is the whitespace-joined sample IDs from the +`#CHROM` line (or `SAMPLES` when the VCF declares none). Its *name* varies per +input, so it **cannot be referenced by a fixed name** — which is one of the +reasons genotype RDF is emitted by the wrapper instead. + +### Rule 3 — think before consuming the helper tables + +The four built-in sample maps are recognised by the wrapper, which then keeps +`sample_calls.tsv` and `sample_format_values.tsv` header-only and streams +genotype RDF itself. A mapping that consumes them in any other way forces both +to be materialized in full — one row per variant × sample, and one per +variant × sample × FORMAT key. On a cohort VCF that is the largest intermediate +the pipeline can produce. + +Such a mapping is **rejected** in `--sample-representation condensed`, which +would otherwise emit the expanded and condensed genotype graphs simultaneously +and restore exactly the inflation condensed mode exists to avoid. Remove the +helper-table consumers, or use expanded mode. A custom mapping with no +helper-table consumers works in both modes. + +## 3. What a custom mapping does *not* control + +This is the part most easily missed. Three families of triple are emitted by the +wrapper, not by RML, and they appear in your graph regardless of your mapping: + +| Triples | Turned off with | +| --- | --- | +| Genotypes (`SampleCall`/`FormatFieldValue`, or the condensed equivalents) | nothing — one emitter always runs; choose which with `--sample-representation` | +| `QUAL`, and structured `InfoFieldValue` nodes | `--info-representation raw` still emits QUAL | +| Header-line subclasses and lifted FILTER/ALT/contig/INFO/FORMAT attributes | `--header-representation basic` | + +The reasons are in [`architecture.md`](architecture.md#4-where-the-split-leaks-and-why): +RML cannot choose a datatype or a class per row, and the alternatives would +require materializing helper tables. It is a real constraint on the extension +point, not an oversight, but a mapping author who assumes `--rules` is the sole +source of truth for the graph will be surprised. + +## 4. What a custom mapping costs in validation + +The semantic validation suite has three layers. A custom mapping affects one of +them. + +Queries `q09`–`q13` — the predicate census, the class census, and the three +identity digests — assume the shipped mapping's predicate inventory and IRI +templates. A custom mapping changes both **by design**. + +`validation_runner.py` has a `--mapping-policy report-only` setting for exactly +this case: it records those five queries without failing on them, leaving the +run to rest on the aggregate comparisons (`q01`–`q08`). + +> **Known gap.** The `vcf-rdfizer` wrapper does **not** currently forward +> `--mapping-policy`, so the runner always executes with the default `strict`. +> A custom mapping validated through `vcf-rdfizer --validate` or +> `--mode validation` will therefore report `MISMATCH` on `q09`–`q13` even when +> the conversion is correct. Until the flag is exposed, either read past those +> five rows in `comparison.json`, or invoke the runner directly inside the +> container with `--mapping-policy report-only`. Tracked in +> [`roadmap.md`](roadmap.md). + +Once that gap is closed, the trade-off is the intended one: with a custom +mapping you keep distributional checking and lose the completeness and +permutation checks. That is correct behaviour — the alternative is a false +failure on every custom run — but it means a custom mapping is validated less +thoroughly than the default one, and you should know that before treating a +`PASS` as equivalent. + +Graph-integrity preflights (blank nodes, empty terms, duplicate statements) and +SHACL are mapping-independent and still apply in full. + +## 5. Worked start + +```bash +vcf-rdfizer-rules init -o my_rules.ttl +$EDITOR my_rules.ttl +vcf-rdfizer-rules check my_rules.ttl + +vcf-rdfizer --mode full \ + -i ./cohort.vcf.gz \ + --rules my_rules.ttl \ + --rdf-storage-mode plain \ + -o ./results +``` + +Recommendations that come from experience rather than from the checker: + +1. **Preserve the stable subject templates** (`file://{SOURCE_FILE}`, + `file://{SOURCE_FILE}#record/{ROW_ID}`, `file://{SOURCE_FILE}#call/{ROW_ID}`) + if you want joins with the wrapper-emitted triples to keep working. The + genotype, QUAL, INFO and header emitters all mint IRIs against those + templates and will not follow a mapping that changes them. +2. **Add, don't replace.** A mapping that extends the default with additional + `rr:TriplesMap` blocks keeps the full validation suite meaningful for the + parts it did not change; one that rebuilds the graph from scratch does not. +3. **Test on a small VCF first.** `test/test_vcf_files/test-100.vcf` converts in + seconds and exercises the same code path as a cohort file. + +## 6. SHACL + +The SHACL constraints are maintained in the vocabulary repository, not here: +[`vcf-rdfizer-vocabulary.shacl.ttl`](https://github.com/ecrum19/VCF-RDFizer-vocabulary/blob/main/shacl/vcf-rdfizer-vocabulary.shacl.ttl). +Point `--shacl-shapes` at them to check a graph structurally, independently of +the VCF. Note the known contradiction inside the published shapes, recorded in +[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). + +--- + +## See also + +- [Conversion](conversion.md) — what the default mapping produces +- [Architecture](architecture.md) — why some triples bypass RML +- [Validation](validation.md) — what changes under a custom mapping +- [`rules/README.md`](../rules/README.md) — the mapping directory itself diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..fdf3c14 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,161 @@ +# Roadmap and future development + +What is planned, what is merely known-to-be-wrong, and what has been assessed +and deliberately rejected. Every item links to the document where it is +described in full. + +This is a working document, not a commitment. Items are ordered by how much they +block something else, not by effort. + +--- + +## Blocking publication + +### 1. Vocabulary coverage for the condensed representation + +The conversion emits 17 terms that `https://w3id.org/vcf-rdfizer/vocab#` does +not define — `CohortCallMatrix`, `FormatValueVector`, `SampleSet`, `VCFSample`, +`VCFTextVector`, `representationProfile`, `sampleIndex` and the rest. They do +not dereference, so **condensed graphs are not ontology-backed**. + +The work is in the vocabulary repository, not here. Until it lands, the +condensed representation is usable but not citable as linked data. +Full list: [`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). + +### 2. The SHACL / missing-value contradiction + +`vcfr:missingValuePolicy` requires `"."^^vcfr:Null` for a missing token; +`VCFRecordShape` constrains `vcfr:alt` to `sh:datatype xsd:string`. A record with +`ALT=.` cannot satisfy both, so it is impossible for the tool to be conformant. + +The fix is a decision in the vocabulary: either relax the `alt`/`ref`/`chrom` +shapes to `sh:or([xsd:string] [vcfr:Null])` — as the `qual` shape already does — +or drop the missing-value policy for those fields. The conversion currently +follows the policy. +[Detail](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). + +## Known defects + +### 3. Validation mapping policy is not forwarded + +`validation_runner.py` supports `--mapping-policy report-only`, which is exactly +what a custom `--rules` mapping needs: it records the mapping-dependent queries +(`q09`–`q13`) without failing on them. The `vcf-rdfizer` wrapper never passes +it, so the runner always executes `strict` and a correct custom-mapping +conversion is reported as `MISMATCH`. + +Fix: expose a `--mapping-policy` flag on the wrapper, and default it to +`report-only` automatically whenever `--rules` is not the shipped default. +Small change; it makes the custom-mapping extension point actually usable. +[Detail](rml-mappings.md#4-what-a-custom-mapping-costs-in-validation). + +### 4. Validation coverage gaps with named fixes + +From [`vcf-coverage.md`](vcf-coverage.md#remaining-gaps): + +- **`vcfr:contigCount` is counted, not read.** A wrong derived contig total is + undetected. Needs the value in a comparison, not just the predicate in the + census. +- **Header line values are not compared.** `q08` compares how many lines carry + each key and `q10` their types; `vcfr:headerValue` itself is only counted. The + structured attributes that matter are already covered. + +Both are query-set additions with matching mutation-catalogue entries, so +closing either one will *fail* the mutation harness until +[`vcf-coverage.md`](vcf-coverage.md) is updated — by design. + +## New capability + +### 5. Data linking as a plug-in system + +The largest planned addition: let users connect the graph to external resources +(rsIDs, genes, clinical assertions, frequencies) through declarative, +distributable linker plugins rather than by patching the tool. + +Full design, including the three join strategies, the three plugin tiers, the +network safeguards, the provenance model and the build order, is in +[`datalinking-design.md`](datalinking-design.md). Nothing is implemented yet; +the design exists to fix the extension contract before code depends on it. + +The hardest part is not the plugin system — it is allele normalization +(§9 of that document), because an under-matching join looks exactly like a true +negative. + +### 6. Query-time decoding for condensed graphs + +Condensed mode stores each FORMAT key as one tab-separated vector, which is what +makes cohort scale tractable. Reconstructing sample *i*'s value at query time +needs a split operation SPARQL does not portably provide. + +The options have been assessed in detail in +[`sample-representation-guide.md`](sample-representation-guide.md#7-assessment-of-geosparql-and-graphdb-sparql-extensions): + +| Option | Verdict | +| --- | --- | +| GeoSPARQL geometry functions | **No.** Genotype vectors are not geometries; the analogy does not survive contact | +| GraphDB `spif:split` | Useful as a prototype tokenizer and validation aid, not as a production join | +| `spif:split` + `spif:for` | Building blocks, but no ordinal pairing mechanism, so no trustworthy sample↔value join | +| A custom vector-extraction function, or an application-side decoder | **The promising direction** | + +The recommended shape is a function that takes a vector and a sample index (or a +`VCFSample` IRI) and returns both the position and the value, with the six +safeguards listed in that section. Crucially, this belongs in a **separate +decoding/projection layer** — the condensed model itself should not change. + +There is also a scale caveat that any implementation has to respect: expanding +every vector in a cohort-wide query produces millions of bindings before +filtering. A targeted single-position extractor is the right primitive; a +whole-graph expansion is not. + +### 7. Granular privacy policies over the graph + +Today the tool has exactly one disclosure setting: convert everything. There is +no way to release a cohort graph with some participants withheld, some regions +degraded, or some fields suppressed, and no machine-readable record of what a +given artifact was permitted to contain. + +The proposal is an ODRL profile whose assets are **graph selectors** (by class, +predicate, sample, genomic region, declared field or pattern), compiled to a +release plan and enforced at the cheapest available point in the existing +pipeline — TSV pre-filtering, emitter-time filtering, or a post-hoc pass. +Full design in [`privacy-policy-design.md`](privacy-policy-design.md). + +Two findings from that design are worth surfacing here because they affect work +outside it: + +- **`ParsedSampleRecord` does not carry `CHROM` or `POS`.** `_parse_row` reads + columns 0, 1, 7, 9, 10 and −1. Any region-scoped feature evaluated at emission + time needs both, and adding them is two fields and two index reads. +- **Condensed mode is not per-sample enforceable by triple filtering.** One + `FormatValueVector` literal holds every participant's value for a FORMAT key, + so the unit of protection is finer than the unit of storage. Redaction has to + rewrite the literal, and a single masked position is itself disclosive. + +The honest framing, which the design keeps throughout: this is *governed +release*, not anonymization. Genotypes identify people, and no access-control +layer changes that. + +## Deliberately not planned + +Stated so the absence reads as a decision rather than an oversight. + +| Not planned | Why | +| --- | --- | +| A Docker-free installation path | The pinned image is what makes results reproducible across machines | +| Variant normalization inside the conversion | The graph is a faithful transcription of the file; normalization is a linking-time concern | +| Distributed or multi-node execution | Out of scope; chunking already bounds memory on one machine | +| Incremental graph update | Would require identity and provenance machinery the current model does not have | +| Clinical interpretation or pathogenicity assertion | The tool transcribes; it is not a clinical authority | +| Differential privacy on aggregate queries | Formal guarantees need a persistent per-recipient budget ledger; without one it is decoration. See [`privacy-policy-design.md` §9](privacy-policy-design.md#9-beyond-access-control-disclosure-limitation) | +| Deciding whether a release is legally compliant | A tool can implement and evidence a policy; it cannot make a data-protection determination | +| `.bcf` input | Would pull `htslib` into the parsing path; convert with `bcftools` first | + +--- + +## See also + +- [Limitations](limitations.md) — the current state, honestly +- [Data linking design](datalinking-design.md) — the largest planned addition +- [Privacy policy design](privacy-policy-design.md) — governed release over parts of the graph +- [Validation methodology](validation-methodology.md) — how coverage is measured, so gaps stay falsifiable +- [`changelog.md`](../changelog.md) — what has actually shipped diff --git a/docs/sample-representation-guide.md b/docs/sample-representation-guide.md new file mode 100644 index 0000000..287c112 --- /dev/null +++ b/docs/sample-representation-guide.md @@ -0,0 +1,551 @@ +# Expanded and Condensed Knowledge Representations + +*Part of the [VCF-RDFizer documentation](README.md). How these triples are emitted: +[`conversion.md`](conversion.md). Where section 7's assessment leads: +[`roadmap.md`](roadmap.md#6-query-time-decoding-for-condensed-graphs).* + +This guide explains the two ways VCF-RDFizer can represent sample genotype +information in RDF: + +- **expanded**, the default representation; +- **condensed**, selected with `--sample-representation condensed`. + +The word *representation* here means the shape of the RDF graph. It does not +mean that the input VCF is changed or that genotype values are discarded. + +## 1. Why sample representation matters + +A VCF is naturally a wide table. A small part of a multi-sample VCF might look +like this: + +```text +#CHROM POS REF ALT FORMAT SAMPLE_A SAMPLE_B SAMPLE_C +20 100 A G GT:DP:AD 0/1:42:30,12 0/0:18:18,0 1/1:9:9,0 +``` + +There is one variant row, followed by one sample column for every individual. +The `FORMAT` column says that each sample value has three fields: + +- `GT`: genotype; +- `DP`: read depth; +- `AD`: allele depths. + +For a real cohort, the numbers can be much larger. For example, a file with +10,000 variants, 2,500 samples, and three FORMAT fields contains: + +```text +10,000 × 2,500 × 3 = 75,000,000 individual FORMAT values +``` + +RDF can describe each of those values separately, but doing so creates a very +large number of RDF resources and relationships. The two modes let users +choose between a graph that is easy to query one value at a time and a graph +that is much more economical for large cohorts. + +## 2. Expanded representation: one RDF object per sample value + +Expanded mode preserves the original, explicit vocabulary model. For every +variant/sample pair, VCF-RDFizer creates a `vcfr:SampleCall`. For every FORMAT +field in that pair, it creates a `vcfr:FormatFieldValue`. + +For the example above, the conceptual graph includes resources like these +(the full output has one line per RDF statement): + +```text + + vcfr:hasSampleCall . + + + a vcfr:SampleCall ; + vcfr:sampleId "SAMPLE_A" ; + vcfr:hasFormatValue . + + + a vcfr:FormatFieldValue ; + vcfr:fieldValue "0/1" . +``` + +The same structure is repeated for `DP` and `AD`, and then repeated again for +`SAMPLE_B` and `SAMPLE_C`. + +### What expanded mode provides + +Expanded mode makes each value easy to address directly: + +- a consumer can find one sample call without decoding a vector; +- a consumer can ask for one FORMAT field as one RDF resource; +- existing consumers that expect `SampleCall` and `FormatFieldValue` can use + the graph directly; +- the sample identifier is attached to every sample-call resource. + +The VCF file is marked with +`vcfr:representationProfile vcfr:ExpandedRepresentation`, so a consumer can +identify the graph shape explicitly. + +### Expanded growth + +Let: + +- `V` = number of variant records; +- `S` = number of samples; +- `F` = average number of FORMAT fields per record. + +Expanded mode creates approximately: + +```text +V × S SampleCall resources +V × S × F FormatFieldValue resources +``` + +In the simplified case where each sample call has three structural statements +(link, type, and sample ID), and each FORMAT value has three statements (link, +type, and value), the sample portion of the graph contains approximately: + +```text +3 × (V × S) + 3 × (V × S × F) +``` + +statements, before counting the rest of the VCF graph. + +For the worked example (`V=1`, `S=3`, `F=3`): + +| Expanded item | Count | +|---|---:| +| `SampleCall` resources | 3 | +| `FormatFieldValue` resources | 9 | +| Approximate sample-related statements | 36 | +| Representation-profile statement | 1 | + +The exact number can vary when values are missing or when custom mappings add +properties, but the important pattern is the multiplication by both samples +and FORMAT fields. + +## 3. Condensed representation: shared samples plus ordered vectors + +Condensed mode avoids creating a separate RDF resource for every scalar sample +value. It describes the sample columns once, then stores the values for each +variant and FORMAT key in one ordered vector. + +For the same example, the graph first declares a reusable sample set: + +```text + + a vcfr:SampleSet ; + vcfr:hasSample ; + vcfr:hasSample ; + vcfr:hasSample . + + + a vcfr:VCFSample ; + vcfr:sampleName "SAMPLE_A" ; + vcfr:sampleIndex "1" . +``` + +The variant then has one cohort matrix. The matrix points back to the sample +set and has one vector for each FORMAT key: + +```text + + a vcfr:CohortCallMatrix ; + vcfr:appliesToSampleSet ; + vcfr:hasFormatValueVector . + + + a vcfr:FormatValueVector ; + vcfr:valueEncoding vcfr:VCFTextVector ; + vcfr:encodedValues "0/1\t0/0\t1/1" . +``` + +The three tab-separated items in `encodedValues` are in `sampleIndex` order: + +| `sampleIndex` | Sample | `GT` value in the vector | +|---:|---|---| +| 1 | `SAMPLE_A` | `0/1` | +| 2 | `SAMPLE_B` | `0/0` | +| 3 | `SAMPLE_C` | `1/1` | + +The `DP` vector is `"42\t18\t9"`, and the `AD` vector is +`"30,12\t18,0\t9,0"`. A comma inside one VCF value remains inside that item; +it is not treated as a sample separator. If a value is absent, the vector +uses `.` to keep every sample position aligned. + +The file is marked with +`vcfr:representationProfile vcfr:CondensedRepresentation`. + +### What condensed mode provides + +- Sample resources are declared once per file, not once per variant. +- Each variant has one `CohortCallMatrix`, rather than one sample-call + resource for every sample. +- Each variant/FORMAT-key pair has one `FormatValueVector`, rather than one + `FormatFieldValue` resource per sample. +- The original sample order and all lexical values remain available. +- A vector-aware consumer can reconstruct sample `i` by looking up the sample + with `sampleIndex=i` and selecting item `i` from the corresponding vector. + +Condensed mode therefore moves repetition from RDF structure into the literal +payload of a vector. It is a storage and graph-shape optimization, not a loss +of genotype information. + +### Condensed growth + +Using the same variables, condensed mode creates approximately: + +```text +S reusable sample descriptions +V cohort matrices +V × F format-value vectors +F shared FORMAT definitions (per file/header) +``` + +The RDF structure grows roughly with `S + V + (V × F)`, while each vector +literal still contains the `S` values for that variant and key. + +For `V=1`, `S=3`, and `F=3`, the graph has approximately: + +| Condensed item | Count | +|---|---:| +| Reusable `VCFSample` resources | 3 | +| `CohortCallMatrix` resources | 1 | +| `FormatValueVector` resources | 3 | +| Shared FORMAT definitions | 3 | +| Vector literals containing sample values | 3 | + +Because a condensed graph has fixed metadata for the sample set, matrix, and +FORMAT definitions, it can be similar in size to—or even slightly larger than— +an expanded graph for a tiny file. Its advantage appears when many variants reuse +the same sample columns. + +## 4. The scaling difference in numbers + +Assume a cohort has: + +```text +V = 1,000 variants +S = 2,500 samples +F = 3 FORMAT fields +``` + +### Expanded counts + +```text +SampleCall resources: 1,000 × 2,500 = 2,500,000 +FormatFieldValue resources: 1,000 × 2,500 × 3 = 7,500,000 +Total per-value resources: 10,000,000 +``` + +Using the simplified three-statements-per-resource estimate, this is about +30,000,000 sample-related RDF statements. + +### Condensed counts + +```text +Reusable samples: 2,500 +Cohort matrices: 1,000 +Format-value vectors: 1,000 × 3 = 3,000 +FORMAT definitions: 3 +``` + +That is roughly 6,500 resources in the sample representation, plus vector +literals containing the same 7,500,000 scalar values. Counting the explicit +statements emitted by the built-in condensed writer gives about 28,000 +sample-related statements for this simplified case (the exact total depends on +header metadata and other record properties). + +So the RDF *structure* is reduced by roughly: + +```text +30,000,000 ÷ 28,000 ≈ 1,070× +``` + +This does not mean the compressed file will always be 1,070 times smaller. The +75 million values still exist inside vector literals, and their text length, +escaping, dictionary compression, and the rest of the VCF graph affect the +final `.nt`, `.hdt`, or `.cottas` size. The large saving is the removal of +millions of repeated RDF nodes, predicates, and type statements. + +## 5. Why VCF-RDFizer keeps both strategies + +The modes serve different downstream needs. Combining them into one universal +graph would either make small files unnecessarily complicated or make large +cohorts unnecessarily expensive. + +### Expanded is useful when direct RDF querying is the priority + +Expanded mode is a good fit when: + +- the VCF has one sample or a small number of samples; +- downstream software already expects `SampleCall` and + `FormatFieldValue` resources; +- queries need to match one sample value directly, without splitting a vector; +- maximum vocabulary-level transparency is more important than graph size. + +For example, a consumer can navigate directly from a call to the sample whose +`sampleId` is `SAMPLE_A`, then to its `GT` `FormatFieldValue`. Every value is a +normal RDF resource and literal. + +### Condensed is useful when cohort scale is the priority + +Condensed mode is a good fit when: + +- there are hundreds or thousands of samples; +- the same sample columns repeat across many variant rows; +- materializing one RDF object per sample/variant/FORMAT combination causes + excessive memory, disk, or indexing work; +- downstream software can use `sampleIndex` and decode tab-separated vectors. + +For a 2,500-sample cohort, expanded mode creates 2,500 sample-call resources for +*every* variant. Condensed mode creates the 2,500 sample descriptions once and +reuses them for all variants. + +### The semantic trade-off + +| Question | Expanded | Condensed | +|---|---|---| +| Where is each sample value? | Its own RDF resource | An item in an ordered vector literal | +| How often are samples declared? | Repeated per variant/sample call | Once per file/sample set | +| Simple one-value RDF query | Easier | Requires vector lookup/decoding | +| Large-cohort graph size | Grows with `V × S × F` | Structure grows roughly with `S + V × F` | +| All source values retained? | Yes | Yes | +| Vocabulary profile | `ExpandedRepresentation` | `CondensedRepresentation` | + +Neither mode is universally “more correct.” They expose the same underlying +VCF information through different graph contracts. + +## 6. How this is implemented in VCF-RDFizer + +The command-line choice is explicit: + +```bash +# Expanded is the default +vcf-rdfizer --mode full --input cohort.vcf.gz \ + --sample-representation expanded \ + --rdf-storage-mode plain --out results + +# Condensed is intended for large multi-sample cohorts +vcf-rdfizer --mode full --input cohort.vcf.gz \ + --sample-representation condensed \ + --rdf-storage-mode space-optimized \ + --representations hdt --out results +``` + +There is no automatic “switch at N samples” threshold. This is deliberate: +the same command should produce the same graph contract every time, and a +downstream consumer should not unexpectedly receive a different RDF shape +just because a file happened to contain more samples. + +For the built-in rules, both modes read the wide sample payload in +`records.tsv` and stream the selected RDF representation directly. The current +implementation does **not** first materialize the enormous helper +tables for the built-in maps. That implementation optimization is separate +from the semantic choice: + +- expanded still emits the expanded `SampleCall`/`FormatFieldValue` graph; +- condensed still emits the condensed `SampleSet`/`CohortCallMatrix`/ + `FormatValueVector` graph. + +Custom rules that consume `sample_calls.tsv` or +`sample_format_values.tsv` retain the materialized helper-table behavior. Such +custom helper-table mappings are rejected in condensed mode, because running them +alongside the condensed emitter would produce both graph shapes and recreate +the expansion that condensed mode is intended to avoid. + +## 7. Assessment of GeoSPARQL and GraphDB SPARQL extensions + +GraphDB's [SPARQL extensions reference](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html) +contains several different kinds of functionality. They should not all be +treated as ways to read a condensed VCF vector. + +### 7.1 GeoSPARQL geometry functions are not a direct match + +The standard `geof:` functions in the GraphDB reference operate on a +`geomLiteral`, such as a literal with the datatype `geo:wktLiteral` or +`geo:gmlLiteral`. They perform geometry operations such as distance, buffer, +intersection, union, envelope, and spatial relationship tests. GraphDB's +additional `geoext:` functions provide operations such as area, geometry +validity, simplification, and Hausdorff distance. These functions are intended +for points, lines, polygons, and other spatial objects, not arbitrary ordered +text values. See the [GeoSPARQL function table](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html#geosparql-functions) +and [GraphDB GeoSPARQL extensions](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html#geosparql-extension-functions). + +Our condensed value is instead a normal RDF string associated with +`vcfr:VCFTextVector`, for example: + +```text +"0/1\t0/0\t1/1" +``` + +It is not a WKT geometry. + +The analogy with WKT remains useful at the design level: both pack a sequence +into one literal. The GeoSPARQL operations themselves, however, are not +reusable for genotype-vector extraction. + +### 7.2 The useful GraphDB feature is string splitting + +The reference also documents the GraphDB/SPIN magic predicate `spif:split`. +It takes a string and a regular expression and produces one result row for +each split item. The implementation is described as using Java's +`String.split()` method. A prototype query for all tokens in condensed vectors +could look like this: + +```sparql +PREFIX vcfr: +PREFIX spif: + +SELECT ?vector ?token WHERE { + ?vector a vcfr:FormatValueVector ; + vcfr:valueEncoding vcfr:VCFTextVector ; + vcfr:encodedValues ?encoded . + ?token spif:split (STR(?encoded) "\t") . +} +``` + +For a `GT` vector, this would expose `0/1`, `0/0`, and `1/1` as separate query +bindings. It is useful for experimentation, validation, or exporting values +to an application. + +There is an important limitation: splitting gives the values, but the query +also needs the ordinal position of each value in order to identify the sample. +The condensed model says that the first token belongs to `sampleIndex=1`, the +second to `sampleIndex=2`, and so on. The documented `spif:split` pattern does +not itself return that ordinal. GraphDB also documents `spif:for`, which can +generate a sequence of integers, but the reference does not provide a direct +“zip these generated indexes with the split results” operation. This means +that `spif:split` alone is not a complete, reliable sample lookup mechanism. + +The page also lists RDF-list functions such as `list:index` and +`list:length`. Those operate on RDF Collections. `vcfr:encodedValues` is a +single string literal, not an RDF Collection, so these functions do not apply +unless the data model is changed to materialize every vector item as list +structure. That would reintroduce much of the per-item RDF overhead that +condensed mode was designed to avoid. GraphDB's `helper:tuple` and +`helper:iterate` functions similarly operate on internal query-time lists; +they do not automatically parse a persisted VCF vector literal. + +### 7.3 What is useful now and what is not + +| GraphDB feature | Usefulness for `VCFTextVector` | Assessment | +|---|---|---| +| `geof:*` GeoSPARQL functions | Geometry calculations | Not applicable to genotype text vectors | +| `geoext:*` geometry extensions | Geometry validity and transformations | Not applicable unless a future dataset contains real sample geometries | +| `spif:split` | Split tab-separated vector text | Useful prototype, but loses the token ordinal | +| `spif:for` | Generate expected sample positions | Helpful support function, but does not pair positions with split tokens | +| `list:index` / `list:length` | Index RDF Collections | Not applicable to the current string encoding | +| `helper:tuple` / `helper:iterate` | Work with internal query lists | Not a persisted-vector parser | +| A custom `vcfr:` vector function | Return value at a sample index | Most direct future SPARQL integration | +| Application-side parsing | Decode one selected vector after retrieval | Best portable near-term approach | + +GraphDB explicitly labels these as extensions beyond the W3C SPARQL +specification, so a query using `spif:*` or a custom function would be tied to +GraphDB (or to another engine that implements the same extension). That can be +acceptable for a GraphDB deployment, but it should not be presented as a +portable SPARQL solution. These are query-time GraphDB operations; using them +would not change VCF-RDFizer's native HDT creation or indexing path, but their +memory and latency behavior would still need to be benchmarked inside the +GraphDB server. + +### 7.4 Recommended future extraction design + +If query-time sample lookup becomes an important use case, the most useful +addition would be an extractor that understands the VCF-RDFizer metadata and +returns both the position and the value. Conceptually, it could have a +function or magic-predicate contract like: + +```text +vcfr:vectorValue(?vector, ?sampleIndex) → ?value +``` + +or: + +```text +?vector vcfr:valueAt (?sampleIndex ?value) +``` + +For a sample-aware form, the function could accept the `vcfr:VCFSample` IRI +instead of an integer, resolve that resource's `sampleIndex`, and then extract +the matching token. Returning the index as well as the value would make it +possible to validate that a vector has the expected number of positions. + +A production implementation should also consider these safeguards: + +1. Verify that the vector belongs to the expected `SampleSet` through its + `CohortCallMatrix`. +2. Check that the requested index is within the sample-set size. +3. Preserve `.` as the VCF missing-value marker rather than silently turning + it into an unbound result. +4. Treat tab as the vector separator and keep commas inside a value such as + `AD=30,12`. +5. Validate that the vector contains exactly one item per declared sample. +6. Expose the function through a GraphDB plugin only when GraphDB-specific + deployment is acceptable; otherwise provide a small application-side + decoder or a materialized per-value projection. + +There is also a performance question. With 1,000 variants, 2,500 samples, and +three FORMAT keys, condensed mode stores 3,000 vectors but each vector contains +2,500 values. Expanding every vector in a SPARQL query would produce up to + +```text +1,000 × 3 × 2,500 = 7,500,000 query result rows +``` + +before filtering to a particular sample. For a targeted lookup, a function that +extracts one position is preferable to `spif:split` over the entire vector. For +large cohort-wide analyses, an external decoder or a precomputed analytical +table may be more appropriate than forcing a graph database to emit millions +of token bindings. + +### 7.5 Overall conclusion + +The GraphDB work is useful as a source of implementation ideas, but not as a +drop-in GeoSPARQL solution: + +- **GeoSPARQL geometry functions:** no, they should not be used for genotype + vectors. +- **`spif:split`:** yes, as a prototype tokenizer and validation aid. +- **`spif:split` plus `spif:for`:** potentially useful building blocks, but not + sufficient for a trustworthy sample/value join without an ordinal pairing + mechanism. +- **Custom vector extraction function or application decoder:** yes, this is + the most promising future direction. + +The condensed model should therefore remain as it is for storage efficiency, +while future query support should be added as a separate decoding/projection +layer rather than changing genotype vectors into geometries or RDF Lists. + +## 8. Practical choice + +Use this short rule of thumb: + +```text +Small or single-sample VCF + simple RDF queries → expanded +Large multi-sample cohort + storage/scale focus → condensed +``` + +If a consumer is unsure which graph it received, inspect the file’s +`vcfr:representationProfile` value before querying. That profile is the +explicit signal that tells the consumer whether sample values are represented +as individual resources or as ordered vectors. + +## Glossary + +- **Variant record**: one row describing a genomic position and its alleles. +- **Sample**: one individual or biological sample represented by a VCF column. +- **FORMAT key**: a field name such as `GT`, `DP`, or `AD` describing one part + of a sample’s value. +- **RDF resource**: a named graph object that can have properties, such as one + `SampleCall` or one `FormatFieldValue`. +- **Vector**: an ordered list of values. In condensed mode, values are stored + as one tab-separated `VCFTextVector` literal in `sampleIndex` order. +- **Graph shape**: which resources and relationships are present, independent + of the underlying biological values. + + +--- + +## See also + +- [Conversion](conversion.md) — how these triples are emitted, and why not through RML +- [VCF coverage matrix](vcf-coverage.md) — which genotype elements are validated +- [Representations](representations.md) — why condensed matters at cohort scale +- [Roadmap](roadmap.md#6-query-time-decoding-for-condensed-graphs) — where section 7 leads diff --git a/docs/validation-methodology.md b/docs/validation-methodology.md new file mode 100644 index 0000000..47000a6 --- /dev/null +++ b/docs/validation-methodology.md @@ -0,0 +1,227 @@ +# How validation coverage is measured + +*Part of the [VCF-RDFizer documentation](README.md). How to run the validator: +[`validation.md`](validation.md). Current results: +[`vcf-coverage.md`](vcf-coverage.md).* + +The semantic validation suite compares a converted RDF graph against summaries +computed independently from the source VCF. That answers "do these agree?" but +not "what would we have missed?" — and a validator nobody has tried to fool is +an assumption, not evidence. + +This document describes the mutation-testing method used to answer the second +question, and how to reproduce the number. + +## The problem with self-reported coverage + +Before this harness existed, the suite's coverage was described in prose. That +description was wrong in a way nobody had noticed: `QUAL` is extracted from +every VCF into `records.tsv` and then never mapped into RDF, and no validation +check could have detected it, because none of them look at QUAL. A test suite +that passes on a graph missing an entire VCF column is not measuring what its +documentation claims. + +## Method + +Mutation testing treats the validator as the system under test. A *correct* +graph is deliberately corrupted in a specific, named way, and the validator is +asked for its verdict. If the verdict stops being `PASS`, the mutation is +**detected**. The proportion detected is the **mutation score**. + +Three properties make the result trustworthy: + +**The fixture cannot lie to itself.** The VCF, the RDF graph, and the parser +oracle are all derived from one declarative specification in +[`test/validation_fixtures.py`](../test/validation_fixtures.py). The graph's +genotype triples come from the project's own emitters rather than being written +by hand, so the fixture tracks the real implementation. A container test closes +the loop by running the real `parse_vcf` over the fixture VCF and asserting it +equals the derived oracle. + +**The harness measures shipped code.** Mutations are evaluated through +`evaluate_validation()` in `validation_runner.py` — the single function that +also decides real runs. There is no second implementation of the decision logic +for the tests to agree with. + +**Known gaps are assertions, not comments.** A mutation the suite cannot catch +carries a `known_undetected` reason, and the harness asserts it is *still* not +detected. Closing a gap therefore **fails a test**, forcing both the catalogue +and [`vcf-coverage.md`](vcf-coverage.md) to be updated. Coverage cannot silently +drift in either direction. + +## The catalogue + +[`test/validation_mutations.py`](../test/validation_mutations.py) holds one +entry per named corruption: an id, the VCF element it targets, the graph edit, +the check expected to catch it, and — where applicable — why it is not caught. + +Mutations cover dropped and duplicated records, corrupted CHROM/POS/ALT values, +retyped literals, permuted values between records, dropped and corrupted FILTER +strings, flipped genotypes, dropped sample calls, non-GT FORMAT values, QUAL, +INFO, header lines, file metadata, missing-token policy, representation +profiles, and spurious predicates. + +A mutation may target a triple the shipped mapping does not emit yet: the +fixture opts that shape in, so the harness answers "if we emitted this, would +we notice it breaking?" ahead of the fix. That is how the QUAL gap was measured +before QUAL was mapped. + +## Three independent layers + +The suite is not one check but three, deliberately independent so that a defect +has to evade all of them: + +1. **Aggregate comparison** — six deterministic VCF summaries recomputed from + the graph and compared exactly. Catches distributional error. +2. **Census and digest** — the graph's predicate and class inventory compared + against what the VCF implies (`q09`, `q10`), and per-record and per-value + identity digests (`q11`–`q13`). Catches missing, extraneous, permuted and + altered data that leaves distributions intact. +3. **Graph integrity** — blank nodes, empty or whitespace-only terms, and + duplicated statements. Catches malformed emission with no reference to the + VCF's contents. +4. **SHACL** — the shapes the vocabulary publishes, checked with `pyshacl` via + `--shacl-shapes`. Catches structural and datatype error, also independent of + the VCF. + +Duplicate detection is the one check that cannot be a query. A SPARQL store +holds a set, so a repeated line is collapsed on load and is invisible to every +other check here. It is found by comparing the statements the parser read +against the distinct triples the store holds; the difference is the number of +redundant statements. When either number is unavailable the check reports +`NOT_EVALUATED`, never `PASS` - a check that could not run must not look like a +clean result. + +The SHACL layer earned its place immediately: it found three conformance +violations the other two could not see, two of which were fixed (QUAL typed as +a plain literal instead of `xsd:decimal`/`vcfr:Null`; `##fileDate` untyped +instead of `xsd:date`) and one of which is a contradiction inside the +vocabulary itself, recorded in [`vcf-coverage.md`](vcf-coverage.md). + +SHACL is opt-in because `pyshacl` loads the graph into memory and does not +scale to a cohort-sized aggregate. + +### Identity digests, and why they are histograms + +`q11`–`q13` hash each record, INFO value and FORMAT value **together with its +own IRI**, then bucket on the first byte of the hash. Two properties matter: + +- **Binding identity into the hash is the whole point.** Digesting values alone + would give a permutation the same multiset and change nothing. +- **A histogram needs no ordering guarantee.** `GROUP_CONCAT` would have been + the obvious alternative and is not portable, because SPARQL does not define + its order. Bucketing is order-independent by construction, and keeps the + result at most 256 rows for a graph of any size. A mismatch is localized by + re-querying only the differing buckets. + +Fields are separated by U+001F, which cannot occur in a VCF field, so no shift +of a field boundary can forge a match. + +## Four engines, two layers + +The host layer runs queries under **rdflib**, in process, needing no Docker, so +the whole mutation catalogue runs in the normal test loop. rdflib is also an +independent SPARQL implementation, which incidentally guards against queries +that only work on one engine. + +The container layer is the authority: +[`test/cross_engine_agreement.py`](../test/cross_engine_agreement.py) runs every +validation query under **Comunica, QLever, native HDT and native COTTAS** inside +the image, across both representations, and asserts they return identical +values. It then runs the shipped validation decision under each engine +separately, because engines agreeing with each other while all being wrong is a +real failure mode that only the Python oracle rules out. + +That second layer is not ceremony. QLever canonicalises numeric literals at +index time, reporting `"100"^^xsd:integer` as `xsd:int`. The POS datatype +preflight originally required exactly `xsd:integer`, so it flagged every record +under QLever while passing under Comunica - every QLever run would have ended +`BLOCKED_BY_PREFLIGHT`. Only cross-engine execution surfaces that class of bug. + +It has since caught two more. Comunica's HDT engine treats a bare filesystem +path as a link to dereference, so the source must be addressed as +`hdt@`; and cyvcf2's `raw_header` is htslib's *normalised* header, which +injects `##FILTER=` into files that +never declared it - making the oracle expect a `FilterDefinition` the graph +could not contain. The oracle now reads the header block from the file itself, +which is the same text the conversion reads. + +**When adding a query**, prefer datatype-*family* checks and lexical +comparisons over anything that assumes a store's internal representation, and +run the agreement script before trusting it. + +### Measuring, not just comparing + +Because every engine answers the same query set against the same graph in the +same container, a multi-engine run is also a controlled benchmark, and the +suite records it: per-query wall time for each engine, setup time (a QLever +index build, or an HDT/COTTAS artifact build) kept separate from query time, +and the **oracle** - what it costs to compute the same answers directly from +the VCF with cyvcf2. + +That last figure makes the comparison meaningful rather than merely internal. +The suite computes every expected value twice, once by parsing and once by +querying, so the two costs are for identical work on identical input. The +report is written as `benchmark.json` plus a long-format `benchmark.csv`, one +row per engine and query. See [`validation.md`](validation.md#timings-and-comparing-sparql-against-the-parser). + +## Reproducing the score + +```bash +pip install rdflib + +VCF_RDFIZER_MUTATION_REPORT=mutation-score.json \ + python -m unittest test.test_validation_mutation_unit -v +``` + +`mutation-score.json` records the total, the detected count, the score, and a +per-mutation row with its status and any recorded gap reason. + +Cross-engine agreement, inside the image: + +```bash +docker build -t vcf-rdfizer:local . +docker run --rm -v "$PWD:/repo:ro" vcf-rdfizer:local \ + /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py +``` + +All four engines are compared by default; pass a comma-separated subset as the +first argument to narrow it. + +Both run in CI via +[`.github/workflows/validation-mutation.yml`](../.github/workflows/validation-mutation.yml). + +## Interpreting the number + +The mutation score is relative to the catalogue, not to the space of all +possible conversion bugs. A high score means "almost every corruption we +thought to write down is caught" — it is a lower bound on blindness, not a +proof of correctness. Growing the catalogue is as valuable as raising the +score, and a score that rises without the catalogue growing means nothing. Its value is that it is falsifiable, reproducible, and +moves in a direction you can point at. + +Current score and the full gap list: [`vcf-coverage.md`](vcf-coverage.md). + +## What a validation PASS means + +A `PASS` now means more than it did: with the census and the identity digests, +the graph must contain exactly the predicates and classes the VCF implies, and +every record, INFO value and FORMAT value must hash to the same bucket as its +counterpart in the source. It reliably catches dropped +records, misclassified variants, flipped genotypes, corrupted FILTER strings, +allele-count errors, missing header lines, and altered file metadata. + +It is **not** proof of a faithful record-by-record round-trip. Treat it as a +regression gate. The precise limits are enumerated in +[`vcf-coverage.md`](vcf-coverage.md) and in the "What is *not* tested" section +of [`validation.md`](validation.md). + + +--- + +## See also + +- [Validation](validation.md) — running the suite, and what it does and does not test +- [VCF coverage matrix](vcf-coverage.md) — the current score and the full gap list +- [Data linking design](datalinking-design.md) — how third-party linksets would plug into this harness +- [Roadmap](roadmap.md) — the coverage gaps with named fixes diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..5b45242 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,516 @@ +# Semantic VCF/RDF validation + +*Part of the [VCF-RDFizer documentation](README.md). How coverage is measured: +[`validation-methodology.md`](validation-methodology.md). Element-by-element +results: [`vcf-coverage.md`](vcf-coverage.md).* + +`vcf-rdfizer --mode validation` checks that a converted RDF graph reproduces +six deterministic VCF summaries. The same validator can be added to a full run +with `--validate`; in that case it runs once for each input after RDF creation +and compression. It computes one result from the source VCF +using `cyvcf2` (and `bcftools` for exact FILTER strings when available), runs +the equivalent SPARQL queries against the graph, then compares canonical +integer results exactly. Two axes are configurable and independent: which +artifact is validated (N-Triples, HDT, or COTTAS) and which SPARQL engine runs +the queries (Comunica, QLever, native HDT, native COTTAS - or several at once, +which cross-checks them and benchmarks them against each other and against the +parser). Read "What is *not* tested" below before treating a `PASS` as a +correctness proof. + +The source artifact is mounted read-only. Anything that is not already plain +N-Triples is decoded under `/work` **inside the Docker container**; a plain +`.nt` input is read in place. The validator uses the resulting stream for +Raptor syntax validation and for the SPARQL queries, and removes any temporary +decode before the container exits. No decoded RDF is written beneath `--out`; +only reports are retained. + +## Run it + +Use the representation selected when the RDF was created. + +```bash +# Expanded (the default VCF-RDFizer graph shape) +vcf-rdfizer --mode validation \ + --input ./vcf_data/HG004_GRCh38.vcf.gz \ + --rdf ./results/HG004_GRCh38/HG004_GRCh38.nt.gz \ + --sample-representation expanded \ + --out ./validation-results + +# Condensed multi-sample graph +vcf-rdfizer --mode validation \ + --input ./vcf_data/1000G_phase3_chr20.vcf.gz \ + --rdf ./results/1000G_phase3_chr20/1000G_phase3_chr20.nt.gz \ + --sample-representation condensed \ + --out ./validation-results +``` + +To include validation in the conversion itself, use the full mode's opt-in +stage. It runs after the aggregate and selected compression artifacts are +available, once per VCF input: + +```bash +vcf-rdfizer --mode full \ + --input ./cohort.vcf.gz \ + --sample-representation condensed \ + --rdf-storage-mode space-optimized \ + --rdf-compression none \ + --representations hdt \ + --validate \ + --out ./results +``` + +For standalone validation, the input VCF and the RDF artifact must originate +from the same conversion. + +## Which artifact is validated + +`--rdf` accepts any artifact the pipeline produces: + +| Extension | How it is read | +|---|---| +| `.nt` | Read in place; nothing is copied | +| `.nt.gz` | Expanded into the container scratch directory | +| `.nt.br` | Expanded with `brotli -d` | +| `.hdt` | Decoded with `hdt2rdf` | +| `.cottas` | Decoded with `cottas_tool.py decompress` (pycottas) | +| `.cottas.gz`, `.cottas.br` | Unwrapped, then decoded as above | + +Format is inferred from the filename; `--rdf-format` overrides that for an +artifact with an unusual name. + +Validating an `.hdt` or `.cottas` decodes it back to N-Triples and then runs the +full semantic suite over the result. That is a strictly stronger statement than +the triple-count round-trip `validate_compression.py` performs during +compression: it proves the artifact decodes to a graph that still reproduces +every VCF summary, not merely to the right number of triples. The decode needs +scratch space of roughly the uncompressed graph size under the container's +`/work`, and nothing decoded is ever written beneath `--out`. + +```bash +# Validate a compressed representation directly +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz \ + --rdf ./results/cohort/cohort.hdt \ + --sample-representation condensed \ + --out ./validation-results +``` + +In full mode, `--validate-artifacts` chooses which produced artifacts to check. +Each one is validated independently and gets its own report directory, so a run +can prove the aggregate, the HDT, and the COTTAS file all agree with the VCF: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz \ + --rdf-storage-mode space-optimized \ + --representations hdt,cottas \ + --validate --validate-artifacts all \ + --out ./results +``` + +Accepted values are `aggregate` (the default), `hdt`, `cottas`, `all`, or a +comma-separated subset. A representation that was not selected, or whose +artifact is missing after a recoverable index warning, is skipped rather than +reported as a failure - the run already records why it is absent. + +## Which SPARQL engine runs the queries + +`--validation-engine` selects the backend. Every engine answers the same +queries and feeds the same comparison layer, so the choice is a scale and +performance decision, never a semantic one; every report records which engine +produced it. + +| Engine | Queries | Setup | Use it when | +|---|---|---|---| +| `comunica` (default) | The N-Triples file directly | none | The graph fits comfortably in RAM | +| `qlever` | An on-disk [QLever](https://github.com/ad-freiburg/qlever) index, served on a container-local port | index build | The graph no longer fits in memory, or the aggregate queries are too slow | +| `hdt` | A `.hdt` artifact **in place**, through Comunica's HDT engine | reuses the run's HDT, or builds one | Checking that the compressed artifact is queryable, not just decodable | +| `cottas` | A `.cottas` artifact **in place**, through `pycottas`'s rdflib store | reuses the run's COTTAS, or builds one | Same, for COTTAS | + +### Validating a compressed artifact without decoding it + +`--validation-target hdt,cottas` validates those artifacts by decoding them +back to N-Triples first. That proves the decode is faithful; it says nothing +about querying them, and it measures the wrong thing entirely in a performance +comparison. + +`--validation-engine hdt` and `--validation-engine cottas` instead query the +artifact where it lies. When the run's own artifact is already in the engine's +format it is used directly - the honest measurement. Otherwise the engine +builds one in container scratch from the materialized N-Triples, and that build +is timed as **setup**, kept out of the query total. Each report records which +of the two happened, in `artifactOrigin`. + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz --rdf ./results/cohort/cohort.hdt \ + --validation-engine hdt \ + --out ./validation-results +``` + +HDT is queried with `comunica-sparql-hdt`, installed in the image alongside +Comunica; the source is addressed as `hdt@`, because a bare path would be +treated as a link to dereference. COTTAS is queried in-process through +`pycottas.COTTASStore`, an rdflib `Store` over the Parquet artifact, under the +same interpreter that already owns pycottas. + +### Several engines in one run + +`--validation-engine` accepts a comma-separated list, or `all`: + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --validation-engine all \ + --out ./validation-results +``` + +Every requested engine answers the whole query set. Two things follow: + +- **Cross-checking.** The normalized results of every engine are compared + against each other and written to `engine-agreement.json`. Engines + disagreeing is a finding in its own right - see below. +- **Benchmarking.** Every engine is timed identically, against the same graph, + in the same container, in one run. + +The first engine named is the **primary**. Its reports keep the single-engine +layout at the top of the results directory, so existing consumers are +unaffected; each engine additionally gets `engines//` with its own +`preflight.json`, `sparql.json`, `comparison.json` and `query-executions.json`. +The run's `status` is `PASS` only if every engine passed, and `summary.json` +carries `engineStatuses` with the per-engine verdict. + +### Timings, and comparing SPARQL against the parser + +Every run writes `benchmark.json` and a long-format `benchmark.csv` - one row +per engine and query, ready to plot without reshaping: + +``` +engine,query_id,status,wall_seconds,oracle_wall_seconds,engine_setup_seconds,artifact_origin +``` + +`benchmark.json` adds the breakdown: per-engine setup and total query time, the +slowest query per engine, N-Triples materialization and SHACL time, and the +**oracle** - what it costs to compute the same answers directly from the VCF +with cyvcf2, split into parse and census. That last figure is the point of +comparison: the validation suite computes every expected value twice, once by +parsing and once by querying, so a run measures a SPARQL engine against a +purpose-built parser on identical work. + +An engine that produced no timed query reports `null` rather than `0`, because +zero seconds reads as "instant" rather than "never ran". + +The parent run's `metrics.csv` carries the summary columns +`validation_engines`, `validation_oracle_seconds`, +`validation_engine_query_seconds` and `validation_engine_setup_seconds`, the +last two as `engine=seconds` pairs. + +An indicative single-container run over a small fixture (312 triples, expanded) +gives the shape of the difference rather than a benchmark result: + +| Engine | Setup | 27 queries | Notes | +|---|---|---|---| +| oracle (cyvcf2) | - | 0.003 s | the parser computing the same answers | +| qlever | 0.17 s | 0.31 s | index built once, then served | +| cottas | 0.28 s | 0.97 s | in-process, no subprocess per query | +| hdt | 0.003 s | 23.9 s | one Comunica process per query | +| comunica | - | 25.1 s | one Comunica process per query | + +Comunica's cost here is almost entirely per-process startup, which a +fixture-sized graph cannot amortize; the ordering says nothing about how these +engines behave on a cohort-sized graph. + +```bash +vcf-rdfizer --mode validation \ + --input ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ + --validation-engine qlever --qlever-memory-gb 32 \ + --out ./validation-results +``` + +QLever tuning, all optional: + +| Option | Meaning | +|---|---| +| `--qlever-memory-gb N` | Index and server memory budget (default 4) | +| `--qlever-port N` | Container-local port (default 7019; never published) | +| `--qlever-startup-timeout N` | Seconds to wait for the server after indexing (default 900) | +| `--validation-query-timeout N` | Per-query timeout, every engine (default 3600) | +| `--qlever-index-arg ARG` | Extra argument for the index builder (repeatable) | +| `--qlever-server-arg ARG` | Extra argument for the server (repeatable) | + +QLever's command-line interface has changed across releases. If a future +QLever image disagrees with the defaults, the whole command line can be +replaced without changing code, using `{index}`, `{input}`, `{memory}`, and +`{port}` placeholders: + +```bash +docker run -e QLEVER_INDEX_COMMAND='qlever-index -i {index} -f {input} -F nt -m {memory}' ... +docker run -e QLEVER_SERVER_COMMAND='qlever-server -i {index} -p {port} -m {memory}' ... +``` + +The exact argv that ran is recorded in each report's `manifest.json` under +`engine.commands`. The QLever index lives only in container scratch and is +removed as soon as the queries finish. + +QLever is copied into the image from the upstream `adfreiburg/qlever` image at +build time (`qlever-index` and `qlever-server`, verified against build +`bfd5741`); pin a version with +`--build-arg QLEVER_IMAGE=adfreiburg/qlever:`. That image is built on a +different Ubuntu release, so its release-specific Boost, ICU, jemalloc and +io_uring libraries are copied alongside the binaries into `/opt/qlever/lib`, +and only QLever's own processes are pointed there - they cannot shadow +anything the rest of the image links against. A build-time `ldd` check records +whether they resolve, and the validator reports that note if the engine cannot +start. Comunica remains the default, so an image whose QLever binaries do not +link still validates normally. + +### Engine equivalence, and one place they differed + +Every engine is held to producing identical results. That is verified, not +assumed: [`test/cross_engine_agreement.py`](../test/cross_engine_agreement.py) +runs the full query set under all four engines, against the same expanded and +condensed graphs, and additionally runs the shipped validation decision under +each - so an engine must agree both with the other engines and with the Python +oracle. Engines agreeing with each other and all being wrong is a real failure +mode; only the oracle rules it out. + +A multi-engine production run performs the first half of that check on the +real graph and records it in `engine-agreement.json`. + +One difference had to be fixed to make that true. QLever canonicalises numeric +literals at index time, so `"100"^^xsd:integer` is reported by `DATATYPE()` as +`xsd:int`. `preflight_position_datatype` originally required exactly +`xsd:integer`, which meant it flagged every record on QLever while passing on +Comunica - every run would have been `BLOCKED_BY_PREFLIGHT`. The query now +accepts the XSD integer family, which is what the check actually means. It +still reports a plain-string or `xsd:decimal` POS as an anomaly on both +engines. + +If you add a query, prefer datatype-family checks and lexical comparisons over +assertions about a store's internal numeric representation. + +`--validation-id NAME` changes the report directory name. The default is the +source VCF basename without `.vcf` or `.vcf.gz`. Existing result directories +are never overwritten. `--filter-oracle {auto,bcftools,cyvcf2}` controls the +FILTER-field oracle; `auto` uses `bcftools` when it is available in the image. + +Validation progress uses the same JSONL sidecar protocol as conversion and +partitioned compression. It emits a `validation` task covering every +preflight, exact-count and core query, then records each query start and completion under the +run's temporary `.progress/` area while the host displays it through the +normal Rich/plain progress session. `--quiet` suppresses that terminal display +and the validator's per-query/summary stdout, but still writes command logs, +stage reports, and metrics. `--no-progress` disables sidecar creation as well. + +## What is tested + +The common record-level queries are used for both graph shapes: + +| Query | Exact comparison | +|---|---| +| Q1 | record counts per source contig and zero-based 1 Mb window | +| Q2 | record-level ALT/REF shape classes | +| Q3 | biallelic A/C/G/T SNV transition and transversion counts | +| Q4 | FILTER broad status and exact lexical value | +| Q5 | per-sample genotype class counts | +| Q6 | genotype-derived single-ALT `(AN, AC, siteCount)` distribution | + +| Q7 | file-level `##fileformat`, `##reference` and `##source` declarations | +| Q8 | how many `##` meta-information lines carry each header key | +| Q9 | every predicate in the graph and its triple count | +| Q10 | every asserted class and its resource count | +| Q11 | per-record identity digest, bucketed | +| Q12 | per-INFO-value identity digest, bucketed | +| Q13 | per-FORMAT-value identity digest, bucketed | + +Q9 and Q10 are the completeness check: comparing the graph's inventory against +what the VCF implies catches a predicate that is missing, one with the wrong +cardinality, and one that should not be there at all. Q11-Q13 close the +permutation gap - see +[`validation-methodology.md`](validation-methodology.md#identity-digests-and-why-they-are-histograms). + +Q9-Q13 assume the shipped RML mapping's predicate inventory and IRI templates. +A custom `--rules` changes both by design. `validation_runner.py` has a +`--mapping-policy report-only` setting for exactly that case, which records +those five queries without failing on them and lets the aggregate comparisons +carry the run. + +> **Known gap.** The `vcf-rdfizer` wrapper does not currently forward +> `--mapping-policy`, so the runner always runs `strict` and a *correct* custom +> mapping is reported as `MISMATCH` on Q9-Q13. See +> [`rml-mappings.md`](rml-mappings.md#4-what-a-custom-mapping-costs-in-validation) +> and [`roadmap.md`](roadmap.md#3-validation-mapping-policy-is-not-forwarded). + +Q5/Q6 are only required when the VCF has both samples and a `GT` FORMAT field. +The suite additionally checks N-Triples syntax, record cardinality, POS +datatype, representation profile, and representation-specific sample/GT +inventory before interpreting the result sets. + +### Graph integrity + +Three checks run before any comparison, because a graph that fails one is not +worth comparing: + +| Check | What it catches | +|---|---| +| `preflight_blank_nodes` | Any blank node. Every class in the vocabulary declares an IRI template, so a blank node means a term map produced no IRI - and the record and value digests could not address such a node. | +| `preflight_empty_values` | Empty or whitespace-only literals and IRIs. An empty literal means a value was lost rather than marked missing; an empty IRI means a template substitution collapsed. | +| `preflight_duplicate_triples` | The same statement emitted more than once. | + +Duplicates need explaining, because no SPARQL query can see them: a store holds +a **set**, so a repeated line is collapsed on load and every other check here +counts it once. The only way to detect it is to compare the statements the +parser read against the distinct triples the store holds. Raptor reports the +first, `preflight_distinct_triple_count` the second, and the difference is +exactly the number of redundant statements. This is worth having because +duplicated RDF parts are a known failure mode of the conversion - +`run_conversion.sh` already carries a defensive dedupe for it - and a duplicated +aggregate costs twice the storage for no added information. + +All three are blocking. When an input a check needs is unavailable it reports +`NOT_EVALUATED` rather than `PASS`, so a check that could not run is never +mistaken for a clean result, and never fails the run either. + +Each structural anomaly preflight runs twice: a `LIMIT 100` query returning +example rows for diagnosis, and a companion aggregate returning the **exact** +anomaly count. Reports carry both, plus `sampleTruncated`, so a graph with ten +million anomalies is never confused with one that has a hundred. + +`--strict-conformance` promotes a missing-token conformance failure (a plain +`"."` literal not typed as `vcfr:Null`) from a report-only observation to a +validation failure. + +### SHACL + +`--shacl-shapes PATH` adds an independent structural layer: the graph is checked +against a SHACL shapes file (for example the vocabulary's published +`shacl/vcf-rdfizer-vocabulary.shacl.ttl`) with `pyshacl`, inside the container. +A violation blocks the run, because a structurally wrong graph makes the +aggregate comparisons uninterpretable. + +It is **off by default**: `pyshacl` loads the whole graph into memory, so it is +suitable for a single-sample graph or a sample of a cohort, not for a +cohort-scale aggregate. The report records the conformance verdict, the exact +violation count, the distinct property paths involved, and the first 50 +violations; the full text is written alongside it. + +```bash +vcf-rdfizer --mode validation -i ./sample.vcf.gz --rdf ./results/sample/sample.nt.gz \ + --shacl-shapes ./vocabulary/shacl/vcf-rdfizer-vocabulary.shacl.ttl \ + -o ./validation-results +``` + +For the expanded representation, Q5/Q6 traverse `SampleCall` and +`FormatFieldValue` resources. For the condensed representation, their matching +queries traverse `SampleSet`, `CohortCallMatrix`, and `FormatValueVector`, then +extract each tab-delimited GT value by its `sampleIndex`. This makes the +condensed tests semantically equivalent without inflating the persisted RDF +back into per-sample value resources. + +## What is *not* tested + +The suite is three independent layers - exact aggregate comparisons, a +predicate/class census with per-record and per-value identity digests, and +graph-integrity checks - plus optional SHACL. That is a broad net, but it is +still a net, and its holes are worth stating plainly. Every claim below is +backed by a named mutation in +[`test/validation_mutations.py`](../test/validation_mutations.py), so a change +that closes a gap fails a test rather than passing silently. + +**`vcfr:contigCount` is counted, not read.** The census asserts the predicate is +present the expected number of times; nothing compares its *value*. A derived +contig total that is simply wrong passes. This is the one mutation in the +catalogue that is still undetected. + +**Header line values are compared only through their structured form.** The +attributes that matter - `filterId`, `altId`, `contigId`, contig length/md5/ +assembly, and the INFO/FORMAT declarations - are lifted into their own +properties and compared. The raw `vcfr:headerValue` literal itself is counted +but never read. + +**Multi-valued INFO fields keep only their lexical value.** A `Number=1` INFO +value additionally carries a typed `fieldValueInteger`/`fieldValueDecimal`; a +`Number=A/R/G/.` field does not, because the vocabulary's IRI template gives one +node per key. The full lexical value is still digested, so corruption is caught +- but no per-element typing is checked. + +**The census assumes the shipped mapping.** A custom `--rules` changes the +predicate inventory and the IRI templates by design, so `q09`-`q13` fall back to +report-only and the run rests on the aggregate comparisons alone. That is +correct behaviour, not a bug, but it means a custom mapping is validated more +weakly than the default one. + +**Condensed graphs are not ontology-backed.** The condensed representation uses +17 terms the published vocabulary does not define, so SHACL cannot meaningfully +check it and dereferencing those terms returns nothing. The list is in +[`vcf-coverage.md`](vcf-coverage.md); closing it is work in the vocabulary +repository. + +**A high mutation score is a lower bound on blindness, not a proof.** It says +"almost every corruption we thought to write down is caught". Corruptions nobody +wrote down are, by construction, not measured. + +In short: a `PASS` means the graph contains exactly the predicates and classes +the VCF implies, that every record, INFO value and FORMAT value hashes to the +same bucket as its counterpart in the source, that the summary statistics agree +exactly, and that the graph carries no blank nodes, empty terms or duplicated +statements. It reliably catches dropped records, misclassified variants, flipped +genotypes, corrupted FILTER strings, allele-count errors, permuted record +fields, missing header lines, and altered file metadata. It is still not proof +of a faithful record-by-record round-trip. Treat it as a regression gate. + +These limits are measured rather than asserted. The current mutation score and +the full element-by-element breakdown are in +[`vcf-coverage.md`](vcf-coverage.md); the method is described in +[`validation-methodology.md`](validation-methodology.md). + +## Results and cleanup evidence + +Results are written to the run's canonical metrics tree: + +```text +/run_metrics/__/reports/validation// +``` + +Detailed results live beneath `reports/validation/`, so they are indexed by +the same `summary.json` used for conversion and compression metrics. + +Important files include `summary.json`, `manifest.json`, `parser.json`, +`rdf-validation.json`, `materialization.json`, `preflight.json`, `sparql.json`, +`comparison.json`, `benchmark.json` and `benchmark.csv`. Raw SPARQL Results +JSON, stderr, and query resource logs are in `raw//`; normalized results +are in `normalized/`. `manifest.json` records the engine that ran (including +QLever's exact argv), the source artifact format and checksum, and every decode +step; `materialization.json` records how a compressed or indexed artifact was +turned into N-Triples and how many triples that yielded. + +A multi-engine run adds `engines//` - one `preflight.json`, +`sparql.json`, `comparison.json` and `query-executions.json` per engine - and +`engine-agreement.json`, which records whether every engine returned the same +normalized results and, if not, which queries differed. The top-level reports +remain those of the primary (first-named) engine. + +When several artifacts are validated in one full run, each gets its own +directory: the aggregate keeps `/`, and the representations use +`__hdt/` and `__cottas/`. + +The parent VCF-RDFizer run metrics include +`run_metrics/__/stages/validation/.json`. +That stage report and the detailed `reports/validation/.../summary.json` record +whether a gzip aggregate was decompressed inside the container and that no +validation scratch RDF was retained on the host. + +`PASS` means every required query and invariant matched. `MISMATCH` means that +both paths ran but differ. `BLOCKED_BY_PREFLIGHT` means RDF syntax or core graph +structure failed. `EXECUTION_FAILED` means a parser or query engine could not +complete. + + +--- + +## See also + +- [Validation methodology](validation-methodology.md) — how this suite's coverage is measured +- [VCF coverage matrix](vcf-coverage.md) — element by element, with the mutation that proves each row +- [Representations](representations.md) — the round-trip check that runs during compression +- [Output and metrics](output-and-metrics.md) — where reports land in the run tree +- [Limitations](limitations.md) — consolidated, across the whole tool diff --git a/docs/vcf-coverage.md b/docs/vcf-coverage.md new file mode 100644 index 0000000..37d1a6d --- /dev/null +++ b/docs/vcf-coverage.md @@ -0,0 +1,150 @@ +# VCF coverage matrix + +*Part of the [VCF-RDFizer documentation](README.md). What the conversion emits: +[`conversion.md`](conversion.md). How this table is measured: +[`validation-methodology.md`](validation-methodology.md).* + +What of a VCF file VCF-RDFizer represents in RDF, and what the semantic +validation suite actually verifies. One row per VCF element. This is the +tracking artifact for the coverage work and the source table for publication. + +Two independent questions per row, deliberately kept apart: + +- **Represented** — does the conversion emit RDF for this element at all? +- **Validated** — would a corruption of it be *detected*? Backed by a named + mutation in [`test/validation_mutations.py`](../test/validation_mutations.py), + not by inspection. + +Last measured mutation score: **76/78 (97%)** across 42 distinct mutations. +Regenerate with: + +```bash +VCF_RDFIZER_MUTATION_REPORT=mutation-score.json \ + python -m unittest test.test_validation_mutation_unit +``` + +--- + +## Fixed fields (the eight mandatory columns) + +| VCF element | `records.tsv` | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | --- | +| CHROM | `CHROM` | `vcfr:chrom` | yes | `q01`, `q11` | `corrupt_chrom` | +| POS | `POS` | `vcfr:pos` (`xsd:integer`) | yes | `q01`, `q11`, `preflight_position_datatype` | `corrupt_pos`, `drop_pos`, `retype_pos_as_string` | +| POS ↔ record binding | — | — | yes | `q11_record_digest` | `permute_pos` | +| ID | `ID` | `vcfr:recordId` | yes | `q11_record_digest` | (covered by digest) | +| REF | `REF` | `vcfr:ref` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | +| ALT | `ALT` | `vcfr:alt` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | +| REF/ALT ↔ record binding | — | — | yes | `q11_record_digest` | `permute_ref_alt` | +| QUAL | `QUAL` | `vcfr:qual` (`xsd:decimal` / `vcfr:Null`) | yes | `q09`, `q11` | `drop_qual`, `drop_all_qual`, `corrupt_qual` | +| FILTER | `FILTER` | `vcfr:filter` | yes | `q04` (exact lexical), `q11` | `drop_filter`, `corrupt_filter_lexical` | +| INFO (raw) | `INFO` | `vcfr:infoRaw` | yes | `q11_record_digest` | `corrupt_info_raw` | +| INFO (structured) | `INFO` | `vcfr:hasInfoValue` → `vcfr:InfoFieldValue` → `vcfr:declaredBy` | yes | `q09`, `q12_info_value_digest` | `drop_info_value`, `corrupt_info_value`, `retype_info_value` | + +INFO values carry `vcfr:fieldValue` plus a typed `fieldValueInteger` / +`fieldValueDecimal` when the declaration says `Number=1` and the value parses; +a Flag entry carries `vcfr:fieldValueBoolean true`. A multi-valued field +(`Number=A/R/G/.`) keeps only the lexical value, because the vocabulary's IRI +template gives one node per key. + +## Genotype fields + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| FORMAT declaration | `vcfr:formatRaw` | yes | `q09_predicate_census` | — | +| Sample identity | `vcfr:sampleId` / `vcfr:sampleName` | yes | `preflight_sample_gt_inventory`, `q09` | `drop_sample_call` | +| GT values | `vcfr:hasFormatValue` → `vcfr:fieldValue` | yes | `q05`, `q06`, `q13` | `flip_genotype` | +| Non-GT FORMAT (DP, GQ, AD, PL…) | `vcfr:FormatFieldValue` / `vcfr:FormatValueVector` | yes | `q13_format_value_digest` | `drop_format_value_dp`, `corrupt_format_value_dp`, `corrupt_format_vector` | + +## Header section + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| `##` line count and keys | `vcfr:HeaderLine` + `vcfr:headerKey` | yes | `q08_header_line_census` | `drop_header_line` | +| Line type | `FileFormatHeaderLine`, `INFOHeaderLine`, `ContigHeaderLine`, … | yes | `q10_class_census` | `untype_header_line` | +| `##fileformat` | `vcfr:fileFormat` | yes | `q07_file_metadata` | `corrupt_file_metadata` | +| `##reference` | `vcfr:referenceGenome` | yes | `q07_file_metadata` | `drop_reference_genome` | +| `##source` | `vcfr:sourceSoftware` | yes | `q07_file_metadata` | — | +| `##fileDate` | `vcfr:fileDate` (`xsd:date` when the form allows) | yes | `q09_predicate_census` | `drop_file_date` | +| `##FILTER` | `vcfr:FilterDefinition` + `vcfr:filterId` | yes | `q09`, `q10` | `drop_filter_definition` | +| `##ALT` | `vcfr:AltDefinition` + `vcfr:altId` | yes | `q09`, `q10` | `drop_alt_definition` | +| `##contig` | `vcfr:ContigHeaderLine` + `contigId`/`contigLength`/`contigMd5`/`contigAssembly` | yes | `q09`, `q10` | `drop_contig_attribute` | +| `##INFO` / `##FORMAT` definitions | `vcfr:InfoFieldDefinition` / `vcfr:FormatFieldDefinition` with `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription` | yes | `q09`, `q10` | `drop_info_definition` | +| Header line *values* | `vcfr:headerValue` | yes | `q09` (count only) | — | +| `vcfr:contigCount` | derived scalar | yes | **count only, not value** | `corrupt_contig_count` | + +An unrecognized `##` key keeps only the base `vcfr:HeaderLine` type: inventing +a subclass would put a term in the graph that the vocabulary does not define. + +## Graph-level properties + +| Property | Validated by | Mutation | +| --- | --- | --- | +| N-Triples syntax | Raptor (`rapper -c`) | — | +| SHACL shape conformance | `--shacl-shapes` (opt-in, independent layer) | — | +| Per-record property cardinality | `preflight_record_cardinality` + exact count | `drop_pos` | +| Missing-token policy | `preflight_missing_token_conformance`, fatal under `--strict-conformance` | `plain_dot_literal` | +| Representation profile | `preflight_representation_profile` | `wrong_representation_profile` | +| Record count | q01/q02/q04 totals | `drop_record`, `duplicate_record` | +| **No extraneous triples** | `q09_predicate_census` (extra rows) | `spurious_predicate` | +| **No blank nodes** | `preflight_blank_nodes` | `introduce_blank_node`, `blank_node_object` | +| **No empty or whitespace-only terms** | `preflight_empty_values` | `empty_literal`, `whitespace_only_literal` | +| **No duplicate statements** | `preflight_duplicate_triples` (parsed vs distinct) | `duplicate_triple`, `duplicate_whole_graph` | +| Class inventory | `q10_class_census` | `untype_header_line` | +| Per-record identity | `q11_record_digest` | `permute_pos`, `permute_ref_alt` | + +## Remaining gaps + +1. **`vcfr:contigCount` is counted, not read.** A wrong derived contig total is + not detected. Closing it needs the value in a comparison, not just the + predicate in the census. +2. **Header line values are not compared.** `q08` compares how many lines carry + each key and `q10` their types, but `vcfr:headerValue` itself is only + counted. The structured attributes that matter (`filterId`, `contigId`, …) + *are* covered. +3. **The census assumes the shipped mapping.** A custom `--rules` changes the + inventory and IRI templates by design, so `q09`–`q13` fall back to + report-only. That is correct, but it means a custom mapping is validated + only by the aggregate comparisons. + +## Vocabulary alignment + +The tool emits 17 terms that `https://w3id.org/vcf-rdfizer/vocab#` does **not** +define — all belonging to the condensed representation: + +`CohortCallMatrix`, `CondensedRepresentation`, `ExpandedRepresentation`, +`FormatValueVector`, `SampleSet`, `VCFSample`, `VCFTextVector`, +`appliesToSampleSet`, `encodedValues`, `hasCallMatrix`, `hasFormatValueVector`, +`hasSample`, `hasSampleSet`, `representationProfile`, `sampleIndex`, +`sampleName`, `valueEncoding` + +Dereferencing any of these returns nothing, so **condensed graphs are not +ontology-backed**. This is the one remaining publication blocker and is work in +the vocabulary repository, not here. + +### An open conflict inside the vocabulary + +Running SHACL surfaced a contradiction between two parts of the published +vocabulary that the tool cannot satisfy simultaneously: + +- `vcfr:missingValuePolicy` says a missing token SHOULD be `"."^^vcfr:Null`. +- `VCFRecordShape` constrains `vcfr:alt` to `sh:datatype xsd:string`. + +A record with `ALT=.` therefore violates the shape no matter which rule the +conversion follows. The tool currently follows the missing-value policy. This +needs a decision in the vocabulary: either relax the `alt`/`ref`/`chrom` shapes +to `sh:or([xsd:string] [vcfr:Null])`, as the `qual` shape already does, or drop +the missing-value policy for those fields. + +Two shape violations found the same way have already been fixed here: QUAL is +now `xsd:decimal`/`vcfr:Null` rather than a plain literal, and `##fileDate` is +typed `xsd:date` when its form allows. + +--- + +## See also + +- [Validation methodology](validation-methodology.md) — how the coverage in this table is measured +- [Validation](validation.md) — running the validator +- [Conversion](conversion.md) — how each represented element is emitted +- [Roadmap](roadmap.md) — the vocabulary and coverage gaps above, with their planned fixes diff --git a/pyproject.toml b/pyproject.toml index c39ec03..695dcc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,9 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Bio-Informatics", ] -dependencies = [] +dependencies = [ + "rich>=13.7.0", +] [project.urls] Homepage = "https://github.com/ecrum19/VCF-RDFizer" @@ -32,15 +34,19 @@ Issues = "https://github.com/ecrum19/VCF-RDFizer/issues" [project.scripts] vcf-rdfizer = "vcf_rdfizer:main" +vcf-rdfizer-rules = "vcf_rdfizer_rules:main" [project.optional-dependencies] dev = [ "build>=1.2.2", "twine>=5.1.1", + # Test-only: an in-process SPARQL engine for the validation mutation harness, + # so it needs no Docker. The suite skips those tests when it is absent. + "rdflib>=7.0.0", ] [tool.setuptools] -py-modules = ["vcf_rdfizer"] +py-modules = ["vcf_rdfizer", "vcf_rdfizer_gzip", "vcf_rdfizer_rules"] packages = ["vcf_rdfizer_data", "vcf_rdfizer_data.rules"] include-package-data = true diff --git a/rules/README.md b/rules/README.md index 75ce2a6..32edbe1 100644 --- a/rules/README.md +++ b/rules/README.md @@ -2,6 +2,18 @@ This directory contains RML mappings used by the conversion pipeline. +Writing your own mapping? Use the `vcf-rdfizer-rules` CLI, which documents the +available TSV columns, scaffolds a starting point, and validates a mapping +against the wrapper's contract before you spend a run on it: + +```bash +vcf-rdfizer-rules columns # what a mapping can reference +vcf-rdfizer-rules init -o my.ttl # annotated copy of default_rules.ttl +vcf-rdfizer-rules check my.ttl # validate before running the pipeline +``` + +See "Custom RML Mappings" in the top-level `README.md` for the full contract. + ## Files - `default_rules.ttl` @@ -13,9 +25,15 @@ This directory contains RML mappings used by the conversion pipeline. - `/data/tsv/records.tsv` - `/data/tsv/sample_calls.tsv` - `/data/tsv/sample_format_values.tsv` - - `sample_calls.tsv` and `sample_format_values.tsv` are derived by the Python wrapper - from `records.tsv` at runtime so FORMAT fields (e.g., `GT:DP:AD`) can be - mapped to per-sample values consistently. + - For the built-in sample maps, `sample_calls.tsv` and + `sample_format_values.tsv` are header-only compatibility sources. In + `--sample-representation expanded`, the Python wrapper streams their equivalent + `SampleCall` and `FormatFieldValue` triples directly from `records.tsv`. In + `--sample-representation condensed`, it instead streams `SampleSet`, + `CohortCallMatrix`, and `FormatValueVector` resources. Only one emitter runs. + - Custom mappings with additional consumers of either helper source retain + materialized TSV generation in expanded mode. They are rejected in condensed mode + to prevent simultaneous expanded and condensed output. - The Python wrapper rewrites these template paths per input VCF to: - `/data/tsv/.file_metadata.tsv` - `/data/tsv/.header_lines.tsv` @@ -32,9 +50,14 @@ This directory contains RML mappings used by the conversion pipeline. 5. Run the wrapper with your custom mapping: ```bash -python3 vcf_rdfizer.py --input --rules rules/my_rules.ttl +python3 vcf_rdfizer.py --input --rules rules/my_rules.ttl \ + --rdf-storage-mode plain --out ./results ``` +Custom rules that do not consume either sample helper table can be used with +`--sample-representation condensed`; the wrapper adds the condensed sample graph +after RMLStreamer emits the custom record-level graph. + ## SHACL Notes The related SHACL constraints are maintained in the vocabulary repository: @@ -47,3 +70,5 @@ The default mapping is structured to align with those classes/properties, especi - `vcfr:VCFHeader` + `vcfr:hasHeaderLine` - `vcfr:VCFRecord` core fields (`chrom`, `pos`, `ref`, `alt`) - `vcfr:VariantCall` with raw call attributes +- expanded `vcfr:SampleCall` / `vcfr:FormatFieldValue` resources, or condensed + `vcfr:CohortCallMatrix` / `vcfr:FormatValueVector` resources diff --git a/rules/default_rules.ttl b/rules/default_rules.ttl index ba357f4..d229b3c 100644 --- a/rules/default_rules.ttl +++ b/rules/default_rules.ttl @@ -11,9 +11,12 @@ # - /data/tsv/.file_metadata.tsv # - /data/tsv/.header_lines.tsv # - /data/tsv/.records.tsv -# - /data/tsv/.sample_calls.tsv (derived by wrapper) -# - /data/tsv/.sample_format_values.tsv (derived by wrapper) +# - /data/tsv/.sample_calls.tsv (header-only compatibility source) +# - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. +# The wrapper selects exactly one direct sample emitter: expanded SampleCall / +# FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / +# FormatValueVector triples. The helper tables stay header-only in either mode. # # Output format: # - The default conversion now targets triples (N-Triples) without named graph terms. @@ -44,6 +47,9 @@ rr:predicate vcfr:fileFormat ; rr:objectMap [ rml:reference "FILE_FORMAT" ] ] ; + # fileDate is emitted by the wrapper: the published shape requires xsd:date, + # but ##fileDate has no mandated format in the VCF specification, so the + # value has to be inspected before it can be typed. rr:predicateObjectMap [ rr:predicate vcfr:sourceSoftware ; rr:objectMap [ rml:reference "SOURCE_SOFTWARE" ] @@ -212,6 +218,9 @@ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ; rr:class vcfr:VariantCall ] ; + # QUAL is emitted by the wrapper, not here: the published SHACL shape + # requires xsd:decimal or vcfr:Null depending on whether the value is + # present, and RML cannot choose a datatype per row. rr:predicateObjectMap [ rr:predicate vcfr:filter ; rr:objectMap [ rml:reference "FILTER" ] diff --git a/RELEASING.md b/scripts/RELEASING.md similarity index 100% rename from RELEASING.md rename to scripts/RELEASING.md diff --git a/src/compression.sh b/src/compression.sh deleted file mode 100644 index a1bf33b..0000000 --- a/src/compression.sh +++ /dev/null @@ -1,643 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# ------------------------------------------------------------------------------ -# RDF compression runner -# ------------------------------------------------------------------------------ -# This script can be used standalone, but the Python wrapper is the primary -# caller. It performs selected compression methods over RDF outputs and updates -# run_metrics/metrics.csv with method-level timing/size/exit-code fields. -# -# NOTE: Compound HDT-first compression is orchestrated by the Python wrapper. -# Here we maintain explicit metric columns so CSV schema stays aligned. -# ------------------------------------------------------------------------------ - -# ---------- Config ---------- -# Root output directory; contains one or more output subdirs -OUT_ROOT_DIR=${OUT_ROOT_DIR:-run_output} -# Optional single output name (compress only this subdir) -OUT_NAME=${OUT_NAME:-} - -# Metrics directory -LOGDIR=${LOGDIR:-run_metrics} - -# rdf2hdt binary (HDT-cpp) -HDT=${RDF2HDT:-${RDF2HDT_BIN:-/usr/local/bin/rdf2hdt}} - -# Base URI for rdf2hdt (reserved for future use) -BASE_URI=${BASE_URI:-http://example.org/base} - -RUN_ID=${RUN_ID:-$(date +%Y%m%dT%H%M%S)} -TIMESTAMP=${TIMESTAMP:-$(date +"%Y-%m-%dT%H:%M:%S")} - -mkdir -p "$LOGDIR" "$OUT_ROOT_DIR" - -METRICS_CSV="$LOGDIR/metrics.csv" -METRICS_HEADER="run_id,timestamp,output_name,output_dir,exit_code_java,wall_seconds_java,user_seconds_java,sys_seconds_java,max_rss_kb_java,input_mapping_size_bytes,input_vcf_size_bytes,output_dir_size_bytes,output_triples,jar,mapping_file,output_path,combined_rdf_size_bytes,gzip_size_bytes,brotli_size_bytes,hdt_size_bytes,exit_code_gzip,exit_code_brotli,exit_code_hdt,wall_seconds_gzip,user_seconds_gzip,sys_seconds_gzip,max_rss_kb_gzip,wall_seconds_brotli,user_seconds_brotli,sys_seconds_brotli,max_rss_kb_brotli,wall_seconds_hdt,user_seconds_hdt,sys_seconds_hdt,max_rss_kb_hdt,compression_methods,hdt_source,gzip_on_hdt_size_bytes,brotli_on_hdt_size_bytes,exit_code_gzip_on_hdt,exit_code_brotli_on_hdt,wall_seconds_gzip_on_hdt,user_seconds_gzip_on_hdt,sys_seconds_gzip_on_hdt,max_rss_kb_gzip_on_hdt,wall_seconds_brotli_on_hdt,user_seconds_brotli_on_hdt,sys_seconds_brotli_on_hdt,max_rss_kb_brotli_on_hdt" - -# ---------- Compression selection ---------- -# Usage: compression.sh [-m gzip,brotli,hdt|none] -# Or set COMPRESSION_METHODS env var (default: gzip,brotli,hdt) -COMPRESSION_METHODS=${COMPRESSION_METHODS:-gzip,brotli,hdt} - -while getopts ":m:h" opt; do - case "$opt" in - m) COMPRESSION_METHODS="$OPTARG" ;; - h) - echo "Usage: $0 [-m gzip,brotli,hdt|none]" - exit 0 - ;; - \?) - echo "Error: invalid option -$OPTARG" >&2 - exit 2 - ;; - :) - echo "Error: option -$OPTARG requires an argument" >&2 - exit 2 - ;; - esac -done - -COMPRESSION_METHODS_CSV=${COMPRESSION_METHODS//,/|} - -DO_GZIP=0 -DO_BROTLI=0 -DO_HDT=0 - -if [[ -n "${COMPRESSION_METHODS// }" && "$COMPRESSION_METHODS" != "none" ]]; then - IFS=',' read -r -a METHODS_ARR <<< "$COMPRESSION_METHODS" - for method in "${METHODS_ARR[@]}"; do - m="${method// /}" - case "$m" in - gzip) DO_GZIP=1 ;; - brotli) DO_BROTLI=1 ;; - hdt) DO_HDT=1 ;; - "" ) ;; - *) - echo "Error: unsupported compression method '$m'. Use gzip,brotli,hdt, or none." >&2 - exit 2 - ;; - esac - done -fi - -ANY_COMPRESS=$((DO_GZIP + DO_BROTLI + DO_HDT)) - -if (( DO_HDT == 1 )); then - if [[ ! -x "$HDT" ]]; then - echo "Error: rdf2hdt binary not found or not executable at '$HDT'." >&2 - exit 2 - fi -fi - -# Resolve output directories to compress -OUTPUT_DIRS=() -if [[ -n "$OUT_NAME" ]]; then - OUTPUT_DIRS=("$OUT_ROOT_DIR/$OUT_NAME") -else - while IFS= read -r discovered_dir; do - OUTPUT_DIRS+=("$discovered_dir") - done < <(find "$OUT_ROOT_DIR" -maxdepth 1 -type d ! -path "$OUT_ROOT_DIR" | sort) - if (( ${#OUTPUT_DIRS[@]} == 0 )); then - OUTPUT_DIRS=("$OUT_ROOT_DIR") - fi -fi - -if (( ${#OUTPUT_DIRS[@]} == 0 )); then - echo "Error: no output directories found in '$OUT_ROOT_DIR'." >&2 - exit 2 -fi - -# ---------- Helper functions ---------- -# Return byte size for file or directory (GNU + BSD compatible). -stat_size() { - local path="$1" - - # --- CASE 1: Regular file --- - if [[ -f "$path" ]]; then - # Linux (GNU coreutils) - if stat -c%s "$path" >/dev/null 2>&1; then - stat -c%s "$path" - # macOS/BSD - elif stat -f%z "$path" >/dev/null 2>&1; then - stat -f%z "$path" - else - wc -c < "$path" | tr -d ' ' - fi - return - fi - - # --- CASE 2: Directory --- - if [[ -d "$path" ]]; then - # Linux (GNU du) - if du -sb "$path" >/dev/null 2>&1; then - du -sb "$path" | awk '{print $1}' - # macOS/BSD (no -b) - elif du -sk "$path" >/dev/null 2>&1; then - local kb - kb=$(du -sk "$path" | awk '{print $1}') - echo $((kb * 1024)) - else - echo 0 - fi - return - fi - - echo 0 -} - -have_gnu_time() { [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; } - -# Count triples via number of non-comment lines ending in ".". -count_triples_json() { - local path="$1" - local total=0 - - echo "{" - shopt -s nullglob - - for f in "$path"/*; do - if [[ -f "$f" ]]; then - local count - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$f" || true) | wc -l | tr -d ' ' ) - total=$((total + count)) - printf " \"%s\": %s,\n" "$f" "$count" - fi - done - - shopt -u nullglob - printf " \"TOTAL\": %s\n" "$total" - echo "}" -} - -# Convert elapsed clock text from `time` output to numeric seconds. -elapsed_to_seconds() { - awk -F':' '{ - if (NF==3) { h=$1+0; m=$2+0; s=$3+0; printf("%.3f", h*3600 + m*60 + s) } - else if (NF==2) { m=$1+0; s=$2+0; printf("%.3f", m*60 + s) } - else { s=$1+0; printf("%.3f", s) } - }' -} - -# CSV header -if [[ ! -f "$METRICS_CSV" ]]; then - echo "$METRICS_HEADER" > "$METRICS_CSV" -else - EXISTING_HEADER=$(head -n 1 "$METRICS_CSV") - if [[ "$EXISTING_HEADER" != "$METRICS_HEADER" ]]; then - BACKUP="$LOGDIR/metrics_csv_bak_${RUN_ID}.csv" - cp "$METRICS_CSV" "$BACKUP" - echo "WARNING: metrics header mismatch; backed up to $BACKUP and creating new metrics file." >&2 - echo "$METRICS_HEADER" > "$METRICS_CSV" - fi -fi - -# ---------- Main loop over output dirs ---------- -OVERALL_EXIT=0 - -for OUT in "${OUTPUT_DIRS[@]}"; do - if [[ ! -d "$OUT" ]]; then - echo "WARNING: output directory '$OUT' not found, skipping." >&2 - OVERALL_EXIT=1 - continue - fi - mkdir -p "$OUT" - - BASENAME=$(basename "$OUT") - SAFE_BASENAME=$(printf "%s" "$BASENAME" | tr -cs 'A-Za-z0-9._-' '_') - if [[ -z "$SAFE_BASENAME" ]]; then - SAFE_BASENAME="rdf" - fi - - TIME_LOG_GZIP_DIR="$LOGDIR/compression_time/gzip/${SAFE_BASENAME}" - TIME_LOG_BROTLI_DIR="$LOGDIR/compression_time/brotli/${SAFE_BASENAME}" - TIME_LOG_HDT_DIR="$LOGDIR/compression_time/hdt/${SAFE_BASENAME}" - METRICS_JSON_DIR="$LOGDIR/compression_metrics/${SAFE_BASENAME}" - mkdir -p "$TIME_LOG_GZIP_DIR" "$TIME_LOG_BROTLI_DIR" "$TIME_LOG_HDT_DIR" "$METRICS_JSON_DIR" - TIME_LOG_GZIP="$TIME_LOG_GZIP_DIR/${RUN_ID}.txt" - TIME_LOG_BROTLI="$TIME_LOG_BROTLI_DIR/${RUN_ID}.txt" - TIME_LOG_HDT="$TIME_LOG_HDT_DIR/${RUN_ID}.txt" - METRICS_JSON="$METRICS_JSON_DIR/${RUN_ID}.json" - - HDT_SOURCE="not_used" - GZIP_ON_HDT_SIZE=0 - BROTLI_ON_HDT_SIZE=0 - EXIT_CODE_GZIP_ON_HDT=0 - EXIT_CODE_BROTLI_ON_HDT=0 - WALL_SEC_GZIP_ON_HDT="null" - USER_SEC_GZIP_ON_HDT="null" - SYS_SEC_GZIP_ON_HDT="null" - MAX_RSS_KB_GZIP_ON_HDT="null" - WALL_SEC_BROTLI_ON_HDT="null" - USER_SEC_BROTLI_ON_HDT="null" - SYS_SEC_BROTLI_ON_HDT="null" - MAX_RSS_KB_BROTLI_ON_HDT="null" - - # Collect current output footprint/triples before compression. - OUT_SIZE=$(stat_size "$OUT") - TRIPLES_JSON=$(count_triples_json "$OUT") - TOTAL_TRIPLES=$(echo "$TRIPLES_JSON" | grep '"TOTAL"' | awk -F': ' '{print $2}' | tr -d '", ') - - shopt -s nullglob - NT_FILES=("$OUT"/*.nt) - shopt -u nullglob - PRIMARY_NT="$OUT/${BASENAME}.nt" - - if (( ${#NT_FILES[@]} == 0 )); then - echo "WARNING: no .nt files found in '$OUT'; skipping compression for this output." >&2 - SOURCE_RDF="" - SOURCE_EXT="" - RDF_SIZE=0 - GZ_PATH="" - GZ_SIZE=0 - BROTLI_PATH="" - BROTLI_SIZE=0 - HDT_PATH="" - HDT_SIZE=0 - EXIT_CODE_GZIP=$(( DO_GZIP == 1 ? 1 : 0 )) - EXIT_CODE_BROTLI=$(( DO_BROTLI == 1 ? 1 : 0 )) - EXIT_CODE_HDT=$(( DO_HDT == 1 ? 1 : 0 )) - WALL_SEC_GZIP="null" - USER_SEC_GZIP="null" - SYS_SEC_GZIP="null" - MAX_RSS_KB_GZIP="null" - WALL_SEC_BROTLI="null" - USER_SEC_BROTLI="null" - SYS_SEC_BROTLI="null" - MAX_RSS_KB_BROTLI="null" - WALL_SEC_HDT="null" - USER_SEC_HDT="null" - SYS_SEC_HDT="null" - MAX_RSS_KB_HDT="null" - if (( ANY_COMPRESS > 0 )); then - OVERALL_EXIT=1 - fi - else - if (( ANY_COMPRESS > 0 )); then - if [[ -f "$PRIMARY_NT" ]]; then - SOURCE_RDF="$PRIMARY_NT" - SOURCE_EXT="nt" - elif (( ${#NT_FILES[@]} == 1 )); then - SOURCE_RDF="${NT_FILES[0]}" - SOURCE_EXT="nt" - else - echo "WARNING: unable to determine a unique primary RDF file in '$OUT'." >&2 - SOURCE_RDF="" - SOURCE_EXT="" - fi - - if [[ -z "$SOURCE_RDF" ]]; then - RDF_SIZE=0 - EXIT_CODE_GZIP=$(( DO_GZIP == 1 ? 1 : 0 )) - EXIT_CODE_BROTLI=$(( DO_BROTLI == 1 ? 1 : 0 )) - EXIT_CODE_HDT=$(( DO_HDT == 1 ? 1 : 0 )) - WALL_SEC_GZIP="null" - USER_SEC_GZIP="null" - SYS_SEC_GZIP="null" - MAX_RSS_KB_GZIP="null" - WALL_SEC_BROTLI="null" - USER_SEC_BROTLI="null" - SYS_SEC_BROTLI="null" - MAX_RSS_KB_BROTLI="null" - WALL_SEC_HDT="null" - USER_SEC_HDT="null" - SYS_SEC_HDT="null" - MAX_RSS_KB_HDT="null" - OVERALL_EXIT=1 - else - RDF_SIZE=$(stat_size "$SOURCE_RDF") - fi - else - SOURCE_RDF="" - SOURCE_EXT="" - RDF_SIZE=0 - fi - - # ----- gzip raw RDF with timing ----- - if (( DO_GZIP == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then - GZ_PATH="$OUT/${BASENAME}.${SOURCE_EXT}.gz" - EXIT_CODE_GZIP=0 - - if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG_GZIP" -- gzip -c "$SOURCE_RDF" > "$GZ_PATH" || EXIT_CODE_GZIP=$? - else - { time -p gzip -c "$SOURCE_RDF" > "$GZ_PATH"; } >"$TIME_LOG_GZIP" 2>&1 || EXIT_CODE_GZIP=$? - fi - - GZ_SIZE=$(stat_size "$GZ_PATH") - - WALL_SEC_GZIP="" - USER_SEC_GZIP="" - SYS_SEC_GZIP="" - MAX_RSS_KB_GZIP="" - - if have_gnu_time; then - ELAPSED=$(awk -F': ' '/Elapsed \(wall clock\) time/ {print $2}' "$TIME_LOG_GZIP") - WALL_SEC_GZIP=$(printf "%s" "$ELAPSED" | elapsed_to_seconds) - - USER_SEC_GZIP=$(awk -F': ' '/User time \(seconds\)/ {print $2}' "$TIME_LOG_GZIP") - SYS_SEC_GZIP=$(awk -F': ' '/System time \(seconds\)/ {print $2}' "$TIME_LOG_GZIP") - MAX_RSS_KB_GZIP=$(awk -F': ' '/Maximum resident set size/ {print $2}' "$TIME_LOG_GZIP") - else - WALL_SEC_GZIP=$(awk '/^real/ {print $2}' "$TIME_LOG_GZIP") - USER_SEC_GZIP=$(awk '/^user/ {print $2}' "$TIME_LOG_GZIP") - SYS_SEC_GZIP=$(awk '/^sys/ {print $2}' "$TIME_LOG_GZIP") - MAX_RSS_KB_GZIP="" - fi - - [[ -z "$MAX_RSS_KB_GZIP" ]] && MAX_RSS_KB_GZIP="null" - if [[ "$EXIT_CODE_GZIP" -ne 0 ]]; then - OVERALL_EXIT=1 - fi - else - GZ_PATH="" - GZ_SIZE=0 - EXIT_CODE_GZIP=0 - WALL_SEC_GZIP="null" - USER_SEC_GZIP="null" - SYS_SEC_GZIP="null" - MAX_RSS_KB_GZIP="null" - fi - - # ----- brotli raw RDF with timing ----- - if (( DO_BROTLI == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then - BROTLI_PATH="$OUT/${BASENAME}.${SOURCE_EXT}.br" - EXIT_CODE_BROTLI=0 - - if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG_BROTLI" -- brotli -q 7 -c "$SOURCE_RDF" > "$BROTLI_PATH" || EXIT_CODE_BROTLI=$? - else - { time -p brotli -q 7 -c "$SOURCE_RDF" > "$BROTLI_PATH"; } >"$TIME_LOG_BROTLI" 2>&1 || EXIT_CODE_BROTLI=$? - fi - - BROTLI_SIZE=$(stat_size "$BROTLI_PATH") - - WALL_SEC_BROTLI="" - USER_SEC_BROTLI="" - SYS_SEC_BROTLI="" - MAX_RSS_KB_BROTLI="" - - if have_gnu_time; then - ELAPSED=$(awk -F': ' '/Elapsed \(wall clock\) time/ {print $2}' "$TIME_LOG_BROTLI") - WALL_SEC_BROTLI=$(printf "%s" "$ELAPSED" | elapsed_to_seconds) - USER_SEC_BROTLI=$(awk -F': ' '/User time \(seconds\)/ {print $2}' "$TIME_LOG_BROTLI") - SYS_SEC_BROTLI=$(awk -F': ' '/System time \(seconds\)/ {print $2}' "$TIME_LOG_BROTLI") - MAX_RSS_KB_BROTLI=$(awk -F': ' '/Maximum resident set size/ {print $2}' "$TIME_LOG_BROTLI") - else - WALL_SEC_BROTLI=$(awk '/^real/ {print $2}' "$TIME_LOG_BROTLI") - USER_SEC_BROTLI=$(awk '/^user/ {print $2}' "$TIME_LOG_BROTLI") - SYS_SEC_BROTLI=$(awk '/^sys/ {print $2}' "$TIME_LOG_BROTLI") - MAX_RSS_KB_BROTLI="" - fi - - [[ -z "$MAX_RSS_KB_BROTLI" ]] && MAX_RSS_KB_BROTLI="null" - if [[ "$EXIT_CODE_BROTLI" -ne 0 ]]; then - OVERALL_EXIT=1 - fi - else - BROTLI_PATH="" - BROTLI_SIZE=0 - EXIT_CODE_BROTLI=0 - WALL_SEC_BROTLI="null" - USER_SEC_BROTLI="null" - SYS_SEC_BROTLI="null" - MAX_RSS_KB_BROTLI="null" - fi - - # ----- Convert raw RDF to HDT with timing ----- - if (( DO_HDT == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then - HDT_PATH="$OUT/$BASENAME.hdt" - HDT_SOURCE="generated" - EXIT_CODE_HDT=0 - - HDT_CMD="\"$HDT\" \"$SOURCE_RDF\" \"$HDT_PATH\"" - - if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG_HDT" -- bash -lc "$HDT_CMD" || EXIT_CODE_HDT=$? - else - { time -p bash -lc "$HDT_CMD"; } >"$TIME_LOG_HDT" 2>&1 || EXIT_CODE_HDT=$? - fi - - HDT_SIZE=$(stat_size "$HDT_PATH") - - WALL_SEC_HDT="" - USER_SEC_HDT="" - SYS_SEC_HDT="" - MAX_RSS_KB_HDT="" - - if have_gnu_time; then - ELAPSED=$(awk -F': ' '/Elapsed \(wall clock\) time/ {print $2}' "$TIME_LOG_HDT") - WALL_SEC_HDT=$(printf "%s" "$ELAPSED" | elapsed_to_seconds) - - USER_SEC_HDT=$(awk -F': ' '/User time \(seconds\)/ {print $2}' "$TIME_LOG_HDT") - SYS_SEC_HDT=$(awk -F': ' '/System time \(seconds\)/ {print $2}' "$TIME_LOG_HDT") - MAX_RSS_KB_HDT=$(awk -F': ' '/Maximum resident set size/ {print $2}' "$TIME_LOG_HDT") - else - WALL_SEC_HDT=$(awk '/^real/ {print $2}' "$TIME_LOG_HDT") - USER_SEC_HDT=$(awk '/^user/ {print $2}' "$TIME_LOG_HDT") - SYS_SEC_HDT=$(awk '/^sys/ {print $2}' "$TIME_LOG_HDT") - MAX_RSS_KB_HDT="" - fi - - [[ -z "$MAX_RSS_KB_HDT" ]] && MAX_RSS_KB_HDT="null" - if [[ "$EXIT_CODE_HDT" -ne 0 ]]; then - if [[ "$EXIT_CODE_HDT" -eq 127 ]]; then - echo "ERROR: HDT conversion failed with exit code 127 (command not found)." >&2 - echo "ERROR: Check dependencies used by '$HDT' and the timing log '$TIME_LOG_HDT'." >&2 - if [[ -f "$TIME_LOG_HDT" ]]; then - tail -n 20 "$TIME_LOG_HDT" >&2 || true - fi - fi - OVERALL_EXIT=1 - fi - else - HDT_PATH="" - HDT_SIZE=0 - EXIT_CODE_HDT=0 - WALL_SEC_HDT="null" - USER_SEC_HDT="null" - SYS_SEC_HDT="null" - MAX_RSS_KB_HDT="null" - fi - fi - - # Persist per-output compression metrics as JSON. - cat > "$METRICS_JSON" < "$tmp_csv" - mv "$tmp_csv" "$METRICS_CSV" - - echo "Done for $OUT." - echo " JSON metrics: $METRICS_JSON" - echo -done - -echo "Compression finished." -echo "CSV summary: $METRICS_CSV" - -if [[ "$OVERALL_EXIT" -ne 0 ]]; then - echo "Compression completed with one or more errors." >&2 -fi -exit "$OVERALL_EXIT" diff --git a/src/cottas_tool.py b/src/cottas_tool.py index c8ddc23..a119227 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -"""Small Docker-side adapter for the pycottas conversion and merge API.""" +"""Docker-side adapter for bounded-memory COTTAS conversion and merging.""" import argparse +import heapq +import json import os import sys import tempfile @@ -9,9 +11,14 @@ from pathlib import Path +DEFAULT_COTTAS_MERGE_BATCH_ROWS = 2048 +COTTAS_OUTPUT_BATCH_ROWS = 16 * 1024 +COTTAS_MERGE_PROGRESS_ROWS = 250_000 + + @contextmanager def cottas_scratch_workspace(): - """Run one pycottas operation with an isolated DuckDB working directory.""" + """Run one COTTAS operation with an isolated DuckDB working directory.""" scratch_root = Path(os.environ.get("COTTAS_SCRATCH_DIR", "/work")).resolve() scratch_root.mkdir(parents=True, exist_ok=True) original_working_directory = Path.cwd() @@ -27,6 +34,297 @@ def cottas_scratch_workspace(): os.chdir(original_working_directory) +def cottas_merge_batch_rows() -> int: + """Return a small, bounded Parquet batch size for the streaming merge.""" + raw_rows = os.environ.get( + "COTTAS_MERGE_BATCH_ROWS", str(DEFAULT_COTTAS_MERGE_BATCH_ROWS) + ).strip() + try: + batch_rows = int(raw_rows) + except ValueError as exc: + raise ValueError("COTTAS_MERGE_BATCH_ROWS must be a positive integer") from exc + if batch_rows <= 0: + raise ValueError("COTTAS_MERGE_BATCH_ROWS must be a positive integer") + return batch_rows + + +def emit_merge_progress( + progress_path: Path | None, + phase: str, + *, + completed: int, + total: int, + detail: str, +) -> None: + """Append a best-effort COTTAS merge heartbeat for the host progress UI.""" + if progress_path is None: + return + payload = { + "stage": "cottas-merge", + "phase": phase, + "completed": completed, + "total": total, + "unit": "triples", + "detail": detail, + } + try: + with progress_path.open("a", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + except OSError: + # A missing/unwritable optional sidecar must not invalidate an index. + pass + + +def cottas_file_index(parquet_file) -> str | None: + """Read COTTAS's embedded Parquet sort-index metadata when available.""" + metadata = getattr(parquet_file.metadata, "metadata", None) or {} + value = metadata.get(b"index") or metadata.get("index") + if value is None: + return None + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + return str(value).lower() + + +class CottasTripleStream: + """Read one sorted COTTAS Parquet file in a bounded number of rows.""" + + def __init__(self, parquet_module, path: Path, index: str, batch_rows: int): + self.path = path + self.index = index + self.parquet_file = parquet_module.ParquetFile(path) + self.schema = self.parquet_file.schema_arrow + missing_columns = {"s", "p", "o"} - set(self.schema.names) + if missing_columns: + raise RuntimeError( + f"COTTAS input {path.name} is missing columns: {', '.join(sorted(missing_columns))}" + ) + source_index = cottas_file_index(self.parquet_file) + if source_index != index: + found = source_index or "missing" + raise RuntimeError( + f"COTTAS input {path.name} is indexed as {found!r}, not {index!r}; " + "a streaming merge requires every input to use the requested index" + ) + self.fields = tuple(self.schema.field(name) for name in ("s", "p", "o")) + positions = {"s": 0, "p": 1, "o": 2} + self._sort_positions = tuple(positions[column] for column in index) + self._batches = self.parquet_file.iter_batches( + batch_size=batch_rows, + columns=["s", "p", "o"], + use_threads=False, + ) + self._values: tuple[list, list, list] | None = None + self._row = 0 + self.current: tuple | None = None + self.sort_key: tuple | None = None + self._previous_sort_key: tuple | None = None + self.exhausted = False + self.advance() + + @property + def row_count(self) -> int: + return int(self.parquet_file.metadata.num_rows) + + def advance(self) -> None: + """Move to one next triple, validating the COTTAS sort-order contract.""" + while self._values is None or self._row >= len(self._values[0]): + try: + batch = next(self._batches) + except StopIteration: + self.current = None + self.sort_key = None + self.exhausted = True + return + values = tuple(column.to_pylist() for column in batch.columns) + if not values[0]: + continue + self._values = values + self._row = 0 + + triple = (self._values[0][self._row], self._values[1][self._row], self._values[2][self._row]) + self._row += 1 + if any(value is None for value in triple): + raise RuntimeError(f"COTTAS input {self.path.name} contains a null RDF term") + sort_key = tuple(triple[position] for position in self._sort_positions) + if self._previous_sort_key is not None and sort_key < self._previous_sort_key: + raise RuntimeError( + f"COTTAS input {self.path.name} is not sorted by its declared {self.index!r} index" + ) + self._previous_sort_key = sort_key + self.current = triple + self.sort_key = sort_key + + def close(self) -> None: + """Release a Parquet file handle before the surrounding workspace exits.""" + close = getattr(self.parquet_file, "close", None) + if callable(close): + close() + + +def streaming_cottas_merge( + input_paths: list[str], + output_path: str, + *, + index: str, + remove_input_files: bool, + progress_path: Path | None = None, +) -> None: + """Merge already-indexed COTTAS files without a global sort or spill area. + + Chunk conversion writes every COTTAS input in the requested lexical index + order. A k-way heap therefore needs only one small Parquet batch from each + input; equal triples meet at the heap head and are written once. This + preserves the RDF set and COTTAS index semantics while avoiding DuckDB's + full-data external sort, whose temporary files can exceed the original RDF + size for large multi-sample VCFs. + """ + if not input_paths: + raise ValueError("at least one COTTAS input is required for a merge") + if not index or set(index.lower()) != {"s", "p", "o"} or len(index) != 3: + raise ValueError("COTTAS merge index must be a permutation of spo") + + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as exc: + raise RuntimeError( + "PyArrow is required for bounded-memory COTTAS merging: " + f"{exc}" + ) from exc + + normalized_index = index.lower() + batch_rows = cottas_merge_batch_rows() + streams: list[CottasTripleStream] = [] + temporary_path: Path | None = None + writer = None + try: + streams = [ + CottasTripleStream(pq, Path(path), normalized_index, batch_rows) + for path in input_paths + ] + fields = streams[0].fields + expected_types = tuple(field.type for field in fields) + for stream in streams[1:]: + if tuple(field.type for field in stream.fields) != expected_types: + raise RuntimeError( + "COTTAS inputs do not share the same RDF term column types" + ) + + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output.name}.merge-", + suffix=".cottas", + dir=str(output.parent), + ) + os.close(descriptor) + temporary_path = Path(temporary_name) + temporary_path.unlink() + output_schema = pa.schema( + list(fields), metadata={b"index": normalized_index.encode("utf-8")} + ) + writer = pq.ParquetWriter( + temporary_path, + output_schema, + compression="zstd", + compression_level=22, + version="2.6", + ) + total_rows = sum(stream.row_count for stream in streams) + emit_merge_progress( + progress_path, + "started", + completed=0, + total=total_rows, + detail=f"{len(streams):,} sorted COTTAS chunks", + ) + heap = [ + (stream.sort_key, stream_number, stream.current) + for stream_number, stream in enumerate(streams) + if not stream.exhausted + ] + heapq.heapify(heap) + output_rows: list[tuple] = [] + previous_triple = None + processed_rows = 0 + written_rows = 0 + last_progress_rows = 0 + while heap: + _, stream_number, triple = heapq.heappop(heap) + processed_rows += 1 + if triple != previous_triple: + output_rows.append(triple) + previous_triple = triple + # Keep Parquet row groups substantially larger than the per-input + # read batch. That avoids creating tens of thousands of tiny row + # groups for a cohort-sized graph without changing input memory. + if len(output_rows) >= COTTAS_OUTPUT_BATCH_ROWS: + arrays = [ + pa.array([row[column] for row in output_rows], type=fields[column].type) + for column in range(3) + ] + writer.write_batch(pa.RecordBatch.from_arrays(arrays, schema=output_schema)) + written_rows += len(output_rows) + output_rows.clear() + + stream = streams[stream_number] + stream.advance() + if not stream.exhausted: + heapq.heappush(heap, (stream.sort_key, stream_number, stream.current)) + if processed_rows - last_progress_rows >= COTTAS_MERGE_PROGRESS_ROWS: + emit_merge_progress( + progress_path, + "merging", + completed=processed_rows, + total=total_rows, + detail=f"{written_rows + len(output_rows):,} distinct triples written", + ) + last_progress_rows = processed_rows + + if output_rows: + arrays = [ + pa.array([row[column] for row in output_rows], type=fields[column].type) + for column in range(3) + ] + writer.write_batch(pa.RecordBatch.from_arrays(arrays, schema=output_schema)) + written_rows += len(output_rows) + writer.close() + writer = None + os.replace(temporary_path, output) + temporary_path = None + emit_merge_progress( + progress_path, + "complete", + completed=processed_rows, + total=total_rows, + detail=f"{written_rows:,} distinct triples written", + ) + except Exception as exc: + # This context is surfaced by the host wrapper after its ephemeral + # Docker volume is removed, so a malformed or unexpectedly indexed + # input does not become a generic non-zero exit code. + pyarrow_version = getattr(pa, "__version__", "unknown") + raise RuntimeError( + "streaming COTTAS merge failed " + f"(pyarrow={pyarrow_version}; inputs={len(input_paths)}; " + f"index={normalized_index}; batch_rows={batch_rows}; " + f"output={output_path}): {exc}" + ) from exc + finally: + if writer is not None: + writer.close() + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + for stream in streams: + stream.close() + + if remove_input_files: + for input_path in input_paths: + Path(input_path).unlink(missing_ok=True) + + def main() -> int: parser = argparse.ArgumentParser(description="VCF-RDFizer COTTAS adapter") subparsers = parser.add_subparsers(dest="command", required=True) @@ -42,6 +340,30 @@ def main() -> int: merge.add_argument("cottas_path") merge.add_argument("index", nargs="?", default="spo") + merge_many = subparsers.add_parser( + "merge-many", + help="merge multiple COTTAS files in one indexed pass (explicit use)", + ) + merge_many.add_argument( + "--input-cottas-files", + nargs="+", + required=True, + help="COTTAS inputs to merge", + ) + merge_many.add_argument("--output-cottas-file", required=True) + merge_many.add_argument("--index", default="spo") + merge_many.add_argument( + "--progress-path", + help="optional JSONL sidecar for bounded streaming-merge progress", + ) + + reindex = subparsers.add_parser( + "reindex", + help="rebuild the embedded COTTAS query index in place", + ) + reindex.add_argument("cottas_path") + reindex.add_argument("index", nargs="?", default="spo") + decompress = subparsers.add_parser("decompress", help="convert COTTAS to RDF") decompress.add_argument("cottas_path") decompress.add_argument("rdf_path") @@ -76,11 +398,63 @@ def main() -> int: pycottas.cottas2rdf(cottas_path, rdf_path) return 0 + if args.command == "reindex": + cottas_path = Path(args.cottas_path).resolve() + if not cottas_path.is_file(): + print(f"COTTAS file not found: {cottas_path}", file=sys.stderr) + return 2 + + # COTTAS indexes are part of the Parquet artifact rather than sibling + # files. Rebuild into a temporary file in the same directory, then + # replace the original only after the streaming Parquet rewrite + # completes successfully. + temporary_path = None + try: + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{cottas_path.name}.reindex-", + suffix=".cottas", + dir=str(cottas_path.parent), + ) + os.close(file_descriptor) + temporary_path = Path(temporary_name) + temporary_path.unlink() + with cottas_scratch_workspace(): + streaming_cottas_merge( + [str(cottas_path)], + str(temporary_path), + index=args.index, + remove_input_files=False, + ) + if not temporary_path.is_file() or temporary_path.stat().st_size == 0: + raise RuntimeError("pycottas did not create a non-empty reindexed file") + os.replace(temporary_path, cottas_path) + temporary_path = None + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + return 0 + + if args.command == "merge-many": + input_paths = [str(Path(path).resolve()) for path in args.input_cottas_files] + cottas_path = str(Path(args.output_cottas_file).resolve()) + if len(input_paths) < 2: + print("merge-many requires at least two input COTTAS files", file=sys.stderr) + return 2 + with cottas_scratch_workspace(): + streaming_cottas_merge( + input_paths, + cottas_path, + index=args.index, + remove_input_files=True, + progress_path=(Path(args.progress_path) if args.progress_path else None), + ) + return 0 + left_path = str(Path(args.left_path).resolve()) right_path = str(Path(args.right_path).resolve()) cottas_path = str(Path(args.cottas_path).resolve()) with cottas_scratch_workspace(): - pycottas.cat( + streaming_cottas_merge( [left_path, right_path], cottas_path, index=args.index, diff --git a/src/ensure_hdt_index.sh b/src/ensure_hdt_index.sh index 93a875a..5bdebde 100644 --- a/src/ensure_hdt_index.sh +++ b/src/ensure_hdt_index.sh @@ -12,20 +12,68 @@ if [[ ! -f "$HDT_PATH" ]]; then exit 2 fi -HDT_JAVA_HOME=${HDT_JAVA_HOME:-/opt/hdt-java} -HDT_SEARCH_BIN=${HDT_SEARCH_BIN:-$HDT_JAVA_HOME/bin/hdtSearch.sh} -if [[ ! -x "$HDT_SEARCH_BIN" ]]; then - echo "Missing HDT Java search launcher: $HDT_SEARCH_BIN" >&2 +HDTC_BIN=${HDTC_BIN:-/usr/local/bin/hdtc} +if [[ ! -x "$HDTC_BIN" ]]; then + echo "Missing Java-free HDT indexer: $HDTC_BIN" >&2 exit 127 fi -# hdtSearch uses HDT Java's mapIndexedHDT(), which creates the sibling index -# lazily. Sending only `exit` initializes the index without running a query or -# materializing an unbounded result set. -printf 'exit\n' | "$HDT_SEARCH_BIN" "$HDT_PATH" >/dev/null +# hdtc implements the canonical HDT v1-1 index format without a JVM. It streams +# BitmapTriples through disk-backed external sorters, so the memory setting is +# a soft buffer budget rather than a Java heap requirement. Keep sort runs in a +# disposable directory; callers may point the work root at a larger/faster disk. +HDT_INDEX_WORK_ROOT=${HDT_INDEX_WORK_ROOT:-/work} +HDT_INDEX_MEMORY_LIMIT=${HDT_INDEX_MEMORY_LIMIT:-512M} +if [[ ! -d "$HDT_INDEX_WORK_ROOT" || ! -w "$HDT_INDEX_WORK_ROOT" ]]; then + echo "HDT index work directory is not writable: $HDT_INDEX_WORK_ROOT" >&2 + exit 2 +fi + +INDEX_WORK_DIR=$(mktemp -d "$HDT_INDEX_WORK_ROOT/vcf-rdfizer-hdt-index.XXXXXXXX") +INDEX_BACKUP_DIR="$INDEX_WORK_DIR/existing-indexes" +mkdir -p "$INDEX_BACKUP_DIR" +INDEX_REGEN_COMPLETE=0 +cleanup() { + cleanup_status=$? + set +e + if [[ "$INDEX_REGEN_COMPLETE" -eq 0 ]]; then + shopt -s nullglob + # Remove every incomplete replacement before restoring the known-good + # sidecars. hdtc writes the final sibling directly, so interruption can + # otherwise leave a partial file behind. + for generated in "${HDT_PATH}".index.*; do + rm -f -- "$generated" + done + for backup in "$INDEX_BACKUP_DIR"/*; do + mv -- "$backup" "$(dirname "$HDT_PATH")/$(basename "$backup")" + done + fi + rm -rf -- "$INDEX_WORK_DIR" + trap - EXIT + exit "$cleanup_status" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +# Move all versioned sidecars out of the way so this command really regenerates +# the index. If indexing fails, the EXIT trap restores the previous sidecars. +shopt -s nullglob +for existing_index in "${HDT_PATH}".index.*; do + if [[ -e "$existing_index" ]]; then + mv -- "$existing_index" "$INDEX_BACKUP_DIR/$(basename "$existing_index")" + fi +done + +# This is a dedicated index command: it does not query, rewrite, or decode the +# source HDT. The output is the hdt-java/hdt-cpp-compatible sibling sidecar. +"$HDTC_BIN" --quiet index "$HDT_PATH" \ + --memory-limit "$HDT_INDEX_MEMORY_LIMIT" \ + --temp-dir "$INDEX_WORK_DIR" shopt -s nullglob -# HDT Java 3.0.10 writes the v1-1 index as ``.hdt.index.v1-1``. +# hdtc writes the canonical v1-1 index as ``.hdt.index.v1-1``. INDEX_CANDIDATES=("${HDT_PATH}".index.*) INDEX_PATH="" for candidate in "${INDEX_CANDIDATES[@]}"; do @@ -36,8 +84,9 @@ for candidate in "${INDEX_CANDIDATES[@]}"; do done if [[ -z "$INDEX_PATH" ]]; then - echo "HDT Java did not create a non-empty index beside: $HDT_PATH" >&2 + echo "hdtc did not create a non-empty index beside: $HDT_PATH" >&2 exit 1 fi +INDEX_REGEN_COMPLETE=1 echo "HDT index ready: $INDEX_PATH" diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py index ff7079a..d015f4e 100644 --- a/src/partitioned_compression.py +++ b/src/partitioned_compression.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse +import errno import gzip import json import os @@ -23,12 +24,71 @@ HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} -HDT_CAT_CANDIDATES = ( - "hdtCat", - "hdtCat.sh", - "/opt/hdt-java/bin/hdtCat.sh", - "/opt/hdt-java/bin/hdtCat", -) +STDERR_TAIL_BYTES = 16 * 1024 +DEFAULT_COTTAS_MERGE_BATCH_ROWS = "2048" +PROGRESS_HEARTBEAT_BYTES = 64 * 1024 * 1024 + + +def prepare_progress_path(path: Path | None) -> None: + """Best-effort setup for the optional host-mounted progress sidecar.""" + if path is None: + return + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError: + pass + + +def emit_progress( + path: Path | None, + stage: str, + phase: str, + *, + completed: int | float | None = None, + total: int | float | None = None, + unit: str | None = None, + detail: str | None = None, +) -> None: + """Append one small JSONL event; progress must never break conversion.""" + if path is None: + return + payload = {"stage": stage, "phase": phase} + for key, value in ( + ("completed", completed), + ("total", total), + ("unit", unit), + ("detail", detail), + ): + if value is not None: + payload[key] = value + try: + with path.open("a", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + except OSError: + pass + + +def progress_descriptor(name: str) -> tuple[str, str, int | None]: + """Map internal stage names to a small set of terminal progress tasks.""" + for prefix, stage in ( + ("hdt-build-", "hdt-chunks"), + ("cottas-build-", "cottas-chunks"), + ): + if name.startswith(prefix): + try: + return stage, "chunks", int(name[len(prefix) :]) + except ValueError: + break + if name.startswith("hdt-merge-"): + return "hdt-merge", "stage", None + if name.startswith("cottas-merge"): + return "cottas-merge", "stage", None + if name.startswith("hdt-"): + return "hdt", "stage", None + if name.startswith("cottas-"): + return "cottas", "stage", None + return name, "stage", None def is_triple_line(line: bytes) -> bool: @@ -48,6 +108,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--max-chunk-bytes", required=True, type=int) parser.add_argument("--expected-triples", type=int) parser.add_argument("--result-path", required=True) + parser.add_argument( + "--progress-path", + help="optional JSONL sidecar for host-side terminal progress", + ) + parser.add_argument( + "--allow-index-failures", + action="store_true", + help="continue with other representations when HDT/COTTAS indexing fails", + ) return parser.parse_args() @@ -58,41 +127,71 @@ def iter_rdf_lines(path: Path): yield from handle -def plan_chunks( +def stream_chunks( source: Path, chunk_dir: Path, *, target_bytes: int, min_bytes: int, max_bytes: int, -) -> tuple[list[Path], dict]: - """Create chunks on complete N-Triples records in one sequential pass.""" + progress_path: Path | None = None, + progress_total: int | None = None, +) -> tuple[object, dict]: + """Yield complete-record chunks while building a mutable chunk plan. + + The old implementation returned only after expanding the *entire* source + into raw ``.nt`` chunks. A space-optimized aggregate is normally gzip + compressed, so that made the Docker workspace hold another full, + uncompressed copy of the aggregate before either converter could reclaim a + single byte. The caller now converts and unlinks each yielded chunk before + asking for the next one. + """ if target_bytes <= 0 or min_bytes <= 0 or max_bytes <= 0: raise ValueError("RDF chunk sizes must be positive") if min_bytes > target_bytes or target_bytes > max_bytes: raise ValueError("RDF chunk sizes must satisfy min <= target <= max") chunk_dir.mkdir(parents=True, exist_ok=True) - chunk_paths: list[Path] = [] - chunk_metadata: list[dict] = [] - handle = None - chunk_path = None - chunk_size = 0 - chunk_start_offset = 0 - chunk_start_record = 0 - logical_offset = 0 - record_count = 0 - chunk_index = 0 - - def close_chunk(): - nonlocal handle, chunk_path, chunk_size - if handle is None or chunk_path is None: - return - handle.close() - chunk_paths.append(chunk_path) - chunk_metadata.append( - { - "chunk_id": len(chunk_metadata), + prepare_progress_path(progress_path) + plan = { + "source_file_count": 1, + "source_paths": [str(source)], + "chunk_count": 0, + "chunk_input_bytes": 0, + "record_count": 0, + "target_chunk_bytes": target_bytes, + "min_chunk_bytes": min_bytes, + "max_chunk_bytes": max_bytes, + "chunks": [], + } + + def generate(): + handle = None + chunk_path = None + chunk_size = 0 + chunk_start_offset = 0 + chunk_start_record = 0 + logical_offset = 0 + record_count = 0 + chunk_index = 0 + last_progress_offset = 0 + + def open_chunk(): + nonlocal handle, chunk_path, chunk_size, chunk_start_offset, chunk_start_record, chunk_index + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_index += 1 + handle = chunk_path.open("wb") + chunk_size = 0 + chunk_start_offset = logical_offset + chunk_start_record = record_count + + def close_chunk(): + nonlocal handle, chunk_path, chunk_size + if handle is None or chunk_path is None: + return None + handle.close() + metadata = { + "chunk_id": len(plan["chunks"]), "path": str(chunk_path), "start_record": chunk_start_record, "end_record": record_count, @@ -101,52 +200,103 @@ def close_chunk(): "record_count": record_count - chunk_start_record, "payload_bytes": chunk_size, } - ) - handle = None - chunk_path = None - chunk_size = 0 + plan["chunks"].append(metadata) + plan["chunk_count"] = len(plan["chunks"]) + emit_progress( + progress_path, + "rdf-scan", + "chunk", + completed=record_count, + total=progress_total, + unit="triples", + detail=( + f"{plan['chunk_count']:,} chunks · " + f"{logical_offset:,} bytes read" + ), + ) + completed_path = chunk_path + handle = None + chunk_path = None + chunk_size = 0 + return completed_path, metadata - try: - for line in iter_rdf_lines(source): - if not line.endswith(b"\n"): - raise ValueError(f"RDF source contains a non-line-terminated record: {source}") - line_size = len(line) - if handle is None: - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - elif chunk_size > 0 and ( - (chunk_size >= target_bytes and chunk_size >= min_bytes) - or chunk_size + line_size > max_bytes - ): - close_chunk() - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - - handle.write(line) - chunk_size += line_size - logical_offset += line_size - if is_triple_line(line): - record_count += 1 - finally: - close_chunk() - - return chunk_paths, { - "source_file_count": 1, - "source_paths": [str(source)], - "chunk_count": len(chunk_paths), - "chunk_input_bytes": logical_offset, - "record_count": record_count, - "target_chunk_bytes": target_bytes, - "min_chunk_bytes": min_bytes, - "max_chunk_bytes": max_bytes, - "chunks": chunk_metadata, - } + try: + for line in iter_rdf_lines(source): + if not line.endswith(b"\n"): + raise ValueError(f"RDF source contains a non-line-terminated record: {source}") + line_size = len(line) + if handle is None: + open_chunk() + elif chunk_size > 0 and ( + (chunk_size >= target_bytes and chunk_size >= min_bytes) + or chunk_size + line_size > max_bytes + ): + completed_chunk = close_chunk() + if completed_chunk is not None: + yield completed_chunk + open_chunk() + + handle.write(line) + chunk_size += line_size + logical_offset += line_size + if is_triple_line(line): + record_count += 1 + plan["chunk_input_bytes"] = logical_offset + plan["record_count"] = record_count + if ( + logical_offset - last_progress_offset >= PROGRESS_HEARTBEAT_BYTES + ): + emit_progress( + progress_path, + "rdf-scan", + "heartbeat", + completed=record_count, + total=progress_total, + unit="triples", + detail=f"{logical_offset:,} bytes read", + ) + last_progress_offset = logical_offset + + completed_chunk = close_chunk() + if completed_chunk is not None: + yield completed_chunk + finally: + # A write/decompression error can leave one unyielded, partial + # chunk. It has no consumer, so remove it before the volume is + # released. + if handle is not None: + try: + handle.close() + finally: + if chunk_path is not None: + chunk_path.unlink(missing_ok=True) + + return generate(), plan + + +def plan_chunks( + source: Path, + chunk_dir: Path, + *, + target_bytes: int, + min_bytes: int, + max_bytes: int, +) -> tuple[list[Path], dict]: + """Materialize chunks for callers that explicitly need every path. + + The production compressor uses :func:`stream_chunks` so it never retains a + full uncompressed copy of a gzip aggregate. This compatibility helper is + deliberately kept for diagnostics and standalone callers. + """ + stream, plan = stream_chunks( + source, + chunk_dir, + target_bytes=target_bytes, + min_bytes=min_bytes, + max_bytes=max_bytes, + ) + chunk_paths = [path for path, _metadata in stream] + return chunk_paths, plan def resolve_executable(candidates: tuple[str, ...], label: str) -> str: @@ -158,13 +308,100 @@ def resolve_executable(candidates: tuple[str, ...], label: str) -> str: def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: - """Locate HDT Java's non-empty versioned index sidecar.""" + """Locate the non-empty canonical HDT versioned index sidecar.""" for candidate in sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")): if candidate.is_file() and candidate.stat().st_size > 0: return candidate return None +def hdtc_merge_temp_dir(work_dir: Path, merged_path: Path) -> Path: + """Return the isolated disk workspace for one hdtc merge stage.""" + return work_dir / f".{merged_path.stem}.hdtc-work" + + +def hdtc_merge_command( + hdtc_bin: str, + left: Path, + right: Path, + merged: Path, + *, + work_dir: Path, + memory_limit: str, +) -> list[str]: + """Build a bounded-memory native merge command for two HDT chunks. + + ``hdtc create`` accepts existing ``.hdt`` inputs, so it performs the same + logical merge as hdt-java's ``hdtCat`` without constructing Java HashMaps. + Its temporary files are kept in a per-stage directory so they can be + removed immediately after each pairwise merge. + """ + return [ + hdtc_bin, + "--quiet", + "create", + str(left), + str(right), + "--output", + str(merged), + "--memory-limit", + memory_limit, + "--temp-dir", + str(hdtc_merge_temp_dir(work_dir, merged)), + ] + + +def cottas_merge_many_command( + python_bin: str, + inputs: list[Path], + merged: Path, + *, + progress_path: Path | None = None, +) -> list[str]: + """Build one bounded-memory, multi-input COTTAS merge command. + + The adapter performs a streaming k-way merge of the already indexed chunk + files. Unlike ``pycottas.cat`` and an external-sort query, it does not + construct a global in-memory hash table or a full-data temporary spill + area. + """ + return [ + python_bin, + "/opt/vcf-rdfizer/cottas_tool.py", + "merge-many", + "--input-cottas-files", + *(str(path) for path in inputs), + "--output-cottas-file", + str(merged), + "--index", + "spo", + *(["--progress-path", str(progress_path)] if progress_path else []), + ] + + +def cottas_merge_command( + python_bin: str, + left: Path, + right: Path, + merged: Path, +) -> list[str]: + """Build a compatible two-input bounded-memory COTTAS merge command. + + The normal partitioned workflow uses ``merge-many`` to merge all sorted + chunks in one pass. This command remains available for explicit callers + that need a two-input merge. + """ + return [ + python_bin, + "/opt/vcf-rdfizer/cottas_tool.py", + "merge", + str(left), + str(right), + str(merged), + "spo", + ] + + def parse_time_log(path: Path) -> dict: if not path.exists(): return {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None} @@ -191,8 +428,10 @@ def number(pattern: str, integer: bool = False): class StageRunner: """Execute stages and accumulate both detailed and method-level metrics.""" - def __init__(self, work_dir: Path): + def __init__(self, work_dir: Path, progress_path: Path | None = None): self.work_dir = work_dir + self.progress_path = progress_path + prepare_progress_path(progress_path) self.stages: list[dict] = [] def run( @@ -203,24 +442,86 @@ def run( stdout_path: Path | None = None, ) -> dict: time_path = self.work_dir / f".{name}.time" + stderr_path = self.work_dir / f".{name}.stderr" + progress_stage, progress_unit, progress_ordinal = progress_descriptor(name) + emit_progress( + self.progress_path, + progress_stage, + "started", + completed=progress_ordinal, + unit=progress_unit, + detail=name, + ) if time_path.exists(): time_path.unlink() + if stderr_path.exists(): + stderr_path.unlink() started = time.perf_counter() + workspace_free_before = None + workspace_total = None + try: + workspace_usage = shutil.disk_usage(self.work_dir) + workspace_free_before = workspace_usage.free + workspace_total = workspace_usage.total + except OSError: + pass time_bin = "/usr/bin/time" if Path("/usr/bin/time").exists() else None + if time_bin: + # macOS ships a BSD ``time`` at this path; only GNU time supports + # the ``-v`` metrics format used by ``parse_time_log``. + probe = subprocess.run( + [time_bin, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if probe.returncode != 0 or b"GNU time" not in probe.stdout + probe.stderr: + time_bin = None if time_bin: timed_command = [time_bin, "-v", "-o", str(time_path), *command] else: timed_command = command stdout_handle = None + stderr_handle = None try: if stdout_path is not None: stdout_path.parent.mkdir(parents=True, exist_ok=True) stdout_handle = stdout_path.open("wb") - completed = subprocess.run(timed_command, stdout=stdout_handle, check=False) + stderr_handle = stderr_path.open("wb") + completed = subprocess.run( + timed_command, + stdout=stdout_handle, + stderr=stderr_handle, + check=False, + ) finally: if stdout_handle is not None: stdout_handle.close() + if stderr_handle is not None: + stderr_handle.close() + emit_progress( + self.progress_path, + progress_stage, + "complete" if completed.returncode == 0 else "failed", + completed=( + progress_ordinal + 1 + if progress_ordinal is not None and completed.returncode == 0 + else progress_ordinal + ), + unit=progress_unit, + detail=name, + ) + stderr_tail = "" + if stderr_path.exists(): + try: + with stderr_path.open("rb") as handle: + handle.seek(0, os.SEEK_END) + handle.seek(max(0, handle.tell() - STDERR_TAIL_BYTES)) + stderr_tail = handle.read().decode("utf-8", errors="replace").strip() + finally: + stderr_path.unlink(missing_ok=True) result = { + "stage_name": name, "exit_code": completed.returncode, "wall_seconds": time.perf_counter() - started, "output_path": "" if output_path is None else str(output_path), @@ -228,6 +529,18 @@ def run( if output_path is not None and output_path.is_file() else 0, } + try: + workspace_usage = shutil.disk_usage(self.work_dir) + result["workspace_free_bytes_after"] = workspace_usage.free + result["workspace_total_bytes"] = workspace_usage.total + except OSError: + pass + if workspace_free_before is not None: + result["workspace_free_bytes_before"] = workspace_free_before + if workspace_total is not None: + result.setdefault("workspace_total_bytes", workspace_total) + if stderr_tail: + result["stderr_tail"] = stderr_tail result.update(parse_time_log(time_path)) self.stages.append({"name": name, **result}) time_path.unlink(missing_ok=True) @@ -258,6 +571,41 @@ def finalize_totals(total: dict) -> dict: } +def failure_message(stage_result: dict | None, fallback: str) -> str: + """Turn a failed subprocess result into an actionable pipeline error. + + ``StageRunner`` intentionally stores a bounded stderr tail in the result + payload. The outer CLI has no access to the ephemeral Docker volume once + the container exits, so omitting that text here turns a concrete DuckDB + error (for example an unwritable spill directory) into a useless + ``exit_code=1`` report. + """ + if not stage_result: + return fallback + exit_code = stage_result.get("exit_code") + diagnostics = [] + if exit_code is not None: + diagnostics.append(f"exit_code={exit_code}") + try: + numeric_exit_code = int(exit_code) + except (TypeError, ValueError): + numeric_exit_code = None + if numeric_exit_code == -9: + diagnostics.append( + "the process was killed by SIGKILL (usually the kernel/Docker OOM killer)" + ) + elif numeric_exit_code == 137: + diagnostics.append("the process was killed (often Docker memory/OOM pressure)") + elif numeric_exit_code == 143: + diagnostics.append("the process was terminated (SIGTERM)") + stderr_tail = " ".join(str(stage_result.get("stderr_tail") or "").split()) + if stderr_tail: + # Preserve the end of a traceback, which contains the concrete + # exception, without making top-level CLI output unbounded. + diagnostics.append(f"stderr={stderr_tail[-2048:]}") + return f"{fallback} ({'; '.join(diagnostics)})" if diagnostics else fallback + + def merge_pairwise( paths: list[Path], *, @@ -265,6 +613,7 @@ def merge_pairwise( runner: StageRunner, merge_command, total: dict, + cleanup_merged_workspace=None, ) -> tuple[Path | None, int]: rounds = 0 current = list(paths) @@ -278,11 +627,15 @@ def merge_pairwise( continue right = current[pair_index + 1] merged = left.parent / f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}{left.suffix}" - stage = runner.run( - f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}", - merge_command(left, right, merged), - merged, - ) + try: + stage = runner.run( + f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}", + merge_command(left, right, merged), + merged, + ) + finally: + if cleanup_merged_workspace is not None: + cleanup_merged_workspace(merged) add_totals(total, stage) if stage["exit_code"] != 0: return None, rounds @@ -301,11 +654,83 @@ def main() -> int: methods = [method.strip() for method in args.methods.split(",") if method.strip()] work_dir = Path("/work") chunk_dir = work_dir / "rdf_chunks" - runner = StageRunner(work_dir) + progress_path = Path(args.progress_path) if args.progress_path else None + runner = StageRunner(work_dir, progress_path=progress_path) results: dict[str, dict] = {} + index_warnings: list[dict] = [] hdt_total = {"exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False} cottas_total = {"exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False} + output_hdt = output_dir / f"{args.output_name}.hdt" + output_cottas = output_dir / f"{args.output_name}.cottas" + cottas_failed = False + cottas_warning = None + + def write_result(payload: dict): + """Best-effort result handoff that never hides the original failure.""" + try: + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except OSError as write_error: + print( + f"Warning: unable to write partitioned-compression result to {result_path}: {write_error}", + file=sys.stderr, + ) + + def record_index_warning( + index_format: str, + stage: str, + artifact: Path, + message: str, + stage_result: dict | None = None, + ) -> dict: + warning = { + "format": index_format, + "stage": stage, + "status": "index_unavailable", + "artifact_path": str(artifact), + "message": " ".join(str(message).split()), + } + if stage_result: + warning["stage_name"] = stage_result.get("stage_name", stage) + warning["exit_code"] = stage_result.get("exit_code") + for key in ( + "workspace_free_bytes_before", + "workspace_free_bytes_after", + "workspace_total_bytes", + "max_rss_kb", + "cottas_merge_batch_rows", + ): + if key in stage_result: + warning[key] = stage_result[key] + stderr_tail = str(stage_result.get("stderr_tail") or "").strip() + if stderr_tail: + warning["stderr_tail"] = stderr_tail + index_warnings.append(warning) + print( + f"Warning: {index_format.upper()} index generation failed for '{artifact}'; " + f"continuing with the remaining pipeline. {warning['message']}", + file=sys.stderr, + ) + return warning + + def skipped_cottas_result(method: str) -> dict: + artifact = { + "cottas": output_cottas, + "cottas_gzip": output_dir / f"{args.output_name}.cottas.gz", + "cottas_brotli": output_dir / f"{args.output_name}.cottas.br", + }[method] + return { + "exit_code": 1, + "wall_seconds": 0.0, + "user_seconds": 0.0, + "sys_seconds": 0.0, + "max_rss_kb": 0, + "output_path": str(artifact), + "output_size_bytes": 0, + "source": "index_unavailable", + "details": {"index_status": "failed", "index_warning": cottas_warning}, + } try: if not source.is_file(): @@ -313,23 +738,17 @@ def main() -> int: if not methods or not all(method in HDT_METHODS | COTTAS_METHODS for method in methods): raise ValueError(f"Unsupported partitioned method list: {methods}") - chunks, plan = plan_chunks( - source, - chunk_dir, - target_bytes=args.target_chunk_bytes, - min_bytes=args.min_chunk_bytes, - max_bytes=args.max_chunk_bytes, - ) - if not chunks: - raise ValueError("RDF source contains no complete records") - source_triples = int(plan["record_count"]) - if args.expected_triples is not None and source_triples != args.expected_triples: - raise ValueError( - "source triple count does not match the upstream conversion count: " - f"source={source_triples}, expected={args.expected_triples}" - ) hdt_paths: list[Path] = [] cottas_paths: list[Path] = [] + + def cleanup_cottas_intermediates() -> None: + """Release COTTAS chunks/merge outputs after a failed attempt.""" + for path in list(cottas_paths): + path.unlink(missing_ok=True) + cottas_paths.clear() + for path in work_dir.glob("cottas-merge-*.cottas"): + path.unlink(missing_ok=True) + needs_hdt = any(method in HDT_METHODS for method in methods) hdt_bin = ( resolve_executable( @@ -343,10 +762,41 @@ def main() -> int: if needs_hdt else None ) - hdt_cat = resolve_executable(HDT_CAT_CANDIDATES, "HDTCat") if needs_hdt else None + hdtc_bin = ( + resolve_executable( + ( + os.environ.get("HDTC_BIN", ""), + "/usr/local/bin/hdtc", + "hdtc", + ), + "hdtc HDT merger", + ) + if needs_hdt + else None + ) + hdt_merge_memory_limit = os.environ.get("HDT_MERGE_MEMORY_LIMIT", "512M").strip() + if needs_hdt and not hdt_merge_memory_limit: + raise ValueError("HDT_MERGE_MEMORY_LIMIT must be a non-empty hdtc memory size") cottas_python = os.environ.get("COTTAS_PYTHON_BIN") or shutil.which("python3") if any(method in COTTAS_METHODS for method in methods) and not cottas_python: raise RuntimeError("Missing Python runtime for COTTAS") + emit_progress( + progress_path, + "rdf-scan", + "started", + completed=0, + total=args.expected_triples, + unit="triples", + ) + chunk_stream, plan = stream_chunks( + source, + chunk_dir, + target_bytes=args.target_chunk_bytes, + min_bytes=args.min_chunk_bytes, + max_bytes=args.max_chunk_bytes, + progress_path=progress_path, + progress_total=args.expected_triples, + ) def validate_artifact( *, @@ -407,38 +857,108 @@ def validate_artifact( ) return report - for index, chunk in enumerate(chunks): - if needs_hdt: - chunk_hdt = work_dir / f"chunk-{index:05d}.hdt" - stage = runner.run(f"hdt-build-{index:05d}", [hdt_bin, str(chunk), str(chunk_hdt)], chunk_hdt) - add_totals(hdt_total, stage) - if stage["exit_code"] != 0: - raise RuntimeError("partitioned HDT chunk conversion failed") - hdt_paths.append(chunk_hdt) - if any(method in COTTAS_METHODS for method in methods): - chunk_cottas = work_dir / f"chunk-{index:05d}.cottas" - stage = runner.run( - f"cottas-build-{index:05d}", - [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "convert", str(chunk), str(chunk_cottas), "spo"], - chunk_cottas, - ) - add_totals(cottas_total, stage) - if stage["exit_code"] != 0: - raise RuntimeError("partitioned COTTAS chunk conversion failed") - cottas_paths.append(chunk_cottas) - chunk.unlink(missing_ok=True) + # Convert each uncompressed chunk before requesting the next one from + # the gzip reader. This bounds raw-RDF workspace use to one chunk, + # rather than the entire decompressed aggregate. + for index, (chunk, _chunk_metadata) in enumerate(chunk_stream): + try: + if needs_hdt: + chunk_hdt = work_dir / f"chunk-{index:05d}.hdt" + stage = runner.run( + f"hdt-build-{index:05d}", + [hdt_bin, str(chunk), str(chunk_hdt)], + chunk_hdt, + ) + add_totals(hdt_total, stage) + if stage["exit_code"] != 0: + raise RuntimeError("partitioned HDT chunk conversion failed") + hdt_paths.append(chunk_hdt) + if any(method in COTTAS_METHODS for method in methods) and not cottas_failed: + chunk_cottas = work_dir / f"chunk-{index:05d}.cottas" + stage = runner.run( + f"cottas-build-{index:05d}", + [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "convert", str(chunk), str(chunk_cottas), "spo"], + chunk_cottas, + ) + add_totals(cottas_total, stage) + if stage["exit_code"] != 0: + if not args.allow_index_failures: + raise RuntimeError("partitioned COTTAS chunk conversion failed") + cottas_failed = True + cottas_total["exit_code"] = 0 + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + failure_message( + stage, + "partitioned COTTAS conversion/index creation failed for a chunk", + ), + stage_result=stage, + ) + chunk_cottas.unlink(missing_ok=True) + cleanup_cottas_intermediates() + else: + cottas_paths.append(chunk_cottas) + finally: + chunk.unlink(missing_ok=True) + + emit_progress( + progress_path, + "rdf-scan", + "complete", + completed=plan["record_count"], + total=args.expected_triples, + unit="triples", + detail=f"{plan['chunk_count']:,} chunks", + ) + if needs_hdt: + emit_progress( + progress_path, + "hdt-chunks", + "complete", + completed=len(hdt_paths), + total=plan["chunk_count"], + unit="chunks", + ) + if any(method in COTTAS_METHODS for method in methods): + emit_progress( + progress_path, + "cottas-chunks", + "failed" if cottas_failed else "complete", + completed=len(cottas_paths), + total=plan["chunk_count"], + unit="chunks", + ) + if not plan["chunks"]: + raise ValueError("RDF source contains no complete records") + source_triples = int(plan["record_count"]) + if args.expected_triples is not None and source_triples != args.expected_triples: + raise ValueError( + "source triple count does not match the upstream conversion count: " + f"source={source_triples}, expected={args.expected_triples}" + ) - output_hdt = output_dir / f"{args.output_name}.hdt" if hdt_paths: final_hdt, hdt_rounds = merge_pairwise( hdt_paths, prefix="hdt", runner=runner, - merge_command=lambda left, right, merged: [hdt_cat, str(left), str(right), str(merged)], + merge_command=lambda left, right, merged: hdtc_merge_command( + hdtc_bin, + left, + right, + merged, + work_dir=work_dir, + memory_limit=hdt_merge_memory_limit, + ), total=hdt_total, + cleanup_merged_workspace=lambda merged: shutil.rmtree( + hdtc_merge_temp_dir(work_dir, merged), ignore_errors=True + ), ) if final_hdt is None: - raise RuntimeError("HDTCat merge failed") + raise RuntimeError("hdtc HDT merge failed") shutil.copyfile(final_hdt, output_hdt) final_hdt.unlink(missing_ok=True) index_stage = runner.run( @@ -448,13 +968,23 @@ def validate_artifact( add_totals(hdt_total, index_stage) output_index = find_hdt_index_sidecar(output_hdt) if output_index is not None: - # The filename is versioned by HDT Java, so record the actual + # The canonical filename is versioned, so record the actual # sidecar after the helper completes instead of assuming .index. index_stage["output_path"] = str(output_index) index_stage["output_size_bytes"] = output_index.stat().st_size runner.stages[-1].update(index_stage) if index_stage["exit_code"] != 0 or output_index is None: - raise RuntimeError("final HDT index initialization failed") + if not args.allow_index_failures: + raise RuntimeError("final HDT index initialization failed") + hdt_total["exit_code"] = 0 + hdt_index_warning = record_index_warning( + "hdt", + "hdt-index", + output_hdt, + "the HDT artifact remains available, but its query index was not created", + ) + else: + hdt_index_warning = None hdt_validation = validate_artifact( name="hdt-validate", artifact=output_hdt, @@ -471,42 +1001,105 @@ def validate_artifact( **plan, "merge_rounds": hdt_rounds, "index_path": str(output_index), - "index_size_bytes": output_index.stat().st_size, + "index_size_bytes": output_index.stat().st_size if output_index else 0, + "index_status": "failed" if hdt_index_warning else "ready", + "index_warning": hdt_index_warning, "validation": hdt_validation, }, } - output_cottas = output_dir / f"{args.output_name}.cottas" - if cottas_paths: - final_cottas, cottas_rounds = merge_pairwise( - cottas_paths, - prefix="cottas", - runner=runner, - merge_command=lambda left, right, merged: [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "merge", str(left), str(right), str(merged), "spo"], - total=cottas_total, - ) + if cottas_paths and not cottas_failed: + # Each COTTAS chunk is already sorted by the `spo` index. The + # adapter k-way merges those ordered Parquet streams with one small + # batch per input, avoiding both pycottas.cat's global hash table + # and DuckDB external-sort spill files that exceeded 41 GiB for a + # 52-chunk condensed cohort. + cottas_stage = None + cottas_rounds = 0 + if len(cottas_paths) == 1: + final_cottas = cottas_paths[0] + else: + cottas_merged_path = work_dir / "cottas-merge-final.cottas" + cottas_stage = runner.run( + "cottas-merge-stream", + cottas_merge_many_command( + cottas_python, + cottas_paths, + cottas_merged_path, + progress_path=progress_path, + ), + cottas_merged_path, + ) + cottas_stage["cottas_merge_batch_rows"] = os.environ.get( + "COTTAS_MERGE_BATCH_ROWS", DEFAULT_COTTAS_MERGE_BATCH_ROWS + ) + runner.stages[-1].update(cottas_stage) + add_totals(cottas_total, cottas_stage) + cottas_rounds = 1 + final_cottas = ( + cottas_merged_path + if cottas_stage["exit_code"] == 0 and cottas_merged_path.is_file() + else None + ) if final_cottas is None: - raise RuntimeError("COTTAS merge failed") - shutil.copyfile(final_cottas, output_cottas) - final_cottas.unlink(missing_ok=True) - cottas_validation = validate_artifact( - name="cottas-validate", - artifact=output_cottas, - artifact_format="cottas", - python_bin=cottas_python, - ) - results["cottas"] = { - **finalize_totals(cottas_total), - "output_path": str(output_cottas), - "output_size_bytes": output_cottas.stat().st_size, - "source": "partitioned_generated", - "details": { - **plan, - "merge_rounds": cottas_rounds, - "index": "spo", - "validation": cottas_validation, - }, - } + if not args.allow_index_failures: + raise RuntimeError( + failure_message(cottas_stage, "COTTAS merge/index creation failed") + ) + cottas_failed = True + cottas_total["exit_code"] = 0 + cleanup_cottas_intermediates() + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + failure_message(cottas_stage, "COTTAS merge/index creation failed"), + stage_result=cottas_stage, + ) + else: + shutil.copyfile(final_cottas, output_cottas) + final_cottas.unlink(missing_ok=True) + cleanup_cottas_intermediates() + try: + cottas_validation = validate_artifact( + name="cottas-validate", + artifact=output_cottas, + artifact_format="cottas", + python_bin=cottas_python, + ) + except RuntimeError as exc: + if not args.allow_index_failures: + raise + cottas_failed = True + cottas_total["exit_code"] = 0 + output_cottas.unlink(missing_ok=True) + cottas_warning = record_index_warning( + "cottas", + "cottas-index", + output_cottas, + str(exc), + ) + if not cottas_failed: + results["cottas"] = { + **finalize_totals(cottas_total), + "output_path": str(output_cottas), + "output_size_bytes": output_cottas.stat().st_size, + "source": "partitioned_generated", + "details": { + **plan, + "merge_rounds": cottas_rounds, + "merge_strategy": "pyarrow_streaming_k_way", + "merge_batch_rows": os.environ.get( + "COTTAS_MERGE_BATCH_ROWS", + DEFAULT_COTTAS_MERGE_BATCH_ROWS, + ), + "index": "spo", + "validation": cottas_validation, + }, + } + + if any(method in COTTAS_METHODS for method in methods) and cottas_failed: + results["cottas"] = skipped_cottas_result("cottas") for method in methods: if method == "hdt_gzip": @@ -516,9 +1109,15 @@ def validate_artifact( artifact = output_dir / f"{args.output_name}.hdt.br" stage = runner.run("hdt-brotli", ["brotli", "-q", "7", "-c", str(output_hdt)], artifact, artifact) elif method == "cottas_gzip": + if cottas_failed: + results[method] = skipped_cottas_result(method) + continue artifact = output_dir / f"{args.output_name}.cottas.gz" stage = runner.run("cottas-gzip", ["gzip", "-c", str(output_cottas)], artifact, artifact) elif method == "cottas_brotli": + if cottas_failed: + results[method] = skipped_cottas_result(method) + continue artifact = output_dir / f"{args.output_name}.cottas.br" stage = runner.run("cottas-brotli", ["brotli", "-q", "7", "-c", str(output_cottas)], artifact, artifact) else: @@ -527,13 +1126,26 @@ def validate_artifact( raise RuntimeError(f"{method} packaging failed") results[method] = stage - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text(json.dumps({"exit_code": 0, "methods": results, "stages": runner.stages}, indent=2) + "\n", encoding="utf-8") + write_result( + { + "exit_code": 0, + "methods": results, + "stages": runner.stages, + "index_warnings": index_warnings, + } + ) return 0 except Exception as exc: - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text(json.dumps({"exit_code": 1, "methods": results, "stages": runner.stages, "error": str(exc)}, indent=2) + "\n", encoding="utf-8") - print(f"partitioned compression failed: {exc}", file=sys.stderr) + error = str(exc) + if isinstance(exc, OSError) and exc.errno == errno.ENOSPC: + error = ( + "temporary partitioned-compression workspace (/work) ran out of storage. " + "Chunks are streamed one at a time, but the workspace must still hold " + "one raw chunk plus the in-progress HDT/COTTAS artifacts. Reduce " + "--chunk-target-bytes and --chunk-max-bytes, or increase Docker's disk limit." + ) + write_result({"exit_code": 1, "methods": results, "stages": runner.stages, "error": error}) + print(f"partitioned compression failed: {error}", file=sys.stderr) return 1 diff --git a/src/run_conversion.sh b/src/run_conversion.sh index 0fdb0ff..3affc27 100644 --- a/src/run_conversion.sh +++ b/src/run_conversion.sh @@ -57,11 +57,15 @@ cleanup_parts_dir() { rm -rf "$PARTS_DIR" } trap cleanup_parts_dir EXIT -TIME_LOG_DIR="$LOGDIR/conversion_time/${SAFE_OUT_NAME}" -METRICS_JSON_DIR="$LOGDIR/conversion_metrics/${SAFE_OUT_NAME}" +PROGRESS_FILE=${PROGRESS_FILE:-} +PROGRESS_READY=0 +# One invocation owns its metrics directory, so output names—not timestamps— +# make the stage files both stable and immediately discoverable. +TIME_LOG_DIR="$LOGDIR/timings/conversion" +METRICS_JSON_DIR="$LOGDIR/stages/conversion" mkdir -p "$TIME_LOG_DIR" "$METRICS_JSON_DIR" -TIME_LOG="$TIME_LOG_DIR/${RUN_ID}.txt" -METRICS_JSON="$METRICS_JSON_DIR/${RUN_ID}.json" +TIME_LOG="$TIME_LOG_DIR/${SAFE_OUT_NAME}.txt" +METRICS_JSON="$METRICS_JSON_DIR/${SAFE_OUT_NAME}.json" METRICS_CSV="$LOGDIR/metrics.csv" TSV_EXIT_CODE=${TSV_EXIT_CODE:-0} TSV_WALL_SECONDS=${TSV_WALL_SECONDS:-null} @@ -73,21 +77,35 @@ TSV_OUTPUT_PATH=${TSV_OUTPUT_PATH:-} METRICS_HEADER="run_id,timestamp,output_name,output_dir,exit_code_java,wall_seconds_java,user_seconds_java,sys_seconds_java,max_rss_kb_java,input_mapping_size_bytes,input_vcf_size_bytes,output_dir_size_bytes,output_triples,jar,mapping_file,output_path,exit_code_tsv,wall_seconds_tsv,user_seconds_tsv,sys_seconds_tsv,max_rss_kb_tsv,tsv_output_size_bytes,tsv_output_path" +# Detect the local `stat` dialect once. `stat_size` is called for every +# RMLStreamer part on every progress tick, so probing the dialect per call +# doubles the number of processes this script spawns for no benefit. +STAT_FILE_FLAVOR="" +detect_stat_file_flavor() { + if [[ -n "$STAT_FILE_FLAVOR" ]]; then + return + fi + if stat -c%s . >/dev/null 2>&1; then + STAT_FILE_FLAVOR="gnu" + elif stat -f%z . >/dev/null 2>&1; then + STAT_FILE_FLAVOR="bsd" + else + STAT_FILE_FLAVOR="wc" + fi +} + # Return byte size for file or directory (GNU + BSD compatible). stat_size() { local path="$1" # --- CASE 1: Regular file --- if [[ -f "$path" ]]; then - # Linux (GNU coreutils) - if stat -c%s "$path" >/dev/null 2>&1; then - stat -c%s "$path" - # macOS/BSD - elif stat -f%z "$path" >/dev/null 2>&1; then - stat -f%z "$path" - else - wc -c < "$path" | tr -d ' ' - fi + detect_stat_file_flavor + case "$STAT_FILE_FLAVOR" in + gnu) stat -c%s "$path" ;; + bsd) stat -f%z "$path" ;; + *) wc -c < "$path" | tr -d ' ' ;; + esac return fi @@ -111,20 +129,81 @@ stat_size() { echo 0 } +# Write sparse machine-readable progress events for the host-side Rich display. +# Progress is deliberately best-effort: a UI sidecar must never make a data +# conversion fail. +progress_emit() { + local stage="$1" + local phase="$2" + local completed="${3:-null}" + local total="${4:-null}" + local unit="${5:-}" + local parts="${6:-null}" + if [[ -z "$PROGRESS_FILE" ]]; then + return 0 + fi + if (( PROGRESS_READY == 0 )); then + mkdir -p "$(dirname "$PROGRESS_FILE")" >/dev/null 2>&1 || return 0 + PROGRESS_READY=1 + fi + printf '{"stage":"%s","phase":"%s","completed":%s,"total":%s,"unit":"%s","parts":%s}\n' \ + "$stage" "$phase" "$completed" "$total" "$unit" "$parts" \ + >> "$PROGRESS_FILE" 2>/dev/null || true +} + +# RMLStreamer writes part files while it runs. Summing their metadata once per +# second gives useful throughput feedback without reading or counting RDF. +# One directory walk fills both the byte total and the part count, and the +# leading-dot test uses parameter expansion rather than a `basename` process. +PART_BYTES=0 +PART_COUNT=0 +scan_parts() { + local file name size + PART_BYTES=0 + PART_COUNT=0 + shopt -s nullglob + for file in "$PARTS_DIR"/*; do + name="${file##*/}" + if [[ ! -f "$file" || "$name" == .* ]]; then + continue + fi + size=$(stat_size "$file") + PART_BYTES=$((PART_BYTES + size)) + PART_COUNT=$((PART_COUNT + 1)) + done + shopt -u nullglob +} + # Report comparable input VCF bytes. # - .vcf -> on-disk bytes # - .vcf.gz -> decompressed bytes # - dir -> sum of normalized sizes for contained .vcf/.vcf.gz files -normalized_vcf_size() { +# +# `vcf_rdfizer_gzip.py` answers this from the file's own structure for BGZF +# (what bgzip/bcftools/tabix emit) and for ordinary single-member gzip, so the +# common case costs milliseconds instead of a full decompression pass. It falls +# back to inflating internally when the structure cannot settle the answer, and +# the shell falls back to `gzip -dc` if the helper is unavailable. The method +# used is recorded alongside the size so the metric stays auditable. +VCF_SIZE_HELPER=${VCF_SIZE_HELPER:-/opt/vcf-rdfizer/vcf_rdfizer_gzip.py} + +# Prints " ". Callers run this in a command substitution, so the +# method has to travel out with the value rather than through a global. +vcf_size_with_method() { local path="$1" local total=0 + local method="" entry entry_size entry_method measured if [[ -f "$path" ]]; then if [[ "$path" == *.vcf.gz ]]; then - gzip -dc "$path" | wc -c | tr -d ' ' + if [[ -f "$VCF_SIZE_HELPER" ]] && measured=$(python3 "$VCF_SIZE_HELPER" --print-method "$path" 2>/dev/null); then + printf '%s\n' "$measured" + return + fi + printf '%s inflate-shell\n' "$(gzip -dc "$path" | wc -c | tr -d ' ')" return fi - stat_size "$path" + printf '%s stat\n' "$(stat_size "$path")" return fi @@ -134,15 +213,29 @@ normalized_vcf_size() { if [[ ! -f "$file" ]]; then continue fi - size=$(normalized_vcf_size "$file") - total=$((total + size)) + entry=$(vcf_size_with_method "$file") + entry_size="${entry%% *}" + entry_method="${entry#* }" + total=$((total + entry_size)) + if [[ -z "$method" ]]; then + method="$entry_method" + elif [[ "$method" != "$entry_method" ]]; then + method="mixed" + fi done shopt -u nullglob - echo "$total" + printf '%s %s\n' "$total" "${method:-none}" return fi - echo 0 + printf '0 none\n' +} + +# Backwards-compatible size-only accessor for direct callers of this script. +normalized_vcf_size() { + local result + result=$(vcf_size_with_method "$1") + printf '%s\n' "${result%% *}" } have_gnu_time() { [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; } @@ -163,17 +256,31 @@ hash_file_sha256() { } # Count triples via non-comment RDF lines ending in '.'. +# +# `grep -c` counts in one process and, in the C locale, works byte-wise instead +# of validating UTF-8 for every line. On a cohort-scale aggregate this is +# several times faster than piping through `awk` or `wc -l`, and it is the last +# unavoidable full pass over the RDF output. `grep` exits 1 when nothing +# matched, so the `|| true` sits inside the pipeline's last stage to keep a +# genuine `gzip` failure visible under `set -o pipefail`. +TRIPLE_LINE_REGEX='^[[:space:]]*[^#].*\.[[:space:]]*$' + +count_triple_lines() { + local path="$1" + if [[ "$path" == *.gz ]]; then + gzip -dc "$path" | { LC_ALL=C grep -cE "$TRIPLE_LINE_REGEX" || true; } + return + fi + { LC_ALL=C grep -cE "$TRIPLE_LINE_REGEX" "$path" || true; } +} + count_triples_json() { local path="$1" local total=0 if [[ -f "$path" ]]; then local count - if [[ "$path" == *.gz ]]; then - count=$(gzip -dc "$path" | awk '/^[[:space:]]*[^#].*\.[[:space:]]*$/ { count++ } END { print count + 0 }') - else - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$path" || true) | wc -l | tr -d ' ' ) - fi + count=$(count_triple_lines "$path") echo "{" printf " \"%s\": %s,\n" "$path" "$count" printf " \"TOTAL\": %s\n" "$count" @@ -187,7 +294,7 @@ count_triples_json() { for f in "$path"/*; do if [[ -f "$f" ]]; then local count - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$f" || true) | wc -l | tr -d ' ' ) + count=$(count_triple_lines "$f") total=$((total + count)) printf " \"%s\": %s,\n" "$f" "$count" fi @@ -201,12 +308,14 @@ count_triples_json() { # Replace plain VCF null marker literals (`"."`) with typed null literals. # Ontology alignment: # "." -> "."^^vcfr:Null -annotate_null_literals_nt() { - local nt_path="$1" - local tmp_path - tmp_path="$(mktemp "${nt_path}.nullfix.XXXXXX")" - sed -E 's/"\."([[:space:]]+\.)/"."^^\1/g' "$nt_path" > "$tmp_path" - mv "$tmp_path" "$nt_path" +# +# The rewrite is per-line and has no cross-line context, so it is applied to +# each RMLStreamer part as it is streamed into the aggregate. Running it as a +# separate pass over the finished aggregate instead would read and rewrite the +# complete RDF output a second time. +NULL_LITERAL_SED='s/"\."([[:space:]]+\.)/"."^^\1/g' +annotate_null_literals_stream() { + sed -E "$NULL_LITERAL_SED" "$1" } @@ -250,14 +359,38 @@ fi # ---------- Pre-run ---------- IN_SIZE=$(stat_size "$IN") -VCF_SIZE=$(normalized_vcf_size "$IN_VCF") +VCF_SIZE_INFO=$(vcf_size_with_method "$IN_VCF") +VCF_SIZE="${VCF_SIZE_INFO%% *}" +VCF_SIZE_METHOD="${VCF_SIZE_INFO#* }" # ---------- Run RMLStreamer with timing ---------- EXIT_CODE=0 -if have_gnu_time; then - /usr/bin/time -v -o "$TIME_LOG" -- "${JAVA_CMD[@]}" || EXIT_CODE=$? +run_rmlstreamer() { + if have_gnu_time; then + /usr/bin/time -v -o "$TIME_LOG" -- "${JAVA_CMD[@]}" + else + { time -p "${JAVA_CMD[@]}"; } >"$TIME_LOG" 2>&1 + fi +} + +if [[ -n "$PROGRESS_FILE" ]]; then + progress_emit "rmlstreamer" "started" 0 null bytes 0 + run_rmlstreamer & + RMLSTREAMER_PID=$! + while kill -0 "$RMLSTREAMER_PID" >/dev/null 2>&1; do + scan_parts + progress_emit "rmlstreamer" "heartbeat" "$PART_BYTES" null bytes "$PART_COUNT" + sleep 1 + done + wait "$RMLSTREAMER_PID" || EXIT_CODE=$? + scan_parts + if (( EXIT_CODE == 0 )); then + progress_emit "rmlstreamer" "complete" "$PART_BYTES" null bytes "$PART_COUNT" + else + progress_emit "rmlstreamer" "failed" "$PART_BYTES" null bytes "$PART_COUNT" + fi else - { time -p "${JAVA_CMD[@]}"; } >"$TIME_LOG" 2>&1 || EXIT_CODE=$? + run_rmlstreamer || EXIT_CODE=$? fi # Normalize output files to .nt for downstream line-oriented processing. The @@ -278,6 +411,11 @@ done # disk spikes. shopt -s nullglob PART_FILES=("$PARTS_DIR"/*.nt) +PART_TOTAL=${#PART_FILES[@]} +PART_INDEX=0 +if (( PART_TOTAL > 0 )); then + progress_emit "rdf-aggregate" "started" 0 "$PART_TOTAL" parts 0 +fi if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then MERGED_RDF="${MERGED_NT}.gz" MERGED_TMP="${MERGED_RDF}.partial.$$" @@ -296,15 +434,18 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then FIRST_SEEN=$(awk -F'\t' -v hash="$PART_HASH" '$1 == hash { print $2; exit }' "$SEEN_MAP_FILE") echo "WARNING: skipping duplicate RDF part '$PART_NT' (same content as '$FIRST_SEEN')." >&2 rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" continue fi printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" printf "%s\t%s\n" "$PART_HASH" "$PART_NT" >> "$SEEN_MAP_FILE" - annotate_null_literals_nt "$PART_NT" # Concatenated gzip members form one valid sequential gzip stream while # allowing each completed RMLStreamer part to be deleted immediately. - gzip -c "$PART_NT" >> "$MERGED_TMP" + annotate_null_literals_stream "$PART_NT" | gzip -c >> "$MERGED_TMP" rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" done rm -f "$SEEN_HASH_FILE" "$SEEN_MAP_FILE" mv "$MERGED_TMP" "$MERGED_RDF" @@ -327,37 +468,33 @@ if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then FIRST_SEEN=$(awk -F'\t' -v hash="$PART_HASH" '$1 == hash { print $2; exit }' "$SEEN_MAP_FILE") echo "WARNING: skipping duplicate RDF part '$PART_NT' (same content as '$FIRST_SEEN')." >&2 rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" continue fi printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" printf "%s\t%s\n" "$PART_HASH" "$PART_NT" >> "$SEEN_MAP_FILE" - cat "$PART_NT" >> "$MERGED_NT" + annotate_null_literals_stream "$PART_NT" >> "$MERGED_NT" rm -f "$PART_NT" + PART_INDEX=$((PART_INDEX + 1)) + progress_emit "rdf-aggregate" "part" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" done rm -f "$SEEN_HASH_FILE" "$SEEN_MAP_FILE" else : > "$MERGED_NT" OUTPUT_PATH="$MERGED_NT" -fi - shopt -u nullglob - if [[ "$RDF_STORAGE_MODE" != "space-optimized" ]]; then - OUTPUT_PATH="$MERGED_NT" fi +shopt -u nullglob +if [[ "$RDF_STORAGE_MODE" != "space-optimized" ]]; then + OUTPUT_PATH="$MERGED_NT" +fi +if (( PART_TOTAL > 0 )); then + progress_emit "rdf-aggregate" "complete" "$PART_INDEX" "$PART_TOTAL" parts "$PART_INDEX" +fi rm -rf "$PARTS_DIR" trap - EXIT -# Apply ontology-compliant null datatype annotation to produced RDF files. -if [[ -f "$OUTPUT_PATH" && "$OUTPUT_PATH" != *.gz ]]; then - annotate_null_literals_nt "$OUTPUT_PATH" -elif [[ -d "$OUTPUT_PATH" ]]; then - shopt -s nullglob - for RDF_NT in "$OUTPUT_PATH"/*.nt; do - annotate_null_literals_nt "$RDF_NT" - done - shopt -u nullglob -fi - OUT_SIZE=$(stat_size "$OUTPUT_PATH") TRIPLES_JSON=$(count_triples_json "$OUTPUT_PATH") @@ -398,6 +535,7 @@ cat > "$METRICS_JSON" <&2 - exit 1 -fi - -RECORDS_TSV="$1" -HEADERS_TSV="${2:-/data/tsv/header_lines.tsv}" -METADATA_TSV="${3:-/data/tsv/file_metadata.tsv}" -if [ "$#" -eq 4 ]; then - RULES_FILE="$4" -fi - -# Make sure rules file exists -if [ ! -f "$RULES_FILE" ]; then - echo "Error: $RULES_FILE not found in current directory." >&2 - exit 1 -fi - -escape_replacement() { - # Escape characters meaningful to sed replacement text. - printf '%s' "$1" | sed -e 's/[&#]/\\&/g' -} - -ESCAPED_RECORDS_TSV=$(escape_replacement "$RECORDS_TSV") -ESCAPED_HEADERS_TSV=$(escape_replacement "$HEADERS_TSV") -ESCAPED_METADATA_TSV=$(escape_replacement "$METADATA_TSV") - -# Replace default TSV locations in the mapping. -sed -i.bak -E \ - -e "s#(csvw:url \")/data/tsv/records\\.tsv(\";)#\1${ESCAPED_RECORDS_TSV}\2#g" \ - -e "s#(csvw:url \")/data/tsv/header_lines\\.tsv(\";)#\1${ESCAPED_HEADERS_TSV}\2#g" \ - -e "s#(csvw:url \")/data/tsv/file_metadata\\.tsv(\";)#\1${ESCAPED_METADATA_TSV}\2#g" \ - "$RULES_FILE" - -echo "Updated TSV source paths in $RULES_FILE" -echo " records: $RECORDS_TSV" -echo " headers: $HEADERS_TSV" -echo " metadata: $METADATA_TSV" -echo "Backup created as ${RULES_FILE}.bak" diff --git a/src/validate_compression.py b/src/validate_compression.py index e86b002..dd46b6d 100644 --- a/src/validate_compression.py +++ b/src/validate_compression.py @@ -82,8 +82,8 @@ def validate(args: argparse.Namespace) -> dict: } if args.format == "hdt": - # Loading through the bundled Java launcher also verifies that the HDT - # structure is readable and eagerly creates the query index. + # The bundled Java-free helper streams the HDT and eagerly creates the + # query index. hdt2rdf below independently checks decoded readability. if not args.skip_index_check: index_helper = Path("/opt/vcf-rdfizer/ensure_hdt_index.sh") if not index_helper.is_file(): diff --git a/src/validation/queries/common/preflight_blank_nodes.rq b/src/validation/queries/common/preflight_blank_nodes.rq new file mode 100644 index 0000000..bc217a3 --- /dev/null +++ b/src/validation/queries/common/preflight_blank_nodes.rq @@ -0,0 +1,18 @@ +PREFIX vcfr: + +# Blank nodes anywhere in the graph. +# +# Every class in the VCF-RDFizer vocabulary declares an vcfr:iriTemplate, so a +# blank node means a term map produced no IRI. That is always a defect here: +# blank-node identity is scoped to the file, so the record and value digests +# could not address such a node, and neither could anything downstream that +# merges or re-serializes the graph. +# +# A predicate cannot be a blank node in RDF, so only subject and object are +# examined. +SELECT ?s ?p ?o +WHERE { + ?s ?p ?o . + FILTER(ISBLANK(?s) || ISBLANK(?o)) +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_blank_nodes_count.rq b/src/validation/queries/common/preflight_blank_nodes_count.rq new file mode 100644 index 0000000..8cfd553 --- /dev/null +++ b/src/validation/queries/common/preflight_blank_nodes_count.rq @@ -0,0 +1,8 @@ +PREFIX vcfr: + +# Exact number of triples involving a blank node. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?s ?p ?o . + FILTER(ISBLANK(?s) || ISBLANK(?o)) +} diff --git a/src/validation/queries/common/preflight_distinct_triple_count.rq b/src/validation/queries/common/preflight_distinct_triple_count.rq new file mode 100644 index 0000000..6b23075 --- /dev/null +++ b/src/validation/queries/common/preflight_distinct_triple_count.rq @@ -0,0 +1,17 @@ +PREFIX vcfr: + +# How many distinct triples the graph contains. +# +# A SPARQL store holds a set, so a duplicated line in the N-Triples file is +# invisible to every other query here: the store collapses it on load. The only +# way to see duplication is to compare this against the number of statements the +# parser actually read, which the Raptor step records. The difference is exactly +# the number of redundant lines. +# +# This matters because duplicate RDF parts are a known failure mode of the +# conversion - run_conversion.sh already carries a defensive dedupe for it - and +# a duplicated aggregate is twice the size for no added information. +SELECT (COUNT(*) AS ?distinctTripleCount) +WHERE { + ?s ?p ?o . +} diff --git a/src/validation/queries/common/preflight_empty_values.rq b/src/validation/queries/common/preflight_empty_values.rq new file mode 100644 index 0000000..2d0effb --- /dev/null +++ b/src/validation/queries/common/preflight_empty_values.rq @@ -0,0 +1,23 @@ +PREFIX vcfr: + +# Empty or whitespace-only terms. +# +# The pipeline should never materialize one. RML emits no triple for an empty +# reference, and every direct emitter either guards on a non-empty value or +# substitutes the VCF missing token, which becomes "."^^vcfr:Null. An empty +# literal therefore means a value was lost rather than marked missing, and an +# empty IRI means a template substitution collapsed. +# +# Whitespace-only counts as empty: a literal of spaces or tabs carries no more +# information than "" and is just as certainly a defect. +SELECT ?s ?p ?o ?issue +WHERE { + ?s ?p ?o . + BIND( + IF(ISLITERAL(?o) && REGEX(STR(?o), "^\\s*$"), "EMPTY_LITERAL", + IF(STR(?s) = "" || STR(?p) = "" || (ISIRI(?o) && STR(?o) = ""), "EMPTY_IRI", "") + ) AS ?issue + ) + FILTER(?issue != "") +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_empty_values_count.rq b/src/validation/queries/common/preflight_empty_values_count.rq new file mode 100644 index 0000000..022c095 --- /dev/null +++ b/src/validation/queries/common/preflight_empty_values_count.rq @@ -0,0 +1,11 @@ +PREFIX vcfr: + +# Exact number of triples carrying an empty or whitespace-only term. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?s ?p ?o . + FILTER( + (ISLITERAL(?o) && REGEX(STR(?o), "^\\s*$")) + || STR(?s) = "" || STR(?p) = "" || (ISIRI(?o) && STR(?o) = "") + ) +} diff --git a/src/validation/queries/common/preflight_missing_token_conformance.rq b/src/validation/queries/common/preflight_missing_token_conformance.rq new file mode 100644 index 0000000..bfbec42 --- /dev/null +++ b/src/validation/queries/common/preflight_missing_token_conformance.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +SELECT ?s ?p ?o (DATATYPE(?o) AS ?datatype) +WHERE { + ?s ?p ?o . + FILTER(ISLITERAL(?o) && STR(?o) = ".") + FILTER(DATATYPE(?o) != vcfr:Null) +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_missing_token_conformance_count.rq b/src/validation/queries/common/preflight_missing_token_conformance_count.rq new file mode 100644 index 0000000..a909af3 --- /dev/null +++ b/src/validation/queries/common/preflight_missing_token_conformance_count.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +# Exact number of missing tokens serialized as a plain "." literal. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?s ?p ?o . + FILTER(ISLITERAL(?o) && STR(?o) = ".") + FILTER(DATATYPE(?o) != vcfr:Null) +} diff --git a/src/validation/queries/common/preflight_position_datatype.rq b/src/validation/queries/common/preflight_position_datatype.rq new file mode 100644 index 0000000..132a55e --- /dev/null +++ b/src/validation/queries/common/preflight_position_datatype.rq @@ -0,0 +1,21 @@ +PREFIX vcfr: +PREFIX xsd: + +# POS must carry an integer datatype, never a plain string or a fractional type. +# +# The accepted set is the XSD integer family rather than xsd:integer alone, +# because a store may canonicalise an integer literal to a narrower derived +# type at index time. QLever, for instance, reports "100"^^xsd:integer as +# xsd:int, which would otherwise make this preflight fail every run on that +# engine while passing on Comunica. Anything outside the family - xsd:string, +# xsd:decimal, xsd:double, a plain literal - is still reported. +SELECT ?record ?pos (DATATYPE(?pos) AS ?datatype) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:pos ?pos . + FILTER(DATATYPE(?pos) NOT IN ( + xsd:integer, xsd:int, xsd:long, xsd:short, xsd:byte, + xsd:nonNegativeInteger, xsd:positiveInteger, + xsd:unsignedLong, xsd:unsignedInt, xsd:unsignedShort, xsd:unsignedByte + )) +} +LIMIT 100 diff --git a/src/validation/queries/common/preflight_position_datatype_count.rq b/src/validation/queries/common/preflight_position_datatype_count.rq new file mode 100644 index 0000000..e773e8e --- /dev/null +++ b/src/validation/queries/common/preflight_position_datatype_count.rq @@ -0,0 +1,13 @@ +PREFIX vcfr: +PREFIX xsd: + +# Exact number of POS values outside the XSD integer family. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:pos ?pos . + FILTER(DATATYPE(?pos) NOT IN ( + xsd:integer, xsd:int, xsd:long, xsd:short, xsd:byte, + xsd:nonNegativeInteger, xsd:positiveInteger, + xsd:unsignedLong, xsd:unsignedInt, xsd:unsignedShort, xsd:unsignedByte + )) +} diff --git a/src/validation/queries/common/preflight_record_cardinality.rq b/src/validation/queries/common/preflight_record_cardinality.rq new file mode 100644 index 0000000..5b683cb --- /dev/null +++ b/src/validation/queries/common/preflight_record_cardinality.rq @@ -0,0 +1,19 @@ +PREFIX vcfr: + +SELECT ?record + (COUNT(DISTINCT ?chrom) AS ?chromCount) + (COUNT(DISTINCT ?pos) AS ?posCount) + (COUNT(DISTINCT ?ref) AS ?refCount) + (COUNT(DISTINCT ?alt) AS ?altCount) + (COUNT(DISTINCT ?call) AS ?callCount) +WHERE { + ?record a vcfr:VCFRecord . + OPTIONAL { ?record vcfr:chrom ?chrom } + OPTIONAL { ?record vcfr:pos ?pos } + OPTIONAL { ?record vcfr:ref ?ref } + OPTIONAL { ?record vcfr:alt ?alt } + OPTIONAL { ?record vcfr:hasCall ?call } +} +GROUP BY ?record +HAVING(COUNT(DISTINCT ?chrom) != 1 || COUNT(DISTINCT ?pos) != 1 || COUNT(DISTINCT ?ref) != 1 || COUNT(DISTINCT ?alt) != 1 || COUNT(DISTINCT ?call) != 1) +LIMIT 100 diff --git a/src/validation/queries/common/preflight_record_cardinality_count.rq b/src/validation/queries/common/preflight_record_cardinality_count.rq new file mode 100644 index 0000000..1f25ced --- /dev/null +++ b/src/validation/queries/common/preflight_record_cardinality_count.rq @@ -0,0 +1,19 @@ +PREFIX vcfr: + +# Exact number of records with anomalous property cardinality. +# The companion query returns at most 100 example rows for diagnosis; this one +# reports the true severity, which a LIMITed sample cannot. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + SELECT ?record + WHERE { + ?record a vcfr:VCFRecord . + OPTIONAL { ?record vcfr:chrom ?chrom } + OPTIONAL { ?record vcfr:pos ?pos } + OPTIONAL { ?record vcfr:ref ?ref } + OPTIONAL { ?record vcfr:alt ?alt } + OPTIONAL { ?record vcfr:hasCall ?call } + } + GROUP BY ?record + HAVING(COUNT(DISTINCT ?chrom) != 1 || COUNT(DISTINCT ?pos) != 1 || COUNT(DISTINCT ?ref) != 1 || COUNT(DISTINCT ?alt) != 1 || COUNT(DISTINCT ?call) != 1) +} diff --git a/src/validation/queries/common/q01_record_density_1mb.rq b/src/validation/queries/common/q01_record_density_1mb.rq new file mode 100644 index 0000000..e37b999 --- /dev/null +++ b/src/validation/queries/common/q01_record_density_1mb.rq @@ -0,0 +1,10 @@ +PREFIX vcfr: +PREFIX xsd: + +SELECT ?chrom ?windowIndex (COUNT(DISTINCT ?record) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:chrom ?chrom ; vcfr:pos ?pos . + BIND(FLOOR((xsd:integer(?pos) - 1) / 1000000) AS ?windowIndex) +} +GROUP BY ?chrom ?windowIndex +ORDER BY ?chrom ?windowIndex diff --git a/src/validation/queries/common/q02_variant_shape_counts.rq b/src/validation/queries/common/q02_variant_shape_counts.rq new file mode 100644 index 0000000..6f29eb0 --- /dev/null +++ b/src/validation/queries/common/q02_variant_shape_counts.rq @@ -0,0 +1,34 @@ +PREFIX vcfr: + +SELECT ?variantClass (COUNT(DISTINCT ?record) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:ref ?refLiteral ; vcfr:alt ?altLiteral . + BIND(UCASE(STR(?refLiteral)) AS ?ref) + BIND(UCASE(STR(?altLiteral)) AS ?alt) + BIND( + IF(?alt = ".", "NO_ALT", + IF(CONTAINS(?alt, ","), "MULTIALLELIC", + IF( + ?alt = "*" || CONTAINS(?alt, "[") || CONTAINS(?alt, "]") || + (STRSTARTS(?alt, "<") && STRENDS(?alt, ">")), + "SYMBOLIC_OR_BREAKEND", + IF( + !REGEX(?ref, "^[ACGTN]+$") || !REGEX(?alt, "^[ACGTN]+$"), + "OTHER", + IF( + STRLEN(?ref) = 1 && STRLEN(?alt) = 1, + "SNV", + IF( + STRLEN(?ref) = STRLEN(?alt), + "MNV_OR_EQUAL_LENGTH_SUBSTITUTION", + IF(STRLEN(?ref) < STRLEN(?alt), "INSERTION_SHAPE", "DELETION_SHAPE") + ) + ) + ) + ) + ) + ) AS ?variantClass + ) +} +GROUP BY ?variantClass +ORDER BY ?variantClass diff --git a/src/validation/queries/common/q03_titv.rq b/src/validation/queries/common/q03_titv.rq new file mode 100644 index 0000000..a768a91 --- /dev/null +++ b/src/validation/queries/common/q03_titv.rq @@ -0,0 +1,12 @@ +PREFIX vcfr: + +SELECT + (COUNT(DISTINCT ?record) AS ?biallelicSnvCount) + (SUM(IF((?ref = "A" && ?alt = "G") || (?ref = "G" && ?alt = "A") || (?ref = "C" && ?alt = "T") || (?ref = "T" && ?alt = "C"), 1, 0)) AS ?transitionCount) + (SUM(IF((?ref = "A" && ?alt = "G") || (?ref = "G" && ?alt = "A") || (?ref = "C" && ?alt = "T") || (?ref = "T" && ?alt = "C"), 0, 1)) AS ?transversionCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:ref ?refLiteral ; vcfr:alt ?altLiteral . + BIND(UCASE(STR(?refLiteral)) AS ?ref) + BIND(UCASE(STR(?altLiteral)) AS ?alt) + FILTER(REGEX(?ref, "^[ACGT]$") && REGEX(?alt, "^[ACGT]$") && ?ref != ?alt) +} diff --git a/src/validation/queries/common/q04_filter_distribution.rq b/src/validation/queries/common/q04_filter_distribution.rq new file mode 100644 index 0000000..9545410 --- /dev/null +++ b/src/validation/queries/common/q04_filter_distribution.rq @@ -0,0 +1,11 @@ +PREFIX vcfr: + +SELECT ?filterStatus ?filterLexical (COUNT(DISTINCT ?record) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; vcfr:hasCall ?call . + ?call vcfr:filter ?filterLiteral . + BIND(STR(?filterLiteral) AS ?filterLexical) + BIND(IF(?filterLexical = "PASS", "PASS", IF(?filterLexical = ".", "NOT_APPLIED", "FAILED")) AS ?filterStatus) +} +GROUP BY ?filterStatus ?filterLexical +ORDER BY ?filterStatus ?filterLexical diff --git a/src/validation/queries/common/q07_file_metadata.rq b/src/validation/queries/common/q07_file_metadata.rq new file mode 100644 index 0000000..26155d3 --- /dev/null +++ b/src/validation/queries/common/q07_file_metadata.rq @@ -0,0 +1,17 @@ +PREFIX vcfr: + +# The file-level declarations carried over from the VCF meta-information lines. +# +# COALESCE binds an explicit sentinel rather than leaving the variable unbound, +# so a missing declaration is reported as a value mismatch against the parser +# instead of failing result normalization with a less useful error. +SELECT + (COALESCE(?ff, "(absent)") AS ?fileFormat) + (COALESCE(?rg, "(absent)") AS ?referenceGenome) + (COALESCE(?ss, "(absent)") AS ?sourceSoftware) +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:fileFormat ?ffLiteral BIND(STR(?ffLiteral) AS ?ff) } + OPTIONAL { ?file vcfr:referenceGenome ?rgLiteral BIND(STR(?rgLiteral) AS ?rg) } + OPTIONAL { ?file vcfr:sourceSoftware ?ssLiteral BIND(STR(?ssLiteral) AS ?ss) } +} diff --git a/src/validation/queries/common/q08_header_line_census.rq b/src/validation/queries/common/q08_header_line_census.rq new file mode 100644 index 0000000..4bfa223 --- /dev/null +++ b/src/validation/queries/common/q08_header_line_census.rq @@ -0,0 +1,12 @@ +PREFIX vcfr: + +# One row per '##' meta-information key, with how many lines carry it. +# Counting per key rather than in total means a dropped INFO or FILTER +# declaration is localized instead of only shifting a single number. +SELECT ?headerKey (COUNT(DISTINCT ?line) AS ?lineCount) +WHERE { + ?line a vcfr:HeaderLine ; vcfr:headerKey ?headerKeyLiteral . + BIND(STR(?headerKeyLiteral) AS ?headerKey) +} +GROUP BY ?headerKey +ORDER BY ?headerKey diff --git a/src/validation/queries/common/q09_predicate_census.rq b/src/validation/queries/common/q09_predicate_census.rq new file mode 100644 index 0000000..cd62500 --- /dev/null +++ b/src/validation/queries/common/q09_predicate_census.rq @@ -0,0 +1,15 @@ +PREFIX vcfr: + +# Every predicate in the graph and how many triples use it. +# +# This is the completeness check. Comparing it against counts derived from the +# VCF catches three things at once: a predicate that is missing entirely, one +# with the wrong cardinality, and - because an unexpected predicate has no +# expected count - one that should not be in the graph at all. Result size is +# the number of distinct predicates, so it stays small on any input. +SELECT ?predicate (COUNT(*) AS ?tripleCount) +WHERE { + ?s ?predicate ?o . +} +GROUP BY ?predicate +ORDER BY ?predicate diff --git a/src/validation/queries/common/q10_class_census.rq b/src/validation/queries/common/q10_class_census.rq new file mode 100644 index 0000000..872a642 --- /dev/null +++ b/src/validation/queries/common/q10_class_census.rq @@ -0,0 +1,11 @@ +PREFIX vcfr: + +# Every asserted rdf:type and how many resources carry it. Complements the +# predicate census: together they bound both the properties and the resources +# a conversion is allowed to produce. +SELECT ?class (COUNT(DISTINCT ?resource) AS ?resourceCount) +WHERE { + ?resource a ?class . +} +GROUP BY ?class +ORDER BY ?class diff --git a/src/validation/queries/common/q11_record_digest.rq b/src/validation/queries/common/q11_record_digest.rq new file mode 100644 index 0000000..69d686b --- /dev/null +++ b/src/validation/queries/common/q11_record_digest.rq @@ -0,0 +1,43 @@ +PREFIX vcfr: + +# Per-record identity, aggregated into a fixed-size histogram. +# +# Every other core query is a GROUP BY count, so values permuted between +# records are invisible to all of them: swap POS between two records in the +# same 1 Mb window and nothing changes. This closes that gap. +# +# The record IRI is hashed together with its values. That binding is the whole +# point: digesting the values alone would give a permutation the same multiset +# and change nothing. Bucketing on the first byte of the hash keeps the result +# at most 256 rows for any graph size, and a mismatch is localized by +# re-querying only the differing buckets. +# +# Fields are separated by U+001F (unit separator), written as a SPARQL UCHAR +# escape. It cannot appear in a VCF field, so no combination of values can be +# made to collide by shifting a field boundary. +SELECT ?bucket (COUNT(*) AS ?recordCount) +WHERE { + ?record a vcfr:VCFRecord ; + vcfr:chrom ?chrom ; + vcfr:pos ?pos ; + vcfr:recordId ?recordIdLiteral ; + vcfr:ref ?ref ; + vcfr:alt ?alt ; + vcfr:hasCall ?call . + ?call vcfr:qual ?qual ; + vcfr:filter ?filter ; + vcfr:infoRaw ?info . + BIND(SUBSTR(SHA256(CONCAT( + STR(?record), "\u001F", + STR(?chrom), "\u001F", + STR(?pos), "\u001F", + STR(?recordIdLiteral), "\u001F", + STR(?ref), "\u001F", + STR(?alt), "\u001F", + STR(?qual), "\u001F", + STR(?filter), "\u001F", + STR(?info) + )), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/queries/common/q12_info_value_digest.rq b/src/validation/queries/common/q12_info_value_digest.rq new file mode 100644 index 0000000..48b2a4b --- /dev/null +++ b/src/validation/queries/common/q12_info_value_digest.rq @@ -0,0 +1,17 @@ +PREFIX vcfr: + +# Structured INFO values, bound to the node that identifies record and key. +# +# The census counts INFO value nodes and the record digest covers the raw INFO +# string, but neither notices a structured value that disagrees with the key it +# was parsed from. Hashing the value together with its own IRI closes that, and +# because the result is a histogram it needs no ordering guarantee from the +# engine - GROUP_CONCAT would not be portable here. +SELECT ?bucket (COUNT(*) AS ?valueCount) +WHERE { + ?value a vcfr:InfoFieldValue ; vcfr:fieldValue ?lexLiteral . + BIND(STR(?lexLiteral) AS ?lex) + BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/queries/condensed/preflight_representation_profile.rq b/src/validation/queries/condensed/preflight_representation_profile.rq new file mode 100644 index 0000000..362062c --- /dev/null +++ b/src/validation/queries/condensed/preflight_representation_profile.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +SELECT ?file ?profile +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:CondensedRepresentation) +} +LIMIT 100 diff --git a/src/validation/queries/condensed/preflight_representation_profile_count.rq b/src/validation/queries/condensed/preflight_representation_profile_count.rq new file mode 100644 index 0000000..2c90126 --- /dev/null +++ b/src/validation/queries/condensed/preflight_representation_profile_count.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +# Exact number of VCFFile resources not declaring the condensed profile. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:CondensedRepresentation) +} diff --git a/src/validation/queries/condensed/preflight_sample_gt_inventory.rq b/src/validation/queries/condensed/preflight_sample_gt_inventory.rq new file mode 100644 index 0000000..9fbde64 --- /dev/null +++ b/src/validation/queries/condensed/preflight_sample_gt_inventory.rq @@ -0,0 +1,16 @@ +PREFIX vcfr: + +SELECT + (COUNT(DISTINCT ?sample) AS ?sampleCount) + (COUNT(DISTINCT ?sampleId) AS ?sampleIdCount) + (COUNT(DISTINCT ?gtVector) AS ?gtVectorCount) +WHERE { + OPTIONAL { + ?sampleSet a vcfr:SampleSet ; vcfr:hasSample ?sample . + ?sample vcfr:sampleName ?sampleId . + } + OPTIONAL { + ?gtVector a vcfr:FormatValueVector ; vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . + ?definition vcfr:fieldId "GT" . + } +} diff --git a/src/validation/queries/condensed/q05_sample_genotype_counts.rq b/src/validation/queries/condensed/q05_sample_genotype_counts.rq new file mode 100644 index 0000000..32bf44a --- /dev/null +++ b/src/validation/queries/condensed/q05_sample_genotype_counts.rq @@ -0,0 +1,27 @@ +PREFIX vcfr: + +SELECT ?sampleId ?genotypeClass (COUNT(DISTINCT ?record) AS ?callCount) +WHERE { + ?file a vcfr:VCFFile ; vcfr:hasRecord ?record ; vcfr:hasSampleSet ?sampleSet . + ?record a vcfr:VCFRecord ; vcfr:hasCall ?call . + ?sampleSet vcfr:hasSample ?sample . + ?sample vcfr:sampleName ?sampleId ; vcfr:sampleIndex ?sampleIndex . + OPTIONAL { + ?sample vcfr:sampleIndex ?sampleIndexForVector . + ?call vcfr:hasCallMatrix ?matrix . + ?matrix vcfr:appliesToSampleSet ?sampleSet ; vcfr:hasFormatValueVector ?gtVector . + ?gtVector vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . + ?definition vcfr:fieldId "GT" . + BIND(?sampleIndexForVector - 1 AS ?zeroIndex) + BIND(REPLACE(STR(?encoded), CONCAT("^(?:[^\\t]*\\t){", STR(?zeroIndex), "}([^\\t]*).*$"), "$1") AS ?gtLiteral) + } + BIND(IF(BOUND(?gtLiteral), REPLACE(STR(?gtLiteral), "[|]", "/"), "") AS ?gt) + BIND(IF(!BOUND(?gtLiteral), "NO_GT_FIELD", + IF(CONTAINS(?gt, "."), "MISSING", + IF(REGEX(?gt, "^[0-9]+$"), IF(?gt = "0", "HAPLOID_REF", "HAPLOID_ALT"), + IF(REGEX(?gt, "^[0-9]+/[0-9]+$"), + IF(STRBEFORE(?gt, "/") = STRAFTER(?gt, "/"), IF(STRBEFORE(?gt, "/") = "0", "HOM_REF", "HOM_ALT"), "HET"), + "OTHER_PLOIDY")))) AS ?genotypeClass) +} +GROUP BY ?sampleId ?genotypeClass +ORDER BY ?sampleId ?genotypeClass diff --git a/src/validation/queries/condensed/q06_ac_an_distribution.rq b/src/validation/queries/condensed/q06_ac_an_distribution.rq new file mode 100644 index 0000000..5c26b70 --- /dev/null +++ b/src/validation/queries/condensed/q06_ac_an_distribution.rq @@ -0,0 +1,33 @@ +PREFIX vcfr: + +SELECT ?an ?ac (COUNT(DISTINCT ?record) AS ?siteCount) +WHERE { + { + SELECT ?record (SUM(?anContribution) AS ?an) (SUM(?acContribution) AS ?ac) + WHERE { + { + SELECT DISTINCT ?record ?sample ?gt + WHERE { + ?file a vcfr:VCFFile ; vcfr:hasRecord ?record ; vcfr:hasSampleSet ?sampleSet . + ?record a vcfr:VCFRecord ; vcfr:alt ?altLiteral ; vcfr:hasCall ?call . + FILTER(STR(?altLiteral) != "." && !CONTAINS(STR(?altLiteral), ",")) + ?sampleSet vcfr:hasSample ?sample . + ?sample vcfr:sampleIndex ?sampleIndex . + ?call vcfr:hasCallMatrix ?matrix . + ?matrix vcfr:appliesToSampleSet ?sampleSet ; vcfr:hasFormatValueVector ?gtVector . + ?gtVector vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . + ?definition vcfr:fieldId "GT" . + BIND(?sampleIndex - 1 AS ?zeroIndex) + BIND(REPLACE(STR(?encoded), CONCAT("^(?:[^\\t]*\\t){", STR(?zeroIndex), "}([^\\t]*).*$"), "$1") AS ?gtLiteral) + BIND(REPLACE(STR(?gtLiteral), "[|]", "/") AS ?gt) + } + } + BIND(IF(REGEX(?gt, "^[01]$"), 1, IF(REGEX(?gt, "^[01]/[01]$"), 2, 0)) AS ?anContribution) + BIND(IF(?gt = "1", 1, IF(?gt = "0/1" || ?gt = "1/0", 1, IF(?gt = "1/1", 2, 0))) AS ?acContribution) + } + GROUP BY ?record + } + FILTER(?an > 0) +} +GROUP BY ?an ?ac +ORDER BY ?an ?ac diff --git a/src/validation/queries/condensed/q13_format_value_digest.rq b/src/validation/queries/condensed/q13_format_value_digest.rq new file mode 100644 index 0000000..c8e4ff6 --- /dev/null +++ b/src/validation/queries/condensed/q13_format_value_digest.rq @@ -0,0 +1,15 @@ +PREFIX vcfr: + +# The condensed equivalent of the expanded per-cell FORMAT digest. +# +# Condensed graphs store one sample-ordered vector per record and FORMAT key, +# so the vector IRI plus its encoded values carries the same information as the +# per-cell nodes of the expanded shape. +SELECT ?bucket (COUNT(*) AS ?valueCount) +WHERE { + ?value a vcfr:FormatValueVector ; vcfr:encodedValues ?lexLiteral . + BIND(STR(?lexLiteral) AS ?lex) + BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/queries/expanded/preflight_representation_profile.rq b/src/validation/queries/expanded/preflight_representation_profile.rq new file mode 100644 index 0000000..3adeb40 --- /dev/null +++ b/src/validation/queries/expanded/preflight_representation_profile.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +SELECT ?file ?profile +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:ExpandedRepresentation) +} +LIMIT 100 diff --git a/src/validation/queries/expanded/preflight_representation_profile_count.rq b/src/validation/queries/expanded/preflight_representation_profile_count.rq new file mode 100644 index 0000000..9174a06 --- /dev/null +++ b/src/validation/queries/expanded/preflight_representation_profile_count.rq @@ -0,0 +1,9 @@ +PREFIX vcfr: + +# Exact number of VCFFile resources not declaring the expanded profile. +SELECT (COUNT(*) AS ?anomalyCount) +WHERE { + ?file a vcfr:VCFFile . + OPTIONAL { ?file vcfr:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfr:ExpandedRepresentation) +} diff --git a/src/validation/queries/expanded/preflight_sample_gt_inventory.rq b/src/validation/queries/expanded/preflight_sample_gt_inventory.rq new file mode 100644 index 0000000..aa59ccb --- /dev/null +++ b/src/validation/queries/expanded/preflight_sample_gt_inventory.rq @@ -0,0 +1,16 @@ +PREFIX vcfr: + +SELECT + (COUNT(DISTINCT ?sampleCall) AS ?sampleCallCount) + (COUNT(DISTINCT ?sampleId) AS ?sampleIdCount) + (COUNT(DISTINCT ?gtValueNode) AS ?gtValueNodeCount) +WHERE { + OPTIONAL { + ?sampleCall a vcfr:SampleCall ; vcfr:sampleId ?sampleId . + OPTIONAL { + ?sampleCall vcfr:hasFormatValue ?gtValueNode . + FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) + ?gtValueNode vcfr:fieldValue ?gtLiteral . + } + } +} diff --git a/src/validation/queries/expanded/q05_sample_genotype_counts.rq b/src/validation/queries/expanded/q05_sample_genotype_counts.rq new file mode 100644 index 0000000..bed78a2 --- /dev/null +++ b/src/validation/queries/expanded/q05_sample_genotype_counts.rq @@ -0,0 +1,20 @@ +PREFIX vcfr: + +SELECT ?sampleId ?genotypeClass (COUNT(DISTINCT ?sampleCall) AS ?callCount) +WHERE { + ?sampleCall a vcfr:SampleCall ; vcfr:sampleId ?sampleId . + OPTIONAL { + ?sampleCall vcfr:hasFormatValue ?gtValueNode . + FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) + ?gtValueNode vcfr:fieldValue ?gtLiteral . + } + BIND(IF(BOUND(?gtLiteral), REPLACE(STR(?gtLiteral), "[|]", "/"), "") AS ?gt) + BIND(IF(!BOUND(?gtLiteral), "NO_GT_FIELD", + IF(CONTAINS(?gt, "."), "MISSING", + IF(REGEX(?gt, "^[0-9]+$"), IF(?gt = "0", "HAPLOID_REF", "HAPLOID_ALT"), + IF(REGEX(?gt, "^[0-9]+/[0-9]+$"), + IF(STRBEFORE(?gt, "/") = STRAFTER(?gt, "/"), IF(STRBEFORE(?gt, "/") = "0", "HOM_REF", "HOM_ALT"), "HET"), + "OTHER_PLOIDY")))) AS ?genotypeClass) +} +GROUP BY ?sampleId ?genotypeClass +ORDER BY ?sampleId ?genotypeClass diff --git a/src/validation/queries/expanded/q06_ac_an_distribution.rq b/src/validation/queries/expanded/q06_ac_an_distribution.rq new file mode 100644 index 0000000..1a0fc93 --- /dev/null +++ b/src/validation/queries/expanded/q06_ac_an_distribution.rq @@ -0,0 +1,28 @@ +PREFIX vcfr: + +SELECT ?an ?ac (COUNT(DISTINCT ?record) AS ?siteCount) +WHERE { + { + SELECT ?record (SUM(?anContribution) AS ?an) (SUM(?acContribution) AS ?ac) + WHERE { + { + SELECT DISTINCT ?record ?sampleCall ?gt + WHERE { + ?record a vcfr:VCFRecord ; vcfr:alt ?altLiteral ; vcfr:hasCall ?call . + FILTER(STR(?altLiteral) != "." && !CONTAINS(STR(?altLiteral), ",")) + ?call vcfr:hasSampleCall ?sampleCall . + ?sampleCall vcfr:hasFormatValue ?gtValueNode . + FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) + ?gtValueNode vcfr:fieldValue ?gtLiteral . + BIND(REPLACE(STR(?gtLiteral), "[|]", "/") AS ?gt) + } + } + BIND(IF(REGEX(?gt, "^[01]$"), 1, IF(REGEX(?gt, "^[01]/[01]$"), 2, 0)) AS ?anContribution) + BIND(IF(?gt = "1", 1, IF(?gt = "0/1" || ?gt = "1/0", 1, IF(?gt = "1/1", 2, 0))) AS ?acContribution) + } + GROUP BY ?record + } + FILTER(?an > 0) +} +GROUP BY ?an ?ac +ORDER BY ?an ?ac diff --git a/src/validation/queries/expanded/q13_format_value_digest.rq b/src/validation/queries/expanded/q13_format_value_digest.rq new file mode 100644 index 0000000..647abed --- /dev/null +++ b/src/validation/queries/expanded/q13_format_value_digest.rq @@ -0,0 +1,15 @@ +PREFIX vcfr: + +# Every FORMAT value, bound to the node that identifies record, sample and key. +# +# Only GT is compared by q05/q06, so a mangled DP, GQ or AD would otherwise +# pass. The FormatFieldValue IRI already encodes record, sample and key, so +# hashing it with the value makes each cell individually accountable. +SELECT ?bucket (COUNT(*) AS ?valueCount) +WHERE { + ?value a vcfr:FormatFieldValue ; vcfr:fieldValue ?lexLiteral . + BIND(STR(?lexLiteral) AS ?lex) + BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) +} +GROUP BY ?bucket +ORDER BY ?bucket diff --git a/src/validation/validation_runner.py b/src/validation/validation_runner.py new file mode 100644 index 0000000..ecc2db5 --- /dev/null +++ b/src/validation/validation_runner.py @@ -0,0 +1,2811 @@ +#!/usr/bin/env python3 +"""Validate a VCF-RDFizer RDF graph against its source VCF. + +The validator computes six deterministic summaries from the VCF with cyvcf2 +(and bcftools for exact FILTER strings), runs the equivalent SPARQL queries +against the graph, and compares canonical integer results exactly. + +Two axes are independent, and both are recorded in every report: + +**Artifact format** - ``.nt``, ``.nt.gz``, ``.nt.br``, ``.hdt``, ``.cottas``, +``.cottas.gz``, ``.cottas.br``. Anything that is not already plain N-Triples is +decoded into the container scratch directory first, so validating an ``.hdt`` +proves it decodes to a graph that still satisfies every semantic check - a +strictly stronger statement than the triple-count round-trip that +``validate_compression.py`` performs during compression. A plain ``.nt`` source +is read in place and never copied. Nothing decoded is written beneath ``--out``. + +**SPARQL engine** - ``comunica`` (default) queries the file with no setup but +holds the graph in memory; ``qlever`` builds an on-disk index and answers over +a container-local HTTP server, which is what makes cohort-scale graphs +queryable. Both answer identical queries, so the choice is never semantic. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import json +import os +import platform +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.parse +from collections import Counter +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any +from urllib.parse import quote_plus + +# cyvcf2 only exists inside the image. Importing it lazily keeps the pure +# normalization/comparison layer below importable on the host, which is what +# the mutation tests in test/test_validation_logic_unit.py exercise. +try: # pragma: no cover - present in the container, absent on the host + import cyvcf2 + from cyvcf2 import VCF +except ImportError: # pragma: no cover + cyvcf2 = None + VCF = None + + +SCRIPT_DIR = Path(__file__).resolve().parent +QUERY_ROOT = SCRIPT_DIR / "queries" +CORE_QUERIES = ( + "q01_record_density_1mb", + "q02_variant_shape_counts", + "q03_titv", + "q04_filter_distribution", + "q05_sample_genotype_counts", + "q06_ac_an_distribution", + "q07_file_metadata", + "q08_header_line_census", + "q09_predicate_census", + "q10_class_census", + "q11_record_digest", + "q12_info_value_digest", + "q13_format_value_digest", +) +#: Queries that must return exactly one row, normalized to a dict rather than +#: a list so the comparison reads as a field-by-field check. +SINGLE_ROW_QUERIES = frozenset({"q03_titv", "q07_file_metadata"}) +PREFLIGHT_QUERIES = ( + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_missing_token_conformance", + "preflight_representation_profile", + "preflight_sample_gt_inventory", + "preflight_blank_nodes", + "preflight_empty_values", + "preflight_distinct_triple_count", +) +#: Anomaly-style preflights return at most 100 example rows so a broken graph +#: cannot produce an unbounded report. That makes the sample useless as a +#: severity measure, so each one is paired with an aggregate that returns the +#: exact count. Both are reported: the count says how bad, the sample says how. +ANOMALY_PREFLIGHT_QUERIES = ( + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_missing_token_conformance", + "preflight_representation_profile", + "preflight_blank_nodes", + "preflight_empty_values", +) +PREFLIGHT_COUNT_QUERIES = tuple(f"{name}_count" for name in ANOMALY_PREFLIGHT_QUERIES) +ANOMALY_SAMPLE_LIMIT = 100 +TRANSITIONS = {("A", "G"), ("G", "A"), ("C", "T"), ("T", "C")} + +QUERY_SPECS = { + "q01_record_density_1mb": (("chrom", "windowIndex"), ("recordCount",)), + "q02_variant_shape_counts": (("variantClass",), ("recordCount",)), + "q03_titv": ((), ("biallelicSnvCount", "transitionCount", "transversionCount")), + "q04_filter_distribution": (("filterStatus", "filterLexical"), ("recordCount",)), + "q05_sample_genotype_counts": (("sampleId", "genotypeClass"), ("callCount",)), + "q06_ac_an_distribution": (("an", "ac"), ("siteCount",)), + "q07_file_metadata": ((), ("fileFormat", "referenceGenome", "sourceSoftware")), + "q08_header_line_census": (("headerKey",), ("lineCount",)), + "q09_predicate_census": (("predicate",), ("tripleCount",)), + "q10_class_census": (("class",), ("resourceCount",)), + "q11_record_digest": (("bucket",), ("recordCount",)), + "q12_info_value_digest": (("bucket",), ("valueCount",)), + "q13_format_value_digest": (("bucket",), ("valueCount",)), +} +QUERY_SCHEMAS = { + "q01_record_density_1mb": (("chrom", "windowIndex", "recordCount"), {"windowIndex", "recordCount"}, ("chrom", "windowIndex")), + "q02_variant_shape_counts": (("variantClass", "recordCount"), {"recordCount"}, ("variantClass",)), + "q03_titv": (("biallelicSnvCount", "transitionCount", "transversionCount"), {"biallelicSnvCount", "transitionCount", "transversionCount"}, ()), + "q04_filter_distribution": (("filterStatus", "filterLexical", "recordCount"), {"recordCount"}, ("filterStatus", "filterLexical")), + "q05_sample_genotype_counts": (("sampleId", "genotypeClass", "callCount"), {"callCount"}, ("sampleId", "genotypeClass")), + "q06_ac_an_distribution": (("an", "ac", "siteCount"), {"an", "ac", "siteCount"}, ("an", "ac")), + "q07_file_metadata": (("fileFormat", "referenceGenome", "sourceSoftware"), set(), ()), + "q08_header_line_census": (("headerKey", "lineCount"), {"lineCount"}, ("headerKey",)), + "q09_predicate_census": (("predicate", "tripleCount"), {"tripleCount"}, ("predicate",)), + "q10_class_census": (("class", "resourceCount"), {"resourceCount"}, ("class",)), + "q11_record_digest": (("bucket", "recordCount"), {"recordCount"}, ("bucket",)), + "q12_info_value_digest": (("bucket", "valueCount"), {"valueCount"}, ("bucket",)), + "q13_format_value_digest": (("bucket", "valueCount"), {"valueCount"}, ("bucket",)), +} +#: Checks that are only meaningful for the shipped RML mapping: they assume its +#: predicate inventory and its IRI templates. A custom mapping changes both by +#: design, so the wrapper switches these to report-only in that case. +DEFAULT_MAPPING_QUERIES = ( + "q09_predicate_census", + "q10_class_census", + "q11_record_digest", + "q12_info_value_digest", + "q13_format_value_digest", +) +MAPPING_POLICIES = ("strict", "report-only") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +class ValidationProgress: + """Best-effort JSONL progress writer shared with the host progress UI. + + Validation can spend most of its time inside a single SPARQL query. A + small event before and after each query gives the existing host-side + ``ProgressSession`` a useful total without retaining query output or RDF + data in memory. Failures to write the optional sidecar are deliberately + ignored so observability can never change validation semantics. + """ + + def __init__(self, path: Path | None, total: int): + self.path = path + self.total = total + if path is not None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.unlink(missing_ok=True) + except OSError: + pass + + def emit( + self, + phase: str, + *, + completed: int | None = None, + query_id: str | None = None, + detail: str | None = None, + ) -> None: + if self.path is None: + return + payload: dict[str, Any] = { + "stage": "validation", + "phase": phase, + "total": self.total, + "unit": "queries", + } + if completed is not None: + payload["completed"] = completed + if query_id is not None: + payload["query"] = query_id + if detail is not None: + payload["detail"] = detail + try: + with self.path.open("a", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + except OSError: + pass + + +def read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def tool_version(command: list[str], *, table_label: str | None = None) -> str | None: + try: + result = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + timeout=20, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + output = result.stdout.strip() + if table_label: + for line in output.splitlines(): + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if len(cells) >= 2 and cells[0] == table_label: + return cells[1] + return output.splitlines()[0] if output else None + + +def alt_lexical(variant: Any) -> str: + return ",".join("." if item is None else str(item) for item in (variant.ALT or [None])) + + +def classify_variant_shape(ref_value: str, alt_value: str) -> str: + ref, alt = str(ref_value).upper(), str(alt_value).upper() + if alt == ".": + return "NO_ALT" + if "," in alt: + return "MULTIALLELIC" + if alt == "*" or "[" in alt or "]" in alt or (alt.startswith("<") and alt.endswith(">")): + return "SYMBOLIC_OR_BREAKEND" + if not re.fullmatch(r"[ACGTN]+", ref) or not re.fullmatch(r"[ACGTN]+", alt): + return "OTHER" + if len(ref) == len(alt) == 1: + return "SNV" + if len(ref) == len(alt): + return "MNV_OR_EQUAL_LENGTH_SUBSTITUTION" + return "INSERTION_SHAPE" if len(ref) < len(alt) else "DELETION_SHAPE" + + +def filter_status(value: str) -> str: + return "PASS" if value == "PASS" else "NOT_APPLIED" if value == "." else "FAILED" + + +def exact_filter_lexical(variant: Any) -> str: + values = list(variant.FILTERS or []) + if values: + return ";".join(str(value) for value in values) + fields = str(variant).rstrip("\r\n").split("\t", 8) + if len(fields) < 7: + raise ValueError("Could not recover FILTER from serialized VCF record") + return fields[6] + + +def filters_with_bcftools(vcf_path: Path) -> Counter[tuple[str, str]]: + process = subprocess.run( + ["bcftools", "query", "-f", "%FILTER\\n", str(vcf_path)], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + if process.returncode: + raise RuntimeError(f"bcftools FILTER extraction failed: {process.stderr.strip()}") + values = [line.rstrip("\r") for line in process.stdout.splitlines()] + if any(value == "" for value in values): + raise ValueError("bcftools returned an empty FILTER value") + return Counter((filter_status(value), value) for value in values) + + +def genotype_alleles(raw_gt: Any) -> tuple[int | None, ...] | None: + if raw_gt is None: + return None + return tuple(None if value is None or int(value) < 0 else int(value) for value in raw_gt[:-1]) + + +def classify_genotype(alleles: tuple[int | None, ...] | None, *, has_gt: bool) -> str: + if not has_gt: + return "NO_GT_FIELD" + if alleles is None or not alleles or any(value is None for value in alleles): + return "MISSING" + complete = tuple(int(value) for value in alleles if value is not None) + if len(complete) == 1: + return "HAPLOID_REF" if complete[0] == 0 else "HAPLOID_ALT" + if len(complete) == 2: + if complete[0] == complete[1]: + return "HOM_REF" if complete[0] == 0 else "HOM_ALT" + return "HET" + return "OTHER_PLOIDY" + + +#: VCF meta-information keys the mapping lifts onto the VCFFile resource, in +#: the case-insensitive form `vcf_as_tsv.sh` matches them. +FILE_METADATA_KEYS = { + "fileformat": "fileFormat", + "reference": "referenceGenome", + "source": "sourceSoftware", + "filedate": "fileDate", +} +METADATA_ABSENT = "(absent)" + + +# --------------------------------------------------------------------------- +# Graph census expectations +# --------------------------------------------------------------------------- +# The census compares the graph's predicate and class inventory against counts +# derived from the VCF. It is the completeness check: a predicate missing, one +# with the wrong cardinality, and one that should not exist at all are all the +# same comparison. +# +# Expectations are derived from the VCF and the emitters' documented shapes - +# never from default_rules.ttl. Deriving them from the mapping would make the +# mapping test itself, which is exactly how QUAL stayed invisible for so long. +# A custom mapping therefore invalidates them, and the wrapper switches the +# policy to report-only in that case. + +VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" + + +def _nonzero(counts: dict[str, int]) -> dict[str, int]: + return {key: value for key, value in counts.items() if value} + + +#: Field separator for the record digest. U+001F cannot appear in a VCF field, +#: so no shift of a field boundary can make two different records collide. The +#: query in q11_record_digest.rq writes it as a SPARQL \\u001F escape. +def rml_uri_component(value: str) -> str: + """Encode a template value the way RMLStreamer 2.5.0 does. + + Mirrors ``_rml_uri_component`` in ``vcf_rdfizer.py``; that module is not on + the container's import path, and ``test_validation_logic_unit.py`` asserts + the two agree so they cannot drift. + """ + encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") + # urllib leaves '~' unescaped; java.net.URLEncoder does not. + return encoded.replace("+", "%20").replace("~", "%7E") + + +DIGEST_SEPARATOR = "\u001f" +#: Hash prefix length in hex characters. Two gives 256 buckets: enough to make +#: an accidental collision of a *changed* record with its own bucket unlikely +#: to hide anything, while keeping the result fixed-size for any graph. +DIGEST_BUCKET_CHARS = 2 + + +def record_digest_bucket(fields: list[str]) -> str: + """Bucket one record exactly as q11_record_digest.rq does.""" + joined = DIGEST_SEPARATOR.join(fields) + return hashlib.sha256(joined.encode("utf-8")).hexdigest()[:DIGEST_BUCKET_CHARS] + + +def expected_census( + parser: dict[str, Any], representation: str, *, info_representation: str = "structured", + header_representation: str = "structured", +) -> dict[str, list[dict[str, Any]]]: + """Predicate and class counts the graph must contain, and nothing else.""" + records = parser["totalRecords"] + samples = parser["sampleCount"] + header_lines = parser["headerLineCount"] + + classes: dict[str, int] = { + f"{VCFR}VCFFile": 1, + f"{VCFR}VCFHeader": 1, + f"{VCFR}HeaderLine": header_lines, + f"{VCFR}VCFRecord": records, + f"{VCFR}VariantCall": records, + } + predicates: dict[str, int] = { + f"{VCFR}hasHeader": 1, + f"{VCFR}hasHeaderLine": header_lines, + f"{VCFR}headerKey": header_lines, + f"{VCFR}headerValue": parser["headerValueCount"], + f"{VCFR}hasRecord": records, + f"{VCFR}chrom": records, + f"{VCFR}pos": records, + f"{VCFR}recordId": records, + f"{VCFR}ref": records, + f"{VCFR}alt": records, + f"{VCFR}hasCall": records, + f"{VCFR}qual": records, + f"{VCFR}filter": records, + f"{VCFR}infoRaw": records, + f"{VCFR}formatRaw": parser["recordsWithFormatColumn"], + } + # RMLStreamer emits nothing for an absent value, so an undeclared + # meta-information line contributes no triple rather than an empty one. + for field, predicate in ( + ("fileFormat", f"{VCFR}fileFormat"), + ("referenceGenome", f"{VCFR}referenceGenome"), + ("sourceSoftware", f"{VCFR}sourceSoftware"), + ): + predicates[predicate] = 0 if parser[field] == METADATA_ABSENT else 1 + + if samples: + predicates[f"{VCFR}representationProfile"] = 1 + if representation == "expanded": + classes[f"{VCFR}SampleCall"] = records * samples + classes[f"{VCFR}FormatFieldValue"] = parser["formatValueSlots"] + predicates[f"{VCFR}hasSampleCall"] = records * samples + predicates[f"{VCFR}sampleId"] = records * samples + predicates[f"{VCFR}hasFormatValue"] = parser["formatValueSlots"] + predicates[f"{VCFR}fieldValue"] = parser["nonEmptyFormatValues"] + else: + definitions = parser["distinctFormatKeyCount"] + classes[f"{VCFR}SampleSet"] = 1 + classes[f"{VCFR}VCFSample"] = samples + classes[f"{VCFR}CohortCallMatrix"] = parser["recordsWithFormatKeys"] + classes[f"{VCFR}FormatValueVector"] = parser["formatKeyOccurrences"] + classes[f"{VCFR}FormatFieldDefinition"] = definitions + predicates[f"{VCFR}hasSampleSet"] = 1 + predicates[f"{VCFR}hasSample"] = samples + predicates[f"{VCFR}sampleName"] = samples + predicates[f"{VCFR}sampleIndex"] = samples + predicates[f"{VCFR}hasCallMatrix"] = parser["recordsWithFormatKeys"] + predicates[f"{VCFR}appliesToSampleSet"] = parser["recordsWithFormatKeys"] + predicates[f"{VCFR}hasFormatValueVector"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}declaredBy"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}valueEncoding"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}encodedValues"] = parser["formatKeyOccurrences"] + predicates[f"{VCFR}fieldId"] = definitions + predicates[f"{VCFR}fieldNumber"] = definitions + predicates[f"{VCFR}fieldDescription"] = definitions + + predicates[f"{VCFR}fileDate"] = 0 if parser["fileDate"] == METADATA_ABSENT else 1 + + if header_representation == "structured": + for class_name, count in parser["headerLineClassCounts"].items(): + classes[f"{VCFR}{class_name}"] = classes.get(f"{VCFR}{class_name}", 0) + count + # A FILTER/ALT line carries both its header-line subclass and its + # definition class, so it contributes two rdf:type triples. + classes[f"{VCFR}FilterDefinition"] = parser["filterDefinitionCount"] + classes[f"{VCFR}AltDefinition"] = parser["altDefinitionCount"] + predicates[f"{VCFR}filterId"] = parser["filterDefinitionCount"] + predicates[f"{VCFR}altId"] = parser["altDefinitionCount"] + predicates[f"{VCFR}contigId"] = parser["contigCount"] + predicates[f"{VCFR}contigCount"] = 1 if parser["contigCount"] else 0 + for attribute, predicate in ( + ("length", "contigLength"), ("md5", "contigMd5"), ("assembly", "contigAssembly"), + ): + predicates[f"{VCFR}{predicate}"] = parser["contigAttributeCounts"].get(attribute, 0) + predicates[f"{VCFR}fieldDescription"] = ( + predicates.get(f"{VCFR}fieldDescription", 0) + + parser["describedDefinitionCount"] + ) + + if info_representation == "structured": + values = parser["infoValueCount"] + definitions = parser["infoDefinitionCount"] + classes[f"{VCFR}InfoFieldValue"] = values + classes[f"{VCFR}InfoFieldDefinition"] = definitions + predicates[f"{VCFR}hasInfoValue"] = values + predicates[f"{VCFR}fieldValueBoolean"] = parser["infoFlagCount"] + predicates[f"{VCFR}fieldValueInteger"] = parser["infoTypedIntegerCount"] + predicates[f"{VCFR}fieldValueDecimal"] = parser["infoTypedDecimalCount"] + predicates[f"{VCFR}fieldType"] = definitions + # declaredBy and the field* descriptors are shared with the condensed + # FORMAT definitions, so accumulate rather than overwrite. + for predicate, count in ( + (f"{VCFR}declaredBy", values), + (f"{VCFR}fieldId", definitions), + (f"{VCFR}fieldNumber", definitions), + (f"{VCFR}fieldDescription", definitions), + ): + predicates[predicate] = predicates.get(predicate, 0) + count + predicates[f"{VCFR}fieldValue"] = ( + predicates.get(f"{VCFR}fieldValue", 0) + values - parser["infoFlagCount"] + ) + + classes = _nonzero(classes) + predicates = _nonzero(predicates) + # Every typed resource contributes exactly one rdf:type triple here. + predicates[RDF_TYPE] = sum(classes.values()) + + return { + "q09_predicate_census": [ + {"predicate": iri, "tripleCount": count} + for iri, count in sorted(predicates.items()) + ], + "q10_class_census": [ + {"class": iri, "resourceCount": count} for iri, count in sorted(classes.items()) + ], + } + + +def sample_uri_ids(sample_ids: list[str]) -> list[str]: + """Derive each sample's IRI component the way the emitters do. + + Mirrors ``_sample_id_to_uri_id`` in ``vcf_rdfizer.py``, including the + occurrence suffix that keeps two samples whose ids sanitize to the same + string distinguishable. + """ + counts: dict[str, int] = {} + out: list[str] = [] + for index, sample_id in enumerate(sample_ids, start=1): + base = re.sub(r"[^A-Za-z0-9._~-]+", "_", sample_id).strip("_") or f"sample_{index}" + counts[base] = counts.get(base, 0) + 1 + out.append(f"{base}_{counts[base]}" if counts[base] > 1 else base) + return out + + +#: '##' key (lower-cased) -> vocabulary subclass. Mirrors HEADER_LINE_CLASSES +#: in vcf_rdfizer.py; the unit tests assert the two stay identical. +HEADER_LINE_CLASSES = { + "fileformat": "FileFormatHeaderLine", + "filedate": "FileDateHeaderLine", + "source": "SourceHeaderLine", + "reference": "ReferenceHeaderLine", + "info": "INFOHeaderLine", + "format": "FORMATHeaderLine", + "filter": "FILTERHeaderLine", + "alt": "ALTHeaderLine", + "contig": "ContigHeaderLine", +} + + +def parse_structured_header_fields(value: str) -> dict[str, str]: + """Parse '' attributes, respecting quoting. + + Mirrors ``_parse_structured_header_fields`` in ``vcf_rdfizer.py``; the unit + tests assert the two agree. + """ + inner = value.strip() + if inner.startswith("<") and inner.endswith(">"): + inner = inner[1:-1] + tokens: list[str] = [] + token: list[str] = [] + in_quotes = escaped = False + for character in inner: + if escaped: + token.append(character) + escaped = False + elif character == "\\" and in_quotes: + token.append(character) + escaped = True + elif character == '"': + token.append(character) + in_quotes = not in_quotes + elif character == "," and not in_quotes: + tokens.append("".join(token)) + token = [] + else: + token.append(character) + tokens.append("".join(token)) + fields: dict[str, str] = {} + for item in tokens: + if "=" not in item: + continue + key, raw_value = item.split("=", 1) + parsed = raw_value.strip() + if len(parsed) >= 2 and parsed[0] == parsed[-1] == '"': + parsed = parsed[1:-1].replace('\\"', '"').replace("\\\\", "\\") + fields[key.strip()] = parsed + return fields + + +def parse_info_entries(info: str) -> list[tuple[str, str | None]]: + """Split an INFO column into ``(key, value)`` pairs; a bare key is a Flag. + + Mirrors ``parse_info_entries`` in ``vcf_rdfizer.py``. The two are asserted + to agree in ``test_validation_logic_unit.py``. + """ + if info in ("", "."): + return [] + entries: list[tuple[str, str | None]] = [] + for item in info.split(";"): + if not item: + continue + key, separator, value = item.partition("=") + entries.append((key, value if separator else None)) + return entries + + +def info_declared_types(raw_header: str) -> dict[str, str]: + """Map each declared INFO key to its VCF Type, for typed-value counting.""" + types: dict[str, str] = {} + for line in raw_header.splitlines(): + if not line.startswith("##INFO="): + continue + body = line[len("##INFO=") :].strip() + if body.startswith("<") and body.endswith(">"): + body = body[1:-1] + fields: dict[str, str] = {} + for match in re.finditer(r'(\w+)=("(?:[^"\\]|\\.)*"|[^,]*)', body): + value = match.group(2) + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + fields[match.group(1)] = value + key = fields.get("ID", "").strip() + if key: + types.setdefault(key, fields.get("Type") or "String") + return types + + +def read_vcf_header_text(vcf_path: Path) -> str: + """Read the header block straight from the VCF file. + + Returns the leading ``#`` lines verbatim - the ``##`` meta-information + lines and the ``#CHROM`` column line - which is the same span cyvcf2's + ``raw_header`` covers, so callers of either see the same shape. + + Deliberately not cyvcf2's ``raw_header``: htslib normalises the header it + exposes, injecting declarations the file does not contain - notably + ``##FILTER=``. The conversion + reads the file's own text, so the oracle must too, or it expects header + resources the graph could never contain. + """ + opener = gzip.open if vcf_path.name.endswith(".gz") else open + lines: list[str] = [] + with opener(vcf_path, "rt", encoding="utf-8", errors="replace") as handle: + for line in handle: + if not line.startswith("#"): + break + lines.append(line.rstrip("\n")) + return "\n".join(lines) + + +def parse_header_metadata(raw_header: str) -> dict[str, Any]: + """Summarise the '##' meta-information block of a VCF. + + The `#CHROM` line is deliberately excluded: it is not a meta-information + line and the mapping emits no HeaderLine resource for it, so counting it + here would guarantee a spurious mismatch. + """ + metadata: dict[str, Any] = {value: METADATA_ABSENT for value in FILE_METADATA_KEYS.values()} + keys: Counter[str] = Counter() + for line in raw_header.splitlines(): + if not line.startswith("##"): + continue + body = line[2:] + key, _, value = body.partition("=") + keys[key] += 1 + field = FILE_METADATA_KEYS.get(key.lower()) + # A repeated declaration keeps the first, matching the awk parser. + if field is not None and metadata[field] == METADATA_ABSENT: + metadata[field] = value + # Header-representation counters. The class map mirrors HEADER_LINE_CLASSES + # in vcf_rdfizer.py; the two are asserted to agree in the unit tests. + header_classes: Counter[str] = Counter() + filter_definitions = alt_definitions = contigs = described = 0 + contig_attributes: Counter[str] = Counter() + for line in raw_header.splitlines(): + if not line.startswith("##"): + continue + key, _, value = line[2:].partition("=") + class_name = HEADER_LINE_CLASSES.get(key.lower()) + if class_name is None: + continue + header_classes[class_name] += 1 + if key.lower() not in {"filter", "alt", "contig"}: + continue + fields = parse_structured_header_fields(value) + if not (fields.get("ID") or "").strip(): + continue + if key.lower() == "contig": + contigs += 1 + for attribute in ("length", "md5", "assembly"): + if fields.get(attribute): + contig_attributes[attribute] += 1 + continue + if key.lower() == "filter": + filter_definitions += 1 + else: + alt_definitions += 1 + if fields.get("Description"): + described += 1 + metadata["headerLineClassCounts"] = dict(header_classes) + metadata["filterDefinitionCount"] = filter_definitions + metadata["altDefinitionCount"] = alt_definitions + metadata["contigCount"] = contigs + metadata["contigAttributeCounts"] = dict(contig_attributes) + metadata["describedDefinitionCount"] = described + metadata["headerLineCount"] = sum(keys.values()) + metadata["headerValueCount"] = sum( + 1 for line in raw_header.splitlines() + if line.startswith("##") and line[2:].partition("=")[2] != "" + ) + metadata["q08_header_line_census"] = [ + {"headerKey": key, "lineCount": int(count)} for key, count in sorted(keys.items()) + ] + return metadata + + +def attach_census_expectations( + parser: dict[str, Any], representation: str, *, info_representation: str = "structured", + header_representation: str = "structured", +) -> dict[str, Any]: + """Add the expected predicate/class inventory to a parser summary. + + Kept separate from ``parse_vcf`` because the expectation depends on the + representation, which is a validation-run choice rather than a property of + the VCF. + """ + parser.update(expected_census( + parser, representation, info_representation=info_representation, + header_representation=header_representation, + )) + key = ( + "_expandedFormatValueDigest" if representation == "expanded" + else "_condensedFormatValueDigest" + ) + parser["q13_format_value_digest"] = parser.get(key, []) + return parser + + +def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: + if VCF is None: + raise RuntimeError( + "cyvcf2 is unavailable; the validator must run inside the " + "VCF-RDFizer container image" + ) + use_bcftools = filter_oracle == "bcftools" or ( + filter_oracle == "auto" and shutil.which("bcftools") is not None + ) + if filter_oracle == "bcftools" and not shutil.which("bcftools"): + raise RuntimeError("--filter-oracle=bcftools requested but bcftools is unavailable") + filters = filters_with_bcftools(vcf_path) if use_bcftools else Counter() + reader = VCF(str(vcf_path), strict_gt=True) + samples = list(reader.samples) + header_metadata = parse_header_metadata(read_vcf_header_text(vcf_path)) + density: Counter[tuple[str, int]] = Counter() + shapes: Counter[str] = Counter() + genotypes: Counter[tuple[str, str]] = Counter() + ac_an: Counter[tuple[int, int]] = Counter() + total_records = gt_records = single_alt_records = q06_eligible = 0 + transition_count = transversion_count = biallelic_snv_count = 0 + # Census inputs. These mirror SampleRecordStream._parse_row, which widens + # FORMAT to the longest sample payload when a record drops trailing fields. + digest_buckets: Counter[str] = Counter() + info_value_digest: Counter[str] = Counter() + expanded_format_digest: Counter[str] = Counter() + condensed_format_digest: Counter[str] = Counter() + sample_components = [rml_uri_component(uri_id) for uri_id in sample_uri_ids(samples)] + declared_info_types = info_declared_types(read_vcf_header_text(vcf_path)) + info_definitions: set[str] = set() + info_values = info_flags = info_typed_integers = info_typed_decimals = 0 + source_component = rml_uri_component(vcf_path.name) + records_with_format_column = records_with_format_keys = 0 + format_key_occurrences = format_value_slots = non_empty_format_values = 0 + distinct_format_keys: set[str] = set() + try: + for variant in reader: + total_records += 1 + density[(str(variant.CHROM), (int(variant.POS) - 1) // 1_000_000)] += 1 + alt = alt_lexical(variant) + shapes[classify_variant_shape(variant.REF, alt)] += 1 + ref_upper, alt_upper = str(variant.REF).upper(), alt.upper() + if re.fullmatch(r"[ACGT]", ref_upper) and re.fullmatch(r"[ACGT]", alt_upper) and ref_upper != alt_upper: + biallelic_snv_count += 1 + if (ref_upper, alt_upper) in TRANSITIONS: + transition_count += 1 + else: + transversion_count += 1 + if not use_bcftools: + raw_filter = exact_filter_lexical(variant) + filters[(filter_status(raw_filter), raw_filter)] += 1 + raw_format = variant.FORMAT + if not raw_format: + format_keys: list[str] = [] + elif isinstance(raw_format, str): + format_keys = raw_format.split(":") + else: + format_keys = [str(item) for item in raw_format] + has_gt = "GT" in format_keys + if has_gt: + gt_records += 1 + + # `vcfr:formatRaw` comes from the FORMAT column itself, so it is + # present whenever that column is non-empty - samples or not. + if format_keys: + records_with_format_column += 1 + + columns = str(variant).rstrip("\r\n").split("\t") + # The record digest is computed from the raw line so it matches the + # lexical values the mapping puts in the graph, character for + # character, with no round-trip through cyvcf2's typed accessors. + record_iri = ( + f"file://{source_component}#record/" + f"{rml_uri_component(str(total_records))}" + ) + digest_buckets[record_digest_bucket([record_iri, *columns[:8]])] += 1 + + # Structured INFO counts. The typed-value rules mirror + # `_typed_info_object` in vcf_rdfizer.py: only a single-valued + # Integer/Float that actually parses gains a typed predicate. + row_component = rml_uri_component(str(total_records)) + for key, value in parse_info_entries(columns[7] if len(columns) > 7 else ""): + info_values += 1 + info_definitions.add(key) + if value is None: + info_flags += 1 + continue + info_iri = ( + f"file://{source_component}#call/{row_component}" + f"/info/{rml_uri_component(key)}" + ) + info_value_digest[record_digest_bucket([info_iri, value])] += 1 + declared = declared_info_types.get(key, "String") + if "," in value or value == ".": + continue + try: + if declared == "Integer": + int(value) + info_typed_integers += 1 + elif declared == "Float": + float(value) + info_typed_decimals += 1 + except ValueError: + pass + payload_fields = [ + payload.split(":") if payload else [] for payload in columns[9:] + ] + # SampleRecordStream widens FORMAT to the longest sample payload and + # names any surplus field FIELD_, so the emitted key set can be + # wider than the declared one. Mirror that exactly. + width = max([len(format_keys), *(len(fields) for fields in payload_fields)], default=0) + widened_keys = [ + format_keys[index] if index < len(format_keys) and format_keys[index] + else f"FIELD_{index + 1}" + for index in range(width) + ] + if samples and widened_keys: + records_with_format_keys += 1 + distinct_format_keys.update(widened_keys) + format_key_occurrences += width + format_value_slots += width * len(samples) + non_empty_format_values += sum( + 1 + for fields in payload_fields + for index in range(width) + if index < len(fields) and fields[index] != "" + ) + # Both representations are accumulated: only one is compared, + # chosen by the run's representation, but computing both keeps + # this loop single-pass. + call_iri = f"file://{source_component}#call/{row_component}" + for key_index, key in enumerate(widened_keys): + key_component = rml_uri_component(key) + for sample_index, fields in enumerate(payload_fields): + cell = fields[key_index] if key_index < len(fields) else "" + if not cell: + continue + value_iri = ( + f"file://{source_component}#sample/{row_component}" + f"/{sample_components[sample_index]}/fmt/{key_component}" + ) + expanded_format_digest[ + record_digest_bucket([value_iri, cell]) + ] += 1 + encoded = "\t".join( + (fields[key_index] if key_index < len(fields) and fields[key_index] + else ".") + for fields in payload_fields + ) + vector_iri = f"{call_iri}/matrix/fmt/{key_component}" + condensed_format_digest[ + record_digest_bucket([vector_iri, encoded]) + ] += 1 + alleles = [None] * len(samples) + if samples and has_gt: + raw_genotypes = list(variant.genotypes) + if len(raw_genotypes) != len(samples): + raise ValueError(f"Sample/genotype length mismatch at {variant.CHROM}:{variant.POS}") + alleles = [genotype_alleles(raw) for raw in raw_genotypes] + for sample, call in zip(samples, alleles, strict=True): + genotypes[(sample, classify_genotype(call, has_gt=has_gt))] += 1 + if alt != "." and "," not in alt: + single_alt_records += 1 + if has_gt: + an = ac = 0 + for call in alleles: + if call is None or any(value is None for value in call): + continue + complete = tuple(int(value) for value in call if value is not None) + if len(complete) not in (1, 2) or any(value not in (0, 1) for value in complete): + continue + an += len(complete) + ac += sum(complete) + if an: + ac_an[(an, ac)] += 1 + q06_eligible += 1 + finally: + reader.close() + if sum(filters.values()) != total_records: + raise ValueError("FILTER oracle record count differs from the VCF record count") + q05 = [ + {"sampleId": sample, "genotypeClass": genotype_class, "callCount": int(count)} + for (sample, genotype_class), count in sorted(genotypes.items()) + ] + return { + "source": str(vcf_path), + "sourceSha256": sha256_file(vcf_path), + "filterOracle": "bcftools" if use_bcftools else "cyvcf2-serialization", + "headerLineCount": header_metadata["headerLineCount"], + "fileDate": header_metadata["fileDate"], + "headerLineClassCounts": header_metadata["headerLineClassCounts"], + "filterDefinitionCount": header_metadata["filterDefinitionCount"], + "altDefinitionCount": header_metadata["altDefinitionCount"], + "contigCount": header_metadata["contigCount"], + "contigAttributeCounts": header_metadata["contigAttributeCounts"], + "describedDefinitionCount": header_metadata["describedDefinitionCount"], + "headerValueCount": header_metadata["headerValueCount"], + "recordsWithFormatColumn": records_with_format_column, + "recordsWithFormatKeys": records_with_format_keys, + "formatKeyOccurrences": format_key_occurrences, + "formatValueSlots": format_value_slots, + "nonEmptyFormatValues": non_empty_format_values, + "distinctFormatKeyCount": len(distinct_format_keys), + "q12_info_value_digest": [ + {"bucket": bucket, "valueCount": int(count)} + for bucket, count in sorted(info_value_digest.items()) + ], + "_expandedFormatValueDigest": [ + {"bucket": bucket, "valueCount": int(count)} + for bucket, count in sorted(expanded_format_digest.items()) + ], + "_condensedFormatValueDigest": [ + {"bucket": bucket, "valueCount": int(count)} + for bucket, count in sorted(condensed_format_digest.items()) + ], + "infoValueCount": info_values, + "infoDefinitionCount": len(info_definitions), + "infoFlagCount": info_flags, + "infoTypedIntegerCount": info_typed_integers, + "infoTypedDecimalCount": info_typed_decimals, + "q11_record_digest": [ + {"bucket": bucket, "recordCount": int(count)} + for bucket, count in sorted(digest_buckets.items()) + ], + "fileFormat": header_metadata["fileFormat"], + "referenceGenome": header_metadata["referenceGenome"], + "sourceSoftware": header_metadata["sourceSoftware"], + "q07_file_metadata": { + "fileFormat": header_metadata["fileFormat"], + "referenceGenome": header_metadata["referenceGenome"], + "sourceSoftware": header_metadata["sourceSoftware"], + }, + "q08_header_line_census": header_metadata["q08_header_line_census"], + "sampleCount": len(samples), + "samples": samples, + "totalRecords": total_records, + "gtRecordCount": gt_records, + "singleAltRecordCount": single_alt_records, + "q06EligibleSiteCount": q06_eligible, + "q01_record_density_1mb": [ + {"chrom": chrom, "windowIndex": window, "recordCount": int(count)} + for (chrom, window), count in sorted(density.items()) + ], + "q02_variant_shape_counts": [ + {"variantClass": kind, "recordCount": int(count)} for kind, count in sorted(shapes.items()) + ], + "q03_titv": { + "biallelicSnvCount": biallelic_snv_count, + "transitionCount": transition_count, + "transversionCount": transversion_count, + "tiTvRatio": transition_count / transversion_count if transversion_count else None, + }, + "q04_filter_distribution": [ + {"filterStatus": status, "filterLexical": lexical, "recordCount": int(count)} + for (status, lexical), count in sorted(filters.items()) + ], + "q05_sample_genotype_counts": q05, + "q06_ac_an_distribution": [ + {"an": an, "ac": ac, "siteCount": int(count), "af": ac / an} + for (an, ac), count in sorted(ac_an.items()) + ], + } + + +# --------------------------------------------------------------------------- +# Source artifact materialization +# --------------------------------------------------------------------------- +# Semantic validation always runs against N-Triples. A compressed or indexed +# artifact is therefore decoded into the container-local scratch directory +# first, which is what makes "validate the HDT" mean "prove the HDT decodes to +# a graph that still satisfies every check", rather than only that it decodes +# to the right number of triples (which validate_compression.py already does). +# +# Decoding needs scratch space of roughly the uncompressed graph size. The +# scratch directory is removed before the process exits in all cases. + +RDF_FORMATS = ("nt", "nt.gz", "nt.br", "hdt", "cottas", "cottas.gz", "cottas.br") +#: Formats that are read in place, with nothing written to scratch. +DIRECT_FORMATS = frozenset({"nt"}) +FORMAT_SUFFIXES = ( + (".nt.gz", "nt.gz"), + (".nt.br", "nt.br"), + (".nt", "nt"), + (".cottas.gz", "cottas.gz"), + (".cottas.br", "cottas.br"), + (".cottas", "cottas"), + (".hdt", "hdt"), +) +COTTAS_TOOL = Path("/opt/vcf-rdfizer/cottas_tool.py") + + +def detect_rdf_format(path: Path) -> str | None: + """Infer the artifact format from a filename, or None when unrecognised.""" + name = path.name + for suffix, fmt in FORMAT_SUFFIXES: + if name.endswith(suffix): + return fmt + return None + + +def _resolve_binary(env_var: str, *candidates: str) -> str: + """Find a container binary the same way the compression stages do.""" + override = os.environ.get(env_var, "").strip() + if override and (Path(override).is_file() or shutil.which(override)): + return override + for candidate in candidates: + found = shutil.which(candidate) or (candidate if Path(candidate).is_file() else None) + if found: + return found + raise RuntimeError(f"Required binary not found in container: {candidates[0]}") + + +def _run_step( + command: list[str], *, label: str, log_dir: Path, env: dict[str, str] | None = None +) -> None: + """Run one decode step, preserving its output for diagnosis on failure.""" + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / f"{label}.log" + with log_path.open("wb") as log: + result = subprocess.run( + command, check=False, stdout=log, stderr=subprocess.STDOUT, env=env + ) + if result.returncode != 0: + tail = "" + try: + tail = log_path.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + raise RuntimeError( + f"{label} failed with exit code {result.returncode}. Output tail: {tail}" + ) + + +def materialize_ntriples( + source: Path, + rdf_format: str, + scratch: Path, + *, + log_dir: Path, +) -> tuple[Path, dict[str, Any]]: + """Return ``(ntriples_path, provenance)`` for any supported artifact. + + A plain ``.nt`` source is used in place and never copied. + """ + provenance: dict[str, Any] = { + "sourceFormat": rdf_format, + "materialized": rdf_format not in DIRECT_FORMATS, + "steps": [], + } + if rdf_format == "nt": + return source, provenance + + target = scratch / "input.nt" + if rdf_format == "nt.gz": + with gzip.open(source, "rb") as handle, target.open("wb") as out: + shutil.copyfileobj(handle, out, length=1024 * 1024) + provenance["steps"].append({"tool": "python-gzip", "output": str(target)}) + return target, provenance + + if rdf_format == "nt.br": + brotli = _resolve_binary("BROTLI_BIN", "brotli") + _run_step([brotli, "-d", "-c", "-o", str(target), str(source)], + label="brotli-decompress", log_dir=log_dir) + provenance["steps"].append({"tool": brotli, "output": str(target)}) + return target, provenance + + if rdf_format == "hdt": + hdt2rdf = _resolve_binary("HDT2RDF_BIN", "hdt2rdf", "/usr/local/bin/hdt2rdf") + _run_step([hdt2rdf, str(source), str(target)], label="hdt2rdf", log_dir=log_dir) + provenance["steps"].append({"tool": hdt2rdf, "output": str(target)}) + return target, provenance + + if rdf_format in {"cottas", "cottas.gz", "cottas.br"}: + cottas_input = source + if rdf_format != "cottas": + # pycottas needs a seekable Parquet file, so a packaged artifact is + # unwrapped into scratch before it is decoded. + cottas_input = scratch / "input.cottas" + if rdf_format == "cottas.gz": + with gzip.open(source, "rb") as handle, cottas_input.open("wb") as out: + shutil.copyfileobj(handle, out, length=1024 * 1024) + provenance["steps"].append({"tool": "python-gzip", "output": str(cottas_input)}) + else: + brotli = _resolve_binary("BROTLI_BIN", "brotli") + _run_step([brotli, "-d", "-c", "-o", str(cottas_input), str(source)], + label="brotli-unwrap", log_dir=log_dir) + provenance["steps"].append({"tool": brotli, "output": str(cottas_input)}) + python_bin = os.environ.get("COTTAS_PYTHON_BIN") or sys.executable + _run_step( + [python_bin, str(COTTAS_TOOL), "decompress", str(cottas_input), str(target)], + label="cottas-decompress", + log_dir=log_dir, + ) + provenance["steps"].append({"tool": f"{python_bin} cottas_tool.py decompress", + "output": str(target)}) + if cottas_input != source: + cottas_input.unlink(missing_ok=True) + return target, provenance + + raise ValueError(f"Unsupported RDF artifact format: {rdf_format}") + + +#: Violations retained in the report. A badly broken graph can produce one per +#: record, so the full text is never inlined into a JSON report. +SHACL_SAMPLE_LIMIT = 50 + + +def validate_shacl(source: Path, shapes: Path, results_dir: Path) -> dict[str, Any]: + """Validate the graph against SHACL shapes, if pyshacl is available. + + This is an independent structural layer: it checks the shapes the + vocabulary publishes, rather than comparing counts against the VCF, so it + catches a different class of defect from everything else here. + + pyshacl loads the graph into memory, so this is opt-in and unsuitable for a + cohort-scale aggregate. It is reported as EXECUTION_FAILED rather than a + conformance failure when the tool is missing, so an absent optional + dependency can never look like a bad graph. + """ + report_path = results_dir / "shacl.json" + try: + from pyshacl import validate as pyshacl_validate + except ImportError as error: + result = { + "status": "EXECUTION_FAILED", + "error": f"pyshacl is not installed in this image: {error}", + "shapes": str(shapes), + } + write_json(report_path, result) + return result + + started = time.monotonic() + try: + conforms, _graph, text = pyshacl_validate( + str(source), + shacl_graph=str(shapes), + data_graph_format="nt", + shacl_graph_format="turtle", + inference="none", + advanced=False, + ) + except Exception as error: # noqa: BLE001 - reported, never fatal here + result = { + "status": "EXECUTION_FAILED", + "error": f"SHACL validation could not run: {error}", + "shapes": str(shapes), + } + write_json(report_path, result) + return result + + violations = [ + line.strip() for line in text.splitlines() + if line.strip().startswith("Constraint Violation") + ] + paths = sorted({ + line.split("Result Path:", 1)[1].strip() + for line in text.splitlines() if "Result Path:" in line + }) + log_path = results_dir / "shacl-report.txt" + log_path.write_text(text, encoding="utf-8") + result = { + "status": "PASS" if conforms else "FAIL", + "conforms": bool(conforms), + "shapes": str(shapes), + "violationCount": len(violations), + "violationPaths": paths, + "report": str(log_path), + "wallSeconds": time.monotonic() - started, + "sampleLimitedTo": SHACL_SAMPLE_LIMIT, + "sample": violations[:SHACL_SAMPLE_LIMIT], + } + write_json(report_path, result) + return result + + +def validate_ntriples(source: Path, results_dir: Path) -> dict[str, Any]: + rapper = shutil.which("rapper") + if not rapper: + return {"status": "EXECUTION_FAILED", "error": "rapper is not installed"} + result = subprocess.run( + [rapper, "-i", "ntriples", "-c", str(source)], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + log = results_dir / "rdf-validation" / "rapper.txt" + log.parent.mkdir(parents=True, exist_ok=True) + output = result.stdout + result.stderr + log.write_text(output, encoding="utf-8") + # Rapper reports the parsed triple count; keeping it makes a decoded + # HDT/COTTAS artifact directly comparable against its source graph. + match = re.search(r"Parsing returned (\d+) triples", output) + return { + "status": "PASS" if result.returncode == 0 else "FAIL", + "log": str(log), + "exitCode": result.returncode, + "tripleCount": int(match.group(1)) if match else None, + } + + +# --------------------------------------------------------------------------- +# SPARQL engines +# --------------------------------------------------------------------------- +# Every engine answers the same queries and writes SPARQL Results JSON to the +# same place, so the comparison layer is engine-agnostic and a run's engine +# choice is only ever a performance/scale decision, never a semantic one. +# +# comunica: zero setup, loads the whole graph into the Node heap. Fine for the +# scale most single-sample runs reach, and the default. +# qlever: builds an on-disk index, then answers over HTTP. Slower to start, +# but the only option once a graph stops fitting in memory. + +#: Every backend that can answer the validation queries. `comunica` and +#: `qlever` read the materialized N-Triples; `hdt` and `cottas` query the +#: compressed representation *directly*, without decoding it first, which is +#: the property those formats exist for. Several may be requested in one run: +#: they all answer the same queries, so the comparison is meaningful and the +#: recorded timings are directly comparable. +SPARQL_ENGINES = ("comunica", "qlever", "hdt", "cottas") +#: Engines that query a compressed artifact natively, and the artifact format +#: each one needs. When the run's source is a different format, the artifact is +#: built in scratch first and the build is timed separately from the queries, +#: so an index build is never mistaken for query cost. +NATIVE_ENGINE_FORMATS = {"hdt": "hdt", "cottas": "cottas"} +# QLever's binaries are copied out of the upstream image, which is built on a +# different Ubuntu release, so their Boost/ICU/jemalloc sonames come with them +# in a private directory. Pointing only QLever's own processes at it keeps +# those libraries from shadowing anything the rest of the image uses. +QLEVER_LIB_DIR = "/opt/qlever/lib" +QLEVER_BIN_DIR = "/opt/qlever/bin" +DEFAULT_QLEVER_PORT = 7019 +DEFAULT_QLEVER_MEMORY_GB = 4 +DEFAULT_QLEVER_STARTUP_TIMEOUT = 900 +DEFAULT_QUERY_TIMEOUT = 3600 + + +QLEVER_STATUS_FILE = Path("/opt/vcf-rdfizer/qlever-status.txt") + + +def qlever_build_status() -> str: + """Read the image's build-time note about whether QLever links here.""" + try: + status = QLEVER_STATUS_FILE.read_text(encoding="utf-8").strip() + except OSError: + return "QLever build status unknown (marker file absent)." + return "QLever binaries linked cleanly at image build time." if status == "ok" else status + + +def _expand_template(template: str, **fields: str) -> list[str]: + """Split a shell-style override and substitute ``{name}`` placeholders.""" + return [part.format(**fields) for part in shlex.split(template)] + + +class QueryEngine: + """Common lifecycle and result envelope for a SPARQL backend.""" + + name = "abstract" + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + self.source = source + self.raw_dir = raw_dir + self.scratch = scratch + self.options = options + self.query_timeout = int(options.get("query_timeout") or DEFAULT_QUERY_TIMEOUT) + #: Seconds spent preparing the backend (index build, format conversion, + #: server startup) before any query ran. Reported separately from query + #: time so a benchmark can attribute cost correctly. + self.setup_seconds: float | None = None + + def prepare(self) -> None: + """Time ``start`` so setup cost is recorded for every engine alike.""" + started = time.monotonic() + self.start() + self.setup_seconds = time.monotonic() - started + + def __enter__(self) -> "QueryEngine": + self.prepare() + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + self.stop() + return False + + def start(self) -> None: + """Prepare the engine. Raise RuntimeError when it cannot be used.""" + + def stop(self) -> None: + """Release anything ``start`` acquired. Must be safe to call twice.""" + + def describe(self) -> dict[str, Any]: + return {"engine": self.name, "setupSeconds": self.setup_seconds} + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raise NotImplementedError + + def _envelope( + self, + query_id: str, + query_path: Path, + *, + returncode: int, + started: float, + raw_path: Path, + stderr_path: Path, + time_path: Path | None = None, + error: str | None = None, + ) -> dict[str, Any]: + envelope = { + "status": "PASS" if returncode == 0 else "EXECUTION_FAILED", + "engine": self.name, + "exitCode": returncode, + "wallSeconds": time.monotonic() - started, + "query": str(query_path), + "rawResult": str(raw_path), + "stderr": str(stderr_path), + "resourceMetrics": str(time_path) if time_path and time_path.exists() else None, + } + if error is not None: + envelope["error"] = error + return envelope + + +class ComunicaEngine(QueryEngine): + """Query the N-Triples file directly with comunica-sparql-file.""" + + name = "comunica" + + def start(self) -> None: + self.executable = shutil.which("comunica-sparql-file") + if not self.executable: + raise RuntimeError( + "comunica-sparql-file is not installed in this image; rebuild it or " + "select --engine qlever" + ) + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "setupSeconds": self.setup_seconds, + "version": tool_version( + ["comunica-sparql-file", "--version"], table_label="Comunica Engine" + ), + "mode": "in-memory file query", + } + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + time_path = self.raw_dir / f"{query_id}.time.txt" + command = [ + self.executable, + str(self.source), + "-f", + str(query_path), + "-t", + "application/sparql-results+json", + ] + timed = ( + ["/usr/bin/time", "-v", "-o", str(time_path), *command] + if tool_version(["/usr/bin/time", "--version"]) + else command + ) + started = time.monotonic() + try: + with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + result = subprocess.run( + timed, check=False, stdout=stdout, stderr=stderr, + timeout=self.query_timeout, + ) + returncode = result.returncode + error = None + except subprocess.TimeoutExpired: + returncode = 124 + error = f"query exceeded {self.query_timeout}s" + return self._envelope( + query_id, query_path, returncode=returncode, started=started, + raw_path=raw_path, stderr_path=stderr_path, time_path=time_path, error=error, + ) + + +class QleverEngine(QueryEngine): + """Build a QLever index over the graph, then answer queries over HTTP. + + QLever has no one-shot "query this file" mode, so the index build and the + short-lived server are both owned by this object and torn down in ``stop``. + The index lives in the container scratch directory and never reaches the + host. + """ + + name = "qlever" + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + super().__init__(source, raw_dir=raw_dir, scratch=scratch, options=options) + self.port = int(options.get("port") or DEFAULT_QLEVER_PORT) + self.memory_gb = int(options.get("memory_gb") or DEFAULT_QLEVER_MEMORY_GB) + self.startup_timeout = int( + options.get("startup_timeout") or DEFAULT_QLEVER_STARTUP_TIMEOUT + ) + self.index_dir = self.scratch / "qlever-index" + self.index_base = self.index_dir / "vcf-rdfizer" + self.log_dir = raw_dir / "engine" + self.server: subprocess.Popen | None = None + self.index_seconds: float | None = None + self.commands: dict[str, list[str]] = {} + self.server_binary: str | None = None + self.extra_index_args = list(options.get("extra_index_args") or []) + shlex.split( + os.environ.get("QLEVER_EXTRA_INDEX_ARGS", "") + ) + self.extra_server_args = list(options.get("extra_server_args") or []) + shlex.split( + os.environ.get("QLEVER_EXTRA_SERVER_ARGS", "") + ) + + def _binary(self, env_var: str, name: str) -> str: + try: + return _resolve_binary( + env_var, f"{QLEVER_BIN_DIR}/{name}", name, f"/usr/local/bin/{name}" + ) + except RuntimeError as error: + raise RuntimeError(f"{error}. {qlever_build_status()}") from error + + def _environment(self) -> dict[str, str]: + """Prepend QLever's private library directory for its processes only.""" + environment = dict(os.environ) + existing = environment.get("LD_LIBRARY_PATH", "") + environment["LD_LIBRARY_PATH"] = ( + f"{QLEVER_LIB_DIR}:{existing}" if existing else QLEVER_LIB_DIR + ) + return environment + + def index_command(self, index_builder: str) -> list[str]: + """QLever's CLI has changed across releases, so the default argv is a + starting point rather than a contract: ``--qlever-index-arg`` (or + ``QLEVER_EXTRA_INDEX_ARGS``) appends to it, and + ``QLEVER_INDEX_COMMAND`` replaces it entirely. ``{index}``, ``{input}`` + and ``{memory}`` are substituted in a replacement template. + """ + template = os.environ.get("QLEVER_INDEX_COMMAND", "").strip() + if template: + return _expand_template( + template, index=str(self.index_base), input=str(self.source), + memory=f"{self.memory_gb}G", + ) + return [ + index_builder, + "-i", str(self.index_base), + "-f", str(self.source), + "-F", "nt", + "-m", f"{self.memory_gb}G", + *self.extra_index_args, + ] + + def server_command(self, server_main: str) -> list[str]: + """See :meth:`index_command`; ``QLEVER_SERVER_COMMAND`` replaces this.""" + template = os.environ.get("QLEVER_SERVER_COMMAND", "").strip() + if template: + return _expand_template( + template, index=str(self.index_base), port=str(self.port), + memory=f"{self.memory_gb}G", + ) + return [ + server_main, + "-i", str(self.index_base), + "-p", str(self.port), + "-m", f"{self.memory_gb}G", + # qlever-server's own default query timeout is 30s, which the + # cohort-scale aggregate queries here routinely exceed. Keep the + # server-side limit aligned with the client-side one. + "-s", f"{self.query_timeout}s", + *self.extra_server_args, + ] + + def start(self) -> None: + index_builder = self._binary("QLEVER_INDEX_BUILDER_BIN", "qlever-index") + server_main = self._binary("QLEVER_SERVER_BIN", "qlever-server") + self.index_dir.mkdir(parents=True, exist_ok=True) + self.log_dir.mkdir(parents=True, exist_ok=True) + + self.server_binary = server_main + self.commands["index"] = self.index_command(index_builder) + self.commands["server"] = self.server_command(server_main) + + environment = self._environment() + started = time.monotonic() + _run_step( + self.commands["index"], label="qlever-index", + log_dir=self.log_dir, env=environment, + ) + self.index_seconds = time.monotonic() - started + + server_log = self.log_dir / "qlever-server.log" + self.server = subprocess.Popen( + self.commands["server"], + stdout=server_log.open("wb"), + stderr=subprocess.STDOUT, + env=environment, + ) + self._await_ready(server_log) + + def _await_ready(self, server_log: Path) -> None: + """Poll the server until it answers a trivial query, or give up.""" + deadline = time.monotonic() + self.startup_timeout + last_error = "server did not become ready" + while time.monotonic() < deadline: + if self.server is not None and self.server.poll() is not None: + tail = "" + try: + tail = server_log.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + raise RuntimeError( + f"QLever server exited with code {self.server.returncode}. Log tail: {tail}" + ) + try: + self._post("SELECT * WHERE { ?s ?p ?o } LIMIT 1", timeout=10) + return + except Exception as error: # noqa: BLE001 - readiness probe + last_error = str(error) + time.sleep(0.5) + raise RuntimeError( + f"QLever server was not ready within {self.startup_timeout}s: {last_error}" + ) + + def _post(self, query: str, *, timeout: int) -> bytes: + import urllib.error + import urllib.request + + request = urllib.request.Request( + f"http://127.0.0.1:{self.port}/", + data=urllib.parse.urlencode({"query": query}).encode("utf-8"), + headers={ + "Accept": "application/sparql-results+json", + "Content-Type": "application/x-www-form-urlencoded", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "setupSeconds": self.setup_seconds, + "version": tool_version([self.server_binary, "--help"]) if self.server_binary else None, + "mode": "on-disk index served over HTTP", + "indexDirectory": str(self.index_dir), + "indexBuildSeconds": self.index_seconds, + "memoryLimitGb": self.memory_gb, + "port": self.port, + # The exact argv is recorded because QLever's CLI is overridable and + # varies by release; a report should say what actually ran. + "commands": {name: list(argv) for name, argv in self.commands.items()}, + "buildStatus": qlever_build_status(), + } + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + started = time.monotonic() + try: + payload = self._post( + query_path.read_text(encoding="utf-8"), timeout=self.query_timeout + ) + except Exception as error: # noqa: BLE001 - reported as EXECUTION_FAILED + stderr_path.write_text(str(error), encoding="utf-8") + raw_path.write_bytes(b"") + return self._envelope( + query_id, query_path, returncode=1, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=str(error), + ) + raw_path.write_bytes(payload) + stderr_path.write_bytes(b"") + return self._envelope( + query_id, query_path, returncode=0, started=started, + raw_path=raw_path, stderr_path=stderr_path, + ) + + def stop(self) -> None: + if self.server is not None and self.server.poll() is None: + self.server.terminate() + try: + self.server.wait(timeout=30) + except subprocess.TimeoutExpired: + self.server.kill() + self.server.wait(timeout=30) + self.server = None + # The index is large and container-local; remove it as soon as the + # queries are done rather than waiting for the scratch teardown. + shutil.rmtree(self.index_dir, ignore_errors=True) + + +class NativeArtifactEngine(QueryEngine): + """Base for engines that query a compressed artifact without decoding it. + + The point of HDT and COTTAS is that they are queryable in place. Validating + them by decoding to N-Triples first proves the decode is faithful but says + nothing about querying them, and measures the wrong thing entirely for a + performance comparison. + + When the run's own artifact is already in this engine's format it is used + directly, which is the honest measurement. Otherwise one is built in scratch + from the materialized N-Triples, and that build is timed as setup rather + than as query cost. + """ + + #: Artifact format this engine reads, e.g. "hdt". + artifact_format = "" + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + super().__init__(source, raw_dir=raw_dir, scratch=scratch, options=options) + self.artifact: Path | None = None + self.artifact_origin = "unknown" + self.log_dir = raw_dir / "engine" + + def _resolve_artifact(self) -> Path: + """Use the run's own artifact when it matches, else build one.""" + supplied = self.options.get("artifact_path") + if supplied is not None and self.options.get("artifact_format") == self.artifact_format: + self.artifact_origin = "run artifact" + return Path(supplied) + self.artifact_origin = "built from N-Triples for this engine" + target = self.scratch / f"{self.name}-engine.{self.artifact_format}" + self.log_dir.mkdir(parents=True, exist_ok=True) + self.build_artifact(target) + return target + + def build_artifact(self, target: Path) -> None: + raise NotImplementedError + + def start(self) -> None: + self.artifact = self._resolve_artifact() + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "setupSeconds": self.setup_seconds, + "mode": f"native {self.artifact_format} query, no decode", + "artifact": str(self.artifact) if self.artifact else None, + "artifactOrigin": self.artifact_origin, + "artifactSizeBytes": ( + self.artifact.stat().st_size + if self.artifact and self.artifact.is_file() else None + ), + } + + +class HdtEngine(NativeArtifactEngine): + """Query a .hdt artifact in place with Comunica's HDT engine.""" + + name = "hdt" + artifact_format = "hdt" + + def build_artifact(self, target: Path) -> None: + rdf2hdt = _resolve_binary("RDF2HDT_BIN", "rdf2hdt", "/usr/local/bin/rdf2hdt") + _run_step( + [rdf2hdt, str(self.source), str(target)], + label="hdt-engine-build", log_dir=self.log_dir, + ) + + def start(self) -> None: + self.executable = shutil.which("comunica-sparql-hdt") + if not self.executable: + raise RuntimeError( + "comunica-sparql-hdt is not installed in this image, so HDT cannot " + "be queried natively. Rebuild the image, or validate the HDT " + "artifact by decoding it (--rdf file.hdt --engine comunica)" + ) + super().start() + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + command = [ + self.executable, + # Comunica needs the source type declared: a bare path is treated + # as a link to dereference and fails with "could not dereference". + f"hdt@{self.artifact}", + "-f", str(query_path), + "-t", "application/sparql-results+json", + ] + started = time.monotonic() + try: + with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + result = subprocess.run( + command, check=False, stdout=stdout, stderr=stderr, + timeout=self.query_timeout, + ) + returncode, error = result.returncode, None + except subprocess.TimeoutExpired: + returncode, error = 124, f"query exceeded {self.query_timeout}s" + return self._envelope( + query_id, query_path, returncode=returncode, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=error, + ) + + +class CottasEngine(NativeArtifactEngine): + """Query a .cottas artifact in place through pycottas's rdflib store. + + pycottas exposes ``COTTASStore``, an rdflib Store backed by the Parquet + artifact, so this runs in-process rather than shelling out. The validator + already runs under the interpreter that owns pycottas. + """ + + name = "cottas" + artifact_format = "cottas" + + def build_artifact(self, target: Path) -> None: + import pycottas + + pycottas.rdf2cottas(str(self.source), str(target)) + + def start(self) -> None: + try: + import pycottas # noqa: F401 + import rdflib # noqa: F401 + except ImportError as error: + raise RuntimeError( + f"pycottas and rdflib are required to query COTTAS natively: {error}" + ) from error + super().start() + + import pycottas + import rdflib + + self.graph = rdflib.Graph(store=pycottas.COTTASStore(str(self.artifact))) + + def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + stderr_path = self.raw_dir / f"{query_id}.stderr.txt" + started = time.monotonic() + try: + result = self.graph.query(query_path.read_text(encoding="utf-8")) + raw_path.write_bytes(result.serialize(format="json")) + stderr_path.write_bytes(b"") + returncode, error = 0, None + except Exception as failure: # noqa: BLE001 - reported as EXECUTION_FAILED + raw_path.write_text("{}", encoding="utf-8") + stderr_path.write_text(str(failure), encoding="utf-8") + returncode, error = 1, str(failure) + return self._envelope( + query_id, query_path, returncode=returncode, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=error, + ) + + def stop(self) -> None: + graph = getattr(self, "graph", None) + if graph is not None: + try: + graph.close() + except Exception: # noqa: BLE001 - teardown must not mask a result + pass + self.graph = None + + +ENGINE_CLASSES = { + "comunica": ComunicaEngine, + "qlever": QleverEngine, + "hdt": HdtEngine, + "cottas": CottasEngine, +} + + +def build_engine( + name: str, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any] +) -> QueryEngine: + try: + engine_class = ENGINE_CLASSES[name] + except KeyError as error: + raise ValueError( + f"Unknown SPARQL engine {name!r}; choose one of {', '.join(SPARQL_ENGINES)}" + ) from error + return engine_class(source, raw_dir=raw_dir, scratch=scratch, options=options) + + +def bindings(path: Path) -> list[dict[str, Any]]: + try: + value = read_json(path)["results"]["bindings"] + except (KeyError, TypeError) as error: + raise ValueError(f"Invalid SPARQL Results JSON: {path}") from error + if not isinstance(value, list): + raise ValueError(f"SPARQL bindings are not a list: {path}") + return value + + +def binding_int(binding: dict[str, Any], field: str) -> int: + return int(binding[field]["value"]) + + +def normalize(query_id: str, path: Path) -> Any: + fields, integer_fields, sort_fields = QUERY_SCHEMAS[query_id] + rows: list[dict[str, Any]] = [] + for number, binding in enumerate(bindings(path), start=1): + row: dict[str, Any] = {} + for field in fields: + try: + lexical = binding[field]["value"] + except (KeyError, TypeError) as error: + raise ValueError(f"Row {number} has no {field!r}") from error + if field in integer_fields: + try: + numeric = Decimal(lexical) + except InvalidOperation as error: + raise ValueError(f"{field} is not numeric: {lexical!r}") from error + if not numeric.is_finite() or numeric != numeric.to_integral_value(): + raise ValueError(f"{field} is not an integer: {lexical!r}") + row[field] = int(numeric) + else: + row[field] = str(lexical) + rows.append(row) + if sort_fields: + rows.sort(key=lambda row: tuple(row[field] for field in sort_fields)) + keys = [tuple(row[field] for field in sort_fields) for row in rows] + if len(keys) != len(set(keys)): + raise ValueError(f"{query_id} returned duplicate canonical keys") + if query_id in SINGLE_ROW_QUERIES: + if len(rows) != 1: + raise ValueError(f"{query_id} must return exactly one row, got {len(rows)}") + row = rows[0] + if query_id == "q03_titv": + row["tiTvRatio"] = ( + row["transitionCount"] / row["transversionCount"] + if row["transversionCount"] else None + ) + return row + if query_id == "q06_ac_an_distribution": + for row in rows: + row["af"] = row["ac"] / row["an"] + return rows + + +def anomaly_count(executions: dict[str, dict[str, Any]], query_id: str) -> Any: + """Exact anomaly total from the companion aggregate, or None if unavailable. + + Returns ``None`` rather than the sample size, so a report never presents a + truncated count as if it were exact. + """ + execution = executions.get(f"{query_id}_count") + if execution is None or execution.get("status") != "PASS": + return None + try: + rows = bindings(Path(execution["rawResult"])) + except (OSError, ValueError, json.JSONDecodeError): + return None + if len(rows) != 1: + return None + try: + return binding_int(rows[0], "anomalyCount") + except (KeyError, TypeError, ValueError): + return None + + +def duplicate_triple_report( + executions: dict[str, dict[str, Any]], parsed_triple_count: int | None +) -> dict[str, Any]: + """Compare statements parsed against distinct triples stored. + + A SPARQL store deduplicates on load, so no query can see a repeated line. + The parser counted every statement it read; the store counted the distinct + ones. Their difference is exactly the number of redundant statements. + + Reports NOT_EVALUATED rather than PASS when either number is unavailable, so + a missing input can never be mistaken for a clean result. + """ + execution = executions.get("preflight_distinct_triple_count") + if parsed_triple_count is None: + return { + "status": "NOT_EVALUATED", + "reason": "the parser did not report a statement count", + } + if execution is None or execution.get("status") != "PASS": + return { + "status": "NOT_EVALUATED", + "reason": "the distinct-triple query did not run", + } + try: + rows = bindings(Path(execution["rawResult"])) + distinct = binding_int(rows[0], "distinctTripleCount") + except (OSError, ValueError, KeyError, IndexError, TypeError, json.JSONDecodeError) as error: + return {"status": "NOT_EVALUATED", "reason": f"unreadable result: {error}"} + + duplicates = int(parsed_triple_count) - distinct + return { + "status": "PASS" if duplicates == 0 else "FAIL", + "parsedTripleCount": int(parsed_triple_count), + "distinctTripleCount": distinct, + "duplicateTripleCount": duplicates, + } + + +def preflight( + executions: dict[str, dict[str, Any]], + parser: dict[str, Any], + representation: str, + *, + parsed_triple_count: int | None = None, +) -> dict[str, Any]: + report: dict[str, Any] = {} + report["preflight_duplicate_triples"] = duplicate_triple_report( + executions, parsed_triple_count + ) + for query_id in PREFLIGHT_QUERIES: + execution = executions[query_id] + if execution["status"] != "PASS": + report[query_id] = {"status": "EXECUTION_FAILED", "execution": execution} + continue + returned = bindings(Path(execution["rawResult"])) + exact = anomaly_count(executions, query_id) + if query_id == "preflight_distinct_triple_count": + # Consumed by duplicate_triple_report above; nothing to judge alone. + report[query_id] = {"status": "PASS", "rows": len(returned)} + elif query_id in { + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_blank_nodes", + "preflight_empty_values", + }: + report[query_id] = { + "status": "PASS" if not returned else "FAIL", + "anomalyCount": exact, + "anomalyCountReturned": len(returned), + "limitedTo": ANOMALY_SAMPLE_LIMIT, + "sampleTruncated": len(returned) >= ANOMALY_SAMPLE_LIMIT, + } + elif query_id == "preflight_representation_profile": + # VCF-RDFizer intentionally omits a sample representation profile + # when a VCF has no sample columns, because no sample graph is + # emitted. That is not a representation mismatch. + profile_missing_is_expected = parser["sampleCount"] == 0 + report[query_id] = { + "status": "PASS" if not returned or profile_missing_is_expected else "FAIL", + "anomalyCount": exact, + "anomalyCountReturned": len(returned), + "limitedTo": ANOMALY_SAMPLE_LIMIT, + "sampleTruncated": len(returned) >= ANOMALY_SAMPLE_LIMIT, + "note": "No sample representation is emitted for a sample-free VCF." if returned and profile_missing_is_expected else None, + } + elif query_id == "preflight_missing_token_conformance": + report[query_id] = { + "status": "PASS" if not returned else "EXPECTED_CONFORMANCE_FAILURE", + "anomalyCount": exact, + "plainDotCountReturned": len(returned), + "limitedTo": ANOMALY_SAMPLE_LIMIT, + "sampleTruncated": len(returned) >= ANOMALY_SAMPLE_LIMIT, + } + elif len(returned) != 1: + report[query_id] = {"status": "FAIL", "error": f"Expected one aggregate row, got {len(returned)}"} + elif representation == "expanded": + actual = {field: binding_int(returned[0], field) for field in ("sampleCallCount", "sampleIdCount", "gtValueNodeCount")} + expected = { + "sampleCallCount": parser["sampleCount"] * parser["totalRecords"], + "sampleIdCount": parser["sampleCount"], + "gtValueNodeCount": parser["sampleCount"] * parser["gtRecordCount"], + } + report[query_id] = {"status": "PASS" if actual == expected else "FAIL", "expected": expected, "actual": actual} + else: + actual = {field: binding_int(returned[0], field) for field in ("sampleCount", "sampleIdCount", "gtVectorCount")} + expected = {"sampleCount": parser["sampleCount"], "sampleIdCount": parser["sampleCount"], "gtVectorCount": parser["gtRecordCount"]} + report[query_id] = {"status": "PASS" if actual == expected else "FAIL", "expected": expected, "actual": actual} + return report + + +def compare_rows(query_id: str, expected: Any, actual: Any) -> dict[str, Any]: + keys, values = QUERY_SPECS[query_id] + selected = keys + values + if not keys: + expected_value = {field: expected[field] for field in values} + actual_value = {field: actual[field] for field in values} + return { + "status": "PASS" if expected_value == actual_value else "MISMATCH", + "expected": expected_value, + "actual": actual_value, + } + def index(rows: list[dict[str, Any]]) -> dict[tuple[Any, ...], dict[str, Any]]: + indexed: dict[tuple[Any, ...], dict[str, Any]] = {} + for row in rows: + key = tuple(row[field] for field in keys) + if key in indexed: + raise ValueError(f"Duplicate {query_id} key: {key!r}") + indexed[key] = {field: row[field] for field in selected} + return indexed + expected_index, actual_index = index(expected), index(actual) + missing_keys, extra_keys = set(expected_index) - set(actual_index), set(actual_index) - set(expected_index) + differing = [ + {"expected": expected_index[key], "actual": actual_index[key]} + for key in sorted(set(expected_index) & set(actual_index)) + if any(expected_index[key][field] != actual_index[key][field] for field in values) + ] + return { + "status": "PASS" if not missing_keys and not extra_keys and not differing else "MISMATCH", + "missingRows": [expected_index[key] for key in sorted(missing_keys)], + "extraRows": [actual_index[key] for key in sorted(extra_keys)], + "differingRows": differing, + } + + +def invariant_checks(payload: dict[str, Any], parser: dict[str, Any], *, check_q06_exact: bool) -> list[dict[str, Any]]: + checks: list[tuple[str, bool, str]] = [] + total = parser["totalRecords"] + for query_id, field in (("q01_record_density_1mb", "recordCount"), ("q02_variant_shape_counts", "recordCount"), ("q04_filter_distribution", "recordCount")): + count = sum(int(row[field]) for row in payload[query_id]) + checks.append((f"{query_id}_total", count == total, f"{count} == {total}")) + q3 = payload["q03_titv"] + checks.append(("q03_partition", q3["transitionCount"] + q3["transversionCount"] == q3["biallelicSnvCount"], "transition + transversion == biallelic SNV")) + if parser["sampleCount"] and parser["gtRecordCount"]: + per_sample: Counter[str] = Counter() + for row in payload["q05_sample_genotype_counts"]: + per_sample[row["sampleId"]] += int(row["callCount"]) + for sample, count in sorted(per_sample.items()): + checks.append((f"q05_total_{sample}", count == total, f"{count} == {total}")) + q06_total = sum(int(row["siteCount"]) for row in payload["q06_ac_an_distribution"]) + checks.append(("q06_subset", q06_total <= parser["singleAltRecordCount"], f"{q06_total} <= {parser['singleAltRecordCount']}")) + if check_q06_exact: + checks.append(("q06_eligible_total", q06_total == parser["q06EligibleSiteCount"], f"{q06_total} == {parser['q06EligibleSiteCount']}")) + for row in payload["q06_ac_an_distribution"]: + checks.append((f"q06_bounds_{row['an']}_{row['ac']}", row["an"] > 0 and 0 <= row["ac"] <= row["an"] and row["siteCount"] > 0, "AN/AC bounds")) + return [{"name": name, "status": "PASS" if passed else "FAIL", "detail": detail} for name, passed, detail in checks] + + +def compare( + parser: dict[str, Any], sparql: dict[str, Any], *, mapping_policy: str = "strict" +) -> dict[str, Any]: + required = bool(parser["sampleCount"] and parser["gtRecordCount"]) + query_results = {query_id: compare_rows(query_id, parser[query_id], sparql[query_id]) for query_id in CORE_QUERIES} + if not required: + for query_id in ("q05_sample_genotype_counts", "q06_ac_an_distribution"): + query_results[query_id] = {"status": "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT", "diagnosticComparison": query_results[query_id]} + if mapping_policy == "report-only": + # A custom mapping emits a different inventory and different IRIs by + # design, so these are reported for inspection but cannot fail a run. + for query_id in DEFAULT_MAPPING_QUERIES: + query_results[query_id] = { + "status": "NOT_APPLICABLE_CUSTOM_MAPPING", + "diagnosticComparison": query_results[query_id], + } + parser_invariants = invariant_checks(parser, parser, check_q06_exact=True) + sparql_invariants = invariant_checks(sparql, parser, check_q06_exact=False) + allowed = { + "PASS", + "NOT_APPLICABLE_VERIFIED_NO_SAMPLES_OR_GT", + "NOT_APPLICABLE_CUSTOM_MAPPING", + } + passed = all(value["status"] in allowed for value in query_results.values()) and all(item["status"] == "PASS" for item in parser_invariants + sparql_invariants) + return {"status": "PASS" if passed else "MISMATCH", "queries": query_results, "invariants": {"parser": parser_invariants, "sparql": sparql_invariants}} + + +# Queries whose failure means the graph's core structure is wrong, so the +# aggregate comparison below them cannot be interpreted. +BLOCKING_PREFLIGHT_QUERIES = ( + "preflight_record_cardinality", + "preflight_position_datatype", + "preflight_representation_profile", + # Graph-integrity checks. A blank node breaks the IRI-template identity the + # digests rely on; an empty term means a value was lost rather than marked + # missing; a duplicated statement means the conversion emitted the same + # data twice. None of these leave a comparison worth interpreting. + "preflight_blank_nodes", + "preflight_empty_values", + "preflight_duplicate_triples", +) + + +def evaluate_validation( + executions: dict[str, dict[str, Any]], + parser: dict[str, Any], + representation: str, + *, + strict_conformance: bool = False, + mapping_policy: str = "strict", + parsed_triple_count: int | None = None, +) -> dict[str, Any]: + """Turn raw query executions into a validation verdict. + + This is the single place the PASS / MISMATCH / BLOCKED_BY_PREFLIGHT + decision is made. ``run_validation`` uses it for real runs and the mutation + harness uses it directly, so the harness measures the shipped decision + logic rather than a reimplementation of it. + """ + report = preflight( + executions, parser, representation, parsed_triple_count=parsed_triple_count + ) + sparql: dict[str, Any] = {} + failures: dict[str, str] = {} + for query_id in CORE_QUERIES: + try: + sparql[query_id] = normalize(query_id, Path(executions[query_id]["rawResult"])) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: + failures[query_id] = str(error) + if failures: + return { + "status": "EXECUTION_FAILED", + "preflight": report, + "normalizationFailures": failures, + "sparql": sparql, + "comparison": None, + } + + comparison = compare(parser, sparql, mapping_policy=mapping_policy) + # NOT_EVALUATED is not a failure: it means an input the check needs was + # unavailable, which must not be reported as a bad graph. + blocking = any( + report[name]["status"] not in {"PASS", "NOT_EVALUATED"} + for name in BLOCKING_PREFLIGHT_QUERIES + ) + inventory_failed = report["preflight_sample_gt_inventory"]["status"] != "PASS" + conformance_failed = strict_conformance and ( + report["preflight_missing_token_conformance"]["status"] != "PASS" + ) + if blocking: + status = "BLOCKED_BY_PREFLIGHT" + elif inventory_failed or conformance_failed or comparison["status"] != "PASS": + status = "MISMATCH" + else: + status = "PASS" + return { + "status": status, + "preflight": report, + "normalizationFailures": {}, + "sparql": sparql, + "comparison": comparison, + "strictConformanceApplied": bool(strict_conformance), + "mappingPolicy": mapping_policy, + } + + +# --------------------------------------------------------------------------- +# Benchmarking +# --------------------------------------------------------------------------- +# A validation run already measures everything a performance comparison needs: +# it computes the answers twice, once from the VCF with a conventional parser +# and once from RDF with a SPARQL engine, over the same data and to the same +# result. Recording those timings turns each run into a directly comparable +# oracle-versus-SPARQL measurement, and a multi-engine run into a comparison +# between the engines as well. + +BENCHMARK_CSV_HEADER = [ + "engine", + "query_id", + "status", + "wall_seconds", + "oracle_wall_seconds", + "engine_setup_seconds", + "artifact_origin", +] + + +def build_benchmark( + per_engine: dict[str, dict[str, Any]], + engine_descriptions: dict[str, Any], + *, + oracle_seconds: dict[str, float] | None, + materialization_seconds: float | None, + shacl_seconds: float | None, + query_ids: tuple[str, ...], +) -> dict[str, Any]: + """Assemble per-query and per-engine timings into one report.""" + oracle_seconds = oracle_seconds or {} + engines: dict[str, Any] = {} + for name, verdict in per_engine.items(): + executions = verdict.get("executions") or verdict.get("queryExecutions") or {} + queries = { + query_id: { + "status": execution.get("status"), + "wallSeconds": execution.get("wallSeconds"), + } + for query_id, execution in executions.items() + } + timed = [ + entry["wallSeconds"] for entry in queries.values() + if isinstance(entry["wallSeconds"], (int, float)) + ] + description = engine_descriptions.get(name, {}) + engines[name] = { + "status": verdict.get("status"), + "setupSeconds": description.get("setupSeconds"), + "artifactOrigin": description.get("artifactOrigin"), + "artifactSizeBytes": description.get("artifactSizeBytes"), + "querySeconds": sum(timed) if timed else None, + "slowestQuery": ( + max(queries.items(), key=lambda item: item[1]["wallSeconds"] or 0)[0] + if timed else None + ), + "queries": queries, + } + + return { + "oracle": { + # The parser side of the comparison: what it costs to compute the + # same answers from the VCF directly. + "totalSeconds": oracle_seconds.get("total"), + "vcfParseSeconds": oracle_seconds.get("parse"), + "censusSeconds": oracle_seconds.get("census"), + }, + "preparation": { + "materializationSeconds": materialization_seconds, + "shaclSeconds": shacl_seconds, + }, + "engines": engines, + "totals": { + "oracleSeconds": oracle_seconds.get("total"), + "engineQuerySeconds": { + name: value["querySeconds"] for name, value in engines.items() + }, + "engineSetupSeconds": { + name: value["setupSeconds"] for name, value in engines.items() + }, + }, + "queryIds": list(query_ids), + } + + +def write_benchmark_csv(path: Path, benchmark: dict[str, Any]) -> Path: + """Write one row per engine and query, for direct analysis.""" + path.parent.mkdir(parents=True, exist_ok=True) + oracle_total = benchmark["oracle"]["totalSeconds"] + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=BENCHMARK_CSV_HEADER) + writer.writeheader() + for name, engine in benchmark["engines"].items(): + for query_id, entry in engine["queries"].items(): + writer.writerow({ + "engine": name, + "query_id": query_id, + "status": entry["status"], + "wall_seconds": entry["wallSeconds"], + # Repeated on each row so the CSV is usable without a join. + "oracle_wall_seconds": oracle_total, + "engine_setup_seconds": engine["setupSeconds"], + "artifact_origin": engine["artifactOrigin"] or "", + }) + return path + + +def compare_engines(per_engine: dict[str, dict[str, Any]]) -> dict[str, Any]: + """Check that every engine that ran produced the same normalized results. + + Engines disagreeing is a finding in its own right: the graph may be correct + and one engine wrong, as when QLever's literal canonicalisation made a + datatype preflight fail on it alone. A multi-engine run is the cheapest + place to notice that, so it is checked rather than assumed. + """ + usable = { + name: verdict["sparql"] + for name, verdict in per_engine.items() + if verdict.get("status") != "EXECUTION_FAILED" and verdict.get("sparql") + } + if len(usable) < 2: + return { + "agree": True, + "comparedEngines": sorted(usable), + "note": "fewer than two engines produced results, so there is nothing to compare", + "differences": {}, + } + reference_name = sorted(usable)[0] + reference = usable[reference_name] + differences: dict[str, Any] = {} + for name in sorted(usable): + if name == reference_name: + continue + differing = [ + query_id for query_id in reference + if usable[name].get(query_id) != reference[query_id] + ] + if differing: + differences[name] = differing + return { + "agree": not differences, + "comparedEngines": sorted(usable), + "referenceEngine": reference_name, + "differences": differences, + } + + +def build_manifest( + args: argparse.Namespace, + query_dir: Path, + parser: dict[str, Any], + *, + engine_description: dict[str, Any], + materialization: dict[str, Any], +) -> dict[str, Any]: + query_paths = sorted({query_path(query_dir, query_id) for query_id in PREFLIGHT_QUERIES + CORE_QUERIES}) + source_rdf_entry = { + "path": str(args.rdf), + "sha256": sha256_file(args.rdf), + "format": args.rdf_format, + } + manifest = { + "datasetId": args.dataset_id, + "representation": args.representation, + "commandLine": sys.argv, + "sourceVcf": {"path": str(args.vcf), "sha256": parser["sourceSha256"]}, + "sourceRdf": source_rdf_entry, + "engine": engine_description, + "materialization": materialization, + "temporaryRdf": { + "decompressedInsideContainer": bool(materialization.get("materialized")), + "persisted": False, + "cleanupConfirmed": True, + }, + "tools": { + "python": platform.python_version(), + "cyvcf2": getattr(cyvcf2, "__version__", None), + "bcftools": tool_version(["bcftools", "--version"]), "node": tool_version(["node", "--version"]), + "comunicaQuerySparqlFile": tool_version(["comunica-sparql-file", "--version"], table_label="Comunica Engine"), + "rapper": tool_version(["rapper", "--version"]), + }, + "queries": {path.stem: {"path": str(path), "sha256": sha256_file(path)} for path in query_paths}, + } + # Preserve the original key for consumers of standalone gzip-validation + # manifests while exposing the format-neutral ``sourceRdf`` entry. + if args.rdf_format == "nt.gz": + manifest["sourceRdfGzip"] = dict(source_rdf_entry) + return manifest + + +def query_path(representation_dir: Path, query_id: str) -> Path: + """Resolve a representation-specific query, falling back to common RDF queries.""" + candidate = representation_dir / f"{query_id}.rq" + return candidate if candidate.is_file() else QUERY_ROOT / "common" / f"{query_id}.rq" + + +def run_validation(args: argparse.Namespace) -> int: + results_dir = args.results_dir.resolve() + raw_dir, normalized_dir = results_dir / "raw", results_dir / "normalized" + raw_dir.mkdir(parents=True, exist_ok=True) + normalized_dir.mkdir(parents=True, exist_ok=True) + query_dir = QUERY_ROOT / args.representation + query_ids = PREFLIGHT_QUERIES + PREFLIGHT_COUNT_QUERIES + CORE_QUERIES + missing = [name for name in query_ids if not query_path(query_dir, name).is_file()] + if missing: + raise RuntimeError(f"Missing {args.representation} validation query files: {', '.join(missing)}") + progress = ValidationProgress(getattr(args, "progress_path", None), len(query_ids)) + quiet = bool(getattr(args, "quiet", False)) + progress.emit( + "started", + completed=0, + detail=f"{args.representation} validation started", + ) + summary: dict[str, Any] | None = None + try: + with tempfile.TemporaryDirectory(prefix="vcf-rdfizer-validation-", dir=args.scratch_dir) as scratch: + scratch_path = Path(scratch) + progress.emit( + "progress", + completed=0, + detail=f"materializing {args.rdf_format} artifact inside container", + ) + materialize_started = time.monotonic() + decoded, materialization = materialize_ntriples( + args.rdf, + args.rdf_format, + scratch_path, + log_dir=raw_dir / "materialization", + ) + materialization["wallSeconds"] = time.monotonic() - materialize_started + materialization["ntriplesPath"] = str(decoded) + progress.emit( + "progress", + completed=0, + detail="parsing source VCF", + ) + oracle_started = time.monotonic() + parser = parse_vcf(args.vcf, filter_oracle=args.filter_oracle) + parse_seconds = time.monotonic() - oracle_started + census_started = time.monotonic() + parser = attach_census_expectations(parser, args.representation) + oracle_seconds = { + "parse": parse_seconds, + "census": time.monotonic() - census_started, + "total": time.monotonic() - oracle_started, + } + parser["oracleSeconds"] = oracle_seconds + write_json(results_dir / "parser.json", parser) + progress.emit( + "progress", + completed=0, + detail="validating RDF syntax and cardinality", + ) + rdf_validation = validate_ntriples(decoded, results_dir) + shacl_result = None + if args.shacl_shapes is not None: + progress.emit("progress", completed=0, detail="validating SHACL shapes") + shacl_result = validate_shacl(decoded, args.shacl_shapes, results_dir) + write_json(results_dir / "rdf-validation.json", rdf_validation) + materialization["decodedTripleCount"] = rdf_validation.get("tripleCount") + write_json(results_dir / "materialization.json", materialization) + engine_options = { + "port": args.qlever_port, + "memory_gb": args.qlever_memory_gb, + "startup_timeout": args.qlever_startup_timeout, + "query_timeout": args.query_timeout, + "extra_index_args": list(args.qlever_index_arg), + "extra_server_args": list(args.qlever_server_arg), + } + engine_options["artifact_path"] = str(args.rdf) + engine_options["artifact_format"] = args.rdf_format + manifest = build_manifest( + args, query_dir, parser, + engine_description={"engines": list(args.engines), "options": engine_options}, + materialization=materialization, + ) + write_json(results_dir / "manifest.json", manifest) + if rdf_validation["status"] != "PASS": + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "BLOCKED_BY_PREFLIGHT", "rdfValidation": rdf_validation} + return 1 + if shacl_result is not None and shacl_result["status"] == "FAIL": + # A shape violation means the graph is structurally wrong, so + # the aggregate comparisons below it cannot be interpreted. + summary = { + "datasetId": args.dataset_id, "representation": args.representation, + "status": "BLOCKED_BY_PREFLIGHT", "shacl": shacl_result, + } + return 1 + + # Each requested engine answers the whole query set independently. + # They are compared against the same oracle and against each other, + # and every engine's timings are recorded, which is what makes a + # multi-engine run usable as a benchmark. + per_engine: dict[str, dict[str, Any]] = {} + engine_descriptions: dict[str, Any] = {} + for engine_name in args.engines: + engine_raw_dir = raw_dir / engine_name + engine_raw_dir.mkdir(parents=True, exist_ok=True) + engine = build_engine( + engine_name, decoded, raw_dir=engine_raw_dir, scratch=scratch_path, + options=engine_options, + ) + progress.emit("progress", completed=0, detail=f"preparing {engine_name} engine") + if not quiet: + print(f"[{args.dataset_id}] preparing {engine_name} engine", flush=True) + try: + engine.prepare() + except (RuntimeError, OSError) as error: + per_engine[engine_name] = { + "status": "EXECUTION_FAILED", + "error": f"{engine_name} engine could not be prepared: {error}", + } + engine_descriptions[engine_name] = {"engine": engine_name, "error": str(error)} + continue + + executions: dict[str, dict[str, Any]] = {} + try: + engine_descriptions[engine_name] = engine.describe() + for completed, query_id in enumerate(query_ids, start=1): + progress.emit( + "progress", completed=completed - 1, query_id=query_id, + detail=f"{engine_name}: {args.representation}/{query_id}", + ) + if not quiet: + print( + f"[{args.dataset_id}] running {args.representation}/{query_id}" + f" ({engine_name})", + flush=True, + ) + executions[query_id] = engine.execute( + query_id, query_path(query_dir, query_id) + ) + progress.emit( + "progress", completed=completed, query_id=query_id, + detail=f"{engine_name}: completed {query_id}", + ) + finally: + engine.stop() + + engine_dir = results_dir / "engines" / engine_name + engine_dir.mkdir(parents=True, exist_ok=True) + write_json(engine_dir / "query-executions.json", executions) + if any(item["status"] != "PASS" for item in executions.values()): + per_engine[engine_name] = { + "status": "EXECUTION_FAILED", "queryExecutions": executions, + } + continue + + verdict = evaluate_validation( + executions, parser, args.representation, + strict_conformance=args.strict_conformance, + mapping_policy=args.mapping_policy, + parsed_triple_count=rdf_validation.get("tripleCount"), + ) + write_json(engine_dir / "preflight.json", verdict["preflight"]) + if verdict["status"] != "EXECUTION_FAILED": + write_json(engine_dir / "sparql.json", verdict["sparql"]) + write_json(engine_dir / "comparison.json", verdict["comparison"]) + verdict["executions"] = executions + per_engine[engine_name] = verdict + + benchmark = build_benchmark( + per_engine, engine_descriptions, + oracle_seconds=oracle_seconds, + materialization_seconds=materialization.get("wallSeconds"), + shacl_seconds=(shacl_result or {}).get("wallSeconds"), + query_ids=query_ids, + ) + write_json(results_dir / "benchmark.json", benchmark) + write_benchmark_csv(results_dir / "benchmark.csv", benchmark) + + agreement = compare_engines(per_engine) + write_json(results_dir / "engine-agreement.json", agreement) + + # The primary engine's artifacts keep their historical locations so + # existing single-engine consumers are unaffected. + primary = args.engines[0] + primary_verdict = per_engine[primary] + report = primary_verdict.get("preflight", {}) + write_json(results_dir / "preflight.json", report) + if primary_verdict["status"] == "EXECUTION_FAILED": + summary = { + "datasetId": args.dataset_id, "representation": args.representation, + "status": "EXECUTION_FAILED", + "engine": primary, "engines": list(args.engines), + "normalizationFailures": primary_verdict.get("normalizationFailures", {}), + "queryExecutions": primary_verdict.get("queryExecutions"), + "error": primary_verdict.get("error"), + "preflight": report, "benchmark": benchmark["totals"], + } + return 1 + sparql = primary_verdict["sparql"] + for query_id, rows in sparql.items(): + write_json(normalized_dir / f"{query_id}.json", rows) + write_json(results_dir / "sparql.json", sparql) + comparison = primary_verdict["comparison"] + write_json(results_dir / "comparison.json", comparison) + + statuses = {name: value["status"] for name, value in per_engine.items()} + failed_engines = [name for name, value in statuses.items() if value != "PASS"] + status = statuses[primary] + if not failed_engines and not agreement["agree"]: + # Every engine validated, but they disagree with each other. The + # graph may be fine and an engine wrong; either way the run + # cannot be called a pass. + status = "ENGINE_DISAGREEMENT" + elif failed_engines: + status = statuses[primary] if statuses[primary] != "PASS" else "MISMATCH" + summary = { + "datasetId": args.dataset_id, "representation": args.representation, "status": status, + "engine": primary, "engines": list(args.engines), + "engineStatuses": statuses, "engineAgreement": agreement["agree"], + "sourceFormat": args.rdf_format, + "strictConformance": bool(args.strict_conformance), + "shacl": shacl_result, + "mappingPolicy": args.mapping_policy, + "recordCount": parser["totalRecords"], "sampleCount": parser["sampleCount"], "gtRecordCount": parser["gtRecordCount"], + "preflight": report, "comparisonStatus": comparison["status"], + "benchmark": benchmark["totals"], + "results": {"manifest": str(results_dir / "manifest.json"), "parser": str(results_dir / "parser.json"), "sparql": str(results_dir / "sparql.json"), "comparison": str(results_dir / "comparison.json"), "benchmark": str(results_dir / "benchmark.csv")}, + } + return 0 if status == "PASS" else 1 + except Exception as error: + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": str(error)} + return 1 + finally: + if summary is None: + summary = {"datasetId": args.dataset_id, "representation": args.representation, "status": "EXECUTION_FAILED", "error": "validation ended without a result"} + summary.setdefault("engine", args.engines[0]) + summary.setdefault("engines", list(args.engines)) + summary.setdefault("sourceFormat", args.rdf_format) + summary["temporaryRdf"] = { + "decompressedInsideContainer": args.rdf_format not in DIRECT_FORMATS, + "persisted": False, + "cleanupConfirmed": True, + } + write_json(results_dir / "summary.json", summary) + status = str(summary.get("status", "EXECUTION_FAILED")) + if status == "EXECUTION_FAILED": + progress.emit("failed", detail="validation execution failed") + else: + progress.emit( + "complete", + completed=progress.total if status in {"PASS", "MISMATCH"} else 0, + detail=f"validation {status.lower()}", + ) + if not quiet: + print(json.dumps(summary, indent=2), flush=True) + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vcf", type=Path, required=True) + rdf_group = parser.add_mutually_exclusive_group(required=True) + rdf_group.add_argument( + "--rdf", + type=Path, + help=( + "RDF artifact to validate: .nt, .nt.gz, .nt.br, .hdt, .cottas, " + ".cottas.gz, or .cottas.br" + ), + ) + # Retained so existing callers and reports keep working. + rdf_group.add_argument("--rdf-gz", type=Path, help="Deprecated alias for --rdf (.nt.gz)") + rdf_group.add_argument("--rdf-nt", type=Path, help="Deprecated alias for --rdf (.nt)") + parser.add_argument( + "--rdf-format", + choices=("auto", *RDF_FORMATS), + default="auto", + help="Override artifact format detection (default: infer from the filename)", + ) + parser.add_argument("--representation", choices=("expanded", "condensed"), required=True) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--dataset-id", required=True) + parser.add_argument("--filter-oracle", choices=("auto", "bcftools", "cyvcf2"), default="auto") + parser.add_argument("--scratch-dir", type=Path, default=Path("/work")) + parser.add_argument( + "--engine", + default="comunica", + help=( + "SPARQL engine(s), comma-separated, or 'all'. comunica queries the " + "N-Triples in memory; qlever builds an on-disk index and serves it; " + "hdt and cottas query those compressed artifacts natively, without " + "decoding them. Several may be given: each answers the whole query " + "set, results are compared across engines, and every engine's " + "timings are recorded for benchmarking. The first is the primary, " + "whose reports keep the single-engine layout " + f"(choices: {', '.join(SPARQL_ENGINES)}; default: comunica)" + ), + ) + parser.add_argument( + "--query-timeout", + type=int, + default=DEFAULT_QUERY_TIMEOUT, + help=f"Per-query timeout in seconds (default: {DEFAULT_QUERY_TIMEOUT})", + ) + parser.add_argument( + "--qlever-memory-gb", + type=int, + default=DEFAULT_QLEVER_MEMORY_GB, + help=f"QLever index/server memory budget in GiB (default: {DEFAULT_QLEVER_MEMORY_GB})", + ) + parser.add_argument( + "--qlever-port", + type=int, + default=DEFAULT_QLEVER_PORT, + help=f"Container-local QLever port (default: {DEFAULT_QLEVER_PORT})", + ) + parser.add_argument( + "--qlever-startup-timeout", + type=int, + default=DEFAULT_QLEVER_STARTUP_TIMEOUT, + help=( + "Seconds to wait for the QLever server to answer after indexing " + f"(default: {DEFAULT_QLEVER_STARTUP_TIMEOUT})" + ), + ) + parser.add_argument( + "--shacl-shapes", + type=Path, + default=None, + help=( + "Validate the graph against a SHACL shapes file as an independent " + "structural layer. Off by default: pyshacl loads the whole graph " + "into memory, so it does not scale to a cohort-sized aggregate" + ), + ) + parser.add_argument( + "--mapping-policy", + choices=MAPPING_POLICIES, + default="strict", + help=( + "How to treat checks that assume the shipped RML mapping (the " + "predicate/class census and the record-identity digest): strict " + "requires an exact match; report-only records them without failing, " + "which is what a custom mapping needs (default: strict)" + ), + ) + parser.add_argument( + "--strict-conformance", + action="store_true", + help=( + "Treat a missing-token conformance failure (a plain '.' literal not " + "typed as vcfr:Null) as a validation failure instead of a report-only " + "observation" + ), + ) + parser.add_argument( + "--qlever-index-arg", + action="append", + default=[], + metavar="ARG", + help=( + "Extra argument for QLever's IndexBuilderMain (repeatable). " + "QLEVER_INDEX_COMMAND replaces the whole command line instead." + ), + ) + parser.add_argument( + "--qlever-server-arg", + action="append", + default=[], + metavar="ARG", + help=( + "Extra argument for QLever's ServerMain (repeatable). " + "QLEVER_SERVER_COMMAND replaces the whole command line instead." + ), + ) + parser.add_argument( + "--progress-path", + type=Path, + help="optional JSONL sidecar consumed by the host progress display", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="suppress per-query and summary output on stdout", + ) + return parser + + +def parse_engine_list(raw: str) -> list[str]: + """Parse a comma-separated engine list, preserving order and de-duplicating. + + Order matters: the first engine is the primary, whose reports keep the + single-engine layout that existing consumers read. + """ + value = (raw or "").strip() + if value == "all": + return list(SPARQL_ENGINES) + engines: list[str] = [] + for token in value.split(","): + engine = token.strip() + if not engine: + continue + if engine not in SPARQL_ENGINES: + raise ValueError( + f"unknown SPARQL engine '{engine}'; choose from " + f"{', '.join(SPARQL_ENGINES)}, or 'all'" + ) + if engine not in engines: + engines.append(engine) + if not engines: + raise ValueError("--engine requires at least one engine") + return engines + + +def resolve_args(parser: argparse.ArgumentParser, argv: list[str] | None = None) -> argparse.Namespace: + """Parse and normalise arguments, collapsing the RDF aliases into --rdf.""" + args = parser.parse_args(argv) + try: + args.engines = parse_engine_list(args.engine) + except ValueError as error: + parser.error(str(error)) + # Retained as the primary engine's name for reports and callers that read it. + args.engine = args.engines[0] + args.vcf = args.vcf.resolve() + + supplied = args.rdf or args.rdf_gz or args.rdf_nt + args.rdf = supplied.resolve() + if args.rdf_gz is not None and args.rdf_format == "auto": + args.rdf_format = "nt.gz" + if args.rdf_nt is not None and args.rdf_format == "auto": + args.rdf_format = "nt" + if args.rdf_format == "auto": + detected = detect_rdf_format(args.rdf) + if detected is None: + parser.error( + f"Could not infer an RDF format from {args.rdf.name!r}; pass " + f"--rdf-format with one of: {', '.join(RDF_FORMATS)}" + ) + args.rdf_format = detected + + if not args.vcf.is_file(): + parser.error(f"VCF does not exist: {args.vcf}") + if not args.rdf.is_file(): + parser.error(f"RDF artifact does not exist: {args.rdf}") + if not re.fullmatch(r"[A-Za-z0-9._-]+", args.dataset_id): + parser.error("--dataset-id may contain only letters, digits, dot, underscore, and hyphen") + if not args.scratch_dir.is_dir(): + parser.error(f"Scratch directory does not exist: {args.scratch_dir}") + for name, value in ( + ("--query-timeout", args.query_timeout), + ("--qlever-memory-gb", args.qlever_memory_gb), + ("--qlever-startup-timeout", args.qlever_startup_timeout), + ): + if value <= 0: + parser.error(f"{name} must be a positive integer") + if not 1 <= args.qlever_port <= 65535: + parser.error("--qlever-port must be between 1 and 65535") + if args.shacl_shapes is not None: + args.shacl_shapes = args.shacl_shapes.resolve() + if not args.shacl_shapes.is_file(): + parser.error(f"SHACL shapes file does not exist: {args.shacl_shapes}") + if args.progress_path is not None: + args.progress_path = args.progress_path.resolve() + return args + + +def parse_args() -> argparse.Namespace: + return resolve_args(build_arg_parser()) + + +if __name__ == "__main__": + raise SystemExit(run_validation(parse_args())) diff --git a/test/README.md b/test/README.md index 108d4e2..c66c5d7 100644 --- a/test/README.md +++ b/test/README.md @@ -24,10 +24,62 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - Verifies output normalization to `.nt`. - Verifies unified metrics CSV row creation and schema consistency. -- `test/test_compression_unit.py` - - Replaces `gzip`, `brotli`, and `rdf2hdt` with fake executables for the standalone helper. - - Verifies helper compression artifact generation and metrics row update. - - Verifies no-op behavior (no compression outputs, metrics still updated). +- `test/test_partitioned_compression_unit.py` and `test/test_cottas_tool.py` + - Exercise the container-side chunking, merge, and COTTAS adapter logic. + +- `test/test_rules_helper_unit.py` + - Verifies the `vcf-rdfizer-rules` contract checks (source paths, column + references, sample-representation compatibility, helper-table warnings). + - Pins the documented TSV column lists to the headers `src/vcf_as_tsv.sh` + actually writes, so the two cannot drift apart. + +- `test/test_validation_mutation_unit.py` (+ `validation_fixtures.py`, + `validation_mutations.py`) + - Mutation testing for the semantic validation suite: corrupts a correct + graph in 42 named ways and asserts which corruptions the validator + detects, producing a reproducible mutation score (currently 76/78). + - Requires `rdflib` (test-only, in the `dev` extra); the tests skip cleanly + without it. See `docs/validation-methodology.md`. + - The fixture derives the VCF, the RDF graph and the parser oracle from one + declarative spec, and builds its graph with the project's own emitters, so + the two halves cannot drift apart. + +- `test/cross_engine_agreement.py` + - Not a unittest module: run inside the image to assert every validation + query returns identical values under all four engines (Comunica, QLever, + native HDT, native COTTAS), across both representations. + - Also runs the shipped validation decision under each engine, so an engine + must agree with the Python oracle and not merely with the other engines. + - Pass a comma-separated subset as the first argument to narrow it. + +- `test/test_validation_logic_unit.py` + - Mutation tests over the validator's pure comparison layer, run on the host + without cyvcf2 or Docker. + - Records both what a validation `PASS` detects and the coverage gaps it does + not, so closing a gap fails a test rather than passing unnoticed. + +- `test/test_validation_engines_unit.py` + - Verifies artifact format detection and decode paths (`.nt`, `.nt.gz`, + `.nt.br`, `.hdt`, `.cottas[.gz|.br]`) with the container tools faked. + - Verifies Comunica and QLever engine construction, QLever's + index/serve/teardown lifecycle and overridable command lines, and the + wrapper's validation-target resolution. + +- `test/test_validation_benchmark_unit.py` + - Verifies multi-engine selection (`--validation-engine a,b` and `all`) in + both the host wrapper and the container runner, and that the two layers + cannot drift apart on which engines exist. + - Verifies the native HDT/COTTAS engines: reusing the run's own artifact + versus building one, Comunica's `hdt@` typed-source prefix, and that + a failing query is reported rather than raised. + - Verifies the benchmark report and its long-format CSV, and the + cross-engine agreement comparison. + +- `test/test_gzip_size_unit.py` + - Verifies uncompressed-size measurement for BGZF, single-member gzip, and + concatenated members, each against a full-inflate ground truth. + - Verifies that an unresolvable file falls back rather than reporting a wrong + size, including the 32-bit `ISIZE` wrap and multi-member trailers. ## CI matrix behavior diff --git a/test/cross_engine_agreement.py b/test/cross_engine_agreement.py new file mode 100644 index 0000000..4762497 --- /dev/null +++ b/test/cross_engine_agreement.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Assert every validation query returns the same values on every engine. + +Run inside the VCF-RDFizer image, where every engine exists: + + docker run --rm -v "$PWD:/repo:ro" \\ + /opt/pycottas-venv/bin/python /repo/test/cross_engine_agreement.py + +All four engines are compared by default. Pass a comma-separated subset as the +first argument (or in VCF_RDFIZER_AGREEMENT_ENGINES) to narrow it, which is +useful when one engine is slow: Comunica answers each query in about a second +against N-Triples or HDT, while QLever and COTTAS answer in milliseconds once +their artifact exists. + +This exists because an engine disagreement is silent and severe: QLever +canonicalises numeric literals at index time, which once made a POS datatype +preflight fail every QLever run while passing on Comunica. Only the values the +validator actually consumes are compared - a store is free to report its own +datatype IRI for a count, and the normalization layer is datatype-agnostic by +design. + +Exits non-zero on the first disagreement, printing both sides. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) + + +def load_runner(): + path = REPO_ROOT / "src" / "validation" / "validation_runner.py" + spec = importlib.util.spec_from_file_location("validation_runner_agreement", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() +DEFAULT_ENGINES = V.SPARQL_ENGINES + + +def selected_engines(argv: list[str]) -> tuple[str, ...]: + """Engines to compare: the argument, then the environment, then all of them.""" + raw = "" + if len(argv) > 1: + raw = argv[1] + else: + raw = os.environ.get("VCF_RDFIZER_AGREEMENT_ENGINES", "") + if not raw.strip(): + return tuple(DEFAULT_ENGINES) + return tuple(V.parse_engine_list(raw)) + + +def evaluate(engine_name: str, representation: str, source: Path, scratch: Path) -> dict: + results: dict[str, object] = {} + raw_dir = scratch / f"raw-{engine_name}-{representation}" + raw_dir.mkdir(parents=True, exist_ok=True) + # A private scratch per run: the native engines name their built artifact + # after the engine, so a shared directory would have HDT reading the graph + # it built for the other representation. + run_scratch = scratch / f"scratch-{engine_name}-{representation}" + run_scratch.mkdir(parents=True, exist_ok=True) + engine = V.build_engine( + engine_name, source, raw_dir=raw_dir, scratch=run_scratch, + options={"memory_gb": 2, "startup_timeout": 300, "query_timeout": 300}, + ) + with engine: + for query_id in V.PREFLIGHT_QUERIES + V.PREFLIGHT_COUNT_QUERIES + V.CORE_QUERIES: + execution = engine.execute(query_id, V.query_path(V.QUERY_ROOT / representation, query_id)) + if execution["status"] != "PASS": + results[query_id] = { + "__executionFailed__": Path(execution["stderr"]).read_text( + encoding="utf-8", errors="replace")[:400] + } + continue + raw = Path(execution["rawResult"]) + if query_id in V.QUERY_SCHEMAS: + results[query_id] = V.normalize(query_id, raw) + else: + results[query_id] = [ + {name: binding["value"] for name, binding in row.items()} + for row in V.bindings(raw) + ] + return results + + +def verdict_for(engine_name: str, representation: str, source: Path, scratch: Path) -> dict: + """Run the shipped validation decision for one engine against the fixture.""" + from test import validation_fixtures as fixtures + + raw_dir = scratch / f"verdict-{engine_name}-{representation}" + raw_dir.mkdir(parents=True, exist_ok=True) + run_scratch = scratch / f"verdict-scratch-{engine_name}-{representation}" + run_scratch.mkdir(parents=True, exist_ok=True) + engine = V.build_engine( + engine_name, source, raw_dir=raw_dir, scratch=run_scratch, + options={"memory_gb": 2, "startup_timeout": 300, "query_timeout": 300}, + ) + with engine: + executions = { + query_id: engine.execute(query_id, V.query_path(V.QUERY_ROOT / representation, query_id)) + for query_id in V.PREFLIGHT_QUERIES + V.PREFLIGHT_COUNT_QUERIES + V.CORE_QUERIES + } + # Supply the parsed statement count so the duplicate check is exercised + # here too; for well-formed N-Triples it equals the non-empty line count, + # which is what rapper reports on the real path. + parsed = sum( + 1 for line in source.read_text(encoding="utf-8").splitlines() if line.strip() + ) + return V.evaluate_validation( + executions, fixtures.parser_summary(representation), representation, + parsed_triple_count=parsed, + ) + + +def main(argv: list[str] | None = None) -> int: + from test import validation_fixtures as fixtures + + engines = selected_engines(argv if argv is not None else sys.argv) + print("Comparing: " + ", ".join(engines)) + disagreements = 0 + with tempfile.TemporaryDirectory(dir="/work") as td: + scratch = Path(td) + for representation in ("expanded", "condensed"): + source = scratch / f"{representation}.nt" + source.write_text(fixtures.build_graph(representation), encoding="utf-8") + per_engine = { + engine: evaluate(engine, representation, source, scratch) for engine in engines + } + print(f"\n=== {representation} ===") + for query_id in per_engine[engines[0]]: + values = [per_engine[engine][query_id] for engine in engines] + if all(value == values[0] for value in values): + print(f" {query_id:44s} agree") + continue + disagreements += 1 + print(f" {query_id:44s} *** DIFFER ***") + for engine, value in zip(engines, values): + print(f" {engine:9s} {json.dumps(value)[:300]}") + + # Engines agreeing with each other is not enough: the digests and + # censuses are compared against values Python computes, so each + # engine must also agree with that oracle. Running the real + # decision proves it end to end. + for engine_name in engines: + verdict = verdict_for(engine_name, representation, source, scratch) + status = verdict["status"] + print(f" {'full validation verdict: ' + engine_name:44s} {status}") + if status != "PASS": + disagreements += 1 + print(f" {json.dumps(verdict.get('comparison'))[:500]}") + + if disagreements: + print(f"\n{disagreements} disagreement(s) found.") + return 1 + print("\nAll validation queries agree across " + ", ".join(engines) + ",") + print("and every engine agrees with the Python oracle.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test_compression_unit.py b/test/test_compression_unit.py deleted file mode 100644 index 30b0d65..0000000 --- a/test/test_compression_unit.py +++ /dev/null @@ -1,377 +0,0 @@ -import csv -import os -import subprocess -import tempfile -import unittest -from pathlib import Path - -from test.helpers import VerboseTestCase, env_with_path, make_executable, seed_conversion_metrics_row - - -REPO_ROOT = Path(__file__).resolve().parents[1] -SCRIPT = REPO_ROOT / "src" / "compression.sh" - - -def prepare_fake_tools(bin_dir: Path, fail_gzip: bool = False, fail_brotli: bool = False, fail_hdt: bool = False): - gzip_fail = "exit 9\n" if fail_gzip else "" - brotli_fail = "exit 8\n" if fail_brotli else "" - hdt_fail = "exit 7\n" if fail_hdt else "" - make_executable( - bin_dir / "gzip", - """#!/usr/bin/env bash -set -euo pipefail -{gzip_fail}file="" -for arg in "$@"; do - if [[ "$arg" != -* ]]; then - file="$arg" - fi -done -cat "$file" -""".format(gzip_fail=gzip_fail), - ) - make_executable( - bin_dir / "brotli", - """#!/usr/bin/env bash -set -euo pipefail -{brotli_fail}file="" -for arg in "$@"; do - if [[ "$arg" != -* ]]; then - file="$arg" - fi -done -cat "$file" -""".format(brotli_fail=brotli_fail), - ) - hdt = bin_dir / "rdf2hdt.sh" - make_executable( - hdt, - """#!/usr/bin/env bash -set -euo pipefail -{hdt_fail}cp "$1" "$2" -""".format(hdt_fail=hdt_fail), - ) - return hdt - - -def read_metrics_row(metrics_csv: Path, run_id: str, output_name: str): - with metrics_csv.open() as f: - rows = list(csv.DictReader(f)) - for row in rows: - if row["run_id"] == run_id and row["output_name"] == output_name: - return row - raise AssertionError(f"Metrics row not found for run_id={run_id}, output_name={output_name}") - - -class CompressionUnitTests(VerboseTestCase): - def test_compression_errors_for_invalid_cli_option(self): - """Invalid CLI option: compression script exits non-zero with an option error.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - result = subprocess.run( - ["bash", str(SCRIPT), "-z"], - cwd=tmp_path, - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("invalid option", result.stderr) - - def test_compression_errors_for_unsupported_method(self): - """Unsupported compression method: script exits non-zero with clear guidance.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - (out_root / "rdf").mkdir(parents=True) - env = {"OUT_ROOT_DIR": str(out_root), "LOGDIR": str(tmp_path / "metrics")} - env.update({"PATH": os.environ["PATH"]}) - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "snappy"], - env=env, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 2) - self.assertIn("unsupported compression method", result.stderr) - - def test_compression_errors_when_hdt_requested_but_binary_missing(self): - """HDT requested with missing binary: script exits non-zero before processing.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - env = {"OUT_ROOT_DIR": str(out_root), "OUT_NAME": "rdf", "LOGDIR": str(tmp_path / "metrics")} - env.update({"PATH": os.environ["PATH"], "RDF2HDT": str(tmp_path / "missing.sh")}) - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "hdt"], - env=env, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 2) - self.assertIn("rdf2hdt binary not found", result.stderr) - - def test_compression_updates_existing_metrics_row_with_mocked_tools(self): - """Compression mode gzip|brotli|hdt updates existing metrics row and writes artifacts.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n .\n") - - logdir = tmp_path / "metrics" - run_id = "run-compress-1" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "gzip,brotli,hdt"], - env=env, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((output / "rdf.nt.gz").exists()) - self.assertTrue((output / "rdf.nt.br").exists()) - self.assertTrue((output / "rdf.hdt").exists()) - self.assertTrue((logdir / "compression_time" / "gzip" / "rdf" / "run-compress-1.txt").exists()) - self.assertTrue((logdir / "compression_time" / "brotli" / "rdf" / "run-compress-1.txt").exists()) - self.assertTrue((logdir / "compression_time" / "hdt" / "rdf" / "run-compress-1.txt").exists()) - self.assertTrue((logdir / "compression_metrics" / "rdf" / "run-compress-1.json").exists()) - - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertEqual(row["run_id"], run_id) - self.assertEqual(row["output_name"], "rdf") - self.assertEqual(row["exit_code_java"], "0") - self.assertEqual(row["compression_methods"], "gzip|brotli|hdt") - self.assertEqual(row["exit_code_gzip"], "0") - self.assertEqual(row["exit_code_brotli"], "0") - self.assertEqual(row["exit_code_hdt"], "0") - self.assertGreater(int(row["combined_rdf_size_bytes"]), 0) - - def test_compression_prefers_nt_when_present(self): - """Compression mode uses primary .nt output when multiple .nt files are present.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - (output / "other.nt").write_text("

.\n") - - logdir = tmp_path / "metrics" - run_id = "run-prefers-nt" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "gzip,brotli,hdt"], - env=env, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((output / "rdf.nt.gz").exists()) - self.assertTrue((output / "rdf.nt.br").exists()) - self.assertTrue((output / "rdf.hdt").exists()) - - def test_compression_none_updates_metrics_without_generating_outputs(self): - """Compression mode none leaves no compressed artifacts and records zero sizes.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - - logdir = tmp_path / "metrics" - run_id = "run-compress-2" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "none"], - env=env, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertFalse((output / "rdf.nt.gz").exists()) - self.assertFalse((output / "rdf.nt.br").exists()) - self.assertFalse((output / "rdf.hdt").exists()) - - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertEqual(row["compression_methods"], "none") - self.assertEqual(row["combined_rdf_size_bytes"], "0") - self.assertEqual(row["gzip_size_bytes"], "0") - self.assertEqual(row["brotli_size_bytes"], "0") - self.assertEqual(row["hdt_size_bytes"], "0") - - def test_compression_fails_when_no_nt_files_are_available_for_requested_methods(self): - """Requested compression with no .nt files returns non-zero and records method failures.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - logdir = tmp_path / "metrics" - run_id = "run-no-nt" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - result = subprocess.run( - ["bash", str(SCRIPT), "-m", "gzip,brotli,hdt"], - env=env, - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertEqual(row["exit_code_gzip"], "1") - self.assertEqual(row["exit_code_brotli"], "1") - self.assertEqual(row["exit_code_hdt"], "1") - - def test_compression_backs_up_metrics_file_on_header_mismatch(self): - """Header mismatch in metrics.csv creates a backup and rewrites a compatible header.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - logdir = tmp_path / "metrics" - logdir.mkdir(parents=True, exist_ok=True) - (logdir / "metrics.csv").write_text("bad,header\nx,y\n") - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin) - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": "run-hdr", - "TIMESTAMP": "2026-01-01T00:00:00", - "RDF2HDT": str(hdt_path), - } - ) - result = subprocess.run(["bash", str(SCRIPT), "-m", "gzip"], env=env, capture_output=True, text=True) - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((logdir / "metrics_csv_bak_run-hdr.csv").exists()) - - def test_compression_reports_failure_when_gzip_fails(self): - """gzip failure path returns non-zero and records gzip exit code.""" - self._assert_method_failure(methods="gzip", fail_gzip=True, expected_field="exit_code_gzip") - - def test_compression_reports_failure_when_brotli_fails(self): - """brotli failure path returns non-zero and records brotli exit code.""" - self._assert_method_failure(methods="brotli", fail_brotli=True, expected_field="exit_code_brotli") - - def test_compression_reports_failure_when_hdt_fails(self): - """hdt failure path returns non-zero and records hdt exit code.""" - self._assert_method_failure(methods="hdt", fail_hdt=True, expected_field="exit_code_hdt") - - def _assert_method_failure(self, methods: str, expected_field: str, fail_gzip: bool = False, fail_brotli: bool = False, fail_hdt: bool = False): - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - out_root = tmp_path / "out" - output = out_root / "rdf" - output.mkdir(parents=True) - (output / "rdf.nt").write_text("

.\n") - logdir = tmp_path / "metrics" - run_id = f"run-{methods}-fail" - timestamp = "2026-01-01T00:00:00" - metrics_csv = logdir / "metrics.csv" - seed_conversion_metrics_row(metrics_csv, run_id, timestamp, "rdf", output) - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - hdt_path = prepare_fake_tools(fake_bin, fail_gzip=fail_gzip, fail_brotli=fail_brotli, fail_hdt=fail_hdt) - env = env_with_path(fake_bin) - env.update( - { - "OUT_ROOT_DIR": str(out_root), - "OUT_NAME": "rdf", - "LOGDIR": str(logdir), - "RUN_ID": run_id, - "TIMESTAMP": timestamp, - "RDF2HDT": str(hdt_path), - } - ) - result = subprocess.run(["bash", str(SCRIPT), "-m", methods], env=env, capture_output=True, text=True) - self.assertNotEqual(result.returncode, 0) - row = read_metrics_row(metrics_csv, run_id, "rdf") - self.assertNotEqual(row[expected_field], "0") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index 293ab0d..81f3c82 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -64,22 +64,17 @@ def fake_rdf2cottas(rdf_path, cottas_path, *, index, disk): self.assertTrue(all(path.parent == scratch_root for path in observed_workspaces)) self.assertFalse(any(scratch_root.iterdir())) - def test_merge_uses_the_same_isolated_scratch_policy(self): - """Pairwise merges do not inherit a DuckDB database from chunk conversion.""" + def test_merge_uses_the_bounded_streaming_adapter(self): + """Two input merges delegate to the non-sorting streaming implementation.""" module = load_cottas_tool() - observed_workspaces = [] + calls = [] - def fake_cat(paths, cottas_path, *, index, remove_input_files): - self.assertTrue(all(Path(path).is_absolute() for path in paths)) - self.assertTrue(Path(cottas_path).is_absolute()) - self.assertEqual(index, "spo") - self.assertTrue(remove_input_files) - database_path = Path.cwd() / "pycottas.duckdb" - if database_path.exists(): - raise RuntimeError("Table with name quads already exists") - database_path.write_text("temporary DuckDB state\n") - Path(cottas_path).write_text("merged COTTAS output\n") - observed_workspaces.append(Path.cwd()) + def fake_streaming_merge(paths, output_path, *, index, remove_input_files, progress_path=None): + calls.append((paths, output_path, index, remove_input_files, progress_path)) + Path(output_path).write_text("merged COTTAS output\n") + if remove_input_files: + for path in paths: + Path(path).unlink() with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -90,8 +85,13 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): left.write_text("left\n") right.write_text("right\n") - with mock.patch.dict(sys.modules, {"pycottas": types.SimpleNamespace(cat=fake_cat)}), mock.patch.dict( + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace()}, + ), mock.patch.dict( os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "streaming_cottas_merge", side_effect=fake_streaming_merge ), mock.patch.object( sys, "argv", @@ -100,7 +100,63 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): self.assertEqual(module.main(), 0) self.assertTrue(output.is_file()) - self.assertEqual(len(observed_workspaces), 1) + self.assertEqual( + calls, + [([str(left.resolve()), str(right.resolve())], str(output.resolve()), "spo", True, None)], + ) + self.assertFalse(left.exists()) + self.assertFalse(right.exists()) + self.assertFalse(any(scratch_root.iterdir())) + + def test_merge_many_passes_all_inputs_to_streaming_merge(self): + """The production merge passes every sorted chunk through one bounded pass.""" + module = load_cottas_tool() + calls = [] + + def fake_streaming_merge(paths, output_path, *, index, remove_input_files, progress_path=None): + calls.append((paths, output_path, index, remove_input_files, progress_path)) + Path(output_path).write_text("merged COTTAS output\n") + if remove_input_files: + for path in paths: + Path(path).unlink() + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = (root / "scratch").resolve() + inputs = [] + for number in range(3): + path = root / f"chunk-{number}.cottas" + path.write_text(f"chunk {number}\n") + inputs.append(path) + output = root / "merged.cottas" + + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace()}, + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "streaming_cottas_merge", side_effect=fake_streaming_merge + ), mock.patch.object( + sys, + "argv", + [ + "cottas_tool.py", + "merge-many", + "--input-cottas-files", + *(str(path) for path in inputs), + "--output-cottas-file", + str(output), + ], + ): + self.assertEqual(module.main(), 0) + + self.assertTrue(output.is_file()) + self.assertEqual( + calls, + [([str(path.resolve()) for path in inputs], str(output.resolve()), "spo", True, None)], + ) + self.assertTrue(all(not path.exists() for path in inputs)) self.assertFalse(any(scratch_root.iterdir())) def test_decompress_uses_pycottas_and_isolated_scratch(self): @@ -139,3 +195,72 @@ def fake_cottas2rdf(cottas_path, rdf_path): self.assertEqual(output.read_text(), "

.\n") self.assertEqual(len(observed_workspaces), 1) self.assertFalse(any(scratch_root.iterdir())) + + def test_reindex_rewrites_atomically_without_removing_input(self): + """Reindex uses a streaming rewrite and replaces only on success.""" + module = load_cottas_tool() + calls = [] + + def fake_streaming_merge(paths, cottas_path, *, index, remove_input_files, progress_path=None): + calls.append((paths, cottas_path, index, remove_input_files)) + self.assertNotEqual(Path(cottas_path), source) + self.assertTrue(Path(cottas_path).name.startswith(f".{source.name}.reindex-")) + Path(cottas_path).write_text("reindexed COTTAS\n") + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = root / "scratch" + source = root / "input.cottas" + source.write_text("original COTTAS\n") + + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace()}, + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "streaming_cottas_merge", side_effect=fake_streaming_merge + ), mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "reindex", str(source)], + ): + self.assertEqual(module.main(), 0) + + self.assertEqual(source.read_text(), "reindexed COTTAS\n") + self.assertEqual( + calls, + [([str(source.resolve())], mock.ANY, "spo", False)], + ) + self.assertEqual(list(root.glob(".input.cottas.reindex-*.cottas")), []) + + def test_reindex_keeps_original_when_streaming_merge_fails(self): + """A failed COTTAS rebuild does not replace the existing artifact.""" + module = load_cottas_tool() + + def failing_streaming_merge(*args, **kwargs): + raise RuntimeError("simulated reindex failure") + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = root / "scratch" + source = root / "input.cottas" + source.write_text("original COTTAS\n") + + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace()}, + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + module, "streaming_cottas_merge", side_effect=failing_streaming_merge + ), mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "reindex", str(source)], + ): + with self.assertRaisesRegex(RuntimeError, "simulated reindex failure"): + module.main() + + self.assertEqual(source.read_text(), "original COTTAS\n") + self.assertEqual(list(root.glob(".input.cottas.reindex-*.cottas")), []) diff --git a/test/test_gzip_size_unit.py b/test/test_gzip_size_unit.py new file mode 100644 index 0000000..00663cc --- /dev/null +++ b/test/test_gzip_size_unit.py @@ -0,0 +1,241 @@ +import gzip +import struct +import tempfile +import unittest +import zlib +from pathlib import Path + +import vcf_rdfizer +import vcf_rdfizer_gzip +from test.helpers import VerboseTestCase + +VCF_LINE = "20\t%d\trs%d\tA\tG\t50\tPASS\tAC=1;AN=2;DP=%d\tGT:AD:DP:GQ:PL\t0|1:35,0:35:99:0,99,1013\n" + + +def vcf_body(count: int, start: int = 0) -> bytes: + return "".join( + VCF_LINE % (i, i, i % 90) for i in range(start, start + count) + ).encode("utf-8") + + +def write_bgzf(path: Path, payload: bytes, block_size: int = 65280) -> None: + """Write a real BGZF (bgzip) file: gzip members carrying a BC extra subfield.""" + with path.open("wb") as out: + for offset in range(0, len(payload), block_size): + chunk = payload[offset : offset + block_size] + compressor = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS) + body = compressor.compress(chunk) + compressor.flush() + total = len(body) + 26 + header = ( + bytes([31, 139, 8, 4, 0, 0, 0, 0, 0, 255]) + + struct.pack(" int: + total = 0 + with gzip.open(path, "rb") as handle: + while True: + chunk = handle.read(1 << 20) + if not chunk: + return total + total += len(chunk) + + +class GzipSizeUnitTests(VerboseTestCase): + def assert_matches_truth(self, path: Path, expected_method: str | None = None): + size, method = vcf_rdfizer_gzip.uncompressed_size(path) + self.assertEqual(size, inflate_truth(path), f"wrong size via {method}") + if expected_method is not None: + self.assertEqual(method, expected_method) + return size, method + + def test_bgzf_size_is_exact_without_inflating(self): + """A bgzip file is measured from its block headers alone.""" + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "cohort.vcf.gz" + payload = vcf_body(40_000) + write_bgzf(path, payload) + size, _ = self.assert_matches_truth(path, "bgzf") + self.assertEqual(size, len(payload)) + + def test_bgzf_single_block_and_eof_marker_only(self): + """A one-block file and a bare EOF marker both measure correctly.""" + with tempfile.TemporaryDirectory() as td: + small = Path(td) / "small.vcf.gz" + write_bgzf(small, b"tiny\n") + self.assert_matches_truth(small, "bgzf") + + empty = Path(td) / "empty.vcf.gz" + write_bgzf(empty, b"") + size, method = vcf_rdfizer_gzip.uncompressed_size(empty) + self.assertEqual((size, method), (0, "bgzf")) + + def test_single_member_gzip_within_the_sample_budget_is_measured(self): + """A small plain gzip is measured outright rather than extrapolated.""" + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "small.vcf.gz" + path.write_bytes(gzip.compress(vcf_body(2_000))) + self.assert_matches_truth(path, "gzip-sample") + + def test_large_single_member_gzip_resolves_via_the_trailer(self): + """Past the sample budget the ISIZE trailer is resolved by sampled ratio.""" + with tempfile.TemporaryDirectory() as td: + # Comfortably past the sample budget so the member cannot finish + # inside it, which is what forces the trailer-resolution path. + payload = vcf_body(400_000) + self.assertGreater( + len(payload), 2 * vcf_rdfizer_gzip.RATIO_SAMPLE_BYTES + ) + for level in (1, 6, 9): + path = Path(td) / f"big-{level}.vcf.gz" + path.write_bytes(gzip.compress(payload, level)) + self.assert_matches_truth(path, "gzip-trailer") + + def test_trailer_resolution_recovers_a_size_past_the_32_bit_wrap(self): + """ISIZE is modulo 2**32; the resolved size adds back the missing wraps.""" + compressed_size = 634_493_967 + true_size = 4_600_694_700 + wrapped = true_size % vcf_rdfizer_gzip.ISIZE_MODULUS + self.assertNotEqual(wrapped, true_size) + + # Reproduce the resolution step against a file whose true size cannot be + # expressed by the trailer, without materialising 4.3 GiB on disk. + ratio = true_size / compressed_size + projected = ratio * compressed_size + upper_bound = compressed_size * vcf_rdfizer_gzip.MAX_DEFLATE_RATIO + accepted = [ + candidate + for candidate in range( + wrapped, + int(upper_bound) + vcf_rdfizer_gzip.ISIZE_MODULUS, + vcf_rdfizer_gzip.ISIZE_MODULUS, + ) + if candidate <= upper_bound + and abs(candidate - projected) + <= vcf_rdfizer_gzip.RATIO_TOLERANCE * projected + ] + self.assertEqual(accepted, [true_size]) + + def test_concatenated_members_fall_back_instead_of_trusting_the_trailer(self): + """A trailer covering only the last member must never be used as-is.""" + with tempfile.TemporaryDirectory() as td: + first = vcf_body(120_000) + second = vcf_body(120_000, start=120_000) + + equal = Path(td) / "equal.vcf.gz" + equal.write_bytes(gzip.compress(first) + gzip.compress(second)) + self.assert_matches_truth(equal, "inflate") + + lopsided = Path(td) / "lopsided.vcf.gz" + lopsided.write_bytes(gzip.compress(first) + gzip.compress(b"tail\n")) + self.assert_matches_truth(lopsided, "inflate") + + many = Path(td) / "many.vcf.gz" + many.write_bytes( + b"".join( + gzip.compress(first[i : i + 50_000]) + for i in range(0, len(first), 50_000) + ) + ) + self.assert_matches_truth(many, "inflate") + + def test_optional_gzip_header_fields_are_skipped(self): + """FEXTRA, FNAME, and FCOMMENT headers do not confuse the sampler.""" + with tempfile.TemporaryDirectory() as td: + payload = vcf_body(2_000) + deflate_and_trailer = gzip.compress(payload)[10:] + extra = struct.pack("<2sH", b"ZZ", 3) + b"abc" + header = ( + bytes([31, 139, 8, 0x04 | 0x08 | 0x10, 0, 0, 0, 0, 0, 255]) + + struct.pack("&2; exit 1"], + ) + self.assertEqual(result["exit_code"], 1) + self.assertIn("No space left on device", result["stderr_tail"]) + self.assertFalse((work_dir / ".cottas-merge-r01-00000.stderr").exists()) + + def test_stage_runner_emits_compact_progress_events(self): + """Stage progress reuses one task for chunk builds.""" + runner_module = load_runner_module() + with tempfile.TemporaryDirectory() as td: + work_dir = Path(td) / "work" + work_dir.mkdir() + progress_path = work_dir / ".progress" / "partitioned.jsonl" + stage_runner = runner_module.StageRunner(work_dir, progress_path=progress_path) + result = stage_runner.run( + "hdt-build-00000", + ["sh", "-c", "exit 0"], + ) + events = [json.loads(line) for line in progress_path.read_text().splitlines()] + self.assertEqual(result["exit_code"], 0) + self.assertEqual(events[0]["stage"], "hdt-chunks") + self.assertEqual(events[0]["phase"], "started") + self.assertEqual(events[-1]["phase"], "complete") + self.assertEqual(events[-1]["completed"], 1) + self.assertEqual(events[-1]["unit"], "chunks") + + def test_failure_message_includes_a_bounded_stderr_tail(self): + """A code-1 COTTAS failure reports DuckDB's actual error to the user.""" + runner = load_runner_module() + message = runner.failure_message( + { + "exit_code": 1, + "stderr_tail": "Traceback\n" + "RuntimeError: disk-backed COTTAS merge failed: " + "IO Error: No space left on device", + }, + "COTTAS merge/index creation failed", + ) + self.assertIn("exit_code=1", message) + self.assertIn("No space left on device", message) + + def test_stream_chunks_only_retains_the_chunk_being_consumed(self): + """Gzip chunking does not stage a second full uncompressed aggregate.""" + runner = load_runner_module() + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "source.nt.gz" + source_bytes = b"".join( + f"

.\n".encode("utf-8") + for index in range(12) + ) + with gzip.open(source, "wb") as handle: + handle.write(source_bytes) + + chunk_dir = tmp_path / "chunks" + progress_path = tmp_path / ".progress" / "partitioned.jsonl" + stream, plan = runner.stream_chunks( + source, + chunk_dir, + target_bytes=40, + min_bytes=20, + max_bytes=60, + progress_path=progress_path, + progress_total=12, + ) + emitted = [] + for chunk, metadata in stream: + self.assertEqual(list(chunk_dir.glob("*.nt")), [chunk]) + self.assertTrue(chunk.read_bytes().endswith(b"\n")) + self.assertEqual(metadata["payload_bytes"], chunk.stat().st_size) + emitted.append(chunk.read_bytes()) + chunk.unlink() + + self.assertEqual(b"".join(emitted), source_bytes) + self.assertEqual(plan["record_count"], 12) + self.assertEqual(plan["chunk_count"], len(plan["chunks"])) + self.assertGreater(plan["chunk_count"], 1) + self.assertEqual(list(chunk_dir.glob("*.nt")), []) + progress_events = [ + json.loads(line) for line in progress_path.read_text().splitlines() + ] + self.assertTrue(all(event["stage"] == "rdf-scan" for event in progress_events)) + self.assertEqual(progress_events[-1]["completed"], 12) + self.assertEqual(progress_events[-1]["total"], 12) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_rules_helper_unit.py b/test/test_rules_helper_unit.py new file mode 100644 index 0000000..2f7dce9 --- /dev/null +++ b/test/test_rules_helper_unit.py @@ -0,0 +1,241 @@ +import csv +import json +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO +from pathlib import Path + +import vcf_rdfizer +import vcf_rdfizer_rules +from test.helpers import VerboseTestCase + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_RULES = REPO_ROOT / "rules" / "default_rules.ttl" +VCF_AS_TSV = REPO_ROOT / "src" / "vcf_as_tsv.sh" + +SAMPLE_VCF = """\ +##fileformat=VCFv4.2 +##fileDate=20260101 +##source=unittest +##reference=GRCh38 +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHG001\tHG002 +20\t100\trs1\tA\tG\t50\tPASS\tAC=1\tGT\t0|1\t1/1 +""" + + +def invoke(argv): + """Run the rules CLI, returning (exit_code, stdout, stderr).""" + out, err = StringIO(), StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = vcf_rdfizer_rules.main(argv) + return code, out.getvalue(), err.getvalue() + + +class RulesHelperUnitTests(VerboseTestCase): + def test_documented_columns_match_what_vcf_as_tsv_actually_writes(self): + """The documented TSV schemas cannot drift from the generator script.""" + if sys.platform.startswith("win"): + self.skipTest("vcf_as_tsv.sh requires a POSIX shell") + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text(SAMPLE_VCF, encoding="utf-8") + out_dir = tmp_path / "tsv" + result = subprocess.run( + ["bash", str(VCF_AS_TSV), str(vcf_path), str(out_dir)], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + emitted = {} + for name in ("records", "header_lines", "file_metadata"): + path = out_dir / f"sample.{name}.tsv" + with path.open(newline="", encoding="utf-8") as handle: + emitted[name] = next(csv.reader(handle, delimiter="\t")) + + # The records header's final column is the whitespace-joined sample + # ids, which is exactly why it is modelled as a dynamic tail. + records = vcf_rdfizer_rules.TSV_SOURCES["records"] + self.assertEqual(emitted["records"][:-1], list(records.columns)) + self.assertEqual(emitted["records"][-1], "HG001 HG002") + + for name in ("header_lines", "file_metadata"): + self.assertEqual( + emitted[name], + list(vcf_rdfizer_rules.TSV_SOURCES[name].columns), + f"{name}.tsv header drifted from the documented columns", + ) + + def test_helper_table_columns_come_from_the_wrapper_constants(self): + """The two helper tables reuse the wrapper's own header definitions.""" + self.assertEqual( + list(vcf_rdfizer_rules.TSV_SOURCES["sample_calls"].columns), + list(vcf_rdfizer.SAMPLE_CALLS_HEADER), + ) + self.assertEqual( + list(vcf_rdfizer_rules.TSV_SOURCES["sample_format_values"].columns), + list(vcf_rdfizer.SAMPLE_FORMAT_HEADER), + ) + + def test_documented_source_paths_match_the_wrapper_rewrite_targets(self): + """Every advertised source path is one render_rules_for_triplet rewrites.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + rendered = tmp_path / "rendered.ttl" + vcf_rdfizer.render_rules_for_triplet( + DEFAULT_RULES, + rendered, + "s.records.tsv", + "s.header_lines.tsv", + "s.file_metadata.tsv", + "s.sample_calls.tsv", + "s.sample_format_values.tsv", + ) + text = rendered.read_text(encoding="utf-8") + for path in vcf_rdfizer_rules.CANONICAL_SOURCE_PATHS: + self.assertNotIn( + f'csvw:url "{path}"', + text, + f"{path} survived rendering, so the wrapper does not rewrite it", + ) + + def test_check_accepts_the_shipped_default_mapping(self): + """The default mapping satisfies the contract the checker enforces.""" + code, stdout, _ = invoke(["check", str(DEFAULT_RULES)]) + self.assertEqual(code, 0) + self.assertIn("Result: OK", stdout) + self.assertIn("--sample-representation condensed: usable", stdout) + + def test_check_rejects_non_canonical_source_paths(self): + """A hard-coded TSV path the wrapper cannot rewrite is an error.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8").replace( + "/data/tsv/records.tsv", "/data/tsv/my_records.tsv" + ), + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("Result: FAILED", stdout) + self.assertIn("/data/tsv/my_records.tsv", stdout) + + def test_check_rejects_columns_no_tsv_provides(self): + """A misspelled column reference is caught before the pipeline runs.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8").replace( + 'rml:reference "CHROM"', 'rml:reference "CHROMOSOME"' + ), + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("CHROMOSOME", stdout) + + def test_check_reports_template_variables_as_referenced_columns(self): + """Columns used only inside rr:template are checked too.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8").replace( + "file://{SOURCE_FILE}#call/{ROW_ID}", + "file://{SOURCE_FILE}#call/{NO_SUCH_COLUMN}", + ), + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("NO_SUCH_COLUMN", stdout) + + def test_check_warns_about_materialized_helper_tables(self): + """A custom helper-table consumer is flagged, and blocks condensed mode.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "custom.ttl" + rules.write_text( + DEFAULT_RULES.read_text(encoding="utf-8") + + """ +<#ExtraSampleMap> a rr:TriplesMap; + rml:logicalSource [ + rml:source [ a csvw:Table; csvw:url "/data/tsv/sample_calls.tsv" ]; + rml:referenceFormulation ql:CSV + ]; + rr:subjectMap [ rr:template "file://{SOURCE_FILE}#extra/{ROW_ID}" ]; + rr:predicateObjectMap [ + rr:predicate vcfr:rawPayload; + rr:objectMap [ rml:reference "SAMPLE_PAYLOAD" ] + ]. +""", + encoding="utf-8", + ) + code, stdout, _ = invoke(["check", str(rules)]) + # Materialization is a cost, not a contract violation. + self.assertEqual(code, 0) + self.assertIn("materialized", stdout) + self.assertIn("--sample-representation condensed: NOT usable", stdout) + + def test_check_rejects_a_file_that_is_not_an_rml_mapping(self): + """A Turtle file with no logical source fails rather than silently passing.""" + with tempfile.TemporaryDirectory() as td: + rules = Path(td) / "plain.ttl" + rules.write_text( + "@prefix ex: .\nex:a ex:b ex:c .\n", encoding="utf-8" + ) + code, stdout, _ = invoke(["check", str(rules)]) + self.assertEqual(code, 1) + self.assertIn("does not look like an RML mapping", stdout) + + def test_check_emits_machine_readable_json(self): + """--json exposes the same findings for scripted use.""" + code, stdout, _ = invoke(["check", str(DEFAULT_RULES), "--json"]) + payload = json.loads(stdout) + self.assertEqual(code, 0) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["errors"], []) + self.assertEqual( + sorted(payload["sources"]), sorted(vcf_rdfizer_rules.CANONICAL_SOURCE_PATHS) + ) + + def test_init_scaffolds_a_checkable_mapping_and_refuses_to_clobber(self): + """init produces a file that passes check, and never overwrites silently.""" + with tempfile.TemporaryDirectory() as td: + output = Path(td) / "mine.ttl" + code, _, _ = invoke(["init", "-o", str(output)]) + self.assertEqual(code, 0) + self.assertTrue(output.is_file()) + self.assertIn("Custom VCF-RDFizer RML mapping", output.read_text(encoding="utf-8")) + + check_code, stdout, _ = invoke(["check", str(output)]) + self.assertEqual(check_code, 0, stdout) + + clobber_code, _, stderr = invoke(["init", "-o", str(output)]) + self.assertEqual(clobber_code, 2) + self.assertIn("already exists", stderr) + + force_code, _, _ = invoke(["init", "-o", str(output), "--force"]) + self.assertEqual(force_code, 0) + + def test_columns_lists_every_source_and_supports_json(self): + """columns documents all five sources the pipeline generates.""" + code, stdout, _ = invoke(["columns"]) + self.assertEqual(code, 0) + for name, source in vcf_rdfizer_rules.TSV_SOURCES.items(): + self.assertIn(name, stdout) + self.assertIn(source.container_path, stdout) + + code, stdout, _ = invoke(["columns", "--json"]) + payload = json.loads(stdout) + self.assertEqual(code, 0) + self.assertEqual(sorted(payload), sorted(vcf_rdfizer_rules.TSV_SOURCES)) + self.assertEqual(payload["records"]["dynamic_tail_column"], "SAMPLES") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_run_conversion_unit.py b/test/test_run_conversion_unit.py index 79b1ab3..d02a90a 100644 --- a/test/test_run_conversion_unit.py +++ b/test/test_run_conversion_unit.py @@ -60,6 +60,7 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): "LOGDIR": str(metrics_dir), "RUN_ID": "run123", "TIMESTAMP": "2026-01-01T00:00:00", + "PROGRESS_FILE": str(metrics_dir / ".progress" / "rml.jsonl"), } ) @@ -69,8 +70,8 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): merged_nt = out_dir / "rdf" / "rdf.nt" self.assertTrue(merged_nt.exists()) self.assertIn("

.", merged_nt.read_text()) - self.assertTrue((metrics_dir / "conversion_time" / "rdf" / "run123.txt").exists()) - self.assertTrue((metrics_dir / "conversion_metrics" / "rdf" / "run123.json").exists()) + self.assertTrue((metrics_dir / "timings" / "conversion" / "rdf.txt").exists()) + self.assertTrue((metrics_dir / "stages" / "conversion" / "rdf.json").exists()) metrics_csv = metrics_dir / "metrics.csv" self.assertTrue(metrics_csv.exists()) @@ -83,6 +84,15 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): self.assertEqual(row["output_name"], "rdf") self.assertEqual(row["exit_code_java"], "0") + progress_path = metrics_dir / ".progress" / "rml.jsonl" + progress_events = [ + json.loads(line) for line in progress_path.read_text().splitlines() + ] + self.assertEqual(progress_events[0]["stage"], "rmlstreamer") + self.assertEqual(progress_events[0]["phase"], "started") + self.assertEqual(progress_events[-1]["stage"], "rdf-aggregate") + self.assertEqual(progress_events[-1]["phase"], "complete") + def test_run_conversion_preserves_unrelated_files_in_existing_output_directory(self): """RMLStreamer parts do not require deleting the sample output directory.""" with tempfile.TemporaryDirectory() as td: @@ -592,7 +602,7 @@ def test_run_conversion_space_optimized_streams_gzip_members_and_deletes_parts(s self.assertIn("

.", aggregate_text) self.assertIn("

.", aggregate_text) self.assertEqual(list((out_dir / "rdf").glob("part-*.nt")), []) - metrics = metrics_dir / "conversion_metrics" / "rdf" / "run-space.json" + metrics = metrics_dir / "stages" / "conversion" / "rdf.json" payload = json.loads(metrics.read_text()) self.assertEqual(payload["rdf_storage"]["mode"], "space-optimized") self.assertTrue(payload["rdf_storage"]["compressed"]) diff --git a/test/test_update_rules_unit.py b/test/test_update_rules_unit.py deleted file mode 100644 index 6dc741c..0000000 --- a/test/test_update_rules_unit.py +++ /dev/null @@ -1,81 +0,0 @@ -import subprocess -import tempfile -import unittest -from pathlib import Path - -from test.helpers import VerboseTestCase - - -REPO_ROOT = Path(__file__).resolve().parents[1] -SCRIPT = REPO_ROOT / "src" / "update_rules.sh" - - -class UpdateRulesUnitTests(VerboseTestCase): - def test_update_rules_errors_with_wrong_argument_count(self): - """No argument: script exits non-zero and prints usage.""" - with tempfile.TemporaryDirectory() as td: - result = subprocess.run(["bash", str(SCRIPT)], cwd=td, capture_output=True, text=True) - self.assertNotEqual(result.returncode, 0) - self.assertIn("Usage:", result.stderr) - - def test_update_rules_errors_when_rules_file_missing(self): - """Missing default rules file: script exits non-zero with a clear error.""" - with tempfile.TemporaryDirectory() as td: - result = subprocess.run( - ["bash", str(SCRIPT), "/data/tsv/records.tsv"], - cwd=td, - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("rules/default_rules.ttl not found", result.stderr) - - def test_update_rules_updates_csvw_url_and_creates_backup(self): - """Valid input updates records/header/metadata csvw:url paths and writes backup.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - rules_dir = tmp_path / "rules" - rules_dir.mkdir() - rules = rules_dir / "default_rules.ttl" - rules.write_text( - 'ex:a csvw:url "/data/tsv/records.tsv";\n' - 'ex:b csvw:url "/data/tsv/header_lines.tsv";\n' - 'ex:c csvw:url "/data/tsv/file_metadata.tsv";\n' - ) - - result = subprocess.run( - ["bash", str(SCRIPT), "/tmp/new_records.tsv", "/tmp/new_headers.tsv", "/tmp/new_metadata.tsv"], - cwd=tmp_path, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - content = rules.read_text() - self.assertIn('csvw:url "/tmp/new_records.tsv";', content) - self.assertIn('csvw:url "/tmp/new_headers.tsv";', content) - self.assertIn('csvw:url "/tmp/new_metadata.tsv";', content) - self.assertTrue((rules_dir / "default_rules.ttl.bak").exists()) - - def test_update_rules_escapes_ampersand_in_filename(self): - """Ampersands in replacement paths are preserved literally.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - rules_dir = tmp_path / "rules" - rules_dir.mkdir() - rules = rules_dir / "default_rules.ttl" - rules.write_text('ex:a csvw:url "/data/tsv/records.tsv";\n') - - result = subprocess.run( - ["bash", str(SCRIPT), "/tmp/A&B.tsv"], - cwd=tmp_path, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertIn('csvw:url "/tmp/A&B.tsv";', rules.read_text()) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_validation_benchmark_unit.py b/test/test_validation_benchmark_unit.py new file mode 100644 index 0000000..47fe899 --- /dev/null +++ b/test/test_validation_benchmark_unit.py @@ -0,0 +1,331 @@ +"""Multi-engine selection, native artifact querying, and the benchmark report. + +One validation run may answer the whole query set under several engines. These +tests cover the three things that makes possible: parsing the engine list, the +two native-artifact engines that query HDT and COTTAS without decoding them, +and the benchmark that turns the resulting timings into a comparison. +""" + +import csv +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import vcf_rdfizer +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_benchmark", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +class EngineListTests(VerboseTestCase): + def test_a_single_engine_still_parses_to_a_one_element_list(self): + """The single-engine case is the multi-engine case with one entry.""" + self.assertEqual(V.parse_engine_list("comunica"), ["comunica"]) + self.assertEqual(vcf_rdfizer.parse_validation_engines("comunica"), ["comunica"]) + + def test_several_engines_keep_the_order_they_were_requested_in(self): + """The first engine is the primary, so order is not incidental.""" + self.assertEqual( + V.parse_engine_list("qlever,comunica"), ["qlever", "comunica"] + ) + self.assertEqual( + vcf_rdfizer.parse_validation_engines("qlever,comunica"), + ["qlever", "comunica"], + ) + + def test_repeats_and_surrounding_whitespace_are_absorbed(self): + """'comunica, qlever, comunica' asks for two engines, not three.""" + self.assertEqual( + V.parse_engine_list(" comunica , qlever , comunica "), + ["comunica", "qlever"], + ) + self.assertEqual( + vcf_rdfizer.parse_validation_engines(" comunica , qlever , comunica "), + ["comunica", "qlever"], + ) + + def test_all_expands_to_every_supported_engine(self): + """The benchmarking shorthand must cover the full set, in both layers.""" + self.assertEqual(V.parse_engine_list("all"), list(V.SPARQL_ENGINES)) + self.assertEqual( + vcf_rdfizer.parse_validation_engines("all"), + list(vcf_rdfizer.VALIDATION_ENGINE_CHOICES), + ) + + def test_the_host_and_container_agree_on_the_engine_set(self): + """A wrapper that offers an engine the runner rejects is unusable.""" + self.assertEqual( + tuple(vcf_rdfizer.VALIDATION_ENGINE_CHOICES), tuple(V.SPARQL_ENGINES) + ) + self.assertEqual(set(V.ENGINE_CLASSES), set(V.SPARQL_ENGINES)) + + def test_an_unknown_or_empty_engine_is_rejected_by_both_layers(self): + """Failing at parse time beats failing after materialization.""" + for raw in ("virtuoso", "comunica,virtuoso", "", " ", ","): + with self.assertRaises(ValueError, msg=raw): + V.parse_engine_list(raw) + with self.assertRaises(ValueError, msg=raw): + vcf_rdfizer.parse_validation_engines(raw) + + def test_the_cli_resolves_engines_and_keeps_engine_as_the_primary(self): + """Existing consumers read args.engine; it stays the first requested.""" + parser = V.build_arg_parser() + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + (tmp_path / "in.vcf").write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + (tmp_path / "in.nt").write_text("", encoding="utf-8") + args = V.resolve_args(parser, [ + "--vcf", str(tmp_path / "in.vcf"), + "--rdf", str(tmp_path / "in.nt"), + "--results-dir", str(tmp_path / "out"), + "--representation", "expanded", + "--dataset-id", "fixture", + "--scratch-dir", str(tmp_path), + "--engine", "qlever,cottas", + ]) + self.assertEqual(args.engines, ["qlever", "cottas"]) + self.assertEqual(args.engine, "qlever") + + +class NativeArtifactEngineTests(VerboseTestCase): + """HDT and COTTAS are queried in place, not decoded to N-Triples first.""" + + def _engine(self, name, tmp_path, options=None): + return V.build_engine( + name, + tmp_path / "cohort.nt", + raw_dir=tmp_path / "raw", + scratch=tmp_path / "scratch", + options=options or {}, + ) + + def test_the_native_engines_declare_no_decode_in_their_description(self): + """The mode string is what a reader of the report sees; it must be true.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + for name, artifact_format in (("hdt", "hdt"), ("cottas", "cottas")): + engine = self._engine(name, tmp_path) + self.assertIsInstance(engine, V.NativeArtifactEngine) + self.assertEqual(engine.artifact_format, artifact_format) + self.assertIn("no decode", engine.describe()["mode"]) + + def test_a_matching_run_artifact_is_queried_directly(self): + """Building a second copy would measure the build, not the artifact.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + supplied = tmp_path / "cohort.hdt" + supplied.write_bytes(b"fake-hdt") + engine = self._engine("hdt", tmp_path, options={ + "artifact_path": supplied, "artifact_format": "hdt", + }) + with mock.patch.object(V.HdtEngine, "build_artifact") as build: + resolved = engine._resolve_artifact() + build.assert_not_called() + self.assertEqual(resolved, supplied) + self.assertEqual(engine.artifact_origin, "run artifact") + + def test_a_mismatched_run_artifact_makes_the_engine_build_its_own(self): + """Validating a .nt aggregate under --engine hdt still needs an HDT.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + engine = self._engine("hdt", tmp_path, options={ + "artifact_path": tmp_path / "cohort.cottas", "artifact_format": "cottas", + }) + with mock.patch.object(V.HdtEngine, "build_artifact") as build: + resolved = engine._resolve_artifact() + build.assert_called_once() + self.assertEqual(resolved.suffix, ".hdt") + self.assertEqual(resolved.parent, tmp_path / "scratch") + self.assertEqual(engine.artifact_origin, "built from N-Triples for this engine") + + def test_hdt_queries_are_addressed_with_comunicas_typed_source_prefix(self): + """A bare path is treated as a link to dereference and fails. + + Verified against comunica-sparql-hdt 5.0.1: '/tmp/g.hdt' reports + "Could not dereference", while 'hdt@/tmp/g.hdt' answers the query. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + raw_dir = tmp_path / "raw" + raw_dir.mkdir() + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }", encoding="utf-8") + engine = self._engine("hdt", tmp_path) + engine.executable = "/usr/bin/comunica-sparql-hdt" + engine.artifact = tmp_path / "cohort.hdt" + recorded = {} + + def fake_run(command, **kwargs): + recorded["command"] = command + return mock.Mock(returncode=0) + + with mock.patch.object(V.subprocess, "run", side_effect=fake_run): + envelope = engine.execute("q01", query) + + self.assertEqual(recorded["command"][1], f"hdt@{tmp_path / 'cohort.hdt'}") + self.assertIn("application/sparql-results+json", recorded["command"]) + self.assertEqual(envelope["status"], "PASS") + self.assertEqual(envelope["engine"], "hdt") + + def test_hdt_without_the_native_binary_explains_the_decode_alternative(self): + """An unusable engine must say what the user can do instead.""" + with tempfile.TemporaryDirectory() as td: + engine = self._engine("hdt", Path(td)) + with mock.patch.object(V.shutil, "which", return_value=None): + with self.assertRaises(RuntimeError) as raised: + engine.start() + self.assertIn("--engine comunica", str(raised.exception)) + + def test_a_failing_cottas_query_is_reported_not_raised(self): + """One bad query must not abort the other engines in a benchmark run.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + raw_dir = tmp_path / "raw" + raw_dir.mkdir() + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }", encoding="utf-8") + engine = self._engine("cottas", tmp_path) + engine.graph = mock.Mock() + engine.graph.query.side_effect = RuntimeError("parquet is unreadable") + envelope = engine.execute("q01", query) + + self.assertEqual(envelope["status"], "EXECUTION_FAILED") + self.assertIn("parquet is unreadable", envelope["error"]) + self.assertEqual(json.loads(Path(envelope["rawResult"]).read_text()), {}) + self.assertIn( + "parquet is unreadable", + Path(envelope["stderr"]).read_text(encoding="utf-8"), + ) + + def test_an_unknown_engine_name_names_the_supported_ones(self): + with tempfile.TemporaryDirectory() as td: + with self.assertRaises(ValueError) as raised: + self._engine("virtuoso", Path(td)) + for name in V.SPARQL_ENGINES: + self.assertIn(name, str(raised.exception)) + + +def _verdict(status, wall_seconds, sparql=None): + return { + "status": status, + "executions": { + query_id: {"status": "PASS", "wallSeconds": seconds} + for query_id, seconds in wall_seconds.items() + }, + "sparql": sparql if sparql is not None else {"q01": [{"n": 1}]}, + } + + +class BenchmarkReportTests(VerboseTestCase): + def _benchmark(self): + return V.build_benchmark( + { + "comunica": _verdict("PASS", {"q01": 1.0, "q02": 3.0}), + "qlever": _verdict("PASS", {"q01": 0.01, "q02": 0.02}), + }, + { + "comunica": {"setupSeconds": 0.001, "artifactOrigin": None, + "artifactSizeBytes": None}, + "qlever": {"setupSeconds": 2.5, "artifactOrigin": None, + "artifactSizeBytes": None}, + }, + oracle_seconds={"total": 0.4, "parse": 0.3, "census": 0.1}, + materialization_seconds=0.7, + shacl_seconds=None, + query_ids=("q01", "q02"), + ) + + def test_query_time_is_summed_per_engine_and_the_slowest_named(self): + """The point of the report is comparing engines, so both must be right.""" + benchmark = self._benchmark() + self.assertAlmostEqual(benchmark["engines"]["comunica"]["querySeconds"], 4.0) + self.assertAlmostEqual(benchmark["engines"]["qlever"]["querySeconds"], 0.03) + self.assertEqual(benchmark["engines"]["comunica"]["slowestQuery"], "q02") + + def test_the_oracle_is_timed_so_sparql_can_be_compared_against_it(self): + """The comparison the user asked for is engine versus parser.""" + benchmark = self._benchmark() + self.assertEqual(benchmark["oracle"]["totalSeconds"], 0.4) + self.assertEqual(benchmark["oracle"]["vcfParseSeconds"], 0.3) + self.assertEqual(benchmark["oracle"]["censusSeconds"], 0.1) + self.assertEqual(benchmark["totals"]["oracleSeconds"], 0.4) + self.assertEqual(benchmark["preparation"]["materializationSeconds"], 0.7) + + def test_an_engine_with_no_timed_query_reports_none_not_zero(self): + """Zero seconds would read as 'instant' rather than 'never ran'.""" + benchmark = V.build_benchmark( + {"hdt": {"status": "EXECUTION_FAILED", "executions": {}, "sparql": {}}}, + {"hdt": {"setupSeconds": 0.1}}, + oracle_seconds=None, + materialization_seconds=None, + shacl_seconds=None, + query_ids=("q01",), + ) + self.assertIsNone(benchmark["engines"]["hdt"]["querySeconds"]) + self.assertIsNone(benchmark["engines"]["hdt"]["slowestQuery"]) + self.assertIsNone(benchmark["oracle"]["totalSeconds"]) + + def test_the_csv_carries_one_row_per_engine_and_query(self): + """A long-format CSV is directly usable in a plot without reshaping.""" + benchmark = self._benchmark() + with tempfile.TemporaryDirectory() as td: + path = V.write_benchmark_csv(Path(td) / "benchmark.csv", benchmark) + rows = list(csv.DictReader(path.open(encoding="utf-8"))) + self.assertEqual(len(rows), 4) + self.assertEqual(list(rows[0]), V.BENCHMARK_CSV_HEADER) + self.assertEqual( + {(row["engine"], row["query_id"]) for row in rows}, + {("comunica", "q01"), ("comunica", "q02"), + ("qlever", "q01"), ("qlever", "q02")}, + ) + # Repeated on every row so the file needs no join to be usable. + self.assertEqual({row["oracle_wall_seconds"] for row in rows}, {"0.4"}) + + +class EngineAgreementTests(VerboseTestCase): + def test_engines_returning_the_same_rows_agree(self): + result = V.compare_engines({ + "comunica": _verdict("PASS", {"q01": 1.0}, sparql={"q01": [{"n": 2}]}), + "qlever": _verdict("PASS", {"q01": 0.1}, sparql={"q01": [{"n": 2}]}), + }) + self.assertTrue(result["agree"]) + self.assertEqual(result["differences"], {}) + self.assertEqual(result["comparedEngines"], ["comunica", "qlever"]) + + def test_a_disagreement_names_the_engine_and_the_queries(self): + """This is how QLever's literal canonicalisation would have been caught.""" + result = V.compare_engines({ + "comunica": _verdict("PASS", {"q01": 1.0}, sparql={"q01": [{"n": 2}], "q02": []}), + "qlever": _verdict("PASS", {"q01": 0.1}, sparql={"q01": [{"n": 3}], "q02": []}), + }) + self.assertFalse(result["agree"]) + self.assertEqual(result["differences"], {"qlever": ["q01"]}) + self.assertEqual(result["referenceEngine"], "comunica") + + def test_an_engine_that_failed_to_execute_is_left_out_of_the_comparison(self): + """Comparing against an empty result set would be a false disagreement.""" + result = V.compare_engines({ + "comunica": _verdict("PASS", {"q01": 1.0}, sparql={"q01": [{"n": 2}]}), + "hdt": {"status": "EXECUTION_FAILED", "executions": {}, "sparql": {}}, + }) + self.assertEqual(result["comparedEngines"], ["comunica"]) + self.assertTrue(result["agree"]) + self.assertIn("fewer than two", result["note"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validation_engines_unit.py b/test/test_validation_engines_unit.py new file mode 100644 index 0000000..45afa60 --- /dev/null +++ b/test/test_validation_engines_unit.py @@ -0,0 +1,534 @@ +"""Engine selection and artifact materialization for semantic validation. + +The container tools (hdt2rdf, pycottas, comunica, QLever) are replaced with +fakes, so these tests verify the command lines and control flow the validator +builds rather than the third-party tools themselves. +""" + +import gzip +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import vcf_rdfizer +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_engines", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() +TRIPLES = b"

.\n

.\n" + + +class ArtifactFormatTests(VerboseTestCase): + def test_every_pipeline_artifact_suffix_is_recognised(self): + """Format detection covers each artifact the pipeline can produce.""" + expected = { + "cohort.nt": "nt", + "cohort.nt.gz": "nt.gz", + "cohort.nt.br": "nt.br", + "cohort.hdt": "hdt", + "cohort.cottas": "cottas", + "cohort.cottas.gz": "cottas.gz", + "cohort.cottas.br": "cottas.br", + } + for name, fmt in expected.items(): + self.assertEqual(V.detect_rdf_format(Path(name)), fmt, name) + self.assertIsNone(V.detect_rdf_format(Path("cohort.ttl"))) + + def test_wrapper_and_runner_agree_on_formats(self): + """The host and container must not disagree about what is supported.""" + wrapper = {fmt for _suffix, fmt in vcf_rdfizer.VALIDATION_RDF_SUFFIXES} + self.assertEqual(wrapper, set(V.RDF_FORMATS)) + for name in ("a.nt", "a.nt.gz", "a.nt.br", "a.hdt", "a.cottas", "a.cottas.gz", "a.cottas.br"): + self.assertEqual( + vcf_rdfizer.detect_validation_rdf_format(Path(name)), + V.detect_rdf_format(Path(name)), + name, + ) + + def test_plain_ntriples_is_read_in_place(self): + """A .nt source is never copied into scratch.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + scratch = tmp_path / "scratch" + scratch.mkdir() + decoded, provenance = V.materialize_ntriples( + source, "nt", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(decoded, source) + self.assertFalse(provenance["materialized"]) + self.assertEqual(list(scratch.iterdir()), []) + + def test_gzip_is_expanded_into_scratch(self): + """A .nt.gz source is decoded to scratch and reports its provenance.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt.gz" + with gzip.open(source, "wb") as handle: + handle.write(TRIPLES) + scratch = tmp_path / "scratch" + scratch.mkdir() + decoded, provenance = V.materialize_ntriples( + source, "nt.gz", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(decoded.read_bytes(), TRIPLES) + self.assertTrue(provenance["materialized"]) + self.assertEqual(provenance["sourceFormat"], "nt.gz") + + def test_hdt_is_decoded_with_hdt2rdf(self): + """An .hdt source is decoded by the container's hdt2rdf binary.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.hdt" + source.write_bytes(b"fake-hdt") + scratch = tmp_path / "scratch" + scratch.mkdir() + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + Path(command[2]).write_bytes(TRIPLES) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(V, "_resolve_binary", return_value="/usr/local/bin/hdt2rdf"), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + decoded, provenance = V.materialize_ntriples( + source, "hdt", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(calls[0][0], "/usr/local/bin/hdt2rdf") + self.assertEqual(calls[0][1], str(source)) + self.assertEqual(decoded.read_bytes(), TRIPLES) + self.assertTrue(provenance["materialized"]) + + def test_cottas_is_decoded_with_the_cottas_tool(self): + """A .cottas source is decoded through cottas_tool.py decompress.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.cottas" + source.write_bytes(b"fake-cottas") + scratch = tmp_path / "scratch" + scratch.mkdir() + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + Path(command[-1]).write_bytes(TRIPLES) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.dict(os.environ, {"COTTAS_PYTHON_BIN": "/opt/py/bin/python"}), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + decoded, _ = V.materialize_ntriples( + source, "cottas", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(calls[0][0], "/opt/py/bin/python") + self.assertIn("decompress", calls[0]) + self.assertEqual(decoded.read_bytes(), TRIPLES) + + def test_packaged_cottas_is_unwrapped_then_decoded(self): + """A .cottas.gz is gunzipped before pycottas, and the unwrap is cleaned up.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.cottas.gz" + with gzip.open(source, "wb") as handle: + handle.write(b"fake-cottas") + scratch = tmp_path / "scratch" + scratch.mkdir() + + def fake_run(command, **kwargs): + self.assertTrue(Path(command[-2]).is_file(), "unwrapped input must exist") + Path(command[-1]).write_bytes(TRIPLES) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(V.subprocess, "run", side_effect=fake_run): + decoded, provenance = V.materialize_ntriples( + source, "cottas.gz", scratch, log_dir=tmp_path / "log" + ) + self.assertEqual(decoded.read_bytes(), TRIPLES) + self.assertFalse((scratch / "input.cottas").exists()) + self.assertEqual(len(provenance["steps"]), 2) + + def test_a_failed_decode_reports_the_tool_output(self): + """A decode failure raises with the tool's own output, not a bare code.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.hdt" + source.write_bytes(b"broken") + scratch = tmp_path / "scratch" + scratch.mkdir() + + def fake_run(command, stdout=None, **kwargs): + stdout.write(b"hdt2rdf: corrupt header\n") + return subprocess.CompletedProcess(command, 3) + + with mock.patch.object(V, "_resolve_binary", return_value="hdt2rdf"), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + with self.assertRaises(RuntimeError) as caught: + V.materialize_ntriples(source, "hdt", scratch, log_dir=tmp_path / "log") + self.assertIn("corrupt header", str(caught.exception)) + + +class EngineTests(VerboseTestCase): + def test_engine_registry_matches_the_wrapper_choices(self): + """The host CLI cannot offer an engine the container does not implement.""" + self.assertEqual( + set(vcf_rdfizer.VALIDATION_ENGINE_CHOICES), set(V.ENGINE_CLASSES) + ) + self.assertEqual(set(V.SPARQL_ENGINES), set(V.ENGINE_CLASSES)) + + def test_unknown_engine_is_rejected(self): + with tempfile.TemporaryDirectory() as td: + with self.assertRaises(ValueError): + V.build_engine( + "sparqlmagic", Path(td) / "a.nt", + raw_dir=Path(td), scratch=Path(td), options={}, + ) + + def test_comunica_engine_builds_the_expected_command(self): + """The default engine still queries the file directly.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }\n", encoding="utf-8") + engine = V.build_engine( + "comunica", source, raw_dir=tmp_path, scratch=tmp_path, options={} + ) + captured = [] + + def fake_run(command, **kwargs): + captured.append(command) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(V.shutil, "which", return_value="/usr/bin/comunica-sparql-file"), \ + mock.patch.object(V, "tool_version", return_value=None), \ + mock.patch.object(V.subprocess, "run", side_effect=fake_run): + engine.start() + result = engine.execute("q02_variant_shape_counts", query) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["engine"], "comunica") + self.assertIn(str(source), captured[0]) + self.assertIn("application/sparql-results+json", captured[0]) + + def test_comunica_reports_a_missing_binary_clearly(self): + """An image without Comunica must say so, not fail obscurely.""" + with tempfile.TemporaryDirectory() as td: + engine = V.build_engine( + "comunica", Path(td) / "a.nt", raw_dir=Path(td), scratch=Path(td), options={} + ) + with mock.patch.object(V.shutil, "which", return_value=None): + with self.assertRaises(RuntimeError) as caught: + engine.start() + self.assertIn("qlever", str(caught.exception)) + + def test_qlever_builds_an_index_then_serves_it(self): + """QLever indexes into scratch, waits for readiness, and answers over HTTP.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + query = tmp_path / "q.rq" + query.write_text("SELECT * WHERE { ?s ?p ?o }\n", encoding="utf-8") + engine = V.build_engine( + "qlever", source, raw_dir=tmp_path, scratch=tmp_path, + options={"memory_gb": 8, "port": 7031}, + ) + steps = [] + server = mock.MagicMock() + server.poll.return_value = None + + with mock.patch.object(V, "_resolve_binary", side_effect=lambda env, *names: names[0]), \ + mock.patch.object(V, "_run_step", side_effect=lambda cmd, **kw: steps.append(cmd)), \ + mock.patch.object(V.subprocess, "Popen", return_value=server), \ + mock.patch.object(V, "tool_version", return_value="QLever 1.0"), \ + mock.patch.object(V.QleverEngine, "_post", return_value=b'{"results":{"bindings":[]}}'): + engine.start() + result = engine.execute("q03_titv", query) + description = engine.describe() + engine.stop() + + # Flags verified against qlever-index/qlever-server bfd5741. + self.assertEqual(steps[0][0], f"{V.QLEVER_BIN_DIR}/qlever-index") + self.assertIn("-f", steps[0]) + self.assertIn(str(source), steps[0]) + self.assertEqual(steps[0][steps[0].index("-F") + 1], "nt") + self.assertEqual(steps[0][steps[0].index("-m") + 1], "8G") + server_argv = engine.commands["server"] + self.assertEqual(server_argv[0], f"{V.QLEVER_BIN_DIR}/qlever-server") + # qlever-server defaults to a 30s query timeout, far below what the + # aggregate queries need, so it must be set explicitly. + self.assertIn("-s", server_argv) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["engine"], "qlever") + self.assertEqual(json.loads(Path(result["rawResult"]).read_bytes())["results"]["bindings"], []) + self.assertEqual(description["port"], 7031) + self.assertIn("index", description["commands"]) + server.terminate.assert_called_once() + self.assertFalse(engine.index_dir.exists(), "index must not outlive the engine") + + def test_qlever_surfaces_a_server_that_died_during_startup(self): + """A crashed server is reported with its log tail, not a silent timeout.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.nt" + source.write_bytes(TRIPLES) + engine = V.build_engine( + "qlever", source, raw_dir=tmp_path, scratch=tmp_path, + options={"startup_timeout": 5}, + ) + server = mock.MagicMock() + server.poll.return_value = 1 + server.returncode = 1 + + def fake_popen(command, stdout=None, **kwargs): + stdout.write(b"ServerMain: index not found\n") + stdout.flush() + return server + + with mock.patch.object(V, "_resolve_binary", side_effect=lambda env, *names: names[0]), \ + mock.patch.object(V, "_run_step"), \ + mock.patch.object(V.subprocess, "Popen", side_effect=fake_popen): + with self.assertRaises(RuntimeError) as caught: + engine.start() + self.assertIn("index not found", str(caught.exception)) + + def test_qlever_command_lines_are_overridable(self): + """QLever's CLI varies by release, so both argv are user-overridable.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + engine = V.build_engine( + "qlever", tmp_path / "cohort.nt", raw_dir=tmp_path, scratch=tmp_path, + options={"memory_gb": 2, "extra_index_args": ["--parser-buffer-size", "20"]}, + ) + self.assertIn("--parser-buffer-size", engine.index_command("qlever-index")) + with mock.patch.dict( + os.environ, + {"QLEVER_INDEX_COMMAND": "qlever index -b {index} -i {input} -m {memory}"}, + ): + argv = engine.index_command("qlever-index") + self.assertEqual(argv[:3], ["qlever", "index", "-b"]) + self.assertIn(str(tmp_path / "cohort.nt"), argv) + self.assertIn("2G", argv) + + +class RunnerArgumentTests(VerboseTestCase): + def _resolve(self, argv): + return V.resolve_args(V.build_arg_parser(), argv) + + def _base_args(self, tmp_path: Path, rdf_name: str) -> list[str]: + vcf = tmp_path / "cohort.vcf" + vcf.write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + rdf = tmp_path / rdf_name + rdf.write_bytes(TRIPLES) + return [ + "--vcf", str(vcf), + "--rdf", str(rdf), + "--representation", "expanded", + "--results-dir", str(tmp_path / "results"), + "--dataset-id", "cohort", + "--scratch-dir", str(tmp_path), + ] + + def test_format_is_inferred_from_the_filename(self): + with tempfile.TemporaryDirectory() as td: + args = self._resolve(self._base_args(Path(td), "cohort.hdt")) + self.assertEqual(args.rdf_format, "hdt") + self.assertEqual(args.engine, "comunica") + + def test_explicit_format_overrides_detection(self): + with tempfile.TemporaryDirectory() as td: + argv = self._base_args(Path(td), "cohort.bin") + ["--rdf-format", "nt"] + self.assertEqual(self._resolve(argv).rdf_format, "nt") + + def test_unrecognised_extension_is_rejected_with_guidance(self): + with tempfile.TemporaryDirectory() as td: + with self.assertRaises(SystemExit): + self._resolve(self._base_args(Path(td), "cohort.bin")) + + def test_deprecated_aliases_still_work(self): + """Existing callers that pass --rdf-nt/--rdf-gz keep working.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf = tmp_path / "cohort.vcf" + vcf.write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + rdf = tmp_path / "cohort.nt.gz" + with gzip.open(rdf, "wb") as handle: + handle.write(TRIPLES) + args = self._resolve([ + "--vcf", str(vcf), + "--rdf-gz", str(rdf), + "--representation", "condensed", + "--results-dir", str(tmp_path / "results"), + "--dataset-id", "cohort", + "--scratch-dir", str(tmp_path), + ]) + self.assertEqual(args.rdf, rdf.resolve()) + self.assertEqual(args.rdf_format, "nt.gz") + + def test_engine_tuning_is_validated(self): + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + argv = self._base_args(tmp_path, "cohort.nt") + [ + "--engine", "qlever", "--qlever-memory-gb", "16", "--qlever-port", "7040", + ] + args = self._resolve(argv) + self.assertEqual((args.engine, args.qlever_memory_gb, args.qlever_port), + ("qlever", 16, 7040)) + with self.assertRaises(SystemExit): + self._resolve(self._base_args(tmp_path, "cohort.nt") + ["--qlever-port", "99999"]) + with self.assertRaises(SystemExit): + self._resolve(self._base_args(tmp_path, "cohort.nt") + ["--query-timeout", "0"]) + + +if __name__ == "__main__": + unittest.main() + + +class WrapperValidationTargetTests(VerboseTestCase): + def test_targets_resolve_only_to_artifacts_that_exist(self): + """A representation that was not produced is skipped, not failed.""" + with tempfile.TemporaryDirectory() as td: + out_dir = Path(td) / "cohort" + out_dir.mkdir() + aggregate = out_dir / "cohort.nt.gz" + with gzip.open(aggregate, "wb") as handle: + handle.write(TRIPLES) + (out_dir / "cohort.hdt").write_bytes(b"fake-hdt") + + targets = vcf_rdfizer.resolve_validation_targets( + requested=["aggregate", "hdt", "cottas"], + output_dir=out_dir, + output_name="cohort", + aggregate_path=aggregate, + selected_methods=["hdt", "cottas"], + ) + self.assertEqual([t["name"] for t in targets], ["aggregate", "hdt"]) + self.assertEqual(targets[0]["format"], "nt.gz") + self.assertEqual(targets[1]["format"], "hdt") + + def test_targets_skip_representations_that_were_not_selected(self): + """Asking for hdt without selecting it produces no target.""" + with tempfile.TemporaryDirectory() as td: + out_dir = Path(td) / "cohort" + out_dir.mkdir() + aggregate = out_dir / "cohort.nt" + aggregate.write_bytes(TRIPLES) + (out_dir / "cohort.hdt").write_bytes(b"fake-hdt") + targets = vcf_rdfizer.resolve_validation_targets( + requested=["hdt"], + output_dir=out_dir, + output_name="cohort", + aggregate_path=aggregate, + selected_methods=["gzip"], + ) + self.assertEqual(targets, []) + + def test_target_option_parsing(self): + self.assertEqual(vcf_rdfizer.parse_validation_targets("all"), ["aggregate", "hdt", "cottas"]) + self.assertEqual(vcf_rdfizer.parse_validation_targets("hdt,cottas,hdt"), ["hdt", "cottas"]) + self.assertEqual(vcf_rdfizer.parse_validation_targets("none"), []) + with self.assertRaises(ValueError): + vcf_rdfizer.parse_validation_targets("hdt,bogus") + + def test_validation_command_carries_engine_and_format(self): + """The wrapper passes the engine and artifact format to the container.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "cohort.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n", encoding="utf-8") + hdt_path = tmp_path / "cohort.hdt" + hdt_path.write_bytes(b"fake-hdt") + commands = [] + + with mock.patch.object(vcf_rdfizer, "run", side_effect=lambda cmd, **kw: commands.append(cmd) or 0): + vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=hdt_path, + representation="condensed", + validation_id="cohort", + results_dir=tmp_path / "results", + metrics_dir=tmp_path / "metrics", + run_id="RID", + timestamp="TS", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + engine="qlever", + engine_options={"qlever_memory_gb": 12, "qlever_index_args": ["--x", "1"]}, + wrapper_log_path=tmp_path / "wrapper.log", + ) + command = commands[0] + self.assertIn("--rdf-format", command) + self.assertEqual(command[command.index("--rdf-format") + 1], "hdt") + self.assertEqual(command[command.index("--engine") + 1], "qlever") + self.assertEqual(command[command.index("--qlever-memory-gb") + 1], "12") + self.assertIn("--qlever-index-arg", command) + + def test_unsupported_artifact_is_rejected_by_the_wrapper(self): + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "cohort.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n", encoding="utf-8") + bogus = tmp_path / "cohort.ttl" + bogus.write_text("", encoding="utf-8") + with self.assertRaises(ValueError): + vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=bogus, + representation="expanded", + validation_id="cohort", + results_dir=tmp_path / "results", + metrics_dir=tmp_path / "metrics", + run_id="RID", + timestamp="TS", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + wrapper_log_path=tmp_path / "wrapper.log", + ) + + +class QleverEnvironmentTests(VerboseTestCase): + def test_qlever_processes_get_their_private_library_path(self): + """QLever's copied Boost/ICU/jemalloc must not shadow the rest of the image.""" + with tempfile.TemporaryDirectory() as td: + engine = V.build_engine( + "qlever", Path(td) / "a.nt", raw_dir=Path(td), scratch=Path(td), options={} + ) + with mock.patch.dict(os.environ, {"LD_LIBRARY_PATH": "/usr/local/lib"}): + environment = engine._environment() + self.assertEqual( + environment["LD_LIBRARY_PATH"], f"{V.QLEVER_LIB_DIR}:/usr/local/lib" + ) + self.assertTrue(V.QLEVER_LIB_DIR.startswith("/opt/qlever")) + + def test_position_datatype_preflight_accepts_the_xsd_integer_family(self): + """QLever canonicalises xsd:integer to xsd:int, so both must pass. + + Verified against qlever-index/qlever-server bfd5741: without the wider + accepted set this preflight failed every run on QLever while passing on + Comunica. xsd:string and xsd:decimal are still reported as anomalies. + """ + text = (V.QUERY_ROOT / "common" / "preflight_position_datatype.rq").read_text( + encoding="utf-8" + ) + self.assertIn("NOT IN", text) + accepted_list = text.split("NOT IN", 1)[1].split("))", 1)[0] + for accepted in ("xsd:integer", "xsd:int", "xsd:long", "xsd:nonNegativeInteger"): + self.assertIn(accepted, accepted_list) + for rejected in ("xsd:string", "xsd:decimal", "xsd:double"): + self.assertNotIn(rejected, accepted_list) diff --git a/test/test_validation_logic_unit.py b/test/test_validation_logic_unit.py new file mode 100644 index 0000000..a945cec --- /dev/null +++ b/test/test_validation_logic_unit.py @@ -0,0 +1,517 @@ +"""What the semantic validator's comparison layer does and does not detect. + +These tests drive the pure normalization/comparison functions directly, so they +run on the host without cyvcf2, Comunica, or Docker. Each one mutates a +graph-derived result and asserts the outcome, which turns the validator's +coverage into something recorded rather than assumed. + +The ``test_blind_spot_*`` cases are deliberate: they document real gaps, so a +future change that closes one will fail here and prompt an update instead of +passing unnoticed. +""" + +import copy +import gzip +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +from test import validation_fixtures as fixtures +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_logic", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +def parser_fixture() -> dict: + """The canonical fixture's oracle. + + Shared with the graph-level mutation harness rather than hand-written here, + so there is exactly one description of what the fixture VCF contains. + """ + return copy.deepcopy(fixtures.parser_summary("expanded")) + + +def sparql_fixture() -> dict: + """A graph-derived result that agrees with the parser exactly.""" + source = parser_fixture() + return {key: copy.deepcopy(source[key]) for key in V.CORE_QUERIES} + + +class ValidationDetectionTests(VerboseTestCase): + def test_matching_results_pass(self): + """An agreeing graph and parser produce a PASS with no failed invariant.""" + report = V.compare(parser_fixture(), sparql_fixture()) + self.assertEqual(report["status"], "PASS") + self.assertTrue(all(q["status"] == "PASS" for q in report["queries"].values())) + + def test_detects_a_dropped_record(self): + """Losing one record breaks both the row comparison and the totals.""" + sparql = sparql_fixture() + sparql["q01_record_density_1mb"][0]["recordCount"] -= 1 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["status"], "MISMATCH") + self.assertEqual(report["queries"]["q01_record_density_1mb"]["status"], "MISMATCH") + self.assertIn( + "FAIL", + [item["status"] for item in report["invariants"]["sparql"]], + ) + + def test_detects_a_misclassified_variant_shape(self): + """Moving a record between shape classes is caught even though the total holds.""" + sparql = sparql_fixture() + rows = sparql["q02_variant_shape_counts"] + self.assertGreaterEqual(len(rows), 2, "fixture needs two shape classes") + rows[0]["recordCount"] += 1 + rows[1]["recordCount"] -= 1 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q02_variant_shape_counts"]["status"], "MISMATCH") + # The record total is unchanged, so only the distribution catches this. + totals = {item["name"]: item["status"] for item in report["invariants"]["sparql"]} + self.assertEqual(totals["q02_variant_shape_counts_total"], "PASS") + + def test_detects_a_flipped_genotype(self): + """A genotype decoded into the wrong class changes the q05 distribution.""" + sparql = sparql_fixture() + # A class the fixture does not already use, so this is a flip rather + # than a collision with an existing row. + sparql["q05_sample_genotype_counts"][0]["genotypeClass"] = "OTHER_PLOIDY" + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q05_sample_genotype_counts"]["status"], "MISMATCH") + + def test_detects_a_corrupted_filter_string(self): + """FILTER is compared by exact lexical value, not only by status class.""" + sparql = sparql_fixture() + sparql["q04_filter_distribution"][0]["filterLexical"] = "q20" + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q04_filter_distribution"]["status"], "MISMATCH") + + def test_detects_a_transition_transversion_swap(self): + """Ti/Tv is compared field-by-field, so a swap is not hidden by the sum.""" + sparql = sparql_fixture() + sparql["q03_titv"]["transitionCount"] = 1 + sparql["q03_titv"]["transversionCount"] = 2 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q03_titv"]["status"], "MISMATCH") + + def test_detects_an_allele_count_error(self): + """A wrong AC/AN pairing shows up as missing and extra q06 rows.""" + sparql = sparql_fixture() + sparql["q06_ac_an_distribution"][0]["ac"] = 2 + report = V.compare(parser_fixture(), sparql) + self.assertEqual(report["queries"]["q06_ac_an_distribution"]["status"], "MISMATCH") + + def test_detects_an_extra_spurious_record(self): + """A record the VCF does not contain appears as an extra row.""" + sparql = sparql_fixture() + sparql["q01_record_density_1mb"].append( + {"chrom": "chrUnplaced", "windowIndex": 0, "recordCount": 1} + ) + report = V.compare(parser_fixture(), sparql) + comparison = report["queries"]["q01_record_density_1mb"] + self.assertEqual(comparison["status"], "MISMATCH") + self.assertTrue(comparison["extraRows"]) + + def test_impossible_allele_counts_fail_the_bounds_invariant(self): + """AC > AN is rejected by an invariant even without a parser disagreement.""" + payload = parser_fixture() + payload["q06_ac_an_distribution"] = [{"an": 2, "ac": 5, "siteCount": 1, "af": 2.5}] + checks = V.invariant_checks(payload, parser_fixture(), check_q06_exact=False) + self.assertIn("FAIL", [item["status"] for item in checks]) + + # -- Recorded blind spots ------------------------------------------------- + + def test_permuted_records_are_caught_only_by_the_digest(self): + """The aggregates still cannot see a permutation; q11 is what catches it. + + Every other core query is a GROUP BY count, so moving a value between + two records in the same bucket leaves them all identical. Removing the + digest from the comparison must therefore restore a PASS. + """ + parser = parser_fixture() + sparql = sparql_fixture() + # A permutation changes which digests exist, but no aggregate. + sparql["q11_record_digest"] = [ + {"bucket": "00", "recordCount": parser["totalRecords"]} + ] + report = V.compare(parser, sparql) + self.assertEqual(report["status"], "MISMATCH") + self.assertEqual(report["queries"]["q11_record_digest"]["status"], "MISMATCH") + aggregates_only = { + name: result for name, result in report["queries"].items() + if name != "q11_record_digest" + } + self.assertTrue( + all(result["status"] != "MISMATCH" for result in aggregates_only.values()), + "an aggregate query should not have noticed a permutation", + ) + + def test_core_query_set_is_pinned(self): + """The query set is part of the contract; changing it changes coverage.""" + self.assertEqual( + set(V.CORE_QUERIES), + { + "q01_record_density_1mb", + "q02_variant_shape_counts", + "q03_titv", + "q04_filter_distribution", + "q05_sample_genotype_counts", + "q06_ac_an_distribution", + "q07_file_metadata", + "q08_header_line_census", + "q09_predicate_census", + "q10_class_census", + "q11_record_digest", + "q12_info_value_digest", + "q13_format_value_digest", + }, + ) + + def test_digest_covers_every_fixed_field(self): + """ID, QUAL and INFO are only covered through the record digest.""" + query = (V.QUERY_ROOT / "common" / "q11_record_digest.rq").read_text(encoding="utf-8") + for term in ("vcfr:chrom", "vcfr:pos", "vcfr:recordId", "vcfr:ref", + "vcfr:alt", "vcfr:qual", "vcfr:filter", "vcfr:infoRaw"): + self.assertIn(term, query, f"{term} missing from the record digest") + + def test_runner_and_wrapper_agree_on_template_encoding(self): + """The digest rebuilds record IRIs, so both encoders must match.""" + import vcf_rdfizer + + for value in ("plain", "a b", "x~y", "\u00e9", "cohort.vcf.gz", "1"): + self.assertEqual( + V.rml_uri_component(value), + vcf_rdfizer._rml_uri_component(value), + value, + ) + + def test_anomaly_severity_is_measured_by_a_companion_aggregate(self): + """The LIMIT 100 sample is for diagnosis; an aggregate gives exact severity.""" + for query_id in V.ANOMALY_PREFLIGHT_QUERIES: + sample = V.query_path(V.QUERY_ROOT / "expanded", query_id).read_text(encoding="utf-8") + self.assertIn("LIMIT 100", sample) + exact = V.query_path(V.QUERY_ROOT / "expanded", f"{query_id}_count") + self.assertTrue(exact.is_file(), f"missing exact-count companion for {query_id}") + body = "\n".join( + line for line in exact.read_text(encoding="utf-8").splitlines() + if not line.lstrip().startswith("#") + ) + self.assertIn("anomalyCount", body) + # An exact count must not be capped, or it measures nothing. + self.assertNotIn("LIMIT", body) + + +class NormalizationTests(VerboseTestCase): + def _write_bindings(self, tmp_path: Path, rows: list[dict]) -> Path: + path = tmp_path / "result.json" + path.write_text(json.dumps({"results": {"bindings": rows}}), encoding="utf-8") + return path + + def test_non_integer_counts_are_rejected(self): + """A count that is not an integer fails rather than being coerced.""" + with tempfile.TemporaryDirectory() as td: + path = self._write_bindings( + Path(td), + [{"variantClass": {"value": "SNV"}, "recordCount": {"value": "3.5"}}], + ) + with self.assertRaises(ValueError): + V.normalize("q02_variant_shape_counts", path) + + def test_duplicate_canonical_keys_are_rejected(self): + """Two rows for the same key would silently hide one; that is an error.""" + with tempfile.TemporaryDirectory() as td: + path = self._write_bindings( + Path(td), + [ + {"variantClass": {"value": "SNV"}, "recordCount": {"value": "1"}}, + {"variantClass": {"value": "SNV"}, "recordCount": {"value": "2"}}, + ], + ) + with self.assertRaises(ValueError): + V.normalize("q02_variant_shape_counts", path) + + def test_missing_projection_variable_is_rejected(self): + """An engine that omits a selected variable fails loudly.""" + with tempfile.TemporaryDirectory() as td: + path = self._write_bindings(Path(td), [{"variantClass": {"value": "SNV"}}]) + with self.assertRaises(ValueError): + V.normalize("q02_variant_shape_counts", path) + + def test_engine_independent_normalization(self): + """Both engines' SPARQL Results JSON normalizes to the same rows.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + comunica_style = self._write_bindings( + tmp_path, + [{"variantClass": {"type": "literal", "value": "SNV"}, + "recordCount": {"type": "literal", "value": "3"}}], + ) + rows = V.normalize("q02_variant_shape_counts", comunica_style) + qlever_style = tmp_path / "qlever.json" + qlever_style.write_text( + json.dumps( + { + "head": {"vars": ["variantClass", "recordCount"]}, + "results": { + "bindings": [ + { + "variantClass": {"type": "literal", "value": "SNV"}, + "recordCount": { + "type": "literal", + "value": "3", + "datatype": "http://www.w3.org/2001/XMLSchema#int", + }, + } + ] + }, + } + ), + encoding="utf-8", + ) + self.assertEqual(rows, V.normalize("q02_variant_shape_counts", qlever_style)) + + +if __name__ == "__main__": + unittest.main() + + +class CensusAndDigestTests(VerboseTestCase): + """The completeness and identity checks added in Phases 2-5.""" + + def test_census_rejects_an_unexpected_predicate(self): + """A predicate the VCF does not imply is an extra row, not a silent pass.""" + parser = parser_fixture() + sparql = sparql_fixture() + sparql["q09_predicate_census"].append( + {"predicate": "https://example.org/invented", "tripleCount": 1} + ) + report = V.compare(parser, sparql) + self.assertEqual(report["queries"]["q09_predicate_census"]["status"], "MISMATCH") + self.assertTrue(report["queries"]["q09_predicate_census"]["extraRows"]) + + def test_census_is_report_only_for_a_custom_mapping(self): + """A custom mapping changes the inventory by design, so it cannot fail.""" + parser = parser_fixture() + sparql = sparql_fixture() + sparql["q09_predicate_census"].append( + {"predicate": "https://example.org/invented", "tripleCount": 1} + ) + report = V.compare(parser, sparql, mapping_policy="report-only") + self.assertEqual(report["status"], "PASS") + self.assertEqual( + report["queries"]["q09_predicate_census"]["status"], + "NOT_APPLICABLE_CUSTOM_MAPPING", + ) + + def test_census_covers_qual_and_structured_info(self): + """The elements added in Phases 2, 4 and 5 are in the expected inventory.""" + predicates = {row["predicate"] for row in parser_fixture()["q09_predicate_census"]} + for term in ("qual", "hasInfoValue", "fileDate", "contigId", "filterId", "altId"): + self.assertIn(f"{V.VCFR}{term}", predicates, term) + + def test_digest_separator_prevents_field_boundary_collisions(self): + """Shifting a field boundary must not be able to forge a matching digest.""" + self.assertEqual(V.DIGEST_SEPARATOR, chr(0x1F)) + joined_left = V.DIGEST_SEPARATOR.join(["ab", "c"]) + joined_right = V.DIGEST_SEPARATOR.join(["a", "bc"]) + self.assertNotEqual(joined_left, joined_right) + + def test_header_class_map_matches_the_wrapper(self): + """The oracle and the emitter must agree on every '##' line's subclass.""" + import vcf_rdfizer + + self.assertEqual(V.HEADER_LINE_CLASSES, vcf_rdfizer.HEADER_LINE_CLASSES) + + def test_info_entry_parsers_agree(self): + """Flag entries and '=' splitting must be identical on both sides.""" + import vcf_rdfizer + + for info in ("AC=1;DB", ".", "", "A=1;B=2,3;C", "KEY=a=b"): + self.assertEqual( + V.parse_info_entries(info), vcf_rdfizer.parse_info_entries(info), info + ) + + def test_qual_is_typed_as_the_published_shape_requires(self): + """The shape demands xsd:decimal or vcfr:Null, never a plain literal.""" + import vcf_rdfizer + + self.assertIn("XMLSchema#decimal", vcf_rdfizer._qual_object("12.5")) + self.assertIn("vocab#Null", vcf_rdfizer._qual_object(".")) + # A non-numeric QUAL is preserved rather than dropped, and the SHACL + # layer reports it. + self.assertEqual(vcf_rdfizer._qual_object("bogus"), '"bogus"') + + def test_file_date_is_typed_when_its_form_allows(self): + """##fileDate has no mandated format, so typing is conditional.""" + import vcf_rdfizer + + self.assertIn("XMLSchema#date", vcf_rdfizer.file_date_object("20260101")) + self.assertIn("2026-01-01", vcf_rdfizer.file_date_object("20260101")) + self.assertEqual(vcf_rdfizer.file_date_object("Jan 2026"), '"Jan 2026"') + self.assertIsNone(vcf_rdfizer.file_date_object("")) + + +class ShaclLayerTests(VerboseTestCase): + """SHACL is an independent structural layer, and an optional one.""" + + def test_missing_pyshacl_is_an_execution_failure_not_a_conformance_failure(self): + """An absent optional dependency must never look like a bad graph.""" + import builtins + import tempfile + import unittest.mock + + real_import = builtins.__import__ + + def blocked(name, *args, **kwargs): + if name == "pyshacl": + raise ImportError("blocked for test") + return real_import(name, *args, **kwargs) + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "g.nt" + source.write_text("

.\n", encoding="utf-8") + shapes = tmp_path / "s.ttl" + shapes.write_text("", encoding="utf-8") + with unittest.mock.patch.object(builtins, "__import__", blocked): + result = V.validate_shacl(source, shapes, tmp_path) + self.assertEqual(result["status"], "EXECUTION_FAILED") + self.assertIn("pyshacl", result["error"]) + self.assertNotIn("conforms", result) + + +class HeaderSourceTests(VerboseTestCase): + """The oracle reads the VCF's own header text, not htslib's version of it.""" + + def test_the_header_block_is_read_verbatim_and_stops_at_the_first_record(self): + """Every '##' line in order, then '#CHROM', and nothing after it.""" + with tempfile.TemporaryDirectory() as td: + vcf_path = fixtures.write_vcf(Path(td) / "fixture.vcf") + lines = V.read_vcf_header_text(vcf_path).splitlines() + meta = [f"##{key}={value}" for key, value in fixtures.HEADER_LINES] + self.assertEqual(lines[:len(meta)], meta) + self.assertEqual(len(lines), len(meta) + 1) + self.assertTrue(lines[-1].startswith("#CHROM\t")) + + def test_htslibs_injected_declarations_are_not_treated_as_header_lines(self): + """cyvcf2's raw_header is normalised; the conversion's input is not. + + htslib injects '##FILTER=' + into the header it exposes even when the file never declared it. Taking + the oracle's header from there makes it expect a FilterDefinition the + graph could not contain, and every header check fails. Reading the file + keeps the oracle and the conversion looking at the same bytes. + """ + injected = '##FILTER=' + with tempfile.TemporaryDirectory() as td: + vcf_path = fixtures.write_vcf(Path(td) / "fixture.vcf") + file_lines = { + line for line in V.read_vcf_header_text(vcf_path).splitlines() + if line.startswith("##") + } + source_lines = { + line for line in vcf_path.read_text(encoding="utf-8").splitlines() + if line.startswith("##") + } + self.assertEqual(file_lines, source_lines) + self.assertNotIn(injected, file_lines) + + def test_a_gzipped_vcf_header_reads_the_same_as_a_plain_one(self): + """Aggregates arrive compressed, so the oracle must handle both.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = fixtures.write_vcf(tmp_path / "fixture.vcf") + gz_path = tmp_path / "fixture.vcf.gz" + with gzip.open(gz_path, "wt", encoding="utf-8") as handle: + handle.write(vcf_path.read_text(encoding="utf-8")) + self.assertEqual( + V.read_vcf_header_text(gz_path), V.read_vcf_header_text(vcf_path) + ) + + +class GraphIntegrityTests(VerboseTestCase): + """Blank nodes, empty terms, and duplicated statements.""" + + def _execution(self, tmp_path: Path, rows: list) -> dict: + path = tmp_path / "distinct.json" + path.write_text(json.dumps({"results": {"bindings": rows}}), encoding="utf-8") + return {"preflight_distinct_triple_count": {"status": "PASS", "rawResult": str(path)}} + + def test_duplicates_are_the_gap_between_parsed_and_distinct(self): + """A store deduplicates on load, so only the parser sees a repeat.""" + with tempfile.TemporaryDirectory() as td: + executions = self._execution( + Path(td), [{"distinctTripleCount": {"value": "100"}}] + ) + clean = V.duplicate_triple_report(executions, 100) + self.assertEqual(clean["status"], "PASS") + self.assertEqual(clean["duplicateTripleCount"], 0) + + duplicated = V.duplicate_triple_report(executions, 137) + self.assertEqual(duplicated["status"], "FAIL") + self.assertEqual(duplicated["duplicateTripleCount"], 37) + self.assertEqual(duplicated["parsedTripleCount"], 137) + self.assertEqual(duplicated["distinctTripleCount"], 100) + + def test_a_missing_parsed_count_is_not_reported_as_clean(self): + """An unavailable input must never look like an absence of duplicates.""" + with tempfile.TemporaryDirectory() as td: + executions = self._execution( + Path(td), [{"distinctTripleCount": {"value": "100"}}] + ) + result = V.duplicate_triple_report(executions, None) + self.assertEqual(result["status"], "NOT_EVALUATED") + self.assertNotIn("duplicateTripleCount", result) + + def test_a_missing_distinct_query_is_not_reported_as_clean(self): + result = V.duplicate_triple_report({}, 100) + self.assertEqual(result["status"], "NOT_EVALUATED") + + def test_not_evaluated_does_not_block_a_run(self): + """A check that could not run must not be treated as a failure.""" + self.assertIn("preflight_duplicate_triples", V.BLOCKING_PREFLIGHT_QUERIES) + parser = parser_fixture() + sparql = sparql_fixture() + report = V.compare(parser, sparql) + self.assertEqual(report["status"], "PASS") + + def test_integrity_checks_are_blocking(self): + """Each of the three is treated as a structural defect, not a mismatch.""" + for name in ("preflight_blank_nodes", "preflight_empty_values", + "preflight_duplicate_triples"): + self.assertIn(name, V.BLOCKING_PREFLIGHT_QUERIES, name) + + def test_blank_node_query_examines_subject_and_object_only(self): + """A predicate cannot be a blank node in RDF, so it is not tested.""" + query = (V.QUERY_ROOT / "common" / "preflight_blank_nodes.rq").read_text(encoding="utf-8") + self.assertIn("ISBLANK(?s)", query) + self.assertIn("ISBLANK(?o)", query) + self.assertNotIn("ISBLANK(?p)", query) + + def test_empty_value_query_covers_literals_and_iris(self): + """Both a lost value and a collapsed template substitution are caught.""" + query = (V.QUERY_ROOT / "common" / "preflight_empty_values.rq").read_text(encoding="utf-8") + self.assertIn("EMPTY_LITERAL", query) + self.assertIn("EMPTY_IRI", query) + # Whitespace-only carries no more information than empty. + self.assertIn("REGEX", query) + + def test_every_anomaly_preflight_has_an_exact_count_companion(self): + """Severity must be measurable for each new check too.""" + for query_id in V.ANOMALY_PREFLIGHT_QUERIES: + companion = V.query_path(V.QUERY_ROOT / "expanded", f"{query_id}_count") + self.assertTrue(companion.is_file(), f"missing companion for {query_id}") + self.assertIn("preflight_blank_nodes", V.ANOMALY_PREFLIGHT_QUERIES) + self.assertIn("preflight_empty_values", V.ANOMALY_PREFLIGHT_QUERIES) diff --git a/test/test_validation_mutation_unit.py b/test/test_validation_mutation_unit.py new file mode 100644 index 0000000..302a6a7 --- /dev/null +++ b/test/test_validation_mutation_unit.py @@ -0,0 +1,217 @@ +"""Mutation testing for the semantic validation suite. + +Each catalogued mutation is applied to a correct graph, the real validation +queries are evaluated over the result, and the shipped decision logic +(``evaluate_validation``) is asked for a verdict. A mutation is "detected" when +that verdict stops being PASS. + +Queries run under rdflib here so the harness needs no Docker and stays fast +enough for the normal test loop. rdflib is a third independent engine, which +also cross-checks that the queries are not accidentally engine-specific; the +container job in ``.github/workflows/validation-mutation.yml`` replays the same +catalogue under Comunica and QLever as the authority. + +The run writes ``mutation-score.json`` next to the repository root when +``VCF_RDFIZER_MUTATION_REPORT`` is set, which is the artifact the coverage +documentation quotes. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import tempfile +import unittest +from pathlib import Path + +from test import validation_fixtures as fixtures +from test import validation_mutations as mutations +from test.helpers import VerboseTestCase + +try: + import rdflib +except ImportError: # pragma: no cover - optional test-only dependency + rdflib = None + +REPO_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = REPO_ROOT / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_mutation", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +class RdflibEngine: + """Evaluate the validation queries in-process and emit SPARQL Results JSON. + + The output is written in exactly the shape the real engines produce, so the + normalization and comparison layers below are the shipped ones, unmodified. + """ + + def __init__(self, graph_text: str, raw_dir: Path): + self.graph = rdflib.Graph() + self.graph.parse(data=graph_text, format="nt") + self.raw_dir = raw_dir + self.raw_dir.mkdir(parents=True, exist_ok=True) + + def execute(self, query_id: str, query_path: Path) -> dict: + raw_path = self.raw_dir / f"{query_id}.sparql.json" + try: + result = self.graph.query(query_path.read_text(encoding="utf-8")) + raw_path.write_bytes(result.serialize(format="json")) + return {"status": "PASS", "exitCode": 0, "rawResult": str(raw_path), + "engine": "rdflib"} + except Exception as error: # noqa: BLE001 - surfaced as EXECUTION_FAILED + raw_path.write_text("{}", encoding="utf-8") + return {"status": "EXECUTION_FAILED", "exitCode": 1, + "rawResult": str(raw_path), "engine": "rdflib", "error": str(error)} + + +def validate_graph( + graph_text: str, representation: str, *, strict_conformance: bool = False, + include_qual: bool = True, mapping_policy: str = "strict", +) -> dict: + """Run the whole validation decision over one graph.""" + parser = fixtures.parser_summary(representation, include_qual=include_qual) + query_dir = V.QUERY_ROOT / representation + with tempfile.TemporaryDirectory() as td: + engine = RdflibEngine(graph_text, Path(td) / "raw") + executions = { + query_id: engine.execute(query_id, V.query_path(query_dir, query_id)) + for query_id in V.PREFLIGHT_QUERIES + V.PREFLIGHT_COUNT_QUERIES + V.CORE_QUERIES + } + if any(item["status"] != "PASS" for item in executions.values()): + failed = {k: v.get("error", "") for k, v in executions.items() if v["status"] != "PASS"} + return {"status": "EXECUTION_FAILED", "queryErrors": failed} + # rapper is a container tool, so the harness supplies the parsed + # statement count itself. For well-formed N-Triples that is exactly the + # number of non-empty lines, which is what rapper would report. + parsed = sum(1 for line in graph_text.splitlines() if line.strip()) + return V.evaluate_validation( + executions, parser, representation, strict_conformance=strict_conformance, + mapping_policy=mapping_policy, parsed_triple_count=parsed, + ) + + +@unittest.skipIf(rdflib is None, "rdflib is required for the host mutation harness") +class MutationHarnessTests(VerboseTestCase): + """The harness itself must agree with a correct graph before it can judge.""" + + def test_unmutated_expanded_graph_passes(self): + """A correct expanded graph validates cleanly against the fixture oracle.""" + verdict = validate_graph(fixtures.build_graph("expanded"), "expanded") + self.assertEqual(verdict["status"], "PASS", json.dumps(verdict.get("comparison"), indent=2)[:2000]) + + def test_unmutated_condensed_graph_passes(self): + """The same holds for the condensed representation.""" + verdict = validate_graph(fixtures.build_graph("condensed"), "condensed") + self.assertEqual(verdict["status"], "PASS", json.dumps(verdict.get("comparison"), indent=2)[:2000]) + + def test_fixture_graph_and_oracle_describe_the_same_data(self): + """A guard that the fixture's two halves cannot drift apart silently.""" + oracle = fixtures.parser_summary("expanded") + self.assertEqual(oracle["totalRecords"], len(fixtures.RECORDS)) + self.assertEqual(oracle["headerLineCount"], len(fixtures.HEADER_LINES)) + self.assertEqual( + sum(row["recordCount"] for row in oracle["q01_record_density_1mb"]), + oracle["totalRecords"], + ) + + +@unittest.skipIf(rdflib is None, "rdflib is required for the host mutation harness") +class MutationDetectionTests(VerboseTestCase): + """Every catalogued mutation, and whether the suite notices it.""" + + @classmethod + def setUpClass(cls): + cls.graphs: dict[tuple, str] = {} + cls.results: list[dict] = [] + + @classmethod + def graph_for(cls, representation: str, options: tuple) -> str: + """Build (and cache) the fixture graph a mutation needs.""" + key = (representation, options) + if key not in cls.graphs: + cls.graphs[key] = fixtures.build_graph(representation, **dict(options)) + return cls.graphs[key] + + @classmethod + def tearDownClass(cls): + detected = [r for r in cls.results if r["detected"]] + report = { + "total": len(cls.results), + "detected": len(detected), + "score": round(len(detected) / len(cls.results), 4) if cls.results else 0.0, + "knownUndetected": [r["id"] for r in cls.results if not r["detected"]], + "mutations": cls.results, + } + destination = os.environ.get("VCF_RDFIZER_MUTATION_REPORT") + if destination: + Path(destination).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print( + f"\n mutation score: {report['detected']}/{report['total']} " + f"({report['score']:.0%} detected)" + ) + + def _check(self, mutation: mutations.Mutation, representation: str): + options = dict(mutation.graph_options) + original = self.graph_for(representation, mutation.graph_options) + mutated = mutation.apply(original) + self.assertNotEqual(mutated, original, f"{mutation.id} did not change the graph") + # The conformance mutation is only a failure under the strict policy. + strict = mutation.id == "plain_dot_literal" + verdict = validate_graph( + mutated, representation, strict_conformance=strict, + include_qual=options.get("include_qual", True), + mapping_policy=mutation.mapping_policy, + ) + detected = verdict["status"] != "PASS" + self.results.append({ + "id": mutation.id, + "representation": representation, + "vcfElement": mutation.vcf_element, + "expectedDetectedBy": mutation.expected_detected_by, + "detected": detected, + "status": verdict["status"], + "knownUndetected": mutation.known_undetected, + "graphOptions": dict(mutation.graph_options), + }) + if mutation.known_undetected: + self.assertFalse( + detected, + f"{mutation.id} is now DETECTED. This gap has been closed - remove " + f"`known_undetected` from the catalogue and update " + f"docs/vcf-coverage.md. Recorded reason was: {mutation.known_undetected}", + ) + else: + self.assertTrue( + detected, + f"{mutation.id} was NOT detected (status={verdict['status']}). " + f"Expected {mutation.expected_detected_by} to catch it.", + ) + + +def _attach_mutation_tests(): + """Generate one test per (mutation, representation) so failures are named.""" + for representation in ("expanded", "condensed"): + for mutation in mutations.for_representation(representation): + def test(self, _m=mutation, _r=representation): + self._check(_m, _r) + + test.__doc__ = f"[{representation}] {mutation.description}" + setattr(MutationDetectionTests, f"test_{representation}_{mutation.id}", test) + + +_attach_mutation_tests() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_vcf_as_tsv_unit.py b/test/test_vcf_as_tsv_unit.py index f293f28..8a3c4a0 100644 --- a/test/test_vcf_as_tsv_unit.py +++ b/test/test_vcf_as_tsv_unit.py @@ -1,3 +1,4 @@ +import csv import gzip import subprocess import tempfile @@ -113,6 +114,37 @@ def test_vcf_as_tsv_collapses_multi_sample_header_and_values_to_single_column(se "multi.vcf\t1\t1\t10\trs1\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:42 0/0:18", ) + def test_vcf_as_tsv_preserves_thousands_of_sample_columns(self): + """A 1000 Genomes-sized sample header and payload remain aligned.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_file = tmp_path / "many-samples.vcf" + output_dir = tmp_path / "tsv" + sample_ids = [f"S{i:04d}" for i in range(2504)] + payloads = [f"{i % 2}/{(i + 1) % 2}:{10 + (i % 90)}" for i in range(2504)] + input_file.write_text( + "##fileformat=VCFv4.2\n" + + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + + "\t".join(sample_ids) + + "\n20\t10\trs1\tA\tG\t50\tPASS\t.\tGT:DP\t" + + "\t".join(payloads) + + "\n", + encoding="utf-8", + ) + + result = subprocess.run( + ["bash", str(SCRIPT), str(input_file), str(output_dir)], + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + with (output_dir / "many-samples.records.tsv").open(newline="", encoding="utf-8") as handle: + rows = list(csv.reader(handle, delimiter="\t")) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0][-1].split(), sample_ids) + self.assertEqual(rows[1][-1].split(), payloads) + def test_vcf_as_tsv_single_gz_file_mode(self): """Single .vcf.gz input: decompresses and writes per-VCF split TSV outputs.""" with tempfile.TemporaryDirectory() as td: diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 9fd09fc..5dea3b7 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -9,6 +9,7 @@ import subprocess import sys import tempfile +import types import unittest from contextlib import redirect_stderr, redirect_stdout from io import StringIO @@ -218,6 +219,20 @@ def prepare_inputs(base: Path): return input_dir, rules_path +def load_validation_runner_module(): + """Load the container validator without requiring cyvcf2 on the host.""" + validator_path = Path(__file__).parents[1] / "src" / "validation" / "validation_runner.py" + fake_cyvcf2 = types.ModuleType("cyvcf2") + fake_cyvcf2.VCF = object + fake_cyvcf2.__version__ = "test" + spec = importlib.util.spec_from_file_location("validation_runner_test", validator_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + with mock.patch.dict(sys.modules, {"cyvcf2": fake_cyvcf2}): + spec.loader.exec_module(module) + return module + + def mocked_triplets(): return [ { @@ -243,20 +258,262 @@ def output_name_from_command(cmd): def latest_metrics_run_dir(metrics_root: Path) -> Path: """Return the single/latest per-run metrics directory.""" - run_dirs = sorted( - ( - path - for path in metrics_root.iterdir() - if path.is_dir() and re.match(r"^\d{8}T\d{6}$", path.name) - ), - key=lambda path: path.name, - ) + run_dirs = [] + for path in metrics_root.iterdir(): + if not path.is_dir(): + continue + match = re.match(r"^.+__(\d{8}T\d{6})$", path.name) + if match: + run_dirs.append((match.group(1), path)) + run_dirs.sort(key=lambda item: item[0]) if not run_dirs: raise AssertionError(f"No per-run metrics directories found under {metrics_root}") - return run_dirs[-1] + return run_dirs[-1][1] class WrapperUnitTests(VerboseTestCase): + def test_metrics_directory_uses_recognizable_input_label(self): + """Metrics runs keep the full input stem alongside their timestamp.""" + metrics_root = Path("/tmp/metrics") + label = vcf_rdfizer.metrics_run_label( + [Path("/data/1000G_phase3_chr20.vcf.gz")] + ) + self.assertEqual(label, "1000G_phase3_chr20") + self.assertEqual( + vcf_rdfizer.metrics_run_directory( + metrics_root, label, "20260904T123456" + ), + metrics_root / "1000G_phase3_chr20__20260904T123456", + ) + + def test_hdt_index_memory_limit_is_forwarded_to_docker(self): + """A host hdtc memory override is passed to indexing containers.""" + with mock.patch.dict(os.environ, {"HDT_INDEX_MEMORY_LIMIT": "2G"}): + self.assertEqual( + vcf_rdfizer.docker_hdt_index_env_args(), + ["-e", "HDT_INDEX_MEMORY_LIMIT=2G"], + ) + + def test_hdt_merge_memory_limit_is_forwarded_to_docker(self): + """A host hdtc merge-memory override is passed to partitioned containers.""" + with mock.patch.dict(os.environ, {"HDT_MERGE_MEMORY_LIMIT": "768M"}): + self.assertEqual( + vcf_rdfizer.docker_hdt_merge_env_args(), + ["-e", "HDT_MERGE_MEMORY_LIMIT=768M"], + ) + + def test_cottas_merge_batch_rows_are_forwarded_to_docker(self): + """The bounded COTTAS streaming-merge batch size reaches Docker.""" + with mock.patch.dict( + os.environ, + { + "COTTAS_MERGE_BATCH_ROWS": "1024", + }, + ): + self.assertEqual( + vcf_rdfizer.docker_cottas_merge_env_args(), + [ + "-e", + "COTTAS_MERGE_BATCH_ROWS=1024", + ], + ) + + def test_plain_progress_is_shown_when_rich_or_a_tty_is_unavailable(self): + """Long compression work remains visible through redirected terminals.""" + previous_progress_setting = vcf_rdfizer._PROGRESS_ALLOWED + try: + vcf_rdfizer._PROGRESS_ALLOWED = True + with tempfile.TemporaryDirectory() as td, mock.patch.dict( + os.environ, + {"VCF_RDFIZER_NO_PROGRESS": "", "CI": ""}, + clear=False, + ), mock.patch.object(vcf_rdfizer, "Progress", None), mock.patch.object( + vcf_rdfizer, "Console", None + ): + progress_path = Path(td) / "partitioned.jsonl" + terminal = StringIO() + with redirect_stderr(terminal), vcf_rdfizer.ProgressSession( + progress_path, "Partitioned compression: cohort" + ) as session: + progress_path.write_text( + json.dumps( + { + "stage": "cottas-merge", + "phase": "started", + "unit": "stage", + "detail": "cottas-merge-disk", + } + ) + + "\n" + ) + session.poll_events() + + displayed = terminal.getvalue() + self.assertIn("Partitioned compression: cohort: started", displayed) + self.assertIn("cottas-merge started", displayed) + self.assertIn("cottas-merge-disk", displayed) + finally: + vcf_rdfizer._PROGRESS_ALLOWED = previous_progress_setting + + def test_quiet_progress_consumes_sidecar_without_terminal_output(self): + """Quiet mode keeps event handling active but renders no progress lines.""" + previous_allowed = vcf_rdfizer._PROGRESS_ALLOWED + previous_events = vcf_rdfizer._PROGRESS_EVENTS_ALLOWED + previous_quiet = vcf_rdfizer._QUIET + try: + vcf_rdfizer._PROGRESS_ALLOWED = False + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = True + vcf_rdfizer._QUIET = True + with tempfile.TemporaryDirectory() as td, mock.patch.dict( + os.environ, + {"VCF_RDFIZER_NO_PROGRESS": "", "CI": ""}, + clear=False, + ), mock.patch.object(vcf_rdfizer, "Progress", None), mock.patch.object( + vcf_rdfizer, "Console", None + ): + progress_path = Path(td) / "validation.jsonl" + terminal = StringIO() + with redirect_stderr(terminal), vcf_rdfizer.ProgressSession( + progress_path, "Validation: cohort" + ) as session: + progress_path.write_text( + json.dumps( + { + "stage": "validation", + "phase": "started", + "completed": 0, + "total": 2, + } + ) + + "\n" + ) + session.poll_events() + + self.assertEqual(terminal.getvalue(), "") + finally: + vcf_rdfizer._PROGRESS_ALLOWED = previous_allowed + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = previous_events + vcf_rdfizer._QUIET = previous_quiet + + def test_validation_forwards_progress_sidecar_and_quiet_flag(self): + """Validation uses the shared sidecar protocol and propagates --quiet.""" + previous_allowed = vcf_rdfizer._PROGRESS_ALLOWED + previous_events = vcf_rdfizer._PROGRESS_EVENTS_ALLOWED + previous_quiet = vcf_rdfizer._QUIET + try: + vcf_rdfizer._PROGRESS_ALLOWED = False + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = True + vcf_rdfizer._QUIET = True + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text("

.\n") + metrics_dir = tmp_path / "metrics" + results_dir = metrics_dir / "reports" / "validation" / "sample" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), redirect_stdout( + StringIO() + ): + rc = vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=rdf_path, + representation="expanded", + validation_id="sample", + results_dir=results_dir, + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + wrapper_log_path=tmp_path / "wrapper.log", + ) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + command = commands[0] + self.assertIn("--progress-path", command) + self.assertIn( + "/data/metrics/.progress/validation-sample.jsonl", command + ) + self.assertIn("--quiet", command) + finally: + vcf_rdfizer._PROGRESS_ALLOWED = previous_allowed + vcf_rdfizer._PROGRESS_EVENTS_ALLOWED = previous_events + vcf_rdfizer._QUIET = previous_quiet + + def test_validation_runner_emits_query_progress_in_quiet_mode(self): + """The container validator still writes lifecycle events when stdout is quiet.""" + validator = load_validation_runner_module() + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text("

.\n") + scratch_dir = tmp_path / "scratch" + scratch_dir.mkdir() + results_dir = tmp_path / "results" + progress_path = tmp_path / ".progress" / "validation.jsonl" + args = argparse.Namespace( + results_dir=results_dir, + representation="expanded", + progress_path=progress_path, + quiet=True, + scratch_dir=scratch_dir, + rdf=rdf_path, + rdf_format="nt", + engine="comunica", + engines=["comunica"], + mapping_policy="strict", + strict_conformance=False, + shacl_shapes=None, + query_timeout=60, + qlever_memory_gb=4, + qlever_port=7019, + qlever_startup_timeout=60, + qlever_index_arg=[], + qlever_server_arg=[], + vcf=vcf_path, + filter_oracle="cyvcf2", + dataset_id="sample", + ) + query_file = tmp_path / "query.rq" + query_file.write_text("SELECT * WHERE { ?s ?p ?o }\n") + parser = { + "totalRecords": 1, + "sampleCount": 0, + "gtRecordCount": 0, + } + failing_engine = mock.MagicMock() + failing_engine.describe.return_value = {"engine": "comunica"} + failing_engine.execute.return_value = {"status": "FAILED"} + with mock.patch.object(validator, "query_path", return_value=query_file), mock.patch.object( + validator, "parse_vcf", return_value=parser + ), mock.patch.object( + validator, "validate_ntriples", return_value={"status": "PASS"} + ), mock.patch.object(validator, "build_manifest", return_value={}), mock.patch.object( + validator, "build_engine", return_value=failing_engine + ), redirect_stdout(StringIO()) as output: + rc = validator.run_validation(args) + + self.assertEqual(rc, 1) + self.assertEqual(output.getvalue(), "") + events = [ + json.loads(line) for line in progress_path.read_text().splitlines() + ] + self.assertEqual(events[0]["phase"], "started") + self.assertEqual(events[0]["unit"], "queries") + self.assertEqual(events[1]["phase"], "progress") + self.assertEqual(events[1]["completed"], 0) + self.assertEqual(events[-1]["phase"], "failed") + def test_validator_counts_plain_and_gzip_ntriples(self): """The Docker validator's fallback source count handles .nt and .nt.gz.""" validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" @@ -540,7 +797,7 @@ def fake_run(cmd, cwd=None, env=None): if emulate_validation_command(cmd): return 0 script = str(cmd[-1]) if cmd else "" - time_match = re.search(r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", script) + time_match = re.search(r"-o\s+(/data/metrics/timings/tsv/[^\s;]+)", script) if metrics_mount and time_match: time_log = Path(metrics_mount) / time_match.group(1).replace("/data/metrics/", "", 1) time_log.parent.mkdir(parents=True, exist_ok=True) @@ -576,12 +833,12 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(metrics["sys_seconds"], 1.5) self.assertEqual(metrics["max_rss_kb"], 2048) - raw_json = metrics_dir / "raw_metrics" / "tsv_metrics" / "sample" / "run-tsv-elapsed.json" + raw_json = metrics_dir / "stages" / "tsv" / "sample.json" payload = json.loads(raw_json.read_text()) self.assertEqual(payload["timing"]["wall_seconds"], 360.0) - def test_run_compression_methods_persists_raw_metrics_and_time_logs(self): - """Per-file compression timing/metrics are retained under raw_metrics.""" + def test_run_compression_methods_persists_stage_metrics_and_time_logs(self): + """Per-file compression timing/metrics use the canonical stages/timings tree.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) out_dir = tmp_path / "out" / "sample" @@ -634,29 +891,24 @@ def fake_run(cmd, cwd=None, env=None): safe_rdf = vcf_rdfizer.safe_metrics_name("sample.nt") hdt_time = ( metrics_dir - / "raw_metrics" - / "compression_time" + / "timings" + / "compression" / safe_output - / safe_rdf - / "hdt" - / "run-1.txt" + / "hdt.txt" ) gzip_time = ( metrics_dir - / "raw_metrics" - / "compression_time" + / "timings" + / "compression" / safe_output - / safe_rdf - / "gzip" - / "run-1.txt" + / "gzip.txt" ) raw_json = ( metrics_dir - / "raw_metrics" - / "compression_metrics" + / "stages" + / "compression_operations" / safe_output - / safe_rdf - / "run-1.json" + / f"{safe_rdf}.json" ) self.assertTrue(hdt_time.exists()) @@ -713,8 +965,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(method_results["hdt"]["sys_seconds"], 4.5) self.assertEqual(method_results["hdt"]["max_rss_kb"], 8192) - def test_run_compression_methods_records_implicit_hdt_in_raw_metrics(self): - """Compound HDT methods include the implicit HDT stage in raw metrics JSON.""" + def test_run_compression_methods_records_implicit_hdt_in_stage_metrics(self): + """Compound HDT methods include the implicit HDT stage in the operation report.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) out_dir = tmp_path / "out" / "sample" @@ -767,11 +1019,10 @@ def fake_run(cmd, cwd=None, env=None): safe_rdf = vcf_rdfizer.safe_metrics_name("sample.nt") raw_json = ( metrics_dir - / "raw_metrics" - / "compression_metrics" + / "stages" + / "compression_operations" / safe_output - / safe_rdf - / "run-2.json" + / f"{safe_rdf}.json" ) self.assertTrue(raw_json.exists()) @@ -840,17 +1091,127 @@ def fake_run(cmd, cwd=None, env=None): raw_json = ( metrics_dir - / "raw_metrics" - / "compression_metrics" + / "stages" + / "compression_operations" / "sample" - / "sample.nt" - / "run-cottas-package.json" + / "sample.nt.json" ) self.assertTrue(raw_json.exists()) payload = json.loads(raw_json.read_text()) self.assertIn("cottas_gzip", payload["methods"]) self.assertIn("cottas_brotli", payload["methods"]) + def test_full_compression_recovers_from_hdt_index_failure(self): + """A readable HDT remains packageable when only index creation fails.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir = tmp_path / "input" + out_dir = tmp_path / "out" + target_dir = out_dir / "sample" + input_dir.mkdir() + target_dir.mkdir(parents=True) + rdf_path = input_dir / "sample.nt" + rdf_path.write_text("

.\n") + hdt_path = target_dir / "sample.hdt" + hdt_path.write_text("readable-hdt\n") + warnings = [] + validation_calls = [] + + def fake_run(cmd, cwd=None, env=None): + rendered = str(cmd[-1]) if cmd else "" + if "validate_compression.py" not in rendered: + if "gzip -c /data/out/sample/sample.hdt" in rendered: + (target_dir / "sample.hdt.gz").write_text("packaged-hdt\n") + return 0 + validation_calls.append(rendered) + result_match = re.search( + r"--result-path\s+['\"]?(/data/out/[^\s'\";]+)", + rendered, + ) + result_path = out_dir / result_match.group(1).replace("/data/out/", "", 1) + result_path.parent.mkdir(parents=True, exist_ok=True) + if "--skip-index-check" in rendered: + result_path.write_text( + json.dumps( + { + "valid": True, + "source_triples": 1, + "decoded_triples": 1, + "count_match": True, + } + ) + ) + return 0 + result_path.write_text( + json.dumps( + { + "valid": False, + "source_triples": 1, + "decoded_triples": 1, + "count_match": False, + "error": "HDT index/readability check failed", + } + ) + ) + return 1 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + target_out_dir=target_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["hdt", "hdt_gzip"], + wrapper_log_path=tmp_path / "wrapper.log", + status_indent=None, + index_warnings=warnings, + ) + + self.assertTrue(ok) + self.assertEqual(len(validation_calls), 2) + self.assertEqual(method_results["hdt"]["index_status"], "failed") + self.assertEqual(method_results["hdt_gzip"]["exit_code"], 0) + self.assertEqual(warnings[0]["format"], "hdt") + + def test_full_compression_continues_after_cottas_index_failure(self): + """COTTAS failure is recorded while independent raw RDF compression continues.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir = tmp_path / "input" + out_dir = tmp_path / "out" + target_dir = out_dir / "sample" + input_dir.mkdir() + target_dir.mkdir(parents=True) + rdf_path = input_dir / "sample.nt" + rdf_path.write_text("

.\n") + warnings = [] + + def fake_run(cmd, cwd=None, env=None): + rendered = str(cmd[-1]) if cmd else "" + if "cottas_tool.py convert" in rendered: + return 1 + if "gzip -c /data/in/sample.nt" in rendered: + (target_dir / "sample.nt.gz").write_text("gzip\n") + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + target_out_dir=target_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["cottas", "gzip"], + wrapper_log_path=tmp_path / "wrapper.log", + status_indent=None, + index_warnings=warnings, + ) + + self.assertTrue(ok) + self.assertEqual(method_results["cottas"]["exit_code"], 1) + self.assertEqual(method_results["gzip"]["exit_code"], 0) + self.assertEqual(warnings[0]["format"], "cottas") + self.assertTrue((target_dir / "sample.nt.gz").exists()) + def test_compression_validation_mismatch_fails_before_success(self): """A decoded triple-count mismatch fails compression and preserves RDF input.""" with tempfile.TemporaryDirectory() as td: @@ -903,70 +1264,6 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(method_results["hdt"]["exit_code"], 0) self.assertTrue(rdf_path.exists()) - def test_plan_partitioned_hdt_chunks_groups_small_inputs_and_splits_large_ones(self): - """Partition planning coalesces small RDF parts and line-splits oversized ones.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - rdf_dir = tmp_path / "rdf" - chunk_dir = tmp_path / "chunks" - rdf_dir.mkdir() - - small_a = rdf_dir / "part-a.nt" - small_b = rdf_dir / "part-b.nt" - large = rdf_dir / "part-c.nt" - small_a.write_text("

.\n") - small_b.write_text("

.\n") - large.write_text(("

.\n".lstrip()) * 8) - - chunk_inputs, plan = vcf_rdfizer.plan_partitioned_hdt_chunks( - [small_a, small_b, large], - chunk_dir, - target_bytes=40, - min_bytes=10, - max_bytes=60, - ) - - self.assertGreaterEqual(len(chunk_inputs), 2) - self.assertEqual(plan["source_file_count"], 3) - self.assertEqual(plan["chunk_count"], len(chunk_inputs)) - self.assertTrue(all(path.exists() for path in chunk_inputs)) - first_chunk_text = chunk_inputs[0].read_text() - self.assertIn("

.", first_chunk_text) - self.assertIn("

.", first_chunk_text) - - def test_record_safe_chunk_planner_reads_gzip_and_writes_boundary_guide(self): - """Gzip-backed chunking preserves complete records and records exact ranges.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - source = tmp_path / "aggregate.nt.gz" - source_payload = ( - b"

.\n" - b"

.\n" - b"

.\n" - ) - import gzip - - with gzip.open(source, "wb") as handle: - handle.write(source_payload) - - chunk_dir = tmp_path / "chunks" - guide = tmp_path / "chunks.json" - chunk_paths, plan = vcf_rdfizer.plan_record_safe_rdf_chunks( - [source], - chunk_dir, - target_bytes=20, - min_bytes=10, - max_bytes=30, - guide_path=guide, - ) - - self.assertTrue(guide.exists()) - self.assertEqual(plan["record_count"], 3) - self.assertEqual(plan["chunk_count"], len(chunk_paths)) - self.assertEqual(b"".join(path.read_bytes() for path in chunk_paths), source_payload) - self.assertTrue(all(path.read_bytes().endswith(b"\n") for path in chunk_paths)) - self.assertEqual(json.loads(guide.read_text())["chunks"], plan["chunks"]) - def test_run_partitioned_hdt_methods_merges_chunks_and_generates_index(self): """Partitioned HDT/COTTAS pipelines share chunks in one container.""" with tempfile.TemporaryDirectory() as td: @@ -1030,6 +1327,7 @@ def test_containerized_partitioned_pipeline_cleans_ephemeral_volume(self): source = tmp_path / "input.nt" source.write_text("

.\n") out_dir = tmp_path / "out" / "sample" + metrics_dir = tmp_path / "metrics" commands = [] def fake_run(cmd, cwd=None, env=None): @@ -1046,6 +1344,9 @@ def fake_run(cmd, cwd=None, env=None): image_ref="example/vcf-rdfizer:latest", methods=["hdt", "cottas"], wrapper_log_path=tmp_path / "wrapper.log", + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", output_name="sample", target_chunk_bytes=40, min_chunk_bytes=10, @@ -1064,6 +1365,15 @@ def fake_run(cmd, cwd=None, env=None): cmd for cmd in commands if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd ) self.assertIn(f"{source.parent.resolve()}:/data/in:ro", runner_command) + report = json.loads( + (metrics_dir / "stages" / "partitioned" / "sample.json").read_text() + ) + self.assertEqual(report["runtime_environment"], "docker-volume") + self.assertEqual(report["container_result"]["exit_code"], 0) + self.assertEqual( + report["container_result"]["methods"]["hdt"]["details"]["workspace"], + "docker-volume", + ) def test_containerized_partitioned_pipeline_removes_volume_after_failure(self): """A failed container run still removes its named workspace volume.""" @@ -1072,6 +1382,7 @@ def test_containerized_partitioned_pipeline_removes_volume_after_failure(self): source = tmp_path / "input.nt" source.write_text("

.\n") out_dir = tmp_path / "out" + metrics_dir = tmp_path / "metrics" commands = [] def fake_run(cmd, cwd=None, env=None): @@ -1097,6 +1408,9 @@ def fake_run(cmd, cwd=None, env=None): image_ref="example/vcf-rdfizer:latest", methods=["cottas"], wrapper_log_path=tmp_path / "wrapper.log", + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", output_name="sample", target_chunk_bytes=40, min_chunk_bytes=10, @@ -1107,6 +1421,10 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(method_results, {}) self.assertTrue(any("volume" in cmd and "rm" in cmd for cmd in commands)) self.assertFalse(any(path.name.startswith(".") for path in out_dir.glob("*"))) + report = json.loads( + (metrics_dir / "stages" / "partitioned" / "sample.json").read_text() + ) + self.assertEqual(report["container_result"]["exit_code"], 1) def test_main_full_mode_records_tsv_metrics_and_raw_artifacts(self): """Full mode stores TSV timing/metrics artifacts and writes TSV fields into metrics.csv.""" @@ -1131,7 +1449,7 @@ def fake_run(cmd, cwd=None, env=None): ) script = str(cmd[-1]) if cmd else "" time_match = re.search( - r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", + r"-o\s+(/data/metrics/timings/tsv/[^\s;]+)", script, ) if metrics_mount and time_match: @@ -1184,10 +1502,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(rc, 0) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - run_id = run_metrics_dir.name - - tsv_time = run_metrics_dir / "raw_metrics" / "tsv_time" / "sample" / f"{run_id}.txt" - tsv_json = run_metrics_dir / "raw_metrics" / "tsv_metrics" / "sample" / f"{run_id}.json" + tsv_time = run_metrics_dir / "timings" / "tsv" / "sample.txt" + tsv_json = run_metrics_dir / "stages" / "tsv" / "sample.json" self.assertTrue(tsv_time.exists()) self.assertTrue(tsv_json.exists()) @@ -1200,6 +1516,103 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("wall_seconds_tsv", row) self.assertNotEqual(row.get("tsv_output_size_bytes", ""), "") + def test_main_full_mode_runs_validation_and_harmonizes_reports_and_metrics(self): + """--validate adds one canonical validation stage and CSV row to a full run.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + validation_calls = [] + + def fake_run(cmd, cwd=None, env=None): + if "/opt/vcf-rdfizer/vcf_as_tsv.sh" in cmd: + tsv_dir = out_dir / ".intermediate" / "tsv" + tsv_dir.mkdir(parents=True, exist_ok=True) + (tsv_dir / "sample.records.tsv").write_text("SOURCE_FILE\tROW_ID\nsample.vcf\t1\n") + (tsv_dir / "sample.header_lines.tsv").write_text("SOURCE_FILE\tLINE\nsample.vcf\t##x\n") + (tsv_dir / "sample.file_metadata.tsv").write_text("SOURCE_FILE\tKEY\tVALUE\nsample.vcf\tk\tv\n") + elif "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + sample_dir = out_dir / "sample" + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "sample.nt").write_text("

.\n") + return 0 + + def fake_validation(**kwargs): + validation_calls.append(kwargs) + results_dir = kwargs["results_dir"] + results_dir.mkdir(parents=True, exist_ok=True) + summary_path = results_dir / "summary.json" + summary_path.write_text(json.dumps({"status": "PASS", "temporaryRdf": {"persisted": False}})) + payload = { + "run_id": kwargs["run_id"], + "timestamp": kwargs["timestamp"], + "stage": "validation", + "vcf_path": str(kwargs["vcf_path"]), + "rdf_path": str(kwargs["rdf_path"]), + "rdf_format": "nt", + "representation": kwargs["representation"], + "results_dir": str(results_dir), + "summary_path": str(summary_path), + "exit_code": 0, + "status": "PASS", + "timing": {"wall_seconds": 0.25, "user_seconds": 0.1, "sys_seconds": 0.02, "max_rss_kb": 512}, + "temporary_rdf": {"decompressed_inside_container": False, "persisted_on_host": False, "cleanup_confirmed_by_runner": True}, + } + kwargs["stage_result"].update(payload) + stage_path = kwargs["metrics_dir"] / "stages" / "validation" / "sample.json" + stage_path.parent.mkdir(parents=True, exist_ok=True) + stage_path.write_text(json.dumps(payload)) + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ), mock.patch.object( + vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets() + ), mock.patch.object( + vcf_rdfizer, "run_validation_mode", side_effect=fake_validation + ): + rc = invoke_main( + [ + "--input", + str(input_dir), + "--rules", + str(rules_path), + "--rdf-storage-mode", + "plain", + "--compression", + "none", + "--validate", + "--out", + str(out_dir), + "--keep-tsv", + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(len(validation_calls), 1) + self.assertEqual(validation_calls[0]["rdf_path"].name, "sample.nt") + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + report_dir = run_metrics_dir / "reports" / "validation" / "sample" + self.assertTrue((report_dir / "summary.json").exists()) + self.assertTrue((run_metrics_dir / "stages" / "validation" / "sample.json").exists()) + with (run_metrics_dir / "metrics.csv").open(newline="", encoding="utf-8") as handle: + row = next(csv.DictReader(handle)) + self.assertEqual(row["validation_status"], "PASS") + self.assertEqual(row["validation_exit_code"], "0") + self.assertEqual( + Path(row["validation_rdf_path"]).resolve(), + (out_dir / "sample" / "sample.nt").resolve(), + ) + summary = json.loads((run_metrics_dir / "summary.json").read_text()) + self.assertIn("reports/validation/sample/summary.json", summary["reports"]) + def test_build_sample_support_tsvs_expands_per_sample_and_per_format_rows(self): """records.tsv is expanded into helper tables for sample calls and format key/value pairs.""" with tempfile.TemporaryDirectory() as td: @@ -1261,6 +1674,276 @@ def test_build_sample_support_tsvs_sanitizes_sample_id_for_uri_paths(self): self.assertIn("sample.vcf\t2\t1\tSAMPLE-A\tSAMPLE-A\t0/1:42", sample_calls_rows) self.assertIn("sample.vcf\t2\t2\tSAMPLE/B\tSAMPLE_B\t0/0:18", sample_calls_rows) + def test_default_sample_maps_stream_without_expanded_helper_tables(self): + """Canonical sample maps emit RDF directly while compatibility TSVs stay empty.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "sample.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tSAMPLE-A SAMPLE/B\n" + "sample.vcf\t2\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:42 ./.:.\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text(" .\n", encoding="utf-8") + sample_calls_tsv = tmp_path / "sample.sample_calls.tsv" + sample_format_tsv = tmp_path / "sample.sample_format_values.tsv" + + vcf_rdfizer.write_sample_support_headers(sample_calls_tsv, sample_format_tsv) + stats = vcf_rdfizer.append_canonical_sample_rdf( + records_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(sample_calls_tsv.read_text().splitlines(), ["\t".join(vcf_rdfizer.SAMPLE_CALLS_HEADER)]) + self.assertEqual(sample_format_tsv.read_text().splitlines(), ["\t".join(vcf_rdfizer.SAMPLE_FORMAT_HEADER)]) + self.assertEqual(stats["records"], 1) + self.assertEqual(stats["sample_calls"], 2) + self.assertEqual(stats["format_values"], 4) + self.assertEqual(stats["triples"], 19) + rdf_lines = rdf_path.read_text(encoding="utf-8").splitlines() + self.assertEqual(len(rdf_lines), 20) + self.assertIn( + " " + " .", + rdf_lines, + ) + self.assertIn( + " " + " .", + rdf_lines, + ) + self.assertIn( + '"."^^ .', + rdf_lines[-1], + ) + + def test_sample_streaming_accepts_a_csv_field_larger_than_python_default(self): + """Thousands of long sample payloads do not hit csv.field_size_limit.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + sample_ids = [f"S{i:04d}" for i in range(2504)] + payload = "A" * 60 + records_tsv = tmp_path / "large.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + + " ".join(sample_ids) + + "\nlarge.vcf\t1\t20\t1\t.\tA\tG\t.\tPASS\t.\tGT\t" + + " ".join([payload] * len(sample_ids)) + + "\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "large.nt.gz" + with gzip.open(rdf_path, "wt", encoding="utf-8") as handle: + handle.write(" .\n") + + stats = vcf_rdfizer.append_canonical_sample_rdf( + records_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(stats["sample_calls"], 2504) + self.assertEqual(stats["format_values"], 2504) + self.assertEqual(stats["triples"], 1 + (2504 * 6)) + with gzip.open(rdf_path, "rt", encoding="utf-8") as handle: + self.assertEqual(sum(1 for _line in handle), 2 + (2504 * 6)) + + def test_sample_support_strategy_preserves_custom_helper_consumers(self): + """Only the exact canonical helper maps use direct RDF streaming.""" + default_rules = Path(__file__).parents[1] / "rules" / "default_rules.ttl" + self.assertEqual(vcf_rdfizer.sample_support_strategy(default_rules), "stream") + with tempfile.TemporaryDirectory() as td: + custom_rules = Path(td) / "custom.ttl" + custom_rules.write_text( + default_rules.read_text(encoding="utf-8") + + '\n<#Extra> csvw:url "/data/tsv/sample_calls.tsv" .\n', + encoding="utf-8", + ) + self.assertEqual(vcf_rdfizer.sample_support_strategy(custom_rules), "expanded") + + def test_sample_workflow_resolver_selects_one_compatible_branch(self): + """Expanded and condensed plans are mutually exclusive and rules-aware.""" + default_rules = Path(__file__).parents[1] / "rules" / "default_rules.ttl" + + expanded = vcf_rdfizer.resolve_sample_workflow("expanded", default_rules) + condensed = vcf_rdfizer.resolve_sample_workflow("condensed", default_rules) + + self.assertEqual(expanded.helper_strategy, "header-only") + self.assertEqual(expanded.emitter, "expanded") + self.assertEqual(condensed.helper_strategy, "header-only") + self.assertEqual(condensed.emitter, "condensed") + + with tempfile.TemporaryDirectory() as td: + custom_rules = Path(td) / "custom.ttl" + custom_rules.write_text( + default_rules.read_text(encoding="utf-8") + + '\n<#Extra> csvw:url "/data/tsv/sample_calls.tsv" .\n', + encoding="utf-8", + ) + self.assertEqual( + vcf_rdfizer.resolve_sample_workflow("expanded", custom_rules).helper_strategy, + "expanded", + ) + with self.assertRaisesRegex(ValueError, "cannot be combined"): + vcf_rdfizer.resolve_sample_workflow("condensed", custom_rules) + + def test_condensed_sample_emitter_uses_shared_samples_and_ordered_vectors(self): + """Condensed mode emits one SampleSet and FORMAT vector per key, not expanded calls.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "cohort.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1 S2 S3\n" + "cohort.vcf\t7\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT:DP:AD\t" + "0/1:42:30,12 ./.:.:. 1/1:9\n", + encoding="utf-8", + ) + headers_tsv = tmp_path / "cohort.header_lines.tsv" + headers_tsv.write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "cohort.vcf\t1\tFORMAT\t\tx\n" + "cohort.vcf\t2\tFORMAT\t\tx\n" + "cohort.vcf\t3\tFORMAT\t\tx\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "cohort.nt" + rdf_path.write_text(" .\n", encoding="utf-8") + + stats = vcf_rdfizer.append_condensed_sample_rdf( + records_tsv, + headers_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(stats["representation"], "condensed") + self.assertEqual(stats["samples"], 3) + self.assertEqual(stats["matrices"], 1) + self.assertEqual(stats["format_vectors"], 3) + self.assertEqual(stats["format_definitions"], 3) + self.assertEqual(stats["triples"], 45) + rdf_text = rdf_path.read_text(encoding="utf-8") + self.assertIn("vocab#CondensedRepresentation", rdf_text) + self.assertIn("vocab#SampleSet", rdf_text) + self.assertIn("#samples/S3> ", rdf_text) + self.assertIn("#call/7/matrix/fmt/GT", rdf_text) + self.assertIn('"0/1\\t./.\\t1/1"', rdf_text) + self.assertIn('"30,12\\t.\\t."', rdf_text) + self.assertIn("#header/line/1> .", rdf_text) + self.assertIn("vocab#fieldNumber> \"1\"", rdf_text) + self.assertIn("vocab#fieldDescription> \"Genotype\"", rdf_text) + self.assertNotIn("vocab#hasSampleCall", rdf_text) + self.assertNotIn("vocab#FormatFieldValue", rdf_text) + + def test_structured_format_header_parser_preserves_quoted_commas(self): + """FORMAT descriptions with commas and escaped quotes remain one attribute.""" + fields = vcf_rdfizer._parse_structured_header_fields( + '' + ) + + self.assertEqual(fields["ID"], "GT") + self.assertEqual(fields["Number"], "1") + self.assertEqual(fields["Description"], 'Genotype, with "quoted" text') + + def test_condensed_sample_emitter_rolls_back_on_sample_count_mismatch(self): + """Malformed sample alignment fails atomically without a partial condensed graph.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "bad.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1\n" + "bad.vcf\t1\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1 0/0\n", + encoding="utf-8", + ) + headers_tsv = tmp_path / "bad.header_lines.tsv" + headers_tsv.write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "bad.nt" + original = " .\n" + rdf_path.write_text(original, encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "2 sample payloads"): + vcf_rdfizer.append_condensed_sample_rdf( + records_tsv, + headers_tsv, + rdf_path, + progress_interval_records=0, + ) + + self.assertEqual(rdf_path.read_text(encoding="utf-8"), original) + + def test_main_condensed_mode_runs_only_the_condensed_emitter(self): + """The full CLI routes default rules to condensed output without expanded sample triples.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_file = tmp_path / "cohort.vcf" + input_file.write_text( + "##fileformat=VCFv4.2\n" + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1\tS2\n" + "1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\t0/0\n", + encoding="utf-8", + ) + out_dir = tmp_path / "out" + + def fake_tsv_conversion(**kwargs): + tsv_dir = kwargs["tsv_dir"] + prefix = kwargs["prefix"] + tsv_dir.mkdir(parents=True, exist_ok=True) + (tsv_dir / f"{prefix}.records.tsv").write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1 S2\n" + "cohort.vcf\t1\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1 0/0\n", + encoding="utf-8", + ) + (tsv_dir / f"{prefix}.header_lines.tsv").write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "cohort.vcf\t1\tFORMAT\t\tx\n", + encoding="utf-8", + ) + (tsv_dir / f"{prefix}.file_metadata.tsv").write_text( + "SOURCE_FILE\tFILE_FORMAT\tFILE_DATE\tSOURCE_SOFTWARE\tREFERENCE_GENOME\tHEADER_COUNT\tRECORD_COUNT\n" + "cohort.vcf\tVCFv4.2\t\t\t\t1\t1\n", + encoding="utf-8", + ) + return {"exit_code": 0, "output_size_bytes": 1} + + with mock.patch.object(vcf_rdfizer, "run", return_value=0), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ), mock.patch.object( + vcf_rdfizer, + "run_tsv_conversion_with_metrics", + side_effect=fake_tsv_conversion, + ): + rc = invoke_main( + [ + "--input", + str(input_file), + "--out", + str(out_dir), + "--sample-representation", + "condensed", + "--compression", + "none", + "--keep-tsv", + ] + ) + + self.assertEqual(rc, 0) + rdf_text = (out_dir / "cohort" / "cohort.nt").read_text(encoding="utf-8") + self.assertIn("vocab#CondensedRepresentation", rdf_text) + self.assertIn("vocab#CohortCallMatrix", rdf_text) + self.assertNotIn("vocab#SampleCall", rdf_text) + self.assertNotIn("vocab#FormatFieldValue", rdf_text) + helper_rows = ( + out_dir / ".intermediate" / "tsv" / "cohort.sample_calls.tsv" + ).read_text(encoding="utf-8").splitlines() + self.assertEqual(helper_rows, ["\t".join(vcf_rdfizer.SAMPLE_CALLS_HEADER)]) + def test_render_rules_for_triplet_rewrites_helper_tsv_placeholders(self): """Rule rendering rewrites records/header/metadata and helper TSV placeholders.""" with tempfile.TemporaryDirectory() as td: @@ -1332,13 +2015,15 @@ def test_help_flag_prints_usage_guide(self): self.assertEqual(exc.exception.code, 0) text = out_buf.getvalue() self.assertIn("Examples:", text) - self.assertIn("-m {full,compress,decompress,tsv,index}", text) + self.assertIn("-m {full,compress,decompress,tsv,index,validation}", text) self.assertIn("-i INPUT", text) self.assertIn("--keep-rmlstreamer-rdf-output", text) self.assertIn("--remove-rdf-storage-output", text) self.assertIn("--rdf-compression", text) self.assertIn("--representations", text) self.assertIn("--artifact-compression", text) + self.assertIn("--cottas", text) + self.assertIn("--quiet", text) self.assertNotIn("--keep-rdf", text) self.assertNotIn("--compression", text) @@ -1474,7 +2159,7 @@ def fake_run(cmd, cwd=None, env=None): ) script = str(cmd[-1]) if cmd else "" time_match = re.search( - r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", + r"-o\s+(/data/metrics/timings/tsv/[^\s;]+)", script, ) if metrics_mount and time_match: @@ -1517,9 +2202,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertNotIn("/opt/vcf-rdfizer/run_conversion.sh", " ".join(map(str, commands[0]))) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - run_id = run_metrics_dir.name - tsv_time = run_metrics_dir / "raw_metrics" / "tsv_time" / "sample" / f"{run_id}.txt" - tsv_json = run_metrics_dir / "raw_metrics" / "tsv_metrics" / "sample" / f"{run_id}.json" + tsv_time = run_metrics_dir / "timings" / "tsv" / "sample.txt" + tsv_json = run_metrics_dir / "stages" / "tsv" / "sample.json" self.assertTrue(tsv_time.exists()) self.assertTrue(tsv_json.exists()) @@ -1708,6 +2392,13 @@ def fake_run(cmd, cwd=None, env=None): rows = list(csv.DictReader(handle)) self.assertEqual(rows[-1]["mode"], "compress") self.assertEqual(rows[-1]["status"], "success") + self.assertRegex(run_metrics_dir.name, r"^sample__\d{8}T\d{6}$") + manifest = json.loads((run_metrics_dir / "run.json").read_text()) + summary = json.loads((run_metrics_dir / "summary.json").read_text()) + self.assertEqual(manifest["source_label"], "sample") + self.assertEqual(manifest["inputs"][0]["path"], str(rdf_path.resolve())) + self.assertEqual(summary["status"], "success") + self.assertIn("logs/wrapper.log", summary["logs"]) def test_main_full_mode_prints_triplets_and_logs_total(self): """Full mode prints produced triples and records them in runtime timing log.""" @@ -1731,11 +2422,14 @@ def fake_run(cmd, cwd=None, env=None): sample_dir.mkdir(parents=True, exist_ok=True) (sample_dir / f"{out_name}.nt").write_text("

.\n") payload = {"artifacts": {"output_triples": {"TOTAL": 17}}} - run_metrics_dir = metrics_dir / run_id - run_metrics_dir.mkdir(parents=True, exist_ok=True) - conversion_metrics_dir = run_metrics_dir / "conversion_metrics" / out_name + metrics_mount = next( + part.split(":", 1)[0] + for part in cmd + if isinstance(part, str) and part.endswith(":/data/metrics") + ) + conversion_metrics_dir = Path(metrics_mount) / "stages" / "conversion" conversion_metrics_dir.mkdir(parents=True, exist_ok=True) - (conversion_metrics_dir / f"{run_id}.json").write_text( + (conversion_metrics_dir / f"{out_name}.json").write_text( json.dumps(payload), encoding="utf-8", ) @@ -1895,6 +2589,135 @@ def test_main_compress_mode_requires_rdf_argument(self): rc = invoke_main(["--mode", "compress"]) self.assertEqual(rc, 2) + def test_main_validation_mode_runs_container_local_nt_gzip_validation(self): + """Validation mounts only .nt.gz input and invokes the internal runner.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt.gz" + with gzip.open(rdf_path, "wt", encoding="utf-8") as handle: + handle.write("

.\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True): + rc = invoke_main( + [ + "--mode", + "validation", + "--input", + str(vcf_path), + "--rdf", + str(rdf_path), + "--sample-representation", + "condensed", + "--out", + str(out_dir), + ] + ) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + command = commands[0] + self.assertIn("/opt/vcf-rdfizer/validation/validation_runner.py", command) + self.assertIn("/data/rdf/sample.nt.gz", command) + self.assertIn("condensed", command) + results_dir = next((out_dir / "run_metrics").glob("*/reports/validation/sample")) + self.assertTrue(results_dir.is_dir()) + stage = next((out_dir / "run_metrics").glob("*/stages/validation/sample.json")) + payload = json.loads(stage.read_text()) + self.assertTrue(payload["temporary_rdf"]["decompressed_inside_container"]) + self.assertFalse(payload["temporary_rdf"]["persisted_on_host"]) + with stage.parent.parent.parent.joinpath("metrics.csv").open( + newline="", encoding="utf-8" + ) as handle: + row = next(csv.DictReader(handle)) + self.assertEqual(row["validation_status"], "PASS") + self.assertEqual(row["validation_exit_code"], "0") + + def test_main_validation_mode_reports_existing_results_without_traceback(self): + """A populated results directory exits 2 with a message, not an exception.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt.gz" + with gzip.open(rdf_path, "wt", encoding="utf-8") as handle: + handle.write("

.\n") + metrics_dir = tmp_path / "metrics" + existing = metrics_dir / "reports" / "validation" / "sample" + existing.mkdir(parents=True) + (existing / "summary.json").write_text("{}", encoding="utf-8") + + stderr = StringIO() + with mock.patch.object( + vcf_rdfizer, "metrics_run_directory", return_value=metrics_dir + ), redirect_stderr(stderr): + rc = invoke_main( + [ + "--mode", + "validation", + "--input", + str(vcf_path), + "--rdf", + str(rdf_path), + "--out", + str(tmp_path / "out"), + ] + ) + + self.assertEqual(rc, 2) + self.assertIn("Refusing to overwrite existing validation results", stderr.getvalue()) + + def test_run_validation_mode_accepts_plain_nt_for_full_runs(self): + """The shared validation runner selects --rdf-nt for a plain aggregate.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + vcf_path = tmp_path / "sample.vcf" + vcf_path.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n") + rdf_path = tmp_path / "sample.nt" + rdf_path.write_text("

.\n") + metrics_dir = tmp_path / "metrics" + results_dir = metrics_dir / "reports" / "validation" / "sample" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + stage_result = {} + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + rc = vcf_rdfizer.run_validation_mode( + vcf_path=vcf_path, + rdf_path=rdf_path, + representation="expanded", + validation_id="sample", + results_dir=results_dir, + metrics_dir=metrics_dir, + run_id="20260904T123456", + timestamp="2026-09-04T12:34:56", + image_ref="example/vcf-rdfizer:latest", + filter_oracle="auto", + wrapper_log_path=tmp_path / "wrapper.log", + stage_result=stage_result, + ) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + self.assertIn("--rdf", commands[0]) + self.assertIn("--rdf-format", commands[0]) + self.assertIn("nt", commands[0]) + self.assertIn("/data/rdf/sample.nt", commands[0]) + self.assertEqual(stage_result["rdf_format"], "nt") + self.assertFalse(stage_result["temporary_rdf"]["decompressed_inside_container"]) + def test_main_rejects_spark_partitions_outside_full_mode(self): """--spark-partitions is rejected for non-full modes.""" rc = invoke_main( @@ -2002,7 +2825,7 @@ def test_main_full_mode_keyboard_interrupt_returns_130_and_writes_progress_log(s self.assertEqual(rc, 130) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - progress_log = run_metrics_dir / "progress.log" + progress_log = run_metrics_dir / "logs" / "progress.log" self.assertTrue(progress_log.exists()) self.assertIn("Run interrupted by user signal", progress_log.read_text()) @@ -2216,7 +3039,12 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(len(commands), 1) self.assertIn("ensure_hdt_index.sh", commands[0][-1]) self.assertTrue(any(arg.endswith(":/data/hdt") for arg in commands[0])) - metrics = latest_metrics_run_dir(out_dir / "run_metrics") / "hdt_index_metrics.json" + metrics = ( + latest_metrics_run_dir(out_dir / "run_metrics") + / "stages" + / "index" + / "hdt-sample.hdt.json" + ) payload = json.loads(metrics.read_text()) self.assertEqual(payload["index_status"], "generated") self.assertEqual( @@ -2224,36 +3052,214 @@ def fake_run(cmd, cwd=None, env=None): (tmp_path / "sample.hdt.index.v1-1").resolve(), ) - def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): - """The Docker-side helper initializes the index without issuing a query.""" + def test_main_index_mode_regenerates_existing_hdt_sidecar(self): + """Index mode permits an existing HDT sidecar and invokes regeneration.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) - java_home = tmp_path / "java-home" - bin_dir = java_home / "bin" - bin_dir.mkdir(parents=True) - search = bin_dir / "hdtSearch.sh" - search.write_text( + hdt_path = tmp_path / "sample.hdt" + hdt_path.write_bytes(b"fake-hdt") + existing_index = tmp_path / "sample.hdt.index.v1-1" + existing_index.write_text("old-index\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + rendered = str(cmd[-1]) + if "ensure_hdt_index.sh" in rendered: + existing_index.write_text("new-index\n") + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ): + rc = invoke_main( + [ + "--mode", + "index", + "--hdt", + str(hdt_path), + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(existing_index.read_text(), "new-index\n") + payload = json.loads( + ( + latest_metrics_run_dir(out_dir / "run_metrics") + / "stages" + / "index" + / "hdt-sample.hdt.json" + ).read_text() + ) + self.assertEqual(payload["index_status"], "regenerated") + + def test_main_index_mode_reindexes_existing_cottas(self): + """Index mode invokes the COTTAS adapter and records embedded-index metadata.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + cottas_path = tmp_path / "sample.cottas" + cottas_path.write_text("old-cottas\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + rendered = str(cmd[-1]) + if "cottas_tool.py" in rendered and " reindex " in rendered: + cottas_path.write_text("reindexed-cottas\n") + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ): + rc = invoke_main( + [ + "--mode", + "index", + "--cottas", + str(cottas_path), + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(cottas_path.read_text(), "reindexed-cottas\n") + self.assertEqual(len(commands), 1) + self.assertIn("cottas_tool.py reindex", commands[0][-1]) + self.assertTrue(any(arg.endswith(":/data/cottas") for arg in commands[0])) + payload = json.loads( + ( + latest_metrics_run_dir(out_dir / "run_metrics") + / "stages" + / "index" + / "cottas-sample.cottas.json" + ).read_text() + ) + self.assertEqual(payload["index_format"], "cottas") + self.assertEqual(payload["index_location"], "embedded") + self.assertEqual(payload["index_status"], "regenerated") + self.assertTrue( + (latest_metrics_run_dir(out_dir / "run_metrics") / "summary.json").exists() + ) + + def test_main_index_mode_requires_exactly_one_index_input(self): + """Index mode rejects missing or ambiguous HDT/COTTAS inputs.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + hdt_path = tmp_path / "sample.hdt" + cottas_path = tmp_path / "sample.cottas" + hdt_path.write_bytes(b"hdt") + cottas_path.write_bytes(b"cottas") + + for options in ( + [], + ["--hdt", str(hdt_path), "--cottas", str(cottas_path)], + ): + rc = invoke_main(["--mode", "index", *options, "--out", str(tmp_path / "out")]) + self.assertEqual(rc, 2) + + def test_hdt_index_helper_uses_hdtc_and_verifies_sidecar(self): + """The helper uses hdtc's Java-free bounded-memory index command.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + work_root = tmp_path / "index-work" + work_root.mkdir() + invocation_path = tmp_path / "invocation.txt" + hdtc = tmp_path / "hdtc" + hdtc.write_text( "#!/usr/bin/env bash\n" - "read -r command\n" - "[[ \"$command\" == \"exit\" ]] || exit 3\n" - "printf 'index\\n' > \"${1}.index.v1-1\"\n", + "printf '%s\\n' \"$@\" > \"$HDT_TEST_INVOCATION\"\n" + "[[ \"$1\" == \"--quiet\" && \"$2\" == \"index\" ]] || exit 3\n" + "hdt_path=$3\n" + "printf 'index\\n' > \"${hdt_path}.index.v1-1\"\n", encoding="utf-8", ) - search.chmod(0o755) + hdtc.chmod(0o755) hdt_path = tmp_path / "sample.hdt" hdt_path.write_bytes(b"fake-hdt") + existing_index = Path(str(hdt_path) + ".index.v1-1") + existing_index.write_text("old-index\n", encoding="utf-8") helper = Path(__file__).parents[1] / "src" / "ensure_hdt_index.sh" result = subprocess.run( ["bash", str(helper), str(hdt_path)], - env={**os.environ, "HDT_JAVA_HOME": str(java_home)}, + env={ + **os.environ, + "HDTC_BIN": str(hdtc), + "HDT_INDEX_WORK_ROOT": str(work_root), + "HDT_INDEX_MEMORY_LIMIT": "768M", + "HDT_TEST_INVOCATION": str(invocation_path), + }, capture_output=True, text=True, ) self.assertEqual(result.returncode, 0, msg=result.stderr) self.assertTrue(Path(str(hdt_path) + ".index.v1-1").exists()) + self.assertEqual(existing_index.read_text(encoding="utf-8"), "index\n") self.assertIn(f"HDT index ready: {hdt_path}.index.v1-1", result.stdout) + invocation = invocation_path.read_text(encoding="utf-8").splitlines() + self.assertEqual( + invocation[0:5], + ["--quiet", "index", str(hdt_path), "--memory-limit", "768M"], + ) + self.assertEqual(invocation[5], "--temp-dir") + self.assertTrue(Path(invocation[6]).parent.samefile(work_root)) + self.assertEqual(list(work_root.iterdir()), []) + + def test_hdt_index_helper_restores_existing_sidecar_on_failure(self): + """Failed HDT regeneration restores the sidecar that was moved aside.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + work_root = tmp_path / "index-work" + work_root.mkdir() + hdtc = tmp_path / "hdtc" + hdtc.write_text( + "#!/usr/bin/env bash\n" + "hdt_path=$3\n" + "printf 'partial-index\\n' > \"${hdt_path}.index.v1-1\"\n" + "exit 7\n", + encoding="utf-8", + ) + hdtc.chmod(0o755) + hdt_path = tmp_path / "sample.hdt" + hdt_path.write_bytes(b"fake-hdt") + existing_index = Path(str(hdt_path) + ".index.v1-1") + existing_index.write_text("old-index\n", encoding="utf-8") + helper = Path(__file__).parents[1] / "src" / "ensure_hdt_index.sh" + + result = subprocess.run( + ["bash", str(helper), str(hdt_path)], + env={ + **os.environ, + "HDTC_BIN": str(hdtc), + "HDT_INDEX_WORK_ROOT": str(work_root), + }, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 7, msg=result.stderr) + self.assertEqual(existing_index.read_text(encoding="utf-8"), "old-index\n") + self.assertEqual(list(work_root.iterdir()), []) def test_main_decompress_mode_rejects_unknown_extension(self): """Decompression mode rejects unsupported compressed RDF extensions.""" @@ -2436,9 +3442,91 @@ def fake_run(cmd, cwd=None, env=None): self.assertEqual(len(runner_commands), 1) self.assertIn("target=/work", " ".join(runner_commands[0])) self.assertIn("--methods hdt", " ".join(runner_commands[0])) + self.assertIn("--allow-index-failures", runner_commands[0]) self.assertFalse((out_dir / "sample" / ".compression_partitioned").exists()) self.assertTrue((out_dir / "sample" / "sample.hdt").exists()) + def test_main_full_mode_records_cottas_index_warning_and_continues(self): + """Full mode succeeds and writes an index-warning report for skipped COTTAS output.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + stdout = StringIO() + stderr = StringIO() + + def fake_run(cmd, cwd=None, env=None): + if "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + sample_dir = out_dir / "sample" + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "sample.nt").write_text("

.\n") + return 0 + + def fake_partitioned(**kwargs): + warning = { + "format": "cottas", + "stage": "cottas-index", + "status": "index_unavailable", + "artifact_path": str(out_dir / "sample" / "sample.cottas"), + "message": "COTTAS index creation failed", + } + kwargs["index_warnings"].append(warning) + return True, { + "cottas": { + "exit_code": 1, + "output_path": str(out_dir / "sample" / "sample.cottas"), + "output_size_bytes": 0, + "index_status": "failed", + "index_warning": warning, + } + } + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with ( + mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), + mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), + mock.patch.object(vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets()), + mock.patch.object( + vcf_rdfizer, + "run_partitioned_representation_methods_for_rdf_files", + side_effect=fake_partitioned, + ), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + rc = invoke_main( + [ + "--input", + str(input_dir), + "--rules", + str(rules_path), + "--rdf-storage-mode", + "plain", + "--rdf-compression", + "none", + "--representations", + "cottas", + "--artifact-compression", + "none", + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + warning_path = run_metrics_dir / "reports" / "index_warnings.json" + self.assertTrue(warning_path.exists()) + self.assertEqual(json.loads(warning_path.read_text())["warning_count"], 1) + self.assertIn("Conversion process finished with index warnings.", stdout.getvalue()) + self.assertIn("index_warnings.json", stdout.getvalue()) + self.assertTrue((out_dir / "sample" / "sample.nt").exists()) + def test_main_full_mode_plain_storage_sets_storage_mode(self): """Plain storage passes the canonical storage mode to conversion.""" with tempfile.TemporaryDirectory() as td: @@ -2668,7 +3756,7 @@ def fake_run(cmd, cwd=None, env=None): self.assertTrue((out_dir / "sample_b" / "sample_b.hdt").exists()) run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") - failed_report = run_metrics_dir / "failed_inputs.csv" + failed_report = run_metrics_dir / "reports" / "failed_inputs.csv" self.assertTrue(failed_report.exists()) report_text = failed_report.read_text() self.assertIn("sample_a", report_text) @@ -2898,8 +3986,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("sample", csv_text) self.assertIn("hdt", csv_text) - json_file = run_metrics_dir / "compression_metrics" / "sample" / f"{run_metrics_dir.name}.json" - time_file = run_metrics_dir / "compression_time" / "hdt" / "sample" / f"{run_metrics_dir.name}.txt" + json_file = run_metrics_dir / "stages" / "compression" / "sample.json" + time_file = run_metrics_dir / "timings" / "compression" / "sample" / "hdt.txt" self.assertTrue(json_file.exists()) self.assertTrue(time_file.exists()) diff --git a/test/validation_fixtures.py b/test/validation_fixtures.py new file mode 100644 index 0000000..458ffc7 --- /dev/null +++ b/test/validation_fixtures.py @@ -0,0 +1,611 @@ +"""One declarative fixture, three derived artifacts. + +The mutation harness needs a VCF, the RDF graph that VCF should convert to, and +the parser summary the validator's oracle should compute from it. Deriving all +three from a single specification below is what keeps them honest: a fixture +change cannot make the graph and the oracle disagree by accident. + +``test_validation_container_unit.py`` closes the loop in CI by running the real +``parse_vcf`` over :func:`write_vcf` output and asserting it equals +:func:`parser_summary`, so the hand-derived oracle is pinned to the real one. + +The records are chosen to exercise every branch the validation queries have: +a transition SNV, a transversion SNV, a deletion shape, a multi-allelic site, +a no-ALT site, PASS and non-PASS FILTER values, a missing genotype, a +non-diploid genotype, and a non-GT FORMAT field that nothing currently checks. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer" +XSD_POSITIVE_INTEGER = "http://www.w3.org/2001/XMLSchema#positiveInteger" + +def _runner(): + """Load the shipped validation runner (it lives outside the package path).""" + import importlib.util + + path = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + spec = importlib.util.spec_from_file_location("validation_runner_fixture", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +SOURCE_FILE = "fixture.vcf" +SAMPLES = ("HG001", "HG002") +FILE_FORMAT = "VCFv4.2" +FILE_DATE = "20260101" +SOURCE_SOFTWARE = "vcf-rdfizer-fixture" +REFERENCE_GENOME = "GRCh38" + +#: ``##`` meta-information lines, in file order. The ``#CHROM`` line is not one +#: of these; ``header_count`` below counts only the ``##`` lines. +HEADER_LINES: tuple[tuple[str, str], ...] = ( + ("fileformat", FILE_FORMAT), + ("fileDate", FILE_DATE), + ("source", SOURCE_SOFTWARE), + ("reference", REFERENCE_GENOME), + ("FILTER", ''), + ("INFO", ''), + ("INFO", ''), + ("FORMAT", ''), + ("FORMAT", ''), + # Exercise the structured header paths: a contig with every optional + # attribute, one with none, and a symbolic ALT declaration. + ("contig", ""), + ("contig", ""), + ("ALT", ''), + # An unrecognized key: it must keep only the base HeaderLine type. + ("phasing", "partial"), +) + + +@dataclass(frozen=True) +class FixtureRecord: + """One VCF data line and everything derived from it.""" + + row_id: str + chrom: str + pos: int + record_id: str + ref: str + alt: str + qual: str + filter_value: str + info: str + format_keys: tuple[str, ...] + #: One ``:``-joined payload per sample, aligned to ``format_keys``. + sample_payloads: tuple[str, ...] = field(default_factory=tuple) + + +RECORDS: tuple[FixtureRecord, ...] = ( + # Transition SNV, PASS, both samples called. + FixtureRecord("1", "20", 100, "rs100", "A", "G", "50", "PASS", "AC=1;DB", + ("GT", "DP"), ("0|1:30", "0/0:28")), + # Transversion SNV in the same 1 Mb window as record 1: a POS swap between + # these two is invisible to every aggregate query. + FixtureRecord("2", "20", 300, ".", "A", "C", "99", "PASS", "AC=2", + ("GT", "DP"), ("1/1:41", "0/1:35")), + # Deletion shape in a different window, failing FILTER, missing genotype. + FixtureRecord("3", "20", 2000200, "rs200", "AAT", "A", "12.5", "q10", "AC=1", + ("GT", "DP"), ("0/1:15", "./.:0")), + # Multi-allelic: excluded from q06, classified MULTIALLELIC by q02. + FixtureRecord("4", "21", 500, ".", "C", "T,G", ".", "PASS", "AC=1,1", + ("GT", "DP"), ("1/2:22", "0/1:19")), + # No ALT, haploid genotype: exercises NO_ALT and HAPLOID_* branches. + FixtureRecord("5", "21", 900, ".", "G", ".", "7", "q10", "DB", + ("GT", "DP"), ("0:11", "1:9")), + # A second transition SNV sharing record 1's contig and 1 Mb window. A + # REF/ALT permutation between records 1 and 6 leaves every aggregate + # identical, which is the blind spot the record digest is meant to close. + FixtureRecord("6", "20", 800, ".", "C", "T", "60", "PASS", "AC=1", + ("GT", "DP"), ("0/1:33", "0/1:31")), +) + +TRANSITIONS = {("A", "G"), ("G", "A"), ("C", "T"), ("T", "C")} + + +# --------------------------------------------------------------------------- +# VCF +# --------------------------------------------------------------------------- +def vcf_text() -> str: + lines = [f"##{key}={value}" for key, value in HEADER_LINES] + lines.append( + "#" + "\t".join( + ["CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", *SAMPLES] + ) + ) + for record in RECORDS: + lines.append("\t".join([ + record.chrom, str(record.pos), record.record_id, record.ref, record.alt, + record.qual, record.filter_value, record.info, + ":".join(record.format_keys), *record.sample_payloads, + ])) + return "\n".join(lines) + "\n" + + +def write_vcf(path: Path) -> Path: + path.write_text(vcf_text(), encoding="utf-8") + return path + + +def records_tsv_text() -> str: + """The `vcf_as_tsv.sh` output for this fixture, used by the RDF emitters.""" + header = ["SOURCE_FILE", "ROW_ID", "CHROM", "POS", "ID", "REF", "ALT", "QUAL", + "FILTER", "INFO", "FORMAT", " ".join(SAMPLES)] + rows = ["\t".join(header)] + for record in RECORDS: + rows.append("\t".join([ + SOURCE_FILE, record.row_id, record.chrom, str(record.pos), record.record_id, + record.ref, record.alt, record.qual, record.filter_value, record.info, + ":".join(record.format_keys), " ".join(record.sample_payloads), + ])) + return "\n".join(rows) + "\n" + + +def header_lines_tsv_text() -> str: + rows = ["\t".join(["SOURCE_FILE", "HEADER_INDEX", "HEADER_KEY", "HEADER_VALUE", "RAW_LINE"])] + for index, (key, value) in enumerate(HEADER_LINES, start=1): + rows.append("\t".join([SOURCE_FILE, str(index), key, value, f"{key}={value}"])) + return "\n".join(rows) + "\n" + + +# --------------------------------------------------------------------------- +# RDF graph +# --------------------------------------------------------------------------- +def _literal(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + if value == ".": + return f'"{escaped}"^^<{VCFR}Null>' + return f'"{escaped}"' + + +def _info_entries(record: "FixtureRecord") -> list[tuple[str, str | None]]: + """Split an INFO column into (key, value) pairs; a bare key is a Flag.""" + entries: list[tuple[str, str | None]] = [] + if record.info in ("", "."): + return entries + for item in record.info.split(";"): + if "=" in item: + key, value = item.split("=", 1) + entries.append((key, value)) + else: + entries.append((item, None)) + return entries + + +def base_triples(*, include_qual: bool = True, include_info: bool = True) -> list[str]: + """The triples RMLStreamer produces from `default_rules.ttl`. + + ``include_qual`` is on because the shipped mapping now emits ``vcfr:qual``; + turning it off models the pre-fix mapping, which is how the harness shows + that dropping QUAL is now detected. ``include_info`` models the structured + INFO representation. + """ + file_uri = f"file://{SOURCE_FILE}" + header_uri = f"{file_uri}#header" + out = [ + f"<{file_uri}> <{RDF_TYPE}> <{VCFR}VCFFile> .", + f"<{file_uri}> <{VCFR}fileFormat> {_literal(FILE_FORMAT)} .", + f"<{file_uri}> <{VCFR}sourceSoftware> {_literal(SOURCE_SOFTWARE)} .", + f"<{file_uri}> <{VCFR}referenceGenome> {_literal(REFERENCE_GENOME)} .", + + f"<{file_uri}> <{VCFR}hasHeader> <{header_uri}> .", + f"<{header_uri}> <{RDF_TYPE}> <{VCFR}VCFHeader> .", + ] + for index, (key, value) in enumerate(HEADER_LINES, start=1): + line_uri = f"{file_uri}#header/line/{index}" + out += [ + f"<{header_uri}> <{VCFR}hasHeaderLine> <{line_uri}> .", + f"<{line_uri}> <{RDF_TYPE}> <{VCFR}HeaderLine> .", + f"<{line_uri}> <{VCFR}headerKey> {_literal(key)} .", + f"<{line_uri}> <{VCFR}headerValue> {_literal(value)} .", + ] + for record in RECORDS: + record_uri = f"{file_uri}#record/{record.row_id}" + call_uri = f"{file_uri}#call/{record.row_id}" + out += [ + f"<{file_uri}> <{VCFR}hasRecord> <{record_uri}> .", + f"<{record_uri}> <{RDF_TYPE}> <{VCFR}VCFRecord> .", + f"<{record_uri}> <{VCFR}chrom> {_literal(record.chrom)} .", + f'<{record_uri}> <{VCFR}pos> "{record.pos}"^^<{XSD_INTEGER}> .', + f"<{record_uri}> <{VCFR}recordId> {_literal(record.record_id)} .", + f"<{record_uri}> <{VCFR}ref> {_literal(record.ref)} .", + f"<{record_uri}> <{VCFR}alt> {_literal(record.alt)} .", + f"<{record_uri}> <{VCFR}hasCall> <{call_uri}> .", + f"<{call_uri}> <{RDF_TYPE}> <{VCFR}VariantCall> .", + f"<{call_uri}> <{VCFR}filter> {_literal(record.filter_value)} .", + f"<{call_uri}> <{VCFR}infoRaw> {_literal(record.info)} .", + f"<{call_uri}> <{VCFR}formatRaw> {_literal(':'.join(record.format_keys))} .", + ] + return out + + +def build_graph( + representation: str = "expanded", *, include_qual: bool = True, + include_info: bool = True, include_headers: bool = True, +) -> str: + """Return the N-Triples graph for this fixture, sample triples included. + + The sample triples come from the project's own emitters rather than being + written out here, so the fixture tracks the real implementation. + """ + import tempfile + + import vcf_rdfizer + + if representation not in {"expanded", "condensed"}: + raise ValueError(f"unknown representation: {representation}") + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / f"{SOURCE_FILE}.records.tsv" + records_tsv.write_text(records_tsv_text(), encoding="utf-8") + headers_tsv = tmp_path / f"{SOURCE_FILE}.header_lines.tsv" + headers_tsv.write_text(header_lines_tsv_text(), encoding="utf-8") + + graph = tmp_path / "graph.nt" + graph.write_text( + "\n".join(base_triples(include_qual=include_qual, include_info=include_info)) + "\n", + encoding="utf-8", + ) + if include_qual or include_info: + vcf_rdfizer.append_record_detail_rdf( + records_tsv, headers_tsv, graph, + emit_qual=include_qual, emit_info=include_info, + progress_interval_records=0, + ) + if representation == "expanded": + vcf_rdfizer.append_expanded_sample_rdf(records_tsv, graph, progress_interval_records=0) + else: + vcf_rdfizer.append_condensed_sample_rdf( + records_tsv, headers_tsv, graph, progress_interval_records=0 + ) + if include_headers: + vcf_rdfizer.append_header_representation_rdf(headers_tsv, graph) + return graph.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Parser oracle +# --------------------------------------------------------------------------- +def _classify_shape(ref: str, alt: str) -> str: + import re + + ref, alt = ref.upper(), alt.upper() + if alt == ".": + return "NO_ALT" + if "," in alt: + return "MULTIALLELIC" + if alt == "*" or "[" in alt or "]" in alt or (alt.startswith("<") and alt.endswith(">")): + return "SYMBOLIC_OR_BREAKEND" + if not re.fullmatch(r"[ACGTN]+", ref) or not re.fullmatch(r"[ACGTN]+", alt): + return "OTHER" + if len(ref) == len(alt) == 1: + return "SNV" + if len(ref) == len(alt): + return "MNV_OR_EQUAL_LENGTH_SUBSTITUTION" + return "INSERTION_SHAPE" if len(ref) < len(alt) else "DELETION_SHAPE" + + +def _classify_genotype(raw: str) -> str: + normalized = raw.replace("|", "/") + if "." in normalized: + return "MISSING" + alleles = normalized.split("/") + if len(alleles) == 1: + return "HAPLOID_REF" if alleles[0] == "0" else "HAPLOID_ALT" + if len(alleles) == 2: + if alleles[0] == alleles[1]: + return "HOM_REF" if alleles[0] == "0" else "HOM_ALT" + return "HET" + return "OTHER_PLOIDY" + + +def _genotype_of(record: FixtureRecord, sample_index: int) -> str: + payload = record.sample_payloads[sample_index].split(":") + return payload[record.format_keys.index("GT")] + + +def _format_shape() -> dict: + """FORMAT counters the census needs, mirroring SampleRecordStream widening.""" + records_with_column = records_with_keys = 0 + occurrences = slots = non_empty = 0 + distinct: set[str] = set() + for record in RECORDS: + if record.format_keys: + records_with_column += 1 + payload_fields = [p.split(":") if p else [] for p in record.sample_payloads] + width = max([len(record.format_keys), *(len(f) for f in payload_fields)], default=0) + keys = [ + record.format_keys[i] if i < len(record.format_keys) and record.format_keys[i] + else f"FIELD_{i + 1}" + for i in range(width) + ] + if SAMPLES and keys: + records_with_keys += 1 + distinct.update(keys) + occurrences += width + slots += width * len(SAMPLES) + non_empty += sum( + 1 for f in payload_fields for i in range(width) + if i < len(f) and f[i] != "" + ) + return { + "recordsWithFormatColumn": records_with_column, + "recordsWithFormatKeys": records_with_keys, + "formatKeyOccurrences": occurrences, + "formatValueSlots": slots, + "nonEmptyFormatValues": non_empty, + "distinctFormatKeyCount": len(distinct), + } + + +def _header_shape() -> dict: + """Header counters the census needs, derived with the shipped parser.""" + runner = _runner() + classes: Counter[str] = Counter() + filters = alts = contigs = described = 0 + attributes: Counter[str] = Counter() + for key, value in HEADER_LINES: + class_name = runner.HEADER_LINE_CLASSES.get(key.lower()) + if class_name is None: + continue + classes[class_name] += 1 + if key.lower() not in {"filter", "alt", "contig"}: + continue + fields = runner.parse_structured_header_fields(value) + if not (fields.get("ID") or "").strip(): + continue + if key.lower() == "contig": + contigs += 1 + for attribute in ("length", "md5", "assembly"): + if fields.get(attribute): + attributes[attribute] += 1 + continue + if key.lower() == "filter": + filters += 1 + else: + alts += 1 + if fields.get("Description"): + described += 1 + return { + "headerLineClassCounts": dict(classes), + "filterDefinitionCount": filters, + "altDefinitionCount": alts, + "contigCount": contigs, + "contigAttributeCounts": dict(attributes), + "describedDefinitionCount": described, + } + + +def _info_shape() -> dict: + """INFO counters the census needs, mirroring the emitter's typing rules.""" + declared = {} + for key, value in HEADER_LINES: + if key != "INFO": + continue + import re as _re + + fields = dict(_re.findall(r'(\w+)=("[^"]*"|[^,>]*)', value)) + ident = fields.get("ID", "").strip() + if ident: + declared[ident] = fields.get("Type", "String").strip('"') + values = flags = typed_int = typed_dec = 0 + keys: set[str] = set() + for record in RECORDS: + for key, value in _info_entries(record): + values += 1 + keys.add(key) + if value is None: + flags += 1 + continue + if "," in value or value == ".": + continue + kind = declared.get(key, "String") + try: + if kind == "Integer": + int(value); typed_int += 1 + elif kind == "Float": + float(value); typed_dec += 1 + except ValueError: + pass + return { + "infoValueCount": values, + "infoDefinitionCount": len(keys), + "infoFlagCount": flags, + "infoTypedIntegerCount": typed_int, + "infoTypedDecimalCount": typed_dec, + } + + +def parser_summary( + representation: str = "expanded", *, include_qual: bool = True, + include_info: bool = True, include_headers: bool = True, +) -> dict: + """The summary ``parse_vcf`` must produce for this fixture. + + ``representation`` selects the census expectation, which depends on which + genotype emitter ran. ``include_qual`` mirrors ``build_graph`` so a graph + built without QUAL is compared against an expectation that also omits it. + """ + import re + + density: Counter[tuple[str, int]] = Counter() + shapes: Counter[str] = Counter() + filters: Counter[tuple[str, str]] = Counter() + genotypes: Counter[tuple[str, str]] = Counter() + ac_an: Counter[tuple[int, int]] = Counter() + transitions = transversions = biallelic_snvs = single_alt = q06_eligible = 0 + + for record in RECORDS: + density[(record.chrom, (record.pos - 1) // 1_000_000)] += 1 + shapes[_classify_shape(record.ref, record.alt)] += 1 + ref_upper, alt_upper = record.ref.upper(), record.alt.upper() + if (re.fullmatch(r"[ACGT]", ref_upper) and re.fullmatch(r"[ACGT]", alt_upper) + and ref_upper != alt_upper): + biallelic_snvs += 1 + if (ref_upper, alt_upper) in TRANSITIONS: + transitions += 1 + else: + transversions += 1 + status = ("PASS" if record.filter_value == "PASS" + else "NOT_APPLIED" if record.filter_value == "." else "FAILED") + filters[(status, record.filter_value)] += 1 + + for index, sample in enumerate(SAMPLES): + genotypes[(sample, _classify_genotype(_genotype_of(record, index)))] += 1 + + if record.alt != "." and "," not in record.alt: + single_alt += 1 + an = ac = 0 + for index in range(len(SAMPLES)): + alleles = _genotype_of(record, index).replace("|", "/").split("/") + if any(a == "." for a in alleles): + continue + if len(alleles) not in (1, 2) or any(a not in ("0", "1") for a in alleles): + continue + an += len(alleles) + ac += sum(int(a) for a in alleles) + if an: + ac_an[(an, ac)] += 1 + q06_eligible += 1 + + summary = { + "sampleCount": len(SAMPLES), + "samples": list(SAMPLES), + "totalRecords": len(RECORDS), + "gtRecordCount": sum(1 for record in RECORDS if "GT" in record.format_keys), + "singleAltRecordCount": single_alt, + "q06EligibleSiteCount": q06_eligible, + "headerLineCount": len(HEADER_LINES), + "headerValueCount": sum(1 for _key, value in HEADER_LINES if value != ""), + **_format_shape(), + **_info_shape(), + "fileFormat": FILE_FORMAT, + "referenceGenome": REFERENCE_GENOME, + "sourceSoftware": SOURCE_SOFTWARE, + "fileDate": FILE_DATE, + **_header_shape(), + "q07_file_metadata": { + "fileFormat": FILE_FORMAT, + "referenceGenome": REFERENCE_GENOME, + "sourceSoftware": SOURCE_SOFTWARE, + }, + "q08_header_line_census": [ + {"headerKey": key, "lineCount": count} + for key, count in sorted(Counter(key for key, _ in HEADER_LINES).items()) + ], + "q01_record_density_1mb": [ + {"chrom": chrom, "windowIndex": window, "recordCount": count} + for (chrom, window), count in sorted(density.items()) + ], + "q02_variant_shape_counts": [ + {"variantClass": name, "recordCount": count} for name, count in sorted(shapes.items()) + ], + "q03_titv": { + "biallelicSnvCount": biallelic_snvs, + "transitionCount": transitions, + "transversionCount": transversions, + "tiTvRatio": transitions / transversions if transversions else None, + }, + "q04_filter_distribution": [ + {"filterStatus": status, "filterLexical": lexical, "recordCount": count} + for (status, lexical), count in sorted(filters.items()) + ], + "q05_sample_genotype_counts": [ + {"sampleId": sample, "genotypeClass": genotype_class, "callCount": count} + for (sample, genotype_class), count in sorted(genotypes.items()) + ], + "q06_ac_an_distribution": [ + {"an": an, "ac": ac, "siteCount": count, "af": ac / an} + for (an, ac), count in sorted(ac_an.items()) + ], + } + # The census and digest expectations come from the shipped derivations + # rather than hand-written copies, so the fixture cannot drift from them. + runner = _runner() + summary.update(runner.expected_census( + summary, representation, + info_representation="structured" if include_info else "raw", + header_representation="structured" if include_headers else "basic", + )) + source_component = runner.rml_uri_component(SOURCE_FILE) + digest: Counter[str] = Counter() + for record in RECORDS: + record_iri = ( + f"file://{source_component}#record/" + f"{runner.rml_uri_component(record.row_id)}" + ) + digest[runner.record_digest_bucket([ + record_iri, record.chrom, str(record.pos), record.record_id, + record.ref, record.alt, + record.qual if include_qual else "", + record.filter_value, record.info, + ])] += 1 + summary["q11_record_digest"] = [ + {"bucket": bucket, "recordCount": count} for bucket, count in sorted(digest.items()) + ] + + # Value-level digests, derived the same way the runner derives them. + sample_components = [ + runner.rml_uri_component(uri) + for uri in runner.sample_uri_ids(list(SAMPLES)) + ] + info_digest: Counter[str] = Counter() + format_digest: Counter[str] = Counter() + for record in RECORDS: + row_component = runner.rml_uri_component(record.row_id) + call_iri = f"file://{source_component}#call/{row_component}" + if include_info: + for key, value in _info_entries(record): + if value is None: + continue + info_iri = f"{call_iri}/info/{runner.rml_uri_component(key)}" + info_digest[runner.record_digest_bucket([info_iri, value])] += 1 + payload_fields = [p.split(":") if p else [] for p in record.sample_payloads] + width = max([len(record.format_keys), *(len(f) for f in payload_fields)], default=0) + keys = [ + record.format_keys[i] if i < len(record.format_keys) and record.format_keys[i] + else f"FIELD_{i + 1}" + for i in range(width) + ] + for key_index, key in enumerate(keys): + key_component = runner.rml_uri_component(key) + if representation == "expanded": + for sample_index, fields in enumerate(payload_fields): + cell = fields[key_index] if key_index < len(fields) else "" + if not cell: + continue + value_iri = ( + f"file://{source_component}#sample/{row_component}" + f"/{sample_components[sample_index]}/fmt/{key_component}" + ) + format_digest[runner.record_digest_bucket([value_iri, cell])] += 1 + else: + encoded = "\t".join( + (fields[key_index] if key_index < len(fields) and fields[key_index] + else ".") + for fields in payload_fields + ) + vector_iri = f"{call_iri}/matrix/fmt/{key_component}" + format_digest[runner.record_digest_bucket([vector_iri, encoded])] += 1 + summary["q12_info_value_digest"] = [ + {"bucket": b, "valueCount": c} for b, c in sorted(info_digest.items()) + ] + summary["q13_format_value_digest"] = [ + {"bucket": b, "valueCount": c} for b, c in sorted(format_digest.items()) + ] + if not include_qual: + summary["q09_predicate_census"] = [ + row for row in summary["q09_predicate_census"] + if not row["predicate"].endswith("#qual") + ] + return summary diff --git a/test/validation_mutations.py b/test/validation_mutations.py new file mode 100644 index 0000000..21d8285 --- /dev/null +++ b/test/validation_mutations.py @@ -0,0 +1,474 @@ +"""A catalogue of graph mutations, and what the validator should make of each. + +Every entry names a specific way a conversion could be wrong, and declares +whether the validation suite catches it. Entries with ``known_undetected`` set +are recorded gaps: the harness asserts they are still *not* detected, so +closing a gap makes a test fail and forces both the catalogue and the coverage +documentation to be updated. That is what turns "coverage" into a number +instead of an opinion. + +Mutations operate on N-Triples text so they are engine-independent and can be +replayed under rdflib on the host or Comunica/QLever in the container. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable + +VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +FILE = "file://fixture.vcf" + + +# --------------------------------------------------------------------------- +# N-Triples editing helpers +# --------------------------------------------------------------------------- +def _lines(text: str) -> list[str]: + return [line for line in text.splitlines() if line.strip()] + + +def _join(lines: list[str]) -> str: + return "\n".join(lines) + "\n" + + +def drop_matching(text: str, *, subject: str | None = None, predicate: str | None = None, + limit: int | None = 1) -> str: + """Remove triples matching a subject and/or predicate.""" + out, removed = [], 0 + for line in _lines(text): + matches = ( + (subject is None or line.startswith(f"<{subject}> ")) + and (predicate is None or f"<{predicate}>" in line) + ) + if matches and (limit is None or removed < limit): + removed += 1 + continue + out.append(line) + if removed == 0: + raise AssertionError(f"mutation matched nothing (subject={subject}, predicate={predicate})") + return _join(out) + + +def replace_object(text: str, *, subject: str, predicate: str, new_object: str) -> str: + """Rewrite the object of the first triple matching subject+predicate.""" + out, done = [], False + for line in _lines(text): + if not done and line.startswith(f"<{subject}> ") and f"<{predicate}>" in line: + out.append(f"<{subject}> <{predicate}> {new_object} .") + done = True + continue + out.append(line) + if not done: + raise AssertionError(f"mutation matched nothing ({subject} {predicate})") + return _join(out) + + +def swap_objects(text: str, *, subject_a: str, subject_b: str, predicate: str) -> str: + """Exchange the objects of the same predicate between two subjects.""" + objects: dict[str, str] = {} + for line in _lines(text): + for subject in (subject_a, subject_b): + if line.startswith(f"<{subject}> ") and f"<{predicate}>" in line: + objects[subject] = line.split(f"<{predicate}>", 1)[1].rsplit(" .", 1)[0].strip() + if len(objects) != 2: + raise AssertionError(f"swap needs both subjects to have {predicate}") + text = replace_object(text, subject=subject_a, predicate=predicate, + new_object=objects[subject_b]) + return replace_object(text, subject=subject_b, predicate=predicate, + new_object=objects[subject_a]) + + +def append_lines(text: str, *new: str) -> str: + return _join(_lines(text) + list(new)) + + +def duplicate_subject(text: str, *, subject: str, new_subject: str) -> str: + """Copy every triple of a subject under a new IRI (a spurious extra record).""" + copies = [ + line.replace(f"<{subject}>", f"<{new_subject}>", 1) + for line in _lines(text) + if line.startswith(f"<{subject}> ") + ] + if not copies: + raise AssertionError(f"nothing to duplicate for {subject}") + return append_lines(text, *copies) + + +# --------------------------------------------------------------------------- +# Catalogue +# --------------------------------------------------------------------------- +@dataclass(frozen=True) +class Mutation: + """One way a converted graph can be wrong.""" + + id: str + description: str + #: Which part of the VCF this corrupts, for the coverage matrix. + vcf_element: str + apply: Callable[[str], str] + #: The check expected to catch it, for documentation. + expected_detected_by: str + #: Set when the suite provably cannot catch this yet, with the reason. + known_undetected: str | None = None + #: Representations this mutation is meaningful for. + representations: tuple[str, ...] = ("expanded", "condensed") + #: Fixture options the mutation needs present in the graph. A mutation that + #: targets a triple the shipped mapping does not yet emit still measures + #: something real - "if we emitted this, would we notice it breaking?" - so + #: the fixture is asked to include it rather than the mutation being skipped. + graph_options: tuple[tuple[str, bool], ...] = () + #: Census policy to validate under. A graph carrying triples the census does + #: not model yet would otherwise fail for that reason alone, masking whether + #: the mutation itself is detectable. + mapping_policy: str = "strict" + + +MUTATIONS: tuple[Mutation, ...] = ( + Mutation( + id="drop_record", + description="Remove every triple of one VCFRecord.", + vcf_element="record", + apply=lambda t: drop_matching(t, subject=f"{FILE}#record/1", limit=None), + expected_detected_by="q01/q02 record totals", + ), + Mutation( + id="drop_pos", + description="Remove one record's POS triple.", + vcf_element="POS", + apply=lambda t: drop_matching(t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos"), + expected_detected_by="preflight_record_cardinality", + ), + Mutation( + id="corrupt_pos", + description="Move a record into a different 1 Mb window.", + vcf_element="POS", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos", + new_object='"9100100"^^'), + expected_detected_by="q01_record_density_1mb", + ), + Mutation( + id="permute_pos", + description="Swap POS between two records in the same contig and 1 Mb window.", + vcf_element="record identity", + apply=lambda t: swap_objects( + t, subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/2", + predicate=f"{VCFR}pos"), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="permute_ref_alt", + description="Swap REF and ALT between two transition SNVs in the same window.", + vcf_element="record identity", + apply=lambda t: swap_objects( + swap_objects(t, subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/6", + predicate=f"{VCFR}ref"), + subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/6", + predicate=f"{VCFR}alt"), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="corrupt_chrom", + description="Change one record's CHROM.", + vcf_element="CHROM", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}chrom", new_object='"X"'), + expected_detected_by="q01_record_density_1mb", + ), + Mutation( + id="corrupt_alt", + description="Change one record's ALT so its shape class changes.", + vcf_element="ALT", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}alt", new_object='"GGG"'), + expected_detected_by="q02_variant_shape_counts", + ), + Mutation( + id="retype_pos_as_string", + description="Emit POS as a plain string instead of an integer.", + vcf_element="POS datatype", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos", new_object='"100"'), + expected_detected_by="preflight_position_datatype", + ), + Mutation( + id="duplicate_record", + description="Emit a second copy of a record under a new IRI.", + vcf_element="record", + apply=lambda t: duplicate_subject( + t, subject=f"{FILE}#record/1", new_subject=f"{FILE}#record/1-copy"), + expected_detected_by="q01/q02 record totals", + ), + Mutation( + id="introduce_blank_node", + description="Emit a record as a blank node instead of an IRI.", + vcf_element="graph integrity", + apply=lambda t: append_lines( + t, f'_:orphan <{VCFR}chrom> "20" .'), + expected_detected_by="preflight_blank_nodes", + ), + Mutation( + id="blank_node_object", + description="Point a record at a blank node instead of its call resource.", + vcf_element="graph integrity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}hasCall", + new_object="_:call1"), + expected_detected_by="preflight_blank_nodes", + ), + Mutation( + id="empty_literal", + description="Emit an empty literal where a value was expected.", + vcf_element="graph integrity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1", predicate=f"{VCFR}chrom", new_object='""'), + expected_detected_by="preflight_empty_values", + ), + Mutation( + id="whitespace_only_literal", + description="Emit a literal containing only whitespace.", + vcf_element="graph integrity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/2", predicate=f"{VCFR}chrom", new_object='" "'), + expected_detected_by="preflight_empty_values", + ), + Mutation( + id="duplicate_triple", + description="Emit the same statement twice, as a duplicated RDF part would.", + vcf_element="graph integrity", + apply=lambda t: append_lines( + t, f'<{FILE}#record/1> <{VCFR}chrom> "20" .'), + expected_detected_by="preflight_duplicate_triples", + ), + Mutation( + id="duplicate_whole_graph", + description="Concatenate the graph with itself, as a duplicated part file would.", + vcf_element="graph integrity", + apply=lambda t: t + t, + expected_detected_by="preflight_duplicate_triples", + ), + Mutation( + id="spurious_predicate", + description="Add a triple using a predicate the vocabulary does not define.", + vcf_element="graph completeness", + apply=lambda t: append_lines( + t, f'<{FILE}#record/1> <{VCFR}notARealProperty> "x" .'), + expected_detected_by="q09_predicate_census (extra row)", + ), + Mutation( + id="drop_filter", + description="Remove one FILTER triple.", + vcf_element="FILTER", + apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFR}filter"), + expected_detected_by="q04_filter_distribution", + ), + Mutation( + id="corrupt_filter_lexical", + description="Change a FILTER value while keeping its broad status class.", + vcf_element="FILTER", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/3", predicate=f"{VCFR}filter", new_object='"q20"'), + expected_detected_by="q04_filter_distribution", + ), + Mutation( + id="plain_dot_literal", + description="Emit a missing token as a plain '.' instead of '.'^^vcfr:Null.", + vcf_element="missing-value policy", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/2", predicate=f"{VCFR}recordId", new_object='"."'), + expected_detected_by="preflight_missing_token_conformance (--strict-conformance)", + ), + Mutation( + id="flip_genotype", + description="Change one sample's GT value.", + vcf_element="FORMAT/GT", + apply=lambda t: replace_object( + t, subject=f"{FILE}#sample/1/HG001/fmt/GT", predicate=f"{VCFR}fieldValue", + new_object='"1/1"'), + expected_detected_by="q05_sample_genotype_counts", + representations=("expanded",), + ), + Mutation( + id="drop_sample_call", + description="Remove one SampleCall entirely.", + vcf_element="sample call", + apply=lambda t: drop_matching(t, subject=f"{FILE}#sample/1/HG001", limit=None), + expected_detected_by="q05 per-sample totals", + representations=("expanded",), + ), + Mutation( + id="drop_format_value_dp", + description="Remove a non-GT FORMAT value node (DP).", + vcf_element="FORMAT/DP", + apply=lambda t: drop_matching(t, subject=f"{FILE}#sample/1/HG001/fmt/DP", limit=None), + expected_detected_by="q09_predicate_census", + representations=("expanded",), + ), + Mutation( + id="corrupt_format_value_dp", + description="Change a DP value.", + vcf_element="FORMAT/DP", + apply=lambda t: replace_object( + t, subject=f"{FILE}#sample/1/HG001/fmt/DP", predicate=f"{VCFR}fieldValue", + new_object='"999"'), + expected_detected_by="q13_format_value_digest", + representations=("expanded",), + ), + Mutation( + id="drop_qual", + description="Remove a QUAL triple.", + vcf_element="QUAL", + apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFR}qual"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_all_qual", + description="Emit no QUAL at all, as the mapping did before it was fixed.", + vcf_element="QUAL", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}qual", limit=None), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_qual", + description="Change a QUAL value.", + vcf_element="QUAL", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1", predicate=f"{VCFR}qual", new_object='"0"'), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="drop_info_value", + description="Remove a structured INFO value node.", + vcf_element="INFO", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}hasInfoValue"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_info_value", + description="Change a structured INFO value.", + vcf_element="INFO", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1/info/AC", predicate=f"{VCFR}fieldValue", + new_object='"99"'), + expected_detected_by="q12_info_value_digest", + ), + Mutation( + id="drop_info_definition", + description="Remove an INFO field declaration resource.", + vcf_element="INFO declaration", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#header/line/6", predicate=f"{VCFR}fieldType"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="retype_info_value", + description="Drop the typed integer form of an INFO value.", + vcf_element="INFO typing", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}fieldValueInteger"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_info_raw", + description="Change a record's raw INFO string.", + vcf_element="INFO", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1", predicate=f"{VCFR}infoRaw", new_object='"AC=99"'), + expected_detected_by="q11_record_digest", + ), + Mutation( + id="corrupt_format_vector", + description="Change one sample's value inside a condensed FORMAT vector.", + vcf_element="FORMAT/DP", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/1/matrix/fmt/DP", predicate=f"{VCFR}encodedValues", + new_object='"999\t28"'), + expected_detected_by="q13_format_value_digest", + representations=("condensed",), + ), + Mutation( + id="drop_header_line", + description="Remove one HeaderLine resource.", + vcf_element="header line", + apply=lambda t: drop_matching(t, subject=f"{FILE}#header/line/5", limit=None), + expected_detected_by="q08_header_line_census (Phase 1c)", + ), + Mutation( + id="untype_header_line", + description="Strip a header line's vocabulary subclass.", + vcf_element="header line typing", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#header/line/10", + predicate="http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + expected_detected_by="q10_class_census", + ), + Mutation( + id="drop_contig_attribute", + description="Remove a contig's declared length.", + vcf_element="contig declaration", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}contigLength"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_filter_definition", + description="Remove a FILTER declaration's id.", + vcf_element="FILTER declaration", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}filterId"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_alt_definition", + description="Remove a symbolic ALT declaration's id.", + vcf_element="ALT declaration", + apply=lambda t: drop_matching(t, predicate=f"{VCFR}altId"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_file_date", + description="Remove the declared file date.", + vcf_element="file metadata", + apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFR}fileDate"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_contig_count", + description="Declare the wrong number of contigs.", + vcf_element="contig declaration", + apply=lambda t: replace_object( + t, subject=FILE, predicate=f"{VCFR}contigCount", + new_object='"99"^^'), + expected_detected_by="header census", + known_undetected=( + "contigCount is a derived scalar: the census counts that one triple " + "exists but never reads its value." + ), + ), + Mutation( + id="corrupt_file_metadata", + description="Change the declared fileformat.", + vcf_element="file metadata", + apply=lambda t: replace_object( + t, subject=FILE, predicate=f"{VCFR}fileFormat", new_object='"VCFv9.9"'), + expected_detected_by="q07_file_metadata (Phase 1c)", + ), + Mutation( + id="drop_reference_genome", + description="Remove the declared reference genome.", + vcf_element="file metadata", + apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFR}referenceGenome"), + expected_detected_by="q07_file_metadata (Phase 1c)", + ), + Mutation( + id="wrong_representation_profile", + description="Declare the wrong sample representation profile.", + vcf_element="representation profile", + apply=lambda t: replace_object( + t, subject=FILE, predicate=f"{VCFR}representationProfile", + new_object=f"<{VCFR}ExpandedRepresentation>"), + expected_detected_by="preflight_representation_profile", + representations=("condensed",), + ), +) + + +def for_representation(representation: str) -> tuple[Mutation, ...]: + return tuple(m for m in MUTATIONS if representation in m.representations) diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 9a1600a..3a15403 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -6,16 +6,56 @@ 2) convert VCF -> TSV 3) run RMLStreamer conversion 4) run selected compression/decompression operations -5) persist run and compression metrics +5) optionally regenerate the query index for an existing HDT or COTTAS file +6) persist run and compression metrics The implementation is intentionally split into small helpers so failures can be diagnosed at a specific stage and future workflow changes stay localized. + +Division of labour +------------------ +This file is the *host* side and stays free of heavy data processing: it plans +work, launches containers, and reads their reports back. Everything that walks +the RDF itself lives in the image (``src/``): + +- ``src/vcf_as_tsv.sh`` VCF -> per-input records/header/metadata TSV +- ``src/run_conversion.sh`` RMLStreamer run + part merge into one aggregate +- ``src/partitioned_compression.py`` record-safe chunking, HDT/COTTAS merge +- ``src/cottas_tool.py`` COTTAS convert/merge/reindex/decompress +- ``src/validate_compression.py`` round-trip triple-count check per artifact +- ``src/validation/`` semantic VCF-vs-RDF SPARQL validation + +The one deliberate exception is genotype RDF emission (see the "Multi-sample +(genotype) representations" section): doing it through RML would require +materializing variants x samples helper tables first. + +Section map (search for the banner comments) +-------------------------------------------- +Command execution and Docker environment helpers .. `run`, `CommandLogger`, + `ProgressSession`, `check_docker` +Input discovery and naming helpers ................ `resolve_input_snapshot` +General formatting and file-system utility helpers `format_bytes`, `ensure_dir` +Triple counting and run-level metrics reporting ... `count_triples_in_nt_files` +Console summaries, artifact naming, ... ........... `planned_output_paths` +Destructive filesystem operations ................. `remove_*_with_docker_fallback` +Preflight input collection and disk estimation .... `estimate_pipeline_sizes` +Per-input TSV discovery ........................... `discover_tsv_triplets` +Multi-sample (genotype) representations ........... `append_expanded_sample_rdf`, + `append_condensed_sample_rdf`, `SampleRecordStream` +RML mapping rendering and Docker image resolution . `render_rules_for_triplet` +Compression plan parsing and strategy selection ... `build_compression_methods` +Run metrics layout: naming, manifest, and summary . `write_run_manifest` +Metrics serialization helpers ..................... `update_metrics_csv_*` +Mode runners ...................................... `run_full_mode`, + `run_tsv_mode`, `run_compress_mode`, `run_decompress_mode`, + `run_index_mode`, `run_validation_mode`, `main` """ import argparse import csv import gzip import importlib.resources as importlib_resources +import io import json import os import re @@ -25,13 +65,46 @@ import subprocess import sys import time +from dataclasses import dataclass from datetime import datetime from pathlib import Path +from urllib.parse import quote_plus + +try: + import vcf_rdfizer_gzip +except ImportError: # pragma: no cover - shipped alongside this module + vcf_rdfizer_gzip = None + +try: + from rich.console import Console + from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, + ) +except ImportError: # pragma: no cover - package metadata installs Rich + Console = None + Progress = None + BarColumn = None + SpinnerColumn = None + TaskProgressColumn = None + TextColumn = None + TimeElapsedColumn = None + TimeRemainingColumn = None RMLSTREAMER_JAR_CONTAINER = "/opt/rmlstreamer/RMLStreamer-v2.5.0-standalone.jar" _COMMAND_LOGGER = None _DOCKER_USE_SUDO = False +_ACTIVE_PROGRESS = None +_PROGRESS_ALLOWED = True +_PROGRESS_EVENTS_ALLOWED = True +_QUIET = False +PROGRESS_POLL_INTERVAL_SECONDS = 0.25 COMPRESSED_VCF_EXPANSION_FACTOR = 5.0 TSV_OVERHEAD_FACTOR = 1.10 @@ -84,6 +157,24 @@ COMPRESSION_COMMON_COLUMNS = ["combined_rdf_size_bytes", "compression_methods"] +VALIDATION_METRICS_COLUMNS = [ + "validation_status", + "validation_exit_code", + "validation_wall_seconds", + "validation_user_seconds", + "validation_sys_seconds", + "validation_max_rss_kb", + "validation_results_path", + "validation_rdf_path", + # Benchmark columns. The oracle and the engine compute the same answers + # from the same data, so these two are directly comparable; the engine + # column is a "|"-joined list when several engines ran. + "validation_engines", + "validation_oracle_seconds", + "validation_engine_query_seconds", + "validation_engine_setup_seconds", +] + COMPRESSION_METHOD_COLUMNS = { "gzip": [ "gzip_size_bytes", @@ -184,7 +275,109 @@ DEFAULT_CHUNK_MIN_BYTES = 128 * 1024 * 1024 DEFAULT_CHUNK_MAX_BYTES = 1024 * 1024 * 1024 HDT_INDEX_HELPER_CONTAINER = "/opt/vcf-rdfizer/ensure_hdt_index.sh" +COTTAS_TOOL_CONTAINER = "/opt/vcf-rdfizer/cottas_tool.py" PARTITIONED_COMPRESSION_RUNNER_CONTAINER = "/opt/vcf-rdfizer/partitioned_compression.py" +SAMPLE_CALLS_HEADER = [ + "SOURCE_FILE", + "ROW_ID", + "SAMPLE_INDEX", + "SAMPLE_ID", + "SAMPLE_URI_ID", + "SAMPLE_PAYLOAD", +] +SAMPLE_FORMAT_HEADER = [ + "SOURCE_FILE", + "ROW_ID", + "SAMPLE_INDEX", + "SAMPLE_ID", + "SAMPLE_URI_ID", + "FORMAT_INDEX", + "FORMAT_KEY", + "FORMAT_VALUE", +] +CANONICAL_SAMPLE_RULE_MARKERS = ( + "<#VariantCallToSampleLinkMap>", + "<#SampleCallMap>", + "<#SampleCallToFormatValueLinkMap>", + "<#FormatFieldValueMap>", +) +CANONICAL_SAMPLE_RULE_FRAGMENTS = ( + 'rr:template "file://{SOURCE_FILE}#call/{ROW_ID}"', + "rr:predicate vcfr:hasSampleCall", + 'rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}"', + "rr:class vcfr:SampleCall", + "rr:predicate vcfr:sampleId", + 'rml:reference "SAMPLE_ID"', + "rr:predicate vcfr:hasFormatValue", + 'rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}"', + "rr:class vcfr:FormatFieldValue", + "rr:predicate vcfr:fieldValue", + 'rml:reference "FORMAT_VALUE"', +) +VCFR_NAMESPACE = "https://w3id.org/vcf-rdfizer/vocab#" +RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +XSD_POSITIVE_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#positiveInteger" +SAMPLE_RDF_BUFFER_BYTES = 8 * 1024 * 1024 +# An N-Triples subject is always an IRI reference or a blank node label, so a +# line starting with one of these bytes and ending in " ." is a statement. +_NTRIPLES_SUBJECT_STARTS = (b"<", b"_") +SAMPLE_REPRESENTATION_CHOICES = {"expanded", "condensed"} +# How the INFO column is represented. "raw" is the historical behaviour (an +# opaque vcfr:infoRaw string). "structured" additionally emits one +# vcfr:InfoFieldValue per record and key, which is what makes INFO queryable. +INFO_REPRESENTATION_CHOICES = ("raw", "structured") +DEFAULT_INFO_REPRESENTATION = "structured" +# Semantic validation accepts any artifact the pipeline can produce. Anything +# that is not already plain N-Triples is decoded inside the container first. +VALIDATION_RDF_SUFFIXES = ( + (".nt.gz", "nt.gz"), + (".nt.br", "nt.br"), + (".nt", "nt"), + (".cottas.gz", "cottas.gz"), + (".cottas.br", "cottas.br"), + (".cottas", "cottas"), + (".hdt", "hdt"), +) +VALIDATION_ENGINE_CHOICES = ("comunica", "qlever", "hdt", "cottas") +DEFAULT_VALIDATION_ENGINE = "comunica" + + +def parse_validation_engines(raw: str) -> list[str]: + """Parse --validation-engine into an ordered, de-duplicated engine list. + + Several engines may be requested in one run. Each answers the whole query + set, so their results are cross-checked and their timings are directly + comparable. The first is the primary, whose reports keep the single-engine + layout that existing consumers read. + """ + value = (raw or "").strip() + if value == "all": + return list(VALIDATION_ENGINE_CHOICES) + engines: list[str] = [] + for token in value.split(","): + engine = token.strip() + if not engine: + continue + if engine not in VALIDATION_ENGINE_CHOICES: + allowed = ",".join(VALIDATION_ENGINE_CHOICES) + raise ValueError( + f"Unsupported value '{engine}' for --validation-engine. " + f"Use {allowed}, or all." + ) + if engine not in engines: + engines.append(engine) + if not engines: + raise ValueError("--validation-engine requires at least one engine") + return engines +# Which produced artifacts a full run should semantically validate. "aggregate" +# is the .nt/.nt.gz RMLStreamer output; the others are the selected +# representations, each decoded back to N-Triples before it is checked. +VALIDATION_TARGET_CHOICES = ("aggregate", "hdt", "cottas") +DEFAULT_VALIDATION_TARGETS = "aggregate" +# This is an internal rules-compatibility value, not a third public +# representation. It means that custom helper TSV rows must be materialized. +SAMPLE_HELPER_STRATEGY_MATERIALIZED = "expanded" +METRICS_LAYOUT_VERSION = 2 # --------------------------------------------------------------------------- @@ -206,17 +399,29 @@ def run(self, cmd, cwd=None, env=None): self._handle.write(f"cwd={cwd}\n") self._handle.flush() - result = subprocess.run( - cmd, - cwd=cwd, - env=env, - stdout=self._handle, - stderr=self._handle, - text=True, - ) - self._handle.write(f"[exit {result.returncode}]\n") + if _ACTIVE_PROGRESS is not None and _ACTIVE_PROGRESS.enabled: + process = subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=self._handle, + stderr=self._handle, + text=True, + ) + exit_code = _ACTIVE_PROGRESS.wait_for_process(process) + else: + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + stdout=self._handle, + stderr=self._handle, + text=True, + ) + exit_code = result.returncode + self._handle.write(f"[exit {exit_code}]\n") self._handle.flush() - return result.returncode + return exit_code def close(self): if not self._handle.closed: @@ -244,6 +449,230 @@ def success_symbol() -> str: return ui_symbol("✅", "[ok]") +def progress_ui_enabled() -> bool: + """Return whether transient Rich progress output should be displayed.""" + if not _PROGRESS_ALLOWED or Progress is None or Console is None: + return False + if os.environ.get("VCF_RDFIZER_NO_PROGRESS"): + return False + if os.environ.get("CI"): + return False + stream = getattr(sys, "stderr", None) + isatty = getattr(stream, "isatty", None) + return bool(callable(isatty) and isatty()) + + +def progress_events_enabled() -> bool: + """Return whether progress sidecars should be consumed at all. + + Rich needs a TTY to redraw a spinner, but compression is often launched + through ``tee``, a scheduler, or a remote shell with redirected stderr. + Keep collecting the same low-volume events in those cases so + ``ProgressSession`` can render readable line-based status instead. + """ + if not _PROGRESS_EVENTS_ALLOWED or (not _PROGRESS_ALLOWED and not _QUIET): + return False + if os.environ.get("VCF_RDFIZER_NO_PROGRESS"): + return False + return not os.environ.get("CI") + + +class ProgressSession: + """Render low-volume progress events from one Docker operation with Rich. + + The container writes newline-delimited JSON to ``path``. The host polls the + small sidecar while the Docker process runs, keeping command logs and + binary subprocess stdout separate from terminal UI output. + """ + + def __init__(self, path: Path | None, label: str): + self.path = path + self.label = label + self.enabled = progress_events_enabled() + # ``--quiet`` keeps producing/consuming sidecar events so the normal + # progress bookkeeping remains available to logs and orchestration, + # but disables every terminal rendering path. + self.render_enabled = self.enabled and _PROGRESS_ALLOWED + self.rich_enabled = self.render_enabled and progress_ui_enabled() + self._offset = 0 + self._progress = None + self._starter_task = None + self._tasks: dict[str, int] = {} + self._previous = None + self._plain_event_states: dict[str, tuple[str, object, object]] = {} + + def __enter__(self): + global _ACTIVE_PROGRESS + self._previous = _ACTIVE_PROGRESS + _ACTIVE_PROGRESS = self + if self.path is not None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.unlink(missing_ok=True) + + if not self.render_enabled: + return self + if not self.rich_enabled: + eprint(f"{self.label}: started") + return self + + console = Console(stderr=True, highlight=False) + self._progress = Progress( + SpinnerColumn(), + TextColumn("{task.description}", style="progress.description", markup=False), + BarColumn(), + TaskProgressColumn(), + TextColumn("{task.fields[detail]}", markup=False), + TimeElapsedColumn(), + TimeRemainingColumn(), + console=console, + refresh_per_second=8, + transient=True, + auto_refresh=False, + ) + self._starter_task = self._progress.add_task( + self.label, + total=None, + detail="starting", + ) + self._progress.start() + return self + + @staticmethod + def _number(value): + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return value + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None + + @staticmethod + def _detail(event: dict) -> str: + completed = ProgressSession._number(event.get("completed")) + unit = event.get("unit") + if completed is None: + detail = "" + elif unit == "bytes": + detail = f"{format_bytes(int(completed))}" + elif unit == "triples": + detail = f"{int(completed):,} triples" + elif unit == "chunks": + detail = f"{int(completed):,} chunks" + elif unit == "parts": + detail = f"{int(completed):,} parts" + elif unit == "queries": + detail = f"{int(completed):,} queries" + else: + detail = f"{int(completed):,}" + + parts = ProgressSession._number(event.get("parts")) + if parts is not None and unit != "parts": + detail = f"{detail} · {int(parts):,} parts" if detail else f"{int(parts):,} parts" + extra = event.get("detail") + if extra: + detail = f"{detail} · {extra}" if detail else str(extra) + return detail + + def _update_event(self, event: dict): + if not self.render_enabled: + return + stage = str(event.get("stage") or "work") + phase = str(event.get("phase") or "working") + if self._progress is None: + # A redirected terminal cannot host a redrawable Rich spinner. + # Emit compact, state-changing lines instead so lengthy + # compression still visibly advances in a terminal or log. + completed = event.get("completed") + total = event.get("total") + state = (phase, completed, total) + if self._plain_event_states.get(stage) != state: + self._plain_event_states[stage] = state + detail = self._detail(event) + suffix = f" — {detail}" if detail else "" + eprint(f" {self.label}: {stage} {phase}{suffix}") + return + task_id = self._tasks.get(stage) + if task_id is None: + if self._starter_task is not None: + self._progress.remove_task(self._starter_task) + self._starter_task = None + total = self._number(event.get("total")) + task_id = self._progress.add_task( + f"{stage}: {phase}", + total=total if total is not None else None, + detail=self._detail(event), + ) + self._tasks[stage] = task_id + return + + update = { + "description": f"{stage}: {phase}", + "detail": self._detail(event), + } + if "total" in event: + total = self._number(event.get("total")) + update["total"] = total if total is not None else None + completed = self._number(event.get("completed")) + if completed is not None: + update["completed"] = completed + self._progress.update(task_id, **update) + + def poll_events(self): + """Consume complete JSONL events without retaining the event stream.""" + if not self.enabled or self.path is None or not self.path.exists(): + return + try: + with self.path.open("rb") as handle: + handle.seek(self._offset) + data = handle.read() + except OSError: + return + + consumed = 0 + for raw_line in data.splitlines(keepends=True): + if not raw_line.endswith(b"\n"): + break + consumed += len(raw_line) + try: + event = json.loads(raw_line.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + if isinstance(event, dict): + self._update_event(event) + self._offset += consumed + + def wait_for_process(self, process): + """Wait while polling progress, without a second monitor thread.""" + while process.poll() is None: + self.poll_events() + if self._progress is not None: + self._progress.refresh() + time.sleep(PROGRESS_POLL_INTERVAL_SECONDS) + self.poll_events() + if self._progress is not None: + self._progress.refresh() + return process.returncode + + def __exit__(self, exc_type, exc_value, traceback): + global _ACTIVE_PROGRESS + self.poll_events() + if self._progress is not None: + self._progress.stop() + elif self.render_enabled: + eprint(f"{self.label}: finished") + if self.path is not None: + try: + self.path.unlink() + except OSError: + pass + _ACTIVE_PROGRESS = self._previous + return False + + class RunTracker: """Track run progress and intermediate artifacts for safe interruption cleanup.""" @@ -354,7 +783,25 @@ def run(cmd, cwd=None, env=None): """ if _COMMAND_LOGGER is not None: return _COMMAND_LOGGER.run(cmd, cwd=cwd, env=env) - return subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True).returncode + if _ACTIVE_PROGRESS is not None and _ACTIVE_PROGRESS.enabled: + process = subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return _ACTIVE_PROGRESS.wait_for_process(process) + # Only the exit code is returned, so discard the streams at the fd level. + # `capture_output=True` would buffer an entire container's output in memory + # before throwing it away. + return subprocess.run( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode def docker_cmd_prefix(*, use_sudo: bool | None = None): @@ -379,6 +826,28 @@ def docker_run_base(*, as_user: bool = True): return base +def docker_hdt_index_env_args() -> list[str]: + """Forward an optional host-side hdtc memory budget into Docker.""" + memory_limit = os.environ.get("HDT_INDEX_MEMORY_LIMIT", "").strip() + if not memory_limit: + return [] + return ["-e", f"HDT_INDEX_MEMORY_LIMIT={memory_limit}"] + + +def docker_hdt_merge_env_args() -> list[str]: + """Forward an optional host-side hdtc merge memory budget into Docker.""" + memory_limit = os.environ.get("HDT_MERGE_MEMORY_LIMIT", "").strip() + if not memory_limit: + return [] + return ["-e", f"HDT_MERGE_MEMORY_LIMIT={memory_limit}"] + + +def docker_cottas_merge_env_args() -> list[str]: + """Forward an optional bounded COTTAS streaming-merge batch size.""" + batch_rows = os.environ.get("COTTAS_MERGE_BATCH_ROWS", "").strip() + return ["-e", f"COTTAS_MERGE_BATCH_ROWS={batch_rows}"] if batch_rows else [] + + def _can_write_dir(path: Path) -> bool: """Best-effort write probe for directories.""" try: @@ -519,28 +988,6 @@ def list_vcfs_in_dir(path: Path): return files -def resolve_input(input_path: Path): - """Legacy input resolver (single mount + container input path).""" - if not input_path.exists(): - raise ValueError(f"Input path not found: {input_path}") - - if input_path.is_file(): - if not is_vcf_file(input_path): - raise ValueError("Input file must end with .vcf or .vcf.gz") - input_dir = input_path.parent - container_input = f"/data/in/{input_path.name}" - return input_dir, container_input - - if input_path.is_dir(): - vcfs = list_vcfs_in_dir(input_path) - if not vcfs: - raise ValueError("No .vcf or .vcf.gz files found in the input directory") - container_input = "/data/in" - return input_path, container_input - - raise ValueError("Input path must be a file or a directory") - - def vcf_output_prefix(path: Path) -> str: """Derive stable sample prefix from VCF filename.""" name = path.name @@ -635,7 +1082,7 @@ def file_size_bytes(path: Path): def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: - """Return HDT Java's non-empty versioned index sidecar.""" + """Return the non-empty canonical HDT versioned index sidecar.""" for candidate in sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")): size = file_size_bytes(candidate) if size is not None and size > 0: @@ -643,261 +1090,54 @@ def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: return None -def write_nt_chunk(chunk_path: Path, source_paths: list[Path]) -> int: - """Concatenate one or more RDF files into a chunk-local `.nt` input.""" - ensure_dir(chunk_path.parent) - total_bytes = 0 - with chunk_path.open("w", encoding="utf-8") as out_handle: - for source_path in source_paths: - with source_path.open("r", encoding="utf-8", errors="replace") as in_handle: - for line in in_handle: - out_handle.write(line) - total_bytes += len(line.encode("utf-8")) - return total_bytes - - -def split_nt_file_for_hdt( - source_path: Path, - chunk_dir: Path, - *, - target_bytes: int, - max_bytes: int, -) -> list[Path]: - """Split an oversized RDF file into line-preserving chunk files for HDT conversion.""" - ensure_dir(chunk_dir) - chunk_paths: list[Path] = [] - chunk_handle = None - chunk_path = None - chunk_size = 0 - - def open_chunk(index: int): - path = chunk_dir / f"{source_path.stem}.split-{index:05d}.nt" - return path, path.open("w", encoding="utf-8") - - try: - with source_path.open("r", encoding="utf-8", errors="replace") as in_handle: - chunk_index = 0 - for line in in_handle: - line_size = len(line.encode("utf-8")) - if chunk_handle is None: - chunk_path, chunk_handle = open_chunk(chunk_index) - chunk_paths.append(chunk_path) - chunk_size = 0 - chunk_index += 1 - elif chunk_size > 0 and ( - chunk_size >= target_bytes or chunk_size + line_size > max_bytes - ): - chunk_handle.close() - chunk_path, chunk_handle = open_chunk(chunk_index) - chunk_paths.append(chunk_path) - chunk_size = 0 - chunk_index += 1 - - chunk_handle.write(line) - chunk_size += line_size - finally: - if chunk_handle is not None and not chunk_handle.closed: - chunk_handle.close() - - return chunk_paths or [source_path] - - -def iter_rdf_binary_lines(path: Path): - """Yield RDF records from plain or gzip-compressed line-oriented RDF.""" - opener = gzip.open if path.name.endswith(".gz") else Path.open - with opener(path, "rb") as handle: - for line in handle: - yield line - +def open_rdf_binary(path: Path): + """Open plain or gzip line-oriented RDF for buffered binary line iteration. -def plan_record_safe_rdf_chunks( - source_paths: list[Path], - chunk_dir: Path, - *, - target_bytes: int, - min_bytes: int, - max_bytes: int, - guide_path: Path | None = None, -) -> tuple[list[Path], dict]: - """Create bounded RDF chunks without splitting a line-level statement. - - The guide is written as boundaries are discovered during this single - sequential pass. A separate pre-scan would read/decompress the complete - aggregate twice, so the guide and chunk files are produced together. - Logical offsets are uncompressed offsets and therefore work for both plain - and gzip-backed aggregate sources. + ``GzipFile.readline`` is markedly slower than a ``BufferedReader`` wrapped + around the same stream, which matters when the caller walks every record of + a cohort-scale aggregate. """ - if not source_paths: - return [], {"source_file_count": 0, "chunk_count": 0, "chunk_input_bytes": 0} - if target_bytes <= 0 or min_bytes <= 0 or max_bytes <= 0: - raise ValueError("RDF chunk sizes must be positive.") - if min_bytes > target_bytes or target_bytes > max_bytes: - raise ValueError("RDF chunk sizes must satisfy min <= target <= max.") - - ensure_dir(chunk_dir) - chunk_paths: list[Path] = [] - guide_chunks: list[dict] = [] - chunk_handle = None - chunk_path = None - chunk_size = 0 - chunk_start_offset = 0 - chunk_start_record = 0 - logical_offset = 0 - record_count = 0 - total_bytes = 0 - chunk_index = 0 - - def close_chunk(): - nonlocal chunk_handle, chunk_path, chunk_size - if chunk_handle is None or chunk_path is None: - return - chunk_handle.close() - chunk_paths.append(chunk_path) - guide_chunks.append( - { - "chunk_id": len(guide_chunks), - "path": str(chunk_path), - "start_record": chunk_start_record, - "end_record": record_count, - "start_uncompressed_byte": chunk_start_offset, - "end_uncompressed_byte": logical_offset, - "record_count": record_count - chunk_start_record, - "payload_bytes": chunk_size, - } - ) - chunk_handle = None - chunk_path = None - chunk_size = 0 + if path.name.endswith(".gz"): + return io.BufferedReader(gzip.open(path, "rb")) + return path.open("rb") - try: - for source_path in source_paths: - for line in iter_rdf_binary_lines(source_path): - if not line.endswith(b"\n"): - raise ValueError( - f"RDF source contains a non-line-terminated record: {source_path}" - ) - line_size = len(line) - if chunk_handle is None: - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - chunk_handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - elif chunk_size > 0 and ( - (chunk_size >= target_bytes and chunk_size >= min_bytes) - or chunk_size + line_size > max_bytes - ): - close_chunk() - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_index += 1 - chunk_handle = chunk_path.open("wb") - chunk_start_offset = logical_offset - chunk_start_record = record_count - - chunk_handle.write(line) - chunk_size += line_size - logical_offset += line_size - total_bytes += line_size - record_count += 1 - finally: - close_chunk() - - plan = { - "source_file_count": len(source_paths), - "source_paths": [str(path) for path in source_paths], - "chunk_count": len(chunk_paths), - "chunk_input_bytes": total_bytes, - "record_count": record_count, - "target_chunk_bytes": target_bytes, - "min_chunk_bytes": min_bytes, - "max_chunk_bytes": max_bytes, - "chunks": guide_chunks, - } - if guide_path is not None: - ensure_dir(guide_path.parent) - plan["guide_path"] = str(guide_path) - guide_path.write_text(json.dumps(plan, indent=2) + "\n", encoding="utf-8") - return chunk_paths, plan +def is_triple_line(line: bytes) -> bool: + """Return whether a serialized line represents an N-Triples statement. -def plan_partitioned_hdt_chunks( - rdf_paths: list[Path], - chunk_dir: Path, - *, - target_bytes: int, - min_bytes: int, - max_bytes: int, -) -> tuple[list[Path], dict]: - """Plan chunk-local `.nt` inputs for partitioned HDT generation. - - The goal is to keep HDT conversion work units small enough to be fast, - while also avoiding the pathological "many tiny HDTs" case. Existing RDF - part files are treated as the first split boundary, and only oversized - parts are re-split on line boundaries. + This mirrors the container-side predicate in ``validate_compression.py`` and + ``partitioned_compression.py`` so a host-side fallback count can never + disagree with the authoritative count produced inside Docker. The leading + check short-circuits the ordinary `` ... .`` line without + allocating a stripped copy of it. """ - ensure_dir(chunk_dir) - - prepared_inputs: list[tuple[Path, int]] = [] - for rdf_path in rdf_paths: - size = int(file_size_bytes(rdf_path) or 0) - if size <= max_bytes: - prepared_inputs.append((rdf_path, size)) - continue - for split_path in split_nt_file_for_hdt( - rdf_path, - chunk_dir / "_split_inputs", - target_bytes=target_bytes, - max_bytes=max_bytes, - ): - prepared_inputs.append((split_path, int(file_size_bytes(split_path) or 0))) - - chunk_groups: list[list[tuple[Path, int]]] = [] - current_group: list[tuple[Path, int]] = [] - current_size = 0 - for path, size in prepared_inputs: - if not current_group: - current_group = [(path, size)] - current_size = size - continue - if current_size < min_bytes or current_size + size <= target_bytes: - current_group.append((path, size)) - current_size += size - continue - chunk_groups.append(current_group) - current_group = [(path, size)] - current_size = size - if current_group: - chunk_groups.append(current_group) - - chunk_inputs: list[Path] = [] - chunk_input_bytes = 0 - for chunk_index, group in enumerate(chunk_groups): - chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" - chunk_input_bytes += write_nt_chunk(chunk_path, [path for path, _size in group]) - chunk_inputs.append(chunk_path) - - plan = { - "source_file_count": len(rdf_paths), - "prepared_input_count": len(prepared_inputs), - "chunk_count": len(chunk_inputs), - "chunk_input_bytes": chunk_input_bytes, - } - return chunk_inputs, plan + if line.endswith(b".\n") and line[:1] in _NTRIPLES_SUBJECT_STARTS: + return True + stripped = line.strip() + return bool(stripped) and not stripped.startswith(b"#") and stripped.endswith(b".") +# --------------------------------------------------------------------------- +# Triple counting and run-level metrics reporting +# The container stages own the authoritative counts; the helpers here read +# them back and provide host-side fallbacks. +# --------------------------------------------------------------------------- def count_triples_in_nt_files(paths: list[Path]) -> int | None: - """Count triples in RDF line-oriented files as a fallback when metrics are missing.""" + """Count triples in RDF line-oriented files as a fallback when metrics are missing. + + The scan stays at the byte level: decoding a cohort-scale aggregate to + ``str`` only to run a regex per line costs several times more than the + read itself. + """ total = 0 matched_any = False - pattern = re.compile(r"^\s*[^#].*\.\s*$") for path in paths: if not path.exists() or not path.is_file(): continue try: - opener = gzip.open if path.name.endswith(".gz") else Path.open - with opener(path, "rt", encoding="utf-8", errors="replace") as handle: + with open_rdf_binary(path) as handle: for line in handle: - if pattern.match(line): + if is_triple_line(line): total += 1 matched_any = True except OSError: @@ -926,6 +1166,7 @@ def read_conversion_total_triples(metrics_dir: Path, output_name: str, run_id: s """Read TOTAL triple count for one conversion output from conversion metrics JSON.""" safe_name = safe_metrics_name(output_name) candidates = [ + metrics_dir / "stages" / "conversion" / f"{safe_name}.json", metrics_dir / "conversion_metrics" / safe_name / f"{run_id}.json", metrics_dir / "conversion_metrics" / safe_name / run_id, # Backward compatibility with older artifact names: @@ -965,12 +1206,13 @@ def collect_full_mode_total_triples(metrics_dir: Path, run_id: str): total = 0 found = False candidate_files = [] + candidate_files.extend(sorted((metrics_dir / "stages" / "conversion").glob("*.json"))) candidate_files.extend(sorted(metrics_dir.glob("conversion_metrics/*/*"))) # Backward compatibility with older artifact names: candidate_files.extend(sorted(metrics_dir.glob(f"conversion-metrics-*-{run_id}.json"))) for metrics_json in candidate_files: - if ( + if metrics_json.parent.name != "conversion" and ( metrics_json.name != run_id and metrics_json.name != f"{run_id}.json" and not metrics_json.name.endswith(f"-{run_id}.json") @@ -1037,8 +1279,9 @@ def append_wrapper_timing_log( def write_failed_inputs_report(*, metrics_dir: Path, failures: list[dict]): """Write per-input failure summary for multi-input modes.""" - ensure_dir(metrics_dir) - report_path = metrics_dir / "failed_inputs.csv" + report_dir = metrics_dir / "reports" + ensure_dir(report_dir) + report_path = report_dir / "failed_inputs.csv" header = [ "input_index", "input_vcf", @@ -1062,6 +1305,23 @@ def write_failed_inputs_report(*, metrics_dir: Path, failures: list[dict]): return report_path +def write_index_warnings_report(*, metrics_dir: Path, run_id: str, warnings: list[dict]): + """Write non-fatal full-run HDT/COTTAS index warnings as JSON.""" + report_dir = metrics_dir / "reports" + ensure_dir(report_dir) + report_path = report_dir / "index_warnings.json" + payload = { + "run_id": run_id, + "warning_count": len(warnings), + "warnings": warnings, + } + report_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return report_path + + +# --------------------------------------------------------------------------- +# Console summaries, artifact naming, and output-collision planning +# --------------------------------------------------------------------------- def print_nt_hdt_summary( *, output_root: Path, @@ -1189,7 +1449,7 @@ def planned_output_paths( ) if any(method in HDT_COMPRESSION_METHODS for method in methods): planned.add(target_dir / f"{output_name}.hdt") - # The pinned HDT Java package generates this versioned sidecar. + # The pinned Java-free indexer generates this canonical sidecar. planned.add(target_dir / f"{output_name}.hdt.index.v1-1") if any(method in COTTAS_COMPRESSION_METHODS for method in methods): planned.add(target_dir / f"{output_name}.cottas") @@ -1244,6 +1504,11 @@ def compression_method_label_for_path(path: Path, method: str) -> str: return labels.get(method, method) +# --------------------------------------------------------------------------- +# Destructive filesystem operations +# Container stages can leave artifacts owned by another uid, so every +# deletion retries inside the image before it is reported as a failure. +# --------------------------------------------------------------------------- def remove_file_with_docker_fallback( *, path: Path, @@ -1384,6 +1649,9 @@ def cleanup_interrupted_full_run( return removed, failed +# --------------------------------------------------------------------------- +# Preflight input collection and disk-footprint estimation +# --------------------------------------------------------------------------- def existing_parent(path: Path) -> Path: """Return the closest existing parent path (used for disk free-space anchor).""" cur = path @@ -1412,20 +1680,42 @@ def collect_input_vcfs(input_path: Path): return [] +def uncompressed_vcf_bytes(vcf: Path) -> tuple[float, str]: + """Return ``(uncompressed_bytes, method)`` for one VCF input. + + A gzip input's real uncompressed size is read from its structure when that + is possible without inflating it (see :mod:`vcf_rdfizer_gzip`); otherwise + this falls back to the coarse expansion factor rather than spending a full + decompression pass on a preflight estimate. + """ + size = vcf.stat().st_size + if not vcf.name.endswith(".vcf.gz"): + return float(size), "stat" + if vcf_rdfizer_gzip is not None: + try: + measured, method = vcf_rdfizer_gzip.uncompressed_size_without_inflating(vcf) + except OSError: + measured, method = None, "unknown" + # A `.vcf.gz` whose content is not really gzip reports method "stat"; + # that is not a measurement of anything, so keep the conservative + # expansion factor rather than under-stating a disk-space warning. + if measured is not None and method != "stat": + return float(measured), method + return size * COMPRESSED_VCF_EXPANSION_FACTOR, "estimated" + + def estimate_pipeline_sizes(vcf_files, out_dir: Path): """Estimate rough TSV/RDF footprint for preflight disk-space warnings.""" input_bytes = 0 est_tsv_bytes = 0 est_rdf_low_bytes = 0 est_rdf_high_bytes = 0 + methods: set[str] = set() for vcf in vcf_files: - size = vcf.stat().st_size - input_bytes += size - if vcf.name.endswith(".vcf.gz"): - expanded_vcf = size * COMPRESSED_VCF_EXPANSION_FACTOR - else: - expanded_vcf = float(size) + input_bytes += vcf.stat().st_size + expanded_vcf, method = uncompressed_vcf_bytes(vcf) + methods.add(method) est_tsv_bytes += expanded_vcf * TSV_OVERHEAD_FACTOR est_rdf_low_bytes += expanded_vcf * RDF_EXPANSION_LOW_FACTOR @@ -1441,11 +1731,15 @@ def estimate_pipeline_sizes(vcf_files, out_dir: Path): "rdf_high_bytes": int(est_rdf_high_bytes), "free_disk_bytes": int(free_disk_bytes), "disk_anchor": out_anchor, + # "estimated" means at least one gzip input fell back to the coarse + # expansion factor; anything else means every input size was measured. + "input_size_methods": sorted(methods), + "uncompressed_input_estimated": "estimated" in methods, } # --------------------------------------------------------------------------- -# Mapping/rules and Docker image management helpers +# Per-input TSV discovery # --------------------------------------------------------------------------- def slugify(value: str) -> str: """Normalize a value for safe filesystem naming.""" @@ -1483,196 +1777,1284 @@ def discover_tsv_triplets(tsv_dir: Path): ) -def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_format_tsv: Path): - """Materialize per-sample helper TSVs from records.tsv. - - These helper tables keep records.tsv user-facing and simple while providing - deterministic row-wise inputs for rules that map: - - one `SampleCall` per sample/record - - one `FormatFieldValue` per sample/record/FORMAT key - """ +# --------------------------------------------------------------------------- +# Multi-sample (genotype) representations +# Expanded and condensed genotype RDF is emitted here rather than by +# RMLStreamer: the equivalent RML maps would first have to materialize +# variants x samples (x FORMAT keys) helper TSV rows. +# See docs/sample-representation-guide.md for the emitted shapes. +# --------------------------------------------------------------------------- +def write_sample_support_headers(sample_calls_tsv: Path, sample_format_tsv: Path): + """Create empty sample helper tables with their canonical TSV headers.""" sample_calls_tsv.parent.mkdir(parents=True, exist_ok=True) sample_format_tsv.parent.mkdir(parents=True, exist_ok=True) - - def sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: - candidate = re.sub(r"[^A-Za-z0-9._~-]+", "_", sample_id).strip("_") - if not candidate: - candidate = f"sample_{fallback_index}" - return candidate - with sample_calls_tsv.open("w", newline="", encoding="utf-8") as sample_calls_handle, \ sample_format_tsv.open("w", newline="", encoding="utf-8") as sample_format_handle: - sample_calls_writer = csv.writer(sample_calls_handle, delimiter="\t") - sample_format_writer = csv.writer(sample_format_handle, delimiter="\t") - - sample_calls_writer.writerow( - [ - "SOURCE_FILE", - "ROW_ID", - "SAMPLE_INDEX", - "SAMPLE_ID", - "SAMPLE_URI_ID", - "SAMPLE_PAYLOAD", - ] - ) - sample_format_writer.writerow( - [ - "SOURCE_FILE", - "ROW_ID", - "SAMPLE_INDEX", - "SAMPLE_ID", - "SAMPLE_URI_ID", - "FORMAT_INDEX", - "FORMAT_KEY", - "FORMAT_VALUE", - ] - ) + csv.writer(sample_calls_handle, delimiter="\t").writerow(SAMPLE_CALLS_HEADER) + csv.writer(sample_format_handle, delimiter="\t").writerow(SAMPLE_FORMAT_HEADER) - if not records_tsv.exists(): - return - with records_tsv.open(newline="", encoding="utf-8") as records_handle: - reader = csv.reader(records_handle, delimiter="\t") - header = next(reader, None) - if not header: - return +def sample_support_strategy(rules_path: Path) -> str: + """Choose no, streamed, or materialized sample handling for one mapping file. - sample_header = header[-1].strip() if len(header) >= 12 else "" - declared_sample_ids = [token for token in sample_header.split() if token] + The built-in four sample maps can be emitted directly as N-Triples without + writing their enormous Cartesian helper TSVs. A custom mapping with extra + helper-table consumers retains the materialized TSV behavior. + """ + text = rules_path.read_text(encoding="utf-8") + calls_refs = text.count('/data/tsv/sample_calls.tsv') + format_refs = text.count('/data/tsv/sample_format_values.tsv') + if calls_refs == 0 and format_refs == 0: + return "none" + if ( + calls_refs == 2 + and format_refs == 2 + and all(marker in text for marker in CANONICAL_SAMPLE_RULE_MARKERS) + and all(fragment in text for fragment in CANONICAL_SAMPLE_RULE_FRAGMENTS) + ): + return "stream" + return SAMPLE_HELPER_STRATEGY_MATERIALIZED - for row in reader: - if not row: - continue - if len(row) < len(header): - row = row + [""] * (len(header) - len(row)) - source_file = row[0] if len(row) > 0 else "" - row_id = row[1] if len(row) > 1 else "" - format_raw = row[10] if len(row) > 10 else "" - samples_raw = row[-1] if len(row) >= 12 else "" +@dataclass(frozen=True) +class SampleWorkflow: + """One mutually exclusive sample-representation execution plan.""" - format_keys = [token for token in format_raw.split(":")] if format_raw else [] - sample_payloads = [token for token in samples_raw.split()] if samples_raw else [] + representation: str + helper_strategy: str + emitter: str | None - total_samples = max(len(declared_sample_ids), len(sample_payloads)) - if total_samples == 0: - continue - sample_uri_seen: dict[str, int] = {} - for sample_idx in range(total_samples): - sample_id = ( - declared_sample_ids[sample_idx] - if sample_idx < len(declared_sample_ids) - else f"SAMPLE_{sample_idx + 1}" - ) - sample_payload = ( - sample_payloads[sample_idx] - if sample_idx < len(sample_payloads) - else "" - ) - sample_index_value = str(sample_idx + 1) - sample_uri_id_base = sample_id_to_uri_id(sample_id, sample_idx + 1) - sample_uri_seen[sample_uri_id_base] = sample_uri_seen.get(sample_uri_id_base, 0) + 1 - if sample_uri_seen[sample_uri_id_base] > 1: - sample_uri_id = f"{sample_uri_id_base}_{sample_uri_seen[sample_uri_id_base]}" - else: - sample_uri_id = sample_uri_id_base +def resolve_sample_workflow(representation: str, rules_path: Path) -> SampleWorkflow: + """Resolve rules compatibility into exactly one sample workflow. - sample_calls_writer.writerow( - [ - source_file, - row_id, - sample_index_value, - sample_id, - sample_uri_id, - sample_payload, - ] - ) + Expanded mode preserves custom helper-table mappings. Condensed mode emits its + RDF directly from records.tsv; it cannot safely coexist with custom rules + that consume materialized helper tables because that would execute both + representations and reintroduce semantic inflation. + """ + if representation not in SAMPLE_REPRESENTATION_CHOICES: + choices = ", ".join(sorted(SAMPLE_REPRESENTATION_CHOICES)) + raise ValueError( + f"unsupported sample representation '{representation}'; choose {choices}" + ) - value_tokens = sample_payload.split(":") if sample_payload else [] - total_fields = max(len(format_keys), len(value_tokens)) - for format_idx in range(total_fields): - format_key = ( - format_keys[format_idx] - if format_idx < len(format_keys) and format_keys[format_idx] - else f"FIELD_{format_idx + 1}" - ) - format_value = ( - value_tokens[format_idx] - if format_idx < len(value_tokens) - else "" - ) - sample_format_writer.writerow( - [ - source_file, - row_id, - sample_index_value, - sample_id, - sample_uri_id, - str(format_idx + 1), - format_key, - format_value, - ] - ) + rules_strategy = sample_support_strategy(rules_path) + if representation == "expanded": + if rules_strategy == "stream": + return SampleWorkflow("expanded", "header-only", "expanded") + if rules_strategy == SAMPLE_HELPER_STRATEGY_MATERIALIZED: + return SampleWorkflow("expanded", SAMPLE_HELPER_STRATEGY_MATERIALIZED, None) + return SampleWorkflow("expanded", "none", None) + if rules_strategy == SAMPLE_HELPER_STRATEGY_MATERIALIZED: + raise ValueError( + "--sample-representation condensed cannot be combined with custom rules " + "that consume materialized sample_calls.tsv or sample_format_values.tsv tables. " + "Remove those materialized helper-table consumers or use expanded mode." + ) + helper_strategy = "header-only" if rules_strategy == "stream" else "none" + return SampleWorkflow("condensed", helper_strategy, "condensed") -def render_rules_for_triplet( - template_rules: Path, - output_rules: Path, - records_name: str, - headers_name: str, - metadata_name: str, - sample_calls_name: str, - sample_format_name: str, -): - """Render per-input mapping rules by substituting TSV placeholders.""" - text = template_rules.read_text() - text = text.replace('/data/tsv/records.tsv', f'/data/tsv/{records_name}') - text = text.replace('/data/tsv/header_lines.tsv', f'/data/tsv/{headers_name}') - text = text.replace('/data/tsv/file_metadata.tsv', f'/data/tsv/{metadata_name}') - text = text.replace('/data/tsv/sample_calls.tsv', f'/data/tsv/{sample_calls_name}') - text = text.replace('/data/tsv/sample_format_values.tsv', f'/data/tsv/{sample_format_name}') - output_rules.write_text(text) +def _set_max_csv_field_size(): + """Allow chromosome-scale multi-sample payload columns in Python's CSV reader.""" + limit = sys.maxsize + while True: + try: + csv.field_size_limit(limit) + return + except OverflowError: + limit //= 10 + + +def _sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: + candidate = re.sub(r"[^A-Za-z0-9._~-]+", "_", sample_id).strip("_") + return candidate or f"sample_{fallback_index}" + + +# Characters that ``quote_plus(..., safe="*-._")`` leaves untouched. Sample +# ids, FORMAT keys, and numeric row ids are almost always drawn from this set, +# so the fast path below skips percent-encoding entirely for them. +_URI_COMPONENT_PASSTHROUGH_RE = re.compile(r"[A-Za-z0-9*\-._]*\Z") +# Characters that require N-Triples escaping. VCF genotype/FORMAT payloads +# essentially never contain them, so the common case avoids five str.replace +# scans of the value. +_NTRIPLES_ESCAPE_RE = re.compile(r'[\\"\n\r\t]') + + +def _rml_uri_component(value: str) -> str: + """Match RMLStreamer's Java URLEncoder-based template substitution.""" + if _URI_COMPONENT_PASSTHROUGH_RE.match(value) is not None: + return value + encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") + # urllib follows current RFC rules and always leaves '~' unescaped, whereas + # java.net.URLEncoder (used by RMLStreamer 2.5.0) encodes it. + return encoded.replace("+", "%20").replace("~", "%7E") + + +def _ntriples_string_literal(value: str) -> str: + """Serialize an RDF 1.1 plain/xsd:string literal for N-Triples.""" + if _NTRIPLES_ESCAPE_RE.search(value) is None: + return f'"{value}"' + escaped = ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + return f'"{escaped}"' + + +def _ntriples_literal(value: str) -> str: + literal = _ntriples_string_literal(value) + if value == ".": + literal += f"^^<{VCFR_NAMESPACE}Null>" + return literal + + +@dataclass(frozen=True) +class SampleColumn: + """One reusable VCF sample column.""" + + index: int + sample_id: str + uri_id: str + + +@dataclass(frozen=True) +class ParsedSampleRecord: + """One VCF record with FORMAT keys aligned to all sample columns.""" + + source_file: str + row_id: str + #: Raw QUAL and INFO columns, needed by the record-detail emitter. + qual: str + info: str + format_keys: tuple[str, ...] + sample_payloads: tuple[str, ...] + sample_values: tuple[tuple[str, ...], ...] + + +class SampleRecordStream: + """Read a records.tsv sample block once and expose a stable sample schema.""" + + def __init__(self, records_tsv: Path): + self.records_tsv = records_tsv + self.columns: tuple[SampleColumn, ...] = () + self.source_file = "" + self._handle = None + self._reader = None + self._header: list[str] = [] + self._pending_row: list[str] | None = None + # A cohort VCF repeats the same FORMAT string on nearly every record. + # Cache the derived key tuple (and its duplicate check) per distinct + # FORMAT/width combination instead of rebuilding it per record. + self._format_key_cache: dict[tuple[str, int], tuple[str, ...]] = {} + + def __enter__(self): + _set_max_csv_field_size() + self._handle = self.records_tsv.open(newline="", encoding="utf-8") + self._reader = csv.reader(self._handle, delimiter="\t") + self._header = next(self._reader, None) or [] + self._pending_row = self._next_nonempty_row() + if self._pending_row: + self.source_file = self._pending_row[0] if self._pending_row else "" + + sample_header = self._header[-1].strip() if len(self._header) >= 12 else "" + declared_ids = ( + [] + if sample_header == "SAMPLES" + else [token for token in sample_header.split() if token] + ) + if not declared_ids and self._pending_row is not None and len(self._header) >= 12: + samples_raw = self._pending_row[-1] if self._pending_row else "" + payload_count = len(samples_raw.split()) if samples_raw else 0 + declared_ids = [f"SAMPLE_{index}" for index in range(1, payload_count + 1)] + + uri_id_counts: dict[str, int] = {} + columns: list[SampleColumn] = [] + for index, sample_id in enumerate(declared_ids, start=1): + uri_id_base = _sample_id_to_uri_id(sample_id, index) + uri_id_counts[uri_id_base] = uri_id_counts.get(uri_id_base, 0) + 1 + occurrence = uri_id_counts[uri_id_base] + uri_id = f"{uri_id_base}_{occurrence}" if occurrence > 1 else uri_id_base + columns.append(SampleColumn(index, sample_id, uri_id)) + self.columns = tuple(columns) + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self._handle is not None: + self._handle.close() + self._handle = None + self._reader = None -def resolve_default_rules_path(repo_root: Path) -> Path: - """Resolve default rules path for both source checkout and installed package. + def _next_nonempty_row(self) -> list[str] | None: + if self._reader is None: + return None + for row in self._reader: + if row: + return row + return None - Resolution order: - 1) `/rules/default_rules.ttl` (editable/local checkout) - 2) packaged data file in `vcf_rdfizer_data/rules/default_rules.ttl` (wheel/sdist install) - """ - local_default = (repo_root / "rules" / "default_rules.ttl").resolve() - if local_default.exists() and local_default.is_file(): - return local_default + def __iter__(self): + if self._reader is None: + raise RuntimeError("SampleRecordStream must be used as a context manager") + pending = self._pending_row + self._pending_row = None + if pending is not None: + yield self._parse_row(pending) + for row in self._reader: + if row: + yield self._parse_row(row) + + def _parse_row(self, row: list[str]) -> ParsedSampleRecord: + if len(row) < len(self._header): + row = row + [""] * (len(self._header) - len(row)) + + source_file = row[0] if len(row) > 0 else "" + if self.source_file and source_file != self.source_file: + raise ValueError( + f"records TSV mixes SOURCE_FILE values '{self.source_file}' and " + f"'{source_file}'" + ) + row_id = row[1] if len(row) > 1 else "" + qual_raw = row[7] if len(row) > 7 else "" + info_raw = row[9] if len(row) > 9 else "" + format_raw = row[10] if len(row) > 10 else "" + samples_raw = row[-1] if len(row) >= 12 else "" + declared_format_keys = format_raw.split(":") if format_raw else [] + sample_payloads = samples_raw.split() if samples_raw else [] + if len(sample_payloads) > len(self.columns): + raise ValueError( + f"record {row_id or '(unknown)'} contains {len(sample_payloads)} sample " + f"payloads but the TSV header declares {len(self.columns)} sample columns" + ) + sample_payloads.extend([""] * (len(self.columns) - len(sample_payloads))) - try: - packaged = importlib_resources.files("vcf_rdfizer_data").joinpath( - "rules/default_rules.ttl" + raw_sample_values = [ + payload.split(":") if payload else [] for payload in sample_payloads + ] + total_fields = max( + [len(declared_format_keys), *(len(values) for values in raw_sample_values)], + default=0, ) - with importlib_resources.as_file(packaged) as packaged_path: - packaged_resolved = packaged_path.resolve() - if packaged_resolved.exists() and packaged_resolved.is_file(): - return packaged_resolved - except (ModuleNotFoundError, FileNotFoundError): - pass + cache_key = (format_raw, total_fields) + format_keys = self._format_key_cache.get(cache_key) + if format_keys is None: + format_keys = tuple( + declared_format_keys[index] + if index < len(declared_format_keys) and declared_format_keys[index] + else f"FIELD_{index + 1}" + for index in range(total_fields) + ) + if len(set(format_keys)) != len(format_keys): + raise ValueError( + f"record {row_id or '(unknown)'} contains duplicate FORMAT keys: " + + ":".join(format_keys) + ) + self._format_key_cache[cache_key] = format_keys - raise ValueError( - "default rules file not found. Provide --rules explicitly or reinstall package." - ) + sample_values = tuple( + tuple(values[index] if index < len(values) else "" for index in range(total_fields)) + for values in raw_sample_values + ) + return ParsedSampleRecord( + source_file=source_file, + row_id=row_id, + qual=qual_raw, + info=info_raw, + format_keys=format_keys, + sample_payloads=tuple(sample_payloads), + sample_values=sample_values, + ) -def docker_image_exists(image: str) -> bool: - """Return True when Docker image reference exists locally.""" - return run([*docker_cmd_prefix(), "image", "inspect", image]) == 0 +def _append_rdf_atomically(rdf_path: Path, stats: dict, producer): + """Append generated N-Triples and restore the original artifact on failure.""" + original_size = rdf_path.stat().st_size + opener = gzip.open if rdf_path.name.endswith(".gz") else Path.open + output_handle = None + buffer = bytearray() + # ``emit`` runs once per emitted triple (billions of times for a cohort + # aggregate), so the counter is a local int and only reaches ``stats`` + # once the producer has finished. + emitted = 0 + + def emit(line: str): + nonlocal buffer, emitted + buffer.extend(line.encode("utf-8")) + emitted += 1 + if len(buffer) >= SAMPLE_RDF_BUFFER_BYTES: + output_handle.write(buffer) + buffer = bytearray() + try: + output_handle = opener(rdf_path, "ab") + producer(emit) + stats["triples"] += emitted + if buffer: + output_handle.write(buffer) + output_handle.close() + output_handle = None + stats["appended_bytes"] = rdf_path.stat().st_size - original_size + return stats + except BaseException: + if output_handle is not None: + try: + output_handle.close() + except OSError: + pass + with rdf_path.open("r+b") as rollback_handle: + rollback_handle.truncate(original_size) + raise -def docker_build_image(image: str, repo_root: Path): - """Build Docker image from repository Dockerfile.""" - return run([*docker_cmd_prefix(), "build", "-t", image, "."], cwd=str(repo_root)) +def append_expanded_sample_rdf( + records_tsv: Path, + rdf_path: Path, + *, + progress_interval_records: int = 10_000, +) -> dict: + """Append the expanded SampleCall/FormatFieldValue representation. -def docker_pull_image(image: str): + This produces the same canonical SampleCall and FormatFieldValue triples as + the default RML maps without materializing V*S and V*S*F helper TSV rows. + """ + stats = { + "representation": "expanded", + "records": 0, + "sample_calls": 0, + "format_values": 0, + "triples": 0, + "appended_bytes": 0, + } + if not records_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for sample streaming: {rdf_path}") + + with SampleRecordStream(records_tsv) as record_stream: + if not record_stream.columns or not record_stream.source_file: + return stats + + def produce(emit): + source_component = _rml_uri_component(record_stream.source_file) + file_uri = f"file://{source_component}" + emit( + f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " + f"<{VCFR_NAMESPACE}ExpandedRepresentation> .\n" + ) + # Sample columns and their serialized sampleId literal are fixed for + # the whole file; FORMAT keys are drawn from a handful of distinct + # values. Encoding them once per file instead of once per + # record/sample removes the dominant cost of this loop. + sample_prefixes = [ + ( + _rml_uri_component(column.uri_id), + _ntriples_literal(column.sample_id), + ) + for column in record_stream.columns + ] + format_components: dict[str, str] = {} + sample_call_count = 0 + format_value_count = 0 + + for record in record_stream: + row_component = _rml_uri_component(record.row_id) + call_uri = f"{file_uri}#call/{row_component}" + format_keys = record.format_keys + record_sample_values = record.sample_values + + for sample_index, (sample_component, sample_id_literal) in enumerate( + sample_prefixes + ): + sample_uri = f"{file_uri}#sample/{row_component}/{sample_component}" + sample_values = record_sample_values[sample_index] + + emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasSampleCall> <{sample_uri}> .\n") + emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleCall> .\n") + emit( + f"<{sample_uri}> <{VCFR_NAMESPACE}sampleId> " + f"{sample_id_literal} .\n" + ) + sample_call_count += 1 + + for format_index, format_key in enumerate(format_keys): + format_value = sample_values[format_index] + format_component = format_components.get(format_key) + if format_component is None: + format_component = _rml_uri_component(format_key) + format_components[format_key] = format_component + format_uri = f"{sample_uri}/fmt/{format_component}" + emit(f"<{sample_uri}> <{VCFR_NAMESPACE}hasFormatValue> <{format_uri}> .\n") + emit(f"<{format_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatFieldValue> .\n") + if format_value: + emit( + f"<{format_uri}> <{VCFR_NAMESPACE}fieldValue> " + f"{_ntriples_literal(format_value)} .\n" + ) + format_value_count += 1 + + stats["records"] += 1 + stats["sample_calls"] = sample_call_count + stats["format_values"] = format_value_count + if progress_interval_records > 0 and stats["records"] % progress_interval_records == 0: + print( + " * Sample RDF streaming: " + f"{stats['records']:,} variants, {sample_call_count:,} calls", + flush=True, + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def append_canonical_sample_rdf( + records_tsv: Path, + rdf_path: Path, + *, + progress_interval_records: int = 10_000, +) -> dict: + """Backward-compatible name for the expanded sample RDF emitter.""" + return append_expanded_sample_rdf( + records_tsv, + rdf_path, + progress_interval_records=progress_interval_records, + ) + + +@dataclass(frozen=True) +class FormatDefinition: + """Structured attributes and RDF identity for one FORMAT declaration.""" + + uri: str + field_number: str + description: str + #: The declared VCF Type (Integer/Float/Flag/Character/String). Defaults to + #: String so an undeclared field still has a usable value type. + value_type: str = "String" + + +def _parse_structured_header_fields(value: str) -> dict[str, str]: + """Parse comma-delimited VCF header attributes while respecting quotes.""" + inner = value.strip() + if inner.startswith("<") and inner.endswith(">"): + inner = inner[1:-1] + + tokens: list[str] = [] + token: list[str] = [] + in_quotes = False + escaped = False + for character in inner: + if escaped: + token.append(character) + escaped = False + elif character == "\\" and in_quotes: + token.append(character) + escaped = True + elif character == '"': + token.append(character) + in_quotes = not in_quotes + elif character == "," and not in_quotes: + tokens.append("".join(token)) + token = [] + else: + token.append(character) + tokens.append("".join(token)) + + fields: dict[str, str] = {} + for item in tokens: + if "=" not in item: + continue + key, raw_value = item.split("=", 1) + parsed_value = raw_value.strip() + if len(parsed_value) >= 2 and parsed_value[0] == parsed_value[-1] == '"': + parsed_value = parsed_value[1:-1] + parsed_value = parsed_value.replace('\\"', '"').replace("\\\\", "\\") + fields[key.strip()] = parsed_value + return fields + + +def _load_field_definitions( + header_lines_tsv: Path, header_key: str +) -> dict[str, FormatDefinition]: + """Map declared field IDs to structured definitions backed by HeaderLine IRIs. + + Serves both ``##FORMAT`` and ``##INFO`` declarations, which share the + ```` syntax and the same vocabulary shape + (``vcfr:FieldDefinition`` with ``fieldId``/``fieldNumber``/``fieldType``). + """ + definitions: dict[str, FormatDefinition] = {} + if not header_lines_tsv.is_file(): + return definitions + wanted = header_key.upper() + _set_max_csv_field_size() + with header_lines_tsv.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + if (row.get("HEADER_KEY") or "").upper() != wanted: + continue + fields = _parse_structured_header_fields(row.get("HEADER_VALUE") or "") + field_id = fields.get("ID", "").strip() + if not field_id: + continue + source_component = _rml_uri_component(row.get("SOURCE_FILE") or "") + index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") + definitions.setdefault( + field_id, + FormatDefinition( + uri=f"file://{source_component}#header/line/{index_component}", + field_number=fields.get("Number") or ".", + description=( + fields.get("Description") + or f"{wanted} field {field_id} (source declaration has no Description)" + ), + value_type=fields.get("Type") or "String", + ), + ) + return definitions + + +def _load_format_definitions(header_lines_tsv: Path) -> dict[str, FormatDefinition]: + """Backward-compatible accessor for the FORMAT declarations.""" + return _load_field_definitions(header_lines_tsv, "FORMAT") + + +#: Maps a declared VCF INFO/FORMAT Type to its vocabulary value-type class. +VCF_VALUE_TYPE_CLASSES = { + "Integer": "IntegerType", + "Float": "FloatType", + "Flag": "FlagType", + "Character": "CharacterType", + "String": "StringType", +} +#: Typed value predicates used when a single-valued field declares a numeric or +#: flag type. Multi-valued fields (Number=A/R/G/.) keep only the lexical value, +#: because the vocabulary gives one InfoFieldValue node per key. +XSD_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#integer" +XSD_DECIMAL_URI = "http://www.w3.org/2001/XMLSchema#decimal" +XSD_BOOLEAN_URI = "http://www.w3.org/2001/XMLSchema#boolean" + + +def parse_info_entries(info: str) -> list[tuple[str, str | None]]: + """Split an INFO column into ``(key, value)`` pairs. + + An entry without ``=`` is a Flag: a presence assertion with no value, which + the vocabulary models as ``vcfr:fieldValueBoolean true``. + """ + if info in ("", "."): + return [] + entries: list[tuple[str, str | None]] = [] + for item in info.split(";"): + if not item: + continue + key, separator, value = item.partition("=") + entries.append((key, value if separator else None)) + return entries + + +def _typed_info_object(value: str, declared_type: str) -> tuple[str, str] | None: + """Return ``(predicate_local_name, literal)`` for a typed single value.""" + if "," in value or value == ".": + return None + try: + if declared_type == "Integer": + return "fieldValueInteger", f'"{int(value)}"^^<{XSD_INTEGER_URI}>' + if declared_type == "Float": + # Serialize the source lexical form rather than a reparsed float, so + # the graph never gains or loses precision relative to the VCF. + float(value) + return "fieldValueDecimal", f'"{value}"^^<{XSD_DECIMAL_URI}>' + except ValueError: + # A value that contradicts its declared type is kept as a plain literal + # rather than dropped; the conversion must not silently lose data. + return None + return None + + +# How the VCF meta-information block is represented. "basic" is the historical +# behaviour: every '##' line becomes an untyped vcfr:HeaderLine carrying its raw +# key and value. "structured" additionally types each line with the vocabulary's +# subclass and lifts the attributes of FILTER, ALT and contig declarations into +# their own properties, so the header becomes queryable rather than just present. +HEADER_REPRESENTATION_CHOICES = ("basic", "structured") +DEFAULT_HEADER_REPRESENTATION = "structured" + +#: '##' key (lower-cased) -> the vocabulary subclass for that line. +HEADER_LINE_CLASSES = { + "fileformat": "FileFormatHeaderLine", + "filedate": "FileDateHeaderLine", + "source": "SourceHeaderLine", + "reference": "ReferenceHeaderLine", + "info": "INFOHeaderLine", + "format": "FORMATHeaderLine", + "filter": "FILTERHeaderLine", + "alt": "ALTHeaderLine", + "contig": "ContigHeaderLine", +} +#: contig attribute -> vocabulary predicate. +CONTIG_ATTRIBUTES = { + "length": "contigLength", + "md5": "contigMd5", + "assembly": "contigAssembly", +} + + +XSD_DATE_URI = "http://www.w3.org/2001/XMLSchema#date" +#: ##fileDate has no mandated format. These are the two forms seen in practice +#: that map unambiguously onto xsd:date, which the SHACL shape requires. +FILE_DATE_PATTERNS = ( + (re.compile(r"^(\d{4})(\d{2})(\d{2})$"), "{0}-{1}-{2}"), + (re.compile(r"^(\d{4})-(\d{2})-(\d{2})$"), "{0}-{1}-{2}"), +) + + +def file_date_object(value: str) -> str | None: + """Serialize ##fileDate as xsd:date when its form allows, else lexically. + + Returns None for an absent value so no triple is emitted, matching RML's + behaviour for an empty reference. + """ + value = (value or "").strip() + if not value or value == ".": + return None + for pattern, template in FILE_DATE_PATTERNS: + match = pattern.match(value) + if match: + return f'"{template.format(*match.groups())}"^^<{XSD_DATE_URI}>' + # An unrecognized form is preserved verbatim rather than dropped; the SHACL + # layer reports it as non-conformant. + return _ntriples_string_literal(value) + + +def append_header_representation_rdf( + header_lines_tsv: Path, + rdf_path: Path, +) -> dict: + """Append typed header lines and structured FILTER/ALT/contig declarations. + + The default mapping emits every '##' line as an untyped ``vcfr:HeaderLine`` + with a raw key and value. The vocabulary already defines a subclass per line + type and dedicated properties for the FILTER, ALT and contig attributes; + this emits them, which is what makes the meta-information block queryable. + + Emitted directly rather than through RML because the attributes live inside + a single ```` value that RML cannot decompose. + """ + stats = { + "representation": "header", + "header_lines": 0, + "typed_lines": 0, + "filter_definitions": 0, + "alt_definitions": 0, + "contigs": 0, + "file_dates": 0, + "triples": 0, + "appended_bytes": 0, + } + if not header_lines_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for header streaming: {rdf_path}") + + _set_max_csv_field_size() + with header_lines_tsv.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle, delimiter="\t")) + + def produce(emit): + source_file = rows[0].get("SOURCE_FILE", "") if rows else "" + file_uri = f"file://{_rml_uri_component(source_file)}" + contig_count = 0 + for row in rows: + key = (row.get("HEADER_KEY") or "").strip() + value = row.get("HEADER_VALUE") or "" + index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") + line_uri = f"file://{_rml_uri_component(row.get('SOURCE_FILE') or '')}" \ + f"#header/line/{index_component}" + stats["header_lines"] += 1 + + line_class = HEADER_LINE_CLASSES.get(key.lower()) + if line_class is None: + # An unrecognized '##' key keeps only the base HeaderLine type + # the mapping already emitted; inventing a subclass for it would + # put a term in the graph that the vocabulary does not define. + continue + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}{line_class}> .\n") + stats["typed_lines"] += 1 + + if key.lower() not in {"filter", "alt", "contig"}: + continue + fields = _parse_structured_header_fields(value) + identifier = (fields.get("ID") or "").strip() + if not identifier: + continue + description = fields.get("Description") + + if key.lower() == "filter": + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FilterDefinition> .\n") + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}filterId> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + stats["filter_definitions"] += 1 + elif key.lower() == "alt": + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}AltDefinition> .\n") + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}altId> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + stats["alt_definitions"] += 1 + else: + contig_count += 1 + stats["contigs"] += 1 + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}contigId> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + for attribute, predicate in CONTIG_ATTRIBUTES.items(): + attribute_value = fields.get(attribute) + if attribute_value: + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}{predicate}> " + f"{_ntriples_string_literal(attribute_value)} .\n" + ) + continue + + if description: + emit( + f"<{line_uri}> <{VCFR_NAMESPACE}fieldDescription> " + f"{_ntriples_string_literal(description)} .\n" + ) + + for row in rows: + if (row.get("HEADER_KEY") or "").strip().lower() != "filedate": + continue + date_object = file_date_object(row.get("HEADER_VALUE") or "") + if date_object is not None: + emit(f"<{file_uri}> <{VCFR_NAMESPACE}fileDate> {date_object} .\n") + stats["file_dates"] += 1 + break + + if contig_count: + emit( + f"<{file_uri}> <{VCFR_NAMESPACE}contigCount> " + f'"{contig_count}"^^<{XSD_INTEGER_URI}> .\n' + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def _qual_object(value: str) -> str: + """Serialize QUAL as the SHACL shape requires: xsd:decimal, or vcfr:Null. + + The shape is ``sh:or([sh:datatype xsd:decimal] [sh:datatype vcfr:Null])``, + which RML cannot satisfy because the datatype depends on the row. A value + that is neither numeric nor the missing token is kept as a plain literal + rather than dropped: a non-conformant graph is more useful than a lossy one, + and the SHACL layer reports it. + """ + if value in ("", "."): + return f'"."^^<{VCFR_NAMESPACE}Null>' + try: + float(value) + except ValueError: + return _ntriples_string_literal(value) + # Serialize the source lexical form so no precision is gained or lost. + return f'"{value}"^^<{XSD_DECIMAL_URI}>' + + +def append_record_detail_rdf( + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, + *, + emit_qual: bool = True, + emit_info: bool = True, + progress_interval_records: int = 10_000, +) -> dict: + """Append per-record detail the RML mapping cannot express. + + Covers QUAL (whose datatype depends on the value) and the structured INFO + representation (which would otherwise need a materialized helper table of + variants x INFO keys). Both are per-record, so they share one pass over + ``records.tsv``. + + The default mapping emits INFO only as an opaque ``vcfr:infoRaw`` string. + This adds the structured form the vocabulary already defines - one + ``vcfr:InfoFieldValue`` per record and key, at the IRI template the + vocabulary declares - so INFO becomes queryable rather than just present. + + Emitted directly rather than through RML for the same reason the genotype + representations are: RML would need a materialized helper table of + variants x INFO keys, and it cannot switch value datatype on a declared + ``Type``. + """ + stats = { + "representation": "record-detail", + "records": 0, + "qual_values": 0, + "info_values": 0, + "info_definitions": 0, + "triples": 0, + "appended_bytes": 0, + } + if not records_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for INFO streaming: {rdf_path}") + + definitions = _load_field_definitions(header_lines_tsv, "INFO") + + with SampleRecordStream(records_tsv) as record_stream: + if not record_stream.source_file: + return stats + + def produce(emit): + source_component = _rml_uri_component(record_stream.source_file) + file_uri = f"file://{source_component}" + emitted_definitions: set[str] = set() + key_components: dict[str, str] = {} + value_count = 0 + definition_count = 0 + + qual_count = 0 + for record in record_stream: + row_component = _rml_uri_component(record.row_id) + call_uri = f"{file_uri}#call/{row_component}" + if emit_qual: + emit( + f"<{call_uri}> <{VCFR_NAMESPACE}qual> " + f"{_qual_object(record.qual)} .\n" + ) + qual_count += 1 + for key, value in (parse_info_entries(record.info) if emit_info else ()): + key_component = key_components.get(key) + if key_component is None: + key_component = _rml_uri_component(key) + key_components[key] = key_component + info_uri = f"{call_uri}/info/{key_component}" + + definition = definitions.get(key) + if definition is None: + definition = FormatDefinition( + uri=f"{file_uri}#header/info/{key_component}", + field_number=".", + description=( + f"Synthesized definition for undeclared INFO key {key}" + ), + value_type="Flag" if value is None else "String", + ) + if definition.uri not in emitted_definitions: + emitted_definitions.add(definition.uri) + definition_count += 1 + type_class = VCF_VALUE_TYPE_CLASSES.get( + definition.value_type, "StringType" + ) + emit( + f"<{definition.uri}> <{RDF_TYPE_URI}> " + f"<{VCFR_NAMESPACE}InfoFieldDefinition> .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldId> " + f"{_ntriples_string_literal(key)} .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldNumber> " + f"{_ntriples_string_literal(definition.field_number)} .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldDescription> " + f"{_ntriples_string_literal(definition.description)} .\n" + ) + emit( + f"<{definition.uri}> <{VCFR_NAMESPACE}fieldType> " + f"<{VCFR_NAMESPACE}{type_class}> .\n" + ) + + emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasInfoValue> <{info_uri}> .\n") + emit( + f"<{info_uri}> <{RDF_TYPE_URI}> " + f"<{VCFR_NAMESPACE}InfoFieldValue> .\n" + ) + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}declaredBy> " + f"<{definition.uri}> .\n" + ) + if value is None: + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}fieldValueBoolean> " + f'"true"^^<{XSD_BOOLEAN_URI}> .\n' + ) + else: + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}fieldValue> " + f"{_ntriples_literal(value)} .\n" + ) + typed = _typed_info_object(value, definition.value_type) + if typed is not None: + predicate, literal = typed + emit( + f"<{info_uri}> <{VCFR_NAMESPACE}{predicate}> " + f"{literal} .\n" + ) + value_count += 1 + + stats["records"] += 1 + stats["qual_values"] = qual_count + stats["info_values"] = value_count + stats["info_definitions"] = definition_count + if ( + progress_interval_records > 0 + and stats["records"] % progress_interval_records == 0 + ): + print( + f" * INFO RDF streaming: {stats['records']:,} variants, " + f"{value_count:,} values", + flush=True, + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def append_condensed_sample_rdf( + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, + *, + progress_interval_records: int = 10_000, +) -> dict: + """Append sample-ordered cohort matrices and FORMAT value vectors.""" + stats = { + "representation": "condensed", + "records": 0, + "samples": 0, + "matrices": 0, + "format_vectors": 0, + "format_definitions": 0, + "triples": 0, + "appended_bytes": 0, + } + if not records_tsv.is_file(): + return stats + if not rdf_path.is_file(): + raise FileNotFoundError(f"RDF aggregate not found for sample streaming: {rdf_path}") + + definitions = _load_format_definitions(header_lines_tsv) + with SampleRecordStream(records_tsv) as record_stream: + if not record_stream.columns or not record_stream.source_file: + return stats + + def produce(emit): + source_component = _rml_uri_component(record_stream.source_file) + file_uri = f"file://{source_component}" + sample_set_uri = f"{file_uri}#samples" + emitted_definitions: set[str] = set() + # FORMAT keys repeat on every record; encode each distinct key once. + format_components: dict[str, str] = {} + + emit( + f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " + f"<{VCFR_NAMESPACE}CondensedRepresentation> .\n" + ) + emit(f"<{file_uri}> <{VCFR_NAMESPACE}hasSampleSet> <{sample_set_uri}> .\n") + emit(f"<{sample_set_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleSet> .\n") + + for sample_column in record_stream.columns: + sample_component = _rml_uri_component(sample_column.uri_id) + sample_uri = f"{sample_set_uri}/{sample_component}" + emit(f"<{sample_set_uri}> <{VCFR_NAMESPACE}hasSample> <{sample_uri}> .\n") + emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}VCFSample> .\n") + emit( + f"<{sample_uri}> <{VCFR_NAMESPACE}sampleName> " + f"{_ntriples_string_literal(sample_column.sample_id)} .\n" + ) + emit( + f"<{sample_uri}> <{VCFR_NAMESPACE}sampleIndex> " + f'"{sample_column.index}"^^<{XSD_POSITIVE_INTEGER_URI}> .\n' + ) + stats["samples"] += 1 + + for record in record_stream: + if not record.format_keys: + continue + row_component = _rml_uri_component(record.row_id) + call_uri = f"{file_uri}#call/{row_component}" + matrix_uri = f"{call_uri}/matrix" + emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasCallMatrix> <{matrix_uri}> .\n") + emit(f"<{matrix_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}CohortCallMatrix> .\n") + emit( + f"<{matrix_uri}> <{VCFR_NAMESPACE}appliesToSampleSet> " + f"<{sample_set_uri}> .\n" + ) + stats["matrices"] += 1 + + for format_index, format_key in enumerate(record.format_keys): + format_component = format_components.get(format_key) + if format_component is None: + format_component = _rml_uri_component(format_key) + format_components[format_key] = format_component + vector_uri = f"{matrix_uri}/fmt/{format_component}" + definition = definitions.get(format_key) + if definition is None: + definition = FormatDefinition( + uri=f"{file_uri}#header/format/{format_component}", + field_number=".", + description=( + f"Synthesized definition for undeclared FORMAT key {format_key}" + ), + ) + definition_uri = definition.uri + if definition_uri not in emitted_definitions: + emit( + f"<{definition_uri}> <{RDF_TYPE_URI}> " + f"<{VCFR_NAMESPACE}FormatFieldDefinition> .\n" + ) + emit( + f"<{definition_uri}> <{VCFR_NAMESPACE}fieldId> " + f"{_ntriples_string_literal(format_key)} .\n" + ) + emit( + f"<{definition_uri}> <{VCFR_NAMESPACE}fieldNumber> " + f"{_ntriples_string_literal(definition.field_number)} .\n" + ) + emit( + f"<{definition_uri}> <{VCFR_NAMESPACE}fieldDescription> " + f"{_ntriples_string_literal(definition.description)} .\n" + ) + emitted_definitions.add(definition_uri) + stats["format_definitions"] += 1 + + encoded_values = "\t".join( + values[format_index] or "." for values in record.sample_values + ) + emit( + f"<{matrix_uri}> <{VCFR_NAMESPACE}hasFormatValueVector> " + f"<{vector_uri}> .\n" + ) + emit(f"<{vector_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatValueVector> .\n") + emit( + f"<{vector_uri}> <{VCFR_NAMESPACE}declaredBy> " + f"<{definition_uri}> .\n" + ) + emit( + f"<{vector_uri}> <{VCFR_NAMESPACE}valueEncoding> " + f"<{VCFR_NAMESPACE}VCFTextVector> .\n" + ) + emit( + f"<{vector_uri}> <{VCFR_NAMESPACE}encodedValues> " + f"{_ntriples_string_literal(encoded_values)} .\n" + ) + stats["format_vectors"] += 1 + + stats["records"] += 1 + if progress_interval_records > 0 and stats["records"] % progress_interval_records == 0: + print( + " * Condensed sample RDF streaming: " + f"{stats['records']:,} variants, {stats['format_vectors']:,} vectors", + flush=True, + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def emit_record_detail( + info_representation: str, + *, + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, +) -> dict | None: + """Append QUAL and, when selected, the structured INFO representation. + + QUAL is always emitted: the RML mapping cannot type it per row, so this is + the only place it can come from. + """ + if info_representation not in INFO_REPRESENTATION_CHOICES: + raise ValueError(f"unknown INFO representation: {info_representation}") + return append_record_detail_rdf( + records_tsv, header_lines_tsv, rdf_path, + emit_qual=True, emit_info=info_representation == "structured", + ) + + +def emit_sample_representation( + workflow: SampleWorkflow, + *, + records_tsv: Path, + header_lines_tsv: Path, + rdf_path: Path, +) -> dict | None: + """Execute the workflow's sole direct RDF emitter, if it has one.""" + if workflow.emitter is None: + return None + if workflow.emitter == "expanded": + return append_expanded_sample_rdf(records_tsv, rdf_path) + if workflow.emitter == "condensed": + return append_condensed_sample_rdf(records_tsv, header_lines_tsv, rdf_path) + raise RuntimeError(f"unknown sample RDF emitter: {workflow.emitter}") + + +def update_conversion_metrics_after_sample_stream( + *, + metrics_dir: Path, + output_name: str, + run_id: str, + rdf_path: Path, + total_triples: int, + sample_stats: dict, +): + """Bring conversion JSON/CSV metrics in sync after direct sample emission.""" + safe_name = safe_metrics_name(output_name) + metrics_json = metrics_dir / "stages" / "conversion" / f"{safe_name}.json" + output_size = int(rdf_path.stat().st_size) + if metrics_json.is_file(): + try: + payload = json.loads(metrics_json.read_text(encoding="utf-8")) + artifacts = payload.setdefault("artifacts", {}) + prior_triples = artifacts.get("output_triples") + if isinstance(prior_triples, dict): + for key in list(prior_triples): + prior_triples[key] = int(total_triples) + prior_triples["TOTAL"] = int(total_triples) + else: + artifacts["output_triples"] = int(total_triples) + artifacts["output_size_bytes"] = output_size + payload["sample_representation"] = sample_stats + # Retained for consumers of pre-condensed conversion metrics. + payload["sample_streaming"] = sample_stats + metrics_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except (OSError, json.JSONDecodeError): + pass + + metrics_csv = metrics_dir / "metrics.csv" + if not metrics_csv.is_file(): + return + try: + with metrics_csv.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + fieldnames = list(reader.fieldnames or []) + rows = list(reader) + changed = False + for row in rows: + if row.get("run_id") == run_id and row.get("output_name") == output_name: + row["output_triples"] = str(int(total_triples)) + row["output_dir_size_bytes"] = str(output_size) + changed = True + if changed: + with metrics_csv.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + except OSError: + pass + + +def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_format_tsv: Path): + """Materialize per-sample helper TSVs from records.tsv. + + These helper tables keep records.tsv user-facing and simple while providing + deterministic row-wise inputs for rules that map: + - one `SampleCall` per sample/record + - one `FormatFieldValue` per sample/record/FORMAT key + """ + write_sample_support_headers(sample_calls_tsv, sample_format_tsv) + with sample_calls_tsv.open("a", newline="", encoding="utf-8") as sample_calls_handle, \ + sample_format_tsv.open("a", newline="", encoding="utf-8") as sample_format_handle: + sample_calls_writer = csv.writer(sample_calls_handle, delimiter="\t") + sample_format_writer = csv.writer(sample_format_handle, delimiter="\t") + + if not records_tsv.exists(): + return + + with SampleRecordStream(records_tsv) as record_stream: + for record in record_stream: + for sample_offset, sample_column in enumerate(record_stream.columns): + sample_calls_writer.writerow( + [ + record.source_file, + record.row_id, + str(sample_column.index), + sample_column.sample_id, + sample_column.uri_id, + record.sample_payloads[sample_offset], + ] + ) + + for format_offset, format_key in enumerate(record.format_keys): + sample_format_writer.writerow( + [ + record.source_file, + record.row_id, + str(sample_column.index), + sample_column.sample_id, + sample_column.uri_id, + str(format_offset + 1), + format_key, + record.sample_values[sample_offset][format_offset], + ] + ) + + +# --------------------------------------------------------------------------- +# RML mapping rendering and Docker image resolution +# --------------------------------------------------------------------------- +def render_rules_for_triplet( + template_rules: Path, + output_rules: Path, + records_name: str, + headers_name: str, + metadata_name: str, + sample_calls_name: str, + sample_format_name: str, +): + """Render per-input mapping rules by substituting TSV placeholders.""" + text = template_rules.read_text() + text = text.replace('/data/tsv/records.tsv', f'/data/tsv/{records_name}') + text = text.replace('/data/tsv/header_lines.tsv', f'/data/tsv/{headers_name}') + text = text.replace('/data/tsv/file_metadata.tsv', f'/data/tsv/{metadata_name}') + text = text.replace('/data/tsv/sample_calls.tsv', f'/data/tsv/{sample_calls_name}') + text = text.replace('/data/tsv/sample_format_values.tsv', f'/data/tsv/{sample_format_name}') + output_rules.write_text(text) + + +def resolve_default_rules_path(repo_root: Path) -> Path: + """Resolve default rules path for both source checkout and installed package. + + Resolution order: + 1) `/rules/default_rules.ttl` (editable/local checkout) + 2) packaged data file in `vcf_rdfizer_data/rules/default_rules.ttl` (wheel/sdist install) + """ + local_default = (repo_root / "rules" / "default_rules.ttl").resolve() + if local_default.exists() and local_default.is_file(): + return local_default + + try: + packaged = importlib_resources.files("vcf_rdfizer_data").joinpath( + "rules/default_rules.ttl" + ) + with importlib_resources.as_file(packaged) as packaged_path: + packaged_resolved = packaged_path.resolve() + if packaged_resolved.exists() and packaged_resolved.is_file(): + return packaged_resolved + except (ModuleNotFoundError, FileNotFoundError): + pass + + raise ValueError( + "default rules file not found. Provide --rules explicitly or reinstall package." + ) + + +def docker_image_exists(image: str) -> bool: + """Return True when Docker image reference exists locally.""" + return run([*docker_cmd_prefix(), "image", "inspect", image]) == 0 + + +def docker_build_image(image: str, repo_root: Path): + """Build Docker image from repository Dockerfile.""" + return run([*docker_cmd_prefix(), "build", "-t", image, "."], cwd=str(repo_root)) + + +def docker_pull_image(image: str): """Pull Docker image from registry.""" return run([*docker_cmd_prefix(), "pull", image]) @@ -1694,6 +3076,12 @@ def resolve_image_ref(image: str, image_version: str | None): return f"{image}:{image_version}", True +# --------------------------------------------------------------------------- +# Compression plan parsing and strategy selection +# The public CLI takes three orthogonal options (--rdf-compression, +# --representations, --artifact-compression); they are translated here into +# the flat internal stage names used by the execution helpers. +# --------------------------------------------------------------------------- def parse_compression_methods(raw: str): """Parse internal compression stage names used by the execution helpers.""" value = (raw or "").strip() @@ -1801,61 +3189,245 @@ def compression_uses_hdt(methods: list[str]) -> bool: return any(method in HDT_COMPRESSION_METHODS for method in methods) -def should_use_partitioned_hdt( - *, - mode: str, - methods: list[str], - hdt_strategy: str, - rdf_storage_mode: str | None = None, -) -> bool: - """Resolve whether the HDT pipeline should use chunked generation + HDTCat.""" - if not compression_uses_hdt(methods): - return False - if hdt_strategy == "single": - return False - if hdt_strategy == "partitioned": - return True - # `auto` uses partitioned conversion for the aggregate storage modes, while - # ordinary compression-only inputs remain single-pass unless requested. - return mode == "full" and rdf_storage_mode in RDF_STORAGE_MODES +def should_use_partitioned_hdt( + *, + mode: str, + methods: list[str], + hdt_strategy: str, + rdf_storage_mode: str | None = None, +) -> bool: + """Resolve whether the HDT pipeline should use chunked generation + hdtc merge.""" + if not compression_uses_hdt(methods): + return False + if hdt_strategy == "single": + return False + if hdt_strategy == "partitioned": + return True + # `auto` uses partitioned conversion for the aggregate storage modes, while + # ordinary compression-only inputs remain single-pass unless requested. + return mode == "full" and rdf_storage_mode in RDF_STORAGE_MODES + + +# --------------------------------------------------------------------------- +# Run metrics layout: naming, manifest, and summary +# --------------------------------------------------------------------------- +def safe_metrics_name(value: str) -> str: + """Sanitize names used in metrics artifact filenames.""" + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("_") + return safe or "rdf" + + +def input_artifact_stem(path: Path) -> str: + """Return a stable, human-readable label for a source artifact. + + Metric directories are intended for people first, so remove only the + recognized VCF/RDF/representation suffixes rather than repeatedly applying + :attr:`Path.stem` (which turns ``cohort.vcf.gz`` into ``cohort.vcf``). + """ + name = path.name + for suffix in ( + ".vcf.gz", + ".vcf", + ".nt.gz", + ".nt.br", + ".nt", + ".cottas.gz", + ".cottas.br", + ".cottas", + ".hdt", + ".gz", + ".br", + ): + if name.endswith(suffix): + return name[: -len(suffix)] or "input" + return path.stem or "input" + + +def metrics_run_label(source_paths: list[Path], source_root: Path | None = None) -> str: + """Return the input-identifying label for one metrics-run directory.""" + if len(source_paths) == 1: + return safe_metrics_name(input_artifact_stem(source_paths[0])) + + root_label = input_artifact_stem(source_root) if source_root is not None else "inputs" + return safe_metrics_name(f"batch-{root_label}-{len(source_paths)}-inputs") + + +def metrics_run_directory(metrics_root: Path, source_label: str, run_id: str) -> Path: + """Build the canonical input-labelled directory for one invocation.""" + return metrics_root / f"{safe_metrics_name(source_label)}__{run_id}" + + +def read_metrics_csv_rows(path: Path) -> list[dict]: + """Read a metrics CSV without allowing a damaged optional report to fail a run.""" + if not path.is_file(): + return [] + try: + with path.open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + except OSError: + return [] + + +def write_run_manifest( + *, + metrics_dir: Path, + run_id: str, + timestamp: str, + mode: str, + source_label: str, + source_paths: list[Path], + out_root: Path, + options: dict, +): + """Write the static, human-readable identity and configuration of a run.""" + ensure_dir(metrics_dir) + payload = { + "metrics_layout_version": METRICS_LAYOUT_VERSION, + "run_id": run_id, + "timestamp": timestamp, + "mode": mode, + "source_label": source_label, + "output_root": str(out_root), + "metrics_directory": str(metrics_dir), + "inputs": [ + { + "path": str(path), + "file_name": path.name, + "size_bytes": file_size_bytes(path), + } + for path in source_paths + ], + "options": options, + } + path = metrics_dir / "run.json" + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return path + + +def update_run_manifest(metrics_dir: Path, **updates) -> None: + """Merge late-bound runtime details (such as the resolved image) into run.json.""" + path = metrics_dir / "run.json" + try: + payload = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + payload.update(updates) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except (OSError, json.JSONDecodeError): + # Metadata must improve diagnosability, never invalidate a completed run. + pass + + +def write_run_summary( + *, + metrics_dir: Path, + run_id: str, + timestamp: str, + mode: str, + exit_code: int, + elapsed_seconds: float, + total_triples: int | None, +): + """Write one discoverable end-of-run summary for all workflow modes. + + Individual stage reports retain their native detail; this file provides the + compact landing page that links their location and repeats the tabular rows + most often used for analysis. + """ + stage_dir = metrics_dir / "stages" + stage_reports = [ + path.relative_to(metrics_dir).as_posix() + for path in sorted(stage_dir.rglob("*.json")) + ] if stage_dir.is_dir() else [] + report_dir = metrics_dir / "reports" + reports = [ + path.relative_to(metrics_dir).as_posix() + for path in sorted(report_dir.rglob("*")) + if path.is_file() + ] if report_dir.is_dir() else [] + payload = { + "metrics_layout_version": METRICS_LAYOUT_VERSION, + "run_id": run_id, + "timestamp": timestamp, + "completed_at": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + "mode": mode, + "status": "success" if int(exit_code) == 0 else "failure", + "exit_code": int(exit_code), + "execution": { + "wall_seconds": round(float(elapsed_seconds), 6), + "wall_human": format_duration(elapsed_seconds), + "total_triples": total_triples, + }, + "summary_tables": { + "conversion_and_compression": read_metrics_csv_rows(metrics_dir / "metrics.csv"), + "tsv": read_metrics_csv_rows(metrics_dir / "tsv_metrics.csv"), + "wrapper": read_metrics_csv_rows(metrics_dir / "wrapper_execution_times.csv"), + }, + "stage_reports": stage_reports, + "reports": reports, + "logs": [ + path.relative_to(metrics_dir).as_posix() + for path in sorted((metrics_dir / "logs").rglob("*")) + if path.is_file() + ] if (metrics_dir / "logs").is_dir() else [], + } + path = metrics_dir / "summary.json" + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return path + + +def progress_event_path(metrics_dir: Path | None, *components: str) -> Path | None: + """Return a hidden, per-operation progress sidecar path.""" + if metrics_dir is None: + return None + safe_components = [safe_metrics_name(component) for component in components] + filename = "-".join(safe_components or ["run"]) + ".jsonl" + return metrics_dir / ".progress" / filename -def safe_metrics_name(value: str) -> str: - """Sanitize names used in metrics artifact filenames.""" - safe = re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("_") - return safe or "rdf" +def container_progress_path(path: Path | None, metrics_dir: Path | None) -> str | None: + """Translate a host progress sidecar into its mounted container path.""" + if path is None or metrics_dir is None: + return None + try: + relative = path.resolve().relative_to(metrics_dir.resolve()) + except ValueError: + return None + return f"/data/metrics/{relative.as_posix()}" -def metrics_header_for_methods(selected_methods: list[str]) -> list[str]: +def metrics_header_for_methods( + selected_methods: list[str], *, include_validation: bool = False +) -> list[str]: """Build a run-specific metrics.csv header with only relevant columns.""" methods = list(selected_methods or []) header = list(CONVERSION_METRICS_HEADER) - if not methods: + if methods: + header.extend(COMPRESSION_COMMON_COLUMNS) + if "gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["gzip"]) + if "brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["brotli"]) + + uses_cottas = any(method in COTTAS_COMPRESSION_METHODS for method in methods) + if uses_cottas: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas"]) + + uses_hdt = any(method in HDT_COMPRESSION_METHODS for method in methods) + if uses_hdt: + header.extend(COMPRESSION_METHOD_COLUMNS["hdt"]) + header.append(HDT_SOURCE_COLUMN) + if "hdt_gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["hdt_gzip"]) + if "hdt_brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["hdt_brotli"]) + if "cottas_gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas_gzip"]) + if "cottas_brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas_brotli"]) + + if include_validation: + header.extend(VALIDATION_METRICS_COLUMNS) + if not methods and not include_validation: return header - - header.extend(COMPRESSION_COMMON_COLUMNS) - if "gzip" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["gzip"]) - if "brotli" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["brotli"]) - - uses_cottas = any(method in COTTAS_COMPRESSION_METHODS for method in methods) - if uses_cottas: - header.extend(COMPRESSION_METHOD_COLUMNS["cottas"]) - - uses_hdt = any(method in HDT_COMPRESSION_METHODS for method in methods) - if uses_hdt: - header.extend(COMPRESSION_METHOD_COLUMNS["hdt"]) - header.append(HDT_SOURCE_COLUMN) - if "hdt_gzip" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["hdt_gzip"]) - if "hdt_brotli" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["hdt_brotli"]) - if "cottas_gzip" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["cottas_gzip"]) - if "cottas_brotli" in methods: - header.extend(COMPRESSION_METHOD_COLUMNS["cottas_brotli"]) - return unique_in_order(header) @@ -1873,6 +3445,7 @@ def update_metrics_csv_with_compression( selected_methods: list[str], method_results: dict[str, dict], tsv_metrics: dict | None = None, + validation_result: dict | None = None, ): """Upsert compression-related columns in `metrics.csv` for one output artifact. @@ -1880,7 +3453,9 @@ def update_metrics_csv_with_compression( HDT-first metrics (gzip_on_hdt / brotli_on_hdt) to avoid ambiguity. """ metrics_csv.parent.mkdir(parents=True, exist_ok=True) - target_header = metrics_header_for_methods(selected_methods) + target_header = metrics_header_for_methods( + selected_methods, include_validation=validation_result is not None + ) rows = [] existing_header = [] @@ -1918,7 +3493,6 @@ def update_metrics_csv_with_compression( row["combined_rdf_size_bytes"] = str(int(combined_size_bytes)) if "compression_methods" in row: row["compression_methods"] = "|".join(selected_methods) if selected_methods else "none" - defaults = { "exit_code_tsv": "0", "wall_seconds_tsv": "null", @@ -1971,6 +3545,23 @@ def update_metrics_csv_with_compression( "sys_seconds_brotli_on_hdt": "null", "max_rss_kb_brotli_on_hdt": "null", } + if validation_result is not None: + defaults.update( + { + "validation_engines": "", + "validation_oracle_seconds": "null", + "validation_engine_query_seconds": "", + "validation_engine_setup_seconds": "", + "validation_status": "NOT_RUN", + "validation_exit_code": "null", + "validation_wall_seconds": "null", + "validation_user_seconds": "null", + "validation_sys_seconds": "null", + "validation_max_rss_kb": "null", + "validation_results_path": "", + "validation_rdf_path": "", + } + ) for key, value in defaults.items(): if key in row: row[key] = value @@ -2081,6 +3672,55 @@ def assign_validation(method: str, result: dict): row["exit_code_brotli_on_cottas"] = str(int(cottas_brotli_result.get("exit_code") or 0)) assign_timing("brotli_on_cottas", cottas_brotli_result) + if validation_result is not None: + validation_timing = validation_result.get("timing") or {} + if "validation_status" in row: + row["validation_status"] = str( + validation_result.get("status") or "EXECUTION_FAILED" + ) + if "validation_exit_code" in row: + exit_code = validation_result.get("exit_code") + row["validation_exit_code"] = "null" if exit_code is None else str(int(exit_code)) + for key, metric_key in ( + ("validation_wall_seconds", "wall_seconds"), + ("validation_user_seconds", "user_seconds"), + ("validation_sys_seconds", "sys_seconds"), + ): + if key in row: + value = validation_timing.get(metric_key) + row[key] = "null" if value is None else f"{float(value):.6f}" + if "validation_max_rss_kb" in row: + rss = validation_timing.get("max_rss_kb") + row["validation_max_rss_kb"] = "null" if rss is None else str(int(rss)) + if "validation_results_path" in row: + row["validation_results_path"] = str(validation_result.get("results_dir") or "") + if "validation_rdf_path" in row: + row["validation_rdf_path"] = str( + validation_result.get("rdf_path") + or validation_result.get("rdf_gzip_path") + or "" + ) + benchmark = validation_result.get("benchmark") or {} + if "validation_engines" in row: + row["validation_engines"] = "|".join(validation_result.get("engines") or []) + if "validation_oracle_seconds" in row: + oracle = benchmark.get("oracleSeconds") + row["validation_oracle_seconds"] = ( + "null" if oracle is None else f"{float(oracle):.6f}" + ) + for column, key in ( + ("validation_engine_query_seconds", "engineQuerySeconds"), + ("validation_engine_setup_seconds", "engineSetupSeconds"), + ): + if column in row: + # "engine=seconds" pairs, so one column holds every engine's + # figure without the header depending on which engines ran. + values = benchmark.get(key) or {} + row[column] = "|".join( + f"{name}={float(value):.6f}" + for name, value in values.items() if value is not None + ) + with metrics_csv.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=target_header) writer.writeheader() @@ -2097,15 +3737,17 @@ def write_compression_metrics_artifacts( combined_size_bytes: int, selected_methods: list[str], method_results: dict[str, dict], + index_warnings: list[dict] | None = None, + validation_result: dict | None = None, ): - """Write per-output compression artifacts (time files + structured JSON).""" + """Write the final per-output compression summary and method timing files.""" metrics_dir.mkdir(parents=True, exist_ok=True) safe_name = safe_metrics_name(output_name) for method, result in method_results.items(): - time_log_dir = metrics_dir / "compression_time" / method / safe_name + time_log_dir = metrics_dir / "timings" / "compression" / safe_name time_log_dir.mkdir(parents=True, exist_ok=True) - time_log = time_log_dir / f"{run_id}.txt" + time_log = time_log_dir / f"{safe_metrics_name(method)}.txt" lines = [ f"method={method}", f"exit_code={result.get('exit_code', 1)}", @@ -2153,6 +3795,11 @@ def timing_payload(result: dict): "compression_methods": ",".join(selected_methods) if selected_methods else "none", "combined_rdf_path": str(source_rdf_path), "combined_rdf_size_bytes": int(combined_size_bytes), + "index_warnings": list(index_warnings or []), + # This preserves every method-specific detail returned by a container + # (validation, chunk plan, index information, and workspace metrics), + # rather than reducing it to the CSV's scalar columns. + "methods": method_results, "hdt_source": str(hdt_result.get("source") or "not_used"), "gzip_raw_rdf": { "output_gz_path": gzip_result.get("output_path", ""), @@ -2207,10 +3854,12 @@ def timing_payload(result: dict): "timing": timing_payload(cottas_brotli_result), }, } + if validation_result is not None: + payload["semantic_validation"] = validation_result - metrics_json_dir = metrics_dir / "compression_metrics" / safe_name + metrics_json_dir = metrics_dir / "stages" / "compression" metrics_json_dir.mkdir(parents=True, exist_ok=True) - metrics_json = metrics_json_dir / f"{run_id}.json" + metrics_json = metrics_json_dir / f"{safe_name}.json" metrics_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") @@ -2224,11 +3873,13 @@ def write_raw_compression_metrics_artifact( source_rdf_path: Path, selected_methods: list[str], method_results: dict[str, dict], + index_warnings: list[dict] | None = None, + auxiliary_stages: dict[str, dict] | None = None, ): - """Persist per-RDF-file compression metrics under `raw_metrics/`.""" + """Persist the operation-level compression detail for one RDF source.""" safe_output = safe_metrics_name(output_name) safe_rdf = safe_metrics_name(rdf_name) - raw_json_dir = metrics_dir / "raw_metrics" / "compression_metrics" / safe_output / safe_rdf + raw_json_dir = metrics_dir / "stages" / "compression_operations" / safe_output raw_json_dir.mkdir(parents=True, exist_ok=True) payload = { @@ -2238,6 +3889,8 @@ def write_raw_compression_metrics_artifact( "rdf_name": rdf_name, "source_rdf_path": str(source_rdf_path), "compression_methods": ",".join(selected_methods) if selected_methods else "none", + "index_warnings": list(index_warnings or []), + "auxiliary_stages": dict(auxiliary_stages or {}), "methods": {}, } @@ -2256,10 +3909,37 @@ def write_raw_compression_metrics_artifact( "validation": result.get("validation") or details.get("validation"), } - raw_json = raw_json_dir / f"{run_id}.json" + raw_json = raw_json_dir / f"{safe_rdf}.json" raw_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") +def write_partitioned_container_stage_report( + *, + metrics_dir: Path, + output_name: str, + source_rdf_path: Path, + payload: dict, +): + """Preserve the detailed stage handoff from the ephemeral Docker volume. + + The partitioned runner records every chunk build, merge, validation, disk + watermark, and GNU-time resource measurement. Its workspace is deleted at + the end of the operation, so this report is the durable location for those + deeper container metrics on both success and failure. + """ + report_dir = metrics_dir / "stages" / "partitioned" + ensure_dir(report_dir) + report = { + "runtime_environment": "docker-volume", + "source_rdf_path": str(source_rdf_path), + "output_name": output_name, + "container_result": payload, + } + report_path = report_dir / f"{safe_metrics_name(output_name)}.json" + report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + return report_path + + def validate_mode_dirs(paths): """Validate that expected directory arguments are not file paths.""" for p in paths: @@ -2296,9 +3976,9 @@ def write_tsv_metrics_artifacts( output_paths: list[Path], output_size_bytes: int, ): - """Persist raw TSV-step metrics under `raw_metrics/`.""" + """Persist one TSV container stage report in the canonical stage tree.""" safe_prefix = safe_metrics_name(prefix) - json_dir = metrics_dir / "raw_metrics" / "tsv_metrics" / safe_prefix + json_dir = metrics_dir / "stages" / "tsv" json_dir.mkdir(parents=True, exist_ok=True) payload = { "run_id": run_id, @@ -2317,7 +3997,7 @@ def write_tsv_metrics_artifacts( "output_size_bytes": int(output_size_bytes), }, } - (json_dir / f"{run_id}.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + (json_dir / f"{safe_prefix}.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") def run_tsv_conversion_with_metrics( @@ -2333,10 +4013,10 @@ def run_tsv_conversion_with_metrics( ): """Run VCF->TSV conversion and collect per-input timing/resource metrics.""" safe_prefix = safe_metrics_name(prefix) - raw_time_dir = metrics_dir / "raw_metrics" / "tsv_time" / safe_prefix + raw_time_dir = metrics_dir / "timings" / "tsv" raw_time_dir.mkdir(parents=True, exist_ok=True) - time_log_host = raw_time_dir / f"{run_id}.txt" - time_log_container = f"/data/metrics/raw_metrics/tsv_time/{safe_prefix}/{run_id}.txt" + time_log_host = raw_time_dir / f"{safe_prefix}.txt" + time_log_container = f"/data/metrics/timings/tsv/{safe_prefix}.txt" wrapped_command = ( "set -euo pipefail; " @@ -2364,7 +4044,8 @@ def run_tsv_conversion_with_metrics( ] started = time.perf_counter() - exit_code = run(cmd) + with ProgressSession(None, f"TSV conversion: {prefix}"): + exit_code = run(cmd) elapsed = time.perf_counter() - started timing = parse_time_log_metrics(time_log_host) @@ -2583,7 +4264,9 @@ def ensure_image_available( return 2 print(f"{step_label}: Ensuring Docker image is available") print(" - Building Docker image") - if docker_build_image(image_ref, repo_root) != 0: + with ProgressSession(None, "Building Docker image"): + image_exit_code = docker_build_image(image_ref, repo_root) + if image_exit_code != 0: eprint(f"Error: docker build failed. See log: {wrapper_log_path}") return 1 print(f"{step_label}: Ensuring Docker image is available {success_symbol()}") @@ -2596,7 +4279,9 @@ def ensure_image_available( if version_requested: print(f"{step_label}: Ensuring Docker image is available") print(f" - Pulling image: {image_ref}") - if docker_pull_image(image_ref) != 0: + with ProgressSession(None, "Pulling Docker image"): + image_exit_code = docker_pull_image(image_ref) + if image_exit_code != 0: eprint(f"Error: image version '{image_ref}' not found. See log: {wrapper_log_path}") return 2 print(f"{step_label}: Ensuring Docker image is available {success_symbol()}") @@ -2609,12 +4294,16 @@ def ensure_image_available( print(f"{step_label}: Ensuring Docker image is available") if has_local_dockerfile: print(" - Image missing locally, building") - if docker_build_image(image_ref, repo_root) != 0: + with ProgressSession(None, "Building Docker image"): + image_exit_code = docker_build_image(image_ref, repo_root) + if image_exit_code != 0: eprint(f"Error: docker build failed. See log: {wrapper_log_path}") return 1 else: print(f" - Image missing locally, pulling: {image_ref}") - if docker_pull_image(image_ref) != 0: + with ProgressSession(None, "Pulling Docker image"): + image_exit_code = docker_pull_image(image_ref) + if image_exit_code != 0: eprint(f"Error: image '{image_ref}' could not be pulled. See log: {wrapper_log_path}") return 2 print(f"{step_label}: Ensuring Docker image is available {success_symbol()}") @@ -2635,6 +4324,7 @@ def run_compression_methods_for_rdf( timestamp: str | None = None, output_name: str | None = None, expected_triples: int | None = None, + index_warnings: list[dict] | None = None, ): """Run selected compression stages for a single RDF file. @@ -2669,6 +4359,7 @@ def run_compression_methods_for_rdf( target_out_container = f"/data/out/{relative_out.as_posix()}" method_results: dict[str, dict] = {} + allow_index_failure = index_warnings is not None hdt_name = f"{input_stem}.hdt" hdt_path = target_out_dir / hdt_name hdt_container = f"{target_out_container}/{hdt_name}" @@ -2678,9 +4369,10 @@ def run_compression_methods_for_rdf( cottas_path = target_out_dir / cottas_name cottas_container = f"{target_out_container}/{cottas_name}" cottas_is_ready = False + cottas_failure_warning: dict | None = None metrics_output_name = output_name or target_out_dir.name safe_output_name = safe_metrics_name(metrics_output_name) - safe_rdf_name = safe_metrics_name(rdf_path.name) + auxiliary_stage_results: dict[str, dict] = {} def run_container_command( *, @@ -2688,6 +4380,7 @@ def run_container_command( artifact_name: str, command: str, record_method: bool = True, + quiet_failure: bool = False, ): """Execute one compression command in Docker and capture timing/size.""" timing_name = f".{input_stem}.{method}.time" @@ -2704,17 +4397,17 @@ def run_container_command( ) cmd = [ *docker_run_base(), + *docker_hdt_index_env_args(), "-v", f"{str(in_dir)}:/data/in:ro", "-v", f"{str(out_dir)}:/data/out", - image_ref, - "bash", - "-lc", - wrapped_command, ] + cmd.extend([image_ref, "bash", "-lc", wrapped_command]) started = time.perf_counter() - exit_code = run(cmd) + progress_label = f"{method.replace('_', ' ').title()}: {input_stem}" + with ProgressSession(None, progress_label): + exit_code = run(cmd) elapsed = time.perf_counter() - started timing = parse_time_log_metrics(timing_host) output_path = target_out_dir / artifact_name @@ -2731,17 +4424,20 @@ def run_container_command( } if record_method: method_results[method] = result - if record_method and metrics_dir is not None and run_id is not None and timing_host.exists(): + else: + # Validation/index checks consume container resources too. Retain + # their measurements in the operation report rather than dropping + # them because they do not produce a final representation. + auxiliary_stage_results[method] = result + if metrics_dir is not None and timing_host.exists(): raw_time_dir = ( metrics_dir - / "raw_metrics" - / "compression_time" + / "timings" + / "compression" / safe_output_name - / safe_rdf_name - / method ) raw_time_dir.mkdir(parents=True, exist_ok=True) - raw_time_path = raw_time_dir / f"{run_id}.txt" + raw_time_path = raw_time_dir / f"{safe_metrics_name(method)}.txt" try: shutil.copyfile(timing_host, raw_time_path) except OSError: @@ -2753,11 +4449,36 @@ def run_container_command( pass if record_method and method == "hdt": method_results[method]["source"] = "generated" - if exit_code != 0 and record_method: - eprint(f"Error: {method} compression failed. See log: {wrapper_log_path}") + if exit_code != 0: + if record_method and not quiet_failure: + eprint(f"Error: {method} compression failed. See log: {wrapper_log_path}") return False return True + def record_index_warning( + *, + index_format: str, + artifact_path: Path, + message: str, + stage: str, + ) -> dict: + """Record one recoverable representation/index problem for a full run.""" + compact = " ".join(str(message).split()) + warning = { + "format": index_format, + "stage": stage, + "status": "index_unavailable", + "artifact_path": str(artifact_path), + "message": compact, + } + if warning not in index_warnings: + index_warnings.append(warning) + eprint( + f"Warning: {index_format.upper()} index generation failed for " + f"'{artifact_path}'; continuing with the remaining pipeline. {compact}" + ) + return warning + def validate_container_artifact( *, method: str, @@ -2766,56 +4487,84 @@ def validate_container_artifact( artifact_container: str, ) -> tuple[bool, dict]: """Validate a base representation before any packaging or cleanup.""" - report_name = f".{input_stem}.{artifact_format}.validation.json" - report_path = target_out_dir / report_name - report_container = f"{target_out_container}/{report_name}" - command_parts = [ - "set -euo pipefail;", - f"rm -f {shlex.quote(report_container)};", - 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}";', - 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ', - 'echo "Missing Python executable in container" >&2; exit 127; fi;', - 'VALIDATOR="/opt/vcf-rdfizer/validate_compression.py";', - 'if [[ ! -f "$VALIDATOR" ]]; then ', - 'echo "Missing compression validator in container" >&2; exit 127; fi;', - '"$PYTHON_BIN" "$VALIDATOR"', - f"--source {shlex.quote(input_container)}", - f"--artifact {shlex.quote(artifact_container)}", - f"--format {shlex.quote(artifact_format)}", - f"--result-path {shlex.quote(report_container)}", - ] - if expected_triples is not None: - command_parts.append(f"--expected-triples {int(expected_triples)}") - command = " ".join(command_parts) - if not run_container_command( - method=f"{method}-validation", - artifact_name=report_name, - command=command, - record_method=False, - ): - report = { - "valid": False, - "count_match": False, - "error": f"{artifact_format.upper()} validation command failed", - } - else: + def perform_validation(*, skip_index_check: bool) -> tuple[bool, dict]: + report_name = f".{input_stem}.{artifact_format}.validation.json" + report_path = target_out_dir / report_name + report_container = f"{target_out_container}/{report_name}" + command_parts = [ + "set -euo pipefail;", + f"rm -f {shlex.quote(report_container)};", + 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}";', + 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ', + 'echo "Missing Python executable in container" >&2; exit 127; fi;', + 'VALIDATOR="/opt/vcf-rdfizer/validate_compression.py";', + 'if [[ ! -f "$VALIDATOR" ]]; then ', + 'echo "Missing compression validator in container" >&2; exit 127; fi;', + '"$PYTHON_BIN" "$VALIDATOR"', + f"--source {shlex.quote(input_container)}", + f"--artifact {shlex.quote(artifact_container)}", + f"--format {shlex.quote(artifact_format)}", + f"--result-path {shlex.quote(report_container)}", + ] + if expected_triples is not None: + command_parts.append(f"--expected-triples {int(expected_triples)}") + if skip_index_check: + command_parts.append("--skip-index-check") + command = " ".join(command_parts) + command_ok = run_container_command( + method=f"{method}-validation", + artifact_name=report_name, + command=command, + record_method=False, + quiet_failure=skip_index_check, + ) try: report = json.loads(report_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: report = { "valid": False, "count_match": False, - "error": f"validator did not produce a valid report: {exc}", + "error": ( + f"{artifact_format.upper()} validation command failed" + if not command_ok + else f"validator did not produce a valid report: {exc}" + ), } - report_path.unlink(missing_ok=True) - valid = bool(report.get("valid")) and bool(report.get("count_match")) - if not valid: - eprint( - f"Error: {artifact_format.upper()} validation failed for {artifact_name}: " - f"{report.get('error', 'decoded triple count mismatch')}. " - f"See log: {wrapper_log_path}" + report_path.unlink(missing_ok=True) + execution = auxiliary_stage_results.get(f"{method}-validation") + if execution is not None: + report["execution"] = execution + return ( + bool(report.get("valid")) and bool(report.get("count_match")), + report, ) - return valid, report + + valid, report = perform_validation(skip_index_check=False) + if valid: + return True, report + + if allow_index_failure and artifact_format == "hdt": + readable, readable_report = perform_validation(skip_index_check=True) + if readable: + warning = record_index_warning( + index_format="hdt", + artifact_path=target_out_dir / artifact_name, + stage="hdt-index", + message=report.get( + "error", + "HDT validation succeeded when the index check was skipped", + ), + ) + readable_report["index_status"] = "failed" + readable_report["index_warning"] = warning + return True, readable_report + + eprint( + f"Error: {artifact_format.upper()} validation failed for {artifact_name}: " + f"{report.get('error', 'decoded triple count mismatch')}. " + f"See log: {wrapper_log_path}" + ) + return False, report def ensure_hdt_available(): """Ensure `.hdt` exists for HDT-based compound methods.""" @@ -2843,8 +4592,14 @@ def ensure_hdt_available(): artifact_container=hdt_container, ) if not valid: + # ``validate_container_artifact`` already downgrades a + # recoverable HDT index problem to a warning and returns True, + # so reaching this point means the artifact itself is unusable. return False method_results["hdt"]["validation"] = report + if report.get("index_status"): + method_results["hdt"]["index_status"] = report["index_status"] + method_results["hdt"]["index_warning"] = report.get("index_warning") hdt_is_ready = True return True hdt_command = ( @@ -2873,15 +4628,20 @@ def ensure_hdt_available(): if not valid: return False method_results["hdt"]["validation"] = report + if report.get("index_status"): + method_results["hdt"]["index_status"] = report["index_status"] + method_results["hdt"]["index_warning"] = report.get("index_warning") hdt_is_ready = True hdt_source = "generated" return True def ensure_cottas_available(): """Ensure `.cottas` exists for COTTAS packaging stages.""" - nonlocal cottas_is_ready + nonlocal cottas_is_ready, cottas_failure_warning if cottas_is_ready: return True + if cottas_failure_warning is not None: + return False if cottas_path.exists(): method_results.setdefault( "cottas", @@ -2920,7 +4680,15 @@ def ensure_cottas_available(): method="cottas", artifact_name=cottas_name, command=cottas_command, + quiet_failure=allow_index_failure, ): + if allow_index_failure: + cottas_failure_warning = record_index_warning( + index_format="cottas", + artifact_path=cottas_path, + stage="cottas-index", + message="COTTAS conversion/index creation did not produce a usable artifact", + ) return False valid, report = validate_container_artifact( method="cottas", @@ -2929,11 +4697,48 @@ def ensure_cottas_available(): artifact_container=cottas_container, ) if not valid: + if allow_index_failure: + cottas_failure_warning = record_index_warning( + index_format="cottas", + artifact_path=cottas_path, + stage="cottas-index", + message=report.get( + "error", + "COTTAS validation failed after conversion/index creation", + ), + ) return False method_results["cottas"]["validation"] = report cottas_is_ready = True return True + def mark_cottas_method_unavailable(method: str): + """Mark COTTAS and dependent packaging methods as skipped after a failure.""" + warning = cottas_failure_warning + if warning is None: + return + result = method_results.setdefault( + method, + { + "exit_code": 1, + "wall_seconds": 0.0, + "user_seconds": 0.0, + "sys_seconds": 0.0, + "max_rss_kb": 0, + "output_path": str( + { + "cottas": cottas_path, + "cottas_gzip": target_out_dir / f"{input_stem}.cottas.gz", + "cottas_brotli": target_out_dir / f"{input_stem}.cottas.br", + }.get(method, cottas_path) + ), + "output_size_bytes": 0, + }, + ) + result["exit_code"] = 1 + result["index_status"] = "failed" + result["index_warning"] = warning + for method in methods: if method == "gzip": artifact_name = f"{input_stem}.{input_ext}.gz" @@ -2989,7 +4794,10 @@ def ensure_cottas_available(): if method == "cottas": if not ensure_cottas_available(): - return False, method_results + if not allow_index_failure: + return False, method_results + mark_cottas_method_unavailable(method) + continue if status_indent is not None: suffix = " (reused existing COTTAS)" if method_results[method].get("source") == "existing" else "" print(f"{status_indent}- {method}: {cottas_name} {success_symbol()}{suffix}") @@ -3039,7 +4847,10 @@ def ensure_cottas_available(): if method == "cottas_gzip": if not ensure_cottas_available(): - return False, method_results + if not allow_index_failure: + return False, method_results + mark_cottas_method_unavailable(method) + continue artifact_name = f"{input_stem}.cottas.gz" out_container = f"{target_out_container}/{artifact_name}" command = ( @@ -3055,7 +4866,10 @@ def ensure_cottas_available(): if method == "cottas_brotli": if not ensure_cottas_available(): - return False, method_results + if not allow_index_failure: + return False, method_results + mark_cottas_method_unavailable(method) + continue artifact_name = f"{input_stem}.cottas.br" out_container = f"{target_out_container}/{artifact_name}" command = ( @@ -3079,6 +4893,8 @@ def ensure_cottas_available(): source_rdf_path=rdf_path, selected_methods=methods, method_results=method_results, + index_warnings=index_warnings, + auxiliary_stages=auxiliary_stage_results, ) return True, method_results @@ -3099,12 +4915,14 @@ def run_containerized_partitioned_representation_methods( min_chunk_bytes: int, max_chunk_bytes: int, expected_triples: int | None = None, + index_warnings: list[dict] | None = None, ): """Run partitioned compression in an ephemeral Docker-managed volume. - The source and final outputs are the only bind mounts. Chunks, DuckDB - scratch data, HDT/COTTAS merge intermediates, and stage timing files stay - in a named volume that is removed in ``finally`` on both success and + The source and final outputs are the primary bind mounts. When interactive + progress is enabled, a small metrics sidecar is mounted as well. Chunks, + DuckDB scratch data, HDT/COTTAS merge intermediates, and stage timing files + stay in a named volume that is removed in ``finally`` on both success and failure. A short JSON handoff carries the container-side metrics back to the host without exposing the temporary workspace. """ @@ -3118,6 +4936,12 @@ def run_containerized_partitioned_representation_methods( f"vcf-rdfizer-{safe_output_name[:40]}-{os.getpid()}-{time.time_ns()}" ) result_path = out_dir / f".{safe_output_name}.partitioned-results.json" + progress_host_path = ( + progress_event_path(metrics_dir, "partitioned", safe_output_name) + if progress_events_enabled() + else None + ) + progress_container_ref = container_progress_path(progress_host_path, metrics_dir) method_results: dict[str, dict] = {} volume_created = False @@ -3151,15 +4975,22 @@ def run_containerized_partitioned_representation_methods( command = [ *docker_run_base(), + *docker_hdt_index_env_args(), + *docker_hdt_merge_env_args(), + *docker_cottas_merge_env_args(), "--mount", f"type=volume,source={volume_name},target=/work", ] - if source_mount is not None: - command.extend(["-v", source_mount]) + command.extend(["-v", source_mount]) command.extend( [ "-v", f"{out_resolved}:/data/out", + *( + ["-v", f"{metrics_dir.resolve()}:/data/metrics"] + if metrics_dir is not None and progress_container_ref is not None + else [] + ), image_ref, "python3", PARTITIONED_COMPRESSION_RUNNER_CONTAINER, @@ -3180,9 +5011,17 @@ def run_containerized_partitioned_representation_methods( *(["--expected-triples", str(expected_triples)] if expected_triples is not None else []), "--result-path", f"/data/out/{result_path.name}", + *( + ["--progress-path", progress_container_ref] + if progress_container_ref is not None + else [] + ), ] ) - run_exit_code = run(command) + if index_warnings is not None: + command.append("--allow-index-failures") + with ProgressSession(progress_host_path, f"Partitioned compression: {output_name}"): + run_exit_code = run(command) payload = None if result_path.is_file(): try: @@ -3191,6 +5030,22 @@ def run_containerized_partitioned_representation_methods( eprint(f"Error: invalid partitioned compression result: {exc}. See log: {wrapper_log_path}") if payload is not None: + for warning in payload.get("index_warnings", []): + artifact_path = str(warning.get("artifact_path", "")) + if artifact_path.startswith("/data/out/"): + warning["artifact_path"] = str( + out_dir / artifact_path.removeprefix("/data/out/") + ) + if warning not in (index_warnings or []): + if index_warnings is not None: + index_warnings.append(warning) + eprint( + "Warning: " + f"{str(warning.get('format', 'representation')).upper()} index generation " + f"failed for '{warning.get('artifact_path', output_name)}'; " + "continuing with the remaining pipeline. " + f"{warning.get('message', '')}" + ) method_results = payload.get("methods", {}) for method, result in method_results.items(): artifact_path = out_dir / compression_artifact_name_for_method( @@ -3210,6 +5065,19 @@ def run_containerized_partitioned_representation_methods( chunk["path"] = Path(str(chunk["path"])).name details["workspace"] = "docker-volume" details["workspace_cleanup"] = "removed" + if metrics_dir is not None: + try: + write_partitioned_container_stage_report( + metrics_dir=metrics_dir, + output_name=output_name, + source_rdf_path=source_rdf_path, + payload=payload, + ) + except OSError as exc: + eprint( + "Warning: failed to preserve detailed partitioned container metrics: " + f"{exc}" + ) missing_methods = set(methods) - set(method_results) if missing_methods and payload is not None and int(payload.get("exit_code", 1)) == 0: @@ -3235,6 +5103,7 @@ def run_containerized_partitioned_representation_methods( source_rdf_path=out_dir, selected_methods=methods, method_results=method_results, + index_warnings=index_warnings, ) return True, method_results finally: @@ -3269,6 +5138,7 @@ def run_partitioned_representation_methods_for_rdf_files( min_chunk_bytes: int, max_chunk_bytes: int, expected_triples: int | None = None, + index_warnings: list[dict] | None = None, ): """Dispatch aggregate RDF to the ephemeral container pipeline. @@ -3300,6 +5170,7 @@ def run_partitioned_representation_methods_for_rdf_files( min_chunk_bytes=min_chunk_bytes, max_chunk_bytes=max_chunk_bytes, expected_triples=expected_triples, + index_warnings=index_warnings, ) @@ -3307,7 +5178,6 @@ def run_full_mode( *, input_mount_dir: Path, container_inputs: list[str], - input_metrics_target: str, expected_prefixes: list[str], rules_path: Path, out_dir: Path, @@ -3315,6 +5185,16 @@ def run_full_mode( metrics_dir: Path, image_ref: str, out_name: str, + sample_workflow: SampleWorkflow, + info_representation: str = DEFAULT_INFO_REPRESENTATION, + header_representation: str = DEFAULT_HEADER_REPRESENTATION, + run_validation: bool = False, + validation_artifacts: list[str] | None = None, + validation_engine: str | list[str] = DEFAULT_VALIDATION_ENGINE, + validation_engine_options: dict | None = None, + validation_strict_conformance: bool = False, + validation_shacl_shapes: Path | None = None, + filter_oracle: str = "auto", rdf_storage_mode: str, methods: list[str], hdt_strategy: str, @@ -3330,10 +5210,25 @@ def run_full_mode( wrapper_log_path: Path, run_tracker: RunTracker | None = None, ): - """Execute full pipeline: per-input TSV -> RDF -> compression -> metrics.""" - print("Step 3/5: Processing per-input pipeline (TSV -> RDF -> compression)") + """Execute full pipeline: per-input TSV -> RDF -> compression -> validation.""" + pipeline_label = "TSV -> RDF -> compression" + if run_validation: + pipeline_label += " -> validation" + print(f"Step 3/5: Processing per-input pipeline ({pipeline_label})") if spark_partitions is not None: print(f" Spark partition hint: {spark_partitions}") + print(f" Sample representation: {sample_workflow.representation}") + print(f" INFO representation: {info_representation}") + print(f" Header representation: {header_representation}") + validation_artifacts = list(validation_artifacts or ["aggregate"]) + if run_validation: + engine_label = ( + validation_engine if isinstance(validation_engine, str) + else ", ".join(validation_engine) + ) + print( + f" Validation: {', '.join(validation_artifacts)} via {engine_label}" + ) intermediate_dir = tsv_dir.parent ensure_dir(tsv_dir) ensure_dir(out_dir) @@ -3370,6 +5265,7 @@ def run_full_mode( total_triples_produced = 0 saw_triple_counts = False input_failures: list[dict] = [] + index_warnings: list[dict] = [] total_inputs = len(container_inputs) for idx, (container_input, expected_prefix) in enumerate( @@ -3383,6 +5279,9 @@ def run_full_mode( except ValueError: input_vcf = container_input input_failed = False + input_index_warnings: list[dict] = [] + validation_failed = False + validation_result: dict | None = None def fail_current(stage: str, message: str): nonlocal input_failed @@ -3409,13 +5308,10 @@ def fail_current(stage: str, message: str): # Pre-flight write checks for expected TSV outputs to fail fast on # permission/mount problems before starting container work. - for suffix in ( - "records.tsv", - "header_lines.tsv", - "file_metadata.tsv", - "sample_calls.tsv", - "sample_format_values.tsv", - ): + expected_tsv_suffixes = ["records.tsv", "header_lines.tsv", "file_metadata.tsv"] + if sample_workflow.helper_strategy != "none": + expected_tsv_suffixes.extend(["sample_calls.tsv", "sample_format_values.tsv"]) + for suffix in expected_tsv_suffixes: expected_tsv_output = tsv_dir / f"{expected_prefix}.{suffix}" if not ensure_writable_path_or_fix( target_path=expected_tsv_output, @@ -3471,11 +5367,20 @@ def fail_current(stage: str, message: str): sample_calls_tsv = tsv_dir / f"{prefix}.sample_calls.tsv" sample_format_tsv = tsv_dir / f"{prefix}.sample_format_values.tsv" try: - build_sample_support_tsvs( - records_tsv=triplet["records"], - sample_calls_tsv=sample_calls_tsv, - sample_format_tsv=sample_format_tsv, - ) + if sample_workflow.helper_strategy == SAMPLE_HELPER_STRATEGY_MATERIALIZED: + build_sample_support_tsvs( + records_tsv=triplet["records"], + sample_calls_tsv=sample_calls_tsv, + sample_format_tsv=sample_format_tsv, + ) + elif sample_workflow.helper_strategy == "header-only": + # RMLStreamer sees valid, empty canonical sources. Their + # equivalent triples are appended directly after base mapping. + write_sample_support_headers(sample_calls_tsv, sample_format_tsv) + elif sample_workflow.helper_strategy != "none": + raise RuntimeError( + f"unknown sample helper strategy: {sample_workflow.helper_strategy}" + ) except Exception as exc: fail_current( "tsv-derivation", @@ -3512,6 +5417,12 @@ def fail_current(stage: str, message: str): ) continue container_generated_rules = f"/data/rules/{generated_rules.name}" + progress_host_path = ( + progress_event_path(metrics_dir, "rmlstreamer", safe_prefix) + if progress_events_enabled() + else None + ) + progress_container_ref = container_progress_path(progress_host_path, metrics_dir) run_cmd = [ *docker_run_base(), @@ -3598,11 +5509,13 @@ def fail_current(stage: str, message: str): f"IN_VCF={container_input}", "-e", "LOGDIR=/data/metrics", - image_ref, - "bash", - "/opt/vcf-rdfizer/run_conversion.sh", ] - if run(run_cmd) != 0: + if progress_container_ref is not None: + run_cmd.extend(["-e", f"PROGRESS_FILE={progress_container_ref}"]) + run_cmd.extend([image_ref, "bash", "/opt/vcf-rdfizer/run_conversion.sh"]) + with ProgressSession(progress_host_path, f"RMLStreamer: {prefix}"): + rdf_exit_code = run(run_cmd) + if rdf_exit_code != 0: fail_current( "rdf-conversion", f"RMLStreamer step failed for '{prefix}'. See log: {wrapper_log_path}", @@ -3632,8 +5545,105 @@ def fail_current(stage: str, message: str): ) continue + sample_stats = None + if sample_workflow.emitter is not None: + print( + f" * Streaming {sample_workflow.representation} multi-sample RDF " + "(no expanded helper TSVs)" + ) + try: + sample_stats = emit_sample_representation( + sample_workflow, + records_tsv=triplet["records"], + header_lines_tsv=triplet["headers"], + rdf_path=raw_rdf_files[0], + ) + except Exception as exc: + fail_current( + f"{sample_workflow.representation}-sample-rdf-streaming", + f"failed streaming {sample_workflow.representation} sample RDF " + f"for '{prefix}': {exc}. " + f"See log: {wrapper_log_path}", + ) + continue + if triples_produced is not None: + triples_produced += int(sample_stats["triples"]) + if sample_workflow.representation == "expanded": + print( + " * Sample calls streamed: " + f"{sample_stats['sample_calls']:,}; FORMAT values: " + f"{sample_stats['format_values']:,}" + ) + else: + print( + " * Condensed matrices streamed: " + f"{sample_stats['matrices']:,}; FORMAT vectors: " + f"{sample_stats['format_vectors']:,}; reusable samples: " + f"{sample_stats['samples']:,}" + ) + + header_stats = None + if header_representation != "basic": + print(f" * Streaming {header_representation} header RDF") + try: + header_stats = append_header_representation_rdf( + triplet["headers"], raw_rdf_files[0] + ) + except Exception as exc: + fail_current( + "header-rdf-streaming", + f"failed streaming {header_representation} header RDF for " + f"'{prefix}': {exc}. See log: {wrapper_log_path}", + ) + continue + if triples_produced is not None: + triples_produced += int(header_stats["triples"]) + print( + f" * Header lines typed: {header_stats['typed_lines']:,}; " + f"contigs: {header_stats['contigs']:,}" + ) + + info_stats = None + if True: + print(f" * Streaming QUAL and {info_representation} INFO RDF") + try: + info_stats = emit_record_detail( + info_representation, + records_tsv=triplet["records"], + header_lines_tsv=triplet["headers"], + rdf_path=raw_rdf_files[0], + ) + except Exception as exc: + fail_current( + "record-detail-rdf-streaming", + f"failed streaming QUAL/{info_representation} INFO RDF for " + f"'{prefix}': {exc}. See log: {wrapper_log_path}", + ) + continue + if info_stats is not None: + if triples_produced is not None: + triples_produced += int(info_stats["triples"]) + print( + f" * QUAL values streamed: {info_stats['qual_values']:,}; " + f"INFO values: {info_stats['info_values']:,}; " + f"declarations: {info_stats['info_definitions']:,}" + ) + if triples_produced is None: triples_produced = count_triples_in_nt_files(raw_rdf_files) + if header_stats is not None: + sample_stats = {**(sample_stats or {}), "header": header_stats} + if info_stats is not None: + sample_stats = {**(sample_stats or {}), "info": info_stats} + if sample_stats is not None and triples_produced is not None: + update_conversion_metrics_after_sample_stream( + metrics_dir=metrics_dir, + output_name=output_name, + run_id=run_id, + rdf_path=raw_rdf_files[0], + total_triples=triples_produced, + sample_stats=sample_stats, + ) if triples_produced is not None: saw_triple_counts = True total_triples_produced += triples_produced @@ -3669,6 +5679,7 @@ def fail_current(stage: str, message: str): timestamp=timestamp, output_name=output_name, expected_triples=triples_produced, + index_warnings=input_index_warnings, ) if not ok: fail_current( @@ -3694,18 +5705,129 @@ def fail_current(stage: str, message: str): min_chunk_bytes=chunk_min_bytes, max_chunk_bytes=chunk_max_bytes, expected_triples=triples_produced, + index_warnings=input_index_warnings, ) if not ok: fail_current( "compression", f"partitioned compression failed for '{output_name}'. See log: {wrapper_log_path}", ) + for warning in input_index_warnings: + warning.setdefault("input_index", idx) + warning.setdefault("input_vcf", input_vcf) + warning.setdefault("expected_prefix", expected_prefix) + if warning not in index_warnings: + index_warnings.append(warning) if input_failed: continue print(f" * Compression {success_symbol()}") if run_tracker is not None: run_tracker.mark(f"Input {idx}: compression completed for {output_name}") + if run_validation: + # Each requested target is validated independently, so a run can + # prove that the aggregate, the HDT, and the COTTAS artifact all + # reproduce the same VCF summaries. `validation_result` keeps the + # aggregate's report in the metrics row for backward compatibility; + # every target also gets its own stage report and results tree. + validation_targets = resolve_validation_targets( + requested=validation_artifacts, + output_dir=out_dir / output_name, + output_name=output_name, + aggregate_path=raw_rdf_files[0], + selected_methods=selected_methods, + ) + validation_failures: list[str] = [] + for target in validation_targets: + target_id = ( + output_name + if target["name"] == "aggregate" + else f"{output_name}__{target['name']}" + ) + safe_target_id = safe_metrics_name(target_id) + target_results_dir = ( + metrics_dir / "reports" / "validation" / safe_target_id + ) + print( + f" * Semantic validation ({target['name']}, {engine_label}): " + f"{target['path'].name}" + ) + target_result = { + "run_id": run_id, + "timestamp": timestamp, + "stage": "validation", + "validation_id": target_id, + "validation_target": target["name"], + "vcf_path": str(input_vcf), + "rdf_path": str(target["path"]), + "rdf_format": target["format"], + "engine": engine_label, + "representation": sample_workflow.representation, + "results_dir": str(target_results_dir), + "summary_path": str(target_results_dir / "summary.json"), + "input_rdf_size_bytes": int(file_size_bytes(target["path"]) or 0), + "exit_code": 1, + "status": "EXECUTION_FAILED", + "timing": { + "wall_seconds": None, + "user_seconds": None, + "sys_seconds": None, + "max_rss_kb": None, + }, + "temporary_rdf": { + "decompressed_inside_container": target["format"] != "nt", + "persisted_on_host": False, + "cleanup_confirmed_by_runner": False, + }, + } + try: + target_exit_code = run_validation_mode( + vcf_path=Path(input_vcf), + rdf_path=target["path"], + rdf_format=target["format"], + representation=sample_workflow.representation, + validation_id=target_id, + results_dir=target_results_dir, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + image_ref=image_ref, + filter_oracle=filter_oracle, + engine=validation_engine, + engine_options=validation_engine_options, + strict_conformance=validation_strict_conformance, + shacl_shapes=validation_shacl_shapes, + wrapper_log_path=wrapper_log_path, + run_tracker=run_tracker, + stage_result=target_result, + ) + except Exception as exc: + target_result["error"] = str(exc) + try: + stage_path = ( + metrics_dir / "stages" / "validation" / f"{safe_target_id}.json" + ) + stage_path.parent.mkdir(parents=True, exist_ok=True) + stage_path.write_text( + json.dumps(target_result, indent=2) + "\n", encoding="utf-8" + ) + except OSError: + pass + eprint( + f"Error: semantic validation could not be completed for " + f"'{target_id}': {exc}. See log: {wrapper_log_path}" + ) + target_exit_code = 1 + if target["name"] == "aggregate": + validation_result = target_result + if int(target_exit_code) != 0: + validation_failures.append(target["name"]) + if validation_result is None and validation_targets: + # Only non-aggregate targets ran; keep the first for the row. + validation_result = target_result + validation_failed = bool(validation_failures) + validation_exit_code = 1 if validation_failed else 0 + raw_size_before_cleanup_by_file = { raw_rdf_path.name: int(file_size_bytes(raw_rdf_path) or 0) for raw_rdf_path in raw_rdf_files } @@ -3725,6 +5847,8 @@ def fail_current(stage: str, message: str): combined_size_bytes=combined_size_before_cleanup, selected_methods=selected_methods, method_results=aggregated_results, + index_warnings=input_index_warnings, + validation_result=validation_result, ) update_metrics_csv_with_compression( metrics_csv=metrics_dir / "metrics.csv", @@ -3736,6 +5860,7 @@ def fail_current(stage: str, message: str): selected_methods=selected_methods, method_results=aggregated_results, tsv_metrics=tsv_metrics, + validation_result=validation_result, ) except PermissionError as exc: blocked_path = exc.filename or str(metrics_dir) @@ -3747,18 +5872,42 @@ def fail_current(stage: str, message: str): ) return 1 + if validation_failed: + fail_current( + "validation", + "semantic VCF/RDF validation failed for " + f"'{output_name}' ({', '.join(validation_failures)}). " + f"See results: {metrics_dir / 'reports' / 'validation'}", + ) + rdf_storage_removed = False - if selected_methods and ( + if not validation_failed and selected_methods and ( remove_rdf_storage_output or not keep_rmlstreamer_rdf_output ): # Cleanup raw RDF only after every selected compression method has # completed successfully for that specific RDF artifact. cleanup_failed = False + warning_formats = { + warning.get("format") + for warning in input_index_warnings + if warning.get("format") in {"hdt", "cottas"} + } + optional_failed_methods = { + method + for method in selected_methods + if ( + (method in HDT_COMPRESSION_METHODS and "hdt" in warning_formats) + or (method in COTTAS_COMPRESSION_METHODS and "cottas" in warning_formats) + ) + } if use_partitioned_compression: missing_or_failed_hdt = [] for method in partitioned_methods: result = partitioned_representation_results.get(method) - if result is None or int(result.get("exit_code", 1)) != 0: + if ( + method not in optional_failed_methods + and (result is None or int(result.get("exit_code", 1)) != 0) + ): missing_or_failed_hdt.append(method) if missing_or_failed_hdt: fail_current( @@ -3778,7 +5927,10 @@ def fail_current(stage: str, message: str): missing_or_failed = [] for method in methods_to_validate: result = method_results.get(method) - if result is None or int(result.get("exit_code", 1)) != 0: + if ( + method not in optional_failed_methods + and (result is None or int(result.get("exit_code", 1)) != 0) + ): missing_or_failed.append(method) if missing_or_failed: fail_current( @@ -3791,6 +5943,13 @@ def fail_current(stage: str, message: str): cleanup_failed = True break + if optional_failed_methods: + eprint( + f"Warning: retaining raw RDF '{raw_rdf_path.name}' because " + "one or more COTTAS/HDT index-dependent outputs were unavailable." + ) + break + # In space-optimized mode the aggregate `.nt.gz` is itself # the gzip artifact. Do not remove it when gzip was selected. if ( @@ -3816,93 +5975,64 @@ def fail_current(stage: str, message: str): continue rdf_storage_removed = not any(path.exists() for path in raw_rdf_files) - if raw_rdf_files: - output_root = out_dir / output_name - raw_total_size = sum(raw_size_before_cleanup_by_file.values()) - - if keep_rmlstreamer_rdf_output: - raw_note = "retained via --keep-rmlstreamer-rdf-output" - elif rdf_storage_removed and remove_rdf_storage_output: - raw_note = "removed via --remove-rdf-storage-output" - elif rdf_storage_removed and selected_methods: - raw_note = "removed after successful compression" - elif selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: - raw_note = "retained because it is also the selected gzip artifact" - elif selected_methods: - raw_note = "retained" - else: - raw_note = "kept (compression methods set to none)" - - first_path = raw_rdf_files[0] - print(f" * Output directory: {output_root}") - print(f" - RDF aggregate: {first_path.name}") - raw_text = format_bytes(raw_total_size) - print( - f" - {rdf_label_for_path(first_path)}: {raw_text} " - f"({raw_note})" - ) + # Full mode always produces exactly one aggregate per input. + output_root = out_dir / output_name + raw_total_size = sum(raw_size_before_cleanup_by_file.values()) + + if validation_failed: + raw_note = "retained after validation failure" + elif keep_rmlstreamer_rdf_output: + raw_note = "retained via --keep-rmlstreamer-rdf-output" + elif rdf_storage_removed and remove_rdf_storage_output: + raw_note = "removed via --remove-rdf-storage-output" + elif rdf_storage_removed and selected_methods: + raw_note = "removed after successful compression" + elif selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: + raw_note = "retained because it is also the selected gzip artifact" + elif selected_methods: + raw_note = "retained" + else: + raw_note = "kept (compression methods set to none)" - if selected_methods: - for method in selected_methods: - if use_partitioned_compression and method in PARTITIONED_COMPRESSION_METHODS: - result = partitioned_representation_results.get(method) - label = compression_method_label_for_path(first_path, method) - if not result or int(result.get("exit_code", 1)) != 0: - print(f" - {label}: not generated") - else: - print( - f" - {label}: {format_bytes(int(result.get('output_size_bytes') or 0))} " - f"({result.get('output_path', '')})" - ) - continue - method_total = 0 - method_count = 0 - for raw_rdf_path in raw_rdf_files: - result = method_results_by_file.get(raw_rdf_path.name, {}).get(method) - if not result or int(result.get("exit_code", 1)) != 0: - continue - method_total += int(result.get("output_size_bytes") or 0) - method_count += 1 + first_path = raw_rdf_files[0] + print(f" * Output directory: {output_root}") + print(f" - RDF aggregate: {first_path.name}") + raw_text = format_bytes(raw_total_size) + print( + f" - {rdf_label_for_path(first_path)}: {raw_text} " + f"({raw_note})" + ) + if selected_methods: + for method in selected_methods: + if use_partitioned_compression and method in PARTITIONED_COMPRESSION_METHODS: + result = partitioned_representation_results.get(method) label = compression_method_label_for_path(first_path, method) - if method_count == 0: + if not result or int(result.get("exit_code", 1)) != 0: print(f" - {label}: not generated") else: - print(f" - {label}: {format_bytes(method_total)}") - else: - print(" - Compression: none selected") - print(f" - Final RDF size (no compression): {format_bytes(raw_total_size)}") - else: - for raw_rdf_path in raw_rdf_files: - hdt_path = (out_dir / output_name) / f"{raw_rdf_path.stem}.hdt" - rdf_size = file_size_bytes(raw_rdf_path) - nt_note = None - method_results = method_results_by_file.get(raw_rdf_path.name, {}) - if raw_rdf_path.exists() and keep_rmlstreamer_rdf_output: - nt_note = "retained via --keep-rmlstreamer-rdf-output" - elif raw_rdf_path.exists() and selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: - nt_note = "retained because it is also the selected gzip artifact" - elif not raw_rdf_path.exists() and remove_rdf_storage_output: - nt_note = "removed via --remove-rdf-storage-output" - elif not raw_rdf_path.exists() and selected_methods: - nt_note = "removed after successful compression" - elif not raw_rdf_path.exists() and not selected_methods: - nt_note = "kept (compression methods set to none)" + print( + f" - {label}: {format_bytes(int(result.get('output_size_bytes') or 0))} " + f"({result.get('output_path', '')})" + ) + continue + method_total = 0 + method_count = 0 + for raw_rdf_path in raw_rdf_files: + result = method_results_by_file.get(raw_rdf_path.name, {}).get(method) + if not result or int(result.get("exit_code", 1)) != 0: + continue + method_total += int(result.get("output_size_bytes") or 0) + method_count += 1 + + label = compression_method_label_for_path(first_path, method) + if method_count == 0: + print(f" - {label}: not generated") else: - nt_note = "retained" - print_nt_hdt_summary( - output_root=out_dir / output_name, - nt_path=raw_rdf_path, - hdt_path=hdt_path, - indent=" ", - nt_note=nt_note, - nt_size_override=rdf_size, - selected_methods=selected_methods, - method_results=method_results, - ) - if not selected_methods: - total_raw_size = sum(raw_size_before_cleanup_by_file.values()) - print(f" * Final RDF size (no compression): {format_bytes(total_raw_size)}") + print(f" - {label}: {format_bytes(method_total)}") + else: + print(" - Compression: none selected") + print(f" - Final RDF size (no compression): {format_bytes(raw_total_size)}") if not keep_tsv: # Cleanup only the triplet generated for this input iteration. @@ -3934,7 +6064,10 @@ def fail_current(stage: str, message: str): continue if run_tracker is not None: - run_tracker.mark(f"Input {idx}/{total_inputs} completed: {output_name}") + if input_failed: + run_tracker.mark(f"Input {idx}/{total_inputs} finished with failures: {output_name}") + else: + run_tracker.mark(f"Input {idx}/{total_inputs} completed: {output_name}") if not keep_tsv and intermediate_dir.exists(): if not remove_path_with_docker_fallback( @@ -3956,6 +6089,19 @@ def fail_current(stage: str, message: str): elif not selected_methods: print("Total triples produced (full run): unavailable") + index_warning_report = None + if index_warnings: + index_warning_report = write_index_warnings_report( + metrics_dir=metrics_dir, + run_id=run_id, + warnings=index_warnings, + ) + eprint( + f"Index generation warnings were recorded for {len(index_warnings)} item(s): " + f"{index_warning_report}" + ) + print(f"Index warnings: {index_warning_report}") + if input_failures: report_path = write_failed_inputs_report(metrics_dir=metrics_dir, failures=input_failures) eprint( @@ -3970,9 +6116,15 @@ def fail_current(stage: str, message: str): ) return 1 - print("Conversion process finished.") + if index_warning_report is not None: + print("Conversion process finished with index warnings.") + else: + print("Conversion process finished.") if run_tracker is not None: - run_tracker.mark("Full pipeline finished successfully") + run_tracker.mark( + "Full pipeline finished successfully" + + (f" with index warnings; report: {index_warning_report}" if index_warning_report else "") + ) return 0 @@ -4077,6 +6229,31 @@ def run_compress_mode( return 1 target_out_dir = out_dir / input_stem + source_size_bytes = int(file_size_bytes(rdf_path) or 0) + try: + write_compression_metrics_artifacts( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + source_rdf_path=rdf_path, + combined_size_bytes=source_size_bytes, + selected_methods=methods, + method_results=method_results, + ) + update_metrics_csv_with_compression( + metrics_csv=metrics_dir / "metrics.csv", + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + output_dir=target_out_dir, + combined_size_bytes=source_size_bytes, + selected_methods=methods, + method_results=method_results, + ) + except OSError as exc: + eprint(f"Error: unable to write compression metrics: {exc}") + return 1 hdt_path = target_out_dir / f"{input_stem}.hdt" print_nt_hdt_summary( output_root=target_out_dir, @@ -4090,6 +6267,69 @@ def run_compress_mode( return 0 +def detect_validation_rdf_format(path: Path) -> str | None: + """Infer the validator's artifact format from a filename, or None.""" + for suffix, fmt in VALIDATION_RDF_SUFFIXES: + if path.name.endswith(suffix): + return fmt + return None + + +def resolve_validation_targets( + *, + requested: list[str], + output_dir: Path, + output_name: str, + aggregate_path: Path, + selected_methods: list[str], +) -> list[dict]: + """Map requested validation targets onto artifacts that actually exist. + + A representation that was not selected, or whose artifact is missing after + a recoverable index warning, is skipped rather than reported as a failure: + the run already recorded why it is absent. + """ + targets: list[dict] = [] + for name in requested: + if name == "aggregate": + fmt = detect_validation_rdf_format(aggregate_path) + if aggregate_path.is_file() and fmt is not None: + targets.append({"name": name, "path": aggregate_path, "format": fmt}) + continue + method_group = ( + HDT_COMPRESSION_METHODS if name == "hdt" else COTTAS_COMPRESSION_METHODS + ) + if not any(method in method_group for method in selected_methods): + continue + candidate = output_dir / f"{output_name}.{name}" + if candidate.is_file(): + targets.append({"name": name, "path": candidate, "format": name}) + return targets + + +def parse_validation_targets(raw: str) -> list[str]: + """Parse --validate-artifacts into an ordered, de-duplicated target list.""" + value = (raw or "").strip() + if value == "" or value == "none": + return [] + if value == "all": + return list(VALIDATION_TARGET_CHOICES) + targets: list[str] = [] + for token in value.split(","): + target = token.strip() + if not target: + continue + if target not in VALIDATION_TARGET_CHOICES: + allowed = ",".join(VALIDATION_TARGET_CHOICES) + raise ValueError( + f"Unsupported value '{target}' for --validate-artifacts. " + f"Use {allowed}, all, or none." + ) + if target not in targets: + targets.append(target) + return targets + + def detect_compressed_format(path: Path): """Infer compressed RDF format from filename/extension.""" if ( @@ -4126,67 +6366,150 @@ def default_decompressed_name(path: Path, fmt: str): return f"{path.stem}.nt" -def run_hdt_index_mode( +def run_index_mode( *, - hdt_path: Path, + index_path: Path, + index_format: str, metrics_dir: Path, image_ref: str, wrapper_log_path: Path, + run_id: str | None = None, + timestamp: str | None = None, ): - """Eagerly create the HDT Java index beside an existing HDT file.""" - print("Step 3/3: Initializing HDT index") + """Generate or regenerate the query index for one existing artifact. + + HDT writes its index as a versioned sibling sidecar. COTTAS stores its + query index in the Parquet artifact itself, so the Docker-side adapter + rewrites that file atomically with the requested index order. + """ + format_label = index_format.upper() + print(f"Step 3/3: Regenerating {format_label} index") ensure_dir(metrics_dir) - existing_index_path = find_hdt_index_sidecar(hdt_path) - index_existed = existing_index_path is not None - source_container = f"/data/hdt/{hdt_path.name}" - command = ( + + existing_index_path = ( + find_hdt_index_sidecar(index_path) if index_format == "hdt" else None + ) + mount_name = "hdt" if index_format == "hdt" else "cottas" + source_container = f"/data/{mount_name}/{index_path.name}" + if index_format == "hdt": + command = ( + "set -euo pipefail; " + f"{shlex.quote(HDT_INDEX_HELPER_CONTAINER)} {shlex.quote(source_container)}" + ) + else: + command = ( + "set -euo pipefail; " + 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}"; ' + 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ' + 'echo "Missing pycottas Python executable in container" >&2; exit 127; fi; ' + f'"$PYTHON_BIN" {shlex.quote(COTTAS_TOOL_CONTAINER)} reindex ' + f"{shlex.quote(source_container)} spo" + ) + safe_input = safe_metrics_name(index_path.name) + timing_host = metrics_dir / "timings" / "index" / f"{safe_input}.txt" + timing_host.parent.mkdir(parents=True, exist_ok=True) + timing_container = f"/data/metrics/timings/index/{safe_input}.txt" + timed_command = ( "set -euo pipefail; " - f"{shlex.quote(HDT_INDEX_HELPER_CONTAINER)} {shlex.quote(source_container)}" + f"rm -f {shlex.quote(timing_container)}; " + 'if [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; then ' + f"/usr/bin/time -v -o {shlex.quote(timing_container)} -- bash -lc {shlex.quote(command)}; " + "else " + f"{{ time -p bash -lc {shlex.quote(command)}; }} > {shlex.quote(timing_container)} 2>&1; " + "fi" ) + input_size_bytes = int(file_size_bytes(index_path) or 0) cmd = [ *docker_run_base(), + *docker_hdt_index_env_args(), + *docker_cottas_merge_env_args(), + "-v", + f"{str(index_path.parent)}:/data/{mount_name}", "-v", - f"{str(hdt_path.parent)}:/data/hdt", + f"{str(metrics_dir.resolve())}:/data/metrics", image_ref, "bash", "-lc", - command, + timed_command, ] started = time.perf_counter() exit_code = run(cmd) elapsed = time.perf_counter() - started - index_path = find_hdt_index_sidecar(hdt_path) - index_ready = index_path is not None + timing = parse_time_log_metrics(timing_host) + index_path_after = ( + find_hdt_index_sidecar(index_path) + if index_format == "hdt" + else (index_path if file_size_bytes(index_path) else None) + ) + index_ready = index_path_after is not None final_code = int(exit_code) if int(exit_code) != 0 else (0 if index_ready else 1) + index_was_present = existing_index_path is not None or index_format == "cottas" payload = { - "hdt_path": str(hdt_path), - "index_path": str(index_path) if index_path else "", + "run_id": run_id, + "timestamp": timestamp, + "index_format": index_format, + "input_path": str(index_path), + "input_size_bytes": input_size_bytes, + "index_path": str(index_path_after) if index_path_after else "", + "index_location": "sidecar" if index_format == "hdt" else "embedded", "exit_code": final_code, "wall_seconds": elapsed, + "timing": { + "wall_seconds": elapsed, + "user_seconds": timing.get("user_seconds"), + "sys_seconds": timing.get("sys_seconds"), + "max_rss_kb": timing.get("max_rss_kb"), + }, "index_status": ( - "existing" if index_existed else "generated" + "regenerated" if index_was_present else "generated" ) if index_ready else "failed", - "index_size_bytes": file_size_bytes(index_path) if index_path else 0, + "index_size_bytes": file_size_bytes(index_path_after) if index_path_after else 0, } - (metrics_dir / "hdt_index_metrics.json").write_text( + if index_format == "hdt": + # Preserve the field used by the original HDT-only metrics payload. + payload["hdt_path"] = str(index_path) + else: + payload["cottas_path"] = str(index_path) + metrics_path = metrics_dir / "stages" / "index" / f"{index_format}-{safe_input}.json" + metrics_path.parent.mkdir(parents=True, exist_ok=True) + metrics_path.write_text( json.dumps(payload, indent=2) + "\n", encoding="utf-8", ) - if final_code != 0: - eprint(f"Error: HDT index initialization failed. See log: {wrapper_log_path}") + eprint(f"Error: {format_label} index regeneration failed. See log: {wrapper_log_path}") return 1 - print(f"HDT index ready: {index_path}") - print(f"HDT index metrics: {metrics_dir / 'hdt_index_metrics.json'}") + print(f"{format_label} index ready: {index_path_after}") + print(f"Index metrics: {metrics_path}") return 0 +def run_hdt_index_mode( + *, + hdt_path: Path, + metrics_dir: Path, + image_ref: str, + wrapper_log_path: Path, +): + """Backward-compatible wrapper for the HDT-only index helper.""" + return run_index_mode( + index_path=hdt_path, + index_format="hdt", + metrics_dir=metrics_dir, + image_ref=image_ref, + wrapper_log_path=wrapper_log_path, + ) + + def run_decompress_mode( *, compressed_path: Path, decompressed_out: Path, + metrics_dir: Path, + run_id: str, + timestamp: str, image_ref: str, wrapper_log_path: Path, ): @@ -4257,22 +6580,292 @@ def run_decompress_mode( f"{shlex.quote(cottas_input)} {shlex.quote(output_container)}" ) + safe_input = safe_metrics_name(compressed_path.name) + timing_host = metrics_dir / "timings" / "decompression" / f"{safe_input}.txt" + timing_host.parent.mkdir(parents=True, exist_ok=True) + timing_container = f"/data/metrics/timings/decompression/{safe_input}.txt" + timed_command = ( + "set -euo pipefail; " + f"rm -f {shlex.quote(timing_container)}; " + 'if [[ -x /usr/bin/time ]] && /usr/bin/time --version >/dev/null 2>&1; then ' + f"/usr/bin/time -v -o {shlex.quote(timing_container)} -- bash -lc {shlex.quote(command)}; " + "else " + f"{{ time -p bash -lc {shlex.quote(command)}; }} > {shlex.quote(timing_container)} 2>&1; " + "fi" + ) + input_size_bytes = int(file_size_bytes(compressed_path) or 0) cmd = [ *docker_run_base(), "-v", f"{str(compressed_path.parent)}:/data/in:ro", "-v", f"{str(decompressed_out.parent)}:/data/out", + "-v", + f"{str(metrics_dir.resolve())}:/data/metrics", image_ref, "bash", "-lc", - command, + timed_command, ] - if run(cmd) != 0: + started = time.perf_counter() + exit_code = run(cmd) + elapsed = time.perf_counter() - started + timing = parse_time_log_metrics(timing_host) + stage_payload = { + "run_id": run_id, + "timestamp": timestamp, + "format": fmt, + "input_path": str(compressed_path), + "input_size_bytes": input_size_bytes, + "output_path": str(decompressed_out), + "output_size_bytes": int(file_size_bytes(decompressed_out) or 0), + # Decompression is deliberately a single pass. Counting N-Triples + # here would reread a cohort-scale output solely for observability. + "output_triples": None, + "exit_code": int(exit_code), + "timing": { + "wall_seconds": elapsed, + "user_seconds": timing.get("user_seconds"), + "sys_seconds": timing.get("sys_seconds"), + "max_rss_kb": timing.get("max_rss_kb"), + }, + } + report_path = metrics_dir / "stages" / "decompression" / f"{safe_input}.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(stage_payload, indent=2) + "\n", encoding="utf-8") + if exit_code != 0: eprint(f"Error: decompression failed. See log: {wrapper_log_path}") return 1 print(f"Done. Decompressed file: {decompressed_out}") + print(f"Decompression metrics: {report_path}") + return 0 + + +def run_validation_mode( + *, + vcf_path: Path, + rdf_path: Path, + representation: str, + validation_id: str, + results_dir: Path, + metrics_dir: Path, + run_id: str, + timestamp: str, + image_ref: str, + filter_oracle: str, + wrapper_log_path: Path, + engine: str | list[str] = DEFAULT_VALIDATION_ENGINE, + engine_options: dict | None = None, + rdf_format: str | None = None, + strict_conformance: bool = False, + shacl_shapes: Path | None = None, + run_tracker: RunTracker | None = None, + stage_result: dict | None = None, + write_metrics_csv: bool = False, +): + """Run VCF/RDF semantic queries with container-local temporary state. + + Accepts any artifact the pipeline produces (``.nt``, ``.nt.gz``, ``.nt.br``, + ``.hdt``, ``.cottas``, ``.cottas[.gz|.br]``). Anything other than plain + N-Triples is decoded under the container's ``/work`` filesystem, so + validating an ``.hdt`` proves it decodes to a graph that still satisfies + every semantic check. No validation scratch RDF is persisted on the host. + + ``engine`` selects the SPARQL backend (``comunica`` or ``qlever``); both + answer the same queries, so it is a scale decision, not a semantic one. + """ + # An empty directory may be left behind if Docker itself fails before the + # runner starts. Reuse only that empty shell; never overwrite reports. + results_dir.mkdir(parents=True, exist_ok=True) + safe_validation_id = safe_metrics_name(validation_id) + timing_host = metrics_dir / "timings" / "validation" / f"{safe_validation_id}.txt" + timing_host.parent.mkdir(parents=True, exist_ok=True) + timing_container = f"/data/metrics/timings/validation/{safe_validation_id}.txt" + progress_host_path = ( + progress_event_path(metrics_dir, "validation", safe_validation_id) + if progress_events_enabled() + else None + ) + progress_container_ref = container_progress_path(progress_host_path, metrics_dir) + resolved_format = rdf_format or detect_validation_rdf_format(rdf_path) + if resolved_format is None: + raise ValueError( + f"Unsupported RDF artifact for validation: {rdf_path.name}. " + "Expected one of .nt, .nt.gz, .nt.br, .hdt, .cottas, .cottas.gz, .cottas.br" + ) + options = dict(engine_options or {}) + engine_list = [engine] if isinstance(engine, str) else list(engine) + engine_args: list[str] = ["--engine", ",".join(engine_list)] + for flag, key in ( + ("--query-timeout", "query_timeout"), + ("--qlever-memory-gb", "qlever_memory_gb"), + ("--qlever-port", "qlever_port"), + ("--qlever-startup-timeout", "qlever_startup_timeout"), + ): + value = options.get(key) + if value is not None: + engine_args.extend([flag, str(value)]) + for flag, key in ( + ("--qlever-index-arg", "qlever_index_args"), + ("--qlever-server-arg", "qlever_server_args"), + ): + for value in options.get(key) or []: + engine_args.extend([flag, str(value)]) + if strict_conformance: + engine_args.append("--strict-conformance") + shacl_mount: list[str] = [] + if shacl_shapes is not None: + # Mounted read-only in its own directory so the shapes file can live + # anywhere on the host without exposing its parent tree for writing. + shacl_mount = ["-v", f"{shacl_shapes.parent.resolve()}:/data/shacl:ro"] + engine_args.extend(["--shacl-shapes", f"/data/shacl/{shacl_shapes.name}"]) + + cmd = [ + *docker_run_base(), + "--init", + "-v", + f"{str(vcf_path.parent)}:/data/vcf:ro", + "-v", + f"{str(rdf_path.parent)}:/data/rdf:ro", + "-v", + f"{str(results_dir)}:/data/validation", + "-v", + f"{str(metrics_dir.resolve())}:/data/metrics", + *shacl_mount, + image_ref, + "/usr/bin/time", + "-v", + "-o", + timing_container, + "--", + "/opt/pycottas-venv/bin/python", + "/opt/vcf-rdfizer/validation/validation_runner.py", + "--vcf", + f"/data/vcf/{vcf_path.name}", + "--rdf", + f"/data/rdf/{rdf_path.name}", + "--rdf-format", + resolved_format, + *engine_args, + "--representation", + representation, + "--results-dir", + "/data/validation", + "--dataset-id", + validation_id, + "--filter-oracle", + filter_oracle, + "--scratch-dir", + "/work", + *( + ["--progress-path", progress_container_ref] + if progress_container_ref is not None + else [] + ), + *(["--quiet"] if _QUIET else []), + ] + if run_tracker is not None: + run_tracker.mark(f"Validation started: {validation_id}") + started = time.perf_counter() + with ProgressSession(progress_host_path, f"Validation: {validation_id}"): + exit_code = run(cmd) + elapsed = time.perf_counter() - started + timing = parse_time_log_metrics(timing_host) + timing["wall_seconds"] = elapsed + summary_path = results_dir / "summary.json" + summary = None + if summary_path.is_file(): + try: + summary = json.loads(summary_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + summary = None + report_path = metrics_dir / "stages" / "validation" / f"{safe_validation_id}.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + status = summary.get("status") if isinstance(summary, dict) else None + if not status: + status = "PASS" if int(exit_code) == 0 else "EXECUTION_FAILED" + if int(exit_code) != 0 and status == "PASS": + status = "EXECUTION_FAILED" + summary_temporary_rdf = summary.get("temporaryRdf", {}) if isinstance(summary, dict) else {} + payload = { + "run_id": run_id, + "timestamp": timestamp, + "stage": "validation", + "validation_id": validation_id, + "vcf_path": str(vcf_path), + "rdf_path": str(rdf_path), + "rdf_format": resolved_format, + "engine": engine_list[0], + "engines": engine_list, + "strict_conformance": bool(strict_conformance), + "representation": representation, + "results_dir": str(results_dir), + "summary_path": str(summary_path), + # Lift the oracle-versus-SPARQL timings into the stage report so a + # benchmark can be read from the run metrics without opening the + # detailed validation results. + "engine_statuses": summary.get("engineStatuses") if isinstance(summary, dict) else None, + "engine_agreement": summary.get("engineAgreement") if isinstance(summary, dict) else None, + "benchmark": summary.get("benchmark") if isinstance(summary, dict) else None, + "benchmark_csv_path": str(results_dir / "benchmark.csv"), + "input_rdf_size_bytes": int(file_size_bytes(rdf_path) or 0), + "exit_code": int(exit_code), + "status": status, + "timing": { + "wall_seconds": timing.get("wall_seconds"), + "user_seconds": timing.get("user_seconds"), + "sys_seconds": timing.get("sys_seconds"), + "max_rss_kb": timing.get("max_rss_kb"), + }, + "temporary_rdf": { + "decompressed_inside_container": bool( + summary_temporary_rdf.get( + "decompressedInsideContainer", resolved_format != "nt" + ) + ), + "persisted_on_host": bool(summary_temporary_rdf.get("persisted", False)), + "cleanup_confirmed_by_runner": bool( + summary_temporary_rdf.get("cleanupConfirmed", False) + ), + }, + } + if resolved_format == "nt.gz": + # Compatibility alias for consumers of the original standalone report + # schema; ``rdf_path`` is the canonical format-neutral field. + payload["rdf_gzip_path"] = str(rdf_path) + report_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + if stage_result is not None: + stage_result.update(payload) + if write_metrics_csv: + # Standalone validation has no preceding conversion/compression stage, + # but it still exposes the same analysis-ready metrics schema used by a + # full run. Full mode writes its row after compression so its selected + # compression columns are preserved. + try: + update_metrics_csv_with_compression( + metrics_csv=metrics_dir / "metrics.csv", + run_id=run_id, + timestamp=timestamp, + output_name=validation_id, + output_dir=results_dir, + combined_size_bytes=int(payload["input_rdf_size_bytes"]), + selected_methods=[], + method_results={}, + validation_result=payload, + ) + except OSError as exc: + eprint(f"Warning: failed to write validation metrics CSV: {exc}") + if exit_code != 0: + eprint(f"Error: validation failed. See results: {results_dir}") + eprint(f"See log for details: {wrapper_log_path}") + if run_tracker is not None: + run_tracker.mark(f"Validation failed: {validation_id} (exit_code={exit_code})") + return 1 + if run_tracker is not None: + run_tracker.mark(f"Validation completed: {validation_id} ({status})") + print(f"Validation results: {results_dir}") + print(f"Validation metrics: {report_path}") return 0 @@ -4293,6 +6886,12 @@ def main(): " Full pipeline (space-optimized aggregate):\n" " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " "--representations hdt,cottas --rdf-compression none -o ./results\n" + " Full pipeline with semantic validation:\n" + " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --rdf-storage-mode plain " + "--representations none --rdf-compression none --validate -o ./results\n" + " Condensed multi-sample representation:\n" + " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --sample-representation condensed " + "--rdf-storage-mode space-optimized --representations hdt -o ./results\n" " Space-optimized full pipeline with shared HDT/COTTAS chunks:\n" " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " "--representations hdt,cottas --rdf-compression none " @@ -4311,27 +6910,35 @@ def main(): "--rdf-compression gzip --representations hdt --artifact-compression gzip -o ./results\n" " Decompression-only:\n" " vcf_rdfizer.py -m decompress -C ./results/out/sample/sample.nt.gz -o ./results\n" - " Initialize an existing HDT index:\n" + " Semantic VCF/RDF validation (expanded graph):\n" + " vcf_rdfizer.py -m validation -i ./sample.vcf.gz --rdf ./results/sample/sample.nt.gz " + "--sample-representation expanded -o ./validation-results\n" + " Semantic VCF/RDF validation (condensed graph):\n" + " vcf_rdfizer.py -m validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz " + "--sample-representation condensed -o ./validation-results\n" + " Generate or regenerate an index for an existing HDT:\n" " vcf_rdfizer.py -m index -H ./results/sample/sample.hdt -o ./results\n" + " Generate or regenerate an index for an existing COTTAS file:\n" + " vcf_rdfizer.py -m index --cottas ./results/sample/sample.cottas -o ./results\n" ), ) parser.add_argument( "-m", "--mode", - choices=["full", "compress", "decompress", "tsv", "index"], + choices=["full", "compress", "decompress", "tsv", "index", "validation"], default="full", - help="Run mode: full pipeline, TSV benchmark, compression, decompression, or HDT index initialization", + help="Run mode: full pipeline, TSV benchmark, compression, decompression, validation, or index-only regeneration", ) parser.add_argument( "-i", "--input", default=None, - help="VCF file or directory (required for --mode full and --mode tsv)", + help="VCF file or directory (required for --mode full/tsv; file required for --mode validation)", ) parser.add_argument( "--rdf", default=None, - help="Input RDF file (.nt or .nt.gz) for --mode compress", + help="Input RDF file (.nt or .nt.gz) for --mode compress; .nt.gz required for --mode validation", ) parser.add_argument( "-C", @@ -4343,7 +6950,12 @@ def main(): "-H", "--hdt", default=None, - help="Existing HDT file for --mode index; HDT Java creates a versioned .hdt.index.* sidecar beside it", + help="Existing .hdt file for --mode index; creates or regenerates its versioned .hdt.index.* sidecar", + ) + parser.add_argument( + "--cottas", + default=None, + help="Existing .cottas file for --mode index; rebuilds its embedded query index in place", ) parser.add_argument( "-d", @@ -4357,6 +6969,48 @@ def main(): default=None, help="RML mapping rules .ttl (default: /rules/default_rules.ttl)", ) + parser.add_argument( + "--sample-representation", + choices=sorted(SAMPLE_REPRESENTATION_CHOICES), + default="expanded", + help=( + "Genotype representation for full/validation mode: expanded emits one SampleCall and " + "FORMAT value resource per sample; condensed emits a shared SampleSet and " + "one sample-ordered value vector per FORMAT key (default: expanded)" + ), + ) + parser.add_argument( + "--header-representation", + choices=HEADER_REPRESENTATION_CHOICES, + default=DEFAULT_HEADER_REPRESENTATION, + help=( + "VCF meta-information representation: structured types each '##' line " + "with its vocabulary subclass and lifts FILTER/ALT/contig attributes " + "into their own properties; basic keeps only untyped header lines " + f"(default: {DEFAULT_HEADER_REPRESENTATION})" + ), + ) + parser.add_argument( + "--info-representation", + choices=INFO_REPRESENTATION_CHOICES, + default=DEFAULT_INFO_REPRESENTATION, + help=( + "INFO column representation: structured also emits one " + "vcfr:InfoFieldValue per record and key alongside the raw string, " + "making INFO queryable; raw keeps only vcfr:infoRaw " + f"(default: {DEFAULT_INFO_REPRESENTATION})" + ), + ) + parser.add_argument( + "--validate", + "--run-validation", + dest="run_validation", + action="store_true", + help=( + "Run semantic VCF/RDF validation for each input during full mode; " + "reports are stored under the run metrics directory" + ), + ) parser.add_argument( "--rdf-storage-mode", choices=sorted(RDF_STORAGE_MODES), @@ -4442,7 +7096,7 @@ def main(): choices=sorted(HDT_STRATEGY_CHOICES), default=DEFAULT_HDT_STRATEGY, help=( - "HDT generation strategy: auto uses partitioned HDT+HDTCat for full-mode aggregate storage, " + "HDT generation strategy: auto uses partitioned HDT+hdtc merge for full-mode aggregate storage, " "single uses one rdf2hdt run, partitioned forces chunked HDT generation" ), ) @@ -4468,6 +7122,106 @@ def main(): action="store_true", help="Print a rough storage estimate before running conversion", ) + parser.add_argument( + "--no-progress", + action="store_true", + help="Disable progress sidecars and terminal progress updates", + ) + parser.add_argument( + "--quiet", + action="store_true", + help=( + "Suppress terminal progress displays (including validation query updates) " + "while retaining progress logging and metrics" + ), + ) + parser.add_argument( + "--validation-id", + default=None, + help="Validation result identifier (default: source VCF basename)", + ) + parser.add_argument( + "--filter-oracle", + choices=("auto", "bcftools", "cyvcf2"), + default="auto", + help="FILTER oracle for full-mode validation and standalone validation (default: auto)", + ) + parser.add_argument( + "--shacl-shapes", + default=None, + help=( + "Validate each graph against a SHACL shapes file as an independent " + "structural layer (for example the vocabulary's published shapes). " + "Off by default: it loads the whole graph into memory, so it does " + "not scale to a cohort-sized aggregate" + ), + ) + parser.add_argument( + "--strict-conformance", + action="store_true", + help=( + "Fail validation when a missing token is serialized as a plain '.' " + "literal instead of '.'^^vcfr:Null (reported but non-fatal by default)" + ), + ) + parser.add_argument( + "--validation-engine", + default=DEFAULT_VALIDATION_ENGINE, + help=( + "SPARQL engine(s) for validation, comma-separated, or 'all'. " + "comunica queries the graph in memory; qlever builds an on-disk " + "index and serves it; hdt and cottas query those compressed " + "artifacts natively without decoding them. Requesting several runs " + "the whole query set on each, cross-checks their answers, and " + "records comparable timings for benchmarking " + f"(choices: {', '.join(VALIDATION_ENGINE_CHOICES)}; " + f"default: {DEFAULT_VALIDATION_ENGINE})" + ), + ) + parser.add_argument( + "--validate-artifacts", + default=DEFAULT_VALIDATION_TARGETS, + help=( + "Which produced artifacts full-mode --validate should check " + "(comma-separated): aggregate,hdt,cottas, or all " + f"(default: {DEFAULT_VALIDATION_TARGETS}). A representation that was " + "not produced is skipped." + ), + ) + parser.add_argument( + "--validation-query-timeout", + default=None, + help="Per-query timeout in seconds for validation (default: engine default)", + ) + parser.add_argument( + "--qlever-memory-gb", + default=None, + help="QLever index/server memory budget in GiB (default: 4)", + ) + parser.add_argument( + "--qlever-port", + default=None, + help="Container-local port for the QLever server (default: 7019)", + ) + parser.add_argument( + "--qlever-startup-timeout", + default=None, + help="Seconds to wait for the QLever server to answer after indexing (default: 900)", + ) + parser.add_argument( + "--qlever-index-arg", + action="append", + default=[], + metavar="ARG", + help="Extra argument for QLever's index builder (repeatable)", + ) + parser.add_argument( + "--qlever-server-arg", + action="append", + default=[], + metavar="ARG", + help="Extra argument for QLever's server (repeatable)", + ) rdf_output_group = parser.add_mutually_exclusive_group() rdf_output_group.add_argument( "-R", @@ -4483,6 +7237,14 @@ def main(): ) args = parser.parse_args() + global _PROGRESS_ALLOWED, _PROGRESS_EVENTS_ALLOWED, _QUIET + _QUIET = bool(args.quiet) + # ``--no-progress`` is the opt-out for the sidecar itself. ``--quiet`` + # only turns off terminal rendering, leaving the existing event stream + # available to the wrapper log/metrics machinery. + _PROGRESS_ALLOWED = not (args.no_progress or args.quiet) + _PROGRESS_EVENTS_ALLOWED = not args.no_progress + if args.build and args.no_build: eprint("Error: --build and --no-build are mutually exclusive.") return 2 @@ -4494,7 +7256,7 @@ def main(): metrics_root = out_root / "run_metrics" run_id = datetime.now().strftime("%Y%m%dT%H%M%S") timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - metrics_dir = metrics_root / run_id + metrics_dir = None mode = args.mode spark_partitions = None chunk_target_bytes = DEFAULT_CHUNK_TARGET_BYTES @@ -4503,7 +7265,35 @@ def main(): step1_label = "Step 1/5" if mode == "full" else "Step 1/3" + validation_artifacts: list[str] = [] + validation_engines: list[str] = [DEFAULT_VALIDATION_ENGINE] + validation_engine_options: dict = {} + shacl_shapes_path: Path | None = None try: + for option_name, key in ( + ("--validation-query-timeout", "query_timeout"), + ("--qlever-memory-gb", "qlever_memory_gb"), + ("--qlever-port", "qlever_port"), + ("--qlever-startup-timeout", "qlever_startup_timeout"), + ): + raw_value = getattr(args, option_name.lstrip("-").replace("-", "_")) + if raw_value is not None: + validation_engine_options[key] = parse_positive_int( + raw_value, name=option_name + ) + if args.qlever_index_arg: + validation_engine_options["qlever_index_args"] = list(args.qlever_index_arg) + if args.qlever_server_arg: + validation_engine_options["qlever_server_args"] = list(args.qlever_server_arg) + if validation_engine_options.get("qlever_port", 1) > 65535: + raise ValueError("--qlever-port must be between 1 and 65535") + validation_engines = parse_validation_engines(args.validation_engine) + validation_artifacts = parse_validation_targets(args.validate_artifacts) + if args.shacl_shapes is not None: + shacl_shapes_path = Path(args.shacl_shapes).expanduser().resolve() + if not shacl_shapes_path.is_file(): + raise ValueError(f"SHACL shapes file not found: {shacl_shapes_path}") + chunk_target_bytes = parse_positive_int( args.chunk_target_bytes, name="--chunk-target-bytes" ) @@ -4528,7 +7318,7 @@ def main(): ( input_mount_dir, container_inputs, - input_metrics_target, + _input_metrics_target, expected_prefixes, ) = resolve_input_snapshot(input_path) if args.rules is None: @@ -4537,6 +7327,17 @@ def main(): rules_path = Path(args.rules).expanduser().resolve() if not rules_path.exists() or not rules_path.is_file(): raise ValueError(f"rules file not found: {rules_path}") + sample_workflow = resolve_sample_workflow( + args.sample_representation, + rules_path, + ) + if args.validate_artifacts != DEFAULT_VALIDATION_TARGETS and not args.run_validation: + raise ValueError("--validate-artifacts requires --validate") + if not validation_artifacts and args.run_validation: + raise ValueError( + "--validate-artifacts resolved to no targets; choose at least one of " + + ", ".join(VALIDATION_TARGET_CHOICES) + ) validate_mode_dirs([out_root, out_dir, tsv_dir, metrics_root]) if args.legacy_compression is not None: if ( @@ -4591,6 +7392,8 @@ def main(): partitioned=full_uses_partitioning, ) validate_no_output_collisions(output_plans) + elif args.run_validation: + raise ValueError("--validate/--run-validation is only valid in --mode full") elif mode == "tsv": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -4604,6 +7407,34 @@ def main(): expected_prefixes, ) = resolve_input_snapshot(input_path) validate_mode_dirs([out_root, out_dir, tsv_dir, metrics_root]) + elif mode == "validation": + if args.spark_partitions is not None: + raise ValueError("--spark-partitions is only valid in --mode full") + if args.input is None: + raise ValueError("--input is required in --mode validation") + validation_vcf_path = Path(args.input).expanduser().resolve() + if not validation_vcf_path.is_file() or not is_vcf_file(validation_vcf_path): + raise ValueError("Validation input must be an existing .vcf or .vcf.gz file") + if not args.rdf: + raise ValueError("--rdf is required in --mode validation") + validation_rdf_gzip_path = Path(args.rdf).expanduser().resolve() + if not validation_rdf_gzip_path.is_file(): + raise ValueError(f"Validation RDF input not found: {validation_rdf_gzip_path}") + validation_rdf_format = detect_validation_rdf_format(validation_rdf_gzip_path) + if validation_rdf_format is None: + supported = ", ".join(fmt for _suffix, fmt in VALIDATION_RDF_SUFFIXES) + raise ValueError( + "Validation RDF input must be one of: " + supported + ) + if args.validate_artifacts != DEFAULT_VALIDATION_TARGETS: + raise ValueError( + "--validate-artifacts is only valid in --mode full; in validation " + "mode pass the artifact directly with --rdf" + ) + validation_id = args.validation_id or vcf_output_prefix(validation_vcf_path) + if not re.fullmatch(r"[A-Za-z0-9._-]+", validation_id): + raise ValueError("--validation-id may contain only letters, digits, dot, underscore, and hyphen") + validate_mode_dirs([out_root, out_dir, metrics_root]) elif mode == "compress": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -4664,22 +7495,25 @@ def main(): elif mode == "index": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") - if not args.hdt: - raise ValueError("--hdt is required in --mode index") - hdt_path = Path(args.hdt).expanduser().resolve() - if not hdt_path.exists() or not hdt_path.is_file(): - raise ValueError(f"HDT input file not found: {hdt_path}") - if hdt_path.suffix != ".hdt": - raise ValueError("HDT index input must end with .hdt") - validate_mode_dirs([out_root, out_dir, metrics_root]) - existing_indexes = sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")) - if existing_indexes: + if bool(args.hdt) == bool(args.cottas): + raise ValueError( + "provide exactly one of --hdt or --cottas in --mode index" + ) + if args.hdt: + index_format = "hdt" + index_path = Path(args.hdt).expanduser().resolve() + if index_path.suffix != ".hdt": + raise ValueError("HDT index input must end with .hdt") + else: + index_format = "cottas" + index_path = Path(args.cottas).expanduser().resolve() + if index_path.suffix != ".cottas": + raise ValueError("COTTAS index input must end with .cottas") + if not index_path.exists() or not index_path.is_file(): raise ValueError( - "Refusing to overwrite existing output file(s): " - + ", ".join(str(path) for path in existing_indexes) - + ". VCF-RDFizer does not overwrite outputs; rename/remove the " - "existing index and try again." + f"{index_format.upper()} input file not found: {index_path}" ) + validate_mode_dirs([out_root, out_dir, metrics_root]) else: if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -4714,7 +7548,95 @@ def main(): eprint(f"Error: {exc}") return 2 + if mode in {"full", "tsv"}: + metrics_source_paths = [] + for container_input in container_inputs: + try: + relative_input = Path(container_input).relative_to("/data/in") + except ValueError: + relative_input = Path(container_input).name + metrics_source_paths.append((input_mount_dir / relative_input).resolve()) + metrics_source_root = input_path + elif mode == "compress": + metrics_source_paths = [rdf_path] + metrics_source_root = rdf_path + elif mode == "validation": + metrics_source_paths = [validation_vcf_path, validation_rdf_gzip_path] + metrics_source_root = validation_vcf_path + elif mode == "index": + metrics_source_paths = [index_path] + metrics_source_root = index_path + else: + metrics_source_paths = [compressed_path] + metrics_source_root = compressed_path + + source_label = metrics_run_label(metrics_source_paths, metrics_source_root) + metrics_dir = metrics_run_directory(metrics_root, source_label, run_id) + if mode == "validation": + # Keep standalone validation reports in the same discoverable tree as + # full-mode stage artifacts. The validation id remains the leaf name + # so existing stage/report consumers can use one layout for both modes. + validation_results_dir = ( + metrics_dir / "reports" / "validation" / safe_metrics_name(validation_id) + ) + if validation_results_dir.exists() and ( + not validation_results_dir.is_dir() or any(validation_results_dir.iterdir()) + ): + # This runs after the argument-validation try/except above, so it + # must report the same way that block does instead of surfacing an + # uncaught traceback. + eprint( + f"Error: Refusing to overwrite existing validation results: " + f"{validation_results_dir}. " + "Choose --validation-id or --out with a new destination." + ) + return 2 + manifest_options = { + "requested_image": args.image, + "requested_image_version": args.image_version, + "sample_representation": args.sample_representation if mode == "full" else None, + "info_representation": args.info_representation if mode == "full" else None, + "header_representation": args.header_representation if mode == "full" else None, + "rdf_storage_mode": args.rdf_storage_mode if mode == "full" else None, + "compression_methods": ( + full_methods if mode == "full" else methods if mode == "compress" else [] + ), + "hdt_strategy": args.hdt_strategy if mode in {"full", "compress"} else None, + "chunk_target_bytes": chunk_target_bytes if mode in {"full", "compress"} else None, + "chunk_min_bytes": chunk_min_bytes if mode in {"full", "compress"} else None, + "chunk_max_bytes": chunk_max_bytes if mode in {"full", "compress"} else None, + "spark_partitions": spark_partitions if mode == "full" else None, + "run_validation": bool(args.run_validation) if mode == "full" else False, + "validation_artifacts": validation_artifacts if mode == "full" and args.run_validation else None, + "validation_engine": validation_engines if mode in {"full", "validation"} else None, + "validation_strict_conformance": bool(args.strict_conformance) if mode in {"full", "validation"} else None, + "validation_shacl_shapes": str(shacl_shapes_path) if shacl_shapes_path else None, + "validation_engine_options": validation_engine_options or None, + "filter_oracle": args.filter_oracle if mode in {"full", "validation"} else None, + "quiet": bool(args.quiet), + "no_progress": bool(args.no_progress), + "index_format": index_format if mode == "index" else None, + "decompression_format": fmt if mode == "decompress" else None, + "validation_representation": args.sample_representation if mode == "validation" else None, + "validation_rdf_gzip": str(validation_rdf_gzip_path) if mode == "validation" else None, + } + try: + write_run_manifest( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + mode=mode, + source_label=source_label, + source_paths=metrics_source_paths, + out_root=out_root, + options=manifest_options, + ) + except OSError as exc: + eprint(f"Error: unable to create metrics directory '{metrics_dir}': {exc}") + return 1 + print(f"{step1_label}: Validating inputs {success_symbol()}") + print(f" Metrics: {metrics_dir}") if mode == "full" and args.estimate_size: # Optional coarse sizing estimate for disk-risk visibility. @@ -4727,6 +7649,11 @@ def main(): " - Estimated RDF N-Triples size: " f"{format_bytes(estimate['rdf_low_bytes'])} to {format_bytes(estimate['rdf_high_bytes'])}" ) + if estimate["uncompressed_input_estimated"]: + print( + " (a gzip input's uncompressed size could not be read from its " + "structure, so its expansion was assumed)" + ) print( f" - Free disk space at {estimate['disk_anchor']}: {format_bytes(estimate['free_disk_bytes'])}" ) @@ -4736,8 +7663,8 @@ def main(): "You may run out of space." ) - wrapper_log_path = metrics_dir / "wrapper_logs" / f"{run_id}.log" - progress_log_path = metrics_dir / "progress.log" + wrapper_log_path = metrics_dir / "logs" / "wrapper.log" + progress_log_path = metrics_dir / "logs" / "progress.log" execution_started = time.perf_counter() global _COMMAND_LOGGER _COMMAND_LOGGER = CommandLogger(wrapper_log_path) @@ -4763,6 +7690,11 @@ def execute_mode(): try: image_ref, version_requested = resolve_image_ref(args.image, args.image_version) resolved_image_ref = image_ref + update_run_manifest( + metrics_dir, + resolved_image=image_ref, + image_version_requested=version_requested, + ) except ValueError as exc: run_tracker.mark(f"Image resolution failed: {exc}") eprint(f"Error: {exc}") @@ -4808,10 +7740,21 @@ def execute_mode(): (out_write_target, True), (metrics_write_target, True), ] + elif mode == "validation": + validation_write_target = ( + validation_results_dir.parent + if validation_results_dir.parent.exists() + else validation_results_dir.parent.parent + ) + metrics_write_target = metrics_dir if metrics_dir.exists() else metrics_dir.parent + writable_targets = [ + (validation_write_target, True), + (metrics_write_target, True), + ] elif mode == "index": metrics_write_target = metrics_dir if metrics_dir.exists() else metrics_dir.parent writable_targets = [ - (hdt_path.parent, True), + (index_path.parent, True), (metrics_write_target, True), ] else: @@ -4848,7 +7791,6 @@ def execute_mode(): return run_full_mode( input_mount_dir=input_mount_dir, container_inputs=container_inputs, - input_metrics_target=input_metrics_target, expected_prefixes=expected_prefixes, rules_path=rules_path, out_dir=out_dir, @@ -4856,6 +7798,16 @@ def execute_mode(): metrics_dir=metrics_dir, image_ref=image_ref, out_name=args.out_name, + sample_workflow=sample_workflow, + info_representation=args.info_representation, + header_representation=args.header_representation, + run_validation=args.run_validation, + validation_artifacts=validation_artifacts, + validation_engine=validation_engines, + validation_engine_options=validation_engine_options, + validation_strict_conformance=args.strict_conformance, + validation_shacl_shapes=shacl_shapes_path, + filter_oracle=args.filter_oracle, rdf_storage_mode=args.rdf_storage_mode, methods=full_methods, hdt_strategy=args.hdt_strategy, @@ -4901,17 +7853,43 @@ def execute_mode(): chunk_max_bytes=chunk_max_bytes, wrapper_log_path=wrapper_log_path, ) + if mode == "validation": + return run_validation_mode( + vcf_path=validation_vcf_path, + rdf_path=validation_rdf_gzip_path, + rdf_format=validation_rdf_format, + representation=args.sample_representation, + validation_id=validation_id, + results_dir=validation_results_dir, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + image_ref=image_ref, + filter_oracle=args.filter_oracle, + engine=validation_engines, + engine_options=validation_engine_options, + strict_conformance=args.strict_conformance, + shacl_shapes=shacl_shapes_path, + wrapper_log_path=wrapper_log_path, + write_metrics_csv=True, + ) if mode == "index": - return run_hdt_index_mode( - hdt_path=hdt_path, + return run_index_mode( + index_path=index_path, + index_format=index_format, metrics_dir=metrics_dir, image_ref=image_ref, wrapper_log_path=wrapper_log_path, + run_id=run_id, + timestamp=timestamp, ) # Decompression-only mode. return run_decompress_mode( compressed_path=compressed_path, decompressed_out=decompressed_out, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, image_ref=image_ref, wrapper_log_path=wrapper_log_path, ) @@ -4975,6 +7953,19 @@ def _interrupt_handler(_signum, _frame): if run_tracker is not None: run_tracker.mark(f"Run finished (exit_code={result_code})") run_tracker.close() + try: + summary_path = write_run_summary( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + mode=mode, + exit_code=result_code, + elapsed_seconds=elapsed_seconds, + total_triples=total_triples, + ) + print(f"Metrics summary: {summary_path}") + except OSError as exc: + eprint(f"Warning: failed to write metrics summary: {exc}") return result_code diff --git a/vcf_rdfizer_data/rules/default_rules.ttl b/vcf_rdfizer_data/rules/default_rules.ttl index ba357f4..7332230 100644 --- a/vcf_rdfizer_data/rules/default_rules.ttl +++ b/vcf_rdfizer_data/rules/default_rules.ttl @@ -11,9 +11,12 @@ # - /data/tsv/.file_metadata.tsv # - /data/tsv/.header_lines.tsv # - /data/tsv/.records.tsv -# - /data/tsv/.sample_calls.tsv (derived by wrapper) -# - /data/tsv/.sample_format_values.tsv (derived by wrapper) +# - /data/tsv/.sample_calls.tsv (header-only compatibility source) +# - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. +# The wrapper selects exactly one direct sample emitter: expanded SampleCall / +# FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / +# FormatValueVector triples. The helper tables stay header-only in either mode. # # Output format: # - The default conversion now targets triples (N-Triples) without named graph terms. diff --git a/vcf_rdfizer_gzip.py b/vcf_rdfizer_gzip.py new file mode 100644 index 0000000..a8bca50 --- /dev/null +++ b/vcf_rdfizer_gzip.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Uncompressed size of a gzip/BGZF file, without decompressing it when possible. + +VCF-RDFizer records ``input_vcf_size_bytes`` as the *uncompressed* size of the +source VCF so that compression ratios are comparable across plain and +compressed inputs. Obtaining that number by inflating the whole file costs a +full pass over a cohort-scale VCF purely for a metric. + +Three strategies, in order of preference. Every result carries the method that +produced it so a reported size is always auditable: + +``bgzf`` + BGZF (``bgzip``) files - what ``bcftools``/``tabix``/``htslib`` produce and + what indexed ``.vcf.gz`` files in practice are - are a sequence of gzip + members, each holding at most 64 KiB and declaring its own compressed + length in a ``BC`` extra subfield. Walking those declarations and summing + each member's ``ISIZE`` trailer is exact, needs no inflate at all, and is + immune to the 32-bit ``ISIZE`` wrap because every member is small. + +``gzip-trailer`` + A plain single-member gzip stores the uncompressed size in its last four + bytes, but only modulo 2**32, and a concatenated multi-member file exposes + only its final member's value. Both hazards are resolved by inflating a + small prefix to measure this file's actual compression ratio and requiring + exactly one ``ISIZE + k * 2**32`` candidate to agree with it. + +``inflate`` + Full streaming inflate. Always correct, always available, and the fallback + whenever the checks above cannot settle the answer. + +The module is dependency-free so it can run both on the host (for +``--estimate-size``) and inside the image (from ``run_conversion.sh``). +""" + +from __future__ import annotations + +import argparse +import struct +import sys +import zlib +from pathlib import Path + +GZIP_MAGIC = b"\x1f\x8b" +GZIP_DEFLATE = 0x08 +FEXTRA = 0x04 +FNAME = 0x08 +FCOMMENT = 0x10 +FHCRC = 0x02 + +ISIZE_MODULUS = 1 << 32 +# A BGZF member never holds more than 64 KiB, so its ISIZE can never wrap. +BGZF_MAX_BLOCK_ISIZE = 1 << 16 + +# DEFLATE's proven upper bound is 1032:1 (a 258-byte match encoded in ~2 bits). +# Nothing can decompress by more than this, so it bounds the candidate search. +MAX_DEFLATE_RATIO = 1032.0 +# Inflate at most this much before extrapolating a ratio. A VCF body is highly +# homogeneous, so a prefix this size characterises the whole file closely. +RATIO_SAMPLE_BYTES = 16 * 1024 * 1024 +# A candidate must land this close to the sampled projection to be accepted. +# Candidates are 4 GiB apart, so this is a wide margin for a correct answer and +# still rejects a multi-member file whose trailer describes one member only. +RATIO_TOLERANCE = 0.25 + +READ_CHUNK_BYTES = 1024 * 1024 + + +class GzipStructureError(ValueError): + """The file is not shaped the way the chosen fast path requires.""" + + +def looks_like_gzip(path: Path) -> bool: + """Return whether the file starts with the gzip magic bytes.""" + try: + with path.open("rb") as handle: + return handle.read(2) == GZIP_MAGIC + except OSError: + return False + + +def _read_exactly(handle, count: int) -> bytes: + data = handle.read(count) + if len(data) != count: + raise GzipStructureError("unexpected end of gzip stream") + return data + + +def _bgzf_block_size(header: bytes, handle) -> int: + """Return the total on-disk length of one BGZF member, or raise.""" + if header[:2] != GZIP_MAGIC or header[2] != GZIP_DEFLATE: + raise GzipStructureError("not a gzip member") + flags = header[3] + if not flags & FEXTRA: + raise GzipStructureError("gzip member has no FEXTRA field") + (xlen,) = struct.unpack(" int: + """Sum every BGZF member's ISIZE by walking block headers only. + + Raises :class:`GzipStructureError` when the file is not valid BGZF, which + includes an ordinary single-member gzip. + """ + total = 0 + file_size = path.stat().st_size + with path.open("rb") as handle: + offset = 0 + while offset < file_size: + handle.seek(offset) + header = _read_exactly(handle, 12) + block_size = _bgzf_block_size(header, handle) + if block_size < 28 or offset + block_size > file_size: + raise GzipStructureError("BGZF block size runs past end of file") + handle.seek(offset + block_size - 4) + (isize,) = struct.unpack("= BGZF_MAX_BLOCK_ISIZE: + raise GzipStructureError("BGZF block declares an oversized payload") + total += isize + offset += block_size + return total + + +def _skip_gzip_header(handle) -> None: + """Advance past one gzip member header, leaving the deflate stream next.""" + header = _read_exactly(handle, 10) + if header[:2] != GZIP_MAGIC or header[2] != GZIP_DEFLATE: + raise GzipStructureError("not a gzip stream") + flags = header[3] + if flags & FEXTRA: + (xlen,) = struct.unpack(" float: + return self.produced / self.consumed if self.consumed else 0.0 + + @property + def single_member(self) -> bool: + return self.member_complete and self.trailing_bytes == 8 + + +def sample_first_member(path: Path, sample_bytes: int = RATIO_SAMPLE_BYTES) -> MemberSample: + """Inflate a bounded prefix of the first member to characterise the file. + + The cost is capped by ``sample_bytes`` regardless of how large the file is, + which is what keeps the fast path constant-time. + """ + decompressor = zlib.decompressobj(-zlib.MAX_WBITS) + produced = 0 + consumed = 0 + with path.open("rb") as handle: + _skip_gzip_header(handle) + header_bytes = handle.tell() + while produced < sample_bytes: + chunk = handle.read(READ_CHUNK_BYTES) + if not chunk: + break + produced += len(decompressor.decompress(chunk)) + consumed += len(chunk) + if decompressor.eof: + # `unused_data` holds the member's 8-byte trailer plus anything + # that follows it, so its length identifies a concatenation. + trailing = len(decompressor.unused_data) + deflate_bytes = consumed - trailing + return MemberSample( + produced, header_bytes + deflate_bytes, True, trailing + ) + if consumed <= 0 or produced <= 0: + raise GzipStructureError("could not sample the deflate stream") + return MemberSample(produced, header_bytes + consumed, False, 0) + + +def trailer_isize(path: Path) -> int: + """Read the final member's ISIZE field (uncompressed size modulo 2**32).""" + with path.open("rb") as handle: + handle.seek(-4, 2) + return struct.unpack(" tuple[int, str]: + """Resolve the 32-bit ISIZE trailer into a full uncompressed size. + + Raises :class:`GzipStructureError` unless the file is provably a single + member measured in full, or exactly one ``ISIZE + k * 2**32`` candidate + agrees with the ratio measured from the file's own prefix. That single + requirement rules out both a >4 GiB wrap the trailer cannot express and a + concatenated file whose trailer describes only its final member. + """ + compressed_size = path.stat().st_size + if compressed_size <= 18: + raise GzipStructureError("file is too small to carry a gzip trailer") + + sample = sample_first_member(path) + if sample.member_complete: + if sample.single_member: + # The whole file fitted in the sample budget: this is not an + # estimate resolved against a trailer, it is the measured size. + return sample.produced, "gzip-sample" + raise GzipStructureError( + "concatenated gzip members; the trailer covers only the last one" + ) + + projected = sample.ratio * compressed_size + if projected <= 0: + raise GzipStructureError("could not project an uncompressed size") + + isize = trailer_isize(path) + upper_bound = compressed_size * MAX_DEFLATE_RATIO + accepted = [ + candidate + for candidate in range(isize, int(upper_bound) + ISIZE_MODULUS, ISIZE_MODULUS) + if candidate <= upper_bound + and abs(candidate - projected) <= RATIO_TOLERANCE * projected + ] + if len(accepted) != 1: + raise GzipStructureError( + f"gzip trailer is ambiguous ({len(accepted)} plausible sizes)" + ) + return accepted[0], "gzip-trailer" + + +def inflate_uncompressed_size(path: Path) -> int: + """Stream the whole file through zlib and count the output bytes. + + Raises :class:`zlib.error` on a truncated stream so a damaged input is + reported rather than silently measured as short. + """ + total = 0 + decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) + with path.open("rb") as handle: + while True: + chunk = handle.read(READ_CHUNK_BYTES) + if not chunk: + break + total += len(decompressor.decompress(chunk)) + while decompressor.eof and decompressor.unused_data: + # Concatenated members: continue with the next one. + tail = decompressor.unused_data + decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) + total += len(decompressor.decompress(tail)) + total += len(decompressor.flush()) + if not decompressor.eof: + raise zlib.error(f"truncated or corrupt gzip stream: {path}") + return total + + +def uncompressed_size(path: Path, *, exact: bool = False) -> tuple[int, str]: + """Return ``(uncompressed_bytes, method)`` for a plain or gzip file. + + Set ``exact`` to skip the structural fast paths and always inflate. + """ + path = Path(path) + if not looks_like_gzip(path): + return path.stat().st_size, "stat" + + if not exact: + try: + return bgzf_uncompressed_size(path), "bgzf" + except (GzipStructureError, OSError, struct.error): + pass + try: + return resolve_trailer_size(path) + except (GzipStructureError, OSError, zlib.error, struct.error): + pass + + return inflate_uncompressed_size(path), "inflate" + + +def uncompressed_size_without_inflating(path: Path) -> tuple[int | None, str]: + """Structural size only: return ``(None, "unknown")`` rather than inflating. + + Callers that merely want a preflight estimate would gain nothing from a + full decompression pass, so they use this instead of + :func:`uncompressed_size`. + """ + path = Path(path) + if not looks_like_gzip(path): + return path.stat().st_size, "stat" + try: + return bgzf_uncompressed_size(path), "bgzf" + except (GzipStructureError, OSError, struct.error): + pass + try: + return resolve_trailer_size(path) + except (GzipStructureError, OSError, zlib.error, struct.error): + pass + return None, "unknown" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Print the uncompressed size in bytes of a gzip/BGZF file, " + "avoiding a full decompression pass where the file's structure " + "makes that possible." + ) + ) + parser.add_argument("path", help="File to measure") + parser.add_argument( + "--exact", + action="store_true", + help="Always inflate instead of using a structural fast path", + ) + parser.add_argument( + "--print-method", + action="store_true", + help="Print ' ' instead of just the byte count", + ) + args = parser.parse_args(argv) + + try: + size, method = uncompressed_size(Path(args.path), exact=args.exact) + except (OSError, zlib.error) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(f"{size} {method}" if args.print_method else size) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vcf_rdfizer_rules.py b/vcf_rdfizer_rules.py new file mode 100644 index 0000000..ea7b5ef --- /dev/null +++ b/vcf_rdfizer_rules.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +"""Authoring helper for custom VCF-RDFizer RML mappings. + +VCF-RDFizer accepts any RML mapping through ``--rules``, but a custom mapping +has to honour a small contract with the wrapper. This tool makes that contract +discoverable and checkable instead of something you find out on the next +multi-hour run: + +``vcf-rdfizer-rules columns`` + Show the TSV sources the pipeline generates and every column each one has, + so you know what is available to reference. + +``vcf-rdfizer-rules init -o my_rules.ttl`` + Copy the shipped default mapping as an annotated starting point. + +``vcf-rdfizer-rules check my_rules.ttl`` + Validate a mapping against the contract before running the pipeline: the + logical-source paths the wrapper can rewrite, the column names it can + supply, and which ``--sample-representation`` values remain usable. + +The contract itself +------------------- +1. Logical sources must use the five canonical container paths verbatim + (``/data/tsv/records.tsv`` and friends). Full mode processes one VCF at a + time and rewrites exactly those five strings to the per-input file names, so + a mapping that hard-codes anything else silently reads the wrong file. +2. Referenced columns must exist in the TSV that ``vcf_as_tsv.sh`` writes. +3. Consuming ``sample_calls.tsv`` or ``sample_format_values.tsv`` beyond the + four built-in sample maps forces the wrapper to materialize those helper + tables, which costs one row per variant x sample (x FORMAT key), and is + incompatible with ``--sample-representation condensed``. + +The checks are deliberately lexical rather than a full Turtle parse: the tool +stays dependency-free, and the mistakes worth catching before a long run are +wrong paths and misspelled column names. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +try: + from vcf_rdfizer import ( + SAMPLE_CALLS_HEADER, + SAMPLE_FORMAT_HEADER, + resolve_default_rules_path, + resolve_sample_workflow, + sample_support_strategy, + ) +except ImportError as exc: # pragma: no cover - installed together + raise SystemExit(f"vcf-rdfizer-rules requires the vcf_rdfizer module: {exc}") + + +@dataclass(frozen=True) +class TsvSource: + """One TSV table the pipeline generates for RMLStreamer to read.""" + + container_path: str + description: str + columns: tuple[str, ...] + #: A trailing column whose *name* varies per input (the VCF sample header). + dynamic_tail_column: str | None = None + notes: tuple[str, ...] = field(default_factory=tuple) + + +# The first three column lists mirror the headers written by `src/vcf_as_tsv.sh`; +# `test_rules_helper_unit.py` runs that script and asserts they still match, so +# the two cannot drift apart silently. The last two reuse the wrapper's own +# constants directly. +TSV_SOURCES: dict[str, TsvSource] = { + "records": TsvSource( + container_path="/data/tsv/records.tsv", + description="One row per VCF data line.", + columns=( + "SOURCE_FILE", + "ROW_ID", + "CHROM", + "POS", + "ID", + "REF", + "ALT", + "QUAL", + "FILTER", + "INFO", + "FORMAT", + ), + dynamic_tail_column="SAMPLES", + notes=( + "ROW_ID is a 1-based counter over data lines, stable within one file.", + "The final column holds every sample's payload separated by spaces. Its" + " header is the whitespace-joined sample ids from #CHROM, or SAMPLES when" + " the VCF declares none, so it cannot be referenced by a fixed name.", + "Genotypes are emitted by the wrapper from this table directly; see" + " --sample-representation.", + ), + ), + "header_lines": TsvSource( + container_path="/data/tsv/header_lines.tsv", + description="One row per '##' meta-information header line.", + columns=( + "SOURCE_FILE", + "HEADER_INDEX", + "HEADER_KEY", + "HEADER_VALUE", + "RAW_LINE", + ), + notes=( + "HEADER_KEY/HEADER_VALUE split on the first '='; RAW_LINE keeps the" + " original text without the leading '##'.", + ), + ), + "file_metadata": TsvSource( + container_path="/data/tsv/file_metadata.tsv", + description="Exactly one row summarising the source VCF.", + columns=( + "SOURCE_FILE", + "FILE_FORMAT", + "FILE_DATE", + "SOURCE_SOFTWARE", + "REFERENCE_GENOME", + "HEADER_COUNT", + "RECORD_COUNT", + ), + ), + "sample_calls": TsvSource( + container_path="/data/tsv/sample_calls.tsv", + description="Helper table: one row per variant x sample.", + columns=tuple(SAMPLE_CALLS_HEADER), + notes=( + "Header-only for the four built-in sample maps. Referencing it from a" + " custom map makes the wrapper materialize every row.", + ), + ), + "sample_format_values": TsvSource( + container_path="/data/tsv/sample_format_values.tsv", + description="Helper table: one row per variant x sample x FORMAT key.", + columns=tuple(SAMPLE_FORMAT_HEADER), + notes=( + "Header-only for the four built-in sample maps. Referencing it from a" + " custom map makes the wrapper materialize every row, which is the" + " largest table the pipeline can produce.", + ), + ), +} + +CANONICAL_SOURCE_PATHS = {source.container_path for source in TSV_SOURCES.values()} + +CSVW_URL_RE = re.compile(r'csvw:url\s+"([^"]*)"') +REFERENCE_RE = re.compile(r'rml:reference\s+"([^"]*)"') +TEMPLATE_RE = re.compile(r'rr:template\s+"((?:[^"\\]|\\.)*)"') +TEMPLATE_VAR_RE = re.compile(r"(?.records.tsv` +# and so on). A different path is not rewritten and will not be found. +# +# 2. Only reference columns the pipeline actually writes. Run +# `vcf-rdfizer-rules columns` to see them all. +# +# Validate this file before a long run: +# +# vcf-rdfizer-rules check +# +# Then use it with: +# +# vcf-rdfizer --mode full --rules ... +# +""" + + +def all_known_columns() -> dict[str, list[str]]: + """Map each column name to the sources that provide it.""" + owners: dict[str, list[str]] = {} + for name, source in TSV_SOURCES.items(): + for column in source.columns: + owners.setdefault(column, []).append(name) + if source.dynamic_tail_column: + owners.setdefault(source.dynamic_tail_column, []).append(name) + return owners + + +def referenced_columns(text: str) -> set[str]: + """Collect every column name the mapping reads, from references and templates.""" + names = set(REFERENCE_RE.findall(text)) + for template in TEMPLATE_RE.findall(text): + names.update( + variable.strip() for variable in TEMPLATE_VAR_RE.findall(template) if variable.strip() + ) + return names + + +def check_rules(rules_path: Path) -> dict: + """Validate one mapping against the wrapper contract. + + Returns a report with ``errors`` (the run will not work), ``warnings`` + (it will work but at a cost worth knowing about), and ``info``. + """ + report: dict = { + "rules_path": str(rules_path), + "errors": [], + "warnings": [], + "info": [], + "sources": [], + "columns": [], + "sample_representations": {}, + } + text = rules_path.read_text(encoding="utf-8") + + if not TRIPLES_MAP_RE.search(text): + report["errors"].append( + "No rml:logicalSource or rr:subjectMap found; this does not look like an " + "RML mapping." + ) + + urls = CSVW_URL_RE.findall(text) + report["sources"] = sorted(set(urls)) + if not urls: + report["errors"].append( + 'No csvw:url found. Each logical source needs one of: ' + + ", ".join(sorted(CANONICAL_SOURCE_PATHS)) + ) + for url in sorted(set(urls)): + if url not in CANONICAL_SOURCE_PATHS: + report["errors"].append( + f"Logical source '{url}' is not one of the five paths the wrapper " + "rewrites per input, so it will not point at the generated TSV. Use " + "one of: " + ", ".join(sorted(CANONICAL_SOURCE_PATHS)) + ) + + owners = all_known_columns() + used = referenced_columns(text) + report["columns"] = sorted(used) + for column in sorted(used): + if column not in owners: + report["errors"].append( + f"Column '{column}' is not written by any pipeline TSV. Run " + "'vcf-rdfizer-rules columns' for the full list." + ) + + unused_sources = CANONICAL_SOURCE_PATHS - set(urls) + if unused_sources: + report["info"].append( + "Sources not used by this mapping: " + ", ".join(sorted(unused_sources)) + ) + + strategy = sample_support_strategy(rules_path) + if strategy == "none": + report["info"].append( + "No sample helper tables are consumed; the wrapper emits genotype RDF " + "directly and no helper rows are materialized." + ) + elif strategy == "stream": + report["info"].append( + "The four built-in sample maps are present unchanged, so the helper " + "tables stay header-only and genotype RDF is streamed directly." + ) + else: + report["warnings"].append( + "This mapping consumes sample_calls.tsv or sample_format_values.tsv in a " + "way the wrapper cannot stream, so those helper tables will be " + "materialized: one row per variant x sample, and one per variant x " + "sample x FORMAT key. On a cohort VCF that is the largest intermediate " + "the pipeline can produce." + ) + + for representation in ("expanded", "condensed"): + try: + workflow = resolve_sample_workflow(representation, rules_path) + except ValueError as exc: + report["sample_representations"][representation] = { + "usable": False, + "reason": str(exc), + } + continue + report["sample_representations"][representation] = { + "usable": True, + "helper_strategy": workflow.helper_strategy, + "emitter": workflow.emitter or "none (RML emits genotypes)", + } + + report["ok"] = not report["errors"] + return report + + +def print_report(report: dict) -> None: + print(f"Checked: {report['rules_path']}") + print() + print("Logical sources:") + for url in report["sources"]: + mark = "ok " if url in CANONICAL_SOURCE_PATHS else "BAD" + print(f" [{mark}] {url}") + print() + print(f"Referenced columns ({len(report['columns'])}): {', '.join(report['columns']) or '(none)'}") + print() + print("Sample representation compatibility:") + for representation, detail in report["sample_representations"].items(): + if detail["usable"]: + print( + f" --sample-representation {representation}: usable " + f"(helper tables: {detail['helper_strategy']}, emitter: {detail['emitter']})" + ) + else: + print(f" --sample-representation {representation}: NOT usable") + print(f" {detail['reason']}") + + for label, items in (("Errors", report["errors"]), ("Warnings", report["warnings"]), ("Notes", report["info"])): + if not items: + continue + print() + print(f"{label}:") + for item in items: + print(f" - {item}") + + print() + print("Result: OK" if report["ok"] else "Result: FAILED") + + +def command_columns(args: argparse.Namespace) -> int: + selected = TSV_SOURCES if args.source is None else {args.source: TSV_SOURCES[args.source]} + if args.json: + payload = { + name: { + "container_path": source.container_path, + "description": source.description, + "columns": list(source.columns), + "dynamic_tail_column": source.dynamic_tail_column, + "notes": list(source.notes), + } + for name, source in selected.items() + } + print(json.dumps(payload, indent=2)) + return 0 + + for name, source in selected.items(): + print(f"{name} ({source.container_path})") + print(f" {source.description}") + print(" Columns:") + for column in source.columns: + print(f" - {column}") + if source.dynamic_tail_column: + print(f" - or {source.dynamic_tail_column} (name varies per input)") + for note in source.notes: + print(f" Note: {note}") + print() + return 0 + + +def command_init(args: argparse.Namespace) -> int: + output = Path(args.output).expanduser() + if output.exists() and not args.force: + print(f"error: {output} already exists (use --force to overwrite)", file=sys.stderr) + return 2 + try: + default_rules = resolve_default_rules_path(Path(__file__).resolve().parent) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + INIT_HEADER + default_rules.read_text(encoding="utf-8"), encoding="utf-8" + ) + print(f"Wrote {output} (from {default_rules})") + print("Next: edit it, then run") + print(f" vcf-rdfizer-rules check {output}") + return 0 + + +def command_check(args: argparse.Namespace) -> int: + rules_path = Path(args.rules).expanduser() + if not rules_path.is_file(): + print(f"error: rules file not found: {rules_path}", file=sys.stderr) + return 2 + try: + report = check_rules(rules_path) + except OSError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if args.json: + print(json.dumps(report, indent=2)) + else: + print_report(report) + return 0 if report["ok"] else 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="vcf-rdfizer-rules", + description=( + "Inspect, scaffold, and validate custom RML mappings for VCF-RDFizer." + ), + formatter_class=argparse.RawTextHelpFormatter, + epilog=( + "Examples:\n" + " See what columns a mapping can reference:\n" + " vcf-rdfizer-rules columns\n" + " Start a custom mapping from the shipped default:\n" + " vcf-rdfizer-rules init -o my_rules.ttl\n" + " Validate it before a long run:\n" + " vcf-rdfizer-rules check my_rules.ttl\n" + " Then use it:\n" + " vcf-rdfizer --mode full -i cohort.vcf.gz --rules my_rules.ttl \\\n" + " --rdf-storage-mode plain -o ./results\n" + ), + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + columns = subparsers.add_parser( + "columns", help="List the TSV sources and columns a mapping can reference" + ) + columns.add_argument( + "--source", choices=sorted(TSV_SOURCES), default=None, help="Show one source only" + ) + columns.add_argument("--json", action="store_true", help="Emit machine-readable JSON") + columns.set_defaults(func=command_columns) + + init = subparsers.add_parser( + "init", help="Write an annotated copy of the default mapping to start from" + ) + init.add_argument( + "-o", "--output", default="custom_rules.ttl", help="Destination path (default: custom_rules.ttl)" + ) + init.add_argument("--force", action="store_true", help="Overwrite an existing file") + init.set_defaults(func=command_init) + + check = subparsers.add_parser( + "check", help="Validate a mapping against the wrapper contract" + ) + check.add_argument("rules", help="Path to the .ttl mapping to check") + check.add_argument("--json", action="store_true", help="Emit machine-readable JSON") + check.set_defaults(func=command_check) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main())