diff --git a/CITATION.cff b/CITATION.cff index d0afadb..8ad11d2 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ cff-version: 1.2.0 message: "If you use VCF-RDFizer in your research, please cite it using the metadata below." title: "VCF-RDFizer" type: software -version: "2.0.0" +version: "2.1.0" authors: - name: "VCF-RDFizer maintainers" repository-code: "https://github.com/ecrum19/VCF-RDFizer" diff --git a/README.md b/README.md index 78d2f38..810376f 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ inside this directory. - `full`: VCF -> TSV -> RDF -> compression - `tsv`: VCF -> TSV only (benchmarking) - `compress`: compress an existing `.nt` or `.nt.gz` -- `decompress`: decompress `.nt.gz`, `.nt.br`, or `.hdt` +- `decompress`: decompress `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` - `index`: eagerly initialize HDT Java's versioned `.hdt.index.*` sidecar for an existing `.hdt` In `full` mode with multiple VCF inputs, failures are isolated per input: @@ -119,6 +119,11 @@ Packaged `.hdt.gz`, `.hdt.br`, `.cottas.gz`, and `.cottas.br` files are archives not directly queryable indexed files. Keep the unwrapped `.hdt`/`.cottas` file when queries must run without a decompression step. +Use `--mode decompress` to decode either base representation. COTTAS packages +are unpacked inside the Docker container before `pycottas` writes the decoded +N-Triples output, so the temporary unwrapped COTTAS file is not added to the +host filesystem. + ## Full Mode Flags - `-i, --input` required VCF file or directory @@ -157,7 +162,7 @@ when queries must run without a decompression step. ## Decompression Mode Flags -- `-C, --compressed-input` required `.nt.gz`, `.nt.br`, or `.hdt` +- `-C, --compressed-input` required `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, or `.cottas.br` - `-d, --decompress-out` optional explicit output `.nt` path (must be inside `--out`) ## HDT Index Mode Flags @@ -313,6 +318,15 @@ vcf-rdfizer \ --out ./results ``` +COTTAS decompression, including an externally packaged COTTAS file: + +```bash +vcf-rdfizer \ + --mode decompress \ + --compressed-input ./results/sample/sample.cottas.gz \ + --out ./results +``` + Initialize an index for an existing HDT: ```bash @@ -423,7 +437,7 @@ state from one chunk being reused by another and requires no user configuration. HDT Java 3.0.10 does not provide a standalone `hdtGenerateIndex` executable. VCF-RDFizer sends an `exit` command to the supported `hdtSearch.sh` launcher; this opens the HDT through `mapIndexedHDT()` without executing a data query and -creates the sibling `.hdt.index` sidecar before the run is marked successful. +creates the versioned `.hdt.index.v1-1` sidecar before the run is marked successful. For the pinned HDT Java 3.0.10 distribution, this is the HDT v1-1 sidecar `.hdt.index.v1-1`; VCF-RDFizer reports the actual path in its metrics. @@ -431,8 +445,9 @@ The record-safe chunk plan and per-stage timings are retained in the raw partitioned-compression metrics JSON for diagnostics. The temporary chunk files and guide are not retained as host files. -See [`COMPRESSION_CHANGELOG.md`](COMPRESSION_CHANGELOG.md) for the detailed -implementation approach and operational constraints. +The implementation keeps COTTAS conversion scratch state inside the Docker +container and removes temporary unpacked package files when decompression +finishes. ## Rules @@ -460,7 +475,7 @@ Safe termination: If you use VCF-RDFizer in a publication, please cite: -VCF-RDFizer maintainers. (2026). *VCF-RDFizer* (Version 2.0.0) [Computer software]. GitHub. https://github.com/ecrum19/VCF-RDFizer +VCF-RDFizer maintainers. (2026). *VCF-RDFizer* (Version 2.1.0) [Computer software]. GitHub. https://github.com/ecrum19/VCF-RDFizer BibTeX: @@ -469,7 +484,7 @@ BibTeX: author = {{VCF-RDFizer maintainers}}, title = {VCF-RDFizer}, year = {2026}, - version = {2.0.0}, + version = {2.1.0}, url = {https://github.com/ecrum19/VCF-RDFizer}, note = {Computer software} } diff --git a/conda-recipe/README.md b/conda-recipe/README.md index e1569fd..dfd9a3f 100644 --- a/conda-recipe/README.md +++ b/conda-recipe/README.md @@ -7,11 +7,11 @@ do not submit this package to `staged-recipes`. ## Before submitting to conda-forge -1. Commit the version bump, then create and push a Git tag (for example `v2.0.0`). +1. Commit the version bump, then create and push a Git tag (for example `v2.1.0`). 2. Download the source tarball and compute sha256: ```bash curl -L -o vcf-rdfizer.tar.gz \ - https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v2.0.0.tar.gz + https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v2.1.0.tar.gz shasum -a 256 vcf-rdfizer.tar.gz ``` 3. Replace `version` and `sha256` in the feedstock's `recipe/meta.yaml`. diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index e771118..dce0364 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "vcf-rdfizer" %} -{% set version = "2.0.0" %} +{% set version = "2.1.0" %} package: name: {{ name|lower }} diff --git a/pyproject.toml b/pyproject.toml index 4406348..c39ec03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "vcf-rdfizer" -version = "2.0.0" +version = "2.1.0" description = "Docker-first VCF to RDF conversion wrapper with compression/decompression modes" readme = "README.md" requires-python = ">=3.10" diff --git a/src/cottas_tool.py b/src/cottas_tool.py index 6dbca5f..c8ddc23 100644 --- a/src/cottas_tool.py +++ b/src/cottas_tool.py @@ -42,6 +42,10 @@ def main() -> int: merge.add_argument("cottas_path") merge.add_argument("index", nargs="?", default="spo") + decompress = subparsers.add_parser("decompress", help="convert COTTAS to RDF") + decompress.add_argument("cottas_path") + decompress.add_argument("rdf_path") + args = parser.parse_args() try: import pycottas @@ -63,6 +67,15 @@ def main() -> int: ) return 0 + if args.command == "decompress": + cottas_path = str(Path(args.cottas_path).resolve()) + rdf_path = str(Path(args.rdf_path).resolve()) + # Keep DuckDB scratch state in the container-local workspace while + # pycottas writes the decoded RDF directly to the mounted output. + with cottas_scratch_workspace(): + pycottas.cottas2rdf(cottas_path, rdf_path) + return 0 + left_path = str(Path(args.left_path).resolve()) right_path = str(Path(args.right_path).resolve()) cottas_path = str(Path(args.cottas_path).resolve()) diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py index 8da86f6..293ab0d 100644 --- a/test/test_cottas_tool.py +++ b/test/test_cottas_tool.py @@ -102,3 +102,40 @@ def fake_cat(paths, cottas_path, *, index, remove_input_files): self.assertTrue(output.is_file()) self.assertEqual(len(observed_workspaces), 1) self.assertFalse(any(scratch_root.iterdir())) + + def test_decompress_uses_pycottas_and_isolated_scratch(self): + """COTTAS decompression writes RDF while cleaning container-local state.""" + module = load_cottas_tool() + observed_workspaces = [] + + def fake_cottas2rdf(cottas_path, rdf_path): + self.assertTrue(Path(cottas_path).is_absolute()) + self.assertTrue(Path(rdf_path).is_absolute()) + Path(rdf_path).write_text("

.\n") + database_path = Path.cwd() / "pycottas.duckdb" + database_path.write_text("temporary DuckDB state\n") + observed_workspaces.append(Path.cwd()) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = (root / "scratch").resolve() + source = root / "input.cottas" + output = root / "output.nt" + source.write_text("COTTAS input\n") + + with mock.patch.dict( + sys.modules, + {"pycottas": types.SimpleNamespace(cottas2rdf=fake_cottas2rdf)}, + ), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "decompress", str(source), str(output)], + ): + self.assertEqual(module.main(), 0) + + self.assertTrue(output.is_file()) + self.assertEqual(output.read_text(), "

.\n") + self.assertEqual(len(observed_workspaces), 1) + self.assertFalse(any(scratch_root.iterdir())) diff --git a/test/test_vcf_rdfizer_cross_platform_unit.py b/test/test_vcf_rdfizer_cross_platform_unit.py index 7985bf6..35dc3c0 100644 --- a/test/test_vcf_rdfizer_cross_platform_unit.py +++ b/test/test_vcf_rdfizer_cross_platform_unit.py @@ -110,6 +110,23 @@ def test_detect_compressed_format(self): self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.nt.gz")), "gzip") self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.nt.br")), "brotli") self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.hdt")), "hdt") + self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.cottas")), "cottas") + self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.cottas.gz")), "cottas") + self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.cottas.br")), "cottas") + + def test_default_decompressed_name_for_cottas_variants(self): + self.assertEqual( + vcf_rdfizer.default_decompressed_name(Path("sample.cottas"), "cottas"), + "sample.nt", + ) + self.assertEqual( + vcf_rdfizer.default_decompressed_name(Path("sample.cottas.gz"), "cottas"), + "sample.nt", + ) + self.assertEqual( + vcf_rdfizer.default_decompressed_name(Path("sample.cottas.br"), "cottas"), + "sample.nt", + ) def test_bind_mount_parser_preserves_windows_drive_letter(self): """Validation mocks preserve Windows bind-mount source paths.""" diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 6e5732e..9fd09fc 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -2119,6 +2119,61 @@ def fake_run(cmd, cwd=None, env=None): self.assertTrue(any(arg.endswith("/out/sample:/data/out") for arg in commands[0])) self.assertIn("/data/out/sample.nt", commands[0][-1]) + def test_main_decompress_mode_cottas_supports_raw_and_packaged_inputs(self): + """COTTAS decompression accepts raw, gzip-packaged, and Brotli-packaged files.""" + cases = ( + ("sample.cottas", None), + ("sample.cottas.gz", "gzip -dc"), + ("sample.cottas.br", "brotli -d -c"), + ) + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + for index, (filename, unpack_command) in enumerate(cases): + compressed = tmp_path / filename + compressed.write_bytes(b"fake-cottas-bytes") + out_dir = tmp_path / f"out-{index}" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object( + vcf_rdfizer, "run", side_effect=fake_run + ), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ): + rc = invoke_main( + [ + "--mode", + "decompress", + "--compressed-input", + str(compressed), + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(len(commands), 1) + command = commands[0][-1] + self.assertIn("cottas_tool.py decompress", command) + self.assertIn("/data/out/sample.nt", command) + self.assertIn("/data/in/" + filename, command) + if unpack_command is None: + self.assertNotIn("gzip -dc", command) + self.assertNotIn("brotli -d -c", command) + else: + self.assertIn(unpack_command, command) + def test_main_index_mode_initializes_existing_hdt(self): """Index mode runs the container helper and records the sidecar artifact.""" with tempfile.TemporaryDirectory() as td: diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 4c509fa..9a1600a 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -4092,13 +4092,21 @@ def run_compress_mode( def detect_compressed_format(path: Path): """Infer compressed RDF format from filename/extension.""" + if ( + path.name.endswith(".cottas") + or path.name.endswith(".cottas.gz") + or path.name.endswith(".cottas.br") + ): + return "cottas" if path.name.endswith(".nt.gz") or path.suffix == ".gz": return "gzip" if path.name.endswith(".nt.br") or path.suffix == ".br": return "brotli" if path.suffix == ".hdt": return "hdt" - raise ValueError("Compressed input must end with .gz, .br, or .hdt") + raise ValueError( + "Compressed input must end with .nt.gz, .nt.br, .hdt, .cottas, .cottas.gz, or .cottas.br" + ) def default_decompressed_name(path: Path, fmt: str): @@ -4111,6 +4119,10 @@ def default_decompressed_name(path: Path, fmt: str): if path.name.endswith(".nt.br"): return path.name[: -len(".br")] return f"{path.stem}.nt" + if fmt == "cottas": + for suffix in (".cottas.gz", ".cottas.br", ".cottas"): + if path.name.endswith(suffix): + return f"{path.name[: -len(suffix)]}.nt" return f"{path.stem}.nt" @@ -4178,7 +4190,7 @@ def run_decompress_mode( image_ref: str, wrapper_log_path: Path, ): - """Execute decompression-only mode (.gz/.br/.hdt -> RDF).""" + """Execute decompression-only mode (.gz/.br/.hdt/.cottas -> RDF).""" print("Step 3/3: Decompressing RDF input") fmt = detect_compressed_format(compressed_path) ensure_dir(decompressed_out.parent) @@ -4198,7 +4210,7 @@ def run_decompress_mode( f"rm -f {shlex.quote(output_container)}; " f"brotli -d -c {shlex.quote(source_container)} > {shlex.quote(output_container)}" ) - else: + elif fmt == "hdt": command = ( "set -euo pipefail; " f"rm -f {shlex.quote(output_container)}; " @@ -4214,6 +4226,36 @@ def run_decompress_mode( '"$HDT2RDF_BIN" ' f"{shlex.quote(source_container)} {shlex.quote(output_container)}" ) + else: + # COTTAS is seekable, so packaged `.cottas.gz`/`.cottas.br` inputs are + # unwrapped inside the container before pycottas reads them. The + # temporary representation never appears on the host filesystem. + cottas_input = source_container + cleanup = "" + if compressed_path.name.endswith(".cottas.gz"): + cottas_input = "/work/vcf-rdfizer-cottas-input" + cleanup = ( + f"rm -f {shlex.quote(cottas_input)}; " + f"trap 'rm -f {shlex.quote(cottas_input)}' EXIT; " + f"gzip -dc {shlex.quote(source_container)} > {shlex.quote(cottas_input)}; " + ) + elif compressed_path.name.endswith(".cottas.br"): + cottas_input = "/work/vcf-rdfizer-cottas-input" + cleanup = ( + f"rm -f {shlex.quote(cottas_input)}; " + f"trap 'rm -f {shlex.quote(cottas_input)}' EXIT; " + f"brotli -d -c {shlex.quote(source_container)} > {shlex.quote(cottas_input)}; " + ) + command = ( + "set -euo pipefail; " + f"{cleanup}" + 'COTTAS_PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}"; ' + 'if [[ -z "$COTTAS_PYTHON_BIN" ]]; then ' + 'echo "Missing pycottas Python executable in container" >&2; exit 127; ' + "fi; " + f'"$COTTAS_PYTHON_BIN" /opt/vcf-rdfizer/cottas_tool.py decompress ' + f"{shlex.quote(cottas_input)} {shlex.quote(output_container)}" + ) cmd = [ *docker_run_base(), @@ -4295,7 +4337,7 @@ def main(): "-C", "--compressed-input", default=None, - help="Compressed RDF input (.gz/.br/.hdt) for --mode decompress", + help="Compressed RDF input (.nt.gz/.nt.br/.hdt/.cottas[.gz|.br]) for --mode decompress", ) parser.add_argument( "-H",