From 1fc2911d8bd63586778db34f217b3f40c5643280 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:22:02 -0400 Subject: [PATCH 01/19] Add load_metadata_example() function --- pointblank/metadata/_import.py | 99 +++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/pointblank/metadata/_import.py b/pointblank/metadata/_import.py index 6cd0897a3..0fc6aa456 100644 --- a/pointblank/metadata/_import.py +++ b/pointblank/metadata/_import.py @@ -3,9 +3,44 @@ from pathlib import Path from typing import Any +from importlib_resources import files + from pointblank.metadata._types import MetadataImport, MetadataPackage -__all__ = ["import_metadata"] +__all__ = ["import_metadata", "load_metadata_example"] + +# Bundled metadata example files, mapped to a short description and the format that +# `import_metadata()` uses to read them. These ship with the package so the documentation +# examples (and your own experimentation) can run against real files without any external +# downloads. +_METADATA_EXAMPLES: dict[str, tuple[str, str]] = { + "define.xml": ( + "cdisc_define", + "CDISC Define-XML 2.0 document describing the DM and AE domains of study XYZ789.", + ), + "sdtm_ct.xml": ( + "cdisc_ct", + "CDISC SDTM Controlled Terminology package with a handful of common codelists " + "(SEX, RACE, severity, etc.).", + ), + "datapackage.json": ( + "frictionless", + "Frictionless Data Package describing a 'transactions' table with typed columns " + "and constraints.", + ), + "table_schema.json": ( + "table_schema", + "Standalone Frictionless Table Schema for a sensor-readings table.", + ), + "weather_csvw.json": ( + "csvw", + "W3C CSVW (CSV on the Web) metadata for a weather-observations table.", + ), + "dm.xpt": ( + "xpt", + "SAS Transport (XPT) file containing a small SDTM Demographics (DM) dataset.", + ), +} # Mapping of format strings to reader functions _FORMAT_REGISTRY: dict[str, str] = { @@ -39,6 +74,68 @@ _XML_FORMATS: set[str] = {"cdisc_define", "define_xml", "cdisc_ct"} +def load_metadata_example(name: str) -> Path: + """Get the path to a bundled metadata example file. + + Pointblank ships a small collection of metadata example files (CDISC Define-XML, CDISC + Controlled Terminology, Frictionless, CSVW, and SAS Transport) so the documentation examples can + run against real files without any external downloads. This function returns the filesystem path + to one of those bundled files, which you can then pass directly to + [`import_metadata()`](`pointblank.import_metadata`). + + Parameters + ---------- + name + The file name of the example to load. Use `load_metadata_example()` with an invalid name to + see the available options, or consult the table below. + + Returns + ------- + Path + A filesystem path to the bundled example file. + + Available Examples + ------------------ + | Name | Format | Description | + |------|--------|-------------| + | `"define.xml"` | `cdisc_define` | Define-XML 2.0 document with the DM and AE domains | + | `"sdtm_ct.xml"` | `cdisc_ct` | SDTM Controlled Terminology with common codelists | + | `"datapackage.json"` | `frictionless` | Frictionless Data Package for a transactions table | + | `"table_schema.json"` | `table_schema` | Frictionless Table Schema for sensor readings | + | `"weather_csvw.json"` | `csvw` | W3C CSVW metadata for weather observations | + | `"dm.xpt"` | `xpt` | SAS Transport file with a small SDTM Demographics dataset | + + Examples + -------- + Use `load_metadata_example()` to locate a bundled Define-XML document, then import it with + [`import_metadata()`](`pointblank.import_metadata`): + + ```python + import pointblank as pb + + define_path = pb.load_metadata_example("define.xml") + package = pb.import_metadata(define_path, format="cdisc_define") + + for name in package.keys(): + meta = package[name] + print(f"{name}: {meta.dataset_label} ({len(meta.variables)} variables)") + ``` + """ + if name not in _METADATA_EXAMPLES: + available = "\n".join( + f"- `{fname}` ({fmt}): {desc}" + for fname, (fmt, desc) in _METADATA_EXAMPLES.items() + ) + raise ValueError( + f"The metadata example `{name}` is not valid. Choose one of the following:\n" + f"{available}" + ) + + resource = files("pointblank.data") / "metadata_examples" / name + + return Path(str(resource)) + + def _detect_format(path: str | Path) -> str: """Detect the metadata format from a file path. From 573f38173d546a4d81d8d20b28565727104b318e Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:22:08 -0400 Subject: [PATCH 02/19] Update __init__.py --- pointblank/metadata/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pointblank/metadata/__init__.py b/pointblank/metadata/__init__.py index af5e5eb55..9d8c63eca 100644 --- a/pointblank/metadata/__init__.py +++ b/pointblank/metadata/__init__.py @@ -9,7 +9,7 @@ ) from pointblank.metadata._adam_validate import adam_to_metadata, validate_adam from pointblank.metadata._export import export_metadata -from pointblank.metadata._import import import_metadata +from pointblank.metadata._import import import_metadata, load_metadata_example from pointblank.metadata._sdtm_templates import ( SDTMDomainTemplate, SDTMVariableSpec, @@ -39,6 +39,7 @@ "ADaMDatasetTemplate", "ADaMVariableSpec", "import_metadata", + "load_metadata_example", "export_metadata", "get_sdtm_domain", "list_sdtm_domains", From 0f3dee79dd5919fa4ed7c720335b6c98eae8dc04 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:23:47 -0400 Subject: [PATCH 03/19] Update test_metadata.py --- tests/test_metadata.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index fa5786c13..b068224b0 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2987,3 +2987,40 @@ def test_population_flags_invalid_values_fail(self): ] assert len(saffl_checks) > 0 assert saffl_checks[0].n_failed > 0 + + +class TestLoadMetadataExample: + """Tests for the `load_metadata_example()` bundled-file accessor.""" + + def test_returns_existing_path_for_each_example(self): + """Every advertised example resolves to a file that exists on disk.""" + from pointblank.metadata._import import _METADATA_EXAMPLES, load_metadata_example + + for name in _METADATA_EXAMPLES: + path = load_metadata_example(name) + assert isinstance(path, Path) + assert path.exists() + assert path.name == name + + def test_invalid_name_raises_with_available_options(self): + """An unknown example name raises ValueError listing valid options.""" + from pointblank.metadata._import import load_metadata_example + + with pytest.raises(ValueError, match="is not valid"): + load_metadata_example("does_not_exist.xml") + + def test_define_example_imports(self): + """The bundled Define-XML example imports into a usable MetadataPackage.""" + from pointblank.metadata._import import import_metadata, load_metadata_example + + package = import_metadata( + load_metadata_example("define.xml"), format="cdisc_define" + ) + assert isinstance(package, MetadataPackage) + assert "DM" in package.keys() + + def test_exposed_at_top_level(self): + """`load_metadata_example` is exported from the top-level package.""" + import pointblank as pb + + assert hasattr(pb, "load_metadata_example") From 9621417c67841708a185b2a0e7cda50a5f008377 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:23:54 -0400 Subject: [PATCH 04/19] Update __init__.py --- pointblank/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pointblank/__init__.py b/pointblank/__init__.py index 231e8233f..67fbf4301 100644 --- a/pointblank/__init__.py +++ b/pointblank/__init__.py @@ -76,6 +76,7 @@ import_metadata, list_adam_datasets, list_sdtm_domains, + load_metadata_example, sdtm_to_metadata, validate_adam, validate_adam_structure, @@ -190,6 +191,7 @@ "register_adapter", # Metadata standards import/export "import_metadata", + "load_metadata_example", "export_metadata", "MetadataImport", "MetadataPackage", From 4d57bcecac47c65954d111d9f3c003b8cd19a566 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:24:06 -0400 Subject: [PATCH 05/19] Create datapackage.json --- .../data/metadata_examples/datapackage.json | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 pointblank/data/metadata_examples/datapackage.json diff --git a/pointblank/data/metadata_examples/datapackage.json b/pointblank/data/metadata_examples/datapackage.json new file mode 100644 index 000000000..57bb9083d --- /dev/null +++ b/pointblank/data/metadata_examples/datapackage.json @@ -0,0 +1,68 @@ +{ + "name": "quarterly-sales", + "title": "Quarterly Sales Dataset", + "description": "Sales transactions for Q1 2024", + "resources": [ + { + "name": "transactions", + "path": "transactions.csv", + "schema": { + "fields": [ + { + "name": "transaction_id", + "type": "string", + "description": "Unique transaction identifier", + "constraints": {"required": true, "unique": true, "minLength": 5, "maxLength": 20} + }, + { + "name": "customer_id", + "type": "string", + "description": "Customer account number", + "constraints": {"required": true, "minLength": 5, "maxLength": 20} + }, + { + "name": "amount", + "type": "number", + "description": "Transaction amount in USD", + "constraints": {"required": true, "minimum": 0.01, "maximum": 99999.99} + }, + { + "name": "quantity", + "type": "integer", + "description": "Number of items purchased", + "constraints": {"required": true, "minimum": 1, "maximum": 1000} + }, + { + "name": "category", + "type": "string", + "description": "Product category", + "constraints": { + "required": true, + "enum": ["electronics", "clothing", "food", "home", "sports"] + } + }, + { + "name": "sale_date", + "type": "date", + "description": "Date of sale", + "constraints": {"required": true} + }, + { + "name": "discount_pct", + "type": "number", + "description": "Discount percentage applied", + "constraints": {"minimum": 0, "maximum": 50} + }, + { + "name": "email", + "type": "string", + "description": "Customer email address", + "constraints": {"pattern": "^[^@]+@[^@]+\\.[^@]+$"} + } + ], + "primaryKey": ["transaction_id"], + "missingValues": ["", "NA", "N/A"] + } + } + ] +} From 978846b4bc0f6f2dcec04e58735fe2271f95e71b Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:24:19 -0400 Subject: [PATCH 06/19] Create define.xml --- pointblank/data/metadata_examples/define.xml | 156 +++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 pointblank/data/metadata_examples/define.xml diff --git a/pointblank/data/metadata_examples/define.xml b/pointblank/data/metadata_examples/define.xml new file mode 100644 index 000000000..38486e662 --- /dev/null +++ b/pointblank/data/metadata_examples/define.xml @@ -0,0 +1,156 @@ + + + + + + XYZ789 Phase III + A randomized, double-blind, placebo-controlled study + XYZ789 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Male + + + Female + + + Unknown + + + + + + White + + + Black or African American + + + Asian + + + American Indian or Alaska Native + + + Native Hawaiian or Other Pacific Islander + + + + + + Mild + + + Moderate + + + Severe + + + + + + No + + + Yes + + + + + + From c1b94059ef2192d151b658f6bd602cd0d025e719 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:24:25 -0400 Subject: [PATCH 07/19] Create dm.xpt --- pointblank/data/metadata_examples/dm.xpt | Bin 0 -> 2800 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pointblank/data/metadata_examples/dm.xpt diff --git a/pointblank/data/metadata_examples/dm.xpt b/pointblank/data/metadata_examples/dm.xpt new file mode 100644 index 0000000000000000000000000000000000000000..41e1c2d67721b1ab5fa2d30205785ca33dedd90e GIT binary patch literal 2800 zcmb7`+i%k_6vn+PW8(pV_5d&QUXVZ(?DV=yAUT(@j-{RA6gJ*^aYqnMq01)t?{N~J zEpeM>Sd%`VQ=ebHbK-S7ahUCR*mJC&XTM3;S;zd+!Wf@qKRZkpS-CLgjLXWLN*u4Fr{l0zKu`!Qs#(!~VEW#+51Y%z1NLD8c&Or=3$m@!p!pV%rUe?jN-E&!|i?KbM z4uTM~;czZ)gFzI`LLHNffdokvZvzkVx}xV$nbFwGF52kxzIiI=(hq~*Hv;pU;SVv2 z$_E4_c$SKHfCqV9(F>@|XzXPJZ7M6vOiX5T9DSGmlGmp}fdpw4KLZ}*bwwB|Gn&q2 z%RA=2?OS(VJTVb-5sqRDgL#DQK_uS$!Bl8aAVFHi`@n;|uILadGn#tY_KuzIaxYFe zE_5RAzFP1s6&?T&^17k}sLW{kjvn!j@5t+-UIY&!QGQoqK9Jy9Dt-+-$m@#sp)#Xu za=(Li*e54B&aU*gpi}a3LXr%~D)L+4L0(sRM;BK-mo@a5I~U1uF8=6cD`Nn{Ar;qv z2YFo!y=+fB7X36(zvF=`b;E3}^OnsUbpH7ltP_v?xLE%rU1jiTUI zU`nP}3mk(I12|F>VzXk9nDa5^9Ghp%`h0Ho Date: Mon, 29 Jun 2026 23:24:28 -0400 Subject: [PATCH 08/19] Create sdtm_ct.xml --- pointblank/data/metadata_examples/sdtm_ct.xml | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 pointblank/data/metadata_examples/sdtm_ct.xml diff --git a/pointblank/data/metadata_examples/sdtm_ct.xml b/pointblank/data/metadata_examples/sdtm_ct.xml new file mode 100644 index 000000000..f1bd44be9 --- /dev/null +++ b/pointblank/data/metadata_examples/sdtm_ct.xml @@ -0,0 +1,119 @@ + + + + + + CDISC SDTM Controlled Terminology + CDISC Submission Value-Level Terminology, 2024-03-29 + SDTM Terminology + + + + + + + Sex + Sex of the subject. + + Female + Female + A person who belongs to the sex that normally produces ova. + + + Male + Male + A person who belongs to the sex that normally produces sperm. + + + Unknown + Unknown + Not known, not observed, not recorded, or refused. + + + Undifferentiated + Undifferentiated + Sex could not be determined. + + + + + + Severity/Intensity Scale for Adverse Events + + Mild + + + Moderate + + + Severe + + + + + + + No + + + Yes + + + + + + + American Indian or Alaska Native + + + Asian + + + Black or African American + + + Native Hawaiian or Other Pacific Islander + + + White + + + + + + + Oral + + + Intravenous + + + Subcutaneous + + + Topical + + + Intramuscular + + + + + + From dc1c4315f2e2383c90ec5add0487c65edde2c954 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:24:33 -0400 Subject: [PATCH 09/19] Create table_schema.json --- .../data/metadata_examples/table_schema.json | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 pointblank/data/metadata_examples/table_schema.json diff --git a/pointblank/data/metadata_examples/table_schema.json b/pointblank/data/metadata_examples/table_schema.json new file mode 100644 index 000000000..0acb4cac6 --- /dev/null +++ b/pointblank/data/metadata_examples/table_schema.json @@ -0,0 +1,42 @@ +{ + "fields": [ + { + "name": "sensor_id", + "type": "string", + "description": "Unique sensor identifier", + "constraints": {"required": true, "pattern": "^SNS-[0-9]{4}$"} + }, + { + "name": "reading_time", + "type": "datetime", + "description": "ISO 8601 timestamp of reading", + "constraints": {"required": true} + }, + { + "name": "temperature", + "type": "number", + "description": "Temperature in Celsius", + "constraints": {"minimum": -40, "maximum": 85} + }, + { + "name": "pressure_hpa", + "type": "number", + "description": "Atmospheric pressure in hectopascals", + "constraints": {"minimum": 870, "maximum": 1084} + }, + { + "name": "battery_pct", + "type": "integer", + "description": "Battery level percentage", + "constraints": {"required": true, "minimum": 0, "maximum": 100} + }, + { + "name": "status", + "type": "string", + "description": "Sensor operational status", + "constraints": {"enum": ["active", "maintenance", "offline", "error"]} + } + ], + "primaryKey": ["sensor_id", "reading_time"], + "missingValues": ["", "NA"] +} From af9fc1e7fabc06e711abb788d5471e873a2f023b Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:24:45 -0400 Subject: [PATCH 10/19] Create weather_csvw.json --- .../data/metadata_examples/weather_csvw.json | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 pointblank/data/metadata_examples/weather_csvw.json diff --git a/pointblank/data/metadata_examples/weather_csvw.json b/pointblank/data/metadata_examples/weather_csvw.json new file mode 100644 index 000000000..e33919a1e --- /dev/null +++ b/pointblank/data/metadata_examples/weather_csvw.json @@ -0,0 +1,64 @@ +{ + "@context": "http://www.w3.org/ns/csvw", + "url": "weather_observations.csv", + "dc:title": "Weather Station Observations", + "dc:description": "Hourly weather observations from monitoring stations", + "tableSchema": { + "columns": [ + { + "name": "station_id", + "titles": "Station ID", + "datatype": "string", + "required": true + }, + { + "name": "timestamp", + "titles": "Observation Time", + "datatype": {"base": "datetime"}, + "required": true + }, + { + "name": "temperature_c", + "titles": "Temperature (Celsius)", + "datatype": { + "base": "decimal", + "minimum": -50, + "maximum": 60 + }, + "required": true + }, + { + "name": "humidity_pct", + "titles": "Relative Humidity (%)", + "datatype": { + "base": "decimal", + "minimum": 0, + "maximum": 100 + } + }, + { + "name": "wind_speed_kmh", + "titles": "Wind Speed (km/h)", + "datatype": { + "base": "decimal", + "minimum": 0, + "maximum": 400 + } + }, + { + "name": "precipitation_mm", + "titles": "Precipitation (mm)", + "datatype": { + "base": "decimal", + "minimum": 0 + } + }, + { + "name": "condition", + "titles": "Weather Condition", + "datatype": "string" + } + ], + "primaryKey": ["station_id", "timestamp"] + } +} From 08aaaa484afd57412a76942fbf1d24e708212996 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:24:50 -0400 Subject: [PATCH 11/19] Update 03-cdisc-validation.qmd --- .../03-cdisc-validation.qmd | 144 ++++++++++++------ 1 file changed, 98 insertions(+), 46 deletions(-) diff --git a/user_guide/11-metadata-import/03-cdisc-validation.qmd b/user_guide/11-metadata-import/03-cdisc-validation.qmd index e8d8d4308..a4dea73a1 100644 --- a/user_guide/11-metadata-import/03-cdisc-validation.qmd +++ b/user_guide/11-metadata-import/03-cdisc-validation.qmd @@ -54,28 +54,53 @@ a form suitable for validation. ### Importing a Define-XML File The `import_metadata()` function with `format="cdisc_define"` reads a Define-XML file and returns -a `MetadataPackage` containing metadata for all datasets defined in the document: +a `MetadataPackage` containing metadata for all datasets defined in the document. So that you can +run these examples without supplying your own files, Pointblank bundles a small set of sample +metadata documents that you can locate with `load_metadata_example()`: -```python +```{python} import pointblank as pb -# Import all datasets from a Define-XML -package = pb.import_metadata("define.xml", format="cdisc_define") +# Locate the bundled Define-XML example and import all of its datasets +define_path = pb.load_metadata_example("define.xml") +package = pb.import_metadata(define_path, format="cdisc_define") # List the datasets defined in the document -for name, meta in package.datasets.items(): +for name in package.keys(): + meta = package[name] print(f"{name}: {meta.dataset_label} ({len(meta.variables)} variables)") ``` Each dataset in the package is a `MetadataImport` object with full variable-level metadata. You -can access individual datasets by name and generate validation from them: +can access individual datasets by name and generate validation from them. Here we validate a small +Demographics table against the metadata extracted from the Define-XML: + +```{python} +import polars as pl -```python # Get metadata for the Demographics domain dm_meta = package["DM"] -# Generate validation for your Demographics data -validation = dm_meta.to_validate(data=dm_dataframe).interrogate() +# A small Demographics dataset to validate +dm_data = pl.DataFrame({ + "STUDYID": ["XYZ789"] * 4, + "DOMAIN": ["DM"] * 4, + "USUBJID": ["XYZ789-001", "XYZ789-002", "XYZ789-003", "XYZ789-004"], + "SUBJID": ["001", "002", "003", "004"], + "RFSTDTC": ["2024-01-15", "2024-01-20", "2024-02-01", "2024-02-10"], + "RFENDTC": ["2024-06-15", "2024-06-20", "2024-07-01", "2024-07-10"], + "SITEID": ["SITE01", "SITE01", "SITE02", "SITE02"], + "AGE": [45, 62, 38, 55], + "AGEU": ["YEARS"] * 4, + "SEX": ["M", "F", "M", "F"], + "RACE": ["WHITE", "BLACK OR AFRICAN AMERICAN", "ASIAN", "WHITE"], + "ARMCD": ["DRUG", "PLACEBO", "DRUG", "PLACEBO"], + "ARM": ["Active Drug 10mg", "Placebo", "Active Drug 10mg", "Placebo"], +}) + +# Generate validation for the Demographics data from the Define-XML metadata +validation = dm_meta.to_validate(data=dm_data).interrogate() +validation ``` ### What Gets Extracted @@ -102,11 +127,8 @@ Define-XML documents embed the codelists that constrain variable values. When Po Define-XML, all codelists are extracted and linked to their respective variables. The `to_validate()` method then generates `col_vals_in_set()` checks for each variable that references a codelist: -```python -package = pb.import_metadata("define.xml", format="cdisc_define") -dm_meta = package["DM"] - -# Inspect codelists referenced by this domain +```{python} +# Inspect codelists referenced by the Demographics domain for cl_name, codelist in dm_meta.codelists.items(): print(f"{cl_name}: {codelist.to_set()[:5]}...") # first 5 values print(f" Extensible: {codelist.extensible}") @@ -120,26 +142,35 @@ the set as warnings rather than hard failures. Beyond the codelists embedded in Define-XML, CDISC publishes standalone Controlled Terminology packages as XML files. These contain the canonical value sets for concepts like SEX, RACE, -ROUTE OF ADMINISTRATION, and hundreds of others. Pointblank can parse these directly: +ROUTE OF ADMINISTRATION, and hundreds of others. Pointblank can parse these directly. -```python -import pointblank as pb +Importing a CT package returns a `MetadataPackage` whose entries are keyed by codelist name. Each +entry is a `MetadataImport` that holds the codelist itself: + +```{python} +# Import the bundled CDISC CT example +ct_path = pb.load_metadata_example("sdtm_ct.xml") +ct = pb.import_metadata(ct_path, format="cdisc_ct") -# Import a CDISC CT package -ct = pb.import_metadata("SDTM_CT_2024-03-29.xml", format="cdisc_ct") +# List the codelists in the package +print(list(ct.keys())) -# Access individual codelists by C-code -sex_codelist = ct.codelists.get("C66731") -if sex_codelist: - print(f"SEX values: {sex_codelist.to_set()}") - print(f"Extensible: {sex_codelist.extensible}") +# Access a codelist by name +sex_codelist = ct["Sex"].codelists["Sex"] +print(f"SEX values: {sex_codelist.to_set()}") +print(f"Extensible: {sex_codelist.extensible}") +``` -# Use in validation +Once you have a codelist, its permitted values feed directly into a `col_vals_in_set()` check: + +```{python} +# Use the CT-derived value set in a validation validation = ( - pb.Validate(data=demographics_df) + pb.Validate(data=dm_data) .col_vals_in_set(columns="SEX", set=sex_codelist.to_set()) .interrogate() ) +validation ``` Controlled Terminology packages version quarterly (e.g., 2024-03-29, 2024-06-28). Referencing a @@ -450,16 +481,22 @@ seamlessly. ### Importing a Frictionless Schema -```python -import pointblank as pb +The bundled `datapackage.json` example describes a transactions table. Importing it yields a +`MetadataImport` with one `VariableMetadata` per column: + +```{python} +# Import the bundled Frictionless Data Package example +datapackage_path = pb.load_metadata_example("datapackage.json") +meta = pb.import_metadata(datapackage_path, format="frictionless") -# Import from a datapackage.json -meta = pb.import_metadata("datapackage.json", format="frictionless") +print(f"Dataset: {meta.dataset_name}") +for v in meta.variables: + print(f" {v.name:16s} {v.dtype}") +``` -# Or from a standalone Table Schema -meta = pb.import_metadata("schema.json", format="table_schema") +Frictionless constraints map directly onto Pointblank validation steps: -# Frictionless constraints map directly: +```python # - "required": true -> col_vals_not_null() # - "unique": true -> rows_distinct() # - "minimum": 0 -> col_vals_ge(value=0) @@ -471,17 +508,22 @@ meta = pb.import_metadata("schema.json", format="table_schema") The constraint mapping is direct and complete. Every constraint expressible in a Frictionless Table Schema has a corresponding Pointblank validation step, making the translation lossless. +A standalone Table Schema (without the surrounding data package wrapper) imports the same way with +`format="table_schema"`. + ### CSVW (CSV on the Web) The W3C's CSVW standard provides similar capabilities to Frictionless but uses JSON-LD and aligns with linked data principles. Pointblank imports CSVW metadata with the same interface: -```python -meta = pb.import_metadata("metadata.json", format="csvw") +```{python} +# Import the bundled CSVW example +csvw_path = pb.load_metadata_example("weather_csvw.json") +meta = pb.import_metadata(csvw_path, format="csvw") # CSVW column descriptors become VariableMetadata -# datatype constraints become validation steps -validation = meta.to_validate(data=df).interrogate() +print(f"Dataset: {meta.dataset_name} ({len(meta.variables)} variables)") +print("Columns:", ", ".join(v.name for v in meta.variables)) ``` ## Exporting Metadata @@ -489,12 +531,20 @@ validation = meta.to_validate(data=df).interrogate() Pointblank can also export validation metadata in Frictionless format. This is useful when you want to share data quality expectations with tools that understand the Frictionless ecosystem: -```python -import pointblank as pb +```{python} +import tempfile +from pathlib import Path -# Export a MetadataImport as Frictionless Table Schema -meta = pb.import_metadata("clinical_data.xpt", format="xpt") -pb.export_metadata(meta, "table_schema.json", format="frictionless") +# Read metadata from the bundled SAS Transport (XPT) example +xpt_path = pb.load_metadata_example("dm.xpt") +meta = pb.import_metadata(xpt_path, format="xpt") + +# Export it as a Frictionless Table Schema +out_path = Path(tempfile.gettempdir()) / "dm_table_schema.json" +pb.export_metadata(meta, out_path, format="frictionless") + +# Inspect the first part of the exported schema +print(out_path.read_text()[:300]) ``` The exported document contains the column definitions and constraints from the original metadata, @@ -507,12 +557,11 @@ Define-XML provides the authoritative variable definitions, but you might also w against SDTM domain rules and controlled terminology packages. Pointblank supports this by letting you compose validation workflows from different metadata sources: -```python -import pointblank as pb +```{python} from pointblank.metadata import validate_sdtm # Load the Define-XML for variable-level constraints -package = pb.import_metadata("define.xml", format="cdisc_define") +package = pb.import_metadata(pb.load_metadata_example("define.xml"), format="cdisc_define") dm_meta = package["DM"] # Generate validation from Define-XML metadata @@ -522,9 +571,12 @@ validation = dm_meta.to_validate(data=dm_data) # (ISO 8601 checks, sequence number rules, etc.) sdtm_validation = validate_sdtm(data=dm_data, domain="DM") -# Run both and compare results +# Run both and compare how many checks each contributes define_results = validation.interrogate() sdtm_results = sdtm_validation.interrogate() + +print(f"Define-XML checks: {len(define_results.validation_info)}") +print(f"SDTM template checks: {len(sdtm_results.validation_info)}") ``` This layered approach gives you the flexibility to apply different levels of validation depending From 76d019a87d37e993f367e126d3ecbc1ce2a3eeed Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:31:18 -0400 Subject: [PATCH 12/19] Update _import.py --- pointblank/metadata/_import.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pointblank/metadata/_import.py b/pointblank/metadata/_import.py index 0fc6aa456..83aefe798 100644 --- a/pointblank/metadata/_import.py +++ b/pointblank/metadata/_import.py @@ -123,12 +123,10 @@ def load_metadata_example(name: str) -> Path: """ if name not in _METADATA_EXAMPLES: available = "\n".join( - f"- `{fname}` ({fmt}): {desc}" - for fname, (fmt, desc) in _METADATA_EXAMPLES.items() + f"- `{fname}` ({fmt}): {desc}" for fname, (fmt, desc) in _METADATA_EXAMPLES.items() ) raise ValueError( - f"The metadata example `{name}` is not valid. Choose one of the following:\n" - f"{available}" + f"The metadata example `{name}` is not valid. Choose one of the following:\n{available}" ) resource = files("pointblank.data") / "metadata_examples" / name From 588be90acfd3e4f123826ba1c82351ffe5ff061b Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Mon, 29 Jun 2026 23:31:20 -0400 Subject: [PATCH 13/19] Update test_metadata.py --- tests/test_metadata.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b068224b0..94d9ec879 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -3013,9 +3013,7 @@ def test_define_example_imports(self): """The bundled Define-XML example imports into a usable MetadataPackage.""" from pointblank.metadata._import import import_metadata, load_metadata_example - package = import_metadata( - load_metadata_example("define.xml"), format="cdisc_define" - ) + package = import_metadata(load_metadata_example("define.xml"), format="cdisc_define") assert isinstance(package, MetadataPackage) assert "DM" in package.keys() From 7261f9bf6d0a440f0affb6caac3556b9c7d48c80 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Tue, 30 Jun 2026 13:33:07 -0400 Subject: [PATCH 14/19] Add dependencies to `docs` group --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 314790628..a9ee0fb34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,8 @@ docs = [ "pyspark==3.5.6", "openpyxl>=3.0.0", "duckdb>=1.2.0,<1.3.3", # Pin to stable versions avoiding 1.4.0+ RecordBatchReader issues + "lxml>=4.9.0", # CDISC Define-XML / Controlled Terminology examples + "pyreadstat>=1.2.0", # SAS Transport (XPT) metadata example ] [dependency-groups] From b59eda9f1f7e73bfee2dcf7e72dba543869a9b84 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Tue, 30 Jun 2026 14:24:27 -0400 Subject: [PATCH 15/19] Update 03-cdisc-validation.qmd --- user_guide/11-metadata-import/03-cdisc-validation.qmd | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/user_guide/11-metadata-import/03-cdisc-validation.qmd b/user_guide/11-metadata-import/03-cdisc-validation.qmd index a4dea73a1..c45380a5c 100644 --- a/user_guide/11-metadata-import/03-cdisc-validation.qmd +++ b/user_guide/11-metadata-import/03-cdisc-validation.qmd @@ -39,6 +39,13 @@ Or install Pointblank with the CDISC extra: pip install pointblank[cdisc] ``` +Reading metadata from SAS Transport (`.xpt`) files (used in the *Exporting Metadata* example below) +additionally requires the `pyreadstat` library: + +```bash +pip install pyreadstat +``` + The SDTM and ADaM domain templates are built into Pointblank and require no additional dependencies. They encode the structural requirements from the SDTM Implementation Guide 3.4 and the ADaM Implementation Guide 1.1 directly in Python, so you can validate clinical datasets From e2a5afb145b378996f63b185b2edb5f792f1da6c Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Tue, 30 Jun 2026 14:24:30 -0400 Subject: [PATCH 16/19] Update 03-cdisc-validation.qmd --- user_guide/11-metadata-import/03-cdisc-validation.qmd | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/user_guide/11-metadata-import/03-cdisc-validation.qmd b/user_guide/11-metadata-import/03-cdisc-validation.qmd index c45380a5c..713430c2f 100644 --- a/user_guide/11-metadata-import/03-cdisc-validation.qmd +++ b/user_guide/11-metadata-import/03-cdisc-validation.qmd @@ -141,9 +141,14 @@ for cl_name, codelist in dm_meta.codelists.items(): print(f" Extensible: {codelist.extensible}") ``` -Non-extensible codelists require strict adherence: any value not in the codelist is a validation -failure. Extensible codelists permit sponsor-defined additions, so Pointblank treats values outside -the set as warnings rather than hard failures. +The `extensible` flag records an important distinction. Non-extensible codelists require strict +adherence: any value not in the codelist is a conformance issue. Extensible codelists permit +sponsor-defined additions, so a value outside the published set is not necessarily an error. + +Pointblank currently generates the same `col_vals_in_set()` check for both kinds of codelist. For +extensible codelists you will typically want to relax that step to reflect that additions are +allowed, for example by attaching a warning-level threshold to the step rather than treating every +out-of-set value as a hard failure. ## CDISC Controlled Terminology Import From 2f77942cd7efca419b9c07d6725efa754008b02c Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Tue, 30 Jun 2026 14:24:33 -0400 Subject: [PATCH 17/19] Update 03-cdisc-validation.qmd --- .../03-cdisc-validation.qmd | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/user_guide/11-metadata-import/03-cdisc-validation.qmd b/user_guide/11-metadata-import/03-cdisc-validation.qmd index 713430c2f..73a6900de 100644 --- a/user_guide/11-metadata-import/03-cdisc-validation.qmd +++ b/user_guide/11-metadata-import/03-cdisc-validation.qmd @@ -319,6 +319,39 @@ Invalid: 15-Mar-2024 (wrong format) This catches a common data quality issue where dates are entered in locale-specific formats rather than the required ISO 8601 pattern. +### Catching Conformance Problems + +The examples so far have used clean data, so every check passes. The point of validation, of course, +is to surface problems. Here is the same DM domain with three deliberate defects: a missing +`USUBJID` (a required variable), a row where `DOMAIN` is `"AE"` instead of `"DM"`, and a `RFSTDTC` +value entered in the wrong date format: + +```{python} +# A Demographics dataset with three injected conformance issues +dm_data_with_issues = pl.DataFrame({ + "STUDYID": ["STUDY01"] * 4, + "DOMAIN": ["DM", "DM", "AE", "DM"], # row 3: wrong domain code + "USUBJID": ["STUDY01-001", "STUDY01-002", None, "STUDY01-004"], # row 3: required but null + "SUBJID": ["001", "002", "003", "004"], + "RFSTDTC": ["2024-01-15", "03/15/2024", "2024-02-01", "2024-02-10"], # row 2: not ISO 8601 + "SITEID": ["SITE01", "SITE01", "SITE02", "SITE02"], + "AGE": [45, 62, 38, 55], + "AGEU": ["YEARS"] * 4, + "SEX": ["M", "F", "M", "F"], + "RACE": ["WHITE", "ASIAN", "WHITE", "ASIAN"], + "ARMCD": ["DRUG", "PLACEBO", "DRUG", "PLACEBO"], + "ARM": ["Active Drug 10mg", "Placebo", "Active Drug 10mg", "Placebo"], +}) + +validation = validate_sdtm(data=dm_data_with_issues, domain="DM").interrogate() +validation +``` + +Three steps now report a failing unit: the `col_vals_not_null()` check on `USUBJID`, the +`col_vals_in_set()` check on `DOMAIN`, and the ISO 8601 `col_vals_regex()` check on `RFSTDTC`. Each +failing step links to the specific rows so you can trace a conformance issue back to the offending +records. + ### Converting SDTM Templates to MetadataImport If you prefer to work with the standard `MetadataImport` interface (for example, to use From 24c2a7c013b7b3dd7ae0071d3f829752cdf13f87 Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Tue, 30 Jun 2026 14:24:37 -0400 Subject: [PATCH 18/19] Update 03-cdisc-validation.qmd --- .../11-metadata-import/03-cdisc-validation.qmd | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/user_guide/11-metadata-import/03-cdisc-validation.qmd b/user_guide/11-metadata-import/03-cdisc-validation.qmd index 73a6900de..3f519aa49 100644 --- a/user_guide/11-metadata-import/03-cdisc-validation.qmd +++ b/user_guide/11-metadata-import/03-cdisc-validation.qmd @@ -541,14 +541,14 @@ for v in meta.variables: Frictionless constraints map directly onto Pointblank validation steps: -```python -# - "required": true -> col_vals_not_null() -# - "unique": true -> rows_distinct() -# - "minimum": 0 -> col_vals_ge(value=0) -# - "maximum": 100 -> col_vals_le(value=100) -# - "pattern": "..." -> col_vals_regex(pattern="...") -# - "enum": [...] -> col_vals_in_set(set=[...]) -``` +| Frictionless constraint | Pointblank step | +|-------------------------|-----------------| +| `"required": true` | `col_vals_not_null()` | +| `"unique": true` | `rows_distinct()` | +| `"minimum": 0` | `col_vals_ge(value=0)` | +| `"maximum": 100` | `col_vals_le(value=100)` | +| `"pattern": "..."` | `col_vals_regex(pattern="...")` | +| `"enum": [...]` | `col_vals_in_set(set=[...])` | The constraint mapping is direct and complete. Every constraint expressible in a Frictionless Table Schema has a corresponding Pointblank validation step, making the translation lossless. From 86f15a0301bc45ca4c119bf250933ac4a92b04cf Mon Sep 17 00:00:00 2001 From: Richard Iannone Date: Tue, 30 Jun 2026 14:24:53 -0400 Subject: [PATCH 19/19] Update 03-cdisc-validation.qmd --- user_guide/11-metadata-import/03-cdisc-validation.qmd | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/user_guide/11-metadata-import/03-cdisc-validation.qmd b/user_guide/11-metadata-import/03-cdisc-validation.qmd index 3f519aa49..aada68777 100644 --- a/user_guide/11-metadata-import/03-cdisc-validation.qmd +++ b/user_guide/11-metadata-import/03-cdisc-validation.qmd @@ -577,6 +577,7 @@ Pointblank can also export validation metadata in Frictionless format. This is u to share data quality expectations with tools that understand the Frictionless ecosystem: ```{python} +import json import tempfile from pathlib import Path @@ -588,8 +589,10 @@ meta = pb.import_metadata(xpt_path, format="xpt") out_path = Path(tempfile.gettempdir()) / "dm_table_schema.json" pb.export_metadata(meta, out_path, format="frictionless") -# Inspect the first part of the exported schema -print(out_path.read_text()[:300]) +# Inspect the exported schema: how many fields, and the first two definitions +schema = json.loads(out_path.read_text()) +print(f"Exported {len(schema['fields'])} field definitions. First two:") +print(json.dumps(schema["fields"][:2], indent=2)) ``` The exported document contains the column definitions and constraints from the original metadata,