Generate XBRL UTR schemas using jq templates

Makefile

@@ -33,7 +33,59 @@ GENERATED = \
 	schemas/iso/currency/2015/historical/alpha-code.json \
 	schemas/iso/currency/2015/historical/alpha-currency.json \
 	schemas/iso/currency/2015/historical/numeric-code.json \
-	schemas/iso/currency/2015/historical/numeric-currency.json
+	schemas/iso/currency/2015/historical/numeric-currency.json \
+	schemas/xbrl/utr/area-item-type-normative.json \
+	schemas/xbrl/utr/area-item-type.json \
+	schemas/xbrl/utr/duration-item-type-normative.json \
+	schemas/xbrl/utr/duration-item-type.json \
+	schemas/xbrl/utr/electric-charge-item-type-normative.json \
+	schemas/xbrl/utr/electric-charge-item-type.json \
+	schemas/xbrl/utr/electric-current-item-type-normative.json \
+	schemas/xbrl/utr/electric-current-item-type.json \
+	schemas/xbrl/utr/energy-item-type-normative.json \
+	schemas/xbrl/utr/energy-item-type.json \
+	schemas/xbrl/utr/energy-per-monetary-item-type-normative.json \
+	schemas/xbrl/utr/energy-per-monetary-item-type.json \
+	schemas/xbrl/utr/flow-item-type-normative.json \
+	schemas/xbrl/utr/flow-item-type.json \
+	schemas/xbrl/utr/force-item-type-normative.json \
+	schemas/xbrl/utr/force-item-type.json \
+	schemas/xbrl/utr/frequency-item-type-normative.json \
+	schemas/xbrl/utr/frequency-item-type.json \
+	schemas/xbrl/utr/ghg-emissions-item-type-normative.json \
+	schemas/xbrl/utr/ghg-emissions-item-type.json \
+	schemas/xbrl/utr/ghg-emissions-per-monetary-item-type-normative.json \
+	schemas/xbrl/utr/ghg-emissions-per-monetary-item-type.json \
+	schemas/xbrl/utr/length-item-type-normative.json \
+	schemas/xbrl/utr/length-item-type.json \
+	schemas/xbrl/utr/mass-item-type-normative.json \
+	schemas/xbrl/utr/mass-item-type.json \
+	schemas/xbrl/utr/memory-item-type-normative.json \
+	schemas/xbrl/utr/memory-item-type.json \
+	schemas/xbrl/utr/monetary-item-type-normative.json \
+	schemas/xbrl/utr/monetary-item-type.json \
+	schemas/xbrl/utr/per-share-item-type-normative.json \
+	schemas/xbrl/utr/per-share-item-type.json \
+	schemas/xbrl/utr/per-unit-item-type-normative.json \
+	schemas/xbrl/utr/per-unit-item-type.json \
+	schemas/xbrl/utr/plane-angle-item-type-normative.json \
+	schemas/xbrl/utr/plane-angle-item-type.json \
+	schemas/xbrl/utr/power-item-type-normative.json \
+	schemas/xbrl/utr/power-item-type.json \
+	schemas/xbrl/utr/pressure-item-type-normative.json \
+	schemas/xbrl/utr/pressure-item-type.json \
+	schemas/xbrl/utr/pure-item-type-normative.json \
+	schemas/xbrl/utr/pure-item-type.json \
+	schemas/xbrl/utr/shares-item-type-normative.json \
+	schemas/xbrl/utr/shares-item-type.json \
+	schemas/xbrl/utr/temperature-item-type-normative.json \
+	schemas/xbrl/utr/temperature-item-type.json \
+	schemas/xbrl/utr/voltage-item-type-normative.json \
+	schemas/xbrl/utr/voltage-item-type.json \
+	schemas/xbrl/utr/volume-item-type-normative.json \
+	schemas/xbrl/utr/volume-item-type.json \
+	schemas/xbrl/utr/volume-per-monetary-item-type-normative.json \
+	schemas/xbrl/utr/volume-per-monetary-item-type.json
 
 # TODO: Make `jsonschema fmt` automatically detect test files
 all: common test
@@ -42,19 +94,20 @@ all: common test
 
 .PHONY: common
 common: $(GENERATED)
-	$(JSONSCHEMA) metaschema schemas meta --verbose
-	$(JSONSCHEMA) lint schemas meta --verbose
-	$(JSONSCHEMA) validate meta/schemas-root.json --verbose $(SCHEMAS)
-	$(JSONSCHEMA) validate meta/schemas.json --verbose $(SCHEMAS)
-	$(JSONSCHEMA) validate meta/test.json --verbose $(TESTS)
+	$(JSONSCHEMA) metaschema schemas meta
+	$(JSONSCHEMA) lint schemas meta
+	$(JSONSCHEMA) validate meta/schemas-root.json $(SCHEMAS)
+	$(JSONSCHEMA) validate meta/schemas.json $(SCHEMAS)
+	$(JSONSCHEMA) validate meta/test.json $(TESTS)
 	$(SHELLCHECK) scripts/*.sh
-	./scripts/schemas-tests-mirror.sh
+	./scripts/quality-schemas-tests-mirror.sh
+	./scripts/quality-templates-xbrl-utr-mirror.sh
 
 # TODO: Make `jsonschema fmt` automatically detect test files
 .PHONY: lint
 lint: common
-	$(JSONSCHEMA) fmt schemas meta --verbose --check
-	$(JSONSCHEMA) fmt test --verbose --default-dialect "https://json-schema.org/draft/2020-12/schema"
+	$(JSONSCHEMA) fmt schemas meta --check
+	$(JSONSCHEMA) fmt test --default-dialect "https://json-schema.org/draft/2020-12/schema"
 
 .PHONY: test
 test:
@@ -83,14 +136,19 @@ schemas/iso/currency/2015/%.json: build/iso/currency/list-one.json templates/iso
 	$(JQ) --from-file $(word 2,$^) $< > $@
 	$(JSONSCHEMA) fmt $@
 
+build/xbrl/utr/%.json: scripts/xml2json.py vendor/data/xbrl/utr/%.xml
+	$(MKDIRP) $(dir $@)
+	$(PYTHON) $< $(word 2,$^) $@
+schemas/xbrl/utr/%.json: build/xbrl/utr/utr.json templates/xbrl/utr/%.jq
+	$(MKDIRP) $(dir $@)
+	$(JQ) --from-file $(word 2,$^) $< > $@
+	$(JSONSCHEMA) fmt $@
+
 generate-iso-language: generate/iso/language/main.py
 	$(PYTHON) $<
 generate-iso-country: generate/iso/country/main.py
 	$(PYTHON) $<
-build/xbrl/utr/%.json: scripts/xml2json.py vendor/data/xbrl/utr/%.xml
-	$(PYTHON) $< $(word 2,$^) $@
-generate-xbrl-utr: generate/xbrl/utr/main.py build/xbrl/utr/utr.json
-	$(PYTHON) $<
+
 
 # TODO: Add a `jsonschema pkg` command instead
 .PHONY: dist

scripts/quality-templates-xbrl-utr-mirror.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+UTR_DATA="$(pwd)/build/xbrl/utr/utr.json"
+TEMPLATES_DIR="$(pwd)/templates/xbrl/utr"
+EXIT_CODE=0
+
+# Create temporary directory
+TMP="$(mktemp -d)"
+# shellcheck disable=SC2329
+clean() {
+  # shellcheck disable=SC2317
+  rm -rf "$TMP"
+}
+trap clean EXIT
+
+camel_to_kebab() {
+  echo "$1" | sed 's/\([A-Z]\)/-\1/g' | tr '[:upper:]' '[:lower:]' | sed 's/^-//'
+}
+
+if [ ! -f "$UTR_DATA" ]
+then
+  echo "ERROR: UTR data file '$UTR_DATA' does not exist" >&2
+  exit 1
+fi
+
+# Extract all unique itemTypes from the data and convert to kebab-case
+# Store them in the temp file first, then validate
+jq -r '.["{http://www.xbrl.org/2009/utr}utr"]["{http://www.xbrl.org/2009/utr}units"]["{http://www.xbrl.org/2009/utr}unit"][] | .["{http://www.xbrl.org/2009/utr}itemType"]' "$UTR_DATA" | sort -u > "$TMP/item_types.txt"
+
+while IFS= read -r item_type
+do
+  kebab_name=$(camel_to_kebab "$item_type")
+  echo "$kebab_name" >> "$TMP/expected_templates.txt"
+
+  # Check for regular template
+  expected_template="$TEMPLATES_DIR/${kebab_name}.jq"
+  if [ ! -f "$expected_template" ]
+  then
+    echo "ERROR: itemType '$item_type' is missing template '$expected_template'" >&2
+    EXIT_CODE=1
+  fi
+
+  expected_normative="$TEMPLATES_DIR/${kebab_name}-normative.jq"
+  if [ ! -f "$expected_normative" ]
+  then
+    echo "ERROR: itemType '$item_type' is missing normative template '$expected_normative'" >&2
+    EXIT_CODE=1
+  fi
+done < "$TMP/item_types.txt"
+
+# Check the reverse: for each template, verify it corresponds to a known itemType
+find "$TEMPLATES_DIR" -type f -name "*.jq" ! -name "*-normative.jq" > "$TMP/templates.txt"
+
+while IFS= read -r template_file
+do
+  base_name=$(basename "$template_file" .jq)
+
+  # Check if this base name is in our expected list
+  if ! grep -q "^${base_name}$" "$TMP/expected_templates.txt"
+  then
+    echo "ERROR: Template '$template_file' does not correspond to any itemType in the data" >&2
+    EXIT_CODE=1
+  fi
+
+  # Also check that corresponding normative template exists
+  normative_template="${template_file%.jq}-normative.jq"
+  if [ ! -f "$normative_template" ]
+  then
+    echo "ERROR: Template '$template_file' is missing corresponding normative template '$normative_template'" >&2
+    EXIT_CODE=1
+  fi
+done < "$TMP/templates.txt"
+
+exit $EXIT_CODE

templates/xbrl/utr/area-item-type-normative.jq

@@ -0,0 +1,31 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": ("XBRL UTR area Item Type units (" + .["{http://www.xbrl.org/2009/utr}utr"]["@attributes"].lastUpdated + ")"),
+  "description": "Valid units for areaItemType as defined in the XBRL Units Type Registry",
+  "examples": (
+    .["{http://www.xbrl.org/2009/utr}utr"]["{http://www.xbrl.org/2009/utr}units"]["{http://www.xbrl.org/2009/utr}unit"]
+    | map(select(.["{http://www.xbrl.org/2009/utr}itemType"] == "areaItemType" and .["{http://www.xbrl.org/2009/utr}status"] == "REC"))
+
+    | .[0:3]
+    | map(.["{http://www.xbrl.org/2009/utr}unitId"])
+  ),
+  "x-license": "https://github.com/sourcemeta/std/blob/main/LICENSE",
+  "x-links": [
+    "https://www.xbrl.org/specification/utr/rec-2013-11-18/utr-rec-2013-11-18-clean.html",
+    "https://www.xbrl.org/utr/utr.xml"
+  ],
+  "anyOf": (
+    .["{http://www.xbrl.org/2009/utr}utr"]["{http://www.xbrl.org/2009/utr}units"]["{http://www.xbrl.org/2009/utr}unit"]
+    | map(select(.["{http://www.xbrl.org/2009/utr}itemType"] == "areaItemType" and .["{http://www.xbrl.org/2009/utr}status"] == "REC"))
+
+    | map(
+        {
+          "const": .["{http://www.xbrl.org/2009/utr}unitId"],
+          "title": .["{http://www.xbrl.org/2009/utr}unitName"],
+          "description": (.["{http://www.xbrl.org/2009/utr}definition"] | gsub("\\s{2,}"; " ") | if endswith(".") then .[:-1] else . end)
+        } +
+        (if .["{http://www.xbrl.org/2009/utr}symbol"] then {"x-symbol": .["{http://www.xbrl.org/2009/utr}symbol"]} else {} end) +
+        (if .["{http://www.xbrl.org/2009/utr}status"] then {"x-status": .["{http://www.xbrl.org/2009/utr}status"]} else {} end)
+      )
+  )
+}