Feature pipeline refactor 2021 05 (#152)

* Rework of docs focusing on JSON docs and model pipeline
* Improvements to composition toolchain
* Fixed a few small bugs in the metaschema-check. Improved performance of the compose pruning using an accumulator.
* Moved edge-case samples into testing directory
* Made shadowing warning a warning
* Initial commit of an Oxygen Metaschema framework.
* Creation of new compose schematron unit tests.
* Cross-linking XML and JSON syntax pages and other improvements to links
* Now building XML and JSON indexes to reference pages, with links to steps
* Reconfigured docs pipeline (XSLT entry points); adding new files including pipeline steps
* Migrating schema generation tools to new/improved composition pipeline
* Addressing usnistgov/OSCAL#902 thanks for finding this bug
* Enhancements to JSON Schema definition (with better performance too)
* Adding support for json-base-uri as a metaschema property
* Updated JSON schema $id; factoring out common docs XSLT
* Fixing IDs in JSON schema per issue usnistgov/OSCAL#933.
* Addressing datatype validation issues: whitespace collapsing; non-empty values; ncname-workalike in JSON Schema - see usnistgov/OSCAL#911  usnistgov/OSCAL#805 also usnistgov#33 usnistgov#67 usnistgov#68
* Improvements to XSD production; fully aligning 'token' datatype across XSD and JSON Schema implementations.
* Updating bidirectional XML/JSON converter generators (#143)
* Committing a version that handles test data correctly (so far) from rebuilt metaschema composition addressing usnistgov#51 usnistgov#53 usnistgov#76
* Now displaying constraints in documentation at point of definition;
* Docs generation revamp Reworked reference and other pages to sketch - #128 and others

Co-authored-by: Wendell Piez <wendell.piez@nist.gov>

.gitignore

@@ -12,3 +12,4 @@
 
 # other generated files
 nohup.out
+/toolchains/xslt-M4/validate/xspec/metaschema-composition-schematron-result.html

.gitmodules

@@ -7,3 +7,7 @@
 	path = support/schematron
 	url = https://github.com/usnistgov/schematron.git
 	branch = master
+[submodule "support/xspec"]
+	path = support/xspec
+	url = https://github.com/xspec/xspec.git
+	branch = d8bbeca51eaccc8b3e07c82547f6aff4a2072aef

scripts/generate-content-converter.sh

@@ -126,12 +126,12 @@ fi
 result=$(generate_converter "$METASCHEMA" "$GENERATED_CONVERTER" "${GENERATE_SOURCE_FORMAT}" "${GENERATE_TARGET_FORMAT}" 2>&1)
 cmd_exitcode=$?
 if [ $cmd_exitcode -ne 0 ]; then
-
-  echo -e "${P_ERROR}Generating ${GENERATE_SOURCE_FORMAT^^} to ${GENERATE_TARGET_FORMAT^^} converter failed for '${P_END}${METASCHEMA_RELATIVE_PATH}${P_ERROR}'.${P_END}"
-  echo -e "${P_ERROR}${result}${P_END}"
-  exitcode=1
+  >&2 echo -e "${P_ERROR}Generating ${GENERATE_SOURCE_FORMAT^^} to ${GENERATE_TARGET_FORMAT^^} converter failed for '${P_END}${METASCHEMA_RELATIVE_PATH}${P_ERROR}'.${P_END}"
+  >&2 echo -e "${P_ERROR}${result}${P_END}"
+  exit 1;
 else
   echo -e "${P_OK}Generated ${GENERATE_SOURCE_FORMAT^^} to ${GENERATE_TARGET_FORMAT^^} converter for '${P_END}${METASCHEMA_RELATIVE_PATH}${P_OK}' as '${P_END}${GENERATED_CONVERTER_RELATIVE}${P_INFO}'.${P_END}"
 fi
 
+exit 0;
 

scripts/include/init-calabash.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+init_calabash() {
+  local my_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)
+#  source "${my_dir}/common-environment.sh"
+  source "${my_dir}/init-saxon.sh"
+
+  if [[ -z "$CALABASH_HOME" ]]; then
+    echo -e "${P_ERROR}CALABASH_HOME is not set or is empty.${P_END} ${P_INFO}Please set CALABASH_HOME to point to the location of the Calabash XML installation.${P_END}"
+  fi
+}
+
+run_calabash() {
+    local xproc="$1"; shift
+    local extra_params=($@)
+
+    local JARS=()
+    JARS+=("$CALABASH_HOME"/*.jar)
+    JARS+=("$CALABASH_HOME"/lib/*.jar)
+
+    local classpath=$(IFS=:; echo -e "${JARS[*]}")
+
+    java -cp "$classpath" com.xmlcalabash.drivers.Main -D "${extra_params[@]}" "$xproc"
+
+    if [ "$?" -ne 0 ]; then
+        echo -e "${P_ERROR}Error running Calabash.${P_END}"
+        return 3
+    fi
+    return 0
+}

scripts/include/init-saxon.sh

@@ -36,11 +36,14 @@ xsl_transform() {
       set -- "$@" "-s:${source_file}"
     fi
 
-    java -cp "$classpath" net.sf.saxon.Transform "$@" "${extra_params[@]}"
-
-    if [ "$?" -ne 0 ]; then
+    result=$(java -cp "$classpath" net.sf.saxon.Transform "$@" "${extra_params[@]}" 2>&1)
+    cmd_exitcode=$?
+    if [ $cmd_exitcode -ne 0 ]; then
+        echo -e "${P_ERROR}${result}${P_END}"
         echo -e "${P_ERROR}Error running Saxon.${P_END}"
-        return 3
+        return 3;
+    else
+        echo -e "${result}"
     fi
     return 0
 }