Major code cleanup, improved validation reporting, and NIMAS support

While working on NIMAS support I stumbled on a lot of side issues,
which led to a big code cleanup and various improvements to validation
related steps. See the individual commit messages for more details.

There are actually only very few changes I did for NIMAS. After all,
NIMAS is almost identical to DTBook. The main difference is that the
`meta` element must be empty in NIMAS. For this reason, when the input
is NIMAS (which you can indicate with a new option for the
dtbook-to-zedai, dtbook-to-html and dtbook-to-epub3 scripts), no MODS
file is generated in the `px:dtbook-to-zedai` step because it would
otherwise be empty and therefore invalid.

I also fixed one thing in the NIMAS schema that Nicole said was wrong.

I found various issues in `px:dtbook-to-zedai` that can result in an
invalid ZedAI output (for a valid DTBook input). These issues are not
related to NIMAS, and are not addressed yet. For now I disabled the
validation of the intermediary ZedAI in `px:dtbook-to-html` and
`px:dtbook-to-epub3`, because the validation issues in the ZedAI do
not necessarily result in bad HTML so the user should not be bothered
with it.

The reported issues in dtbook-to-rtf and dtbook-to-odt were not
addressed yet. It is quite likely that these are not related to NIMAS
either.

see #58

bom/pom.xml

@@ -117,7 +117,7 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>common-utils</artifactId>
-        <version>2.1.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
@@ -127,7 +127,7 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>fileset-utils</artifactId>
-        <version>5.2.1</version>
+        <version>5.3.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
@@ -152,7 +152,7 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>validation-utils</artifactId>
-        <version>1.0.10</version>
+        <version>2.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
@@ -230,12 +230,12 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>dtbook-utils</artifactId>
-        <version>3.1.1-SNAPSHOT</version>
+        <version>4.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>epub-utils</artifactId>
-        <version>1.2.5-SNAPSHOT</version>
+        <version>2.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
@@ -250,7 +250,7 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>smil-utils</artifactId>
-        <version>3.0.0</version>
+        <version>3.0.1-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
@@ -265,7 +265,7 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>zedai-utils</artifactId>
-        <version>1.1.3</version>
+        <version>1.2.0-SNAPSHOT</version>
       </dependency>
 
       <!--
@@ -299,7 +299,7 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>daisy3-to-epub3</artifactId>
-        <version>2.0.9-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
@@ -314,12 +314,12 @@
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>dtbook-to-epub3</artifactId>
-        <version>2.2.4-SNAPSHOT</version>
+        <version>2.3.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>
         <artifactId>dtbook-to-html</artifactId>
-        <version>3.0.2-SNAPSHOT</version>
+        <version>4.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.daisy.pipeline.modules</groupId>

common/common-utils/pom.xml

@@ -11,7 +11,7 @@
   </parent>
 
   <artifactId>common-utils</artifactId>
-  <version>2.1.0-SNAPSHOT</version>
+  <version>3.0.0-SNAPSHOT</version>
   <packaging>bundle</packaging>
 
   <name>DAISY Pipeline 2 module :: Common Utilities</name>

common/common-utils/src/main/java/org/daisy/pipeline/common/calabash/impl/Error.java

@@ -33,8 +33,9 @@ public XProcStep newStep(XProcRuntime runtime, XAtomicStep step) {
 	}
 
 	/**
-	 * Own version of <code>p:error</code> that reads a <code>c:errors</code> document, creates
-	 * {@link XProcException}s from it, reports them, and throws the last one.
+	 * Own version of <code>p:error</code> that reads one or more <code>c:errors</code> documents
+	 * with one or more <code>c:error</code> elements in each, creates {@link XProcException}s,
+	 * reports them, and throws the last one.
 	 */
 	public static class ErrorStep extends DefaultStep implements XProcStep {
 
@@ -73,7 +74,7 @@ public void run() throws SaxonApiException {
 				if (lastError != null)
 					throw lastError;
 				else
-					throw new RuntimeException("Expected valid c:errors document with at least one c:error");
+					throw new RuntimeException("Expected c:errors document with at least one c:error");
 			} catch (Throwable e) {
 				throw XProcStep.raiseError(e, step);
 			}

common/common-utils/src/main/java/org/daisy/pipeline/common/calabash/impl/ErrorReporter.java

@@ -31,7 +31,7 @@ public class ErrorReporter implements XMLTransformer {
 
 	@Override
 	public Runnable transform(Map<QName,InputValue<?>> input, Map<QName,OutputValue<?>> output) {
-		input = XMLTransformer.validateInput(input, ImmutableMap.of(_SOURCE, InputType.MANDATORY_NODE_SINGLE));
+		input = XMLTransformer.validateInput(input, ImmutableMap.of(_SOURCE, InputType.MANDATORY_NODE_SEQUENCE));
 		output = XMLTransformer.validateOutput(output, null);
 		XMLInputValue<?> source = (XMLInputValue<?>)input.get(_SOURCE);
 		return () -> report(source.asXMLStreamReader());

common/common-utils/src/main/resources/xml/xproc/assert.xpl

@@ -24,13 +24,11 @@
         <p:pipe port="result" step="result"/>
     </p:output>
 
-    <p:option name="test" select="''" cx:as="xs:string"/>                           <!-- boolean -->
-    <p:option name="test-count-min" select="''"/>                                   <!-- positive integer -->
-    <p:option name="test-count-max" select="''"/>                                   <!-- positive integer -->
-    <p:option name="error-code" select="''"/>                                       <!-- QName - if not given, only a warning will be displayed. -->
-    <p:option name="error-code-prefix" select="''"/>                                <!-- NCName -->
-    <p:option name="error-code-namespace" select="''"/>                             <!-- anyURI -->
-    <p:option name="message" required="true" cx:as="xs:string"/>                    <!-- description of what you are asserting. $1, $2 etc will be replaced with the contents of param1, param2 etc. -->
+    <p:option name="test" select="''" cx:as="xs:string"/>             <!-- boolean -->
+    <p:option name="test-count-min" select="''"/>                     <!-- positive integer -->
+    <p:option name="test-count-max" select="''"/>                     <!-- positive integer -->
+    <p:option name="error-code" required="false" cx:as="xs:QName"/>   <!-- if not given, only a warning will be displayed. -->
+    <p:option name="message" required="true" cx:as="xs:string"/>      <!-- description of what you are asserting. $1, $2 etc will be replaced with the contents of param1, param2 etc. -->
     <p:option name="param1" select="''" cx:as="xs:string"/>
     <p:option name="param2" select="''" cx:as="xs:string"/>
     <p:option name="param3" select="''" cx:as="xs:string"/>
@@ -135,13 +133,11 @@
             </p:identity>
         </p:when>
 
-        <p:when test="not($error-code='')">
+        <p:when test="p:value-available('error-code')">
             <!-- assertion failed; throw error -->
             <px:error>
                 <p:with-option name="message" select="/*/@message"/>
                 <p:with-option name="code" select="$error-code"/>
-                <p:with-option name="code-namespace" select="$error-code-namespace"/>
-                <p:with-option name="code-prefix" select="$error-code-prefix"/>
             </px:error>
         </p:when>
 

common/common-utils/src/main/resources/xml/xproc/error.xpl

@@ -16,30 +16,37 @@
         </pre>
     </p:documentation>
 
+    <p:input port="source" primary="true" sequence="true">
+        <p:documentation xmlns="http://www.w3.org/1999/xhtml">
+            <p>Documents on this port will be ignored.</p>
+        </p:documentation>
+        <p:empty/>
+    </p:input>
     <p:input port="error" primary="false" sequence="true">
         <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-            <p>If a <code>c:errors</code> document is supplied on this port, the errors will be
-            reported and the last error will be raised.</p>
+            <p>If one or more <code>c:errors</code> documents are supplied on this port, the errors
+            will be reported and the last error will be raised.</p>
         </p:documentation>
         <p:empty/>
     </p:input>
     <p:output port="result" sequence="true">
+        <p:documentation xmlns="http://www.w3.org/1999/xhtml">
+            <p>Nothing can ever appear on this port since the step will always fail.</p>
+        </p:documentation>
         <p:pipe port="result" step="error"/>
     </p:output>
 
-    <p:option name="code" select="''" cx:as="xs:string">     <!-- QName -->
+    <p:option name="code" required="false" cx:as="xs:QName">
         <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-            Must be set if there is a document on the "error" port. Must not be set if there is no
-            document on the "error" port.
+            Must be set if there are no documents on the "error" port. Must not be set if there are
+            documents on the "error" port.
         </p:documentation>
     </p:option>
-    <p:option name="code-prefix" select="''"/>                <!-- NCName -->
-    <p:option name="code-namespace" select="''"/>             <!-- anyURI -->
-    <p:option name="message" select="''" cx:as="xs:string">
+    <p:option name="message" required="false" cx:as="xs:string">
         <p:documentation xmlns="http://www.w3.org/1999/xhtml">
             Description of the error that occured. $1, $2 etc will be replaced with the contents of
             optios "param1", "param2", etc. Must be set if there is a document on the "error"
-            port. Must not be set if there is no document on the "error" port.
+            port. Must not be set if there are documents on the "error" port.
         </p:documentation>
     </p:option>
     <p:option name="param1" select="''" cx:as="xs:string"/>
@@ -54,9 +61,9 @@
     <!-- in the unlikely event that you need more parameters you'll have to format the message string yourself -->
 
     <p:declare-step type="pxi:error">
-        <p:input port="error" sequence="false">
+        <p:input port="error" sequence="true">
             <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-                <p>A <code>c:errors</code> document.</p>
+                <p>One or more <code>c:errors</code> documents.</p>
             </p:documentation>
         </p:input>
         <p:output port="result" sequence="true"/>
@@ -68,26 +75,19 @@
         </p:input>
     </p:count>
     <p:choose>
-        <p:when test="number(/*)&gt;1">
-            <p:error code="XXX">
-                <p:input port="source">
-                    <p:inline><message>Not more than one document allowed on the "error" port.</message></p:inline>
-                </p:input>
-            </p:error>
-        </p:when>
-        <p:when test="number(/*)=1">
+        <p:when test="number(/*)&gt;=1">
             <p:choose>
-                <p:when test="$message!=''">
+                <p:when test="p:value-available('message')">
                     <p:error code="XXX">
                         <p:input port="source">
-                            <p:inline><message>"message" option must not be set if there is a document on the "error" port.</message></p:inline>
+                            <p:inline><message>"message" option must not be set if there are documents on the "error" port.</message></p:inline>
                         </p:input>
                     </p:error>
                 </p:when>
-                <p:when test="$code!=''">
+                <p:when test="p:value-available('code')">
                     <p:error code="XXX">
                         <p:input port="source">
-                            <p:inline><message>"code" option must not be set if there is a document on the "error" port.</message></p:inline>
+                            <p:inline><message>"code" option must not be set if there are documents on the "error" port.</message></p:inline>
                         </p:input>
                     </p:error>
                 </p:when>
@@ -100,23 +100,21 @@
                 </p:otherwise>
             </p:choose>
         </p:when>
-        <p:when test="$message=''">
+        <p:when test="not(p:value-available('message'))">
             <p:error code="XXX">
                 <p:input port="source">
                     <p:inline><message>"message" option must be set</message></p:inline>
                 </p:input>
             </p:error>
         </p:when>
-        <p:when test="$code=''">
+        <p:when test="not(p:value-available('code'))">
             <p:error code="XXX">
                 <p:input port="source">
                     <p:inline><message>"code" option must be set</message></p:inline>
                 </p:input>
             </p:error>
         </p:when>
         <p:otherwise>
-            <p:variable name="code-localName" select="if (contains($code,':')) then substring-after($code,':') else $code"/>
-            <p:variable name="prefix" select="concat(substring-before($code,':'),$code-prefix)"/>
             <p:string-replace match="/*/text()" name="message">
                 <p:input port="source">
                     <p:inline><message>MESSAGE</message></p:inline>
@@ -136,44 +134,13 @@
                                                 '\$9',replace($param9,'\$','\\\$')),
                                                 '&quot;')"/>
             </p:string-replace>
-            <p:choose>
-                <p:when test="not($code-namespace='') and not($prefix='')">
-                    <p:error>
-                        <p:input port="source">
-                            <p:pipe port="result" step="message"/>
-                        </p:input>
-                        <p:with-option name="code" select="$code-localName"/>
-                        <p:with-option name="code-namespace" select="$code-namespace"/>
-                        <p:with-option name="code-prefix" select="$prefix"/>
-                    </p:error>
-                </p:when>
-                <p:when test="not($code-namespace='')">
-                    <p:error>
-                        <p:input port="source">
-                            <p:pipe port="result" step="message"/>
-                        </p:input>
-                        <p:with-option name="code" select="$code-localName"/>
-                        <p:with-option name="code-namespace" select="$code-namespace"/>
-                    </p:error>
-                </p:when>
-                <p:when test="not($prefix='')">
-                    <p:error>
-                        <p:input port="source">
-                            <p:pipe port="result" step="message"/>
-                        </p:input>
-                        <p:with-option name="code" select="$code-localName"/>
-                        <p:with-option name="code-prefix" select="$prefix"/>
-                    </p:error>
-                </p:when>
-                <p:otherwise>
-                    <p:error>
-                        <p:input port="source">
-                            <p:pipe port="result" step="message"/>
-                        </p:input>
-                        <p:with-option name="code" select="$code-localName"/>
-                    </p:error>
-                </p:otherwise>
-            </p:choose>
+            <p:sink/>
+            <p:error>
+                <p:input port="source">
+                    <p:pipe port="result" step="message"/>
+                </p:input>
+                <p:with-option name="code" select="$code"/>
+            </p:error>
         </p:otherwise>
     </p:choose>
     <p:identity name="error"/>

common/common-utils/src/main/resources/xml/xproc/log-error.xpl

@@ -1,12 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <p:declare-step xmlns:p="http://www.w3.org/ns/xproc" version="1.0"
                 xmlns:px="http://www.daisy.org/ns/pipeline/xproc"
+                xmlns:cx="http://xmlcalabash.com/ns/extensions"
                 exclude-inline-prefixes="#all"
                 type="px:log-error">
 
     <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-        <p>Log a caught XProc error.</p>
-        <p>Logs a message in the job execution log and prints the stack trace in the detailed
+        <p>Log caught XProc errors.</p>
+        <p>Logs messages in the job execution log and prints the stack trace in the detailed
         log.</p>
     </p:documentation>
 
@@ -15,19 +16,20 @@
             <p>Any document sequence.</p>
         </p:documentation>
     </p:input>
-    <p:input port="error" sequence="false">
+    <p:input port="error" sequence="true">
         <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-            <p>A <code>c:errors</code> document.</p>
+            <p>Zero or more <a
+            href="https://www.w3.org/TR/xproc/#cv.errors"><code>c:errors</code></a> documents.</p>
         </p:documentation>
     </p:input>
-    <p:option name="severity" select="'ERROR'">
+    <p:option name="severity" select="'ERROR'" cx:type="ERROR|WARN|INFO|DEBUG|TRACE">
         <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-            <p>Severity of the log message</p>
+            <p>Severity of the log messages</p>
         </p:documentation>
     </p:option>
     <p:output port="result" sequence="true">
         <p:documentation xmlns="http://www.w3.org/1999/xhtml">
-            <p>Copy of source.</p>
+            <p>Copy of <code>source</code>.</p>
         </p:documentation>
     </p:output>
 

common/common-utils/src/test/xprocspec/error.xprocspec

@@ -15,14 +15,12 @@
     </x:context>
     <x:expect label="the error" type="compare">
       <x:document type="inline">
-        <c:error code="ERR" name="!1.19.5.4.4.1" type="p:error" href="src/main/resources/xml/xproc/error.xpl" line="169" column="30">
+        <c:error code="ERR" name="!1.18.4.3" type="p:error" href="src/main/resources/xml/xproc/error.xpl" line="138" column="22">
           <message>foobar</message>
           <px:location>
-            <px:file href="src/main/resources/xml/xproc/error.xpl" line="169" column="30"/>
-            <px:file href="src/main/resources/xml/xproc/error.xpl" line="168" column="30"/>
-            <px:file href="src/main/resources/xml/xproc/error.xpl" line="139" column="23"/>
+            <px:file href="src/main/resources/xml/xproc/error.xpl" line="138" column="22"/>
             <px:file href="src/main/resources/xml/xproc/error.xpl" line="117" column="22"/>
-            <px:file href="src/main/resources/xml/xproc/error.xpl" line="70" column="15"/>
+            <px:file href="src/main/resources/xml/xproc/error.xpl" line="77" column="15"/>
             <!-- these lines can change if the XProcSpec implementation changes! -->
             <px:file href="src/test/xprocspec/error.xprocspec" line="10" column="98"/>
             <px:file href="src/test/xprocspec/error.xprocspec" line="9" column="18"/>

common/fileset-utils/pom.xml

@@ -11,7 +11,7 @@
   </parent>
 
   <artifactId>fileset-utils</artifactId>
-  <version>5.2.2-SNAPSHOT</version>
+  <version>5.3.0-SNAPSHOT</version>
   <packaging>bundle</packaging>
 
   <name>DAISY Pipeline 2 module :: File Set Utilities</name>