Merge pull request #60 from ndw/iss-57-improve-docs

Major refactor of test markup
xproc · Oct 10, 2018 · a6325eb · a6325eb
2 parents b71b79d + 6d0fbc9
commit a6325eb
Show file tree

Hide file tree

Showing 374 changed files with 15,143 additions and 7,079 deletions.
diff --git a/README.md b/README.md
@@ -20,34 +20,104 @@ This repository is organized in the following way:
         └── …                    # Individual tests
 ```
 
-Tests must indicate whether they are expected to pass or fail. The
-simplest structure for a passing test is:
+## Test format
+
+Tests must conform to the `schema/test-suite.rnc` and
+`schema/test-suite.sch` schemas. Most of the constraints are in the
+RELAX NG grammar but a few co-constraints are enforced in the
+Schematron.
+
+The basic structure of a passing test is:
 
 ```
-<test xmlns='http://xproc.org/ns/testsuite/3.0' expected="pass">
-  <pipeline>…</pipeline>
-  <schematron>…</schematron>
-</test>
+<t:test xmlns:t="http://xproc.org/ns/testsuite/3.0"
+        xmlns="http://www.w3.org/1999/xhtml"
+        expected="pass">
 ```
 
-The simplest structure for a failing test is:
+A passing test must assert that the expected result is a pass. Failing
+tests are described below.
+
+The first element in the `test` must be an `info` that includes metadata
+about the test.
 
 ```
-<test xmlns='http://xproc.org/ns/testsuite/3.0'
-      xmlns:err='http://www.w3.org/ns/xproc-error’
-      expected="fail" code=”err:code>
-  <pipeline>…</pipeline>
-</test>
+  <t:info>
+    <t:title>Test Title</t:title>
+    <t:revision-history>
+      <t:revision>
+        <t:date>2018-02-02T17:42:37+01:00</t:date>
+        <t:author>
+          <t:name>Author Name</t:name>
+        </t:author>
+        <t:description>
+          <p>Some description of this revision.</p>
+        </t:description>
+      </t:revision>
+    </t:revision-history>
+   </t:info>
+```
+
+If there are multiple revisions, they should be in date order.
+
+Followed by a description of the test.
+
+```
+   <t:description>
+      <p>Some description of the test.</p>
+   </t:description>
+```
+
+Note that the `description` element must contain some fragment(s) of HTML.
+
+Following the metadata and description is the test pipeline followed,
+optionally, by a Schematron schema to validate the test results.
+
+```
+   <t:pipeline>
+      <p:declare-step xmlns="http://xproc.org/ns/testsuite/3.0" version="3.0">
+         <p:output port="result">
+            <p:pipe step="producer" port="result"/>
+         </p:output>
+      
+         <p:identity name="producer">
+            <p:with-input port="source">
+               <doc xmlns=""/>
+            </p:with-input>
+         </p:identity>
+      </p:declare-step>
+   </t:pipeline>
+   <t:schematron src="../schematron/some-schema.sch"/>
+</t:test>
 ```
 
+A failing test is much the same, except that it must assert that the
+expected result is a fail and provide a (list of) error codes that may
+be reported by a conformant processor.
+
+
+```
+<t:test xmlns:t="http://xproc.org/ns/testsuite/3.0"
+        xmlns="http://www.w3.org/1999/xhtml"
+        xmlns:err="http://www.w3.org/ns/xproc-error"
+        expected="fail"
+        code="err:XS0038">
+  …
+</t:test>  
+```
+
+A failing test may not include a schematron schema to validate the
+test results.
+
 Tests can also include inputs and options:
 
 ```
-<test xmlns='http://xproc.org/ns/testsuite/3.0' expected="pass">
-  <input port='source'>…</input>
-  <option name='optname' select='3'/>
-  <pipeline>…</pipeline>
-  <schematron>…</schematron>
+<t:test xmlns:t='http://xproc.org/ns/testsuite/3.0' expected="pass">
+  <t:info>…</t:info>
+  <t:input port='source'>…</t:input>
+  <t:option name='optname' select='3'/>
+  <t:pipeline>…</t:pipeline>
+  <t:schematron>…</t:schematron>
 </test>
 ```
 
@@ -67,6 +137,43 @@ and tested.
 A group of tests can be combined into a `test-suite` and tests may be divided into
 sections using `div`s.
 
+### Revision histories
+
+In the case of revisions, it is somewhat inconvenient to have to
+repeat the same author information over and over again. If the same
+author is responsible for multiple revisions, a syntacit shortcut is
+possible:
+
+1. On the first `t:author`, specify the author initials.
+2. On a subsequent revision by that author, you can use the
+   `initials` attribute on the `t:revision` as a shortcut for
+   the author.
+
+For example:
+
+```
+<t:revision-history>
+
+<t:revision>
+    <t:date>2018-02-02</t:date>
+    <t:author initials="fa">
+      <t:name>First Author</t:name>
+    </t:author>
+    <t:description>
+      <p>Some description of this revision.</p>
+    </t:description>
+  </t:revision>
+
+  <t:revision initials=”fa”>
+    <t:date>2018-03-14</t:date>
+    <t:description>
+      <p>Some description of this revision.</p>
+    </t:description>
+  </t:revision>
+
+</t:revision-history>
+```
+
 ## Executing tests
 
 The program that runs tests should accept a list of file or directory names
@@ -96,4 +203,3 @@ error if the expression refers to the context.
 ## Test report format
 
 Tests should output the JUnit XML test result format.
-
diff --git a/examples/passing.xml b/examples/passing.xml
@@ -0,0 +1,47 @@
+<t:test xmlns:t="http://xproc.org/ns/testsuite/3.0"
+        xmlns="http://www.w3.org/1999/xhtml"
+        expected="pass">
+
+<t:info>
+  <t:title>Connections 001</t:title>
+  <t:revision-history>
+    <t:revision>
+      <t:date>2018-10-08</t:date>
+      <t:author initials="ndw">
+        <t:name>Norman Walsh</t:name>
+        <t:email>ndw@nwalsh.com</t:email>
+        <t:uri>https://nwalsh.com/</t:uri>
+      </t:author>
+      <t:description>
+        <p>Initial creation</p>
+      </t:description>
+    </t:revision>
+    <t:revision initials="ndw">
+      <t:date>2018-10-09</t:date>
+      <t:description>
+        <p>Updated error code.</p>
+      </t:description>
+    </t:revision>
+  </t:revision-history>
+</t:info>
+
+<t:description>
+  <p>Tests that an implicit primary output port is auto connected to
+  the output of the last step of the subpipeline.</p>
+</t:description>
+
+<t:pipeline>
+  <p:declare-step version="3.0" xmlns:p="http://www.w3.org/ns/xproc">
+    <p:output port="result"/>
+
+    <p:identity>
+      <p:with-input port="source">
+        <doc xmlns=""/>
+      </p:with-input>
+    </p:identity>
+  </p:declare-step>
+</t:pipeline>
+
+<t:schematron src="../schematron/ab-doc-result.sch"/>
+
+</t:test>
diff --git a/schema/test-suite.rnc b/schema/test-suite.rnc
@@ -1,68 +1,88 @@
 namespace rng  = "http://relaxng.org/ns/structure/1.0"
 namespace t = "http://xproc.org/ns/testsuite/3.0"
+namespace h = "http://www.w3.org/1999/xhtml"
 default namespace = "http://xproc.org/ns/testsuite/3.0"
 
 start = test | test-suite | test-div
 
+testattr = attribute features {
+               list { xsd:token+ }
+           } ?
+           & attribute when { text }?
+           & attribute src { text }?
+           & attribute version { text }?
+
 test-suite =
     element t:test-suite {
-        attribute features {
-            list { xsd:token+ }
-        } ?
-      & attribute when { text }?
-      & attribute src { text }?
-      & attribute version { text }?
-      & ((info|title)?, test*, test-div*)
+        testattr,
+        (info, description?, test*, test-div*)
+    }
+
+passingTest =
+    element t:test {
+        testattr
+      & attribute step { text }?
+      & attribute expected { "pass" }
+      & (info, description?, (input* & option* & pipeline? & schematron?))
+    }
+
+failingTest =
+    element t:test {
+        testattr
+      & attribute step { text }?
+      & attribute expected { "fail" }
+      & attribute code { xsd:NMTOKENS }
+      & (info, description?, (input* & option* & pipeline?))
     }
 
 test-div =
     element t:div {
-        (info|title)?, test*, test-div*
+        (info, description?, test*, test-div*)
     }
 
 info =
     element t:info {
-        title & author? & date? & specref?
+        title
+        & revision-history
+        & specref?
     }
 
-title =
-    element title { text }
-
 author =
-    element author { text }
+    element t:author {
+        attribute initials { text }?
+      & name?
+      & email?
+      & uri?
+    }
+
+name = element t:name { text }
+email = element t:email { text }
+uri = element t:uri { xsd:anyURI }
+
+revision-history =
+    element t:revision-history {
+        revision+
+    }
+
+revision =
+    element t:revision {
+        attribute initials { text }?,
+        date,
+        author?,
+        description
+    }
+
+title =
+    element t:title { text }
 
 date =
-    element date { xsd:date|xsd:dateTime }
+    element t:date { xsd:date|xsd:dateTime }
 
 specref =
-    element specref { text }
+    element t:specref { text }
 
 test = passingTest | failingTest
 
-passingTest =
-    element t:test {
-        attribute features {
-            list { xsd:token+ }
-        } ?
-      & attribute when { text }?
-      & attribute src { text }?
-      & attribute step { text }?
-      & attribute expected { "pass" }
-      & ((info|title)? & description? & input* & option* & pipeline? & schematron?)
-    }
-
-failingTest =
-    element t:test {
-        attribute features {
-            list { xsd:token+ }
-        } ?
-      & attribute when { text }?
-      & attribute src { text }?
-      & attribute step { text }?
-      & attribute expected { "fail" }
-      & attribute code { xsd:NMTOKENS }
-      & ((info|title)? & description? & input* & option* & pipeline?)
-    }
 
 pipeline =
     element t:pipeline {
@@ -96,7 +116,13 @@ any =
       & (text | any)*
     }
 
+anyhtml =
+    element h:* {
+        attribute * { text }*
+      & (text | anyhtml)*
+    }
+
 description =
-    element t:description{
-        any*
+    element t:description {
+        anyhtml+
     }
diff --git a/schema/test-suite.sch b/schema/test-suite.sch
@@ -12,6 +12,20 @@
     </s:rule>
   </s:pattern>
 
+  <s:pattern>
+    <s:title>Initials must be defined somewhere</s:title>
+    <s:rule context="t:revision[@initials]">
+      <s:assert test="//t:author[@initials=current()/@initials]">Initials must be defined</s:assert>
+    </s:rule>
+  </s:pattern>
+
+  <s:pattern>
+    <s:title>Name or initials must be provided</s:title>
+    <s:rule context="t:revision">
+      <s:assert test="@initials or t:author/t:name">Name or initials must be provided</s:assert>
+    </s:rule>
+  </s:pattern>
+
   <!-- It would be nice to test that t:test/@code contains valid error codes...
        but I can't figure out how to make that work in Schematron. I also suspect
        that XML Calabash isn't supporting Schematron perfectly, but that's a

diff --git a/test-suite/.DS_Store b/test-suite/.DS_Store