test-catalog.rnc

default namespace = "https://github.com/cmsmcq/ixml-tests"
namespace unqualified = ""

grammar {

	# RNC grammar for test catalog.
	#
	# Revisions:
	# 2022-04-11 : CMSMcQ : Add dynamic-error as expected result
	# 2022-02-14 : CMSMcQ : Move metadata from attributes to elements
	# 2022-02-06 : CMSMcQ : Add a quick and dirty report format.
	# 2021-12-22 : CMSMcQ : Make 'created' optional on individual tests;
	#                       notionally, let it be inherited from test set.
	# 2021-11-11 : CMSMcQ : Revamp result to allow multiple results
	#                       and include assert-not-a-grammar.
	#                       Rewrite some comments.
	# 2021-10-31 : CMSMcQ : Commit some changes:  @name on test-case,
	#                       allow at most one grammar for each test
	#                       set (grammars may be inherited from ancestor
	#                       test sets).
	# 2021-01-25 : CMSMcQ : Sketch this out by hand.
        #
	# To do:
	# - rewrite test-set to allow test cases only if a grammar is
	#   specified on the test-set or some ancestor.
	# - allow description to be (p+ | xhtml:div+) HTML
	# - supply types for tokenized attributes?
	#

# Notational convention:  definitions starting in uppercase (e.g.
# Metadata, Grammar-spec are for content-model expressions.
# Definitions starting in lowercase (e.g. test-catalog) are for
# individual elements, usually with the same name as the element.
#
# (Exception:  element test-set has two definitions, test-set-0
# and test-set-1.)

# The normal starting points are test-catalog and test-report.
# But to allow individual test sets and tests to be reported
# separately, we also allow lower-level result elements as the
# start symbol.

	start = test-catalog | test-report 
              | test-set-results | grammar-result | test-result

# test-catalog, test-report

	# A test catalog is a collection of test sets, with common
	# metadata.
	test-catalog = element test-catalog {
	    attribute name { text },
	    attribute release-date { xsd:date },
	    external-atts,
	    (Metadata
	    &
	    (test-set-0 | test-set-ref)*)
	}

	# A test report is a collection of test set reports, with common
	# metadata.
	test-report = element test-report {
            element metadata { 
    	        (element name { text },
	        element report-date { xsd:date | xsd:dateTime },
	        element processor { text },
	    	element processor-version { text }?,
	    	element catalog-uri { text },
	    	element catalog-date { text }?)
                & 
                Metadata
            },
	    external-atts,
	    (Metadata
	    &
	    test-set-results*)
	}

# Metadata

        # At various levels we allow metadata:
	# prose descriptions, pointers to external
	# documentation, or arbitrary XML elements
	# ('application-specific information').
	
        Metadata = (description | app-info | doc)*

        description = element description {
	  external-atts,
	  p*
	}

        app-info = element app-info {
	  external-atts,
	  any-element*
	}

        doc = element doc {
	  external-atts,
	  attribute href { xsd:anyURI }
	}
	

# test-set, test-set-results

        # A test set is a collection of tests (or possibly subordinate
	# test sets, or both) with common metadata and a common
	# grammar.

        # Test cases are allowed only after a grammar is specified.
	
        # We keep track of whether an ancestor has specified a grammar
	# by having two nonterminals for test sets:  test-set-0 is used
	# when no ancestor has specified a grammar, test-set-1 when
	# at least one grammar has been specified.

        # If no ancestor has specified a grammar, test cases are allowed
	# in this test set only if this test set does specify a grammar.
	# Use test-set-0 or -1 to pass the news along.
	
        test-set-0 = element test-set {
	    attribute name { text },
	    external-atts,
	    (Metadata
	    &
	    (History,
	    ( (test-set-0 | test-set-ref)*
	    | (Grammar-spec, (test-set-1 | test-set-ref | test-case)*) )))
	}
	
        # If an ancestor has specified a grammar, test cases are allowed
	# in this test set even if there is no grammar at this level.
	
        test-set-1 = element test-set {
	    attribute name { text },
	    external-atts,
	    (Metadata
	    &
	    (History,
	    Grammar-spec?,
	    (test-set-1 | test-set-ref | test-case)*))
	}

        test-set-results = element test-set-results {
	    attribute name { text },
	    external-atts,
	    (Metadata
	    &
	    (Grammar-results?,
	    (test-set-results | test-result)*))
	}

        # Grammars can be in invisible XML or in visible XML.
	# They can be inline or external.  They can be marked
	# as a grammar test or not.
	
	Grammar-data = (ixml-grammar
	    | vxml-grammar
	    | ixml-grammar-ref
	    | vxml-grammar-ref)
	    
	Grammar-spec = (Grammar-data, grammar-test?)

        # In the results file, we may omit the grammar, or include
	# it, possibly both reproducing the reference and giving
	# the grammar inline. 
	Grammar-results = (Grammar-data*, grammar-result?)

        # Q. Why is the grammar optional?
	# A. Because in a nested test set we may want to inherit the
	#    grammar from the parent test set.  In a top-level test
	#    set with no direct test-case children, we may just be
	#    pointing to multiple test sets which each provide their
	#    own grammar.  By the time we reach a test case we must
	#    have at least one grammar, but we don't need on at every
	#    level.
        # Q. Why can't there be multiple grammars?
        # A. First, it's error prone:  it would work only if all of them
	#    were guaranteed equivalent.  We don't want to have to check
	#    that, and we don't want the mess that will result if it
	#    turns out not to be true.  Second, it complicates reporting
	#    unnecessarily.  It's simpler when one test case is one
	#    grammar + input + result triple.

	test-set-ref = element test-set-ref {
	    external-atts,
	    attribute href { xsd:anyURI }
	}

	# ixml-grammar:  grammar in invisible-XML form
	ixml-grammar = element ixml-grammar {
	    external-atts,
	    text
	}
	
	ixml-grammar-ref = element ixml-grammar-ref {
	    external-atts,
	    attribute href { xsd:anyURI }
	}

	# vxml-grammar:  grammar in visible-XML form (either a parsed
	# ixml grammar, translated into XML, or something created in
	# XML)
	#
	# N.B. It is tempting to embed a schema for ixml grammars here
	# to enforce the correct XML form.  But we do not require a
	# legal ixml grammar, because it may be a negative test case.
	
	vxml-grammar = element vxml-grammar {
	    external-atts,
	    any-element
	}
	
	vxml-grammar-ref = element vxml-grammar-ref {
	    external-atts,
	    attribute href { xsd:anyURI }
	}

	# grammar-test:  signals that this grammar should be checked
	# and either accepted or declined as a grammar.
	
	grammar-test = element grammar-test {
	    external-atts,
	    (Metadata & (History?, result))
	}

	grammar-result = element grammar-result {
	    attribute result { result-type }, 
	    external-atts,
	    (Metadata & (result-report?))
	}


# test-case

        # test-case:  describes one test case, with metadata, history,
	# and expected result.
	
        test-case = element test-case {
	    attribute name { text }, 
	    external-atts,
	    (Metadata & (History?, Test-string, result))
	}
	
        test-result = element test-result {
	    attribute name { text }, 
	    attribute result { result-type }, 
	    external-atts,
	    (Metadata & 
            (Grammar-data*, (Test-string)*, result-report?)
            )
	}
	
	result-type = 'pass' | 'fail' | 'not-run' | 'other'

        # Test-string:  in-line or external

        Test-string = (test-string | test-string-ref)
	
	test-string = element test-string {
	    external-atts,
	    text
	}

	test-string-ref = element test-string-ref {
	    external-atts,
	    attribute href { xsd:anyURI }
	}

# result

        # result: specifies the expected result of a test;
	# contains an assertion of some kind.
        result = element result {
	    external-atts,
	    Assertion
	}

        result-report = element result {
	    external-atts,
	    Assertion?,
	    Observation?
	}


# Test assertions

        # Several kinds of result are possible.
	#
	# - In the common case we will have one expected XML result.  We
	#   specify it with assert-xml or assert-xml-ref (inline or
	#   external).
	#
	# - For ambiguous sentences, we may and should specify several 
	#   XML results, any of which is acceptable.  So the XML
	#   assertions can repeat, with an implicit OR as their meaning.
	#
	# - In the case of infinite ambiguity, we can and should specify
	#   a finite subset of the expected results, which we add to as
	#   needed.
	#
	# - If the input is not be a sentence in the language defined
	#   by the grammar, we use assert-not-a-sentence.
	#
	# - If the grammar specified is not a conforming ixml grammar, 
	#   then we use assert-not-a-grammar.
	#
	# - If the particular grammar + input pair would produce 
	#   ill-formed output if the normal rules were followed, then
	#   we use assert-dynamic-error.
	#
	# Logically speaking, in the case of a grammar-test, there is no
	# useful distinction between assert-not-a-sentence and
	# assert-not-a-grammar.  Casuists can argue over which makes
	# more sense, but in practice they should be treated as
	# equivalent.  They are usefully different only for normal
	# test cases.
	#
	# Since dynamic errors are allowed to be caught statically,
	# some processors may return assert-not-a-grammar when the test
	# catalog expects assert-dynamic-error.
	#
	# Errors in the grammar and dynamic errors may be associated
	# with error codes.  For the moment, these are optional.

        Assertion = ((assert-xml-ref | assert-xml)+
	            | assert-not-a-sentence
	            | assert-not-a-grammar
		    | assert-dynamic-error)

        Error-Code = attribute error-code { text }?

        assert-xml-ref = element assert-xml-ref {
	    external-atts,
	    attribute href { xsd:anyURI }
	}
        assert-xml = element assert-xml {
	    external-atts,
	    any-element+
	}
        assert-not-a-sentence = element assert-not-a-sentence {
	    external-atts,
	    Metadata
	}
        assert-not-a-grammar = element assert-not-a-grammar {
	    Error-Code,
	    external-atts,
	    Metadata
	}
        assert-dynamic-error = element assert-dynamic-error {
	    Error-Code,
	    external-atts,
	    Metadata
	}

        Observation = ((reported-xml-ref | reported-xml)+
	            | reported-not-a-sentence
	            | reported-not-a-grammar
		    | reported-dynamic-error)
		    
        reported-xml-ref = element reported-xml-ref {
	    external-atts,
	    attribute href { xsd:anyURI }
	}
        reported-xml = element reported-xml {
	    external-atts,
	    any-element+
	}
        reported-not-a-sentence = element reported-not-a-sentence {
	    external-atts,
	    Metadata
	}
        reported-not-a-grammar = element reported-not-a-grammar {
	    Error-Code,
	    external-atts,
	    Metadata
	}
        reported-dynamic-error = element reported-dynamic-error {
	    Error-Code,
	    external-atts,
	    Metadata
	}
		    

# Common constructs

        # History:  creation and modification history
	History = (created, modified*)
	
        who-when = attribute by { text },
	    	   attribute on { xsd:date }

        created = element created {
	    who-when
	}
	
        modified = element modified {
	    who-when,
	    attribute change { text }
	}

        # Elements for simple prose.
	
	p = element p { phrases }
	
	phrases = (text | emph | code)*

	emph = element emph { phrases }

        code = element code { text }

        # Arbitrary XML

	anything = (any-element | any-attribute | text)* 
	any-element = element * { anything }
	any-attribute = attribute * { text }

        external-atts = nsq-att*

	nsq-att = attribute (* - unqualified:*) { text }

}