Update test README (#2000)

This removes the test suite specific commands as output is now written by default to the right files. Other changes: - Write EARL report to file suffixed with `-HEAD` without timestamps by default. This is to make it easier to keep the checked in test reports accurate. - Remove XFAILs for tests that are now passing.
RDFLib · Jun 24, 2022 · 8ccebc8 · 8ccebc8
1 parent eabab5f
commit 8ccebc8
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 64 deletions.
diff --git a/test/README.rst b/test/README.rst
@@ -46,60 +46,19 @@ test_conneg - test content negotiation when reading remote graphs
 EARL Test Reports
 =================
 
-EARL test reports can be generated using the EARL reporter plugin from ``earl.py``.
+EARL test reports are generated using the EARL reporter plugin from
+``test/utils/earl.py``.
 
-When this plugin is enabled it will create an ``earl:Assertion`` for every test that has a ``rdf_test_uri`` parameter which can be either a string or an ``URIRef``.
+This plugin is enabled by default and writes test reports to
+``test_reports/*-HEAD.ttl`` without timestamps by default.
 
-To enable the EARL reporter plugin an output file path must be supplied to pytest with ``--earl-output-file``. The report will be written to this location in turtle format.
+For EARL reporter plugin options see the output of ``pytest --help``.
 
-Some examples of generating test reports:
+To write reports with timestamps:
 
 .. code-block:: bash
 
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-jsonld-local.ttl \
-      test/jsonld/test_localsuite.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-jsonld-v1.1.ttl \
-      test/jsonld/test_onedotone.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-jsonld-v1.0.ttl \
-      test/jsonld/test_testsuite.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-sparql.ttl \
-      test/test_w3c_spec/test_sparql_w3c.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-nquads.ttl \
-      test/test_w3c_spec/test_nquads_w3c.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-nt.ttl \
-      test/test_w3c_spec/test_nt_w3c.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-trig.ttl \
-      test/test_w3c_spec/test_trig_w3c.py
-
-   pytest \
-      --earl-assertor-homepage=http://example.com \
-      --earl-assertor-name 'Example Name' \
-      --earl-output-file=/var/tmp/earl/earl-turtle.ttl \
-      test/test_w3c_spec/test_turtle_w3c.py
+    pytest \
+      --earl-add-datetime \
+      --earl-output-suffix=-timestamped \
+      test/test_w3c_spec/
diff --git a/test/test_w3c_spec/test_sparql11_w3c.py b/test/test_w3c_spec/test_sparql11_w3c.py
@@ -235,12 +235,6 @@
     f"{REMOTE_BASE_IRI}syntax-update-1/manifest#test_54": pytest.mark.xfail(
         reason="Parses sucessfully instead of failing."
     ),
-    f"{REMOTE_BASE_IRI}csv-tsv-res/manifest#csv01": pytest.mark.xfail(
-        reason="Bug in blank node serialization."
-    ),
-    f"{REMOTE_BASE_IRI}csv-tsv-res/manifest#csv02": pytest.mark.xfail(
-        reason="Bug in blank node serialization."
-    ),
 }
 
 

diff --git a/test/utils/earl.py b/test/utils/earl.py
@@ -121,8 +121,8 @@ def pytest_addoption(parser: _pytest.config.argparsing.Parser):
         action="store",
         dest="earl_output_suffix",
         metavar="path",
-        default="-latest",
-        help="suffix to use for prefix-defined test reports, defaults to '-latest'",
+        default="-HEAD",
+        help="suffix to use for prefix-defined test reports, defaults to '-HEAD'",
     )
 
     group.addoption(
@@ -162,10 +162,10 @@ def pytest_addoption(parser: _pytest.config.argparsing.Parser):
     )
 
     group.addoption(
-        "--earl-omit-datetime",
-        action="store_false",
+        "--earl-add-datetime",
+        action="store_true",
         dest="earl_add_datetime",
-        default=True,
+        default=False,
         help="Don't write datetime to result",
     )
 

diff --git a/test_reports/rdflib_w3c_sparql11-HEAD.ttl b/test_reports/rdflib_w3c_sparql11-HEAD.ttl
@@ -691,15 +691,15 @@
     earl:assertedBy <https://github.com/RDFLib/rdflib> ;
     earl:mode earl:automatic ;
     earl:result [ a earl:TestResult ;
-            earl:outcome earl:failed ] ;
+            earl:outcome earl:passed ] ;
     earl:subject <https://github.com/RDFLib/rdflib> ;
     earl:test <http://www.w3.org/2009/sparql/docs/tests/data-sparql11/csv-tsv-res/manifest#csv01> .
 
 [] a earl:Assertion ;
     earl:assertedBy <https://github.com/RDFLib/rdflib> ;
     earl:mode earl:automatic ;
     earl:result [ a earl:TestResult ;
-            earl:outcome earl:failed ] ;
+            earl:outcome earl:passed ] ;
     earl:subject <https://github.com/RDFLib/rdflib> ;
     earl:test <http://www.w3.org/2009/sparql/docs/tests/data-sparql11/csv-tsv-res/manifest#csv02> .