Fix some validator-specific crashes (#395)

* Give validator patched `included` a default value to make it really optional (closes #393)

* Validate relationships data as a list before testing type (closes #397)

* Improve --verbosity help string for validator (closes #396)

* Add concept of InternalErrors to validator

These errors should be handled differently to ValidationError and
ResponseError as they indicate a problem with the validator itself.

* Added validator flags --fail_fast and --skip_optional_tests

* Improvements to pagination validation

- Allow links objects to be passed in pagination
- Enforced maximum recursion depth (5) for pagination tests

* Reduce set of example queries from the specification, pending future improvements (#357)

* Improve error message for single entry endpoint if deserialization fails

* Implemented suggestions from code review

Co-authored-by: Casper Welzel Andersen <CasperWA@users.noreply.github.com>

* Renamed entrypoint optimade_validator->optimade-validator

* Flip arg options the "standard" way around

Use -t for --as-type (instead of -a).

Co-authored-by: Casper Welzel Andersen <casper.andersen@epfl.ch>
Co-authored-by: Andrius Merkys <andrius.merkys@gmail.com>

optimade/models/entries.py

@@ -20,7 +20,13 @@ class TypedRelationship(Relationship):
     # This may be updated when moving to Python 3.8
     @validator("data")
     def check_rel_type(cls, data):
-        if hasattr(cls, "_req_type") and any(obj.type != cls._req_type for obj in data):
+        if not isinstance(data, list):
+            # All relationships at this point are empty-to-many relationships in JSON:API:
+            # https://jsonapi.org/format/1.0/#document-resource-object-linkage
+            raise ValueError("`data` key in a relationship must always store a list.")
+        if hasattr(cls, "_req_type") and any(
+            getattr(obj, "type", None) != cls._req_type for obj in data
+        ):
             raise ValueError("Object stored in relationship data has wrong type")
         return data
 

optimade/validator/__init__.py

@@ -12,20 +12,20 @@ def validate():
     import traceback
 
     parser = argparse.ArgumentParser(
-        prog="optimade_validator",
+        prog="optimade-validator",
         description="""Tests OPTIMADE implementations for compliance with the optimade-python-tools models.
 
     - To test an entire implementation (at say example.com/optimade/v1) for all required/available endpoints:
 
-        $ optimade_validator http://example.com/optimade/v1
+        $ optimade-validator http://example.com/optimade/v1
 
     - To test a particular response of an implementation against a particular model:
 
-        $ optimade_validator http://example.com/optimade/v1/structures/id=1234 --as_type structure
+        $ optimade-validator http://example.com/optimade/v1/structures/id=1234 --as-type structure
 
     - To test a particular response of an implementation against a particular model:
 
-        $ optimade_validator http://example.com/optimade/v1/structures --as_type structures
+        $ optimade-validator http://example.com/optimade/v1/structures --as-type structures
         """,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -39,24 +39,39 @@ def validate():
         ),
     )
     parser.add_argument(
-        "--verbosity", "-v", type=int, default=0, help="The verbosity of the output"
+        "-v",
+        "--verbosity",
+        action="count",
+        default=0,
+        help="""Increase the verbosity of the output.""",
     )
     parser.add_argument(
-        "--as_type",
-        "-a",
+        "-t",
+        "--as-type",
         type=str,
         help=(
             "Validate the request URL with the provided type, rather than scanning the entire implementation e.g. "
-            "optimade_validator `http://example.com/optimade/v1/structures/0 --as_type structure`"
+            "optimade-validator `http://example.com/optimade/v1/structures/0 --as-type structure`"
         ),
     )
     parser.add_argument(
         "--index",
         action="store_true",
         help=(
-            "Flag for whether the specified OPTIMADE implementation is an Index meta-database or not. [default=False]"
+            "Flag for whether the specified OPTIMADE implementation is an Index meta-database or not."
         ),
     )
+    parser.add_argument(
+        "--skip-optional",
+        action="store_true",
+        help=("Flag for whether the skip the tests of optional features."),
+    )
+
+    parser.add_argument(
+        "--fail-fast",
+        action="store_true",
+        help="Whether to exit on first test failure.",
+    )
 
     args = vars(parser.parse_args())
 
@@ -84,6 +99,8 @@ def validate():
         verbosity=args["verbosity"],
         as_type=args["as_type"],
         index=args["index"],
+        run_optional_tests=not args["skip_optional"],
+        fail_fast=args["fail_fast"],
     )
 
     try:

optimade/validator/data/filters.txt

@@ -1,41 +1,15 @@
 _exmpl1_bandgap<2.0 OR _exmpl2_bandgap<2.5
-NOT ( chemical_formula_hill = "Al" AND chemical_formula_anonymous = "A" OR chemical_formula_anonymous = "H2O" AND NOT chemical_formula_hill = "Ti" )
+NOT ( chemical_formula_anonymous = "A" OR chemical_formula_anonymous = "H2O" )
 nelements > 3
-chemical_formula_hill = "H2O" AND chemical_formula_anonymous != "AB"
-_exmpl_aax <= +.1e8 OR nelements >= 10 AND NOT ( _exmpl_x != "Some string" OR NOT _exmpl_a = 7)
-_exmpl_spacegroup="P2"
-_exmpl_cell_volume<100.0
-_exmpl_bandgap > 5.0 AND _exmpl_molecular_weight < 350
-_exmpl_melting_point<300 AND nelements=4 AND elements="Si,O2"
-_exmpl_some_string_property = 42
-5 < _exmpl_a
-identifier CONTAINS x
-identifier STARTS WITH x
-identifier ENDS WITH x
 chemical_formula_anonymous CONTAINS "C2" AND chemical_formula_anonymous STARTS WITH "A2"
 chemical_formula_anonymous STARTS "B2" AND chemical_formula_anonymous ENDS WITH "D2"
-list HAS value
-list HAS ALL values
-list HAS ANY values
-list LENGTH value
-NOT list HAS inverse
-calculations.id HAS "calc-id-96"
-authors.lastname HAS "Schmit"
-identifier IS UNKNOWN
-NOT identifier IS KNOWN
-chemical_formula_hill IS KNOWN AND NOT chemical_formula_anonymous IS UNKNOWN
-NOT a > b OR c = 100 AND f = "C2 H6"
-(NOT (a > b)) OR ( (c = 100) AND (f = "C2 H6") )
-a >= 0 AND NOT b < c OR c = 0
-((a >= 0) AND (NOT (b < c))) OR (c = 0)
 elements HAS ALL "Si", "Al", "O"
 elements HAS ALL "Si", "Al", "O" AND elements LENGTH 3
 nelements=4
 nelements>=2 AND nelements<=7
 chemical_formula_descriptive="(H2O)2 Na"
 chemical_formula_descriptive CONTAINS "H2O"
 chemical_formula_reduced="H2NaO"
-chemical_formula_hill="H2O2"
 chemical_formula_anonymous="A2B"
 nsites=4
 nsites>=2 AND nsites<=7