Update filter examples and validate optional cases

optimade/validator/data/__init__.py

@@ -1,13 +1,9 @@
 """ This submodule loads the filter examples that have been scraped
 from the specification.
 
-- `filters.txt` is a automatically generated file containing all
-examples. It is generated using the `invoke` task `parse_spec_for_filters`.
-- `optional_filters.txt` is human-generated, and allows for filters that
-are inside `filters.txt` to be skipped, if it has been deemed that they
-are optional (external to the parsed file). In the future, it is hoped
-that the spec will be fully parseable and as such this file could be
-generated automatically too.
+- Both `filters.txt` and `optional_filters.txt.` are automatically
+generated files containing all examples from the specification. It
+is generated using the `invoke` task `parse_spec_for_filters`.
 
 When being loaded in, these filter examples are also made more concrete
 by replacing references to vague `values`/`value` with specific examples.
@@ -16,21 +12,37 @@
 
 from pathlib import Path
 
-__all__ = ["MANDATORY_FILTER_EXAMPLES"]
+__all__ = ["MANDATORY_FILTER_EXAMPLES", "OPTIONAL_FILTER_EXAMPLES"]
 
 ALIASES = {"values": '"1", "2", "3"', "value": "1", "inverse": "1"}
 
-with open(Path(__file__).parent.joinpath("optional_filters.txt"), "r") as f:
-    OPTIONAL_FILTER_EXAMPLES = [line.strip() for line in f.readlines()]
 
-with open(Path(__file__).parent.joinpath("filters.txt"), "r") as f:
-    _filters = []
-    for line in f.readlines():
-        if line.strip() in OPTIONAL_FILTER_EXAMPLES:
-            continue
-        for alias in ALIASES:
-            if alias in line:
-                line = line.replace(alias, ALIASES[alias])
-        _filters.append(line)
+def _load_filters_and_apply_aliases(path):
+    """ Load a text file containing example filters with one
+    filter per line, and apply aliases to swap out dummy values
+    with more concrete examples.
 
-    MANDATORY_FILTER_EXAMPLES = _filters
+    Parameters:
+        path (str/Path): the filename to load.
+
+    Returns:
+        list: a list of filters.
+
+    """
+    with open(path, "r") as f:
+        _filters = []
+        for line in f.readlines():
+            for alias in ALIASES:
+                if alias in line:
+                    line = line.replace(alias, ALIASES[alias])
+            _filters.append(line)
+
+    return _filters
+
+
+OPTIONAL_FILTER_EXAMPLES = _load_filters_and_apply_aliases(
+    Path(__file__).parent.joinpath("optional_filters.txt").resolve()
+)
+MANDATORY_FILTER_EXAMPLES = _load_filters_and_apply_aliases(
+    Path(__file__).parent.joinpath("filters.txt").resolve()
+)

optimade/validator/data/filters.txt

@@ -9,8 +9,6 @@ _exmpl_bandgap > 5.0 AND _exmpl_molecular_weight < 350
 _exmpl_melting_point<300 AND nelements=4 AND elements="Si,O2"
 _exmpl_some_string_property = 42
 5 < _exmpl_a
-((NOT (_exmpl_a>_exmpl_b)) AND _exmpl_x>0)
-5 < 7
 identifier CONTAINS x
 identifier STARTS WITH x
 identifier ENDS WITH x
@@ -20,16 +18,7 @@ list HAS value
 list HAS ALL values
 list HAS ANY values
 list LENGTH value
-list HAS ONLY values
 NOT list HAS inverse
-list HAS < 3
-list HAS ALL < 3, > 3
-list:list HAS >=2:<=5
-elements HAS "H" AND elements HAS ALL "H","He","Ga","Ta" AND elements HAS ONLY "H","He","Ga","Ta" AND elements HAS ANY "H", "He", "Ga", "Ta"
-elements HAS ONLY "H","He","Ga","Ta"
-elements:_exmpl_element_counts HAS "H":6 AND elements:_exmpl_element_counts HAS ALL "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6 AND elements:_exmpl_element_counts HAS ANY "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6,"He":7
-_exmpl_element_counts HAS < 3 AND _exmpl_element_counts HAS ANY > 3, = 6, 4, != 8
-elements:_exmpl_element_counts:_exmpl_element_weights HAS ANY > 3:"He":>55.3 , = 6:>"Ti":<37.6 , 8:<"Ga":0
 calculations.id HAS "calc-id-96"
 authors.lastname HAS "Schmit"
 identifier IS UNKNOWN
@@ -43,7 +32,6 @@ elements HAS ALL "Si", "Al", "O"
 elements HAS ALL "Si", "Al", "O" AND elements LENGTH 3
 nelements=4
 nelements>=2 AND nelements<=7
-elements:elements_ratios HAS ALL "Al":>0.3333, "Al":<0.3334
 chemical_formula_descriptive="(H2O)2 Na"
 chemical_formula_descriptive CONTAINS "H2O"
 chemical_formula_reduced="H2NaO"

optimade/validator/data/optional_filters.txt

@@ -1,9 +1,12 @@
+((NOT (_exmpl_a>_exmpl_b)) AND _exmpl_x>0)
+5 < 7
 list HAS ONLY values
 list HAS < 3
 list HAS ALL < 3, > 3
 list:list HAS >=2:<=5
-elements:elements_ratios HAS ALL "Al":>0.3333, "Al":<0.3334
+elements HAS "H" AND elements HAS ALL "H","He","Ga","Ta" AND elements HAS ONLY "H","He","Ga","Ta" AND elements HAS ANY "H", "He", "Ga", "Ta"
+elements HAS ONLY "H","He","Ga","Ta"
 elements:_exmpl_element_counts HAS "H":6 AND elements:_exmpl_element_counts HAS ALL "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6 AND elements:_exmpl_element_counts HAS ANY "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6,"He":7
 _exmpl_element_counts HAS < 3 AND _exmpl_element_counts HAS ANY > 3, = 6, 4, != 8
 elements:_exmpl_element_counts:_exmpl_element_weights HAS ANY > 3:"He":>55.3 , = 6:>"Ti":<37.6 , 8:<"Ga":0
-5 < 7
+elements:elements_ratios HAS ALL "Al":>0.3333, "Al":<0.3334

optimade/validator/validator.py

@@ -18,7 +18,7 @@
 
 from optimade.models import InfoResponse, EntryInfoResponse, IndexInfoResponse
 
-from .data import MANDATORY_FILTER_EXAMPLES
+from .data import MANDATORY_FILTER_EXAMPLES, OPTIONAL_FILTER_EXAMPLES
 from .validator_model_patches import (
     ValidatorLinksResponse,
     ValidatorEntryResponseOne,
@@ -39,6 +39,11 @@
     "references": [],
 }
 
+ENDPOINT_OPTIONAL_QUERIES = {
+    "structures": OPTIONAL_FILTER_EXAMPLES,
+    "references": [],
+}
+
 RESPONSE_CLASSES = {
     "references": ValidatorReferenceResponseMany,
     "references/": ValidatorReferenceResponseOne,
@@ -148,11 +153,14 @@ def test_case(test_fn):
             and a message to print upon success. Should raise `ResponseError`,
             `ValidationError` or `ManualValidationError` if the test case has failed.
 
+    Keyword arguments:
+        optional (bool): whether or not to treat the test as optional.
+
     """
     from functools import wraps
 
     @wraps(test_fn)
-    def wrapper(validator, *args, **kwargs):
+    def wrapper(validator, *args, optional=False, **kwargs):
         try:
             result, msg = test_fn(validator, *args, **kwargs)
         except json.JSONDecodeError as exc:
@@ -169,24 +177,46 @@ def wrapper(validator, *args, **kwargs):
             request = validator.client.last_request
         except AttributeError:
             request = validator.base_url
+
         if result is not None:
-            validator.success_count += 1
+            if not optional:
+                validator.success_count += 1
+            else:
+                validator.optional_success_count += 1
+            message = f"✔: {request} - {msg}"
             if validator.verbosity > 0:
-                print_success(f"✔: {request} - {msg}")
+                if optional:
+                    print(message)
+                else:
+                    print_success(message)
             else:
-                print_success(".", end="", flush=True)
+                if optional:
+                    print(".", end="", flush=True)
+                else:
+                    print_success(".", end="", flush=True)
         else:
-            validator.failure_count += 1
+            if not optional:
+                validator.failure_count += 1
+            else:
+                validator.optional_failure_count += 1
             request = request.replace("\n", "")
             message = f"{msg}".split("\n")
             summary = f"✖: {request} - {test_fn.__name__} - failed with error"
             validator.failure_messages.append((summary, message))
             if validator.verbosity > 0:
-                print_failure(summary)
-                for line in message:
-                    print_warning(f"\t{line}")
+                if optional:
+                    print(summary)
+                    for line in message:
+                        print(f"\t{line}")
+                else:
+                    print_failure(summary)
+                    for line in message:
+                        print_warning(f"\t{line}")
             else:
-                print_failure(f"✖", end="", flush=True)
+                if optional:
+                    print(f"✖", end="", flush=True)
+                else:
+                    print_failure(f"✖", end="", flush=True)
 
         return result
 
@@ -262,6 +292,8 @@ def __init__(  # pylint: disable=too-many-arguments
             {} if self.index else ENDPOINT_MANDATORY_QUERIES
         )
 
+        self.endpoint_optional_queries = {} if self.index else ENDPOINT_OPTIONAL_QUERIES
+
         self.response_classes = (
             RESPONSE_CLASSES_INDEX if self.index else RESPONSE_CLASSES
         )
@@ -273,6 +305,8 @@ def __init__(  # pylint: disable=too-many-arguments
 
         self.success_count = 0
         self.failure_count = 0
+        self.optional_success_count = 0
+        self.optional_failure_count = 0
         self.failure_messages = []
 
     def _setup_log(self):
@@ -315,6 +349,7 @@ def main(self):
 
         # test entire implementation
         print(f"Testing entire implementation at {self.base_url}...")
+        print("\nMandatory tests:")
         self._log.debug("Testing base info endpoint of %s", BASE_INFO_ENDPOINT)
         base_info = self.test_info_or_links_endpoints(BASE_INFO_ENDPOINT)
         self.get_available_endpoints(base_info)
@@ -336,28 +371,40 @@ def main(self):
             # skip empty endpoint query lists
             if self.endpoint_mandatory_queries[endp]:
                 self._log.debug("Testing mandatory query syntax on endpoint %s", endp)
-                self.test_mandatory_query_syntax(
-                    endp, self.endpoint_mandatory_queries[endp]
-                )
+                self.test_query_syntax(endp, self.endpoint_mandatory_queries[endp])
 
         self._log.debug("Testing %s endpoint", LINKS_ENDPOINT)
         self.test_info_or_links_endpoints(LINKS_ENDPOINT)
 
         self.valid = not bool(self.failure_count)
 
+        print("\nOptional tests:")
+        for endp in self.endpoint_optional_queries:
+            # skip empty endpoint query lists
+            if self.endpoint_mandatory_queries[endp]:
+                self._log.debug("Testing optional query syntax on endpoint %s", endp)
+                self.test_query_syntax(
+                    endp, self.endpoint_optional_queries[endp], optional=True
+                )
+
         if not self.valid:
             print(f"\n\nFAILURES\n")
             for message in self.failure_messages:
                 print_failure(message[0])
                 for line in message[1]:
                     print_warning("\t" + line)
 
-        final_message = f"\nPassed {self.success_count} out of {self.success_count + self.failure_count} tests."
+        final_message = f"\n\nPassed {self.success_count} out of {self.success_count + self.failure_count} tests."
         if not self.valid:
             print_failure(final_message)
         else:
             print_success(final_message)
 
+        print(
+            f"Additionally passed {self.optional_success_count} out of "
+            f"{self.optional_success_count + self.optional_failure_count} optional tests."
+        )
+
     def test_info_or_links_endpoints(self, request_str):
         """ Runs the test cases for the info endpoints. """
         response = self.get_endpoint(request_str)
@@ -512,7 +559,7 @@ def get_available_endpoints(self, base_info):
         )
 
     @test_case
-    def get_endpoint(self, request_str):
+    def get_endpoint(self, request_str, optional=False):
         """ Gets the response from the endpoint specified by `request_str`. """
         request_str = f"/{request_str}".replace("\n", "")
         response = self.client.get(request_str)
@@ -530,17 +577,21 @@ def get_endpoint(self, request_str):
 
         return response, "request successful."
 
-    def test_mandatory_query_syntax(self, endpoint, endpoint_queries):
-        """ Perform a list of valid queries and assert that no errors are raised.
+    def test_query_syntax(self, endpoint, endpoint_queries, optional=False):
+        """ Execute a list of valid queries agains the endpoint and assert
+        that no errors are raised.
 
         Parameters:
             endpoint (str): the endpoint to query (e.g. "structures").
             endpoint_queries (list): the list of valid mandatory queries
                 for that endpoint, where the queries do not include the
                 "?filter=" prefix, e.g. ['elements HAS "Na"'].
 
+        Keyword arguments:
+            optional (bool): treat the success of the queries as optional.
+
         """
 
         valid_queries = [f"{endpoint}?filter={query}" for query in endpoint_queries]
         for query in valid_queries:
-            self.get_endpoint(query)
+            self.get_endpoint(query, optional=optional)

tasks.py

@@ -113,6 +113,9 @@ def parse_spec_for_filters(_):
     import requests
 
     filter_path = TOP_DIR.joinpath("optimade/validator/data/filters.txt")
+    optional_filter_path = TOP_DIR.joinpath(
+        "optimade/validator/data/optional_filters.txt"
+    )
 
     specification_flines = (
         requests.get(
@@ -123,12 +126,21 @@ def parse_spec_for_filters(_):
     )
 
     filters = []
+    optional_filters = []
+    optional_triggers = ("OPTIONAL",)
     for line in specification_flines:
         if ":filter:" in line:
             for _split in line.replace("filter=", "").split(":filter:")[1:]:
                 _filter = _split.split("`")[1].strip()
-                filters.append(_filter)
+                if any(trigger in line for trigger in optional_triggers):
+                    optional_filters.append(_filter)
+                else:
+                    filters.append(_filter)
 
     with open(filter_path, "w") as f:
         for _filter in filters:
             f.write(_filter + "\n")
+
+    with open(optional_filter_path, "w") as f:
+        for _filter in optional_filters:
+            f.write(_filter + "\n")

tests/server/test_server_validation.py

@@ -1,6 +1,7 @@
 # pylint: disable=relative-beyond-top-level,import-outside-toplevel
 import os
 import unittest
+from traceback import print_exc
 
 from optimade.validator import ImplementationValidator
 
@@ -13,7 +14,10 @@ class ServerTestWithValidator(SetClient, unittest.TestCase):
 
     def test_with_validator(self):
         validator = ImplementationValidator(client=self.client)
-        validator.main()
+        try:
+            validator.main()
+        except Exception:
+            print_exc()
         self.assertTrue(validator.valid)
 
 
@@ -23,7 +27,10 @@ class IndexServerTestWithValidator(SetClient, unittest.TestCase):
 
     def test_with_validator(self):
         validator = ImplementationValidator(client=self.client, index=True)
-        validator.main()
+        try:
+            validator.main()
+        except Exception:
+            print_exc()
         self.assertTrue(validator.valid)