Return 400 when a test suite failed validation

python/src/etos_api/__init__.py

@@ -14,8 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ETOS API module."""
+
 import os
-from importlib.metadata import PackageNotFoundError, version
+from importlib.metadata import PackageNotFoundError, version as _version
 
 from etos_lib.logging.logger import setup_logging
 from opentelemetry import trace
@@ -42,17 +43,17 @@
 os.environ["ETOS_ENABLE_SENDING_LOGS"] = "false"
 
 try:
-    VERSION = version("etos_api")
+    version = _version("etos_api")
 except PackageNotFoundError:
-    VERSION = "Unknown"
+    version = "Unknown"
 
 DEV = os.getenv("DEV", "false").lower() == "true"
 
 if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
     OTEL_RESOURCE = Resource.create(
         {
             SERVICE_NAME: "etos-api",
-            SERVICE_VERSION: VERSION,
+            SERVICE_VERSION: version,
         },
     )
 
@@ -65,10 +66,10 @@
     PROCESSOR = BatchSpanProcessor(EXPORTER)
     PROVIDER.add_span_processor(PROCESSOR)
     trace.set_tracer_provider(PROVIDER)
-    setup_logging("ETOS API", VERSION, otel_resource=OTEL_RESOURCE)
+    setup_logging("ETOS API", version, otel_resource=OTEL_RESOURCE)
 
     FastAPIInstrumentor().instrument_app(APP, tracer_provider=PROVIDER, excluded_urls=".*/ping")
 else:
-    setup_logging("ETOS API", VERSION)
+    setup_logging("ETOS API", version)
 
 RegisterProviders()

python/src/etos_api/library/validator.py

@@ -164,7 +164,10 @@ async def validate(self, test_suite):
         """
         for suite_json in test_suite:
             test_runners = set()
-            suite = Suite(**suite_json)
+            try:
+                suite = Suite(**suite_json)
+            except ValidationError as exception:
+                raise AssertionError("Not a valid test suite was provided") from exception
             assert suite
 
             for recipe in suite.recipes:

python/tests/library/test_validator.py

@@ -14,13 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Tests for the validator library."""
+
 import logging
 import sys
 from unittest.mock import patch
 
 import pytest
 
-from etos_api.library.validator import SuiteValidator, ValidationError
+from etos_api.library.validator import SuiteValidator
 
 logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
 
@@ -75,7 +76,7 @@ async def test_validate_proper_suite(self, digest_mock):
         try:
             await validator.validate(test_suite)
             exception = False
-        except (AssertionError, ValidationError):
+        except (AssertionError, AssertionError):
             exception = True
         self.logger.info("STEP: Verify that no exceptions were raised.")
         assert exception is False
@@ -88,7 +89,7 @@ async def test_validate_missing_constraints(self):
 
         Test steps::
             1. Validate a suite with a missing constraint.
-            2. Verify that the validator raises ValidationError.
+            2. Verify that the validator raises AssertionError.
         """
         test_suite = [
             {
@@ -118,9 +119,9 @@ async def test_validate_missing_constraints(self):
         try:
             await validator.validate(test_suite)
             exception = False
-        except ValidationError:
+        except AssertionError:
             exception = True
-        self.logger.info("STEP: Verify that the validator raises ValidationError.")
+        self.logger.info("STEP: Verify that the validator raises AssertionError.")
         assert exception is True
 
     async def test_validate_wrong_types(self):
@@ -132,7 +133,7 @@ async def test_validate_wrong_types(self):
         Test steps::
             1. For each constraint.
                 1. Validate constraint with wrong type.
-                2. Verify that the validator raises ValidationError.
+                2. Verify that the validator raises AssertionError.
         """
         base_suite = {
             "name": "TestValidator",
@@ -204,8 +205,8 @@ async def test_validate_wrong_types(self):
         for constraint in constraints:
             self.logger.info("STEP: Validate constraint with wrong type.")
             base_suite["recipes"][0]["constraints"] = constraint
-            self.logger.info("STEP: Verify that the validator raises ValidationError.")
-            with pytest.raises(ValidationError):
+            self.logger.info("STEP: Verify that the validator raises AssertionError.")
+            with pytest.raises(AssertionError):
                 await validator.validate([base_suite])
 
     async def test_validate_too_many_constraints(self):
@@ -216,7 +217,7 @@ async def test_validate_too_many_constraints(self):
 
         Test steps::
             1. Validate a suite with a constraint defined multiple times.
-            2. Verify that the validator raises ValidationError.
+            2. Verify that the validator raises AssertionError.
         """
         test_suite = [
             {
@@ -248,9 +249,9 @@ async def test_validate_too_many_constraints(self):
         try:
             await validator.validate(test_suite)
             exception = False
-        except ValidationError:
+        except AssertionError:
             exception = True
-        self.logger.info("STEP: Verify that the validator raises ValidationError.")
+        self.logger.info("STEP: Verify that the validator raises AssertionError.")
         assert exception is True
 
     async def test_validate_unknown_constraint(self):
@@ -261,7 +262,7 @@ async def test_validate_unknown_constraint(self):
 
         Test steps::
             1. Validate a suite with an unknown constraint.
-            2. Verify that the validator raises ValidationError.
+            2. Verify that the validator raises AssertionError.
         """
         test_suite = [
             {
@@ -295,7 +296,7 @@ async def test_validate_unknown_constraint(self):
             exception = False
         except TypeError:
             exception = True
-        self.logger.info("STEP: Verify that the validator raises ValidationError.")
+        self.logger.info("STEP: Verify that the validator raises AssertionError.")
         assert exception is True
 
     async def test_validate_empty_constraints(self):
@@ -354,5 +355,5 @@ async def test_validate_empty_constraints(self):
         for constraint in constraints:
             base_suite["recipes"][0]["constraints"] = constraint
             self.logger.info("STEP: Validate a suite without the required key.")
-            with pytest.raises(ValidationError):
+            with pytest.raises(AssertionError):
                 await validator.validate([base_suite])

python/tox.ini

@@ -22,7 +22,7 @@ commands =
 deps =
     black
 commands =
-    black --check --diff -l 100 .
+    black --check --diff -l 100 src tests
 
 [testenv:pylint]
 deps =
@@ -36,4 +36,4 @@ deps =
     pydocstyle
     tomli
 commands =
-    pydocstyle .
+    pydocstyle src tests