validate schema of returned timeseries objects

pyproject.toml

@@ -26,6 +26,7 @@ pytest-cov = "^4.0.0"
 coverage = "^7.0.1"
 pytest-mock = "^3.10.0"
 mock = "^5.0.0"
+jsonschema = "^4.17.3"
 
 [build-system]
 requires = ["poetry-core"]

test-requirements.txt

@@ -7,6 +7,7 @@ coverage[toml]==7.0.1 ; python_version >= "3.9" and python_version < "4.0"
 exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11"
 flake8==6.0.0 ; python_version >= "3.9" and python_version < "4.0"
 iniconfig==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
+jsonschema==4.17.3 ; python_version >= "3.9" and python_version < "4.0"
 mccabe==0.7.0 ; python_version >= "3.9" and python_version < "4.0"
 mock==5.0.0 ; python_version >= "3.9" and python_version < "4.0"
 mypy-extensions==0.4.3 ; python_version >= "3.9" and python_version < "4.0"
@@ -16,6 +17,7 @@ platformdirs==2.6.2 ; python_version >= "3.9" and python_version < "4.0"
 pluggy==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
 pycodestyle==2.10.0 ; python_version >= "3.9" and python_version < "4.0"
 pyflakes==3.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pyrsistent==0.19.3 ; python_version >= "3.9" and python_version < "4.0"
 pytest-black==0.3.12 ; python_version >= "3.9" and python_version < "4.0"
 pytest-cov==4.0.0 ; python_version >= "3.9" and python_version < "4.0"
 pytest-mock==3.10.0 ; python_version >= "3.9" and python_version < "4.0"

test/get_test_data.py

@@ -1,4 +1,5 @@
 import json
+import os
 from azure.functions import EventHubEvent
 import datetime
 from dateutil import parser
@@ -27,7 +28,10 @@ def load_test_data():
     Load test data from a JSON file
     @return: a dictionary of test data
     """
-    with open("./test/test_data.json", "r") as f:
+    SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+    test_data_path = os.sep.join([SCRIPT_DIR, "test_data.json"])
+
+    with open(test_data_path, "r") as f:
         test_data = json.load(f)
     return recursive_json_parser(test_data)
 

test/test_json_to_timeseries.py

@@ -21,6 +21,7 @@
     extract_topic,
 )
 
+
 # import test data
 test_data = load_test_data()
 

test/test_shared_code.py

@@ -7,6 +7,11 @@
 from azure.functions import EventHubEvent
 from datetime import datetime, timezone
 
+import json
+from jsonschema import validate
+
+
+
 # add the shared_code directory to the path
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.dirname(SCRIPT_DIR))
@@ -22,6 +27,11 @@
     create_correlation_id,
 )
 
+# load the schema
+schema_path = os.sep.join([SCRIPT_DIR, "timeseries.json"])
+with open(schema_path) as f:
+    schema = json.load(f)
+
 # import test data
 test_data = load_test_data()
 
@@ -60,6 +70,12 @@ def call_converter(
         raise ValueError(f"Unknown converter {converter}")
 
 
+def assert_valid_schema(data: dict, schema: dict):
+    """ Checks whether the given data matches the schema """
+
+    return validate(data, schema)
+
+
 class Test_Converter_Methods:
     class Test_Emon:
         def test_with_ignored_key(self):
@@ -74,31 +90,38 @@ def test_with_emonTx4_key(self):
             expected_value = test_object["expected"]
             for actual, expected in zip(actual_value, expected_value):
                 TestCase().assertDictEqual(actual, expected)
+            assert_valid_schema(actual_value, schema)
+
 
     class Test_Glow:
         def test_glow_to_timescale_with_valid_json_for_electricity_meter(self):
             actual_value = call_converter("glow", test_data["glow_electricitymeter"])
             expected_value = test_data["glow_electricitymeter"]["expected"]
             for actual, expected in zip(actual_value, expected_value):
                 TestCase().assertDictEqual(actual, expected)
+            assert_valid_schema(actual_value, schema)
+
 
         def test_glow_to_timescale_with_valid_json_for_gas_meter(self):
             actual_value = call_converter("glow", test_data["glow_gasmeter"])
             expected_value = test_data["glow_gasmeter"]["expected"]
             for actual, expected in zip(actual_value, expected_value):
                 TestCase().assertDictEqual(actual, expected)
+            assert_valid_schema(actual_value, schema)
 
         def test_glow_to_timescale_with_item_to_ignore(self):
             actual_value = call_converter("glow", test_data["homie_heartbeat"])
-            expected_value = test_data["homie_heartbeat"]["expected"]
-            assert actual_value == expected_value
+            expected_value = test_data["homie_heartbeat"]["expected"] # None
+            assert expected_value is None
+            assert actual_value is None
 
     class Test_Homie:
         def test_homie_to_timescale_with_valid_json_for_mode(self):
             actual_value = call_converter("homie", test_data["homie_mode"])
             expected_value = test_data["homie_mode"]["expected"]
             for actual, expected in zip(actual_value, expected_value):
                 TestCase().assertDictEqual(actual, expected)
+            assert_valid_schema(actual_value, schema)
 
         def test_homie_to_timescale_with_ignored_json_for_heartbeat(self):
             actual_value = call_converter("homie", test_data["homie_heartbeat"])
@@ -113,6 +136,7 @@ def test_homie_to_timescale_with_valid_json_for_measure_temperature(self):
             expected_value = test_data["homie_measure_temperature"]["expected"]
             for actual, expected in zip(actual_value, expected_value):
                 TestCase().assertDictEqual(actual, expected)
+            assert_valid_schema(actual_value, schema)
 
     class Test_Timeseries:
         class Test_get_record_type:
@@ -187,6 +211,7 @@ def test_create_record_recursive_with_single_payload(self):
                 )
                 for actual, expected in zip(actual_value, expected_value):
                     TestCase().assertDictEqual(actual, expected)
+                assert_valid_schema(actual_value, schema)
 
             def test_create_record_recursive_with_empty_payload(self):
                 records = []
@@ -210,6 +235,7 @@ def test_create_record_recursive_with_empty_payload(self):
                 )
                 for actual, expected in zip(actual_value, expected_value):
                     TestCase().assertDictEqual(actual, expected)
+                assert_valid_schema(actual_value, schema)
 
             def test_create_record_recursive_with_dict_of_payloads(self):
                 records = []
@@ -250,6 +276,7 @@ def test_create_record_recursive_with_dict_of_payloads(self):
                 )
                 for actual, expected in zip(actual_value, expected_value):
                     TestCase().assertDictEqual(actual, expected)
+                assert_valid_schema(actual_value, schema)
 
             def test_create_record_recursive_with_dict_of_payloads_ignoring_one(self):
                 records = []
@@ -282,6 +309,7 @@ def test_create_record_recursive_with_dict_of_payloads_ignoring_one(self):
                 )
                 for actual, expected in zip(actual_value, expected_value):
                     TestCase().assertDictEqual(actual, expected)
+                assert_valid_schema(actual_value, schema)
 
             def test_create_record_recursive_with_dict_of_payloads_and_measurement_prefix(
                 self,
@@ -324,6 +352,7 @@ def test_create_record_recursive_with_dict_of_payloads_and_measurement_prefix(
                 )
                 for actual, expected in zip(actual_value, expected_value):
                     TestCase().assertDictEqual(actual, expected)
+                assert_valid_schema(actual_value, schema)
 
             def test_create_record_recursive_with_dict_of_payloads_and_measurement_prefix_ignoring_one(
                 self,
@@ -358,6 +387,7 @@ def test_create_record_recursive_with_dict_of_payloads_and_measurement_prefix_ig
                 )
                 for actual, expected in zip(actual_value, expected_value):
                     TestCase().assertDictEqual(actual, expected)
+                assert_valid_schema(actual_value, schema)
 
 
 class Test_Helpers:

test/timeseries.json

@@ -0,0 +1,50 @@
+{
+  "$id": "https://cynexia.com/schemas/timeseries.json",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "array",
+  "items": [
+    {
+      "type": "object",
+      "properties": {
+        "timestamp": {
+          "type": "string",
+          "format": "date-time",
+          "description": "The time at which the measurement was taken. For example, 2018-01-01T12:00:00Z."
+        },
+        "measurement_subject": {
+          "type": "string",
+          "description": "The device which created the measurement. For example, a sensor ID or a device ID."
+        },
+        "measurement_of": {
+          "type": "string",
+          "description": "The thing which was measured. For example, a measure-temperature or state."
+        },
+        "measurement_value": {
+          "type": ["string", "number", "boolean"],
+          "description": "The value of the measurement. For example, 23.5 or Home."
+        },
+        "measurement_data_type": {
+          "type": "string",
+          "enum": [
+            "string",
+            "number",
+            "boolean"
+          ],
+          "description": "The data type of the measurement value. Must be one of: string, number, or boolean."
+        },
+        "correlation_id": {
+          "type": "string",
+          "description": "A unique ID that can be used to correlate multiple measurements together. For example, a unique ID for a user or a session."
+        }
+      },
+      "required": [
+        "timestamp",
+        "measurement_subject",
+        "measurement_of",
+        "measurement_value",
+        "measurement_data_type"
+      ],
+      "additionalProperties": false
+    }
+  ]
+}