From 50444ea8e0fdf44fbc20a5add877380ecfc58937 Mon Sep 17 00:00:00 2001
From: Trevor <grafnu@users.noreply.github.com>
Date: Mon, 21 Nov 2022 14:13:46 -0800
Subject: [PATCH] Enable metadata schema validation (#504)

---
 bin/test_registrar                            |  4 +-
 .../devices/DWN-1/expected/metadata_norm.json | 11 +++-
 .../devices/DWN-2/expected/metadata_norm.json | 11 +++-
 .../devices/DWN-2/out/generated_config.json   |  2 +-
 .../downgrade.site/registration_summary.json  |  2 +-
 tests/downgrade.site/site_metadata.json       |  9 ++-
 tests/metadata.tests/nosys.json               | 41 +++++++++++++
 tests/metadata.tests/nosys.out                |  2 +
 .../.idea/runConfigurations/Registrar.xml     |  2 +-
 validator/bin/registrar                       |  2 +-
 .../daq/mqtt/registrar/LocalDevice.java       | 11 +++-
 .../daq/mqtt/registrar/LocalDeviceTest.java   | 61 +++++++++++++++++++
 12 files changed, 144 insertions(+), 14 deletions(-)
 create mode 100644 tests/metadata.tests/nosys.json
 create mode 100644 tests/metadata.tests/nosys.out
 create mode 100644 validator/src/test/java/com/google/daq/mqtt/registrar/LocalDeviceTest.java

diff --git a/bin/test_registrar b/bin/test_registrar
index 6ae019f06..5d9b79762 100755
--- a/bin/test_registrar
+++ b/bin/test_registrar
@@ -14,11 +14,11 @@ cat "${TEST_SITE}/registration_summary.json"
 echo
 
 clean_devices=$(fgrep 'Z"' ${TEST_SITE}/registration_summary.json | wc -l)
-
+EXPECTED_CLEAN=3  # AHU-1 has a bad site location, so does not match as clean.
 exit_status=0
 
 echo Found ${clean_devices} clean devices.
-[ "${clean_devices}" == 4 ] || exit_status=1
+[ "${clean_devices}" == $EXPECTED_CLEAN ] || exit_status=1
 
 device_files=$(ls -d ${TEST_SITE}/devices/*)
 
diff --git a/tests/downgrade.site/devices/DWN-1/expected/metadata_norm.json b/tests/downgrade.site/devices/DWN-1/expected/metadata_norm.json
index f40a2bb0d..3a8d6c903 100644
--- a/tests/downgrade.site/devices/DWN-1/expected/metadata_norm.json
+++ b/tests/downgrade.site/devices/DWN-1/expected/metadata_norm.json
@@ -1,19 +1,26 @@
 {
   "timestamp": "2020-05-01T13:39:07Z",
   "version": "1.3.14",
-  "hash": "1064d284",
+  "hash": "c62d55be",
   "cloud": {
     "auth_type": "RS256"
   },
   "system": {
     "location": {
-      "site": "AA-TEST-SITE",
+      "site": "US-SFO-XYY",
       "section": "2-3N8C",
       "position": {
         "x": -111.0,
         "y": 102.3
       }
     },
+    "physical_tag": {
+      "asset": {
+        "guid": "bim://04aEp5ymD_$u5IxhJN2aGi",
+        "site": "US-SFO-XYY",
+        "name": "AHU-1"
+      }
+    },
     "min_loglevel": 300
   },
   "localnet": {
diff --git a/tests/downgrade.site/devices/DWN-2/expected/metadata_norm.json b/tests/downgrade.site/devices/DWN-2/expected/metadata_norm.json
index f40a2bb0d..3a8d6c903 100644
--- a/tests/downgrade.site/devices/DWN-2/expected/metadata_norm.json
+++ b/tests/downgrade.site/devices/DWN-2/expected/metadata_norm.json
@@ -1,19 +1,26 @@
 {
   "timestamp": "2020-05-01T13:39:07Z",
   "version": "1.3.14",
-  "hash": "1064d284",
+  "hash": "c62d55be",
   "cloud": {
     "auth_type": "RS256"
   },
   "system": {
     "location": {
-      "site": "AA-TEST-SITE",
+      "site": "US-SFO-XYY",
       "section": "2-3N8C",
       "position": {
         "x": -111.0,
         "y": 102.3
       }
     },
+    "physical_tag": {
+      "asset": {
+        "guid": "bim://04aEp5ymD_$u5IxhJN2aGi",
+        "site": "US-SFO-XYY",
+        "name": "AHU-1"
+      }
+    },
     "min_loglevel": 300
   },
   "localnet": {
diff --git a/tests/downgrade.site/devices/DWN-2/out/generated_config.json b/tests/downgrade.site/devices/DWN-2/out/generated_config.json
index 721f59f87..a5f2dcbd9 100644
--- a/tests/downgrade.site/devices/DWN-2/out/generated_config.json
+++ b/tests/downgrade.site/devices/DWN-2/out/generated_config.json
@@ -1,6 +1,6 @@
 {
   "timestamp" : "2020-05-01T13:39:07Z",
-  "version" : "1.4.0git ",
+  "version" : "1.4.0",
   "system" : {
     "min_loglevel" : 300,
     "metrics_rate_sec" : 600
diff --git a/tests/downgrade.site/registration_summary.json b/tests/downgrade.site/registration_summary.json
index 734f96c47..423a8ac97 100644
--- a/tests/downgrade.site/registration_summary.json
+++ b/tests/downgrade.site/registration_summary.json
@@ -8,6 +8,6 @@
     "DWN-2" : "devices/DWN-2"
   },
   "Version" : {
-    "main" : "1.3.14-90-g266b9b3"
+    "main" : "1.3.14-92-g6b250906"
   }
 }
\ No newline at end of file
diff --git a/tests/downgrade.site/site_metadata.json b/tests/downgrade.site/site_metadata.json
index 6c35316a4..cc54be628 100644
--- a/tests/downgrade.site/site_metadata.json
+++ b/tests/downgrade.site/site_metadata.json
@@ -1,7 +1,14 @@
 {
   "system": {
     "location": {
-      "site": "AA-TEST-SITE"
+      "site": "US-SFO-XYY"
+    },
+    "physical_tag": {
+      "asset": {
+        "guid": "bim://04aEp5ymD_$u5IxhJN2aGi",
+        "site": "US-SFO-XYY",
+        "name": "AHU-1"
+      }
     }
   }
 }
diff --git a/tests/metadata.tests/nosys.json b/tests/metadata.tests/nosys.json
new file mode 100644
index 000000000..34a4be63c
--- /dev/null
+++ b/tests/metadata.tests/nosys.json
@@ -0,0 +1,41 @@
+{
+  "version": "1.4.0",
+  "timestamp": "2018-08-26T21:39:29.364Z",
+  "description": "Generic test example metadata file",
+  "cloud": {
+    "auth_type": "ES256"
+  },
+  "pointset": {
+    "points": {
+      "return_air_temperature_sensor": {
+        "units": "Degrees-Celsius",
+        "baseline_value": 20,
+        "baseline_tolerance": 2
+      },
+      "room_setpoint": {
+        "writable": true,
+        "units": "Degrees-Celsius",
+        "baseline_value": 20,
+        "baseline_state": "applied"
+      }
+    }
+  },
+  "discovery": {
+    "families": {
+      "bacnet": {
+      }
+    }
+  },
+  "testing": {
+    "targets": {
+      "invalid": {
+        "target_point": "return_air_temperature_sensor",
+        "target_value": -20
+      },
+      "failure": {
+        "target_point": "room_setpoint",
+        "target_value": -20
+      }
+    }
+  }
+}
diff --git a/tests/metadata.tests/nosys.out b/tests/metadata.tests/nosys.out
new file mode 100644
index 000000000..e7346293c
--- /dev/null
+++ b/tests/metadata.tests/nosys.out
@@ -0,0 +1,2 @@
+1 schema violations found
+  object has missing required properties (["system"])
diff --git a/validator/.idea/runConfigurations/Registrar.xml b/validator/.idea/runConfigurations/Registrar.xml
index 6b8b5e030..65e2fa8e9 100644
--- a/validator/.idea/runConfigurations/Registrar.xml
+++ b/validator/.idea/runConfigurations/Registrar.xml
@@ -2,7 +2,7 @@
   <configuration default="false" name="Registrar" type="Application" factoryName="Application">
     <option name="MAIN_CLASS_NAME" value="com.google.daq.mqtt.registrar.Registrar" />
     <module name="validator" />
-    <option name="PROGRAM_PARAMETERS" value="-s sites/udmi_site_model" />
+    <option name="PROGRAM_PARAMETERS" value="-s sites/udmi_site_model -t" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.." />
     <extension name="coverage">
       <pattern>
diff --git a/validator/bin/registrar b/validator/bin/registrar
index ca69ecaf0..fc8fd078a 100755
--- a/validator/bin/registrar
+++ b/validator/bin/registrar
@@ -33,7 +33,7 @@ fi
 
 error=0
 echo java -cp $jarfile $mainclass -r $ROOT -s $site_dir $project_opt $*
-java -cp $jarfile $mainclass -r $ROOT -s $site_dir $project_opt $* || error=$?
+java -cp $jarfile $mainclass -r $ROOT -t -s $site_dir $project_opt $* || error=$?
 
 echo Registrar complete, exit $error
 exit $error
diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java
index 2dc84c089..abb295026 100644
--- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java
+++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java
@@ -301,23 +301,28 @@ private Metadata readMetadataWithValidation(boolean validate) {
       return null;
     }
 
+    JsonNode mergedMetadata = getMergedMetadata(instance);
+
     try {
-      ProcessingReport report = schemas.get(METADATA_JSON).validate(instance);
+      ProcessingReport report = schemas.get(METADATA_JSON).validate(mergedMetadata);
       if (validate) {
         parseMetadataValidateProcessingReport(report);
       }
     } catch (ProcessingException | ValidationException e) {
       exceptionMap.put(EXCEPTION_VALIDATING, e);
     }
+    return JsonUtil.convertTo(Metadata.class, mergedMetadata);
+  }
 
+  JsonNode getMergedMetadata(JsonNode instance) {
     try {
       String intermediary = JsonUtil.stringify(instance);
       if (siteMetadata == null) {
-        return JsonUtil.fromString(Metadata.class, intermediary);
+        return instance;
       } else {
         Map<String, Object> mergedMetadata = GeneralUtils.deepCopy(siteMetadata);
         GeneralUtils.mergeObject(mergedMetadata, asMap(intermediary));
-        return JsonUtil.convertTo(Metadata.class, mergedMetadata);
+        return JsonUtil.convertTo(JsonNode.class, mergedMetadata);
       }
     } catch (Exception e) {
       exceptionMap.put(EXCEPTION_READING, e);
diff --git a/validator/src/test/java/com/google/daq/mqtt/registrar/LocalDeviceTest.java b/validator/src/test/java/com/google/daq/mqtt/registrar/LocalDeviceTest.java
new file mode 100644
index 000000000..4408d670f
--- /dev/null
+++ b/validator/src/test/java/com/google/daq/mqtt/registrar/LocalDeviceTest.java
@@ -0,0 +1,61 @@
+package com.google.daq.mqtt.registrar;
+
+import static com.google.udmi.util.JsonUtil.convertTo;
+import static com.google.udmi.util.JsonUtil.stringify;
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jsonschema.main.JsonSchema;
+import com.google.udmi.util.GeneralUtils;
+import com.google.udmi.util.JsonUtil;
+import java.util.Date;
+import java.util.Map;
+import org.junit.Test;
+import udmi.schema.Metadata;
+
+/**
+ * Unit tests for LocalDevice.
+ */
+public class LocalDeviceTest {
+
+  public static final String DEVICE_ID = "test_device";
+  public static final Map<String, JsonSchema> SCHEMAS = null;
+  public static final int FAKE_TIME = 1298213;
+  public static String MERGE_DATA = "{ \"testing\": 10 }";
+
+  @Test
+  public void getMergedEmptyMetadata() {
+    LocalDevice localDevice = getTestInstance(null);
+    JsonNode toMerge = JsonUtil.fromString(JsonNode.class, MERGE_DATA);
+    String original = stringify(toMerge);
+    JsonNode result = localDevice.getMergedMetadata(toMerge);
+    assertEquals("merged results with no side defaults", original, stringify(result));
+  }
+
+  @Test
+  public void getMergedSiteMetadata() {
+    Metadata siteMetadata = getSiteMetadata();
+    Metadata augmentedData = GeneralUtils.deepCopy(siteMetadata);
+    Metadata toMerge = new Metadata();
+    toMerge.timestamp = new Date(FAKE_TIME + 1);
+    augmentedData.timestamp = toMerge.timestamp;
+    toMerge.description = "testing";
+    augmentedData.description = toMerge.description;
+    LocalDevice localDevice = getTestInstance(siteMetadata);
+    Metadata result = convertTo(Metadata.class,
+        localDevice.getMergedMetadata(convertTo(JsonNode.class, toMerge)));
+    assertEquals("merged results with no side defaults", stringify(augmentedData),
+        stringify(result));
+  }
+
+  private Metadata getSiteMetadata() {
+    Metadata metadata = new Metadata();
+    metadata.timestamp = new Date(FAKE_TIME);
+    metadata.version = "testing";
+    return metadata;
+  }
+
+  private LocalDevice getTestInstance(Metadata siteMetadata) {
+    return new LocalDevice(null, null, DEVICE_ID, SCHEMAS, null, siteMetadata);
+  }
+}
\ No newline at end of file