Merge different configuration objects, and create the basic foundation for profiles-in-validator

.gencode_hash.txt

@@ -1,9 +1,9 @@
-909b90e19ba41e691ea277eb26ff8a3b7d7bad43ecd7ac082683e97401bde9b5  gencode/docs/cloud_iot_config.html
 b35646a2f3d7c30fd34e3ea6ab6a1070b39c469fd1de49cab1093ff32faaf06d  gencode/docs/command_discovery.html
 e1f62882576d0f4332360f2aedb79b37a43973fb31d862c829af924f8f310ec1  gencode/docs/command_mapping.html
 5513a43ef916d0413de0c95aede03042ce8824e98fbb8f3634c12f5dee53cce8  gencode/docs/config.html
 30c7f39e50e290b0a36a3414cf1c90277ebd249f126f85f0a307bc7b74164e08  gencode/docs/config_mapping.html
 5598402bde04860394d91ecf43f5c653ff539cce49d011db4b5b17d596b72c2a  gencode/docs/configuration_endpoint.html
+b7ca70e3176f997f254c32bd8d53dff1462ac8a71c0e166aaa85193d67bff60b  gencode/docs/configuration_execution.html
 00e8646fd8117d47ddf32f2488228cd35b66fd711cb21288dc6bb5a5715978cb  gencode/docs/configuration_pubber.html
 82fbf41e4e88593f22a929fc8ade4d1e8f75a0fc1aa8452ea1d3b19023bee314  gencode/docs/envelope.html
 f1fde870638541bea6de16acd64365c89a7d56ee3270d6fd1bcdbb5bdb141887  gencode/docs/event.html
@@ -15,7 +15,7 @@ a8800ab8384de3b846af0d72cf310acf01644842b557e42eec20b69a0bfa1868  gencode/docs/e
 9ae793a641f2e053c5af734bf813cd2637a675330acb8d9e903ef03b4286e007  gencode/docs/metadata.html
 e101ec26541b7787fa722244fed744713d05649de547160f39744e1e61850ddd  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
-846eaf1735bd2506357173cfc720c8674d95acf2d20e19f37e3d34cad0a345d2  gencode/docs/readme.md
+392f272c1c8cd047eb6906f2f11ffacc7083c32f639da2437d6a9ca861062ecc  gencode/docs/readme.md
 690d56e96c4e180e49e0abcba358ab1842ea06dd3f693918852a9de370c994d5  gencode/docs/reflect_config.html
 9332e44c87dae9261b079424e748d5ee1df08a3c6b39987d254ebf78274e2f34  gencode/docs/reflect_state.html
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
@@ -32,7 +32,6 @@ b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc  gencode/java/u
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e  gencode/java/udmi/schema/BuildingTranslation.java
 b95ab79a531c917a5004b5e5bc70e010b33d5d6ff53be09e5dbf2d1df479b569  gencode/java/udmi/schema/Category.java
-d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b  gencode/java/udmi/schema/CloudIotConfig.java
 a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d  gencode/java/udmi/schema/CloudModel.java
 ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4  gencode/java/udmi/schema/Config.java
 10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05  gencode/java/udmi/schema/DeviceMappingConfig.java
@@ -48,6 +47,7 @@ b90ab40a281c5e383881629bd8a791ba3d30491e06ebc623a476e2735a3f6c3a  gencode/java/u
 885859ea9de29217186c34f3a90b112411d1beddaf58a88b85e7934cd3c92b6f  gencode/java/udmi/schema/Entry.java
 9b92e03774ed0b9159ff3ab462822699ef7f824161b166a881889ea1eb6a62ad  gencode/java/udmi/schema/Envelope.java
 e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e  gencode/java/udmi/schema/Event.java
+69c3c12ce057b6ab8e27ab8cab6fb009bfd1c997652214b49e6f2a26a58f302c  gencode/java/udmi/schema/ExecutionConfiguration.java
 2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed  gencode/java/udmi/schema/FamilyDiscoveryConfig.java
 ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454  gencode/java/udmi/schema/FamilyDiscoveryEvent.java
 0afc15acd72874e5a0c47f546abc0c4569f5bc37838fdcac77bc7bd55cc53a6d  gencode/java/udmi/schema/FamilyDiscoveryState.java
@@ -98,12 +98,11 @@ d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/u
 ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4  gencode/java/udmi/schema/ValidationEvent.java
 f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2  gencode/java/udmi/schema/ValidationState.java
 e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36  gencode/java/udmi/schema/ValidationSummary.java
-0601b86702f0e8959a391342307cd782be91e7dcfa0017089230a38e76661dd5  gencode/python/udmi/schema/__init__.py
+67256cd379f8b456a2d8c3b5e64c9bde740569338f8e32be626ecb65ad6fd23c  gencode/python/udmi/schema/__init__.py
 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db  gencode/python/udmi/schema/ancillary_properties.py
 5ecd6c542f33450cb4ce75d940a6dff4d3bd67d4b9de4aff5ee88abcc301dbff  gencode/python/udmi/schema/building_config.py
 dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6  gencode/python/udmi/schema/building_translation.py
 c33fab16eb86b93692ddbb243a2d6cfe202ca1a0fd76b351a18727a35940d958  gencode/python/udmi/schema/category.py
-a61b1c5732f01b7efda41a773d5786fad755f371193ce4478b458387ca2a8fe8  gencode/python/udmi/schema/cloud_iot_config.py
 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb  gencode/python/udmi/schema/command_discovery.py
 1254c34d973c9099ae99dcea4534e234e9019f49255e2e27d2afa1bc074fd596  gencode/python/udmi/schema/command_mapping.py
 c5a62f92328e2ede167fc1f53bece6e48696a0ab0e37e3a41f65cd98494ba0d7  gencode/python/udmi/schema/common.py
@@ -121,6 +120,7 @@ ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244  gencode/python
 ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857  gencode/python/udmi/schema/config_system.py
 97c2e5fadc6da0d84660f3296de885ab59f4b04154179b6717f77ec366f1544b  gencode/python/udmi/schema/config_system_testing.py
 4637fc257ea4d6ea5a46d98db69c57e27e4ecbf5d86f77036cd1382f1a40f532  gencode/python/udmi/schema/configuration_endpoint.py
+e8a1574074554b6144b178d2adedc76a1c7be5ae911b253deff4460d8d82c6ee  gencode/python/udmi/schema/configuration_execution.py
 3814c88403934dbd3fce77d92a8ad45c68dc7e07c319ba9b4e8f1b1ac7518c07  gencode/python/udmi/schema/configuration_pubber.py
 998ce105f88686f27b85f3630a396ed04b106f830c133a684ea5c505ca95b1c3  gencode/python/udmi/schema/envelope.py
 1eb9019b9d0b4ff7de2df8beb625a4f89292d636ece0c02f160495c537bd6c2c  gencode/python/udmi/schema/event.py

.github/workflows/testing.yml

@@ -135,6 +135,10 @@ jobs:
           cd sites/udmi_site_model/out
           echo Output from `pwd`
           more `find out -type f`
+      - name: validator logs
+        if: ${{ always() }}
+        run: |
+          cat out/validator.out || true
       - name: pubber logs
         if: ${{ always() }}
         run: |

bin/test_trace

@@ -20,6 +20,7 @@ echo
 for file in `find $site_out -type f`; do
     sed -E -i $file \
         -e "s/[0-9-]{10}T[0-9:]{8}Z/1999-10-20T01:02:03Z/" \
+        -e "s/Validator.java:[0-9]+/redacted/" \
         -e 's/\\t[a-zA-Z .()$0-9]+\.java:[0-9]+\)\\n/\\tredacted\\n/g'
 done
 

bin/test_validator

@@ -116,4 +116,5 @@ echo Found out files $outfiles, copying to $TMP_VALIDATOR
 # Include /dev/null so that more has >1 to chomp on and outputs headers.
 more /dev/null $outfiles | sed -E >> $TMP_VALIDATOR \
         -e "s/[0-9-]{10}T[0-9:]{8}Z/1999-10-20T01:02:03Z/" \
+        -e "s/Validator.java:[0-9]+/redacted/" \
         -e 's/\\t[a-zA-Z .()$0-9]+\.java:[0-9]+\)\\n/\\tredacted\\n/g' \

bin/validator

@@ -20,9 +20,6 @@ if [ ! -f $site_path/cloud_iot_config.json ]; then
     false
 fi
 
-echo Building validator...
-$ROOT_DIR/validator/bin/build > /dev/null
-
 echo Running tools version `(cd $ROOT_DIR; git describe)`
 
 if [[ -n $subscription ]]; then

validator/src/main/java/com/google/daq/mqtt/validator/CleanDateFormat.java → common/src/main/java/com/google/udmi/util/CleanDateFormat.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.validator;
+package com.google.udmi.util;
 
 import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import java.text.ParseException;

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -8,6 +8,7 @@
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 public class GeneralUtils {
 
@@ -76,12 +77,33 @@ public static void toJsonFile(File file, Object target) {
     }
   }
 
-  public static <T> T deepCopy(T endpoint1, Class<T> valueType) {
+  @SuppressWarnings("unchecked")
+  public static <T> T deepCopy(T object) {
+    Class<?> targetClass = object.getClass();
     try {
-      return OBJECT_MAPPER.readValue(toJsonString(endpoint1),
-          valueType);
+      return (T) OBJECT_MAPPER.readValue(toJsonString(object), targetClass);
     } catch (Exception e) {
-      throw new RuntimeException("While making deep copy of " + valueType.getName(), e);
+      throw new RuntimeException("While making deep copy of " + targetClass.getName(), e);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public static <T> T mergeObject(Object destination, Object source) {
+    Map<String, Object> target = JsonUtil.asMap(destination);
+    mergeObject(target, JsonUtil.asMap(source));
+    return (T) JsonUtil.convertTo(destination.getClass(), target);
+  }
+
+  @SuppressWarnings("unchecked")
+  public static void mergeObject(Map<String, Object> target, Map<String, Object> source) {
+    for (String key : source.keySet()) {
+      Object targetValue = target.get(key);
+      Object sourceValue = source.get(key);
+      if (targetValue instanceof Map && sourceValue instanceof Map) {
+        mergeObject((Map<String, Object>) targetValue, (Map<String, Object>) sourceValue);
+      } else {
+        target.put(key, sourceValue);
+      }
     }
   }
 }

validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java → common/src/main/java/com/google/udmi/util/JsonUtil.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
@@ -7,7 +7,6 @@
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.daq.mqtt.validator.CleanDateFormat;
 import java.io.File;
 import java.time.Instant;
 import java.util.Date;
@@ -144,7 +143,35 @@ public static void writeFile(Object target, File file) {
     }
   }
 
+  /**
+   * Get a date object parsed from a string representation.
+   *
+   * @param timestamp string representation
+   * @return Date object
+   */
   public static Date getDate(String timestamp) {
     return timestamp == null ? null : Date.from(Instant.parse(timestamp));
   }
+
+  /**
+   * Convert the json string to a generic map object.
+   *
+   * @param input input string
+   * @return input as map object
+   */
+  @SuppressWarnings("unchecked")
+  public static Map<String, Object> asMap(String input) {
+    return convertTo(TreeMap.class, input);
+  }
+
+  /**
+   * Convert the json object to a generic map object.
+   *
+   * @param input input object
+   * @return input as map object
+   */
+  @SuppressWarnings("unchecked")
+  public static Map<String, Object> asMap(Object input) {
+    return convertTo(TreeMap.class, input);
+  }
 }

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -17,10 +17,10 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import udmi.schema.CloudIotConfig;
 import udmi.schema.CloudModel.Auth_type;
 import udmi.schema.EndpointConfiguration;
 import udmi.schema.Envelope;
+import udmi.schema.ExecutionConfiguration;
 import udmi.schema.GatewayModel;
 import udmi.schema.Metadata;
 
@@ -39,17 +39,17 @@ public class SiteModel {
   final String sitePath;
   private Map<String, Metadata> allMetadata;
   private Map<String, Device> allDevices;
-  private CloudIotConfig cloudIotConfig;
+  private ExecutionConfiguration executionConfiguration;
 
   public SiteModel(String sitePath) {
     this.sitePath = sitePath;
   }
 
   public static EndpointConfiguration makeEndpointConfig(String projectId,
-      CloudIotConfig cloudIotConfig, String deviceId) {
+      ExecutionConfiguration executionConfig, String deviceId) {
     EndpointConfiguration endpoint = new EndpointConfiguration();
     endpoint.client_id = getClientId(projectId,
-        cloudIotConfig.cloud_region, cloudIotConfig.registry_id, deviceId);
+        executionConfig.cloud_region, executionConfig.registry_id, deviceId);
     endpoint.hostname = DEFAULT_ENDPOINT_HOSTNAME;
     return endpoint;
   }
@@ -59,18 +59,18 @@ public static String getClientId(String projectId, String cloudRegion, String re
     return String.format(ID_FORMAT, projectId, cloudRegion, registryId, deviceId);
   }
 
-  private static CloudIotConfig makeCloudIotConfig(Envelope attributes) {
-    CloudIotConfig cloudIotConfig = new CloudIotConfig();
-    cloudIotConfig.registry_id = Preconditions.checkNotNull(attributes.deviceRegistryId,
+  private static ExecutionConfiguration makeExecutionConfiguration(Envelope attributes) {
+    ExecutionConfiguration executionConfiguration = new ExecutionConfiguration();
+    executionConfiguration.registry_id = Preconditions.checkNotNull(attributes.deviceRegistryId,
         "deviceRegistryId");
-    cloudIotConfig.cloud_region = Preconditions.checkNotNull(attributes.deviceRegistryLocation,
+    executionConfiguration.cloud_region = Preconditions.checkNotNull(attributes.deviceRegistryLocation,
         "deviceRegistryLocation");
-    return cloudIotConfig;
+    return executionConfiguration;
   }
 
   public static EndpointConfiguration makeEndpointConfig(Envelope attributes) {
-    CloudIotConfig cloudIotConfig = makeCloudIotConfig(attributes);
-    return makeEndpointConfig(attributes.projectId, cloudIotConfig, attributes.deviceId);
+    ExecutionConfiguration executionConfiguration = makeExecutionConfiguration(attributes);
+    return makeEndpointConfig(attributes.projectId, executionConfiguration, attributes.deviceId);
   }
 
   /**
@@ -97,7 +97,7 @@ public static ClientInfo parseClientId(String clientId) {
   }
 
   public EndpointConfiguration makeEndpointConfig(String projectId, String deviceId) {
-    return makeEndpointConfig(projectId, cloudIotConfig, deviceId);
+    return makeEndpointConfig(projectId, executionConfiguration, deviceId);
   }
 
   private Set<String> getDeviceIds() {
@@ -159,7 +159,7 @@ private void loadSiteConfig() {
         "sitePath not defined in configuration");
     File cloudConfig = new File(new File(sitePath), "cloud_iot_config.json");
     try {
-      cloudIotConfig = OBJECT_MAPPER.readValue(cloudConfig, CloudIotConfig.class);
+      executionConfiguration = OBJECT_MAPPER.readValue(cloudConfig, ExecutionConfiguration.class);
     } catch (Exception e) {
       throw new RuntimeException("While reading config file " + cloudConfig.getAbsolutePath(), e);
     }
@@ -197,7 +197,7 @@ private String getDeviceKeyPrefix(String targetId) {
    * @return site registry
    */
   public String getRegistryId() {
-    return cloudIotConfig.registry_id;
+    return executionConfiguration.registry_id;
   }
 
   /**
@@ -206,7 +206,7 @@ public String getRegistryId() {
    * @return update topic
    */
   public String getUpdateTopic() {
-    return cloudIotConfig.update_topic;
+    return executionConfiguration.update_topic;
   }
 
   public Device getDevice(String deviceId) {

common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java

@@ -0,0 +1,91 @@
+package com.google.udmi.util;
+
+import static com.google.udmi.util.GeneralUtils.deepCopy;
+import static com.google.udmi.util.GeneralUtils.mergeObject;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+
+/**
+ * Tests for general utilities.
+ */
+public class GeneralUtilsTest {
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testDeepCopy() {
+    Map<String, Object> original = getBaseMap();
+    Map<String, Object> copy = deepCopy(original);
+    assertEquals("copy is equal", original, copy);
+    ((Map<String, String>) copy.get("C")).put("F", "H");
+    assertNotEquals("copy is not equal", original, copy);
+  }
+
+  private Map<String, Object> getBaseMap() {
+    Map<String, Object> original = new HashMap<>();
+    original.put("A", "B");
+    original.put("C", ImmutableMap.of("D", "E", "F", "G"));
+    return original;
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testMergeObject() {
+    final Map<String, Object> target = deepCopy(getBaseMap());
+    final Map<String, Object> originalTarget = deepCopy(target);
+    final Map<String, Object> source = deepCopy(target);
+    source.put("X", "Y");
+    final Map<String, String> sourceC = (Map<String, String>) source.get("C");
+    sourceC.put("D", "Q");
+    sourceC.put("H", "I");
+    final Map<String, Object> originalSource = deepCopy(source);
+    mergeObject(target, source);
+    assertEquals("unmolested source", originalSource, source);
+    assertNotEquals("changed target", target, originalTarget);
+    assertNull("original X", originalTarget.get("X"));
+    assertEquals("target X", "Y", target.get("X"));
+    final Map<String, String> originalC = (Map<String, String>) originalTarget.get("C");
+    assertEquals("original C.D", "E", originalC.get("D"));
+    assertEquals("original C.F", "G", originalC.get("F"));
+    assertNull("original C.H", originalC.get("H"));
+    final Map<String, String> targetC = (Map<String, String>) target.get("C");
+    assertEquals("target C.D", "Q", targetC.get("D"));
+    assertEquals("target C.F", "G", targetC.get("F"));
+    assertEquals("target C.H", "I", targetC.get("H"));
+  }
+
+  @Test
+  public void testMergeTyped() {
+    Container source = new Container();
+    Container target = new Container();
+    source.anInt = 2;
+    source.bool = true;
+    target.string = "hello";
+    target.bool = false;
+    Container merged = mergeObject(target, source);
+    assertEquals("source a", 2, source.anInt);
+    assertEquals("target a", 0, target.anInt);
+    assertEquals("merged a", 2, merged.anInt);
+    assertNull("source b", source.string);
+    assertEquals("target b", "hello", target.string);
+    assertEquals("merged b", "hello", merged.string);
+    assertEquals("source c", true, source.bool);
+    assertEquals("target c", false, target.bool);
+    assertEquals("merged c", true, merged.bool);
+  }
+
+  /**
+   * Simple test container class.
+   */
+  public static class Container {
+
+    public int anInt;
+    public String string;
+    public Boolean bool;
+  }
+}

etc/validator.out

@@ -25,7 +25,7 @@ sites/udmi_site_model/out/devices/AHU-1/event_pointset.out
   "sub_type" : "event",
   "status" : {
     "message" : "While converting to json node: 1 schema violations found",
-    "detail" : "While converting to json node: 1 schema violations found @Validator.validateMessage(Validator.java:417); 1 schema violations found; object instance has properties which are not allowed by the schema: [\"extraField\"]",
+    "detail" : "While converting to json node: 1 schema violations found @Validator.validateMessage(redacted); 1 schema violations found; object instance has properties which are not allowed by the schema: [\"extraField\"]",
     "category" : "validation.error.simple",
     "timestamp" : "1999-10-20T01:02:03Z",
     "level" : 500