Add persistent store, restart cound, and utility refactoring (#467)

.gencode_hash.txt

@@ -6,15 +6,16 @@ d93322f64b94bc1808c59cde7b83915ff7e14d50401b9c2ae8467730ed15257c  gencode/docs/c
 bbb98edbbc279678856c11024f4681261f0b9b58b07f8de6026be70d2e32074a  gencode/docs/configuration_endpoint.html
 299bc6f4d9842fab2d2cfd003d8a8adf94303c560cf71e8595d85cf661558a4d  gencode/docs/configuration_pubber.html
 82fbf41e4e88593f22a929fc8ade4d1e8f75a0fc1aa8452ea1d3b19023bee314  gencode/docs/envelope.html
-276158157fdf8989edf8ef9a53874246fc6e87412aea410712496de9759e922c  gencode/docs/event.html
+5888a5c257b62851dcaed9cf5e015240e0b1882d88cc1f1329402b285b1eeb4b  gencode/docs/event.html
 a8800ab8384de3b846af0d72cf310acf01644842b557e42eec20b69a0bfa1868  gencode/docs/event_discovery.html
 3019d06826576d287edbf69116ec712c74c8fdfcc5653b51f2573cb9755ebb23  gencode/docs/event_mapping.html
 71fe25d3b17d1dc87b52c049a3235c20527f73e12ffa53fa4800a15e53f73732  gencode/docs/event_pointset.html
-7adf725427f285539b37abdad0c15f32e76bd6e1ec7d775527491597aef32875  gencode/docs/event_system.html
+76a0d1c4baf4b74b65c9be862ca114dbee00552ba59e75978215db1c6c7fc0ee  gencode/docs/event_system.html
 43025f61381de050eae9680c3193a1d878a397d0b5515dd1063a1a993b0c41ea  gencode/docs/event_validation.html
 9ae793a641f2e053c5af734bf813cd2637a675330acb8d9e903ef03b4286e007  gencode/docs/metadata.html
+5468d640a0a4467d21dccd7f0440a7ca84c7de1102596011494ac9d3b3e9f6be  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
-162f8f3b8ca9491e7dc1d2569d4773acae51f93b07fa875a9ece218d9d4f8a89  gencode/docs/readme.md
+846eaf1735bd2506357173cfc720c8674d95acf2d20e19f37e3d34cad0a345d2  gencode/docs/readme.md
 690d56e96c4e180e49e0abcba358ab1842ea06dd3f693918852a9de370c994d5  gencode/docs/reflect_config.html
 9332e44c87dae9261b079424e748d5ee1df08a3c6b39987d254ebf78274e2f34  gencode/docs/reflect_state.html
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
@@ -30,12 +31,13 @@ a7c57d119adcd0cf6363cc5301ce562004222522242e8ffd1d0cd7010f235ae1  gencode/java/u
 b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc  gencode/java/udmi/schema/BlobsetConfig.java
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e  gencode/java/udmi/schema/BuildingTranslation.java
-5655b896f000b0108f438f9b04bfc092764ad5af63b70bbf750ba58b5f5527fa  gencode/java/udmi/schema/Category.java
+b95ab79a531c917a5004b5e5bc70e010b33d5d6ff53be09e5dbf2d1df479b569  gencode/java/udmi/schema/Category.java
 d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b  gencode/java/udmi/schema/CloudIotConfig.java
 a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d  gencode/java/udmi/schema/CloudModel.java
 ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4  gencode/java/udmi/schema/Config.java
 10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05  gencode/java/udmi/schema/DeviceMappingConfig.java
 5ffb3bf92436c469eee16fe1e472efc89d1c466785be6ce30148188439314afe  gencode/java/udmi/schema/DeviceMappingState.java
+42e1f71656b94a2a47c3bd084e970604fbb953dc50bd74fcced7ea06adf4d2c2  gencode/java/udmi/schema/DevicePersistent.java
 d4a82e132b2d223c4a82384d22455283c0b64aba3100fc210e72f9700b9df2d0  gencode/java/udmi/schema/DeviceValidationEvent.java
 587d67a67431349939dffd37b880c44e798a1eb607d54bd6d8a077bbac668067  gencode/java/udmi/schema/DiscoveryCommand.java
 d8a80ab3180d33bfa17564c969018e1d4350a47dbc70c4ae8a5abbfb25cfedc9  gencode/java/udmi/schema/DiscoveryConfig.java
@@ -63,7 +65,7 @@ b815ba1f198fd32b11fbedf71e5fa820bba08713dc5c95603c700dfa21ec5904  gencode/java/u
 7f790aa98c03fcd00826a48fcf4dca7eda592abac994e5073c70be8dbd9b6745  gencode/java/udmi/schema/MappingEvent.java
 d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886  gencode/java/udmi/schema/MappingState.java
 304164da05dc722b6e94cfa68659f0120e2425c94bfb5f5a4c6d796fe13da885  gencode/java/udmi/schema/Metadata.java
-a4e8f69100ab678a8236f481c558d677bbaea3e76c853bbd9262113d2a9c031d  gencode/java/udmi/schema/Metrics.java
+c7659cd5f239dc20ce3cae84a0e38bbfd9de346e5963a14e245e1163a8d37a52  gencode/java/udmi/schema/Metrics.java
 5e1c5411fae4d7c47391ceb5d19ae864fcd484df75ac6b6db39fd2d12647dec8  gencode/java/udmi/schema/Physical_tag.java
 d808259db6bbcd26ecf438844ce286d15d4750906be24588d97acfbe8a4ae315  gencode/java/udmi/schema/PointEnumerationEvent.java
 9855400da64cb7c9501294b72d0190caa646fab043b39a6c7c5e4b0250515f20  gencode/java/udmi/schema/PointPointsetConfig.java
@@ -96,7 +98,7 @@ d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/u
 ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4  gencode/java/udmi/schema/ValidationEvent.java
 f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2  gencode/java/udmi/schema/ValidationState.java
 e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36  gencode/java/udmi/schema/ValidationSummary.java
-07125fcc743ca05483e52fcd806488e98eb3716413c1f928a5d1896ed7f52496  gencode/python/udmi/schema/__init__.py
+0601b86702f0e8959a391342307cd782be91e7dcfa0017089230a38e76661dd5  gencode/python/udmi/schema/__init__.py
 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db  gencode/python/udmi/schema/ancillary_properties.py
 5ecd6c542f33450cb4ce75d940a6dff4d3bd67d4b9de4aff5ee88abcc301dbff  gencode/python/udmi/schema/building_config.py
 dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6  gencode/python/udmi/schema/building_translation.py
@@ -128,7 +130,7 @@ ad33b91a7fabb4eed7e49c30a983a2106c96330facbe0f376f94d06e2263d6d0  gencode/python
 5f41d6252d46f7b63e7a4cac731739b3e672f207eae2c65d448dfefc39b8a2ca  gencode/python/udmi/schema/event_mapping.py
 ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b  gencode/python/udmi/schema/event_pointset.py
 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00  gencode/python/udmi/schema/event_pointset_point.py
-c3bf9959c821ccc8d0847a2e022e847ac3da8309dc6d35681af3d20148464ee4  gencode/python/udmi/schema/event_system.py
+0c4fa4205fedd58f156edfa53a17e795599163ec327bf4e8d048309750798789  gencode/python/udmi/schema/event_system.py
 fb8a8a0e09eca4bb061c6cb2ff91b8eee77203e6ea353d3356e411692e1e4f2a  gencode/python/udmi/schema/event_validation.py
 2bb36986f805c2127f0c300de07bce07897c0c6d8f504bddec6e201d0f0ab1bf  gencode/python/udmi/schema/event_validation_device.py
 42f3429edf4a187be6ebfe9760f5e14821a25afa8f7c65b69e0e34fb29a7932e  gencode/python/udmi/schema/metadata.py
@@ -145,6 +147,7 @@ ee9c02c35438fb7d9aacb15a21ec7b35b533c1000d0bde044ec3923b1fdccca4  gencode/python
 aafe6e70c281152db958adf77a024e3e9fab8293927106297c5ec48c11f54e27  gencode/python/udmi/schema/model_testing.py
 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f  gencode/python/udmi/schema/model_testing_target.py
 a10a7e593d8747ea46c56da2c24c7b0501c10b190c5b66119cab8e71d2bdc0e6  gencode/python/udmi/schema/options_pubber.py
+6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77  gencode/python/udmi/schema/persistent_device.py
 a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python/udmi/schema/properties.py
 1f521678016ad267ad1c817896c7900ba30a85fc694669577b71148193db32c1  gencode/python/udmi/schema/reflect_config.py
 1a4eef286957d77418777eb7f4cd1ee13aa24c2d916ec71a7ff4e56de76e303e  gencode/python/udmi/schema/reflect_state.py

bin/test_redirect

@@ -22,6 +22,10 @@ site_path=sites/udmi_site_model
 device_id=AHU-1 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
 mkdir -p out
 
+out_base=$site_path/devices/$device_id/out # For officially generated files (in repo)
+work_base=$site_path/out/devices/$device_id # For runtime out files (transitory)
+rm -rf $work_base
+
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
 
@@ -72,7 +76,6 @@ cat <<EOF > out/blobs.json
 }
 EOF
 
-out_base=$site_path/devices/$device_id/out
 # Merge JSON files together into new redirect config
 jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json
 jq '.system.mode = "restart"' $out_base/redirect_config.json > $out_base/restart_config.json
@@ -103,4 +106,11 @@ bin/reset_config $site_path $project_id $device_id shutdown_config.json
 echo And let it settle for last start...
 sleep 20
 
+# Quick sanity check that the restart_count persistent store is handled correctly.
+restarts=$(jq .restart_count $work_base/persistent_data.json)
+if [[ $restarts -ne 3 ]]; then
+    echo Restart count incorrect, was $restarts
+    false
+fi
+
 echo Done with redirect test.

bin/test_validator

@@ -38,13 +38,14 @@ pubber/bin/build
 
 bin/reset_config $site_path $project_id $device_id
 
-bin/validator $site_path $project_id &
+echo Starting validator, output in $VALIDATOR_OUT
+bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 &
 vpid=$!
-sleep 10
 echo Started validator pid $vpid
 
-echo Starting validator, output in $VALIDATOR_OUT
-bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 &
+# Just give it a head-start, actual waiting check is the loop below
+sleep 10
+
 for i in `seq 1 $WAITING`; do
     if fgrep "Entering message loop" $VALIDATOR_OUT; then
         break
@@ -58,7 +59,6 @@ if [[ $i -eq $WAITING ]]; then
     cat $VALIDATOR_OUT
     false
 fi
-echo Started validator pid $vpid
 
 echo Writing pubber output to $PUBBER_OUT
 cmd="bin/pubber $site_path $project_id $device_id $serial_no extraField=prlagle"
@@ -80,7 +80,7 @@ if [[ $i -eq $WAITING ]]; then
 fi
 
 echo Waiting for system to run for a bit...
-timeout 60s tail -f $VALIDATOR_OUT || true
+timeout 30s tail -f $VALIDATOR_OUT || true
 
 echo Killing running pubber/validator instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -1,10 +1,21 @@
 package com.google.udmi.util;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
+import java.io.File;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
 
 public class GeneralUtils {
+
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
+      .enable(SerializationFeature.INDENT_OUTPUT)
+      .setDateFormat(new ISO8601DateFormat())
+      .setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
   /**
    * Returns a string of enabled options and values.
    */
@@ -26,5 +37,51 @@ public static String optionsString(Object target) {
     }
     return String.join(" ", options);
   }
-
+
+  public static <T> T fromJsonFile(File path, Class<T> valueType) {
+    try {
+      return OBJECT_MAPPER.readValue(path, valueType);
+    } catch (Exception e) {
+      throw new RuntimeException("While loading json file " + path.getAbsolutePath(), e);
+    }
+  }
+
+  public static <T> T fromJsonString(String body, Class<T> valueType) {
+    try {
+      if (body == null) {
+        return null;
+      }
+      return OBJECT_MAPPER.readValue(body, valueType);
+    } catch (Exception e) {
+      throw new RuntimeException("While loading json string", e);
+    }
+  }
+
+  public static String toJsonString(Object object) {
+    try {
+      if (object == null) {
+        return null;
+      }
+      return OBJECT_MAPPER.writeValueAsString(object);
+    } catch (Exception e) {
+      throw new RuntimeException("While converting object to json", e);
+    }
+  }
+
+  public static void toJsonFile(File file, Object target) {
+    try {
+      OBJECT_MAPPER.writeValue(file, target);
+    } catch (Exception e) {
+      throw new RuntimeException("While writing target " + file.getAbsolutePath(), e);
+    }
+  }
+
+  public static <T> T deepCopy(T endpoint1, Class<T> valueType) {
+    try {
+      return OBJECT_MAPPER.readValue(toJsonString(endpoint1),
+          valueType);
+    } catch (Exception e) {
+      throw new RuntimeException("While making deep copy of " + valueType.getName(), e);
+    }
+  }
 }

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -221,6 +221,18 @@ public String validatorKey() {
     return sitePath + "/validator/rsa_private.pkcs8";
   }
 
+  public File getDeviceWorkingDir(String deviceId) {
+    File file = new File(sitePath + "/out/devices/" + deviceId);
+    if (!file.exists()) {
+      file.mkdirs();
+    }
+    if (!file.isDirectory()) {
+      throw new RuntimeException(
+          "Device working dir is not a valid directory: " + file.getAbsolutePath());
+    }
+    return file;
+  }
+
   public static class ClientInfo {
 
     public String cloudRegion;

docs/specs/categories.md

@@ -50,7 +50,7 @@ implicit expected `level` values, indicated by '(**LEVEL**)' in the hierarchy be
 * _validation_: Handling validation pipeline messages
   * _device_: Conditions specific to processing a given device message.
     * _receive_: (**DEBUG**) Receiving/processing a message for validation.
-    * _result_: (**INFO**) Regarging the actual validation results.
+    * _result_: (**INFO**) Regarding the actual validation results.
   * _summary_: Conditions specific to an overall site summary.
     * _report_: (**INFO**) The validation summary report.
 * _device_: Device specific messages (ignored by UDMI system)