diff --git a/.gencode_hash.txt b/.gencode_hash.txt index f03141fc5..ad66d005e 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -6,15 +6,16 @@ d93322f64b94bc1808c59cde7b83915ff7e14d50401b9c2ae8467730ed15257c gencode/docs/c bbb98edbbc279678856c11024f4681261f0b9b58b07f8de6026be70d2e32074a gencode/docs/configuration_endpoint.html 299bc6f4d9842fab2d2cfd003d8a8adf94303c560cf71e8595d85cf661558a4d gencode/docs/configuration_pubber.html 82fbf41e4e88593f22a929fc8ade4d1e8f75a0fc1aa8452ea1d3b19023bee314 gencode/docs/envelope.html -276158157fdf8989edf8ef9a53874246fc6e87412aea410712496de9759e922c gencode/docs/event.html +5888a5c257b62851dcaed9cf5e015240e0b1882d88cc1f1329402b285b1eeb4b gencode/docs/event.html a8800ab8384de3b846af0d72cf310acf01644842b557e42eec20b69a0bfa1868 gencode/docs/event_discovery.html 3019d06826576d287edbf69116ec712c74c8fdfcc5653b51f2573cb9755ebb23 gencode/docs/event_mapping.html 71fe25d3b17d1dc87b52c049a3235c20527f73e12ffa53fa4800a15e53f73732 gencode/docs/event_pointset.html -7adf725427f285539b37abdad0c15f32e76bd6e1ec7d775527491597aef32875 gencode/docs/event_system.html +76a0d1c4baf4b74b65c9be862ca114dbee00552ba59e75978215db1c6c7fc0ee gencode/docs/event_system.html 43025f61381de050eae9680c3193a1d878a397d0b5515dd1063a1a993b0c41ea gencode/docs/event_validation.html 9ae793a641f2e053c5af734bf813cd2637a675330acb8d9e903ef03b4286e007 gencode/docs/metadata.html +5468d640a0a4467d21dccd7f0440a7ca84c7de1102596011494ac9d3b3e9f6be gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html -162f8f3b8ca9491e7dc1d2569d4773acae51f93b07fa875a9ece218d9d4f8a89 gencode/docs/readme.md +846eaf1735bd2506357173cfc720c8674d95acf2d20e19f37e3d34cad0a345d2 gencode/docs/readme.md 690d56e96c4e180e49e0abcba358ab1842ea06dd3f693918852a9de370c994d5 gencode/docs/reflect_config.html 9332e44c87dae9261b079424e748d5ee1df08a3c6b39987d254ebf78274e2f34 gencode/docs/reflect_state.html 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css @@ -30,12 +31,13 @@ a7c57d119adcd0cf6363cc5301ce562004222522242e8ffd1d0cd7010f235ae1 gencode/java/u b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc gencode/java/udmi/schema/BlobsetConfig.java fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/udmi/schema/BlobsetState.java 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e gencode/java/udmi/schema/BuildingTranslation.java -5655b896f000b0108f438f9b04bfc092764ad5af63b70bbf750ba58b5f5527fa gencode/java/udmi/schema/Category.java +b95ab79a531c917a5004b5e5bc70e010b33d5d6ff53be09e5dbf2d1df479b569 gencode/java/udmi/schema/Category.java d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b gencode/java/udmi/schema/CloudIotConfig.java a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d gencode/java/udmi/schema/CloudModel.java ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4 gencode/java/udmi/schema/Config.java 10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05 gencode/java/udmi/schema/DeviceMappingConfig.java 5ffb3bf92436c469eee16fe1e472efc89d1c466785be6ce30148188439314afe gencode/java/udmi/schema/DeviceMappingState.java +42e1f71656b94a2a47c3bd084e970604fbb953dc50bd74fcced7ea06adf4d2c2 gencode/java/udmi/schema/DevicePersistent.java d4a82e132b2d223c4a82384d22455283c0b64aba3100fc210e72f9700b9df2d0 gencode/java/udmi/schema/DeviceValidationEvent.java 587d67a67431349939dffd37b880c44e798a1eb607d54bd6d8a077bbac668067 gencode/java/udmi/schema/DiscoveryCommand.java d8a80ab3180d33bfa17564c969018e1d4350a47dbc70c4ae8a5abbfb25cfedc9 gencode/java/udmi/schema/DiscoveryConfig.java @@ -63,7 +65,7 @@ b815ba1f198fd32b11fbedf71e5fa820bba08713dc5c95603c700dfa21ec5904 gencode/java/u 7f790aa98c03fcd00826a48fcf4dca7eda592abac994e5073c70be8dbd9b6745 gencode/java/udmi/schema/MappingEvent.java d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886 gencode/java/udmi/schema/MappingState.java 304164da05dc722b6e94cfa68659f0120e2425c94bfb5f5a4c6d796fe13da885 gencode/java/udmi/schema/Metadata.java -a4e8f69100ab678a8236f481c558d677bbaea3e76c853bbd9262113d2a9c031d gencode/java/udmi/schema/Metrics.java +c7659cd5f239dc20ce3cae84a0e38bbfd9de346e5963a14e245e1163a8d37a52 gencode/java/udmi/schema/Metrics.java 5e1c5411fae4d7c47391ceb5d19ae864fcd484df75ac6b6db39fd2d12647dec8 gencode/java/udmi/schema/Physical_tag.java d808259db6bbcd26ecf438844ce286d15d4750906be24588d97acfbe8a4ae315 gencode/java/udmi/schema/PointEnumerationEvent.java 9855400da64cb7c9501294b72d0190caa646fab043b39a6c7c5e4b0250515f20 gencode/java/udmi/schema/PointPointsetConfig.java @@ -96,7 +98,7 @@ d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645 gencode/java/u ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4 gencode/java/udmi/schema/ValidationEvent.java f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2 gencode/java/udmi/schema/ValidationState.java e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36 gencode/java/udmi/schema/ValidationSummary.java -07125fcc743ca05483e52fcd806488e98eb3716413c1f928a5d1896ed7f52496 gencode/python/udmi/schema/__init__.py +0601b86702f0e8959a391342307cd782be91e7dcfa0017089230a38e76661dd5 gencode/python/udmi/schema/__init__.py 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db gencode/python/udmi/schema/ancillary_properties.py 5ecd6c542f33450cb4ce75d940a6dff4d3bd67d4b9de4aff5ee88abcc301dbff gencode/python/udmi/schema/building_config.py dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6 gencode/python/udmi/schema/building_translation.py @@ -128,7 +130,7 @@ ad33b91a7fabb4eed7e49c30a983a2106c96330facbe0f376f94d06e2263d6d0 gencode/python 5f41d6252d46f7b63e7a4cac731739b3e672f207eae2c65d448dfefc39b8a2ca gencode/python/udmi/schema/event_mapping.py ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b gencode/python/udmi/schema/event_pointset.py 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00 gencode/python/udmi/schema/event_pointset_point.py -c3bf9959c821ccc8d0847a2e022e847ac3da8309dc6d35681af3d20148464ee4 gencode/python/udmi/schema/event_system.py +0c4fa4205fedd58f156edfa53a17e795599163ec327bf4e8d048309750798789 gencode/python/udmi/schema/event_system.py fb8a8a0e09eca4bb061c6cb2ff91b8eee77203e6ea353d3356e411692e1e4f2a gencode/python/udmi/schema/event_validation.py 2bb36986f805c2127f0c300de07bce07897c0c6d8f504bddec6e201d0f0ab1bf gencode/python/udmi/schema/event_validation_device.py 42f3429edf4a187be6ebfe9760f5e14821a25afa8f7c65b69e0e34fb29a7932e gencode/python/udmi/schema/metadata.py @@ -145,6 +147,7 @@ ee9c02c35438fb7d9aacb15a21ec7b35b533c1000d0bde044ec3923b1fdccca4 gencode/python aafe6e70c281152db958adf77a024e3e9fab8293927106297c5ec48c11f54e27 gencode/python/udmi/schema/model_testing.py 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f gencode/python/udmi/schema/model_testing_target.py a10a7e593d8747ea46c56da2c24c7b0501c10b190c5b66119cab8e71d2bdc0e6 gencode/python/udmi/schema/options_pubber.py +6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77 gencode/python/udmi/schema/persistent_device.py a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690 gencode/python/udmi/schema/properties.py 1f521678016ad267ad1c817896c7900ba30a85fc694669577b71148193db32c1 gencode/python/udmi/schema/reflect_config.py 1a4eef286957d77418777eb7f4cd1ee13aa24c2d916ec71a7ff4e56de76e303e gencode/python/udmi/schema/reflect_state.py diff --git a/bin/test_redirect b/bin/test_redirect index fa9341919..8a26b5ba0 100755 --- a/bin/test_redirect +++ b/bin/test_redirect @@ -22,6 +22,10 @@ site_path=sites/udmi_site_model device_id=AHU-1 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI. mkdir -p out +out_base=$site_path/devices/$device_id/out # For officially generated files (in repo) +work_base=$site_path/out/devices/$device_id # For runtime out files (transitory) +rm -rf $work_base + serial_no=sequencer-$RANDOM echo Using pubber with serial $serial_no @@ -72,7 +76,6 @@ cat < out/blobs.json } EOF -out_base=$site_path/devices/$device_id/out # Merge JSON files together into new redirect config jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json jq '.system.mode = "restart"' $out_base/redirect_config.json > $out_base/restart_config.json @@ -103,4 +106,11 @@ bin/reset_config $site_path $project_id $device_id shutdown_config.json echo And let it settle for last start... sleep 20 +# Quick sanity check that the restart_count persistent store is handled correctly. +restarts=$(jq .restart_count $work_base/persistent_data.json) +if [[ $restarts -ne 3 ]]; then + echo Restart count incorrect, was $restarts + false +fi + echo Done with redirect test. diff --git a/bin/test_validator b/bin/test_validator index 738cce7ae..fed3ea184 100755 --- a/bin/test_validator +++ b/bin/test_validator @@ -38,13 +38,14 @@ pubber/bin/build bin/reset_config $site_path $project_id $device_id -bin/validator $site_path $project_id & +echo Starting validator, output in $VALIDATOR_OUT +bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 & vpid=$! -sleep 10 echo Started validator pid $vpid -echo Starting validator, output in $VALIDATOR_OUT -bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 & +# Just give it a head-start, actual waiting check is the loop below +sleep 10 + for i in `seq 1 $WAITING`; do if fgrep "Entering message loop" $VALIDATOR_OUT; then break @@ -58,7 +59,6 @@ if [[ $i -eq $WAITING ]]; then cat $VALIDATOR_OUT false fi -echo Started validator pid $vpid echo Writing pubber output to $PUBBER_OUT cmd="bin/pubber $site_path $project_id $device_id $serial_no extraField=prlagle" @@ -80,7 +80,7 @@ if [[ $i -eq $WAITING ]]; then fi echo Waiting for system to run for a bit... -timeout 60s tail -f $VALIDATOR_OUT || true +timeout 30s tail -f $VALIDATOR_OUT || true echo Killing running pubber/validator instances... ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index ccd1ca621..8599b02f2 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -1,10 +1,21 @@ package com.google.udmi.util; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import java.io.File; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; public class GeneralUtils { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT) + .setDateFormat(new ISO8601DateFormat()) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + /** * Returns a string of enabled options and values. */ @@ -26,5 +37,51 @@ public static String optionsString(Object target) { } return String.join(" ", options); } - + + public static T fromJsonFile(File path, Class valueType) { + try { + return OBJECT_MAPPER.readValue(path, valueType); + } catch (Exception e) { + throw new RuntimeException("While loading json file " + path.getAbsolutePath(), e); + } + } + + public static T fromJsonString(String body, Class valueType) { + try { + if (body == null) { + return null; + } + return OBJECT_MAPPER.readValue(body, valueType); + } catch (Exception e) { + throw new RuntimeException("While loading json string", e); + } + } + + public static String toJsonString(Object object) { + try { + if (object == null) { + return null; + } + return OBJECT_MAPPER.writeValueAsString(object); + } catch (Exception e) { + throw new RuntimeException("While converting object to json", e); + } + } + + public static void toJsonFile(File file, Object target) { + try { + OBJECT_MAPPER.writeValue(file, target); + } catch (Exception e) { + throw new RuntimeException("While writing target " + file.getAbsolutePath(), e); + } + } + + public static T deepCopy(T endpoint1, Class valueType) { + try { + return OBJECT_MAPPER.readValue(toJsonString(endpoint1), + valueType); + } catch (Exception e) { + throw new RuntimeException("While making deep copy of " + valueType.getName(), e); + } + } } diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 4b10e912f..8f059914a 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -221,6 +221,18 @@ public String validatorKey() { return sitePath + "/validator/rsa_private.pkcs8"; } + public File getDeviceWorkingDir(String deviceId) { + File file = new File(sitePath + "/out/devices/" + deviceId); + if (!file.exists()) { + file.mkdirs(); + } + if (!file.isDirectory()) { + throw new RuntimeException( + "Device working dir is not a valid directory: " + file.getAbsolutePath()); + } + return file; + } + public static class ClientInfo { public String cloudRegion; diff --git a/docs/specs/categories.md b/docs/specs/categories.md index b66cf539f..27cab5cc1 100644 --- a/docs/specs/categories.md +++ b/docs/specs/categories.md @@ -50,7 +50,7 @@ implicit expected `level` values, indicated by '(**LEVEL**)' in the hierarchy be * _validation_: Handling validation pipeline messages * _device_: Conditions specific to processing a given device message. * _receive_: (**DEBUG**) Receiving/processing a message for validation. - * _result_: (**INFO**) Regarging the actual validation results. + * _result_: (**INFO**) Regarding the actual validation results. * _summary_: Conditions specific to an overall site summary. * _report_: (**INFO**) The validation summary report. * _device_: Device specific messages (ignored by UDMI system) diff --git a/gencode/docs/event.html b/gencode/docs/event.html index ba611e7db..43c1d4eb0 100644 --- a/gencode/docs/event.html +++ b/gencode/docs/event.html @@ -2179,7 +2179,7 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - restart_countType: number
+ restart_countType: integer
diff --git a/gencode/docs/event_system.html b/gencode/docs/event_system.html index bab959225..0784d50fa 100644 --- a/gencode/docs/event_system.html +++ b/gencode/docs/event_system.html @@ -1881,7 +1881,7 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - restart_countType: number
+ restart_countType: integer
diff --git a/gencode/docs/persistent_device.html b/gencode/docs/persistent_device.html new file mode 100644 index 000000000..5fc2bda72 --- /dev/null +++ b/gencode/docs/persistent_device.html @@ -0,0 +1,341 @@ + + + + + + + + + + + + + + + + Device Persistent + +

Device Persistent

+ + +
+ +
Type: object
+

Device persistent data

+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + endpoint
Type: object
+

Parameters to define an MQTT endpoint

+ + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + endpoint + + + + protocol
Type: enum (of string)
+
+

Must be one of:

+
  • "mqtt"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + endpoint + + + + hostname
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + endpoint + + + + port
Type: string Default: "8883"
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + endpoint + + + + client_id
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ +
root + + + + endpoint + + + + additionalProperties
Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + restart_count
Type: integer
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ +
root + + + + additionalProperties
Type: object
+ + + + + + + +
+
+
+
+ + + \ No newline at end of file diff --git a/gencode/docs/readme.md b/gencode/docs/readme.md index b602ed47a..a69eb18ab 100644 --- a/gencode/docs/readme.md +++ b/gencode/docs/readme.md @@ -28,6 +28,7 @@ * [**event**](event.html) - Container object for all event schemas, not directly used. * [**event_mapping**](event_mapping.html) - [Mapping result](../../docs/specs/mapping.md) with implicit enumeration * [**event_validation**](event_validation.html) - Validation device result +* [**persistent_device**](persistent_device.html) - Device persistent data * [**properties**](properties.html) * [**reflect_config**](reflect_config.html) - Config for a reflector client * [**reflect_state**](reflect_state.html) - State of a reflector client diff --git a/gencode/java/udmi/schema/Category.java b/gencode/java/udmi/schema/Category.java index 941490a2d..af93b8ec2 100644 --- a/gencode/java/udmi/schema/Category.java +++ b/gencode/java/udmi/schema/Category.java @@ -135,7 +135,7 @@ public class Category { public static final Level VALIDATION_DEVICE_RECEIVE_LEVEL = DEBUG; static { LEVEL.put(VALIDATION_DEVICE_RECEIVE, DEBUG); } - // Regarging the actual validation results. + // Regarding the actual validation results. public static final String VALIDATION_DEVICE_RESULT = "validation.device.result"; public static final Level VALIDATION_DEVICE_RESULT_LEVEL = INFO; static { LEVEL.put(VALIDATION_DEVICE_RESULT, INFO); } diff --git a/gencode/java/udmi/schema/DevicePersistent.java b/gencode/java/udmi/schema/DevicePersistent.java new file mode 100644 index 000000000..951b018f5 --- /dev/null +++ b/gencode/java/udmi/schema/DevicePersistent.java @@ -0,0 +1,57 @@ + +package udmi.schema; + +import javax.annotation.processing.Generated; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + + +/** + * Device Persistent + *

+ * Device persistent data + * + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "endpoint", + "restart_count" +}) +@Generated("jsonschema2pojo") +public class DevicePersistent { + + /** + * Endpoint Configuration + *

+ * Parameters to define an MQTT endpoint + * + */ + @JsonProperty("endpoint") + @JsonPropertyDescription("Parameters to define an MQTT endpoint") + public EndpointConfiguration endpoint; + @JsonProperty("restart_count") + public Integer restart_count; + + @Override + public int hashCode() { + int result = 1; + result = ((result* 31)+((this.endpoint == null)? 0 :this.endpoint.hashCode())); + result = ((result* 31)+((this.restart_count == null)? 0 :this.restart_count.hashCode())); + return result; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if ((other instanceof DevicePersistent) == false) { + return false; + } + DevicePersistent rhs = ((DevicePersistent) other); + return (((this.endpoint == rhs.endpoint)||((this.endpoint!= null)&&this.endpoint.equals(rhs.endpoint)))&&((this.restart_count == rhs.restart_count)||((this.restart_count!= null)&&this.restart_count.equals(rhs.restart_count)))); + } + +} diff --git a/gencode/java/udmi/schema/Metrics.java b/gencode/java/udmi/schema/Metrics.java index 47cb518c3..a06a6d001 100644 --- a/gencode/java/udmi/schema/Metrics.java +++ b/gencode/java/udmi/schema/Metrics.java @@ -20,7 +20,7 @@ public class Metrics { @JsonProperty("restart_count") - public Double restart_count; + public Integer restart_count; @JsonProperty("mem_total_mb") public Double mem_total_mb; @JsonProperty("mem_free_mb") diff --git a/gencode/python/udmi/schema/__init__.py b/gencode/python/udmi/schema/__init__.py index 40ade7efa..5e25d1f28 100644 --- a/gencode/python/udmi/schema/__init__.py +++ b/gencode/python/udmi/schema/__init__.py @@ -46,6 +46,7 @@ from .model_testing import TestingModel from .model_testing_target import TargetTestingModel from .options_pubber import PubberOptions +from .persistent_device import DevicePersistent from .properties import Properties from .reflect_config import ReflectorConfig from .reflect_state import ReflectorState diff --git a/gencode/python/udmi/schema/event_system.py b/gencode/python/udmi/schema/event_system.py index 4342d2621..bb89bc921 100644 --- a/gencode/python/udmi/schema/event_system.py +++ b/gencode/python/udmi/schema/event_system.py @@ -1,7 +1,7 @@ """Generated class for event_system.json""" -class ObjectB317E398: +class ObjectD1FC597A: """Generated schema class""" def __init__(self): @@ -17,7 +17,7 @@ def __init__(self): def from_dict(source): if not source: return None - result = ObjectB317E398() + result = ObjectD1FC597A() result.restart_count = source.get('restart_count') result.mem_total_mb = source.get('mem_total_mb') result.mem_free_mb = source.get('mem_free_mb') @@ -33,7 +33,7 @@ def map_from(source): return None result = {} for key in source: - result[key] = ObjectB317E398.from_dict(source[key]) + result[key] = ObjectD1FC597A.from_dict(source[key]) return result @staticmethod @@ -79,7 +79,7 @@ def from_dict(source): result.timestamp = source.get('timestamp') result.version = source.get('version') result.logentries = Entry.array_from(source.get('logentries')) - result.metrics = ObjectB317E398.from_dict(source.get('metrics')) + result.metrics = ObjectD1FC597A.from_dict(source.get('metrics')) return result @staticmethod diff --git a/gencode/python/udmi/schema/persistent_device.py b/gencode/python/udmi/schema/persistent_device.py new file mode 100644 index 000000000..ca14a36c3 --- /dev/null +++ b/gencode/python/udmi/schema/persistent_device.py @@ -0,0 +1,43 @@ +"""Generated class for persistent_device.json""" +from .configuration_endpoint import EndpointConfiguration + + +class DevicePersistent: + """Generated schema class""" + + def __init__(self): + self.endpoint = None + self.restart_count = None + + @staticmethod + def from_dict(source): + if not source: + return None + result = DevicePersistent() + result.endpoint = EndpointConfiguration.from_dict(source.get('endpoint')) + result.restart_count = source.get('restart_count') + return result + + @staticmethod + def map_from(source): + if not source: + return None + result = {} + for key in source: + result[key] = DevicePersistent.from_dict(source[key]) + return result + + @staticmethod + def expand_dict(input): + result = {} + for property in input: + result[property] = input[property].to_dict() if input[property] else {} + return result + + def to_dict(self): + result = {} + if self.endpoint: + result['endpoint'] = self.endpoint.to_dict() # 4 + if self.restart_count: + result['restart_count'] = self.restart_count # 5 + return result diff --git a/pubber/.idea/compiler.xml b/pubber/.idea/compiler.xml index 61a9130cd..3794d5969 100644 --- a/pubber/.idea/compiler.xml +++ b/pubber/.idea/compiler.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/pubber/.idea/misc.xml b/pubber/.idea/misc.xml index 38e91b9bf..6fed7b5d7 100644 --- a/pubber/.idea/misc.xml +++ b/pubber/.idea/misc.xml @@ -5,7 +5,7 @@ - + \ No newline at end of file diff --git a/pubber/build.gradle b/pubber/build.gradle index f36a3711e..6ee638b63 100644 --- a/pubber/build.gradle +++ b/pubber/build.gradle @@ -19,7 +19,7 @@ plugins { group 'daq-pubber' version '1.0-SNAPSHOT' -sourceCompatibility = 1.8 +sourceCompatibility = 1.9 sourceSets { main { diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index c8d771323..36c044baa 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -2,21 +2,22 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.udmi.util.GeneralUtils.deepCopy; +import static com.google.udmi.util.GeneralUtils.fromJsonFile; +import static com.google.udmi.util.GeneralUtils.fromJsonString; +import static com.google.udmi.util.GeneralUtils.optionsString; +import static com.google.udmi.util.GeneralUtils.toJsonFile; +import static com.google.udmi.util.GeneralUtils.toJsonString; import static java.lang.Boolean.TRUE; import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toMap; import static udmi.schema.BlobsetConfig.SystemBlobsets.IOT_ENDPOINT_CONFIG; import static udmi.schema.EndpointConfiguration.Protocol.MQTT; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.util.CatchingScheduledThreadPoolExecutor; -import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.SiteModel; import com.google.udmi.util.SiteModel.ClientInfo; import daq.pubber.MqttPublisher.PublisherException; @@ -35,6 +36,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledExecutorService; @@ -58,6 +60,7 @@ import udmi.schema.Category; import udmi.schema.CloudModel.Auth_type; import udmi.schema.Config; +import udmi.schema.DevicePersistent; import udmi.schema.DiscoveryConfig; import udmi.schema.DiscoveryEvent; import udmi.schema.DiscoveryState; @@ -71,6 +74,7 @@ import udmi.schema.FamilyLocalnetModel; import udmi.schema.Level; import udmi.schema.Metadata; +import udmi.schema.Metrics; import udmi.schema.PointEnumerationEvent; import udmi.schema.PointPointsetConfig; import udmi.schema.PointPointsetModel; @@ -93,12 +97,10 @@ public class Pubber { public static final int SCAN_DURATION_SEC = 10; public static final String PUBBER_OUT = "pubber/out"; + public static final String PERSISTENT_STORE_FILE = "persistent_data.json"; + public static final String PERSISTENT_TMP_FORMAT = "/tmp/pubber_%s_" + PERSISTENT_STORE_FILE; private static final String UDMI_VERSION = "1.3.14"; private static final Logger LOG = LoggerFactory.getLogger(Pubber.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() - .enable(SerializationFeature.INDENT_OUTPUT) - .setDateFormat(new ISO8601DateFormat()) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); private static final String HOSTNAME = System.getenv("HOSTNAME"); private static final String CONFIG_TOPIC = "config"; private static final String ERROR_TOPIC = "errors"; @@ -139,6 +141,8 @@ public class Pubber { private static final int CONNECT_RETRIES = 10; private static final AtomicInteger retriesRemaining = new AtomicInteger(CONNECT_RETRIES); private static final long RESTART_DELAY_MS = 1000; + private static final long BYTES_PER_MEGABYTE = 1024 * 1024; + public static final String PUBBER_LOG_CATEGORY = "device.log"; private final File outDir; private final ScheduledExecutorService executor = new CatchingScheduledThreadPoolExecutor(1); private final PubberConfiguration configuration; @@ -152,20 +156,19 @@ public class Pubber { private final String projectId; private final String deviceId; private Config deviceConfig = new Config(); - private int deviceMessageCount = -1; + private int deviceUpdateCount = -1; private MqttPublisher mqttPublisher; private ScheduledFuture periodicSender; private long lastStateTimeMs; private PubSubClient pubSubClient; private Function connectionDone; private boolean publishingLog; - private String appliedEndpoint; private String workingEndpoint; private String attemptedEndpoint; private EndpointConfiguration extractedEndpoint; private SiteModel siteModel; private PrintStream logPrintWriter; - private Entry endpointStatus; + private DevicePersistent persistentData; /** * Start an instance from a configuration file. @@ -175,8 +178,7 @@ public class Pubber { public Pubber(String configPath) { File configFile = new File(configPath); try { - configuration = sanitizeConfiguration( - OBJECT_MAPPER.readValue(configFile, PubberConfiguration.class)); + configuration = sanitizeConfiguration(fromJsonFile(configFile, PubberConfiguration.class)); checkArgument(MQTT.equals(configuration.endpoint.protocol), "protocol mismatch"); ClientInfo clientInfo = SiteModel.parseClientId(configuration.endpoint.client_id); projectId = clientInfo.projectId; @@ -355,9 +357,11 @@ private void initializeDevice() { pullDeviceMessage(); } + initializePersistentStore(); + info(String.format("Starting pubber %s, serial %s, mac %s, gateway %s, options %s", configuration.deviceId, configuration.serialNo, configuration.macAddr, - configuration.gatewayId, GeneralUtils.optionsString(configuration.options))); + configuration.gatewayId, optionsString(configuration.options))); deviceState.system.operational = true; deviceState.system.mode = SystemMode.INITIAL; @@ -385,6 +389,27 @@ private void initializeDevice() { markStateDirty(0); } + private void initializePersistentStore() { + Preconditions.checkState(persistentData == null, "persistent data already loaded"); + File persistentStore = getPersistentStore(); + info("Initializing from persistent store " + persistentStore.getAbsolutePath()); + persistentData = + persistentStore.exists() ? fromJsonFile(persistentStore, DevicePersistent.class) + : new DevicePersistent(); + persistentData.restart_count = Objects.requireNonNullElse(persistentData.restart_count, 0) + 1; + writePersistentStore(); + } + + private void writePersistentStore() { + Preconditions.checkState(persistentData != null, "persistent data not defined"); + toJsonFile(getPersistentStore(), persistentData); + } + + private File getPersistentStore() { + return siteModel == null ? new File(String.format(PERSISTENT_TMP_FORMAT, deviceId)) : + new File(siteModel.getDeviceWorkingDir(deviceId), PERSISTENT_STORE_FILE); + } + private void markStateDirty(long delayMs) { stateDirty.set(true); if (delayMs >= 0) { @@ -410,7 +435,7 @@ private void pullDeviceMessage() { attributes.deviceId = pull.attributes.get("deviceId"); attributes.deviceRegistryId = pull.attributes.get("deviceRegistryId"); attributes.deviceRegistryLocation = pull.attributes.get("deviceRegistryLocation"); - SwarmMessage swarm = OBJECT_MAPPER.readValue(pull.body, SwarmMessage.class); + SwarmMessage swarm = fromJsonString(pull.body, SwarmMessage.class); processSwarmConfig(swarm, attributes); return; } catch (Exception e) { @@ -479,7 +504,7 @@ private synchronized void startPeriodicSend() { Preconditions.checkState(periodicSender == null); int delay = messageDelayMs.get(); info("Starting executor with send message delay " + delay); - periodicSender = executor.scheduleAtFixedRate(this::sendMessages, delay, delay, + periodicSender = executor.scheduleAtFixedRate(this::periodicUpdate, delay, delay, TimeUnit.MILLISECONDS); } @@ -495,17 +520,29 @@ private synchronized void cancelPeriodicSend() { } } - private void sendMessages() { + private void periodicUpdate() { try { + deviceUpdateCount++; updatePoints(); deferredConfigActions(); - sendDeviceMessage(); + sendDevicePoints(); + sendSystemMetrics(); flushDirtyState(); } catch (Exception e) { error("Fatal error during execution", e); } } + private void sendSystemMetrics() { + SystemEvent systemEvent = new SystemEvent(); + systemEvent.metrics = new Metrics(); + systemEvent.metrics.restart_count = persistentData.restart_count; + Runtime runtime = Runtime.getRuntime(); + systemEvent.metrics.mem_free_mb = (double) runtime.freeMemory() / BYTES_PER_MEGABYTE; + systemEvent.metrics.mem_total_mb = (double) runtime.totalMemory() / BYTES_PER_MEGABYTE; + publishDeviceMessage(systemEvent); + } + private void deferredConfigActions() { maybeRedirectEndpoint(); maybeRestartSystem(); @@ -679,7 +716,6 @@ private void initializeMqtt() { } Preconditions.checkState(mqttPublisher == null, "mqttPublisher already defined"); ensureKeyBytes(); - appliedEndpoint = toJson(configuration.endpoint); mqttPublisher = new MqttPublisher(configuration, this::publisherException); if (configuration.gatewayId != null) { mqttPublisher.registerHandler(configuration.gatewayId, CONFIG_TOPIC, @@ -691,17 +727,6 @@ private void initializeMqtt() { this::configHandler, Config.class); } - private String toJson(Object target) { - try { - if (target == null) { - return null; - } - return OBJECT_MAPPER.writeValueAsString(target); - } catch (Exception e) { - throw new RuntimeException("While converting object to string", e); - } - } - private void ensureKeyBytes() { if (configuration.keyBytes != null) { return; @@ -716,7 +741,7 @@ private void connect() { try { mqttPublisher.connect(configuration.deviceId); info("Connection complete."); - workingEndpoint = toJson(configuration.endpoint); + workingEndpoint = toJsonString(configuration.endpoint); } catch (Exception e) { throw new RuntimeException("Connection error", e); } @@ -796,12 +821,9 @@ private void configHandler(Config config) { try { info("Config handler"); File configOut = new File(outDir, traceTimestamp("config") + ".json"); - try { - OBJECT_MAPPER.writeValue(configOut, config); - debug(String.format("Config update%s", getTestingTag(config)), toJson(config)); - } catch (Exception e) { - throw new RuntimeException("While writing config " + configOut.getPath(), e); - } + toJsonFile(configOut, config); + debug(String.format("Config update%s", getTestingTag(config)), + toJsonString(config)); processConfigUpdate(config); configLatch.countDown(); publisherConfigLog("apply", null); @@ -835,8 +857,7 @@ private void extractEndpointBlobConfig() { } try { String iotConfig = extractConfigBlob(IOT_ENDPOINT_CONFIG.value()); - extractedEndpoint = iotConfig == null ? null - : OBJECT_MAPPER.readValue(iotConfig, EndpointConfiguration.class); + extractedEndpoint = fromJsonString(iotConfig, EndpointConfiguration.class); } catch (Exception e) { throw new RuntimeException("While extracting endpoint blob config", e); } @@ -855,9 +876,10 @@ private void removeBlobsetBlobState(SystemBlobsets blobId) { private void maybeRedirectEndpoint() { String redirectRegistry = configuration.options.redirectRegistry; - String currentSignature = toJson(configuration.endpoint); + String currentSignature = toJsonString(configuration.endpoint); String extractedSignature = - redirectRegistry == null ? toJson(extractedEndpoint) : redirectedEndpoint(redirectRegistry); + redirectRegistry == null ? toJsonString(extractedEndpoint) + : redirectedEndpoint(redirectRegistry); if (extractedSignature == null) { attemptedEndpoint = null; @@ -867,7 +889,7 @@ private void maybeRedirectEndpoint() { BlobBlobsetState endpointState = ensureBlobsetState(IOT_ENDPOINT_CONFIG); - if (extractedSignature.equals(currentSignature) + if (extractedSignature.equals(currentSignature) || extractedSignature.equals(attemptedEndpoint)) { return; // No need to redirect anything! } @@ -881,7 +903,6 @@ private void maybeRedirectEndpoint() { publishSynchronousState(); resetConnection(extractedSignature); endpointState.phase = BlobPhase.FINAL; - appliedEndpoint = null; } catch (Exception e) { try { error("Reconfigure failed, attempting connection to last working endpoint", e); @@ -899,10 +920,10 @@ private void maybeRedirectEndpoint() { private String redirectedEndpoint(String redirectRegistry) { try { - EndpointConfiguration endpoint = OBJECT_MAPPER.readValue(toJson(configuration.endpoint), + EndpointConfiguration endpoint = deepCopy(configuration.endpoint, EndpointConfiguration.class); endpoint.client_id = getClientId(redirectRegistry); - return toJson(endpoint); + return toJsonString(endpoint); } catch (Exception e) { throw new RuntimeException("While getting redirected endpoint"); } @@ -910,7 +931,7 @@ private String redirectedEndpoint(String redirectRegistry) { private void resetConnection(String targetEndpoint) { try { - configuration.endpoint = OBJECT_MAPPER.readValue(targetEndpoint, + configuration.endpoint = fromJsonString(targetEndpoint, EndpointConfiguration.class); disconnectMqtt(); initializeMqtt(); @@ -1196,7 +1217,7 @@ private String getTimestamp() { private Date isoConvert(String timestamp) { try { String wrappedString = "\"" + timestamp + "\""; - return OBJECT_MAPPER.readValue(wrappedString, Date.class); + return fromJsonString(wrappedString, Date.class); } catch (Exception e) { throw new RuntimeException("Creating date", e); } @@ -1207,7 +1228,7 @@ private String isoConvert(Date timestamp) { if (timestamp == null) { return "null"; } - String dateString = toJson(timestamp); + String dateString = toJsonString(timestamp); // Strip off the leading and trailing quotes from the JSON string-as-string representation. return dateString.substring(1, dateString.length() - 1); } catch (Exception e) { @@ -1250,9 +1271,9 @@ private byte[] getFileBytes(String dataFile) { } } - private void sendDeviceMessage() { - if ((++deviceMessageCount) % MESSAGE_REPORT_INTERVAL == 0) { - info(String.format("%s sending test message #%d", getTimestamp(), deviceMessageCount)); + private void sendDevicePoints() { + if (deviceUpdateCount % MESSAGE_REPORT_INTERVAL == 0) { + info(String.format("%s sending test message #%d", getTimestamp(), deviceUpdateCount)); } publishDeviceMessage(devicePoints); } @@ -1260,7 +1281,7 @@ private void sendDeviceMessage() { private void pubberLogMessage(String logMessage, Level level, String timestamp, String detail) { Entry logEntry = new Entry(); - logEntry.category = "pubber"; + logEntry.category = PUBBER_LOG_CATEGORY; logEntry.level = level.value(); logEntry.timestamp = isoConvert(timestamp); logEntry.message = logMessage; @@ -1316,7 +1337,8 @@ private void publishStateMessage() { info(String.format("update state %s last_config %s", isoConvert(deviceState.timestamp), isoConvert(deviceState.system.last_config))); try { - debug(String.format("State update%s", getTestingTag(deviceConfig)), toJson(deviceState)); + debug(String.format("State update%s", getTestingTag(deviceConfig)), + toJsonString(deviceState)); } catch (Exception e) { throw new RuntimeException("While converting new device state", e); } @@ -1353,7 +1375,7 @@ private void publishDeviceMessage(Object message, Runnable callback) { String fileName = traceTimestamp(messageBase) + ".json"; File messageOut = new File(outDir, fileName); try { - OBJECT_MAPPER.writeValue(messageOut, message); + toJsonFile(messageOut, message); } catch (Exception e) { throw new RuntimeException("While writing " + messageOut.getAbsolutePath(), e); } diff --git a/schema/event_system.json b/schema/event_system.json index 1a7ac0701..af048ad7a 100644 --- a/schema/event_system.json +++ b/schema/event_system.json @@ -27,7 +27,7 @@ "additionalProperties": false, "properties": { "restart_count": { - "type": "number" + "type": "integer" }, "mem_total_mb": { "type": "number" diff --git a/schema/persistent_device.json b/schema/persistent_device.json new file mode 100644 index 000000000..205007d92 --- /dev/null +++ b/schema/persistent_device.json @@ -0,0 +1,13 @@ +{ + "title": "Device Persistent", + "description": "Device persistent data", + "additionalProperties": true, + "properties": { + "endpoint": { + "$ref": "file:configuration_endpoint.json" + }, + "restart_count": { + "type": "integer" + } + } +}