From 182422712c53c473dcac49c137c61dd535ee4ead Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 2 Dec 2023 02:31:46 -0500 Subject: [PATCH 01/55] Adding new pubber fail option --- .gencode_hash.txt | 6 +-- etc/test_itemized.in | 1 + gencode/docs/configuration_pubber.html | 40 ++++++++++++++++++++ gencode/java/udmi/schema/PubberOptions.java | 6 ++- gencode/python/udmi/schema/options_pubber.py | 4 ++ schema/options_pubber.json | 3 ++ 6 files changed, 56 insertions(+), 4 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 911623dfe4..b4c563ec2e 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -5,7 +5,7 @@ cde866298a8a54168d3582b50ec93160209d07338d3d6c58d215855bc35cb158 gencode/docs/c 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79 gencode/docs/configuration_endpoint.html 6cf94d6cb600c75cde32a64bd78acb3ed3b54adfad08dbf6bb159b467e8925c9 gencode/docs/configuration_execution.html 6f8a3766b840e96881e3573d594e28864917efb288e068d6c82de55deb576694 gencode/docs/configuration_pod.html -fe6dabc7b6634449d457df758bcd441e7ecb87d8461c7e1ce26364f6b7f66f7f gencode/docs/configuration_pubber.html +c70b179e889d92a147cee3ef8d6637aa74c59093894dfdec80ef2deed6705613 gencode/docs/configuration_pubber.html 69bee6bfcd2f786d95cfd563ecb497296361c8945a1422385f78d25008f38ef1 gencode/docs/event.html 0a41807b999f80bd283fcee03e6df5845e13b122d9be7c7d4723b19a334b89d5 gencode/docs/event_discovery.html 79331072f185ca6e875b2e658e7e0dc471cd40be6b973ce9ee3f371aca474e9b gencode/docs/event_mapping.html @@ -100,7 +100,7 @@ fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa gencode/java/u ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a gencode/java/udmi/schema/Position.java 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70 gencode/java/udmi/schema/Properties.java d5e11dc354995681b894bcb62e4e21da7f0be68aaa3643a3dc5aa13610741f29 gencode/java/udmi/schema/PubberConfiguration.java -2fa3172b3ee42593da30c45dd47eb586f962e5b667c98a08410b1d8b525f22b9 gencode/java/udmi/schema/PubberOptions.java +9802e24240f2b16d91953d88814af33f06b597d7bcd8a0ed155df76d5906eaeb gencode/java/udmi/schema/PubberOptions.java 703ed0cecb60f284b52e76dc4d612eb798ba3e8781cc40698a46611b36e6b0c9 gencode/java/udmi/schema/SchemaValidationState.java dc7c549b3358f9b2ceccc8633af3d0fca7eed7c05c94c3396815363b889b4c16 gencode/java/udmi/schema/SequenceValidationState.java 9da49b22341a65580d085fd9d00eaadcaefcf7b813988ef844c617aa4b8a9a4f gencode/java/udmi/schema/SetupUdmiConfig.java @@ -183,7 +183,7 @@ b48ae013d203eb31cc388a084c0cc6f93d4d5534336d4da2ecbccd84a085ea2b gencode/python 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f gencode/python/udmi/schema/model_testing_target.py 7c9a755ae06c44fc2b8117d7918b6401ce08a7218b86033fb3320eda7286e581 gencode/python/udmi/schema/monitoring.py 3a3873c7db8d5fa13b43629031d36417a4ec4838d612a7b8c2fce7c22fdee009 gencode/python/udmi/schema/monitoring_metric.py -6dd7eb5e89d98715d7aa13d8e7362b2e1d821543a8b879a65e8fbf0abf4cc3de gencode/python/udmi/schema/options_pubber.py +651f39bef1ee9250fa48a9b1ae90c435f911443bdd129f06e8aa19cb2c282743 gencode/python/udmi/schema/options_pubber.py 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77 gencode/python/udmi/schema/persistent_device.py a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690 gencode/python/udmi/schema/properties.py e604cf0280fe772de5f4e5ecf10dc6c564b6177eeff9cd9fb8b385af8fe10a95 gencode/python/udmi/schema/state.py diff --git a/etc/test_itemized.in b/etc/test_itemized.in index 6ce600544d..20cbfff5d8 100644 --- a/etc/test_itemized.in +++ b/etc/test_itemized.in @@ -16,6 +16,7 @@ TEST broken_config noState TEST broken_config noLog TEST broken_config badCategory TEST system_last_update extraField=fnooz +TEST system_last_update dupeState TEST state_make_model badState # Test with various device metadata.json options diff --git a/gencode/docs/configuration_pubber.html b/gencode/docs/configuration_pubber.html index 9a60580fe2..7f16fd12db 100644 --- a/gencode/docs/configuration_pubber.html +++ b/gencode/docs/configuration_pubber.html @@ -2177,6 +2177,46 @@

+ + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + options + + + + dupeState
Type: boolean
+ + + + + + +
diff --git a/gencode/java/udmi/schema/PubberOptions.java b/gencode/java/udmi/schema/PubberOptions.java index 92c457a352..c4c452fa70 100644 --- a/gencode/java/udmi/schema/PubberOptions.java +++ b/gencode/java/udmi/schema/PubberOptions.java @@ -40,6 +40,7 @@ "spamState", "tweakState", "badState", + "dupeState", "noLog", "featureEnableSwap", "disableWriteback", @@ -98,6 +99,8 @@ public class PubberOptions { public Boolean tweakState; @JsonProperty("badState") public Boolean badState; + @JsonProperty("dupeState") + public Boolean dupeState; @JsonProperty("noLog") public Boolean noLog; @JsonProperty("featureEnableSwap") @@ -148,6 +151,7 @@ public int hashCode() { result = ((result* 31)+((this.extraField == null)? 0 :this.extraField.hashCode())); result = ((result* 31)+((this.emptyMissing == null)? 0 :this.emptyMissing.hashCode())); result = ((result* 31)+((this.fixedSampleRate == null)? 0 :this.fixedSampleRate.hashCode())); + result = ((result* 31)+((this.dupeState == null)? 0 :this.dupeState.hashCode())); result = ((result* 31)+((this.featureEnableSwap == null)? 0 :this.featureEnableSwap.hashCode())); result = ((result* 31)+((this.extraDevice == null)? 0 :this.extraDevice.hashCode())); result = ((result* 31)+((this.noConfigAck == null)? 0 :this.noConfigAck.hashCode())); @@ -166,7 +170,7 @@ public boolean equals(Object other) { return false; } PubberOptions rhs = ((PubberOptions) other); - return (((((((((((((((((((((((((((((((this.skewClock == rhs.skewClock)||((this.skewClock!= null)&&this.skewClock.equals(rhs.skewClock)))&&((this.noPersist == rhs.noPersist)||((this.noPersist!= null)&&this.noPersist.equals(rhs.noPersist))))&&((this.noLog == rhs.noLog)||((this.noLog!= null)&&this.noLog.equals(rhs.noLog))))&&((this.noHardware == rhs.noHardware)||((this.noHardware!= null)&&this.noHardware.equals(rhs.noHardware))))&&((this.messageTrace == rhs.messageTrace)||((this.messageTrace!= null)&&this.messageTrace.equals(rhs.messageTrace))))&&((this.softwareFirmwareValue == rhs.softwareFirmwareValue)||((this.softwareFirmwareValue!= null)&&this.softwareFirmwareValue.equals(rhs.softwareFirmwareValue))))&&((this.noWriteback == rhs.noWriteback)||((this.noWriteback!= null)&&this.noWriteback.equals(rhs.noWriteback))))&&((this.noLastStart == rhs.noLastStart)||((this.noLastStart!= null)&&this.noLastStart.equals(rhs.noLastStart))))&&((this.tweakState == rhs.tweakState)||((this.tweakState!= null)&&this.tweakState.equals(rhs.tweakState))))&&((this.spamState == rhs.spamState)||((this.spamState!= null)&&this.spamState.equals(rhs.spamState))))&&((this.noState == rhs.noState)||((this.noState!= null)&&this.noState.equals(rhs.noState))))&&((this.badState == rhs.badState)||((this.badState!= null)&&this.badState.equals(rhs.badState))))&&((this.noProxy == rhs.noProxy)||((this.noProxy!= null)&&this.noProxy.equals(rhs.noProxy))))&&((this.missingPoint == rhs.missingPoint)||((this.missingPoint!= null)&&this.missingPoint.equals(rhs.missingPoint))))&&((this.badCategory == rhs.badCategory)||((this.badCategory!= null)&&this.badCategory.equals(rhs.badCategory))))&&((this.extraPoint == rhs.extraPoint)||((this.extraPoint!= null)&&this.extraPoint.equals(rhs.extraPoint))))&&((this.smokeCheck == rhs.smokeCheck)||((this.smokeCheck!= null)&&this.smokeCheck.equals(rhs.smokeCheck))))&&((this.redirectRegistry == rhs.redirectRegistry)||((this.redirectRegistry!= null)&&this.redirectRegistry.equals(rhs.redirectRegistry))))&&((this.noPointState == rhs.noPointState)||((this.noPointState!= null)&&this.noPointState.equals(rhs.noPointState))))&&((this.disableWriteback == rhs.disableWriteback)||((this.disableWriteback!= null)&&this.disableWriteback.equals(rhs.disableWriteback))))&&((this.barfConfig == rhs.barfConfig)||((this.barfConfig!= null)&&this.barfConfig.equals(rhs.barfConfig))))&&((this.extraField == rhs.extraField)||((this.extraField!= null)&&this.extraField.equals(rhs.extraField))))&&((this.emptyMissing == rhs.emptyMissing)||((this.emptyMissing!= null)&&this.emptyMissing.equals(rhs.emptyMissing))))&&((this.fixedSampleRate == rhs.fixedSampleRate)||((this.fixedSampleRate!= null)&&this.fixedSampleRate.equals(rhs.fixedSampleRate))))&&((this.featureEnableSwap == rhs.featureEnableSwap)||((this.featureEnableSwap!= null)&&this.featureEnableSwap.equals(rhs.featureEnableSwap))))&&((this.extraDevice == rhs.extraDevice)||((this.extraDevice!= null)&&this.extraDevice.equals(rhs.extraDevice))))&&((this.noConfigAck == rhs.noConfigAck)||((this.noConfigAck!= null)&&this.noConfigAck.equals(rhs.noConfigAck))))&&((this.badVersion == rhs.badVersion)||((this.badVersion!= null)&&this.badVersion.equals(rhs.badVersion))))&&((this.fixedLogLevel == rhs.fixedLogLevel)||((this.fixedLogLevel!= null)&&this.fixedLogLevel.equals(rhs.fixedLogLevel))))&&((this.configStateDelay == rhs.configStateDelay)||((this.configStateDelay!= null)&&this.configStateDelay.equals(rhs.configStateDelay)))); + return ((((((((((((((((((((((((((((((((this.skewClock == rhs.skewClock)||((this.skewClock!= null)&&this.skewClock.equals(rhs.skewClock)))&&((this.noPersist == rhs.noPersist)||((this.noPersist!= null)&&this.noPersist.equals(rhs.noPersist))))&&((this.noLog == rhs.noLog)||((this.noLog!= null)&&this.noLog.equals(rhs.noLog))))&&((this.noHardware == rhs.noHardware)||((this.noHardware!= null)&&this.noHardware.equals(rhs.noHardware))))&&((this.messageTrace == rhs.messageTrace)||((this.messageTrace!= null)&&this.messageTrace.equals(rhs.messageTrace))))&&((this.softwareFirmwareValue == rhs.softwareFirmwareValue)||((this.softwareFirmwareValue!= null)&&this.softwareFirmwareValue.equals(rhs.softwareFirmwareValue))))&&((this.noWriteback == rhs.noWriteback)||((this.noWriteback!= null)&&this.noWriteback.equals(rhs.noWriteback))))&&((this.noLastStart == rhs.noLastStart)||((this.noLastStart!= null)&&this.noLastStart.equals(rhs.noLastStart))))&&((this.tweakState == rhs.tweakState)||((this.tweakState!= null)&&this.tweakState.equals(rhs.tweakState))))&&((this.spamState == rhs.spamState)||((this.spamState!= null)&&this.spamState.equals(rhs.spamState))))&&((this.noState == rhs.noState)||((this.noState!= null)&&this.noState.equals(rhs.noState))))&&((this.badState == rhs.badState)||((this.badState!= null)&&this.badState.equals(rhs.badState))))&&((this.noProxy == rhs.noProxy)||((this.noProxy!= null)&&this.noProxy.equals(rhs.noProxy))))&&((this.missingPoint == rhs.missingPoint)||((this.missingPoint!= null)&&this.missingPoint.equals(rhs.missingPoint))))&&((this.badCategory == rhs.badCategory)||((this.badCategory!= null)&&this.badCategory.equals(rhs.badCategory))))&&((this.extraPoint == rhs.extraPoint)||((this.extraPoint!= null)&&this.extraPoint.equals(rhs.extraPoint))))&&((this.smokeCheck == rhs.smokeCheck)||((this.smokeCheck!= null)&&this.smokeCheck.equals(rhs.smokeCheck))))&&((this.redirectRegistry == rhs.redirectRegistry)||((this.redirectRegistry!= null)&&this.redirectRegistry.equals(rhs.redirectRegistry))))&&((this.noPointState == rhs.noPointState)||((this.noPointState!= null)&&this.noPointState.equals(rhs.noPointState))))&&((this.disableWriteback == rhs.disableWriteback)||((this.disableWriteback!= null)&&this.disableWriteback.equals(rhs.disableWriteback))))&&((this.barfConfig == rhs.barfConfig)||((this.barfConfig!= null)&&this.barfConfig.equals(rhs.barfConfig))))&&((this.extraField == rhs.extraField)||((this.extraField!= null)&&this.extraField.equals(rhs.extraField))))&&((this.emptyMissing == rhs.emptyMissing)||((this.emptyMissing!= null)&&this.emptyMissing.equals(rhs.emptyMissing))))&&((this.fixedSampleRate == rhs.fixedSampleRate)||((this.fixedSampleRate!= null)&&this.fixedSampleRate.equals(rhs.fixedSampleRate))))&&((this.dupeState == rhs.dupeState)||((this.dupeState!= null)&&this.dupeState.equals(rhs.dupeState))))&&((this.featureEnableSwap == rhs.featureEnableSwap)||((this.featureEnableSwap!= null)&&this.featureEnableSwap.equals(rhs.featureEnableSwap))))&&((this.extraDevice == rhs.extraDevice)||((this.extraDevice!= null)&&this.extraDevice.equals(rhs.extraDevice))))&&((this.noConfigAck == rhs.noConfigAck)||((this.noConfigAck!= null)&&this.noConfigAck.equals(rhs.noConfigAck))))&&((this.badVersion == rhs.badVersion)||((this.badVersion!= null)&&this.badVersion.equals(rhs.badVersion))))&&((this.fixedLogLevel == rhs.fixedLogLevel)||((this.fixedLogLevel!= null)&&this.fixedLogLevel.equals(rhs.fixedLogLevel))))&&((this.configStateDelay == rhs.configStateDelay)||((this.configStateDelay!= null)&&this.configStateDelay.equals(rhs.configStateDelay)))); } } diff --git a/gencode/python/udmi/schema/options_pubber.py b/gencode/python/udmi/schema/options_pubber.py index 4e0956db7b..205d81288f 100644 --- a/gencode/python/udmi/schema/options_pubber.py +++ b/gencode/python/udmi/schema/options_pubber.py @@ -29,6 +29,7 @@ def __init__(self): self.spamState = None self.tweakState = None self.badState = None + self.dupeState = None self.noLog = None self.featureEnableSwap = None self.disableWriteback = None @@ -65,6 +66,7 @@ def from_dict(source): result.spamState = source.get('spamState') result.tweakState = source.get('tweakState') result.badState = source.get('badState') + result.dupeState = source.get('dupeState') result.noLog = source.get('noLog') result.featureEnableSwap = source.get('featureEnableSwap') result.disableWriteback = source.get('disableWriteback') @@ -139,6 +141,8 @@ def to_dict(self): result['tweakState'] = self.tweakState # 5 if self.badState: result['badState'] = self.badState # 5 + if self.dupeState: + result['dupeState'] = self.dupeState # 5 if self.noLog: result['noLog'] = self.noLog # 5 if self.featureEnableSwap: diff --git a/schema/options_pubber.json b/schema/options_pubber.json index f9bce87310..46c948d0ff 100644 --- a/schema/options_pubber.json +++ b/schema/options_pubber.json @@ -76,6 +76,9 @@ "badState": { "type": "boolean" }, + "dupeState": { + "type": "boolean" + }, "noLog": { "type": "boolean" }, From 6a0c3d6be564ecdce6b80ed4249e87d9ddcb1fc7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 2 Dec 2023 03:01:25 -0500 Subject: [PATCH 02/55] Add dupeState --- etc/schema_itemized.out | 17 ++++++++++------- etc/test_itemized.out | 21 +++++++++++---------- pubber/src/main/java/daq/pubber/Pubber.java | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index de48626c61..7e086aa9cc 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -20,13 +20,16 @@ 18 system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed -19 state_make_model -23 valid_serial_no -24 system_last_update -25 system_min_loglevel +19 system_last_update RESULT pass schemas event_pointset_stable STABLE 5/5 All schema validations passed +19 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed +19 system_last_update RESULT pass schemas state_update_stable STABLE 5/5 All schema validations passed +20 state_make_model +24 valid_serial_no +25 system_last_update 26 system_min_loglevel -30 gateway_proxy_events +27 system_min_loglevel 31 gateway_proxy_events -35 gateway_proxy_events +32 gateway_proxy_events 36 gateway_proxy_events -40 device_config_acked +37 gateway_proxy_events +41 device_config_acked diff --git a/etc/test_itemized.out b/etc/test_itemized.out index 686d3ece5b..e0b237035e 100644 --- a/etc/test_itemized.out +++ b/etc/test_itemized.out @@ -21,13 +21,14 @@ 17 CPBLTY fail system broken_config.logging ALPHA 0/1 Timeout after 30s waiting for log category `system.config.receive` level `DEBUG` to be logged 17 RESULT fail system broken_config ALPHA 0/5 expected: but was: 18 RESULT pass system system_last_update STABLE 5/5 Sequence complete -19 RESULT fail system state_make_model BETA 0/5 Timeout waiting for no applicable system status -23 RESULT skip system valid_serial_no ALPHA 0/0 State testing disabled -24 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches config timestamp -25 RESULT fail system system_min_loglevel ALPHA 0/5 Received state update with no-state device -26 RESULT pass system system_min_loglevel ALPHA 5/5 Sequence complete -30 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete -31 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Timeout waiting for Missing data from AHU-22 -35 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete -36 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Received state update with no-state device -40 RESULT skip system device_config_acked BETA 0/0 No config check for proxy device +19 RESULT pass system system_last_update STABLE 5/5 Sequence complete +20 RESULT fail system state_make_model BETA 0/5 Timeout waiting for no applicable system status +24 RESULT skip system valid_serial_no ALPHA 0/0 State testing disabled +25 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches config timestamp +26 RESULT fail system system_min_loglevel ALPHA 0/5 Received state update with no-state device +27 RESULT pass system system_min_loglevel ALPHA 5/5 Sequence complete +31 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete +32 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Timeout waiting for Missing data from AHU-22 +36 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete +37 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Received state update with no-state device +41 RESULT skip system device_config_acked BETA 0/0 No config check for proxy device diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index ea9ddb7a4a..0ad872adb7 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -33,6 +33,7 @@ import static udmi.schema.BlobsetConfig.SystemBlobsets.IOT_ENDPOINT_CONFIG; import static udmi.schema.EndpointConfiguration.Protocol.MQTT; +import com.google.api.services.cloudiot.v1.model.DeviceState; import com.google.common.collect.ImmutableMap; import com.google.daq.mqtt.util.CatchingScheduledThreadPoolExecutor; import com.google.udmi.util.GeneralUtils; @@ -491,6 +492,7 @@ public void update(Object update) { publishSynchronousState(); } else if (checkTarget instanceof SystemState) { deviceState.system = (SystemState) checkValue; + ifTrueThen(options.dupeState, () -> sendDupeState()); } else if (checkTarget instanceof PointsetState) { deviceState.pointset = (PointsetState) checkValue; } else if (checkTarget instanceof LocalnetState) { @@ -504,6 +506,14 @@ public void update(Object update) { markStateDirty(); } + private void sendDupeState() { + State dupeState = new State(); + dupeState.system = deviceState.system; + dupeState.timestamp = deviceState.timestamp; + dupeState.version = deviceState.version; + publishStateMessage(dupeState); + } + @Override public void publish(Object message) { publishDeviceMessage(message); @@ -1416,6 +1426,15 @@ private void publishStateMessage() { } private void publishStateMessage(Object stateToSend) { + try { + stateLock.lock(); + publishStateMessageRaw(stateToSend); + } finally { + stateLock.unlock(); + } + } + + private void publishStateMessageRaw(Object stateToSend) { if (configLatch == null || configLatch.getCount() > 0) { warn("Dropping state update until config received..."); return; From 5096f91b13396ff06ab17e57147ddcfd2f80e0f7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 2 Dec 2023 03:02:56 -0500 Subject: [PATCH 03/55] Inject racoon --- pubber/src/main/java/daq/pubber/Pubber.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index 0ad872adb7..2d1e697219 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -142,7 +142,7 @@ public class Pubber extends ManagerBase implements ManagerHost { .put(SystemEvent.class, getEventsSuffix("system")) .put(PointsetEvent.class, getEventsSuffix("pointset")) .put(ExtraPointsetEvent.class, getEventsSuffix("pointset")) - .put(InjectedMessage.class, getEventsSuffix("invalid")) + .put(InjectedMessage.class, getEventsSuffix("racoon")) .put(InjectedState.class, MqttDevice.STATE_TOPIC) .put(DiscoveryEvent.class, getEventsSuffix("discovery")) .build(); From 8e833c89c28967b0c93afaa868c20f917b88355f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 17:39:26 -0500 Subject: [PATCH 04/55] Add disallowed state entry checking --- .../java/com/google/udmi/util/DiffEntry.java | 23 +++++++++++++ .../com/google/udmi/util/GeneralUtils.java | 4 +-- .../daq/mqtt/sequencer/SequenceBase.java | 33 ++++++++++++++----- .../daq/mqtt/util/ObjectDiffEngine.java | 22 ++++++++----- 4 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 common/src/main/java/com/google/udmi/util/DiffEntry.java diff --git a/common/src/main/java/com/google/udmi/util/DiffEntry.java b/common/src/main/java/com/google/udmi/util/DiffEntry.java new file mode 100644 index 0000000000..4d1f022d33 --- /dev/null +++ b/common/src/main/java/com/google/udmi/util/DiffEntry.java @@ -0,0 +1,23 @@ +package com.google.udmi.util; + +import static java.lang.String.format; + +public record DiffEntry(DiffAction action, String key, String value) { + public enum DiffAction { + ADD("Add"), + SET("Set"), + REMOVE("Remove"); + + final String value; + + DiffAction(String value) { + this.value = value; + } + } + + @Override + public String toString() { + return action == DiffAction.REMOVE ? format("%s `%s`", action().value, key) + : format("%s `%s` = `%s`", action().value, key, value); + } +} diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index cb152c033b..03edddd02d 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -74,8 +74,8 @@ public static String[] arrayOf(String... args) { return args; } - public static String changedLines(List nullableChanges) { - List changes = ofNullable(nullableChanges).orElse(ImmutableList.of()); + public static String changedLines(List nullableChanges) { + List changes = ofNullable(nullableChanges).orElse(ImmutableList.of()); String terminator = changes.size() == 0 ? "." : ":"; String header = format("Changed %d fields%s%s", changes.size(), terminator, SEPARATOR); return (header + INDENTED_LINES.join(changes)).trim(); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 6e82c600bc..6646f03df7 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -3,6 +3,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Predicates.not; import static com.google.common.base.Strings.emptyToNull; import static com.google.daq.mqtt.sequencer.semantic.SemanticValue.actualize; import static com.google.daq.mqtt.util.IotReflectorClient.REFLECTOR_PREFIX; @@ -14,6 +15,7 @@ import static com.google.udmi.util.Common.EXCEPTION_KEY; import static com.google.udmi.util.Common.GATEWAY_ID_KEY; import static com.google.udmi.util.Common.TIMESTAMP_KEY; +import static com.google.udmi.util.GeneralUtils.CSV_JOINER; import static com.google.udmi.util.GeneralUtils.changedLines; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; @@ -61,6 +63,7 @@ import com.google.common.collect.ImmutableMap; import com.google.daq.mqtt.sequencer.semantic.SemanticDate; import com.google.daq.mqtt.sequencer.semantic.SemanticValue; +import com.google.udmi.util.DiffEntry; import com.google.daq.mqtt.util.MessagePublisher; import com.google.daq.mqtt.util.MessagePublisher.QuerySpeed; import com.google.daq.mqtt.util.ObjectDiffEngine; @@ -197,7 +200,7 @@ public class SequenceBase { private static final int LOG_TIMEOUT_SEC = 10; private static final long ONE_SECOND_MS = 1000; private static final int EXIT_CODE_PRESERVE = -9; - private static final String SYSTEM_TESTING_MARKER = " `system.testing"; + private static final String SYSTEM_TESTING_MARKER = "system.testing"; private static final BiMap RESULT_LEVEL_MAP = ImmutableBiMap.of( SequenceResult.START, Level.INFO, SKIP, Level.WARNING, @@ -1138,8 +1141,8 @@ private void captureConfigChange(String reason) { String header = format("Update config%s: ", suffix); debug(header + getTimestamp(deviceConfig.timestamp)); recordRawMessage(deviceConfig, LOCAL_CONFIG_UPDATE); - List allDiffs = SENT_CONFIG_DIFFERNATOR.computeChanges(deviceConfig); - List filteredDiffs = filterTesting(allDiffs); + List allDiffs = SENT_CONFIG_DIFFERNATOR.computeChanges(deviceConfig); + List filteredDiffs = filterTesting(allDiffs); if (!filteredDiffs.isEmpty()) { recordSequence(header); filteredDiffs.forEach(this::recordBullet); @@ -1399,6 +1402,10 @@ private void recordSequence(String step) { } } + private void recordBullet(DiffEntry step) { + recordBullet(step.toString()); + } + private void recordBullet(String step) { if (recordSequence) { info("Device config " + step); @@ -1608,7 +1615,7 @@ private synchronized void handleUpdateMessage(String subTypeRaw, error("Shouldn't be seeing this!"); return; } - List changes = updateDeviceConfig(config); + List changes = updateDeviceConfig(config); debug(format("Updated config %s %s", getTimestamp(config.timestamp), txnId)); if (updateCount == CAPABILITY_SCORE) { info(format("Initial config #%03d", updateCount), stringify(deviceConfig)); @@ -1634,12 +1641,13 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } checkState(deviceSupportsState(), "Received state update with no-state device"); boolean deltaState = RECV_STATE_DIFFERNATOR.isInitialized(); - List stateChanges = RECV_STATE_DIFFERNATOR.computeChanges(converted); + List stateChanges = RECV_STATE_DIFFERNATOR.computeChanges(converted); Instant start = ofNullable(convertedState.timestamp).orElseGet(Date::new).toInstant(); long delta = Duration.between(start, Instant.now()).getSeconds(); debug(format("Updated state after %ds %s %s", delta, timestamp, txnId)); if (deltaState) { info(format("Updated state #%03d", updateCount), changedLines(stateChanges)); + validateIntermediateState(convertedState, stateChanges); } else { info(format("Initial state #%03d", updateCount), stringify(converted)); } @@ -1656,7 +1664,16 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } } - private List updateDeviceConfig(Config config) { + private void validateIntermediateState(State convertedState, List stateChanges) { + List badChanges = stateChanges.stream().filter(not(this::changeAllowed)).toList(); + checkState(badChanges.isEmpty(),"Disallowed state changes: " + CSV_JOINER.join(badChanges)); + } + + private boolean changeAllowed(DiffEntry change) { + return false; + } + + private List updateDeviceConfig(Config config) { if (deviceConfig == null) { return null; } @@ -1731,8 +1748,8 @@ private String configTransactionsListString() { /** * Filter out any testing-oriented messages, since they should not impact behavior. */ - private List filterTesting(List allDiffs) { - return allDiffs.stream().filter(message -> !message.contains(SYSTEM_TESTING_MARKER)) + private List filterTesting(List allDiffs) { + return allDiffs.stream().filter(message -> !message.key().startsWith(SYSTEM_TESTING_MARKER)) .collect(Collectors.toList()); } diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java b/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java index 22270355e4..d6a600856b 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java @@ -1,8 +1,14 @@ package com.google.daq.mqtt.util; +import static com.google.udmi.util.DiffEntry.DiffAction.ADD; +import static com.google.udmi.util.DiffEntry.DiffAction.REMOVE; +import static com.google.udmi.util.DiffEntry.DiffAction.SET; + import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.daq.mqtt.sequencer.semantic.SemanticValue; +import com.google.udmi.util.DiffEntry; +import com.google.udmi.util.DiffEntry.DiffAction; import com.google.udmi.util.JsonUtil; import java.lang.reflect.Field; import java.util.ArrayList; @@ -33,11 +39,11 @@ public ObjectDiffEngine() { * @param updatedObject new object * @return list of differences against the previous object */ - public List computeChanges(Object updatedObject) { + public List computeChanges(Object updatedObject) { // TODO: Hack alert! State should be handled semantically, but for now show raw changes. ignoreSemantics = updatedObject instanceof State; Map updated = extractDefinitions(updatedObject); - List updates = new ArrayList<>(); + List updates = new ArrayList<>(); accumulateDifference("", previous, updated, updates); previous = updated; return updates; @@ -113,7 +119,7 @@ private boolean isBaseType(Object value) { @SuppressWarnings("unchecked") void accumulateDifference(String prefix, Map left, Map right, - List updates) { + List updates) { right.forEach((key, value) -> { String describedKey = describedKey(prefix, key); String raw = describeValue(prefix, key, semanticValue(value)); @@ -125,20 +131,20 @@ void accumulateDifference(String prefix, Map left, Map) leftValue, (Map) value, updates); } } else { - updates.add(String.format("Add `%s` = %s", describedKey, describedValue)); + updates.add(new DiffEntry(ADD, describedKey, describedValue)); } }); if (left != null) { left.forEach((key, value) -> { if (!right.containsKey(key)) { String describedKey = describedKey(prefix, key); - updates.add(String.format("Remove `%s`", describedKey)); + updates.add(new DiffEntry(REMOVE, describedKey, null)); } }); } @@ -209,10 +215,10 @@ public boolean equals(Object oneObject, Object twoObject) { * @param endObject ending object * @return list of differences going from start to end objects */ - public List diff(Object startObject, Object endObject) { + public List diff(Object startObject, Object endObject) { Map left = extractValues(startObject); Map right = extractValues(endObject); - List updates = new ArrayList<>(); + List updates = new ArrayList<>(); accumulateDifference("", left, right, updates); return updates; } From dda8fc6a74dfb50fc3d95aab3018a9dcc16d911f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 18:04:12 -0500 Subject: [PATCH 05/55] Add system filter --- .../com/google/daq/mqtt/sequencer/SequenceBase.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 6646f03df7..82000a36f0 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -61,9 +61,9 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.sequencer.semantic.SemanticDate; import com.google.daq.mqtt.sequencer.semantic.SemanticValue; -import com.google.udmi.util.DiffEntry; import com.google.daq.mqtt.util.MessagePublisher; import com.google.daq.mqtt.util.MessagePublisher.QuerySpeed; import com.google.daq.mqtt.util.ObjectDiffEngine; @@ -73,6 +73,7 @@ import com.google.daq.mqtt.validator.Validator.MessageBundle; import com.google.udmi.util.CleanDateFormat; import com.google.udmi.util.Common; +import com.google.udmi.util.DiffEntry; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.SiteModel; @@ -261,6 +262,8 @@ public class SequenceBase { private final Stack waitingCondition = new Stack<>(); private final SortedMap> validationResults = new TreeMap<>(); private final Map capabilityExceptions = new ConcurrentHashMap<>(); + private final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", + "system.last_config", "system.status.timestamp"); @Rule public Timeout globalTimeout = new Timeout(NORM_TIMEOUT_MS, TimeUnit.MILLISECONDS); @Rule @@ -1665,12 +1668,14 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } private void validateIntermediateState(State convertedState, List stateChanges) { - List badChanges = stateChanges.stream().filter(not(this::changeAllowed)).toList(); - checkState(badChanges.isEmpty(),"Disallowed state changes: " + CSV_JOINER.join(badChanges)); + List badChanges = stateChanges.stream() + .filter(not(this::changeAllowed)).map(DiffEntry::key).toList(); + checkState(badChanges.isEmpty(), "Disallowed state changes: " + CSV_JOINER.join(badChanges)); } private boolean changeAllowed(DiffEntry change) { - return false; + String key = change.key(); + return SYSTEM_STATE_CHANGES.stream().filter(key::startsWith).anyMatch(); } private List updateDeviceConfig(Config config) { From 379b408b413d7f0de14a47847dd558e137d5f61f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 18:59:22 -0500 Subject: [PATCH 06/55] Fix anyMatch --- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 82000a36f0..975c37b1eb 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -227,6 +227,8 @@ public class SequenceBase { private static final String NO_EXTRA_DETAIL = ""; private static final Duration LOG_WAIT_TIME = Duration.ofSeconds(30); private static final Duration DEFAULT_WAIT_TIMEOUT = Duration.ofHours(30); + private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", + "system.last_config", "system.status.timestamp"); protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -262,8 +264,6 @@ public class SequenceBase { private final Stack waitingCondition = new Stack<>(); private final SortedMap> validationResults = new TreeMap<>(); private final Map capabilityExceptions = new ConcurrentHashMap<>(); - private final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", - "system.last_config", "system.status.timestamp"); @Rule public Timeout globalTimeout = new Timeout(NORM_TIMEOUT_MS, TimeUnit.MILLISECONDS); @Rule @@ -1675,7 +1675,7 @@ private void validateIntermediateState(State convertedState, List sta private boolean changeAllowed(DiffEntry change) { String key = change.key(); - return SYSTEM_STATE_CHANGES.stream().filter(key::startsWith).anyMatch(); + return SYSTEM_STATE_CHANGES.stream().anyMatch(key::startsWith); } private List updateDeviceConfig(Config config) { From 2799ceca2e4514232c767355cb9ebfd1dc4a73ea Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 19:10:58 -0500 Subject: [PATCH 07/55] Allow system status changes not above threshold --- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 975c37b1eb..350d2746b4 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -16,6 +16,7 @@ import static com.google.udmi.util.Common.GATEWAY_ID_KEY; import static com.google.udmi.util.Common.TIMESTAMP_KEY; import static com.google.udmi.util.GeneralUtils.CSV_JOINER; +import static com.google.udmi.util.GeneralUtils.catchToElse; import static com.google.udmi.util.GeneralUtils.changedLines; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; @@ -228,7 +229,8 @@ public class SequenceBase { private static final Duration LOG_WAIT_TIME = Duration.ofSeconds(30); private static final Duration DEFAULT_WAIT_TIMEOUT = Duration.ofHours(30); private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", - "system.last_config", "system.status.timestamp"); + "system.last_config", "system.status"); + private static final int DISALLOWED_SYSTAM_STATUS_LEVEL = NOTICE.value(); protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -1668,6 +1670,11 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } private void validateIntermediateState(State convertedState, List stateChanges) { + int stateStatusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); + if (stateStatusLevel >= DISALLOWED_SYSTAM_STATUS_LEVEL) { + throw new RuntimeException(format("Status level %d exceeded allowed threshold %d", + stateStatusLevel, DISALLOWED_SYSTAM_STATUS_LEVEL)); + } List badChanges = stateChanges.stream() .filter(not(this::changeAllowed)).map(DiffEntry::key).toList(); checkState(badChanges.isEmpty(), "Disallowed state changes: " + CSV_JOINER.join(badChanges)); From 11a129af395e504fd85f40776f0d237142b32f05 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 19:21:33 -0500 Subject: [PATCH 08/55] Fix error threshold --- .../com/google/daq/mqtt/sequencer/SequenceBase.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 350d2746b4..205e063b62 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -230,7 +230,7 @@ public class SequenceBase { private static final Duration DEFAULT_WAIT_TIMEOUT = Duration.ofHours(30); private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", "system.last_config", "system.status"); - private static final int DISALLOWED_SYSTAM_STATUS_LEVEL = NOTICE.value(); + private static final int DISALLOWED_STATUS_LEVEL = WARNING.value(); protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -1670,10 +1670,10 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } private void validateIntermediateState(State convertedState, List stateChanges) { - int stateStatusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); - if (stateStatusLevel >= DISALLOWED_SYSTAM_STATUS_LEVEL) { - throw new RuntimeException(format("Status level %d exceeded allowed threshold %d", - stateStatusLevel, DISALLOWED_SYSTAM_STATUS_LEVEL)); + int statusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); + if (statusLevel >= DISALLOWED_STATUS_LEVEL) { + throw new RuntimeException(format("System status level %d exceeded allowed threshold %d", + statusLevel, DISALLOWED_STATUS_LEVEL)); } List badChanges = stateChanges.stream() .filter(not(this::changeAllowed)).map(DiffEntry::key).toList(); From 900b197f066048f567e98abb10ef59ae6d8ac618 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 20:36:19 -0500 Subject: [PATCH 09/55] Don't validate state always --- .../main/java/com/google/daq/mqtt/sequencer/SequenceBase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 205e063b62..b2797b3db3 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -1670,6 +1670,10 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } private void validateIntermediateState(State convertedState, List stateChanges) { + if (!recordSequence) { + return; + } + int statusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); if (statusLevel >= DISALLOWED_STATUS_LEVEL) { throw new RuntimeException(format("System status level %d exceeded allowed threshold %d", From 7174fc5d7cb27fede981be5b85c286ef908cb873 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 3 Dec 2023 20:37:10 -0500 Subject: [PATCH 10/55] Don't always --- .github/workflows/testing.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 49d738b2c5..b9b5deaf14 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -154,16 +154,13 @@ jobs: - name: registrar clean run: bin/test_regclean $TARGET_PROJECT - name: sequence tests clean - if: ${{ always() }} run: bin/test_sequencer clean nocheck $TARGET_PROJECT - name: sequence tests alpha - if: ${{ always() }} run: bin/test_sequencer alpha nocheck $TARGET_PROJECT - name: sequence test post-process if: ${{ always() }} run: egrep ' test .* after .*s ' out/sequencer.log | tee out/timing_sequencer.out - name: itemized sequencer tests - if: ${{ always() }} run: bin/test_itemized $TARGET_PROJECT - name: output files if: ${{ always() }} From ae42d90759c31b880c898f764faedcad1dbc3b14 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 05:20:58 -0500 Subject: [PATCH 11/55] Fix output value --- common/src/main/java/com/google/udmi/util/DiffEntry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/com/google/udmi/util/DiffEntry.java b/common/src/main/java/com/google/udmi/util/DiffEntry.java index 4d1f022d33..77c67bc57b 100644 --- a/common/src/main/java/com/google/udmi/util/DiffEntry.java +++ b/common/src/main/java/com/google/udmi/util/DiffEntry.java @@ -18,6 +18,6 @@ public enum DiffAction { @Override public String toString() { return action == DiffAction.REMOVE ? format("%s `%s`", action().value, key) - : format("%s `%s` = `%s`", action().value, key, value); + : format("%s `%s` = %s", action().value, key, value); } } From 201ef9ddc6624d5ce3231194e89232746fced424 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 05:27:15 -0500 Subject: [PATCH 12/55] Replace always --- .github/workflows/testing.yml | 37 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b9b5deaf14..42d8bea934 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -35,7 +35,7 @@ jobs: - name: bin/run_tests all_tests run: bin/run_tests all_tests - name: all test output - if: ${{ always() }} + if: ${{ !cancelled() }} run: more out/test_* spelling: @@ -154,30 +154,33 @@ jobs: - name: registrar clean run: bin/test_regclean $TARGET_PROJECT - name: sequence tests clean + if: ${{ !cancelled() }} run: bin/test_sequencer clean nocheck $TARGET_PROJECT - name: sequence tests alpha + if: ${{ !cancelled() }} run: bin/test_sequencer alpha nocheck $TARGET_PROJECT - name: sequence test post-process - if: ${{ always() }} + if: ${{ !cancelled() }} run: egrep ' test .* after .*s ' out/sequencer.log | tee out/timing_sequencer.out - name: itemized sequencer tests + if: ${{ !cancelled() }} run: bin/test_itemized $TARGET_PROJECT - name: output files - if: ${{ always() }} + if: ${{ !cancelled() }} run: more out/*.out* | cat - name: pubber logs - if: ${{ always() }} + if: ${{ !cancelled() }} run: more out/pubber.log* pubber/out/*.json | cat - name: itemized test post-process - if: ${{ always() }} + if: ${{ !cancelled() }} run: egrep ' test .* after .*s ' out/sequencer.log-* | tee out/timing_itemized.out - name: support bundle - if: ${{ always() }} + if: ${{ !cancelled() }} run: | bin/support ${{ github.repository_owner }}_${{ github.job }}_ tar -tzvf *udmi-support*.tgz - uses: actions/upload-artifact@v3 - if: ${{ always() }} + if: ${{ !cancelled() }} with: if-no-files-found: error name: udmi-support_${{ github.run_id }} @@ -203,23 +206,25 @@ jobs: - name: registrar clean run: bin/test_regclean $TARGET_PROJECT - name: telemetry validator + if: ${{ !cancelled() }} run: bin/test_validator $TARGET_PROJECT - name: validator result - if: ${{ always() }} + if: ${{ !cancelled() }} run: cat /tmp/validator.out || true - name: message captures - if: ${{ always() }} + if: ${{ !cancelled() }} run: | cd sites/udmi_site_model/out find . -type f | sort | xargs ls -l find . -type f | sort | xargs more | cat - name: nostate sequences + if: ${{ !cancelled() }} run: bin/test_sequencer nostate full $TARGET_PROJECT - name: support bundle - if: ${{ always() }} + if: ${{ !cancelled() }} run: UDMI_REGISTRY_SUFFIX=_b bin/support ${{ github.repository_owner }}_${{ github.job }}_ - uses: actions/upload-artifact@v3 - if: ${{ always() }} + if: ${{ !cancelled() }} with: if-no-files-found: error name: udmi-support_${{ github.run_id }} @@ -249,13 +254,13 @@ jobs: - name: bin/test_redirect run: bin/test_redirect $TARGET_PROJECT - name: pubber.log - if: ${{ always() }} + if: ${{ !cancelled() }} run: more out/pubber.log* | cat - name: support bundle - if: ${{ always() }} + if: ${{ !cancelled() }} run: UDMI_REGISTRY_SUFFIX=_r bin/support ${{ github.repository_owner }}_${{ github.job }}_ - uses: actions/upload-artifact@v3 - if: ${{ always() }} + if: ${{ !cancelled() }} with: if-no-files-found: error name: udmi-support_${{ github.run_id }} @@ -286,8 +291,8 @@ jobs: ls -d sites/udmi_site_model*/out*/devices/AHU-1/tests/* find out* -name 'schema*.out*' -size +0 | xargs more | cat - name: Sequencer validation - if: ${{ always() }} + if: ${{ !cancelled() }} run: bin/test_sequcheck - name: Itemized validation - if: ${{ always() }} + if: ${{ !cancelled() }} run: bin/test_itemcheck From 7b1274816c08af9846dbe3f5d7261060948a47ce Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 06:26:54 -0500 Subject: [PATCH 13/55] Synchronize start state --- .../daq/mqtt/sequencer/SequenceBase.java | 36 ++++++++++++++----- .../sequencer/sequences/BlobsetSequences.java | 6 ++-- .../sequencer/sequences/ConfigSequences.java | 3 +- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index b2797b3db3..06af341ca5 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -5,6 +5,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Predicates.not; import static com.google.common.base.Strings.emptyToNull; +import static com.google.daq.mqtt.sequencer.SequenceBase.Capabilities.LAST_CONFIG; import static com.google.daq.mqtt.sequencer.semantic.SemanticValue.actualize; import static com.google.daq.mqtt.util.IotReflectorClient.REFLECTOR_PREFIX; import static com.google.daq.mqtt.validator.Validator.CONFIG_PREFIX; @@ -226,11 +227,11 @@ public class SequenceBase { STATE_QUERY_CUTOFF_SEC)); private static final String FAKE_DEVICE_ID = "TAP-1"; private static final String NO_EXTRA_DETAIL = ""; + private static final Duration DEFAULT_WAIT_TIME = Duration.ofSeconds(10); private static final Duration LOG_WAIT_TIME = Duration.ofSeconds(30); - private static final Duration DEFAULT_WAIT_TIMEOUT = Duration.ofHours(30); + private static final Duration DEFAULT_LOOP_TIMEOUT = Duration.ofHours(30); private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", "system.last_config", "system.status"); - private static final int DISALLOWED_STATUS_LEVEL = WARNING.value(); protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -273,6 +274,7 @@ public class SequenceBase { protected State deviceState; protected boolean configAcked; protected String lastSerialNo; + private int disallowedStatusLevel; private String extraField; private Instant lastConfigUpdate; private boolean enforceSerial; @@ -722,6 +724,7 @@ public void setUp() { enforceSerial = false; recordMessages = true; recordSequence = false; + disallowedStatusLevel = WARNING.value(); resetConfig(resetRequired); @@ -739,6 +742,9 @@ public void setUp() { clearReceivedEvents(); validationResults.clear(); + forCapability(LAST_CONFIG, + () -> waitFor("state last_config sync", this::lastConfigUpdatedString)); + recordSequence = true; waitingConditionPush("executing test"); @@ -1247,6 +1253,11 @@ protected void checkNotThat(String description, Supplier condition) { recordSequence("Check that " + notDescription); } + private void waitFor(String description, Supplier evaluator) { + waitFor(description, DEFAULT_WAIT_TIME, evaluator); + + } + private void waitFor(String description, Duration maxWait, Supplier evaluator) { AtomicReference detail = new AtomicReference<>(); whileDoing(description, () -> { @@ -1279,8 +1290,7 @@ private String checkLogged(String category, Level exactLevel) { protected void checkNotLogged(String category, Level minLevel) { withRecordSequence(false, () -> { ifTrueThen(deviceSupportsState(), () -> - untilTrue("last_config synchronized", - () -> dateEquals(deviceConfig.timestamp, deviceState.system.last_config))); + untilTrue("last_config synchronized", this::lastConfigUpdated)); processLogMessages(); }); final Instant endTime = lastConfigUpdate.plusSeconds(LOG_TIMEOUT_SEC); @@ -1383,7 +1393,7 @@ private void untilLoop(String description, Supplier evaluator, Supplier } private void messageEvaluateLoop(Supplier evaluator) { - messageEvaluateLoop(DEFAULT_WAIT_TIMEOUT, evaluator); + messageEvaluateLoop(DEFAULT_LOOP_TIMEOUT, evaluator); } private void messageEvaluateLoop(Duration maxWait, Supplier evaluator) { @@ -1669,15 +1679,19 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } } + protected void expectedStatusLevel(Level level) { + disallowedStatusLevel = level.value(); + } + private void validateIntermediateState(State convertedState, List stateChanges) { if (!recordSequence) { return; } int statusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); - if (statusLevel >= DISALLOWED_STATUS_LEVEL) { + if (statusLevel >= disallowedStatusLevel) { throw new RuntimeException(format("System status level %d exceeded allowed threshold %d", - statusLevel, DISALLOWED_STATUS_LEVEL)); + statusLevel, disallowedStatusLevel)); } List badChanges = stateChanges.stream() .filter(not(this::changeAllowed)).map(DiffEntry::key).toList(); @@ -1907,7 +1921,11 @@ private void updateMirrorConfig(String receivedConfig) { } } - protected boolean stateMatchesConfigTimestamp() { + protected String lastConfigUpdatedString() { + return lastConfigUpdated() ? null : ""; + } + + protected boolean lastConfigUpdated() { Date expectedConfig = deviceConfig.timestamp; Date lastConfig = deviceState.system.last_config; return dateEquals(expectedConfig, lastConfig); @@ -2071,7 +2089,7 @@ protected void forCapability(Capabilities capability, Runnable action) { public enum Capabilities { DEVICE_STATE("device_state"), LOGGING("logging"), - ACKNOWLEDGE("acknowledge"); + LAST_CONFIG("last_config"); private final String value; diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java index d95c0e4d55..5257d1075b 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java @@ -200,7 +200,7 @@ public void endpoint_failure_and_restart() { private void check_endpoint_connection_success(boolean doRestart) { // Phase one: initiate connection to alternate registry. - untilTrue("initial last_config matches config timestamp", this::stateMatchesConfigTimestamp); + untilTrue("initial last_config matches config timestamp", this::lastConfigUpdated); setDeviceConfigEndpointBlob(getAlternateEndpointHostname(), altRegistry, false); untilSuccessfulRedirect(BlobPhase.APPLY); @@ -208,7 +208,7 @@ private void check_endpoint_connection_success(boolean doRestart) { // Phase two: verify connection to alternate registry. untilSuccessfulRedirect(BlobPhase.FINAL); untilTrue("alternate last_config matches config timestamp", - this::stateMatchesConfigTimestamp); + this::lastConfigUpdated); untilClearedRedirect(); if (doRestart) { @@ -225,7 +225,7 @@ private void check_endpoint_connection_success(boolean doRestart) { // Phase four: verify restoration of initial registry connection. whileDoing("restoring main connection", () -> { untilSuccessfulRedirect(BlobPhase.FINAL); - untilTrue("restored last_config matches config timestamp", this::stateMatchesConfigTimestamp); + untilTrue("restored last_config matches config timestamp", this::lastConfigUpdated); untilClearedRedirect(); }); } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index 38d232d2bc..222396b61a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -53,7 +53,7 @@ public class ConfigSequences extends SequenceBase { @Summary("Check that last_update state is correctly set in response to a config update.") @ValidateSchema public void system_last_update() { - untilTrue("state last_config matches config timestamp", this::stateMatchesConfigTimestamp); + untilTrue("state last_config matches config timestamp", this::lastConfigUpdated); ensureStateUpdate(); } @@ -120,6 +120,7 @@ public void device_config_acked() { @Capability(value = LOGGING, stage = ALPHA) @Summary("Check that the device correctly handles a broken (non-json) config message.") public void broken_config() { + expectedStatusLevel(Level.ERROR); deviceConfig.system.min_loglevel = Level.DEBUG.value(); updateConfig("starting broken_config"); Date stableConfig = deviceConfig.timestamp; From 45ea3233f589518f43145b6c8fa20f01f83be30f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 07:02:34 -0500 Subject: [PATCH 14/55] Fix startup capabilities --- .../google/daq/mqtt/sequencer/SequenceBase.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 06af341ca5..765046e736 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -575,12 +575,25 @@ private static Map getCapabilities(Description desc) { List list = ofNullable(all).map(array -> Arrays.asList(all.value())) .orElseGet(ArrayList::new); ifNotNullThen(desc.getAnnotation(Capability.class), list::add); - return list.stream().collect(Collectors.toMap(Capability::value, cap -> cap)); + Map collect = list.stream() + .collect(Collectors.toMap(Capability::value, cap -> cap)); + collect.put(LAST_CONFIG, defaultLastConfig()); + return collect; } catch (Exception e) { throw new RuntimeException("While extracting capabilities for " + desc.getMethodName(), e); } } + @Capability(LAST_CONFIG) + private static Capability defaultLastConfig() { + try { + return requireNonNull( + SequenceBase.class.getDeclaredMethod("defaultLastConfig").getAnnotation(Capability.class)); + } catch (Exception e) { + throw new RuntimeException("While getting default capability for LAST_CONFIG", e); + } + } + private Map.Entry emitCapabilityResult(Capabilities capability, Exception state, Capability cap, Bucket bucket, String methodName) { boolean pass = state instanceof CapabilitySuccess; From 3df7e146d0fe1cfa3ff72529b2ece113a1d90f1e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 07:06:08 -0500 Subject: [PATCH 15/55] Remove default LAST_CONFIG hack --- .../daq/mqtt/sequencer/SequenceBase.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 765046e736..5f4e162f44 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -575,25 +575,13 @@ private static Map getCapabilities(Description desc) { List list = ofNullable(all).map(array -> Arrays.asList(all.value())) .orElseGet(ArrayList::new); ifNotNullThen(desc.getAnnotation(Capability.class), list::add); - Map collect = list.stream() + return list.stream() .collect(Collectors.toMap(Capability::value, cap -> cap)); - collect.put(LAST_CONFIG, defaultLastConfig()); - return collect; } catch (Exception e) { throw new RuntimeException("While extracting capabilities for " + desc.getMethodName(), e); } } - @Capability(LAST_CONFIG) - private static Capability defaultLastConfig() { - try { - return requireNonNull( - SequenceBase.class.getDeclaredMethod("defaultLastConfig").getAnnotation(Capability.class)); - } catch (Exception e) { - throw new RuntimeException("While getting default capability for LAST_CONFIG", e); - } - } - private Map.Entry emitCapabilityResult(Capabilities capability, Exception state, Capability cap, Bucket bucket, String methodName) { boolean pass = state instanceof CapabilitySuccess; @@ -828,6 +816,11 @@ private void recordResult(SequenceResult result, Description description, String AtomicInteger total = new AtomicInteger(isSkip ? 0 : base); AtomicInteger score = new AtomicInteger(isPass ? base : 0); + if (!capabilities.containsKey(LAST_CONFIG)) { + debug("Removing implicit system capability LAST_CONFIG"); + capabilityExceptions.remove(LAST_CONFIG); + } + ifTrueThen(isPass, () -> assertEquals("executed test capabilities", capabilities.keySet(), capabilityExceptions.keySet())); From 9991a0c57e4b23f076824015ed5de2189491eb42 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 07:16:16 -0500 Subject: [PATCH 16/55] Fix linty --- .../main/java/com/google/daq/mqtt/sequencer/SequenceBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 5f4e162f44..108a44b345 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -807,8 +807,8 @@ private void recordResult(SequenceResult result, Description description, String Feature feature = description.getAnnotation(Feature.class); Map capabilities = getCapabilities(description); Bucket bucket = getBucket(feature); - String stage = (feature == null ? Feature.DEFAULT_STAGE : feature.stage()).name(); - int base = (feature == null ? Feature.DEFAULT_SCORE : feature.score()); + final String stage = (feature == null ? Feature.DEFAULT_STAGE : feature.stage()).name(); + final int base = (feature == null ? Feature.DEFAULT_SCORE : feature.score()); boolean isSkip = result == SKIP; boolean isPass = result == PASS; From e51190b3cb6941fc8f4f6e2b0782ff602bb14abd Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 07:44:54 -0500 Subject: [PATCH 17/55] Adding prefix allow --- .../com/google/daq/mqtt/sequencer/PointsetBase.java | 1 - .../com/google/daq/mqtt/sequencer/SequenceBase.java | 11 ++++++++++- .../mqtt/sequencer/sequences/BlobsetSequences.java | 6 ++++++ .../mqtt/sequencer/sequences/PointsetSequences.java | 6 ++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/PointsetBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/PointsetBase.java index ab63147399..5a78b292d8 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/PointsetBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/PointsetBase.java @@ -53,5 +53,4 @@ protected TargetTestingModel getTarget(String target) { } return testingMetadata; } - } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 108a44b345..095b0d1b5c 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -267,6 +267,8 @@ public class SequenceBase { private final Stack waitingCondition = new Stack<>(); private final SortedMap> validationResults = new TreeMap<>(); private final Map capabilityExceptions = new ConcurrentHashMap<>(); + private final Set allowedDeviceStateChanges = new HashSet<>(); + @Rule public Timeout globalTimeout = new Timeout(NORM_TIMEOUT_MS, TimeUnit.MILLISECONDS); @Rule @@ -706,6 +708,7 @@ public void setUp() { assumeTrue("Feature bucket not enabled", isBucketEnabled(testBucket)); assertTrue("exceptions map should be empty", capabilityExceptions.isEmpty()); + assertTrue("allowed changes map should be empty", allowedDeviceStateChanges.isEmpty()); if (!deviceSupportsState()) { boolean featureDisabled = ifNotNullGet(testDescription.getAnnotation(Feature.class), @@ -1706,7 +1709,13 @@ private void validateIntermediateState(State convertedState, List sta private boolean changeAllowed(DiffEntry change) { String key = change.key(); - return SYSTEM_STATE_CHANGES.stream().anyMatch(key::startsWith); + return SYSTEM_STATE_CHANGES.stream().anyMatch(key::startsWith) || + allowedDeviceStateChanges.stream().anyMatch(key::startsWith); + } + + protected void allowDeviceStateChange(String changePrefix) { + debug("Allowing device state change: " + changePrefix); + allowedDeviceStateChanges.add(changePrefix); } private List updateDeviceConfig(Config config) { diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java index 5257d1075b..3bab6f05da 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java @@ -18,6 +18,7 @@ import com.google.daq.mqtt.sequencer.semantic.SemanticValue; import java.util.Date; import java.util.HashMap; +import org.junit.Before; import org.junit.Test; import udmi.schema.BlobBlobsetConfig; import udmi.schema.BlobBlobsetConfig.BlobPhase; @@ -44,6 +45,11 @@ public class BlobsetSequences extends SequenceBase { "projects/%s/locations/%s/registries/%s/devices/%s"; private static final String BOGUS_ENDPOINT_HOSTNAME = "twiddily.fiddily.fog"; + @Before + public void setupExpectedParameters() { + allowDeviceStateChange("blobset."); + } + private String generateEndpointConfigClientId(String registryId) { return String.format( ENDPOINT_CONFIG_CLIENT_ID, diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java index d0eee6365a..c75e716106 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map.Entry; import java.util.stream.Collectors; +import org.junit.Before; import org.junit.Test; import udmi.schema.Level; import udmi.schema.PointPointsetConfig; @@ -40,6 +41,11 @@ public class PointsetSequences extends PointsetBase { private static final int DEFAULT_SAMPLE_RATE_SEC = 10; private static final String POINTS_MAP_PATH = "pointset.points"; + @Before + public void setupExpectedParameters() { + allowDeviceStateChange("pointset."); + } + private boolean isErrorState(PointPointsetState pointState) { return ofNullable(catchToNull(() -> pointState.status.level)).orElse(Level.INFO.value()) >= Level.ERROR.value(); From 1b2e21f3facc0cc841597292d66038fa76efe7b8 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 07:58:53 -0500 Subject: [PATCH 18/55] Fix linty --- .../main/java/com/google/daq/mqtt/sequencer/SequenceBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 095b0d1b5c..76bb9d7ff7 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -1709,8 +1709,8 @@ private void validateIntermediateState(State convertedState, List sta private boolean changeAllowed(DiffEntry change) { String key = change.key(); - return SYSTEM_STATE_CHANGES.stream().anyMatch(key::startsWith) || - allowedDeviceStateChanges.stream().anyMatch(key::startsWith); + return SYSTEM_STATE_CHANGES.stream().anyMatch(key::startsWith) + || allowedDeviceStateChanges.stream().anyMatch(key::startsWith); } protected void allowDeviceStateChange(String changePrefix) { From b95c6739eacd5b25c72f5ddeb682cb336f387cc1 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 07:59:48 -0500 Subject: [PATCH 19/55] Fix blobset allowed changes --- .../google/daq/mqtt/sequencer/sequences/BlobsetSequences.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java index 3bab6f05da..b3aed25893 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java @@ -47,7 +47,7 @@ public class BlobsetSequences extends SequenceBase { @Before public void setupExpectedParameters() { - allowDeviceStateChange("blobset."); + allowDeviceStateChange("blobset"); } private String generateEndpointConfigClientId(String registryId) { From 228aa43e28c156ff859fdcc8a38dfa64825138ad Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 08:14:16 -0500 Subject: [PATCH 20/55] Add noLastConfig option --- .gencode_hash.txt | 6 +-- .../com/google/udmi/util/GeneralUtils.java | 4 ++ gencode/docs/configuration_pubber.html | 40 +++++++++++++++++++ gencode/java/udmi/schema/PubberOptions.java | 6 ++- gencode/python/udmi/schema/options_pubber.py | 4 ++ .../main/java/daq/pubber/SystemManager.java | 3 +- schema/options_pubber.json | 3 ++ 7 files changed, 61 insertions(+), 5 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index b4c563ec2e..b0d3e682ac 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -5,7 +5,7 @@ cde866298a8a54168d3582b50ec93160209d07338d3d6c58d215855bc35cb158 gencode/docs/c 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79 gencode/docs/configuration_endpoint.html 6cf94d6cb600c75cde32a64bd78acb3ed3b54adfad08dbf6bb159b467e8925c9 gencode/docs/configuration_execution.html 6f8a3766b840e96881e3573d594e28864917efb288e068d6c82de55deb576694 gencode/docs/configuration_pod.html -c70b179e889d92a147cee3ef8d6637aa74c59093894dfdec80ef2deed6705613 gencode/docs/configuration_pubber.html +dc77a21a05f9f98a23403a1db0c3b468ef50d028e4c4934734d08e15f36b1d57 gencode/docs/configuration_pubber.html 69bee6bfcd2f786d95cfd563ecb497296361c8945a1422385f78d25008f38ef1 gencode/docs/event.html 0a41807b999f80bd283fcee03e6df5845e13b122d9be7c7d4723b19a334b89d5 gencode/docs/event_discovery.html 79331072f185ca6e875b2e658e7e0dc471cd40be6b973ce9ee3f371aca474e9b gencode/docs/event_mapping.html @@ -100,7 +100,7 @@ fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa gencode/java/u ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a gencode/java/udmi/schema/Position.java 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70 gencode/java/udmi/schema/Properties.java d5e11dc354995681b894bcb62e4e21da7f0be68aaa3643a3dc5aa13610741f29 gencode/java/udmi/schema/PubberConfiguration.java -9802e24240f2b16d91953d88814af33f06b597d7bcd8a0ed155df76d5906eaeb gencode/java/udmi/schema/PubberOptions.java +3bff2c98c8d41816b44c7623d542428a6378d3c7a4f9440ae35b395fb1f1e386 gencode/java/udmi/schema/PubberOptions.java 703ed0cecb60f284b52e76dc4d612eb798ba3e8781cc40698a46611b36e6b0c9 gencode/java/udmi/schema/SchemaValidationState.java dc7c549b3358f9b2ceccc8633af3d0fca7eed7c05c94c3396815363b889b4c16 gencode/java/udmi/schema/SequenceValidationState.java 9da49b22341a65580d085fd9d00eaadcaefcf7b813988ef844c617aa4b8a9a4f gencode/java/udmi/schema/SetupUdmiConfig.java @@ -183,7 +183,7 @@ b48ae013d203eb31cc388a084c0cc6f93d4d5534336d4da2ecbccd84a085ea2b gencode/python 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f gencode/python/udmi/schema/model_testing_target.py 7c9a755ae06c44fc2b8117d7918b6401ce08a7218b86033fb3320eda7286e581 gencode/python/udmi/schema/monitoring.py 3a3873c7db8d5fa13b43629031d36417a4ec4838d612a7b8c2fce7c22fdee009 gencode/python/udmi/schema/monitoring_metric.py -651f39bef1ee9250fa48a9b1ae90c435f911443bdd129f06e8aa19cb2c282743 gencode/python/udmi/schema/options_pubber.py +743395d4b764d4fd2614aef85cab680498e5f2560767bc24f424d6594f14d75c gencode/python/udmi/schema/options_pubber.py 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77 gencode/python/udmi/schema/persistent_device.py a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690 gencode/python/udmi/schema/properties.py e604cf0280fe772de5f4e5ecf10dc6c564b6177eeff9cd9fb8b385af8fe10a95 gencode/python/udmi/schema/state.py diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 03edddd02d..a92ffcbceb 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -287,6 +287,10 @@ public static void ifTrueThen(Object conditional, Runnable action, Runnable } } + public static T ifNotTrueGet(Object conditional, Supplier supplier) { + return isTrue(conditional) : null ? supplier.get(); + } + public static boolean isTrue(Object value) { return Boolean.TRUE.equals(value); } diff --git a/gencode/docs/configuration_pubber.html b/gencode/docs/configuration_pubber.html index 7f16fd12db..7236c91d73 100644 --- a/gencode/docs/configuration_pubber.html +++ b/gencode/docs/configuration_pubber.html @@ -1417,6 +1417,46 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + options + + + + noLastConfig
Type: boolean
+ + + + + + +
diff --git a/gencode/java/udmi/schema/PubberOptions.java b/gencode/java/udmi/schema/PubberOptions.java index c4c452fa70..2c8edea97a 100644 --- a/gencode/java/udmi/schema/PubberOptions.java +++ b/gencode/java/udmi/schema/PubberOptions.java @@ -21,6 +21,7 @@ "noConfigAck", "noPersist", "noLastStart", + "noLastConfig", "badCategory", "badVersion", "noProxy", @@ -61,6 +62,8 @@ public class PubberOptions { public Boolean noPersist; @JsonProperty("noLastStart") public Boolean noLastStart; + @JsonProperty("noLastConfig") + public Boolean noLastConfig; @JsonProperty("badCategory") public Boolean badCategory; @JsonProperty("badVersion") @@ -129,6 +132,7 @@ public int hashCode() { int result = 1; result = ((result* 31)+((this.skewClock == null)? 0 :this.skewClock.hashCode())); result = ((result* 31)+((this.noPersist == null)? 0 :this.noPersist.hashCode())); + result = ((result* 31)+((this.noLastConfig == null)? 0 :this.noLastConfig.hashCode())); result = ((result* 31)+((this.noLog == null)? 0 :this.noLog.hashCode())); result = ((result* 31)+((this.noHardware == null)? 0 :this.noHardware.hashCode())); result = ((result* 31)+((this.messageTrace == null)? 0 :this.messageTrace.hashCode())); @@ -170,7 +174,7 @@ public boolean equals(Object other) { return false; } PubberOptions rhs = ((PubberOptions) other); - return ((((((((((((((((((((((((((((((((this.skewClock == rhs.skewClock)||((this.skewClock!= null)&&this.skewClock.equals(rhs.skewClock)))&&((this.noPersist == rhs.noPersist)||((this.noPersist!= null)&&this.noPersist.equals(rhs.noPersist))))&&((this.noLog == rhs.noLog)||((this.noLog!= null)&&this.noLog.equals(rhs.noLog))))&&((this.noHardware == rhs.noHardware)||((this.noHardware!= null)&&this.noHardware.equals(rhs.noHardware))))&&((this.messageTrace == rhs.messageTrace)||((this.messageTrace!= null)&&this.messageTrace.equals(rhs.messageTrace))))&&((this.softwareFirmwareValue == rhs.softwareFirmwareValue)||((this.softwareFirmwareValue!= null)&&this.softwareFirmwareValue.equals(rhs.softwareFirmwareValue))))&&((this.noWriteback == rhs.noWriteback)||((this.noWriteback!= null)&&this.noWriteback.equals(rhs.noWriteback))))&&((this.noLastStart == rhs.noLastStart)||((this.noLastStart!= null)&&this.noLastStart.equals(rhs.noLastStart))))&&((this.tweakState == rhs.tweakState)||((this.tweakState!= null)&&this.tweakState.equals(rhs.tweakState))))&&((this.spamState == rhs.spamState)||((this.spamState!= null)&&this.spamState.equals(rhs.spamState))))&&((this.noState == rhs.noState)||((this.noState!= null)&&this.noState.equals(rhs.noState))))&&((this.badState == rhs.badState)||((this.badState!= null)&&this.badState.equals(rhs.badState))))&&((this.noProxy == rhs.noProxy)||((this.noProxy!= null)&&this.noProxy.equals(rhs.noProxy))))&&((this.missingPoint == rhs.missingPoint)||((this.missingPoint!= null)&&this.missingPoint.equals(rhs.missingPoint))))&&((this.badCategory == rhs.badCategory)||((this.badCategory!= null)&&this.badCategory.equals(rhs.badCategory))))&&((this.extraPoint == rhs.extraPoint)||((this.extraPoint!= null)&&this.extraPoint.equals(rhs.extraPoint))))&&((this.smokeCheck == rhs.smokeCheck)||((this.smokeCheck!= null)&&this.smokeCheck.equals(rhs.smokeCheck))))&&((this.redirectRegistry == rhs.redirectRegistry)||((this.redirectRegistry!= null)&&this.redirectRegistry.equals(rhs.redirectRegistry))))&&((this.noPointState == rhs.noPointState)||((this.noPointState!= null)&&this.noPointState.equals(rhs.noPointState))))&&((this.disableWriteback == rhs.disableWriteback)||((this.disableWriteback!= null)&&this.disableWriteback.equals(rhs.disableWriteback))))&&((this.barfConfig == rhs.barfConfig)||((this.barfConfig!= null)&&this.barfConfig.equals(rhs.barfConfig))))&&((this.extraField == rhs.extraField)||((this.extraField!= null)&&this.extraField.equals(rhs.extraField))))&&((this.emptyMissing == rhs.emptyMissing)||((this.emptyMissing!= null)&&this.emptyMissing.equals(rhs.emptyMissing))))&&((this.fixedSampleRate == rhs.fixedSampleRate)||((this.fixedSampleRate!= null)&&this.fixedSampleRate.equals(rhs.fixedSampleRate))))&&((this.dupeState == rhs.dupeState)||((this.dupeState!= null)&&this.dupeState.equals(rhs.dupeState))))&&((this.featureEnableSwap == rhs.featureEnableSwap)||((this.featureEnableSwap!= null)&&this.featureEnableSwap.equals(rhs.featureEnableSwap))))&&((this.extraDevice == rhs.extraDevice)||((this.extraDevice!= null)&&this.extraDevice.equals(rhs.extraDevice))))&&((this.noConfigAck == rhs.noConfigAck)||((this.noConfigAck!= null)&&this.noConfigAck.equals(rhs.noConfigAck))))&&((this.badVersion == rhs.badVersion)||((this.badVersion!= null)&&this.badVersion.equals(rhs.badVersion))))&&((this.fixedLogLevel == rhs.fixedLogLevel)||((this.fixedLogLevel!= null)&&this.fixedLogLevel.equals(rhs.fixedLogLevel))))&&((this.configStateDelay == rhs.configStateDelay)||((this.configStateDelay!= null)&&this.configStateDelay.equals(rhs.configStateDelay)))); + return (((((((((((((((((((((((((((((((((this.skewClock == rhs.skewClock)||((this.skewClock!= null)&&this.skewClock.equals(rhs.skewClock)))&&((this.noPersist == rhs.noPersist)||((this.noPersist!= null)&&this.noPersist.equals(rhs.noPersist))))&&((this.noLastConfig == rhs.noLastConfig)||((this.noLastConfig!= null)&&this.noLastConfig.equals(rhs.noLastConfig))))&&((this.noLog == rhs.noLog)||((this.noLog!= null)&&this.noLog.equals(rhs.noLog))))&&((this.noHardware == rhs.noHardware)||((this.noHardware!= null)&&this.noHardware.equals(rhs.noHardware))))&&((this.messageTrace == rhs.messageTrace)||((this.messageTrace!= null)&&this.messageTrace.equals(rhs.messageTrace))))&&((this.softwareFirmwareValue == rhs.softwareFirmwareValue)||((this.softwareFirmwareValue!= null)&&this.softwareFirmwareValue.equals(rhs.softwareFirmwareValue))))&&((this.noWriteback == rhs.noWriteback)||((this.noWriteback!= null)&&this.noWriteback.equals(rhs.noWriteback))))&&((this.noLastStart == rhs.noLastStart)||((this.noLastStart!= null)&&this.noLastStart.equals(rhs.noLastStart))))&&((this.tweakState == rhs.tweakState)||((this.tweakState!= null)&&this.tweakState.equals(rhs.tweakState))))&&((this.spamState == rhs.spamState)||((this.spamState!= null)&&this.spamState.equals(rhs.spamState))))&&((this.noState == rhs.noState)||((this.noState!= null)&&this.noState.equals(rhs.noState))))&&((this.badState == rhs.badState)||((this.badState!= null)&&this.badState.equals(rhs.badState))))&&((this.noProxy == rhs.noProxy)||((this.noProxy!= null)&&this.noProxy.equals(rhs.noProxy))))&&((this.missingPoint == rhs.missingPoint)||((this.missingPoint!= null)&&this.missingPoint.equals(rhs.missingPoint))))&&((this.badCategory == rhs.badCategory)||((this.badCategory!= null)&&this.badCategory.equals(rhs.badCategory))))&&((this.extraPoint == rhs.extraPoint)||((this.extraPoint!= null)&&this.extraPoint.equals(rhs.extraPoint))))&&((this.smokeCheck == rhs.smokeCheck)||((this.smokeCheck!= null)&&this.smokeCheck.equals(rhs.smokeCheck))))&&((this.redirectRegistry == rhs.redirectRegistry)||((this.redirectRegistry!= null)&&this.redirectRegistry.equals(rhs.redirectRegistry))))&&((this.noPointState == rhs.noPointState)||((this.noPointState!= null)&&this.noPointState.equals(rhs.noPointState))))&&((this.disableWriteback == rhs.disableWriteback)||((this.disableWriteback!= null)&&this.disableWriteback.equals(rhs.disableWriteback))))&&((this.barfConfig == rhs.barfConfig)||((this.barfConfig!= null)&&this.barfConfig.equals(rhs.barfConfig))))&&((this.extraField == rhs.extraField)||((this.extraField!= null)&&this.extraField.equals(rhs.extraField))))&&((this.emptyMissing == rhs.emptyMissing)||((this.emptyMissing!= null)&&this.emptyMissing.equals(rhs.emptyMissing))))&&((this.fixedSampleRate == rhs.fixedSampleRate)||((this.fixedSampleRate!= null)&&this.fixedSampleRate.equals(rhs.fixedSampleRate))))&&((this.dupeState == rhs.dupeState)||((this.dupeState!= null)&&this.dupeState.equals(rhs.dupeState))))&&((this.featureEnableSwap == rhs.featureEnableSwap)||((this.featureEnableSwap!= null)&&this.featureEnableSwap.equals(rhs.featureEnableSwap))))&&((this.extraDevice == rhs.extraDevice)||((this.extraDevice!= null)&&this.extraDevice.equals(rhs.extraDevice))))&&((this.noConfigAck == rhs.noConfigAck)||((this.noConfigAck!= null)&&this.noConfigAck.equals(rhs.noConfigAck))))&&((this.badVersion == rhs.badVersion)||((this.badVersion!= null)&&this.badVersion.equals(rhs.badVersion))))&&((this.fixedLogLevel == rhs.fixedLogLevel)||((this.fixedLogLevel!= null)&&this.fixedLogLevel.equals(rhs.fixedLogLevel))))&&((this.configStateDelay == rhs.configStateDelay)||((this.configStateDelay!= null)&&this.configStateDelay.equals(rhs.configStateDelay)))); } } diff --git a/gencode/python/udmi/schema/options_pubber.py b/gencode/python/udmi/schema/options_pubber.py index 205d81288f..89f74f9def 100644 --- a/gencode/python/udmi/schema/options_pubber.py +++ b/gencode/python/udmi/schema/options_pubber.py @@ -10,6 +10,7 @@ def __init__(self): self.noConfigAck = None self.noPersist = None self.noLastStart = None + self.noLastConfig = None self.badCategory = None self.badVersion = None self.noProxy = None @@ -47,6 +48,7 @@ def from_dict(source): result.noConfigAck = source.get('noConfigAck') result.noPersist = source.get('noPersist') result.noLastStart = source.get('noLastStart') + result.noLastConfig = source.get('noLastConfig') result.badCategory = source.get('badCategory') result.badVersion = source.get('badVersion') result.noProxy = source.get('noProxy') @@ -103,6 +105,8 @@ def to_dict(self): result['noPersist'] = self.noPersist # 5 if self.noLastStart: result['noLastStart'] = self.noLastStart # 5 + if self.noLastConfig: + result['noLastConfig'] = self.noLastConfig # 5 if self.badCategory: result['badCategory'] = self.badCategory # 5 if self.badVersion: diff --git a/pubber/src/main/java/daq/pubber/SystemManager.java b/pubber/src/main/java/daq/pubber/SystemManager.java index 66e6924051..5d73e19d01 100644 --- a/pubber/src/main/java/daq/pubber/SystemManager.java +++ b/pubber/src/main/java/daq/pubber/SystemManager.java @@ -4,6 +4,7 @@ import static com.google.udmi.util.GeneralUtils.catchToNull; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; +import static com.google.udmi.util.GeneralUtils.ifNotTrueGet; import static com.google.udmi.util.GeneralUtils.ifNotTrueThen; import static com.google.udmi.util.GeneralUtils.isTrue; import static com.google.udmi.util.JsonUtil.getTimestamp; @@ -223,7 +224,7 @@ public void setPersistentData(DevicePersistent persistentData) { void updateConfig(SystemConfig system, Date timestamp) { systemConfig = system; - systemState.last_config = timestamp; + systemState.last_config = ifNotTrueGet(options.noLastConfig, () -> timestamp); updateInterval(ifNotNullGet(system, config -> config.metrics_rate_sec)); updateState(); } diff --git a/schema/options_pubber.json b/schema/options_pubber.json index 46c948d0ff..d037dbaaba 100644 --- a/schema/options_pubber.json +++ b/schema/options_pubber.json @@ -19,6 +19,9 @@ "noLastStart": { "type": "boolean" }, + "noLastConfig": { + "type": "boolean" + }, "badCategory": { "type": "boolean" }, From b61f983aafd7dab3afba14aada0279317aae866f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 08:16:43 -0500 Subject: [PATCH 21/55] Adding allowed state changes --- .../daq/mqtt/sequencer/sequences/DiscoverySequences.java | 6 ++++++ .../daq/mqtt/sequencer/sequences/GatewaySequences.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index 49a14e7fa8..e82b476dd9 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -38,6 +38,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import org.junit.AssumptionViolatedException; +import org.junit.Before; import org.junit.Test; import udmi.schema.Bucket; import udmi.schema.DiscoveryConfig; @@ -57,6 +58,11 @@ public class DiscoverySequences extends SequenceBase { private HashMap previousGenerations; private Set families; + @Before + public void setupExpectedParameters() { + allowDeviceStateChange("discovery"); + } + private static boolean isActive(Entry entry) { return Optional.ofNullable(entry.getValue().stage).orElse(STABLE).compareTo(BETA) >= 0; } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/GatewaySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/GatewaySequences.java index d1d8b477c1..d69139ec00 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/GatewaySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/GatewaySequences.java @@ -13,6 +13,7 @@ import com.google.daq.mqtt.sequencer.Summary; import java.util.HashSet; import java.util.Set; +import org.junit.Before; import org.junit.Test; import udmi.schema.Bucket; import udmi.schema.FeatureEnumeration.FeatureStage; @@ -22,6 +23,11 @@ */ public class GatewaySequences extends SequenceBase { + @Before + public void setupExpectedParameters() { + allowDeviceStateChange("gateway"); + } + @Override public void setUp() { ifTrueSkipTest(catchToTrue(() -> deviceMetadata.gateway.proxy_ids.isEmpty()), "Not a gateway"); From 676e55da94da0e347a3b1d8c5c86364d7245fdc3 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 08:19:40 -0500 Subject: [PATCH 22/55] Allowing system restart changes --- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 5 +++-- .../daq/mqtt/sequencer/sequences/BlobsetSequences.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 76bb9d7ff7..26563e5e49 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -1714,8 +1714,9 @@ private boolean changeAllowed(DiffEntry change) { } protected void allowDeviceStateChange(String changePrefix) { - debug("Allowing device state change: " + changePrefix); - allowedDeviceStateChanges.add(changePrefix); + if (allowedDeviceStateChanges.add(changePrefix)) { + debug("Allowing device state change: " + changePrefix); + } } private List updateDeviceConfig(Config config) { diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java index b3aed25893..0db22bea0a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java @@ -244,6 +244,8 @@ public void system_mode_restart() { } private void check_system_restart() { + allowDeviceStateChange("system.operation."); + // Prepare for the restart. final Date dateZero = new Date(0); untilTrue("last_start is not zero", From 281d1678553b17ee39db5ec4d02ec1d5c96e3e39 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 08:31:37 -0500 Subject: [PATCH 23/55] Fix stupid --- common/src/main/java/com/google/udmi/util/GeneralUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index a92ffcbceb..d7561c310f 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -288,7 +288,7 @@ public static void ifTrueThen(Object conditional, Runnable action, Runnable } public static T ifNotTrueGet(Object conditional, Supplier supplier) { - return isTrue(conditional) : null ? supplier.get(); + return isTrue(conditional) ? null : supplier.get(); } public static boolean isTrue(Object value) { From 8b3a22bc29616a9a40d1119ce7cfec36e1fd2a78 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 09:10:40 -0500 Subject: [PATCH 24/55] Fix some errors --- .../com/google/daq/mqtt/sequencer/SequenceBase.java | 10 +++++----- .../mqtt/sequencer/sequences/WritebackSequences.java | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 26563e5e49..04cc111f38 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -276,7 +276,7 @@ public class SequenceBase { protected State deviceState; protected boolean configAcked; protected String lastSerialNo; - private int disallowedStatusLevel; + private int maxAllowedStatusLevel; private String extraField; private Instant lastConfigUpdate; private boolean enforceSerial; @@ -728,7 +728,7 @@ public void setUp() { enforceSerial = false; recordMessages = true; recordSequence = false; - disallowedStatusLevel = WARNING.value(); + maxAllowedStatusLevel = NOTICE.value(); resetConfig(resetRequired); @@ -1689,7 +1689,7 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } protected void expectedStatusLevel(Level level) { - disallowedStatusLevel = level.value(); + maxAllowedStatusLevel = level.value(); } private void validateIntermediateState(State convertedState, List stateChanges) { @@ -1698,9 +1698,9 @@ private void validateIntermediateState(State convertedState, List sta } int statusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); - if (statusLevel >= disallowedStatusLevel) { + if (statusLevel > maxAllowedStatusLevel) { throw new RuntimeException(format("System status level %d exceeded allowed threshold %d", - statusLevel, disallowedStatusLevel)); + statusLevel, maxAllowedStatusLevel)); } List badChanges = stateChanges.stream() .filter(not(this::changeAllowed)).map(DiffEntry::key).toList(); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/WritebackSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/WritebackSequences.java index bac156da63..548f1a8fba 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/WritebackSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/WritebackSequences.java @@ -9,6 +9,7 @@ import com.google.daq.mqtt.sequencer.Summary; import java.util.List; import java.util.Objects; +import org.junit.Before; import org.junit.Test; import udmi.schema.PointPointsetState.Value_state; import udmi.schema.PointsetEvent; @@ -24,6 +25,11 @@ public class WritebackSequences extends PointsetBase { public static final String APPLIED_STATE = "applied"; public static final String DEFAULT_STATE = null; + @Before + public void setupExpectedParameters() { + allowDeviceStateChange("pointset.points."); + } + /** * Checks `value_state` for the point in the state matches the provided string. * From 9f96e8846ce373028bfcb699a004b892b3463489 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 10:14:09 -0500 Subject: [PATCH 25/55] Allow +0000 timestamps --- .../main/java/com/google/udmi/util/JsonUtil.java | 16 ++++++++++++++-- .../simple/devices/AHU-1/002_event_system.json | 2 +- .../expected/devices/AHU-1/event_system.attr | 2 +- .../expected/devices/AHU-1/event_system.json | 2 +- .../com/google/daq/mqtt/validator/Validator.java | 13 ++++++++----- .../com/google/daq/mqtt/validator/BasicTest.java | 3 ++- 6 files changed, 27 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/com/google/udmi/util/JsonUtil.java b/common/src/main/java/com/google/udmi/util/JsonUtil.java index 3e6dd4312b..7b86a445e7 100644 --- a/common/src/main/java/com/google/udmi/util/JsonUtil.java +++ b/common/src/main/java/com/google/udmi/util/JsonUtil.java @@ -124,13 +124,25 @@ public static T fromStringStrict(Class targetClass, String messageString) } /** - * Get a date object parsed from a string representation. + * Get a Date object parsed from a string representation. * * @param timestamp string representation * @return Date object */ public static Date getDate(String timestamp) { - return timestamp == null ? null : Date.from(Instant.parse(timestamp)); + return timestamp == null ? null : Date.from(getInstant(timestamp)); + } + + /** + * Get an Instant object parsed from a string representation. Also perform some munging on the + * input string to handle standard-yet-not-supported formats. + * + * @param timestamp string representation + * @return Instant object + */ + public static Instant getInstant(String timestamp) { + String replaced = timestamp.replaceFirst("\\+0000$", "Z"); + return timestamp == null ? null : Instant.parse(replaced); } /** diff --git a/tests/traces/simple/devices/AHU-1/002_event_system.json b/tests/traces/simple/devices/AHU-1/002_event_system.json index ea6c810592..2eed54e6a6 100644 --- a/tests/traces/simple/devices/AHU-1/002_event_system.json +++ b/tests/traces/simple/devices/AHU-1/002_event_system.json @@ -9,6 +9,6 @@ "mem_total_mb": 25164.88, "store_total_mb": null }, - "timestamp" : "2022-07-19T05:04:36Z", + "timestamp" : "2022-07-19T05:04:36+0000", "version" : "1.3.14" } diff --git a/tests/traces/simple/expected/devices/AHU-1/event_system.attr b/tests/traces/simple/expected/devices/AHU-1/event_system.attr index fc5fbd4b91..2aa6af54a6 100644 --- a/tests/traces/simple/expected/devices/AHU-1/event_system.attr +++ b/tests/traces/simple/expected/devices/AHU-1/event_system.attr @@ -1,6 +1,6 @@ { "deviceNumId" : "00000062256946", - "publishTime" : "1999-10-20T01:02:03Z", + "publishTime" : "2022-07-19T05:04:36+0000", "subFolder" : "system", "deviceRegistryId" : "ZZ-TRI-FECTA", "subType" : "event", diff --git a/tests/traces/simple/expected/devices/AHU-1/event_system.json b/tests/traces/simple/expected/devices/AHU-1/event_system.json index 8b7a51ab13..af4cc83b1b 100644 --- a/tests/traces/simple/expected/devices/AHU-1/event_system.json +++ b/tests/traces/simple/expected/devices/AHU-1/event_system.json @@ -8,6 +8,6 @@ "metrics" : { "mem_total_mb" : 25164.88 }, - "timestamp" : "1999-10-20T01:02:03Z", + "timestamp" : "2022-07-19T05:04:36+0000", "version" : "1.3.14" } \ No newline at end of file diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index c105d32c61..510f42c1ca 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -25,6 +25,7 @@ import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static com.google.udmi.util.JsonUtil.JSON_SUFFIX; import static com.google.udmi.util.JsonUtil.OBJECT_MAPPER; +import static com.google.udmi.util.JsonUtil.getInstant; import static com.google.udmi.util.JsonUtil.getTimestamp; import static java.lang.String.format; import static java.util.Optional.ofNullable; @@ -118,7 +119,8 @@ public class Validator { public static final String STATE_PREFIX = "state_"; public static final String PROJECT_PROVIDER_PREFIX = "//"; public static final String TIMESTAMP_ZULU_SUFFIX = "Z"; - public static final String TIMESTAMP_UTC_SUFFIX = "+00:00"; + public static final String TIMESTAMP_UTC_SUFFIX_1 = "+00:00"; + public static final String TIMESTAMP_UTC_SUFFIX_2 = "+0000"; private static final String ERROR_FORMAT_INDENT = " "; private static final String SCHEMA_VALIDATION_FORMAT = "Validating %d schemas"; private static final String TARGET_VALIDATION_FORMAT = "Validating %d files against %s"; @@ -527,7 +529,7 @@ private void validateMessage( } if (simulatedMessages) { - mockNow = Instant.parse((String) message.get(TIMESTAMP_KEY)); + mockNow = getInstant((String) message.get(TIMESTAMP_KEY)); ReportingDevice.setMockNow(mockNow); } ReportingDevice reportingDevice = validateMessageCore(message, attributes); @@ -614,9 +616,9 @@ public void validateDeviceMessage(ReportingDevice device, Map me upgradeMessage(schemaName, message); String timestampRaw = (String) message.get("timestamp"); - Instant timestamp = ifNotNullGet(timestampRaw, Instant::parse); + Instant timestamp = ifNotNullGet(timestampRaw, JsonUtil::getInstant); String publishRaw = attributes.get(PUBLISH_TIME_KEY); - Instant publishTime = ifNotNullGet(publishRaw, Instant::parse); + Instant publishTime = ifNotNullGet(publishRaw, JsonUtil::getInstant); try { // TODO: Validate message contests to make sure state sub-blocks don't also have timestamp. @@ -632,7 +634,8 @@ public void validateDeviceMessage(ReportingDevice device, Map me } if (publishTime != null) { if (!timestampRaw.endsWith(TIMESTAMP_ZULU_SUFFIX) - && !timestampRaw.endsWith(TIMESTAMP_UTC_SUFFIX)) { + && !timestampRaw.endsWith(TIMESTAMP_UTC_SUFFIX_1) + && !timestampRaw.endsWith(TIMESTAMP_UTC_SUFFIX_2)) { throw new RuntimeException("Invalid timestamp timezone " + timestampRaw); } long between = Duration.between(publishTime, timestamp).getSeconds(); diff --git a/validator/src/test/java/com/google/daq/mqtt/validator/BasicTest.java b/validator/src/test/java/com/google/daq/mqtt/validator/BasicTest.java index 4b5dbefe92..8126ec38ed 100644 --- a/validator/src/test/java/com/google/daq/mqtt/validator/BasicTest.java +++ b/validator/src/test/java/com/google/daq/mqtt/validator/BasicTest.java @@ -1,6 +1,7 @@ package com.google.daq.mqtt.validator; import static com.google.udmi.util.Common.TIMESTAMP_KEY; +import static com.google.udmi.util.JsonUtil.getInstant; import static com.google.udmi.util.JsonUtil.getTimestamp; import static com.google.udmi.util.JsonUtil.safeSleep; import static org.junit.Assert.assertEquals; @@ -145,7 +146,7 @@ public void lastSeenUpdate() { ValidationState report = getValidationReport(); DeviceValidationEvent deviceValidationEvent = report.devices.get(TestCommon.DEVICE_ID); Date lastSeen = deviceValidationEvent.last_seen; - Instant parse = Instant.parse((String) eventBundle.message.get(TIMESTAMP_KEY)); + Instant parse = getInstant((String) eventBundle.message.get(TIMESTAMP_KEY)); assertEquals("device last seen", getTimestamp(Date.from(parse)), getTimestamp(lastSeen)); } From bcd6e90ad9c31681b85d7dfdc6afd585c9376421 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 10:14:30 -0500 Subject: [PATCH 26/55] Only check validateschema --- .../daq/mqtt/sequencer/SequenceBase.java | 31 +++++++++++++------ .../sequencer/sequences/ConfigSequences.java | 1 + 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 04cc111f38..4b247c45be 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -164,6 +164,7 @@ public class SequenceBase { public static final String SCHEMA_BUCKET = "schemas"; public static final int SCHEMA_SCORE = 5; public static final int CAPABILITY_SCORE = 1; + public static final String STATUS_LEVEL_VIOLATION = "STATUS_LEVEL"; private static final int FUNCTIONS_VERSION_BETA = 11; private static final int FUNCTIONS_VERSION_ALPHA = FUNCTIONS_VERSION_BETA; private static final long CONFIG_BARRIER_MS = 1000; @@ -232,6 +233,7 @@ public class SequenceBase { private static final Duration DEFAULT_LOOP_TIMEOUT = Duration.ofHours(30); private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", "system.last_config", "system.status"); + public static final String DEVICE_STATE_SCHEMA = "device_state"; protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -266,6 +268,7 @@ public class SequenceBase { private final Queue logEntryQueue = new LinkedBlockingDeque<>(); private final Stack waitingCondition = new Stack<>(); private final SortedMap> validationResults = new TreeMap<>(); + private final Map deviceStateViolations = new ConcurrentHashMap<>(); private final Map capabilityExceptions = new ConcurrentHashMap<>(); private final Set allowedDeviceStateChanges = new HashSet<>(); @@ -857,6 +860,12 @@ private void recordSchemaValidations(Description description) { collectSchemaResult(description, schemaName, FAIL, result.detail)); } }); + + SequenceResult result = deviceStateViolations.isEmpty() ? PASS : FAIL; + String message = result == PASS + ? "Only expected device state changes observed" + : "Unexpected device state changes: " + CSV_JOINER.join(deviceStateViolations.keySet()); + collectSchemaResult(description, DEVICE_STATE_SCHEMA, result, message); } private String uniqueKey(Entry entry) { @@ -1693,18 +1702,18 @@ protected void expectedStatusLevel(Level level) { } private void validateIntermediateState(State convertedState, List stateChanges) { - if (!recordSequence) { + if (!recordSequence || !shouldValidateSchema(testDescription)) { return; } int statusLevel = catchToElse(() -> convertedState.system.status.level, Level.TRACE.value()); if (statusLevel > maxAllowedStatusLevel) { - throw new RuntimeException(format("System status level %d exceeded allowed threshold %d", - statusLevel, maxAllowedStatusLevel)); + String message = format("System status level %d exceeded allowed threshold %d", statusLevel, + maxAllowedStatusLevel); + deviceStateViolations.put(STATUS_LEVEL_VIOLATION, message); } - List badChanges = stateChanges.stream() - .filter(not(this::changeAllowed)).map(DiffEntry::key).toList(); - checkState(badChanges.isEmpty(), "Disallowed state changes: " + CSV_JOINER.join(badChanges)); + deviceStateViolations.putAll(stateChanges.stream().filter(not(this::changeAllowed)).collect( + Collectors.toMap(DiffEntry::key, DiffEntry::toString))); } private boolean changeAllowed(DiffEntry change) { @@ -2099,6 +2108,10 @@ protected void forCapability(Capabilities capability, Runnable action) { } } + private boolean shouldValidateSchema(Description description) { + return description.getAnnotation(ValidateSchema.class) != null; + } + /** * Master list of test capabilities. */ @@ -2189,10 +2202,8 @@ protected void finished(Description description) { throw new IllegalStateException("Unexpected test method name"); } - if (testResult == PASS) { - ValidateSchema annotation = description.getAnnotation(ValidateSchema.class); - ifNotNullThen(annotation, a -> recordSchemaValidations(description)); - } + ifTrueThen(testResult == PASS && shouldValidateSchema(description), + () -> recordSchemaValidations(description)); notice("ending test " + testName + " after " + timeSinceStart() + " " + START_END_MARKER); testName = null; diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index 222396b61a..c890e2b858 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -119,6 +119,7 @@ public void device_config_acked() { @Feature(stage = ALPHA, bucket = SYSTEM, score = 4) @Capability(value = LOGGING, stage = ALPHA) @Summary("Check that the device correctly handles a broken (non-json) config message.") + @ValidateSchema public void broken_config() { expectedStatusLevel(Level.ERROR); deviceConfig.system.min_loglevel = Level.DEBUG.value(); From 3bcef41b357ca9fa63916619cd6c79b1995c6835 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 11:18:30 -0500 Subject: [PATCH 27/55] Fix some output tests --- etc/schema.out | 4 ++++ etc/schema_itemized.out | 13 +++++++++++-- .../mqtt/sequencer/sequences/ConfigSequences.java | 4 ++-- .../mqtt/sequencer/sequences/PointsetSequences.java | 3 +-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/etc/schema.out b/etc/schema.out index 94a69a9b11..0d13a3ade6 100644 --- a/etc/schema.out +++ b/etc/schema.out @@ -1,3 +1,7 @@ +RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed +RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found +RESULT pass schemas device_state_beta BETA 5/5 All schema validations passed +RESULT pass schemas device_state_stable STABLE 5/5 All schema validations passed RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed RESULT pass schemas event_pointset_beta BETA 5/5 All schema validations passed RESULT pass schemas event_pointset_stable STABLE 5/5 All schema validations passed diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 7e086aa9cc..33fb76a131 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -4,6 +4,7 @@ 04 pointset_remove_point 05 too_much_state 06 feature_enumeration +07 valid_serial_no RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed 07 valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed 07 valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed 07 valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed @@ -13,13 +14,21 @@ 11 system_mode_restart 12 config_logging 13 broken_config -14 broken_config +14 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed +14 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed +14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed +14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 15 broken_config -16 broken_config +16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 17 broken_config 18 system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found +18 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 All schema validations passed 18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed +19 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 All schema validations passed 19 system_last_update RESULT pass schemas event_pointset_stable STABLE 5/5 All schema validations passed 19 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed 19 system_last_update RESULT pass schemas state_update_stable STABLE 5/5 All schema validations passed diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index c890e2b858..e473ec6afc 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -53,8 +53,8 @@ public class ConfigSequences extends SequenceBase { @Summary("Check that last_update state is correctly set in response to a config update.") @ValidateSchema public void system_last_update() { - untilTrue("state last_config matches config timestamp", this::lastConfigUpdated); ensureStateUpdate(); + untilTrue("state last_config matches config timestamp", this::lastConfigUpdated); } @Test @@ -62,8 +62,8 @@ public void system_last_update() { @ValidateSchema public void valid_serial_no() { ifNullSkipTest(serialNo, "No test serial number provided"); - untilTrue("received serial number matches", () -> serialNo.equals(lastSerialNo)); ensureStateUpdate(); + untilTrue("received serial number matches", () -> serialNo.equals(lastSerialNo)); } @Test(timeout = TWO_MINUTES_MS) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java index c75e716106..ad09ae3657 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java @@ -135,8 +135,7 @@ public void pointset_publish() { ifNullSkipTest(deviceConfig.pointset, "no pointset found in config"); untilTrue("receive a pointset event", - () -> (countReceivedEvents(PointsetEvent.class) > 1 - )); + () -> (countReceivedEvents(PointsetEvent.class) > 1)); } /** From beb3c17a96f23016447e1fc02320bff8cb9d684c Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 11:19:27 -0500 Subject: [PATCH 28/55] Trigger commit From 610649c3e5683db8b93d48e2703635ce57be84a6 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 11:35:36 -0500 Subject: [PATCH 29/55] Adding device_state nostate schema out --- etc/schema_nostate.out | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/schema_nostate.out b/etc/schema_nostate.out index 33c0fe7b9a..33d72ef717 100644 --- a/etc/schema_nostate.out +++ b/etc/schema_nostate.out @@ -1,2 +1,3 @@ +RESULT pass schemas device_state_beta BETA 5/5 All schema validations passed RESULT pass schemas event_pointset_beta BETA 5/5 All schema validations passed RESULT pass schemas event_system_beta BETA 5/5 All schema validations passed From 081448ebda17714433ec95ac4d8a301182f658f8 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 11:36:47 -0500 Subject: [PATCH 30/55] Update itemized expected out --- etc/test_itemized.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/test_itemized.out b/etc/test_itemized.out index e0b237035e..5335380305 100644 --- a/etc/test_itemized.out +++ b/etc/test_itemized.out @@ -24,7 +24,7 @@ 19 RESULT pass system system_last_update STABLE 5/5 Sequence complete 20 RESULT fail system state_make_model BETA 0/5 Timeout waiting for no applicable system status 24 RESULT skip system valid_serial_no ALPHA 0/0 State testing disabled -25 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches config timestamp +25 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for received at least one state update 26 RESULT fail system system_min_loglevel ALPHA 0/5 Received state update with no-state device 27 RESULT pass system system_min_loglevel ALPHA 5/5 Sequence complete 31 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete From 25cc1b773a3f759a4e417fbaff0519343054dc75 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 11:37:51 -0500 Subject: [PATCH 31/55] Fix itemized schema out --- etc/schema_itemized.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 33fb76a131..09d93be5ae 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -19,7 +19,7 @@ 14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed 14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 15 broken_config -16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found 16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed 16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed 16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed From a7f35f882a31d9931e3ccc82f3583dc70fb1789c Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 20:08:35 -0500 Subject: [PATCH 32/55] Update last_config --- etc/sequencer_nostate.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/sequencer_nostate.out b/etc/sequencer_nostate.out index 9d64ffbc6c..09f526c5f9 100644 --- a/etc/sequencer_nostate.out +++ b/etc/sequencer_nostate.out @@ -10,4 +10,4 @@ RESULT skip pointset pointset_request_extraneous BETA 0/0 State testing disabled RESULT skip pointset pointset_sample_rate BETA 0/0 State testing disabled RESULT skip system state_make_model BETA 0/0 State testing disabled RESULT skip system state_software BETA 0/0 State testing disabled -RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches config timestamp +RESULT fail system system_last_update STABLE 0/5 Timeout waiting for received at least one state update From 77da8026ab3da5631a0a92759161126f1bbde7b2 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 11:12:30 -0500 Subject: [PATCH 33/55] Fix schema expected out --- etc/schema_itemized.out | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 09d93be5ae..9af94cec36 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -19,8 +19,7 @@ 14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed 14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 15 broken_config -16 broken_config RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found -16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed 16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed 16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 17 broken_config From 26956b2ef2f9b43435ceffe05545b497a2d0e728 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 11:32:08 -0500 Subject: [PATCH 34/55] Trigger commit From 03138f3aa51a2e464c4269eb405ec6e0a5200509 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 12:53:32 -0500 Subject: [PATCH 35/55] Adding likely missing things --- etc/schema_itemized.out | 1 + .../main/java/com/google/daq/mqtt/sequencer/SequenceBase.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 9af94cec36..33fb76a131 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -20,6 +20,7 @@ 14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 15 broken_config 16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed 16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed 16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed 17 broken_config diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 4b247c45be..b91ac24154 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -231,8 +231,8 @@ public class SequenceBase { private static final Duration DEFAULT_WAIT_TIME = Duration.ofSeconds(10); private static final Duration LOG_WAIT_TIME = Duration.ofSeconds(30); private static final Duration DEFAULT_LOOP_TIMEOUT = Duration.ofHours(30); - private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of("timestamp", - "system.last_config", "system.status"); + private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of( + "timestamp", "system.last_config", "system.status"); public static final String DEVICE_STATE_SCHEMA = "device_state"; protected static Metadata deviceMetadata; protected static String projectId; From 36414816d6edd41801f8bf09e0b7fb9d96ab39f5 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 13:28:56 -0500 Subject: [PATCH 36/55] Trigger commit From e5d6db26337a4c0e32edf65751f13876681b7b8a Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 13:32:21 -0500 Subject: [PATCH 37/55] Update to latest --- udmis/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/udmis/build.gradle b/udmis/build.gradle index 0eeb27454c..67e989ca9c 100644 --- a/udmis/build.gradle +++ b/udmis/build.gradle @@ -92,7 +92,7 @@ dependencies { exclude group: 'com.google.guava', module: 'guava-jdk5' } - implementation ('io.github.clearblade:clearblade-cloud-iot:1.0.2') { + implementation ('io.github.clearblade:clearblade-cloud-iot:1.0.3') { // Exclude transitive dependency causing problems with running unit tests in the IDE. exclude group: 'org.junit.jupiter', module: 'junit-jupiter-api' } From 9d1a070d2b8bf5284306059f480be89a9dd44642 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 05:09:31 -0500 Subject: [PATCH 38/55] REstrict schema validations --- .../daq/mqtt/sequencer/SequenceBase.java | 23 ++++++++++--------- .../daq/mqtt/sequencer/ValidateSchema.java | 6 +++++ .../sequencer/sequences/BlobsetSequences.java | 3 ++- .../sequencer/sequences/ConfigSequences.java | 8 ++++--- .../sequences/PointsetSequences.java | 3 ++- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index b91ac24154..bb79f41509 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -8,8 +8,6 @@ import static com.google.daq.mqtt.sequencer.SequenceBase.Capabilities.LAST_CONFIG; import static com.google.daq.mqtt.sequencer.semantic.SemanticValue.actualize; import static com.google.daq.mqtt.util.IotReflectorClient.REFLECTOR_PREFIX; -import static com.google.daq.mqtt.validator.Validator.CONFIG_PREFIX; -import static com.google.daq.mqtt.validator.Validator.STATE_PREFIX; import static com.google.udmi.util.CleanDateFormat.cleanDate; import static com.google.udmi.util.CleanDateFormat.dateEquals; import static com.google.udmi.util.Common.DEVICE_ID_KEY; @@ -301,6 +299,7 @@ public class SequenceBase { private int startStateCount; private Boolean expectedSystemStatus; private Description testDescription; + private SubFolder testSchema; private static void setupSequencer() { exeConfig = SequenceRunner.ensureExecutionConfig(); @@ -473,13 +472,13 @@ private static String makeMessageBase(Envelope attributes) { } @NotNull - private static Predicate>> isInterestingValidation() { + private Predicate>> isInterestingValidation() { return entry -> isInterestingValidation(entry.getKey()); } - private static boolean isInterestingValidation(String schemaName) { - return !schemaName.startsWith(CONFIG_PREFIX) - && (!schemaName.startsWith(STATE_PREFIX) || schemaName.equals(STATE_UPDATE_MESSAGE_TYPE)); + private boolean isInterestingValidation(String schemaName) { + String targetSchema = format("event_%s", ifNotNullGet(testSchema, SubFolder::value)); + return schemaName.equals(targetSchema) || schemaName.equals(STATE_UPDATE_MESSAGE_TYPE); } private static void emitSequenceResult(SequenceResult result, String bucket, String name, @@ -535,7 +534,7 @@ private static Entry summarizeSchemaResults( logEntry.message = "No messages validated"; logEntry.level = WARNING.value(); } else if (failures.isEmpty()) { - logEntry.message = "All schema validations passed"; + logEntry.message = "Schema validation passed"; logEntry.level = NOTICE.value(); } else { logEntry.message = "Schema violations found"; @@ -1702,7 +1701,7 @@ protected void expectedStatusLevel(Level level) { } private void validateIntermediateState(State convertedState, List stateChanges) { - if (!recordSequence || !shouldValidateSchema(testDescription)) { + if (!recordSequence || !shouldValidateSchema(SubFolder.VALIDATION)) { return; } @@ -2108,8 +2107,8 @@ protected void forCapability(Capabilities capability, Runnable action) { } } - private boolean shouldValidateSchema(Description description) { - return description.getAnnotation(ValidateSchema.class) != null; + private boolean shouldValidateSchema(SubFolder folder) { + return testSchema != null && (testSchema == folder || folder == SubFolder.VALIDATION); } /** @@ -2171,6 +2170,8 @@ protected void starting(@NotNull Description description) { testStage = getTestStage(description); testBucket = getBucket(description); testResult = SequenceResult.START; + testSchema = ifNotNullGet(description.getAnnotation(ValidateSchema.class), + ValidateSchema::value); testDir = new File(new File(deviceOutputDir, TESTS_OUT_DIR), testName); FileUtils.deleteDirectory(testDir); @@ -2202,7 +2203,7 @@ protected void finished(Description description) { throw new IllegalStateException("Unexpected test method name"); } - ifTrueThen(testResult == PASS && shouldValidateSchema(description), + ifTrueThen(testResult == PASS && shouldValidateSchema(SubFolder.VALIDATION), () -> recordSchemaValidations(description)); notice("ending test " + testName + " after " + timeSinceStart() + " " + START_END_MARKER); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/ValidateSchema.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/ValidateSchema.java index f3c68ad792..b6d0315d95 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/ValidateSchema.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/ValidateSchema.java @@ -4,6 +4,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import udmi.schema.Envelope.SubFolder; /** * Indicate that test post-processing should apply schema validation. Don't include by @@ -13,4 +14,9 @@ @Target({ElementType.METHOD}) public @interface ValidateSchema { + /** + * Limit the subFolder that should be validated. + */ + SubFolder value(); + } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java index 0db22bea0a..2a5c725262 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java @@ -28,6 +28,7 @@ import udmi.schema.EndpointConfiguration; import udmi.schema.EndpointConfiguration.Protocol; import udmi.schema.Entry; +import udmi.schema.Envelope.SubFolder; import udmi.schema.Level; import udmi.schema.Operation.SystemMode; @@ -166,7 +167,7 @@ public void endpoint_connection_success_reconnect() { @Feature(stage = ALPHA, bucket = ENDPOINT) @Summary("Failed connection because of bad hash.") - @ValidateSchema + @ValidateSchema(SubFolder.BLOBSET) @Test public void endpoint_connection_bad_hash() { setDeviceConfigEndpointBlob(getAlternateEndpointHostname(), registryId, true); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index e473ec6afc..b58ff0241c 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; import udmi.schema.Entry; +import udmi.schema.Envelope.SubFolder; import udmi.schema.Level; /** @@ -51,7 +52,7 @@ public class ConfigSequences extends SequenceBase { @Test(timeout = ONE_MINUTE_MS) @Feature(stage = STABLE, bucket = SYSTEM, nostate = true) @Summary("Check that last_update state is correctly set in response to a config update.") - @ValidateSchema + @ValidateSchema(SubFolder.SYSTEM) public void system_last_update() { ensureStateUpdate(); untilTrue("state last_config matches config timestamp", this::lastConfigUpdated); @@ -59,7 +60,7 @@ public void system_last_update() { @Test @Feature(stage = ALPHA, bucket = SYSTEM) - @ValidateSchema + @ValidateSchema(SubFolder.SYSTEM) public void valid_serial_no() { ifNullSkipTest(serialNo, "No test serial number provided"); ensureStateUpdate(); @@ -69,6 +70,7 @@ public void valid_serial_no() { @Test(timeout = TWO_MINUTES_MS) @Feature(stage = ALPHA, bucket = SYSTEM, nostate = true) @Summary("Check that the min log-level config is honored by the device.") + @ValidateSchema(SubFolder.SYSTEM) public void system_min_loglevel() { Integer savedLevel = deviceConfig.system.min_loglevel; checkState(SYSTEM_CONFIG_APPLY_LEVEL.value() >= savedLevel, "invalid saved level"); @@ -119,7 +121,7 @@ public void device_config_acked() { @Feature(stage = ALPHA, bucket = SYSTEM, score = 4) @Capability(value = LOGGING, stage = ALPHA) @Summary("Check that the device correctly handles a broken (non-json) config message.") - @ValidateSchema + @ValidateSchema(SubFolder.SYSTEM) public void broken_config() { expectedStatusLevel(Level.ERROR); deviceConfig.system.min_loglevel = Level.DEBUG.value(); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java index ad09ae3657..3055888063 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; +import udmi.schema.Envelope.SubFolder; import udmi.schema.Level; import udmi.schema.PointPointsetConfig; import udmi.schema.PointPointsetEvent; @@ -207,7 +208,7 @@ private String samplingMessagesCheckMessage(SamplingRange samplingRange) { @Test(timeout = THREE_MINUTES_MS) @Summary("Check handling of sample rate and sample limit sec") @Feature(stage = BETA, bucket = POINTSET, nostate = true) - @ValidateSchema + @ValidateSchema(SubFolder.POINTSET) public void pointset_publish_interval() { ifNullSkipTest(deviceConfig.pointset, "no pointset found in config"); From 0c26a39bb50a312b0df61ef606fb7c77b387d7f1 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 05:19:17 -0500 Subject: [PATCH 39/55] Dynamically allow some changes --- .../com/google/daq/mqtt/sequencer/SequenceBase.java | 10 ++++++++-- .../daq/mqtt/sequencer/sequences/ConfigSequences.java | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index bb79f41509..c9ce8cbfea 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -1722,8 +1722,14 @@ private boolean changeAllowed(DiffEntry change) { } protected void allowDeviceStateChange(String changePrefix) { - if (allowedDeviceStateChanges.add(changePrefix)) { - debug("Allowing device state change: " + changePrefix); + if (!allowedDeviceStateChanges.add(changePrefix)) { + throw new AbortMessageLoop("State change prefix already allowed: " + changePrefix); + } + } + + protected void disallowDeviceStateChange(String changePrefix) { + if (!allowedDeviceStateChanges.remove(changePrefix)) { + throw new AbortMessageLoop("Unexpected state change removal: " + changePrefix); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index b58ff0241c..20d8149d5f 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -48,6 +48,7 @@ public class ConfigSequences extends SequenceBase { private static final long LOG_APPLY_DELAY_MS = 1000; // How frequently to send out confg queries for device config acked check. private static final Duration CONFIG_QUERY_INTERVAL = Duration.ofSeconds(30); + public static final String ALL_CHANGES = ""; @Test(timeout = ONE_MINUTE_MS) @Feature(stage = STABLE, bucket = SYSTEM, nostate = true) @@ -124,6 +125,7 @@ public void device_config_acked() { @ValidateSchema(SubFolder.SYSTEM) public void broken_config() { expectedStatusLevel(Level.ERROR); + deviceConfig.system.min_loglevel = Level.DEBUG.value(); updateConfig("starting broken_config"); Date stableConfig = deviceConfig.timestamp; @@ -156,6 +158,8 @@ public void broken_config() { checkNotLogged(SYSTEM_CONFIG_APPLY, SYSTEM_CONFIG_APPLY_LEVEL); }); + // When resetting a config, all sorts of things can change so allow everything. + allowDeviceStateChange(ALL_CHANGES); // Will restore min_loglevel to the default of INFO. resetConfig(); // clears extra_field and interesting status checks @@ -165,6 +169,8 @@ public void broken_config() { () -> dateEquals(deviceConfig.timestamp, deviceState.system.last_config)); }); + disallowDeviceStateChange(ALL_CHANGES); + deviceConfig.system.min_loglevel = Level.DEBUG.value(); untilTrue("last_config updated", () -> !dateEquals(stableConfig, deviceState.system.last_config) From f2584a03ecd58ebf359b3baaae7b46257c7995ad Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 05:23:09 -0500 Subject: [PATCH 40/55] Allow all changes --- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 5 +++++ .../google/daq/mqtt/sequencer/sequences/ConfigSequences.java | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index c9ce8cbfea..4d93f1dced 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -163,6 +163,7 @@ public class SequenceBase { public static final int SCHEMA_SCORE = 5; public static final int CAPABILITY_SCORE = 1; public static final String STATUS_LEVEL_VIOLATION = "STATUS_LEVEL"; + private static final String ALL_CHANGES = ""; private static final int FUNCTIONS_VERSION_BETA = 11; private static final int FUNCTIONS_VERSION_ALPHA = FUNCTIONS_VERSION_BETA; private static final long CONFIG_BARRIER_MS = 1000; @@ -777,6 +778,8 @@ protected void resetConfig() { } protected void resetConfig(boolean fullReset) { + allowDeviceStateChange(ALL_CHANGES); + recordSequence("Force reset config"); withRecordSequence(false, () -> { debug("Starting reset_config full reset " + fullReset); @@ -794,6 +797,8 @@ protected void resetConfig(boolean fullReset) { debug("Done with reset_config"); resetRequired = false; }); + + disallowDeviceStateChange(ALL_CHANGES); } private void waitForConfigSync() { diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index 20d8149d5f..10f9eb490c 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -48,7 +48,6 @@ public class ConfigSequences extends SequenceBase { private static final long LOG_APPLY_DELAY_MS = 1000; // How frequently to send out confg queries for device config acked check. private static final Duration CONFIG_QUERY_INTERVAL = Duration.ofSeconds(30); - public static final String ALL_CHANGES = ""; @Test(timeout = ONE_MINUTE_MS) @Feature(stage = STABLE, bucket = SYSTEM, nostate = true) @@ -158,8 +157,6 @@ public void broken_config() { checkNotLogged(SYSTEM_CONFIG_APPLY, SYSTEM_CONFIG_APPLY_LEVEL); }); - // When resetting a config, all sorts of things can change so allow everything. - allowDeviceStateChange(ALL_CHANGES); // Will restore min_loglevel to the default of INFO. resetConfig(); // clears extra_field and interesting status checks @@ -169,8 +166,6 @@ public void broken_config() { () -> dateEquals(deviceConfig.timestamp, deviceState.system.last_config)); }); - disallowDeviceStateChange(ALL_CHANGES); - deviceConfig.system.min_loglevel = Level.DEBUG.value(); untilTrue("last_config updated", () -> !dateEquals(stableConfig, deviceState.system.last_config) From d95202e75c3a5bff7188196825a25f4100620d3b Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 05:25:51 -0500 Subject: [PATCH 41/55] Fix expected nostate schema out --- etc/schema_nostate.out | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/schema_nostate.out b/etc/schema_nostate.out index 33d72ef717..de6cd983e2 100644 --- a/etc/schema_nostate.out +++ b/etc/schema_nostate.out @@ -1,3 +1,3 @@ -RESULT pass schemas device_state_beta BETA 5/5 All schema validations passed -RESULT pass schemas event_pointset_beta BETA 5/5 All schema validations passed -RESULT pass schemas event_system_beta BETA 5/5 All schema validations passed +RESULT pass schemas device_state_beta BETA 5/5 Schema validation passed +RESULT pass schemas event_pointset_beta BETA 5/5 Schema validation passed +RESULT pass schemas event_system_beta BETA 5/5 Schema validation passed From 9dcdabf1b88e726af97cdb73726d31ce059b58d1 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 05:28:04 -0500 Subject: [PATCH 42/55] Adjusting language --- etc/schema.out | 24 ++++++++++++------------ etc/schema_itemized.out | 36 ++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/etc/schema.out b/etc/schema.out index 0d13a3ade6..3238751a37 100644 --- a/etc/schema.out +++ b/etc/schema.out @@ -1,13 +1,13 @@ -RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed +RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found -RESULT pass schemas device_state_beta BETA 5/5 All schema validations passed -RESULT pass schemas device_state_stable STABLE 5/5 All schema validations passed -RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed -RESULT pass schemas event_pointset_beta BETA 5/5 All schema validations passed -RESULT pass schemas event_pointset_stable STABLE 5/5 All schema validations passed -RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed -RESULT pass schemas event_system_beta BETA 5/5 All schema validations passed -RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed -RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed -RESULT pass schemas state_update_beta BETA 5/5 All schema validations passed -RESULT pass schemas state_update_stable STABLE 5/5 All schema validations passed +RESULT pass schemas device_state_beta BETA 5/5 Schema validation passed +RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed +RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed +RESULT pass schemas event_pointset_beta BETA 5/5 Schema validation passed +RESULT pass schemas event_pointset_stable STABLE 5/5 Schema validation passed +RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed +RESULT pass schemas event_system_beta BETA 5/5 Schema validation passed +RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed +RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed +RESULT pass schemas state_update_beta BETA 5/5 Schema validation passed +RESULT pass schemas state_update_stable STABLE 5/5 Schema validation passed diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 33fb76a131..cdf20adbe0 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -4,34 +4,34 @@ 04 pointset_remove_point 05 too_much_state 06 feature_enumeration -07 valid_serial_no RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed -07 valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed -07 valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed -07 valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed +07 valid_serial_no RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed +07 valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed +07 valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed +07 valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 08 writeback_success 09 writeback_success 10 pointset_sample_rate 11 system_mode_restart 12 config_logging 13 broken_config -14 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed -14 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed -14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed -14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed +14 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed +14 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed +14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed +14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 15 broken_config -16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 All schema validations passed -16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed -16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed -16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed +16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed +16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed +16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed +16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 17 broken_config 18 system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found -18 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 All schema validations passed -18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed -19 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 All schema validations passed -19 system_last_update RESULT pass schemas event_pointset_stable STABLE 5/5 All schema validations passed -19 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed -19 system_last_update RESULT pass schemas state_update_stable STABLE 5/5 All schema validations passed +18 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed +18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed +19 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed +19 system_last_update RESULT pass schemas event_pointset_stable STABLE 5/5 Schema validation passed +19 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed +19 system_last_update RESULT pass schemas state_update_stable STABLE 5/5 Schema validation passed 20 state_make_model 24 valid_serial_no 25 system_last_update From 7f27dc9261dbf20a985cd756bfc4d3fd784a83a0 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 05:55:18 -0500 Subject: [PATCH 43/55] Fixing expected out --- etc/schema.out | 3 --- etc/schema_itemized.out | 10 +++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/etc/schema.out b/etc/schema.out index 3238751a37..ea100f1b6c 100644 --- a/etc/schema.out +++ b/etc/schema.out @@ -2,11 +2,8 @@ RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found RESULT pass schemas device_state_beta BETA 5/5 Schema validation passed RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed -RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed RESULT pass schemas event_pointset_beta BETA 5/5 Schema validation passed -RESULT pass schemas event_pointset_stable STABLE 5/5 Schema validation passed RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed -RESULT pass schemas event_system_beta BETA 5/5 Schema validation passed RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed RESULT pass schemas state_update_beta BETA 5/5 Schema validation passed diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index cdf20adbe0..11dbfadc67 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -5,7 +5,6 @@ 05 too_much_state 06 feature_enumeration 07 valid_serial_no RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed -07 valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed 07 valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 07 valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 08 writeback_success @@ -15,28 +14,25 @@ 12 config_logging 13 broken_config 14 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed -14 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed 14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 15 broken_config -16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed -16 broken_config RESULT pass schemas event_pointset_alpha ALPHA 5/5 Schema validation passed +16 broken_config RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found 16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 17 broken_config -18 system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed 18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed 19 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed -19 system_last_update RESULT pass schemas event_pointset_stable STABLE 5/5 Schema validation passed 19 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed 19 system_last_update RESULT pass schemas state_update_stable STABLE 5/5 Schema validation passed 20 state_make_model 24 valid_serial_no 25 system_last_update 26 system_min_loglevel -27 system_min_loglevel +27 system_min_loglevel RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed +27 system_min_loglevel RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 31 gateway_proxy_events 32 gateway_proxy_events 36 gateway_proxy_events From ad9b6cedf351ca99e2bdf9cbd71a8007f7333600 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 12:09:32 -0500 Subject: [PATCH 44/55] Fixing expected out --- etc/schema_itemized.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 11dbfadc67..2962743b2f 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -17,7 +17,7 @@ 14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 15 broken_config -16 broken_config RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found +16 broken_config RESULT fail schemas device_state_alpha ALPHA 5/5 Schema validation passed 16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 17 broken_config From d50df5f11185b0a8f4fc7bdd0a3869b8e41f9453 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 12:29:10 -0500 Subject: [PATCH 45/55] Fixed schema itemized out --- etc/schema_itemized.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 2962743b2f..5976c40876 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -17,7 +17,7 @@ 14 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 14 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 15 broken_config -16 broken_config RESULT fail schemas device_state_alpha ALPHA 5/5 Schema validation passed +16 broken_config RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed 16 broken_config RESULT pass schemas event_system_alpha ALPHA 5/5 Schema validation passed 16 broken_config RESULT pass schemas state_update_alpha ALPHA 5/5 Schema validation passed 17 broken_config From 785be6f485e4388d08b80c6fa93ff0c7b30c6304 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 6 Dec 2023 17:22:41 -0500 Subject: [PATCH 46/55] Add better logging --- .../daq/mqtt/sequencer/SequenceBase.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 4d93f1dced..c6790c369e 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -163,6 +163,7 @@ public class SequenceBase { public static final int SCHEMA_SCORE = 5; public static final int CAPABILITY_SCORE = 1; public static final String STATUS_LEVEL_VIOLATION = "STATUS_LEVEL"; + public static final String DEVICE_STATE_SCHEMA = "device_state"; private static final String ALL_CHANGES = ""; private static final int FUNCTIONS_VERSION_BETA = 11; private static final int FUNCTIONS_VERSION_ALPHA = FUNCTIONS_VERSION_BETA; @@ -232,7 +233,6 @@ public class SequenceBase { private static final Duration DEFAULT_LOOP_TIMEOUT = Duration.ofHours(30); private static final Set SYSTEM_STATE_CHANGES = ImmutableSet.of( "timestamp", "system.last_config", "system.status"); - public static final String DEVICE_STATE_SCHEMA = "device_state"; protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -472,16 +472,6 @@ private static String makeMessageBase(Envelope attributes) { return format("%s_%s%s", subType, subFolder, deviceSuffix); } - @NotNull - private Predicate>> isInterestingValidation() { - return entry -> isInterestingValidation(entry.getKey()); - } - - private boolean isInterestingValidation(String schemaName) { - String targetSchema = format("event_%s", ifNotNullGet(testSchema, SubFolder::value)); - return schemaName.equals(targetSchema) || schemaName.equals(STATE_UPDATE_MESSAGE_TYPE); - } - private static void emitSequenceResult(SequenceResult result, String bucket, String name, String stage, int score, int total, String message) { emitSequencerOut(format(RESULT_FORMAT, result, bucket, name, stage, score, total, message)); @@ -587,6 +577,16 @@ private static Map getCapabilities(Description desc) { } } + @NotNull + private Predicate>> isInterestingValidation() { + return entry -> isInterestingValidation(entry.getKey()); + } + + private boolean isInterestingValidation(String schemaName) { + String targetSchema = format("event_%s", ifNotNullGet(testSchema, SubFolder::value)); + return schemaName.equals(targetSchema) || schemaName.equals(STATE_UPDATE_MESSAGE_TYPE); + } + private Map.Entry emitCapabilityResult(Capabilities capability, Exception state, Capability cap, Bucket bucket, String methodName) { boolean pass = state instanceof CapabilitySuccess; @@ -1715,9 +1715,12 @@ private void validateIntermediateState(State convertedState, List sta String message = format("System status level %d exceeded allowed threshold %d", statusLevel, maxAllowedStatusLevel); deviceStateViolations.put(STATUS_LEVEL_VIOLATION, message); + warning(message); } - deviceStateViolations.putAll(stateChanges.stream().filter(not(this::changeAllowed)).collect( - Collectors.toMap(DiffEntry::key, DiffEntry::toString))); + Map badChanges = stateChanges.stream().filter(not(this::changeAllowed)) + .collect(Collectors.toMap(DiffEntry::key, DiffEntry::toString)); + badChanges.values().stream().map(x -> "Unexpected state change: " + x).forEach(this::warning); + deviceStateViolations.putAll(badChanges); } private boolean changeAllowed(DiffEntry change) { From 0a494f3122645e029f784ecd85bc20ba1758cd65 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 00:20:28 -0500 Subject: [PATCH 47/55] Adding synchornized on device reset --- .../com/google/daq/mqtt/sequencer/SequenceBase.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index c6790c369e..71d40abc2a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -749,8 +749,7 @@ public void setUp() { clearReceivedEvents(); validationResults.clear(); - forCapability(LAST_CONFIG, - () -> waitFor("state last_config sync", this::lastConfigUpdatedString)); + waitForStateConfigSync(); recordSequence = true; waitingConditionPush("executing test"); @@ -758,6 +757,11 @@ public void setUp() { debug(format("stage begin %s at %s", currentWaitingCondition(), timeSinceStart())); } + private void waitForStateConfigSync() { + forCapability(LAST_CONFIG, + () -> waitFor("state last_config sync", this::lastConfigUpdatedString)); + } + private boolean deviceSupportsState() { return !isTrue(catchToNull(() -> deviceMetadata.testing.nostate)); } @@ -798,6 +802,7 @@ protected void resetConfig(boolean fullReset) { resetRequired = false; }); + waitForStateConfigSync(); disallowDeviceStateChange(ALL_CHANGES); } @@ -1730,12 +1735,14 @@ private boolean changeAllowed(DiffEntry change) { } protected void allowDeviceStateChange(String changePrefix) { + info("Allowing device state change " + changePrefix); if (!allowedDeviceStateChanges.add(changePrefix)) { throw new AbortMessageLoop("State change prefix already allowed: " + changePrefix); } } protected void disallowDeviceStateChange(String changePrefix) { + info("Disallowing device state change " + changePrefix); if (!allowedDeviceStateChanges.remove(changePrefix)) { throw new AbortMessageLoop("Unexpected state change removal: " + changePrefix); } From 2130f7ca5650c94096b63be411369d5223551255 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 05:19:43 -0500 Subject: [PATCH 48/55] Fix last expected out --- etc/schema.out | 1 - validator/sequences/broken_config/sequence.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/schema.out b/etc/schema.out index ea100f1b6c..81a2742d0f 100644 --- a/etc/schema.out +++ b/etc/schema.out @@ -1,5 +1,4 @@ RESULT pass schemas device_state_alpha ALPHA 5/5 Schema validation passed -RESULT fail schemas device_state_alpha ALPHA 5/5 Schema violations found RESULT pass schemas device_state_beta BETA 5/5 Schema validation passed RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed RESULT pass schemas event_pointset_beta BETA 5/5 Schema validation passed diff --git a/validator/sequences/broken_config/sequence.md b/validator/sequences/broken_config/sequence.md index 449261e319..52babad661 100644 --- a/validator/sequences/broken_config/sequence.md +++ b/validator/sequences/broken_config/sequence.md @@ -14,6 +14,7 @@ Check that the device correctly handles a broken (non-json) config message. 1. Wait for log category `system.config.parse` level `ERROR` to be logged 1. Check that log category `system.config.apply` level `NOTICE` not logged 1. Force reset config +1. Wait for state last_config sync 1. Wait for log category `system.config.apply` level `NOTICE` to be logged 1. Wait for restored state synchronized 1. Update config before last_config updated: From 8d3b4dfd846b46956fc86f71c923613f6716d80f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 05:29:27 -0500 Subject: [PATCH 49/55] Fix expected out --- etc/schema_itemized.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 5976c40876..4bcf7cae02 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -24,7 +24,7 @@ 18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found 18 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed 18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed -19 system_last_update RESULT pass schemas device_state_stable STABLE 5/5 Schema validation passed +19 system_last_update RESULT fail schemas device_state_stable STABLE 5/5 Schema violations found 19 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 Schema validation passed 19 system_last_update RESULT pass schemas state_update_stable STABLE 5/5 Schema validation passed 20 state_make_model From aab9c64b31745def2e3e7b019c0b3dfe8795ed7a Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 08:00:01 -0500 Subject: [PATCH 50/55] Tweak test ot be more stable --- .../google/daq/mqtt/sequencer/sequences/ConfigSequences.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index 10f9eb490c..65f52f8083 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -54,8 +54,9 @@ public class ConfigSequences extends SequenceBase { @Summary("Check that last_update state is correctly set in response to a config update.") @ValidateSchema(SubFolder.SYSTEM) public void system_last_update() { - ensureStateUpdate(); - untilTrue("state last_config matches config timestamp", this::lastConfigUpdated); + untilTrue("state last_config matches first config timestamp", this::lastConfigUpdated); + forceConfigUpdate("trigger another config update"); + untilTrue("state last_config matches new config timestamp", this::lastConfigUpdated); } @Test From 855bf3b276467c9b33be5c8155f1753c916a8813 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 08:12:48 -0500 Subject: [PATCH 51/55] Fix expected out --- etc/sequencer_nostate.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/sequencer_nostate.out b/etc/sequencer_nostate.out index 09f526c5f9..5d1a6e6fc4 100644 --- a/etc/sequencer_nostate.out +++ b/etc/sequencer_nostate.out @@ -10,4 +10,4 @@ RESULT skip pointset pointset_request_extraneous BETA 0/0 State testing disabled RESULT skip pointset pointset_sample_rate BETA 0/0 State testing disabled RESULT skip system state_make_model BETA 0/0 State testing disabled RESULT skip system state_software BETA 0/0 State testing disabled -RESULT fail system system_last_update STABLE 0/5 Timeout waiting for received at least one state update +RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches first config timestamp From 9bd34cdf3324640a15067b4ded4cfb6c0e463b58 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 08:28:50 -0500 Subject: [PATCH 52/55] Fix generated out --- bin/test_sequcheck | 7 +++++-- docs/specs/sequences/generated.md | 4 +++- validator/sequences/system_last_update/sequence.md | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bin/test_sequcheck b/bin/test_sequcheck index 801ef4361c..21f5aad42d 100755 --- a/bin/test_sequcheck +++ b/bin/test_sequcheck @@ -26,8 +26,11 @@ diff -u out/schema.out $UDMI_ROOT/etc/schema${suffix}.out || failures+="schema " cp $UDMI_ROOT/docs/specs/sequences/generated.md out/generated.tmp # Save for test/comparison bin/gencode_seq || failures+="gencode " -echo Comparing diff out/generated.tmp docs/specs/sequences/generated.md -diff -u out/generated.tmp $UDMI_ROOT/docs/specs/sequences/generated.md || failures+="gencode_seq " +mv $UDMI_ROOT/docs/specs/sequences/generated.md out/generated.out +mv out/generated.tmp $UDMI_ROOT/docs/specs/sequences/generated.md + +echo Comparing diff out/generated.out docs/specs/sequences/generated.md +diff -u out/generated.out $UDMI_ROOT/docs/specs/sequences/generated.md || failures+="gencode_seq " SAMPLE_TEST_ROW="| system | system_min_loglevel | alpha | pass | Sequence complete |" target=sites/udmi_site_model/out/devices/AHU-1/results.md diff --git a/docs/specs/sequences/generated.md b/docs/specs/sequences/generated.md index fcd6105982..24e1f39137 100644 --- a/docs/specs/sequences/generated.md +++ b/docs/specs/sequences/generated.md @@ -196,4 +196,6 @@ Check that a device publishes correct software information in state messages Check that last_update state is correctly set in response to a config update. -1. Wait for state last_config matches config timestamp +1. Wait for state last_config matches first config timestamp +1. Force config update to trigger another config update +1. Wait for state last_config matches new config timestamp diff --git a/validator/sequences/system_last_update/sequence.md b/validator/sequences/system_last_update/sequence.md index 2909b72ad2..528500e8f5 100644 --- a/validator/sequences/system_last_update/sequence.md +++ b/validator/sequences/system_last_update/sequence.md @@ -3,4 +3,6 @@ Check that last_update state is correctly set in response to a config update. -1. Wait for state last_config matches config timestamp +1. Wait for state last_config matches first config timestamp +1. Force config update to trigger another config update +1. Wait for state last_config matches new config timestamp From 5288c54b5dde3b3363c8bc03231f7f8388c63f15 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 09:18:50 -0500 Subject: [PATCH 53/55] Cleanup lingering processes --- bin/support | 4 ++++ bin/test_itemized | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/bin/support b/bin/support index 12a85fc002..6008dfac6c 100755 --- a/bin/support +++ b/bin/support @@ -37,6 +37,10 @@ if [[ -n $UDMI_REGISTRY_SUFFIX && -n $site_model ]]; then site_model=$site_model${UDMI_REGISTRY_SUFFIX} fi +echo Lingering pubber processes: +ps ax | fgrep pubber | fgrep java || true +echo + echo tar --exclude-vcs -czf /tmp/$ARCHIVE $site_model $OUT_DIR tar --exclude-vcs -czf /tmp/$ARCHIVE $site_model $OUT_DIR mv /tmp/$ARCHIVE . diff --git a/bin/test_itemized b/bin/test_itemized index 1a7a6fb550..4a374e63bb 100755 --- a/bin/test_itemized +++ b/bin/test_itemized @@ -104,7 +104,11 @@ while read -u 7 action test_name pubber_opts; do echo bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name + echo Cleaning up pubber processes + ps ax | fgrep pubber | fgrep java || true kill $(ps ax | fgrep pubber | fgrep java | awk '{print $1}') || true + sleep 2 + ps ax | fgrep pubber | fgrep java || fail Unexpected lingering pubber processes egrep "(RESULT|CPBLTY) [a-z]+ [a-z.]+ $test_name[^ ]* " $SEQUENCER_OUT | while read result; do echo $test_marker ${result/* NOTICE /} >> $RESULTS_OUT From 22b82495f46c2fb3889b93b8f13d8b3e867d5288 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 09:49:05 -0500 Subject: [PATCH 54/55] Add better completion check --- bin/test_itemized | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/test_itemized b/bin/test_itemized index 4a374e63bb..d9196777d1 100755 --- a/bin/test_itemized +++ b/bin/test_itemized @@ -104,11 +104,11 @@ while read -u 7 action test_name pubber_opts; do echo bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name - echo Cleaning up pubber processes - ps ax | fgrep pubber | fgrep java || true + echo Cleaning up pubber processes... + (ps ax | fgrep pubber | fgrep java) || true kill $(ps ax | fgrep pubber | fgrep java | awk '{print $1}') || true sleep 2 - ps ax | fgrep pubber | fgrep java || fail Unexpected lingering pubber processes + ! (ps ax | fgrep pubber | fgrep java) || fail Unexpected lingering pubber processes egrep "(RESULT|CPBLTY) [a-z]+ [a-z.]+ $test_name[^ ]* " $SEQUENCER_OUT | while read result; do echo $test_marker ${result/* NOTICE /} >> $RESULTS_OUT From 0d6f9a39b7e3557fdf074734035b1f230e22d650 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Dec 2023 12:13:19 -0500 Subject: [PATCH 55/55] Out tweaks --- etc/test_itemized.out | 2 +- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/etc/test_itemized.out b/etc/test_itemized.out index 5335380305..6c89aa3548 100644 --- a/etc/test_itemized.out +++ b/etc/test_itemized.out @@ -24,7 +24,7 @@ 19 RESULT pass system system_last_update STABLE 5/5 Sequence complete 20 RESULT fail system state_make_model BETA 0/5 Timeout waiting for no applicable system status 24 RESULT skip system valid_serial_no ALPHA 0/0 State testing disabled -25 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for received at least one state update +25 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches first config timestamp 26 RESULT fail system system_min_loglevel ALPHA 0/5 Received state update with no-state device 27 RESULT pass system system_min_loglevel ALPHA 5/5 Sequence complete 31 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index e52786dad3..d103453597 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -1283,7 +1283,6 @@ protected void checkNotThat(String description, Supplier condition) { private void waitFor(String description, Supplier evaluator) { waitFor(description, DEFAULT_WAIT_TIME, evaluator); - } private void waitFor(String description, Duration maxWait, Supplier evaluator) { @@ -1390,18 +1389,18 @@ private void whileDoing(String description, Runnable action, Consumer private void waitingConditionPop(Instant startTime) { Duration between = Duration.between(startTime, Instant.now()); - debug(format("stage finished %s at %s after %ss", currentWaitingCondition(), + debug(format("Stage finished %s at %s after %ss", currentWaitingCondition(), timeSinceStart(), between.toSeconds())); waitingCondition.pop(); ifTrueThen(!waitingCondition.isEmpty(), - () -> trace(format("stage resume %s at %s", currentWaitingCondition(), timeSinceStart()))); + () -> trace(format("Stage resume %s at %s", currentWaitingCondition(), timeSinceStart()))); } private void waitingConditionPush(String condition) { ifTrueThen(!waitingCondition.isEmpty(), () -> trace(format("stage suspend %s at %s", currentWaitingCondition(), timeSinceStart()))); waitingCondition.push("waiting for " + condition); - info(format("stage start %s at %s", currentWaitingCondition(), timeSinceStart())); + info(format("Stage start %s at %s", currentWaitingCondition(), timeSinceStart())); } private String currentWaitingCondition() {