From 36e12b62a8d462866d30b40adeed769a73c4c8d5 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 6 Oct 2022 11:07:52 -0700 Subject: [PATCH 01/40] Adjust crontab --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 08b8b2867b..c91ceeb0e8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -4,7 +4,7 @@ on: push: pull_request: schedule: - - cron: '0 13 * * *' + - cron: '37 * * * *' jobs: unit: From 1c1279650a80a3de16f5988c30a66ebe8850612e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 11:06:57 -0700 Subject: [PATCH 02/40] Fix some logging things --- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 09993d56a7..5400c04894 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -81,7 +81,6 @@ public abstract class SequenceBase { public static final String SYSTEM_EVENT_MESSAGE_BASE = "event_system"; public static final int CONFIG_UPDATE_DELAY_MS = 2000; public static final int NORM_TIMEOUT_MS = 120 * 1000; - public static final String LOCAL_CONFIG = "local_config"; private static final String EMPTY_MESSAGE = "{}"; private static final String CLOUD_IOT_CONFIG_FILE = "cloud_iot_config.json"; private static final String RESULT_LOG_FILE = "RESULT.log"; @@ -103,6 +102,7 @@ public abstract class SequenceBase { private static final Map UPDATE_COUNTS = new HashMap<>(); private static final long LOG_CLEAR_TIME_MS = 1000; private static final String LOCAL_PREFIX = "local_"; + private static final String LOCAL_CONFIG_UPDATE = LOCAL_PREFIX + "update"; private static final String SEQUENCER_LOG = "sequencer.log"; private static final String SYSTEM_LOG = "system.log"; private static final String SEQUENCE_MD = "sequence.md"; @@ -502,7 +502,7 @@ private void recordRawMessage(Map message, String messageBase) { try { JsonUtil.OBJECT_MAPPER.writeValue(messageFile, message); boolean traceMessage = - traceLogLevel() || (debugLogLevel() && messageBase.equals(LOCAL_CONFIG)); + traceLogLevel() || (debugLogLevel() && messageBase.equals(LOCAL_CONFIG_UPDATE)); String postfix = traceMessage ? (message == null ? ": (null)" : ":\n" + stringify(message)) : ""; if (messageBase.equals(SYSTEM_EVENT_MESSAGE_BASE)) { @@ -650,7 +650,7 @@ private void localConfigChange(String reason) { String suffix = reason == null ? "" : (" " + reason); String header = String.format("Update config%s:", suffix); debug(header + " " + getTimestamp(deviceConfig.timestamp)); - recordRawMessage(deviceConfig, LOCAL_PREFIX + "config"); + recordRawMessage(deviceConfig, LOCAL_CONFIG_UPDATE); List configUpdates = configDiffEngine.computeChanges(deviceConfig); if (configUpdates.isEmpty()) { return; @@ -970,7 +970,7 @@ protected boolean configUpdateComplete() { List differences = configDiffEngine.diff(deviceConfig, sanitizeConfig((Config) receivedConfig)); if (traceLogLevel() && !differences.isEmpty()) { - trace("+- " + Joiner.on("\n+- ").join(differences)); + trace("\n+- " + Joiner.on("\n+- ").join(differences)); } return differences.isEmpty(); } From 0b10a5d5bd39de9e10b690ceb870932b2694d369 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 12:16:20 -0700 Subject: [PATCH 03/40] Add safe sleep delay --- .../java/com/google/daq/mqtt/sequencer/SequenceBase.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 5400c04894..4a90f398a1 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -3,6 +3,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.daq.mqtt.sequencer.semantic.SemanticValue.actualize; import static com.google.daq.mqtt.util.JsonUtil.getTimestamp; +import static com.google.daq.mqtt.util.JsonUtil.safeSleep; import static com.google.daq.mqtt.util.JsonUtil.stringify; import static java.util.Optional.ofNullable; @@ -107,6 +108,7 @@ public abstract class SequenceBase { private static final String SYSTEM_LOG = "system.log"; private static final String SEQUENCE_MD = "sequence.md"; private static final String CONFIG_NONCE_KEY = "debug_config_nonce"; + private static final long DEBUG_START_DELAY_MS = 10 * 1000; protected static Metadata deviceMetadata; protected static String projectId; protected static String deviceId; @@ -392,6 +394,7 @@ private Config readGeneratedConfig() { */ @Before public void setUp() { + safeSleep(DEBUG_START_DELAY_MS); deviceState = new State(); configAcked = false; receivedState.clear(); @@ -745,7 +748,7 @@ protected void checkThat(String description, Supplier condition) { protected List> clearLogs() { info("clearing system logs..."); - JsonUtil.safeSleep(LOG_CLEAR_TIME_MS); + safeSleep(LOG_CLEAR_TIME_MS); lastLog = null; return receivedEvents.remove(SubFolder.SYSTEM); } From 0d775b9d4ea3670001c5b258c3d7398fd16c69b8 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 16:18:58 -0700 Subject: [PATCH 04/40] Make discovery generation more exact --- .../main/java/com/google/daq/mqtt/sequencer/SequenceBase.java | 4 ++-- .../daq/mqtt/sequencer/sequences/DiscoverySequences.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 4a90f398a1..e835c475fc 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -908,7 +908,7 @@ private synchronized void handleReflectorMessage(String subFolderRaw, } Config config = (Config) converted; updateDeviceConfig(config); - info("Updated config with timestamp " + JsonUtil.getTimestamp(config.timestamp)); + debug("Updated config with timestamp " + JsonUtil.getTimestamp(config.timestamp)); debug(String.format("Updated config #%03d:\n%s", updateCount, stringify(converted))); } else if (converted instanceof AugmentedState) { @@ -917,7 +917,7 @@ private synchronized void handleReflectorMessage(String subFolderRaw, deviceState = (State) converted; updateConfigAcked((AugmentedState) converted); validSerialNo(); - info("Updated state has last_config " + JsonUtil.getTimestamp( + debug("Updated state has last_config " + JsonUtil.getTimestamp( deviceState.system.last_config)); } else { error("Unknown update type " + converted.getClass().getSimpleName()); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index d0d392e62a..972272872d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -52,7 +52,7 @@ public void self_enumeration() { ); untilUntrue("enumeration still not active", () -> deviceState.discovery.enumeration.active); List events = getReceivedEvents(DiscoveryEvent.class); - assertTrue("a few events received", events.size() >= 1 && events.size() <= 2); + assertEquals("a single event received", events.size(), 1); DiscoveryEvent discoveryEvent = events.get(0); info("Received discovery generation " + JsonUtil.getTimestamp(discoveryEvent.generation)); assertEquals("matching event generation", startTime, discoveryEvent.generation); From f90eb6e82d95d2dd40c5c8ab343a5bab09e13111 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 18:17:20 -0700 Subject: [PATCH 05/40] Cleanup discovery --- dashboard/functions/index.js | 30 ++++++++++++------- .../sequences/DiscoverySequences.java | 9 ++++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/dashboard/functions/index.js b/dashboard/functions/index.js index c6765767fa..9c05bef9dc 100644 --- a/dashboard/functions/index.js +++ b/dashboard/functions/index.js @@ -268,7 +268,7 @@ function process_state_update(attributes, msgObject) { const commandFolder = `devices/${deviceId}/${STATE_TYPE}/${UPDATE_FOLDER}`; promises.push(sendCommand(REFLECT_REGISTRY, registryId, commandFolder, msgObject)); - + attributes.subFolder = UPDATE_FOLDER; attributes.subType = STATE_TYPE; promises.push(publishPubsubMessage('udmi_target', attributes, msgObject)); @@ -330,18 +330,28 @@ exports.udmi_config = functions.pubsub.topic('udmi_config').onPublish((event) => return Promise.all(promises); }); -function parse_old_config(oldConfig, resetConfig) { - if (!oldConfig || resetConfig) { - console.warn('Resetting config bock, explicit=' + resetConfig); - return {}; - } - +function parse_old_config(configStr, resetConfig) { + let config = {}; try { - return JSON.parse(oldConfig); + config = JSON.parse(configStr || "{}"); } catch(e) { - console.warn('Previous config parse error, ignoring update'); - return null; + if (!resetConfig) { + console.warn('Previous config parse error without reset, ignoring update'); + return null; + } + config = {}; + } + + if (resetConfig) { + const configLastStart = config.system && config.system.last_start; + console.warn('Resetting config bock', configLastStart); + config = { + system: { + last_start: configLastStart + } + } } + return config; } function update_last_start(config, stateStart) { diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index 972272872d..d32f23a017 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -51,9 +51,12 @@ public void self_enumeration() { () -> deviceState.discovery.enumeration.generation.equals(startTime) ); untilUntrue("enumeration still not active", () -> deviceState.discovery.enumeration.active); - List events = getReceivedEvents(DiscoveryEvent.class); - assertEquals("a single event received", events.size(), 1); - DiscoveryEvent discoveryEvent = events.get(0); + List allEvents = getReceivedEvents(DiscoveryEvent.class); + // Filter for enumeration events, since there will sometimes be lingering scan events. + List discoveryEvents = allEvents.stream().filter(event -> event.uniqs != null) + .collect(Collectors.toList()); + assertEquals("a single discovery event received", discoveryEvents.size(), 1); + DiscoveryEvent discoveryEvent = discoveryEvents.get(0); info("Received discovery generation " + JsonUtil.getTimestamp(discoveryEvent.generation)); assertEquals("matching event generation", startTime, discoveryEvent.generation); int discoveredPoints = discoveryEvent.uniqs == null ? 0 : discoveryEvent.uniqs.size(); From 2843c768fd850374f9daa45d02b1441d38da2cc6 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 18:40:25 -0700 Subject: [PATCH 06/40] Add discovery event logging --- .../google/daq/mqtt/sequencer/sequences/DiscoverySequences.java | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index d32f23a017..8d390f6c53 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -55,6 +55,7 @@ public void self_enumeration() { // Filter for enumeration events, since there will sometimes be lingering scan events. List discoveryEvents = allEvents.stream().filter(event -> event.uniqs != null) .collect(Collectors.toList()); + System.err.println("TAP\n" + JsonUtil.stringify(discoveryEvents)); assertEquals("a single discovery event received", discoveryEvents.size(), 1); DiscoveryEvent discoveryEvent = discoveryEvents.get(0); info("Received discovery generation " + JsonUtil.getTimestamp(discoveryEvent.generation)); From 77c0bc5f084978e5ed1d353645abf12fc06c8fbd Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 19:09:22 -0700 Subject: [PATCH 07/40] Proper filter event --- .../google/daq/mqtt/sequencer/sequences/DiscoverySequences.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index 8d390f6c53..b117ed8ea9 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -53,7 +53,7 @@ public void self_enumeration() { untilUntrue("enumeration still not active", () -> deviceState.discovery.enumeration.active); List allEvents = getReceivedEvents(DiscoveryEvent.class); // Filter for enumeration events, since there will sometimes be lingering scan events. - List discoveryEvents = allEvents.stream().filter(event -> event.uniqs != null) + List discoveryEvents = allEvents.stream().filter(event -> event.scan_id == null) .collect(Collectors.toList()); System.err.println("TAP\n" + JsonUtil.stringify(discoveryEvents)); assertEquals("a single discovery event received", discoveryEvents.size(), 1); From 9ab5eedb6356b0ab2d1e1a939b53c4d6bb6fde7d Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 19:10:04 -0700 Subject: [PATCH 08/40] Variable rename --- .../sequencer/sequences/DiscoverySequences.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index b117ed8ea9..a07ad04e46 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -53,14 +53,14 @@ public void self_enumeration() { untilUntrue("enumeration still not active", () -> deviceState.discovery.enumeration.active); List allEvents = getReceivedEvents(DiscoveryEvent.class); // Filter for enumeration events, since there will sometimes be lingering scan events. - List discoveryEvents = allEvents.stream().filter(event -> event.scan_id == null) + List enumEvents = allEvents.stream().filter(event -> event.scan_id == null) .collect(Collectors.toList()); - System.err.println("TAP\n" + JsonUtil.stringify(discoveryEvents)); - assertEquals("a single discovery event received", discoveryEvents.size(), 1); - DiscoveryEvent discoveryEvent = discoveryEvents.get(0); - info("Received discovery generation " + JsonUtil.getTimestamp(discoveryEvent.generation)); - assertEquals("matching event generation", startTime, discoveryEvent.generation); - int discoveredPoints = discoveryEvent.uniqs == null ? 0 : discoveryEvent.uniqs.size(); + System.err.println("TAP\n" + JsonUtil.stringify(enumEvents)); + assertEquals("a single discovery event received", enumEvents.size(), 1); + DiscoveryEvent event = enumEvents.get(0); + info("Received discovery generation " + JsonUtil.getTimestamp(event.generation)); + assertEquals("matching event generation", startTime, event.generation); + int discoveredPoints = event.uniqs == null ? 0 : event.uniqs.size(); assertEquals("discovered points count", deviceMetadata.pointset.points.size(), discoveredPoints); } From e9a400e506bab897a95c9c8bc34428099fbad489 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 19:37:38 -0700 Subject: [PATCH 09/40] Remove debugging --- .../google/daq/mqtt/sequencer/sequences/DiscoverySequences.java | 1 - 1 file changed, 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java index a07ad04e46..c18577cbf7 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/DiscoverySequences.java @@ -55,7 +55,6 @@ public void self_enumeration() { // Filter for enumeration events, since there will sometimes be lingering scan events. List enumEvents = allEvents.stream().filter(event -> event.scan_id == null) .collect(Collectors.toList()); - System.err.println("TAP\n" + JsonUtil.stringify(enumEvents)); assertEquals("a single discovery event received", enumEvents.size(), 1); DiscoveryEvent event = enumEvents.get(0); info("Received discovery generation " + JsonUtil.getTimestamp(event.generation)); From 087cb20a62157b4de68e36be8ebbb3f50c6c7240 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 21:14:17 -0700 Subject: [PATCH 10/40] SiteModel refactor --- .gencode_hash.txt | 10 +- .../java/com/google/udmi/util/SiteModel.java | 30 +- gencode/docs/cloud_iot_config.html | 201 ------- gencode/docs/configuration_execution.html | 531 ++++++++++++++++++ gencode/docs/readme.md | 2 +- gencode/java/udmi/schema/CloudIotConfig.java | 61 -- .../udmi/schema/ExecutionConfiguration.java | 101 ++++ gencode/python/udmi/schema/__init__.py | 2 +- .../python/udmi/schema/cloud_iot_config.py | 54 -- .../udmi/schema/configuration_execution.py | 94 ++++ schema/cloud_iot_config.json | 25 - schema/configuration_execution.json | 55 ++ 12 files changed, 803 insertions(+), 363 deletions(-) delete mode 100644 gencode/docs/cloud_iot_config.html create mode 100644 gencode/docs/configuration_execution.html delete mode 100644 gencode/java/udmi/schema/CloudIotConfig.java create mode 100644 gencode/java/udmi/schema/ExecutionConfiguration.java delete mode 100644 gencode/python/udmi/schema/cloud_iot_config.py create mode 100644 gencode/python/udmi/schema/configuration_execution.py delete mode 100644 schema/cloud_iot_config.json create mode 100644 schema/configuration_execution.json diff --git a/.gencode_hash.txt b/.gencode_hash.txt index fb7cb6c8fa..c611b7b918 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,9 +1,9 @@ -909b90e19ba41e691ea277eb26ff8a3b7d7bad43ecd7ac082683e97401bde9b5 gencode/docs/cloud_iot_config.html b35646a2f3d7c30fd34e3ea6ab6a1070b39c469fd1de49cab1093ff32faaf06d gencode/docs/command_discovery.html e1f62882576d0f4332360f2aedb79b37a43973fb31d862c829af924f8f310ec1 gencode/docs/command_mapping.html 5513a43ef916d0413de0c95aede03042ce8824e98fbb8f3634c12f5dee53cce8 gencode/docs/config.html 30c7f39e50e290b0a36a3414cf1c90277ebd249f126f85f0a307bc7b74164e08 gencode/docs/config_mapping.html 5598402bde04860394d91ecf43f5c653ff539cce49d011db4b5b17d596b72c2a gencode/docs/configuration_endpoint.html +054711545d47108384ff422a64600177ad59226519575990263b39973b481230 gencode/docs/configuration_execution.html 00e8646fd8117d47ddf32f2488228cd35b66fd711cb21288dc6bb5a5715978cb gencode/docs/configuration_pubber.html 82fbf41e4e88593f22a929fc8ade4d1e8f75a0fc1aa8452ea1d3b19023bee314 gencode/docs/envelope.html f1fde870638541bea6de16acd64365c89a7d56ee3270d6fd1bcdbb5bdb141887 gencode/docs/event.html @@ -15,7 +15,7 @@ a8800ab8384de3b846af0d72cf310acf01644842b557e42eec20b69a0bfa1868 gencode/docs/e 9ae793a641f2e053c5af734bf813cd2637a675330acb8d9e903ef03b4286e007 gencode/docs/metadata.html e101ec26541b7787fa722244fed744713d05649de547160f39744e1e61850ddd gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html -846eaf1735bd2506357173cfc720c8674d95acf2d20e19f37e3d34cad0a345d2 gencode/docs/readme.md +392f272c1c8cd047eb6906f2f11ffacc7083c32f639da2437d6a9ca861062ecc gencode/docs/readme.md 690d56e96c4e180e49e0abcba358ab1842ea06dd3f693918852a9de370c994d5 gencode/docs/reflect_config.html 9332e44c87dae9261b079424e748d5ee1df08a3c6b39987d254ebf78274e2f34 gencode/docs/reflect_state.html 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css @@ -32,7 +32,6 @@ b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc gencode/java/u fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/udmi/schema/BlobsetState.java 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e gencode/java/udmi/schema/BuildingTranslation.java b95ab79a531c917a5004b5e5bc70e010b33d5d6ff53be09e5dbf2d1df479b569 gencode/java/udmi/schema/Category.java -d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b gencode/java/udmi/schema/CloudIotConfig.java a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d gencode/java/udmi/schema/CloudModel.java ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4 gencode/java/udmi/schema/Config.java 10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05 gencode/java/udmi/schema/DeviceMappingConfig.java @@ -48,6 +47,7 @@ b90ab40a281c5e383881629bd8a791ba3d30491e06ebc623a476e2735a3f6c3a gencode/java/u 885859ea9de29217186c34f3a90b112411d1beddaf58a88b85e7934cd3c92b6f gencode/java/udmi/schema/Entry.java 9b92e03774ed0b9159ff3ab462822699ef7f824161b166a881889ea1eb6a62ad gencode/java/udmi/schema/Envelope.java e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e gencode/java/udmi/schema/Event.java +8f841abf51c9dfb14f7dd374edc9252d6f6c71de5628459c502ed154a755dc58 gencode/java/udmi/schema/ExecutionConfiguration.java 2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed gencode/java/udmi/schema/FamilyDiscoveryConfig.java ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454 gencode/java/udmi/schema/FamilyDiscoveryEvent.java 0afc15acd72874e5a0c47f546abc0c4569f5bc37838fdcac77bc7bd55cc53a6d gencode/java/udmi/schema/FamilyDiscoveryState.java @@ -98,12 +98,11 @@ d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645 gencode/java/u ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4 gencode/java/udmi/schema/ValidationEvent.java f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2 gencode/java/udmi/schema/ValidationState.java e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36 gencode/java/udmi/schema/ValidationSummary.java -0601b86702f0e8959a391342307cd782be91e7dcfa0017089230a38e76661dd5 gencode/python/udmi/schema/__init__.py +67256cd379f8b456a2d8c3b5e64c9bde740569338f8e32be626ecb65ad6fd23c gencode/python/udmi/schema/__init__.py 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db gencode/python/udmi/schema/ancillary_properties.py 5ecd6c542f33450cb4ce75d940a6dff4d3bd67d4b9de4aff5ee88abcc301dbff gencode/python/udmi/schema/building_config.py dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6 gencode/python/udmi/schema/building_translation.py c33fab16eb86b93692ddbb243a2d6cfe202ca1a0fd76b351a18727a35940d958 gencode/python/udmi/schema/category.py -a61b1c5732f01b7efda41a773d5786fad755f371193ce4478b458387ca2a8fe8 gencode/python/udmi/schema/cloud_iot_config.py 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb gencode/python/udmi/schema/command_discovery.py 1254c34d973c9099ae99dcea4534e234e9019f49255e2e27d2afa1bc074fd596 gencode/python/udmi/schema/command_mapping.py c5a62f92328e2ede167fc1f53bece6e48696a0ab0e37e3a41f65cd98494ba0d7 gencode/python/udmi/schema/common.py @@ -121,6 +120,7 @@ ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244 gencode/python ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857 gencode/python/udmi/schema/config_system.py 97c2e5fadc6da0d84660f3296de885ab59f4b04154179b6717f77ec366f1544b gencode/python/udmi/schema/config_system_testing.py 4637fc257ea4d6ea5a46d98db69c57e27e4ecbf5d86f77036cd1382f1a40f532 gencode/python/udmi/schema/configuration_endpoint.py +11911531cab76feb2ae23f9a703801a8ca301b0d926f6fce04dfae92b7a22e3d gencode/python/udmi/schema/configuration_execution.py 3814c88403934dbd3fce77d92a8ad45c68dc7e07c319ba9b4e8f1b1ac7518c07 gencode/python/udmi/schema/configuration_pubber.py 998ce105f88686f27b85f3630a396ed04b106f830c133a684ea5c505ca95b1c3 gencode/python/udmi/schema/envelope.py 1eb9019b9d0b4ff7de2df8beb625a4f89292d636ece0c02f160495c537bd6c2c gencode/python/udmi/schema/event.py diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 8f059914a7..728f535196 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -17,10 +17,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import udmi.schema.CloudIotConfig; import udmi.schema.CloudModel.Auth_type; import udmi.schema.EndpointConfiguration; import udmi.schema.Envelope; +import udmi.schema.ExecutionConfiguration; import udmi.schema.GatewayModel; import udmi.schema.Metadata; @@ -39,17 +39,17 @@ public class SiteModel { final String sitePath; private Map allMetadata; private Map allDevices; - private CloudIotConfig cloudIotConfig; + private ExecutionConfiguration executionConfiguration; public SiteModel(String sitePath) { this.sitePath = sitePath; } public static EndpointConfiguration makeEndpointConfig(String projectId, - CloudIotConfig cloudIotConfig, String deviceId) { + ExecutionConfiguration executionConfig, String deviceId) { EndpointConfiguration endpoint = new EndpointConfiguration(); endpoint.client_id = getClientId(projectId, - cloudIotConfig.cloud_region, cloudIotConfig.registry_id, deviceId); + executionConfig.cloud_region, executionConfig.registry_id, deviceId); endpoint.hostname = DEFAULT_ENDPOINT_HOSTNAME; return endpoint; } @@ -59,18 +59,18 @@ public static String getClientId(String projectId, String cloudRegion, String re return String.format(ID_FORMAT, projectId, cloudRegion, registryId, deviceId); } - private static CloudIotConfig makeCloudIotConfig(Envelope attributes) { - CloudIotConfig cloudIotConfig = new CloudIotConfig(); - cloudIotConfig.registry_id = Preconditions.checkNotNull(attributes.deviceRegistryId, + private static ExecutionConfiguration makeExecutionConfiguration(Envelope attributes) { + ExecutionConfiguration executionConfiguration = new ExecutionConfiguration(); + executionConfiguration.registry_id = Preconditions.checkNotNull(attributes.deviceRegistryId, "deviceRegistryId"); - cloudIotConfig.cloud_region = Preconditions.checkNotNull(attributes.deviceRegistryLocation, + executionConfiguration.cloud_region = Preconditions.checkNotNull(attributes.deviceRegistryLocation, "deviceRegistryLocation"); - return cloudIotConfig; + return executionConfiguration; } public static EndpointConfiguration makeEndpointConfig(Envelope attributes) { - CloudIotConfig cloudIotConfig = makeCloudIotConfig(attributes); - return makeEndpointConfig(attributes.projectId, cloudIotConfig, attributes.deviceId); + ExecutionConfiguration executionConfiguration = makeExecutionConfiguration(attributes); + return makeEndpointConfig(attributes.projectId, executionConfiguration, attributes.deviceId); } /** @@ -97,7 +97,7 @@ public static ClientInfo parseClientId(String clientId) { } public EndpointConfiguration makeEndpointConfig(String projectId, String deviceId) { - return makeEndpointConfig(projectId, cloudIotConfig, deviceId); + return makeEndpointConfig(projectId, executionConfiguration, deviceId); } private Set getDeviceIds() { @@ -159,7 +159,7 @@ private void loadSiteConfig() { "sitePath not defined in configuration"); File cloudConfig = new File(new File(sitePath), "cloud_iot_config.json"); try { - cloudIotConfig = OBJECT_MAPPER.readValue(cloudConfig, CloudIotConfig.class); + executionConfiguration = OBJECT_MAPPER.readValue(cloudConfig, ExecutionConfiguration.class); } catch (Exception e) { throw new RuntimeException("While reading config file " + cloudConfig.getAbsolutePath(), e); } @@ -197,7 +197,7 @@ private String getDeviceKeyPrefix(String targetId) { * @return site registry */ public String getRegistryId() { - return cloudIotConfig.registry_id; + return executionConfiguration.registry_id; } /** @@ -206,7 +206,7 @@ public String getRegistryId() { * @return update topic */ public String getUpdateTopic() { - return cloudIotConfig.update_topic; + return executionConfiguration.update_topic; } public Device getDevice(String deviceId) { diff --git a/gencode/docs/cloud_iot_config.html b/gencode/docs/cloud_iot_config.html deleted file mode 100644 index ae0f0be4ff..0000000000 --- a/gencode/docs/cloud_iot_config.html +++ /dev/null @@ -1,201 +0,0 @@ - - - - - - - - - - - - - - - - Cloud Iot Config - -

Cloud Iot Config

- - -
- -
Type: object
-

Parameters for configuring a connection to cloud systems

- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- -
root - - - - registry_id
Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - cloud_region
Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - site_name
Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - update_topic
Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - reflect_region
Type: string
- - - - - - - -
-
-
-
- - - \ No newline at end of file diff --git a/gencode/docs/configuration_execution.html b/gencode/docs/configuration_execution.html new file mode 100644 index 0000000000..3280561035 --- /dev/null +++ b/gencode/docs/configuration_execution.html @@ -0,0 +1,531 @@ + + + + + + + + + + + + + + + + Execution Configuration + +

Execution Configuration

+ + +
+ +
Type: object
+

Parameters for configuring the execution run of a UDMI tool

+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + registry_id
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + cloud_region
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + site_name
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + update_topic
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + reflect_region
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + include
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + device_id
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + project_id
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + key_file
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + serial_no
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + log_level
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + udmi_version
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + alt_project
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + alt_registry
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + block_unknown
Type: boolean
+ + + + + + + +
+
+
+
+ + + \ No newline at end of file diff --git a/gencode/docs/readme.md b/gencode/docs/readme.md index a69eb18abb..6906b4b039 100644 --- a/gencode/docs/readme.md +++ b/gencode/docs/readme.md @@ -14,7 +14,6 @@ * [**state**](state.html) - [State](../../docs/messages/state.md) message, defined by [`state.json`] ## Site Model -* [**cloud_iot_config**](cloud_iot_config.html) - Parameters for configuring a connection to cloud systems * [**metadata**](metadata.html) - [Metadata](../../docs/specs/metadata.md) is a description about the device: a specification about how the device should be configured and expectations about what the device should be doing. Defined by `metadata.json` ## Blobs @@ -24,6 +23,7 @@ * [**command_discovery**](command_discovery.html) - [Discovery command](../../docs/specs/discovery.md) for provisioning * [**command_mapping**](command_mapping.html) - [Mapping command](../../docs/specs/mapping.md) for provisioning * [**config_mapping**](config_mapping.html) - Configuration for [mapping](../../docs/specs/mapping.md) +* [**configuration_execution**](configuration_execution.html) - Parameters for configuring the execution run of a UDMI tool * [**configuration_pubber**](configuration_pubber.html) - Parameters to define a pubber runtime instance * [**event**](event.html) - Container object for all event schemas, not directly used. * [**event_mapping**](event_mapping.html) - [Mapping result](../../docs/specs/mapping.md) with implicit enumeration diff --git a/gencode/java/udmi/schema/CloudIotConfig.java b/gencode/java/udmi/schema/CloudIotConfig.java deleted file mode 100644 index fbafa456ea..0000000000 --- a/gencode/java/udmi/schema/CloudIotConfig.java +++ /dev/null @@ -1,61 +0,0 @@ - -package udmi.schema; - -import javax.annotation.processing.Generated; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - * Cloud Iot Config - *

- * Parameters for configuring a connection to cloud systems - * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ - "registry_id", - "cloud_region", - "site_name", - "update_topic", - "reflect_region" -}) -@Generated("jsonschema2pojo") -public class CloudIotConfig { - - @JsonProperty("registry_id") - public String registry_id; - @JsonProperty("cloud_region") - public String cloud_region; - @JsonProperty("site_name") - public String site_name; - @JsonProperty("update_topic") - public String update_topic; - @JsonProperty("reflect_region") - public String reflect_region; - - @Override - public int hashCode() { - int result = 1; - result = ((result* 31)+((this.site_name == null)? 0 :this.site_name.hashCode())); - result = ((result* 31)+((this.cloud_region == null)? 0 :this.cloud_region.hashCode())); - result = ((result* 31)+((this.update_topic == null)? 0 :this.update_topic.hashCode())); - result = ((result* 31)+((this.registry_id == null)? 0 :this.registry_id.hashCode())); - result = ((result* 31)+((this.reflect_region == null)? 0 :this.reflect_region.hashCode())); - return result; - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if ((other instanceof CloudIotConfig) == false) { - return false; - } - CloudIotConfig rhs = ((CloudIotConfig) other); - return ((((((this.site_name == rhs.site_name)||((this.site_name!= null)&&this.site_name.equals(rhs.site_name)))&&((this.cloud_region == rhs.cloud_region)||((this.cloud_region!= null)&&this.cloud_region.equals(rhs.cloud_region))))&&((this.update_topic == rhs.update_topic)||((this.update_topic!= null)&&this.update_topic.equals(rhs.update_topic))))&&((this.registry_id == rhs.registry_id)||((this.registry_id!= null)&&this.registry_id.equals(rhs.registry_id))))&&((this.reflect_region == rhs.reflect_region)||((this.reflect_region!= null)&&this.reflect_region.equals(rhs.reflect_region)))); - } - -} diff --git a/gencode/java/udmi/schema/ExecutionConfiguration.java b/gencode/java/udmi/schema/ExecutionConfiguration.java new file mode 100644 index 0000000000..b211b481f0 --- /dev/null +++ b/gencode/java/udmi/schema/ExecutionConfiguration.java @@ -0,0 +1,101 @@ + +package udmi.schema; + +import javax.annotation.processing.Generated; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + + +/** + * Execution Configuration + *

+ * Parameters for configuring the execution run of a UDMI tool + * + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "registry_id", + "cloud_region", + "site_name", + "update_topic", + "reflect_region", + "include", + "device_id", + "project_id", + "key_file", + "serial_no", + "log_level", + "udmi_version", + "alt_project", + "alt_registry", + "block_unknown" +}) +@Generated("jsonschema2pojo") +public class ExecutionConfiguration { + + @JsonProperty("registry_id") + public String registry_id; + @JsonProperty("cloud_region") + public String cloud_region; + @JsonProperty("site_name") + public String site_name; + @JsonProperty("update_topic") + public String update_topic; + @JsonProperty("reflect_region") + public String reflect_region; + @JsonProperty("include") + public String include; + @JsonProperty("device_id") + public String device_id; + @JsonProperty("project_id") + public String project_id; + @JsonProperty("key_file") + public String key_file; + @JsonProperty("serial_no") + public String serial_no; + @JsonProperty("log_level") + public String log_level; + @JsonProperty("udmi_version") + public String udmi_version; + @JsonProperty("alt_project") + public String alt_project; + @JsonProperty("alt_registry") + public String alt_registry; + @JsonProperty("block_unknown") + public Boolean block_unknown; + + @Override + public int hashCode() { + int result = 1; + result = ((result* 31)+((this.alt_registry == null)? 0 :this.alt_registry.hashCode())); + result = ((result* 31)+((this.include == null)? 0 :this.include.hashCode())); + result = ((result* 31)+((this.block_unknown == null)? 0 :this.block_unknown.hashCode())); + result = ((result* 31)+((this.cloud_region == null)? 0 :this.cloud_region.hashCode())); + result = ((result* 31)+((this.device_id == null)? 0 :this.device_id.hashCode())); + result = ((result* 31)+((this.key_file == null)? 0 :this.key_file.hashCode())); + result = ((result* 31)+((this.udmi_version == null)? 0 :this.udmi_version.hashCode())); + result = ((result* 31)+((this.alt_project == null)? 0 :this.alt_project.hashCode())); + result = ((result* 31)+((this.log_level == null)? 0 :this.log_level.hashCode())); + result = ((result* 31)+((this.registry_id == null)? 0 :this.registry_id.hashCode())); + result = ((result* 31)+((this.site_name == null)? 0 :this.site_name.hashCode())); + result = ((result* 31)+((this.update_topic == null)? 0 :this.update_topic.hashCode())); + result = ((result* 31)+((this.project_id == null)? 0 :this.project_id.hashCode())); + result = ((result* 31)+((this.serial_no == null)? 0 :this.serial_no.hashCode())); + result = ((result* 31)+((this.reflect_region == null)? 0 :this.reflect_region.hashCode())); + return result; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if ((other instanceof ExecutionConfiguration) == false) { + return false; + } + ExecutionConfiguration rhs = ((ExecutionConfiguration) other); + return ((((((((((((((((this.alt_registry == rhs.alt_registry)||((this.alt_registry!= null)&&this.alt_registry.equals(rhs.alt_registry)))&&((this.include == rhs.include)||((this.include!= null)&&this.include.equals(rhs.include))))&&((this.block_unknown == rhs.block_unknown)||((this.block_unknown!= null)&&this.block_unknown.equals(rhs.block_unknown))))&&((this.cloud_region == rhs.cloud_region)||((this.cloud_region!= null)&&this.cloud_region.equals(rhs.cloud_region))))&&((this.device_id == rhs.device_id)||((this.device_id!= null)&&this.device_id.equals(rhs.device_id))))&&((this.key_file == rhs.key_file)||((this.key_file!= null)&&this.key_file.equals(rhs.key_file))))&&((this.udmi_version == rhs.udmi_version)||((this.udmi_version!= null)&&this.udmi_version.equals(rhs.udmi_version))))&&((this.alt_project == rhs.alt_project)||((this.alt_project!= null)&&this.alt_project.equals(rhs.alt_project))))&&((this.log_level == rhs.log_level)||((this.log_level!= null)&&this.log_level.equals(rhs.log_level))))&&((this.registry_id == rhs.registry_id)||((this.registry_id!= null)&&this.registry_id.equals(rhs.registry_id))))&&((this.site_name == rhs.site_name)||((this.site_name!= null)&&this.site_name.equals(rhs.site_name))))&&((this.update_topic == rhs.update_topic)||((this.update_topic!= null)&&this.update_topic.equals(rhs.update_topic))))&&((this.project_id == rhs.project_id)||((this.project_id!= null)&&this.project_id.equals(rhs.project_id))))&&((this.serial_no == rhs.serial_no)||((this.serial_no!= null)&&this.serial_no.equals(rhs.serial_no))))&&((this.reflect_region == rhs.reflect_region)||((this.reflect_region!= null)&&this.reflect_region.equals(rhs.reflect_region)))); + } + +} diff --git a/gencode/python/udmi/schema/__init__.py b/gencode/python/udmi/schema/__init__.py index 5e25d1f284..3df63ea23a 100644 --- a/gencode/python/udmi/schema/__init__.py +++ b/gencode/python/udmi/schema/__init__.py @@ -2,7 +2,6 @@ from .building_config import BuildingConfig from .building_translation import BuildingTranslation from .category import ObjectC50A4669 -from .cloud_iot_config import CloudIotConfig from .command_discovery import DiscoveryCommand from .command_mapping import MappingCommand from .common import Common @@ -20,6 +19,7 @@ from .config_system import SystemConfig from .config_system_testing import TestingSystemConfig from .configuration_endpoint import EndpointConfiguration +from .configuration_execution import ExecutionConfiguration from .configuration_pubber import PubberConfiguration from .envelope import Envelope from .event import Event diff --git a/gencode/python/udmi/schema/cloud_iot_config.py b/gencode/python/udmi/schema/cloud_iot_config.py deleted file mode 100644 index 2bf7b4f51c..0000000000 --- a/gencode/python/udmi/schema/cloud_iot_config.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Generated class for cloud_iot_config.json""" - - -class CloudIotConfig: - """Generated schema class""" - - def __init__(self): - self.registry_id = None - self.cloud_region = None - self.site_name = None - self.update_topic = None - self.reflect_region = None - - @staticmethod - def from_dict(source): - if not source: - return None - result = CloudIotConfig() - result.registry_id = source.get('registry_id') - result.cloud_region = source.get('cloud_region') - result.site_name = source.get('site_name') - result.update_topic = source.get('update_topic') - result.reflect_region = source.get('reflect_region') - return result - - @staticmethod - def map_from(source): - if not source: - return None - result = {} - for key in source: - result[key] = CloudIotConfig.from_dict(source[key]) - return result - - @staticmethod - def expand_dict(input): - result = {} - for property in input: - result[property] = input[property].to_dict() if input[property] else {} - return result - - def to_dict(self): - result = {} - if self.registry_id: - result['registry_id'] = self.registry_id # 5 - if self.cloud_region: - result['cloud_region'] = self.cloud_region # 5 - if self.site_name: - result['site_name'] = self.site_name # 5 - if self.update_topic: - result['update_topic'] = self.update_topic # 5 - if self.reflect_region: - result['reflect_region'] = self.reflect_region # 5 - return result diff --git a/gencode/python/udmi/schema/configuration_execution.py b/gencode/python/udmi/schema/configuration_execution.py new file mode 100644 index 0000000000..17cb4c7291 --- /dev/null +++ b/gencode/python/udmi/schema/configuration_execution.py @@ -0,0 +1,94 @@ +"""Generated class for configuration_execution.json""" + + +class ExecutionConfiguration: + """Generated schema class""" + + def __init__(self): + self.registry_id = None + self.cloud_region = None + self.site_name = None + self.update_topic = None + self.reflect_region = None + self.include = None + self.device_id = None + self.project_id = None + self.key_file = None + self.serial_no = None + self.log_level = None + self.udmi_version = None + self.alt_project = None + self.alt_registry = None + self.block_unknown = None + + @staticmethod + def from_dict(source): + if not source: + return None + result = ExecutionConfiguration() + result.registry_id = source.get('registry_id') + result.cloud_region = source.get('cloud_region') + result.site_name = source.get('site_name') + result.update_topic = source.get('update_topic') + result.reflect_region = source.get('reflect_region') + result.include = source.get('include') + result.device_id = source.get('device_id') + result.project_id = source.get('project_id') + result.key_file = source.get('key_file') + result.serial_no = source.get('serial_no') + result.log_level = source.get('log_level') + result.udmi_version = source.get('udmi_version') + result.alt_project = source.get('alt_project') + result.alt_registry = source.get('alt_registry') + result.block_unknown = source.get('block_unknown') + return result + + @staticmethod + def map_from(source): + if not source: + return None + result = {} + for key in source: + result[key] = ExecutionConfiguration.from_dict(source[key]) + return result + + @staticmethod + def expand_dict(input): + result = {} + for property in input: + result[property] = input[property].to_dict() if input[property] else {} + return result + + def to_dict(self): + result = {} + if self.registry_id: + result['registry_id'] = self.registry_id # 5 + if self.cloud_region: + result['cloud_region'] = self.cloud_region # 5 + if self.site_name: + result['site_name'] = self.site_name # 5 + if self.update_topic: + result['update_topic'] = self.update_topic # 5 + if self.reflect_region: + result['reflect_region'] = self.reflect_region # 5 + if self.include: + result['include'] = self.include # 5 + if self.device_id: + result['device_id'] = self.device_id # 5 + if self.project_id: + result['project_id'] = self.project_id # 5 + if self.key_file: + result['key_file'] = self.key_file # 5 + if self.serial_no: + result['serial_no'] = self.serial_no # 5 + if self.log_level: + result['log_level'] = self.log_level # 5 + if self.udmi_version: + result['udmi_version'] = self.udmi_version # 5 + if self.alt_project: + result['alt_project'] = self.alt_project # 5 + if self.alt_registry: + result['alt_registry'] = self.alt_registry # 5 + if self.block_unknown: + result['block_unknown'] = self.block_unknown # 5 + return result diff --git a/schema/cloud_iot_config.json b/schema/cloud_iot_config.json deleted file mode 100644 index a4028f4daf..0000000000 --- a/schema/cloud_iot_config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "title": "Cloud Iot Config", - "description": "Parameters for configuring a connection to cloud systems", - "$section": "Site Model", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "registry_id": { - "type": "string" - }, - "cloud_region": { - "type": "string" - }, - "site_name": { - "type": "string" - }, - "update_topic": { - "type": "string" - }, - "reflect_region": { - "type": "string" - } - } -} diff --git a/schema/configuration_execution.json b/schema/configuration_execution.json new file mode 100644 index 0000000000..fd345b10ff --- /dev/null +++ b/schema/configuration_execution.json @@ -0,0 +1,55 @@ +{ + "title": "Execution Configuration", + "description": "Parameters for configuring the execution run of a UDMI tool", + "$section": "Tool Configuration", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "registry_id": { + "type": "string" + }, + "cloud_region": { + "type": "string" + }, + "site_name": { + "type": "string" + }, + "update_topic": { + "type": "string" + }, + "reflect_region": { + "type": "string" + }, + "include": { + "type": "string" + }, + "device_id": { + "type": "string" + }, + "project_id": { + "type": "string" + }, + "key_file": { + "type": "string" + }, + "serial_no": { + "type": "string" + }, + "log_level": { + "type": "string" + }, + "udmi_version": { + "type": "string" + }, + "alt_project": { + "type": "string" + }, + "alt_registry": { + "type": "string" + }, + "block_unknown": { + "type": "boolean" + } + } +} From 83528550e8fabbb4a1504102552052fdf34d56e9 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 21:30:54 -0700 Subject: [PATCH 11/40] CloudIoTConfig refactoring --- .../bos/iot/core/proxy/CloudIotManager.java | 10 ++--- .../iot/core/proxy/IotReflectorClient.java | 6 +-- .../bos/iot/core/proxy/ProxyTarget.java | 16 +++---- .../google/daq/mqtt/registrar/Registrar.java | 7 ++- .../daq/mqtt/sequencer/SequenceBase.java | 12 ++--- .../google/daq/mqtt/util/CloudIotConfig.java | 17 ------- .../google/daq/mqtt/util/CloudIotManager.java | 44 ++++++++++--------- .../com/google/daq/mqtt/util/ConfigUtil.java | 6 +-- .../google/daq/mqtt/validator/Reflector.java | 9 ++-- .../google/daq/mqtt/validator/Validator.java | 15 ++++--- 10 files changed, 64 insertions(+), 78 deletions(-) delete mode 100644 validator/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/CloudIotManager.java b/validator/src/main/java/com/google/bos/iot/core/proxy/CloudIotManager.java index 2fc4506d8d..34e7c14233 100644 --- a/validator/src/main/java/com/google/bos/iot/core/proxy/CloudIotManager.java +++ b/validator/src/main/java/com/google/bos/iot/core/proxy/CloudIotManager.java @@ -13,7 +13,6 @@ import com.google.api.services.cloudiot.v1.model.ModifyCloudToDeviceConfigRequest; import com.google.auth.http.HttpCredentialsAdapter; import com.google.auth.oauth2.GoogleCredentials; -import com.google.daq.mqtt.util.CloudIotConfig; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.HashMap; @@ -23,6 +22,7 @@ import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import udmi.schema.ExecutionConfiguration; /** * General functions for managing the connection with IoT Core. @@ -31,13 +31,13 @@ public class CloudIotManager { private static final Logger LOG = LoggerFactory.getLogger(CloudIotManager.class); private static final String APPLICATION_NAME = "iot_core_proxy"; - private final CloudIotConfig iotConfig; + private final ExecutionConfiguration iotConfig; private final CloudIot cloudIot; private final String projectId; private final String projectPath; private final String registryId; - private Map deviceMap = new HashMap<>(); - private CloudIot.Projects.Locations.Registries cloudIotRegistries; + private final Map deviceMap = new HashMap<>(); + private final CloudIot.Projects.Locations.Registries cloudIotRegistries; /** * Create a manager with the given configuration. @@ -45,7 +45,7 @@ public class CloudIotManager { * @param projectId Cloud project ID. * @param iotConfig Configuration parameters. */ - CloudIotManager(String projectId, CloudIotConfig iotConfig) { + CloudIotManager(String projectId, ExecutionConfiguration iotConfig) { this.projectId = projectId; this.iotConfig = iotConfig; projectPath = "projects/" + projectId + "/locations/" + iotConfig.cloud_region; diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java b/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java index 6dc2fb7100..92a7eda67e 100644 --- a/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java +++ b/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.google.api.client.util.Base64; import com.google.common.collect.ImmutableSet; -import com.google.daq.mqtt.util.CloudIotConfig; import com.google.daq.mqtt.util.MessagePublisher; import java.io.File; import java.nio.file.Files; @@ -18,6 +17,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; import java.util.function.BiConsumer; +import udmi.schema.ExecutionConfiguration; /** * Publish messages using the iot core reflector. @@ -47,9 +47,9 @@ public class IotReflectorClient implements MessagePublisher { * * @param projectId target project * @param iotConfig configuration file - * @param keyFile auth key file + * @param keyFile auth key file */ - public IotReflectorClient(String projectId, CloudIotConfig iotConfig, String keyFile) { + public IotReflectorClient(String projectId, ExecutionConfiguration iotConfig, String keyFile) { final byte[] keyBytes; try { keyBytes = getFileBytes(keyFile); diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/ProxyTarget.java b/validator/src/main/java/com/google/bos/iot/core/proxy/ProxyTarget.java index cd3cc67442..8a77a419a5 100644 --- a/validator/src/main/java/com/google/bos/iot/core/proxy/ProxyTarget.java +++ b/validator/src/main/java/com/google/bos/iot/core/proxy/ProxyTarget.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.google.api.services.cloudiot.v1.model.Device; import com.google.cloud.ServiceOptions; -import com.google.daq.mqtt.util.CloudIotConfig; import com.google.daq.mqtt.util.MessagePublisher; import java.time.Duration; import java.time.LocalDateTime; @@ -19,6 +18,7 @@ import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import udmi.schema.ExecutionConfiguration; import udmi.schema.Metadata; class ProxyTarget { @@ -43,11 +43,11 @@ class ProxyTarget { private final String srcRegistryId; private final ProxyConfig proxyConfig; private final Consumer bundleOut; - private CloudIotConfig cloudConfig; - private CloudIotManager cloudIotManager; private final Map initializedTimes = new ConcurrentHashMap<>(); private final Map configTimes = new ConcurrentHashMap<>(); private final Map udmiMetadata = new ConcurrentHashMap<>(); + private ExecutionConfiguration cloudConfig; + private CloudIotManager cloudIotManager; ProxyTarget(Map configMap, String registryId, Consumer bundleOut) { @@ -102,11 +102,11 @@ private String getKeyPrefix(String srcRegistryId) { return String.format("proxy_%s_", srcRegistryId); } - private CloudIotConfig loadCloudConfig(String srcRegistryId) { - CloudIotConfig cloudIotConfig = new CloudIotConfig(); - cloudIotConfig.registry_id = srcRegistryId; - cloudIotConfig.cloud_region = proxyConfig.dstCloudRegion; - return cloudIotConfig; + private ExecutionConfiguration loadCloudConfig(String srcRegistryId) { + ExecutionConfiguration executionConfiguration = new ExecutionConfiguration(); + executionConfiguration.registry_id = srcRegistryId; + executionConfiguration.cloud_region = proxyConfig.dstCloudRegion; + return executionConfiguration; } MessagePublisher getMqttPublisher(String deviceId) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 3cc2a783c9..faf9c24757 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -31,7 +31,6 @@ import java.net.URI; import java.time.Duration; import java.time.Instant; -import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Base64; import java.util.Date; @@ -255,7 +254,7 @@ private void initializeCloudProject() { if (cloudIotManager.getUpdateTopic() != null) { updatePusher = new PubSubPusher(projectId, cloudIotManager.getUpdateTopic()); } - blockUnknown = cloudIotManager.cloudIotConfig.block_unknown; + blockUnknown = cloudIotManager.executionConfiguration.block_unknown; } private String getGenerationString() { @@ -377,8 +376,8 @@ private boolean sendFeedMessage(LocalDevice localDevice) { Map attributes = new HashMap<>(); attributes.put("subFolder", SWARM_SUBFOLDER); attributes.put("deviceId", localDevice.getDeviceId()); - attributes.put("deviceRegistryId", cloudIotManager.cloudIotConfig.registry_id); - attributes.put("deviceRegistryLocation", cloudIotManager.cloudIotConfig.cloud_region); + attributes.put("deviceRegistryId", cloudIotManager.executionConfiguration.registry_id); + attributes.put("deviceRegistryLocation", cloudIotManager.executionConfiguration.cloud_region); SwarmMessage swarmMessage = new SwarmMessage(); swarmMessage.key_base64 = Base64.getEncoder().encodeToString(localDevice.getKeyBytes()); swarmMessage.device_metadata = localDevice.getMetadata(); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index e835c475fc..2f93f2cec6 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -15,7 +15,6 @@ import com.google.common.collect.ImmutableMap; import com.google.daq.mqtt.sequencer.semantic.SemanticDate; import com.google.daq.mqtt.sequencer.semantic.SemanticValue; -import com.google.daq.mqtt.util.CloudIotConfig; import com.google.daq.mqtt.util.Common; import com.google.daq.mqtt.util.ConfigDiffEngine; import com.google.daq.mqtt.util.ConfigUtil; @@ -55,6 +54,7 @@ import udmi.schema.Entry; import udmi.schema.Envelope.SubFolder; import udmi.schema.Envelope.SubType; +import udmi.schema.ExecutionConfiguration; import udmi.schema.Level; import udmi.schema.Metadata; import udmi.schema.PointsetEvent; @@ -278,15 +278,15 @@ private static void setupSequencer() { } File cloudIotConfigFile = new File(new File(siteModel), CLOUD_IOT_CONFIG_FILE); - final CloudIotConfig cloudIotConfig; + final ExecutionConfiguration executionConfiguration; try { - cloudIotConfig = ConfigUtil.readCloudIotConfig(cloudIotConfigFile); + executionConfiguration = ConfigUtil.readExecutionConfiguration(cloudIotConfigFile); } catch (Exception e) { throw new RuntimeException("While loading " + cloudIotConfigFile.getAbsolutePath(), e); } - cloudRegion = cloudIotConfig.cloud_region; - registryId = cloudIotConfig.registry_id; + cloudRegion = executionConfiguration.cloud_region; + registryId = executionConfiguration.registry_id; deviceMetadata = readDeviceMetadata(); @@ -299,7 +299,7 @@ private static void setupSequencer() { System.err.printf("Loading reflector key file from %s%n", new File(key_file).getAbsolutePath()); System.err.printf("Validating against device %s serial %s%n", deviceId, serialNo); - client = new IotReflectorClient(projectId, cloudIotConfig, key_file); + client = new IotReflectorClient(projectId, executionConfiguration, key_file); setReflectorState(); } diff --git a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java deleted file mode 100644 index d5a348b6fe..0000000000 --- a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.google.daq.mqtt.util; - -/** - * Data class encapsulating cloud-based configuration. - */ -@SuppressWarnings("MemberName") -public class CloudIotConfig { - - public String registry_id; - public String cloud_region; - public String reflect_region; - public String site_name; - public String update_topic; - public String alt_project; - public String alt_registry; - public boolean block_unknown = true; -} diff --git a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java index 92405895e1..66d440dff3 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java @@ -1,6 +1,6 @@ package com.google.daq.mqtt.util; -import static com.google.daq.mqtt.util.ConfigUtil.readCloudIotConfig; +import static com.google.daq.mqtt.util.ConfigUtil.readExecutionConfiguration; import com.google.api.services.cloudiot.v1.model.Device; import com.google.api.services.cloudiot.v1.model.DeviceCredential; @@ -10,13 +10,13 @@ import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; -import java.util.AbstractMap.SimpleEntry; import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import udmi.schema.ExecutionConfiguration; /** * Encapsulation of all Cloud IoT interaction functions. @@ -32,7 +32,7 @@ public class CloudIotManager { private static final String KEY_ALGORITHM_KEY = "key_algorithm"; private static final String MOCK_PROJECT = "unit-testing"; - public final CloudIotConfig cloudIotConfig; + public final ExecutionConfiguration executionConfiguration; private final String registryId; private final String projectId; @@ -52,9 +52,9 @@ public CloudIotManager(String projectId, File siteDir) { File cloudConfig = new File(siteDir, CLOUD_IOT_CONFIG_JSON); try { System.err.println("Reading cloud config from " + cloudConfig.getAbsolutePath()); - cloudIotConfig = validate(readCloudIotConfig(cloudConfig), projectId); - registryId = cloudIotConfig.registry_id; - cloudRegion = cloudIotConfig.cloud_region; + executionConfiguration = validate(readExecutionConfiguration(cloudConfig), projectId); + registryId = executionConfiguration.registry_id; + cloudRegion = executionConfiguration.cloud_region; initializeIotProvider(); } catch (Exception e) { throw new RuntimeException( @@ -66,22 +66,24 @@ public CloudIotManager(String projectId, File siteDir) { /** * Validate the given configuration. * - * @param cloudIotConfig configuration to validate - * @param projectId expected project id + * @param executionConfiguration configuration to validate + * @param projectId expected project id * @return validated config (for chaining) */ - public static CloudIotConfig validate(CloudIotConfig cloudIotConfig, String projectId) { - if (projectId.equals(cloudIotConfig.alt_project)) { - System.err.printf("Using alt_registry %s for alt_project %s\n", cloudIotConfig.alt_registry, - cloudIotConfig.alt_project); - cloudIotConfig.alt_project = null; - cloudIotConfig.registry_id = cloudIotConfig.alt_registry; - cloudIotConfig.alt_registry = null; + public static ExecutionConfiguration validate(ExecutionConfiguration executionConfiguration, + String projectId) { + if (projectId.equals(executionConfiguration.alt_project)) { + System.err.printf("Using alt_registry %s for alt_project %s\n", + executionConfiguration.alt_registry, + executionConfiguration.alt_project); + executionConfiguration.alt_project = null; + executionConfiguration.registry_id = executionConfiguration.alt_registry; + executionConfiguration.alt_registry = null; } - Preconditions.checkNotNull(cloudIotConfig.registry_id, "registry_id not defined"); - Preconditions.checkNotNull(cloudIotConfig.cloud_region, "cloud_region not defined"); - Preconditions.checkNotNull(cloudIotConfig.site_name, "site_name not defined"); - return cloudIotConfig; + Preconditions.checkNotNull(executionConfiguration.registry_id, "registry_id not defined"); + Preconditions.checkNotNull(executionConfiguration.cloud_region, "cloud_region not defined"); + Preconditions.checkNotNull(executionConfiguration.site_name, "site_name not defined"); + return executionConfiguration; } /** @@ -233,7 +235,7 @@ public String getProjectId() { * @return Name for the site (building name) */ public String getSiteName() { - return cloudIotConfig.site_name; + return executionConfiguration.site_name; } /** @@ -242,7 +244,7 @@ public String getSiteName() { * @return Topic name to use for sending update messages */ public String getUpdateTopic() { - return cloudIotConfig.update_topic; + return executionConfiguration.update_topic; } /** diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java b/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java index 9705275ef8..1d370f7392 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java @@ -8,10 +8,10 @@ import java.io.File; import java.io.FileInputStream; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.regex.Pattern; +import udmi.schema.ExecutionConfiguration; /** * Collection of utilities for managing configuration. @@ -31,9 +31,9 @@ public abstract class ConfigUtil { * @param configFile file ot parse * @return cloud configuration information */ - public static CloudIotConfig readCloudIotConfig(File configFile) { + public static ExecutionConfiguration readExecutionConfiguration(File configFile) { try { - return OBJECT_MAPPER.readValue(configFile, CloudIotConfig.class); + return OBJECT_MAPPER.readValue(configFile, ExecutionConfiguration.class); } catch (Exception e) { throw new RuntimeException("While reading config file " + configFile.getAbsolutePath(), e); } diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java index ef88026b9d..532428045e 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java @@ -5,7 +5,6 @@ import static com.google.daq.mqtt.util.Common.removeNextArg; import com.google.bos.iot.core.proxy.IotReflectorClient; -import com.google.daq.mqtt.util.CloudIotConfig; import com.google.daq.mqtt.util.CloudIotManager; import com.google.daq.mqtt.util.ConfigUtil; import java.io.File; @@ -13,6 +12,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import udmi.schema.ExecutionConfiguration; /** * General utility for working with UDMI Reflector messages. @@ -22,7 +22,7 @@ public class Reflector { private final List reflectCommands; private String projectId; private String siteDir; - private CloudIotConfig cloudIotConfig; + private ExecutionConfiguration executionConfiguration; private File baseDir; private IotReflectorClient client; private String deviceId; @@ -91,7 +91,7 @@ private void reflect(String topic, String data) { private void initialize() { String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath(); System.err.println("Loading reflector key file from " + keyFile); - client = new IotReflectorClient(projectId, cloudIotConfig, keyFile); + client = new IotReflectorClient(projectId, executionConfiguration, keyFile); } private List parseArgs(List argsList) { @@ -133,7 +133,8 @@ public void setSiteDir(String siteDir) { this.siteDir = siteDir; baseDir = new File(siteDir); File cloudConfig = new File(siteDir, "cloud_iot_config.json"); - cloudIotConfig = CloudIotManager.validate(ConfigUtil.readCloudIotConfig(cloudConfig), + executionConfiguration = CloudIotManager.validate( + ConfigUtil.readExecutionConfiguration(cloudConfig), projectId); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index af45de9590..0cff4864f9 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.daq.mqtt.util.CloudIotConfig; import com.google.daq.mqtt.util.CloudIotManager; import com.google.daq.mqtt.util.ConfigUtil; import com.google.daq.mqtt.util.ExceptionMap; @@ -67,6 +66,7 @@ import udmi.schema.Entry; import udmi.schema.Envelope.SubFolder; import udmi.schema.Envelope.SubType; +import udmi.schema.ExecutionConfiguration; import udmi.schema.Metadata; import udmi.schema.PointsetEvent; import udmi.schema.PointsetState; @@ -125,7 +125,7 @@ public class Validator { private File outBaseDir; private File schemaRoot; private String schemaSpec; - private CloudIotConfig cloudIotConfig; + private ExecutionConfiguration executionConfiguration; private CloudIotManager cloudIotManager; private String siteDir; private MessagePublisher client; @@ -218,7 +218,7 @@ MessageReadingClient getMessageReadingClient() { } private void validateMessageTrace(String messageDir) { - client = new MessageReadingClient(cloudIotConfig.registry_id, messageDir); + client = new MessageReadingClient(executionConfiguration.registry_id, messageDir); dataSinks.add(client); prepForMock(); } @@ -242,7 +242,8 @@ public void setSiteDir(String siteDir) { this.siteDir = siteDir; baseDir = new File(siteDir); File cloudConfig = new File(siteDir, "cloud_iot_config.json"); - cloudIotConfig = CloudIotManager.validate(ConfigUtil.readCloudIotConfig(cloudConfig), + executionConfiguration = CloudIotManager.validate( + ConfigUtil.readExecutionConfiguration(cloudConfig), projectId); initializeExpectedDevices(siteDir); } @@ -331,14 +332,14 @@ private void validatePubSub(String instName) { } private String getRegistryId() { - String registryId = cloudIotConfig.registry_id; + String registryId = executionConfiguration.registry_id; return registryId; } private void validateReflector() { String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath(); System.err.println("Loading reflector key file from " + keyFile); - client = new IotReflectorClient(projectId, cloudIotConfig, keyFile); + client = new IotReflectorClient(projectId, executionConfiguration, keyFile); } void messageLoop() { @@ -565,7 +566,7 @@ private void writeMessageCapture(Map message, Map attributes) { String registryId = attributes.get(DEVICE_REGISTRY_ID_KEY); - if (cloudIotConfig != null && !cloudIotConfig.registry_id.equals(registryId)) { + if (executionConfiguration != null && !executionConfiguration.registry_id.equals(registryId)) { if (ignoredRegistries.add(registryId)) { System.err.println("Ignoring data for not-configured registry " + registryId); } From dac1d65c1ae54afea57988edc6be04fd388c60c4 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 21:59:06 -0700 Subject: [PATCH 12/40] Fix block_unknown --- .../main/java/com/google/daq/mqtt/registrar/Registrar.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index faf9c24757..0666820833 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -73,6 +73,7 @@ public class Registrar { private static final int RUNNER_THREADS = 25; private static final String CONFIG_SUB_TYPE = "config"; private static final String MODEL_SUB_TYPE = "model"; + private static final boolean DEFAULT_BLOCK_UNKNOWN = true; private final Map schemas = new HashMap<>(); private final String generation = getGenerationString(); private CloudIotManager cloudIotManager; @@ -254,7 +255,8 @@ private void initializeCloudProject() { if (cloudIotManager.getUpdateTopic() != null) { updatePusher = new PubSubPusher(projectId, cloudIotManager.getUpdateTopic()); } - blockUnknown = cloudIotManager.executionConfiguration.block_unknown; + blockUnknown = Objects.requireNonNullElse(cloudIotManager.executionConfiguration.block_unknown, + DEFAULT_BLOCK_UNKNOWN); } private String getGenerationString() { From f27574228da4af2e567b782d43412950679930dc Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 22:06:17 -0700 Subject: [PATCH 13/40] ValidatorConfig removal --- .gencode_hash.txt | 6 +++--- gencode/docs/configuration_execution.html | 16 ++++++++-------- .../java/udmi/schema/ExecutionConfiguration.java | 10 +++++----- .../udmi/schema/configuration_execution.py | 8 ++++---- schema/configuration_execution.json | 2 +- .../google/daq/mqtt/sequencer/SequenceBase.java | 7 +++---- .../daq/mqtt/sequencer/SequenceRunner.java | 10 +++++----- .../com/google/daq/mqtt/util/ConfigUtil.java | 4 ++-- .../google/daq/mqtt/util/ValidatorConfig.java | 15 --------------- .../daq/mqtt/sequencer/SequenceRunnerTest.java | 11 +++++------ 10 files changed, 36 insertions(+), 53 deletions(-) delete mode 100644 validator/src/main/java/com/google/daq/mqtt/util/ValidatorConfig.java diff --git a/.gencode_hash.txt b/.gencode_hash.txt index c611b7b918..131fb598d0 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -3,7 +3,7 @@ e1f62882576d0f4332360f2aedb79b37a43973fb31d862c829af924f8f310ec1 gencode/docs/c 5513a43ef916d0413de0c95aede03042ce8824e98fbb8f3634c12f5dee53cce8 gencode/docs/config.html 30c7f39e50e290b0a36a3414cf1c90277ebd249f126f85f0a307bc7b74164e08 gencode/docs/config_mapping.html 5598402bde04860394d91ecf43f5c653ff539cce49d011db4b5b17d596b72c2a gencode/docs/configuration_endpoint.html -054711545d47108384ff422a64600177ad59226519575990263b39973b481230 gencode/docs/configuration_execution.html +b7ca70e3176f997f254c32bd8d53dff1462ac8a71c0e166aaa85193d67bff60b gencode/docs/configuration_execution.html 00e8646fd8117d47ddf32f2488228cd35b66fd711cb21288dc6bb5a5715978cb gencode/docs/configuration_pubber.html 82fbf41e4e88593f22a929fc8ade4d1e8f75a0fc1aa8452ea1d3b19023bee314 gencode/docs/envelope.html f1fde870638541bea6de16acd64365c89a7d56ee3270d6fd1bcdbb5bdb141887 gencode/docs/event.html @@ -47,7 +47,7 @@ b90ab40a281c5e383881629bd8a791ba3d30491e06ebc623a476e2735a3f6c3a gencode/java/u 885859ea9de29217186c34f3a90b112411d1beddaf58a88b85e7934cd3c92b6f gencode/java/udmi/schema/Entry.java 9b92e03774ed0b9159ff3ab462822699ef7f824161b166a881889ea1eb6a62ad gencode/java/udmi/schema/Envelope.java e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e gencode/java/udmi/schema/Event.java -8f841abf51c9dfb14f7dd374edc9252d6f6c71de5628459c502ed154a755dc58 gencode/java/udmi/schema/ExecutionConfiguration.java +69c3c12ce057b6ab8e27ab8cab6fb009bfd1c997652214b49e6f2a26a58f302c gencode/java/udmi/schema/ExecutionConfiguration.java 2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed gencode/java/udmi/schema/FamilyDiscoveryConfig.java ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454 gencode/java/udmi/schema/FamilyDiscoveryEvent.java 0afc15acd72874e5a0c47f546abc0c4569f5bc37838fdcac77bc7bd55cc53a6d gencode/java/udmi/schema/FamilyDiscoveryState.java @@ -120,7 +120,7 @@ ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244 gencode/python ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857 gencode/python/udmi/schema/config_system.py 97c2e5fadc6da0d84660f3296de885ab59f4b04154179b6717f77ec366f1544b gencode/python/udmi/schema/config_system_testing.py 4637fc257ea4d6ea5a46d98db69c57e27e4ecbf5d86f77036cd1382f1a40f532 gencode/python/udmi/schema/configuration_endpoint.py -11911531cab76feb2ae23f9a703801a8ca301b0d926f6fce04dfae92b7a22e3d gencode/python/udmi/schema/configuration_execution.py +e8a1574074554b6144b178d2adedc76a1c7be5ae911b253deff4460d8d82c6ee gencode/python/udmi/schema/configuration_execution.py 3814c88403934dbd3fce77d92a8ad45c68dc7e07c319ba9b4e8f1b1ac7518c07 gencode/python/udmi/schema/configuration_pubber.py 998ce105f88686f27b85f3630a396ed04b106f830c133a684ea5c505ca95b1c3 gencode/python/udmi/schema/envelope.py 1eb9019b9d0b4ff7de2df8beb625a4f89292d636ece0c02f160495c537bd6c2c gencode/python/udmi/schema/event.py diff --git a/gencode/docs/configuration_execution.html b/gencode/docs/configuration_execution.html index 3280561035..bd9c123b6c 100644 --- a/gencode/docs/configuration_execution.html +++ b/gencode/docs/configuration_execution.html @@ -194,18 +194,18 @@

-
+
-
+

- +

-
+
root @@ -215,7 +215,7 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - include

Type: string
+ site_model
Type: string
diff --git a/gencode/java/udmi/schema/ExecutionConfiguration.java b/gencode/java/udmi/schema/ExecutionConfiguration.java index b211b481f0..9f9482799c 100644 --- a/gencode/java/udmi/schema/ExecutionConfiguration.java +++ b/gencode/java/udmi/schema/ExecutionConfiguration.java @@ -20,7 +20,7 @@ "site_name", "update_topic", "reflect_region", - "include", + "site_model", "device_id", "project_id", "key_file", @@ -44,8 +44,8 @@ public class ExecutionConfiguration { public String update_topic; @JsonProperty("reflect_region") public String reflect_region; - @JsonProperty("include") - public String include; + @JsonProperty("site_model") + public String site_model; @JsonProperty("device_id") public String device_id; @JsonProperty("project_id") @@ -69,7 +69,6 @@ public class ExecutionConfiguration { public int hashCode() { int result = 1; result = ((result* 31)+((this.alt_registry == null)? 0 :this.alt_registry.hashCode())); - result = ((result* 31)+((this.include == null)? 0 :this.include.hashCode())); result = ((result* 31)+((this.block_unknown == null)? 0 :this.block_unknown.hashCode())); result = ((result* 31)+((this.cloud_region == null)? 0 :this.cloud_region.hashCode())); result = ((result* 31)+((this.device_id == null)? 0 :this.device_id.hashCode())); @@ -77,6 +76,7 @@ public int hashCode() { result = ((result* 31)+((this.udmi_version == null)? 0 :this.udmi_version.hashCode())); result = ((result* 31)+((this.alt_project == null)? 0 :this.alt_project.hashCode())); result = ((result* 31)+((this.log_level == null)? 0 :this.log_level.hashCode())); + result = ((result* 31)+((this.site_model == null)? 0 :this.site_model.hashCode())); result = ((result* 31)+((this.registry_id == null)? 0 :this.registry_id.hashCode())); result = ((result* 31)+((this.site_name == null)? 0 :this.site_name.hashCode())); result = ((result* 31)+((this.update_topic == null)? 0 :this.update_topic.hashCode())); @@ -95,7 +95,7 @@ public boolean equals(Object other) { return false; } ExecutionConfiguration rhs = ((ExecutionConfiguration) other); - return ((((((((((((((((this.alt_registry == rhs.alt_registry)||((this.alt_registry!= null)&&this.alt_registry.equals(rhs.alt_registry)))&&((this.include == rhs.include)||((this.include!= null)&&this.include.equals(rhs.include))))&&((this.block_unknown == rhs.block_unknown)||((this.block_unknown!= null)&&this.block_unknown.equals(rhs.block_unknown))))&&((this.cloud_region == rhs.cloud_region)||((this.cloud_region!= null)&&this.cloud_region.equals(rhs.cloud_region))))&&((this.device_id == rhs.device_id)||((this.device_id!= null)&&this.device_id.equals(rhs.device_id))))&&((this.key_file == rhs.key_file)||((this.key_file!= null)&&this.key_file.equals(rhs.key_file))))&&((this.udmi_version == rhs.udmi_version)||((this.udmi_version!= null)&&this.udmi_version.equals(rhs.udmi_version))))&&((this.alt_project == rhs.alt_project)||((this.alt_project!= null)&&this.alt_project.equals(rhs.alt_project))))&&((this.log_level == rhs.log_level)||((this.log_level!= null)&&this.log_level.equals(rhs.log_level))))&&((this.registry_id == rhs.registry_id)||((this.registry_id!= null)&&this.registry_id.equals(rhs.registry_id))))&&((this.site_name == rhs.site_name)||((this.site_name!= null)&&this.site_name.equals(rhs.site_name))))&&((this.update_topic == rhs.update_topic)||((this.update_topic!= null)&&this.update_topic.equals(rhs.update_topic))))&&((this.project_id == rhs.project_id)||((this.project_id!= null)&&this.project_id.equals(rhs.project_id))))&&((this.serial_no == rhs.serial_no)||((this.serial_no!= null)&&this.serial_no.equals(rhs.serial_no))))&&((this.reflect_region == rhs.reflect_region)||((this.reflect_region!= null)&&this.reflect_region.equals(rhs.reflect_region)))); + return ((((((((((((((((this.alt_registry == rhs.alt_registry)||((this.alt_registry!= null)&&this.alt_registry.equals(rhs.alt_registry)))&&((this.block_unknown == rhs.block_unknown)||((this.block_unknown!= null)&&this.block_unknown.equals(rhs.block_unknown))))&&((this.cloud_region == rhs.cloud_region)||((this.cloud_region!= null)&&this.cloud_region.equals(rhs.cloud_region))))&&((this.device_id == rhs.device_id)||((this.device_id!= null)&&this.device_id.equals(rhs.device_id))))&&((this.key_file == rhs.key_file)||((this.key_file!= null)&&this.key_file.equals(rhs.key_file))))&&((this.udmi_version == rhs.udmi_version)||((this.udmi_version!= null)&&this.udmi_version.equals(rhs.udmi_version))))&&((this.alt_project == rhs.alt_project)||((this.alt_project!= null)&&this.alt_project.equals(rhs.alt_project))))&&((this.log_level == rhs.log_level)||((this.log_level!= null)&&this.log_level.equals(rhs.log_level))))&&((this.site_model == rhs.site_model)||((this.site_model!= null)&&this.site_model.equals(rhs.site_model))))&&((this.registry_id == rhs.registry_id)||((this.registry_id!= null)&&this.registry_id.equals(rhs.registry_id))))&&((this.site_name == rhs.site_name)||((this.site_name!= null)&&this.site_name.equals(rhs.site_name))))&&((this.update_topic == rhs.update_topic)||((this.update_topic!= null)&&this.update_topic.equals(rhs.update_topic))))&&((this.project_id == rhs.project_id)||((this.project_id!= null)&&this.project_id.equals(rhs.project_id))))&&((this.serial_no == rhs.serial_no)||((this.serial_no!= null)&&this.serial_no.equals(rhs.serial_no))))&&((this.reflect_region == rhs.reflect_region)||((this.reflect_region!= null)&&this.reflect_region.equals(rhs.reflect_region)))); } } diff --git a/gencode/python/udmi/schema/configuration_execution.py b/gencode/python/udmi/schema/configuration_execution.py index 17cb4c7291..b3b97105f7 100644 --- a/gencode/python/udmi/schema/configuration_execution.py +++ b/gencode/python/udmi/schema/configuration_execution.py @@ -10,7 +10,7 @@ def __init__(self): self.site_name = None self.update_topic = None self.reflect_region = None - self.include = None + self.site_model = None self.device_id = None self.project_id = None self.key_file = None @@ -31,7 +31,7 @@ def from_dict(source): result.site_name = source.get('site_name') result.update_topic = source.get('update_topic') result.reflect_region = source.get('reflect_region') - result.include = source.get('include') + result.site_model = source.get('site_model') result.device_id = source.get('device_id') result.project_id = source.get('project_id') result.key_file = source.get('key_file') @@ -71,8 +71,8 @@ def to_dict(self): result['update_topic'] = self.update_topic # 5 if self.reflect_region: result['reflect_region'] = self.reflect_region # 5 - if self.include: - result['include'] = self.include # 5 + if self.site_model: + result['site_model'] = self.site_model # 5 if self.device_id: result['device_id'] = self.device_id # 5 if self.project_id: diff --git a/schema/configuration_execution.json b/schema/configuration_execution.json index fd345b10ff..6c92c768c8 100644 --- a/schema/configuration_execution.json +++ b/schema/configuration_execution.json @@ -21,7 +21,7 @@ "reflect_region": { "type": "string" }, - "include": { + "site_model": { "type": "string" }, "device_id": { diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 2f93f2cec6..cd796eb77a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -19,7 +19,6 @@ import com.google.daq.mqtt.util.ConfigDiffEngine; import com.google.daq.mqtt.util.ConfigUtil; import com.google.daq.mqtt.util.JsonUtil; -import com.google.daq.mqtt.util.ValidatorConfig; import com.google.daq.mqtt.validator.AugmentedState; import com.google.daq.mqtt.validator.AugmentedSystemConfig; import com.google.daq.mqtt.validator.CleanDateFormat; @@ -114,7 +113,7 @@ public abstract class SequenceBase { protected static String deviceId; protected static String cloudRegion; protected static String registryId; - static ValidatorConfig validatorConfig; + static ExecutionConfiguration validatorConfig; private static String udmiVersion; private static String siteModel; private static String serialNo; @@ -244,8 +243,8 @@ private void recordCompletion(String result, Level level, }; private static void ensureValidatorConfig() { - if (SequenceRunner.validationConfig != null) { - validatorConfig = SequenceRunner.validationConfig; + if (SequenceRunner.executionConfiguration != null) { + validatorConfig = SequenceRunner.executionConfiguration; } else { if (CONFIG_PATH == null || CONFIG_PATH.equals("")) { throw new RuntimeException(CONFIG_ENV + " env not defined."); diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceRunner.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceRunner.java index 3194b1de10..e9e08d57f8 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceRunner.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceRunner.java @@ -4,7 +4,6 @@ import com.google.daq.mqtt.WebServerRunner; import com.google.daq.mqtt.sequencer.sequences.ConfigSequences; import com.google.daq.mqtt.util.Common; -import com.google.daq.mqtt.util.ValidatorConfig; import com.google.udmi.util.SiteModel; import java.util.ArrayList; import java.util.Arrays; @@ -17,6 +16,7 @@ import org.junit.runner.Request; import org.junit.runner.Result; import org.junit.runner.notification.Failure; +import udmi.schema.ExecutionConfiguration; import udmi.schema.Level; /** @@ -27,7 +27,7 @@ public class SequenceRunner { private static final String INITIALIZATION_ERROR_PREFIX = "initializationError(org.junit."; private static final int EXIT_STATUS_SUCCESS = 0; private static final int EXIST_STATUS_FAILURE = 1; - static ValidatorConfig validationConfig; + static ExecutionConfiguration executionConfiguration; private final Set sequenceClasses = Common.allClassesInPackage(ConfigSequences.class); private int successes = -1; private List failures; @@ -58,8 +58,8 @@ private static boolean hasActualTestResults(Result result) { || !result.getFailures().get(0).toString().startsWith(INITIALIZATION_ERROR_PREFIX)); } - static SequenceRunner processConfig(ValidatorConfig config) { - validationConfig = config; + static SequenceRunner processConfig(ExecutionConfiguration config) { + executionConfiguration = config; SequenceRunner sequenceRunner = new SequenceRunner(); sequenceRunner.process(List.of()); return sequenceRunner; @@ -77,7 +77,7 @@ public static void handleRequest(Map params) { siteModel.initialize(); String deviceId = params.remove(WebServerRunner.DEVICE_PARAM); - ValidatorConfig config = new ValidatorConfig(); + ExecutionConfiguration config = new ExecutionConfiguration(); config.project_id = projectId; config.site_model = sitePath; config.device_id = deviceId; diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java b/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java index 1d370f7392..5188b4ed13 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/ConfigUtil.java @@ -72,9 +72,9 @@ static AllDeviceExceptions loadExceptions(File siteConfig) { * @param configFile file to read * @return parsed validator config */ - public static ValidatorConfig readValidatorConfig(File configFile) { + public static ExecutionConfiguration readValidatorConfig(File configFile) { try { - return OBJECT_MAPPER.readValue(configFile, ValidatorConfig.class); + return OBJECT_MAPPER.readValue(configFile, ExecutionConfiguration.class); } catch (Exception e) { throw new RuntimeException("While reading config file " + configFile.getAbsolutePath(), e); } diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ValidatorConfig.java b/validator/src/main/java/com/google/daq/mqtt/util/ValidatorConfig.java deleted file mode 100644 index 499c3214bf..0000000000 --- a/validator/src/main/java/com/google/daq/mqtt/util/ValidatorConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.google.daq.mqtt.util; - -/** - * Configuration of a validator instance. - */ -@SuppressWarnings("MemberName") -public class ValidatorConfig { - public String site_model; - public String device_id; - public String project_id; - public String key_file; - public String serial_no; - public String log_level; - public String udmi_version; -} diff --git a/validator/src/test/java/com/google/daq/mqtt/sequencer/SequenceRunnerTest.java b/validator/src/test/java/com/google/daq/mqtt/sequencer/SequenceRunnerTest.java index 741122aec1..86e7d1dc79 100644 --- a/validator/src/test/java/com/google/daq/mqtt/sequencer/SequenceRunnerTest.java +++ b/validator/src/test/java/com/google/daq/mqtt/sequencer/SequenceRunnerTest.java @@ -1,12 +1,11 @@ package com.google.daq.mqtt.sequencer; import static com.google.daq.mqtt.sequencer.SequenceBase.SERIAL_NO_MISSING; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import com.google.daq.mqtt.TestCommon; -import com.google.daq.mqtt.util.ValidatorConfig; import org.junit.Test; +import udmi.schema.ExecutionConfiguration; /** * Unit tests for sequence runner (not to be confused with sequence tests themselves). @@ -15,8 +14,8 @@ public class SequenceRunnerTest { private static final String TEST_DEVICE = "AHU-1"; - private ValidatorConfig getValidatorConfig() { - ValidatorConfig validatorConfig = new ValidatorConfig(); + private ExecutionConfiguration getExecutionConfiguration() { + ExecutionConfiguration validatorConfig = new ExecutionConfiguration(); validatorConfig.project_id = TestCommon.PROJECT_ID; validatorConfig.site_model = TestCommon.SITE_DIR; validatorConfig.log_level = TestCommon.LOG_LEVEL; @@ -27,7 +26,7 @@ private ValidatorConfig getValidatorConfig() { @Test public void processSite() { - ValidatorConfig config = getValidatorConfig(); + ExecutionConfiguration config = getExecutionConfiguration(); SequenceRunner sequenceRunner = SequenceRunner.processConfig(config); // TODO: SequenceRunner is not properly mocked, so everything fails. assertTrue("many failures", sequenceRunner.getFailures().size() > 10); @@ -35,7 +34,7 @@ public void processSite() { @Test public void processDevice() { - ValidatorConfig config = getValidatorConfig(); + ExecutionConfiguration config = getExecutionConfiguration(); config.device_id = TEST_DEVICE; SequenceRunner sequenceRunner = SequenceRunner.processConfig(config); // TODO: SequenceRunner is not properly mocked, so everything fails. From 3936f2089c19c6904a5f2e74a59f351d11c01e2c Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 7 Oct 2022 22:07:51 -0700 Subject: [PATCH 14/40] Fix tests --- .../cloud_iot_config.json | 0 .../cloud_iot_config.out | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{cloud_iot_config.tests => configuration_execution.tests}/cloud_iot_config.json (100%) rename tests/{cloud_iot_config.tests => configuration_execution.tests}/cloud_iot_config.out (100%) diff --git a/tests/cloud_iot_config.tests/cloud_iot_config.json b/tests/configuration_execution.tests/cloud_iot_config.json similarity index 100% rename from tests/cloud_iot_config.tests/cloud_iot_config.json rename to tests/configuration_execution.tests/cloud_iot_config.json diff --git a/tests/cloud_iot_config.tests/cloud_iot_config.out b/tests/configuration_execution.tests/cloud_iot_config.out similarity index 100% rename from tests/cloud_iot_config.tests/cloud_iot_config.out rename to tests/configuration_execution.tests/cloud_iot_config.out From f93e4bad7963d47d3f39ddd07366ee8cea69edb0 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 8 Oct 2022 09:13:05 -0700 Subject: [PATCH 15/40] Initial refactoring --- .../google/daq/mqtt/validator/Validator.java | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 0cff4864f9..329a4ef826 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -6,8 +6,10 @@ import static com.google.daq.mqtt.util.Common.TIMESTAMP_ATTRIBUTE; import static com.google.daq.mqtt.util.Common.removeNextArg; import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION; +import static com.google.daq.mqtt.util.ConfigUtil.readExecutionConfiguration; import static com.google.daq.mqtt.util.JsonUtil.JSON_SUFFIX; import static com.google.daq.mqtt.util.JsonUtil.OBJECT_MAPPER; +import static java.util.Objects.requireNonNull; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; @@ -52,7 +54,6 @@ import java.util.List; import java.util.Map; import java.util.MissingFormatArgumentException; -import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -125,9 +126,8 @@ public class Validator { private File outBaseDir; private File schemaRoot; private String schemaSpec; - private ExecutionConfiguration executionConfiguration; + private ExecutionConfiguration config; private CloudIotManager cloudIotManager; - private String siteDir; private MessagePublisher client; private Map schemaMap; private File traceDir; @@ -171,7 +171,10 @@ public static void main(String[] args) { System.exit(0); } - private void parseArgs(List argList) { + private List parseArgs(List argList) { + if (!argList.isEmpty() && !argList.get(0).startsWith("-")) { + processProfile(new File(argList.remove(0))); + } while (!argList.isEmpty()) { String option = removeNextArg(argList); try { @@ -186,7 +189,7 @@ private void parseArgs(List argList) { setSchemaSpec(removeNextArg(argList)); break; case "-t": - cloudIotManager = new CloudIotManager(projectId, new File(siteDir)); + cloudIotManager = new CloudIotManager(projectId, new File(config.site_model)); validatePubSub(removeNextArg(argList)); break; case "-f": @@ -203,7 +206,7 @@ private void parseArgs(List argList) { break; case "--": // All remaining arguments remain in the return list. - return; + return argList; default: throw new RuntimeException("Unknown cmdline option " + option); } @@ -211,6 +214,13 @@ private void parseArgs(List argList) { throw new RuntimeException("For command line option " + option, e); } } + return argList; + } + + private void processProfile(File profilePath) { + config = ConfigUtil.readExecutionConfiguration(profilePath); + projectId = config.project_id; + setSiteDir(config.site_model); } MessageReadingClient getMessageReadingClient() { @@ -218,7 +228,7 @@ MessageReadingClient getMessageReadingClient() { } private void validateMessageTrace(String messageDir) { - client = new MessageReadingClient(executionConfiguration.registry_id, messageDir); + client = new MessageReadingClient(config.registry_id, messageDir); dataSinks.add(client); prepForMock(); } @@ -236,15 +246,10 @@ Validator prepForMock() { public void setSiteDir(String siteDir) { final File baseDir; if (NO_SITE.equals(siteDir)) { - this.siteDir = null; baseDir = new File("."); } else { - this.siteDir = siteDir; baseDir = new File(siteDir); - File cloudConfig = new File(siteDir, "cloud_iot_config.json"); - executionConfiguration = CloudIotManager.validate( - ConfigUtil.readExecutionConfiguration(cloudConfig), - projectId); + config = CloudIotManager.validate(resolveSiteConfig(config, siteDir), projectId); initializeExpectedDevices(siteDir); } @@ -253,6 +258,14 @@ public void setSiteDir(String siteDir) { dataSinks.add(new FileDataSink(outBaseDir)); } + private ExecutionConfiguration resolveSiteConfig(ExecutionConfiguration config, String siteDir) { + File cloudConfig = new File(siteDir, "cloud_iot_config.json"); + if (config == null) { + return readExecutionConfiguration(cloudConfig); + } + return config; + } + private void setMessageTraceDir(String writeDirArg) { traceDir = new File(writeDirArg); if (traceDir.exists()) { @@ -269,7 +282,7 @@ private void initializeExpectedDevices(String siteDir) { return; } try { - for (String device : Objects.requireNonNull(devicesDir.list())) { + for (String device : requireNonNull(devicesDir.list())) { ReportingDevice reportingDevice = new ReportingDevice(device); try { File deviceDir = new File(devicesDir, device); @@ -332,14 +345,13 @@ private void validatePubSub(String instName) { } private String getRegistryId() { - String registryId = executionConfiguration.registry_id; - return registryId; + return config.registry_id; } private void validateReflector() { - String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath(); + String keyFile = new File(config.site_model, GCP_REFLECT_KEY_PKCS8).getAbsolutePath(); System.err.println("Loading reflector key file from " + keyFile); - client = new IotReflectorClient(projectId, executionConfiguration, keyFile); + client = new IotReflectorClient(projectId, config, keyFile); } void messageLoop() { @@ -566,7 +578,7 @@ private void writeMessageCapture(Map message, Map attributes) { String registryId = attributes.get(DEVICE_REGISTRY_ID_KEY); - if (executionConfiguration != null && !executionConfiguration.registry_id.equals(registryId)) { + if (config != null && !config.registry_id.equals(registryId)) { if (ignoredRegistries.add(registryId)) { System.err.println("Ignoring data for not-configured registry " + registryId); } From 05f712e533626afec33f58ce2fc3db494a76c58b Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 8 Oct 2022 10:07:43 -0700 Subject: [PATCH 16/40] Refactoring tests --- .../com/google/udmi/util/GeneralUtils.java | 18 +++++++ .../daq/mqtt/registrar/LocalDevice.java | 52 +++++++------------ .../google/daq/mqtt/registrar/Registrar.java | 13 +++-- .../com/google/daq/mqtt/util/JsonUtil.java | 20 ++++++- .../google/daq/mqtt/validator/Validator.java | 3 ++ 5 files changed, 64 insertions(+), 42 deletions(-) diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 8599b02f20..72fc76c47f 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -8,6 +8,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class GeneralUtils { @@ -84,4 +85,21 @@ public static T deepCopy(T endpoint1, Class valueType) { throw new RuntimeException("While making deep copy of " + valueType.getName(), e); } } + + @SuppressWarnings("unchecked") + public static void deepMergeDefaults(Map destination, Map source) { + for (String key : source.keySet()) { + Object value2 = source.get(key); + if (destination.containsKey(key)) { + Object value1 = destination.get(key); + // When destination and source both contain key, deep copy maps but not other key/values, + // which would produce config override rather than defaults. + if (value1 instanceof Map && value2 instanceof Map) { + deepMergeDefaults((Map) value1, (Map) value2); + } + } else { + destination.put(key, value2); + } + } + } } diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 736e92340d..9906cfd800 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -1,10 +1,13 @@ package com.google.daq.mqtt.registrar; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static com.google.daq.mqtt.registrar.Registrar.DEVICE_ERRORS_JSON; import static com.google.daq.mqtt.registrar.Registrar.ENVELOPE_JSON; import static com.google.daq.mqtt.registrar.Registrar.GENERATED_CONFIG_JSON; import static com.google.daq.mqtt.registrar.Registrar.METADATA_JSON; import static com.google.daq.mqtt.registrar.Registrar.NORMALIZED_JSON; +import static com.google.daq.mqtt.util.JsonUtil.asMap; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonGenerator; @@ -23,7 +26,6 @@ import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; import com.google.api.services.cloudiot.v1.model.DeviceCredential; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -32,8 +34,10 @@ import com.google.daq.mqtt.util.CloudIotManager; import com.google.daq.mqtt.util.ExceptionMap; import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; +import com.google.daq.mqtt.util.JsonUtil; import com.google.daq.mqtt.util.MessageUpgrader; import com.google.daq.mqtt.util.ValidationException; +import com.google.udmi.util.GeneralUtils; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -264,7 +268,7 @@ public void validateExpected() { ExceptionMap exceptionMap = new ExceptionMap(relativized.toString()); String[] files = deviceDir.list(); - Preconditions.checkNotNull(files, "No files found in " + deviceDir.getAbsolutePath()); + checkNotNull(files, "No files found in " + deviceDir.getAbsolutePath()); Set actualFiles = ImmutableSet.copyOf(files); Set expectedFiles = Sets.union(DEVICE_FILES, keyFiles()); SortedSet missing = new TreeSet<>(Sets.difference(expectedFiles, actualFiles)); @@ -288,22 +292,6 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private void deepMergeDefaults(Map destination, Map source) { - for (String key : source.keySet()) { - Object value2 = source.get(key); - if (destination.containsKey(key)) { - Object value1 = destination.get(key); - // When destination and source both contain key, deep copy maps but not other key/values, - // which would produce config override rather than defaults. - if (value1 instanceof Map && value2 instanceof Map) { - deepMergeDefaults((Map) value1, (Map) value2); - } - } else { - destination.put(key, value2); - } - } - } - private Metadata readMetadataWithValidation(boolean validate) { File metadataFile = new File(deviceDir, METADATA_JSON); final JsonNode instance; @@ -326,14 +314,13 @@ private Metadata readMetadataWithValidation(boolean validate) { } try { - String intermediary = OBJECT_MAPPER.writeValueAsString(instance); + String intermediary = JsonUtil.stringify(instance); if (siteMetadata == null) { - return OBJECT_MAPPER.readValue(intermediary, Metadata.class); + return JsonUtil.convertTo(Metadata.class, intermediary); } else { - final Map metadataBase = OBJECT_MAPPER.readValue(intermediary, - TreeMap.class); - deepMergeDefaults(metadataBase, siteMetadata); - return OBJECT_MAPPER.convertValue(metadataBase, Metadata.class); + final Map metadataBase = asMap(intermediary); + GeneralUtils.deepMergeDefaults(metadataBase, siteMetadata); + return JsonUtil.convertTo(Metadata.class, metadataBase); } } catch (Exception e) { exceptionMap.put(EXCEPTION_READING, e); @@ -492,7 +479,7 @@ boolean isDirectConnect() { } CloudDeviceSettings getSettings() { - return Preconditions.checkNotNull(settings, "Device settings not initialized"); + return checkNotNull(settings, "Device settings not initialized"); } void initializeSettings() { @@ -677,14 +664,12 @@ private void checkConsistency(String expectedSite) { } if (metadata.system.physical_tag != null) { String assetName = metadata.system.physical_tag.asset.name; - Preconditions.checkState( - deviceId.equals(assetName), - String.format( - "system.physical_tag.asset.name %s does not match expected %s", assetName, deviceId)); + checkState(deviceId.equals(assetName), + String.format("system.physical_tag.asset.name %s does not match expected %s", assetName, + deviceId)); String assetSite = metadata.system.physical_tag.asset.site; - Preconditions.checkState( - expectedSite.equals(assetSite), + checkState(expectedSite.equals(assetSite), String.format( "system.physical_tag.asset.site %s does not match expected %s", assetSite, expectedSite)); @@ -692,8 +677,7 @@ private void checkConsistency(String expectedSite) { if (metadata.system.location != null) { String siteName = metadata.system.location.site; - Preconditions.checkState( - expectedSite.equals(siteName), + checkState(expectedSite.equals(siteName), String.format( "system.location.site %s does not match expected %s", siteName, expectedSite)); } @@ -779,7 +763,7 @@ public String getDeviceId() { } public String getDeviceNumId() { - return Preconditions.checkNotNull(deviceNumId, "deviceNumId not set"); + return checkNotNull(deviceNumId, "deviceNumId not set"); } public void setDeviceNumId(String numId) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 0666820833..b23323183e 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -208,7 +208,7 @@ private void writeErrors() throws Exception { device -> { Set> entries = device.getTreeChildren(dem.forDevice(device.getDeviceId())); - entries.stream() + entries .forEach( error -> errorSummary @@ -245,12 +245,11 @@ protected void setSitePath(String sitePath) { private void initializeCloudProject() { cloudIotManager = new CloudIotManager(projectId, siteDir); - System.err.println( - String.format( - "Working with project %s registry %s/%s", - cloudIotManager.getProjectId(), - cloudIotManager.getCloudRegion(), - cloudIotManager.getRegistryId())); + System.err.printf( + "Working with project %s registry %s/%s%n", + cloudIotManager.getProjectId(), + cloudIotManager.getCloudRegion(), + cloudIotManager.getRegistryId()); if (cloudIotManager.getUpdateTopic() != null) { updatePusher = new PubSubPusher(projectId, cloudIotManager.getUpdateTopic()); diff --git a/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java b/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java index 905472eb7a..55e96bd31b 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java @@ -11,6 +11,8 @@ import java.io.File; import java.time.Instant; import java.util.Date; +import java.util.Map; +import java.util.TreeMap; /** * Collection of utilities for working with json things. @@ -107,7 +109,6 @@ public static String stringify(Object target) { * @param clazz class of result * @param file file to load * @param type of result - * * @return loaded object */ public static T loadFile(Class clazz, File file) { @@ -132,7 +133,24 @@ public static void writeFile(Object target, File file) { } } + /** + * Get a date object parsed from a string representation. + * + * @param timestamp string representation + * @return Date object + */ public static Date getDate(String timestamp) { return timestamp == null ? null : Date.from(Instant.parse(timestamp)); } + + /** + * Convert the json string to a map object. + * + * @param input input string + * @return input as map object + */ + @SuppressWarnings("unchecked") + public static Map asMap(String input) { + return convertTo(TreeMap.class, input); + } } diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 329a4ef826..b20b47414a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -1,5 +1,6 @@ package com.google.daq.mqtt.validator; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.daq.mqtt.util.Common.GCP_REFLECT_KEY_PKCS8; import static com.google.daq.mqtt.util.Common.NO_SITE; import static com.google.daq.mqtt.util.Common.STATE_QUERY_TOPIC; @@ -263,6 +264,8 @@ private ExecutionConfiguration resolveSiteConfig(ExecutionConfiguration config, if (config == null) { return readExecutionConfiguration(cloudConfig); } + checkArgument(siteDir.equals(config.site_model), "siteDir mismatch"); + ExecutionConfiguration siteConfig = readExecutionConfiguration(cloudConfig); return config; } From b170ee17841b11f46a6327a64edeaf62a8bd87f0 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 8 Oct 2022 10:45:35 -0700 Subject: [PATCH 17/40] Test common utils --- .../com/google/udmi/util/GeneralUtils.java | 7 ++++++ .../google/udmi/util/GeneralUtilsTest.java | 24 +++++++++++++++++++ validator/build.gradle | 5 ++++ .../com/google/daq/mqtt/util/JsonUtil.java | 13 +++++++++- .../google/daq/mqtt/validator/Validator.java | 3 ++- validator/validator.iml | 5 +++- 6 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 72fc76c47f..573a33d833 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import com.google.daq.mqtt.util.JsonUtil; import java.io.File; import java.lang.reflect.Field; import java.util.ArrayList; @@ -86,6 +87,12 @@ public static T deepCopy(T endpoint1, Class valueType) { } } + @SuppressWarnings("unchecked") + public static T deepMergeDefaults(Object destination, Object source) { + deepMergeDefaults(JsonUtil.asMap(destination), JsonUtil.asMap(source)); + return (T) JsonUtil.convertTo(destination.getClass(), destination); + } + @SuppressWarnings("unchecked") public static void deepMergeDefaults(Map destination, Map source) { for (String key : source.keySet()) { diff --git a/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java b/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java new file mode 100644 index 0000000000..ffd53ea611 --- /dev/null +++ b/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java @@ -0,0 +1,24 @@ +package com.google.udmi.util; + +import org.junit.Test; + +/** + * Tests for general utilities. + */ +public class GeneralUtilsTest { + + @Test + public void deepCopy() { + throw new RuntimeException("Nope"); + } + + @Test + public void deepMergeDefaults() { + throw new RuntimeException("Nope"); + } + + @Test + public void testDeepMergeDefaults() { + throw new RuntimeException("Nope"); + } +} \ No newline at end of file diff --git a/validator/build.gradle b/validator/build.gradle index 8b1ef18dcc..524b683303 100644 --- a/validator/build.gradle +++ b/validator/build.gradle @@ -29,6 +29,11 @@ sourceSets { srcDirs '../common/src/main/java' } } + test { + java { + srcDirs '../common/src/test/java' + } + } } checkstyle { diff --git a/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java b/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java index 55e96bd31b..51d11f06c7 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java @@ -144,7 +144,7 @@ public static Date getDate(String timestamp) { } /** - * Convert the json string to a map object. + * Convert the json string to a generic map object. * * @param input input string * @return input as map object @@ -153,4 +153,15 @@ public static Date getDate(String timestamp) { public static Map asMap(String input) { return convertTo(TreeMap.class, input); } + + /** + * Convert the json object to a generic map object. + * + * @param input input object + * @return input as map object + */ + @SuppressWarnings("unchecked") + public static Map asMap(Object input) { + return convertTo(TreeMap.class, input); + } } diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index b20b47414a..7eb82fec3d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -36,6 +36,7 @@ import com.google.daq.mqtt.util.MessageUpgrader; import com.google.daq.mqtt.util.PubSubClient; import com.google.daq.mqtt.util.ValidationException; +import com.google.udmi.util.GeneralUtils; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -266,7 +267,7 @@ private ExecutionConfiguration resolveSiteConfig(ExecutionConfiguration config, } checkArgument(siteDir.equals(config.site_model), "siteDir mismatch"); ExecutionConfiguration siteConfig = readExecutionConfiguration(cloudConfig); - return config; + return GeneralUtils.deepMergeDefaults(siteConfig, config); } private void setMessageTraceDir(String writeDirArg) { diff --git a/validator/validator.iml b/validator/validator.iml index 2656a703c9..c4a20769fa 100644 --- a/validator/validator.iml +++ b/validator/validator.iml @@ -4,9 +4,12 @@ - + + + + From 0a45c63d47799390f3bc048a66f8a73e93019646 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 8 Oct 2022 10:59:22 -0700 Subject: [PATCH 18/40] Refactoring common libraries --- .../com/google/udmi/util}/CleanDateFormat.java | 2 +- .../java/com/google/udmi/util/GeneralUtils.java | 10 +++++----- .../java/com/google/udmi}/util/JsonUtil.java | 3 +-- .../com/google/udmi/util/GeneralUtilsTest.java | 16 +++++++++++++++- pubber/src/main/java/daq/pubber/Pubber.java | 3 +-- validator/.idea/runConfigurations/Registrar.xml | 2 +- .../google/daq/mqtt/mapping/MappingAgent.java | 5 ++--- .../google/daq/mqtt/mapping/MappingEngine.java | 2 +- .../com/google/daq/mqtt/mapping/MappingSink.java | 4 ++-- .../google/daq/mqtt/registrar/LocalDevice.java | 4 ++-- .../com/google/daq/mqtt/registrar/Registrar.java | 2 +- .../google/daq/mqtt/sequencer/SequenceBase.java | 10 +++++----- .../sequencer/sequences/ConfigSequences.java | 4 ++-- .../sequencer/sequences/DiscoverySequences.java | 4 ++-- .../sequencer/sequences/WritebackSequences.java | 2 +- .../com/google/daq/mqtt/util/PubSubClient.java | 1 + .../com/google/daq/mqtt/validator/Validator.java | 6 +++--- .../google/daq/mqtt/validator/PlaybackTest.java | 2 +- 18 files changed, 47 insertions(+), 35 deletions(-) rename {validator/src/main/java/com/google/daq/mqtt/validator => common/src/main/java/com/google/udmi/util}/CleanDateFormat.java (97%) rename {validator/src/main/java/com/google/daq/mqtt => common/src/main/java/com/google/udmi}/util/JsonUtil.java (98%) diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/CleanDateFormat.java b/common/src/main/java/com/google/udmi/util/CleanDateFormat.java similarity index 97% rename from validator/src/main/java/com/google/daq/mqtt/validator/CleanDateFormat.java rename to common/src/main/java/com/google/udmi/util/CleanDateFormat.java index f9b97f94c5..2b7b4ddd44 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/CleanDateFormat.java +++ b/common/src/main/java/com/google/udmi/util/CleanDateFormat.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.validator; +package com.google.udmi.util; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import java.text.ParseException; diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 573a33d833..2ef738c49f 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; -import com.google.daq.mqtt.util.JsonUtil; import java.io.File; import java.lang.reflect.Field; import java.util.ArrayList; @@ -78,12 +77,13 @@ public static void toJsonFile(File file, Object target) { } } - public static T deepCopy(T endpoint1, Class valueType) { + @SuppressWarnings("unchecked") + public static T deepCopy(T object) { + Class targetClass = object.getClass(); try { - return OBJECT_MAPPER.readValue(toJsonString(endpoint1), - valueType); + return (T) OBJECT_MAPPER.readValue(toJsonString(object), targetClass); } catch (Exception e) { - throw new RuntimeException("While making deep copy of " + valueType.getName(), e); + throw new RuntimeException("While making deep copy of " + targetClass.getName(), e); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java b/common/src/main/java/com/google/udmi/util/JsonUtil.java similarity index 98% rename from validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java rename to common/src/main/java/com/google/udmi/util/JsonUtil.java index 51d11f06c7..005375e3a1 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/JsonUtil.java +++ b/common/src/main/java/com/google/udmi/util/JsonUtil.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import static com.google.common.base.Preconditions.checkNotNull; @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.daq.mqtt.validator.CleanDateFormat; import java.io.File; import java.time.Instant; import java.util.Date; diff --git a/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java b/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java index ffd53ea611..1f910ea6e1 100644 --- a/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java +++ b/common/src/test/java/com/google/udmi/util/GeneralUtilsTest.java @@ -1,5 +1,12 @@ package com.google.udmi.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; import org.junit.Test; /** @@ -8,8 +15,15 @@ public class GeneralUtilsTest { @Test + @SuppressWarnings("unchecked") public void deepCopy() { - throw new RuntimeException("Nope"); + Map original = new HashMap<>(); + original.put("A", "B"); + original.put("C", ImmutableMap.of("D", "E")); + Map copy = GeneralUtils.deepCopy(original); + assertEquals("copy is equal", original, copy); + ((Map) copy.get("C")).put("F", "G"); + assertNotEquals("copy is not equal", original, copy); } @Test diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index c42458f05d..4178c370c0 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -937,8 +937,7 @@ private void maybeRedirectEndpoint() { private String redirectedEndpoint(String redirectRegistry) { try { - EndpointConfiguration endpoint = deepCopy(configuration.endpoint, - EndpointConfiguration.class); + EndpointConfiguration endpoint = deepCopy(configuration.endpoint); endpoint.client_id = getClientId(redirectRegistry); return toJsonString(endpoint); } catch (Exception e) { diff --git a/validator/.idea/runConfigurations/Registrar.xml b/validator/.idea/runConfigurations/Registrar.xml index af305a1e0b..83911ff3a1 100644 --- a/validator/.idea/runConfigurations/Registrar.xml +++ b/validator/.idea/runConfigurations/Registrar.xml @@ -6,7 +6,7 @@