From d7a066495de4d7ef8460b2054c59abc4cebb47eb Mon Sep 17 00:00:00 2001 From: Trevor Date: Wed, 6 Sep 2023 09:00:05 +0100 Subject: [PATCH] Handle legacy state messages (#710) --- .gencode_hash.txt | 6 +- .github/workflows/testing.yml | 3 +- bin/pull_message | 14 +-- bin/run_tests | 2 +- bin/test_regclean | 4 + bin/test_registrar | 3 +- bin/test_sites | 2 +- bin/test_trace | 3 +- .../com/google/udmi/util/GeneralUtils.java | 2 +- .../google/udmi/util}/MessageDowngrader.java | 2 +- .../google/udmi}/util/MessageUpgrader.java | 37 ++++-- .../java/com/google/udmi/util/SiteModel.java | 111 +++++++++--------- docs/specs/metadata.md | 2 +- docs/tools/registrar.md | 8 +- gencode/docs/metadata.html | 34 ------ gencode/java/udmi/schema/Metadata.java | 11 +- gencode/python/udmi/schema/metadata.py | 38 ------ pubber/src/main/java/daq/pubber/Pubber.java | 7 +- schema/metadata.json | 5 - .../expected/{errors.json => errors.map} | 0 .../devices/DWN-2/expected/errors.json | 7 -- .../devices/DWN-2/expected/errors.map | 4 + .../devices/DWN-2/expected/exceptions.txt | 7 -- .../DWN-2/expected/generated_config.json | 27 ----- .../devices/DWN-2/expected/metadata_norm.json | 49 -------- .../expected/{errors.json => errors.map} | 0 .../expected/{errors.json => errors.map} | 0 .../bos/udmi/service/core/StateProcessor.java | 5 +- .../messaging/impl/MessageDispatcherImpl.java | 30 +++-- .../udmi/service/core/ProcessorTestBase.java | 36 +++++- .../udmi/service/core/StateProcessorTest.java | 13 +- .../messaging/impl/MessageTestCore.java | 2 +- udmis/src/test/messages/legacy_state.json | 1 + validator/bin/build | 5 + .../daq/mqtt/registrar/LocalDevice.java | 27 +++-- .../google/daq/mqtt/registrar/Registrar.java | 2 +- .../daq/mqtt/sequencer/SequenceBase.java | 8 +- .../google/daq/mqtt/validator/Validator.java | 14 +-- .../util}/MessageDowngraderTest.java | 3 +- 39 files changed, 229 insertions(+), 305 deletions(-) rename {validator/src/main/java/com/google/daq/mqtt/registrar => common/src/main/java/com/google/udmi/util}/MessageDowngrader.java (98%) rename {validator/src/main/java/com/google/daq/mqtt => common/src/main/java/com/google/udmi}/util/MessageUpgrader.java (88%) rename tests/sites/downgrade/devices/DWN-1/expected/{errors.json => errors.map} (100%) delete mode 100644 tests/sites/downgrade/devices/DWN-2/expected/errors.json create mode 100644 tests/sites/downgrade/devices/DWN-2/expected/errors.map delete mode 100644 tests/sites/downgrade/devices/DWN-2/expected/exceptions.txt delete mode 100644 tests/sites/downgrade/devices/DWN-2/expected/generated_config.json delete mode 100644 tests/sites/downgrade/devices/DWN-2/expected/metadata_norm.json rename tests/sites/missing/devices/AHU-1/expected/{errors.json => errors.map} (100%) rename tests/sites/missing/devices/GAT-123/expected/{errors.json => errors.map} (100%) create mode 100644 udmis/src/test/messages/legacy_state.json rename validator/src/test/java/com/google/{daq/mqtt/registrar => udmi/util}/MessageDowngraderTest.java (96%) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 5c5e06a98f..16f8bfae03 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -12,7 +12,7 @@ d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673 gencode/docs/c 71fe25d3b17d1dc87b52c049a3235c20527f73e12ffa53fa4800a15e53f73732 gencode/docs/event_pointset.html acd50976d1afd771d55177db9a1b9452a0c6a35e3e15b2e444c799991294808d gencode/docs/event_system.html 816481f69d3b1bdeb2224eaad6e3751a991d20eb98294d89f888b1323505209c gencode/docs/event_validation.html -21a66523e339c2d6424761e7dbcc9bf14a1a294a684361ff060a91792e6d4fc0 gencode/docs/metadata.html +ae5762dbe2304e44cd742e7236d2de6987bdc1384b21093490cfcd7e33a5509f gencode/docs/metadata.html 4db86e0b979a1873d73da07d68a432df39282cb8b94ea509514d2b411411d55e gencode/docs/monitoring.html 180b32717db748e164a185b163ef9a97aa83d9d6add306283d5b9852d04af947 gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html @@ -78,7 +78,7 @@ ccc7c234dd522a91d387d55573677681bbc5a6926f0fbf1101c2e9f607cafd63 gencode/java/u 39f4cf5f89dc52f24ea02246e17668ebc85ed0fab351e69138e477bcc048d79c gencode/java/udmi/schema/MappingEvent.java 052b6b7f9ca8173901d9d2fae314f5858e034f20efa5d184780b800bd870cb2b gencode/java/udmi/schema/MappingEventEntity.java d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886 gencode/java/udmi/schema/MappingState.java -7668ef7a34380a13454de3e027972cab9fdafd858b8af6be90116702701556aa gencode/java/udmi/schema/Metadata.java +476f542353b69649eea8a63fe53a01d83a1d62a33eae140241b107525024dda5 gencode/java/udmi/schema/Metadata.java df64e4ddbf543ac70e7c2af9d3fbc20ffe3dff68c6718aa9ceadab7f64d3d171 gencode/java/udmi/schema/Metrics.java 14722df90406cdcd29c7c946e5fcdf6a3e513eb879a7fbb493801fc0d9093504 gencode/java/udmi/schema/Monitoring.java b3b7ef6cae004d5fdb4052f860df0aa583df87bb798a027fc769aeaf16489789 gencode/java/udmi/schema/MonitoringMetric.java @@ -164,7 +164,7 @@ ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b gencode/python d89500d1e1c88ac754ddc5abd13135dd140d40c60568ca9294f6fe384b0b12ef gencode/python/udmi/schema/event_system.py 4361f48caba6a948a5916f67c8168215721cb3cc2384649f0336804785f01f1d gencode/python/udmi/schema/event_validation.py 633c85b0999d969310918bb03e9a40a08960f420b862340045a0290f8f5c4fe5 gencode/python/udmi/schema/event_validation_device.py -30dd2d2029ba3abd6f958edd105033f7fba73fb302895c429f56ccb35e9d755b gencode/python/udmi/schema/metadata.py +c7304b2070e6d930b4d101263b90768b036f7fb7e352a6e5147b83a5523987f4 gencode/python/udmi/schema/metadata.py 6e763776c8fc5b5a980b490873b1ee7613e15c5b1dee151278f7eb941e897901 gencode/python/udmi/schema/model_cloud.py 805cc8dcb29732d1965bbd533b12d2bc4966d584f05b8a478dd2cac98fd99d52 gencode/python/udmi/schema/model_discovery.py 44d057dffcac3a994e7b894f8e1ebb643ba49d7dbfbb0bfea9cc04f97af0bc47 gencode/python/udmi/schema/model_discovery_family.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ab7b3de27b..597d171fa1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -2,6 +2,7 @@ name: UDMI test suite on: push: + workflow_dispatch: pull_request: schedule: - cron: '0 13 * * *' @@ -40,7 +41,7 @@ jobs: run: bin/test_mosquitto - name: bin/run_tests udmis_tests run: bin/run_tests udmis_tests - - name: all test output + - name: test output if: ${{ always() }} run: more out/test_* | cat - name: Log in to the github registry diff --git a/bin/pull_message b/bin/pull_message index d16c6703dc..8ea5ddebe5 100755 --- a/bin/pull_message +++ b/bin/pull_message @@ -21,20 +21,20 @@ gcloud --format=json --project=$project_id pubsub subscriptions pull $subscripti subType=$(jq -r '.[0].message.attributes.subType' $out_file 2> /dev/null) subFolder=$(jq -r '.[0].message.attributes.subFolder' $out_file 2> /dev/null) deviceId=$(jq -r '.[0].message.attributes.deviceId' $out_file 2> /dev/null) -timestamp=$(jq -r '.[0].message.data' $out_file 2>/dev/null | base64 --decode | jq -r .timestamp 2> /dev/null) || true -data=$(jq -r .[0].message.data /tmp/captured.json | base64 --decode) - -out_folder=$ROOT/out/devices/$deviceId -mkdir -p $out_folder +registryId=$(jq -r '.[0].message.attributes.deviceRegistryId' $out_file 2> /dev/null) +timestamp=$(jq -r '.[0].message.publishTime' $out_file 2> /dev/null) +data=$(jq -r .[0].message.data $out_file | base64 --decode) if [[ $subType == null ]]; then subType=event fi if [[ $subFolder != null ]]; then - echo $subFolder $subType $deviceId $timestamp - out_file=$out_folder/${timestamp}_${subFolder}_${subType}.json + out_file=$ROOT/out/registries/$registryId/devices/$deviceId/${timestamp}_${subFolder}_${subType}.json + echo $out_file + mkdir -p $(dirname $out_file) echo $data | jq . > $out_file else echo No matching messages found. + false fi diff --git a/bin/run_tests b/bin/run_tests index ba10e53004..c7a526063b 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -88,7 +88,7 @@ case "$1" in esac if [[ -s $failures ]]; then - echo ====== Test failures: $(tr '\n' ' ' <$failures) + echo ====== Failures for $1: $(tr '\n' ' ' <$failures) false else echo ====== All tests passed for $1 diff --git a/bin/test_regclean b/bin/test_regclean index 2e3f12935c..1e1089da64 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -101,6 +101,10 @@ fi echo $pubber_config: cat $pubber_config +echo Corrupting site model to check error handling... +mkdir -p $site_path/devices/XXX-1 +echo { > $site_path/devices/XXX-1/metadata.json + echo Clean out the registry to make sure devices get removed... bin/registrar $site_arg $registrar_project -d diff --git a/bin/test_registrar b/bin/test_registrar index 385dd523a6..d6249ce264 100755 --- a/bin/test_registrar +++ b/bin/test_registrar @@ -19,7 +19,7 @@ exit_status=clean echo Found ${clean_devices} clean devices. [ "${clean_devices}" == $EXPECTED_CLEAN ] || exit_status=expected_clean -device_files=$(ls -d ${TEST_SITE}/devices/*) +device_files=$(ls -d ${TEST_SITE}/devices/* | fgrep -v XXX) # Individual tests for generated config values echo @@ -58,6 +58,7 @@ function check_norm { sm_devices=$(check_norm) echo Devices with correct site_metadata: ${sm_devices} + [ "${sm_devices}" == "${device_files}" ] || exit_status=site_metadata mv ${TEST_SITE}/site_metadata.json ${TEST_SITE}/_site_metadata.json diff --git a/bin/test_sites b/bin/test_sites index 6ef63488e6..345641e5f7 100755 --- a/bin/test_sites +++ b/bin/test_sites @@ -29,7 +29,7 @@ shift $((OPTIND-1)) function redact_files { for file in $@; do sed -E -i \ - -e 's-While loading.*udmi/tests/-REDACTED/-' \ + -e 's-oading .*udmi/tests/-REDACTED/-' \ -e 's/at .*main\(.*\.java:[0-9]+\)/REDACTED/' \ -e '/at .*\(.*\.java:[0-9]+\)/d' \ -e '/\.\.\. [0-9]+ more/d' \ diff --git a/bin/test_trace b/bin/test_trace index 1218455e55..a613376555 100755 --- a/bin/test_trace +++ b/bin/test_trace @@ -28,7 +28,8 @@ site_out=$site_model/out # Use a static/constant value to make output stable. export UDMI_TOOLS=test_trace -rm -rf $site_out +# Clean things out, including some potentially corrupt state. +rm -rf $site_out $site_model/devices/XXX-1 validator/bin/validate -- schema trace $trace_in $site_model echo Redacting output files... diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 14524838e5..1ea70ed69e 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -415,7 +415,7 @@ public static void writeString(File metadataFile, String metadataString) { } public static String multiTrim(String message) { - return Arrays.stream(message.split("\n")) + return Arrays.stream(ofNullable(message).orElse("").split("\n")) .map(String::trim).collect(Collectors.joining(" ")); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/MessageDowngrader.java b/common/src/main/java/com/google/udmi/util/MessageDowngrader.java similarity index 98% rename from validator/src/main/java/com/google/daq/mqtt/registrar/MessageDowngrader.java rename to common/src/main/java/com/google/udmi/util/MessageDowngrader.java index 56194ab903..9414be7403 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/MessageDowngrader.java +++ b/common/src/main/java/com/google/udmi/util/MessageDowngrader.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.registrar; +package com.google.udmi.util; import static com.google.udmi.util.Common.VERSION_KEY; diff --git a/validator/src/main/java/com/google/daq/mqtt/util/MessageUpgrader.java b/common/src/main/java/com/google/udmi/util/MessageUpgrader.java similarity index 88% rename from validator/src/main/java/com/google/daq/mqtt/util/MessageUpgrader.java rename to common/src/main/java/com/google/udmi/util/MessageUpgrader.java index e914c901bf..a4f5b3c954 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/MessageUpgrader.java +++ b/common/src/main/java/com/google/udmi/util/MessageUpgrader.java @@ -1,20 +1,17 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import static com.google.udmi.util.Common.VERSION_KEY; -import static com.google.udmi.util.GeneralUtils.CSV_JOINER; +import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_RAW; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import com.google.udmi.util.GeneralUtils; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * Container class for upgrading UDMI messages from older versions. @@ -27,6 +24,7 @@ public class MessageUpgrader { public static final String METADATA_SCHEMA = "metadata"; private static final String TARGET_FORMAT = "%d.%d.%d"; private final ObjectNode message; + private final JsonNode original; private final String schemaName; private final int major; private int patch; @@ -41,6 +39,7 @@ public class MessageUpgrader { public MessageUpgrader(String schemaName, JsonNode message) { this.message = (ObjectNode) message; this.schemaName = schemaName; + this.original = message.deepCopy(); JsonNode version = message.get(VERSION_KEY); String verStr = @@ -57,18 +56,31 @@ public MessageUpgrader(String schemaName, JsonNode message) { } } + public MessageUpgrader(String schemaName, Object originalMessage) { + this(schemaName, OBJECT_MAPPER_RAW.valueToTree(originalMessage)); + } + + public boolean wasUpgraded() { + return !original.equals(message); + } + + /** + * Update message to the latest standard, if necessary. + */ + public Object upgrade() { + return upgrade(false); + } + /** * Update message to the latest standard. * * @param forceUpgrade true to force a complete upgrade pass irrespective of original version - * @return true if the message has been altered */ - public boolean upgrade(boolean forceUpgrade) { + public Object upgrade(boolean forceUpgrade) { if (major != 1) { throw new IllegalArgumentException("Starting major version " + major); } - final JsonNode original = message.deepCopy(); boolean upgraded = false; if (forceUpgrade || minor < 0) { @@ -102,11 +114,10 @@ public boolean upgrade(boolean forceUpgrade) { } if (upgraded && message.has(VERSION_KEY)) { - ((ObjectNode) message).put(VERSION_KEY, - String.format(TARGET_FORMAT, major, minor, patch)); + message.put(VERSION_KEY, String.format(TARGET_FORMAT, major, minor, patch)); } - return !original.equals(message); + return message; } private void upgrade_1_3_14() { @@ -193,8 +204,8 @@ private void upgradeFirmware(ObjectNode system) { private TextNode sanitizeFirmwareVersion(JsonNode version) { if (version.isArray()) { List values = new ArrayList<>(); - Iterator elements = ((ArrayNode) version).elements(); - elements.forEachRemaining(item -> values.add(((TextNode) item).asText())); + Iterator elements = version.elements(); + elements.forEachRemaining(item -> values.add(item.asText())); String collect = values.stream().collect(Collectors.joining(", ")); return new TextNode(collect); } diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 84a1915e9d..bea6765c19 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -2,6 +2,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.udmi.util.GeneralUtils.ifNullThen; +import static com.google.udmi.util.JsonUtil.loadFileStrict; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; @@ -115,6 +116,49 @@ public static List listDevices(File devicesDir) { .collect(Collectors.toList()); } + private static boolean validDeviceDirectory(String dirName) { + return !(dirName.startsWith(".") || dirName.endsWith("~")); + } + + public static Metadata loadDeviceMetadata(String sitePath, String deviceId, Class container) { + Preconditions.checkState(sitePath != null, "sitePath not defined"); + File deviceDir = getDeviceDir(sitePath, deviceId); + File deviceMetadataFile = new File(deviceDir, "metadata.json"); + Metadata metadata = captureLoadErrors(deviceMetadataFile); + if (metadata != null) { + // Missing arrays are automatically parsed to an empty list, which is not what + // we want, so hacky go through and convert an empty list to null. + if (metadata.gateway != null && metadata.gateway.proxy_ids.isEmpty()) { + metadata.gateway.proxy_ids = null; + } + } + return metadata; + } + + private static Metadata captureLoadErrors(File deviceMetadataFile) { + try { + return loadFileStrict(Metadata.class, deviceMetadataFile); + } catch (Exception e) { + return new MetadataException(deviceMetadataFile, e); + } + } + + private static File getDeviceDir(String sitePath, String deviceId) { + File devicesFile = new File(new File(sitePath), "devices"); + return new File(devicesFile, deviceId); + } + + public static String getRegistryActual(ExecutionConfiguration iotConfig) { + return getRegistryActual(iotConfig.registry_id, iotConfig.registry_suffix); + } + + public static String getRegistryActual(String registry_id, String registry_suffix) { + if (registry_id == null) { + return null; + } + return registry_id + Optional.ofNullable(registry_suffix).orElse(""); + } + public EndpointConfiguration makeEndpointConfig(String projectId, String deviceId) { return makeEndpointConfig(projectId, executionConfiguration, deviceId); } @@ -123,11 +167,8 @@ private Set getDeviceIds() { Preconditions.checkState(sitePath != null, "sitePath not defined"); File devicesFile = new File(new File(sitePath), "devices"); File[] files = Objects.requireNonNull(devicesFile.listFiles(), "no files in site devices/"); - return Arrays.stream(files).map(File::getName).filter(SiteModel::validDeviceDirectory).collect(Collectors.toSet()); - } - - private static boolean validDeviceDirectory(String dirName) { - return !(dirName.startsWith(".") || dirName.endsWith("~")); + return Arrays.stream(files).map(File::getName).filter(SiteModel::validDeviceDirectory) + .collect(Collectors.toSet()); } private void loadAllDeviceMetadata() { @@ -137,53 +178,17 @@ private void loadAllDeviceMetadata() { } private CloudModel newDevice(String deviceId) { - CloudModel cloudModel = new CloudModel(); - return cloudModel; + return new CloudModel(); } private Metadata loadDeviceMetadata(String deviceId) { return loadDeviceMetadata(sitePath, deviceId, SiteModel.class); } - public static Metadata loadDeviceMetadata(String sitePath, String deviceId, Class container) { - Preconditions.checkState(sitePath != null, "sitePath not defined"); - File deviceDir = getDeviceDir(sitePath, deviceId); - File deviceMetadataFile = new File(deviceDir, "metadata.json"); - try { - Metadata metadata = JsonUtil.loadFile(Metadata.class, deviceMetadataFile); - if (metadata != null) { - metadata.exception = captureLoadErrors(deviceMetadataFile, container); - // Missing arrays are automatically parsed to an empty list, which is not what - // we want, so hacky go through and convert an empty list to null. - if (metadata.gateway != null && metadata.gateway.proxy_ids.isEmpty()) { - metadata.gateway.proxy_ids = null; - } - } - return metadata; - } catch (Exception e) { - throw new RuntimeException( - "While reading metadata file " + deviceMetadataFile.getAbsolutePath(), e); - } - } - - private static Exception captureLoadErrors(File deviceMetadataFile, Class container) { - try { - JsonUtil.loadFileStrict(Metadata.class, deviceMetadataFile); - return null; - } catch (Exception e) { - return e; - } - } - public File getDeviceDir(String deviceId) { return getDeviceDir(sitePath, deviceId); } - private static File getDeviceDir(String sitePath, String deviceId) { - File devicesFile = new File(new File(sitePath), "devices"); - return new File(devicesFile, deviceId); - } - public Metadata getMetadata(String deviceId) { return allMetadata.get(deviceId); } @@ -241,17 +246,6 @@ private String getDeviceKeyPrefix(String targetId) { return auth_type.value().startsWith("RS") ? "rsa" : "ec"; } - public static String getRegistryActual(ExecutionConfiguration iotConfig) { - return getRegistryActual(iotConfig.registry_id, iotConfig.registry_suffix); - } - - public static String getRegistryActual(String registry_id, String registry_suffix) { - if (registry_id == null) { - return null; - } - return registry_id + Optional.ofNullable(registry_suffix).orElse(""); - } - /** * Get the site registry name. * @@ -317,6 +311,17 @@ public ExecutionConfiguration getExecutionConfiguration() { return executionConfiguration; } + public static class MetadataException extends Metadata { + + public final File file; + public final Exception exception; + + public MetadataException(File deviceMetadataFile, Exception metadataException) { + file = deviceMetadataFile; + exception = metadataException; + } + } + public static class ClientInfo { public String cloudRegion; diff --git a/docs/specs/metadata.md b/docs/specs/metadata.md index fd30e1ebbc..a8f0c6a46b 100644 --- a/docs/specs/metadata.md +++ b/docs/specs/metadata.md @@ -27,7 +27,7 @@ The various tools will generate some output files in the same `devices/` directo * `rsa_*`: Device public/private key files (`genkeys` tool). * `metadata_norm.json`: Normalized version of the `metadata.json` file (`registrar` tool). * `generated_config.json`: Default device IoT Core `config` block (`registrar` tool). -* `errors.json`: Detailed error file (`registrar` tool). +* `errors.map`: Detailed error file (`registrar` tool). ## Metadata Structure diff --git a/docs/tools/registrar.md b/docs/tools/registrar.md index 457bcd4a5d..3940ab360a 100644 --- a/docs/tools/registrar.md +++ b/docs/tools/registrar.md @@ -83,10 +83,10 @@ Loading local device SNS-4 Loading local device AHU-1 Skipping remote registry fetch Processed 4 devices -Removing test_site/devices/GAT-123/errors.json -Removing test_site/devices/AHU-1/errors.json -Removing test_site/devices/AHU-22/errors.json -Removing test_site/devices/SNS-4/errors.json +Removing test_site/devices/GAT-123/errors.map +Removing test_site/devices/AHU-1/errors.map +Removing test_site/devices/AHU-22/errors.map +Removing test_site/devices/SNS-4/errors.map Summary: Device Clean: 4 diff --git a/gencode/docs/metadata.html b/gencode/docs/metadata.html index f580f28aed..69d22cfa1d 100644 --- a/gencode/docs/metadata.html +++ b/gencode/docs/metadata.html @@ -165,40 +165,6 @@

- - - - -
-
-
-

- -

-
- -
-
- -
root - - - - exception
Type: object
-

Exception captured during processing or conversion.

- - - - - - -
diff --git a/gencode/java/udmi/schema/Metadata.java b/gencode/java/udmi/schema/Metadata.java index 7f353a4fc6..b09953cef5 100644 --- a/gencode/java/udmi/schema/Metadata.java +++ b/gencode/java/udmi/schema/Metadata.java @@ -22,7 +22,6 @@ "version", "description", "hash", - "exception", "cloud", "system", "gateway", @@ -65,13 +64,6 @@ public class Metadata { @JsonProperty("hash") @JsonPropertyDescription("Automatically generated field that contains the hash of file contents.") public java.lang.String hash; - /** - * Exception captured during processing or conversion. - * - */ - @JsonProperty("exception") - @JsonPropertyDescription("Exception captured during processing or conversion.") - public Exception exception; /** * Cloud Model *

@@ -149,7 +141,6 @@ public class Metadata { @Override public int hashCode() { int result = 1; - result = ((result* 31)+((this.exception == null)? 0 :this.exception.hashCode())); result = ((result* 31)+((this.testing == null)? 0 :this.testing.hashCode())); result = ((result* 31)+((this.description == null)? 0 :this.description.hashCode())); result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); @@ -174,7 +165,7 @@ public boolean equals(Object other) { return false; } Metadata rhs = ((Metadata) other); - return ((((((((((((((this.exception == rhs.exception)||((this.exception!= null)&&this.exception.equals(rhs.exception)))&&((this.testing == rhs.testing)||((this.testing!= null)&&this.testing.equals(rhs.testing))))&&((this.description == rhs.description)||((this.description!= null)&&this.description.equals(rhs.description))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.cloud == rhs.cloud)||((this.cloud!= null)&&this.cloud.equals(rhs.cloud))))&&((this.features == rhs.features)||((this.features!= null)&&this.features.equals(rhs.features))))&&((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system))))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.hash == rhs.hash)||((this.hash!= null)&&this.hash.equals(rhs.hash))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); + return (((((((((((((this.testing == rhs.testing)||((this.testing!= null)&&this.testing.equals(rhs.testing)))&&((this.description == rhs.description)||((this.description!= null)&&this.description.equals(rhs.description))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.cloud == rhs.cloud)||((this.cloud!= null)&&this.cloud.equals(rhs.cloud))))&&((this.features == rhs.features)||((this.features!= null)&&this.features.equals(rhs.features))))&&((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system))))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.hash == rhs.hash)||((this.hash!= null)&&this.hash.equals(rhs.hash))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); } } diff --git a/gencode/python/udmi/schema/metadata.py b/gencode/python/udmi/schema/metadata.py index cd41bce01c..46f07dc420 100644 --- a/gencode/python/udmi/schema/metadata.py +++ b/gencode/python/udmi/schema/metadata.py @@ -1,38 +1,4 @@ """Generated class for metadata.json""" - - -class Object9DDDF843: - """Generated schema class""" - - def __init__(self): - pass - - @staticmethod - def from_dict(source): - if not source: - return None - result = Object9DDDF843() - return result - - @staticmethod - def map_from(source): - if not source: - return None - result = {} - for key in source: - result[key] = Object9DDDF843.from_dict(source[key]) - return result - - @staticmethod - def expand_dict(input): - result = {} - for property in input: - result[property] = input[property].to_dict() if input[property] else {} - return result - - def to_dict(self): - result = {} - return result from .model_cloud import CloudModel from .model_system import SystemModel from .model_gateway import GatewayModel @@ -51,7 +17,6 @@ def __init__(self): self.version = None self.description = None self.hash = None - self.exception = None self.cloud = None self.system = None self.gateway = None @@ -70,7 +35,6 @@ def from_dict(source): result.version = source.get('version') result.description = source.get('description') result.hash = source.get('hash') - result.exception = Object9DDDF843.from_dict(source.get('exception')) result.cloud = CloudModel.from_dict(source.get('cloud')) result.system = SystemModel.from_dict(source.get('system')) result.gateway = GatewayModel.from_dict(source.get('gateway')) @@ -107,8 +71,6 @@ def to_dict(self): result['description'] = self.description # 5 if self.hash: result['hash'] = self.hash # 5 - if self.exception: - result['exception'] = self.exception.to_dict() # 4 if self.cloud: result['cloud'] = self.cloud.to_dict() # 4 if self.system: diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index afb0d1fb6f..12833284ad 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -37,6 +37,7 @@ import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.SiteModel; +import com.google.udmi.util.SiteModel.MetadataException; import daq.pubber.MqttPublisher.InjectedMessage; import daq.pubber.MqttPublisher.InjectedState; import daq.pubber.MqttPublisher.PublisherException; @@ -587,6 +588,10 @@ private void processSwarmConfig(SwarmMessage swarm, Envelope attributes) { } private void processDeviceMetadata(Metadata metadata) { + if (metadata instanceof MetadataException metadataException) { + throw new RuntimeException("While processing metadata file " + metadataException.file, + metadataException.exception); + } if (metadata.cloud != null) { configuration.algorithm = catchToNull(() -> metadata.cloud.auth_type.value()); } @@ -1776,7 +1781,7 @@ private String getTestingTag(Config config) { } private void localLog(Entry entry) { - String message = format("Entry %s%s %s %s %s%s", Level.fromValue(entry.level).name(), + String message = format("Log %s%s %s %s %s%s", Level.fromValue(entry.level).name(), shouldLogLevel(entry.level) ? "" : "*", entry.category, entry.message, isoConvert(entry.timestamp), getTestingTag(deviceConfig)); localLog(message, Level.fromValue(entry.level), isoConvert(entry.timestamp), null); diff --git a/schema/metadata.json b/schema/metadata.json index 02a91b1799..b7ef18a2e5 100644 --- a/schema/metadata.json +++ b/schema/metadata.json @@ -31,11 +31,6 @@ "type": "string", "pattern": "^[0-9a-z]{8}$" }, - "exception": { - "description": "Exception captured during processing or conversion.", - "existingJavaType": "Exception", - "type": "object" - }, "cloud": { "$ref": "file:model_cloud.json#" }, diff --git a/tests/sites/downgrade/devices/DWN-1/expected/errors.json b/tests/sites/downgrade/devices/DWN-1/expected/errors.map similarity index 100% rename from tests/sites/downgrade/devices/DWN-1/expected/errors.json rename to tests/sites/downgrade/devices/DWN-1/expected/errors.map diff --git a/tests/sites/downgrade/devices/DWN-2/expected/errors.json b/tests/sites/downgrade/devices/DWN-2/expected/errors.json deleted file mode 100644 index f3500f995a..0000000000 --- a/tests/sites/downgrade/devices/DWN-2/expected/errors.json +++ /dev/null @@ -1,7 +0,0 @@ -Exceptions for DWN-2 - While loading credentials for local device DWN-2 - Found 0 credentials - REDACTED/sites/downgrade/./devices/DWN-2/metadata.json - Unrecognized field "grumpy" (class udmi.schema.Metadata), not marked as ignorable (13 known properties: "gateway", "hash", "exception", "discovery", "localnet", "cloud", "timestamp", "version", "description", "features", "system", "testing", "pointset"]) at [Source: (File); line: 27, column: 14] (through reference chain: udmi.schema.Metadata["grumpy"]) - expected files - Missing files: [rsa_private.pem, rsa_private.pkcs8, rsa_public.pem] diff --git a/tests/sites/downgrade/devices/DWN-2/expected/errors.map b/tests/sites/downgrade/devices/DWN-2/expected/errors.map new file mode 100644 index 0000000000..00cac2f01f --- /dev/null +++ b/tests/sites/downgrade/devices/DWN-2/expected/errors.map @@ -0,0 +1,4 @@ +Exceptions for DWN-2 + LREDACTED/sites/downgrade/./devices/DWN-2/metadata.json + While lREDACTED/sites/downgrade/./devices/DWN-2/metadata.json + Unrecognized field "grumpy" (class udmi.schema.Metadata), not marked as ignorable (12 known properties: "discovery", "localnet", "gateway", "cloud", "timestamp", "version", "description", "features", "hash", "system", "testing", "pointset"]) at [Source: (File); line: 27, column: 14] (through reference chain: udmi.schema.Metadata["grumpy"]) diff --git a/tests/sites/downgrade/devices/DWN-2/expected/exceptions.txt b/tests/sites/downgrade/devices/DWN-2/expected/exceptions.txt deleted file mode 100644 index 3f452791b2..0000000000 --- a/tests/sites/downgrade/devices/DWN-2/expected/exceptions.txt +++ /dev/null @@ -1,7 +0,0 @@ -Credential -java.lang.RuntimeException: While loading credentials for local device DWN-2 - REDACTED -Caused by: java.lang.RuntimeException: Found 0 credentials -Files -com.google.daq.mqtt.util.ExceptionMap: expected files - REDACTED diff --git a/tests/sites/downgrade/devices/DWN-2/expected/generated_config.json b/tests/sites/downgrade/devices/DWN-2/expected/generated_config.json deleted file mode 100644 index 2f23b2a2e4..0000000000 --- a/tests/sites/downgrade/devices/DWN-2/expected/generated_config.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "timestamp": "2020-05-01T13:39:07Z", - "version": "1.4.2", - "system": { - "min_loglevel": 300, - "metrics_rate_sec": 600, - "operation": { } - }, - "localnet": { - "families": { - "virtual": { - "addr": "0x65" - } - } - }, - "pointset": { - "points": { - "filter_alarm_pressure_status": { - "ref": "BV11.present_value" - }, - "filter_differential_pressure": { }, - "filter_differential_pressure_sensor": { - "ref": "AV12.present_value" - } - } - } -} \ No newline at end of file diff --git a/tests/sites/downgrade/devices/DWN-2/expected/metadata_norm.json b/tests/sites/downgrade/devices/DWN-2/expected/metadata_norm.json deleted file mode 100644 index a39f4edd77..0000000000 --- a/tests/sites/downgrade/devices/DWN-2/expected/metadata_norm.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "timestamp": "2020-05-01T13:39:07Z", - "version": "1.3.14", - "hash": "a4d7a946", - "cloud": { - "auth_type": "RS256" - }, - "system": { - "location": { - "site": "US-SFO-XYY", - "section": "2-3N8C", - "position": { - "x": -111.0, - "y": 102.3 - } - }, - "physical_tag": { - "asset": { - "guid": "bim://04aEp5ymD_$u5IxhJN2aGi", - "site": "US-SFO-XYY", - "name": "AHU-1" - } - }, - "min_loglevel": 300 - }, - "localnet": { - "families": { - "virtual": { - "addr": "0x65" - } - } - }, - "pointset": { - "points": { - "filter_alarm_pressure_status": { - "units": "No-units", - "ref": "BV11.present_value" - }, - "filter_differential_pressure": { - "units": "Bars" - }, - "filter_differential_pressure_sensor": { - "units": "Degrees-Celsius", - "ref": "AV12.present_value" - } - }, - "exclude_units_from_config": true - } -} \ No newline at end of file diff --git a/tests/sites/missing/devices/AHU-1/expected/errors.json b/tests/sites/missing/devices/AHU-1/expected/errors.map similarity index 100% rename from tests/sites/missing/devices/AHU-1/expected/errors.json rename to tests/sites/missing/devices/AHU-1/expected/errors.map diff --git a/tests/sites/missing/devices/GAT-123/expected/errors.json b/tests/sites/missing/devices/GAT-123/expected/errors.map similarity index 100% rename from tests/sites/missing/devices/GAT-123/expected/errors.json rename to tests/sites/missing/devices/GAT-123/expected/errors.map diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java b/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java index 52cfdc5e02..e8e8a6186e 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java @@ -4,10 +4,12 @@ import static com.google.udmi.util.JsonUtil.convertToStrict; import static com.google.udmi.util.JsonUtil.stringify; import static com.google.udmi.util.JsonUtil.toMap; +import static com.google.udmi.util.MessageUpgrader.STATE_SCHEMA; import static udmi.schema.Envelope.SubFolder.UPDATE; import com.google.bos.udmi.service.messaging.MessageContinuation; import com.google.bos.udmi.service.messaging.StateUpdate; +import com.google.udmi.util.MessageUpgrader; import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -31,7 +33,8 @@ public class StateProcessor extends ProcessorBase { @Override protected void defaultHandler(Object originalMessage) { - StateUpdate stateMessage = convertToStrict(StateUpdate.class, originalMessage); + Object upgradedMessage = new MessageUpgrader(STATE_SCHEMA, originalMessage).upgrade(); + StateUpdate stateMessage = convertToStrict(StateUpdate.class, upgradedMessage); shardStateUpdate(getContinuation(originalMessage), stateMessage); updateLastStart(getContinuation(originalMessage).getEnvelope(), stateMessage); } diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageDispatcherImpl.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageDispatcherImpl.java index 1d5c579afa..ca80ffa8c7 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageDispatcherImpl.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageDispatcherImpl.java @@ -33,7 +33,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import java.util.function.Supplier; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; @@ -133,21 +132,33 @@ private static void registerMessageClass(SubType type, SubFolder folder, Class handlerType, Object messageObject) { + /** + * Execute the runnable with the envelope mapped for the message. + */ + @VisibleForTesting + public void withEnvelopeFor(Envelope envelope, Object message, Runnable run) { try { - messageEnvelopes.put(messageObject, envelope); + messageEnvelopes.put(message, envelope); setThreadEnvelope(envelope); - handlers.get(handlerType).accept(messageObject); - synchronized (handlerCounts) { - handlerCounts.computeIfAbsent(handlerType, key -> new AtomicInteger()).incrementAndGet(); - handlerCounts.notify(); - } + run.run(); } finally { - messageEnvelopes.remove(messageObject); + messageEnvelopes.remove(message); setThreadEnvelope(null); } } + private void processHandler(Envelope envelope, Class handlerType, Object messageObject) { + withEnvelopeFor(envelope, messageObject, () -> executeHandler(handlerType, messageObject)); + } + + private void executeHandler(Class handlerType, Object messageObject) { + handlers.get(handlerType).accept(messageObject); + synchronized (handlerCounts) { + handlerCounts.computeIfAbsent(handlerType, key -> new AtomicInteger()).incrementAndGet(); + handlerCounts.notify(); + } + } + /** * Process a received message bundle. */ @@ -355,4 +366,5 @@ public void terminate() { public String toString() { return format("Dispatcher %08x", Objects.hash(this)); } + } \ No newline at end of file diff --git a/udmis/src/test/java/com/google/bos/udmi/service/core/ProcessorTestBase.java b/udmis/src/test/java/com/google/bos/udmi/service/core/ProcessorTestBase.java index c4d458bc43..4cd938a35f 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/core/ProcessorTestBase.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/core/ProcessorTestBase.java @@ -1,6 +1,8 @@ package com.google.bos.udmi.service.core; import static com.google.bos.udmi.service.core.StateProcessor.IOT_ACCESS_COMPONENT; +import static com.google.udmi.util.JsonUtil.writeFile; +import static org.apache.commons.io.FileUtils.deleteDirectory; import static org.mockito.Mockito.mock; import com.google.bos.udmi.service.access.IotAccessBase; @@ -10,6 +12,7 @@ import com.google.bos.udmi.service.messaging.impl.MessageTestBase; import com.google.bos.udmi.service.pod.UdmiServicePod; import com.google.udmi.util.CleanDateFormat; +import java.io.File; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -17,6 +20,7 @@ import udmi.schema.EndpointConfiguration; import udmi.schema.EndpointConfiguration.Protocol; import udmi.schema.Envelope; +import udmi.schema.SetupUdmiConfig; /** * Base class for functional processor tests. @@ -27,8 +31,26 @@ public abstract class ProcessorTestBase extends MessageTestBase { public static final Date TEST_TIMESTAMP = CleanDateFormat.cleanDate(); public static final long ASYNC_PROCESSING_DELAY_MS = 2000; protected final List captured = new ArrayList<>(); - private ProcessorBase processor; protected IotAccessBase provider; + private ProcessorBase processor; + + /** + * Write a deployment file for testing. + */ + public static void writeVersionDeployFile() { + File deployFile = new File(UdmiServicePod.DEPLOY_FILE); + try { + deleteDirectory(deployFile.getParentFile()); + deployFile.getParentFile().mkdirs(); + SetupUdmiConfig deployedVersion = new SetupUdmiConfig(); + deployedVersion.deployed_at = TEST_TIMESTAMP; + deployedVersion.deployed_by = TEST_USER; + deployedVersion.udmi_version = TEST_VERSION; + writeFile(deployedVersion, deployFile); + } catch (Exception e) { + throw new RuntimeException("While writing deploy file " + deployFile.getAbsolutePath(), e); + } + } protected int getDefaultCount() { return getMessageCount(Object.class); @@ -42,6 +64,15 @@ protected int getMessageCount(Class clazz) { return processor.getMessageCount(clazz); } + @NotNull + protected abstract Class getProcessorClass(); + + protected T initializeTestInstance(@SuppressWarnings("unused") Class clazz) { + initializeTestInstance(); + //noinspection unchecked + return (T) processor; + } + protected void initializeTestInstance() { try { UdmiServicePod.resetForTest(); @@ -81,9 +112,6 @@ private void createProcessorInstance() { provider.activate(); } - @NotNull - protected abstract Class getProcessorClass(); - protected void terminateAndWait() { getTestDispatcher().terminate(); getReverseDispatcher().terminate(); diff --git a/udmis/src/test/java/com/google/bos/udmi/service/core/StateProcessorTest.java b/udmis/src/test/java/com/google/bos/udmi/service/core/StateProcessorTest.java index 80c6d6dcaf..e3a49e19a8 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/core/StateProcessorTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/core/StateProcessorTest.java @@ -2,6 +2,7 @@ import static com.google.udmi.util.GeneralUtils.ifNotNullGet; import static com.google.udmi.util.JsonUtil.fromStringStrict; +import static com.google.udmi.util.JsonUtil.loadFileRequired; import static com.google.udmi.util.JsonUtil.safeSleep; import static com.google.udmi.util.JsonUtil.stringify; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -36,6 +37,7 @@ public class StateProcessorTest extends ProcessorTestBase { public static final Date INITIAL_LAST_START = CleanDateFormat.cleanDate(new Date(12981837)); + private static final String LEGACY_STATE_MESSAGE_FILE = "src/test/messages/legacy_state.json"; @NotNull protected Class getProcessorClass() { @@ -64,7 +66,7 @@ private Envelope getTestStateEnvelope() { @NotNull private State getTestStateMessage(boolean includeGateway, boolean includeLastStart) { State stateMessage = new State(); - stateMessage.version = TEST_VERSION + "x"; + stateMessage.version = TEST_VERSION; stateMessage.gateway = includeGateway ? new GatewayState() : null; stateMessage.system = new SystemState(); if (includeLastStart) { @@ -141,6 +143,15 @@ public void multiExpansion() { assertEquals(1, getDefaultCount(), "default handler count"); } + @Test + public void legacyStateMessage() { + StateProcessor processor = initializeTestInstance(StateProcessor.class); + Object message = loadFileRequired(Object.class, LEGACY_STATE_MESSAGE_FILE); + dispatcher.withEnvelopeFor(new Envelope(), message, () -> processor.defaultHandler(message)); + getReverseDispatcher().waitForMessageProcessed(SystemState.class); + terminateAndWait(); + } + /** * Test that a state update with one sub-block results in a received message of the proper type. */ diff --git a/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/MessageTestCore.java b/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/MessageTestCore.java index bc2f6e089d..47002b14b7 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/MessageTestCore.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/MessageTestCore.java @@ -24,7 +24,7 @@ public abstract class MessageTestCore { protected static final String TEST_NAMESPACE = "test-namespace"; protected static final String TEST_SOURCE = "message_from"; protected static final String TEST_DESTINATION = "message_to"; - protected static final String TEST_VERSION = "1.32"; + protected static final String TEST_VERSION = "1.4.1"; protected static final String TEST_REF = "g123456789"; { diff --git a/udmis/src/test/messages/legacy_state.json b/udmis/src/test/messages/legacy_state.json new file mode 100644 index 0000000000..07575e4e5f --- /dev/null +++ b/udmis/src/test/messages/legacy_state.json @@ -0,0 +1 @@ +{"timestamp":"2023-08-08T16:32:31.000Z","version":"1","system":{"last_config":"2023-08-08T13:07:00Z","operational":true,"serial_no":"30000852","hardware":{"make":"EasyIO","model":"FS32","sku":"PLACEHOLDER","rev":"PLACEHOLDER"},"software":{"firmware":"V3.0b53"}},"pointset":{"points":{"temperature_sensor":{"units":"Degrees-Celsius"}}}} diff --git a/validator/bin/build b/validator/bin/build index fe56af5068..feffd710ac 100755 --- a/validator/bin/build +++ b/validator/bin/build @@ -24,6 +24,11 @@ if [[ -z $check && -z $covg && -f $jarfile && $jarfile -nt $newest ]]; then exit 0 fi +if [[ -n $check ]]; then + rm -rf $BASE/sites/udmi_site_model/devices/XXX-1 + $BASE/bin/clone_model +fi + echo Java version $(java --version) echo Building validator in $PWD diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index bd745ec538..0a74400810 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -2,20 +2,19 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; -import static com.google.daq.mqtt.registrar.Registrar.DEVICE_ERRORS_JSON; +import static com.google.daq.mqtt.registrar.Registrar.DEVICE_ERRORS_MAP; import static com.google.daq.mqtt.registrar.Registrar.ENVELOPE_JSON; import static com.google.daq.mqtt.registrar.Registrar.GENERATED_CONFIG_JSON; import static com.google.daq.mqtt.registrar.Registrar.METADATA_JSON; import static com.google.daq.mqtt.registrar.Registrar.NORMALIZED_JSON; -import static com.google.daq.mqtt.util.MessageUpgrader.METADATA_SCHEMA; import static com.google.udmi.util.Common.VERSION_KEY; -import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_RAW; import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_STRICT; import static com.google.udmi.util.GeneralUtils.compressJsonString; import static com.google.udmi.util.GeneralUtils.isTrue; import static com.google.udmi.util.GeneralUtils.writeString; import static com.google.udmi.util.JsonUtil.OBJECT_MAPPER; import static com.google.udmi.util.JsonUtil.asMap; +import static com.google.udmi.util.MessageUpgrader.METADATA_SCHEMA; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -32,11 +31,13 @@ import com.google.daq.mqtt.util.CloudIotManager; import com.google.daq.mqtt.util.ExceptionMap; import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; -import com.google.daq.mqtt.util.MessageUpgrader; import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; +import com.google.udmi.util.MessageDowngrader; +import com.google.udmi.util.MessageUpgrader; import com.google.udmi.util.SiteModel; +import com.google.udmi.util.SiteModel.MetadataException; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; @@ -150,7 +151,7 @@ class LocalDevice { EXPECTED_DIR, OUT_DIR); private static final Set OUT_FILES = ImmutableSet.of( - GENERATED_CONFIG_JSON, DEVICE_ERRORS_JSON, NORMALIZED_JSON, EXCEPTION_LOG_FILE); + GENERATED_CONFIG_JSON, DEVICE_ERRORS_MAP, NORMALIZED_JSON, EXCEPTION_LOG_FILE); private static final Set ALL_KEY_FILES = ImmutableSet.of( RSA_PUBLIC_PEM, @@ -269,6 +270,10 @@ private Metadata readMetadataWithValidation(boolean validate) { try { Metadata loadedMetadata = SiteModel.loadDeviceMetadata(siteDir.getPath(), deviceId, LocalDevice.class); + if (loadedMetadata instanceof MetadataException metadataException) { + throw new RuntimeException("Loading " + metadataException.file.getAbsolutePath(), + metadataException.exception); + } instance = JsonUtil.convertTo(JsonNode.class, loadedMetadata); baseVersion = instance.get(VERSION_KEY); new MessageUpgrader(METADATA_SCHEMA, instance).upgrade(false); @@ -308,9 +313,8 @@ JsonNode getMergedMetadata(JsonNode instance) { private Metadata readMetadata() { Metadata deviceMetadata = readMetadataWithValidation(validateMetadata); - if (deviceMetadata != null && deviceMetadata.exception != null) { - exceptionMap.put(EXCEPTION_CONVERTING, deviceMetadata.exception); - deviceMetadata.exception = null; + if (deviceMetadata instanceof MetadataException metadataException) { + exceptionMap.put(EXCEPTION_CONVERTING, metadataException.exception); } return deviceMetadata; } @@ -359,6 +363,9 @@ private boolean isDeviceKeySource() { public void loadCredentials() { try { deviceCredentials.clear(); + if (metadata == null) { + return; + } if (hasGateway() && hasAuthType()) { throw new RuntimeException("Proxied devices should not have cloud.auth_type defined"); } @@ -400,7 +407,7 @@ private Credential getDeviceCredential(String keyFile) throws IOException { } private Set keyFiles() { - if (!isDirectConnect()) { + if (metadata == null || !isDirectConnect()) { return ImmutableSet.of(); } String authType = getAuthType(); @@ -663,7 +670,7 @@ private String makeNumId(Envelope envelope) { } public void writeErrors(List ignoreErrors) { - File errorsFile = new File(outDir, DEVICE_ERRORS_JSON); + File errorsFile = new File(outDir, DEVICE_ERRORS_MAP); ErrorTree errorTree = getErrorTree(ignoreErrors); if (errorTree != null) { try (PrintStream printStream = new PrintStream(Files.newOutputStream(errorsFile.toPath()))) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 9fcd17b4fc..58bb24cb3f 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -77,7 +77,7 @@ public class Registrar { static final String METADATA_JSON = "metadata.json"; static final String ENVELOPE_JSON = "envelope.json"; static final String NORMALIZED_JSON = "metadata_norm.json"; - static final String DEVICE_ERRORS_JSON = "errors.json"; + static final String DEVICE_ERRORS_MAP = "errors.map"; static final String GENERATED_CONFIG_JSON = "generated_config.json"; private static final String DEVICES_DIR = "devices"; private static final String SCHEMA_SUFFIX = ".json"; diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index 66c6955150..9f009fae09 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -65,6 +65,7 @@ import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.SiteModel; +import com.google.udmi.util.SiteModel.MetadataException; import java.io.File; import java.io.FileOutputStream; import java.io.PrintWriter; @@ -294,9 +295,10 @@ private static void reportLoadingErrors(SiteModel model) { String deviceId = validatorConfig.device_id; checkState(model.allDeviceIds().contains(deviceId), format("device_id %s not found in site model", deviceId)); - Exception exception = model.getMetadata(deviceId).exception; - if (exception != null) { - System.err.println("Device loading error: " + exception.getMessage()); + Metadata metadata = model.getMetadata(deviceId); + if (metadata instanceof MetadataException metadataException) { + System.err.println( + "Device loading error: " + friendlyStackTrace(metadataException.exception)); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index ca6fb4f1c3..3d83b0a6c3 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -47,12 +47,12 @@ import com.google.daq.mqtt.util.FileDataSink; import com.google.daq.mqtt.util.MessagePublisher; import com.google.daq.mqtt.util.MessagePublisher.QuerySpeed; -import com.google.daq.mqtt.util.MessageUpgrader; import com.google.daq.mqtt.util.PubSubClient; import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.Common; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; +import com.google.udmi.util.MessageUpgrader; import com.google.udmi.util.SiteModel; import java.io.File; import java.io.FilenameFilter; @@ -943,7 +943,9 @@ private void validateFile( Map message = JsonUtil.loadMap(inputFile); sanitizeMessage(schemaName, message); JsonNode jsonNode = OBJECT_MAPPER.valueToTree(message); - if (upgradeMessage(schemaName, jsonNode)) { + MessageUpgrader messageUpgrader = new MessageUpgrader(schemaName, jsonNode); + messageUpgrader.upgrade(forceUpgrade); + if (messageUpgrader.wasUpgraded()) { OBJECT_MAPPER.writeValue(outputStream, jsonNode); } else { // If the message was not upgraded, then copy over unmolested to preserve formatting. @@ -993,18 +995,14 @@ private File getOutputPath(String baseFile, String addedFile) { private void upgradeMessage(String schemaName, Map message) { JsonNode jsonNode = OBJECT_MAPPER.convertValue(message, JsonNode.class); - upgradeMessage(schemaName, jsonNode); - Map objectMap = OBJECT_MAPPER.convertValue(jsonNode, + Object upgraded = new MessageUpgrader(schemaName, jsonNode).upgrade(forceUpgrade); + Map objectMap = OBJECT_MAPPER.convertValue(upgraded, new TypeReference<>() { }); message.clear(); message.putAll(objectMap); } - private boolean upgradeMessage(String schemaName, JsonNode jsonNode) { - return new MessageUpgrader(schemaName, jsonNode).upgrade(forceUpgrade); - } - private void validateJsonNode(JsonSchema schema, JsonNode jsonNode) throws ProcessingException { ProcessingReport report = schema.validate(jsonNode, true); if (!report.isSuccess()) { diff --git a/validator/src/test/java/com/google/daq/mqtt/registrar/MessageDowngraderTest.java b/validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java similarity index 96% rename from validator/src/test/java/com/google/daq/mqtt/registrar/MessageDowngraderTest.java rename to validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java index c1899146f8..8a45be2b04 100644 --- a/validator/src/test/java/com/google/daq/mqtt/registrar/MessageDowngraderTest.java +++ b/validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.registrar; +package com.google.udmi.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import com.google.udmi.util.MessageDowngrader; import java.io.File; import org.junit.Test;