From 30eb0da1299ce41fd5c028e7b068f5e06da4a4b5 Mon Sep 17 00:00:00 2001 From: Trevor Date: Mon, 16 Oct 2023 08:02:41 -0700 Subject: [PATCH] Implement no-state sequence testing mode (#738) --- .gencode_hash.txt | 44 +++---- .github/workflows/testing.yml | 22 ++-- bin/augment_metadata | 22 +++- bin/test_itemized | 50 +++++--- bin/test_sequencer | 35 ++++-- .../daq/mqtt/util/ValidationException.java | 49 ++++++++ .../java/com/google/udmi/util/Common.java | 2 + .../com/google/udmi/util/GeneralUtils.java | 26 ++++- .../google/udmi/util/MessageDowngrader.java | 101 ++++++++++++---- .../com/google/udmi/util/MessageUpgrader.java | 17 +-- .../com/google/udmi/util/SchemaVersion.java | 51 ++++++++ etc/schema_itemized.out | 5 +- etc/schema_nostate.out | 2 + etc/sequencer_nostate.out | 35 ++++++ etc/test_itemized.out | 11 +- gencode/docs/config.html | 34 ++++++ gencode/docs/configuration_pubber.html | 40 +++++++ gencode/docs/event.html | 82 +++++++++++++ gencode/docs/event_pointset.html | 34 ++++++ gencode/docs/event_system.html | 34 ++++++ gencode/docs/metadata.html | 110 +++++++++++++++++- gencode/docs/monitoring.html | 48 ++++++++ gencode/docs/state.html | 34 ++++++ gencode/java/udmi/schema/Config.java | 11 +- gencode/java/udmi/schema/Metadata.java | 24 +++- gencode/java/udmi/schema/PointsetEvent.java | 11 +- gencode/java/udmi/schema/PubberOptions.java | 6 +- gencode/java/udmi/schema/State.java | 11 +- gencode/java/udmi/schema/SystemEvent.java | 11 +- gencode/java/udmi/schema/TestingModel.java | 6 +- gencode/python/udmi/schema/config.py | 4 + gencode/python/udmi/schema/event_pointset.py | 4 + gencode/python/udmi/schema/event_system.py | 4 + gencode/python/udmi/schema/metadata.py | 8 ++ gencode/python/udmi/schema/model_testing.py | 4 + gencode/python/udmi/schema/options_pubber.py | 4 + gencode/python/udmi/schema/state.py | 4 + pubber/src/main/java/daq/pubber/Pubber.java | 64 ++++++---- schema/config.json | 4 + schema/event_pointset.json | 4 + schema/event_system.json | 4 + schema/metadata.json | 10 +- schema/model_testing.json | 3 + schema/options_pubber.json | 3 + schema/state.json | 4 + tests/schemas/config/errors.out | 2 +- .../DWN-1/expected/generated_config.json | 3 +- .../devices/DWN-1/expected/metadata_norm.json | 3 +- .../devices/DWN-2/expected/errors.map | 2 +- .../GAT-123/expected/generated_config.json | 3 +- .../GAT-123/expected/metadata_norm.json | 3 +- .../SNS-4/expected/generated_config.json | 3 +- .../devices/SNS-4/expected/metadata_norm.json | 3 +- udmis/bin/container | 2 +- udmis/bin/deploy | 1 - udmis/bin/pod_logs | 4 +- udmis/build.gradle | 2 +- udmis/deploy_udmis_gcloud | 49 -------- .../access/DynamicIotAccessProvider.java | 50 ++++---- .../udmi/service/core/ReflectProcessor.java | 17 +-- .../messaging/impl/TraceMessagePipe.java | 3 +- .../bos/udmi/service/pod/UdmiServicePod.java | 1 + udmis/udmis.iml | 2 +- .../bos/iot/core/proxy/MessageValidator.java | 3 +- .../daq/mqtt/registrar/LocalDevice.java | 3 +- .../google/daq/mqtt/sequencer/Feature.java | 5 + .../daq/mqtt/sequencer/SequenceBase.java | 101 ++++++++++++---- .../sequencer/sequences/ConfigSequences.java | 4 +- .../sequences/PointsetSequences.java | 4 +- .../daq/mqtt/util/ObjectDiffEngine.java | 4 + .../daq/mqtt/util/ValidationException.java | 80 ------------- .../google/daq/mqtt/validator/Validator.java | 37 +++++- .../udmi/util/MessageDowngraderTest.java | 19 ++- 73 files changed, 1164 insertions(+), 345 deletions(-) create mode 100644 common/src/main/java/com/google/daq/mqtt/util/ValidationException.java create mode 100644 common/src/main/java/com/google/udmi/util/SchemaVersion.java create mode 100644 etc/schema_nostate.out create mode 100644 etc/sequencer_nostate.out delete mode 120000 udmis/bin/deploy delete mode 100755 udmis/deploy_udmis_gcloud delete mode 100644 validator/src/main/java/com/google/daq/mqtt/util/ValidationException.java diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 355f6ff7ec..af1ee980ac 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,26 +1,26 @@ b35646a2f3d7c30fd34e3ea6ab6a1070b39c469fd1de49cab1093ff32faaf06d gencode/docs/command_discovery.html d3758eba2529d4a5f1dfd5ed3355a536936b02285ddde7cc75b1f41f4916203a gencode/docs/command_mapping.html -cc618f9c64c36a7ffad5df420b8f192a4b1f1af9a4efca1a1e92056d4a892a47 gencode/docs/config.html +2067b2eab579b6f6667d6890dfeb72cb3ec6fbd779668c8ff7dc8f42e3f9a60b gencode/docs/config.html d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673 gencode/docs/config_mapping.html 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79 gencode/docs/configuration_endpoint.html 33846607c2c7241660b62b0ca1ef1f56cba922e6aa3b1a2813e3a4a0cc94a4ef gencode/docs/configuration_execution.html e081e1438839467d3be35dd866d0cd00f4454fcc5d8e384ba03ca28cd439af87 gencode/docs/configuration_pod.html -75294ff7051586e821a8c090e18eb12e325ccf7fcac86537669dd6e009025979 gencode/docs/configuration_pubber.html -564d652ef4393f0b83d5e35c0be26766023c300414f6cef92ccc26cae440c0d1 gencode/docs/event.html +b197e72b85de1e73e4c8954d85cd28c6958335c9b9dcc2504834bf92cd21f1f1 gencode/docs/configuration_pubber.html +96186777da06f95eae1d16d73555445d23608a9301636ea1ccd17922b3fe4019 gencode/docs/event.html 587e048c161273b927de67b899204bf0e183db64e59ae513f833e5eff406b1ab gencode/docs/event_discovery.html 0f99534574718e07e655e33e76e06b56e6a96a7a42ae1457dc97dabc581d848f gencode/docs/event_mapping.html -71fe25d3b17d1dc87b52c049a3235c20527f73e12ffa53fa4800a15e53f73732 gencode/docs/event_pointset.html -acd50976d1afd771d55177db9a1b9452a0c6a35e3e15b2e444c799991294808d gencode/docs/event_system.html +d2fe30200f9a1c530d144137a7774b5b97f268aae41185bfcb25de274d399250 gencode/docs/event_pointset.html +6849a96fe7608dc5ebda0dc9595dcea57757bc7c5253c30c30409469b6260498 gencode/docs/event_system.html 816481f69d3b1bdeb2224eaad6e3751a991d20eb98294d89f888b1323505209c gencode/docs/event_validation.html -ae5762dbe2304e44cd742e7236d2de6987bdc1384b21093490cfcd7e33a5509f gencode/docs/metadata.html -4db86e0b979a1873d73da07d68a432df39282cb8b94ea509514d2b411411d55e gencode/docs/monitoring.html +0734afc2bddcaa7166c10e070ed6a4805127aa4c90e060915cdb48d5d79f618f gencode/docs/metadata.html +c4fa2845c5ad385a619ec97827370988c059567cb23c8da6894331fed89fefce gencode/docs/monitoring.html 180b32717db748e164a185b163ef9a97aa83d9d6add306283d5b9852d04af947 gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html 1766f84518a315fe57e4a4bf934c0a386ad61d87091754a6bab097c686c16019 gencode/docs/readme.md 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9 gencode/docs/schema_doc.min.js 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a gencode/docs/schema_extras.js -f8854f923ab8dbda6559942948298467fe20982bf71f79b0cb0912d9a7ae5167 gencode/docs/state.html +2294674149e961fb6aaf993dcf37d94681bd81ee073053ca9b8b70df7d3ceb3d gencode/docs/state.html f4a52cdad2cd9a9646da4d84cc1f3962ddd151a8d348255e5cff30eadfee283c gencode/docs/state_mapping.html 567b27a9257816eb9a9063ce089fe0f58c5962f10ea449bb7c653bd6151365a8 gencode/docs/state_udmi.html 2c6557d3d63269a956cd32d1784044b16d72172ab559656bf65b8d3684748c25 gencode/docs/state_validation.html @@ -37,7 +37,7 @@ e28c3899bf2cb08cf456dbca29ccc3d4559c1d2145e252095689b171be82b4c6 gencode/java/u 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e gencode/java/udmi/schema/BuildingTranslation.java 36a165d3c65ca02cd095606a27d7efa21f6d3b7134bb49771b315dcdfa7154b7 gencode/java/udmi/schema/Category.java e15316a81d0acfc965e5517b9b49a7514f6dac024bebf8557e41f8abe0787023 gencode/java/udmi/schema/CloudModel.java -ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4 gencode/java/udmi/schema/Config.java +a8bf763f1db77446d9e97154630b30ce4f92c13ea1c567102067dd132227ddd5 gencode/java/udmi/schema/Config.java cf9762392e02b5d04c6498963222cc0c00f7be6c3cd82bde3d063a5eceba2b65 gencode/java/udmi/schema/Connections.java 8164e93ccc76d78548d456890e064df8427a09bbcd50686b37a10d3ff5ad429f gencode/java/udmi/schema/Credential.java 10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05 gencode/java/udmi/schema/DeviceMappingConfig.java @@ -78,7 +78,7 @@ ccc7c234dd522a91d387d55573677681bbc5a6926f0fbf1101c2e9f607cafd63 gencode/java/u 39f4cf5f89dc52f24ea02246e17668ebc85ed0fab351e69138e477bcc048d79c gencode/java/udmi/schema/MappingEvent.java 052b6b7f9ca8173901d9d2fae314f5858e034f20efa5d184780b800bd870cb2b gencode/java/udmi/schema/MappingEventEntity.java d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886 gencode/java/udmi/schema/MappingState.java -476f542353b69649eea8a63fe53a01d83a1d62a33eae140241b107525024dda5 gencode/java/udmi/schema/Metadata.java +991bf3fa8916e0d1745f4734b9bb5c61409cd860277aaa29701335fdf07919e9 gencode/java/udmi/schema/Metadata.java df64e4ddbf543ac70e7c2af9d3fbc20ffe3dff68c6718aa9ceadab7f64d3d171 gencode/java/udmi/schema/Metrics.java 14722df90406cdcd29c7c946e5fcdf6a3e513eb879a7fbb493801fc0d9093504 gencode/java/udmi/schema/Monitoring.java b3b7ef6cae004d5fdb4052f860df0aa583df87bb798a027fc769aeaf16489789 gencode/java/udmi/schema/MonitoringMetric.java @@ -91,29 +91,29 @@ eb478a6ebd66b150da8d6dc8fdf05a10d09d6d0ea1df8a0578728703718c5551 gencode/java/u 65337109653a93d873dae40eb517857149bae1b20c7b479f41b35c0822d94ba0 gencode/java/udmi/schema/PointPointsetModel.java db8d6dd3498019ad12e0f328b6237d07e52f133f8b08858b712611a52c198009 gencode/java/udmi/schema/PointPointsetState.java 1fd6168df4ee02c8395abc762f6d4d4ddac4eb8ee3b3e329252b2f88c9903a8c gencode/java/udmi/schema/PointsetConfig.java -bb1c7e0df78870b7f7bd78fdfb1ed115cb121613ffcd98ec6b95e41c01624d41 gencode/java/udmi/schema/PointsetEvent.java +c385e937c9952be3405a63bf3292790f965960a91ef0267126f39051a8e0b513 gencode/java/udmi/schema/PointsetEvent.java 1e6be912a048065d678962b23b3b8f17e01c426ac8ff463c57c8c2d02e39ebd3 gencode/java/udmi/schema/PointsetModel.java fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa gencode/java/udmi/schema/PointsetState.java 61afd6c9c0364b8982c90df7664147829bad8858408b2141299ee8bbc35f1612 gencode/java/udmi/schema/PointsetSummary.java ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a gencode/java/udmi/schema/Position.java 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70 gencode/java/udmi/schema/Properties.java 2f48b09437aab980a40ebf06c3334a7b33843b160d7618da20789e3d628e9650 gencode/java/udmi/schema/PubberConfiguration.java -da8a104d7538d81ee97e325b18f509650d86e0feb2540d110aae8172de123179 gencode/java/udmi/schema/PubberOptions.java +6230499555484858a6032d73094b9338b6983cda4aa519f77666fd8bf4f837a6 gencode/java/udmi/schema/PubberOptions.java 703ed0cecb60f284b52e76dc4d612eb798ba3e8781cc40698a46611b36e6b0c9 gencode/java/udmi/schema/SchemaValidationState.java 6da1708f597a0c64ce30dcbe739095fc7ebc091d63b98af88c73f70706af31ca gencode/java/udmi/schema/SequenceValidationState.java 9da49b22341a65580d085fd9d00eaadcaefcf7b813988ef844c617aa4b8a9a4f gencode/java/udmi/schema/SetupUdmiConfig.java e79f3842470cc1f12b1d0a9d6691b147aeb4d6bf29e24872721133352aed1cc9 gencode/java/udmi/schema/SetupUdmiState.java -9dc7e377f3392d35b5b4da3d1bbeb5e8d9a375a13ec6c852c478112cf26d8006 gencode/java/udmi/schema/State.java +f3d880f0ec7af07a8eb22c19b3c23e943fc54af03723d458664a5d9dcce57584 gencode/java/udmi/schema/State.java 47ad54328160a1aabce719a3263d4bfea903e8e8255e04a54ace86eb095b0f0c gencode/java/udmi/schema/StateSystemHardware.java c9b920d355e5f6350fa6eb978afbfe55dd1438df171d2ea215845d5ac0b3e7e9 gencode/java/udmi/schema/StateSystemOperation.java d771dcb6bd06620e4e21419420d252b1f4aa0c55d0be315d6310299cebff56d6 gencode/java/udmi/schema/SystemConfig.java 6f7b213970bfe2b1ac3056b83da8c8f5b4c50eac221dc1d6c6887e193e8bf40c gencode/java/udmi/schema/SystemDiscoveryEvent.java -21ca023d5a9006b1e041fcf644d572270b5743fcd2d2f221364a3046f0c5609c gencode/java/udmi/schema/SystemEvent.java +484939a730497cd27a8eb57c17cd10140ff0710368326b4fe8ba28080b406a79 gencode/java/udmi/schema/SystemEvent.java 2cf23174ef4e2876511fb471d3f9fcb5cefe2fde324db844c2d0d505fd2c8844 gencode/java/udmi/schema/SystemHardware.java b48923eea2caded1badca59376c63d51be6352b9f9fcb375cdef5897c0fc5fda gencode/java/udmi/schema/SystemModel.java 7d8d9f058878d3228a2042b8ffe6a5d897548c32b218307004101699aa17c2b3 gencode/java/udmi/schema/SystemState.java 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d gencode/java/udmi/schema/TargetTestingModel.java -d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645 gencode/java/udmi/schema/TestingModel.java +7db9435bd72c03d4bc4bff720d6cee1b65c7af5d10687579e44d96d2d1683813 gencode/java/udmi/schema/TestingModel.java 7793d6d76a430dc7acf668d92d2df5f8e0625d6228207731f96e220f3b90e659 gencode/java/udmi/schema/TestingSystemConfig.java 2958978ce5b7418320835e7d6731bd8db6700643f02bb48eed4edc3dd90b686f gencode/java/udmi/schema/UdmiConfig.java 92d94860b963a1b024a6cc55c40870852363da7e08547cc1cea6e6274293cc2a gencode/java/udmi/schema/UdmiState.java @@ -129,7 +129,7 @@ dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6 gencode/python 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb gencode/python/udmi/schema/command_discovery.py 2a858a05c8717002c909ebf9e4661abedd5affeada33e2a114d440eccc677e03 gencode/python/udmi/schema/command_mapping.py 741b37355b360f2daeb99a1e6c2ce4f4673168d443b0e7cab8d67f5101460bbb gencode/python/udmi/schema/common.py -b975892df78076dabc797b4c0be87f20b33eacda11f9d1ac1c09be33d4937a87 gencode/python/udmi/schema/config.py +53961cd8a6ddd118143457ca60d765e12b21bcdf98fc2d648ebac0172ffca880 gencode/python/udmi/schema/config.py 79eb0299b3751d93c01a5de65eacc717283e99caf0996f3db48a15247f69c8eb gencode/python/udmi/schema/config_blobset.py 59039eaf2812392329c641ad387f7e27790cf804fc94a014626d2da5ab426e3d gencode/python/udmi/schema/config_blobset_blob.py 7a6411ba2646881ffad5b6ad472c980e48a11b36ab2c9c77d12608ccc7225594 gencode/python/udmi/schema/config_discovery.py @@ -159,12 +159,12 @@ e285b2af68d9cc1f019642f763ae62c35a482e652c1def8715bae5a487af4153 gencode/python dd30f748b5321223933c272d58f5fa6dc319912737255170322f0df3244732a7 gencode/python/udmi/schema/event_mapping.py 6163270aa726ef29bc9285d224ec5c2611aab5d19f6ed09d711e6c816fc7e467 gencode/python/udmi/schema/event_mapping_entities.py 3652a73a7df2d9df18a45b72d948c38a84a61e008d7543d006dad1832640c54f gencode/python/udmi/schema/event_mapping_entity.py -ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b gencode/python/udmi/schema/event_pointset.py +40f4e3cd2ff180b9b8c6235682f2d661342f610e30fc1613e18676387d52bfa5 gencode/python/udmi/schema/event_pointset.py 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00 gencode/python/udmi/schema/event_pointset_point.py -d89500d1e1c88ac754ddc5abd13135dd140d40c60568ca9294f6fe384b0b12ef gencode/python/udmi/schema/event_system.py +7d520df05f839ed4a7472c395c77cf1d2cbf47e3162e87e43ed5c9ae9e979af6 gencode/python/udmi/schema/event_system.py 4361f48caba6a948a5916f67c8168215721cb3cc2384649f0336804785f01f1d gencode/python/udmi/schema/event_validation.py 633c85b0999d969310918bb03e9a40a08960f420b862340045a0290f8f5c4fe5 gencode/python/udmi/schema/event_validation_device.py -c7304b2070e6d930b4d101263b90768b036f7fb7e352a6e5147b83a5523987f4 gencode/python/udmi/schema/metadata.py +02e464aec10ae7f8cf10045f6107e069de4b852a24cd8fae6a4290a01c0d2ac8 gencode/python/udmi/schema/metadata.py 6e763776c8fc5b5a980b490873b1ee7613e15c5b1dee151278f7eb941e897901 gencode/python/udmi/schema/model_cloud.py 805cc8dcb29732d1965bbd533b12d2bc4966d584f05b8a478dd2cac98fd99d52 gencode/python/udmi/schema/model_discovery.py 44d057dffcac3a994e7b894f8e1ebb643ba49d7dbfbb0bfea9cc04f97af0bc47 gencode/python/udmi/schema/model_discovery_family.py @@ -176,14 +176,14 @@ ae6a7bc93b520cdc90cd158f876a58e98a3e000402ebb1ca632f01ee492d408a gencode/python 1ad65c005d865b9259bfa239b2dc4cd1cde13d312ecbc7843bb61cc31d6cabde gencode/python/udmi/schema/model_pointset_point.py ee9c02c35438fb7d9aacb15a21ec7b35b533c1000d0bde044ec3923b1fdccca4 gencode/python/udmi/schema/model_system.py 15b349141ebae651c6c3c5c313b197d49c8b2b44e8ff1b0639848ad42e5c4e63 gencode/python/udmi/schema/model_system_hardware.py -aafe6e70c281152db958adf77a024e3e9fab8293927106297c5ec48c11f54e27 gencode/python/udmi/schema/model_testing.py +b48ae013d203eb31cc388a084c0cc6f93d4d5534336d4da2ecbccd84a085ea2b gencode/python/udmi/schema/model_testing.py 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f gencode/python/udmi/schema/model_testing_target.py 7c9a755ae06c44fc2b8117d7918b6401ce08a7218b86033fb3320eda7286e581 gencode/python/udmi/schema/monitoring.py 3a3873c7db8d5fa13b43629031d36417a4ec4838d612a7b8c2fce7c22fdee009 gencode/python/udmi/schema/monitoring_metric.py -db023d2b8833953a497e467c7e4dbd8fe5019db2c4f8a492bae4d3ac61af4460 gencode/python/udmi/schema/options_pubber.py +4b3a8aa2fabdb41ee5d652f0769cae9f8daa5d60e013d7f2e569112bb09d040c gencode/python/udmi/schema/options_pubber.py 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77 gencode/python/udmi/schema/persistent_device.py a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690 gencode/python/udmi/schema/properties.py -03880d929419402a76e1ff1ae3faec7520eff6b8b632d19dc13de70f0f9873f2 gencode/python/udmi/schema/state.py +e604cf0280fe772de5f4e5ecf10dc6c564b6177eeff9cd9fb8b385af8fe10a95 gencode/python/udmi/schema/state.py 4a908cee3fb8afb559bcbfa594e57dbc515a35e4468e02600751b2fcce05a238 gencode/python/udmi/schema/state_blobset.py 182e07b534403dcc121d980672e41b0fa2ee55c4da1f5c56f0dad5d599450c80 gencode/python/udmi/schema/state_blobset_blob.py 8c6fc5fd63ab50768cb3662228ed893eb8b9d563cd062efaed12f5079a652d0c gencode/python/udmi/schema/state_discovery.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e62ae84102..1e8c7b4cf0 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -242,7 +242,7 @@ jobs: comprehensive: name: Comprehensive Suite runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 35 if: vars.TARGET_PROJECT != '' env: TARGET_PROJECT: ${{ vars.TARGET_PROJECT }} @@ -256,11 +256,9 @@ jobs: - name: base setup run: bin/run_tests install_dependencies - name: registrar clean - run: | - bin/test_regclean $TARGET_PROJECT + run: bin/test_regclean $TARGET_PROJECT - name: telemetry validator - run: | - bin/test_validator $TARGET_PROJECT + run: bin/test_validator $TARGET_PROJECT - name: validator result if: ${{ always() }} run: | @@ -270,13 +268,15 @@ jobs: run: | cd sites/udmi_site_model/out find . -type f | sort | xargs more | cat + - name: nostate sequences + if: ${{ always() }} + run: bin/test_sequencer clean alpha nostate full $TARGET_PROJECT - name: itemized sequencer tests - run: | - bin/test_itemized $TARGET_PROJECT + run: bin/test_itemized $TARGET_PROJECT - name: itemized output logs if: ${{ always() }} run: | - more out/test_itemized.out out/schemas_itemized.out | cat + more out/*_itemized.out | cat more out/sequencer.out-* out/pubber.out-* | cat - name: tool output logs if: ${{ always() }} @@ -319,11 +319,9 @@ jobs: run: bin/run_tests install_dependencies - name: bin/test_proxy if: "${{ env.MQTT_PROXY_HOST != '' }}" - run: | - bin/test_proxy $TARGET_PROJECT $MQTT_PROXY_HOST + run: bin/test_proxy $TARGET_PROJECT $MQTT_PROXY_HOST - name: bin/test_redirect - run: | - bin/test_redirect $TARGET_PROJECT + run: bin/test_redirect $TARGET_PROJECT - name: pubber.out.1 run: | echo ::::::: pubber.out.1 diff --git a/bin/augment_metadata b/bin/augment_metadata index 83cc3cc0c9..d48f6ddf71 100755 --- a/bin/augment_metadata +++ b/bin/augment_metadata @@ -5,14 +5,20 @@ # the testing infrastrucure. Not intended for use outside of CI testing. # -if [[ $# -ne 2 ]]; then - echo Usage: $0 SITE_DIR DEVICE_ID +if [[ $# -lt 2 ]]; then + echo Usage: $0 SITE_DIR DEVICE_ID [PUBBER_OPTS...] false fi site_dir=$(realpath $1) device_id=$2 shift 2 +pubber_opts=$* + +nostate=false +if [[ $pubber_opts =~ noState ]]; then + nostate=true +fi ROOT_DIR=$(dirname $0)/.. cd $ROOT_DIR @@ -35,12 +41,16 @@ ipv4_addr=${ipv4_addr%/*} ipv6_addr=$(ip addr show dev $default_iface | fgrep 'inet6 ' | tail -n 1 | awk '{print $2}') ipv6_addr=${ipv6_addr%/*} -echo Setting $metadata_out ether addr to $ether_addr -echo Setting $metadata_out ipv4 addr to $ipv4_addr -echo Setting $metadata_out ipv6 addr to $ipv6_addr +echo Augmenting to $metadata_out +echo Setting ether addr to $ether_addr +echo Setting ipv4 addr to $ipv4_addr +echo Setting ipv6 addr to $ipv6_addr +echo Setting nostate to $nostate + jq ".localnet.families.ether.addr = \"$ether_addr\"" $metadata_file | jq ".localnet.families.ipv4.addr = \"$ipv4_addr\"" | - jq ".localnet.families.ipv6.addr = \"$ipv6_addr\"" > $metadata_out + jq ".localnet.families.ipv6.addr = \"$ipv6_addr\"" | + jq ".testing.nostate = $nostate" > $metadata_out # Dynamically update the site model based off of dynamic pubber output. pubber_features=$(< $site_dir/out/pubber_features.json) diff --git a/bin/test_itemized b/bin/test_itemized index 1899e11097..117150564a 100755 --- a/bin/test_itemized +++ b/bin/test_itemized @@ -3,12 +3,13 @@ ROOT_DIR=$(dirname $0)/.. cd $ROOT_DIR -if [[ $# != 1 ]]; then - echo Usage: $0 PROJECT_ID +if [[ $# != 1 && $# != 2 ]]; then + echo Usage: $0 PROJECT_ID [target test] false fi PROJECT_ID=$1 shift 1 +TARGET_TEST=$1 SITE_PATH=sites/udmi_site_model DEVICE_ID=AHU-1 @@ -21,27 +22,44 @@ RESULTS_OUT=out/test_itemized.out GOLDEN_FILE=etc/test_itemized.out SCHEMA_OUT=out/schema_itemized.out GOLDEN_SCHEMAS=etc/schema_itemized.out +AUGMENT_OPTS=out/augment_opts.txt echo "export TARGET_PROJECT=$PROJECT_ID" echo "export UDMI_REGISTRY_SUFFIX=$UDMI_REGISTRY_SUFFIX" echo "export UDMI_ALT_REGISTRY=$UDMI_ALT_REGISTRY" -rm -f $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT +rm -f $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT $AUGMENT_OPTS mkdir -p out -touch $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT +touch $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT $AUGMENT_OPTS pubber/bin/build validator/bin/build test_index=0 while read -u 7 action test_name remainder; do - if [[ $action != TEST ]]; then - continue; + if [[ $action == TEST && -n $TARGET_TEST && $TARGET_TEST != $test_name ]]; then + continue + fi + + if [[ $action == WITH ]]; then + echo $test_name $remainder > $AUGMENT_OPTS + elif [[ -z $action || $action =~ ^# ]]; then + true # skip blank lines and comments + elif [[ $action != TEST ]]; then + echo Unknown test action $action + false fi + pubber_opts=${remainder/RESULT*/} - echo -n "TEST $test_name $pubber_opts" >> $RESULTS_OUT - echo "TEST $test_name $pubber_opts" + echo -n "$action $test_name $pubber_opts" >> $RESULTS_OUT + + if [[ $action != TEST ]]; then + echo >> $RESULTS_OUT + continue + fi + + echo itemized test $test_name $pubber_opts [$(< $AUGMENT_OPTS)] # Clean out the persistant data store to ensure a clean state each time. rm -rf sites/udmi_site_model/out/devices/AHU-1/persistent_data.json @@ -67,9 +85,10 @@ while read -u 7 action test_name remainder; do false fi - bin/augment_metadata $SITE_PATH $DEVICE_ID - echo + echo bin/augment_metadata $SITE_PATH $DEVICE_ID $(< $AUGMENT_OPTS) + bin/augment_metadata $SITE_PATH $DEVICE_ID $(< $AUGMENT_OPTS) + echo bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name \> $SEQUENCER_OUT bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name 2>&1 | tee $SEQUENCER_OUT @@ -89,8 +108,11 @@ while read -u 7 action test_name remainder; do done 7< $GOLDEN_FILE -echo Comparing diff $RESULTS_OUT $GOLDEN_FILE -diff -u $RESULTS_OUT $GOLDEN_FILE +if [[ -z $TARGET_TEST ]]; then + echo Comparing diff $RESULTS_OUT $GOLDEN_FILE + diff -bu $RESULTS_OUT $GOLDEN_FILE -echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS -diff -u $SCHEMA_OUT $GOLDEN_SCHEMAS + echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS + uniq < $SCHEMA_OUT | sponge $SCHEMA_OUT + diff -bu $SCHEMA_OUT $GOLDEN_SCHEMAS +fi diff --git a/bin/test_sequencer b/bin/test_sequencer index e5354f579a..ea499639ae 100755 --- a/bin/test_sequencer +++ b/bin/test_sequencer @@ -10,6 +10,8 @@ debug_opts=-v seq_opts= alpha_mode= clean_cache= +pubber_opts= +suffix= if [[ $1 == 'clean' ]]; then shift @@ -23,13 +25,19 @@ if [[ $1 == 'alpha' ]]; then alpha_mode=y fi +if [[ $1 == 'nostate' ]]; then + shift + pubber_opts+=" noState" + suffix=_nostate +fi + if [[ $1 == 'full' ]]; then shift debug_opts= fi if [[ $# -lt 1 ]]; then - echo Usage: $0 [clean] [alpha] [full] PROJECT_ID [tests...] + echo Usage: $0 [clean] [alpha] [nostate] [full] PROJECT_ID [tests...] false fi @@ -66,13 +74,13 @@ bin/clone_model # Clone, and check that version is correct. bin/reset_config $site_path $project_id $device_id # Clean out the persistant data store to ensure a clean state each time. -rm -rf $site_path/out/devices/AHU-1/persistent_data.json +rm -rf $site_path/out/devices/${device_id}/persistent_data.json pubber/bin/build echo Writing pubber output to $PUBBER_OUT -echo bin/pubber $site_path $project_id $device_id $serial_no -bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT 2>&1 & +echo bin/pubber $site_path $project_id $device_id $serial_no $pubber_opts +bin/pubber $site_path $project_id $device_id $serial_no $pubber_opts > $PUBBER_OUT 2>&1 & WAITING=10 for i in `seq 1 $WAITING`; do @@ -90,7 +98,7 @@ if [[ $i -eq $WAITING ]]; then fi # Augment the device metadata to that tests know what to expect... -bin/augment_metadata $site_path $device_id +bin/augment_metadata $site_path $device_id $pubber_opts # Run in debug mode for fail-fast when things go wrong! echo bin/sequencer $debug_opts $seq_opts $site_path $project_id $device_id $serial_no $targets @@ -112,16 +120,21 @@ fi if [[ -n $clean_cache ]]; then echo Cleaning sequencer result cache... rm -rf validator/sequences -else +fi + +if [[ -n $alpha_mode ]]; then bin/sequencer_cache - echo Comparing diff out/sequencer.out etc/sequencer.out - diff -u out/sequencer.out etc/sequencer.out + echo Comparing diff out/sequencer.out etc/sequencer${suffix}.out + diff -u out/sequencer.out etc/sequencer${suffix}.out + + echo Comparing diff out/schema.out etc/schema${suffix}.out + diff -u out/schema.out etc/schema${suffix}.out fi -if [[ -n $alpha_mode ]]; then - echo Comparing diff out/schema.out etc/schema.out - diff -u out/schema.out etc/schema.out +if [[ -n $suffix ]]; then + echo Done with $suffix test_sequencer run. + exit 0 fi cp docs/specs/sequences/generated.md out/ # Save for test/comparison later diff --git a/common/src/main/java/com/google/daq/mqtt/util/ValidationException.java b/common/src/main/java/com/google/daq/mqtt/util/ValidationException.java new file mode 100644 index 0000000000..99e89b7b5b --- /dev/null +++ b/common/src/main/java/com/google/daq/mqtt/util/ValidationException.java @@ -0,0 +1,49 @@ +package com.google.daq.mqtt.util; + +import com.google.common.collect.ImmutableList; + +/** + * Exception during validation. + */ +public class ValidationException extends RuntimeException { + + private final ImmutableList causingExceptions; + + public ValidationException( + String message, ImmutableList causingExceptions) { + super(message); + this.causingExceptions = causingExceptions; + } + + /** + * Create a simple validation exception. + * + * @param message exception message + */ + public ValidationException(String message) { + this(message, ImmutableList.of()); + } + + /** + * Get the exception that started it all. + * + * @return The triggering exception. + */ + public ImmutableList getCausingExceptions() { + return causingExceptions; + } + + /** + * Get all the messages in this exception. + * + * @return List of the messages. + */ + public ImmutableList getAllMessages() { + ImmutableList.Builder messagesBuilder = + ImmutableList.builder().add(getMessage()); + for (ValidationException causingException : causingExceptions) { + messagesBuilder.addAll(causingException.getAllMessages()); + } + return messagesBuilder.build(); + } +} diff --git a/common/src/main/java/com/google/udmi/util/Common.java b/common/src/main/java/com/google/udmi/util/Common.java index f51245e86f..981cc71699 100644 --- a/common/src/main/java/com/google/udmi/util/Common.java +++ b/common/src/main/java/com/google/udmi/util/Common.java @@ -26,6 +26,8 @@ public abstract class Common { public static final String MESSAGE_KEY = "message"; public static final String TIMESTAMP_KEY = "timestamp"; public static final String VERSION_KEY = "version"; + public static final String UPGRADED_FROM = "upgraded_from"; + public static final String DOWNGRADED_FROM = "downgraded_from"; public static final String CLOUD_VERSION_KEY = "cloud_version"; public static final String UDMI_VERSION_KEY = "udmi_version"; public static final String SUBTYPE_PROPERTY_KEY = "subType"; diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 6f19c54f9c..34fbfeec8c 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -16,6 +16,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.hash.Hashing; +import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.ProperPrinter.OutputFormat; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; @@ -134,7 +135,16 @@ public static String encodeBase64(byte[] payload) { public static String friendlyStackTrace(Throwable e) { List messages = new ArrayList<>(); while (e != null) { - messages.add(ofNullable(e.getMessage()).orElse(e.getClass().getSimpleName())); + if (e instanceof ValidationException validationException) { + ImmutableList causes = validationException.getCausingExceptions(); + if (causes.isEmpty()) { + messages.add(validationException.getMessage()); + } else { + causes.forEach(exception -> messages.add(friendlyStackTrace(exception))); + } + } else { + messages.add(ofNullable(e.getMessage()).orElse(e.getClass().getSimpleName())); + } e = e.getCause(); } return CSV_JOINER.join(messages).replace('\n', ' '); @@ -186,12 +196,14 @@ private static String writeToPrettyString(Object data, OutputFormat indent) { * A custom generator can't be set on a base object mapper instance, so need to do it for each * invocation. */ - private static JsonGenerator getPrettyPrinterGenerator(OutputStream outputStream, OutputFormat indent) { + private static JsonGenerator getPrettyPrinterGenerator(OutputStream outputStream, + OutputFormat indent) { try { return OBJECT_MAPPER_STRICT .getFactory() .createGenerator(outputStream) - .setPrettyPrinter(indent == VERBOSE ? ProperPrinter.INDENT_PRINTER : ProperPrinter.NO_INDENT_PRINTER); + .setPrettyPrinter( + indent == VERBOSE ? ProperPrinter.INDENT_PRINTER : ProperPrinter.NO_INDENT_PRINTER); } catch (Exception e) { throw new RuntimeException("While creating pretty printer", e); } @@ -254,6 +266,14 @@ public static boolean isTrue(Object value) { return Boolean.TRUE.equals(value); } + public static boolean isGetTrue(Supplier target) { + try { + return Boolean.TRUE.equals(target.get()); + } catch (Exception e) { + return false; + } + } + public static void catchOrElse(Runnable action, Consumer caught) { try { action.run(); diff --git a/common/src/main/java/com/google/udmi/util/MessageDowngrader.java b/common/src/main/java/com/google/udmi/util/MessageDowngrader.java index 9414be7403..872a21ea39 100644 --- a/common/src/main/java/com/google/udmi/util/MessageDowngrader.java +++ b/common/src/main/java/com/google/udmi/util/MessageDowngrader.java @@ -1,11 +1,18 @@ package com.google.udmi.util; +import static com.google.udmi.util.Common.DOWNGRADED_FROM; import static com.google.udmi.util.Common.VERSION_KEY; +import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_RAW; +import static com.google.udmi.util.GeneralUtils.ifNotNullThen; +import static java.lang.Boolean.TRUE; +import static java.util.Objects.isNull; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; +import java.util.Map; +import udmi.schema.Envelope.SubType; /** * Downgrade a message to a previous UDMI schema version. @@ -15,30 +22,94 @@ public class MessageDowngrader { private static final TextNode LEGACY_VERSION = new TextNode("1"); private static final JsonNode LEGACY_REPLACEMENT = new IntNode(1); private final ObjectNode message; + private final String schema; private int major; private int minor; private int patch; /** - * Create message downgrader. + * Create message down-grader. * * @param schemaName schema to downgrade * @param messageJson message json */ public MessageDowngrader(String schemaName, JsonNode messageJson) { - if (!"config".equals(schemaName)) { - throw new IllegalArgumentException("Can only downgrade config messages"); - } + schema = schemaName; this.message = (ObjectNode) messageJson; } + /** + * Create message down-grader. + * + * @param schemaName schema to downgrade + * @param message message object + */ + public MessageDowngrader(String schemaName, Object message) { + this(schemaName, OBJECT_MAPPER_RAW.valueToTree(message)); + } + + static String convertVersion(JsonNode versionNode) { + if (versionNode == null) { + return "1"; + } + if (versionNode.isTextual()) { + return versionNode.asText(); + } + if (versionNode.isIntegralNumber()) { + return Integer.toString(versionNode.asInt()); + } + throw new IllegalStateException("Unrecognized version node " + versionNode.asText()); + } + /** * Downgrade a message to a target version. * - * @param versionNode target downgrade version (as a JsonNode) + * @param version target downgrade version (as a SchemaVersion) + * + * @return downgraded object */ - public void downgrade(JsonNode versionNode) { - final String version = convertVersion(versionNode); + public Map downgrade(SchemaVersion version) { + return downgrade(new TextNode(version.key())); + } + + /** + * Downgrade a message to a target version. + * + * @param targetVersion target downgrade version (as a JsonNode) + * + * @return downgraded object + */ + public Map downgrade(JsonNode targetVersion) { + return switch (schema) { + case "config" -> downgradeConfig(targetVersion); + case "state" -> downgradeState(targetVersion); + default -> + throw new IllegalArgumentException("Unknown downgrade schema " + schema); + }; + } + + private Map downgradeState(JsonNode targetVersion) { + final String version = convertVersion(targetVersion); + + ObjectNode system = (ObjectNode) message.get("system"); + if (version.equals(SchemaVersion.VERSION_1_4_0.key())) { + ifNotNullThen(system, map -> { + JsonNode operation = map.remove("operation"); + ifNotNullThen(operation, src -> map.set("operational", src.get("operational"))); + }); + } else { + throw new RuntimeException("Unknown target legacy version " + version); + } + return JsonUtil.asMap(message); + } + + private Map downgradeConfig(JsonNode targetVersion) { + downgradeConfigRaw(targetVersion); + return JsonUtil.asMap(message); + } + + private void downgradeConfigRaw(JsonNode targetVersion) { + final String version = convertVersion(targetVersion); String[] components = version.split("-", 2); String[] parts = components[0].split("\\.", 4); major = Integer.parseInt(parts[0]); @@ -62,24 +133,12 @@ public void downgrade(JsonNode versionNode) { } } - JsonNode useVersion = versionNode.equals(LEGACY_VERSION) ? LEGACY_REPLACEMENT : versionNode; + JsonNode useVersion = targetVersion.equals(LEGACY_VERSION) ? LEGACY_REPLACEMENT : targetVersion; + message.set(DOWNGRADED_FROM, message.get(VERSION_KEY)); message.set(VERSION_KEY, useVersion); } - private String convertVersion(JsonNode versionNode) { - if (versionNode == null) { - return "1"; - } - if (versionNode.isTextual()) { - return versionNode.asText(); - } - if (versionNode.isIntegralNumber()) { - return Integer.toString(versionNode.asInt()); - } - throw new IllegalStateException("Unrecognized version node " + versionNode.asText()); - } - private void downgradeLocalnet() { ObjectNode localnet = (ObjectNode) message.get("localnet"); if (localnet == null) { diff --git a/common/src/main/java/com/google/udmi/util/MessageUpgrader.java b/common/src/main/java/com/google/udmi/util/MessageUpgrader.java index a4f5b3c954..daff9e414b 100644 --- a/common/src/main/java/com/google/udmi/util/MessageUpgrader.java +++ b/common/src/main/java/com/google/udmi/util/MessageUpgrader.java @@ -1,8 +1,10 @@ package com.google.udmi.util; +import static com.google.udmi.util.Common.UPGRADED_FROM; import static com.google.udmi.util.Common.VERSION_KEY; import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_RAW; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; +import static com.google.udmi.util.MessageDowngrader.convertVersion; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; @@ -27,6 +29,7 @@ public class MessageUpgrader { private final JsonNode original; private final String schemaName; private final int major; + private final String originalVersion; private int patch; private int minor; @@ -42,18 +45,17 @@ public MessageUpgrader(String schemaName, JsonNode message) { this.original = message.deepCopy(); JsonNode version = message.get(VERSION_KEY); - String verStr = - version != null ? version.isNumber() ? Integer.toString(version.asInt()) : version.asText() - : "1"; - String[] components = verStr.split("-", 2); + String versionString = convertVersion(version); + String[] components = versionString.split("-", 2); String[] parts = components[0].split("\\.", 4); major = Integer.parseInt(parts[0]); minor = parts.length >= 2 ? Integer.parseInt(parts[1]) : -1; patch = parts.length >= 3 ? Integer.parseInt(parts[2]) : -1; if (parts.length >= 4) { - throw new IllegalArgumentException("Unexpected version " + verStr); + throw new IllegalArgumentException("Unexpected src version " + versionString); } + originalVersion = versionString; } public MessageUpgrader(String schemaName, Object originalMessage) { @@ -113,7 +115,8 @@ public Object upgrade(boolean forceUpgrade) { patch = 1; } - if (upgraded && message.has(VERSION_KEY)) { + if (upgraded && message.get(VERSION_KEY) != null) { + message.put(UPGRADED_FROM, originalVersion); message.put(VERSION_KEY, String.format(TARGET_FORMAT, major, minor, patch)); } @@ -192,7 +195,7 @@ private void upgradeStatuses(ObjectNode system) { private void upgradeFirmware(ObjectNode system) { JsonNode firmware = system.remove("firmware"); if (firmware != null) { - JsonNode version = ((ObjectNode) firmware).remove(VERSION_KEY); + JsonNode version = ((ObjectNode) firmware).remove("version"); if (version != null && !system.has("software")) { ObjectNode softwareNode = new ObjectNode(NODE_FACTORY); softwareNode.set("firmware", sanitizeFirmwareVersion(version)); diff --git a/common/src/main/java/com/google/udmi/util/SchemaVersion.java b/common/src/main/java/com/google/udmi/util/SchemaVersion.java new file mode 100644 index 0000000000..098082887c --- /dev/null +++ b/common/src/main/java/com/google/udmi/util/SchemaVersion.java @@ -0,0 +1,51 @@ +package com.google.udmi.util; + +import static com.google.common.base.Preconditions.checkState; + +import java.util.AbstractMap.SimpleEntry; +import java.util.HashMap; +import java.util.Map; + +/** + * Formalized enums of the UDMI schema version. + */ +public enum SchemaVersion { + VERSION_1_4_2("1.4.2", 104020), + VERSION_1_4_0("1.4.0", 104000); + + public static final SchemaVersion CURRENT; + private static final Map CONSTANTS = new HashMap<>(); + + static { + SimpleEntry max = new SimpleEntry<>(null, 0); + for (SchemaVersion c : values()) { + CONSTANTS.put(c.key, c); + if (c.value > max.getValue()) { + max = new SimpleEntry<>(c, c.value); + } + } + CURRENT = max.getKey(); + } + + private final String key; + private final int value; + + SchemaVersion(String key, int value) { + this.key = key; + this.value = value; + } + + public static SchemaVersion fromKey(String key) { + checkState(CONSTANTS.containsKey(key), "unrecognized schema version " + key); + return CONSTANTS.get(key); + } + + public String key() { + return key; + } + + public int value() { + return value; + } + +} diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index 4c1014e9d8..c11a438f10 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -1,13 +1,11 @@ =========== pointset_request_extraneous =========== pointset_remove_point =========== feature_enumeration -=========== broken_config =========== valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5 All schema validations passed RESULT pass schemas event_system_alpha ALPHA 5 All schema validations passed RESULT pass schemas state_update_alpha ALPHA 5 All schema validations passed =========== writeback_success -=========== writeback_success =========== pointset_sample_rate =========== system_mode_restart =========== broken_config @@ -15,3 +13,6 @@ RESULT pass schemas state_update_alpha ALPHA 5 All schema validations passed RESULT fail schemas event_pointset_stable STABLE 5 Schema violations found RESULT pass schemas event_system_stable STABLE 5 All schema validations passed RESULT pass schemas state_update_stable STABLE 5 All schema validations passed +=========== valid_serial_no +=========== system_last_update +=========== system_min_loglevel diff --git a/etc/schema_nostate.out b/etc/schema_nostate.out new file mode 100644 index 0000000000..7e1bff3dcb --- /dev/null +++ b/etc/schema_nostate.out @@ -0,0 +1,2 @@ +RESULT pass schemas event_pointset_stable STABLE 5 All schema validations passed +RESULT pass schemas event_system_stable STABLE 5 All schema validations passed diff --git a/etc/sequencer_nostate.out b/etc/sequencer_nostate.out new file mode 100644 index 0000000000..1e8b993ab7 --- /dev/null +++ b/etc/sequencer_nostate.out @@ -0,0 +1,35 @@ +RESULT skip discovery.scan periodic_scan ALPHA 5 State testing disabled +RESULT skip discovery.scan single_scan ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_connection_bad_hash ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_connection_error ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_connection_retry ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_connection_success_alternate ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_connection_success_reconnect ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_failure_and_restart ALPHA 5 State testing disabled +RESULT skip endpoint endpoint_redirect_and_restart ALPHA 5 State testing disabled +RESULT skip enumeration empty_enumeration PREVIEW 5 State testing disabled +RESULT skip enumeration multi_enumeration ALPHA 5 State testing disabled +RESULT skip enumeration.families family_enumeration ALPHA 5 State testing disabled +RESULT skip enumeration.features feature_enumeration PREVIEW 5 State testing disabled +RESULT skip enumeration.pointset pointset_enumeration ALPHA 5 State testing disabled +RESULT skip pointset pointset_publish BETA 5 State testing disabled +RESULT pass pointset pointset_publish_interval BETA 5 Sequence complete +RESULT skip pointset pointset_remove_point ALPHA 5 State testing disabled +RESULT skip pointset pointset_request_extraneous ALPHA 5 State testing disabled +RESULT skip pointset pointset_sample_rate BETA 5 State testing disabled +RESULT skip system broken_config ALPHA 5 State testing disabled +RESULT skip system config_logging BETA 5 State testing disabled +RESULT skip system device_config_acked BETA 5 State testing disabled +RESULT skip system extra_config BETA 5 State testing disabled +RESULT skip system state_make_model BETA 5 State testing disabled +RESULT skip system state_software BETA 5 State testing disabled +RESULT fail system system_last_update STABLE 5 timeout waiting for state last_config matches config timestamp +RESULT pass system system_min_loglevel ALPHA 5 Sequence complete +RESULT skip system valid_serial_no ALPHA 5 State testing disabled +RESULT skip system.mode system_mode_restart ALPHA 5 State testing disabled +RESULT skip unknown family_ether_addr ALPHA 5 State testing disabled +RESULT skip unknown family_ipv4_addr ALPHA 5 State testing disabled +RESULT skip unknown family_ipv6_addr ALPHA 5 State testing disabled +RESULT skip writeback writeback_failure ALPHA 5 State testing disabled +RESULT skip writeback writeback_invalid ALPHA 5 State testing disabled +RESULT skip writeback writeback_success ALPHA 5 State testing disabled diff --git a/etc/test_itemized.out b/etc/test_itemized.out index 947375f803..60e39880fa 100644 --- a/etc/test_itemized.out +++ b/etc/test_itemized.out @@ -1,11 +1,20 @@ TEST pointset_request_extraneous extraPoint=llama RESULT fail pointset pointset_request_extraneous ALPHA 5 timeout waiting for pointset event contains correct points with present_value TEST pointset_remove_point missingPoint=filter_alarm_pressure_status RESULT fail pointset pointset_remove_point ALPHA 5 timeout waiting for pointset state reports same points as defined in config +TEST pointset_remove_point noState RESULT fail pointset pointset_remove_point ALPHA 5 timeout waiting for no interesting system status TEST feature_enumeration featureEnableSwap RESULT fail enumeration.features feature_enumeration PREVIEW 5 Failed check that feature enumeration matches metadata; missing { enumeration }, extra { unknown } -TEST broken_config configStateDelay RESULT pass system broken_config ALPHA 5 Sequence complete TEST valid_serial_no noLastStart RESULT pass system valid_serial_no ALPHA 5 Sequence complete TEST writeback_success noWriteback RESULT fail writeback writeback_success ALPHA 5 timeout waiting for point filter_differential_pressure_setpoint to have value_state applied TEST writeback_success noPointState RESULT fail writeback writeback_success ALPHA 5 timeout waiting for point filter_differential_pressure_setpoint to have value_state applied TEST pointset_sample_rate fixedSampleRate=10 RESULT fail pointset pointset_sample_rate BETA 5 Failed check that time period between successive pointset events is between 1 and 5 seconds TEST system_mode_restart noPersist RESULT fail system.mode system_mode_restart ALPHA 5 Failed check that restart count increased by one TEST broken_config barfConfig RESULT fail system broken_config ALPHA 5 timeout waiting for has interesting system status +TEST broken_config configStateDelay RESULT pass system broken_config ALPHA 5 Sequence complete +TEST broken_config noState RESULT fail system broken_config ALPHA 5 timeout waiting for initial state synchronized TEST system_last_update extraField=fnooz RESULT pass system system_last_update STABLE 5 Sequence complete + +# Test with various device metadata.json options +WITH noState +TEST valid_serial_no RESULT skip system valid_serial_no ALPHA 5 State testing disabled +TEST system_last_update noState RESULT fail system system_last_update STABLE 5 timeout waiting for state last_config matches config timestamp +TEST system_min_loglevel RESULT fail system system_min_loglevel ALPHA 5 Received state update with no-state device +TEST system_min_loglevel noState RESULT pass system system_min_loglevel ALPHA 5 Sequence complete diff --git a/gencode/docs/config.html b/gencode/docs/config.html index 96ffb04cef..0087d6580d 100644 --- a/gencode/docs/config.html +++ b/gencode/docs/config.html @@ -97,6 +97,40 @@

+ + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + downgraded_from
Type: string
+

Original version of schema pre-downgrade

+ + + + + + +
diff --git a/gencode/docs/configuration_pubber.html b/gencode/docs/configuration_pubber.html index bab95160ac..32a9d54c5b 100644 --- a/gencode/docs/configuration_pubber.html +++ b/gencode/docs/configuration_pubber.html @@ -1783,6 +1783,46 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + options + + + + noState
Type: boolean
+ + + + + + +
diff --git a/gencode/docs/event.html b/gencode/docs/event.html index 7b7655cc4d..049b47bc94 100644 --- a/gencode/docs/event.html +++ b/gencode/docs/event.html @@ -143,6 +143,47 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + system + + + + upgraded_from
Type: string
+

Original version of schema pre-upgrade

+ + + + + + +
@@ -3002,6 +3043,47 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + pointset + + + + upgraded_from
Type: string
+

Original version of schema pre-upgrade

+ + + + + + +
diff --git a/gencode/docs/event_pointset.html b/gencode/docs/event_pointset.html index 41ecae0d6f..71fc8aa885 100644 --- a/gencode/docs/event_pointset.html +++ b/gencode/docs/event_pointset.html @@ -97,6 +97,40 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + upgraded_from
Type: string
+

Original version of schema pre-upgrade

+ + + + + + +
diff --git a/gencode/docs/event_system.html b/gencode/docs/event_system.html index bb88f124f4..d53742dbf4 100644 --- a/gencode/docs/event_system.html +++ b/gencode/docs/event_system.html @@ -97,6 +97,40 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + upgraded_from
Type: string
+

Original version of schema pre-upgrade

+ + + + + + +
diff --git a/gencode/docs/metadata.html b/gencode/docs/metadata.html index 69d22cfa1d..848b38765a 100644 --- a/gencode/docs/metadata.html +++ b/gencode/docs/metadata.html @@ -89,7 +89,41 @@

/> version

Type: string
-

Version of the UDMI schema

+

Version of the UDMI schema for this file

+ + + + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + upgraded_from
Type: string
+

Original version of the UDMI schema for this file

 @@ -165,6 +199,40 @@

+

+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + device_version
Type: string
+

Version of schema supported by the device

+ + + + + + +
@@ -2727,6 +2795,46 @@

+
+
+
+

+ +

+
+ +
+
+ +
root + + + + testing + + + + nostate
Type: boolean
+ + + + + + + +
+
+
+
diff --git a/gencode/docs/monitoring.html b/gencode/docs/monitoring.html index a0dc397870..57abdfbfad 100644 --- a/gencode/docs/monitoring.html +++ b/gencode/docs/monitoring.html @@ -196,6 +196,54 @@

+

+
+
+

+
+
+
+

+ +

+
+ +
+
+ +
root + + + + metric + + + + event_system + + + + upgraded_from
Type: string
+

Original version of schema pre-upgrade

+ + + + + + +
diff --git a/gencode/docs/state.html b/gencode/docs/state.html index 0d40cfccf3..d65663f47d 100644 --- a/gencode/docs/state.html +++ b/gencode/docs/state.html @@ -97,6 +97,40 @@

+

+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + upgraded_from
Type: string
+

Original version of schema pre-upgrade

+ + + + + + +
diff --git a/gencode/java/udmi/schema/Config.java b/gencode/java/udmi/schema/Config.java index f2031e8298..17e15c331d 100644 --- a/gencode/java/udmi/schema/Config.java +++ b/gencode/java/udmi/schema/Config.java @@ -19,6 +19,7 @@ @JsonPropertyOrder({ "timestamp", "version", + "downgraded_from", "system", "gateway", "discovery", @@ -45,6 +46,13 @@ public class Config { @JsonProperty("version") @JsonPropertyDescription("Version of the UDMI schema") public String version; + /** + * Original version of schema pre-downgrade + * + */ + @JsonProperty("downgraded_from") + @JsonPropertyDescription("Original version of schema pre-downgrade") + public String downgraded_from; /** * System Config *

@@ -104,6 +112,7 @@ public int hashCode() { int result = 1; result = ((result* 31)+((this.system == null)? 0 :this.system.hashCode())); result = ((result* 31)+((this.discovery == null)? 0 :this.discovery.hashCode())); + result = ((result* 31)+((this.downgraded_from == null)? 0 :this.downgraded_from.hashCode())); result = ((result* 31)+((this.pointset == null)? 0 :this.pointset.hashCode())); result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); result = ((result* 31)+((this.blobset == null)? 0 :this.blobset.hashCode())); @@ -122,7 +131,7 @@ public boolean equals(Object other) { return false; } Config rhs = ((Config) other); - return (((((((((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system)))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.blobset == rhs.blobset)||((this.blobset!= null)&&this.blobset.equals(rhs.blobset))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); + return ((((((((((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system)))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.downgraded_from == rhs.downgraded_from)||((this.downgraded_from!= null)&&this.downgraded_from.equals(rhs.downgraded_from))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.blobset == rhs.blobset)||((this.blobset!= null)&&this.blobset.equals(rhs.blobset))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); } } diff --git a/gencode/java/udmi/schema/Metadata.java b/gencode/java/udmi/schema/Metadata.java index b09953cef5..48cd5c05f2 100644 --- a/gencode/java/udmi/schema/Metadata.java +++ b/gencode/java/udmi/schema/Metadata.java @@ -20,8 +20,10 @@ @JsonPropertyOrder({ "timestamp", "version", + "upgraded_from", "description", "hash", + "device_version", "cloud", "system", "gateway", @@ -43,13 +45,20 @@ public class Metadata { @JsonPropertyDescription("RFC 3339 timestamp the message was generated") public Date timestamp; /** - * Version of the UDMI schema + * Version of the UDMI schema for this file * (Required) * */ @JsonProperty("version") - @JsonPropertyDescription("Version of the UDMI schema") + @JsonPropertyDescription("Version of the UDMI schema for this file") public java.lang.String version; + /** + * Original version of the UDMI schema for this file + * + */ + @JsonProperty("upgraded_from") + @JsonPropertyDescription("Original version of the UDMI schema for this file") + public java.lang.String upgraded_from; /** * Generic human-readable text describing the device * @@ -64,6 +73,13 @@ public class Metadata { @JsonProperty("hash") @JsonPropertyDescription("Automatically generated field that contains the hash of file contents.") public java.lang.String hash; + /** + * Version of schema supported by the device + * + */ + @JsonProperty("device_version") + @JsonPropertyDescription("Version of schema supported by the device") + public java.lang.String device_version; /** * Cloud Model *

@@ -141,6 +157,7 @@ public class Metadata { @Override public int hashCode() { int result = 1; + result = ((result* 31)+((this.device_version == null)? 0 :this.device_version.hashCode())); result = ((result* 31)+((this.testing == null)? 0 :this.testing.hashCode())); result = ((result* 31)+((this.description == null)? 0 :this.description.hashCode())); result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); @@ -148,6 +165,7 @@ public int hashCode() { result = ((result* 31)+((this.features == null)? 0 :this.features.hashCode())); result = ((result* 31)+((this.system == null)? 0 :this.system.hashCode())); result = ((result* 31)+((this.discovery == null)? 0 :this.discovery.hashCode())); + result = ((result* 31)+((this.upgraded_from == null)? 0 :this.upgraded_from.hashCode())); result = ((result* 31)+((this.pointset == null)? 0 :this.pointset.hashCode())); result = ((result* 31)+((this.hash == null)? 0 :this.hash.hashCode())); result = ((result* 31)+((this.gateway == null)? 0 :this.gateway.hashCode())); @@ -165,7 +183,7 @@ public boolean equals(Object other) { return false; } Metadata rhs = ((Metadata) other); - return (((((((((((((this.testing == rhs.testing)||((this.testing!= null)&&this.testing.equals(rhs.testing)))&&((this.description == rhs.description)||((this.description!= null)&&this.description.equals(rhs.description))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.cloud == rhs.cloud)||((this.cloud!= null)&&this.cloud.equals(rhs.cloud))))&&((this.features == rhs.features)||((this.features!= null)&&this.features.equals(rhs.features))))&&((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system))))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.hash == rhs.hash)||((this.hash!= null)&&this.hash.equals(rhs.hash))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); + return (((((((((((((((this.device_version == rhs.device_version)||((this.device_version!= null)&&this.device_version.equals(rhs.device_version)))&&((this.testing == rhs.testing)||((this.testing!= null)&&this.testing.equals(rhs.testing))))&&((this.description == rhs.description)||((this.description!= null)&&this.description.equals(rhs.description))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.cloud == rhs.cloud)||((this.cloud!= null)&&this.cloud.equals(rhs.cloud))))&&((this.features == rhs.features)||((this.features!= null)&&this.features.equals(rhs.features))))&&((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system))))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.upgraded_from == rhs.upgraded_from)||((this.upgraded_from!= null)&&this.upgraded_from.equals(rhs.upgraded_from))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.hash == rhs.hash)||((this.hash!= null)&&this.hash.equals(rhs.hash))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); } } diff --git a/gencode/java/udmi/schema/PointsetEvent.java b/gencode/java/udmi/schema/PointsetEvent.java index 985ab1aef9..bf5b7bb418 100644 --- a/gencode/java/udmi/schema/PointsetEvent.java +++ b/gencode/java/udmi/schema/PointsetEvent.java @@ -20,6 +20,7 @@ @JsonPropertyOrder({ "timestamp", "version", + "upgraded_from", "partial_update", "points" }) @@ -42,6 +43,13 @@ public class PointsetEvent { @JsonProperty("version") @JsonPropertyDescription("Version of the UDMI schema") public java.lang.String version; + /** + * Original version of schema pre-upgrade + * + */ + @JsonProperty("upgraded_from") + @JsonPropertyDescription("Original version of schema pre-upgrade") + public java.lang.String upgraded_from; /** * Indicates if this is a partial update (only some points may be included) * @@ -62,6 +70,7 @@ public class PointsetEvent { public int hashCode() { int result = 1; result = ((result* 31)+((this.partial_update == null)? 0 :this.partial_update.hashCode())); + result = ((result* 31)+((this.upgraded_from == null)? 0 :this.upgraded_from.hashCode())); result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); result = ((result* 31)+((this.timestamp == null)? 0 :this.timestamp.hashCode())); result = ((result* 31)+((this.points == null)? 0 :this.points.hashCode())); @@ -77,7 +86,7 @@ public boolean equals(Object other) { return false; } PointsetEvent rhs = ((PointsetEvent) other); - return (((((this.partial_update == rhs.partial_update)||((this.partial_update!= null)&&this.partial_update.equals(rhs.partial_update)))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp))))&&((this.points == rhs.points)||((this.points!= null)&&this.points.equals(rhs.points)))); + return ((((((this.partial_update == rhs.partial_update)||((this.partial_update!= null)&&this.partial_update.equals(rhs.partial_update)))&&((this.upgraded_from == rhs.upgraded_from)||((this.upgraded_from!= null)&&this.upgraded_from.equals(rhs.upgraded_from))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp))))&&((this.points == rhs.points)||((this.points!= null)&&this.points.equals(rhs.points)))); } } diff --git a/gencode/java/udmi/schema/PubberOptions.java b/gencode/java/udmi/schema/PubberOptions.java index d484574ca9..5f9f45bb01 100644 --- a/gencode/java/udmi/schema/PubberOptions.java +++ b/gencode/java/udmi/schema/PubberOptions.java @@ -31,6 +31,7 @@ "redirectRegistry", "smokeCheck", "noPointState", + "noState", "featureEnableSwap", "disableWriteback", "noWriteback", @@ -70,6 +71,8 @@ public class PubberOptions { public Boolean smokeCheck; @JsonProperty("noPointState") public Boolean noPointState; + @JsonProperty("noState") + public Boolean noState; @JsonProperty("featureEnableSwap") public Boolean featureEnableSwap; /** @@ -108,6 +111,7 @@ public int hashCode() { result = ((result* 31)+((this.noWriteback == null)? 0 :this.noWriteback.hashCode())); result = ((result* 31)+((this.fixedSampleRate == null)? 0 :this.fixedSampleRate.hashCode())); result = ((result* 31)+((this.noLastStart == null)? 0 :this.noLastStart.hashCode())); + result = ((result* 31)+((this.noState == null)? 0 :this.noState.hashCode())); result = ((result* 31)+((this.featureEnableSwap == null)? 0 :this.featureEnableSwap.hashCode())); result = ((result* 31)+((this.missingPoint == null)? 0 :this.missingPoint.hashCode())); result = ((result* 31)+((this.noConfigAck == null)? 0 :this.noConfigAck.hashCode())); @@ -126,7 +130,7 @@ public boolean equals(Object other) { return false; } PubberOptions rhs = ((PubberOptions) other); - return (((((((((((((((((((((this.noPersist == rhs.noPersist)||((this.noPersist!= null)&&this.noPersist.equals(rhs.noPersist)))&&((this.smokeCheck == rhs.smokeCheck)||((this.smokeCheck!= null)&&this.smokeCheck.equals(rhs.smokeCheck))))&&((this.redirectRegistry == rhs.redirectRegistry)||((this.redirectRegistry!= null)&&this.redirectRegistry.equals(rhs.redirectRegistry))))&&((this.noPointState == rhs.noPointState)||((this.noPointState!= null)&&this.noPointState.equals(rhs.noPointState))))&&((this.disableWriteback == rhs.disableWriteback)||((this.disableWriteback!= null)&&this.disableWriteback.equals(rhs.disableWriteback))))&&((this.noHardware == rhs.noHardware)||((this.noHardware!= null)&&this.noHardware.equals(rhs.noHardware))))&&((this.barfConfig == rhs.barfConfig)||((this.barfConfig!= null)&&this.barfConfig.equals(rhs.barfConfig))))&&((this.extraField == rhs.extraField)||((this.extraField!= null)&&this.extraField.equals(rhs.extraField))))&&((this.messageTrace == rhs.messageTrace)||((this.messageTrace!= null)&&this.messageTrace.equals(rhs.messageTrace))))&&((this.emptyMissing == rhs.emptyMissing)||((this.emptyMissing!= null)&&this.emptyMissing.equals(rhs.emptyMissing))))&&((this.softwareFirmwareValue == rhs.softwareFirmwareValue)||((this.softwareFirmwareValue!= null)&&this.softwareFirmwareValue.equals(rhs.softwareFirmwareValue))))&&((this.noWriteback == rhs.noWriteback)||((this.noWriteback!= null)&&this.noWriteback.equals(rhs.noWriteback))))&&((this.fixedSampleRate == rhs.fixedSampleRate)||((this.fixedSampleRate!= null)&&this.fixedSampleRate.equals(rhs.fixedSampleRate))))&&((this.noLastStart == rhs.noLastStart)||((this.noLastStart!= null)&&this.noLastStart.equals(rhs.noLastStart))))&&((this.featureEnableSwap == rhs.featureEnableSwap)||((this.featureEnableSwap!= null)&&this.featureEnableSwap.equals(rhs.featureEnableSwap))))&&((this.missingPoint == rhs.missingPoint)||((this.missingPoint!= null)&&this.missingPoint.equals(rhs.missingPoint))))&&((this.noConfigAck == rhs.noConfigAck)||((this.noConfigAck!= null)&&this.noConfigAck.equals(rhs.noConfigAck))))&&((this.extraPoint == rhs.extraPoint)||((this.extraPoint!= null)&&this.extraPoint.equals(rhs.extraPoint))))&&((this.fixedLogLevel == rhs.fixedLogLevel)||((this.fixedLogLevel!= null)&&this.fixedLogLevel.equals(rhs.fixedLogLevel))))&&((this.configStateDelay == rhs.configStateDelay)||((this.configStateDelay!= null)&&this.configStateDelay.equals(rhs.configStateDelay)))); + return ((((((((((((((((((((((this.noPersist == rhs.noPersist)||((this.noPersist!= null)&&this.noPersist.equals(rhs.noPersist)))&&((this.smokeCheck == rhs.smokeCheck)||((this.smokeCheck!= null)&&this.smokeCheck.equals(rhs.smokeCheck))))&&((this.redirectRegistry == rhs.redirectRegistry)||((this.redirectRegistry!= null)&&this.redirectRegistry.equals(rhs.redirectRegistry))))&&((this.noPointState == rhs.noPointState)||((this.noPointState!= null)&&this.noPointState.equals(rhs.noPointState))))&&((this.disableWriteback == rhs.disableWriteback)||((this.disableWriteback!= null)&&this.disableWriteback.equals(rhs.disableWriteback))))&&((this.noHardware == rhs.noHardware)||((this.noHardware!= null)&&this.noHardware.equals(rhs.noHardware))))&&((this.barfConfig == rhs.barfConfig)||((this.barfConfig!= null)&&this.barfConfig.equals(rhs.barfConfig))))&&((this.extraField == rhs.extraField)||((this.extraField!= null)&&this.extraField.equals(rhs.extraField))))&&((this.messageTrace == rhs.messageTrace)||((this.messageTrace!= null)&&this.messageTrace.equals(rhs.messageTrace))))&&((this.emptyMissing == rhs.emptyMissing)||((this.emptyMissing!= null)&&this.emptyMissing.equals(rhs.emptyMissing))))&&((this.softwareFirmwareValue == rhs.softwareFirmwareValue)||((this.softwareFirmwareValue!= null)&&this.softwareFirmwareValue.equals(rhs.softwareFirmwareValue))))&&((this.noWriteback == rhs.noWriteback)||((this.noWriteback!= null)&&this.noWriteback.equals(rhs.noWriteback))))&&((this.fixedSampleRate == rhs.fixedSampleRate)||((this.fixedSampleRate!= null)&&this.fixedSampleRate.equals(rhs.fixedSampleRate))))&&((this.noLastStart == rhs.noLastStart)||((this.noLastStart!= null)&&this.noLastStart.equals(rhs.noLastStart))))&&((this.noState == rhs.noState)||((this.noState!= null)&&this.noState.equals(rhs.noState))))&&((this.featureEnableSwap == rhs.featureEnableSwap)||((this.featureEnableSwap!= null)&&this.featureEnableSwap.equals(rhs.featureEnableSwap))))&&((this.missingPoint == rhs.missingPoint)||((this.missingPoint!= null)&&this.missingPoint.equals(rhs.missingPoint))))&&((this.noConfigAck == rhs.noConfigAck)||((this.noConfigAck!= null)&&this.noConfigAck.equals(rhs.noConfigAck))))&&((this.extraPoint == rhs.extraPoint)||((this.extraPoint!= null)&&this.extraPoint.equals(rhs.extraPoint))))&&((this.fixedLogLevel == rhs.fixedLogLevel)||((this.fixedLogLevel!= null)&&this.fixedLogLevel.equals(rhs.fixedLogLevel))))&&((this.configStateDelay == rhs.configStateDelay)||((this.configStateDelay!= null)&&this.configStateDelay.equals(rhs.configStateDelay)))); } } diff --git a/gencode/java/udmi/schema/State.java b/gencode/java/udmi/schema/State.java index bb1dfe7185..966d8d6bdc 100644 --- a/gencode/java/udmi/schema/State.java +++ b/gencode/java/udmi/schema/State.java @@ -19,6 +19,7 @@ @JsonPropertyOrder({ "timestamp", "version", + "upgraded_from", "system", "gateway", "discovery", @@ -45,6 +46,13 @@ public class State { @JsonProperty("version") @JsonPropertyDescription("Version of the UDMI schema") public String version; + /** + * Original version of schema pre-upgrade + * + */ + @JsonProperty("upgraded_from") + @JsonPropertyDescription("Original version of schema pre-upgrade") + public String upgraded_from; /** * System State *

@@ -104,6 +112,7 @@ public int hashCode() { int result = 1; result = ((result* 31)+((this.system == null)? 0 :this.system.hashCode())); result = ((result* 31)+((this.discovery == null)? 0 :this.discovery.hashCode())); + result = ((result* 31)+((this.upgraded_from == null)? 0 :this.upgraded_from.hashCode())); result = ((result* 31)+((this.pointset == null)? 0 :this.pointset.hashCode())); result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); result = ((result* 31)+((this.blobset == null)? 0 :this.blobset.hashCode())); @@ -122,7 +131,7 @@ public boolean equals(Object other) { return false; } State rhs = ((State) other); - return (((((((((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system)))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.blobset == rhs.blobset)||((this.blobset!= null)&&this.blobset.equals(rhs.blobset))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); + return ((((((((((this.system == rhs.system)||((this.system!= null)&&this.system.equals(rhs.system)))&&((this.discovery == rhs.discovery)||((this.discovery!= null)&&this.discovery.equals(rhs.discovery))))&&((this.upgraded_from == rhs.upgraded_from)||((this.upgraded_from!= null)&&this.upgraded_from.equals(rhs.upgraded_from))))&&((this.pointset == rhs.pointset)||((this.pointset!= null)&&this.pointset.equals(rhs.pointset))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.blobset == rhs.blobset)||((this.blobset!= null)&&this.blobset.equals(rhs.blobset))))&&((this.gateway == rhs.gateway)||((this.gateway!= null)&&this.gateway.equals(rhs.gateway))))&&((this.localnet == rhs.localnet)||((this.localnet!= null)&&this.localnet.equals(rhs.localnet))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); } } diff --git a/gencode/java/udmi/schema/SystemEvent.java b/gencode/java/udmi/schema/SystemEvent.java index 3cad1d085e..847ff2d798 100644 --- a/gencode/java/udmi/schema/SystemEvent.java +++ b/gencode/java/udmi/schema/SystemEvent.java @@ -21,6 +21,7 @@ @JsonPropertyOrder({ "timestamp", "version", + "upgraded_from", "last_config", "logentries", "event_count", @@ -45,6 +46,13 @@ public class SystemEvent { @JsonProperty("version") @JsonPropertyDescription("Version of the UDMI schema") public String version; + /** + * Original version of schema pre-upgrade + * + */ + @JsonProperty("upgraded_from") + @JsonPropertyDescription("Original version of schema pre-upgrade") + public String upgraded_from; /** * Last config received * @@ -68,6 +76,7 @@ public class SystemEvent { public int hashCode() { int result = 1; result = ((result* 31)+((this.event_count == null)? 0 :this.event_count.hashCode())); + result = ((result* 31)+((this.upgraded_from == null)? 0 :this.upgraded_from.hashCode())); result = ((result* 31)+((this.metrics == null)? 0 :this.metrics.hashCode())); result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); result = ((result* 31)+((this.timestamp == null)? 0 :this.timestamp.hashCode())); @@ -85,7 +94,7 @@ public boolean equals(Object other) { return false; } SystemEvent rhs = ((SystemEvent) other); - return (((((((this.event_count == rhs.event_count)||((this.event_count!= null)&&this.event_count.equals(rhs.event_count)))&&((this.metrics == rhs.metrics)||((this.metrics!= null)&&this.metrics.equals(rhs.metrics))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp))))&&((this.last_config == rhs.last_config)||((this.last_config!= null)&&this.last_config.equals(rhs.last_config))))&&((this.logentries == rhs.logentries)||((this.logentries!= null)&&this.logentries.equals(rhs.logentries)))); + return ((((((((this.event_count == rhs.event_count)||((this.event_count!= null)&&this.event_count.equals(rhs.event_count)))&&((this.upgraded_from == rhs.upgraded_from)||((this.upgraded_from!= null)&&this.upgraded_from.equals(rhs.upgraded_from))))&&((this.metrics == rhs.metrics)||((this.metrics!= null)&&this.metrics.equals(rhs.metrics))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp))))&&((this.last_config == rhs.last_config)||((this.last_config!= null)&&this.last_config.equals(rhs.last_config))))&&((this.logentries == rhs.logentries)||((this.logentries!= null)&&this.logentries.equals(rhs.logentries)))); } } diff --git a/gencode/java/udmi/schema/TestingModel.java b/gencode/java/udmi/schema/TestingModel.java index cc857d9423..f2f09e6565 100644 --- a/gencode/java/udmi/schema/TestingModel.java +++ b/gencode/java/udmi/schema/TestingModel.java @@ -16,11 +16,14 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ + "nostate", "targets" }) @Generated("jsonschema2pojo") public class TestingModel { + @JsonProperty("nostate") + public Boolean nostate; @JsonProperty("targets") public HashMap targets; @@ -28,6 +31,7 @@ public class TestingModel { public int hashCode() { int result = 1; result = ((result* 31)+((this.targets == null)? 0 :this.targets.hashCode())); + result = ((result* 31)+((this.nostate == null)? 0 :this.nostate.hashCode())); return result; } @@ -40,7 +44,7 @@ public boolean equals(Object other) { return false; } TestingModel rhs = ((TestingModel) other); - return ((this.targets == rhs.targets)||((this.targets!= null)&&this.targets.equals(rhs.targets))); + return (((this.targets == rhs.targets)||((this.targets!= null)&&this.targets.equals(rhs.targets)))&&((this.nostate == rhs.nostate)||((this.nostate!= null)&&this.nostate.equals(rhs.nostate)))); } } diff --git a/gencode/python/udmi/schema/config.py b/gencode/python/udmi/schema/config.py index 939b0a72c8..517103217b 100644 --- a/gencode/python/udmi/schema/config.py +++ b/gencode/python/udmi/schema/config.py @@ -13,6 +13,7 @@ class Config: def __init__(self): self.timestamp = None self.version = None + self.downgraded_from = None self.system = None self.gateway = None self.discovery = None @@ -27,6 +28,7 @@ def from_dict(source): result = Config() result.timestamp = source.get('timestamp') result.version = source.get('version') + result.downgraded_from = source.get('downgraded_from') result.system = SystemConfig.from_dict(source.get('system')) result.gateway = GatewayConfig.from_dict(source.get('gateway')) result.discovery = DiscoveryConfig.from_dict(source.get('discovery')) @@ -57,6 +59,8 @@ def to_dict(self): result['timestamp'] = self.timestamp # 5 if self.version: result['version'] = self.version # 5 + if self.downgraded_from: + result['downgraded_from'] = self.downgraded_from # 5 if self.system: result['system'] = self.system.to_dict() # 4 if self.gateway: diff --git a/gencode/python/udmi/schema/event_pointset.py b/gencode/python/udmi/schema/event_pointset.py index 0fc2bc094e..d76e235b17 100644 --- a/gencode/python/udmi/schema/event_pointset.py +++ b/gencode/python/udmi/schema/event_pointset.py @@ -8,6 +8,7 @@ class PointsetEvent: def __init__(self): self.timestamp = None self.version = None + self.upgraded_from = None self.partial_update = None self.points = None @@ -18,6 +19,7 @@ def from_dict(source): result = PointsetEvent() result.timestamp = source.get('timestamp') result.version = source.get('version') + result.upgraded_from = source.get('upgraded_from') result.partial_update = source.get('partial_update') result.points = PointPointsetEvent.map_from(source.get('points')) return result @@ -44,6 +46,8 @@ def to_dict(self): result['timestamp'] = self.timestamp # 5 if self.version: result['version'] = self.version # 5 + if self.upgraded_from: + result['upgraded_from'] = self.upgraded_from # 5 if self.partial_update: result['partial_update'] = self.partial_update # 5 if self.points: diff --git a/gencode/python/udmi/schema/event_system.py b/gencode/python/udmi/schema/event_system.py index 1de8b98f85..f8714ed42f 100644 --- a/gencode/python/udmi/schema/event_system.py +++ b/gencode/python/udmi/schema/event_system.py @@ -60,6 +60,7 @@ class SystemEvent: def __init__(self): self.timestamp = None self.version = None + self.upgraded_from = None self.last_config = None self.logentries = None self.event_count = None @@ -72,6 +73,7 @@ def from_dict(source): result = SystemEvent() result.timestamp = source.get('timestamp') result.version = source.get('version') + result.upgraded_from = source.get('upgraded_from') result.last_config = source.get('last_config') result.logentries = Entry.array_from(source.get('logentries')) result.event_count = source.get('event_count') @@ -100,6 +102,8 @@ def to_dict(self): result['timestamp'] = self.timestamp # 5 if self.version: result['version'] = self.version # 5 + if self.upgraded_from: + result['upgraded_from'] = self.upgraded_from # 5 if self.last_config: result['last_config'] = self.last_config # 5 if self.logentries: diff --git a/gencode/python/udmi/schema/metadata.py b/gencode/python/udmi/schema/metadata.py index 46f07dc420..aa309c2dfd 100644 --- a/gencode/python/udmi/schema/metadata.py +++ b/gencode/python/udmi/schema/metadata.py @@ -15,8 +15,10 @@ class Metadata: def __init__(self): self.timestamp = None self.version = None + self.upgraded_from = None self.description = None self.hash = None + self.device_version = None self.cloud = None self.system = None self.gateway = None @@ -33,8 +35,10 @@ def from_dict(source): result = Metadata() result.timestamp = source.get('timestamp') result.version = source.get('version') + result.upgraded_from = source.get('upgraded_from') result.description = source.get('description') result.hash = source.get('hash') + result.device_version = source.get('device_version') result.cloud = CloudModel.from_dict(source.get('cloud')) result.system = SystemModel.from_dict(source.get('system')) result.gateway = GatewayModel.from_dict(source.get('gateway')) @@ -67,10 +71,14 @@ def to_dict(self): result['timestamp'] = self.timestamp # 5 if self.version: result['version'] = self.version # 5 + if self.upgraded_from: + result['upgraded_from'] = self.upgraded_from # 5 if self.description: result['description'] = self.description # 5 if self.hash: result['hash'] = self.hash # 5 + if self.device_version: + result['device_version'] = self.device_version # 5 if self.cloud: result['cloud'] = self.cloud.to_dict() # 4 if self.system: diff --git a/gencode/python/udmi/schema/model_testing.py b/gencode/python/udmi/schema/model_testing.py index 665b20e4a3..78157e7dc1 100644 --- a/gencode/python/udmi/schema/model_testing.py +++ b/gencode/python/udmi/schema/model_testing.py @@ -6,6 +6,7 @@ class TestingModel: """Generated schema class""" def __init__(self): + self.nostate = None self.targets = None @staticmethod @@ -13,6 +14,7 @@ def from_dict(source): if not source: return None result = TestingModel() + result.nostate = source.get('nostate') result.targets = TargetTestingModel.map_from(source.get('targets')) return result @@ -34,6 +36,8 @@ def expand_dict(input): def to_dict(self): result = {} + if self.nostate: + result['nostate'] = self.nostate # 5 if self.targets: result['targets'] = TargetTestingModel.expand_dict(self.targets) # 2 return result diff --git a/gencode/python/udmi/schema/options_pubber.py b/gencode/python/udmi/schema/options_pubber.py index 632f9cc4b5..c5f46099c9 100644 --- a/gencode/python/udmi/schema/options_pubber.py +++ b/gencode/python/udmi/schema/options_pubber.py @@ -20,6 +20,7 @@ def __init__(self): self.redirectRegistry = None self.smokeCheck = None self.noPointState = None + self.noState = None self.featureEnableSwap = None self.disableWriteback = None self.noWriteback = None @@ -46,6 +47,7 @@ def from_dict(source): result.redirectRegistry = source.get('redirectRegistry') result.smokeCheck = source.get('smokeCheck') result.noPointState = source.get('noPointState') + result.noState = source.get('noState') result.featureEnableSwap = source.get('featureEnableSwap') result.disableWriteback = source.get('disableWriteback') result.noWriteback = source.get('noWriteback') @@ -101,6 +103,8 @@ def to_dict(self): result['smokeCheck'] = self.smokeCheck # 5 if self.noPointState: result['noPointState'] = self.noPointState # 5 + if self.noState: + result['noState'] = self.noState # 5 if self.featureEnableSwap: result['featureEnableSwap'] = self.featureEnableSwap # 5 if self.disableWriteback: diff --git a/gencode/python/udmi/schema/state.py b/gencode/python/udmi/schema/state.py index b8db83e180..be188ff9e8 100644 --- a/gencode/python/udmi/schema/state.py +++ b/gencode/python/udmi/schema/state.py @@ -13,6 +13,7 @@ class State: def __init__(self): self.timestamp = None self.version = None + self.upgraded_from = None self.system = None self.gateway = None self.discovery = None @@ -27,6 +28,7 @@ def from_dict(source): result = State() result.timestamp = source.get('timestamp') result.version = source.get('version') + result.upgraded_from = source.get('upgraded_from') result.system = SystemState.from_dict(source.get('system')) result.gateway = GatewayState.from_dict(source.get('gateway')) result.discovery = DiscoveryState.from_dict(source.get('discovery')) @@ -57,6 +59,8 @@ def to_dict(self): result['timestamp'] = self.timestamp # 5 if self.version: result['version'] = self.version # 5 + if self.upgraded_from: + result['upgraded_from'] = self.upgraded_from # 5 if self.system: result['system'] = self.system.to_dict() # 4 if self.gateway: diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index 47930df788..3da056101a 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -3,6 +3,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import static com.google.udmi.util.Common.VERSION_KEY; import static com.google.udmi.util.GeneralUtils.catchOrElse; import static com.google.udmi.util.GeneralUtils.catchToNull; import static com.google.udmi.util.GeneralUtils.deepCopy; @@ -12,14 +13,18 @@ import static com.google.udmi.util.GeneralUtils.ifNotNullGet; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static com.google.udmi.util.GeneralUtils.ifNotTrueThen; +import static com.google.udmi.util.GeneralUtils.isGetTrue; +import static com.google.udmi.util.GeneralUtils.isTrue; import static com.google.udmi.util.GeneralUtils.optionsString; import static com.google.udmi.util.GeneralUtils.toJsonFile; import static com.google.udmi.util.GeneralUtils.toJsonString; import static com.google.udmi.util.JsonUtil.safeSleep; import static daq.pubber.MqttDevice.CONFIG_TOPIC; import static daq.pubber.MqttDevice.ERRORS_TOPIC; +import static daq.pubber.MqttDevice.STATE_TOPIC; import static java.lang.Boolean.TRUE; import static java.lang.String.format; +import static java.util.Objects.isNull; import static java.util.Objects.requireNonNull; import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toMap; @@ -36,6 +41,8 @@ import com.google.udmi.util.CleanDateFormat; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; +import com.google.udmi.util.MessageDowngrader; +import com.google.udmi.util.SchemaVersion; import com.google.udmi.util.SiteModel; import com.google.udmi.util.SiteModel.MetadataException; import daq.pubber.MqttPublisher.InjectedMessage; @@ -92,6 +99,7 @@ import udmi.schema.Enumerate; import udmi.schema.Envelope; import udmi.schema.Envelope.SubFolder; +import udmi.schema.Envelope.SubType; import udmi.schema.FamilyDiscoveryConfig; import udmi.schema.FamilyDiscoveryEvent; import udmi.schema.FamilyDiscoveryState; @@ -129,7 +137,7 @@ public class Pubber { public static final String PERSISTENT_TMP_FORMAT = "/tmp/pubber_%s_" + PERSISTENT_STORE_FILE; public static final String PUBBER_LOG_CATEGORY = "device.log"; public static final String DATA_URL_JSON_BASE64 = "data:application/json;base64,"; - static final String UDMI_VERSION = "1.4.2"; + static final String UDMI_VERSION = SchemaVersion.CURRENT.key(); private static final Logger LOG = LoggerFactory.getLogger(Pubber.class); private static final String HOSTNAME = System.getenv("HOSTNAME"); private static final int MIN_REPORT_MS = 200; @@ -219,6 +227,7 @@ public class Pubber { private MqttDevice gatewayTarget; private int systemEventCount; private LocalnetManager localnetManager; + private SchemaVersion targetSchema; /** * Start an instance from a configuration file. @@ -439,7 +448,7 @@ private void initializeDevice() { deviceState.system = new SystemState(); deviceState.system.operation = new StateSystemOperation(); - if (!TRUE.equals(configuration.options.noLastStart)) { + if (!isTrue(configuration.options.noLastStart)) { deviceState.system.operation.last_start = DEVICE_START_TIME; } @@ -491,7 +500,7 @@ protected void initializePersistentStore() { checkState(persistentData == null, "persistent data already loaded"); File persistentStore = getPersistentStore(); - if (TRUE.equals(configuration.options.noPersist)) { + if (isTrue(configuration.options.noPersist)) { info("Resetting persistent store " + persistentStore.getAbsolutePath()); persistentData = newDevicePersistent(); } else { @@ -590,6 +599,9 @@ private void processDeviceMetadata(Metadata metadata) { throw new RuntimeException("While processing metadata file " + metadataException.file, metadataException.exception); } + targetSchema = ifNotNullGet(metadata.device_version, SchemaVersion::fromKey); + ifNotNullThen(targetSchema, version -> warn("Emulating UDMI version " + version.key())); + if (metadata.cloud != null) { configuration.algorithm = catchToNull(() -> metadata.cloud.auth_type.value()); } @@ -690,7 +702,7 @@ private void periodicUpdate() { */ private void sendEmptyMissingBadEvents() { int phase = deviceUpdateCount % MESSAGE_REPORT_INTERVAL; - if (!TRUE.equals(configuration.options.emptyMissing) + if (!isTrue(configuration.options.emptyMissing) || (phase >= INVALID_REPLACEMENTS.size() + 2)) { return; } @@ -765,7 +777,7 @@ private void maybeRestartSystem() { warn(format("Device start time %s before last config start %s, terminating.", isoConvert(DEVICE_START_TIME), isoConvert(configLastStart))); systemLifecycle(SystemMode.TERMINATE); - } else if (TRUE.equals(configuration.options.smokeCheck) + } else if (isTrue(configuration.options.smokeCheck) && CleanDateFormat.dateEquals(DEVICE_START_TIME, configLastStart)) { warn(format("Device start time %s matches, smoke check indicating success!", isoConvert(configLastStart))); @@ -1011,7 +1023,7 @@ private void publisherException(Exception toReport) { private void publisherHandler(String type, String phase, Throwable cause) { if (cause != null) { error("Error receiving message " + type, cause); - if (TRUE.equals(configuration.options.barfConfig)) { + if (isTrue(configuration.options.barfConfig)) { error("Restarting system because of restart-on-error configuration setting"); systemLifecycle(SystemMode.RESTART); } @@ -1034,7 +1046,7 @@ private void registerSystemStatus(Entry report) { * this case appropriately. */ private void publishConfigStateUpdate() { - if (TRUE.equals(configuration.options.configStateDelay)) { + if (isTrue(configuration.options.configStateDelay)) { delayNextStateUpdate(); } publishAsynchronousState(); @@ -1100,7 +1112,7 @@ private void configHandler(Config config) { private void processConfigUpdate(Config config) { final int actualInterval; if (config != null) { - if (config.system == null && TRUE.equals(configuration.options.barfConfig)) { + if (config.system == null && isTrue(configuration.options.barfConfig)) { error("Empty config system block and configured to restart on bad config!"); systemLifecycle(SystemMode.RESTART); } @@ -1319,7 +1331,7 @@ private void updateDiscoveryEnumeration(DiscoveryConfig config) { } private T ifTrue(Boolean condition, Supplier supplier) { - return isTrue(() -> condition) ? supplier.get() : null; + return isGetTrue(() -> condition) ? supplier.get() : null; } private Map enumeratePoints(String deviceId) { @@ -1448,7 +1460,7 @@ private void sendDiscoveryEvent(String family, Date scanGeneration) { .collect(toMap(Map.Entry::getKey, this::eventForTarget)); discoveryEvent.families.computeIfAbsent("iot", key -> new FamilyDiscoveryEvent()).addr = deviceId; - if (isTrue(() -> deviceConfig.discovery.families.get(family).enumerate)) { + if (isGetTrue(() -> deviceConfig.discovery.families.get(family).enumerate)) { discoveryEvent.uniqs = enumeratePoints(deviceId); } publishDeviceMessage(discoveryEvent); @@ -1460,14 +1472,6 @@ private void sendDiscoveryEvent(String family, Date scanGeneration) { } } - private boolean isTrue(Supplier target) { - try { - return target.get(); - } catch (Exception e) { - return false; - } - } - private FamilyDiscoveryEvent eventForTarget(Map.Entry target) { FamilyDiscoveryEvent event = new FamilyDiscoveryEvent(); event.addr = target.getValue().addr; @@ -1704,7 +1708,7 @@ private void publishStateMessage(Object stateToSend) { latch.countDown(); }); try { - if (!latch.await(WAIT_TIME_SEC, TimeUnit.SECONDS)) { + if (shouldSendState() && !latch.await(WAIT_TIME_SEC, TimeUnit.SECONDS)) { throw new RuntimeException("Timeout waiting for state send"); } } catch (Exception e) { @@ -1712,6 +1716,10 @@ private void publishStateMessage(Object stateToSend) { } } + private boolean shouldSendState() { + return !isGetTrue(() -> configuration.options.noState); + } + private void publishDeviceMessage(Object message) { publishDeviceMessage(message, null); } @@ -1723,28 +1731,39 @@ private void publishDeviceMessage(Object message, Runnable callback) { return; } + if (!shouldSendState() && topicSuffix.equals(STATE_TOPIC)) { + info("Squelching state update as per configuration"); + return; + } + if (deviceTarget == null) { error("publisher not active"); return; } augmentDeviceMessage(message); - deviceTarget.publish(topicSuffix, message, callback); + Object downgraded = downgradeMessage(message); + deviceTarget.publish(topicSuffix, downgraded, callback); String messageBase = topicSuffix.replace("/", "_"); String fileName = traceTimestamp(messageBase) + ".json"; File messageOut = new File(outDir, fileName); try { - toJsonFile(messageOut, message); + toJsonFile(messageOut, downgraded); } catch (Exception e) { throw new RuntimeException("While writing " + messageOut.getAbsolutePath(), e); } } + private Object downgradeMessage(Object message) { + MessageDowngrader messageDowngrader = new MessageDowngrader(SubType.STATE.value(), message); + return ifNotNullGet(targetSchema, messageDowngrader::downgrade, message); + } + private String traceTimestamp(String messageBase) { int serial = MESSAGE_COUNTS.computeIfAbsent(messageBase, key -> new AtomicInteger()) .incrementAndGet(); String timestamp = getTimestamp().replace("Z", format(".%03dZ", serial)); - return messageBase + (TRUE.equals(configuration.options.messageTrace) ? ("_" + timestamp) : ""); + return messageBase + (isTrue(configuration.options.messageTrace) ? ("_" + timestamp) : ""); } private boolean publisherActive() { @@ -1839,5 +1858,4 @@ static class ExtraPointsetEvent extends PointsetEvent { // This extraField exists only to trigger schema parsing errors. public Object extraField; } - } diff --git a/schema/config.json b/schema/config.json index b2ff520e8a..bee16bc5c7 100644 --- a/schema/config.json +++ b/schema/config.json @@ -21,6 +21,10 @@ "description": "Version of the UDMI schema", "type": "string" }, + "downgraded_from": { + "description": "Original version of schema pre-downgrade", + "type": "string" + }, "system": { "$ref": "file:config_system.json#" }, diff --git a/schema/event_pointset.json b/schema/event_pointset.json index 58239a946e..88b91e6aaa 100644 --- a/schema/event_pointset.json +++ b/schema/event_pointset.json @@ -17,6 +17,10 @@ "description": "Version of the UDMI schema", "type": "string" }, + "upgraded_from": { + "description": "Original version of schema pre-upgrade", + "type": "string" + }, "partial_update": { "description": "Indicates if this is a partial update (only some points may be included)", "type": "boolean" diff --git a/schema/event_system.json b/schema/event_system.json index ea3fe03e9d..4c5922057b 100644 --- a/schema/event_system.json +++ b/schema/event_system.json @@ -17,6 +17,10 @@ "description": "Version of the UDMI schema", "type": "string" }, + "upgraded_from": { + "description": "Original version of schema pre-upgrade", + "type": "string" + }, "last_config": { "description": "Last config received", "type": "string", diff --git a/schema/metadata.json b/schema/metadata.json index b7ef18a2e5..e74a86a544 100644 --- a/schema/metadata.json +++ b/schema/metadata.json @@ -19,7 +19,11 @@ "examples": ["2019-01-17T14:02:29.364Z"] }, "version": { - "description": "Version of the UDMI schema", + "description": "Version of the UDMI schema for this file", + "type": "string" + }, + "upgraded_from": { + "description": "Original version of the UDMI schema for this file", "type": "string" }, "description": { @@ -31,6 +35,10 @@ "type": "string", "pattern": "^[0-9a-z]{8}$" }, + "device_version": { + "description": "Version of schema supported by the device", + "type": "string" + }, "cloud": { "$ref": "file:model_cloud.json#" }, diff --git a/schema/model_testing.json b/schema/model_testing.json index e7350f8582..7f02020bf1 100644 --- a/schema/model_testing.json +++ b/schema/model_testing.json @@ -6,6 +6,9 @@ "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": false, "properties": { + "nostate": { + "type": "boolean" + }, "targets": { "additionalProperties": false, "existingJavaType": "java.util.HashMap", diff --git a/schema/options_pubber.json b/schema/options_pubber.json index 60b0195212..ebc7a3415e 100644 --- a/schema/options_pubber.json +++ b/schema/options_pubber.json @@ -49,6 +49,9 @@ "noPointState": { "type": "boolean" }, + "noState": { + "type": "boolean" + }, "featureEnableSwap": { "type": "boolean" }, diff --git a/schema/state.json b/schema/state.json index a0cd6cd049..77780b93a1 100644 --- a/schema/state.json +++ b/schema/state.json @@ -22,6 +22,10 @@ "description": "Version of the UDMI schema", "type": "string" }, + "upgraded_from": { + "description": "Original version of schema pre-upgrade", + "type": "string" + }, "system": { "$ref": "file:state_system.json#" }, diff --git a/tests/schemas/config/errors.out b/tests/schemas/config/errors.out index 65c21435ab..e2c83487a7 100644 --- a/tests/schemas/config/errors.out +++ b/tests/schemas/config/errors.out @@ -4,4 +4,4 @@ /pointset/state_etag: string "2a8b71dwqhdhdddddddddddddddddddddddddddddddddddddddddd8" is too long (length: 55, maximum allowed: 32) /pointset/version: instance type (integer) does not match any allowed primitive type (allowed: ["string"]) /pointset: object instance has properties which are not allowed by the schema: ["config_etag","id","properties"] - object instance has properties which are not allowed by the schema: ["points","properties","type"] + object instance has properties which are not allowed by the schema: ["points","properties","type","upgraded_from"] diff --git a/tests/sites/downgrade/devices/DWN-1/expected/generated_config.json b/tests/sites/downgrade/devices/DWN-1/expected/generated_config.json index 60802ff407..5c8d31020c 100644 --- a/tests/sites/downgrade/devices/DWN-1/expected/generated_config.json +++ b/tests/sites/downgrade/devices/DWN-1/expected/generated_config.json @@ -26,5 +26,6 @@ "units": "Degrees-Celsius" } } - } + }, + "downgraded_from": "1.4.2" } \ No newline at end of file diff --git a/tests/sites/downgrade/devices/DWN-1/expected/metadata_norm.json b/tests/sites/downgrade/devices/DWN-1/expected/metadata_norm.json index c9cde5d011..ef8f4548c8 100644 --- a/tests/sites/downgrade/devices/DWN-1/expected/metadata_norm.json +++ b/tests/sites/downgrade/devices/DWN-1/expected/metadata_norm.json @@ -1,7 +1,8 @@ { "timestamp": "2020-05-01T13:39:07Z", "version": "1.4.1", - "hash": "e7a2211b", + "upgraded_from": "1", + "hash": "e6712083", "cloud": { "auth_type": "RS256" }, diff --git a/tests/sites/downgrade/devices/DWN-2/expected/errors.map b/tests/sites/downgrade/devices/DWN-2/expected/errors.map index 00cac2f01f..2ed23053ba 100644 --- a/tests/sites/downgrade/devices/DWN-2/expected/errors.map +++ b/tests/sites/downgrade/devices/DWN-2/expected/errors.map @@ -1,4 +1,4 @@ Exceptions for DWN-2 LREDACTED/sites/downgrade/./devices/DWN-2/metadata.json While lREDACTED/sites/downgrade/./devices/DWN-2/metadata.json - Unrecognized field "grumpy" (class udmi.schema.Metadata), not marked as ignorable (12 known properties: "discovery", "localnet", "gateway", "cloud", "timestamp", "version", "description", "features", "hash", "system", "testing", "pointset"]) at [Source: (File); line: 27, column: 14] (through reference chain: udmi.schema.Metadata["grumpy"]) + Unrecognized field "grumpy" (class udmi.schema.Metadata), not marked as ignorable (14 known properties: "upgraded_from", "gateway", "hash", "device_version", "discovery", "localnet", "cloud", "timestamp", "version", "description", "features", "system", "testing", "pointset"]) at [Source: (File); line: 27, column: 14] (through reference chain: udmi.schema.Metadata["grumpy"]) diff --git a/tests/sites/missing/devices/GAT-123/expected/generated_config.json b/tests/sites/missing/devices/GAT-123/expected/generated_config.json index d61b235ae9..dcac8ba786 100644 --- a/tests/sites/missing/devices/GAT-123/expected/generated_config.json +++ b/tests/sites/missing/devices/GAT-123/expected/generated_config.json @@ -7,5 +7,6 @@ }, "gateway": { "proxy_ids": [ "SNS-4", "AHU-22" ] - } + }, + "downgraded_from": "1.4.2" } \ No newline at end of file diff --git a/tests/sites/missing/devices/GAT-123/expected/metadata_norm.json b/tests/sites/missing/devices/GAT-123/expected/metadata_norm.json index 0e8d71031d..a04cf1580a 100644 --- a/tests/sites/missing/devices/GAT-123/expected/metadata_norm.json +++ b/tests/sites/missing/devices/GAT-123/expected/metadata_norm.json @@ -1,7 +1,8 @@ { "timestamp": "2020-05-01T13:39:07Z", "version": "1.4.1", - "hash": "51901e97", + "upgraded_from": "1", + "hash": "fdda202f", "cloud": { "auth_type": "ES256" }, diff --git a/tests/sites/missing/devices/SNS-4/expected/generated_config.json b/tests/sites/missing/devices/SNS-4/expected/generated_config.json index f2c14ff9ab..e68bab170c 100644 --- a/tests/sites/missing/devices/SNS-4/expected/generated_config.json +++ b/tests/sites/missing/devices/SNS-4/expected/generated_config.json @@ -14,5 +14,6 @@ "units": "Degrees-phase" } } - } + }, + "downgraded_from": "1.4.2" } \ No newline at end of file diff --git a/tests/sites/missing/devices/SNS-4/expected/metadata_norm.json b/tests/sites/missing/devices/SNS-4/expected/metadata_norm.json index 41ed265aac..5674a877e2 100644 --- a/tests/sites/missing/devices/SNS-4/expected/metadata_norm.json +++ b/tests/sites/missing/devices/SNS-4/expected/metadata_norm.json @@ -1,7 +1,8 @@ { "timestamp": "2020-05-01T13:39:07Z", "version": "1.4.1", - "hash": "5ad401e4", + "upgraded_from": "1", + "hash": "302d594c", "system": { "location": { "site": "ZZ-TRI-FECTA", diff --git a/udmis/bin/container b/udmis/bin/container index ae69e346f6..f179f0cfb3 100755 --- a/udmis/bin/container +++ b/udmis/bin/container @@ -59,7 +59,7 @@ current_user=$USER@$HOSTNAME revparse=`git rev-parse HEAD` udmi_ver=g${revparse:0:9} -udmi_ref=$REPOSITORY/udmi:$udmi_ver +udmi_ref=$REPOSITORY/udmis:$udmi_ver version=`git describe` diff --git a/udmis/bin/deploy b/udmis/bin/deploy deleted file mode 120000 index 77c430a5ac..0000000000 --- a/udmis/bin/deploy +++ /dev/null @@ -1 +0,0 @@ -update \ No newline at end of file diff --git a/udmis/bin/pod_logs b/udmis/bin/pod_logs index d33813c23f..6c49d4a7f1 100755 --- a/udmis/bin/pod_logs +++ b/udmis/bin/pod_logs @@ -13,11 +13,11 @@ echo Getting logs for pods $pods rm -f ${podprefix}*.log for pod in $pods; do - kubectl logs $pod > $pod.log & + kubectl logs $pod | sed "s/.*/$pod &/" > $pod.log & done wait -cat ${podprefix}*.log | fgrep "Z " | sort -k 1 > pods.log +cat ${podprefix}*.log | fgrep "Z " | sort -k 2 > pods.log echo Combined logs available in $(realpath pods.log) diff --git a/udmis/build.gradle b/udmis/build.gradle index 9910563eb9..bf5f8adf36 100644 --- a/udmis/build.gradle +++ b/udmis/build.gradle @@ -92,7 +92,7 @@ dependencies { exclude group: 'com.google.guava', module: 'guava-jdk5' } - implementation ('io.github.clearblade:clearblade-cloud-iot:1.0.0') { + implementation ('io.github.clearblade:clearblade-cloud-iot:1.0.1') { // Exclude transitive dependency causing problems with running unit tests in the IDE. exclude group: 'org.junit.jupiter', module: 'junit-jupiter-api' } diff --git a/udmis/deploy_udmis_gcloud b/udmis/deploy_udmis_gcloud deleted file mode 100755 index c96750dc59..0000000000 --- a/udmis/deploy_udmis_gcloud +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e - -if (( $# < 1 )); then - echo Usage: $0 PROJECT_ID [options] - false -fi - -ROOT=$(realpath $(dirname $0)/..) -cd $ROOT/udmis - -PROJECT=$1 -shift - -RUNTIME=nodejs16 -SOURCE=functions/ - - -deployed_by=$USER@$HOSTNAME -deployed_at=`date --utc --iso=seconds` -version=`git describe --dirty` -funchash=`git log -n 1 --oneline -- functions/ | awk '{print $1}'` -funccount=`git diff --name-only -- functions/ | wc -l` -if [[ $funccount != 0 ]]; then - funcdirty=-dirty -fi -funcver=`git describe $funchash`$funcdirty - -echo Deploying version $version to $PROJECT - -cat < functions/version.js -module.exports = { - udmi_version: '$version', - udmi_functions: '$funcver', - deployed_at: '$deployed_at', - deployed_by: '$deployed_by' -} -EOF - -PUBSUB_FUNCTIONS="udmi_target udmi_state udmi_config udmi_reflect" -for func in $PUBSUB_FUNCTIONS; do - echo Deploying pubsub-trigger function $func... - gcloud functions deploy $func --set-env-vars GCP_PROJECT=$PROJECT --trigger-topic=$func --runtime=$RUNTIME --project=$PROJECT --source=$SOURCE "$@" & - sleep 10 -done - -echo Waiting for all deployments to complete... -wait - -echo Done with deploying functions $PUBSUB_FUNCTIONS diff --git a/udmis/src/main/java/com/google/bos/udmi/service/access/DynamicIotAccessProvider.java b/udmis/src/main/java/com/google/bos/udmi/service/access/DynamicIotAccessProvider.java index 10530654e4..f225526d43 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/access/DynamicIotAccessProvider.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/access/DynamicIotAccessProvider.java @@ -5,12 +5,10 @@ import static com.google.udmi.util.GeneralUtils.sortedMapCollector; import static com.google.udmi.util.JsonUtil.getTimestamp; import static java.lang.String.format; +import static java.util.Objects.requireNonNull; import static java.util.Optional.ofNullable; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.graph.ImmutableNetwork; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -42,6 +40,21 @@ public DynamicIotAccessProvider(IotAccess iotAccess) { providerList = Arrays.asList(iotAccess.project_id.split(",")); } + @Override + protected Map fetchRegistryRegions() { + return ImmutableMap.of(); + } + + @Override + protected Set getRegistriesForRegion(String region) { + throw new RuntimeException("Should not be called!"); + } + + @Override + protected boolean isEnabled() { + return true; + } + @Override protected String updateConfig(String registryId, String deviceId, String config, Long version) { throw new RuntimeException("Shouldn't be called for dynamic provider"); @@ -56,7 +69,11 @@ private String determineProvider(String registryId) { } private IotAccessBase getProviderFor(String registryId) { - return providers.get(registryProviders.computeIfAbsent(registryId, this::determineProvider)); + IotAccessBase provider = + providers.get(registryProviders.computeIfAbsent(registryId, this::determineProvider)); + return requireNonNull( + provider, + "could not determine provider for " + registryId); } private String registryPriority(String registryId, Entry provider) { @@ -67,21 +84,6 @@ private String registryPriority(String registryId, Entry return provisionedAt; } - @Override - protected boolean isEnabled() { - return true; - } - - @Override - protected Map fetchRegistryRegions() { - return ImmutableMap.of(); - } - - @Override - protected Set getRegistriesForRegion(String region) { - throw new RuntimeException("Should not be called!"); - } - @Override public void activate() { super.activate(); @@ -136,11 +138,6 @@ public void sendCommandBase(String registryId, String deviceId, SubFolder folder getProviderFor(registryId).sendCommandBase(registryId, deviceId, folder, message); } - @Override - public void updateRegistryRegions(Map regions) { - providers.values().forEach(provider -> provider.updateRegistryRegions(regions)); - } - @Override public void setProviderAffinity(String registryId, String deviceId, String providerId) { if (providerId != null) { @@ -152,4 +149,9 @@ public void setProviderAffinity(String registryId, String deviceId, String provi } super.setProviderAffinity(registryId, deviceId, providerId); } + + @Override + public void updateRegistryRegions(Map regions) { + providers.values().forEach(provider -> provider.updateRegistryRegions(regions)); + } } diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java b/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java index 5b7d22eeaf..c3e6510317 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java @@ -2,6 +2,7 @@ import static com.google.bos.udmi.service.messaging.impl.MessageDispatcherImpl.getMessageClassFor; import static com.google.common.base.Preconditions.checkState; +import static com.google.udmi.util.Common.DEVICE_ID_PROPERTY_KEY; import static com.google.udmi.util.Common.ERROR_KEY; import static com.google.udmi.util.Common.TIMESTAMP_KEY; import static com.google.udmi.util.GeneralUtils.decodeBase64; @@ -12,9 +13,8 @@ import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static com.google.udmi.util.JsonUtil.convertTo; import static com.google.udmi.util.JsonUtil.convertToStrict; -import static com.google.udmi.util.JsonUtil.fromStringStrict; +import static com.google.udmi.util.JsonUtil.fromString; import static com.google.udmi.util.JsonUtil.getTimestamp; -import static com.google.udmi.util.JsonUtil.loadFileStrictRequired; import static com.google.udmi.util.JsonUtil.stringify; import static com.google.udmi.util.JsonUtil.stringifyTerse; import static com.google.udmi.util.JsonUtil.toMap; @@ -50,20 +50,20 @@ public class ReflectProcessor extends ProcessorBase { protected void defaultHandler(Object message) { MessageContinuation continuation = getContinuation(message); Envelope reflection = continuation.getEnvelope(); + Map objectMap = toMap(message); try { if (reflection.subFolder == null) { reflectStateHandler(reflection, extractUdmiState(message)); } else if (reflection.subFolder != SubFolder.UDMI) { throw new IllegalStateException("Unexpected reflect subfolder " + reflection.subFolder); } else { - Map stringObjectMap = toMap(message); - Map payload = extractMessagePayload(stringObjectMap); - Envelope envelope = extractMessageEnvelope(stringObjectMap); + Map payload = extractMessagePayload(objectMap); + Envelope envelope = extractMessageEnvelope(objectMap); reflection.transactionId = envelope.transactionId; processReflection(reflection, envelope, payload); } } catch (Exception e) { - processException(reflection, e); + processException(reflection, objectMap, e); } } @@ -119,13 +119,14 @@ private CloudModel getReflectionResult(Envelope attributes, Map } } - private void processException(Envelope reflection, Exception e) { + private void processException(Envelope reflection, Map objectMap, Exception e) { String stackMessage = friendlyStackTrace(e); warn("Processing exception %s: %s", reflection.transactionId, stackMessage); Map message = new HashMap<>(); message.put(ERROR_KEY, stackMessage); Envelope envelope = new Envelope(); envelope.subFolder = SubFolder.ERROR; + envelope.deviceId = (String) objectMap.get(DEVICE_ID_PROPERTY_KEY); envelope.transactionId = reflection.transactionId; sendReflectCommand(reflection, envelope, message); } @@ -160,7 +161,7 @@ private CloudModel queryDeviceState(Envelope attributes) { String state = ofNullable( iotAccess.fetchState(attributes.deviceRegistryId, attributes.deviceId)).orElse("{}"); debug(format("Processing device %s state query", attributes.deviceId)); - StateUpdate stateUpdate = fromStringStrict(StateUpdate.class, state); + StateUpdate stateUpdate = fromString(StateUpdate.class, state); stateUpdate.configAcked = checkConfigAckTime(attributes, state); processStateUpdate(attributes, stateUpdate); publish(stateUpdate); diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java index c5a78443d5..26e20cbbf4 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java @@ -1,5 +1,6 @@ package com.google.bos.udmi.service.messaging.impl; +import static com.google.udmi.util.Common.DEVICE_ID_PROPERTY_KEY; import static com.google.udmi.util.GeneralUtils.decodeBase64; import static com.google.udmi.util.GeneralUtils.encodeBase64; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; @@ -81,7 +82,7 @@ private Envelope makeEnvelope(Map bundle) { Envelope envelope = new Envelope(); envelope.subFolder = getBundleSubfolder(attributes); envelope.subType = ifNotNullGet(attributes.get("subType"), SubType::fromValue); - envelope.deviceId = attributes.get("deviceId"); + envelope.deviceId = attributes.get(DEVICE_ID_PROPERTY_KEY); envelope.projectId = attributes.get("projectId"); envelope.deviceRegistryId = attributes.get("deviceRegistryId"); envelope.publishTime = ifNotNullGet(ofNullable(attributes.get("publishTime")) diff --git a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java index 78ab03f89c..5ab554bc4d 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java @@ -122,6 +122,7 @@ public static void main(String[] args) { udmiServicePod.activate(); } catch (Exception e) { System.err.println("Exception activating pod: " + friendlyStackTrace(e)); + e.printStackTrace(); System.exit(FATAL_ERROR_CODE); } } diff --git a/udmis/udmis.iml b/udmis/udmis.iml index 3e9053d8c6..a7b7ccfe77 100644 --- a/udmis/udmis.iml +++ b/udmis/udmis.iml @@ -34,7 +34,7 @@ - + diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java b/validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java index ede74a0444..6c7125ca39 100644 --- a/validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java +++ b/validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java @@ -11,6 +11,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.daq.mqtt.util.ValidationException; +import com.google.daq.mqtt.validator.Validator; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -57,7 +58,7 @@ public List validateMessage(String subFolder, String data) { if (report.isSuccess()) { return ImmutableList.of(); } - throw ValidationException.fromProcessingReport(report); + throw Validator.fromProcessingReport(report); } catch (ValidationException e) { return e.getAllMessages(); } catch (IOException | ProcessingException ex) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 642a38d7fa..d49cac525c 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -34,6 +34,7 @@ import com.google.daq.mqtt.util.ExceptionMap; import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; import com.google.daq.mqtt.util.ValidationException; +import com.google.daq.mqtt.validator.Validator; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.MessageDowngrader; @@ -223,7 +224,7 @@ public static void parseMetadataValidateProcessingReport(ProcessingReport report for (ProcessingMessage msg : report) { if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { - throw ValidationException.fromProcessingReport(report); + throw Validator.fromProcessingReport(report); } } } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/Feature.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/Feature.java index fdafae8cbb..8190ea3f83 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/Feature.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/Feature.java @@ -44,4 +44,9 @@ * @return feature score value */ int score() default DEFAULT_SCORE; + + /** + * Indicates if this test can run without state updates. + */ + boolean nostate() default false; } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java index d94a0f93d6..6d2c144d01 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java @@ -10,8 +10,10 @@ import static com.google.daq.mqtt.validator.Validator.STATE_PREFIX; import static com.google.udmi.util.CleanDateFormat.cleanDate; import static com.google.udmi.util.CleanDateFormat.dateEquals; +import static com.google.udmi.util.Common.DEVICE_ID_PROPERTY_KEY; import static com.google.udmi.util.Common.EXCEPTION_KEY; import static com.google.udmi.util.Common.TIMESTAMP_KEY; +import static com.google.udmi.util.Common.getExceptionMessage; import static com.google.udmi.util.GeneralUtils.changedLines; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; @@ -19,6 +21,7 @@ import static com.google.udmi.util.GeneralUtils.ifNotTrueThen; import static com.google.udmi.util.GeneralUtils.ifNullThen; import static com.google.udmi.util.GeneralUtils.ifTrueThen; +import static com.google.udmi.util.GeneralUtils.isTrue; import static com.google.udmi.util.GeneralUtils.stackTraceString; import static com.google.udmi.util.JsonUtil.getTimestamp; import static com.google.udmi.util.JsonUtil.loadFileRequired; @@ -45,7 +48,6 @@ import com.google.bos.iot.core.proxy.IotReflectorClient; import com.google.bos.iot.core.proxy.MockPublisher; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; @@ -200,6 +202,10 @@ public class SequenceBase { private static final AtomicReference stateTransaction = new AtomicReference<>(); private static final int MINIMUM_TEST_SEC = 15; private static final Date RESET_LAST_START = new Date(73642); + private static final long STATE_QUERY_CUTOFF_SEC = 60; + private static final Date stateCutoffThreshold = Date.from(Instant.now().minusSeconds( + STATE_QUERY_CUTOFF_SEC)); + private static final String FAKE_DEVICE_ID = "TAP-1"; protected static Metadata deviceMetadata; protected static String projectId; protected static String cloudRegion; @@ -245,7 +251,7 @@ public class SequenceBase { private Instant lastConfigUpdate; private boolean enforceSerial; private String testName; - private String testDescription; + private String testSummary; private Bucket testBucket; private FeatureStage testStage; private long startTestTimeMs; @@ -261,6 +267,7 @@ public class SequenceBase { private SequenceResult testResult; private int startStateCount; private Boolean expectedSystemStatus; + private Description testDescription; static void ensureValidatorConfig() { if (validatorConfig != null) { @@ -601,7 +608,7 @@ private void withRecordSequence(boolean value, Runnable operation) { } private void writeSequenceMdHeader() { - String block = testDescription == null ? "" : format("%s\n\n", testDescription); + String block = testSummary == null ? "" : format("%s\n\n", testSummary); sequenceMd.printf("\n## %s (%s)\n\n%s", testName, testStage.name(), block); sequenceMd.flush(); } @@ -670,6 +677,13 @@ public void setUp() { assumeTrue("Feature bucket not enabled", isBucketEnabled(testBucket)); + if (!deviceSupportsState()) { + boolean featureDisabled = ifNotNullGet(testDescription.getAnnotation(Feature.class), + feature -> !feature.nostate(), true); + ifTrueSkipTest(featureDisabled, "State testing disabled"); + notice("Running test with state checks disabled"); + } + waitingConditionPush("starting test wrapper"); checkState(reflector().isActive(), "Reflector is not currently active"); @@ -689,7 +703,8 @@ public void setUp() { updateConfig("initial setup"); - untilTrue("device state update", () -> deviceState != null); + ifTrueThen(deviceSupportsState(), + () -> untilTrue("device state update", () -> deviceState != null)); checkThatHasInterestingSystemStatus(false); // Do this late in the sequence to make sure any state is cleared out from previous test. @@ -704,6 +719,10 @@ public void setUp() { debug(format("stage begin %s at %s", waitingConditionPeek(), timeSinceStart())); } + private boolean deviceSupportsState() { + return !isTrue(catchToNull(() -> deviceMetadata.testing.nostate)); + } + protected boolean isBucketEnabled(Bucket bucket) { if (bucket == SYSTEM || enableAllTargets || deviceMetadata.features == null) { return true; @@ -968,11 +987,13 @@ private boolean stateTransactionPending() { } protected void queryState() { + if (!deviceSupportsState()) { + return; + } assertConfigIsNotPending(); - Preconditions.checkState(!stateTransactionPending(), "state transaction already pending"); String txnId = reflector().publish(getDeviceId(), Common.STATE_QUERY_TOPIC, EMPTY_MESSAGE); - stateTransaction.set(txnId); - debug(format("Waiting for device stateTransaction %s", txnId)); + String previous = stateTransaction.getAndSet(txnId); + debug(format("Waiting for device stateTransaction %s (was %s)", txnId, previous)); whileDoing("state query", () -> messageEvaluateLoop(this::stateTransactionPending), e -> debug( format("While waiting for stateTransaction %s: %s", txnId, friendlyStackTrace(e)))); @@ -1177,8 +1198,9 @@ protected void untilLogged(String category, Level exactLevel) { protected void checkNotLogged(String category, Level minLevel) { withRecordSequence(false, () -> { - untilTrue("last_config synchronized", - () -> dateEquals(deviceConfig.timestamp, deviceState.system.last_config)); + ifTrueThen(deviceSupportsState(), () -> + untilTrue("last_config synchronized", + () -> dateEquals(deviceConfig.timestamp, deviceState.system.last_config))); processLogMessages(); }); final Instant endTime = lastConfigUpdate.plusSeconds(LOG_TIMEOUT_SEC); @@ -1392,7 +1414,11 @@ private void validateMessage(Map attributes, Map String deviceId = attributes.get("deviceId"); ReportingDevice reportingDevice = new ReportingDevice(deviceId); - messageValidator.validateDeviceMessage(reportingDevice, message, attributes); + + Map modified = new HashMap<>(attributes); + modified.put(DEVICE_ID_PROPERTY_KEY, FAKE_DEVICE_ID); // Allow for non-standard device IDs. + + messageValidator.validateDeviceMessage(reportingDevice, message, modified); String messageBase = makeMessageBase(attributes); validationResults.computeIfAbsent(messageBase, key -> new ArrayList<>()) .addAll(reportingDevice.getMessageEntries()); @@ -1475,19 +1501,31 @@ private synchronized void handleUpdateMessage(String subTypeRaw, } } else if (converted instanceof State convertedState) { String timestamp = getTimestamp(convertedState.timestamp); - if (deviceState != null && convertedState.timestamp != null - && convertedState.timestamp.before(deviceState.timestamp)) { + if (convertedState.timestamp == null) { + warning("No timestamp in state message, rejecting."); + return; + } + if (deviceState != null && convertedState.timestamp.before(deviceState.timestamp)) { warning(format("Ignoring out-of-order state update %s %s", timestamp, txnId)); return; } + if (deviceState == null && convertedState.timestamp.before(stateCutoffThreshold)) { + String lastStart = getTimestamp( + catchToNull(() -> convertedState.system.operation.last_start)); + warning(format("Ignoring stale state update %s %s %s %s", timestamp, + getTimestamp(stateCutoffThreshold), lastStart, txnId)); + return; + } + checkState(deviceSupportsState(), "Received state update with no-state device"); + boolean deltaState = RECV_STATE_DIFFERNATOR.isInitialized(); List stateChanges = RECV_STATE_DIFFERNATOR.computeChanges(converted); Instant start = ofNullable(convertedState.timestamp).orElseGet(Date::new).toInstant(); long delta = Duration.between(start, Instant.now()).getSeconds(); debug(format("Updated state after %ds %s %s", delta, timestamp, txnId)); - if (updateCount == 1) { - info(format("Initial state #%03d", updateCount), stringify(converted)); - } else { + if (deltaState) { info(format("Updated state #%03d", updateCount), changedLines(stateChanges)); + } else { + info(format("Initial state #%03d", updateCount), stringify(converted)); } deviceState = convertedState; validSerialNo(); @@ -1723,21 +1761,33 @@ protected boolean stateMatchesConfigTimestamp() { return dateEquals(expectedConfig, lastConfig); } - private boolean hasInterestingSystemStatus() { + private Boolean hasInterestingSystemStatus() { if (deviceState.system.status != null) { debug("Status level: " + deviceState.system.status.level); } + + // State missing is neither interesting nor not-interesting... + if (deviceState == null || deviceState.system == null) { + return null; + } + return deviceState.system.status != null && deviceState.system.status.level >= Level.WARNING.value(); } protected void checkThatHasInterestingSystemStatus(boolean isInteresting) { + if (!deviceSupportsState()) { + return; + } BiConsumer> check = isInteresting ? this::checkThat : this::checkNotThat; check.accept(SYSTEM_STATUS_MESSAGE, this::hasInterestingSystemStatus); } protected void untilHasInterestingSystemStatus(boolean isInteresting) { + if (!deviceSupportsState()) { + return; + } expectedSystemStatus = null; BiConsumer> until = isInteresting ? this::untilTrue : this::untilFalse; @@ -1801,6 +1851,10 @@ protected void skipTest(String reason) { throw new AssumptionViolatedException(reason); } + protected void ifTrueSkipTest(Boolean condition, String reason) { + ifTrueThen(condition, () -> skipTest(reason)); + } + protected T ifNullSkipTest(T testable, String reason) { ifNullThen(testable, () -> skipTest(reason)); return testable; @@ -1809,7 +1863,6 @@ protected T ifNullSkipTest(T testable, String reason) { protected T ifCatchNullSkipTest(Supplier evaluator, String reason) { T evaluatorResult = catchToNull(evaluator); return ifNullSkipTest(evaluatorResult, reason); - } /** @@ -1832,7 +1885,8 @@ protected void starting(@NotNull Description description) { putSequencerResult(description, SequenceResult.START); checkState(reflector().isActive(), "Reflector is not currently active"); - testDescription = getTestSummary(description); + testDescription = description; + testSummary = getTestSummary(description); testStage = getTestStage(description); testBucket = getBucket(description); @@ -1906,10 +1960,14 @@ protected void failed(Throwable e, Description description) { message = e.getMessage(); failureType = SequenceResult.SKIP; } else { - message = friendlyStackTrace(e); + Throwable cause = e; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + message = getExceptionMessage(cause); failureType = SequenceResult.FAIL; } - debug("exception message: " + Common.getExceptionMessage(e)); + debug("exception message: " + friendlyStackTrace(e)); trace("ending stack trace", stackTraceString(e)); recordCompletion(failureType, description, message); String action = failureType == SequenceResult.SKIP ? "skipped" : "failed"; @@ -1917,7 +1975,8 @@ protected void failed(Throwable e, Description description) { if (failureType != SequenceResult.SKIP) { resetRequired = true; if (debugLogLevel()) { - error("Forcing exit to preserve failing config/state " + START_END_MARKER); + error("terminating test " + testName + " after " + timeSinceStart() + " " + + START_END_MARKER); System.exit(EXIT_CODE_PRESERVE); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java index b3a240a7e6..117c51b9db 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java @@ -40,7 +40,7 @@ public class ConfigSequences extends SequenceBase { private static final long LOG_APPLY_DELAY_MS = 1000; @Test(timeout = ONE_MINUTE_MS) - @Feature(stage = STABLE, bucket = SYSTEM) + @Feature(stage = STABLE, bucket = SYSTEM, nostate = true) @Summary("Check that last_update state is correctly set in response to a config update.") @ValidateSchema public void system_last_update() { @@ -58,7 +58,7 @@ public void valid_serial_no() { } @Test(timeout = TWO_MINUTES_MS) - @Feature(stage = ALPHA, bucket = SYSTEM) + @Feature(stage = ALPHA, bucket = SYSTEM, nostate = true) @Summary("Check that the min log-level config is honored by the device.") public void system_min_loglevel() { Integer savedLevel = deviceConfig.system.min_loglevel; diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java index 42d88433fb..af423a0220 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/PointsetSequences.java @@ -114,7 +114,6 @@ public void pointset_remove_point() { untilPointsetSanity(); } - /** * Simple check that device publishes pointset events. */ @@ -129,7 +128,6 @@ public void pointset_publish() { )); } - /** * Tests sample_rate_min by measuring the initial interval between the last two messages received, * then setting the config.pointset.sample_rate_min to match half the initial interval and @@ -198,7 +196,7 @@ private String samplingMessagesCheckMessage(SamplingRange samplingRange) { */ @Test(timeout = THREE_MINUTES_MS) @Summary("test sample rate and sample limit sec") - @Feature(stage = BETA, bucket = POINTSET) + @Feature(stage = BETA, bucket = POINTSET, nostate = true) public void pointset_publish_interval() { ifNullSkipTest(deviceConfig.pointset, "no pointset found in config"); diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java b/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java index 75783e4872..daa70e2926 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/ObjectDiffEngine.java @@ -193,4 +193,8 @@ public List diff(Object startObject, Object endObject) { accumulateDifference("", left, right, updates); return updates; } + + public boolean isInitialized() { + return previous != null; + } } \ No newline at end of file diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ValidationException.java b/validator/src/main/java/com/google/daq/mqtt/util/ValidationException.java deleted file mode 100644 index daed3cd136..0000000000 --- a/validator/src/main/java/com/google/daq/mqtt/util/ValidationException.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.google.daq.mqtt.util; - -import static com.google.common.collect.ImmutableList.toImmutableList; - -import com.github.fge.jsonschema.core.report.LogLevel; -import com.github.fge.jsonschema.core.report.ProcessingMessage; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.google.api.client.util.Strings; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import java.util.stream.StreamSupport; - -/** - * Exception during validation. - */ -public class ValidationException extends RuntimeException { - - private final ImmutableList causingExceptions; - - private ValidationException( - String message, ImmutableList causingExceptions) { - super(message); - this.causingExceptions = causingExceptions; - } - - /** - * Create a simple validation exception. - * - * @param message exception message - */ - public ValidationException(String message) { - this(message, ImmutableList.of()); - } - - /** - * From an external processing report. - * - * @param report Report to convert - * @return Converted exception. - */ - public static ValidationException fromProcessingReport(ProcessingReport report) { - Preconditions.checkArgument(!report.isSuccess(), "Report must not be successful"); - ImmutableList causingExceptions = - StreamSupport.stream(report.spliterator(), false) - .filter( - processingMessage -> processingMessage.getLogLevel().compareTo(LogLevel.ERROR) >= 0) - .map(ValidationException::convertMessage).collect(toImmutableList()); - return new ValidationException( - String.format("%d schema violations found", causingExceptions.size()), causingExceptions); - } - - private static ValidationException convertMessage(ProcessingMessage processingMessage) { - String pointer = processingMessage.asJson().get("instance").get("pointer").asText(); - String prefix = Strings.isNullOrEmpty(pointer) ? "" : (pointer + ": "); - return new ValidationException(prefix + processingMessage.getMessage()); - } - - /** - * Get the exception that started it all. - * - * @return The triggering exception. - */ - public ImmutableList getCausingExceptions() { - return causingExceptions; - } - - /** - * Get all the messages in this exception. - * - * @return List of the messages. - */ - public ImmutableList getAllMessages() { - ImmutableList.Builder messagesBuilder = - ImmutableList.builder().add(getMessage()); - for (ValidationException causingException : causingExceptions) { - messagesBuilder.addAll(causingException.getAllMessages()); - } - return messagesBuilder.build(); - } -} diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 634d470827..7376934d9d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -1,12 +1,14 @@ package com.google.daq.mqtt.validator; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.daq.mqtt.registrar.Registrar.BASE_DIR; import static com.google.daq.mqtt.sequencer.SequenceBase.EMPTY_MESSAGE; import static com.google.daq.mqtt.util.ConfigUtil.UDMI_TOOLS; import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION; import static com.google.daq.mqtt.util.ConfigUtil.readExeConfig; import static com.google.daq.mqtt.validator.ReportingDevice.typeFolderPairKey; +import static com.google.udmi.util.Common.DEVICE_ID_PROPERTY_KEY; import static com.google.udmi.util.Common.ERROR_KEY; import static com.google.udmi.util.Common.EXCEPTION_KEY; import static com.google.udmi.util.Common.GCP_REFLECT_KEY_PKCS8; @@ -30,6 +32,8 @@ import com.github.fge.jsonschema.core.exceptions.ProcessingException; import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; import com.github.fge.jsonschema.core.load.download.URIDownloader; +import com.github.fge.jsonschema.core.report.LogLevel; +import com.github.fge.jsonschema.core.report.ProcessingMessage; import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; import com.github.fge.jsonschema.main.JsonSchemaFactory; @@ -82,6 +86,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.apache.commons.io.FileUtils; import udmi.schema.Category; import udmi.schema.DeviceValidationEvent; @@ -216,6 +221,30 @@ private static void sanitizeMessageException(Map message) { } } + /** + * From an external processing report. + * + * @param report Report to convert + * @return Converted exception. + */ + public static ValidationException fromProcessingReport(ProcessingReport report) { + checkArgument(!report.isSuccess(), "Report must not be successful"); + ImmutableList causingExceptions = + StreamSupport.stream(report.spliterator(), false) + .filter( + processingMessage -> processingMessage.getLogLevel().compareTo(LogLevel.ERROR) >= 0) + .map(Validator::convertMessage).collect(toImmutableList()); + return new ValidationException( + String.format("%d schema violations found", causingExceptions.size()), causingExceptions); + } + + private static ValidationException convertMessage(ProcessingMessage processingMessage) { + String pointer = processingMessage.asJson().get("instance").get("pointer").asText(); + String prefix = + com.google.api.client.util.Strings.isNullOrEmpty(pointer) ? "" : (pointer + ": "); + return new ValidationException(prefix + processingMessage.getMessage()); + } + private List parseArgs(List argList) { if (!argList.isEmpty() && !argList.get(0).startsWith("-")) { processProfile(new File(argList.remove(0))); @@ -597,7 +626,7 @@ public void validateDeviceMessage(ReportingDevice device, Map me try { validateMessage(schemaMap.get(ENVELOPE_SCHEMA_ID), attributes); } catch (Exception e) { - System.err.println("Error validating attributes: " + e); + System.err.println("Error validating attributes: " + friendlyStackTrace(e)); device.addError(e, attributes, Category.VALIDATION_DEVICE_RECEIVE); } @@ -606,7 +635,7 @@ public void validateDeviceMessage(ReportingDevice device, Map me validateMessage(schemaMap.get(schemaName), message); } catch (Exception e) { System.err.printf("Error validating schema %s: %s%n", schemaName, - GeneralUtils.friendlyStackTrace(e)); + friendlyStackTrace(e)); device.addError(e, attributes, Category.VALIDATION_DEVICE_SCHEMA); } } @@ -620,7 +649,7 @@ public void validateDeviceMessage(ReportingDevice device, Map me device.validateMessageType(messageObject, JsonUtil.getDate(timeString), attributes); }); } catch (Exception e) { - System.err.println("Error validating contents: " + e.getMessage()); + System.err.println("Error validating contents: " + friendlyStackTrace(e)); device.addError(e, attributes, Category.VALIDATION_DEVICE_CONTENT); } } else { @@ -1007,7 +1036,7 @@ private void upgradeMessage(String schemaName, Map message) { private void validateJsonNode(JsonSchema schema, JsonNode jsonNode) throws ProcessingException { ProcessingReport report = schema.validate(jsonNode, true); if (!report.isSuccess()) { - throw ValidationException.fromProcessingReport(report); + throw fromProcessingReport(report); } } diff --git a/validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java b/validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java index 8a45be2b04..ac486a9cc8 100644 --- a/validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java +++ b/validator/src/test/java/com/google/udmi/util/MessageDowngraderTest.java @@ -1,5 +1,6 @@ package com.google.udmi.util; +import static java.lang.Boolean.TRUE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -9,9 +10,12 @@ import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; -import com.google.udmi.util.MessageDowngrader; import java.io.File; +import java.util.Map; import org.junit.Test; +import udmi.schema.State; +import udmi.schema.StateSystemOperation; +import udmi.schema.SystemState; /** * Unit tests for MessageDowngrader. @@ -29,9 +33,16 @@ public class MessageDowngraderTest { private static final String LOCALNET_VERSION = "1.3.14"; private static final IntNode OLD_VERSION = new IntNode(1); - @Test(expected = IllegalArgumentException.class) - public void stateFail() { - new MessageDowngrader(STATE_SCHEMA, new TextNode("hello")); + @Test + public void stateDowngrade() { + State state = new State(); + state.system = new SystemState(); + state.system.operation = new StateSystemOperation(); + state.system.operation.operational = TRUE; + MessageDowngrader messageDowngrader = new MessageDowngrader(STATE_SCHEMA, state); + Map downgrade = messageDowngrader.downgrade(SchemaVersion.VERSION_1_4_0); + Object operational = GeneralUtils.getSubMap(downgrade, "system").get("operational"); + assertTrue("downgraded operational not TRUE", (Boolean) operational); } @Test