Implement no-state sequence testing mode (#738)

faucetsdn · Oct 16, 2023 · 30eb0da · 30eb0da
1 parent 65c4343
commit 30eb0da
Show file tree

Hide file tree

Showing 73 changed files with 1,164 additions and 345 deletions.
diff --git a/.gencode_hash.txt b/.gencode_hash.txt
@@ -1,26 +1,26 @@
 b35646a2f3d7c30fd34e3ea6ab6a1070b39c469fd1de49cab1093ff32faaf06d  gencode/docs/command_discovery.html
 d3758eba2529d4a5f1dfd5ed3355a536936b02285ddde7cc75b1f41f4916203a  gencode/docs/command_mapping.html
-cc618f9c64c36a7ffad5df420b8f192a4b1f1af9a4efca1a1e92056d4a892a47  gencode/docs/config.html
+2067b2eab579b6f6667d6890dfeb72cb3ec6fbd779668c8ff7dc8f42e3f9a60b  gencode/docs/config.html
 d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673  gencode/docs/config_mapping.html
 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79  gencode/docs/configuration_endpoint.html
 33846607c2c7241660b62b0ca1ef1f56cba922e6aa3b1a2813e3a4a0cc94a4ef  gencode/docs/configuration_execution.html
 e081e1438839467d3be35dd866d0cd00f4454fcc5d8e384ba03ca28cd439af87  gencode/docs/configuration_pod.html
-75294ff7051586e821a8c090e18eb12e325ccf7fcac86537669dd6e009025979  gencode/docs/configuration_pubber.html
-564d652ef4393f0b83d5e35c0be26766023c300414f6cef92ccc26cae440c0d1  gencode/docs/event.html
+b197e72b85de1e73e4c8954d85cd28c6958335c9b9dcc2504834bf92cd21f1f1  gencode/docs/configuration_pubber.html
+96186777da06f95eae1d16d73555445d23608a9301636ea1ccd17922b3fe4019  gencode/docs/event.html
 587e048c161273b927de67b899204bf0e183db64e59ae513f833e5eff406b1ab  gencode/docs/event_discovery.html
 0f99534574718e07e655e33e76e06b56e6a96a7a42ae1457dc97dabc581d848f  gencode/docs/event_mapping.html
-71fe25d3b17d1dc87b52c049a3235c20527f73e12ffa53fa4800a15e53f73732  gencode/docs/event_pointset.html
-acd50976d1afd771d55177db9a1b9452a0c6a35e3e15b2e444c799991294808d  gencode/docs/event_system.html
+d2fe30200f9a1c530d144137a7774b5b97f268aae41185bfcb25de274d399250  gencode/docs/event_pointset.html
+6849a96fe7608dc5ebda0dc9595dcea57757bc7c5253c30c30409469b6260498  gencode/docs/event_system.html
 816481f69d3b1bdeb2224eaad6e3751a991d20eb98294d89f888b1323505209c  gencode/docs/event_validation.html
-ae5762dbe2304e44cd742e7236d2de6987bdc1384b21093490cfcd7e33a5509f  gencode/docs/metadata.html
-4db86e0b979a1873d73da07d68a432df39282cb8b94ea509514d2b411411d55e  gencode/docs/monitoring.html
+0734afc2bddcaa7166c10e070ed6a4805127aa4c90e060915cdb48d5d79f618f  gencode/docs/metadata.html
+c4fa2845c5ad385a619ec97827370988c059567cb23c8da6894331fed89fefce  gencode/docs/monitoring.html
 180b32717db748e164a185b163ef9a97aa83d9d6add306283d5b9852d04af947  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
 1766f84518a315fe57e4a4bf934c0a386ad61d87091754a6bab097c686c16019  gencode/docs/readme.md
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-f8854f923ab8dbda6559942948298467fe20982bf71f79b0cb0912d9a7ae5167  gencode/docs/state.html
+2294674149e961fb6aaf993dcf37d94681bd81ee073053ca9b8b70df7d3ceb3d  gencode/docs/state.html
 f4a52cdad2cd9a9646da4d84cc1f3962ddd151a8d348255e5cff30eadfee283c  gencode/docs/state_mapping.html
 567b27a9257816eb9a9063ce089fe0f58c5962f10ea449bb7c653bd6151365a8  gencode/docs/state_udmi.html
 2c6557d3d63269a956cd32d1784044b16d72172ab559656bf65b8d3684748c25  gencode/docs/state_validation.html
@@ -37,7 +37,7 @@ e28c3899bf2cb08cf456dbca29ccc3d4559c1d2145e252095689b171be82b4c6  gencode/java/u
 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e  gencode/java/udmi/schema/BuildingTranslation.java
 36a165d3c65ca02cd095606a27d7efa21f6d3b7134bb49771b315dcdfa7154b7  gencode/java/udmi/schema/Category.java
 e15316a81d0acfc965e5517b9b49a7514f6dac024bebf8557e41f8abe0787023  gencode/java/udmi/schema/CloudModel.java
-ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4  gencode/java/udmi/schema/Config.java
+a8bf763f1db77446d9e97154630b30ce4f92c13ea1c567102067dd132227ddd5  gencode/java/udmi/schema/Config.java
 cf9762392e02b5d04c6498963222cc0c00f7be6c3cd82bde3d063a5eceba2b65  gencode/java/udmi/schema/Connections.java
 8164e93ccc76d78548d456890e064df8427a09bbcd50686b37a10d3ff5ad429f  gencode/java/udmi/schema/Credential.java
 10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05  gencode/java/udmi/schema/DeviceMappingConfig.java
@@ -78,7 +78,7 @@ ccc7c234dd522a91d387d55573677681bbc5a6926f0fbf1101c2e9f607cafd63  gencode/java/u
 39f4cf5f89dc52f24ea02246e17668ebc85ed0fab351e69138e477bcc048d79c  gencode/java/udmi/schema/MappingEvent.java
 052b6b7f9ca8173901d9d2fae314f5858e034f20efa5d184780b800bd870cb2b  gencode/java/udmi/schema/MappingEventEntity.java
 d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886  gencode/java/udmi/schema/MappingState.java
-476f542353b69649eea8a63fe53a01d83a1d62a33eae140241b107525024dda5  gencode/java/udmi/schema/Metadata.java
+991bf3fa8916e0d1745f4734b9bb5c61409cd860277aaa29701335fdf07919e9  gencode/java/udmi/schema/Metadata.java
 df64e4ddbf543ac70e7c2af9d3fbc20ffe3dff68c6718aa9ceadab7f64d3d171  gencode/java/udmi/schema/Metrics.java
 14722df90406cdcd29c7c946e5fcdf6a3e513eb879a7fbb493801fc0d9093504  gencode/java/udmi/schema/Monitoring.java
 b3b7ef6cae004d5fdb4052f860df0aa583df87bb798a027fc769aeaf16489789  gencode/java/udmi/schema/MonitoringMetric.java
@@ -91,29 +91,29 @@ eb478a6ebd66b150da8d6dc8fdf05a10d09d6d0ea1df8a0578728703718c5551  gencode/java/u
 65337109653a93d873dae40eb517857149bae1b20c7b479f41b35c0822d94ba0  gencode/java/udmi/schema/PointPointsetModel.java
 db8d6dd3498019ad12e0f328b6237d07e52f133f8b08858b712611a52c198009  gencode/java/udmi/schema/PointPointsetState.java
 1fd6168df4ee02c8395abc762f6d4d4ddac4eb8ee3b3e329252b2f88c9903a8c  gencode/java/udmi/schema/PointsetConfig.java
-bb1c7e0df78870b7f7bd78fdfb1ed115cb121613ffcd98ec6b95e41c01624d41  gencode/java/udmi/schema/PointsetEvent.java
+c385e937c9952be3405a63bf3292790f965960a91ef0267126f39051a8e0b513  gencode/java/udmi/schema/PointsetEvent.java
 1e6be912a048065d678962b23b3b8f17e01c426ac8ff463c57c8c2d02e39ebd3  gencode/java/udmi/schema/PointsetModel.java
 fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa  gencode/java/udmi/schema/PointsetState.java
 61afd6c9c0364b8982c90df7664147829bad8858408b2141299ee8bbc35f1612  gencode/java/udmi/schema/PointsetSummary.java
 ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a  gencode/java/udmi/schema/Position.java
 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70  gencode/java/udmi/schema/Properties.java
 2f48b09437aab980a40ebf06c3334a7b33843b160d7618da20789e3d628e9650  gencode/java/udmi/schema/PubberConfiguration.java
-da8a104d7538d81ee97e325b18f509650d86e0feb2540d110aae8172de123179  gencode/java/udmi/schema/PubberOptions.java
+6230499555484858a6032d73094b9338b6983cda4aa519f77666fd8bf4f837a6  gencode/java/udmi/schema/PubberOptions.java
 703ed0cecb60f284b52e76dc4d612eb798ba3e8781cc40698a46611b36e6b0c9  gencode/java/udmi/schema/SchemaValidationState.java
 6da1708f597a0c64ce30dcbe739095fc7ebc091d63b98af88c73f70706af31ca  gencode/java/udmi/schema/SequenceValidationState.java
 9da49b22341a65580d085fd9d00eaadcaefcf7b813988ef844c617aa4b8a9a4f  gencode/java/udmi/schema/SetupUdmiConfig.java
 e79f3842470cc1f12b1d0a9d6691b147aeb4d6bf29e24872721133352aed1cc9  gencode/java/udmi/schema/SetupUdmiState.java
-9dc7e377f3392d35b5b4da3d1bbeb5e8d9a375a13ec6c852c478112cf26d8006  gencode/java/udmi/schema/State.java
+f3d880f0ec7af07a8eb22c19b3c23e943fc54af03723d458664a5d9dcce57584  gencode/java/udmi/schema/State.java
 47ad54328160a1aabce719a3263d4bfea903e8e8255e04a54ace86eb095b0f0c  gencode/java/udmi/schema/StateSystemHardware.java
 c9b920d355e5f6350fa6eb978afbfe55dd1438df171d2ea215845d5ac0b3e7e9  gencode/java/udmi/schema/StateSystemOperation.java
 d771dcb6bd06620e4e21419420d252b1f4aa0c55d0be315d6310299cebff56d6  gencode/java/udmi/schema/SystemConfig.java
 6f7b213970bfe2b1ac3056b83da8c8f5b4c50eac221dc1d6c6887e193e8bf40c  gencode/java/udmi/schema/SystemDiscoveryEvent.java
-21ca023d5a9006b1e041fcf644d572270b5743fcd2d2f221364a3046f0c5609c  gencode/java/udmi/schema/SystemEvent.java
+484939a730497cd27a8eb57c17cd10140ff0710368326b4fe8ba28080b406a79  gencode/java/udmi/schema/SystemEvent.java
 2cf23174ef4e2876511fb471d3f9fcb5cefe2fde324db844c2d0d505fd2c8844  gencode/java/udmi/schema/SystemHardware.java
 b48923eea2caded1badca59376c63d51be6352b9f9fcb375cdef5897c0fc5fda  gencode/java/udmi/schema/SystemModel.java
 7d8d9f058878d3228a2042b8ffe6a5d897548c32b218307004101699aa17c2b3  gencode/java/udmi/schema/SystemState.java
 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d  gencode/java/udmi/schema/TargetTestingModel.java
-d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/udmi/schema/TestingModel.java
+7db9435bd72c03d4bc4bff720d6cee1b65c7af5d10687579e44d96d2d1683813  gencode/java/udmi/schema/TestingModel.java
 7793d6d76a430dc7acf668d92d2df5f8e0625d6228207731f96e220f3b90e659  gencode/java/udmi/schema/TestingSystemConfig.java
 2958978ce5b7418320835e7d6731bd8db6700643f02bb48eed4edc3dd90b686f  gencode/java/udmi/schema/UdmiConfig.java
 92d94860b963a1b024a6cc55c40870852363da7e08547cc1cea6e6274293cc2a  gencode/java/udmi/schema/UdmiState.java
@@ -129,7 +129,7 @@ dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6  gencode/python
 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb  gencode/python/udmi/schema/command_discovery.py
 2a858a05c8717002c909ebf9e4661abedd5affeada33e2a114d440eccc677e03  gencode/python/udmi/schema/command_mapping.py
 741b37355b360f2daeb99a1e6c2ce4f4673168d443b0e7cab8d67f5101460bbb  gencode/python/udmi/schema/common.py
-b975892df78076dabc797b4c0be87f20b33eacda11f9d1ac1c09be33d4937a87  gencode/python/udmi/schema/config.py
+53961cd8a6ddd118143457ca60d765e12b21bcdf98fc2d648ebac0172ffca880  gencode/python/udmi/schema/config.py
 79eb0299b3751d93c01a5de65eacc717283e99caf0996f3db48a15247f69c8eb  gencode/python/udmi/schema/config_blobset.py
 59039eaf2812392329c641ad387f7e27790cf804fc94a014626d2da5ab426e3d  gencode/python/udmi/schema/config_blobset_blob.py
 7a6411ba2646881ffad5b6ad472c980e48a11b36ab2c9c77d12608ccc7225594  gencode/python/udmi/schema/config_discovery.py
@@ -159,12 +159,12 @@ e285b2af68d9cc1f019642f763ae62c35a482e652c1def8715bae5a487af4153  gencode/python
 dd30f748b5321223933c272d58f5fa6dc319912737255170322f0df3244732a7  gencode/python/udmi/schema/event_mapping.py
 6163270aa726ef29bc9285d224ec5c2611aab5d19f6ed09d711e6c816fc7e467  gencode/python/udmi/schema/event_mapping_entities.py
 3652a73a7df2d9df18a45b72d948c38a84a61e008d7543d006dad1832640c54f  gencode/python/udmi/schema/event_mapping_entity.py
-ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b  gencode/python/udmi/schema/event_pointset.py
+40f4e3cd2ff180b9b8c6235682f2d661342f610e30fc1613e18676387d52bfa5  gencode/python/udmi/schema/event_pointset.py
 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00  gencode/python/udmi/schema/event_pointset_point.py
-d89500d1e1c88ac754ddc5abd13135dd140d40c60568ca9294f6fe384b0b12ef  gencode/python/udmi/schema/event_system.py
+7d520df05f839ed4a7472c395c77cf1d2cbf47e3162e87e43ed5c9ae9e979af6  gencode/python/udmi/schema/event_system.py
 4361f48caba6a948a5916f67c8168215721cb3cc2384649f0336804785f01f1d  gencode/python/udmi/schema/event_validation.py
 633c85b0999d969310918bb03e9a40a08960f420b862340045a0290f8f5c4fe5  gencode/python/udmi/schema/event_validation_device.py
-c7304b2070e6d930b4d101263b90768b036f7fb7e352a6e5147b83a5523987f4  gencode/python/udmi/schema/metadata.py
+02e464aec10ae7f8cf10045f6107e069de4b852a24cd8fae6a4290a01c0d2ac8  gencode/python/udmi/schema/metadata.py
 6e763776c8fc5b5a980b490873b1ee7613e15c5b1dee151278f7eb941e897901  gencode/python/udmi/schema/model_cloud.py
 805cc8dcb29732d1965bbd533b12d2bc4966d584f05b8a478dd2cac98fd99d52  gencode/python/udmi/schema/model_discovery.py
 44d057dffcac3a994e7b894f8e1ebb643ba49d7dbfbb0bfea9cc04f97af0bc47  gencode/python/udmi/schema/model_discovery_family.py
@@ -176,14 +176,14 @@ ae6a7bc93b520cdc90cd158f876a58e98a3e000402ebb1ca632f01ee492d408a  gencode/python
 1ad65c005d865b9259bfa239b2dc4cd1cde13d312ecbc7843bb61cc31d6cabde  gencode/python/udmi/schema/model_pointset_point.py
 ee9c02c35438fb7d9aacb15a21ec7b35b533c1000d0bde044ec3923b1fdccca4  gencode/python/udmi/schema/model_system.py
 15b349141ebae651c6c3c5c313b197d49c8b2b44e8ff1b0639848ad42e5c4e63  gencode/python/udmi/schema/model_system_hardware.py
-aafe6e70c281152db958adf77a024e3e9fab8293927106297c5ec48c11f54e27  gencode/python/udmi/schema/model_testing.py
+b48ae013d203eb31cc388a084c0cc6f93d4d5534336d4da2ecbccd84a085ea2b  gencode/python/udmi/schema/model_testing.py
 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f  gencode/python/udmi/schema/model_testing_target.py
 7c9a755ae06c44fc2b8117d7918b6401ce08a7218b86033fb3320eda7286e581  gencode/python/udmi/schema/monitoring.py
 3a3873c7db8d5fa13b43629031d36417a4ec4838d612a7b8c2fce7c22fdee009  gencode/python/udmi/schema/monitoring_metric.py
-db023d2b8833953a497e467c7e4dbd8fe5019db2c4f8a492bae4d3ac61af4460  gencode/python/udmi/schema/options_pubber.py
+4b3a8aa2fabdb41ee5d652f0769cae9f8daa5d60e013d7f2e569112bb09d040c  gencode/python/udmi/schema/options_pubber.py
 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77  gencode/python/udmi/schema/persistent_device.py
 a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python/udmi/schema/properties.py
-03880d929419402a76e1ff1ae3faec7520eff6b8b632d19dc13de70f0f9873f2  gencode/python/udmi/schema/state.py
+e604cf0280fe772de5f4e5ecf10dc6c564b6177eeff9cd9fb8b385af8fe10a95  gencode/python/udmi/schema/state.py
 4a908cee3fb8afb559bcbfa594e57dbc515a35e4468e02600751b2fcce05a238  gencode/python/udmi/schema/state_blobset.py
 182e07b534403dcc121d980672e41b0fa2ee55c4da1f5c56f0dad5d599450c80  gencode/python/udmi/schema/state_blobset_blob.py
 8c6fc5fd63ab50768cb3662228ed893eb8b9d563cd062efaed12f5079a652d0c  gencode/python/udmi/schema/state_discovery.py

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -242,7 +242,7 @@ jobs:
   comprehensive:
     name: Comprehensive Suite
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 35
     if: vars.TARGET_PROJECT != ''
     env:
       TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}
@@ -256,11 +256,9 @@ jobs:
       - name: base setup
         run: bin/run_tests install_dependencies
       - name: registrar clean
-        run: |
-          bin/test_regclean $TARGET_PROJECT
+        run: bin/test_regclean $TARGET_PROJECT
       - name: telemetry validator
-        run: |
-          bin/test_validator $TARGET_PROJECT
+        run: bin/test_validator $TARGET_PROJECT
       - name: validator result
         if: ${{ always() }}
         run: |
@@ -270,13 +268,15 @@ jobs:
         run: |
           cd sites/udmi_site_model/out
           find . -type f | sort | xargs more | cat
+      - name: nostate sequences
+        if: ${{ always() }}
+        run: bin/test_sequencer clean alpha nostate full $TARGET_PROJECT
       - name: itemized sequencer tests
-        run: |
-          bin/test_itemized $TARGET_PROJECT
+        run: bin/test_itemized $TARGET_PROJECT
       - name: itemized output logs
         if: ${{ always() }}
         run: |
-          more out/test_itemized.out out/schemas_itemized.out | cat
+          more out/*_itemized.out | cat
           more out/sequencer.out-* out/pubber.out-* | cat
       - name: tool output logs
         if: ${{ always() }}
@@ -319,11 +319,9 @@ jobs:
         run: bin/run_tests install_dependencies
       - name: bin/test_proxy
         if: "${{ env.MQTT_PROXY_HOST != '' }}"
-        run: |
-          bin/test_proxy $TARGET_PROJECT $MQTT_PROXY_HOST
+        run: bin/test_proxy $TARGET_PROJECT $MQTT_PROXY_HOST
       - name: bin/test_redirect
-        run: |
-          bin/test_redirect $TARGET_PROJECT
+        run: bin/test_redirect $TARGET_PROJECT
       - name: pubber.out.1
         run: |
           echo ::::::: pubber.out.1

diff --git a/bin/augment_metadata b/bin/augment_metadata
@@ -5,14 +5,20 @@
 # the testing infrastrucure. Not intended for use outside of CI testing.
 #
 
-if [[ $# -ne 2 ]]; then
-    echo Usage: $0 SITE_DIR DEVICE_ID
+if [[ $# -lt 2 ]]; then
+    echo Usage: $0 SITE_DIR DEVICE_ID [PUBBER_OPTS...]
     false
 fi
 
 site_dir=$(realpath $1)
 device_id=$2
 shift 2
+pubber_opts=$*
+
+nostate=false
+if [[ $pubber_opts =~ noState ]]; then
+    nostate=true
+fi
 
 ROOT_DIR=$(dirname $0)/..
 cd $ROOT_DIR
@@ -35,12 +41,16 @@ ipv4_addr=${ipv4_addr%/*}
 ipv6_addr=$(ip addr show dev $default_iface | fgrep 'inet6 ' | tail -n 1 | awk '{print $2}')
 ipv6_addr=${ipv6_addr%/*}
 
-echo Setting $metadata_out ether addr to $ether_addr
-echo Setting $metadata_out ipv4 addr to $ipv4_addr
-echo Setting $metadata_out ipv6 addr to $ipv6_addr
+echo Augmenting to $metadata_out
+echo Setting ether addr to $ether_addr
+echo Setting ipv4 addr to $ipv4_addr
+echo Setting ipv6 addr to $ipv6_addr
+echo Setting nostate to $nostate
+
 jq ".localnet.families.ether.addr = \"$ether_addr\"" $metadata_file |
     jq ".localnet.families.ipv4.addr = \"$ipv4_addr\"" |
-    jq ".localnet.families.ipv6.addr = \"$ipv6_addr\"" > $metadata_out
+    jq ".localnet.families.ipv6.addr = \"$ipv6_addr\"" |
+    jq ".testing.nostate = $nostate" > $metadata_out
 
 # Dynamically update the site model based off of dynamic pubber output.
 pubber_features=$(< $site_dir/out/pubber_features.json)

diff --git a/bin/test_itemized b/bin/test_itemized
@@ -3,12 +3,13 @@
 ROOT_DIR=$(dirname $0)/..
 cd $ROOT_DIR
 
-if [[ $# != 1 ]]; then
-    echo Usage: $0 PROJECT_ID
+if [[ $# != 1 && $# != 2 ]]; then
+    echo Usage: $0 PROJECT_ID [target test]
     false
 fi
 PROJECT_ID=$1
 shift 1
+TARGET_TEST=$1
 
 SITE_PATH=sites/udmi_site_model
 DEVICE_ID=AHU-1
@@ -21,27 +22,44 @@ RESULTS_OUT=out/test_itemized.out
 GOLDEN_FILE=etc/test_itemized.out
 SCHEMA_OUT=out/schema_itemized.out
 GOLDEN_SCHEMAS=etc/schema_itemized.out
+AUGMENT_OPTS=out/augment_opts.txt
 
 echo "export TARGET_PROJECT=$PROJECT_ID"
 echo "export UDMI_REGISTRY_SUFFIX=$UDMI_REGISTRY_SUFFIX"
 echo "export UDMI_ALT_REGISTRY=$UDMI_ALT_REGISTRY"
 
-rm -f $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT
+rm -f $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT $AUGMENT_OPTS
 mkdir -p out
-touch $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT
+touch $RESULTS_OUT $SEQUENCER_OUT $PUBBER_OUT $SCHEMA_OUT $AUGMENT_OPTS
 
 pubber/bin/build
 validator/bin/build
 
 test_index=0
 while read -u 7 action test_name remainder; do
-    if [[ $action != TEST ]]; then
-        continue;
+    if [[ $action == TEST && -n $TARGET_TEST && $TARGET_TEST != $test_name ]]; then
+        continue
+    fi
+
+    if [[ $action == WITH ]]; then
+        echo $test_name $remainder > $AUGMENT_OPTS
+    elif [[ -z $action || $action =~ ^# ]]; then
+        true # skip blank lines and comments
+    elif [[ $action != TEST ]]; then
+        echo Unknown test action $action
+        false
     fi
+
     pubber_opts=${remainder/RESULT*/}
 
-    echo -n "TEST $test_name $pubber_opts" >> $RESULTS_OUT
-    echo "TEST $test_name $pubber_opts"
+    echo -n "$action $test_name $pubber_opts" >> $RESULTS_OUT
+
+    if [[ $action != TEST ]]; then
+        echo >> $RESULTS_OUT
+        continue
+    fi
+
+    echo itemized test $test_name $pubber_opts [$(< $AUGMENT_OPTS)]
 
     # Clean out the persistant data store to ensure a clean state each time.
     rm -rf sites/udmi_site_model/out/devices/AHU-1/persistent_data.json
@@ -67,9 +85,10 @@ while read -u 7 action test_name remainder; do
         false
     fi
 
-    bin/augment_metadata $SITE_PATH $DEVICE_ID
-
     echo
+    echo bin/augment_metadata $SITE_PATH $DEVICE_ID $(< $AUGMENT_OPTS)
+    bin/augment_metadata $SITE_PATH $DEVICE_ID $(< $AUGMENT_OPTS)
+
     echo bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name \> $SEQUENCER_OUT
     bin/sequencer $seq_opts $SITE_PATH $PROJECT_ID $DEVICE_ID $serial_no $test_name 2>&1 | tee $SEQUENCER_OUT
 
@@ -89,8 +108,11 @@ while read -u 7 action test_name remainder; do
 
 done 7< $GOLDEN_FILE
 
-echo Comparing diff $RESULTS_OUT $GOLDEN_FILE
-diff -u $RESULTS_OUT $GOLDEN_FILE
+if [[ -z $TARGET_TEST ]]; then
+    echo Comparing diff $RESULTS_OUT $GOLDEN_FILE
+    diff -bu $RESULTS_OUT $GOLDEN_FILE
 
-echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS
-diff -u $SCHEMA_OUT $GOLDEN_SCHEMAS
+    echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS
+    uniq < $SCHEMA_OUT | sponge $SCHEMA_OUT
+    diff -bu $SCHEMA_OUT $GOLDEN_SCHEMAS
+fi