Fix error reporting and some outputs (#764)

.gencode_hash.txt

@@ -5,7 +5,7 @@ d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673  gencode/docs/c
 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79  gencode/docs/configuration_endpoint.html
 6cf94d6cb600c75cde32a64bd78acb3ed3b54adfad08dbf6bb159b467e8925c9  gencode/docs/configuration_execution.html
 d0f6ea11943f46c955891b3b49a25a9bffcf558fcabb9ed2b0f7667d8413953b  gencode/docs/configuration_pod.html
-26d2fe3ecc9f95dff52cdd9eae3be621886ef264305f31f64aa8f254e7ffbf99  gencode/docs/configuration_pubber.html
+1fbd42e4591a609d7f8869c6d1942477d96221ee01ee9bf4aae0bc23d570f301  gencode/docs/configuration_pubber.html
 96186777da06f95eae1d16d73555445d23608a9301636ea1ccd17922b3fe4019  gencode/docs/event.html
 587e048c161273b927de67b899204bf0e183db64e59ae513f833e5eff406b1ab  gencode/docs/event_discovery.html
 0f99534574718e07e655e33e76e06b56e6a96a7a42ae1457dc97dabc581d848f  gencode/docs/event_mapping.html
@@ -99,7 +99,7 @@ fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa  gencode/java/u
 ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a  gencode/java/udmi/schema/Position.java
 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70  gencode/java/udmi/schema/Properties.java
 d5e11dc354995681b894bcb62e4e21da7f0be68aaa3643a3dc5aa13610741f29  gencode/java/udmi/schema/PubberConfiguration.java
-3575b20afc5e1b38f0ac34799968270b301c3320915bd74b607e02d45c9e6eb4  gencode/java/udmi/schema/PubberOptions.java
+87fd10ad778d7d2c1d800343de76430f80ea7701ffb60859bd4a6102fae3d243  gencode/java/udmi/schema/PubberOptions.java
 703ed0cecb60f284b52e76dc4d612eb798ba3e8781cc40698a46611b36e6b0c9  gencode/java/udmi/schema/SchemaValidationState.java
 6da1708f597a0c64ce30dcbe739095fc7ebc091d63b98af88c73f70706af31ca  gencode/java/udmi/schema/SequenceValidationState.java
 9da49b22341a65580d085fd9d00eaadcaefcf7b813988ef844c617aa4b8a9a4f  gencode/java/udmi/schema/SetupUdmiConfig.java
@@ -182,7 +182,7 @@ b48ae013d203eb31cc388a084c0cc6f93d4d5534336d4da2ecbccd84a085ea2b  gencode/python
 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f  gencode/python/udmi/schema/model_testing_target.py
 7c9a755ae06c44fc2b8117d7918b6401ce08a7218b86033fb3320eda7286e581  gencode/python/udmi/schema/monitoring.py
 3a3873c7db8d5fa13b43629031d36417a4ec4838d612a7b8c2fce7c22fdee009  gencode/python/udmi/schema/monitoring_metric.py
-47feb547b1630546bfa7ed5a8de1c708f0f769b42fe1aac35f4b1cdacda865eb  gencode/python/udmi/schema/options_pubber.py
+07ffd16404e2788249448da9fda6572b6930c5241441dedcc803fab5bf236a10  gencode/python/udmi/schema/options_pubber.py
 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77  gencode/python/udmi/schema/persistent_device.py
 a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python/udmi/schema/properties.py
 e604cf0280fe772de5f4e5ecf10dc6c564b6177eeff9cd9fb8b385af8fe10a95  gencode/python/udmi/schema/state.py

.github/workflows/testing.yml

@@ -146,9 +146,9 @@ jobs:
       - name: registrar clean
         run: bin/test_regclean $TARGET_PROJECT
       - name: sequence tests clean
-        run: bin/test_sequencer clean nocheck $TARGET_PROJECT
+        run: bin/test_sequencer nocheck $TARGET_PROJECT
       - name: sequence tests alpha
-        run: bin/test_sequencer alpha nocheck $TARGET_PROJECT
+        run: bin/test_sequencer noclean alpha nocheck $TARGET_PROJECT
       - name: sequence test post-process
         if: ${{ always() }}
         run: egrep ' test .* after .*s ' out/sequencer.log > out/timing_sequencer.out
@@ -206,7 +206,7 @@ jobs:
           find . -type f | sort | xargs ls -l
           find . -type f | sort | xargs more | cat
       - name: nostate sequences
-        run: bin/test_sequencer clean nostate full $TARGET_PROJECT
+        run: bin/test_sequencer nostate full $TARGET_PROJECT
       - name: support bundle
         if: ${{ always() }}
         run: UDMI_REGISTRY_SUFFIX=_b bin/support ${{ github.repository_owner }}_${{ github.job }}_

bin/sequencer_cache

@@ -15,8 +15,10 @@
 #
 
 RUNDIR=$PWD
-ROOT_DIR=$(realpath $(dirname $0)/..)
-cd $ROOT_DIR
+UDMI_ROOT=$(realpath $(dirname $0)/..)
+cd $UDMI_ROOT
+
+source etc/shell_common.sh
 
 mkdir -p $RUNDIR/out
 TEST_LIST=$RUNDIR/out/sequencer_tests.txt
@@ -108,12 +110,9 @@ if [[ -n $validate_saved ]]; then
         | xargs -I% diff -u \
             $TEST_ROOT/tests/%/sequence.md \
             $CACHE_ROOT/%/sequence.md
-    root_count=$(ls -1 $TEST_ROOT/tests/ | wc -l)
-    cache_count=$(ls -1 $CACHE_ROOT/ | wc -l)
-    if [[ $root_count != $cache_count ]]; then
-        echo Test count mismatch, was $root_count expected $cache_count
-        false
-    fi
+    ls $CACHE_ROOT/ > out/cached_sequences.out
+    ls $TEST_ROOT/tests/ > out/device_sequences.out
+    diff -u out/cached_sequences.out out/device_sequences.out || fail Unexpected difference with cached results.
     echo Verification successful for $root_count tests.
 fi
 

bin/test_sequencer

@@ -3,19 +3,21 @@
 # Force consistent sort order
 export LC_ALL=C
 
-ROOT_DIR=$(dirname $0)/..
-cd $ROOT_DIR
+UDMI_ROOT=$(dirname $0)/..
+cd $UDMI_ROOT
+
+source etc/shell_common.sh
 
 debug_opts=-v
 seq_opts=
-clean_cache=
+clean_cache=y
 pubber_opts=
 suffix=
 out_check=y
 
-if [[ $1 == 'clean' ]]; then
+if [[ $1 == 'noclean' ]]; then
     shift
-    clean_cache=y
+    clean_cache=
 fi
 
 if [[ $1 == 'alpha' ]]; then
@@ -47,6 +49,9 @@ fi
 project_id=$1
 shift
 targets=$*
+if [[ -n $targets ]]; then
+    clean_cache=
+fi
 
 [[ -n $GITHUB_RUN_NUMBER ]] && echo "Workflow run number $GITHUB_RUN_NUMBER" || true
 echo "export TARGET_PROJECT=$project_id"
@@ -115,6 +120,13 @@ echo bin/sequencer $debug_opts $seq_opts $site_path $project_id $device_id $seri
 bin/sequencer $debug_opts $seq_opts $site_path $project_id $device_id $serial_no $targets
 echo Completed execution of sequencer test run.
 
+echo Sanity check for completed tests...
+target_count=$(fgrep "Running target" out/sequencer.log | wc -l)
+result_count=$(cat out/sequencer.out | wc -l)
+if [[ $target_count != $result_count ]]; then
+    fail Only $result_count results found for expected $target_count targets.
+fi
+
 echo Checking reported cloud version info
 jq .cloud_version.udmi_ref $output_file
 

common/src/main/java/com/google/udmi/util/MessageUpgrader.java

@@ -45,17 +45,24 @@ public MessageUpgrader(String schemaName, JsonNode message) {
     this.original = message.deepCopy();
 
     JsonNode version = message.get(VERSION_KEY);
-    String versionString = convertVersion(version);
-    String[] components = versionString.split("-", 2);
-    String[] parts = components[0].split("\\.", 4);
-    major = Integer.parseInt(parts[0]);
-    minor = parts.length >= 2 ? Integer.parseInt(parts[1]) : -1;
-    patch = parts.length >= 3 ? Integer.parseInt(parts[2]) : -1;
+    originalVersion = convertVersion(version);
 
-    if (parts.length >= 4) {
-      throw new IllegalArgumentException("Unexpected src version " + versionString);
+    try {
+      String[] components = originalVersion.split("-", 2);
+      String[] parts = components[0].split("\\.", 4);
+      if (parts.length >= 4) {
+        throw new IllegalArgumentException("More than 3 version components");
+      }
+      try {
+        major = Integer.parseInt(parts[0]);
+        minor = parts.length >= 2 ? Integer.parseInt(parts[1]) : -1;
+        patch = parts.length >= 3 ? Integer.parseInt(parts[2]) : -1;
+      } catch (NumberFormatException e) {
+        throw new RuntimeException("Bad version string number format");
+      }
+    } catch (Exception e) {
+      throw new RuntimeException("While parsing version string " + originalVersion, e);
     }
-    originalVersion = versionString;
   }
 
   public MessageUpgrader(String schemaName, Object originalMessage) {

docs/specs/sequences/generated.md

@@ -83,11 +83,11 @@ Check that the device correctly handles an extra out-of-schema field
     * Set `system.min_loglevel` = `100`
 1. Wait for last_config not null
 1. Wait for system operational
-1. Check that no interesting system status
+1. Check that no applicable system status
 1. Wait for log category `system.config.receive` level `DEBUG` was logged
 1. Wait for last_config updated
 1. Wait for system operational
-1. Check that no interesting system status
+1. Check that no applicable system status
 1. Wait for log category `system.config.parse` level `DEBUG` was logged
 1. Wait for log category `system.config.apply` level `NOTICE` was logged
 1. Wait for log category `system.config.receive` level `DEBUG` was logged

etc/schema_itemized.out

@@ -1,4 +1,5 @@
 broken_config
+config_logging
 device_config_acked
 feature_enumeration
 gateway_proxy_events

etc/test_itemized.in

@@ -8,9 +8,11 @@ TEST writeback_success noWriteback
 TEST writeback_success noPointState
 TEST pointset_sample_rate fixedSampleRate=10
 TEST system_mode_restart noPersist
+TEST config_logging badVersion
 TEST broken_config barfConfig
 TEST broken_config configStateDelay
 TEST broken_config noState
+TEST broken_config badCategory
 TEST system_last_update extraField=fnooz
 
 # Test with various device metadata.json options

etc/test_itemized.out

@@ -1,6 +1,8 @@
 RESULT pass system broken_config ALPHA 5 Sequence complete
-RESULT fail system broken_config ALPHA 5 timeout waiting for has interesting system status
-RESULT fail system broken_config ALPHA 5 timeout waiting for no interesting system status
+RESULT fail system broken_config ALPHA 5 timeout waiting for has applicable system status
+RESULT fail system broken_config ALPHA 5 timeout waiting for log category `system.config.receive` level `DEBUG` was logged
+RESULT fail system broken_config ALPHA 5 timeout waiting for no applicable system status
+RESULT fail system config_logging BETA 5 Bad version string number format
 RESULT skip system device_config_acked BETA 5 No config check for proxy device
 RESULT fail system device_config_acked BETA 5 timeout waiting for config acked
 RESULT fail enumeration.features feature_enumeration PREVIEW 5 Failed check that feature enumeration matches metadata; missing { enumeration }, extra { unknown }