From fc6e0d0774c49e14250775aba608a4af1bb315e7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 4 Dec 2023 21:21:49 -0500 Subject: [PATCH 1/9] Fixing cmdline --- bin/registrar | 9 +++++---- .../com/google/daq/mqtt/util/IotReflectorClient.java | 2 +- .../java/com/google/daq/mqtt/util/MessagePublisher.java | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bin/registrar b/bin/registrar index aeeba4a40..ba4b0b44d 100755 --- a/bin/registrar +++ b/bin/registrar @@ -16,7 +16,7 @@ shift cd $UDMI_ROOT -if [[ -z $1 || $1 =~ ^- ]]; then +if [[ -z $1 || $1 =~ ^- || -f $site_path ]]; then [[ -f $site_path ]] || echo No project ID supplied with site directory. else project_id=$1 @@ -25,8 +25,7 @@ fi iot_config=$site_path/cloud_iot_config.json if [[ -d $site_path && ! -f $iot_config ]]; then - echo Config file not found: $iot_config - false + fail Config file not found: $iot_config elif [[ -d $site_path && -z $project_id ]]; then echo No project defined, dry run only. fi @@ -49,7 +48,9 @@ echo Running tools version `git describe` quoted_namespace=null quoted_bridge=null iot_provider=null -if [[ $project_id =~ ^// ]]; then +if [[ -f $site_path ]]; then + echo Project_id is $project_id +elif [[ $project_id =~ ^// ]]; then no_prefix=${project_id#//} iot_provider=\"${no_prefix%%/*}\" no_provider=${no_prefix#*/} diff --git a/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java b/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java index b881e1905..b1453bc52 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java @@ -148,7 +148,7 @@ public Set fetchDeviceIds(String forGatewayId) { private CloudModel fetchCloudModel(String deviceId) { try { Map message = transaction(deviceId, CLOUD_QUERY_TOPIC, EMPTY_MESSAGE, - MessagePublisher.QuerySpeed.LONG); + QuerySpeed.ETERNITY); return convertToStrict(CloudModel.class, message); } catch (Exception e) { if (e.getMessage().contains("NOT_FOUND")) { diff --git a/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java b/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java index 8ad7912a0..70d3b76c6 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java @@ -76,7 +76,8 @@ default Credential getCredential() { enum QuerySpeed { QUICK(1), SHORT(15), - LONG(30); + LONG(30), + ETERNITY(90); private final int seconds; From f27e330e0685ccfffa74c7f75af4a1cacbda614a Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 00:08:51 -0500 Subject: [PATCH 2/9] Add solo regclean test --- .github/workflows/testing.yml | 2 +- bin/test_regclean | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 49d738b2c..c1c529d7c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -204,7 +204,7 @@ jobs: - name: base setup run: bin/run_tests install_dependencies - name: registrar clean - run: bin/test_regclean $TARGET_PROJECT + run: bin/test_regclean solo $TARGET_PROJECT - name: telemetry validator run: bin/test_validator $TARGET_PROJECT - name: validator result diff --git a/bin/test_regclean b/bin/test_regclean index f6d4a9137..33966e366 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -6,11 +6,18 @@ export LC_ALL=C ROOT_DIR=$(dirname $0)/.. cd $ROOT_DIR -if [[ $# -ne 1 ]]; then +solo_arg= + +if [[ $# -lt 1 ]]; then echo Usage: $0 PROJECT_ID[@BROKER] false fi +if [[ $1 == solo ]]; then + solo_arg=AHU-1 + shift +fi + project_id=$1 shift @@ -119,6 +126,18 @@ pubber/bin/run $pubber_config echo Successful pubber check passed on clean registry. +if [[ -n $solo_arg ]]; then + echo Deleting singular entry... + echo bin/registrar $site_arg $registrar_project -d $solo_arg + bin/registrar $site_arg $registrar_project -d $solo_arg | tee out/registrar.log + fgrep "Deleted 1 devices" out/registrar.log + cp /tmp/registrar_config.json /tmp/alt_config.json + bin/registrar /tmp/alt_config.json $solo_arg | tee out/registrar.log + fgrep "Updating 0 existing devices" out/registrar.log +fi + +false + if [[ -n $UDMI_ALT_REGISTRY ]]; then echo Registering alt registry... site_file=$site_arg/cloud_iot_config.json From 7308bc5ffe0098291b11ee5cb4593fbbd5eb0ad7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 05:51:08 -0500 Subject: [PATCH 3/9] Add solo regclean testing --- bin/test_regclean | 26 +++++++++++++++++--------- etc/shell_common.sh | 4 ++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/bin/test_regclean b/bin/test_regclean index 33966e366..d7d8e75ad 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -3,8 +3,10 @@ # Force consistent sort order export LC_ALL=C -ROOT_DIR=$(dirname $0)/.. -cd $ROOT_DIR +UDMI_ROOT=$(dirname $0)/.. +cd $UDMI_ROOT + +source etc/shell_common.sh solo_arg= @@ -129,20 +131,26 @@ echo Successful pubber check passed on clean registry. if [[ -n $solo_arg ]]; then echo Deleting singular entry... echo bin/registrar $site_arg $registrar_project -d $solo_arg - bin/registrar $site_arg $registrar_project -d $solo_arg | tee out/registrar.log - fgrep "Deleted 1 devices" out/registrar.log + bin/registrar $site_arg $registrar_project -d $solo_arg + search "Deleting device $solo_arg (1/1)" out/registrar.log + search "Deleted 1 devices" out/registrar.log cp /tmp/registrar_config.json /tmp/alt_config.json - bin/registrar /tmp/alt_config.json $solo_arg | tee out/registrar.log - fgrep "Updating 0 existing devices" out/registrar.log + bin/registrar /tmp/alt_config.json $solo_arg + search "Processing 1 new devices..." out/registrar.log + search "Processed $solo_arg (1/1) " out/registrar.log + search "Processed 1 (skipped 0) devices in " out/registrar.log + search "Updating 0 existing devices" out/registrar.log + search "Processed 0 (skipped 0) devices in " out/registrar.log + echo Done with solo update test for $solo_arg fi -false - if [[ -n $UDMI_ALT_REGISTRY ]]; then - echo Registering alt registry... + echo Registering alt registry $UDMI_ALT_REGISTRY... site_file=$site_arg/cloud_iot_config.json cp $site_file $site_file-tmp jq ".registry_id = \"$UDMI_ALT_REGISTRY\"" $site_file-tmp > $site_file + echo bin/registrar $site_arg $registrar_project -u bin/registrar $site_arg $registrar_project -u mv $site_file-tmp $site_file + echo Done with registering alt registry fi diff --git a/etc/shell_common.sh b/etc/shell_common.sh index bb20d67c5..ce686f946 100644 --- a/etc/shell_common.sh +++ b/etc/shell_common.sh @@ -14,6 +14,10 @@ function find_site_model_root { done } +function search { + fgrep "$1" $2 || fail Could not find in $2: $1 +} + function fail { echo error: $* false From e8a1ae0b376829e10c78ead2ec4f0945b0201a70 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 06:07:16 -0500 Subject: [PATCH 4/9] Add better output checking --- bin/test_regclean | 8 ++++---- etc/shell_common.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/test_regclean b/bin/test_regclean index d7d8e75ad..b8003e1e0 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -132,15 +132,15 @@ if [[ -n $solo_arg ]]; then echo Deleting singular entry... echo bin/registrar $site_arg $registrar_project -d $solo_arg bin/registrar $site_arg $registrar_project -d $solo_arg - search "Deleting device $solo_arg (1/1)" out/registrar.log + search "Deleting device $solo_arg \(1/1)" out/registrar.log search "Deleted 1 devices" out/registrar.log cp /tmp/registrar_config.json /tmp/alt_config.json bin/registrar /tmp/alt_config.json $solo_arg search "Processing 1 new devices..." out/registrar.log - search "Processed $solo_arg (1/1) " out/registrar.log - search "Processed 1 (skipped 0) devices in " out/registrar.log + search "Processed $solo_arg \(1/1) in [0-9.]+s \(add)" out/registrar.log + search "Processed 1 \(skipped 0) devices in " out/registrar.log search "Updating 0 existing devices" out/registrar.log - search "Processed 0 (skipped 0) devices in " out/registrar.log + search "Processed 0 \(skipped 0) devices in " out/registrar.log echo Done with solo update test for $solo_arg fi diff --git a/etc/shell_common.sh b/etc/shell_common.sh index ce686f946..169cecf00 100644 --- a/etc/shell_common.sh +++ b/etc/shell_common.sh @@ -15,7 +15,7 @@ function find_site_model_root { } function search { - fgrep "$1" $2 || fail Could not find in $2: $1 + egrep "$1" $2 || fail Could not find in $2: $1 } function fail { From f3cba19e2a521ff00d24647f406ba8f60af2a7ba Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 06:43:04 -0500 Subject: [PATCH 5/9] Clean up error checking --- bin/registrar | 2 ++ bin/test_regclean | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/bin/registrar b/bin/registrar index ba4b0b44d..634eb22f6 100755 --- a/bin/registrar +++ b/bin/registrar @@ -93,3 +93,5 @@ echo validator/bin/registrar $config_file $* 2>&1 | tee $OUT_DIR/registrar.log bin/check_version + +search "Registrar complete, exit 0" $OUT_DIR/registrar.log diff --git a/bin/test_regclean b/bin/test_regclean index b8003e1e0..7d1b49a13 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -ex # Force consistent sort order export LC_ALL=C @@ -129,13 +129,22 @@ pubber/bin/run $pubber_config echo Successful pubber check passed on clean registry. if [[ -n $solo_arg ]]; then - echo Deleting singular entry... + echo Deleting singular entry $solo_arg... echo bin/registrar $site_arg $registrar_project -d $solo_arg bin/registrar $site_arg $registrar_project -d $solo_arg search "Deleting device $solo_arg \(1/1)" out/registrar.log search "Deleted 1 devices" out/registrar.log - cp /tmp/registrar_config.json /tmp/alt_config.json - bin/registrar /tmp/alt_config.json $solo_arg + + echo Munging input config file... + jq ".reflector_endpoint.client_id = \"$UDMI_ALT_REGISTRY\" | .iot_provider = null" \ + /tmp/registrar_config.json > /tmp/alt_config.json + echo bin/registrar /tmp/alt_config.json + ( ! bin/registrar /tmp/alt_config.json ) || fail Was expecting bad config to fail. + + echo bin/registrar /tmp/alt_config.json $registrar_project $solo_arg + bin/registrar /tmp/alt_config.json $registrar_project $solo_arg + + echo Checking creation of one device search "Processing 1 new devices..." out/registrar.log search "Processed $solo_arg \(1/1) in [0-9.]+s \(add)" out/registrar.log search "Processed 1 \(skipped 0) devices in " out/registrar.log From 42b96c08307b666cc5d4e502c5a947e6c0f96b06 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 07:06:19 -0500 Subject: [PATCH 6/9] Clean up project specification --- bin/registrar | 11 ++++++++--- bin/test_regclean | 2 +- .../com/google/daq/mqtt/util/MessagePublisher.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/registrar b/bin/registrar index 634eb22f6..f18bb8f82 100755 --- a/bin/registrar +++ b/bin/registrar @@ -49,8 +49,14 @@ quoted_namespace=null quoted_bridge=null iot_provider=null if [[ -f $site_path ]]; then + if [[ $1 =~ ^// || $1 =~ @ ]]; then + project_id=$1 + shift + fi echo Project_id is $project_id -elif [[ $project_id =~ ^// ]]; then +fi + +if [[ $project_id =~ ^// ]]; then no_prefix=${project_id#//} iot_provider=\"${no_prefix%%/*}\" no_provider=${no_prefix#*/} @@ -70,8 +76,7 @@ else fi if [[ -f $site_path ]]; then - [[ $site_path -ef $config_file ]] || rm -f $config_file - config_file=$site_path + [[ ! $site_path -ef $config_file ]] || cp $site_path $config_file else echo Writing config to $config_file cat < $config_file diff --git a/bin/test_regclean b/bin/test_regclean index 7d1b49a13..c07f1a0cf 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # Force consistent sort order export LC_ALL=C diff --git a/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java b/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java index 70d3b76c6..0f1e57fd2 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java @@ -35,7 +35,7 @@ static MessagePublisher from(ExecutionConfiguration iotConfig, BiConsumer messageHandler, Consumer errorHandler) { IotAccess.IotProvider iotProvider = ofNullable(iotConfig.iot_provider).orElse(IMPLICIT); if (iotConfig.reflector_endpoint != null && iotProvider != IMPLICIT) { - throw new RuntimeException("Explicit endpoint conflicts with iot_provider " + iotProvider); + iotConfig.reflector_endpoint = null; } if (PUBSUB == iotConfig.iot_provider) { return PubSubReflector.from(iotConfig, messageHandler, errorHandler); From 7f25125b12c855f15aaf18df59dfee9e7ebe6ab3 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 07:21:41 -0500 Subject: [PATCH 7/9] Fixing CI testing --- bin/registrar | 2 +- bin/test_regclean | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/bin/registrar b/bin/registrar index f18bb8f82..907fad8bb 100755 --- a/bin/registrar +++ b/bin/registrar @@ -76,7 +76,7 @@ else fi if [[ -f $site_path ]]; then - [[ ! $site_path -ef $config_file ]] || cp $site_path $config_file + [[ $site_path -ef $config_file ]] || cp $site_path $config_file else echo Writing config to $config_file cat < $config_file diff --git a/bin/test_regclean b/bin/test_regclean index c07f1a0cf..dbf8755c1 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -135,14 +135,8 @@ if [[ -n $solo_arg ]]; then search "Deleting device $solo_arg \(1/1)" out/registrar.log search "Deleted 1 devices" out/registrar.log - echo Munging input config file... - jq ".reflector_endpoint.client_id = \"$UDMI_ALT_REGISTRY\" | .iot_provider = null" \ - /tmp/registrar_config.json > /tmp/alt_config.json - echo bin/registrar /tmp/alt_config.json - ( ! bin/registrar /tmp/alt_config.json ) || fail Was expecting bad config to fail. - - echo bin/registrar /tmp/alt_config.json $registrar_project $solo_arg - bin/registrar /tmp/alt_config.json $registrar_project $solo_arg + echo bin/registrar /tmp/registrar_config.json $solo_arg + bin/registrar /tmp/registrar_config.json $solo_arg echo Checking creation of one device search "Processing 1 new devices..." out/registrar.log From 2a9dcb05840ae6a8861e82d8c58ebe785a51de9c Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 07:58:59 -0500 Subject: [PATCH 8/9] Fix test case --- bin/registrar | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/registrar b/bin/registrar index 907fad8bb..d0dcea168 100755 --- a/bin/registrar +++ b/bin/registrar @@ -76,7 +76,8 @@ else fi if [[ -f $site_path ]]; then - [[ $site_path -ef $config_file ]] || cp $site_path $config_file + [[ $site_path -ef $config_file ]] || rm -f $config_file + config_file=$site_path else echo Writing config to $config_file cat < $config_file From 23fdda3d9551a72d0770956aff61a6b73ac6f4fc Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 5 Dec 2023 08:48:14 -0500 Subject: [PATCH 9/9] Add sleep --- bin/test_regclean | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/test_regclean b/bin/test_regclean index dbf8755c1..54fa3918d 100755 --- a/bin/test_regclean +++ b/bin/test_regclean @@ -135,6 +135,9 @@ if [[ -n $solo_arg ]]; then search "Deleting device $solo_arg \(1/1)" out/registrar.log search "Deleted 1 devices" out/registrar.log + echo Sleeping to avoid IoT Core race condition... + sleep 20 + echo bin/registrar /tmp/registrar_config.json $solo_arg bin/registrar /tmp/registrar_config.json $solo_arg