Fix for broken pubber proxy device and corresponding integration test (…

…#475)

.github/workflows/testing.yml

@@ -45,6 +45,7 @@ jobs:
   redirect:
     name: Endpoint Redirection
     runs-on: ubuntu-latest
+    needs: udmi  # Access to UDMI-REFLECTOR is mutually exclusive
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v2
@@ -95,7 +96,6 @@ jobs:
     name: Integration Tests
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    needs: redirect  # Access to UDMI-REFLECTOR is mutually exclusive
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v1
@@ -105,6 +105,15 @@ jobs:
         run: |
           bin/setup_base
           bin/clone_model
+      - name: telemetry validator
+        env:
+          GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
+        if: "${{ env.GCP_TARGET_PROJECT != '' }}"
+        run: |
+          bin/test_validator $GCP_TARGET_PROJECT
+          more /tmp/validator.out
+          echo Comparing test run results with golden file:
+          diff -u /tmp/validator.out etc/validator.out
       - name: sequence tests
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
@@ -117,15 +126,6 @@ jobs:
           ls -1 sites/udmi_site_model/out/devices/AHU-1/tests/ | xargs -I% diff -u \
               sites/udmi_site_model/out/devices/AHU-1/tests/%/sequence.md \
               validator/sequences/%/sequence.md
-      - name: telemetry validator
-        env:
-          GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
-        if: "${{ env.GCP_TARGET_PROJECT != '' }}"
-        run: |
-          bin/test_validator $GCP_TARGET_PROJECT
-          more /tmp/validator.out
-          echo Comparing test run results with golden file:
-          diff -u /tmp/validator.out etc/validator.out
       - name: output logs
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}

bin/test_validator

@@ -15,13 +15,11 @@ shift
 
 site_path=sites/udmi_site_model
 device_id=AHU-1
+proxy_id=AHU-22
 mkdir -p out
 
-serial_no=validator-$RANDOM
-echo Using pubber with serial $serial_no
-
-PUBBER_OUT=pubber.out
-VALIDATOR_OUT=validator.out
+PUBBER_OUT=out/pubber.out
+VALIDATOR_OUT=out/validator.out
 WAITING=10
 
 echo Killing running pubber/validator instances...
@@ -34,9 +32,13 @@ rm -rf $site_path/out/devices
 mkdir -p $site_path/validator
 cp $site_path/devices/AHU-1/rsa_private.pkcs8 $site_path/validator/rsa_private.pkcs8
 
+# Run explicitly so that it's not done in the background when the program is launched.
+validator/bin/build
 pubber/bin/build
 
+# Have to reset pubber configs before running validator to avoid reflector registry conflict
 bin/reset_config $site_path $project_id $device_id
+bin/reset_config $site_path $project_id $proxy_id
 
 echo Starting validator, output in $VALIDATOR_OUT
 bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 &
@@ -60,24 +62,36 @@ if [[ $i -eq $WAITING ]]; then
     false
 fi
 
-echo Writing pubber output to $PUBBER_OUT
-cmd="bin/pubber $site_path $project_id $device_id $serial_no extraField=prlagle"
-echo $cmd
-$cmd > $PUBBER_OUT 2>&1 &
-
-for i in `seq 1 $WAITING`; do
-    if fgrep "Connection complete" $PUBBER_OUT; then
-        break
+function pubber_bg {
+    did=$1
+    shift
+    outfile=$PUBBER_OUT.$did
+    serial_no=validator-$RANDOM
+    echo Writing pubber output to $outfile, serial no $serial_no
+    cmd="bin/pubber $site_path $project_id $did $serial_no $@"
+    echo $cmd
+    $cmd > $outfile 2>&1 &
+
+    # Give a little bit of time to settle before deterministic check
+    sleep 10
+
+    for i in `seq 1 $WAITING`; do
+        if fgrep "Connection complete" $outfile; then
+            break
+        fi
+        echo Waiting for pubber startup $((WAITING - i))...
+        sleep 2
+    done
+
+    if [[ $i -eq $WAITING ]]; then
+        echo pubber startup failed:
+        cat $outfile
+        return 1
     fi
-    echo Waiting for pubber startup $((WAITING - i))...
-    sleep 2
-done
+}
 
-if [[ $i -eq $WAITING ]]; then
-    echo pubber startup failed:
-    cat $PUBBER_OUT
-    false
-fi
+pubber_bg AHU-1 extraField=prlagle
+pubber_bg AHU-22
 
 echo Waiting for system to run for a bit...
 timeout 30s tail -f $VALIDATOR_OUT || true
@@ -86,15 +100,20 @@ echo Killing running pubber/validator instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 ps ax | fgrep validator | fgrep java | awk '{print $1}' | xargs kill || true
 
+TMP_VALIDATOR=/tmp/validator.out
+
+find $site_path/out/devices -name \*.json | sort > $TMP_VALIDATOR
+echo Found `wc -l $TMP_VALIDATOR` json output files.
+
 outfiles=`find $site_path/out/devices -name \*.out | sort` || true
 if [[ -z $outfiles ]]; then
     echo No .out files found in $site_path/out/devices
     false
 fi
 
-echo Found out files $outfiles, copying to /tmp/validator.out
+echo Found out files $outfiles, copying to $TMP_VALIDATOR
 
 # Include /dev/null so that more has >1 to chomp on and outputs headers.
-more /dev/null $outfiles | sed -E > /tmp/validator.out \
+more /dev/null $outfiles | sed -E >> $TMP_VALIDATOR \
         -e "s/[0-9-]{10}T[0-9:]{8}Z/1999-10-20T01:02:03Z/" \
         -e 's/\\t[a-zA-Z .()$0-9]+\.java:[0-9]+\)\\n/\\tredacted\\n/g' \

etc/validator.out

@@ -1,3 +1,17 @@
+sites/udmi_site_model/out/devices/AHU-1/config.json
+sites/udmi_site_model/out/devices/AHU-1/event_pointset.json
+sites/udmi_site_model/out/devices/AHU-1/event_system.json
+sites/udmi_site_model/out/devices/AHU-1/persistent_data.json
+sites/udmi_site_model/out/devices/AHU-1/state.json
+sites/udmi_site_model/out/devices/AHU-1/state_pointset.json
+sites/udmi_site_model/out/devices/AHU-1/state_system.json
+sites/udmi_site_model/out/devices/AHU-22/config.json
+sites/udmi_site_model/out/devices/AHU-22/event_pointset.json
+sites/udmi_site_model/out/devices/AHU-22/event_system.json
+sites/udmi_site_model/out/devices/AHU-22/persistent_data.json
+sites/udmi_site_model/out/devices/AHU-22/state.json
+sites/udmi_site_model/out/devices/AHU-22/state_pointset.json
+sites/udmi_site_model/out/devices/AHU-22/state_system.json
 ::::::::::::::
 /dev/null
 ::::::::::::::

pubber/src/main/java/daq/pubber/MqttPublisher.java

@@ -106,8 +106,11 @@ public class MqttPublisher {
   private String getClientId(String deviceId) {
     // Create our MQTT client. The mqttClientId is a unique string that identifies this device. For
     // Google Cloud IoT, it must be in the format below.
-    if (configuration.endpoint.client_id != null) {
-      return configuration.endpoint.client_id;
+    String configuredClientId = configuration.endpoint.client_id;
+    if (configuredClientId != null) {
+      ClientInfo clientInfo = SiteModel.parseClientId(configuredClientId);
+      return SiteModel.getClientId(clientInfo.projectId, clientInfo.cloudRegion,
+          clientInfo.registryId, deviceId);
     }
     return SiteModel.getClientId(projectId, cloudRegion, registryId, deviceId);
   }