Support additional event hub consumer groups for testing environments

builds/e2e/longhaul.yaml

@@ -89,6 +89,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'longhaul_linux_amd64'
 
 ################################################################################
   - job: linux_arm32v7_moby
@@ -154,6 +155,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.arm32v7)'
+          analyzer.consumerGroupId: 'longhaul_linux_arm32v7'
 
 ################################################################################
   - job: linux_arm64v8_docker
@@ -219,6 +221,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.arm64v8)'
+          analyzer.consumerGroupId: 'longhaul_linux_arm64v8'
 
 ################################################################################
   - job: windows_amd64_moby
@@ -286,6 +289,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'longhaul_windows_amd64'
 
 ################################################################################
   - job: windows_servercore_amd64_moby
@@ -353,6 +357,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'longhaul_windows_servercore_amd64'
 
 ################################################################################
   - job: iotuap_amd64_moby
@@ -419,3 +424,4 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'longhaul_iotuap_amd64'

builds/e2e/stresstest.yaml

@@ -89,6 +89,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'stress_linux_amd64'
 
 ################################################################################
   - job: linux_arm32v7_moby
@@ -154,6 +155,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: $(loadGen.message.frequency.arm32v7)
+          analyzer.consumerGroupId: 'stress_linux_arm32v7'
 
 ################################################################################
   - job: linux_arm64v8_docker
@@ -219,6 +221,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: $(loadGen.message.frequency.arm64v8)
+          analyzer.consumerGroupId: 'stress_linux_arm64v8'
 
 ################################################################################
   - job: windows_amd64_moby
@@ -286,6 +289,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'stress_windows_amd64'
 
 ################################################################################
   - job: windows_servercore_amd64_moby
@@ -353,6 +357,7 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'stress_windows_servercore_amd64'
 
 ################################################################################
   - job: iotuap_amd64_moby
@@ -419,3 +424,4 @@ jobs:
           snitch.storage.account: '$(snitch.storage.account)'
           snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
           loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          analyzer.consumerGroupId: 'stress_iotuap_amd64'

builds/e2e/templates/longhaul-deploy-windows.yaml

@@ -34,5 +34,5 @@ steps:
         #Requires -RunAsAdministrator
         $testName="LongHaul"
         $ArtifactInfo=convertfrom-stringdata (get-content $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/artifactInfo.txt -raw)
-        $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/windows/test/Run-E2ETest.ps1 -E2ETestFolder "$(Agent.HomeDirectory)/.." -ReleaseLabel "${{ parameters['release.label'] }}" -ArtifactImageBuildNumber "$($ArtifactInfo.BuildNumber)" -TestName "$testName" -ContainerRegistry "${{ parameters['container.registry'] }}" -ContainerRegistryUsername "${{ parameters['container.registry.username'] }}" -ContainerRegistryPassword "${{ parameters['container.registry.password'] }}" -IoTHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -EventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -SnitchBuildNumber "${{ parameters['snitch.build.number'] }}" -SnitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -SnitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -SnitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -LoadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}"
+        $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/windows/test/Run-E2ETest.ps1 -E2ETestFolder "$(Agent.HomeDirectory)/.." -ReleaseLabel "${{ parameters['release.label'] }}" -ArtifactImageBuildNumber "$($ArtifactInfo.BuildNumber)" -TestName "$testName" -ContainerRegistry "${{ parameters['container.registry'] }}" -ContainerRegistryUsername "${{ parameters['container.registry.username'] }}" -ContainerRegistryPassword "${{ parameters['container.registry.password'] }}" -IoTHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -EventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -SnitchBuildNumber "${{ parameters['snitch.build.number'] }}" -SnitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -SnitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -SnitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -LoadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}" -EventHubConsumerGroupId "${{ parameters['analyzer.consumerGroupId'] }}"
       workingDirectory: "$(Agent.HomeDirectory)/.."

builds/e2e/templates/longhaul-deploy.yaml

@@ -13,6 +13,7 @@ parameters:
   snitch.storage.masterKey: ''
   loadGen.message.frequency: ''
   loadGen.transportType: 'mqtt'
+  analyzer.consumerGroupId: ''
 
 steps:
   - task: CopyFiles@2
@@ -36,5 +37,5 @@ steps:
         testName="LongHaul"
         . $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/artifactInfo.txt
         chmod +x $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/linux/runE2ETest.sh
-        sudo $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/linux/runE2ETest.sh -testDir "$(Agent.HomeDirectory)/.." -releaseLabel "${{ parameters['release.label'] }}" -artifactImageBuildNumber "$BuildNumber" -testName "$testName" -containerRegistry "${{ parameters['container.registry'] }}" -containerRegistryUsername "${{ parameters['container.registry.username'] }}" -containerRegistryPassword "${{ parameters['container.registry.password'] }}" -iotHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -eventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -snitchBuildNumber "${{ parameters['snitch.build.number'] }}" -snitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -snitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -snitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -loadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}" -cleanAll
+        sudo $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/linux/runE2ETest.sh -testDir "$(Agent.HomeDirectory)/.." -releaseLabel "${{ parameters['release.label'] }}" -artifactImageBuildNumber "$BuildNumber" -testName "$testName" -containerRegistry "${{ parameters['container.registry'] }}" -containerRegistryUsername "${{ parameters['container.registry.username'] }}" -containerRegistryPassword "${{ parameters['container.registry.password'] }}" -iotHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -eventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -snitchBuildNumber "${{ parameters['snitch.build.number'] }}" -snitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -snitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -snitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -loadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}" -eventHubConsumerGroupId "${{ parameters['analyzer.consumerGroupId'] }}" -cleanAll
       workingDirectory: "$(Agent.HomeDirectory)/.."

builds/e2e/templates/stresstest-deploy-windows.yaml

@@ -40,7 +40,7 @@ steps:
         #Requires -RunAsAdministrator
         $testName="Stress"
         $ArtifactInfo=convertfrom-stringdata (get-content $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/artifactInfo.txt -raw)
-        $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/windows/test/Run-E2ETest.ps1 -E2ETestFolder "$(Agent.HomeDirectory)/.." -ReleaseLabel "${{ parameters['release.label'] }}" -ArtifactImageBuildNumber "$($ArtifactInfo.BuildNumber)" -TestName "$testName" -ContainerRegistry "${{ parameters['container.registry'] }}" -ContainerRegistryUsername "${{ parameters['container.registry.username'] }}" -ContainerRegistryPassword "${{ parameters['container.registry.password'] }}" -IoTHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -EventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -SnitchBuildNumber "${{ parameters['snitch.build.number'] }}" -SnitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -SnitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -SnitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -LoadGen1TransportType "${{ parameters['loadGen1.transportType'] }}" -LoadGen2TransportType "${{ parameters['loadGen2.transportType'] }}" -LoadGen3TransportType "${{ parameters['loadGen3.transportType'] }}" -LoadGen4TransportType "${{ parameters['loadGen4.transportType'] }}" -AmqpSettingsEnabled "${{ parameters['amqp.settings.enabled'] }}" -MqttSettingsEnabled "${{ parameters['mqtt.settings.enabled'] }}" -LoadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}"
+        $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/windows/test/Run-E2ETest.ps1 -E2ETestFolder "$(Agent.HomeDirectory)/.." -ReleaseLabel "${{ parameters['release.label'] }}" -ArtifactImageBuildNumber "$($ArtifactInfo.BuildNumber)" -TestName "$testName" -ContainerRegistry "${{ parameters['container.registry'] }}" -ContainerRegistryUsername "${{ parameters['container.registry.username'] }}" -ContainerRegistryPassword "${{ parameters['container.registry.password'] }}" -IoTHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -EventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -SnitchBuildNumber "${{ parameters['snitch.build.number'] }}" -SnitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -SnitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -SnitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -LoadGen1TransportType "${{ parameters['loadGen1.transportType'] }}" -LoadGen2TransportType "${{ parameters['loadGen2.transportType'] }}" -LoadGen3TransportType "${{ parameters['loadGen3.transportType'] }}" -LoadGen4TransportType "${{ parameters['loadGen4.transportType'] }}" -AmqpSettingsEnabled "${{ parameters['amqp.settings.enabled'] }}" -MqttSettingsEnabled "${{ parameters['mqtt.settings.enabled'] }}" -LoadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}" -EventHubConsumerGroupId "${{ parameters['analyzer.consumerGroupId'] }}"
         # schedule a task to stop iotedge service (terminate the test) in 4:30h
         $ScheduleDatetime = [DateTime]::Now.AddMinutes(270)
         $ScheduleDate=$ScheduleDatetime.ToString("MM/dd/yyyy")

builds/e2e/templates/stresstest-deploy.yaml

@@ -18,6 +18,7 @@ parameters:
   loadGen2.transportType: 'Amqp_WebSocket_Only'
   loadGen3.transportType: 'Mqtt_Tcp_Only'
   loadGen4.transportType: 'Mqtt_WebSocket_Only'
+  analyzer.consumerGroupId: ''
 
 steps:
   - task: CopyFiles@2
@@ -42,7 +43,7 @@ steps:
         . $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/artifactInfo.txt
         echo "snitcher duration in mins=$(snitcher.test.duration.in.mins)"
         chmod +x $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/linux/runE2ETest.sh
-        sudo $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/linux/runE2ETest.sh -testDir "$(Agent.HomeDirectory)/.." -releaseLabel "${{ parameters['release.label'] }}" -artifactImageBuildNumber "$BuildNumber" -testName "$testName" -containerRegistry "${{ parameters['container.registry'] }}" -containerRegistryUsername "${{ parameters['container.registry.username'] }}" -containerRegistryPassword "${{ parameters['container.registry.password'] }}" -iotHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -eventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -snitchBuildNumber "${{ parameters['snitch.build.number'] }}" -snitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -snitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -snitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -loadGen1TransportType "${{ parameters['loadGen1.transportType'] }}" -loadGen2TransportType "${{ parameters['loadGen2.transportType'] }}" -loadGen3TransportType "${{ parameters['loadGen3.transportType'] }}" -loadGen4TransportType "${{ parameters['loadGen4.transportType'] }}" -amqpSettingsEnabled "${{ parameters['amqp.settings.enabled'] }}" -mqttSettingsEnabled "${{ parameters['mqtt.settings.enabled'] }}" -loadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}" -snitchTestDurationInSecs "$(($(snitcher.test.duration.in.mins)*60))" -cleanAll
+        sudo $(Agent.HomeDirectory)/../artifacts/${{ parameters['images.artifact.name'] }}/scripts/linux/runE2ETest.sh -testDir "$(Agent.HomeDirectory)/.." -releaseLabel "${{ parameters['release.label'] }}" -artifactImageBuildNumber "$BuildNumber" -testName "$testName" -containerRegistry "${{ parameters['container.registry'] }}" -containerRegistryUsername "${{ parameters['container.registry.username'] }}" -containerRegistryPassword "${{ parameters['container.registry.password'] }}" -iotHubConnectionString "${{ parameters['iotHub.connectionString'] }}" -eventHubConnectionString "${{ parameters['eventHub.connectionString'] }}" -snitchBuildNumber "${{ parameters['snitch.build.number'] }}" -snitchStorageAccount "${{ parameters['snitch.storage.account'] }}" -snitchStorageMasterKey "${{ parameters['snitch.storage.masterKey'] }}" -snitchAlertUrl "${{ parameters['snitch.alert.url'] }}" -loadGen1TransportType "${{ parameters['loadGen1.transportType'] }}" -loadGen2TransportType "${{ parameters['loadGen2.transportType'] }}" -loadGen3TransportType "${{ parameters['loadGen3.transportType'] }}" -loadGen4TransportType "${{ parameters['loadGen4.transportType'] }}" -amqpSettingsEnabled "${{ parameters['amqp.settings.enabled'] }}" -mqttSettingsEnabled "${{ parameters['mqtt.settings.enabled'] }}" -loadGenMessageFrequency "${{ parameters['loadGen.message.frequency'] }}" -snitchTestDurationInSecs "$(($(snitcher.test.duration.in.mins)*60))" -eventHubConsumerGroupId "${{ parameters['analyzer.consumerGroupId'] }}" -cleanAll
         #Remove all pending jobs
         atrm $(atq | cut -f1)
         durationToStop=$(($(snitcher.test.duration.in.mins)+30))

edge-modules/MessagesAnalyzer/Program.cs

@@ -19,7 +19,7 @@ class Program
 
         static async Task Main(string[] args)
         {
-            Log.LogInformation($"Starting analyzer for deviceId: {Settings.Current.DeviceId}, exclude-modules: {string.Join(", ", Settings.Current.ExcludedModuleIds.ToArray())}");
+            Log.LogInformation($"Starting analyzer for [deviceId: {Settings.Current.DeviceId}] with [consumerGroupId: {Settings.Current.ConsumerGroupId}], exclude-modules: {string.Join(", ", Settings.Current.ExcludedModuleIds.ToArray())}");
 
             await ReceiveMessages();
 

scripts/linux/runE2ETest.sh

@@ -170,7 +170,7 @@ function prepare_test_from_artifacts() {
                 local escapedSnitchAlertUrl
                 local escapedBuildId
                 sed -i -e "s@<Analyzer.EventHubConnectionString>@$EVENTHUB_CONNECTION_STRING@g" "$deployment_working_file"
-                sed -i -e "s@<Analyzer.ConsumerGroupId>@${EVENT_HUB_CONSUMER_GROUP:-\$Default}@g" "$deployment_working_file"
+                sed -i -e "s@<Analyzer.ConsumerGroupId>@${EVENT_HUB_CONSUMER_GROUP_ID:-\$Default}@g" "$deployment_working_file"
                 sed -i -e "s@<LoadGen.MessageFrequency>@$LOADGEN_MESSAGE_FREQUENCY@g" "$deployment_working_file"
                 escapedSnitchAlertUrl="${SNITCH_ALERT_URL//&/\\&}"
                 escapedBuildId="${ARTIFACT_IMAGE_BUILD_NUMBER//./}"
@@ -350,7 +350,7 @@ function process_args() {
             DPS_MASTER_SYMMETRIC_KEY="$arg"
             saveNextArg=0
         elif [ $saveNextArg -eq 29 ]; then
-            EVENT_HUB_CONSUMER_GROUP="$arg"
+            EVENT_HUB_CONSUMER_GROUP_ID="$arg"
             saveNextArg=0
         else
             case "$arg" in
@@ -383,7 +383,7 @@ function process_args() {
                 '-installRootCAKeyPassword' ) saveNextArg=26;;
                 '-dpsScopeId' ) saveNextArg=27;;
                 '-dpsMasterSymmetricKey' ) saveNextArg=28;;
-                '-eventHubConsumerGroup' ) saveNextArg=29;;
+                '-eventHubConsumerGroupId' ) saveNextArg=29;;
                 '-cleanAll' ) CLEAN_ALL=1;;
                 * ) usage;;
             esac

scripts/windows/test/Run-E2ETest.ps1

@@ -44,6 +44,9 @@
     .PARAMETER EventHubConnectionString
         Event hub connection string for receive D2C messages
 
+    .PARAMETER EventHubConsumerGroupId
+        Event hub consumer group id needed to avoid overloading the default Event Hub endpoint
+
     .PARAMETER ProxyUri
         (Optional) The URI of an HTTPS proxy server; if specified, all communications to IoT Hub will go through this proxy.
 
@@ -206,6 +209,8 @@ Param (
     [ValidateNotNullOrEmpty()]
     [string] $EventHubConnectionString = $(Throw "Event hub connection string is required"),
 
+    [string] $EventHubConsumerGroupId = $null,
+
     [string] $EdgeE2ERootCACertRSAFile = $null,
 
     [string] $EdgeE2ERootCAKeyRSAFile = $null,
@@ -431,6 +436,7 @@ Function PrepareTestFromArtifacts
                 }
 
                 (Get-Content $DeploymentWorkingFilePath).replace('<Analyzer.EventHubConnectionString>',$EventHubConnectionString) | Set-Content $DeploymentWorkingFilePath
+                (Get-Content $DeploymentWorkingFilePath).replace('<Analyzer.ConsumerGroupId>',$EventHubConsumerGroupId) | Set-Content $DeploymentWorkingFilePath
                 (Get-Content $DeploymentWorkingFilePath).replace('<LoadGen.MessageFrequency>',$LoadGenMessageFrequency) | Set-Content $DeploymentWorkingFilePath
                 $escapedBuildId= $ArtifactImageBuildNumber -replace "\.",""
                 (Get-Content $DeploymentWorkingFilePath).replace('<Snitch.AlertUrl>',$SnitchAlertUrl) | Set-Content $DeploymentWorkingFilePath