Add test for multiple function hosts

performance/SqlTriggerBindingPerformance.cs

@@ -32,7 +32,7 @@ await this.WaitForProductChanges(
                 () => { this.InsertProducts(1, count); return Task.CompletedTask; },
                 id => $"Product {id}",
                 id => id * 100,
-                GetBatchProcessingTimeout(1, count));
+                this.GetBatchProcessingTimeout(1, count));
         }
 
         [IterationCleanup]

test/Integration/IntegrationTestBase.cs

@@ -1,33 +1,39 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
-using Microsoft.Data.SqlClient;
-using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common;
 using System;
+using System.Collections.Generic;
 using System.Data.Common;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
 using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.WebUtilities;
+using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common;
+using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common;
+using Microsoft.Data.SqlClient;
 using Xunit;
 using Xunit.Abstractions;
-using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common;
-using Microsoft.AspNetCore.WebUtilities;
-using System.Collections.Generic;
-using System.Linq;
+
 using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry;
 
 namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration
 {
     public class IntegrationTestBase : IDisposable
     {
         /// <summary>
-        /// Host process for Azure Function CLI
+        /// The first Function Host process that was started. Null if no process has been started yet.
+        /// </summary>
+        protected Process FunctionHost => this.FunctionHostList.FirstOrDefault();
+
+        /// <summary>
+        /// Host processes for Azure Function CLI.
         /// </summary>
-        protected Process FunctionHost { get; private set; }
+        protected List<Process> FunctionHostList { get; } = new List<Process>();
 
         /// <summary>
         /// Host process for Azurite local storage emulator. This is required for non-HTTP trigger functions:
@@ -184,12 +190,16 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag
             {
                 throw new FileNotFoundException("Working directory not found at " + workingDirectory);
             }
+
+            // Use a different port for each new host process, starting with the default port number: 7071.
+            int port = this.Port + this.FunctionHostList.Count;
+
             var startInfo = new ProcessStartInfo
             {
                 // The full path to the Functions CLI is required in the ProcessStartInfo because UseShellExecute is set to false.
                 // We cannot both use shell execute and redirect output at the same time: https://docs.microsoft.com//dotnet/api/system.diagnostics.processstartinfo.redirectstandardoutput#remarks
                 FileName = GetFunctionsCoreToolsPath(),
-                Arguments = $"start --verbose --port {this.Port} --functions {functionName}",
+                Arguments = $"start --verbose --port {port} --functions {functionName}",
                 WorkingDirectory = workingDirectory,
                 WindowStyle = ProcessWindowStyle.Hidden,
                 RedirectStandardOutput = true,
@@ -205,24 +215,26 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag
             startInfo.EnvironmentVariables[TelemetryOptoutEnvVar] = "1";
 
             this.LogOutput($"Starting {startInfo.FileName} {startInfo.Arguments} in {startInfo.WorkingDirectory}");
-            this.FunctionHost = new Process
+
+            var functionHost = new Process
             {
                 StartInfo = startInfo
             };
 
+            this.FunctionHostList.Add(functionHost);
+
             // Register all handlers before starting the functions host process.
             var taskCompletionSource = new TaskCompletionSource<bool>();
-            this.FunctionHost.OutputDataReceived += this.TestOutputHandler;
-            this.FunctionHost.OutputDataReceived += SignalStartupHandler;
+            functionHost.OutputDataReceived += SignalStartupHandler;
             this.FunctionHost.OutputDataReceived += customOutputHandler;
 
-            this.FunctionHost.ErrorDataReceived += this.TestOutputHandler;
+            functionHost.Start();
+            functionHost.OutputDataReceived += this.GetTestOutputHandler(functionHost.Id);
+            functionHost.ErrorDataReceived += this.GetTestOutputHandler(functionHost.Id);
+            functionHost.BeginOutputReadLine();
+            functionHost.BeginErrorReadLine();
 
-            this.FunctionHost.Start();
-            this.FunctionHost.BeginOutputReadLine();
-            this.FunctionHost.BeginErrorReadLine();
-
-            this.LogOutput($"Waiting for Azure Function host to start...");
+            this.LogOutput("Waiting for Azure Function host to start...");
 
             const int FunctionHostStartupTimeoutInSeconds = 60;
             bool isCompleted = taskCompletionSource.Task.Wait(TimeSpan.FromSeconds(FunctionHostStartupTimeoutInSeconds));
@@ -233,7 +245,7 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag
             const int BufferTimeInSeconds = 5;
             Task.Delay(TimeSpan.FromSeconds(BufferTimeInSeconds)).Wait();
 
-            this.LogOutput($"Azure Function host started!");
+            this.LogOutput("Azure Function host started!");
             this.FunctionHost.OutputDataReceived -= SignalStartupHandler;
 
             void SignalStartupHandler(object sender, DataReceivedEventArgs e)
@@ -293,11 +305,16 @@ private void LogOutput(string output)
             }
         }
 
-        private void TestOutputHandler(object sender, DataReceivedEventArgs e)
+        private DataReceivedEventHandler GetTestOutputHandler(int processId)
         {
-            if (e != null && !string.IsNullOrEmpty(e.Data))
+            return TestOutputHandler;
+
+            void TestOutputHandler(object sender, DataReceivedEventArgs e)
             {
-                this.LogOutput(e.Data);
+                if (!string.IsNullOrEmpty(e.Data))
+                {
+                    this.LogOutput($"[{processId}] {e.Data}");
+                }
             }
         }
 
@@ -377,14 +394,17 @@ public void Dispose()
                 this.LogOutput($"Failed to close connection. Error: {e1.Message}");
             }
 
-            try
-            {
-                this.FunctionHost?.Kill();
-                this.FunctionHost?.Dispose();
-            }
-            catch (Exception e2)
+            foreach (Process functionHost in this.FunctionHostList)
             {
-                this.LogOutput($"Failed to stop function host, Error: {e2.Message}");
+                try
+                {
+                    functionHost.Kill();
+                    functionHost.Dispose();
+                }
+                catch (Exception e2)
+                {
+                    this.LogOutput($"Failed to stop function host, Error: {e2.Message}");
+                }
             }
 
             try