Separate Unit Tests from Integration Tests (#122)

microsoft · Sep 19, 2023 · 75abc46 · 75abc46
1 parent 817667f
commit 75abc46
Show file tree

Hide file tree

Showing 52 changed files with 461 additions and 156 deletions.
diff --git a/TypeChat.sln b/TypeChat.sln
@@ -41,17 +41,20 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.Dialog", "src\type
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HealthData", "examples\HealthData\HealthData.csproj", "{F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.Tests", "tests\TypeChat.Tests\TypeChat.Tests.csproj", "{68ED885B-36C6-454B-BC94-742A2B2BE21B}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.Tests.Pre6", "tests\TypeChat.Pre6.Tests\TypeChat.Tests.Pre6.csproj", "{AF0A444A-F902-40B3-8FD6-60BE130A1143}"
-	ProjectSection(ProjectDependencies) = postProject
-		{68ED885B-36C6-454B-BC94-742A2B2BE21B} = {68ED885B-36C6-454B-BC94-742A2B2BE21B}
-	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat", "src\typechat\TypeChat.csproj", "{EF1231C5-56C4-439F-84F0-FE5D5315D4AA}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.IntegrationTests", "tests\TypeChat.IntegrationTests\TypeChat.IntegrationTests.csproj", "{918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.TestLib", "tests\TypeChat.TestLib\TypeChat.TestLib.csproj", "{B7F278E8-CE85-4987-B292-F15238A80A75}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SchemaHierarchy", "examples\SchemaHierarchy\SchemaHierarchy.csproj", "{6FDA463B-4A1E-4E3E-B508-A14910FAF579}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.UnitTests", "tests\TypeChat.UnitTests\TypeChat.UnitTests.csproj", "{C585B613-3BAF-4999-8FA5-7BF425AEB1E2}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{EAEA9E40-7370-407E-B2DA-3F6E265262BF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -110,10 +113,6 @@ Global
 		{F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{68ED885B-36C6-454B-BC94-742A2B2BE21B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{68ED885B-36C6-454B-BC94-742A2B2BE21B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{68ED885B-36C6-454B-BC94-742A2B2BE21B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{68ED885B-36C6-454B-BC94-742A2B2BE21B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{AF0A444A-F902-40B3-8FD6-60BE130A1143}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{AF0A444A-F902-40B3-8FD6-60BE130A1143}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AF0A444A-F902-40B3-8FD6-60BE130A1143}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -122,10 +121,22 @@ Global
 		{EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B7F278E8-CE85-4987-B292-F15238A80A75}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B7F278E8-CE85-4987-B292-F15238A80A75}.Release|Any CPU.Build.0 = Release|Any CPU
 		{6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -140,7 +151,11 @@ Global
 		{5F8B1030-2D87-4942-BCB3-271B4541BAF3} = {179655A3-3581-45F4-BE46-1BF4FABA7F97}
 		{9598D98A-3688-45E5-9C51-1C4F385BD693} = {179655A3-3581-45F4-BE46-1BF4FABA7F97}
 		{F4B127FE-CAB9-483E-AA79-6A2F8665C5FC} = {179655A3-3581-45F4-BE46-1BF4FABA7F97}
+		{AF0A444A-F902-40B3-8FD6-60BE130A1143} = {EAEA9E40-7370-407E-B2DA-3F6E265262BF}
+		{918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE} = {EAEA9E40-7370-407E-B2DA-3F6E265262BF}
+		{B7F278E8-CE85-4987-B292-F15238A80A75} = {EAEA9E40-7370-407E-B2DA-3F6E265262BF}
 		{6FDA463B-4A1E-4E3E-B508-A14910FAF579} = {179655A3-3581-45F4-BE46-1BF4FABA7F97}
+		{C585B613-3BAF-4999-8FA5-7BF425AEB1E2} = {EAEA9E40-7370-407E-B2DA-3F6E265262BF}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {8A9AF678-E669-4AF2-9644-7EBB970DAC41}

diff --git a/src/typechat.app/Config.cs b/src/typechat.app/Config.cs
@@ -74,12 +74,15 @@ public static void ApplyEnvFile(string filePath)
         string line = null;
         while ((line = reader.ReadLine()) != null)
         {
-            if (line.StartsWith('#'))
+            if (line.StartsWith("#"))
             {
                 continue;
             }
-
+#if NET6_0_OR_GREATER
             string[] envVars = line.Split('=', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+#else
+            string[] envVars = line.Split('=', (char)StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToArray();
+#endif
             if (envVars != null)
             {
                 UpdateVariable(envVars);

diff --git a/src/typechat.app/ConsoleApp.cs b/src/typechat.app/ConsoleApp.cs
@@ -128,9 +128,13 @@ bool IsStop(string? line)
         return _stopStrings.Contains(line, StringComparer.OrdinalIgnoreCase);
     }
 
-    public async Task<string?> ReadLineAsync(CancellationToken cancelToken)
+    public async Task<string?> ReadLineAsync(CancellationToken cancelToken = default)
     {
+#if NET7_0_OR_GREATER
         string? line = await Console.In.ReadLineAsync(cancelToken).ConfigureAwait(false);
+#else
+        string? line = await Console.In.ReadLineAsync().ConfigureAwait(false);
+#endif
         return (line != null) ? line.Trim() : line;
     }
 

diff --git a/src/typechat.app/HierarchicalJsonTranslator.cs b/src/typechat.app/HierarchicalJsonTranslator.cs
@@ -23,10 +23,8 @@ public class HierarchicalJsonTranslator : IJsonTranslator
     /// <param name="embeddingModel">embedding model to use for translators</param>
     public HierarchicalJsonTranslator(LanguageModel model, TextEmbeddingModel embeddingModel)
     {
-        if (model == null)
-        {
-            throw new ArgumentNullException(nameof(model));
-        }
+        ArgumentVerify.ThrowIfNull(model, nameof(model));
+        ArgumentVerify.ThrowIfNull(embeddingModel, nameof(embeddingModel));
         _model = model;
         _requestRouter = new VectorTextIndex<IJsonTranslator>(embeddingModel);
     }

diff --git a/src/typechat.app/TypeChat.App.csproj b/src/typechat.app/TypeChat.App.csproj
@@ -1,13 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0;net60;net70</TargetFrameworks>
+    <LangVersion>10</LangVersion>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>annotations</Nullable>
     <AssemblyName>TypeChat.App</AssemblyName>
     <RootNamespace>Microsoft.TypeChat</RootNamespace>
+    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Include="..\typechat\ArgumentVerify.cs" Link="ArgumentVerify.cs" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\typechat.dialog\TypeChat.Dialog.csproj" />
     <ProjectReference Include="..\typechat.program\TypeChat.Program.csproj" />
@@ -16,9 +22,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>
 

diff --git a/tests/TypeChat.Tests/Extensions.cs → ...s/TypeChat.IntegrationTests/Extensions.cs b/tests/TypeChat.Tests/Extensions.cs → ...s/TypeChat.IntegrationTests/Extensions.cs
@@ -58,7 +58,11 @@ public static float[] FloatArray(this Random random, int count)
         float[] array = new float[count];
         for (int i = 0; i < count; ++i)
         {
+#if NET6_0_OR_GREATER
             array[i] = random.NextSingle();
+#else
+            array[i] = (float)random.NextDouble();
+#endif
         }
 
         return array;

diff --git a/tests/TypeChat.Tests/Includes.cs → tests/TypeChat.IntegrationTests/Includes.cs b/tests/TypeChat.Tests/Includes.cs → tests/TypeChat.IntegrationTests/Includes.cs
diff --git a/tests/TypeChat.Tests/Readme.md → tests/TypeChat.IntegrationTests/Readme.md b/tests/TypeChat.Tests/Readme.md → tests/TypeChat.IntegrationTests/Readme.md
@@ -1,5 +1,5 @@
 # Running Tests
-Some unit tests require an Open AI key and will just skip if one is not available. 
+The integration tests require an Open AI key and will just skip if one is not available. 
 To supply a key:
 - Create appSettings.Development.json in this folder and override the ApiKey property defined in appSettings.json
 - In Visual Studio, set File properties to Copy Always

diff --git a/tests/TypeChat.Tests/TestAgent.cs → tests/TypeChat.IntegrationTests/TestAgent.cs b/tests/TypeChat.Tests/TestAgent.cs → tests/TypeChat.IntegrationTests/TestAgent.cs
@@ -41,13 +41,10 @@ public void TestMessageStream()
         }
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task TestEndToEnd()
     {
-        if (!CanRunEndToEndTest(_config))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
 
         AgentWithHistory<Order> agent = new AgentWithHistory<Order>(_config.CreateTranslator<Order>());
         agent.CreateMessageForHistory = (r) => null; // Don't remember responses. 

diff --git a/tests/TypeChat.Tests/TestClassification.cs → ...at.IntegrationTests/TestClassification.cs b/tests/TypeChat.Tests/TestClassification.cs → ...at.IntegrationTests/TestClassification.cs
@@ -26,13 +26,11 @@ public void TestRouter()
         Assert.False(router.Routes.ContainsKey("Foo"));
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task TestRouting()
     {
-        if (!CanRunEndToEndTest(_config, nameof(TestRouting)))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
+
         TextRequestRouter<string> router = CreateRouter();
         string query = "I want to buy a Sherlock Holmes novel";
         string route = await router.RouteRequestAsync(query);

diff --git a/tests/TypeChat.Tests/TestEndToEnd.cs → ...TypeChat.IntegrationTests/TestEndToEnd.cs b/tests/TypeChat.Tests/TestEndToEnd.cs → ...TypeChat.IntegrationTests/TestEndToEnd.cs
@@ -1,5 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using Xunit;
+using Xunit.Sdk;
+
 namespace Microsoft.TypeChat.Tests;
 
 public class TestEndToEnd : TypeChatTest, IClassFixture<Config>
@@ -12,24 +15,18 @@ public TestEndToEnd(Config config, ITestOutputHelper output)
         _config = config;
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task TranslateSentiment()
     {
-        if (!CanRunEndToEndTest(_config, nameof(TranslateSentiment)))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
         await TranslateSentiment(new LanguageModel(_config.OpenAI));
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task TranslateSentiment_CompletionModel()
     {
-        if (!CanRunEndToEndTest(_config, nameof(TranslateSentiment_CompletionModel)))
-        {
-            Trace.WriteLine("No Open AI. Skipping");
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
+
         await TranslateSentiment(new TextCompletionModel(_config.OpenAI));
     }
 
@@ -48,13 +45,10 @@ async Task TranslateSentiment(ILanguageModel model)
         Assert.NotNull(response.Sentiment);
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task Translate_Polymorphic()
     {
-        if (!CanRunEndToEndTest(_config, nameof(Translate_Polymorphic)))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
 
         var translator = new JsonTranslator<Drawing>(new LanguageModel(_config.OpenAI));
         string request = "Add a circle of radius 4.5 at 30, 30 and\n" +
@@ -77,14 +71,10 @@ public async Task Translate_Polymorphic()
 
     /// <summary>
     /// This one loads the schema from a TS file
-    [Fact]
+    [SkippableFact]
     public async Task TranslateWithTSFileSchema()
     {
-        if (!CanRunEndToEndTest(_config))
-        {
-            Trace.WriteLine("No Open AI. Skipping");
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
 
         SchemaText schema = SchemaText.Load("./SentimentSchema.ts");
         var translator = new JsonTranslator<SentimentResponse>(
@@ -100,25 +90,19 @@ public async Task TranslateWithTSFileSchema()
         Assert.NotNull(response.Sentiment);
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task ProgramMath()
     {
-        if (!CanRunEndToEndTest(_config))
-        {
-            Trace.WriteLine("No Open AI. Skipping");
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
+
         await ProgramMath(new LanguageModel(_config.OpenAI));
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task ProgramMath_Completion()
     {
-        if (!CanRunEndToEndTest(_config))
-        {
-            Trace.WriteLine("No Open AI. Skipping");
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
+
         await ProgramMath(new TextCompletionModel(_config.OpenAI));
     }
 

diff --git a/tests/TypeChat.Tests/TestKernel.cs → ...s/TypeChat.IntegrationTests/TestKernel.cs b/tests/TypeChat.Tests/TestKernel.cs → ...s/TypeChat.IntegrationTests/TestKernel.cs
@@ -12,13 +12,11 @@ public TestKernel(ITestOutputHelper output, Config config)
         _config = config;
     }
 
-    [Fact]
+    [SkippableFact]
     public void TestKernelBuild()
     {
-        if (!CanRunEndToEndTest(_config, nameof(TestKernelBuild)))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
+
         KernelBuilder kb = new KernelBuilder();
         kb.WithChatModels(_config.OpenAI, Config.ModelNames.Gpt35Turbo, Config.ModelNames.Gpt4);
         IKernel kernel = kb.Build();
@@ -32,27 +30,21 @@ public void TestKernelBuild()
         Assert.Equal(languageModel.ModelInfo.Name, Config.ModelNames.Gpt4);
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task TestLanguageModel()
     {
-        if (!CanRunEndToEndTest(_config, nameof(TestLanguageModel)))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
 
         TextCompletionModel lm = new TextCompletionModel(_config.OpenAI);
         string response = await lm.CompleteAsync("Is Venus a planet?");
         Assert.NotNull(response);
         Assert.NotEmpty(response);
     }
 
-    [Fact]
+    [SkippableFact]
     public async Task TestChatModel()
     {
-        if (!CanRunEndToEndTest(_config, nameof(TestLanguageModel)))
-        {
-            return;
-        }
+        Skip.If(!CanRunEndToEndTest(_config));
 
         LanguageModel cm = new LanguageModel(_config.OpenAI);
         Prompt prompt = "Is Venus a planet?";