Skip to content

Commit b679b3d

Browse files
shurivichandrewlock
shurivich
and
andrewlock
authored
Implementation of Exploration Tests (DataDog#2089)
* Implementation of Exploration Tests * add memory dump + clean * remove redundant 'git' * Update tracer/build/_build/Build.ExplorationTests.cs Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com> * Update tracer/build/_build/Build.ExplorationTests.cs Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com> * Update Build.ExplorationTests.cs * Update DotNetSettingsExtensions.cs * Replace targets with method functions * Replace 'ExplorationTestName' with enum + rename parameters * Refactor environment variables * Refactor BuildVariables with extensions + fix ilspy git repo * readable if * Rename Profiler -> ContinuousProfiler * Rename monitoringType -> explorationTestUseCase * Renamings * fix display name * build tracer home during exploration test + fix ilspy exploration test * fix property * rename env variables + add MoveLogsToBuildData method * rename PrepareExplorationTestEnvironment -> SetUpExplorationTest * add skip BuildTracerHome * fix CI * add x86 .net sdk * more time to wait * clone quietly * -c advice.detachedHead=false * no-warnings * customize build process for different cases * try..finally * move helper classes * add "DD_SERVICE" and "DD_VERSION" Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com> * remove "DD_INTEGRATIONS" Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com> * disable ilspy * remove publish test results * Revert "remove publish test results" This reverts commit cb2d053. * remover publish test results Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com>
1 parent d7b21b4 commit b679b3d

File tree

7 files changed

+469
-57
lines changed

7 files changed

+469
-57
lines changed

‎.azure-pipelines/ultimate-pipeline.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,64 @@ stages:
841841
baseImage: $(baseImage)
842842
command: "CheckBuildLogsForErrors"
843843

844+
- stage: exploration_tests_windows
845+
condition: and(succeeded(), eq(variables['isScheduledBuild'], 'False'), eq(variables['runExplorationTests'], 'True'))
846+
dependsOn: build_windows
847+
848+
jobs:
849+
- template: steps/update-github-status-jobs.yml
850+
parameters:
851+
jobs: [ExplorationTest]
852+
853+
- job: generator
854+
pool:
855+
vmImage: ubuntu-18.04
856+
steps:
857+
- bash: |
858+
matrix="{"
859+
for explorationTestUseCase in "debugger" "continuousProfiler"; do
860+
for explorationTestName in "eshoponweb" "protobuf" "cake" "swashbuckle" "paket" ; do
861+
name="${explorationTestUseCase//./_}_${explorationTestName}"
862+
matrix="${matrix} '${name}':"
863+
matrix="${matrix} { 'explorationTestUseCase': '${explorationTestUseCase}', "
864+
matrix="${matrix} 'explorationTestName': '${explorationTestName}' },"
865+
done
866+
done
867+
matrix="${matrix::-1} }"
868+
echo "##vso[task.setVariable variable=values;isOutput=true]${matrix}"
869+
name: mtrx
870+
871+
- job: ExplorationTest
872+
timeoutInMinutes: 100
873+
dependsOn: generator
874+
strategy:
875+
matrix: $[ dependencies.generator.outputs['mtrx.values'] ]
876+
pool:
877+
vmImage: windows-2019
878+
879+
# Enable the Datadog Agent service for this job
880+
services:
881+
dd_agent: dd_agent
882+
883+
steps:
884+
- template: steps/install-dotnet-sdks.yml
885+
parameters:
886+
includeX86: true
887+
- template: steps/restore-working-directory.yml
888+
889+
- script: tracer\build.cmd RunExplorationTests_$(explorationTestUseCase) -ExplorationTestName $(explorationTestName)
890+
displayName: RunExplorationTest
891+
892+
- publish: tracer/build_data
893+
displayName: Uploading exploration_tests_windows tracer logs
894+
artifact: exploration_tests_windows_tracer_logs_$(explorationTestUseCase)_$(explorationTestName)
895+
condition: succeededOrFailed()
896+
continueOnError: true
897+
898+
- script: tracer\build.cmd CheckBuildLogsForErrors
899+
displayName: Check logs for errors
900+
901+
844902
- stage: benchmarks
845903
condition: and(succeeded(), ne(variables['isNgenTestBuild'], 'True'))
846904
dependsOn: build_windows

‎tracer/build/_build/Build.ExplorationTests.cs‎

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using Nuke.Common;
6+
using Nuke.Common.IO;
7+
using Nuke.Common.Tools.DotNet;
8+
using Nuke.Common.Tooling;
9+
using Nuke.Common.Tools.Git;
10+
using static Nuke.Common.Tools.DotNet.DotNetTasks;
11+
12+
partial class Build
13+
{
14+
AbsolutePath ExplorationTestsDirectory => RootDirectory / "exploration-tests";
15+
16+
[Parameter("Indicates name of exploration test to run. If not specified, will run all tests sequentially.")]
17+
readonly ExplorationTestName? ExplorationTestName;
18+
19+
[Parameter("Indicates whether exploration tests should skip repository cloning. Useful for local development. Default false.")]
20+
readonly bool ExplorationTestSkipClone;
21+
22+
[Parameter("Indicates whether exploration tests should run on latest repository commit. Useful if you want to update tested repositories to the latest tags. Default false.",
23+
List = false)]
24+
readonly bool ExplorationTestCloneLatest;
25+
26+
Target RunExplorationTests_Debugger
27+
=> _ => _
28+
.Description("Run exploration tests for debugger.")
29+
.After(Clean, BuildTracerHome)
30+
.Executes(() =>
31+
{
32+
try
33+
{
34+
SetUpExplorationTest_Debugger();
35+
36+
var envVariables = GetEnvVariables(ExplorationTestUseCase.Debugger);
37+
RunExplorationTestsGitCloneAndUnitTest(envVariables);
38+
RunExplorationTestAssertions_Debugger();
39+
}
40+
finally
41+
{
42+
MoveLogsToBuildData();
43+
}
44+
45+
void SetUpExplorationTest_Debugger()
46+
{
47+
Logger.Info($"Prepare environment variables for profiler.");
48+
//TODO TBD
49+
}
50+
51+
void RunExplorationTestAssertions_Debugger()
52+
{
53+
Logger.Info($"Running assertions tests for debugger.");
54+
//TODO TBD
55+
}
56+
})
57+
;
58+
59+
60+
Target RunExplorationTests_ContinuousProfiler
61+
=> _ => _
62+
.Description("Run exploration tests for profiler.")
63+
.After(Clean, BuildTracerHome)
64+
.Executes(() =>
65+
{
66+
try
67+
{
68+
SetUpExplorationTest_ContinuousProfiler();
69+
70+
var envVariables = GetEnvVariables(ExplorationTestUseCase.ContinuousProfiler);
71+
RunExplorationTestsGitCloneAndUnitTest(envVariables);
72+
RunExplorationTestAssertions_ContinuousProfiler();
73+
}
74+
finally
75+
{
76+
MoveLogsToBuildData();
77+
}
78+
79+
void SetUpExplorationTest_ContinuousProfiler()
80+
{
81+
Logger.Info($"Prepare environment variables for profiler.");
82+
//TODO TBD
83+
}
84+
85+
void RunExplorationTestAssertions_ContinuousProfiler()
86+
{
87+
Logger.Info($"Running assertions tests for profiler.");
88+
//TODO TBD
89+
}
90+
})
91+
;
92+
93+
Dictionary<string, string> GetEnvVariables(ExplorationTestUseCase usecase)
94+
{
95+
var envVariables = new Dictionary<string, string>();
96+
switch (usecase)
97+
{
98+
case ExplorationTestUseCase.Debugger:
99+
envVariables.AddDebuggerEnvironmentVariables(TracerHomeDirectory);
100+
break;
101+
case ExplorationTestUseCase.ContinuousProfiler:
102+
envVariables.AddContinuousProfilerEnvironmentVariables(TracerHomeDirectory);
103+
break;
104+
default:
105+
throw new ArgumentOutOfRangeException(nameof(usecase), usecase, null);
106+
}
107+
108+
envVariables["DD_SERVICE"]="exploration_tests";
109+
envVariables["DD_VERSION"]=Version;
110+
111+
envVariables.AddExtraEnvVariables(ExtraEnvVars);
112+
return envVariables;
113+
}
114+
115+
void RunExplorationTestsGitCloneAndUnitTest(Dictionary<string, string> envVariables)
116+
{
117+
if (ExplorationTestName.HasValue)
118+
{
119+
Logger.Info($"Provided exploration test name is {ExplorationTestName}.");
120+
121+
var testDescription = ExplorationTestDescription.GetExplorationTestDescription(ExplorationTestName.Value);
122+
123+
GitCloneAndRunUnitTest(testDescription, envVariables);
124+
}
125+
else
126+
{
127+
Logger.Info($"Exploration test name is not provided. Running all.");
128+
129+
foreach (var testDescription in ExplorationTestDescription.GetAllExplorationTestDescriptions())
130+
{
131+
GitCloneAndRunUnitTest(testDescription, envVariables);
132+
}
133+
}
134+
}
135+
136+
void GitCloneAndRunUnitTest(ExplorationTestDescription testDescription, Dictionary<string, string> envVariables)
137+
{
138+
Logger.Info($"Running exploration test {testDescription.Name}.");
139+
140+
if (Framework != null && !testDescription.IsFrameworkSupported(Framework))
141+
{
142+
Logger.Warn($"This framework '{Framework}' is not listed in the project's target frameworks of {testDescription.Name}");
143+
return;
144+
}
145+
146+
var projectPath = GitClone(testDescription);
147+
Logger.Info($"Test project path is {projectPath}");
148+
149+
DotNetBuild(
150+
x => x
151+
.SetProjectFile(projectPath)
152+
.SetConfiguration(BuildConfiguration)
153+
.SetProcessArgumentConfigurator(arguments => arguments.Add("-consoleLoggerParameters:ErrorsOnly"))
154+
.When(Framework != null, settings => settings.SetFramework(Framework))
155+
);
156+
157+
DotNetTest(
158+
x =>
159+
{
160+
x = x
161+
.When(!testDescription.IsTestedByVSTest, settings => settings.SetProjectFile(projectPath))
162+
.When(testDescription.IsTestedByVSTest
163+
, settings =>
164+
{
165+
var frameworkFolder = Framework ?? "*";
166+
var projectName = Path.GetFileName(projectPath);
167+
168+
return settings.SetProjectFile($"{projectPath}/bin/{BuildConfiguration}/{frameworkFolder}/{projectName}.exe");
169+
})
170+
.EnableNoRestore()
171+
.EnableNoBuild()
172+
.SetConfiguration(BuildConfiguration)
173+
.When(Framework != null, settings => settings.SetFramework(Framework))
174+
.SetProcessEnvironmentVariables(envVariables)
175+
.SetIgnoreFilter(testDescription.TestsToIgnore)
176+
.WithMemoryDumpAfter(5)
177+
;
178+
179+
return x;
180+
});
181+
}
182+
183+
string GitClone(ExplorationTestDescription testDescription)
184+
{
185+
if (!ExplorationTestSkipClone)
186+
{
187+
var depth = testDescription.IsGitShallowCloneSupported ? "--depth 1" : "";
188+
var submodules = testDescription.IsGitSubmodulesRequired ? "--recurse-submodules" : "";
189+
var source = ExplorationTestCloneLatest ? $"-b {testDescription.GitRepositoryTag} {testDescription.GitRepositoryUrl}" : testDescription.GitRepositoryUrl;
190+
var target = $"{ExplorationTestsDirectory}/{testDescription.Name}";
191+
192+
var cloneCommand = $"clone -q -c advice.detachedHead=false {depth} {submodules} {source} {target}";
193+
GitTasks.Git(cloneCommand);
194+
}
195+
196+
var projectPath = $"{ExplorationTestsDirectory}/{testDescription.Name}/{testDescription.PathToUnitTestProject}";
197+
if (!Directory.Exists(projectPath))
198+
{
199+
throw new DirectoryNotFoundException($"Test path '{projectPath}' doesn't exist.");
200+
}
201+
202+
return projectPath;
203+
}
204+
}
205+
206+
207+
public enum ExplorationTestUseCase
208+
{
209+
Debugger, ContinuousProfiler
210+
}
211+
212+
public enum ExplorationTestName
213+
{
214+
eShopOnWeb, protobuf, cake, swashbuckle, paket, ilspy
215+
}
216+
217+
class ExplorationTestDescription
218+
{
219+
public string Name { get; set; }
220+
221+
public string GitRepositoryUrl { get; set; }
222+
public string GitRepositoryTag { get; set; }
223+
public bool IsGitShallowCloneSupported { get; set; }
224+
public bool IsGitSubmodulesRequired { get; set; }
225+
226+
public string PathToUnitTestProject { get; set; }
227+
public bool IsTestedByVSTest { get; set; }
228+
public string[] TestsToIgnore { get; set; }
229+
230+
231+
public TargetFramework[] SupportedFrameworks { get; set; }
232+
233+
public bool IsFrameworkSupported(TargetFramework targetFramework)
234+
{
235+
return SupportedFrameworks.Any(framework => framework.Equals(targetFramework));
236+
}
237+
238+
public static ExplorationTestDescription[] GetAllExplorationTestDescriptions()
239+
{
240+
return Enum.GetValues<ExplorationTestName>()
241+
.Select(GetExplorationTestDescription)
242+
.ToArray()
243+
;
244+
}
245+
246+
public static ExplorationTestDescription GetExplorationTestDescription(ExplorationTestName name)
247+
{
248+
var description = name switch
249+
{
250+
ExplorationTestName.eShopOnWeb => new ExplorationTestDescription()
251+
{
252+
Name = "eShopOnWeb",
253+
GitRepositoryUrl = "https://github.com/dotnet-architecture/eShopOnWeb.git",
254+
GitRepositoryTag = "netcore2.1",
255+
IsGitShallowCloneSupported = true,
256+
PathToUnitTestProject = "tests/UnitTests",
257+
SupportedFrameworks = new[] { TargetFramework.NETCOREAPP2_1 },
258+
},
259+
ExplorationTestName.protobuf => new ExplorationTestDescription()
260+
{
261+
Name = "protobuf",
262+
GitRepositoryUrl = "https://github.com/protocolbuffers/protobuf.git",
263+
GitRepositoryTag = "v3.19.1",
264+
IsGitShallowCloneSupported = true,
265+
IsGitSubmodulesRequired = true,
266+
PathToUnitTestProject = "csharp/src/Google.Protobuf.Test",
267+
SupportedFrameworks = new[] { TargetFramework.NETCOREAPP2_1, TargetFramework.NET5_0, },
268+
},
269+
ExplorationTestName.cake => new ExplorationTestDescription()
270+
{
271+
Name = "cake",
272+
GitRepositoryUrl = "https://github.com/cake-build/cake.git",
273+
GitRepositoryTag = "v1.3.0",
274+
IsGitShallowCloneSupported = true,
275+
PathToUnitTestProject = "src/Cake.Common.Tests",
276+
SupportedFrameworks = new[] { TargetFramework.NETCOREAPP3_1, TargetFramework.NET5_0, TargetFramework.NET6_0 },
277+
},
278+
ExplorationTestName.swashbuckle => new ExplorationTestDescription()
279+
{
280+
Name = "Swashbuckle.AspNetCore",
281+
GitRepositoryUrl = "https://github.com/domaindrivendev/Swashbuckle.AspNetCore.git",
282+
GitRepositoryTag = "v6.2.3",
283+
IsGitShallowCloneSupported = true,
284+
PathToUnitTestProject = "test/Swashbuckle.AspNetCore.SwaggerGen.Test",
285+
SupportedFrameworks = new[] { TargetFramework.NET6_0 },
286+
},
287+
ExplorationTestName.paket => new ExplorationTestDescription()
288+
{
289+
Name = "Paket",
290+
GitRepositoryUrl = "https://github.com/fsprojects/Paket.git",
291+
GitRepositoryTag = "6.2.1",
292+
IsGitShallowCloneSupported = true,
293+
PathToUnitTestProject = "tests/Paket.Tests",
294+
TestsToIgnore = new[] { "Loading assembly metadata works" },
295+
SupportedFrameworks = new[] { TargetFramework.NET461, TargetFramework.NETCOREAPP3_1 },
296+
},
297+
ExplorationTestName.ilspy => new ExplorationTestDescription()
298+
{
299+
Name = "ILSpy",
300+
GitRepositoryUrl = "https://github.com/icsharpcode/ILSpy.git",
301+
GitRepositoryTag = "v7.1",
302+
IsGitSubmodulesRequired = true,
303+
PathToUnitTestProject = "ICSharpCode.Decompiler.Tests",
304+
IsTestedByVSTest = true,
305+
TestsToIgnore = new[] { "UseMc", "_net45", "ImplicitConversions", "ExplicitConversions", "ICSharpCode_Decompiler", "NewtonsoftJson_pcl_debug", "NRefactory_CSharp", "Random_TestCase_1", "AsyncForeach", "AsyncStreams", "AsyncUsing", "CS9_ExtensionGetEnumerator", "IndexRangeTest", "InterfaceTests", "UsingVariables" },
306+
SupportedFrameworks = new[] { TargetFramework.NET461 },
307+
},
308+
_ => throw new ArgumentOutOfRangeException(nameof(name), name, null)
309+
};
310+
311+
return description;
312+
}
313+
}
314+

0 commit comments

Comments
 (0)