Skip to content

C#: Download latest dotnet SDK when missing #15692

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Feb 23, 2024
Merged

C#: Download latest dotnet SDK when missing #15692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
976c627
C#: Download latest dotnet SDK when missing
tamasvajk Feb 22, 2024
8e64880
Fix and add unit tests
tamasvajk Feb 22, 2024
c0d82cb
Minor improvement to not start dotnet process when it is known to fail
tamasvajk Feb 22, 2024
648c06c
Simplify dotnet SDK check in autobuilder
tamasvajk Feb 22, 2024
e176b32
Remove environment dictionary passing
tamasvajk Feb 22, 2024
50f9354
Remove redundant `using`
tamasvajk Feb 22, 2024
20f795c
Code quality improvements
tamasvajk Feb 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 33 additions & 4 deletions csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,8 @@ public void TestVcVarsAllBatFiles()
[Fact]
public void TestLinuxBuildlessExtractionSuccess()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
Expand All @@ -567,12 +569,14 @@ public void TestLinuxBuildlessExtractionSuccess()
actions.EnumerateDirectories[@"C:\Project"] = "";

var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 1);
TestAutobuilderScript(autobuilder, 0, 2);
}

[Fact]
public void TestLinuxBuildlessExtractionFailed()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
Expand All @@ -582,12 +586,14 @@ public void TestLinuxBuildlessExtractionFailed()
actions.EnumerateDirectories[@"C:\Project"] = "";

var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 10, 1);
TestAutobuilderScript(autobuilder, 10, 2);
}

[Fact]
public void TestLinuxBuildlessExtractionSolution()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
Expand All @@ -597,7 +603,28 @@ public void TestLinuxBuildlessExtractionSolution()
actions.EnumerateDirectories[@"C:\Project"] = "";

var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 1);
TestAutobuilderScript(autobuilder, 0, 2);
}

[Fact]
public void TestLinuxBuildlessExtractionNoDotnet()
{
actions.RunProcess["dotnet --list-sdks"] = 1;
actions.RunProcessOut["dotnet --list-sdks"] = "";
actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 8.0.101 --install-dir scratch/.dotnet"] = 0;
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --dotnet scratch/.dotnet"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
actions.EnumerateDirectories[@"C:\Project"] = "";
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
actions.CreateDirectories.Add(@"scratch/.dotnet");

var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 4);
}

private void SkipVsWhere()
Expand Down Expand Up @@ -888,6 +915,8 @@ public void TestSkipNugetMsBuild()
[Fact]
public void TestSkipNugetBuildless()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
Expand All @@ -897,7 +926,7 @@ public void TestSkipNugetBuildless()
actions.EnumerateDirectories[@"C:\Project"] = "";

var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 1);
TestAutobuilderScript(autobuilder, 0, 2);
}


Expand Down
2 changes: 1 addition & 1 deletion csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public override BuildScript GetBuildScript()
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
attempt = DotNetRule.WithDotNet(this, (dotNetPath, env) =>
attempt = DotNetRule.WithDotNet(this, ensureDotNetAvailable: true, (dotNetPath, env) =>
{
// No need to check that the extractor has been executed in buildless mode
return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
Expand Down
8 changes: 8 additions & 0 deletions csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace Semmle.Autobuild.CSharp
{
internal static class Constants
{
// The version number should be kept in sync with the version .NET version used for building the application.
public const string LatestDotNetSdkVersion = "8.0.101";
}
}
53 changes: 37 additions & 16 deletions csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
builder.Log(Severity.Info, "Attempting to build using .NET Core");
}

return WithDotNet(builder, (dotNetPath, environment) =>
return WithDotNet(builder, ensureDotNetAvailable: false, (dotNetPath, environment) =>
{
var ret = GetInfoCommand(builder.Actions, dotNetPath, environment);
foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
Expand Down Expand Up @@ -79,29 +79,29 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
/// variables needed by the installed .NET Core (<code>null</code> when no variables
/// are needed).
/// </summary>
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, bool ensureDotNetAvailable, Func<string?, IDictionary<string, string>?, BuildScript> f)
{
var installDir = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var _), ".dotnet");
var installScript = DownloadDotNet(builder, installDir);
var installScript = DownloadDotNet(builder, installDir, ensureDotNetAvailable);
return BuildScript.Bind(installScript, installed =>
{
Dictionary<string, string>? env;
var env = new Dictionary<string, string>
{
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
{ "MSBUILDDISABLENODEREUSE", "1" }
};
if (installed == 0)
{
// The installation succeeded, so use the newly installed .NET Core
var path = builder.Actions.GetEnvironmentVariable("PATH");
var delim = builder.Actions.IsWindows() ? ";" : ":";
env = new Dictionary<string, string>{
{ "DOTNET_MULTILEVEL_LOOKUP", "false" }, // prevent look up of other .NET Core SDKs
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
{ "MSBUILDDISABLENODEREUSE", "1" },
{ "PATH", installDir + delim + path }
};
env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET Core SDKs
env.Add("PATH", installDir + delim + path);
}
else
{
// The .NET SDK was not installed, either because the installation failed or because it was already installed.
installDir = null;
env = null;
}

return f(installDir, env);
Expand All @@ -117,14 +117,14 @@ public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builde
/// are needed).
/// </summary>
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<IDictionary<string, string>?, BuildScript> f)
=> WithDotNet(builder, (_1, env) => f(env));
=> WithDotNet(builder, ensureDotNetAvailable: false, (_, env) => f(env));

/// <summary>
/// Returns a script for downloading relevant versions of the
/// .NET Core SDK. The SDK(s) will be installed at <code>installDir</code>
/// (provided that the script succeeds).
/// </summary>
private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir)
private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir, bool ensureDotNetAvailable)
{
if (!string.IsNullOrEmpty(builder.Options.DotNetVersion))
// Specific version supplied in configuration: always use that
Expand Down Expand Up @@ -152,7 +152,17 @@ private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> b
validGlobalJson = true;
}

return validGlobalJson ? installScript : BuildScript.Failure;
if (validGlobalJson)
{
return installScript;
}

if (ensureDotNetAvailable)
{
return DownloadDotNetVersion(builder, installDir, Constants.LatestDotNetSdkVersion, needExactVersion: false);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why can't we simply skip the --version argument to the install script? That should install the latest version.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't tried that, but I think it wouldn't work. You can pass latest to -Version, but you need to also specify the -Channel. Also, I think it's somewhat safer to control the exact fallback version, and then the version that we use seems to be a good candidate.

}

return BuildScript.Failure;
}

/// <summary>
Expand All @@ -161,14 +171,25 @@ private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> b
///
/// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
/// </summary>
private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version)
private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version, bool needExactVersion = true)
{
return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) =>
{
if (sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
{
// The requested SDK is already installed (and no other SDKs are installed), so
// no need to reinstall
return BuildScript.Failure;
}
else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
{
// there's at least one SDK installed, so no need to reinstall
return BuildScript.Failure;
}
else if (!needExactVersion && sdksRet != 0)
{
builder.Log(Severity.Info, "No .NET Core SDK found.");
}

builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version);

Expand Down
38 changes: 12 additions & 26 deletions csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System.Linq;
using Semmle.Autobuild.Shared;
using Semmle.Autobuild.Shared;

namespace Semmle.Autobuild.CSharp
{
Expand All @@ -17,36 +16,23 @@ internal StandaloneBuildRule(string? dotNetPath)

public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
{
BuildScript GetCommand()
if (builder.CodeQLExtractorLangRoot is null
|| builder.CodeQlPlatform is null)
{
string standalone;
if (builder.CodeQLExtractorLangRoot is not null && builder.CodeQlPlatform is not null)
{
standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
}
else
{
return BuildScript.Failure;
}

var cmd = new CommandBuilder(builder.Actions);
cmd.RunCommand(standalone);

if (!string.IsNullOrEmpty(this.dotNetPath))
{
cmd.Argument("--dotnet");
cmd.QuoteArgument(this.dotNetPath);
}

return cmd.Script;
return BuildScript.Failure;
}

if (!builder.Options.Buildless)
var standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
var cmd = new CommandBuilder(builder.Actions);
cmd.RunCommand(standalone);

if (!string.IsNullOrEmpty(this.dotNetPath))
{
return BuildScript.Failure;
cmd.Argument("--dotnet");
cmd.QuoteArgument(this.dotNetPath);
}

return GetCommand();
return cmd.Script;
}
}
}