Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use MSBuild Logging APIs for Task output instead of StdOut #933

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.IO.Redist" Version="6.1.0" />
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.12.19" />
@@ -58,4 +59,4 @@
<PackageVersion Include="System.Threading.Tasks.Dataflow" Version="8.0.1" />
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.6.0" />
</ItemGroup>
</Project>
</Project>
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@
<PackageReference Include="Microsoft.ComponentDetection.Contracts" />
<PackageReference Include="Newtonsoft.Json" />
<PackageReference Include="packageurl-dotnet" />
<PackageReference Include="Microsoft.Extensions.Options" />
</ItemGroup>

<ItemGroup>
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
using Microsoft.Sbom.Api.Utils;
using Microsoft.Sbom.Common;
using Microsoft.Sbom.Common.Config;
using Microsoft.Sbom.Contracts;
using Microsoft.Sbom.Extensions;
using Constants = Microsoft.Sbom.Api.Utils.Constants;
using ILogger = Serilog.ILogger;
@@ -34,6 +35,7 @@ public abstract class ComponentDetectionBaseWalker
private readonly ISbomConfigProvider sbomConfigs;
private readonly IFileSystemUtils fileSystemUtils;
private readonly ILicenseInformationFetcher licenseInformationFetcher;
private readonly RuntimeConfiguration? runtimeConfiguration;
private readonly IPackageDetailsFactory packageDetailsFactory;

public ConcurrentDictionary<string, string> LicenseDictionary = new ConcurrentDictionary<string, string>();
@@ -48,7 +50,8 @@ public ComponentDetectionBaseWalker(
ISbomConfigProvider sbomConfigs,
IFileSystemUtils fileSystemUtils,
IPackageDetailsFactory packageDetailsFactory,
ILicenseInformationFetcher licenseInformationFetcher)
ILicenseInformationFetcher licenseInformationFetcher,
RuntimeConfiguration? runtimeConfiguration)
{
this.log = log ?? throw new ArgumentNullException(nameof(log));
this.componentDetector = componentDetector ?? throw new ArgumentNullException(nameof(componentDetector));
@@ -57,6 +60,7 @@ public ComponentDetectionBaseWalker(
this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils));
this.packageDetailsFactory = packageDetailsFactory ?? throw new ArgumentNullException(nameof(packageDetailsFactory));
this.licenseInformationFetcher = licenseInformationFetcher ?? throw new ArgumentNullException(nameof(licenseInformationFetcher));
this.runtimeConfiguration = runtimeConfiguration;
}

public (ChannelReader<ScannedComponent> output, ChannelReader<ComponentDetectorException> error) GetComponents(string buildComponentDirPath)
@@ -114,6 +118,10 @@ async Task Scan(string path)
var cmdLineParams = configuration.ToComponentDetectorCommandLineParams(cliArgumentBuilder);

var scanSettings = cliArgumentBuilder.BuildScanSettingsFromParsedArgs(cmdLineParams);
if (runtimeConfiguration != null)
{
scanSettings.NoSummary = runtimeConfiguration.NoComponentGovernanceSummary;
}

var scanResult = await componentDetector.ScanAsync(scanSettings);

5 changes: 3 additions & 2 deletions src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@
using Microsoft.Sbom.Api.Utils;
using Microsoft.Sbom.Common;
using Microsoft.Sbom.Common.Config;
using Microsoft.Sbom.Contracts;
using Microsoft.Sbom.Extensions;
using ILogger = Serilog.ILogger;

@@ -19,8 +20,8 @@ namespace Microsoft.Sbom.Api.Executors;
/// </summary>
public class PackagesWalker : ComponentDetectionBaseWalker
{
public PackagesWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher)
: base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, packageDetailsFactory, licenseInformationFetcher)
public PackagesWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher, RuntimeConfiguration runtimeConfiguration = null)
: base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, packageDetailsFactory, licenseInformationFetcher, runtimeConfiguration)
{
}

5 changes: 3 additions & 2 deletions src/Microsoft.Sbom.Api/Executors/SbomComponentsWalker.cs
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@
using Microsoft.Sbom.Api.Utils;
using Microsoft.Sbom.Common;
using Microsoft.Sbom.Common.Config;
using Microsoft.Sbom.Contracts;
using Microsoft.Sbom.Extensions;
using ILogger = Serilog.ILogger;

@@ -19,8 +20,8 @@ namespace Microsoft.Sbom.Api.Executors;
/// </summary>
public class SbomComponentsWalker : ComponentDetectionBaseWalker
{
public SbomComponentsWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher)
: base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, packageDetailsFactory, licenseInformationFetcher)
public SbomComponentsWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs, IFileSystemUtils fileSystemUtils, IPackageDetailsFactory packageDetailsFactory, ILicenseInformationFetcher licenseInformationFetcher, RuntimeConfiguration runtimeConfiguration = null)
: base(log, componentDetector, configuration, sbomConfigs, fileSystemUtils, packageDetailsFactory, licenseInformationFetcher, runtimeConfiguration)
{
}

4 changes: 2 additions & 2 deletions src/Microsoft.Sbom.Api/Workflows/SbomGenerationWorkflow.cs
Original file line number Diff line number Diff line change
@@ -150,7 +150,7 @@ public virtual async Task<bool> RunAsync()
recorder.RecordTotalErrors(validErrors);
}

// Delete the generated _manifest folder if generation failed.
// Delete the generated _manifest folder if generation failed.
if (deleteSbomDir || validErrors.Any())
{
DeleteManifestFolder(sbomDir);
@@ -242,7 +242,7 @@ private void RemoveExistingManifestDirectory()
$"overwrite this folder.");
}

log.Warning(
log.Information(
$"Deleting pre-existing folder {rootManifestFolderPath} as {Constants.DeleteManifestDirBoolVariableName}" +
$" is 'true'.");
fileSystemUtils.DeleteDir(rootManifestFolderPath, true);
Original file line number Diff line number Diff line change
@@ -49,4 +49,9 @@ public class RuntimeConfiguration
/// Gets or sets a value indicating whether if set to false, we will not follow symlinks while traversing the build drop folder.
/// </summary>
public bool FollowSymlinks { get; set; } = true;

/// <summary>
/// Gets or sets a value indicating whether if set to true, we will not print a summary of the component governance to stdout.
/// </summary>
public bool NoComponentGovernanceSummary { get; set; } = false;
}
Original file line number Diff line number Diff line change
@@ -46,6 +46,14 @@ namespace Microsoft.Sbom.Extensions.DependencyInjection;

public static class ServiceCollectionExtensions
{
/// <summary>
/// Initializes the services required to use the SBOM Tool using a default <see cref="InputConfiguration"/> and <see cref="LogEventLevel"/>.
/// </summary>
/// <param name="services">The services to which registrations are added.</param>
/// <param name="inputConfiguration">The default configuration to use, which will be registered as a singleton and will overwrite the currently-set <see cref="Configuration"/>.</param>
/// <param name="logLevel">The log verbosity level with which to use for the minimal logger used for internal services. These services require an <see cref="ILogger"/> but may themselves be used to <strong>initialize</strong> an <see cref="ILogger"/>, so need separate configuration.</param>
/// <remarks>The <see cref="InputConfiguration"/> is registered as a singleton in the container, and the <paramref name="logLevel"/> is used to create a default logger for internal utility classes, irrespective of any other <see cref="ILogger"/> registered in the container. This method does not register an instance of <see cref="ILogger"/>, however one is required for the system to function. Callers need to provide their own implementation.</remarks>
/// <returns></returns>
public static IServiceCollection AddSbomConfiguration(this IServiceCollection services, InputConfiguration inputConfiguration, LogEventLevel logLevel = LogEventLevel.Information)
{
ArgumentNullException.ThrowIfNull(inputConfiguration);
@@ -59,16 +67,17 @@ public static IServiceCollection AddSbomConfiguration(this IServiceCollection se
return services;
}

/// <summary>
/// Initializes the services required to use the SBOM Tool using a default <see cref="LogEventLevel"/>.
/// </summary>
/// <param name="services">The services to which registrations are added.</param>
/// <param name="logLevel">The log verbosity level with which to use for the minimal logger used for internal services. These services require an <see cref="ILogger"/> but may themselves be used to <strong>initialize</strong> an <see cref="ILogger"/>, so need separate configuration.</param>
/// <remarks>This method does not register an instance of <see cref="ILogger"/>, however one is required for the system to function. Callers need to provide their own implementation.</remarks>
public static IServiceCollection AddSbomTool(this IServiceCollection services, LogEventLevel logLevel = LogEventLevel.Information)
{
services
.AddSingleton<IConfiguration, Configuration>()
.AddTransient(_ => FileSystemUtilsProvider.CreateInstance(CreateLogger(logLevel)))
.AddTransient(x =>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is that this will create a breaking change for API users, as they will now have to inject a logger similarly to how we are now doing it in Program.cs, is that right? This is okay with me as our next release will be a breaking change anyways, but tagging @DaveTryon for visibility

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oof, good call - I didn't realize that the dependency injection project was public API surface area. Let me noodle on how that could be fixed.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've pushed a commit that minimizes the API surface area changes, and adds a ton of xmldoc that instructs users on the new requirement of registering their own ILogger. That won't stop someone that just updates and then runs tests from failing, but maybe that will help mitigate somewhat.

{
logLevel = x.GetService<InputConfiguration>()?.Verbosity?.Value ?? logLevel;
return Log.Logger = CreateLogger(logLevel);
})
.AddTransient(x => FileSystemUtilsProvider.CreateInstance(CreateLogger(logLevel)))
.AddTransient<IWorkflow<SbomParserBasedValidationWorkflow>, SbomParserBasedValidationWorkflow>()
.AddTransient<IWorkflow<SbomGenerationWorkflow>, SbomGenerationWorkflow>()
.AddTransient<IWorkflow<SbomRedactionWorkflow>, SbomRedactionWorkflow>()
@@ -125,7 +134,8 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L
.AddSingleton<IFileTypeUtils, FileTypeUtils>()
.AddSingleton<ISignValidationProvider, SignValidationProvider>()
.AddSingleton<IManifestParserProvider, ManifestParserProvider>()
.AddSingleton(x => {
.AddSingleton(x =>
{
var comparer = x.GetRequiredService<IOSUtils>().GetFileSystemStringComparer();
return new FileHashesDictionary(new ConcurrentDictionary<string, FileHashes>(comparer));
})
@@ -180,6 +190,11 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L
return services;
}

/// <summary>
/// This method integrates Serilog into the <see cref="Microsoft.Extensions.Logging"/> infrastructure.
/// </summary>
/// <param name="services"></param>
/// <returns></returns>
public static IServiceCollection ConfigureLoggingProviders(this IServiceCollection services)
{
var providers = new LoggerProviderCollection();
@@ -200,7 +215,13 @@ public static IServiceCollection ConfigureLoggingProviders(this IServiceCollecti
return services;
}

private static ILogger CreateLogger(LogEventLevel logLevel)
/// <summary>
/// Creates a logger with the specified <paramref name="logLevel"/> that writes to a file and optionally to stderr.
/// This logger converts all errors from ComponentDetection assemblies to warnings, and does not write tabular output to stdout/stderr, only the configured file.
/// </summary>
/// <param name="logLevel"></param>
/// <returns></returns>
public static ILogger CreateLogger(LogEventLevel logLevel = LogEventLevel.Information)
{
return new RemapComponentDetectionErrorsToWarningsLogger(
new LoggerConfiguration()
Loading
Oops, something went wrong.
Loading
Oops, something went wrong.