diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln index 66daf9b57..7bdf0c011 100644 --- a/NuGet.Jobs.sln +++ b/NuGet.Jobs.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 -VisualStudioVersion = 15.0.27121.1 +VisualStudioVersion = 15.0.27130.2027 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Common", "src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj", "{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}" EndProject @@ -113,6 +113,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job", "sr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PackageHash", "src\PackageHash\PackageHash.csproj", "{40843020-6F0A-48F0-AC28-42FFE3A5C21E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageCompatibility.Core", "src\Validation.PackageCompatibility.Core\Validation.PackageCompatibility.Core.csproj", "{3958ECDD-DE42-401B-8BC6-553E49F2D42A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -289,6 +291,10 @@ Global {40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.Build.0 = Debug|Any CPU {40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.ActiveCfg = Release|Any CPU {40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.Build.0 = Release|Any CPU + {3958ECDD-DE42-401B-8BC6-553E49F2D42A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3958ECDD-DE42-401B-8BC6-553E49F2D42A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3958ECDD-DE42-401B-8BC6-553E49F2D42A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3958ECDD-DE42-401B-8BC6-553E49F2D42A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -336,6 +342,7 @@ Global {B4B7564A-965B-447B-927F-6749E2C08880} = {6A776396-02B1-475D-A104-26940ADB04AB} {FA87D075-A934-4443-8D0B-5DB32640B6D7} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} {40843020-6F0A-48F0-AC28-42FFE3A5C21E} = {FA5644B5-4F08-43F6-86B3-039374312A47} + {3958ECDD-DE42-401B-8BC6-553E49F2D42A} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B} diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs index cc5fea3ae..e81f36d57 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Job.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Job.cs @@ -3,8 +3,9 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net; +using System.Net.Http; +using System.Reflection; using System.Threading.Tasks; using AnglicanGeek.MarkdownMailer; using Autofac; @@ -18,18 +19,20 @@ using Microsoft.WindowsAzure.Storage; using NuGet.Jobs; using NuGet.Jobs.Configuration; +using NuGet.Jobs.Validation; using NuGet.Jobs.Validation.Common; using NuGet.Jobs.Validation.PackageSigning.Messages; using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Services.Configuration; using NuGet.Services.KeyVault; -using NuGet.Services.Logging; using NuGet.Services.ServiceBus; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageCertificates; +using NuGet.Services.Validation.PackageCompatibility; using NuGet.Services.Validation.PackageSigning; using NuGet.Services.Validation.Vcs; using NuGetGallery.Services; +using Validation.PackageCompatibility.Core.Storage; namespace NuGet.Services.Validation.Orchestrator { @@ -42,17 +45,20 @@ public class Job : JobBase private const string VcsSectionName = "Vcs"; private const string PackageSigningSectionName = "PackageSigning"; private const string PackageCertificatesSectionName = "PackageCertificates"; + private const string PackageCompatibilitySectionName = "PackageCompatibility"; private const string RunnerConfigurationSectionName = "RunnerConfiguration"; private const string GalleryDbConfigurationSectionName = "GalleryDb"; private const string ValidationDbConfigurationSectionName = "ValidationDb"; private const string ServiceBusConfigurationSectionName = "ServiceBus"; private const string SmtpConfigurationSectionName = "Smtp"; private const string EmailConfigurationSectionName = "Email"; + private const string PackageDownloadTimeoutName = "PackageDownloadTimeout"; private const string VcsBindingKey = VcsSectionName; private const string PackageVerificationTopicClientBindingKey = "PackageVerificationTopicClient"; private const string PackageSigningBindingKey = PackageSigningSectionName; private const string PackageCertificatesBindingKey = PackageCertificatesSectionName; + private const string PackageCompatibilityBindingKey = PackageCompatibilitySectionName; private const string ValidationStorageBindingKey = "ValidationStorage"; private const string OrchestratorBindingKey = "Orchestrator"; @@ -182,6 +188,7 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo services.AddTransient, PackageValidationMessageDataSerializationAdapter>(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient, SignatureValidationMessageSerializer>(); services.AddTransient(); @@ -207,10 +214,29 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo ? (IMailSender)new DiskMailSender() : (IMailSender)new MailSender(mailSenderConfiguration); }); + + services.AddSingleton(p => + { + var assembly = Assembly.GetEntryAssembly(); + var assemblyName = assembly.GetName().Name; + var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; + + var client = new HttpClient(new WebRequestHandler + { + AllowPipelining = true, + AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), + }); + + client.Timeout = configurationRoot.GetValue(PackageDownloadTimeoutName); + client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); + + return client; + }); services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddSingleton(new TelemetryClient()); } @@ -218,7 +244,6 @@ private static IServiceProvider CreateProvider(IServiceCollection services) { var containerBuilder = new ContainerBuilder(); containerBuilder.Populate(services); - /// Initialize dependencies for the . There is some additional complexity here /// because the implementations require ambiguous types (such as a and a /// which there may be more than one configuration of). @@ -309,6 +334,7 @@ private static IServiceProvider CreateProvider(IServiceCollection services) ConfigurePackageSigningValidator(containerBuilder); ConfigurePackageCertificatesValidator(containerBuilder); + ConfigurePackageCompatibilityValidator(containerBuilder); return new AutofacServiceProvider(containerBuilder.Build()); } @@ -345,6 +371,29 @@ private static void ConfigurePackageSigningValidator(ContainerBuilder builder) .As(); } + + private static void ConfigurePackageCompatibilityValidator(ContainerBuilder builder) + { + // Configure the validator state service for the package compatibility validator. + builder + .RegisterType() + .WithParameter( + (pi, ctx) => pi.ParameterType == typeof(Type), + (pi, ctx) => typeof(PackageCompatibilityValidator)) + .Keyed(PackageCompatibilityBindingKey); + + // Configure the package compatibility service + builder + .RegisterType() + .As(); + + // Configure the package compatibility validator + builder + .RegisterType() + .WithKeyedParameter(typeof(IValidatorStateService), PackageCompatibilityBindingKey) + .As(); + } + private static void ConfigurePackageCertificatesValidator(ContainerBuilder builder) { // Configure the validator state service for the package certificates validator. diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index 526832518..b46178d25 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -61,6 +61,7 @@ + @@ -120,6 +121,10 @@ {2539ddf3-0cc5-4a03-b5f9-39b47744a7bd} Validation.Common + + {3958ECDD-DE42-401B-8BC6-553E49F2D42A} + Validation.PackageCompatibility.Core + {91C060DA-736F-4DA9-A57F-CB3AC0E6CB10} Validation.PackageSigning.Core diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageCompatibility/PackageCompatibilityValidator.cs b/src/NuGet.Services.Validation.Orchestrator/PackageCompatibility/PackageCompatibilityValidator.cs new file mode 100644 index 000000000..bf5e677a0 --- /dev/null +++ b/src/NuGet.Services.Validation.Orchestrator/PackageCompatibility/PackageCompatibilityValidator.cs @@ -0,0 +1,109 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Common; +using NuGet.Jobs.Validation; +using NuGet.Jobs.Validation.PackageSigning.Storage; +using Validation.PackageCompatibility.Core.Messages; +using Validation.PackageCompatibility.Core.Storage; + +namespace NuGet.Services.Validation.PackageCompatibility +{ + /// + /// Configuration for initializing the . + /// + public class PackageCompatibilityValidator : IValidator + { + private IValidatorStateService _validatorStateService; + private IPackageCompatibilityService _packageCompatibilityService; + private readonly ILogger _logger; + + private IPackageDownloader _packageDownloader; + + public PackageCompatibilityValidator( + IValidatorStateService validatorStateService, + IPackageCompatibilityService packageCompatibilityService, + IPackageDownloader packageDownloader, + ILogger logger) + { + _validatorStateService = validatorStateService; + _packageCompatibilityService = packageCompatibilityService; + _packageDownloader = packageDownloader; + _logger = logger; + } + + public async Task GetResultAsync(IValidationRequest request) + { + var validatorStatus = await _validatorStateService.GetStatusAsync(request); + + return validatorStatus.ToValidationResult(); + } + + public async Task StartAsync(IValidationRequest request) + { + try + { + var validatorStatus = await _validatorStateService.GetStatusAsync(request); + + if (validatorStatus.State != ValidationStatus.NotStarted) + { + _logger.LogWarning( + "Package Compatibility validation with validationId {ValidationId} ({PackageId} {PackageVersion}) has already started.", + request.ValidationId, + request.PackageId, + request.PackageVersion); + + return validatorStatus.ToValidationResult(); + } + + try + { + await Validate(request, CancellationToken.None); + } + catch (Exception e) + { + _logger.LogWarning(0, e, "Validation failed in the validator for the following ValidationId {ValidationId}", request.ValidationId); + } + + // Treat every validation as succeeded, as we don't want to block + validatorStatus.State = ValidationStatus.Succeeded; + + await _validatorStateService.SaveStatusAsync(validatorStatus); + + return validatorStatus.ToValidationResult(); + } + catch (Exception e) + { + _logger.LogWarning(0, e, "Validation failed for the following ValidationId {ValidationId}", request.ValidationId); + } + + return new ValidationResult(ValidationStatus.Succeeded); + } + + private async Task Validate(IValidationRequest request, CancellationToken cancellationToken) + { + using (var packageStream = await _packageDownloader.DownloadAsync(new Uri(request.NupkgUrl), cancellationToken)) + using (var package = new Packaging.PackageArchiveReader(packageStream)) + { + var warnings = new List(); + + foreach (var rule in Packaging.Rules.DefaultPackageRuleSet.Rules) + { + warnings.AddRange(rule.Validate(package)); + } + + await _packageCompatibilityService.SetPackageCompatibilityState(request.ValidationId, warnings); + } + } + + public Task CleanUpAsync(IValidationRequest request) + { + return Task.CompletedTask; + } + } +} diff --git a/src/NuGet.Services.Validation.Orchestrator/settings.json b/src/NuGet.Services.Validation.Orchestrator/settings.json index 22c9c2f70..ed02e2c1e 100644 --- a/src/NuGet.Services.Validation.Orchestrator/settings.json +++ b/src/NuGet.Services.Validation.Orchestrator/settings.json @@ -55,6 +55,13 @@ "SubscriptionName": "" } }, + "PackageCompatibility": { + "ServiceBus": { + "ConnectionString": "", + "TopicPath": "", + "SubscriptionName": "" + } + }, "RunnerConfiguration": { "ProcessRecycleInterval": "1:00:00:00", "ShutdownWaitInterval": "00:01:00" @@ -86,6 +93,7 @@ "AnnouncementsUrl": "https://github.com/NuGet/Announcements/issues", "TwitterUrl": "https://twitter.com/nuget" }, + "PackageDownloadTimeout": "10:00", "KeyVault_VaultName": "", "KeyVault_ClientId": "", "KeyVault_CertificateThumbprint": "", diff --git a/src/Validation.PackageCompatibility.Core/IPackageCompatibilityService.cs b/src/Validation.PackageCompatibility.Core/IPackageCompatibilityService.cs new file mode 100644 index 000000000..f057ef8cf --- /dev/null +++ b/src/Validation.PackageCompatibility.Core/IPackageCompatibilityService.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using NuGet.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Validation.PackageCompatibility.Core.Storage +{ + public interface IPackageCompatibilityService + { + Task SetPackageCompatibilityState( + Guid validationId, + IEnumerable messages); + } +} diff --git a/src/Validation.PackageCompatibility.Core/Messages/PackageCompatibilityValidationMessage.cs b/src/Validation.PackageCompatibility.Core/Messages/PackageCompatibilityValidationMessage.cs new file mode 100644 index 000000000..69dd4d451 --- /dev/null +++ b/src/Validation.PackageCompatibility.Core/Messages/PackageCompatibilityValidationMessage.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Validation.PackageCompatibility.Core.Messages +{ + public class PackageCompatibilityValidationMessage + { + public PackageCompatibilityValidationMessage(string packageId, string packageVersion, Uri nupkgUri, Guid validationId) + { + if (validationId == Guid.Empty) + { + throw new ArgumentOutOfRangeException(nameof(validationId)); + } + ValidationId = validationId; + PackageId = packageId ?? throw new ArgumentNullException(nameof(packageId)); + PackageVersion = packageVersion ?? throw new ArgumentNullException(nameof(packageVersion)); + NupkgUri = nupkgUri ?? throw new ArgumentNullException(nameof(nupkgUri)); + } + + public string PackageId { get; } + public string PackageVersion { get; } + public Uri NupkgUri { get; } + public Guid ValidationId { get; } + } +} diff --git a/src/Validation.PackageCompatibility.Core/PackageCompatibilityService.cs b/src/Validation.PackageCompatibility.Core/PackageCompatibilityService.cs new file mode 100644 index 000000000..0597e3a0c --- /dev/null +++ b/src/Validation.PackageCompatibility.Core/PackageCompatibilityService.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Microsoft.Extensions.Logging; +using NuGet.Common; +using NuGet.Services.Validation; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Validation.PackageCompatibility.Core.Storage +{ + public class PackageCompatibilityService : IPackageCompatibilityService + { + private readonly IValidationEntitiesContext _validationContext; + private readonly ILogger _logger; + + public PackageCompatibilityService( + IValidationEntitiesContext validationContext, + ILogger logger) + { + _validationContext = validationContext ?? throw new ArgumentNullException(nameof(validationContext)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task SetPackageCompatibilityState( + Guid validationId, + IEnumerable messages) + { + foreach (var log in messages) + { + _validationContext.PackageCompatibilityIssues.Add( + new PackageCompatibilityIssue() + { + ClientIssueCode = log.Code.ToString(), + Message = log.Message, + PackageValidationKey = validationId + } + ); + } + await _validationContext.SaveChangesAsync(); + } + } +} diff --git a/src/Validation.PackageCompatibility.Core/Properties/AssemblyInfo.cs b/src/Validation.PackageCompatibility.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..b8580d9a5 --- /dev/null +++ b/src/Validation.PackageCompatibility.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Validation.PackageCompatibility.Core")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany(".NET Foundation")] +[assembly: AssemblyProduct("Validation.PackageCompatibility.Core")] +[assembly: AssemblyCopyright("Copyright © .NET Foundation 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("3958ecdd-de42-401b-8bc6-553e49f2d42a")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/src/Validation.PackageCompatibility.Core/Validation.PackageCompatibility.Core.csproj b/src/Validation.PackageCompatibility.Core/Validation.PackageCompatibility.Core.csproj new file mode 100644 index 000000000..3cd50f70c --- /dev/null +++ b/src/Validation.PackageCompatibility.Core/Validation.PackageCompatibility.Core.csproj @@ -0,0 +1,66 @@ + + + + + Debug + AnyCPU + {3958ECDD-DE42-401B-8BC6-553E49F2D42A} + Library + Properties + Validation.PackageCompatibility.Core + Validation.PackageCompatibility.Core + v4.6 + 512 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + {fa87d075-a934-4443-8d0b-5db32640b6d7} + Validation.Common.Job + + + + + 4.7.0-preview1-4929 + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/src/Validation.PackageSigning.ExtractAndValidateSignature/Validation.PackageSigning.ExtractAndValidateSignature.csproj b/src/Validation.PackageSigning.ExtractAndValidateSignature/Validation.PackageSigning.ExtractAndValidateSignature.csproj index f172679ef..72e05e9c9 100644 --- a/src/Validation.PackageSigning.ExtractAndValidateSignature/Validation.PackageSigning.ExtractAndValidateSignature.csproj +++ b/src/Validation.PackageSigning.ExtractAndValidateSignature/Validation.PackageSigning.ExtractAndValidateSignature.csproj @@ -85,6 +85,11 @@ Validation.PackageSigning.Core + + + 4.7.0-preview1-4886 + + ..\..\build