-
Notifications
You must be signed in to change notification settings - Fork 642
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[NuGet Symbol Server] Add Symbol push validation support #6276
Changes from all commits
242351e
5004758
517d47a
20f3d00
0eb9568
ade5f9f
60e5433
f72c497
743e2ca
a908c03
e04a585
e3bf803
ffb555c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
// Copyright (c) .NET Foundation. All rights reserved. | ||
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
||
namespace NuGetGallery | ||
{ | ||
/// <summary> | ||
/// The common entity type to be shared by <see cref="Package"/> and <see cref="SymbolPackage"/> | ||
/// This allows us the generic type instantiation in dependency injection for the commonly required code. | ||
/// </summary> | ||
public interface IPackageEntity | ||
{ | ||
string Id { get; } | ||
|
||
string Version { get; } | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,12 +3,11 @@ | |
|
||
using System; | ||
using System.ComponentModel.DataAnnotations; | ||
using System.ComponentModel.DataAnnotations.Schema; | ||
|
||
namespace NuGetGallery | ||
{ | ||
public class SymbolPackage | ||
: IEntity, IEquatable<SymbolPackage> | ||
: IEntity, IPackageEntity, IEquatable<SymbolPackage> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Outside the scope of this PR... but why does There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
{ | ||
public int Key { get; set; } | ||
|
||
|
@@ -51,6 +50,10 @@ public class SymbolPackage | |
/// </summary> | ||
public byte[] RowVersion { get; set; } | ||
|
||
public string Id => Package?.Id; | ||
|
||
public string Version => Package?.Version; | ||
|
||
public bool Equals(SymbolPackage other) | ||
{ | ||
return other.Key == Key; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -112,6 +112,7 @@ | |
<Compile Include="Entities\AccountDelete.cs" /> | ||
<Compile Include="Entities\MembershipRequest.cs" /> | ||
<Compile Include="Entities\OrganizationMigrationRequest.cs" /> | ||
<Compile Include="Entities\IPackageEntity.cs" /> | ||
<Compile Include="Entities\SymbolPackage.cs" /> | ||
<Compile Include="Entities\PackageDelete.cs" /> | ||
<Compile Include="Entities\EmailMessage.cs" /> | ||
|
@@ -254,10 +255,10 @@ | |
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Services.Validation"> | ||
<Version>2.26.0-master-33196</Version> | ||
<Version>2.28.0-master-37018</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Services.Validation.Issues"> | ||
<Version>2.26.0-master-33196</Version> | ||
<Version>2.28.0-master-37018</Version> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is there a stable version to update to? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't see one in manage nuget packages. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe our (undocumented) process for ServerCommon is to kick off a stable build when making changes there. I had similiar feedback in another PR, which is why I mention it. |
||
</PackageReference> | ||
<PackageReference Include="NuGet.Versioning"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -41,6 +41,14 @@ namespace NuGetGallery | |
{ | ||
public class DefaultDependenciesModule : Module | ||
{ | ||
public static class BindingKeys | ||
{ | ||
public const string PackageValidationTopic = "PackageValidationBindingKey"; | ||
public const string SymbolsPackageValidationTopic = "SymbolsPackageValidationBindingKey"; | ||
public const string PackageValidationEnqueuer = "PackageValidationEnqueuerBindingKey"; | ||
public const string SymbolsPackageValidationEnqueuer = "SymbolsPackageValidationEnqueuerBindingKey"; | ||
} | ||
|
||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Maintainability", "CA1502:CyclomaticComplexity", Justification = "This code is more maintainable in the same function.")] | ||
protected override void Load(ContainerBuilder builder) | ||
{ | ||
|
@@ -468,36 +476,71 @@ private static DbConnection CreateDbConnection(ISqlConnectionFactory connectionF | |
.RegisterType<ServiceBusMessageSerializer>() | ||
.As<IServiceBusMessageSerializer>(); | ||
|
||
// We need to setup two enqueuers for Package validation and symbol validation each publishes | ||
// to a different topic for validation. | ||
builder | ||
.RegisterType<PackageValidationEnqueuer>() | ||
.WithParameter(new ResolvedParameter( | ||
(pi, ctx) => pi.ParameterType == typeof(ITopicClient), | ||
(pi, ctx) => ctx.ResolveKeyed<ITopicClient>(BindingKeys.PackageValidationTopic))) | ||
.Keyed<IPackageValidationEnqueuer>(BindingKeys.PackageValidationEnqueuer) | ||
.As<IPackageValidationEnqueuer>(); | ||
|
||
builder | ||
.RegisterType<PackageValidationEnqueuer>() | ||
.WithParameter(new ResolvedParameter( | ||
(pi, ctx) => pi.ParameterType == typeof(ITopicClient), | ||
(pi, ctx) => ctx.ResolveKeyed<ITopicClient>(BindingKeys.SymbolsPackageValidationTopic))) | ||
.Keyed<IPackageValidationEnqueuer>(BindingKeys.SymbolsPackageValidationEnqueuer) | ||
.As<IPackageValidationEnqueuer>(); | ||
|
||
if (configuration.Current.AsynchronousPackageValidationEnabled) | ||
{ | ||
ConfigureValidationEntitiesContext(builder, diagnostics, configuration, secretInjector); | ||
|
||
builder | ||
.RegisterType<AsynchronousPackageValidationInitiator>() | ||
.As<IPackageValidationInitiator>(); | ||
.Register(c => { | ||
return new AsynchronousPackageValidationInitiator<Package>( | ||
c.ResolveKeyed<IPackageValidationEnqueuer>(BindingKeys.PackageValidationEnqueuer), | ||
c.Resolve<IAppConfiguration>(), | ||
c.Resolve<IDiagnosticsService>()); | ||
}).As<IPackageValidationInitiator<Package>>(); | ||
|
||
builder | ||
.Register(c => { | ||
return new AsynchronousPackageValidationInitiator<SymbolPackage>( | ||
c.ResolveKeyed<IPackageValidationEnqueuer>(BindingKeys.SymbolsPackageValidationEnqueuer), | ||
c.Resolve<IAppConfiguration>(), | ||
c.Resolve<IDiagnosticsService>()); | ||
}).As<IPackageValidationInitiator<SymbolPackage>>(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So was There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I missed instantiating the |
||
|
||
// we retrieve the values here (on main thread) because otherwise it would run in another thread | ||
// and potentially cause a deadlock on async operation. | ||
var validationConnectionString = configuration.ServiceBus.Validation_ConnectionString; | ||
var validationTopicName = configuration.ServiceBus.Validation_TopicName; | ||
var symbolsValidationConnectionString = configuration.ServiceBus.SymbolsValidation_ConnectionString; | ||
var symbolsValidationTopicName = configuration.ServiceBus.SymbolsValidation_TopicName; | ||
|
||
builder | ||
.Register(c => new TopicClientWrapper(validationConnectionString, validationTopicName)) | ||
.As<ITopicClient>() | ||
.SingleInstance() | ||
.Keyed<ITopicClient>(BindingKeys.PackageValidationTopic) | ||
.OnRelease(x => x.Close()); | ||
|
||
builder | ||
.Register(c => new TopicClientWrapper( | ||
validationConnectionString, | ||
validationTopicName)) | ||
.Register(c => new TopicClientWrapper(symbolsValidationConnectionString, symbolsValidationTopicName)) | ||
.As<ITopicClient>() | ||
.SingleInstance() | ||
.Keyed<ITopicClient>(BindingKeys.SymbolsPackageValidationTopic) | ||
.OnRelease(x => x.Close()); | ||
} | ||
else | ||
{ | ||
// This will register all the instances of ImmediatePackageValidator<T> as IPackageValidationInitiator<T> where T is a typeof(IPackageEntity) | ||
builder | ||
.RegisterType<ImmediatePackageValidator>() | ||
.As<IPackageValidationInitiator>(); | ||
.RegisterGeneric(typeof(ImmediatePackageValidator<>)) | ||
.As(typeof(IPackageValidationInitiator<>)); | ||
} | ||
|
||
builder.RegisterType<ValidationAdminService>() | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2190,21 +2190,9 @@ | |
<PackageReference Include="Newtonsoft.Json"> | ||
<Version>9.0.1</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Common"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Configuration"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Frameworks"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Packaging"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Packaging.Core"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Protocol"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
|
@@ -2220,21 +2208,9 @@ | |
<PackageReference Include="NuGet.Services.Platform.Client"> | ||
<Version>3.0.29-r-master</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Services.ServiceBus"> | ||
<Version>2.26.0-master-33196</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Services.Sql"> | ||
<Version>2.27.0</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Services.Validation"> | ||
<Version>2.26.0-master-33196</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Services.Validation.Issues"> | ||
<Version>2.26.0-master-33196</Version> | ||
</PackageReference> | ||
<PackageReference Include="NuGet.Versioning"> | ||
<Version>4.8.0-preview4.5287</Version> | ||
</PackageReference> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nice! I assume these dependencies are pulled in from Gallery.Core? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes. |
||
<PackageReference Include="Owin"> | ||
<Version>1.0.0</Version> | ||
</PackageReference> | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -13,9 +13,10 @@ namespace NuGetGallery | |
/// Initiates asynchronous validation on a package by enqueuing a message containing the package identity and a new | ||
/// <see cref="Guid"/>. The <see cref="Guid"/> represents a unique validation request. | ||
/// </summary> | ||
public class AsynchronousPackageValidationInitiator : IPackageValidationInitiator | ||
public class AsynchronousPackageValidationInitiator<TPackageEntity> : IPackageValidationInitiator<TPackageEntity> | ||
where TPackageEntity: IPackageEntity | ||
{ | ||
private readonly IPackageValidationEnqueuer _enqueuer; | ||
private readonly IPackageValidationEnqueuer _validationEnqueuer; | ||
private readonly IAppConfiguration _appConfiguration; | ||
private readonly IDiagnosticsSource _diagnosticsSource; | ||
|
||
|
@@ -24,36 +25,51 @@ public class AsynchronousPackageValidationInitiator : IPackageValidationInitiato | |
IAppConfiguration appConfiguration, | ||
IDiagnosticsService diagnosticsService) | ||
{ | ||
_enqueuer = enqueuer ?? throw new ArgumentNullException(nameof(enqueuer)); | ||
_validationEnqueuer = enqueuer ?? throw new ArgumentNullException(nameof(enqueuer)); | ||
_appConfiguration = appConfiguration ?? throw new ArgumentNullException(nameof(appConfiguration)); | ||
|
||
if (diagnosticsService == null) | ||
{ | ||
throw new ArgumentNullException(nameof(IDiagnosticsService)); | ||
} | ||
|
||
_diagnosticsSource = diagnosticsService.SafeGetSource(nameof(AsynchronousPackageValidationInitiator)); | ||
_diagnosticsSource = diagnosticsService.SafeGetSource(nameof(AsynchronousPackageValidationInitiator<TPackageEntity>)); | ||
} | ||
|
||
public async Task<PackageStatus> StartValidationAsync(Package package) | ||
public async Task<PackageStatus> StartValidationAsync(TPackageEntity package) | ||
{ | ||
if (_appConfiguration.ReadOnlyMode) | ||
{ | ||
throw new ReadOnlyModeException(Strings.CannotEnqueueDueToReadOnly); | ||
} | ||
|
||
ValidatingType validatingType; | ||
if (package is Package) | ||
{ | ||
validatingType = ValidatingType.Package; | ||
} | ||
else if (package is SymbolPackage) | ||
{ | ||
validatingType = ValidatingType.SymbolPackage; | ||
} | ||
else | ||
{ | ||
throw new ArgumentException($"Unknown IPackageEntity type: {nameof(package)}"); | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why add There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. see @skofman1's comment. I think it doesn't make sense to put the |
||
|
||
var data = new PackageValidationMessageData( | ||
package.PackageRegistration.Id, | ||
package.Id, | ||
package.Version, | ||
Guid.NewGuid()); | ||
Guid.NewGuid(), | ||
validatingType); | ||
|
||
var activityName = $"Enqueuing asynchronous package validation: " + | ||
$"{data.PackageId} {data.PackageVersion} ({data.ValidationTrackingId})"; | ||
$"{data.PackageId} {data.PackageVersion} {data.ValidatingType} ({data.ValidationTrackingId})"; | ||
using (_diagnosticsSource.Activity(activityName)) | ||
{ | ||
var postponeProcessingTill = DateTimeOffset.UtcNow + _appConfiguration.AsynchronousPackageValidationDelay; | ||
|
||
await _enqueuer.StartValidationAsync(data, postponeProcessingTill); | ||
await _validationEnqueuer.StartValidationAsync(data, postponeProcessingTill); | ||
} | ||
|
||
if (_appConfiguration.BlockingAsynchronousPackageValidationEnabled) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Are there any shared code paths as part of this PR that would require
NormalizedVersion
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not the code I am touching. I wouldn't refactor much for anything else right now.