Repeatable build using Project lock file implementation (#2342)

generating/ updating project lock file

consuming project lock file during restore in VS

Validation of package's sha512 and added error messages as appropriate

Test infra for legacy PackageReference func tests

Base workflow is completed

Added telemetry events for project lock file feature

src/NuGet.Clients/NuGet.PackageManagement.VisualStudio/Projects/LegacyPackageReferenceProject.cs

@@ -363,7 +363,11 @@ private async Task<PackageSpec> GetPackageSpecAsync(ISettings settings)
                     ProjectWideWarningProperties = WarningProperties.GetWarningProperties(
                         treatWarningsAsErrors: _vsProjectAdapter.TreatWarningsAsErrors, 
                         noWarn: _vsProjectAdapter.NoWarn,
-                        warningsAsErrors: _vsProjectAdapter.WarningsAsErrors)
+                        warningsAsErrors: _vsProjectAdapter.WarningsAsErrors),
+                    RestoreLockProperties = new RestoreLockProperties(
+                        await _vsProjectAdapter.GetRestorePackagesWithLockFileAsync(),
+                        await _vsProjectAdapter.GetNuGetLockFilePathAsync(),
+                        await _vsProjectAdapter.IsRestoreLockedAsync())
                 }
             };
         }

src/NuGet.Clients/NuGet.PackageManagement.VisualStudio/Projects/VsProjectAdapter.cs

@@ -363,6 +363,31 @@ public async Task<NuGetFramework> GetTargetFrameworkAsync()
             return NuGetFramework.UnsupportedFramework;
         }
 
+        public async Task<string> GetRestorePackagesWithLockFileAsync()
+        {
+            await _threadingService.JoinableTaskFactory.SwitchToMainThreadAsync();
+
+            var value = await BuildProperties.GetPropertyValueAsync(ProjectBuildProperties.RestorePackagesWithLockFile);
+
+            return value;
+        }
+
+        public async Task<string> GetNuGetLockFilePathAsync()
+        {
+            await _threadingService.JoinableTaskFactory.SwitchToMainThreadAsync();
+
+            return await BuildProperties.GetPropertyValueAsync(ProjectBuildProperties.NuGetLockFilePath);
+        }
+
+        public async Task<bool> IsRestoreLockedAsync()
+        {
+            await _threadingService.JoinableTaskFactory.SwitchToMainThreadAsync();
+
+            var value = await BuildProperties.GetPropertyValueAsync(ProjectBuildProperties.RestoreLockedMode);
+
+            return MSBuildStringUtility.IsTrue(value);
+        }
+
         private async Task<string> GetTargetFrameworkStringAsync()
         {
             await _threadingService.JoinableTaskFactory.SwitchToMainThreadAsync();

src/NuGet.Clients/NuGet.SolutionRestoreManager/SolutionRestoreJob.cs

@@ -329,15 +329,17 @@ private async Task RestoreAsync(bool forceRestore, RestoreOperationSource restor
                             var providerCache = new RestoreCommandProvidersCache();
                             Action<SourceCacheContext> cacheModifier = (cache) => { };
 
+                            var isRestoreOriginalAction = true;
                             var restoreSummaries = await DependencyGraphRestoreUtility.RestoreAsync(
                                 _solutionManager,
+                                dgSpec,
                                 cacheContext,
                                 providerCache,
                                 cacheModifier,
                                 sources,
                                 _nuGetProjectContext.OperationId,
                                 forceRestore,
-                                dgSpec,
+                                isRestoreOriginalAction,
                                 l,
                                 t);
 

src/NuGet.Clients/NuGet.SolutionRestoreManager/VsSolutionRestoreService.cs

@@ -57,7 +57,9 @@ public sealed class VsSolutionRestoreService : IVsSolutionRestoreService, IVsSol
         private const string TreatWarningsAsErrors = nameof(TreatWarningsAsErrors);
         private const string WarningsAsErrors = nameof(WarningsAsErrors);
         private const string NoWarn = nameof(NoWarn);
-
+        private const string RestorePackagesWithLockFile = nameof(RestorePackagesWithLockFile);
+        private const string NuGetLockFilePath = nameof(NuGetLockFilePath);
+        private const string RestoreLockedMode = nameof(RestoreLockedMode);
 
         private static readonly Version Version20 = new Version(2, 0, 0, 0);
 
@@ -286,7 +288,11 @@ private static PackageSpec ToPackageSpec(ProjectNames projectNames, IVsProjectRe
                         treatWarningsAsErrors: GetSingleOrDefaultPropertyValue(projectRestoreInfo.TargetFrameworks, TreatWarningsAsErrors, e => e),
                         warningsAsErrors: GetSingleOrDefaultNuGetLogCodes(projectRestoreInfo.TargetFrameworks, WarningsAsErrors, e => MSBuildStringUtility.GetNuGetLogCodes(e)),
                         noWarn: GetSingleOrDefaultNuGetLogCodes(projectRestoreInfo.TargetFrameworks, NoWarn, e => MSBuildStringUtility.GetNuGetLogCodes(e))),
-                    CacheFilePath = NoOpRestoreUtilities.GetProjectCacheFilePath(cacheRoot: outputPath, projectPath: projectFullPath)
+                    CacheFilePath = NoOpRestoreUtilities.GetProjectCacheFilePath(cacheRoot: outputPath, projectPath: projectFullPath),
+                    RestoreLockProperties = new RestoreLockProperties(
+                        GetRestorePackagesWithLockFile(projectRestoreInfo.TargetFrameworks),
+                        GetNuGetLockFilePath(projectRestoreInfo.TargetFrameworks),
+                        IsLockFileFreezeOnRestore(projectRestoreInfo.TargetFrameworks))
                 },
                 RuntimeGraph = GetRuntimeGraph(projectRestoreInfo),
                 RestoreSettings = new ProjectRestoreSettings() { HideWarningsAndErrors = true }
@@ -316,6 +322,21 @@ private static string GetRestoreProjectPath(IVsTargetFrameworks tfms)
             return GetSingleNonEvaluatedPropertyOrNull(tfms, RestorePackagesPath, e => e);
         }
 
+        private static string GetRestorePackagesWithLockFile(IVsTargetFrameworks tfms)
+        {
+            return GetSingleNonEvaluatedPropertyOrNull(tfms, RestorePackagesWithLockFile, v => v);
+        }
+
+        private static string GetNuGetLockFilePath(IVsTargetFrameworks tfms)
+        {
+            return GetSingleNonEvaluatedPropertyOrNull(tfms, NuGetLockFilePath, v => v);
+        }
+
+        private static bool IsLockFileFreezeOnRestore(IVsTargetFrameworks tfms)
+        {
+            return GetSingleNonEvaluatedPropertyOrNull(tfms, RestoreLockedMode, MSBuildStringUtility.IsTrue);
+        }
+
         /// <summary>
         /// The result will contain CLEAR and no sources specified in RestoreSources if the clear keyword is in it.
         /// If there are additional sources specified, the value AdditionalValue will be set in the result and then all the additional sources will follow

src/NuGet.Clients/NuGet.VisualStudio.Common/IVsProjectAdapter.cs

@@ -128,5 +128,23 @@ public interface IVsProjectAdapter
         /// Project's target framework
         /// </summary>
         Task<NuGetFramework> GetTargetFrameworkAsync();
+
+        /// <summary>
+        /// RestorePackagesWithLockFile project property.
+        /// </summary>
+        /// <returns></returns>
+        Task<string> GetRestorePackagesWithLockFileAsync();
+
+        /// <summary>
+        /// NuGetLockFilePath project property.
+        /// </summary>
+        /// <returns></returns>
+        Task<string> GetNuGetLockFilePathAsync();
+
+        /// <summary>
+        /// RestoreLockedMode project property.
+        /// </summary>
+        /// <returns></returns>
+        Task<bool> IsRestoreLockedAsync();
     }
 }

src/NuGet.Core/NuGet.Build.Tasks/NuGet.targets

@@ -119,7 +119,8 @@ Copyright (c) .NET Foundation. All rights reserved.
       RestoreRecursive="$(RestoreRecursive)"
       RestoreForce="$(RestoreForce)"
       HideWarningsAndErrors="$(HideWarningsAndErrors)"
-      Interactive="$(NuGetInteractive)"/>
+      Interactive="$(NuGetInteractive)"
+      ReevaluateRestoreGraph="$(ReevaluateRestoreGraph)"/>
   </Target>
 
   <!--
@@ -657,6 +658,9 @@ Copyright (c) .NET Foundation. All rights reserved.
         <TreatWarningsAsErrors>$(TreatWarningsAsErrors)</TreatWarningsAsErrors>
         <WarningsAsErrors>$(WarningsAsErrors)</WarningsAsErrors>
         <NoWarn>$(NoWarn)</NoWarn>
+        <RestorePackagesWithLockFile>$(RestorePackagesWithLockFile)</RestorePackagesWithLockFile>
+        <NuGetLockFilePath>$(NuGetLockFilePath)</NuGetLockFilePath>
+        <RestoreLockedMode>$(RestoreLockedMode)</RestoreLockedMode>
       </_RestoreGraphEntry>
     </ItemGroup>
 

src/NuGet.Core/NuGet.Build.Tasks/RestoreTask.cs

@@ -74,6 +74,11 @@ public class RestoreTask : Microsoft.Build.Utilities.Task, ICancelableTask, IDis
         /// </summary>
         public bool Interactive { get; set; }
 
+        /// <summary>
+        /// Reevaluate resotre graph even with a lock file, skip no op as well.
+        /// </summary>
+        public bool ReevaluateRestoreGraph { get; set; }
+
         public override bool Execute()
         {
             var log = new MSBuildLogger(Log);
@@ -86,6 +91,7 @@ public override bool Execute()
             log.LogDebug($"(in) RestoreRecursive '{RestoreRecursive}'");
             log.LogDebug($"(in) RestoreForce '{RestoreForce}'");
             log.LogDebug($"(in) HideWarningsAndErrors '{HideWarningsAndErrors}'");
+            log.LogDebug($"(in) Reevaluate '{ReevaluateRestoreGraph}'");
 
             try
             {
@@ -159,7 +165,8 @@ private async Task<bool> ExecuteAsync(Common.ILogger log)
                     PreLoadedRequestProviders = providers,
                     CachingSourceProvider = sourceProvider,
                     AllowNoOp = !RestoreForce,
-                    HideWarningsAndErrors = HideWarningsAndErrors
+                    HideWarningsAndErrors = HideWarningsAndErrors,
+                    ReevaluateRestoreGraph = ReevaluateRestoreGraph
                 };
 
                 if (restoreContext.DisableParallel)

src/NuGet.Core/NuGet.Commands/PackagesLockFileBuilder.cs

@@ -0,0 +1,93 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using NuGet.Common;
+using NuGet.LibraryModel;
+using NuGet.Packaging.Core;
+using NuGet.ProjectModel;
+using NuGet.Shared;
+
+namespace NuGet.Commands
+{
+    public class PackagesLockFileBuilder
+    {
+        public PackagesLockFile CreateNuGetLockFile(LockFile assetsFile)
+        {
+            var lockFile = new PackagesLockFile();
+
+            var libraryLookup = assetsFile.Libraries.Where(e => e.Type == LibraryType.Package)
+                .ToDictionary(e => new PackageIdentity(e.Name, e.Version));
+
+            foreach (var target in assetsFile.Targets)
+            {
+                var nuGettarget = new PackagesLockFileTarget()
+                {
+                    TargetFramework = target.TargetFramework,
+                    RuntimeIdentifier = target.RuntimeIdentifier
+                };
+
+                var framework = assetsFile.PackageSpec.TargetFrameworks.FirstOrDefault(
+                    f => EqualityUtility.EqualsWithNullCheck(f.FrameworkName, target.TargetFramework));
+
+                var libraries = target.Libraries;
+
+                // check if this is RID-based graph then only add those libraries which differ from original TFM.
+                if (!string.IsNullOrEmpty(target.RuntimeIdentifier))
+                {
+                    var onlyTFM = assetsFile.Targets.First(t => EqualityUtility.EqualsWithNullCheck(t.TargetFramework, target.TargetFramework));
+
+                    libraries = target.Libraries.Where(lib => !onlyTFM.Libraries.Any(tfmLib => tfmLib.Equals(lib))).ToList();
+                }
+
+                foreach (var library in libraries.Where(e => e.Type == LibraryType.Package))
+                {
+                    var identity = new PackageIdentity(library.Name, library.Version);
+
+                    var dependency = new LockFileDependency()
+                    {
+                        Id = library.Name,
+                        ResolvedVersion = library.Version,
+                        Sha512 = libraryLookup[identity].Sha512,
+                        Dependencies = library.Dependencies
+                    };
+
+                    var framework_dep = framework?.Dependencies.FirstOrDefault(
+                        dep => StringComparer.OrdinalIgnoreCase.Equals(dep.Name, library.Name));
+
+                    if (framework_dep != null)
+                    {
+                        dependency.Type = PackageDependencyType.Direct;
+                        dependency.RequestedVersion = framework_dep.LibraryRange.VersionRange;
+                    }
+                    else
+                    {
+                        dependency.Type = PackageDependencyType.Transitive;
+                    }
+
+                    nuGettarget.Dependencies.Add(dependency);
+                }
+
+                foreach (var projectReference in libraries.Where(e => e.Type == LibraryType.Project || e.Type == LibraryType.ExternalProject))
+                {
+                    var dependency = new LockFileDependency()
+                    {
+                        Id = projectReference.Name,
+                        Dependencies = projectReference.Dependencies,
+                        Type = PackageDependencyType.Project
+                    };
+
+                    nuGettarget.Dependencies.Add(dependency);
+                }
+
+                nuGettarget.Dependencies = nuGettarget.Dependencies.OrderBy(d => d.Type).ToList();
+
+                lockFile.Targets.Add(nuGettarget);
+            }
+
+            return lockFile;
+        }
+
+    }
+}

src/NuGet.Core/NuGet.Commands/RestoreCommand/RequestFactory/DependencyGraphSpecRequestProvider.cs

@@ -154,8 +154,7 @@ private static ExternalProjectReference GetExternalProject(PackageSpec rootProje
                 //  Project.json is special cased to put assets file and generated .props and targets in the project folder
                 RestoreOutputPath = project.PackageSpec.RestoreMetadata.ProjectStyle == ProjectStyle.ProjectJson ? rootPath : project.PackageSpec.RestoreMetadata.OutputPath,
                 DependencyGraphSpec = projectDgSpec,
-                MSBuildProjectExtensionsPath = projectPackageSpec.RestoreMetadata.OutputPath,
-                ParentId = restoreArgs.ParentId
+                MSBuildProjectExtensionsPath = projectPackageSpec.RestoreMetadata.OutputPath
             };
 
             var restoreLegacyPackagesDirectory = project.PackageSpec?.RestoreMetadata?.LegacyPackagesDirectory

src/NuGet.Core/NuGet.Commands/RestoreCommand/RequestFactory/NoOpRestoreResult.cs

@@ -18,7 +18,7 @@ public class NoOpRestoreResult : RestoreResult
         public NoOpRestoreResult(bool success, LockFile lockFile, LockFile previousLockFile, string lockFilePath, CacheFile cacheFile, string cacheFilePath, ProjectStyle projectStyle, TimeSpan elapsedTime) :
             base(success : success, restoreGraphs : null, compatibilityCheckResults : new List<CompatibilityCheckResult>() , 
                 msbuildFiles : null, lockFile : lockFile, previousLockFile : previousLockFile, lockFilePath: lockFilePath,
-                cacheFile: cacheFile, cacheFilePath: cacheFilePath, projectStyle: projectStyle, elapsedTime: elapsedTime)
+                cacheFile: cacheFile, cacheFilePath: cacheFilePath, packagesLockFilePath:null, packagesLockFile:null, projectStyle: projectStyle, elapsedTime: elapsedTime)
         {
         }
 

src/NuGet.Core/NuGet.Commands/RestoreCommand/RequestFactory/RestoreArgs.cs

@@ -62,6 +62,10 @@ public class RestoreArgs
 
         public Guid ParentId { get; set; }
 
+        public bool IsRestoreOriginalAction { get; set; } = true;
+
+        public bool ReevaluateRestoreGraph { get; set; }
+
         // Cache directory -> ISettings
         private ConcurrentDictionary<string, ISettings> _settingsCache
             = new ConcurrentDictionary<string, ISettings>(StringComparer.Ordinal);
@@ -227,6 +231,9 @@ public void ApplyStandardProperties(RestoreRequest request)
 
             request.AllowNoOp = !request.CacheContext.NoCache && AllowNoOp;
             request.HideWarningsAndErrors = HideWarningsAndErrors;
+            request.ParentId = ParentId;
+            request.IsRestoreOriginalAction = IsRestoreOriginalAction;
+            request.ReevaluateRestoreGraph = ReevaluateRestoreGraph;
         }
     }
 }