diff --git a/src/CBT.NuGet.AggregatePackage/CBT.NuGet.AggregatePackage.nuproj b/src/CBT.NuGet.AggregatePackage/CBT.NuGet.AggregatePackage.nuproj new file mode 100644 index 0000000..f78cc97 --- /dev/null +++ b/src/CBT.NuGet.AggregatePackage/CBT.NuGet.AggregatePackage.nuproj @@ -0,0 +1,20 @@ + + + + Provides ability to aggregate nuget packages. + true + 525e50eb-c6d9-497d-bc54-c625ab62b7d0 + CBT Module NuGet Aggregate Packages + CBT NuGet Aggregate Package Restore + + + + + + [1.0,) + + + + + + \ No newline at end of file diff --git a/src/CBT.NuGet.AggregatePackage/CBT/Module/After.CBT.NuGet.props b/src/CBT.NuGet.AggregatePackage/CBT/Module/After.CBT.NuGet.props new file mode 100644 index 0000000..147a471 --- /dev/null +++ b/src/CBT.NuGet.AggregatePackage/CBT/Module/After.CBT.NuGet.props @@ -0,0 +1,5 @@ + + + + + diff --git a/src/CBT.NuGet.AggregatePackage/CBT/Module/After.Microsoft.Common.targets b/src/CBT.NuGet.AggregatePackage/CBT/Module/After.Microsoft.Common.targets new file mode 100644 index 0000000..b39c9e0 --- /dev/null +++ b/src/CBT.NuGet.AggregatePackage/CBT/Module/After.Microsoft.Common.targets @@ -0,0 +1,23 @@ + + + + + $(RestoreNuGetPackagesDependsOn);AggregateNugetPackages + + + + + + + + + + diff --git a/src/CBT.NuGet.AggregatePackage/CBT/Module/CBT.Nuget.AggregatePackage.props b/src/CBT.NuGet.AggregatePackage/CBT/Module/CBT.Nuget.AggregatePackage.props new file mode 100644 index 0000000..22b2733 --- /dev/null +++ b/src/CBT.NuGet.AggregatePackage/CBT/Module/CBT.Nuget.AggregatePackage.props @@ -0,0 +1,36 @@ + + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + + + + + true + $(MSBuildProjectDirectory)\CBT.AggregatePackages.props + $(NuGetPackagesPath)\.agg + $(NuGetPackagesPath) + + + + + + $(IntermediateOutputPath)\AggregatePackages.props + $(CBTNuGetTasksAssemblyPath.GetType().Assembly.GetType('System.AppDomain').GetProperty('CurrentDomain').GetValue(null).CreateInstanceFromAndUnwrap($(CBTNuGetTasksAssemblyPath), 'CBT.NuGet.Tasks.AggregatePackages').Execute('$(CBTAggregateDestPackageRoot)', '$(CBTAggregatePackage)', '$(CBTNuGetAggregatePackagePropertyFile)', '$(CBTNugetAggregatePackageImmutableRoots)')) + + + + + CBT.Nuget.AggregatePackage.1000 + + + + + + + + + diff --git a/src/CBT.NuGet.AggregatePackage/CBT/Module/module.config b/src/CBT.NuGet.AggregatePackage/CBT/Module/module.config new file mode 100644 index 0000000..c12b06a --- /dev/null +++ b/src/CBT.NuGet.AggregatePackage/CBT/Module/module.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/CBT.NuGet.AggregatePackage/version.json b/src/CBT.NuGet.AggregatePackage/version.json new file mode 100644 index 0000000..458e29c --- /dev/null +++ b/src/CBT.NuGet.AggregatePackage/version.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://raw.githubusercontent.com/AArnott/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", + "version": "1.0-dev", + "buildNumberOffset": 0, + "publicReleaseRefSpec": [ + "^refs/tags/CBT\\.NuGet\\.AggregatePackage.*" + ], + "cloudBuild": { + "setVersionVariables": false + } +} \ No newline at end of file diff --git a/src/CBT.NuGet.Package/CBT/Module/CBT.NuGet.props b/src/CBT.NuGet.Package/CBT/Module/CBT.NuGet.props index 1de0f70..ac05e88 100644 --- a/src/CBT.NuGet.Package/CBT/Module/CBT.NuGet.props +++ b/src/CBT.NuGet.Package/CBT/Module/CBT.NuGet.props @@ -31,6 +31,12 @@ CBT.Nuget.1002 - + + + + + + + \ No newline at end of file diff --git a/src/CBT.NuGet.Package/CBT/Module/build.props b/src/CBT.NuGet.Package/CBT/Module/build.props index c1b004d..1feb8b2 100644 --- a/src/CBT.NuGet.Package/CBT/Module/build.props +++ b/src/CBT.NuGet.Package/CBT/Module/build.props @@ -48,10 +48,4 @@ - - - - - - \ No newline at end of file diff --git a/src/CBT.NuGet/CBT.NuGet.csproj b/src/CBT.NuGet/CBT.NuGet.csproj index 43ee9a7..59bb582 100644 --- a/src/CBT.NuGet/CBT.NuGet.csproj +++ b/src/CBT.NuGet/CBT.NuGet.csproj @@ -23,9 +23,12 @@ + + + diff --git a/src/CBT.NuGet/Internal/AggregatePackage.cs b/src/CBT.NuGet/Internal/AggregatePackage.cs new file mode 100644 index 0000000..0bae552 --- /dev/null +++ b/src/CBT.NuGet/Internal/AggregatePackage.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace CBT.NuGet.Internal +{ + internal sealed class AggregatePackage + { + public enum AggregateOperation + { + Add = '*', + Remove = '!', + } + + internal class PackageOperation + { + private string folder = string.Empty; + internal AggregateOperation Operation { get; set; } + + internal string Folder + { + get + { + return folder; + } + set + { + folder = Path.GetFullPath(value); + } + } + } + + private string[] immutableRootPaths = null; + + public AggregatePackage(string outPropertyId, ICollection packagesToAggregate, string destinationRoot, string immutableRoots) + { + OutPropertyId = outPropertyId; + PackagesToAggregate = packagesToAggregate; + immutableRootPaths = immutableRoots.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(i => Path.GetFullPath(i.Trim())).Where(j => !string.IsNullOrWhiteSpace(j)).ToArray(); + OutPropertyValue = Path.Combine(destinationRoot, $"{outPropertyId}.{GetOutputPropertyHash()}"); + } + + public string OutPropertyId { get; private set; } + + public string OutPropertyValue { get; private set; } + + public ICollection PackagesToAggregate { get; private set; } + + private string GetOutputPropertyHash() + { + StringBuilder valueToHash = new StringBuilder(); + foreach (var pkg in PackagesToAggregate) + { + valueToHash.Append(pkg.Folder.ToLower()); + // Include last write time on all files to ensure that if the source folder was updated a new aggregate package will be created. + // if a file was removed then it will not be in the value to hash so the hash will still change to trigger a new aggregation as desired. + // We don't care about directories in aggregation since removing directories is not possible given current implementation. + // This is in the scenario where the user aggregates against a nuget package and a checked in folder under source control that contains a config file or whatever. + // Since this is a string contains match an immutable root match could be problematic in corner cases. d:\tmp\src and d:\tmp\src2 will both match d:\tmp\sr as an immutable root. + valueToHash.Append(Directory.GetLastWriteTime(pkg.Folder).ToString()); + if ( !immutableRootPaths.Any() || !immutableRootPaths.Where(i => pkg.Folder.IndexOf(i, StringComparison.OrdinalIgnoreCase) >= 0).Any()) + { + foreach (var f in Directory.EnumerateFiles(pkg.Folder, "*.*", SearchOption.AllDirectories)) + { + valueToHash.Append(File.GetLastWriteTime(f).ToString()); + } + } + } + + string hashValue = Convert.ToBase64String((new SHA1CryptoServiceProvider()).ComputeHash(Encoding.UTF8.GetBytes(valueToHash.ToString()))); + Regex pattern = new Regex("[/=+]"); + hashValue = pattern.Replace(hashValue, "x"); + return string.Format("{0:X}", hashValue); + } + + } +} \ No newline at end of file diff --git a/src/CBT.NuGet/Internal/FileUtilities.cs b/src/CBT.NuGet/Internal/FileUtilities.cs new file mode 100644 index 0000000..f99fc30 --- /dev/null +++ b/src/CBT.NuGet/Internal/FileUtilities.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace CBT.NuGet.Internal +{ + class FileUtilities + { + // Need to find a proper directory copy routine. + public static void DirectoryCopy(string sourceDirName, string destDirName, bool copySubDirs, bool overWriteFiles) + { + if (!Directory.Exists(sourceDirName)) + { + throw new DirectoryNotFoundException("Source Directory does not exist or could not be found: " + sourceDirName); + } + DirectoryInfo dir = new DirectoryInfo(sourceDirName); + DirectoryInfo[] dirs = dir.GetDirectories(); + if (!Directory.Exists(destDirName)) + { + Directory.CreateDirectory(destDirName); + } + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo file in files) + { + string tempPath = Path.Combine(destDirName, file.Name); + file.CopyTo(tempPath, overWriteFiles); + } + if (copySubDirs) + { + foreach (DirectoryInfo subdir in dirs) + { + string temppath = Path.Combine(destDirName, subdir.Name); + DirectoryCopy(subdir.FullName, temppath, copySubDirs, overWriteFiles); + } + } + } + + public static void DirectoryRemove(string sourceDirName, string destDirName, bool recurseSubDirs) + { + DirectoryInfo dir = new DirectoryInfo(sourceDirName); + if (!dir.Exists) + { + throw new DirectoryNotFoundException("Source Directory does not exist or could not be found: " + sourceDirName); + } + DirectoryInfo[] dirs = dir.GetDirectories(); + + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo file in files) + { + string tempPath = Path.Combine(destDirName, file.Name); + if (File.Exists(tempPath)) + { + File.SetAttributes(tempPath, File.GetAttributes(tempPath) & ~FileAttributes.ReadOnly); + File.Delete(tempPath); + } + } + if (recurseSubDirs) + { + foreach (DirectoryInfo subdir in dirs) + { + string temppath = Path.Combine(destDirName, subdir.Name); + DirectoryRemove(subdir.FullName, temppath, recurseSubDirs); + } + } + } + + // Mutex isn't platform agnostic need to consider options. + public static string ComputeMutexName(string sessionString) + { + // get a hash of the file path; reason: there's a limit on name length for named mutexes + using (var algo = SHA256.Create()) + { + // Global: make it work across TS sessions; not that we should need this, but just to be super-extra safe + return "Global\\" + Convert.ToBase64String(algo.ComputeHash(Encoding.UTF8.GetBytes(sessionString.ToUpperInvariant()))); + } + } + + public static void AcquireMutex(Mutex mutex) + { + try + { + bool owner = mutex.WaitOne(); // Perhaps specify a timeout value so builds don't hang. + if (!owner) + throw new TimeoutException("Timeout waiting for mutex"); + } + catch (AbandonedMutexException) + { + // Unlikely we care. + Trace.TraceWarning("mutex for Aggregate package was abandoned."); + } + } + } +} diff --git a/src/CBT.NuGet/Tasks/AggregatePackages.cs b/src/CBT.NuGet/Tasks/AggregatePackages.cs new file mode 100644 index 0000000..c1f7570 --- /dev/null +++ b/src/CBT.NuGet/Tasks/AggregatePackages.cs @@ -0,0 +1,232 @@ +using CBT.NuGet.Internal; +using Microsoft.Build.Construction; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using static CBT.NuGet.Internal.AggregatePackage; + +namespace CBT.NuGet.Tasks +{ + /// + /// Generate NuGet properties. + /// + /// Generate properties that contain the path and version of a given nuget package. + /// + public sealed class AggregatePackages : Task + { + /// + /// Gets or sets the msbuild item of that contains the packages to aggregate. + /// Example Input: + /// + /// + /// adds the folder to the aggregate unless folder starts with ! then it removes it. + /// | separates list of folders used for aggregation. + /// NugetPath_LSBuild_Corext=$(NugetPath_LSBuild_Corext)|$(NugetPath_Microsoft_LSBuild_Extensions_SQLIS)|!$(NugetPath_Microsoft_LSBuild_Excluded_Extensions_SQLIS);MyAggPkg=$(NugetPath_Azure_Corext)|$(NugetPath_Azure_Corext_AGG) + /// + /// + [Required] + public string PackagesToAggregate { get; set; } + + /// + /// Gets or sets the full path of the props file that is written to. + /// + [Required] + public string PropsFile { get; set; } + + /// + /// Gets or sets the root path of the packages to be aggregated. + /// + [Required] + public string AggregateDestRoot { get; set; } + + /// + /// Gets or sets the root paths of folders that are considered to be immutable and that the content will never change for that unique folder name. Example a nuget package. + /// + [Required] + public string ImmutableRoots { get; set; } + + public override bool Execute() + { + + IDictionary propertiesToCreate = new Dictionary(); + + foreach (var pkg in ParsePackagesToAggregate()) + { + try + { + if (!CreateAggregatePackage(pkg)) + { + Log.LogError("Failed to create aggregate package {0} for input of {1}", pkg.OutPropertyId, PackagesToAggregate); + } + } + catch (DirectoryNotFoundException) + { + Log.LogError("Aggregate package {0} not found after aggregation.", pkg.OutPropertyValue); + } + if (propertiesToCreate.ContainsKey(pkg.OutPropertyId)) + { + Log.LogWarning("Duplicate Aggregate package {0} specified. Using first defined.", pkg.OutPropertyId); + continue; + } + propertiesToCreate.Add(pkg.OutPropertyId, pkg.OutPropertyValue); + } + + try + { + CreatePropsFile(propertiesToCreate, PropsFile); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + + if (Log.HasLoggedErrors) + { + Log.LogError("Define aggregate packages in the format of 'MYAGGPROPERTY=c:\\pkg1|c:\\pkg2|!c:\\pkg3;MYAAGPROPERTY2=c:\\pkg1|c:\\pkg2' where ; seperates aggregate packages and | seperates paths to be aggregated for a package and ! denotes content that should be excluded from the aggregate."); + } + + return !Log.HasLoggedErrors; + } + + public bool Execute(string aggregateDestRoot, string packagesToAggregate, string propsFile, string immutableRoots) + { + BuildEngine = new CBTBuildEngine(); + AggregateDestRoot = aggregateDestRoot; + PackagesToAggregate = packagesToAggregate; + PropsFile = propsFile; + ImmutableRoots = immutableRoots; + return Execute(); + } + + private void CreatePropsFile(IDictionary propertyPairs, string propsFile) + { + ProjectRootElement project = ProjectRootElement.Create(); + + ProjectPropertyGroupElement propertyGroup = project.AddPropertyGroup(); + propertyGroup.SetProperty("MSBuildAllProjects", "$(MSBuildAllProjects);$(MSBuildThisFileFullPath)"); + + foreach (var kvp in propertyPairs) + { + propertyGroup.SetProperty(kvp.Key, kvp.Value); + } + + project.Save(propsFile); + } + + internal IEnumerable ParsePackagesToAggregate() + { + // foo=pkg|pkg2|!pkg3;foo2=pkg|pkg2|!pkg3 + + foreach (var item in PackagesToAggregate.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries) + .Select(i => i.Trim()) + .Where(i => !String.IsNullOrWhiteSpace(i)) + .Select(i => i.Split(new[] { '=' }, 2, StringSplitOptions.RemoveEmptyEntries)) + .Select(i => new + { + PropertyName = i.First(), + Options = i.Length == 2 ? i.Last() : null + }) + ) + { + IList packageOperations = ParsePackageOperations(item.Options).ToList(); + + if (packageOperations.Count == 0) + { + // Invalid item because nothing is on the right side of the equal sign or there was no equal sign + Log.LogError($"No valid paths were found to aggregate for '{item.PropertyName}' with options '{item.Options}'"); + continue; + } + + yield return new AggregatePackage(item.PropertyName, packageOperations, AggregateDestRoot, ImmutableRoots); + } + } + + private IEnumerable ParsePackageOperations(string options) + { + // pkg1|pkg2|!pkg + + if (String.IsNullOrWhiteSpace(options)) + { + yield break; + } + + foreach (var option in options.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries) + .Select(i => i.Trim()) + .Where(i => !String.IsNullOrWhiteSpace(i))) + { + AggregateOperation aggregateOperation = option.First() == (char)AggregateOperation.Remove ? AggregateOperation.Remove : AggregateOperation.Add; + + string folder = option.TrimStart((char)AggregateOperation.Remove).Trim(); + + if (!Directory.Exists(folder)) + { + Log.LogError($"Path to aggregate '{folder}' does not exist."); + continue; + } + + yield return new PackageOperation { Operation = aggregateOperation, Folder = folder }; + } + } + + internal bool CreateAggregatePackage(AggregatePackage package) + { + // The assumption is that in order for the source of an aggregate to change the package source folder must of changed for a new version. + // And therefor it is assumed this never needs to be regenerated (assumed corruption would be cleaned up manually). + if (Directory.Exists(package.OutPropertyValue)) + { + Log.LogMessage(MessageImportance.Low, $"{package.OutPropertyValue} already created. Skipping"); + return true; + } + + using (var mutex = new Mutex(false, FileUtilities.ComputeMutexName(package.OutPropertyValue))) + { + bool owner = false; + try + { + var outTmpDir = package.OutPropertyValue + ".tmp"; + FileUtilities.AcquireMutex(mutex); + owner = true; + // check again to see if aggregate package is already created while waiting. + if (Directory.Exists(package.OutPropertyValue)) + { + Log.LogMessage(MessageImportance.Low, $"{package.OutPropertyValue} already created. Skipping"); + return true; + } + + if (Directory.Exists(outTmpDir)) + { + Log.LogMessage(MessageImportance.Low, $"{outTmpDir} not cleaned up from previous build cleaning now."); + Directory.Delete(outTmpDir, true); + } + foreach (var srcPkg in package.PackagesToAggregate) + { + if (srcPkg.Operation.Equals(AggregatePackage.AggregateOperation.Add)) + { + Log.LogMessage(MessageImportance.Low, $"Adding {srcPkg.Folder} to aggregate of {package.OutPropertyValue}"); + FileUtilities.DirectoryCopy(srcPkg.Folder, outTmpDir, true, true); + } + if (srcPkg.Operation.Equals(AggregatePackage.AggregateOperation.Remove)) + { + Log.LogMessage(MessageImportance.Low, $"Removing {srcPkg.Folder} from aggregate of {package.OutPropertyValue}"); + FileUtilities.DirectoryRemove(srcPkg.Folder, outTmpDir, true); + } + } + Directory.Move(outTmpDir, package.OutPropertyValue); + } + finally + { + if (owner) + { + mutex.ReleaseMutex(); + } + } + } + return Directory.Exists(package.OutPropertyValue); + } + + } +}