Create build manifest in PushToBlobFeed

src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj

@@ -7,6 +7,7 @@
     <CopyNuGetImplementations>true</CopyNuGetImplementations>
     <ImportedProjectRelativePath>..\Microsoft.DotNet.Build.Tasks.Feed\</ImportedProjectRelativePath>
     <ProjectGuid>{17C66BCE-EB35-44CA-893C-8AAFB67708E9}</ProjectGuid>
+    <TargetFrameworkProjectSuffix>.net45</TargetFrameworkProjectSuffix>
   </PropertyGroup>
   <Import Project="$(ImportedProjectRelativePath)Microsoft.DotNet.Build.Tasks.Feed.csproj" />
   <PropertyGroup>
@@ -18,6 +19,7 @@
     <TargetingPackReference Include="Microsoft.Build.Utilities.v4.0" />
     <TargetingPackReference Include="System" />
     <TargetingPackReference Include="System.Net.Http" />
+    <TargetingPackReference Include="System.Collections" />
     <TargetingPackReference Include="System.Core" />
     <TargetingPackReference Include="System.IO" />
     <TargetingPackReference Include="System.IO.Compression" />

src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs

@@ -66,5 +66,23 @@ public async Task<bool> CheckIfBlobExists(string blobPath)
                 }
             }
         }
+
+        public async Task<string> DownloadBlobAsString(string blobPath)
+        {
+            string url = $"{FeedContainerUrl}/{blobPath}";
+            using (HttpClient client = new HttpClient())
+            {
+                client.DefaultRequestHeaders.Clear();
+                var request = AzureHelper.RequestMessage("GET", url, AccountName, AccountKey)();
+                using (HttpResponseMessage response = await client.SendAsync(request))
+                {
+                    if (response.IsSuccessStatusCode)
+                    {
+                        return await response.Content.ReadAsStringAsync();
+                    }
+                    return null;
+                }
+            }
+        }
     }
 }

src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj

@@ -55,6 +55,12 @@
     <None Include="PackageFiles\Microsoft.DotNet.Build.Tasks.Feed.targets" />
     <None Include="project.json" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.DotNet.VersionTools$(TargetFrameworkProjectSuffix)\Microsoft.DotNet.VersionTools$(TargetFrameworkProjectSuffix).csproj">
+      <Project>{8d524fa5-a8c5-4ebd-ba8b-2a4fed03ee58}</Project>
+      <Name>Microsoft.DotNet.VersionTools$(TargetFrameworkProjectSuffix)</Name>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
   <Target Name="AfterBuild">
     <ItemGroup>

src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs

@@ -3,18 +3,25 @@
 // See the LICENSE file in the project root for more information.
 
 using Microsoft.Build.Framework;
+using Microsoft.DotNet.VersionTools.Automation;
+using Microsoft.DotNet.VersionTools.BuildManifest.Model;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Xml.Linq;
 using MSBuild = Microsoft.Build.Utilities;
 
 namespace Microsoft.DotNet.Build.Tasks.Feed
 {
     public class PushToBlobFeed : MSBuild.Task
     {
+        private static readonly char[] ManifestDataPairSeparators = { ';' };
+        private const string DisableManifestPushConfigurationBlob = "disable-manifest-push";
+        private const string AssetsVirtualDir = "assets/";
+
         [Required]
         public string ExpectedFeedUrl { get; set; }
 
@@ -34,6 +41,22 @@ public class PushToBlobFeed : MSBuild.Task
 
         public int UploadTimeoutInMinutes { get; set; } = 5;
 
+        public bool SkipCreateManifest { get; set; }
+
+        public string ManifestName { get; set; } = "anonymous";
+        public string ManifestBuildId { get; set; } = "no build id provided";
+        public string ManifestBranch { get; set; }
+        public string ManifestCommit { get; set; }
+
+        /// <summary>
+        /// When publishing build outputs to an orchestrated blob feed, do not change this property.
+        /// 
+        /// The virtual dir to place the manifest XML file in, under the assets/ virtual dir. The
+        /// default value is the well-known location that orchestration searches to find all
+        /// manifest XML files and combine them into the orchestrated build output manifest.
+        /// </summary>
+        public string ManifestAssetOutputDir { get; set; } = "orchestration-metadata/manifests/";
+
         public override bool Execute()
         {
             return ExecuteAsync().GetAwaiter().GetResult();
@@ -53,6 +76,9 @@ public async Task<bool> ExecuteAsync()
                 {
                     BlobFeedAction blobFeedAction = new BlobFeedAction(ExpectedFeedUrl, AccountKey, Log);
 
+                    IEnumerable<BlobArtifactModel> blobArtifacts = Enumerable.Empty<BlobArtifactModel>();
+                    IEnumerable<PackageArtifactModel> packageArtifacts = Enumerable.Empty<PackageArtifactModel>();
+
                     if (!SkipCreateContainer)
                     {
                         await blobFeedAction.CreateContainerAsync(BuildEngine, PublishFlatContainer);
@@ -61,6 +87,7 @@ public async Task<bool> ExecuteAsync()
                     if (PublishFlatContainer)
                     {
                         await PublishToFlatContainerAsync(ItemsToPush, blobFeedAction);
+                        blobArtifacts = ConcatBlobArtifacts(blobArtifacts, ItemsToPush);
                     }
                     else
                     {
@@ -69,12 +96,24 @@ public async Task<bool> ExecuteAsync()
                             string fileName = Path.GetFileName(i.ItemSpec);
                             i.SetMetadata("RelativeBlobPath", $"symbols/{fileName}");
                             return i;
-                        });
+                        }).ToArray();
 
-                        var packageItems = ItemsToPush.Where(i => !symbolItems.Contains(i)).Select(i => i.ItemSpec);
+                        ITaskItem[] packageItems = ItemsToPush
+                            .Where(i => !symbolItems.Contains(i))
+                            .ToArray();
 
-                        await blobFeedAction.PushToFeed(packageItems, Overwrite);
+                        var packagePaths = packageItems.Select(i => i.ItemSpec);
+
+                        await blobFeedAction.PushToFeed(packagePaths, Overwrite);
                         await PublishToFlatContainerAsync(symbolItems, blobFeedAction);
+
+                        packageArtifacts = ConcatPackageArtifacts(packageArtifacts, packageItems);
+                        blobArtifacts = ConcatBlobArtifacts(blobArtifacts, symbolItems);
+                    }
+
+                    if (!SkipCreateManifest)
+                    {
+                        await PushBuildManifestAsync(blobFeedAction, blobArtifacts, packageArtifacts);
                     }
                 }
             }
@@ -86,6 +125,76 @@ public async Task<bool> ExecuteAsync()
             return !Log.HasLoggedErrors;
         }
 
+        private async Task PushBuildManifestAsync(
+            BlobFeedAction blobFeedAction,
+            IEnumerable<BlobArtifactModel> blobArtifacts,
+            IEnumerable<PackageArtifactModel> packageArtifacts)
+        {
+            bool disabledByBlob = await blobFeedAction.feed.CheckIfBlobExists(
+                $"{blobFeedAction.feed.RelativePath}{DisableManifestPushConfigurationBlob}");
+
+            if (disabledByBlob)
+            {
+                Log.LogMessage(
+                    MessageImportance.Normal,
+                    $"Skipping manifest push: feed has '{DisableManifestPushConfigurationBlob}'.");
+                return;
+            }
+
+            string blobPath = $"{AssetsVirtualDir}{ManifestAssetOutputDir}{ManifestName}.xml";
+
+            string existingStr = await blobFeedAction.feed.DownloadBlobAsString(
+                $"{blobFeedAction.feed.RelativePath}{blobPath}");
+
+            BuildModel buildModel;
+
+            if (existingStr != null)
+            {
+                buildModel = BuildModel.Parse(XElement.Parse(existingStr));
+            }
+            else
+            {
+                buildModel = new BuildModel(
+                    new BuildIdentity(
+                        ManifestName,
+                        ManifestBuildId,
+                        ManifestBranch,
+                        ManifestCommit));
+            }
+
+            buildModel.Artifacts.Blobs.AddRange(blobArtifacts);
+            buildModel.Artifacts.Packages.AddRange(packageArtifacts);
+
+            string tempFile = null;
+            try
+            {
+                tempFile = Path.GetTempFileName();
+
+                File.WriteAllText(tempFile, buildModel.ToXml().ToString());
+
+                var item = new MSBuild.TaskItem(tempFile, new Dictionary<string, string>
+                {
+                    ["RelativeBlobPath"] = blobPath
+                });
+
+                using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients))
+                {
+                    await blobFeedAction.UploadAssets(
+                        item,
+                        clientThrottle,
+                        UploadTimeoutInMinutes,
+                        allowOverwrite: true);
+                }
+            }
+            finally
+            {
+                if (tempFile != null)
+                {
+                    File.Delete(tempFile);
+                }
+            }
+        }
+
         private async Task PublishToFlatContainerAsync(IEnumerable<ITaskItem> taskItems, BlobFeedAction blobFeedAction)
         {
             if (taskItems.Any())
@@ -97,5 +206,74 @@ private async Task PublishToFlatContainerAsync(IEnumerable<ITaskItem> taskItems,
                 }
             }
         }
+
+        private static IEnumerable<PackageArtifactModel> ConcatPackageArtifacts(
+            IEnumerable<PackageArtifactModel> artifacts,
+            IEnumerable<ITaskItem> items)
+        {
+            return artifacts.Concat(items
+                .Select(CreatePackageArtifactModel));
+        }
+
+        private static IEnumerable<BlobArtifactModel> ConcatBlobArtifacts(
+            IEnumerable<BlobArtifactModel> artifacts,
+            IEnumerable<ITaskItem> items)
+        {
+            return artifacts.Concat(items
+                .Select(CreateBlobArtifactModel)
+                .Where(blob => blob != null));
+        }
+
+        private static PackageArtifactModel CreatePackageArtifactModel(ITaskItem item)
+        {
+            NupkgInfo info = new NupkgInfo(item.ItemSpec);
+
+            return new PackageArtifactModel
+            {
+                Attributes = ParseCustomAttributes(item),
+                Id = info.Id,
+                Version = info.Version
+            };
+        }
+
+        private static BlobArtifactModel CreateBlobArtifactModel(ITaskItem item)
+        {
+            string path = item.GetMetadata("RelativeBlobPath");
+
+            // Only include assets in the manifest if they're in "assets/".
+            if (path?.StartsWith(AssetsVirtualDir, StringComparison.Ordinal) == true)
+            {
+                return new BlobArtifactModel
+                {
+                    Attributes = ParseCustomAttributes(item),
+                    Id = path.Substring(AssetsVirtualDir.Length)
+                };
+            }
+            return null;
+        }
+
+        private static Dictionary<string, string> ParseCustomAttributes(ITaskItem item)
+        {
+            Dictionary<string, string> customAttributes = item
+                .GetMetadata("ManifestArtifactData")
+                ?.Split(ManifestDataPairSeparators, StringSplitOptions.RemoveEmptyEntries)
+                .Select(pair =>
+                {
+                    int keyValueSeparatorIndex = pair.IndexOf('=');
+                    if (keyValueSeparatorIndex > 0)
+                    {
+                        return new
+                        {
+                            Key = pair.Substring(0, keyValueSeparatorIndex).Trim(),
+                            Value = pair.Substring(keyValueSeparatorIndex + 1).Trim()
+                        };
+                    }
+                    return null;
+                })
+                .Where(pair => pair != null)
+                .ToDictionary(pair => pair.Key, pair => pair.Value);
+
+            return customAttributes ?? new Dictionary<string, string>();
+        }
     }
 }