prepare-artifacts.proj

<Project Sdk="Microsoft.Build.NoTargets">
  <PropertyGroup>
    <!-- Set IsStableBuild to mimic https://github.com/dotnet/arcade/blob/694d59f090b743f894779d04a7ffe11cbaf352e7/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj#L30-L31 -->
    <IsStableBuild>false</IsStableBuild>
    <IsStableBuild Condition="'$(DotNetFinalVersionKind)' == 'release'">true</IsStableBuild>

    <ChecksumExtension>.sha512</ChecksumExtension>
    <DownloadDirectory>$(ArtifactsDir)PackageDownload/</DownloadDirectory>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
    <PackageReference Include="Microsoft.DotNet.VersionTools.Tasks" Version="$(MicrosoftDotNetVersionToolsTasksVersion)" />
  </ItemGroup>

  <!-- When doing post build sign, pull in the arcade sign.props file, which will
       create some initial sign groups and then pulls in the repo's custom Signing.props overrides.
       Before importing the signing props. Set PrepareArtifactst=true. Depending on context (SDK project vs. not),
       the initial imports of livebuilds.targets and Configuration.props are already imported and thus cannot
       be imported again. Without those imports, the globbing of files to sign will not work properly. -->
  <PropertyGroup>
    <PrepareArtifacts>true</PrepareArtifacts>
  </PropertyGroup>
  <Import Project="../tools/Sign.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />

  <Import Project="$(RepositoryEngineeringDir)Publishing.props" Condition="Exists('$(RepositoryEngineeringDir)Publishing.props')" />

  <UsingTask TaskName="GenerateChecksums" AssemblyFile="$(InstallerTasksAssemblyPath)" />

  <PropertyGroup>
    <SuppressFinalPackageVersion>true</SuppressFinalPackageVersion>
  </PropertyGroup>

  <!-- Pulled from arcade's publish.proj see https://github.com/dotnet/arcade/issues/5790 for
       issue on making this common in arcade so it doesn't have to be duplicated here -->
  <PropertyGroup>
    <CollectionUri>$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)</CollectionUri>

    <!-- When we have dev.azure.com/<account>/ -->
    <AzureDevOpsAccount Condition="$(CollectionUri.IndexOf('dev.azure.com')) >= 0">$(CollectionUri.Split('/')[3])</AzureDevOpsAccount>

    <!-- When we have <account>.visualstudio.com -->
    <AzureDevOpsAccount Condition="$(CollectionUri.IndexOf('visualstudio.com')) >= 0">$(CollectionUri.Split('.')[0].Split('/')[2])</AzureDevOpsAccount>
  </PropertyGroup>

  <!--
    The new Maestro/BAR build model keeps separate Azure DevOps and GitHub build information.
    The GitHub information will be extracted based on the Azure DevOps repository.
  -->
  <ItemGroup>
    <ManifestBuildData Include="InitialAssetsLocation=$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)$(SYSTEM_TEAMPROJECT)/_apis/build/builds/$(BUILD_BUILDID)/artifacts" />
    <ManifestBuildData Include="AzureDevOpsBuildId=$(BUILD_BUILDID)" />
    <ManifestBuildData Include="AzureDevOpsBuildDefinitionId=$(SYSTEM_DEFINITIONID)" />
    <ManifestBuildData Include="AzureDevOpsAccount=$(AzureDevOpsAccount)" />
    <ManifestBuildData Include="AzureDevOpsProject=$(SYSTEM_TEAMPROJECT)" />
    <ManifestBuildData Include="AzureDevOpsBuildNumber=$(BUILD_BUILDNUMBER)" />
    <ManifestBuildData Include="AzureDevOpsRepository=$(BUILD_REPOSITORY_URI)" />
    <ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
  </ItemGroup>

  <!-- 
    Set metadata for assets that are not marked as NonShipping. 
    This is used to determine if the asset should be shipped as part of .NET release.
  -->
  <ItemDefinitionGroup>
    <ItemsToPush>
      <ManifestArtifactData Condition="'$(ProducesDotNetReleaseShippingAssets)' == 'true'">DotNetReleaseShipping=true</ManifestArtifactData>
    </ItemsToPush>
  </ItemDefinitionGroup>

  <!--
    Run Arcade's signing project directly. The 'eng/Signing.props' extensibility props file checks
    if '$(<StageName>)' == 'true' and points Arcade to the correct files.
  -->
  <Target Name="SignArtifacts"
          Condition="'$(PostBuildSign)' != 'true'" >
    <MSBuild
      Projects="$(ArcadeSdkSignProject)"
      Targets="Sign"
      Properties="
        DownloadDirectory=$(DownloadDirectory);
        PrepareArtifacts=$(PrepareArtifacts)" />
  </Target>

  <!--
    Take assets from the build jobs, prepare them for publishing (signing, arrangement) then upload
    them to the pipeline for Arcade and custom steps to publish.
  -->
  <Target Name="PreparePublishToAzureBlobFeed"
          AfterTargets="Build"
          DependsOnTargets="GetProductVersions;FindDownloadedArtifacts;SignArtifacts;CreateChecksums">
    <PropertyGroup>
      <AssetManifestFilename>Manifest.xml</AssetManifestFilename>
      <AssetManifestFile>$(ArtifactsLogDir)AssetManifest/$(AssetManifestFilename)</AssetManifestFile>

      <!-- Create temp dir to store generated asset manifest, per Arcade guidance. -->
      <TempWorkingDir>$(ArtifactsObjDir)TempWorkingDir\$([System.Guid]::NewGuid())\</TempWorkingDir>

      <ProductVersionTxtContents Condition="'$(StabilizePackageVersion)'=='true'">$(ProductionVersion)</ProductVersionTxtContents>
      <ProductVersionTxtContents Condition="'$(StabilizePackageVersion)'!='true'">$(ProductVersion)</ProductVersionTxtContents>
    </PropertyGroup>

    <!-- Generate productVersion.txt containing the value of $(PackageVersion) -->
    <WriteLinesToFile
      File="$(ArtifactsShippingPackagesDir)productVersion.txt"
      Lines="$(ProductVersionTxtContents)"
      Overwrite="true"
      Encoding="ASCII" />

    <!-- Generate runtime-productVersion.txt containing the value of $(PackageVersion) -->
    <WriteLinesToFile
      File="$(ArtifactsShippingPackagesDir)runtime-productVersion.txt"
      Lines="$(ProductVersionTxtContents)"
      Overwrite="true"
      Encoding="ASCII" />

    <ItemGroup>
      <ItemsToPush Remove="@(ItemsToPush)" />

      <ItemsToPush Include="@(ShippingNupkgToPublishFile)" />
      <ItemsToPush Include="@(NonShippingNupkgToPublishFile)" ManifestArtifactData="NonShipping=true" />
      <ItemsToPush Include="@(SymbolNupkgToPublishFile)" />

      <ItemsToPush
        Include="@(UploadToBlobStorageFile)"
        Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)">
        <RelativeBlobPath>$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
        <PublishFlatContainer>true</PublishFlatContainer>
      </ItemsToPush>

      <ItemsToPush Include="@(GeneratedChecksumFile)">
        <RelativeBlobPath>$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
        <Category>Checksum</Category>
        <PublishFlatContainer>true</PublishFlatContainer>
      </ItemsToPush>

      <ItemsToPush Include="$(ArtifactsShippingPackagesDir)productVersion.txt">
        <RelativeBlobPath>$(InstallersRelativePath)productVersion.txt</RelativeBlobPath>
        <PublishFlatContainer>true</PublishFlatContainer>
      </ItemsToPush>

      <ItemsToPush Include="$(ArtifactsShippingPackagesDir)runtime-productVersion.txt">
        <RelativeBlobPath>$(InstallersRelativePath)runtime-productVersion.txt</RelativeBlobPath>
        <PublishFlatContainer>true</PublishFlatContainer>
      </ItemsToPush>

      <ItemsToPush Include="@(WorkloadsVSInsertionFile)">
        <RelativeBlobPath>$(InstallersRelativePath)workloads/%(Filename)%(Extension)</RelativeBlobPath>
        <PublishFlatContainer>true</PublishFlatContainer>
      </ItemsToPush>

      <!-- Source build intermediated packages will be pushed and signed by the sourcebuild leg. -->
      <ItemsToPush Remove="@(ItemsToPush)" Condition="$([System.String]::new('%(Identity)').Contains('Microsoft.SourceBuild.Intermediate'))" />
      <ItemsToSignPostBuild Remove="@(ItemsToSignPostBuild)" Condition="$([System.String]::new('%(Identity)').Contains('Microsoft.SourceBuild.Intermediate'))" />
    </ItemGroup>

    <!-- Push items to AzDO as build artifacts, generating the asset manifest as a side effect. -->
    <PushToAzureDevOpsArtifacts
      AzureDevOpsCollectionUri="$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)"
      AzureDevOpsProject="$(SYSTEM_TEAMPROJECT)"
      AzureDevOpsBuildId="$(BUILD_BUILDID)"
      ItemsToSign="@(ItemsToSignPostBuild)"
      StrongNameSignInfo="@(StrongNameSignInfo)"
      FileSignInfo="@(FileSignInfo)"
      FileExtensionSignInfo="@(FileExtensionSignInfo)"
      ItemsToPush="@(ItemsToPush)"
      ManifestBuildData="@(ManifestBuildData)"
      ManifestRepoUri="$(BUILD_REPOSITORY_NAME)"
      ManifestBranch="$(BUILD_SOURCEBRANCH)"
      ManifestBuildId="$(BUILD_BUILDNUMBER)"
      ManifestCommit="$(BUILD_SOURCEVERSION)"
      IsStableBuild="$(IsStableBuild)"
      AssetManifestPath="$(AssetManifestFile)"
      AssetsTemporaryDirectory="$(TempWorkingDir)"
      PublishingVersion ="3" />

    <!-- Copy the generated manifest to the build's artifacts -->
    <Copy SourceFiles="$(AssetManifestFile)" DestinationFolder="$(TempWorkingDir)" />

    <Message Importance="High" Text="Uploading $(AssetManifestFilename) to pipeline" />
    <Message
      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(TempWorkingDir)$(AssetManifestFilename)"
      Importance="High" />

    <Message Importance="High" Text="Complete!" />
  </Target>

  <!--
    Look through the downloaded artifacts to figure out how to ship them. Creates item groups for
    other types of publishing to use.

    DownloadDirectory is expected to have some directory inside, which then contains a dir for each
    build job name. For example, this nupkg would be found:

      $(DownloadDirectory)IntermediateUnsignedArtifacts\Windows_x64\Shipping\a.nupkg
  -->
  <Target Name="FindDownloadedArtifacts">

    <ItemGroup>
      <DownloadedArtifactFile Include="$(DownloadDirectory)**" />

      <Error
        Condition="'@(DownloadedArtifactFile)' == ''"
        Text="No downloaded artifacts found." />

      <DownloadedSymbolNupkgFile Include="$(DownloadDirectory)**\*.symbols.nupkg" />
      <DownloadedWixPdbFile Include="$(DownloadDirectory)**\*.wixpdb" />
      <DownloadedWorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*" />
      <DownloadedNupkgFile
        Include="$(DownloadDirectory)**\*.nupkg"
        Exclude="@(DownloadedSymbolNupkgFile)" />

      <!-- Add files that are not affected by filtering. -->
      <UploadToBlobStorageFile
        Include="@(DownloadedArtifactFile)"
        Exclude="@(DownloadedSymbolNupkgFile);@(DownloadedNupkgFile);@(DownloadedWixPdbFile);@(DownloadedWorkloadsVSInsertionFile)" />

      <!--
        Filter out the RID-specific (Runtime) nupkgs and RID-agnostic nupkgs. RID-specific packages
        are published from every job. RID-agnostic nupkgs are built with the same ID/version by
        every job, so one specific job's outputs must be picked to sign and publish.
      -->

      <!-- RID-specific framework packs. -->
      <RuntimeNupkgFile
        Include="
          $(DownloadDirectory)**\Microsoft.*.Runtime.*.nupkg;
          $(DownloadDirectory)**\Microsoft.*.App.Host.*.nupkg;
          $(DownloadDirectory)**\Microsoft.*.App.Crossgen2.*.nupkg"
        Exclude="@(DownloadedSymbolNupkgFile)" />

      <!-- VS insertion packages, carrying RID-specific installers. -->
      <RuntimeNupkgFile
        Include="$(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
        Exclude="@(DownloadedSymbolNupkgFile)" />

      <!--
        Workloads VS insertion artifacts produced by src/workloads/workloads.csproj. Only grab
        the zip artifacts as they're grouped by SDK feature band which correlates with specific VS versions.
      -->
      <WorkloadsVSInsertionFile
        Include="
          $(DownloadDirectory)*\workloads-vs\**\*.zip"/>

      <!--
        Runtime packages associated with some identity packages. Need to exclude "runtime.native.*"
        because Libraries produces some "runtime.native.Foo" packages with
        "runtime.<rid>.runtime.native.Foo" identity packages.
      -->
      <RuntimeNupkgFile
        Include="$(DownloadDirectory)**\runtime.*.nupkg"
        Exclude="
          $(DownloadDirectory)**\runtime.native.*.nupkg;
          @(DownloadedSymbolNupkgFile)" />

      <!--
        Packages that aren't matched above as RID-specific are considered RID-agnostic. Also include
        the AllConfigurations packages from the Libraries build.
      -->
      <RidAgnosticNupkgToPublishFile
        Include="
          $(DownloadDirectory)**\Microsoft.NET.Workload.Mono.Toolchain.*Manifest-*.nupkg;
          $(DownloadDirectory)*\$(PublishRidAgnosticPackagesFromPlatform)\**\*.nupkg;
          $(DownloadDirectory)*\*AllConfigurations\**\*.nupkg"
        Exclude="@(RuntimeNupkgFile);@(DownloadedSymbolNupkgFile)" />

      <TransportPackagesToPublishFile
        Include="$(DownloadDirectory)**\*Transport*.nupkg"
        Exclude="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(DownloadedSymbolNupkgFile)" />

      <NupkgToPublishFile Include="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(TransportPackagesToPublishFile)" />

      <!--
        Assuming all symbol packages ship and can be found by turning .nupkg => .symbols.nupkg, find
        them. Don't check for missing symbol packages here: some nupkgs don't have them for valid
        reasons, such as the VS insertion packages that transport MSIs. Symbol package validation
        will check for symbol completeness with file-by-file granularity rather than looking for
        missing symbols.nupkg files: https://github.com/dotnet/arcade/issues/2499.

        Handles several conventions:
        * NonShipping packages have symbol nupkgs that are Shipping.
        * Shipping packages have symbol packages in a "symbols" subdirectory.
      -->
      <PotentialSymbolNupkgToPublishFile
        Include="
          @(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'));
          @(NupkgToPublishFile->Replace('\NonShipping\', '\NonShipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'));
          @(NupkgToPublishFile->Replace('\Shipping\', '\Shipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'))" />

      <SymbolNupkgToPublishFile
        Include="@(PotentialSymbolNupkgToPublishFile -> Distinct())"
        Condition="Exists('%(Identity)')" />

      <!-- Split nupkgs into shipping/nonshipping for BAR categorization. -->
      <ShippingNupkgToPublishFile
        Include="@(NupkgToPublishFile)"
        Condition="$([System.String]::new('%(Identity)').Contains('\Shipping\'))" />

      <NonShippingNupkgToPublishFile
        Include="@(NupkgToPublishFile)"
        Exclude="@(ShippingNupkgToPublishFile)" />

    </ItemGroup>

    <Error
      Condition="'@(SymbolNupkgToPublishFile)' == ''"
      Text="No symbol packages found." />

    <!--
      Duplicate RuntimeNupkgFile items mean artifact upload will fail, but only after another hour
      of signing. Detect this early. It's possible to automatically "fix" this with Distinct(),
      however the patterns should be fairly specific: this is likely a build infra mistake that
      should be corrected.
    -->
    <Error
      Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)"
      Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
  </Target>

  <Target Name="CreateChecksums">
    <ItemGroup>
      <ArtifactsForGeneratingChecksums Include="@(UploadToBlobStorageFile)"
                                       DestinationPath="%(FullPath)$(ChecksumExtension)" />
      <GeneratedChecksumFile Include="@(ArtifactsForGeneratingChecksums -> '%(DestinationPath)')" />
    </ItemGroup>

    <GenerateChecksums Items="@(ArtifactsForGeneratingChecksums)" />
  </Target>

</Project>