Merge branch 'main' into lock-on-session-start

eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml

@@ -22,3 +22,4 @@ steps:
     displayName: Verify REST API spec location for "${{ parameters.PackageName }}"
     env:
       GH_TOKEN: $(azuresdk-github-pat)
+    condition: and(succeededOrFailed(), ne(variables['Skip.Verify-RestApiSpecLocation'], 'true'))

eng/pipelines/mirror-container-images.yml

@@ -10,7 +10,7 @@ parameters:
         changes: bash -c "apt update -y && apt upgrade -y"
       - source: ghcr.io/chaos-mesh/chaos-mesh:v2.6.3
         mirror: azsdkengsys.azurecr.io/mirror/chaos-mesh/chaos-mesh:v2.6.3
-        changes: bash -c "apk update && apk upgrade"
+        changes: sh -c "apk update && apk upgrade"
       - source: ghcr.io/chaos-mesh/chaos-dashboard:v2.6.3
         mirror: azsdkengsys.azurecr.io/mirror/chaos-mesh/chaos-dashboard:v2.6.3
         changes: bash -c "apt update -y && apt upgrade -y"
@@ -36,11 +36,14 @@ extends:
 
             steps:
               - ${{ each image in parameters.Images }}:
-                - task: Docker@2
+                - task: AzureCLI@2
                   displayName: Login to ${{ split(image.mirror, '.')[0] }}
                   inputs:
-                    command: login
-                    containerRegistry: ${{ split(image.mirror, '.')[0] }}
+                    azureSubscription: "Azure SDK Engineering System"
+                    scriptType: pscore
+                    scriptLocation: inlineScript
+                    inlineScript: |
+                      az acr login --name ${{ split(image.mirror, '.')[0] }}
                 - task: Powershell@2
                   displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
                   inputs:

eng/pipelines/pipeline-owners-extraction.yml

@@ -21,12 +21,24 @@ stages:
       Project: internal
       DotNetDevOpsFeed: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json"
       OutputPath: '$(Agent.BuildDirectory)/pipelineOwners.json'
-      RepoLabelUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/repository-labels-blob?$(azuresdkartifacts-azure-sdk-write-teams-sas)"
-      TeamUserUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob?$(azuresdkartifacts-azure-sdk-write-teams-sas)"
-      UserOrgUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/user-org-visibility-blob?$(azuresdkartifacts-azure-sdk-write-teams-sas)"
+      RepoLabelUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/repository-labels-blob"
+      TeamUserUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob"
+      UserOrgUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/user-org-visibility-blob"
       RepoListFile: "$(Build.SourcesDirectory)/tools/github/data/repositories.txt"
 
     steps:
+      - task: AzureCLI@2
+        displayName: 'Fetch and store team/user data'
+        inputs:
+          azureSubscription: 'Azure SDK Artifacts'
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: |
+            dotnet run -rUri "$(RepoLabelUri)" -tUri "$(TeamUserUri)" -uUri "$(UserOrgUri)" -rlFile "$(RepoListFile)"
+          workingDirectory: tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore
+        env:
+          GITHUB_TOKEN: $(azuresdkartifacts-azure-sdk-write-teams-github-pat)
+
       - task: DotNetCoreCLI@2
         displayName: 'Install Pipeline Owners Extractor'
         inputs:
@@ -46,9 +58,3 @@ stages:
         artifact: pipelineOwners
         condition: succeededOrFailed()
 
-      - pwsh: |
-          dotnet run -rUri "$(RepoLabelUri)" -tUri "$(TeamUserUri)" -uUri "$(UserOrgUri)" -rlFile "$(RepoListFile)"
-        displayName: 'Fetch and store team/user data'
-        workingDirectory: tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore
-        env:
-          GITHUB_TOKEN: $(azuresdkartifacts-azure-sdk-write-teams-github-pat)

eng/pipelines/templates/steps/sdk-testgen-set-env.yml

@@ -1,15 +1,15 @@
-# Template for setup rush env
-steps:
-  - task: NodeTool@0
-    inputs:
-      versionSpec: "16.15"
-    displayName: "Install Node.js"
-  - task: GoTool@0
-    inputs:
-      version: '1.22.1'
-    displayName: "Install Go"
-  - script: |
-      npm install -g "@microsoft/rush"
-      rush install
-    displayName: "Install dependencies"
+# Template for setup rush env
+steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: "18.x"
+    displayName: "Install Node.js"
+  - task: GoTool@0
+    inputs:
+      version: '1.22.1'
+    displayName: "Install Go"
+  - script: |
+      npm install -g "@microsoft/rush"
+      rush install
+    displayName: "Install dependencies"
     workingDirectory: $(Build.SourcesDirectory)/tools/sdk-testgen

tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Program.cs

@@ -21,22 +21,21 @@ static void Main(string[] args)
             stopWatch.Start();
 
             // The storage URIs are in the azure-sdk-write-teams-container-blobs pipeline variable group.
-            // The URIs do not contain the SAS.
             var teamUserBlobStorageUriOption = new Option<string>
                 (name: "--teamUserBlobStorageURI",
-                description: "The team/user blob storage URI without the SAS.");
+                description: "The team/user blob storage URI.");
             teamUserBlobStorageUriOption.AddAlias("-tUri");
             teamUserBlobStorageUriOption.IsRequired = true;
 
             var userOrgVisibilityBlobStorageUriOption = new Option<string>
                 (name: "--userOrgVisibilityBlobStorageURI",
-                description: "The user/org blob storage URI without the SAS.");
+                description: "The user/org blob storage URI.");
             userOrgVisibilityBlobStorageUriOption.AddAlias("-uUri");
             userOrgVisibilityBlobStorageUriOption.IsRequired = true;
 
             var repoLabelBlobStorageUriOption = new Option<string>
                 (name: "--repoLabelBlobStorageURI",
-                description: "The repo/label blob storage URI without the SAS.");
+                description: "The repo/label blob storage URI.");
             repoLabelBlobStorageUriOption.AddAlias("-rUri");
             repoLabelBlobStorageUriOption.IsRequired = true;
 
@@ -68,6 +67,13 @@ static void Main(string[] args)
                 description: "Generate the baseline error file.");
             generateBaselineOption.AddAlias("-gbl");
 
+            var baseBranchBaselineFileOption = new Option<string>
+                (name: "--baseBranchBaselineFile",
+                description: "The full path to base branch baseline file to be generated or used. The file will be generated if -gbl is set and used to further filter errors if -fbl is set.");
+            baseBranchBaselineFileOption.AddAlias("-bbf");
+            baseBranchBaselineFileOption.IsRequired = false;
+            baseBranchBaselineFileOption.SetDefaultValue(null);
+
             var rootCommand = new RootCommand
             {
                 teamUserBlobStorageUriOption,
@@ -76,7 +82,8 @@ static void Main(string[] args)
                 repoRootOption,
                 repoNameOption,
                 filterBaselineErrorsOption,
-                generateBaselineOption
+                generateBaselineOption,
+                baseBranchBaselineFileOption
             };
 
             int returnCode = 1;
@@ -98,13 +105,15 @@ static void Main(string[] args)
                     string repoName = context.ParseResult.GetValueForOption(repoNameOption);
                     bool filterBaselineErrors = context.ParseResult.GetValueForOption(filterBaselineErrorsOption);
                     bool generateBaseline = context.ParseResult.GetValueForOption(generateBaselineOption);
+                    string baseBranchBaselineFile = context.ParseResult.GetValueForOption(baseBranchBaselineFileOption);
                     returnCode = LintCodeownersFile(teamUserBlobStorageUri,
                                                     userOrgVisibilityBlobStorageUri,
                                                     repoLabelBlobStorageUri,
                                                     repoRoot,
                                                     repoName,
                                                     filterBaselineErrors,
-                                                    generateBaseline);
+                                                    generateBaseline,
+                                                    baseBranchBaselineFile);
                 });
 
             rootCommand.Invoke(args);
@@ -124,6 +133,18 @@ static void Main(string[] args)
         /// Verify the arguments and call to process the CODEOWNERS file. If errors are being filtered with a 
         /// baseline, or used to regenerate the baseline, that's done in here. Note that filtering errors and
         /// regenerating the baseline cannot both be done in the same run.
+        /// 
+        /// The baseBranchBaselineFile
+        /// This file will be primarily used in PR validation where two calls will be made. be made. The first
+        /// call will use the -gbl option and generate the secondary file to a different location. It can't use
+        /// the standard CODEOWNERS_baseline_error.txt file because it'll be being used in combination with this
+        /// file. The second call, to verify CODEOWNERS changes in the PR, will verify against the default
+        /// CODEOWNERS_baseline_error.txt and, if there are any remaining errors, check to see if those exist in the
+        /// secondary baseline file. The reason for doing this is prevent PRs from being blocked if there are issues
+        /// in the baseline branch. The typical scenario here will be people leaving the company, the base branch's
+        /// CODEOWNERS hasn't yet been updated to reflect this and because of that any PRs with CODEOWNERS changes
+        /// would get blocked. If the remaining errors in the PR validation exist in the base branch's errors then
+        /// the linter will return a pass instead of a failure.
         /// </summary>
         /// <param name="teamUserBlobStorageUri">URI of the team/user data in blob storate</param>
         /// <param name="userOrgVisibilityBlobStorageUri">URI of the org visibility in blob storage</param>
@@ -132,6 +153,7 @@ static void Main(string[] args)
         /// <param name="repoName">The repository name, including org. Eg. Azure/azure-sdk</param>
         /// <param name="filterBaselineErrors">Boolean, if true then errors should be filtered using the repository's baseline.</param>
         /// <param name="generateBaseline">Boolean, if true then regenerate the baseline file from the error encountered during parsing.</param>
+        /// <param name="baseBranchBaselineFile">The name of the base branch baseline file to generate or use.</param>
         /// <returns>integer, used to set the return code</returns>
         /// <exception cref="ArgumentException">Thrown if any arguments, or argument combinations, are invalid.</exception>
         static int LintCodeownersFile(string teamUserBlobStorageUri, 
@@ -140,7 +162,8 @@ static void Main(string[] args)
                                       string repoRoot, 
                                       string repoName,
                                       bool   filterBaselineErrors,
-                                      bool   generateBaseline)
+                                      bool   generateBaseline,
+                                      string baseBranchBaselineFile)
         {
             // Don't allow someone to create and use a baseline in the same run
             if (filterBaselineErrors && generateBaseline)
@@ -165,7 +188,20 @@ static void Main(string[] args)
             {
                 throw new ArgumentException($"The repository label data for {repoName} does not exist. Should this be running in this repository?");
             }
-
+
+            bool useBaseBranchBaselineFile = false;
+            if (!string.IsNullOrEmpty(baseBranchBaselineFile))
+            {
+                if ((filterBaselineErrors && File.Exists(baseBranchBaselineFile)) || generateBaseline)
+                {
+                    useBaseBranchBaselineFile = true;
+                }
+                else
+                {
+                    throw new ArgumentException($"The base branch baseline file {baseBranchBaselineFile} does not exist.");
+                }
+            }
+
             string codeownersBaselineFile = Path.Combine(repoRoot, ".github", BaselineConstants.BaselineErrorFile);
             bool codeownersBaselineFileExists = false;
             // If the baseline is to be used, verify that it exists.
@@ -192,7 +228,15 @@ static void Main(string[] args)
             // Regenerate the baseline file if that option was selected
             if (generateBaseline)
             {
-                BaselineUtils baselineUtils = new BaselineUtils(codeownersBaselineFile);
+                BaselineUtils baselineUtils = null;
+                if (useBaseBranchBaselineFile)
+                {
+                    baselineUtils = new BaselineUtils(baseBranchBaselineFile);
+                }
+                else
+                {
+                    baselineUtils = new BaselineUtils(codeownersBaselineFile);
+                }
                 baselineUtils.GenerateBaseline(errors);
             }
 
@@ -215,11 +259,20 @@ static void Main(string[] args)
                         errors = baselineUtils.FilterErrorsUsingBaseline(errors);
                     }
                 }
+
+                // After the file has been filered with the standard CODEOWNERS baseline file, if there are
+                // still remaining errors and there is a base branch baseline file, further filter with that
+                // file.
+                if (useBaseBranchBaselineFile && errors.Count > 0)
+                {
+                    BaselineUtils baselineUtils = new BaselineUtils(baseBranchBaselineFile);
+                    errors = baselineUtils.FilterErrorsUsingBaseline(errors);
+                }
             }
 
             int returnCode = 0;
-            // If there are errors, ensure the returnCode is non-zero and output the errors.
-            if (errors.Count > 0)
+            // If there are errors, and this isn't a baseline generation, ensure the returnCode is non-zero and output the errors.
+            if ((errors.Count > 0) && !generateBaseline)
             {
                 returnCode = 1;
 

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/GitHubEventClient.cs

@@ -4,6 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using Azure.Storage.Blobs;
+using Azure.Identity;
 using Octokit;
 using GitHubTeamUserStore.Constants;
 
@@ -158,14 +159,40 @@ public async Task<IReadOnlyList<Team>> GetAllChildTeams(Team team)
         /// <summary>
         /// Upload the data to blob storage. Uses the BlobUriBuilder to get the blob information to created the
         /// Blob clients and upload the data.
+        /// Credentials:
+        /// Instead of using DefaultAzureCredential [1] we use ChainedTokenCredential [2] which works
+        /// as DefaultAzureCredential, but most importantly, it excludes ManagedIdentityCredential.
+        /// We do so because there is an undesired managed identity available when we run this
+        /// code in CI/CD pipelines, which takes priority over the desired AzureCliCredential coming
+        /// from the calling AzureCLI@2 task.
+        /// 
+        /// Running Locally:
+        /// Your user needs to have Storage Blob Data Contributor access. This is done through
+        /// https://ms.portal.azure.com/, selecting the azuresdkartifacts storage account, selecting Access Control (IAM)
+        /// and adding Storage Blob Data Contributor then following the buttons at the bottom to assign this to your user.
+        /// In Visual Studio select Tools-Options and then search for Azure and select Azure Service Authentication and
+        /// authenticate. Once that's done the DefaultAzureCredential will use those creds.
+        /// 
+        /// Running in a pipeline:
+        /// Requires using the AzureCLI or AzurePowerShell task and azure subscription, which was already setup, 
+        /// is 'Azure SDK Artifacts' in both cases the exact line is as follows
+        /// azureSubscription: 'Azure SDK Artifacts'
+        /// The DefaultAzureCredential will use the creds setup in the task
         /// </summary>
         /// <param name="rawJson">The json string, representing the information that will be uploaded to blob storage.</param>
         /// <param name="blobUriBuilder">BlobUriBuilder which contains the blob storage information.</param>
         /// <returns></returns>
-        /// <exception cref="ApplicationException">If there is no AZURE_SDK_TEAM_USER_STORE_SAS in the environment</exception>
         public async Task UploadDataToBlobStorage(string rawJson, BlobUriBuilder blobUriBuilder)
         {
-            BlobServiceClient blobServiceClient = new BlobServiceClient(blobUriBuilder.ToUri());
+            var cred = new ChainedTokenCredential(
+                                new EnvironmentCredential(),
+                                new VisualStudioCredential(),
+                                new AzureCliCredential(),
+                                new AzurePowerShellCredential(),
+                                new InteractiveBrowserCredential()
+                            );
+            BlobServiceClient blobServiceClient = new BlobServiceClient(blobUriBuilder.ToUri(), cred);
+
             BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient(blobUriBuilder.BlobContainerName);
             BlobClient blobClient = blobContainerClient.GetBlobClient(blobUriBuilder.BlobName);
             await blobClient.UploadAsync(BinaryData.FromString(rawJson), overwrite: true);

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/GitHubTeamUserStore.csproj

@@ -8,7 +8,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.16.0" />
+    <PackageReference Include="Azure.Identity" Version="1.11.3" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageReference Include="Octokit" Version="5.0.2" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>

tools/js-sdk-release-tools/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@azure-tools/js-sdk-release-tools",
-  "version": "2.7.8",
+  "version": "2.7.9",
   "description": "",
   "scripts": {
     "start": "node dist/changelogToolCli.js",