feat(ado-ext-telemetry): Embed upload artifact into task (#1085)

* embed artifact upload in ado extension task

* update documentation

* fix report output dir in logging command

* adding ability to run task multiple times without artifact failure

* fix task.json

* update artifact path

* add outputDir information to the running multiple times usage documentation

* adding uploadResultAsArtifact parameter to allow skipping auto-upload of artifact

* remove unnecessary stdOut transformations

* update markdown and console output to handle uploadResultArtifact false case

* test update

* update helpMarkdown and visibleRule for artifactName

docs/ado-extension-usage.md

@@ -48,11 +48,6 @@ steps:
           # Provide either siteDir or url
           # siteDir: '$(System.DefaultWorkingDirectory)/path-to-built-website/'
           # url: 'your-website-url'
-
-    - publish: '$(System.DefaultWorkingDirectory)/_accessibility-reports'
-      displayName: Upload report artifact
-      condition: succeededOrFailed()
-      artifact: 'accessibility-reports'
 ```
 
 ### Scan a URL
@@ -147,30 +142,31 @@ Here is an example of a YAML file that is configured to take advantage of a base
   inputs:
       url: 'http://localhost:12345/'
       baselineFile: '$(Build.SourcesDirectory)/baselines/my-web-site.baseline'
-
-- publish: '$(System.DefaultWorkingDirectory)/_accessibility-reports'
-  displayName: Upload report artifacts
-  condition: succeededOrFailed()
-  artifact: 'accessibility-reports'
 ```
 
-## Viewing results
+## Report Artifacts
+
+By default, an HTML report containing detailed results is automatically uploaded as a pipeline artifact named `accessibility-reports`. You can opt out of this automatic artifact upload by setting the `uploadResultAsArtifact` parameter to `false`. You can also specify a custom artifact name by setting the `artifactName` parameter in your YAML file. If not opted out, a link to the artifacts will also appear in both the task log and in the Extensions tab of the pipeline run.
 
--   An HTML report containing detailed results is saved to disk. To view it, you need to have added the step to upload artifacts to your yml file (see [Basic template](#basic-template)). Navigate to Artifacts from the build. Under `accessibility-reports`, you'll find the downloadable report labeled `index.html`.
+To view the report, navigate to Artifacts from the build. Under `accessibility-reports`, or the artifact name manually specified, you'll find the downloadable report labeled `index.html`.
 
--   If the workflow was triggered by a pull request, the action should leave a comment on the Azure DevOps pull request with results. The extension does not leave comments on repos in GitHub.
+## Summary results
+
+-   Summary results are output in both the task log and in the Extensions tab of the pipeline run. To view the task log, click into the job and then click on the `accessibilityinsights` task.
 
 ## Blocking pull requests
 
 You can choose to block pull requests if the extension finds accessibility issues.
 
 1. Ensure the extension is [triggered on each pull request](https://docs.microsoft.com/en-us/azure/devops/pipelines/customize-pipeline?view=azure-devops#customize-ci-triggers).
 2. Ensure that you have set the `failOnAccessibilityError` input variable to `true`.
-3. Ensure that the `Upload report artifact` step runs even in failure cases using [**succeededOrFailed()**](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/conditions?view=azure-devops&tabs=yaml)
+
+## Running multiple times in a single pipeline
+
+If you want to run the extension multiple times in a single pipeline, you will need to ensure that unique `artifactName` and `outputDir` inputs are specified in your YAML file for each task. Artifact names and the output directory must be unique across all tasks in a pipeline.
 
 ## Troubleshooting
 
 -   If the action didn't trigger as you expected, check the `trigger` or `pr` sections of your yml file. Make sure any listed branch names are correct for your repository.
 -   If the action fails to complete, you can check the build logs for execution errors. Using the template above, these logs will be in the `Scan for accessibility issues` step.
--   If you can't find an artifact, note that your workflow must include a `publish` step to add the report folder to your check results. See the [Basic template](#basic-template) above and [Azure DevOps documentation on publishing artifacts](https://docs.microsoft.com/en-us/azure/devops/pipelines/artifacts/pipeline-artifacts?view=azure-devops&tabs=yaml#publish-artifacts).
 -   If the scan takes longer than 90 seconds, you can override the default timeout by providing a value for `scanTimeout` in milliseconds.

packages/ado-extension/src/output-hooks/ado-stdout-transformer.spec.ts

@@ -24,16 +24,6 @@ describe(adoStdoutTransformer, () => {
         expect(output).toBe(expectedOutput);
     });
 
-    // Note: these are special logging commands in ADO that can't be added to the output text of the test because ADO attempts to evaluate them and fails or throws warnings.
-    it.each`
-        input                          | expectedOutput                 | safeTextIdentifier
-        ${'##vso[task.uploadsummary]'} | ${'##vso[task.uploadsummary]'} | ${'task.uploadsummary'}
-        ${'##vso[task.logissue]abc'}   | ${'##vso[task.logissue]abc'}   | ${'task.logissue'}
-    `(`ADO Special logging command '$safeTextIdentifier' returns as expected`, ({ input, expectedOutput }) => {
-        const output = adoStdoutTransformer(input);
-        expect(output).toBe(expectedOutput);
-    });
-
     it.each`
         input
         ${'##vso[task.debug]abc'}

packages/ado-extension/src/output-hooks/ado-stdout-transformer.ts

@@ -13,14 +13,6 @@ const regexTransformations: RegexTransformation[] = [
         regex: new RegExp('^##vso\\[task.debug\\]'),
         method: useUnmodifiedString,
     },
-    {
-        regex: new RegExp('^##vso\\[task.uploadsummary\\]'),
-        method: useUnmodifiedString,
-    },
-    {
-        regex: new RegExp('^##vso\\[task.logissue\\]'),
-        method: useUnmodifiedString,
-    },
     {
         regex: new RegExp('^Processing page .*'),
         method: useUnmodifiedString,

packages/ado-extension/src/progress-reporter/console/ado-console-comment-creator.spec.ts

@@ -19,6 +19,18 @@ describe(AdoConsoleCommentCreator, () => {
     let fsMock: IMock<typeof fs>;
     const reportOutDir = 'reportOutDir';
     const fileName = `${reportOutDir}/results.md`;
+    const artifactName = 'accessibility-reports';
+    const reportStub: CombinedReportParameters = {
+        results: {
+            urlResults: {
+                failedUrls: 1,
+            },
+        },
+    } as CombinedReportParameters;
+    const baselineInfoStub = {};
+    const reportMarkdownStub = '#ReportMarkdownStub';
+
+    const expectedLogOutput = reportMarkdownStub;
 
     beforeEach(() => {
         adoTaskConfigMock = Mock.ofType<ADOTaskConfig>(undefined, MockBehavior.Strict);
@@ -30,60 +42,73 @@ describe(AdoConsoleCommentCreator, () => {
 
     describe('constructor', () => {
         it('should initialize', () => {
-            buildAdoConsoleCommentCreatorWithMocks();
+            buildAdoConsoleCommentCreatorWithMocks(false);
 
             verifyAllMocks();
         });
     });
 
     describe('completeRun', () => {
         it('should output results to the console', async () => {
-            const reportStub: CombinedReportParameters = {
-                results: {
-                    urlResults: {
-                        failedUrls: 1,
-                    },
-                },
-            } as CombinedReportParameters;
-            const baselineInfoStub = {};
-            const reportMarkdownStub = '#ReportMarkdownStub';
-
-            const expectedLogOutput = reportMarkdownStub;
-
             adoTaskConfigMock
-                .setup((atcm) => atcm.getBaselineFile())
-                .returns(() => undefined)
+                .setup((atcm) => atcm.getVariable('System.JobAttempt'))
+                .returns(() => '1')
                 .verifiable(Times.once());
 
             adoTaskConfigMock
-                .setup((atcm) => atcm.getReportOutDir())
-                .returns(() => reportOutDir)
+                .setup((atcm) => atcm.getUploadResultAsArtifact())
+                .returns(() => true)
                 .verifiable(Times.once());
 
-            reportMarkdownConvertorMock
-                .setup((o) => o.convert(reportStub, undefined, baselineInfoStub))
-                .returns(() => expectedLogOutput)
+            loggerMock.setup((lm) => lm.logInfo(`##vso[task.uploadsummary]${fileName}`)).verifiable(Times.once());
+            loggerMock
+                .setup((lm) => lm.logInfo(`##vso[artifact.upload artifactname=${artifactName}]${reportOutDir}`))
                 .verifiable(Times.once());
 
-            reportConsoleLogConvertorMock
-                .setup((o) => o.convert(reportStub, undefined, baselineInfoStub))
-                .returns(() => expectedLogOutput)
+            adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks();
+            await adoConsoleCommentCreator.completeRun(reportStub);
+
+            verifyAllMocks();
+        });
+
+        it('Successfully adds suffix to output name if job attmept > 1', async () => {
+            adoTaskConfigMock
+                .setup((atcm) => atcm.getVariable('System.JobAttempt'))
+                .returns(() => '2')
+                .verifiable(Times.once());
+
+            adoTaskConfigMock
+                .setup((atcm) => atcm.getUploadResultAsArtifact())
+                .returns(() => true)
                 .verifiable(Times.once());
 
-            loggerMock.setup((lm) => lm.logInfo(expectedLogOutput)).verifiable(Times.once());
             loggerMock.setup((lm) => lm.logInfo(`##vso[task.uploadsummary]${fileName}`)).verifiable(Times.once());
-            fsMock.setup((fsm) => fsm.writeFileSync(fileName, expectedLogOutput)).verifiable();
+            loggerMock
+                .setup((lm) => lm.logInfo(`##vso[artifact.upload artifactname=${artifactName}-2]${reportOutDir}`))
+                .verifiable(Times.once());
 
             adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks();
             await adoConsoleCommentCreator.completeRun(reportStub);
 
             verifyAllMocks();
         });
+
+        it('skips upload artifact step if uploadResultAsArtifact is false', async () => {
+            adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks();
+
+            adoTaskConfigMock
+                .setup((atcm) => atcm.getUploadResultAsArtifact())
+                .returns(() => false)
+                .verifiable(Times.once());
+
+            loggerMock.setup((lm) => lm.logInfo(`##vso[task.uploadsummary]${fileName}`)).verifiable(Times.never());
+            await adoConsoleCommentCreator.completeRun(reportStub);
+        });
     });
 
     describe('failRun', () => {
         it('does nothing interesting', async () => {
-            const adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks();
+            const adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks(false);
 
             await adoConsoleCommentCreator.failRun();
 
@@ -99,23 +124,55 @@ describe(AdoConsoleCommentCreator, () => {
         });
 
         it('returns true after failRun() is called', async () => {
-            const adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks();
+            const adoConsoleCommentCreator = buildAdoConsoleCommentCreatorWithMocks(false);
 
             await adoConsoleCommentCreator.failRun();
 
             await expect(adoConsoleCommentCreator.didScanSucceed()).resolves.toBe(true);
         });
     });
 
-    const buildAdoConsoleCommentCreatorWithMocks = (): AdoConsoleCommentCreator =>
-        new AdoConsoleCommentCreator(
+    const buildAdoConsoleCommentCreatorWithMocks = (setupSharedMocks = true): AdoConsoleCommentCreator => {
+        if (setupSharedMocks) {
+            adoTaskConfigMock
+                .setup((atcm) => atcm.getBaselineFile())
+                .returns(() => undefined)
+                .verifiable(Times.once());
+
+            adoTaskConfigMock
+                .setup((atcm) => atcm.getReportOutDir())
+                .returns(() => reportOutDir)
+                .verifiable(Times.exactly(2));
+
+            adoTaskConfigMock
+                .setup((atcm) => atcm.getArtifactName())
+                .returns(() => artifactName)
+                .verifiable(Times.once());
+
+            reportMarkdownConvertorMock
+                .setup((o) => o.convert(reportStub, undefined, baselineInfoStub))
+                .returns(() => expectedLogOutput)
+                .verifiable(Times.once());
+
+            reportConsoleLogConvertorMock
+                .setup((o) => o.convert(reportStub, undefined, baselineInfoStub))
+                .returns(() => expectedLogOutput)
+                .verifiable(Times.once());
+
+            loggerMock.setup((lm) => lm.logInfo(expectedLogOutput)).verifiable(Times.once());
+            loggerMock.setup((lm) => lm.logInfo(`##vso[task.uploadsummary]${fileName}`)).verifiable(Times.once());
+            fsMock.setup((fsm) => fsm.writeFileSync(fileName, expectedLogOutput)).verifiable();
+        }
+
+        return new AdoConsoleCommentCreator(
             adoTaskConfigMock.object,
             reportMarkdownConvertorMock.object,
             reportConsoleLogConvertorMock.object,
             loggerMock.object,
             adoTaskConfigMock.object,
             fsMock.object,
         );
+    };
 
     const verifyAllMocks = () => {
         adoTaskConfigMock.verifyAll();

packages/ado-extension/src/progress-reporter/console/ado-console-comment-creator.ts

@@ -31,6 +31,7 @@ export class AdoConsoleCommentCreator extends ProgressReporter {
     public async completeRun(combinedReportResult: CombinedReportParameters, baselineEvaluation?: BaselineEvaluation): Promise<void> {
         const baselineInfo = this.getBaselineInfo(baselineEvaluation);
         this.outputResultsMarkdownToBuildSummary(combinedReportResult, baselineInfo);
+        this.uploadReportArtifacts();
         this.logResultsToConsole(combinedReportResult, baselineInfo);
 
         return Promise.resolve();
@@ -62,6 +63,25 @@ export class AdoConsoleCommentCreator extends ProgressReporter {
         this.logger.logInfo(`##vso[task.uploadsummary]${fileName}`);
     }
 
+    private uploadReportArtifacts(): void {
+        const uploadResultAsArtifactEnabled: boolean = this.taskConfig.getUploadResultAsArtifact();
+        if (uploadResultAsArtifactEnabled) {
+            const outputDirectory = this.taskConfig.getReportOutDir();
+            const jobAttemptBuildVariable = this.taskConfig.getVariable('System.JobAttempt');
+            const artifactName = this.taskConfig.getArtifactName();
+
+            let artifactNameSuffix = '';
+            let jobAttemptNumber = 1;
+
+            if (jobAttemptBuildVariable !== undefined) {
+                jobAttemptNumber = parseInt(jobAttemptBuildVariable);
+                artifactNameSuffix = jobAttemptNumber > 1 ? `-${jobAttemptBuildVariable}` : '';
+            }
+
+            this.logger.logInfo(`##vso[artifact.upload artifactname=${artifactName + artifactNameSuffix}]${outputDirectory}`);
+        }
+    }
+
     private logResultsToConsole(combinedReportResult: CombinedReportParameters, baselineInfo?: BaselineInfo): void {
         const reportMarkdown = this.reportConsoleLogConvertor.convert(combinedReportResult, undefined, baselineInfo);
 

packages/ado-extension/src/task-config/ado-task-config.spec.ts

@@ -53,6 +53,8 @@ describe(ADOTaskConfig, () => {
         ${'baselineFile'}             | ${'./baselineFile'} | ${getPlatformAgnosticPath(__dirname + '/baselineFile')} | ${() => taskConfig.getBaselineFile()}
         ${'failOnAccessibilityError'} | ${true}             | ${true}                                                 | ${() => taskConfig.getFailOnAccessibilityError()}
         ${'singleWorker'}             | ${true}             | ${true}                                                 | ${() => taskConfig.getSingleWorker()}
+        ${'artifactName'}             | ${'artifact-name'}  | ${'artifact-name'}                                      | ${() => taskConfig.getArtifactName()}
+        ${'uploadResultAsArtifact'}   | ${true}             | ${true}                                                 | ${() => taskConfig.getUploadResultAsArtifact()}
     `(
         `input value '$inputValue' returned as '$expectedValue' for '$inputOption' parameter`,
         ({ inputOption, getInputFunc, inputValue, expectedValue }) => {
@@ -164,4 +166,17 @@ describe(ADOTaskConfig, () => {
 
         expect(actualCommitHash).toEqual(commitHash);
     });
+
+    it('should call get variable from task library', () => {
+        const variableName = 'variableName';
+        const variableValue = 'variableValue';
+        adoTaskMock
+            .setup((o) => o.getVariable(variableName))
+            .returns(() => variableValue)
+            .verifiable(Times.once());
+
+        const actualVariableValue = taskConfig.getVariable(variableName);
+
+        expect(actualVariableValue).toEqual(variableValue);
+    });
 });

packages/ado-extension/src/task-config/ado-task-config.ts

@@ -121,6 +121,19 @@ export class ADOTaskConfig extends TaskConfig {
         return this.processObj.env.SYSTEM_TEAMPROJECT ?? undefined;
     }
 
+    public getArtifactName(): string {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        return this.adoTaskObj.getInput('artifactName')!;
+    }
+
+    public getUploadResultAsArtifact(): boolean {
+        return this.adoTaskObj.getBoolInput('uploadResultAsArtifact');
+    }
+
+    public getVariable(definedVariableName: string): string | undefined {
+        return this.adoTaskObj.getVariable(definedVariableName);
+    }
+
     private getAbsolutePath(path: string | undefined): string | undefined {
         if (isEmpty(path)) {
             return undefined;

packages/ado-extension/task.json

@@ -121,6 +121,23 @@
             "required": true,
             "defaultValue": true,
             "helpMarkDown": "To get deterministic scanning results, either specify the singleWorker parameter or ensure that the value specified for the maxUrls parameter is larger than the total number of urls in the web site being scanned."
+        },
+        {
+            "name": "uploadResultAsArtifact",
+            "type": "boolean",
+            "label": "Upload result as artifact",
+            "defaultValue": true,
+            "required": true,
+            "helpMarkDown": "Automatically upload the result as an artifact to the build. Set to false if you need to upload the artifact manually in a separate task or publish step."
+        },
+        {
+            "name": "artifactName",
+            "type": "string",
+            "label": "Artifact Name",
+            "required": false,
+            "defaultValue": "accessibility-reports",
+            "helpMarkDown": "Name of the report artifact to be uploaded to the build. Ignored if uploadResultAsArtifact is false.",
+            "visibleRule": "uploadResultAsArtifact = true"
         }
     ],
     "execution": {

packages/shared/src/console-output/__snapshots__/result-console-log-builder.spec.ts.snap

@@ -43,6 +43,25 @@ You can review the log to troubleshoot the issue. Fix it and re-run the pipeline
 "
 `;
 
+exports[`ResultConsoleLogBuilder uploadResultAsArtifact is false skips artifact link line when artifactsUrl returns undefined 1`] = `
+"
+Accessibility Insights
+
+No failures detected
+No failures were detected by automatic scanning.
+
+Next step: Manually assess keyboard accessibility with Accessibility Insights Tab Stops (https://accessibilityinsights.io/docs/en/web/getstarted/fastpass/#complete-the-manual-test-for-tab-stops)
+
+-------------------
+Scan summary
+URLs: 0 with failures, 1 passed, 7 not scannable
+Rules: 0 with failures, 1 passed, 2 not applicable
+
+-------------------
+This scan used axe-core axeVersion (https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent.
+"
+`;
+
 exports[`ResultConsoleLogBuilder with baseline builds content when baseline failures and scanned failures are the same (no new failures) 1`] = `
 "
 Accessibility Insights