feat(pr-baseline-2): Modify ADO PR comments to include baseline info

packages/ado-extension/src/progress-reporter/pull-request/ado-pull-request-comment-creator.spec.ts

@@ -7,7 +7,7 @@ import * as GitApi from 'azure-devops-node-api/GitApi';
 import * as nodeApi from 'azure-devops-node-api';
 import * as GitInterfaces from 'azure-devops-node-api/interfaces/GitInterfaces';
 
-import { It, Mock, Times, IMock, MockBehavior } from 'typemoq';
+import { Mock, Times, IMock, MockBehavior } from 'typemoq';
 import {
     AdoPullRequestCommentCreator as ADOPullRequestCommentCreator,
     AdoPullRequestCommentCreator,
@@ -17,6 +17,7 @@ import { CombinedReportParameters } from 'accessibility-insights-report';
 
 import { Logger, ReportMarkdownConvertor } from '@accessibility-insights-action/shared';
 import { BaselineEvaluation, BaselineFileContent } from '@accessibility-insights-action/shared/dist/baseline-types';
+import { BaselineInfo } from '@accessibility-insights-action/shared/dist/baseline-info';
 
 describe(ADOPullRequestCommentCreator, () => {
     let adoTaskMock: IMock<typeof adoTask>;
@@ -193,6 +194,7 @@ describe(ADOPullRequestCommentCreator, () => {
             gitApiMock.setup((o) => o.createThread(newThread, repoId, prId)).verifiable(Times.once());
             setupIsSupportedReturnsTrue();
             setupFailOnAccessibilityError(false);
+            setupBaselineFileParameterDoesNotExist();
             setupInitializeWithoutServiceConnectionName();
             setupInitializeSetConnection(webApiMock.object);
             prCommentCreator = buildPrCommentCreatorWithMocks();
@@ -217,6 +219,7 @@ describe(ADOPullRequestCommentCreator, () => {
             gitApiMock.setup((o) => o.createComment(newComment, repoId, prId, threadId)).verifiable(Times.once());
             setupIsSupportedReturnsTrue();
             setupFailOnAccessibilityError(true);
+            setupBaselineFileParameterDoesNotExist();
             setupInitializeWithoutServiceConnectionName();
             setupInitializeSetConnection(webApiMock.object);
             prCommentCreator = buildPrCommentCreatorWithMocks();
@@ -244,6 +247,7 @@ describe(ADOPullRequestCommentCreator, () => {
             gitApiMock.setup((o) => o.updateComment(expectedComment, repoId, prId, threadId, commentId)).verifiable(Times.once());
             setupIsSupportedReturnsTrue();
             setupFailOnAccessibilityError(true);
+            setupBaselineFileParameterDoesNotExist();
             setupInitializeWithoutServiceConnectionName();
             setupInitializeSetConnection(webApiMock.object);
             prCommentCreator = buildPrCommentCreatorWithMocks();
@@ -263,15 +267,25 @@ describe(ADOPullRequestCommentCreator, () => {
             const baselineEvaluationStub = {
                 suggestedBaselineUpdate: {} as BaselineFileContent,
             } as BaselineEvaluation;
+            const baselineInfo: BaselineInfo = {
+                baselineFileName: 'baseline-file',
+                baselineEvaluation: baselineEvaluationStub,
+            };
 
             setupReturnPrThread(repoId, prId, reportStub, reportMd, threadsStub);
+            reportMarkdownConvertorMock.reset();
+            reportMarkdownConvertorMock
+                .setup((o) => o.convert(reportStub, ADOPullRequestCommentCreator.CURRENT_COMMENT_TITLE, baselineInfo))
+                .returns(() => ADOPullRequestCommentCreator.CURRENT_COMMENT_TITLE + reportMd)
+                .verifiable(Times.once());
             loggerMock.setup((o) => o.logInfo(`Didn't find an existing thread, making a new thread`)).verifiable(Times.once());
             gitApiMock.setup((o) => o.createThread(newThread, repoId, prId)).verifiable(Times.once());
             setupIsSupportedReturnsTrue();
             setupFailOnAccessibilityError(false);
             setupBaselineFileParameterExists();
             setupInitializeWithoutServiceConnectionName();
             setupInitializeSetConnection(webApiMock.object);
+
             prCommentCreator = buildPrCommentCreatorWithMocks();
 
             await expect(prCommentCreator.completeRun(reportStub, baselineEvaluationStub)).rejects.toThrowError(
@@ -342,7 +356,14 @@ describe(ADOPullRequestCommentCreator, () => {
     const setupBaselineFileParameterExists = () => {
         adoTaskConfigMock
             .setup((o) => o.getBaselineFile())
-            .returns(() => It.isAnyString())
+            .returns(() => 'baseline-file')
+            .verifiable(Times.atLeastOnce());
+    };
+
+    const setupBaselineFileParameterDoesNotExist = () => {
+        adoTaskConfigMock
+            .setup((o) => o.getBaselineFile())
+            .returns(() => undefined)
             .verifiable(Times.atLeastOnce());
     };
 
@@ -488,8 +509,9 @@ describe(ADOPullRequestCommentCreator, () => {
         threadsStub: GitInterfaces.GitPullRequestCommentThread[],
     ) => {
         makeGitApiMockThenable();
+        const baselineInfoStub = {};
         reportMarkdownConvertorMock
-            .setup((o) => o.convert(reportStub, ADOPullRequestCommentCreator.CURRENT_COMMENT_TITLE))
+            .setup((o) => o.convert(reportStub, ADOPullRequestCommentCreator.CURRENT_COMMENT_TITLE, baselineInfoStub))
             .returns(() => ADOPullRequestCommentCreator.CURRENT_COMMENT_TITLE + reportMd)
             .verifiable(Times.once());
         adoTaskMock

packages/ado-extension/src/progress-reporter/pull-request/ado-pull-request-comment-creator.ts

@@ -14,6 +14,7 @@ import * as GitApi from 'azure-devops-node-api/GitApi';
 import * as GitInterfaces from 'azure-devops-node-api/interfaces/GitInterfaces';
 import * as VsoBaseInterfaces from 'azure-devops-node-api/interfaces/common/VsoBaseInterfaces';
 import { BaselineEvaluation, BaselineFileContent } from '@accessibility-insights-action/shared/src/baseline-types';
+import { BaselineInfo } from '@accessibility-insights-action/shared/src/baseline-info';
 
 @injectable()
 export class AdoPullRequestCommentCreator extends ProgressReporter {
@@ -97,6 +98,7 @@ export class AdoPullRequestCommentCreator extends ProgressReporter {
         const reportMarkdown = this.reportMarkdownConvertor.convert(
             combinedReportResult,
             AdoPullRequestCommentCreator.CURRENT_COMMENT_TITLE,
+            this.getBaselineInfo(baselineEvaluation),
         );
         this.traceMarkdown(reportMarkdown);
 
@@ -175,6 +177,16 @@ export class AdoPullRequestCommentCreator extends ProgressReporter {
         throw message;
     }
 
+    private getBaselineInfo(baselineEvaluation?: BaselineEvaluation): BaselineInfo {
+        const baselineFileName = this.adoTaskConfig.getBaselineFile();
+
+        if (!baselineFileName) {
+            return {} as BaselineInfo;
+        }
+
+        return { baselineFileName, baselineEvaluation };
+    }
+
     private failOnAccessibilityError(combinedReportResult: CombinedReportParameters): void {
         if (this.adoTaskConfig.getFailOnAccessibilityError() && combinedReportResult.results.urlResults.failedUrls > 0) {
             throw new Error('Failed Accessibility Error');

packages/shared/src/baseline-info.ts

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { BaselineEvaluation } from './baseline-types';
+
+export type BaselineInfo = {
+    baselineFileName?: string;
+    baselineEvaluation?: BaselineEvaluation;
+};

packages/shared/src/baseline-types.ts

@@ -24,6 +24,7 @@ export type BaselineEvaluation = {
     fixedViolationsByRule: CountsByRule;
     totalNewViolations: number;
     totalFixedViolations: number;
+    totalBaselineViolations?: number;
 };
 
 export type BaselineResult = {

packages/shared/src/mark-down/__snapshots__/result-markdown-builder.spec.ts.snap

@@ -39,3 +39,170 @@ You can review the log to troubleshoot the issue. Fix it and re-run the workflow
 ---
 "
 `;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are baseline failures and new failures 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+
+**4 failure instances not in baseline**
+* (3) **rule id**:  rule description
+* (1) **rule id 2**:  rule description 2
+
+**1 failure instances in baseline**
+(not shown; see [baselining docs](temporarily-empty) for how to include new failures into the baseline)
+
+See all failures and scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 5 with failures, 1 passed, 7 not scannable
+**Rules**: 4 with failures, 1 passed, 2 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are baseline failures and no new failures 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+**:white_check_mark: No failures not in baseline**
+No failures were detected by automatic scanning except those which exist in the baseline.
+
+**:point_right: Next step:** Manually assess keyboard accessibility with [Accessibility Insights Tab Stops](https://accessibilityinsights.io/docs/en/web/getstarted/fastpass/#complete-the-manual-test-for-tab-stops)
+
+**1 failure instances in baseline**
+(not shown; see [baselining docs](temporarily-empty))
+
+See all failures and scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 0 with failures, 0 passed, 0 not scannable
+**Rules**: 0 with failures, 0 passed, 0 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are new failures and no baseline detected 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+
+**4 failure instances**
+* (3) **rule id**:  rule description
+* (1) **rule id 2**:  rule description 2
+
+**Baseline not detected**
+To update the baseline with these changes, copy the updated baseline file to [scan arguments](temporarily-empty). See [baselining docs](temporarily-empty) for more.
+
+See all failures and scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 5 with failures, 1 passed, 7 not scannable
+**Rules**: 4 with failures, 1 passed, 2 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are new failures and no baseline failures 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+
+**4 failure instances**
+* (3) **rule id**:  rule description
+* (1) **rule id 2**:  rule description 2
+
+**Baseline not detected**
+To update the baseline with these changes, copy the updated baseline file to [scan arguments](temporarily-empty). See [baselining docs](temporarily-empty) for more.
+
+See all failures and scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 5 with failures, 1 passed, 7 not scannable
+**Rules**: 4 with failures, 1 passed, 2 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are no baseline failures and no new failures 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+**:white_check_mark: No failures detected**
+No failures were detected by automatic scanning.
+
+**:point_right: Next step:** Manually assess keyboard accessibility with [Accessibility Insights Tab Stops](https://accessibilityinsights.io/docs/en/web/getstarted/fastpass/#complete-the-manual-test-for-tab-stops)
+
+
+
+See scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 0 with failures, 1 passed, 7 not scannable
+**Rules**: 0 with failures, 1 passed, 2 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are no new failures and baseline failures are undefined 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+**:white_check_mark: No failures detected**
+No failures were detected by automatic scanning.
+
+**:point_right: Next step:** Manually assess keyboard accessibility with [Accessibility Insights Tab Stops](https://accessibilityinsights.io/docs/en/web/getstarted/fastpass/#complete-the-manual-test-for-tab-stops)
+
+**Baseline not configured**
+A baseline lets you mark known failures so it's easier to identify new failures as they're introduced. See [baselining docs](temporarily-empty) for more.
+
+See scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 0 with failures, 0 passed, 0 not scannable
+**Rules**: 0 with failures, 0 passed, 0 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there are no new failures and no baseline detected 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+**:white_check_mark: No failures detected**
+No failures were detected by automatic scanning.
+
+**:point_right: Next step:** Manually assess keyboard accessibility with [Accessibility Insights Tab Stops](https://accessibilityinsights.io/docs/en/web/getstarted/fastpass/#complete-the-manual-test-for-tab-stops)
+
+**Baseline not detected**
+To update the baseline with these changes, copy the updated baseline file to [scan arguments](temporarily-empty). See [baselining docs](temporarily-empty) for more.
+
+See all failures and scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 0 with failures, 0 passed, 0 not scannable
+**Rules**: 0 with failures, 0 passed, 0 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;
+
+exports[`ResultMarkdownBuilder with baseline builds content when there is no baseline configured 1`] = `
+"### ![Accessibility Insights](https://accessibilityinsights.io/img/a11yinsights-blue.svg) Accessibility Insights Action
+
+**4 failure instances**
+* (3) **rule id**:  rule description
+* (1) **rule id 2**:  rule description 2
+
+**Baseline not configured**
+A baseline lets you mark known failures so it's easier to identify new failures as they're introduced. See [baselining docs](temporarily-empty) for more.
+
+See all failures and scan details by downloading the report from [run artifacts](temporarily-empty)
+
+---
+#### Scan summary
+**URLs**: 5 with failures, 1 passed, 7 not scannable
+**Rules**: 4 with failures, 1 passed, 2 not applicable
+
+---
+This scan used [axe-core axeVersion](https://github.com/dequelabs/axe-core/releases/tag/vaxeVersion) with userAgent."
+`;

packages/shared/src/mark-down/report-markdown-convertor.spec.ts

@@ -6,6 +6,7 @@ import { IMock, Mock } from 'typemoq';
 import { ReportMarkdownConvertor } from './report-markdown-convertor';
 import { ResultMarkdownBuilder } from './result-markdown-builder';
 import { CombinedReportParameters } from 'accessibility-insights-report';
+import { BaselineEvaluation } from '../baseline-types';
 
 describe(ReportMarkdownConvertor, () => {
     let resultMarkdownBuilderMock: IMock<ResultMarkdownBuilder>;
@@ -34,17 +35,29 @@ describe(ReportMarkdownConvertor, () => {
         expect(reportMarkdownConvertor).not.toBeNull();
     });
 
-    it('convert report', () => {
-        resultMarkdownBuilderMock.setup((o) => o.buildContent(combinedReportResult, undefined)).verifiable();
+    describe('convert', () => {
+        it('report', () => {
+            resultMarkdownBuilderMock.setup((o) => o.buildContent(combinedReportResult, undefined, undefined)).verifiable();
 
-        reportMarkdownConvertor.convert(combinedReportResult);
-    });
+            reportMarkdownConvertor.convert(combinedReportResult);
+        });
+
+        it('report with title', () => {
+            const title = 'some title';
+            resultMarkdownBuilderMock.setup((o) => o.buildContent(combinedReportResult, title, undefined)).verifiable();
+
+            reportMarkdownConvertor.convert(combinedReportResult, title);
+        });
 
-    it('convert report with title', () => {
-        const title = 'some title';
-        resultMarkdownBuilderMock.setup((o) => o.buildContent(combinedReportResult, title)).verifiable();
+        it('report with baseline', () => {
+            const baselineInfo = {
+                baselineFileName: 'some filename',
+                baselineEvaluationStub: {} as BaselineEvaluation,
+            };
+            resultMarkdownBuilderMock.setup((o) => o.buildContent(combinedReportResult, undefined, baselineInfo)).verifiable();
 
-        reportMarkdownConvertor.convert(combinedReportResult, title);
+            reportMarkdownConvertor.convert(combinedReportResult, undefined, baselineInfo);
+        });
     });
 
     it('get error markdown', () => {

packages/shared/src/mark-down/report-markdown-convertor.ts

@@ -4,13 +4,14 @@
 import { inject, injectable } from 'inversify';
 import { ResultMarkdownBuilder } from './result-markdown-builder';
 import { CombinedReportParameters } from 'accessibility-insights-report';
+import { BaselineInfo } from '../baseline-info';
 
 @injectable()
 export class ReportMarkdownConvertor {
     constructor(@inject(ResultMarkdownBuilder) private readonly checkResultMarkdownBuilder: ResultMarkdownBuilder) {}
 
-    public convert(combinedReportResult: CombinedReportParameters, title?: string): string {
-        return this.checkResultMarkdownBuilder.buildContent(combinedReportResult, title);
+    public convert(combinedReportResult: CombinedReportParameters, title?: string, baselineInfo?: BaselineInfo): string {
+        return this.checkResultMarkdownBuilder.buildContent(combinedReportResult, title, baselineInfo);
     }
 
     public getErrorMarkdown(): string {