AutoApproveNewFeatureBranchBasedOnHistory added

src/test-runs/test-runs.service.spec.ts

@@ -514,8 +514,10 @@ describe('TestRunsService', () => {
     service.getDiff = getDiffMock;
     const saveDiffResultMock = jest.fn();
     service.saveDiffResult = saveDiffResultMock.mockResolvedValueOnce(testRunWithResult);
-    const tryAutoApproveBasedOnHistory = jest.fn();
-    service['tryAutoApproveBasedOnHistory'] = tryAutoApproveBasedOnHistory.mockResolvedValueOnce(testRunWithResult);
+    const tryAutoApproveByPastBaselines = jest.fn();
+    service['tryAutoApproveByPastBaselines'] = tryAutoApproveByPastBaselines.mockResolvedValueOnce(testRunWithResult);
+    const tryAutoApproveByNewBaselines = jest.fn();
+    service['tryAutoApproveByNewBaselines'] = tryAutoApproveByNewBaselines.mockResolvedValueOnce(testRunWithResult);
 
     const result = await service.create(testVariation, createTestRequestDto);
 
@@ -566,7 +568,21 @@ describe('TestRunsService', () => {
       },
     ]);
     expect(saveDiffResultMock).toHaveBeenCalledWith(testRun.id, diffResult);
-    expect(tryAutoApproveBasedOnHistory).toHaveBeenCalledWith(testVariation, testRunWithResult, image, [
+    expect(tryAutoApproveByPastBaselines).toHaveBeenCalledWith(testVariation, testRunWithResult, [
+      {
+        x: 3,
+        y: 4,
+        width: 500,
+        height: 600,
+      },
+      {
+        x: 1,
+        y: 2,
+        width: 100,
+        height: 200,
+      },
+    ]);
+    expect(tryAutoApproveByNewBaselines).toHaveBeenCalledWith(testVariation, testRunWithResult, [
       {
         x: 3,
         y: 4,

src/test-runs/test-runs.service.ts

@@ -5,7 +5,7 @@ import { CreateTestRequestDto } from './dto/create-test-request.dto';
 import { IgnoreAreaDto } from './dto/ignore-area.dto';
 import { StaticService } from '../shared/static/static.service';
 import { PrismaService } from '../prisma/prisma.service';
-import { TestRun, TestStatus, TestVariation } from '@prisma/client';
+import { Baseline, TestRun, TestStatus, TestVariation } from '@prisma/client';
 import { DiffResult } from './diffResult';
 import { EventsGateway } from '../shared/events/events.gateway';
 import { CommentDto } from '../shared/dto/comment.dto';
@@ -233,7 +233,8 @@ export class TestRunsService {
 
     let testRunWithResult = await this.saveDiffResult(testRun.id, diffResult);
 
-    testRunWithResult = await this.tryAutoApproveBasedOnHistory(testVariation, testRunWithResult, image, ignoreAreas);
+    testRunWithResult = await this.tryAutoApproveByPastBaselines(testVariation, testRunWithResult, ignoreAreas);
+    testRunWithResult = await this.tryAutoApproveByNewBaselines(testVariation, testRunWithResult, ignoreAreas);
 
     this.eventsGateway.testRunCreated(testRunWithResult);
     return testRunWithResult;
@@ -334,69 +335,86 @@ export class TestRunsService {
     return image.data;
   }
 
-  private async tryAutoApproveBasedOnHistory(
+  /**
+   * Reason: not rebased code from feature branch is compared agains new main branch baseline thus diff is expected
+   * Tries to find past baseline in main branch and autoApprove in case matched
+   * @param testVariation
+   * @param testRun
+   * @param ignoreAreas
+   */
+  private async tryAutoApproveByPastBaselines(
     testVariation: TestVariation,
     testRun: TestRun,
-    image: PNG,
     ignoreAreas: IgnoreAreaDto[]
   ): Promise<TestRun> {
-    if (process.env.AUTO_APPROVE_BASED_ON_HISTORY && testRun.status !== TestStatus.ok) {
-      this.logger.log(`Try auto approve testRun: ${testRun.id}`);
-
-      const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({
-        where: {
-          ...getTestVariationUniqueData(testVariation),
-          baselineName: testVariation.baselineName,
-          status: TestStatus.approved,
-        },
-      });
+    if (
+      !process.env.AUTO_APPROVE_BASED_ON_HISTORY ||
+      testRun.status === TestStatus.ok ||
+      testRun.branchName === testRun.baselineBranchName
+    ) {
+      return testRun;
+    }
 
-      let autoApproved = false;
-      for (const approvedTestRun of alreadyApprovedTestRuns) {
-        this.logger.log(
-          `Found already approved baseline for testRun: ${testRun.id}
-        testVariation: ${approvedTestRun.testVariationId}
-        branch: ${approvedTestRun.branchName}
-        testRun: ${approvedTestRun.id}
-        build: ${approvedTestRun.buildId}`
-        );
+    this.logger.log(`Try AutoApproveByPastBaselines testRun: ${testRun.id}`);
+    const testVariationHistory = await this.testVariationService.getDetails(testVariation.id);
+    // skip first baseline as it was used by default in general flow
+    for (const baseline of testVariationHistory.baselines.slice(1)) {
+      if (this.shouldAutoApprove(baseline, testRun, ignoreAreas)) {
+        return this.approve(testRun.id, false, true);
+      }
+    }
 
-        const approvedTestVariation = await this.prismaService.testVariation.findUnique({
-          where: {
-            id: approvedTestRun.testVariationId,
-          },
-        });
+    return testRun;
+  }
+
+  /**
+   * Reason: branch got another one merged thus diff is expected
+   * Tries to find latest baseline in test variation
+   * that has already approved test agains the same baseline image
+   * and autoApprove in case matched
+   * @param testVariation
+   * @param testRun
+   * @param image
+   * @param ignoreAreas
+   */
+  private async tryAutoApproveByNewBaselines(
+    testVariation: TestVariation,
+    testRun: TestRun,
+    ignoreAreas: IgnoreAreaDto[]
+  ): Promise<TestRun> {
+    if (!process.env.AUTO_APPROVE_BASED_ON_HISTORY || testRun.status === TestStatus.ok) {
+      return testRun;
+    }
+    this.logger.log(`Try AutoApproveByNewBaselines testRun: ${testRun.id}`);
 
-        const approvedBaseline = this.staticService.getImage(approvedTestVariation.baselineName);
-        const diffResult = this.getDiff(approvedBaseline, image, testRun.diffTollerancePercent, ignoreAreas);
+    const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({
+      where: {
+        ...getTestVariationUniqueData(testVariation),
+        baselineName: testVariation.baselineName,
+        status: TestStatus.approved,
+      },
+    });
 
-        if (diffResult.status === TestStatus.ok) {
-          autoApproved = true;
-          const baseline = await this.prismaService.baseline.findFirst({
-            where: {
-              testVariationId: approvedTestVariation.id,
-              baselineName: approvedTestVariation.baselineName,
-            },
-            include: {
-              testRun: true,
-            },
-          });
-          this.logger.log(
-            `Found reason to auto approve testRun: ${testRun.id}
-          testVariation: ${baseline.testVariationId}
-          baseline: ${baseline.id}
-          branch: ${approvedTestVariation.branchName}
-          testRun: ${baseline.testRunId}
-          build: ${baseline.testRun.buildId}`
-          );
-        }
-      }
+    for (const approvedTestRun of alreadyApprovedTestRuns) {
+      const approvedTestVariation = await this.testVariationService.getDetails(approvedTestRun.testVariationId);
+      const baseline = approvedTestVariation.baselines.shift();
 
-      if (autoApproved) {
+      if (this.shouldAutoApprove(baseline, testRun, ignoreAreas)) {
         return this.approve(testRun.id, false, true);
       }
-      this.logger.log(`Cannot auto approve testRun: ${testRun.id}`);
     }
+
     return testRun;
   }
+
+  private shouldAutoApprove(baseline: Baseline, testRun: TestRun, ignoreAreas: Array<IgnoreAreaDto>): boolean {
+    const approvedImage = this.staticService.getImage(baseline.baselineName);
+    const image = this.staticService.getImage(testRun.imageName);
+    const diffResult = this.getDiff(approvedImage, image, testRun.diffTollerancePercent, ignoreAreas);
+
+    if (diffResult.status === TestStatus.ok) {
+      this.logger.log(`TestRun ${testRun.id} could be auto approved based on Baseline ${baseline.id}`);
+      return true;
+    }
+  }
 }

test/preconditions.ts

@@ -2,6 +2,10 @@ import { INestApplication } from '@nestjs/common';
 import { UsersService } from 'src/users/users.service';
 import uuidAPIKey from 'uuid-apikey';
 import request, { Test } from 'supertest';
+import { BuildsService } from 'src/builds/builds.service';
+import { TestRunsService } from 'src/test-runs/test-runs.service';
+import { readFileSync } from 'fs';
+import { TestRunResultDto } from 'src/test-runs/dto/testRunResult.dto';
 
 export const generateUser = (
   password: string
@@ -41,3 +45,21 @@ export const haveUserLogged = async (usersService: UsersService) => {
     password,
   });
 };
+
+export const haveTestRunCreated = async (
+  buildsService: BuildsService,
+  testRunsService: TestRunsService,
+  projectId: string,
+  branchName: string,
+  imagePath: string
+): Promise<TestRunResultDto> => {
+  const build = await buildsService.create({ project: projectId, branchName });
+  return testRunsService.postTestRun({
+    projectId: build.projectId,
+    branchName: build.branchName,
+    imageBase64: readFileSync(imagePath).toString('base64'),
+    buildId: build.id,
+    name: 'Image name',
+    merge: false,
+  });
+};

test/test-runs.e2e-spec.ts

@@ -0,0 +1,109 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { INestApplication } from '@nestjs/common';
+import { AppModule } from '../src/app.module';
+import { UsersService } from '../src/users/users.service';
+import { haveTestRunCreated, haveUserLogged } from './preconditions';
+import { UserLoginResponseDto } from '../src/users/dto/user-login-response.dto';
+import { TestRunsService } from '../src/test-runs/test-runs.service';
+import { ProjectsService } from '../src/projects/projects.service';
+import { Project, TestStatus } from '@prisma/client';
+import { BuildsService } from '../src/builds/builds.service';
+
+jest.setTimeout(20000);
+
+describe('TestRuns (e2e)', () => {
+  let app: INestApplication;
+  let testRunsService: TestRunsService;
+  let usersService: UsersService;
+  let projecstService: ProjectsService;
+  let buildsService: BuildsService;
+  let user: UserLoginResponseDto;
+  let project: Project;
+
+  beforeAll(async () => {
+    const moduleFixture: TestingModule = await Test.createTestingModule({
+      imports: [AppModule],
+    }).compile();
+
+    app = moduleFixture.createNestApplication();
+    testRunsService = moduleFixture.get<TestRunsService>(TestRunsService);
+    usersService = moduleFixture.get<UsersService>(UsersService);
+    projecstService = moduleFixture.get<ProjectsService>(ProjectsService);
+    buildsService = moduleFixture.get<BuildsService>(BuildsService);
+
+    await app.init();
+  });
+
+  beforeEach(async () => {
+    user = await haveUserLogged(usersService);
+    project = await projecstService.create({ name: 'TestRun E2E test', mainBranchName: 'master' });
+  });
+
+  afterEach(async () => {
+    await projecstService.remove(project.id);
+    await usersService.delete(user.id);
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  describe('POST /', () => {
+    const image_v1 = './test/image.png';
+    const image_v2 = './test/image_edited.png';
+    it('Auto approve not rebased feature branch', async () => {
+      const testRun1 = await haveTestRunCreated(
+        buildsService,
+        testRunsService,
+        project.id,
+        project.mainBranchName,
+        image_v1
+      );
+      await testRunsService.approve(testRun1.id, false, false);
+      const testRun2 = await haveTestRunCreated(
+        buildsService,
+        testRunsService,
+        project.id,
+        project.mainBranchName,
+        image_v2
+      );
+      await testRunsService.approve(testRun2.id, false, false);
+
+      const testRun = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1);
+
+      expect(testRun.status).toBe(TestStatus.autoApproved);
+    });
+
+    it('Auto approve merged feature into feature branch', async () => {
+      const testRun1 = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature1', image_v1);
+      await testRunsService.approve(testRun1.id, false, false);
+
+      const testRun = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature2', image_v1);
+
+      expect(testRun.status).toBe(TestStatus.autoApproved);
+    });
+
+    it('Auto approve merged feature into main branch', async () => {
+      const testRun1 = await haveTestRunCreated(
+        buildsService,
+        testRunsService,
+        project.id,
+        project.mainBranchName,
+        image_v1
+      );
+      await testRunsService.approve(testRun1.id, false, false);
+      const testRun2 = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v2);
+      await testRunsService.approve(testRun2.id, false, false);
+
+      const testRun = await haveTestRunCreated(
+        buildsService,
+        testRunsService,
+        project.id,
+        project.mainBranchName,
+        image_v2
+      );
+
+      expect(testRun.status).toBe(TestStatus.autoApproved);
+    });
+  });
+});