diff --git a/src/test-runs/test-runs.service.spec.ts b/src/test-runs/test-runs.service.spec.ts index 7aec62b8..597a6b17 100644 --- a/src/test-runs/test-runs.service.spec.ts +++ b/src/test-runs/test-runs.service.spec.ts @@ -514,8 +514,10 @@ describe('TestRunsService', () => { service.getDiff = getDiffMock; const saveDiffResultMock = jest.fn(); service.saveDiffResult = saveDiffResultMock.mockResolvedValueOnce(testRunWithResult); - const tryAutoApproveBasedOnHistory = jest.fn(); - service['tryAutoApproveBasedOnHistory'] = tryAutoApproveBasedOnHistory.mockResolvedValueOnce(testRunWithResult); + const tryAutoApproveByPastBaselines = jest.fn(); + service['tryAutoApproveByPastBaselines'] = tryAutoApproveByPastBaselines.mockResolvedValueOnce(testRunWithResult); + const tryAutoApproveByNewBaselines = jest.fn(); + service['tryAutoApproveByNewBaselines'] = tryAutoApproveByNewBaselines.mockResolvedValueOnce(testRunWithResult); const result = await service.create(testVariation, createTestRequestDto); @@ -566,7 +568,21 @@ describe('TestRunsService', () => { }, ]); expect(saveDiffResultMock).toHaveBeenCalledWith(testRun.id, diffResult); - expect(tryAutoApproveBasedOnHistory).toHaveBeenCalledWith(testVariation, testRunWithResult, image, [ + expect(tryAutoApproveByPastBaselines).toHaveBeenCalledWith(testVariation, testRunWithResult, [ + { + x: 3, + y: 4, + width: 500, + height: 600, + }, + { + x: 1, + y: 2, + width: 100, + height: 200, + }, + ]); + expect(tryAutoApproveByNewBaselines).toHaveBeenCalledWith(testVariation, testRunWithResult, [ { x: 3, y: 4, diff --git a/src/test-runs/test-runs.service.ts b/src/test-runs/test-runs.service.ts index e3853874..dad02035 100644 --- a/src/test-runs/test-runs.service.ts +++ b/src/test-runs/test-runs.service.ts @@ -5,7 +5,7 @@ import { CreateTestRequestDto } from './dto/create-test-request.dto'; import { IgnoreAreaDto } from './dto/ignore-area.dto'; import { StaticService } from '../shared/static/static.service'; import { PrismaService } from '../prisma/prisma.service'; -import { TestRun, TestStatus, TestVariation } from '@prisma/client'; +import { Baseline, TestRun, TestStatus, TestVariation } from '@prisma/client'; import { DiffResult } from './diffResult'; import { EventsGateway } from '../shared/events/events.gateway'; import { CommentDto } from '../shared/dto/comment.dto'; @@ -233,7 +233,8 @@ export class TestRunsService { let testRunWithResult = await this.saveDiffResult(testRun.id, diffResult); - testRunWithResult = await this.tryAutoApproveBasedOnHistory(testVariation, testRunWithResult, image, ignoreAreas); + testRunWithResult = await this.tryAutoApproveByPastBaselines(testVariation, testRunWithResult, ignoreAreas); + testRunWithResult = await this.tryAutoApproveByNewBaselines(testVariation, testRunWithResult, ignoreAreas); this.eventsGateway.testRunCreated(testRunWithResult); return testRunWithResult; @@ -334,69 +335,86 @@ export class TestRunsService { return image.data; } - private async tryAutoApproveBasedOnHistory( + /** + * Reason: not rebased code from feature branch is compared agains new main branch baseline thus diff is expected + * Tries to find past baseline in main branch and autoApprove in case matched + * @param testVariation + * @param testRun + * @param ignoreAreas + */ + private async tryAutoApproveByPastBaselines( testVariation: TestVariation, testRun: TestRun, - image: PNG, ignoreAreas: IgnoreAreaDto[] ): Promise { - if (process.env.AUTO_APPROVE_BASED_ON_HISTORY && testRun.status !== TestStatus.ok) { - this.logger.log(`Try auto approve testRun: ${testRun.id}`); - - const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({ - where: { - ...getTestVariationUniqueData(testVariation), - baselineName: testVariation.baselineName, - status: TestStatus.approved, - }, - }); + if ( + !process.env.AUTO_APPROVE_BASED_ON_HISTORY || + testRun.status === TestStatus.ok || + testRun.branchName === testRun.baselineBranchName + ) { + return testRun; + } - let autoApproved = false; - for (const approvedTestRun of alreadyApprovedTestRuns) { - this.logger.log( - `Found already approved baseline for testRun: ${testRun.id} - testVariation: ${approvedTestRun.testVariationId} - branch: ${approvedTestRun.branchName} - testRun: ${approvedTestRun.id} - build: ${approvedTestRun.buildId}` - ); + this.logger.log(`Try AutoApproveByPastBaselines testRun: ${testRun.id}`); + const testVariationHistory = await this.testVariationService.getDetails(testVariation.id); + // skip first baseline as it was used by default in general flow + for (const baseline of testVariationHistory.baselines.slice(1)) { + if (this.shouldAutoApprove(baseline, testRun, ignoreAreas)) { + return this.approve(testRun.id, false, true); + } + } - const approvedTestVariation = await this.prismaService.testVariation.findUnique({ - where: { - id: approvedTestRun.testVariationId, - }, - }); + return testRun; + } + + /** + * Reason: branch got another one merged thus diff is expected + * Tries to find latest baseline in test variation + * that has already approved test agains the same baseline image + * and autoApprove in case matched + * @param testVariation + * @param testRun + * @param image + * @param ignoreAreas + */ + private async tryAutoApproveByNewBaselines( + testVariation: TestVariation, + testRun: TestRun, + ignoreAreas: IgnoreAreaDto[] + ): Promise { + if (!process.env.AUTO_APPROVE_BASED_ON_HISTORY || testRun.status === TestStatus.ok) { + return testRun; + } + this.logger.log(`Try AutoApproveByNewBaselines testRun: ${testRun.id}`); - const approvedBaseline = this.staticService.getImage(approvedTestVariation.baselineName); - const diffResult = this.getDiff(approvedBaseline, image, testRun.diffTollerancePercent, ignoreAreas); + const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({ + where: { + ...getTestVariationUniqueData(testVariation), + baselineName: testVariation.baselineName, + status: TestStatus.approved, + }, + }); - if (diffResult.status === TestStatus.ok) { - autoApproved = true; - const baseline = await this.prismaService.baseline.findFirst({ - where: { - testVariationId: approvedTestVariation.id, - baselineName: approvedTestVariation.baselineName, - }, - include: { - testRun: true, - }, - }); - this.logger.log( - `Found reason to auto approve testRun: ${testRun.id} - testVariation: ${baseline.testVariationId} - baseline: ${baseline.id} - branch: ${approvedTestVariation.branchName} - testRun: ${baseline.testRunId} - build: ${baseline.testRun.buildId}` - ); - } - } + for (const approvedTestRun of alreadyApprovedTestRuns) { + const approvedTestVariation = await this.testVariationService.getDetails(approvedTestRun.testVariationId); + const baseline = approvedTestVariation.baselines.shift(); - if (autoApproved) { + if (this.shouldAutoApprove(baseline, testRun, ignoreAreas)) { return this.approve(testRun.id, false, true); } - this.logger.log(`Cannot auto approve testRun: ${testRun.id}`); } + return testRun; } + + private shouldAutoApprove(baseline: Baseline, testRun: TestRun, ignoreAreas: Array): boolean { + const approvedImage = this.staticService.getImage(baseline.baselineName); + const image = this.staticService.getImage(testRun.imageName); + const diffResult = this.getDiff(approvedImage, image, testRun.diffTollerancePercent, ignoreAreas); + + if (diffResult.status === TestStatus.ok) { + this.logger.log(`TestRun ${testRun.id} could be auto approved based on Baseline ${baseline.id}`); + return true; + } + } } diff --git a/test/image_edited.png b/test/image_edited.png new file mode 100644 index 00000000..a6c4624a Binary files /dev/null and b/test/image_edited.png differ diff --git a/test/preconditions.ts b/test/preconditions.ts index 8638a05f..32ceeb07 100644 --- a/test/preconditions.ts +++ b/test/preconditions.ts @@ -2,6 +2,10 @@ import { INestApplication } from '@nestjs/common'; import { UsersService } from 'src/users/users.service'; import uuidAPIKey from 'uuid-apikey'; import request, { Test } from 'supertest'; +import { BuildsService } from 'src/builds/builds.service'; +import { TestRunsService } from 'src/test-runs/test-runs.service'; +import { readFileSync } from 'fs'; +import { TestRunResultDto } from 'src/test-runs/dto/testRunResult.dto'; export const generateUser = ( password: string @@ -41,3 +45,21 @@ export const haveUserLogged = async (usersService: UsersService) => { password, }); }; + +export const haveTestRunCreated = async ( + buildsService: BuildsService, + testRunsService: TestRunsService, + projectId: string, + branchName: string, + imagePath: string +): Promise => { + const build = await buildsService.create({ project: projectId, branchName }); + return testRunsService.postTestRun({ + projectId: build.projectId, + branchName: build.branchName, + imageBase64: readFileSync(imagePath).toString('base64'), + buildId: build.id, + name: 'Image name', + merge: false, + }); +}; diff --git a/test/test-runs.e2e-spec.ts b/test/test-runs.e2e-spec.ts new file mode 100644 index 00000000..8dbce73b --- /dev/null +++ b/test/test-runs.e2e-spec.ts @@ -0,0 +1,109 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { INestApplication } from '@nestjs/common'; +import { AppModule } from '../src/app.module'; +import { UsersService } from '../src/users/users.service'; +import { haveTestRunCreated, haveUserLogged } from './preconditions'; +import { UserLoginResponseDto } from '../src/users/dto/user-login-response.dto'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { Project, TestStatus } from '@prisma/client'; +import { BuildsService } from '../src/builds/builds.service'; + +jest.setTimeout(20000); + +describe('TestRuns (e2e)', () => { + let app: INestApplication; + let testRunsService: TestRunsService; + let usersService: UsersService; + let projecstService: ProjectsService; + let buildsService: BuildsService; + let user: UserLoginResponseDto; + let project: Project; + + beforeAll(async () => { + const moduleFixture: TestingModule = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + + app = moduleFixture.createNestApplication(); + testRunsService = moduleFixture.get(TestRunsService); + usersService = moduleFixture.get(UsersService); + projecstService = moduleFixture.get(ProjectsService); + buildsService = moduleFixture.get(BuildsService); + + await app.init(); + }); + + beforeEach(async () => { + user = await haveUserLogged(usersService); + project = await projecstService.create({ name: 'TestRun E2E test', mainBranchName: 'master' }); + }); + + afterEach(async () => { + await projecstService.remove(project.id); + await usersService.delete(user.id); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('POST /', () => { + const image_v1 = './test/image.png'; + const image_v2 = './test/image_edited.png'; + it('Auto approve not rebased feature branch', async () => { + const testRun1 = await haveTestRunCreated( + buildsService, + testRunsService, + project.id, + project.mainBranchName, + image_v1 + ); + await testRunsService.approve(testRun1.id, false, false); + const testRun2 = await haveTestRunCreated( + buildsService, + testRunsService, + project.id, + project.mainBranchName, + image_v2 + ); + await testRunsService.approve(testRun2.id, false, false); + + const testRun = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); + + expect(testRun.status).toBe(TestStatus.autoApproved); + }); + + it('Auto approve merged feature into feature branch', async () => { + const testRun1 = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature1', image_v1); + await testRunsService.approve(testRun1.id, false, false); + + const testRun = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature2', image_v1); + + expect(testRun.status).toBe(TestStatus.autoApproved); + }); + + it('Auto approve merged feature into main branch', async () => { + const testRun1 = await haveTestRunCreated( + buildsService, + testRunsService, + project.id, + project.mainBranchName, + image_v1 + ); + await testRunsService.approve(testRun1.id, false, false); + const testRun2 = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v2); + await testRunsService.approve(testRun2.id, false, false); + + const testRun = await haveTestRunCreated( + buildsService, + testRunsService, + project.id, + project.mainBranchName, + image_v2 + ); + + expect(testRun.status).toBe(TestStatus.autoApproved); + }); + }); +});