From 15ff7df0e1e5dcc1befb123eee3df26ce9b079cb Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Fri, 9 Jul 2021 15:18:30 +0300 Subject: [PATCH 1/3] Clean up old test variations https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/275 --- package-lock.json | 40 +++++ package.json | 2 + .../README.md | 74 +++++++++ .../schema.prisma | 140 ++++++++++++++++++ .../steps.json | 37 +++++ prisma/migrations/migrate.lock | 3 +- prisma/schema.prisma | 15 +- src/_data_/index.ts | 1 + src/app.module.ts | 4 +- src/projects/dto/project.dto.ts | 6 +- src/projects/projects.service.ts | 1 + src/shared/shared.module.ts | 8 +- src/shared/tasks/tasks.service.ts | 38 +++++ .../test-variations.service.ts | 6 +- 14 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 prisma/migrations/20210709115233-gh-275-max-branch-lifetime/README.md create mode 100644 prisma/migrations/20210709115233-gh-275-max-branch-lifetime/schema.prisma create mode 100644 prisma/migrations/20210709115233-gh-275-max-branch-lifetime/steps.json create mode 100644 src/shared/tasks/tasks.service.ts diff --git a/package-lock.json b/package-lock.json index 45d42eb1..8cf3a128 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1085,6 +1085,15 @@ } } }, + "@nestjs/schedule": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-0.4.3.tgz", + "integrity": "sha512-EowkpwD09lrdMFzjt/kNQgaq6MG0GrNadNoOEebbfyA0qBU0UTg7J+FVullTR9+5HtaegqqAuWDvZlVoGHYz+Q==", + "requires": { + "cron": "1.7.2", + "uuid": "8.3.2" + } + }, "@nestjs/schematics": { "version": "7.2.5", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-7.2.5.tgz", @@ -1309,6 +1318,16 @@ "integrity": "sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==", "dev": true }, + "@types/cron": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@types/cron/-/cron-1.7.2.tgz", + "integrity": "sha512-AEpNLRcsVSc5AdseJKNHpz0d4e8+ow+abTaC0fKDbAU86rF1evoFF0oC2fV9FdqtfVXkG2LKshpLTJCFOpyvTg==", + "dev": true, + "requires": { + "@types/node": "*", + "moment": ">=2.14.0" + } + }, "@types/engine.io": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/@types/engine.io/-/engine.io-3.1.4.tgz", @@ -3019,6 +3038,14 @@ "yaml": "^1.7.2" } }, + "cron": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/cron/-/cron-1.7.2.tgz", + "integrity": "sha512-+SaJ2OfeRvfQqwXQ2kgr0Y5pzBR/lijf5OpnnaruwWnmI799JfWr2jN2ItOV9s3A/+TFOt6mxvKzQq5F0Jp6VQ==", + "requires": { + "moment-timezone": "^0.5.x" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -6963,6 +6990,19 @@ "minimist": "^1.2.5" } }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + }, + "moment-timezone": { + "version": "0.5.33", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", + "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", diff --git a/package.json b/package.json index c6eeec6b..65850ff4 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@nestjs/passport": "^7.1.0", "@nestjs/platform-express": "^7.4.2", "@nestjs/platform-socket.io": "^7.6.17", + "@nestjs/schedule": "^0.4.3", "@nestjs/swagger": "^4.8.0", "@nestjs/websockets": "^7.4.2", "@prisma/client": "2.12.1", @@ -58,6 +59,7 @@ "@prisma/cli": "2.12.1", "@types/bcryptjs": "^2.4.2", "@types/cache-manager": "^2.10.3", + "@types/cron": "^1.7.2", "@types/express": "^4.17.7", "@types/jest": "26.0.14", "@types/lodash": "^4.14.168", diff --git a/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/README.md b/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/README.md new file mode 100644 index 00000000..60fbdee7 --- /dev/null +++ b/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/README.md @@ -0,0 +1,74 @@ +# Migration `20210709115233-gh-275-max-branch-lifetime` + +This migration has been generated by Pavel Strunkin at 7/9/2021, 2:52:33 PM. +You can check out the [state of the schema](./schema.prisma) after the migration. + +## Database Steps + +```sql +ALTER TABLE "Baseline" ADD COLUMN "userId" TEXT + +ALTER TABLE "Project" ADD COLUMN "maxBranchLifetime" INTEGER NOT NULL DEFAULT 30 + +ALTER TABLE "Baseline" ADD FOREIGN KEY("userId")REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE +``` + +## Changes + +```diff +diff --git schema.prisma schema.prisma +migration 20210705154453-baseline-author..20210709115233-gh-275-max-branch-lifetime +--- datamodel.dml ++++ datamodel.dml +@@ -3,9 +3,9 @@ + } + datasource db { + provider = "postgresql" +- url = "***" ++ url = "***" + } + model Build { + id String @id @default(uuid()) +@@ -31,8 +31,9 @@ + mainBranchName String @default("master") + builds Build[] + buildsCounter Int @default(0) + maxBuildAllowed Int @default(100) ++ maxBranchLifetime Int @default(30) + testVariations TestVariation[] + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + // config +@@ -104,24 +105,24 @@ + testRunId String? + testRun TestRun? @relation(fields: [testRunId], references: [id]) + userId String? + user User? @relation(fields: [userId], references: [id]) +- updatedAt DateTime @updatedAt ++ updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + } + model User { +- id String @id @default(uuid()) +- email String @unique ++ id String @id @default(uuid()) ++ email String @unique + password String + firstName String? + lastName String? +- apiKey String @unique +- isActive Boolean @default(true) ++ apiKey String @unique ++ isActive Boolean @default(true) + builds Build[] + baselines Baseline[] +- updatedAt DateTime @updatedAt +- createdAt DateTime @default(now()) ++ updatedAt DateTime @updatedAt ++ createdAt DateTime @default(now()) + } + enum TestStatus { + failed +``` + + diff --git a/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/schema.prisma b/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/schema.prisma new file mode 100644 index 00000000..b19e84ac --- /dev/null +++ b/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/schema.prisma @@ -0,0 +1,140 @@ +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = "***" +} + +model Build { + id String @id @default(uuid()) + ciBuildId String? + number Int? + branchName String? + status String? + testRuns TestRun[] + projectId String + project Project @relation(fields: [projectId], references: [id]) + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + user User? @relation(fields: [userId], references: [id]) + userId String? + isRunning Boolean? + + @@unique([projectId, ciBuildId]) +} + +model Project { + id String @id @default(uuid()) + name String + mainBranchName String @default("master") + builds Build[] + buildsCounter Int @default(0) + maxBuildAllowed Int @default(100) + maxBranchLifetime Int @default(30) + testVariations TestVariation[] + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + // config + autoApproveFeature Boolean @default(false) + imageComparison ImageComparison @default(pixelmatch) + imageComparisonConfig String @default("{ \"threshold\": 0.1, \"ignoreAntialiasing\": true, \"allowDiffDimensions\": false }") + + @@unique([name]) +} + +model TestRun { + id String @id @default(uuid()) + imageName String + diffName String? + diffPercent Float? + diffTollerancePercent Float @default(0) + pixelMisMatchCount Int? + status TestStatus + buildId String + build Build @relation(fields: [buildId], references: [id]) + testVariationId String? + testVariation TestVariation? @relation(fields: [testVariationId], references: [id]) + merge Boolean @default(false) + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + // Test variation data + name String @default("") + browser String? + device String? + os String? + viewport String? + customTags String? @default("") + baselineName String? + comment String? + baseline Baseline? + branchName String @default("master") + baselineBranchName String? + ignoreAreas String @default("[]") + tempIgnoreAreas String @default("[]") +} + +model TestVariation { + id String @id @default(uuid()) + name String + branchName String @default("master") + browser String @default("") + device String @default("") + os String @default("") + viewport String @default("") + customTags String @default("") + baselineName String? + ignoreAreas String @default("[]") + projectId String + project Project @relation(fields: [projectId], references: [id]) + testRuns TestRun[] + baselines Baseline[] + comment String? + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + + @@unique([projectId, name, browser, device, os, viewport, customTags, branchName]) +} + +model Baseline { + id String @id @default(uuid()) + baselineName String + testVariationId String + testVariation TestVariation @relation(fields: [testVariationId], references: [id]) + testRunId String? + testRun TestRun? @relation(fields: [testRunId], references: [id]) + userId String? + user User? @relation(fields: [userId], references: [id]) + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) +} + +model User { + id String @id @default(uuid()) + email String @unique + password String + firstName String? + lastName String? + apiKey String @unique + isActive Boolean @default(true) + builds Build[] + baselines Baseline[] + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) +} + +enum TestStatus { + failed + new + ok + unresolved + approved + autoApproved +} + +enum ImageComparison { + pixelmatch + lookSame + odiff +} diff --git a/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/steps.json b/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/steps.json new file mode 100644 index 00000000..15027e91 --- /dev/null +++ b/prisma/migrations/20210709115233-gh-275-max-branch-lifetime/steps.json @@ -0,0 +1,37 @@ +{ + "version": "0.3.14-fixed", + "steps": [ + { + "tag": "CreateField", + "model": "Project", + "field": "maxBranchLifetime", + "type": "Int", + "arity": "Required" + }, + { + "tag": "CreateDirective", + "location": { + "path": { + "tag": "Field", + "model": "Project", + "field": "maxBranchLifetime" + }, + "directive": "default" + } + }, + { + "tag": "CreateArgument", + "location": { + "tag": "Directive", + "path": { + "tag": "Field", + "model": "Project", + "field": "maxBranchLifetime" + }, + "directive": "default" + }, + "argument": "", + "value": "30" + } + ] +} \ No newline at end of file diff --git a/prisma/migrations/migrate.lock b/prisma/migrations/migrate.lock index b7de43ee..42b0eea1 100644 --- a/prisma/migrations/migrate.lock +++ b/prisma/migrations/migrate.lock @@ -20,4 +20,5 @@ 20210517203552-add-custom-tags 20210605124856-image-compare-config-as-json 20210612140950-limit-build-number -20210705154453-baseline-author \ No newline at end of file +20210705154453-baseline-author +20210709115233-gh-275-max-branch-lifetime \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 5b966c57..00a8d5d3 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -32,6 +32,7 @@ model Project { builds Build[] buildsCounter Int @default(0) maxBuildAllowed Int @default(100) + maxBranchLifetime Int @default(30) testVariations TestVariation[] updatedAt DateTime @updatedAt createdAt DateTime @default(now()) @@ -105,22 +106,22 @@ model Baseline { testRun TestRun? @relation(fields: [testRunId], references: [id]) userId String? user User? @relation(fields: [userId], references: [id]) - updatedAt DateTime @updatedAt + updatedAt DateTime @updatedAt createdAt DateTime @default(now()) } model User { - id String @id @default(uuid()) - email String @unique + id String @id @default(uuid()) + email String @unique password String firstName String? lastName String? - apiKey String @unique - isActive Boolean @default(true) + apiKey String @unique + isActive Boolean @default(true) builds Build[] baselines Baseline[] - updatedAt DateTime @updatedAt - createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) } enum TestStatus { diff --git a/src/_data_/index.ts b/src/_data_/index.ts index a64f615d..af90a3cd 100644 --- a/src/_data_/index.ts +++ b/src/_data_/index.ts @@ -6,6 +6,7 @@ export const TEST_PROJECT: Project = { name: 'Test Project', buildsCounter: 2, maxBuildAllowed: 100, + maxBranchLifetime: 30, mainBranchName: 'master', createdAt: new Date(), updatedAt: new Date(), diff --git a/src/app.module.ts b/src/app.module.ts index 90ac0690..0aff8613 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -10,11 +10,13 @@ import { ConfigModule } from '@nestjs/config'; import { APP_FILTER, APP_INTERCEPTOR } from '@nestjs/core'; import { HttpExceptionFilter } from './http-exception.filter'; import { CompareModule } from './compare/compare.module'; +import { ScheduleModule } from '@nestjs/schedule'; @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true }), CacheModule.register(), + ScheduleModule.forRoot(), AuthModule, UsersModule, BuildsModule, @@ -32,7 +34,7 @@ import { CompareModule } from './compare/compare.module'; { provide: APP_INTERCEPTOR, useClass: CacheInterceptor, - } + }, ], }) export class AppModule {} diff --git a/src/projects/dto/project.dto.ts b/src/projects/dto/project.dto.ts index bc5aaecd..304d7006 100644 --- a/src/projects/dto/project.dto.ts +++ b/src/projects/dto/project.dto.ts @@ -36,7 +36,11 @@ export class ProjectDto implements Project { @ApiProperty() @IsNumber() maxBuildAllowed: number; - + + @ApiProperty() + @IsNumber() + maxBranchLifetime: number; + @ApiProperty() @IsJSON() imageComparisonConfig: string; diff --git a/src/projects/projects.service.ts b/src/projects/projects.service.ts index 66c737cb..cb6b8b4e 100644 --- a/src/projects/projects.service.ts +++ b/src/projects/projects.service.ts @@ -57,6 +57,7 @@ export class ProjectsService { autoApproveFeature: projectDto.autoApproveFeature, imageComparison: projectDto.imageComparison, maxBuildAllowed: projectDto.maxBuildAllowed, + maxBranchLifetime: projectDto.maxBranchLifetime, imageComparisonConfig: projectDto.imageComparisonConfig, }, }); diff --git a/src/shared/shared.module.ts b/src/shared/shared.module.ts index ab7e0b38..3a3cfb93 100644 --- a/src/shared/shared.module.ts +++ b/src/shared/shared.module.ts @@ -1,13 +1,15 @@ -import { Global, Module } from '@nestjs/common'; +import { forwardRef, Global, Module } from '@nestjs/common'; import { StaticService } from './static/static.service'; import { EventsGateway } from '../shared/events/events.gateway'; import { PrismaService } from '../prisma/prisma.service'; +import { TasksService } from './tasks/tasks.service'; +import { TestVariationsModule } from 'src/test-variations/test-variations.module'; @Global() @Module({ - providers: [StaticService, EventsGateway, PrismaService], + providers: [StaticService, EventsGateway, PrismaService, TasksService], exports: [StaticService, EventsGateway, PrismaService], - imports: [], + imports: [forwardRef(() => TestVariationsModule)], controllers: [], }) export class SharedModule {} diff --git a/src/shared/tasks/tasks.service.ts b/src/shared/tasks/tasks.service.ts new file mode 100644 index 00000000..0251f88c --- /dev/null +++ b/src/shared/tasks/tasks.service.ts @@ -0,0 +1,38 @@ +import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { PrismaService } from 'src/prisma/prisma.service'; +import { TestVariationsService } from 'src/test-variations/test-variations.service'; + +@Injectable() +export class TasksService { + private readonly logger = new Logger(TasksService.name); + + constructor( + private prismaService: PrismaService, + @Inject(forwardRef(() => TestVariationsService)) + private testVariationService: TestVariationsService + ) {} + + @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) + async cleanOldTestVariations() { + const projects = await this.prismaService.project.findMany(); + + for (const project of projects) { + const dateRemoveAfter: Date = new Date(); + dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); + + const testVariations = await this.prismaService.testVariation.findMany({ + where: { + updatedAt: { lte: dateRemoveAfter }, + }, + }); + this.logger.debug( + `Removing ${testVariations.length} TestVariations for ${project.name} later than ${dateRemoveAfter}` + ); + + for (const testVariation of testVariations) { + await this.testVariationService.delete(testVariation.id); + } + } + } +} diff --git a/src/test-variations/test-variations.service.ts b/src/test-variations/test-variations.service.ts index 3fc4fb52..206d1247 100644 --- a/src/test-variations/test-variations.service.ts +++ b/src/test-variations/test-variations.service.ts @@ -1,8 +1,7 @@ -import { Injectable, Inject, forwardRef } from '@nestjs/common'; +import { Injectable, Inject, forwardRef, Logger } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; import { TestVariation, Baseline, Project, Prisma, Build, TestRun } from '@prisma/client'; import { StaticService } from '../shared/static/static.service'; -import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; import { BuildsService } from '../builds/builds.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { PNG } from 'pngjs'; @@ -13,6 +12,8 @@ import { TestVariationUpdateDto } from './dto/test-variation-update.dto'; @Injectable() export class TestVariationsService { + private readonly logger = new Logger(TestVariationsService.name); + constructor( private prismaService: PrismaService, private staticService: StaticService, @@ -219,6 +220,7 @@ export class TestVariationsService { } async delete(id: string): Promise { + this.logger.debug(`Going to remove TestVariation ${id}`); const testVariation = await this.getDetails(id); // delete Baselines From b0dbd4c85a3eb974c43db2dc7bf7530885765662 Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Fri, 9 Jul 2021 15:55:45 +0300 Subject: [PATCH 2/3] Update tasks.service.ts --- src/shared/tasks/tasks.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/tasks/tasks.service.ts b/src/shared/tasks/tasks.service.ts index 0251f88c..fbdd7eb2 100644 --- a/src/shared/tasks/tasks.service.ts +++ b/src/shared/tasks/tasks.service.ts @@ -1,7 +1,7 @@ import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; -import { PrismaService } from 'src/prisma/prisma.service'; -import { TestVariationsService } from 'src/test-variations/test-variations.service'; +import { PrismaService } from '../../prisma/prisma.service'; +import { TestVariationsService } from '../../test-variations/test-variations.service'; @Injectable() export class TasksService { From 5642dbb7a5d421a31bef09f8714f0e86c66475f3 Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Fri, 9 Jul 2021 16:05:53 +0300 Subject: [PATCH 3/3] Update shared.module.ts --- src/shared/shared.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared/shared.module.ts b/src/shared/shared.module.ts index 3a3cfb93..6a82bd87 100644 --- a/src/shared/shared.module.ts +++ b/src/shared/shared.module.ts @@ -3,7 +3,7 @@ import { StaticService } from './static/static.service'; import { EventsGateway } from '../shared/events/events.gateway'; import { PrismaService } from '../prisma/prisma.service'; import { TasksService } from './tasks/tasks.service'; -import { TestVariationsModule } from 'src/test-variations/test-variations.module'; +import { TestVariationsModule } from '../test-variations/test-variations.module'; @Global() @Module({