diff --git a/generated/schema.graphql b/generated/schema.graphql index ea07224f..a72bbfab 100644 --- a/generated/schema.graphql +++ b/generated/schema.graphql @@ -8192,9 +8192,19 @@ input match_lineups_variance_order_by { columns and relationships of "match_map_demos" """ type match_map_demos { + """ + A computed field, executes function "demo_download_url" + """ + download_url: String file: String! id: uuid! + + """An object relationship""" + match: matches! match_id: uuid! + + """An object relationship""" + match_map: match_maps! match_map_id: uuid! size: Int! } @@ -8281,9 +8291,12 @@ input match_map_demos_bool_exp { _and: [match_map_demos_bool_exp!] _not: match_map_demos_bool_exp _or: [match_map_demos_bool_exp!] + download_url: String_comparison_exp file: String_comparison_exp id: uuid_comparison_exp + match: matches_bool_exp match_id: uuid_comparison_exp + match_map: match_maps_bool_exp match_map_id: uuid_comparison_exp size: Int_comparison_exp } @@ -8311,13 +8324,19 @@ input type for inserting data into table "match_map_demos" input match_map_demos_insert_input { file: String id: uuid + match: matches_obj_rel_insert_input match_id: uuid + match_map: match_maps_obj_rel_insert_input match_map_id: uuid size: Int } """aggregate max on columns""" type match_map_demos_max_fields { + """ + A computed field, executes function "demo_download_url" + """ + download_url: String file: String id: uuid match_id: uuid @@ -8338,6 +8357,10 @@ input match_map_demos_max_order_by { """aggregate min on columns""" type match_map_demos_min_fields { + """ + A computed field, executes function "demo_download_url" + """ + download_url: String file: String id: uuid match_id: uuid @@ -8378,9 +8401,12 @@ input match_map_demos_on_conflict { """Ordering options when selecting data from "match_map_demos".""" input match_map_demos_order_by { + download_url: order_by file: order_by id: order_by + match: matches_order_by match_id: order_by + match_map: match_maps_order_by match_map_id: order_by size: order_by } @@ -9553,7 +9579,7 @@ type match_maps { ): match_map_demos_aggregate! """ - A computed field, executes function "demo_download_url" + A computed field, executes function "match_map_demo_download_url" """ demos_download_url: String @@ -10127,7 +10153,7 @@ type match_maps_max_fields { created_at: timestamptz """ - A computed field, executes function "demo_download_url" + A computed field, executes function "match_map_demo_download_url" """ demos_download_url: String @@ -10177,7 +10203,7 @@ type match_maps_min_fields { created_at: timestamptz """ - A computed field, executes function "demo_download_url" + A computed field, executes function "match_map_demo_download_url" """ demos_download_url: String diff --git a/generated/schema.ts b/generated/schema.ts index 036e78f9..a30662af 100644 --- a/generated/schema.ts +++ b/generated/schema.ts @@ -2691,9 +2691,15 @@ export interface match_lineups_variance_fields { /** columns and relationships of "match_map_demos" */ export interface match_map_demos { + /** A computed field, executes function "demo_download_url" */ + download_url: (Scalars['String'] | null) file: Scalars['String'] id: Scalars['uuid'] + /** An object relationship */ + match: matches match_id: Scalars['uuid'] + /** An object relationship */ + match_map: match_maps match_map_id: Scalars['uuid'] size: Scalars['Int'] __typename: 'match_map_demos' @@ -2738,6 +2744,8 @@ export type match_map_demos_constraint = 'match_demos_pkey' /** aggregate max on columns */ export interface match_map_demos_max_fields { + /** A computed field, executes function "demo_download_url" */ + download_url: (Scalars['String'] | null) file: (Scalars['String'] | null) id: (Scalars['uuid'] | null) match_id: (Scalars['uuid'] | null) @@ -2749,6 +2757,8 @@ export interface match_map_demos_max_fields { /** aggregate min on columns */ export interface match_map_demos_min_fields { + /** A computed field, executes function "demo_download_url" */ + download_url: (Scalars['String'] | null) file: (Scalars['String'] | null) id: (Scalars['uuid'] | null) match_id: (Scalars['uuid'] | null) @@ -3127,7 +3137,7 @@ export interface match_maps { demos: match_map_demos[] /** An aggregate relationship */ demos_aggregate: match_map_demos_aggregate - /** A computed field, executes function "demo_download_url" */ + /** A computed field, executes function "match_map_demo_download_url" */ demos_download_url: (Scalars['String'] | null) /** A computed field, executes function "match_map_demo_total_size" */ demos_total_size: (Scalars['Int'] | null) @@ -3242,7 +3252,7 @@ export type match_maps_constraint = 'match_maps_match_id_order_key' | 'match_map /** aggregate max on columns */ export interface match_maps_max_fields { created_at: (Scalars['timestamptz'] | null) - /** A computed field, executes function "demo_download_url" */ + /** A computed field, executes function "match_map_demo_download_url" */ demos_download_url: (Scalars['String'] | null) /** A computed field, executes function "match_map_demo_total_size" */ demos_total_size: (Scalars['Int'] | null) @@ -3266,7 +3276,7 @@ export interface match_maps_max_fields { /** aggregate min on columns */ export interface match_maps_min_fields { created_at: (Scalars['timestamptz'] | null) - /** A computed field, executes function "demo_download_url" */ + /** A computed field, executes function "match_map_demo_download_url" */ demos_download_url: (Scalars['String'] | null) /** A computed field, executes function "match_map_demo_total_size" */ demos_total_size: (Scalars['Int'] | null) @@ -16252,9 +16262,15 @@ export interface match_lineups_variance_order_by {coach_steam_id?: (order_by | n /** columns and relationships of "match_map_demos" */ export interface match_map_demosGenqlSelection{ + /** A computed field, executes function "demo_download_url" */ + download_url?: boolean | number file?: boolean | number id?: boolean | number + /** An object relationship */ + match?: matchesGenqlSelection match_id?: boolean | number + /** An object relationship */ + match_map?: match_mapsGenqlSelection match_map_id?: boolean | number size?: boolean | number __typename?: boolean | number @@ -16316,7 +16332,7 @@ export interface match_map_demos_avg_order_by {size?: (order_by | null)} /** Boolean expression to filter rows from the table "match_map_demos". All fields are combined with a logical 'AND'. */ -export interface match_map_demos_bool_exp {_and?: (match_map_demos_bool_exp[] | null),_not?: (match_map_demos_bool_exp | null),_or?: (match_map_demos_bool_exp[] | null),file?: (String_comparison_exp | null),id?: (uuid_comparison_exp | null),match_id?: (uuid_comparison_exp | null),match_map_id?: (uuid_comparison_exp | null),size?: (Int_comparison_exp | null)} +export interface match_map_demos_bool_exp {_and?: (match_map_demos_bool_exp[] | null),_not?: (match_map_demos_bool_exp | null),_or?: (match_map_demos_bool_exp[] | null),download_url?: (String_comparison_exp | null),file?: (String_comparison_exp | null),id?: (uuid_comparison_exp | null),match?: (matches_bool_exp | null),match_id?: (uuid_comparison_exp | null),match_map?: (match_maps_bool_exp | null),match_map_id?: (uuid_comparison_exp | null),size?: (Int_comparison_exp | null)} /** input type for incrementing numeric columns in table "match_map_demos" */ @@ -16324,11 +16340,13 @@ export interface match_map_demos_inc_input {size?: (Scalars['Int'] | null)} /** input type for inserting data into table "match_map_demos" */ -export interface match_map_demos_insert_input {file?: (Scalars['String'] | null),id?: (Scalars['uuid'] | null),match_id?: (Scalars['uuid'] | null),match_map_id?: (Scalars['uuid'] | null),size?: (Scalars['Int'] | null)} +export interface match_map_demos_insert_input {file?: (Scalars['String'] | null),id?: (Scalars['uuid'] | null),match?: (matches_obj_rel_insert_input | null),match_id?: (Scalars['uuid'] | null),match_map?: (match_maps_obj_rel_insert_input | null),match_map_id?: (Scalars['uuid'] | null),size?: (Scalars['Int'] | null)} /** aggregate max on columns */ export interface match_map_demos_max_fieldsGenqlSelection{ + /** A computed field, executes function "demo_download_url" */ + download_url?: boolean | number file?: boolean | number id?: boolean | number match_id?: boolean | number @@ -16345,6 +16363,8 @@ export interface match_map_demos_max_order_by {file?: (order_by | null),id?: (or /** aggregate min on columns */ export interface match_map_demos_min_fieldsGenqlSelection{ + /** A computed field, executes function "demo_download_url" */ + download_url?: boolean | number file?: boolean | number id?: boolean | number match_id?: boolean | number @@ -16375,7 +16395,7 @@ export interface match_map_demos_on_conflict {constraint: match_map_demos_constr /** Ordering options when selecting data from "match_map_demos". */ -export interface match_map_demos_order_by {file?: (order_by | null),id?: (order_by | null),match_id?: (order_by | null),match_map_id?: (order_by | null),size?: (order_by | null)} +export interface match_map_demos_order_by {download_url?: (order_by | null),file?: (order_by | null),id?: (order_by | null),match?: (matches_order_by | null),match_id?: (order_by | null),match_map?: (match_maps_order_by | null),match_map_id?: (order_by | null),size?: (order_by | null)} /** primary key columns input for table: match_map_demos */ @@ -16994,7 +17014,7 @@ export interface match_mapsGenqlSelection{ order_by?: (match_map_demos_order_by[] | null), /** filter the rows returned */ where?: (match_map_demos_bool_exp | null)} }) - /** A computed field, executes function "demo_download_url" */ + /** A computed field, executes function "match_map_demo_download_url" */ demos_download_url?: boolean | number /** A computed field, executes function "match_map_demo_total_size" */ demos_total_size?: boolean | number @@ -17319,7 +17339,7 @@ export interface match_maps_insert_input {created_at?: (Scalars['timestamptz'] | /** aggregate max on columns */ export interface match_maps_max_fieldsGenqlSelection{ created_at?: boolean | number - /** A computed field, executes function "demo_download_url" */ + /** A computed field, executes function "match_map_demo_download_url" */ demos_download_url?: boolean | number /** A computed field, executes function "match_map_demo_total_size" */ demos_total_size?: boolean | number @@ -17348,7 +17368,7 @@ export interface match_maps_max_order_by {created_at?: (order_by | null),ended_a /** aggregate min on columns */ export interface match_maps_min_fieldsGenqlSelection{ created_at?: boolean | number - /** A computed field, executes function "demo_download_url" */ + /** A computed field, executes function "match_map_demo_download_url" */ demos_download_url?: boolean | number /** A computed field, executes function "match_map_demo_total_size" */ demos_total_size?: boolean | number diff --git a/generated/types.ts b/generated/types.ts index 02831ca3..a9fd0289 100644 --- a/generated/types.ts +++ b/generated/types.ts @@ -11043,15 +11043,24 @@ export default { ] }, "match_map_demos": { + "download_url": [ + 11 + ], "file": [ 11 ], "id": [ 2033 ], + "match": [ + 995 + ], "match_id": [ 2033 ], + "match_map": [ + 877 + ], "match_map_id": [ 2033 ], @@ -11220,15 +11229,24 @@ export default { "_or": [ 780 ], + "download_url": [ + 13 + ], "file": [ 13 ], "id": [ 2034 ], + "match": [ + 1004 + ], "match_id": [ 2034 ], + "match_map": [ + 886 + ], "match_map_id": [ 2034 ], @@ -11255,9 +11273,15 @@ export default { "id": [ 2033 ], + "match": [ + 1013 + ], "match_id": [ 2033 ], + "match_map": [ + 895 + ], "match_map_id": [ 2033 ], @@ -11269,6 +11293,9 @@ export default { ] }, "match_map_demos_max_fields": { + "download_url": [ + 11 + ], "file": [ 11 ], @@ -11309,6 +11336,9 @@ export default { ] }, "match_map_demos_min_fields": { + "download_url": [ + 11 + ], "file": [ 11 ], @@ -11374,15 +11404,24 @@ export default { ] }, "match_map_demos_order_by": { + "download_url": [ + 1148 + ], "file": [ 1148 ], "id": [ 1148 ], + "match": [ + 1015 + ], "match_id": [ 1148 ], + "match_map": [ + 897 + ], "match_map_id": [ 1148 ], diff --git a/hasura/metadata/databases/default/tables/public_match_map_demos.yaml b/hasura/metadata/databases/default/tables/public_match_map_demos.yaml index ac3b8af8..5075feaa 100644 --- a/hasura/metadata/databases/default/tables/public_match_map_demos.yaml +++ b/hasura/metadata/databases/default/tables/public_match_map_demos.yaml @@ -1,6 +1,13 @@ table: name: match_map_demos schema: public +object_relationships: + - name: match + using: + foreign_key_constraint_on: match_id + - name: match_map + using: + foreign_key_constraint_on: match_map_id computed_fields: - name: download_url definition: diff --git a/src/matches/enums/MatchQueues.ts b/src/matches/enums/MatchQueues.ts index b437b297..488f7023 100644 --- a/src/matches/enums/MatchQueues.ts +++ b/src/matches/enums/MatchQueues.ts @@ -1,4 +1,5 @@ export enum MatchQueues { + CleanDemos = "clean-demos", MatchServers = "match-servers", ScheduledMatches = "scheduled-matches", EloCalculation = "elo-calculation", diff --git a/src/matches/jobs/CleanDemos.ts b/src/matches/jobs/CleanDemos.ts new file mode 100644 index 00000000..66e80454 --- /dev/null +++ b/src/matches/jobs/CleanDemos.ts @@ -0,0 +1,166 @@ +import { Job } from "bullmq"; +import { Logger } from "@nestjs/common"; +import { WorkerHost } from "@nestjs/bullmq"; +import { MatchQueues } from "../enums/MatchQueues"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { HasuraService } from "../../hasura/hasura.service"; +import { S3Service } from "../../s3/s3.service"; + +@UseQueue("Matches", MatchQueues.CleanDemos) +export class CleanDemos extends WorkerHost { + private maxStorageInBytes: number; + private minRetentionInDays: number; + private totalStoredBytes: number; + + constructor( + private readonly s3: S3Service, + private readonly logger: Logger, + private readonly hasura: HasuraService, + ) { + super(); + } + + async process(job: Job): Promise { + const { settings } = await this.hasura.query({ + settings: { + __args: { + where: { + _or: [ + { + name: { + _eq: "s3_min_retention", + }, + }, + { + name: { + _eq: "s3_max_storage", + }, + }, + ], + }, + }, + name: true, + value: true, + }, + }); + + this.minRetentionInDays = parseInt( + settings.find(function (setting) { + return setting.name === "s3_min_retention"; + })?.value || "1", + ); + + const maxStorageInGB = parseInt( + settings.find(function (setting) { + return setting.name === "s3_max_storage"; + })?.value || "10", + ); + + this.maxStorageInBytes = maxStorageInGB * 1024 * 1024 * 1024; + + const { match_map_demos_aggregate } = await this.hasura.query({ + match_map_demos_aggregate: { + aggregate: { + sum: { + size: true, + }, + }, + }, + }); + + this.totalStoredBytes = match_map_demos_aggregate.aggregate.sum.size; + + return await this.deleteOldDemos(); + } + + private async deleteOldDemos() { + if (this.totalStoredBytes < this.maxStorageInBytes) { + return 0; + } + + const demosToDelete = await this.findDemosToDelete(); + + if (demosToDelete.length > 0) { + this.logger.log( + `Marked ${demosToDelete.length} demos for deletion, freeing up ${this.formatStorageSize(demosToDelete.reduce((total, demo) => total + demo.size, 0))}`, + ); + for (const demo of demosToDelete) { + await this.s3.remove(demo.file); + await this.hasura.mutation({ + delete_match_map_demos_by_pk: { + __args: { id: demo.id }, + __typename: true, + }, + }); + break; + } + } + + return demosToDelete.length; + } + + private async findDemosToDelete(): Promise< + Array<{ + id: string; + size: number; + file: string; + }> + > { + const finishedAfter = new Date( + Date.now() - this.minRetentionInDays * 24 * 60 * 60 * 1000, + ).toISOString(); + + const { match_map_demos } = await this.hasura.query({ + match_map_demos: { + __args: { + limit: 10, + where: { + match: { + ended_at: { + _lt: finishedAfter, + }, + }, + }, + order_by: [ + { + match: { + ended_at: "asc", + }, + }, + ], + }, + id: true, + size: true, + file: true, + }, + }); + + const demosToDelete = []; + + for (const demo of match_map_demos) { + demosToDelete.push(demo); + this.totalStoredBytes -= demo.size; + + if (this.totalStoredBytes > this.maxStorageInBytes) { + break; + } + } + + if (this.totalStoredBytes > this.maxStorageInBytes) { + const moreDemosToDelete = await this.findDemosToDelete(); + demosToDelete.push(...moreDemosToDelete); + } + + return demosToDelete; + } + + private formatStorageSize(bytes: number): string { + const megabytes = bytes / (1024 * 1024); + const gigabytes = megabytes / 1024; + + if (gigabytes >= 1) { + return `${gigabytes.toFixed(2)} GB`; + } + return `${megabytes.toFixed(2)} MB`; + } +} diff --git a/src/matches/matches.module.ts b/src/matches/matches.module.ts index 2cf41e0d..4e45d8e2 100644 --- a/src/matches/matches.module.ts +++ b/src/matches/matches.module.ts @@ -44,6 +44,7 @@ import { ChatModule } from "src/chat/chat.module"; import { HasuraService } from "src/hasura/hasura.service"; import { EloCalculation } from "./jobs/EloCalculation"; import { PostgresService } from "src/postgres/postgres.service"; +import { CleanDemos } from "./jobs/CleanDemos"; @Module({ imports: [ @@ -66,6 +67,12 @@ import { PostgresService } from "src/postgres/postgres.service"; { name: MatchQueues.ScheduledMatches, }, + { + name: MatchQueues.CleanDemos, + }, + { + name: MatchQueues.EloCalculation, + }, ), BullBoardModule.forFeature( { @@ -76,14 +83,15 @@ import { PostgresService } from "src/postgres/postgres.service"; name: MatchQueues.ScheduledMatches, adapter: BullMQAdapter, }, + { + name: MatchQueues.CleanDemos, + adapter: BullMQAdapter, + }, + { + name: MatchQueues.EloCalculation, + adapter: BullMQAdapter, + }, ), - BullModule.registerQueue({ - name: MatchQueues.EloCalculation, - }), - BullBoardModule.forFeature({ - name: MatchQueues.EloCalculation, - adapter: BullMQAdapter, - }), ], controllers: [MatchesController, DemosController, BackupRoundsController], exports: [MatchAssistantService], @@ -98,6 +106,7 @@ import { PostgresService } from "src/postgres/postgres.service"; RemoveCancelledMatches, CancelInvalidTournaments, CleanAbandonedMatches, + CleanDemos, EloCalculation, ...getQueuesProcessors("Matches"), ...Object.values(MatchEvents), @@ -107,6 +116,7 @@ import { PostgresService } from "src/postgres/postgres.service"; export class MatchesModule implements NestModule { constructor( private readonly hasuraService: HasuraService, + @InjectQueue(MatchQueues.CleanDemos) cleanDemosQueue: Queue, @InjectQueue(MatchQueues.MatchServers) matchServersQueue: Queue, @InjectQueue(MatchQueues.ScheduledMatches) scheduleMatchQueue: Queue, private readonly postgres: PostgresService, @@ -175,6 +185,16 @@ export class MatchesModule implements NestModule { }, ); + void cleanDemosQueue.add( + CleanDemos.name, + {}, + { + repeat: { + pattern: "0 * * * *", + }, + }, + ); + void this.generatePlayerRatings(); }