Make player props their own separate entity

src/entities/event.ts

@@ -41,7 +41,7 @@ export default class Event {
 
     this.props.forEach((prop) => {
       if (eventMetaProps.includes(prop.key)) {
-        this.playerAlias.player.props.push(prop)
+        this.playerAlias.player.addProp(prop.key, prop.value)
       }
     })
   }

src/entities/index.ts

@@ -26,8 +26,10 @@ import OrganisationPricingPlanAction from './organisation-pricing-plan-action'
 import Integration from './integration'
 import SteamworksIntegrationEvent from './steamworks-integration-event'
 import SteamworksLeaderboardMapping from './steamworks-leaderboard-mapping'
+import PlayerProp from './player-prop'
 
 export default [
+  PlayerProp,
   SteamworksLeaderboardMapping,
   SteamworksIntegrationEvent,
   Integration,

src/entities/player-prop.ts

@@ -0,0 +1,23 @@
+import { Entity, ManyToOne, PrimaryKey, Property } from '@mikro-orm/core'
+import Player from './player'
+
+@Entity()
+export default class PlayerProp {
+  @PrimaryKey()
+  id: number
+
+  @ManyToOne(() => Player)
+  player: Player
+
+  @Property()
+  key: string
+
+  @Property()
+  value: string
+
+  constructor(player: Player, key: string, value: string) {
+    this.player = player
+    this.key = key
+    this.value = value
+  }
+}

src/entities/player.ts

@@ -1,8 +1,8 @@
-import { Collection, Embedded, Entity, ManyToOne, OneToMany, PrimaryKey, Property } from '@mikro-orm/core'
+import { Collection, Entity, ManyToOne, OneToMany, PrimaryKey, Property } from '@mikro-orm/core'
 import Game from './game'
 import { v4 } from 'uuid'
 import PlayerAlias from './player-alias'
-import Prop from './prop'
+import PlayerProp from './player-prop'
 
 @Entity()
 export default class Player {
@@ -12,8 +12,8 @@ export default class Player {
   @OneToMany(() => PlayerAlias, (alias) => alias.player)
   aliases: Collection<PlayerAlias> = new Collection<PlayerAlias>(this)
 
-  @Embedded(() => Prop, { array: true })
-  props: Prop[] = []
+  @OneToMany(() => PlayerProp, (prop) => prop.player, { eager: true, orphanRemoval: true })
+  props: Collection<PlayerProp> = new Collection<PlayerProp>(this)
 
   @ManyToOne(() => Game)
   game: Game
@@ -32,13 +32,21 @@ export default class Player {
   }
 
   isDevBuild() {
-    return this.props.some((prop) => prop.key === 'META_DEV_BUILD')
+    return this.props.getItems().some((prop) => prop.key === 'META_DEV_BUILD')
+  }
+
+  addProp(key: string, value: string) {
+    this.props.add(new PlayerProp(this, key, value))
+  }
+
+  setProps(props: { key: string, value: string }[]) {
+    this.props.set(props.map(({ key, value }) => new PlayerProp(this, key, value)))
   }
 
   toJSON() {
     return {
       id: this.id,
-      props: this.props,
+      props: this.props.getItems().map(({ key, value }) => ({ key, value })),
       aliases: this.aliases,
       devBuild: this.isDevBuild(),
       createdAt: this.createdAt,

src/lib/integrations/steamworks-integration.ts

@@ -8,7 +8,6 @@ import LeaderboardEntry from '../../entities/leaderboard-entry'
 import SteamworksLeaderboardMapping from '../../entities/steamworks-leaderboard-mapping'
 import PlayerAlias, { PlayerAliasService } from '../../entities/player-alias'
 import Player from '../../entities/player'
-import Prop from '../../entities/prop'
 import { performance } from 'perf_hooks'
 import GameStat from '../../entities/game-stat'
 import PlayerGameStat from '../../entities/player-game-stat'
@@ -338,7 +337,7 @@ export async function syncSteamworksLeaderboards(em: EntityManager, integration:
         // if the alias doesnt exist then neither does the entry, so create both
       } else {
         const player = new Player(integration.game)
-        player.props.push(new Prop('importedFromSteam', new Date().toISOString()))
+        player.addProp('importedFromSteam', new Date().toISOString())
 
         const playerAlias = new PlayerAlias()
         playerAlias.player = player

src/middlewares/dev-data-middleware.ts

@@ -1,6 +1,8 @@
-import { expr, ObjectQuery } from '@mikro-orm/core'
+import { QBFilterQuery } from '@mikro-orm/core'
+import { EntityManager } from '@mikro-orm/mysql'
 import { Context, Next } from 'koa'
 import Player from '../entities/player'
+import PlayerProp from '../entities/player-prop'
 
 export default async (ctx: Context, next: Next): Promise<void> => {
   if (Number(ctx.headers['x-talo-include-dev-data'])) {
@@ -10,6 +12,10 @@ export default async (ctx: Context, next: Next): Promise<void> => {
   await next()
 }
 
-export const devDataPlayerFilter: ObjectQuery<Player> = {
-  [expr((alias) => `json_search(${alias}.props, 'one', 'META_DEV_BUILD', null, '$[*].key')`)]: null
+export function devDataPlayerFilter(em: EntityManager): QBFilterQuery<Player> {
+  return {
+    $nin: em.qb(PlayerProp).select('player_id', true).where({
+      key: 'META_DEV_BUILD'
+    }).getKnexQuery()
+  }
 }

src/migrations/.snapshot-gs_dev.json

@@ -878,15 +878,6 @@
           "nullable": false,
           "mappedType": "string"
         },
-        "props": {
-          "name": "props",
-          "type": "json",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "json"
-        },
         "game_id": {
           "name": "game_id",
           "type": "int",
@@ -2823,6 +2814,82 @@
           "deleteRule": "cascade"
         }
       }
+    },
+    {
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "int",
+          "unsigned": true,
+          "autoincrement": true,
+          "primary": true,
+          "nullable": false,
+          "mappedType": "integer"
+        },
+        "player_id": {
+          "name": "player_id",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "string"
+        },
+        "key": {
+          "name": "key",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "string"
+        },
+        "value": {
+          "name": "value",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "string"
+        }
+      },
+      "name": "player_prop",
+      "indexes": [
+        {
+          "columnNames": [
+            "player_id"
+          ],
+          "composite": false,
+          "keyName": "player_prop_player_id_index",
+          "primary": false,
+          "unique": false
+        },
+        {
+          "keyName": "PRIMARY",
+          "columnNames": [
+            "id"
+          ],
+          "composite": false,
+          "primary": true,
+          "unique": true
+        }
+      ],
+      "checks": [],
+      "foreignKeys": {
+        "player_prop_player_id_foreign": {
+          "constraintName": "player_prop_player_id_foreign",
+          "columnNames": [
+            "player_id"
+          ],
+          "localTableName": "player_prop",
+          "referencedColumnNames": [
+            "id"
+          ],
+          "referencedTableName": "player",
+          "updateRule": "cascade"
+        }
+      }
     }
   ]
 }

src/migrations/20220910200720CreatePlayerPropsTable.ts

@@ -0,0 +1,22 @@
+import { Migration } from '@mikro-orm/migrations'
+
+export class CreatePlayerPropsTable extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('create table `player_prop` (`id` int unsigned not null auto_increment primary key, `player_id` varchar(255) not null, `key` varchar(255) not null, `value` varchar(255) not null) default character set utf8mb4 engine = InnoDB;')
+    this.addSql('alter table `player_prop` add index `player_prop_player_id_index`(`player_id`);')
+
+    this.addSql('alter table `player_prop` add constraint `player_prop_player_id_foreign` foreign key (`player_id`) references `player` (`id`) on update cascade;')
+
+    this.addSql('insert into `player_prop` (`player_id`, `key`, `value`) select `p`.`id`, `r`.* from `player` as `p`, json_table(`p`.`props`, \'$[*]\' columns (`key` varchar(255) path \'$.key\', `value` varchar(255) path \'$.value\')) as `r`')
+
+    this.addSql('alter table `player` drop `props`;')
+  }
+
+  async down(): Promise<void> {
+    this.addSql('drop table if exists `player_prop`;')
+
+    this.addSql('alter table `player` add `props` json not null;')
+  }
+
+}

src/migrations/index.ts

@@ -16,6 +16,7 @@ import { CreatePricingPlansTable } from './20220603123117CreatePricingPlansTable
 import { CreateIntegrationsTable } from './20220717215205CreateIntegrationsTable'
 import { CreateSteamIntegrationTables } from './20220723122554CreateSteamIntegrationTables'
 import { PlayerAliasServiceUseEnum } from './20220730134520PlayerAliasServiceUseEnum'
+import { CreatePlayerPropsTable } from './20220910200720CreatePlayerPropsTable'
 
 export default [
   {
@@ -89,5 +90,9 @@ export default [
   {
     name: 'PlayerAliasServiceUseEnum',
     class: PlayerAliasServiceUseEnum
+  },
+  {
+    name: 'CreatePlayerPropsTable',
+    class: CreatePlayerPropsTable
   }
 ]

src/services/api/player-api.service.ts

@@ -7,6 +7,7 @@ import PlayerAPIPolicy from '../../policies/api/player-api.policy'
 import APIService from './api-service'
 import uniqWith from 'lodash.uniqwith'
 import PlayerAPIDocs from '../../docs/player-api.docs'
+import PlayerProp from '../../entities/player-prop'
 
 @Routes([
   {
@@ -126,12 +127,12 @@ export default class PlayerAPIService extends APIService {
 
     player2Aliases.forEach((alias) => alias.player = player1)
 
-    const mergedProps = uniqWith([
+    const mergedProps: PlayerProp[] = uniqWith([
       ...player2.props,
       ...player1.props
     ], (a, b) => a.key === b.key)
 
-    player1.props = Array.from(mergedProps)
+    player1.props.set(Array.from(mergedProps))
 
     await em.removeAndFlush(player2)
 

src/services/data-export.service.ts

@@ -1,4 +1,4 @@
-import { EntityManager, FilterQuery, MikroORM, ObjectQuery } from '@mikro-orm/core'
+import { Collection, FilterQuery, MikroORM } from '@mikro-orm/core'
 import { HasPermission, Routes, Service, Request, Response, Validate, ValidationCondition } from 'koa-clay'
 import DataExport, { DataExportAvailableEntities, DataExportStatus } from '../entities/data-export'
 import DataExportPolicy from '../policies/data-export.policy'
@@ -23,9 +23,14 @@ import handlePricingPlanAction from '../lib/billing/handlePricingPlanAction'
 import { PricingPlanActionType } from '../entities/pricing-plan-action'
 import queueEmail from '../lib/messaging/queueEmail'
 import OrganisationPricingPlanAction from '../entities/organisation-pricing-plan-action'
+import { EntityManager } from '@mikro-orm/mysql'
+import pick from 'lodash.pick'
+import PlayerProp from '../entities/player-prop'
+
+type PropCollection = Collection<PlayerProp, Player>
 
 interface EntityWithProps {
-  props: Prop[]
+  props: Prop[] | PropCollection
 }
 
 interface UpdatedDataExportStatus {
@@ -76,7 +81,7 @@ export default class DataExportService extends Service {
       const filename = `export-${dataExport.game.id}-${dataExport.createdAt.getTime()}.zip`
       const filepath = './storage/' + filename
 
-      const zip: AdmZip = await this.createZip(dataExport, orm.em, includeDevData)
+      const zip: AdmZip = await this.createZip(dataExport, orm.em as EntityManager, includeDevData)
       zip.writeZip(filepath)
 
       dataExport.status = DataExportStatus.QUEUED
@@ -136,7 +141,7 @@ export default class DataExportService extends Service {
 
     if (!includeDevData) {
       where.playerAlias = {
-        player: devDataPlayerFilter
+        player: devDataPlayerFilter(em)
       }
     }
 
@@ -147,7 +152,7 @@ export default class DataExportService extends Service {
     let where: FilterQuery<Player> = { game: dataExport.game }
 
     if (!includeDevData) {
-      where = { ...where, ...devDataPlayerFilter }
+      where = Object.assign(where, devDataPlayerFilter(em))
     }
 
     return await em.getRepository(Player).find(where)
@@ -159,10 +164,7 @@ export default class DataExportService extends Service {
     }
 
     if (!includeDevData) {
-      where.player = {
-        ...(where.player as ObjectQuery<Player>),
-        ...devDataPlayerFilter
-      }
+      where.player = Object.assign(where.player, devDataPlayerFilter(em))
     }
 
     return await em.getRepository(PlayerAlias).find(where)
@@ -175,7 +177,7 @@ export default class DataExportService extends Service {
 
     if (!includeDevData) {
       where.playerAlias = {
-        player: devDataPlayerFilter
+        player: devDataPlayerFilter(em)
       }
     }
 
@@ -205,7 +207,7 @@ export default class DataExportService extends Service {
     }
 
     if (!includeDevData) {
-      where.player = devDataPlayerFilter
+      where.player = devDataPlayerFilter(em)
     }
 
     return await em.getRepository(PlayerGameStat).find(where)
@@ -279,9 +281,15 @@ export default class DataExportService extends Service {
     }
   }
 
+  private getProps(object: ExportableEntityWithProps): { key: string, value: string }[] {
+    let props = object.props
+    if (props instanceof Collection) props = props.getItems()
+    return props.map((prop) => pick(prop, ['key', 'value']))
+  }
+
   private transformColumn(column: string, object: ExportableEntity): string {
     if (column.startsWith('props')) {
-      const value = (object as ExportableEntityWithProps).props.find((prop) => column.endsWith(prop.key))?.value ?? ''
+      const value = this.getProps(object as ExportableEntityWithProps).find((prop) => column.endsWith(prop.key))?.value ?? ''
       return value
     }
 
@@ -309,7 +317,7 @@ export default class DataExportService extends Service {
     columns = columns.filter((col) => col !== 'props')
 
     const allProps = objects.reduce((acc: string[], curr: EntityWithProps): string[] => {
-      return [...acc, ...curr.props.map((prop) => `props.${prop.key}`)]
+      return [...acc, ...this.getProps(curr as ExportableEntityWithProps).map((prop) => `props.${prop.key}`)]
     }, []).sort((a, b) => a.localeCompare(b))
 
     columns = [...new Set([...columns, ...allProps])]