Dynamic Rarity

.github/workflows/release.yml

@@ -21,3 +21,8 @@ jobs:
 
       - name: Sentry Build
         run: yarn release
+        env:
+          SENTRY_AUTH_TOKEN: ${{secrets.SENTRY_AUTH_TOKEN}}
+          SENTRY_DSN: ${{secrets.SENTRY_DSN}}
+          SENTRY_ORG: ${{secrets.SENTRY_ORG}}
+          SENTRY_PROJECT: ${{secrets.SENTRY_PROJECT}}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "reactmap",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "React based frontend map.",
   "main": "ReactMap.mjs",
   "author": "TurtIeSocks <58572875+TurtIeSocks@users.noreply.github.com>",

public/base-locales/en.json

@@ -37,7 +37,7 @@
   "generations": "Generations",
   "types": "Types",
   "forms": "Forms",
-  "rarity": "Rarity",
+  "rarity": "Current Rarity",
   "others": "Others",
   "categories": "Categories",
   "rank": "Rank",
@@ -547,6 +547,7 @@
   "quest_condition": "Quest Condition",
   "always_show_labels": "Always Show Labels",
   "scan_areas_options": "Scan Areas Options",
+  "historic_rarity": "Historic Rarity",
   "poi": "Points of Interest",
   "300m_range": "300m Range",
   "lure_range": "Lure Range"

server/scripts/generateMasterfile.js

@@ -1,57 +1,79 @@
-/* eslint-disable no-console */
-/* eslint-disable no-restricted-syntax */
 const fs = require('fs')
-const path = require('path')
-const { rarity } = require('../src/services/config')
+const { resolve } = require('path')
+const { rarity: customRarity } = require('../src/services/config')
 const fetchJson = require('../src/services/api/fetchJson')
 const defaultRarity = require('../src/data/defaultRarity.json')
 
-const getRarityLevel = (id, pkmn) => {
-  let pkmnRarity
-  for (const [tier, pokemon] of Object.entries(defaultRarity)) {
-    if (rarity?.[tier]?.length) {
-      if (rarity[tier].includes(parseInt(id))) {
-        pkmnRarity = tier
-      }
-    } else if (pokemon.includes(parseInt(id))) {
-      pkmnRarity = tier
-    }
+const rarityObj = {}
+Object.entries(defaultRarity).forEach(([tier, pokemon]) => {
+  if (customRarity?.[tier]?.length) {
+    customRarity[tier].forEach((mon) => (rarityObj[mon] = tier))
+  } else {
+    pokemon.forEach((mon) => (rarityObj[mon] = tier))
   }
-  if (pkmn.legendary) pkmnRarity = 'legendary'
-  if (pkmn.mythical) pkmnRarity = 'mythical'
-  if (pkmn.ultraBeast) pkmnRarity = 'ultraBeast'
-  return pkmnRarity
-}
+})
 
-const generate = async (save) => {
+const generate = async (
+  save = false,
+  historicRarity = new Map(),
+  dbRarity = new Map(),
+) => {
   try {
     const masterfile = await fetchJson(
       'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/master/master-latest-react-map.json',
     )
 
-    Object.values(masterfile.pokemon).forEach((pokemon) => {
-      pokemon.rarity = getRarityLevel(pokemon.pokedexId, pokemon)
-      pokemon.types = pokemon.types || []
-      delete pokemon.mythical
-      delete pokemon.legendary
-    })
+    const newMf = {
+      ...masterfile,
+      pokemon: Object.fromEntries(
+        Object.values(masterfile.pokemon).map((pokemon) => {
+          const { legendary, mythical, ultraBeast, ...rest } = pokemon
+          const historic = historicRarity.get(pokemon.pokedexId) || 'never'
+
+          let rarity =
+            (dbRarity.size
+              ? dbRarity.get(`${pokemon.pokedexId}-${pokemon.defaultFormId}`)
+              : rarityObj[pokemon.pokedexId]) || 'never'
+          if (legendary) rarity = 'legendary'
+          if (mythical) rarity = 'mythical'
+          if (ultraBeast) rarity = 'ultraBeast'
+          if (rarityObj[pokemon.pokedexId] === 'regional') rarity = 'regional'
+
+          const forms = Object.fromEntries(
+            Object.entries(pokemon.forms || {}).map(([formId, form]) => [
+              formId,
+              {
+                ...form,
+                rarity:
+                  +formId === pokemon.defaultFormId
+                    ? rarity
+                    : dbRarity.get(`${pokemon.pokedexId}-${formId}`) || 'never',
+              },
+            ]),
+          )
+          return [pokemon.pokedexId, { ...rest, forms, rarity, historic }]
+        }),
+      ),
+    }
 
     if (save) {
       fs.writeFileSync(
-        path.resolve(`${__dirname}/../src/data/masterfile.json`),
-        JSON.stringify(masterfile, null, 2),
+        resolve(`${__dirname}/../src/data/masterfile.json`),
+        JSON.stringify(newMf, null, 2),
         'utf8',
         () => {},
       )
     }
-    return masterfile
+    return newMf
   } catch (e) {
+    // eslint-disable-next-line no-console
     console.warn('[WARN] Unable to generate new masterfile, using existing.', e)
   }
 }
 
 module.exports.generate = generate
 
 if (require.main === module) {
+  // eslint-disable-next-line no-console
   generate(true).then(() => console.log('Masterfile generated'))
 }

server/src/configs/custom-environment-variables.json

@@ -1427,6 +1427,20 @@
     }
   },
   "rarity": {
+    "percents": {
+      "uncommon": {
+        "__name": "RARITY_PERCENTS_UNCOMMON",
+        "__format": "number"
+      },
+      "rare": {
+        "__name": "RARITY_PERCENTS_RARE",
+        "__format": "number"
+      },
+      "ultraRare": {
+        "__name": "RARITY_PERCENTS_ULTRA_RARE",
+        "__format": "number"
+      }
+    },
     "common": {
       "__name": "RARITY_COMMON",
       "__format": "json"
@@ -1450,6 +1464,10 @@
     "event": {
       "__name": "RARITY_EVENT",
       "__format": "json"
+    },
+    "never": {
+      "__name": "RARITY_NEVER",
+      "__format": "json"
     }
   },
   "manualAreas": {

server/src/configs/default.json

@@ -704,12 +704,18 @@
     }
   },
   "rarity": {
+    "percents": {
+      "uncommon": 0.25,
+      "rare": 0.1,
+      "ultraRare": 0.01
+    },
     "common": [],
     "uncommon": [],
     "rare": [],
     "ultraRare": [],
     "regional": [],
-    "event": []
+    "event": [],
+    "never": []
   },
   "manualAreas": []
 }

server/src/data/defaultRarity.json

@@ -6,15 +6,16 @@
     2,5,8,11,14,17,20,22,24,26,30,33,36,38,40,42,44,47,49,51,53,55,57,59,61,64,67,70,73,75,78,80,82,85,87,89,91,93,97,99,101,103,105,110,112,117,119,121,124,125,126,127,139,141,153,156,159,162,164,166,168,171,178,180,184,185,188,195,202,203,205,206,210,211,213,217,219,221,224,226,227,229,234,253,256,259,262,264,266,268,271,274,277,279,281,284,286,288,294,297,301,302,305,308,310,317,319,323,326,329,332,340,342,344,346,348,354,356,362,364,388,391,394,397,400,402,419,426,428,432,435,437,450,452,454,457,460,496,499,502,505,507,510,520,522,524,528,536,541,544,547,549,552,558,563,569,573,578,581,583,586,587,591,595,603,606,614,615,620,651,654,657,660,662,668
   ],
   "rare": [
-    3,6,9,12,15,18,28,31,34,45,62,65,68,71,76,94,106,107,108,113,114,130,131,132,134135,136,137,142,143,144,154,157,160,169,176,181,189,232,235,237,241,242,254,257,260,267,269,272,275,282,289,295,306,330,358,365,389,392,395,398,408,409,410,411,413,414,497,500,503,508,518,521,523,525,526,530,531,537,542,545,553,554,555,564,565,566,567,571,574,575,576,579,584,589,593,594,596,597,598,604,617,618,621,622,623,625,630,636,637,652,655,658
+    3,6,9,12,15,18,28,31,34,45,62,65,68,71,76,94,106,107,108,113,114,130,131,132,134,135,136,137,142,143,144,154,157,160,169,176,181,189,232,235,237,241,242,254,257,260,267,269,272,275,282,289,295,306,330,358,365,389,392,395,398,408,409,410,411,413,414,497,500,503,508,518,521,523,525,526,530,531,537,542,545,553,554,555,564,565,566,567,571,574,575,576,579,584,589,593,594,596,597,598,604,617,618,621,622,623,625,630,636,637,652,655,658
   ],
   "ultraRare": [
     147,148,149,201,246,247,248,371,372,373,374,375,376,443,444,445,607,608,609,610,611,612,633,634,635,714,715
   ],
   "regional": [
     83,115,122,128,214,222,313,314,324,335,336,337,338,357,369,417,422,423,439,441,455,480,481,482,511,512,513,514,515,516,538,539,550,556,561,626,631,632,707
   ],
-  "event": [
+  "never": [
     172,173,174,175,182,186,192,196,197,199,208,212,225,230,233,236,238,239,240,290,291,292,298,303,321,327,334,350,352,359,360,367,368,403,404,405,406,407,416,424,429,430,433,438,439,440,442,446,447,448,458,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,532,533,534,559,560,599,600,601,627,628,677,678
-  ]
+  ],
+  "event": []
 }

server/src/index.js

@@ -196,20 +196,24 @@ app.use((err, req, res, next) => {
 
 Db.determineType().then(async () => {
   await Promise.all([
-    Event.getUicons(config.icons.styles),
-    Event.getMasterfile(),
-    Event.getInvasions(),
-    Event.getWebhooks(config),
+    Db.initRarity(),
     Event.setAvailable('gyms', 'Gym', Db),
     Event.setAvailable('pokestops', 'Pokestop', Db),
     Event.setAvailable('pokemon', 'Pokemon', Db),
     Event.setAvailable('nests', 'Nest', Db),
-  ]).then(() => {
-    Event.addAvailable()
-    app.listen(config.port, config.interface, () => {
-      console.log(
-        `[INIT] Server is now listening at http://${config.interface}:${config.port}`,
-      )
+  ]).then(async () => {
+    await Promise.all([
+      Event.getUicons(config.icons.styles),
+      Event.getMasterfile(Db.historical, Db.rarity),
+      Event.getInvasions(),
+      Event.getWebhooks(config),
+    ]).then(() => {
+      Event.addAvailable()
+      app.listen(config.port, config.interface, () => {
+        console.log(
+          `[INIT] Server is now listening at http://${config.interface}:${config.port}`,
+        )
+      })
     })
   })
 })

server/src/models/Pokemon.js

@@ -416,17 +416,32 @@ module.exports = class Pokemon extends Model {
 
   static async getAvailable({ isMad }) {
     const ts = Math.floor(new Date().getTime() / 1000)
-    const results = await this.query()
-      .select('pokemon_id', 'form')
+    const availableQuery = this.query()
+      .select(['pokemon_id', 'form'])
       .where(
         isMad ? 'disappear_time' : 'expire_timestamp',
         '>=',
         isMad ? this.knex().fn.now() : ts,
       )
       .groupBy('pokemon_id', 'form')
       .orderBy('pokemon_id', 'form')
+    const rarityQuery = this.query()
+      .select(['pokemon_id AS id', 'form as formId'])
+      .count('pokemon_id AS count')
+      .groupBy('pokemon_id', 'form')
+      .where(
+        isMad ? 'disappear_time' : 'expire_timestamp',
+        '>=',
+        isMad ? this.knex().fn.now() : ts,
+      )
+
+    const [available, rarity] = await Promise.all([availableQuery, rarityQuery])
+
     return {
-      available: results.map((pkmn) => `${pkmn.pokemon_id}-${pkmn.form}`),
+      available: available.map((pkmn) => `${pkmn.pokemon_id}-${pkmn.form}`),
+      rarity: Object.fromEntries(
+        rarity.map((pkmn) => [`${pkmn.id}-${pkmn.formId}`, pkmn.count]),
+      ),
     }
   }
 

server/src/services/DbCheck.js

@@ -4,12 +4,22 @@ const { raw } = require('objection')
 const extend = require('extend')
 
 module.exports = class DbCheck {
-  constructor(validModels, dbSettings, queryDebug, apiSettings, distanceUnit) {
+  constructor(
+    validModels,
+    dbSettings,
+    queryDebug,
+    apiSettings,
+    distanceUnit,
+    rarityPercents,
+  ) {
     this.validModels = validModels.flatMap((s) => s.useFor)
     this.singleModels = ['User', 'Badge', 'Session']
     this.searchLimit = apiSettings.searchLimit
+    this.rarityPercents = rarityPercents
     this.models = {}
     this.questConditions = {}
+    this.rarity = new Map()
+    this.historical = new Map()
     this.connections = dbSettings.schemas
       .filter((s) => s.useFor.length)
       .map((schema, i) => {
@@ -108,6 +118,48 @@ module.exports = class DbCheck {
     )
   }
 
+  setRarity(id, percent, historical = false) {
+    if (percent === 0) {
+      this[historical ? 'historical' : 'rarity'].set(id, 'never')
+    } else if (percent < this.rarityPercents.ultraRare) {
+      this[historical ? 'historical' : 'rarity'].set(id, 'ultraRare')
+    } else if (percent < this.rarityPercents.rare) {
+      this[historical ? 'historical' : 'rarity'].set(id, 'rare')
+    } else if (percent < this.rarityPercents.uncommon) {
+      this[historical ? 'historical' : 'rarity'].set(id, 'uncommon')
+    } else {
+      this[historical ? 'historical' : 'rarity'].set(id, 'common')
+    }
+  }
+
+  async initRarity() {
+    const results = await Promise.all(
+      this.models.Pokemon.map(async (source) =>
+        source.isMad
+          ? []
+          : source.SubModel.query()
+              .select('pokemon_id', raw('SUM(count) as total'))
+              .from('pokemon_stats')
+              .groupBy('pokemon_id'),
+      ),
+    )
+    const consolidated = {}
+    let total = 0
+    results.forEach((result) => {
+      result.forEach((row) => {
+        if (consolidated[row.pokemon_id]) {
+          consolidated[row.pokemon_id] += +row.total
+        } else {
+          consolidated[row.pokemon_id] = +row.total
+        }
+        total += +row.total
+      })
+    })
+    Object.entries(consolidated).forEach(([id, count]) => {
+      this.setRarity(+id, (count / total) * 100, true)
+    })
+  }
+
   bindConnections(models) {
     try {
       Object.entries(this.models).forEach(([model, sources]) => {
@@ -239,6 +291,23 @@ module.exports = class DbCheck {
             ]),
           )
         }
+        if (model === 'Pokemon') {
+          const base = {}
+          let total = 0
+          results.forEach((result) => {
+            Object.entries(result.rarity).forEach(([key, count]) => {
+              if (key in base) {
+                base[key] += count
+              } else {
+                base[key] = count
+              }
+              total += count
+            })
+          })
+          Object.entries(base).forEach(([id, count]) => {
+            this.setRarity(id, (count / total) * 100)
+          })
+        }
         if (results.length === 1) return results[0].available
         if (results.length > 1) {
           const returnSet = new Set()