fix(top): use proper db ordering (#2744)

* fix(top): use proper db ordering Fixes #2730 * add timestamps to test * fix bits
sogebot · Oct 7, 2019 · db1e7c1 · db1e7c1
1 parent b0b8e4e
commit db1e7c1
Show file tree

Hide file tree

Showing 11 changed files with 65 additions and 61 deletions.
diff --git a/src/bot/databases/mongodb.js b/src/bot/databases/mongodb.js
@@ -71,28 +71,17 @@ class IMongoDB extends Interface {
 
     if (this.createIndexes || this.forceRemoveIndexes) {
       const collections = await db.listCollections().toArray()
-      const dropIndexes = [
-        'users.bits', 'users.tips', 'users.points', 'users.online', 'users.messages', 'users.watched',
-        'cache', 'customTranslations', 'users', 'stats'
-      ]
-      for (let table of dropIndexes) {
-        if (_.find(collections, (o) => o.name === table)) await db.collection(table).dropIndexes()
-        await db.createCollection(table)
+      for (let k of Object.keys(collections)) {
+        await db.collection(collections[k].name).dropIndexes()
+        await db.createCollection(collections[k].name)
       }
 
       if (this.createIndexes) {
-        await db.collection('users.bits').createIndex('timestamp')
-        await db.collection('users.tips').createIndex('timestamp')
         await db.collection('users').createIndex('username')
         await db.collection('users').createIndex(
           { id: 1 },
           { partialFilterExpression: { id: { $exists: true } }, unique: true }
         )
-        await db.collection('users.online').createIndex('username')
-        await db.collection('users.points').createIndex('id', { unique: true })
-        await db.collection('users.messages').createIndex('id', { unique: true })
-        await db.collection('users.watched').createIndex('id', { unique: true })
-        await db.collection('cache').createIndex('key', { unique: true })
         await db.collection('customTranslations').createIndex('key')
         await db.collection('stats').createIndex('whenOnline')
         await db.collection('cache.hosts').createIndex('username', { unique: true })

diff --git a/src/bot/databases/nedb.js b/src/bot/databases/nedb.js
@@ -46,20 +46,10 @@ class INeDB extends Interface {
       this.table[table].persistence.setAutocompactionInterval(60000)
 
       switch (table) {
-        case 'users.bits':
-        case 'users.tips':
-          this.table[table].removeIndex('timestamp')
-          break
-        case 'users':
-        case 'users.online':
-        case 'users.points':
-        case 'users.messages':
-        case 'users.watched':
         case 'cache.hosts':
           this.table[table].removeIndex('username')
           this.table[table].removeIndex('id')
           break
-        case 'cache':
         case 'customTranslations':
           this.table[table].removeIndex('key')
           break
@@ -71,25 +61,6 @@ class INeDB extends Interface {
       // create indexes
       if (this.createIndexes) {
         switch (table) {
-          case 'users.bits':
-          case 'users.tips':
-            this.table[table].ensureIndex({ fieldName: 'timestamp' })
-            break
-          case 'users':
-            this.table[table].ensureIndex({ fieldName: 'username' })
-            this.table[table].ensureIndex({ fieldName: 'id', unique: true })
-            break
-          case 'cache.hosts':
-            this.table[table].ensureIndex({ fieldName: 'username', unique: true })
-            break
-          case 'users.online':
-            this.table[table].ensureIndex({ fieldName: 'username' })
-            break
-          case 'users.points':
-          case 'users.messages':
-          case 'users.watched':
-            this.table[table].ensureIndex({ fieldName: 'id', unique: true })
-            break
           case 'cache':
           case 'customTranslations':
             this.table[table].ensureIndex({ fieldName: 'key', unique: true })

diff --git a/src/bot/integrations/donationalerts.ts b/src/bot/integrations/donationalerts.ts
@@ -113,7 +113,15 @@ class Donationalerts extends Integration {
         if (!data._is_test_alert) {
           const id = await global.users.getIdByName(data.username.toLowerCase(), false);
           if (id) {
-            global.db.engine.insert('users.tips', { id, amount: Number(data.amount), message: data.message, currency: data.currency, timestamp: _.now() });
+            global.db.engine.insert('users.tips', {
+              id,
+              amount: Number(data.amount),
+              currency: data.currency,
+              _amount: global.currency.exchange(Number(data.amount), data.currency, 'EUR'), // recounting amount to EUR to have simplified ordering
+              _currency: 'EUR', // we are forcing _currency to have simplified ordering
+              message: data.message,
+              timestamp: _.now(),
+            });
           }
           if (global.api.isStreamOnline) {
             global.api.stats.currentTips = parseFloat(global.currency.exchange(data.amount, data.currency, global.currency.mainCurrency));

diff --git a/src/bot/integrations/qiwi.ts b/src/bot/integrations/qiwi.ts
@@ -61,7 +61,14 @@ class Qiwi extends Integration {
 
       const id = username ? await global.users.getIdByName(username, false) : null;
       if (id) {
-        global.db.engine.insert('users.tips', { id, amount, message, currency, timestamp: Date.now() });
+        global.db.engine.insert('users.tips', {
+          id,
+          amount,
+          currency,
+          _amount: global.currency.exchange(Number(amount), currency, 'EUR'), // recounting amount to EUR to have simplified ordering
+          _currency: 'EUR', // we are forcing _currency to have simplified ordering
+          message,
+          timestamp: Date.now() });
       }
 
       if (global.api.isStreamOnline) {

diff --git a/src/bot/integrations/streamlabs.ts b/src/bot/integrations/streamlabs.ts
@@ -68,7 +68,15 @@ class Streamlabs extends Integration {
         if (!event.isTest) {
           const id = await global.users.getIdByName(event.from.toLowerCase(), false);
           if (id) {
-            global.db.engine.insert('users.tips', { id, amount: Number(event.amount), message: event.message, currency: event.currency, timestamp: _.now() });
+            global.db.engine.insert('users.tips', {
+              id,
+              amount: Number(event.amount),
+              currency: event.currency,
+              _amount: global.currency.exchange(Number(event.amount), event.currency, 'EUR'), // recounting amount to EUR to have simplified ordering
+              _currency: 'EUR', // we are forcing _currency to have simplified ordering
+              message: event.message,
+              timestamp: _.now(),
+            });
           }
           if (global.api.isStreamOnline) {
             global.api.stats.currentTips = parseFloat(global.currency.exchange(event.amount, event.currency, global.currency.mainCurrency));

diff --git a/src/bot/systems/top.ts b/src/bot/systems/top.ts
@@ -115,18 +115,10 @@ class Top extends System {
         message = global.translate('systems.top.time').replace(/\$amount/g, 10);
         break;
       case TYPE.TIPS:
-        const users = {};
-        const tips = await global.db.engine.find('users.tips');
-
-        message = global.translate('systems.top.tips').replace(/\$amount/g, 10);
-        for (const tip of tips) {
-          const username = await global.users.getNameById(tip.id);
-          if (_.isNil(users[username])) {
-            users[username] = { username, value: 0 };
-          }
-          users[username].value += global.currency.exchange(Number(tip.amount), tip.currency, global.currency.mainCurrency);
+        for (const tip of (await global.db.engine.find('users.tips', { _sort: '_amount', _sum: '_amount', _total, _group: 'id' }))) {
+          sorted.push({ username: await global.users.getNameById(tip._id), value: global.currency.exchange(Number(tip._amount), 'EUR' /* we know that _currency is EUR */, global.currency.mainCurrency) });
         }
-        sorted = _.orderBy(users, 'value', 'desc');
+        message = global.translate('systems.top.tips').replace(/\$amount/g, 10);
         break;
       case TYPE.POINTS:
         if (!global.systems.points.isEnabled()) {

diff --git a/src/bot/users.js b/src/bot/users.js
@@ -24,6 +24,12 @@ class Users extends Core {
 
     if (isMainThread) {
       global.db.engine.index('users.chat', [{ index: 'id' }]);
+      global.db.engine.index('users.tips', [{ index: 'timestamp', unique: true }, { index: 'id' }]);
+      global.db.engine.index('users.bits', [{ index: 'timestamp', unique: true }, { index: 'id' }]);
+      global.db.engine.index('users.points', [{ index: 'id', unique: true }]);
+      global.db.engine.index('users.messages', [{ index: 'id', unique: true }]);
+      global.db.engine.index('users.watched', [{ index: 'id', unique: true }]);
+      global.db.engine.index('users.online', [{ index: 'username' }]);
       this.updateWatchTime(true);
       this.updateChatTime();
     }
@@ -491,6 +497,8 @@ class Users extends Core {
           b.tips = Number(b.tips); // force retype amount to be sure we really have number (ui is sending string)
           if (b.new) {
             delete b.new; delete b._id
+            b._amount = global.currency.exchange(Number(b.amount), b.currency, 'EUR'); // recounting amount to EUR to have simplified ordering
+            b._currency = 'EUR'; // we are forcing _currency to have simplified ordering
             const tip = await global.db.engine.insert('users.tips', b)
             newTipsIds.push(String(tip._id));
           } else {

diff --git a/test/tests/permissions/check.js b/test/tests/permissions/check.js
@@ -37,8 +37,8 @@ describe('Permissions - check()', () => {
     }
     await global.db.engine.insert('users.points', { id: 7, points: 100 });
     await global.db.engine.insert('users.watched', { id: 8, watched: 100 * (60 * 60 * 1000 /*hours*/) });
-    await global.db.engine.insert('users.tips', { id: 9, currency: 'EUR', amount: 100 });
-    await global.db.engine.insert('users.bits', { id: 10, amount: 100 });
+    await global.db.engine.insert('users.tips', { id: 9, currency: 'EUR', amount: 100, _amount: 100, _currency: 'EUR', timestamp: Math.random() });
+    await global.db.engine.insert('users.bits', { id: 10, amount: 100, timestamp: Math.random() });
     await global.db.engine.insert('users.messages', { id: 11, messages: 100 });
   });
 

diff --git a/test/tests/top/bits.js b/test/tests/top/bits.js
@@ -31,7 +31,7 @@ describe('Top - !top bits', () => {
       });
       if (i != 0) {
         await global.db.engine.insert('users.bits', {
-          id, amount: i,
+          id, amount: i, timestamp: Math.random()
         });
       }
     }

diff --git a/test/tests/top/tips.js b/test/tests/top/tips.js
@@ -31,7 +31,7 @@ describe('Top - !top tips', () => {
       });
       if (i != 0) {
         await global.db.engine.insert('users.tips', {
-          id, amount: i, currency: 'EUR', message: 'test',
+          id, amount: i, _amount: i, currency: 'EUR', _currency: 'EUR', message: 'test', timestamp: Math.random()
         });
       }
     }

diff --git a/tools/migrate.js b/tools/migrate.js
@@ -27,6 +27,27 @@ const end = function (updated) {
 };
 
 const migration = {
+  15: async () => {
+    header('Add _amount, _currency to tips for faster ordering');
+    let updated = 0;
+    const rates = {
+      AUD: '16.065', BRL: '5.893', BGN: '13.206', CNY: '3.344', DKK: '3.459', EUR: '25.830', PHP: '0.445', HKD: '2.947',
+      HRK: '3.482', INR: '0.333', IDR: '0.002', ISK: '0.186', ILS: '6.363', JPY: '0.213', ZAR: '1.586',
+      CAD: '17.109', KRW: '0.020', HUF: '0.080', MYR: '5.522', MXN: '1.168', XDR: '31.814', NOK: '2.642',
+      NZD: '15.159', PLN: '6.030', RON: '5.452', RUB: '0.353', SGD: '16.852', SEK: '2.433', CHF: '23.140',
+      THB: '0.735', TRY: '3.925', USD: '23.093', GBP: '29.147', CZK: '1',
+    }
+
+    const items = await global.db.engine.find('users.tips');
+    for (const item of items) {
+      const from = item.currency;
+      const to = 'EUR';
+      const amount = (item.amount * rates[from]) / rates[to];
+      await global.db.engine.update('users.tips', { _id: String(item._id) }, { _currency: "EUR", _amount: amount });
+      updated++;
+    }
+    end(updated);
+  },
   14: async () => {
     header('Add id for alias');
     let updated = 0;