Reduce CPU usage loading large numbers of items (#1768)

src/BackendConnector.js

@@ -2,13 +2,9 @@ import * as utils from './utils.js';
 import baseLogger from './logger.js';
 import EventEmitter from './EventEmitter.js';
 
-function remove(arr, what) {
-  let found = arr.indexOf(what);
-
-  while (found !== -1) {
-    arr.splice(found, 1);
-    found = arr.indexOf(what);
-  }
+function removePending(q, name) {
+  delete q.pending[name];
+  q.pendingCount--;
 }
 
 class Connector extends EventEmitter {
@@ -35,10 +31,10 @@ class Connector extends EventEmitter {
 
   queueLoad(languages, namespaces, options, callback) {
     // find what needs to be loaded
-    const toLoad = [];
-    const pending = [];
-    const toLoadLanguages = [];
-    const toLoadNamespaces = [];
+    const toLoad = {};
+    const pending = {};
+    const toLoadLanguages = {};
+    const toLoadNamespaces = {};
 
     languages.forEach((lng) => {
       let hasAllNamespaces = true;
@@ -51,35 +47,36 @@ class Connector extends EventEmitter {
         } else if (this.state[name] < 0) {
           // nothing to do for err
         } else if (this.state[name] === 1) {
-          if (pending.indexOf(name) < 0) pending.push(name);
+          if (pending[name] !== undefined) pending[name] = true;
         } else {
           this.state[name] = 1; // pending
 
           hasAllNamespaces = false;
 
-          if (pending.indexOf(name) < 0) pending.push(name);
-          if (toLoad.indexOf(name) < 0) toLoad.push(name);
-          if (toLoadNamespaces.indexOf(ns) < 0) toLoadNamespaces.push(ns);
+          pending[name] = true;
+          toLoad[name] = true;
+          toLoadNamespaces[ns] = true;
         }
       });
 
-      if (!hasAllNamespaces) toLoadLanguages.push(lng);
+      if (!hasAllNamespaces) toLoadLanguages[lng] = true;
     });
 
-    if (toLoad.length || pending.length) {
+    if (Object.keys(toLoad).length || Object.keys(pending).length) {
       this.queue.push({
         pending,
+        pendingCount: Object.keys(pending).length,
         loaded: {},
         errors: [],
         callback,
       });
     }
 
     return {
-      toLoad,
-      pending,
-      toLoadLanguages,
-      toLoadNamespaces,
+      toLoad: Object.keys(toLoad),
+      pending: Object.keys(pending),
+      toLoadLanguages: Object.keys(toLoadLanguages),
+      toLoadNamespaces: Object.keys(toLoadNamespaces),
     };
   }
 
@@ -103,17 +100,18 @@ class Connector extends EventEmitter {
     // callback if ready
     this.queue.forEach((q) => {
       utils.pushPath(q.loaded, [lng], ns);
-      remove(q.pending, name);
+      removePending(q, name);
 
       if (err) q.errors.push(err);
 
-      if (q.pending.length === 0 && !q.done) {
+      if (q.pendingCount === 0 && !q.done) {
         // only do once per loaded -> this.emit('loaded', q.loaded);
         Object.keys(q.loaded).forEach((l) => {
-          if (!loaded[l]) loaded[l] = [];
-          if (q.loaded[l].length) {
-            q.loaded[l].forEach((ns) => {
-              if (loaded[l].indexOf(ns) < 0) loaded[l].push(ns);
+          if (!loaded[l]) loaded[l] = {};
+          const loadedKeys = Object.keys(loaded[l]);
+          if (loadedKeys.length) {
+            loadedKeys.forEach((ns) => {
+              if (loadedKeys[ns] !== undefined) loaded[l][ns] = true;
             });
           }
         });

test/backend/backendConnector.perf.spec.js

@@ -0,0 +1,39 @@
+import BackendConnector from '../../src/BackendConnector.js';
+import BackendMock from './backendMock.js';
+import Interpolator from '../../src/Interpolator.js';
+import ResourceStore from '../../src/ResourceStore.js';
+import { expect } from 'chai';
+
+describe('BackendConnector performance test', () => {
+  let connector;
+
+  before(() => {
+    connector = new BackendConnector(
+      new BackendMock(),
+      new ResourceStore(),
+      {
+        interpolator: new Interpolator(),
+      },
+      {
+        backend: { loadPath: 'http://localhost:9876/locales/{{lng}}/{{ns}}.json' },
+      },
+    );
+  });
+
+  describe('#load', () => {
+    it('should load 10,000 items in under the 2 second timeout', (done) => {
+      const namespaces = [];
+      for (let i = 0; i < 10000; i++) {
+        namespaces.push(`namespace${i}`);
+      }
+      connector.load(['en'], namespaces, function (err) {
+        expect(err).to.be.not.ok;
+        expect(connector.store.getResourceBundle('en', 'namespace1')).to.eql({
+          status: 'nok',
+          retries: 0,
+        });
+        done();
+      });
+    });
+  });
+});