Merge 82843d4 into 6a79d09

bin/top-npm-users.js

@@ -15,4 +15,4 @@ require('yargs')
   .help('h')
   .alias('h', 'help')
   .demand(1, 'a command must be provided')
-  .argv
+  .parse()

index.js

@@ -15,62 +15,77 @@ function TopUsers (opts) {
     outputDirectory: './output/',
     jsonData: 'top-npm-users.json',
     templateName: 'top-npm-users.md',
-    registryDb: 'https://skimdb.npmjs.com/registry',
+    registryDb: 'https://replicate.npmjs.com/registry',
     downloadCounts: {},
     ChangesStream: require('changes-stream')
   }, opts)
 }
 
-TopUsers.prototype.calculate = function () {
-  var _this = this
+let backoff = 5000
+const backoffIncrement = 5000
+const queueConcurrency = 4
 
-  this._q = queue(function (task, callback) {
+TopUsers.prototype.calculate = function () {
+  this._q = queue((task, callback) => {
     request.get({
       json: true,
-      url: _this.countsUrl + encodeURIComponent(task.name)
-    }, function (err, res, obj) {
-      if (err) return callback(err)
-      if (res.statusCode !== 200) return callback(Error('unexpected status = ' + res.statusCode))
-      _this.dirty = true
-      _this._updateUserCounts(task, obj.downloads)
-      return callback()
+      url: this.countsUrl + encodeURIComponent(task.name)
+    }, (err, res, obj) => {
+      if (res && [200, 404].indexOf(res.statusCode) === -1) {
+        err = Error('unexpected status = ' + res.statusCode)
+      }
+      if (err) {
+        console.warn(err.message)
+        setTimeout(() => {
+          console.info(`retrying task ${task.name} backoff now ${backoff}`)
+          task.failures++
+          if (task.failures < 3) this._q.push(task)
+          return callback()
+        }, backoff)
+        backoff += backoffIncrement
+      } else {
+        backoff = backoffIncrement
+        this.dirty = true
+        this._updateUserCounts(task, obj.downloads)
+        return callback()
+      }
     })
-  }, 5)
+  }, queueConcurrency)
 
   var changes = new this.ChangesStream({
     include_docs: true,
     db: this.registryDb
   })
 
-  changes.on('readable', function () {
+  changes.on('readable', () => {
     var change = changes.read()
 
     if (change.seq % 100 === 0) console.log('sequence #' + change.seq)
 
     if (change.doc && change.doc.maintainers) {
-      _this._q.push({
+      this._q.push({
         maintainers: change.doc.maintainers,
-        name: change.doc.name
-      }, function (err) {
+        name: change.doc.name,
+        failures: 0
+      }, (err) => {
         if (err) console.log(err.message)
       })
     }
   })
 
-  this._saveInterval = setInterval(function () {
-    if (_this.dirty) {
-      console.log('saving download counts (q = ' + _this._q.length() + ')')
-      mkdirp.sync(_this.outputDirectory)
-      fs.writeFileSync(_this.outputDirectory + _this.jsonData, JSON.stringify(_this.downloadCounts, null, 2), 'utf-8')
+  this._saveInterval = setInterval(() => {
+    if (this.dirty) {
+      console.log('saving download counts (q = ' + this._q.length() + ')')
+      mkdirp.sync(this.outputDirectory)
+      fs.writeFileSync(this.outputDirectory + this.jsonData, JSON.stringify(this.downloadCounts, null, 2), 'utf-8')
     }
   }, this.saveInterval)
 }
 
 TopUsers.prototype._updateUserCounts = function (task, downloads) {
-  var _this = this
-  task.maintainers.forEach(function (maintainer) {
-    if (!_this.downloadCounts[maintainer.name]) _this.downloadCounts[maintainer.name] = 0
-    _this.downloadCounts[maintainer.name] += downloads || 0
+  task.maintainers.forEach((maintainer) => {
+    if (!this.downloadCounts[maintainer.name]) this.downloadCounts[maintainer.name] = 0
+    this.downloadCounts[maintainer.name] += downloads || 0
   })
 }
 
@@ -82,7 +97,7 @@ TopUsers.prototype.render = function () {
   var result = template({
     end: moment().format('ll'),
     start: moment().subtract(1, 'month').format('ll'),
-    users: this._top100Users().map(function (u, i) {
+    users: this._top100Users().map((u, i) => {
       return {
         name: u.name,
         downloads: u.downloads.toLocaleString(),
@@ -99,14 +114,14 @@ TopUsers.prototype._top100Users = function () {
   var userMap = JSON.parse(fs.readFileSync(this.outputDirectory + this.jsonData, 'utf-8'))
   var users = []
 
-  Object.keys(userMap).forEach(function (k) {
+  Object.keys(userMap).forEach((k) => {
     users.push({
       name: k,
       downloads: userMap[k]
     })
   })
 
-  users.sort(function (a, b) {
+  users.sort((a, b) => {
     return b.downloads - a.downloads
   })
 

package.json

@@ -24,20 +24,20 @@
   },
   "homepage": "https://github.com/bcoe/top-npm-users#readme",
   "devDependencies": {
-    "chai": "^3.4.1",
-    "nock": "^3.6.0",
-    "rimraf": "^2.5.0",
-    "standard": "^5.4.1",
-    "tap": "^4.0.0"
+    "chai": "^4.1.2",
+    "nock": "^9.2.5",
+    "rimraf": "^2.6.2",
+    "standard": "^11.0.1",
+    "tap": "^11.1.4"
   },
   "dependencies": {
-    "async": "^1.5.0",
-    "changes-stream": "^1.1.0",
-    "handlebars": "^4.0.5",
-    "lodash": "^3.10.1",
+    "async": "^2.6.0",
+    "changes-stream": "^2.2.0",
+    "handlebars": "^4.0.11",
+    "lodash": "^4.17.10",
     "mkdirp": "^0.5.1",
-    "moment": "^2.10.6",
-    "request": "^2.67.0",
-    "yargs": "^3.31.0"
+    "moment": "^2.22.1",
+    "request": "^2.85.0",
+    "yargs": "^11.0.0"
   }
 }