Add an interactive update system (#84)

* Document update system.

* Add a postinstall script which calls ./bin/update.js.

* Remove dependency on request and async and rewrite the updater script

Also, bundle punycode as a dependency

* Test update command during Continuous Integration tests.

.travis.yml

@@ -11,6 +11,10 @@ notifications:
   email:
     on_failure: change
 
+install:
+- rm rules.json
+- npm install --tldjs-update-rules
+
 script:
 - xvfb-run -a npm test
 - npm run test-coverage

README.md

@@ -159,27 +159,25 @@ tld.getSubdomain('vhost.localhost');  // returns 'vhost'
 
 Many libraries offer a list of TLDs. But, are they up-to-date? And how to update them?
 
+`tldjs` bundles a list of known TLDs but this list can become outdated.
+This is especially true if the package have not been updated on npm for a while.
+
 Hopefully for you, even if I'm flying over the world, if I've lost my Internet connection or even if
 you do manage your own list, you can update it by yourself, painlessly.
 
-How? By typing this in your console
+How? By passing the `--tldjs-update-rules` to your `npm install` command:
 
 ```bash
-npm run build
-```
-
-Alternatively, you can launch the updater through its API:
+# anytime you reinstall your project
+npm install --tldjs-update-rules
 
-```js
-var updater = require('tldjs/lib/updater');
-updater.run(function done(){
-  // do something when update is performed
-});
+# or if you add the dependency to your project
+npm install --save tldjs --tldjs-update-rules
 ```
 
-A fresh copy will be made available as `./rules.json`.
-
-Open an issue to request an update in all package systems (or do a PR with a bugfix version bump).
+Open an issue to request an update of the bundled rules.
+Or else, fork the project and open a PR after having run `npm version patch`.
+Once merged, the `tldjs` package will be published on npmjs.com.
 
 
 # Contributing

bin/postinstall.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+
+var updater = require('./update.js');
+var SHOULD_UPDATE = process.env.npm_config_tldjs_update_rules === 'true';
+
+
+if (SHOULD_UPDATE) {
+  console.log('tldjs: updating rules from %s.', updater.providerUrl);
+
+  updater.run(function(err){
+    if (err) {
+      console.error(err.message);
+      process.exit(err.code);
+    }
+
+    console.log('tldjs: rules list updated.');
+    process.exit(0);
+  });
+}

bin/update.js

@@ -5,8 +5,12 @@
 var pathJoin = require('path').join;
 var updater = require(pathJoin(__dirname, '..', 'lib', 'updater'));
 
-console.log('Requesting tld data...');
+module.exports = updater;
 
-updater.run(function(){
-  console.log('Update complete.');
-});
+if (process.mainModule === module) {
+  console.log('Requesting tld data...');
+
+  updater.run(function(){
+    console.log('Update complete.');
+  });
+}

lib/updater.js

@@ -1,40 +1,48 @@
 "use strict";
 
 var join = require('path').join;
-var request = require('request');
-var async = require('async');
+var http = require('https');
 var fs = require('fs');
 
 var pkg = require('../package.json');
 
 var providerUrl = pkg.tldjs.providers['publicsuffix-org'];
 var parser = require('./parsers/publicsuffix-org.js');
+var exportTask = require('./exports/standard-json.js');
 
 module.exports = {
+  providerUrl: providerUrl,
   run: function runUpdater(done){
     done = typeof done === 'function' ? done : function(){};
 
-    request.get(providerUrl, function (err, response, body) {
-      var queue, tlds;
+    var req = http.request(providerUrl, function (res) {
+      var body = '';
 
-      if (err) {
-        throw new Error(err);
+      if (res.statusCode !== 200) {
+        res.destroy();
+        return done(new Error('tldjs: remote server responded with HTTP status ' + res.statusCode));
       }
 
-      tlds = parser.parse(body);
-      queue = async.queue(function(exportTask, callback){
+      res.setEncoding('utf8');
+
+      res.on('data', function(d) {
+        body += d;
+      });
+
+      res.on('end', function() {
+        var tlds = parser.parse(body);
         var task_result = exportTask(tlds);
         var filename = task_result[0];
         var data = task_result[1];
 
         data = JSON.stringify(data);
 
-        fs.writeFile(join(__dirname, '..', filename), data, 'utf-8', callback);
-      }, 5);
-
-      queue.drain = done;
-
-      queue.push(require(join(__dirname, 'exports', 'index.js')));
+        fs.writeFile(join(__dirname, '..', filename), data, 'utf-8', done);
+      });
     });
+
+    req.setTimeout(5000);
+    req.on('error', done);
+    req.end();
   }
 };

package.json

@@ -18,14 +18,14 @@
   "license": "MIT",
   "scripts": {
     "lint": "jshint --config .jshintrc lib/**/*.js",
-    "pretest": "npm run build",
     "test": "mocha -R dot -r env-test",
     "posttest": "npm run lint && npm run test-browser",
     "test-coverage": "mocha -R travis-cov -r blanket -r env-test",
     "coverage": "npm run test-coverage -- > coverage.html",
     "test-watch": "mocha -R dot -r env-test --watch",
     "test-browser": "testling",
-    "build": "node ./bin/update.js",
+    "update": "node ./bin/update.js",
+    "postinstall": "node ./bin/postinstall.js",
     "generate-changelog": "github-changes -o oncletom -r 'tld.js' -n ${npm_package_version} --only-pulls --use-commit-body",
     "version": "npm run generate-changelog && git add CHANGELOG.md rules.json"
   },
@@ -55,17 +55,17 @@
       "android/4.2"
     ]
   },
-  "dependencies": {},
+  "dependencies": {
+    "punycode": "^1.4.1"
+  },
   "devDependencies": {
-    "async": "^1.4.2",
     "blanket": "1.1.9",
     "env-test": "^1.0.0",
     "expect.js": "^0.3.1",
     "github-changes": "^1.0.0",
     "jshint": "^2.5.1",
     "mocha": "^2.3.3",
     "phantomjs": "^1.9.18",
-    "request": "^2.36.0",
     "testling": "^1.7.0",
     "travis-cov": "^0.2.5"
   },