Dependents support. fixed #190

bin/restore_module_deps.js

@@ -0,0 +1,74 @@
+/**!
+ * cnpmjs.org - bin/restore_module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+var Module = require('../proxy/module');
+var ModuleDeps = require('../proxy/module_deps');
+
+var addCount = 0;
+
+function restore(id, callback) {
+  var sql = 'SELECT id, name, package FROM module WHERE id > ? ORDER BY id ASC LIMIT 1000';
+  mysql.query(sql, [id], function (err, rows) {
+    if (err) {
+      return callback(err);
+    }
+    if (rows.length === 0) {
+      return callback(null, []);
+    }
+
+    console.log('[%s] got %d rows', id, rows.length);
+
+    rows.forEach(function (r) {
+      Module.parseRow(r);
+      if (!r.package) {
+        return;
+      }
+      var deps = Object.keys(r.package.dependencies || {});
+      if (!Array.isArray(deps) || !deps.length) {
+        return;
+      }
+      deps.forEach(function (dep) {
+        ModuleDeps.add(dep, r.name, function (err) {
+          // console.log('[%s] add %s <= %s, error: %s', id, dep, r.name, err);
+        });
+      });
+      addCount += deps.length;
+    });
+    setTimeout(function () {
+      console.log('[%s] add %d relations', id, addCount);
+      callback(null, rows);
+    }, 1000);
+  });
+}
+
+var id = 0;
+function run() {
+  restore(id, function (err, rows) {
+    if (err) {
+      throw err;
+    }
+    if (rows.length === 0) {
+      console.log('finished, last id: %s, exit.', id);
+      process.exit(0);
+    }
+
+    id = rows[rows.length - 1].id;
+    run();
+  });
+}
+
+run();

bin/test.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name_deps` (`name`,`deps`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';

controllers/web/package.js

@@ -26,6 +26,7 @@ var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var ModuleDeps = require('../../proxy/module_deps');
 var setDownloadURL = require('../../lib/common').setDownloadURL;
 
 exports.display = function (req, res, next) {
@@ -48,14 +49,24 @@ exports.display = function (req, res, next) {
 
   down.total(name, ep.done('download'));
 
-  ep.all('pkg', 'download', function (pkg, download) {
+  ModuleDeps.list(name, ep.done(function (rows) {
+    ep.emit('dependents', rows.map(function (r) {
+      return r.deps;
+    }));
+  }));
+
+  ep.all('pkg', 'download', 'dependents', function (pkg, download, dependents) {
     if (!pkg || !pkg.package) {
       return next();
     }
 
     pkg.package.fromNow = moment(pkg.publish_time).fromNow();
     pkg = pkg.package;
     pkg.readme = marked(pkg.readme || '');
+    if (!pkg.readme) {
+      pkg.readme = pkg.description || '';
+    }
+
     if (pkg.maintainers) {
       for (var i = 0; i < pkg.maintainers.length; i++) {
         var maintainer = pkg.maintainers[i];
@@ -90,9 +101,10 @@ exports.display = function (req, res, next) {
     for (var k in download) {
       download[k] = humanize(download[k]);
     }
-
     setDownloadURL(pkg, req, config.registryHost);
 
+    pkg.dependents = dependents;
+
     res.render('package', {
       title: 'Package - ' + pkg.name,
       package: pkg,

docs/db.sql

@@ -112,3 +112,13 @@ CREATE TABLE `download_total` (
  UNIQUE KEY `date_name` (`date`, `name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
 -- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+
+CREATE TABLE `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name_deps` (`name`,`deps`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';

proxy/module.js

@@ -175,6 +175,7 @@ function parseRow(row) {
     }
   }
 }
+exports.parseRow = parseRow;
 
 function stringifyPackage(pkg) {
   return encodeURIComponent(JSON.stringify(pkg));

proxy/module_deps.js

@@ -0,0 +1,41 @@
+/**!
+ * cnpmjs.org - proxy/module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+
+var LIST_DEPS_SQL = 'SELECT deps FROM module_deps WHERE name=?;';
+
+exports.list = function (name, callback) {
+  mysql.query(LIST_DEPS_SQL, [name], callback);
+};
+
+var INSERT_DEPS_SQL = 'INSERT INTO module_deps(gmt_create, name, deps) \
+  VALUES(now(), ?, ?);';
+
+exports.add = function (name, deps, callback) {
+  mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
+    if (err && err.code === 'ER_DUP_ENTRY') {
+      err = null;
+    }
+    callback(err);
+  });
+};
+
+var DELETE_DEPS_SQL = 'DELETE FROM module_deps WHERE name=? AND deps=?;';
+
+exports.remove = function (name, deps, callback) {
+  mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
+};

proxy/sync_module_worker.js

@@ -24,12 +24,13 @@ var crypto = require('crypto');
 var eventproxy = require('eventproxy');
 var urllib = require('urllib');
 var utility = require('utility');
+var ms = require('ms');
 var nfs = require('../common/nfs');
 var npm = require('./npm');
 var common = require('../lib/common');
 var Module = require('./module');
+var ModuleDeps = require('./module_deps');
 var Log = require('./module_log');
-var ms = require('ms');
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
@@ -409,9 +410,12 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
     callback(err);
   });
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s',
+  var dependencies = Object.keys(sourcePackage.dependencies || {});
+
+  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s, dependencies: %d',
     sourcePackage.name, versionIndex, sourcePackage.version,
-    sourcePackage.dist, that.noDep, that._publish);
+    sourcePackage.dist, that.noDep, that._publish,
+    dependencies.length);
   if (!that.noDep) {
     for (var k in sourcePackage.dependencies) {
       that.add(k);
@@ -422,6 +426,11 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
     }
   }
 
+  // add deps relations
+  dependencies.forEach(function (depName) {
+    ModuleDeps.add(depName, sourcePackage.name, utility.noop);
+  });
+
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
   urllib.request(downurl, options, ep.done(function (_, response) {

test/proxy/module_deps.test.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - test/proxy/module_deps.test.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var pedding = require('pedding');
+var ModuleDeps = require('../../proxy/module_deps');
+
+describe('proxy/module_deps.test.js', function () {
+  before(function (done) {
+    done = pedding(2, done);
+    ModuleDeps.remove('testmodule', 'foo', done);
+    ModuleDeps.remove('testmodule', 'bar', done);
+  });
+
+  describe('add()', function () {
+    it('should add foo, bar success', function (done) {
+      done = pedding(2, done);
+      ModuleDeps.add('testmodule', 'foo', function (err) {
+        should.not.exist(err);
+        // add again should work
+        ModuleDeps.add('testmodule', 'foo', function (err) {
+          should.not.exist(err);
+          done();
+        });
+      });
+
+      ModuleDeps.add('testmodule', 'bar', done);
+    });
+  });
+
+  describe('list()', function () {
+    it('should list testmodule deps', function (done) {
+      ModuleDeps.list('testmodule', function (err, rows) {
+        should.not.exist(err);
+        should.exist(rows);
+        rows.should.be.an.Array;
+        rows.should.length(2);
+        rows.forEach(function (row) {
+          row.should.have.property('deps');
+        });
+        done();
+      });
+    });
+  });
+});

view/web/package.html

@@ -22,10 +22,10 @@ <h1>
 
     <ul class="nav nav-tabs">
       <% if (package.readme) { %>
-        <li class="active"><a href="#readme">README</a></li>
-        <li><a href="#meta">Meta</a></li>
+        <li class="active"><a href="#readme">Readme.md</a></li>
+        <li><a href="#meta">package.json</a></li>
       <% } else { %>
-        <li class="active"><a href="#meta">Meta</a></li>
+        <li class="active"><a href="#meta">package.json</a></li>
       <% } %>
     </ul>
 
@@ -141,7 +141,33 @@ <h1>
             var deps = Object.keys(package.dependencies || {})
             var l = deps.length
           %>
-          <th>Dependencies<%= l > 5 ? ' (' + l + ')' : '' %></th>
+          <th>Dependencies<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <tr>
+          <%
+            var deps = package.dependents || [];
+            var l = deps.length
+          %>
+          <th>Dependents<%= l > 0 ? ' (' + l + ')' : '' %></th>
           <td>
             <%
             if (l === 0) {
@@ -167,7 +193,7 @@ <h1>
           var l = starredBy.length
         %>
         <tr>
-          <th>Starred by<%= l > 5 ? ' (' + l + ')' : '' %></th>
+          <th>Starred by<%= l > 0 ? ' (' + l + ')' : '' %></th>
           <td>
             <%
             var max = 20