Merge branch 'mbrevoort'

README.md

@@ -6,9 +6,19 @@ www/ is the code for search.npmjs.org, eventually maybe www.npmjs.org
 
 # Installing
 
-You'll need CouchDB version 1.0.0 or higher.  We're using some newish features.
+You'll need CouchDB version 1.0.2 or higher.  We're using some newish features.
 I recommend getting one from http://iriscouch.com/
 
+Once you have CouchDB installed, create two new databases:
+
+		registry
+		search
+
+Clone the repository if you haven't already, and cd into it:
+
+		git clone https://github.com/isaacs/npmjs.org.git
+		cd npmjs.org
+
 Now install couchapp:
 
     sudo npm install couchapp -g
@@ -23,25 +33,54 @@ You may need to put a username and password in the URL:
     couchapp push www/app.js http://user:pass@localhost:5984/search
     couchapp push registry/app.js http://user:pass@localhost:5984/registry
 
-# Optional: top-of-host urls
+To synchronize from the public NPM registry to your private registry, create a replication
+task from http://isaacs.couchone.com/registry --> local database registry. This can be done
+through the CouchBase administrative UI or via an HTTP call to '/_replicate like so:
+
+		curl -X POST -H "Content-Type:application/json" \
+		    http://localhost:5984/_replicate -d \
+		    '{"source":"http://isaacs.couchone.com/registry/", "target":"registry"}'
+
+To run the search app, you will also want to create a synchronization task between
+local database: registry --> local database: search
+
+		curl -X POST -H "Content-Type:application/json" \
+				http://localhost:5984/_replicate -d  \
+				'{"source":"registry", "target":"search"}'
+
+# Using the registry with the npm client
 
 With the setup so far, you can point the npm client at the registry by
 putting this in your ~/.npmrc file:
 
     registry = http://localhost:5984/registry/_design/app/_rewrite
 
+You can also set the NPM registry config property like:
+
+		npm config set registry http://localhost:5984/registry/_design/app/_rewrite
+
+Or you can simple override the registry config on each call:
+
+		npm --registry http://localhost:5984/registry/_design/app/_rewrite install <package>
+
+# Optional: top-of-host urls
+
 To be snazzier, add a vhost config:
 
     [vhosts]
     registry.mydomain.com:5984 = /registry/_design/app/_rewrite
-    search.mydomain.com:5984 = /search/_design/app/_rewrite
+    search.mydomain.com:5984 = /search/_design/ui/_rewrite
 
 
 Where `registry.mydomain.com` and `search.mydomain.com` are
 the hostnames where you're running the thing, and `5984` is the
 port that CouchDB is running on. If you're running on port 80,
 then omit the port altogether.
 
+Then for example you can reference the repository like so:
+
+		npm config set registry http://registry.mydomain.com:5984
+
 # API
 
 ### GET /packagename

registry/app.js

@@ -87,6 +87,8 @@ ddoc.rewrites =
   , { from: "/-/jsonp/:jsonp", to:"_list/index/listAll", method: "GET" }
 
   , { from: "/-/all", to:"_list/index/listAll", method: "GET" }
+  , { from: "/-/search/:start/:end", to: "_list/index/search", method: "GET", query: { startkey: ':start', endkey: ':end' }}
+  , { from: "/-/search/:start/:end/:limit", to: "_list/index/search", method: "GET", query: { startkey: ':start', endkey: ':end', limit: ':limit' }}
   , { from: "/-/all/-/jsonp/:jsonp", to:"_list/index/listAll", method: "GET" }
 
   , { from: "/-/short", to:"_list/short/listAll", method: "GET" }
@@ -250,6 +252,69 @@ ddoc.views.listAll = {
   map : function (doc) { return emit(doc._id, doc) }
 }
 
+// copied from the www project
+ddoc.views.search = { map: function(doc) {
+  var descriptionBlacklist =
+    [ "for"
+    , "and"
+    , "in"
+    , "are"
+    , "is"
+    , "it"
+    , "do"
+    , "of"
+    , "on"
+    , "the"
+    , "to"
+    , "as"
+    ]
+
+  if (doc.name) { // There aren't any better attributes for check if isPackage()
+    if (doc.name) {
+      var names = [doc.name];
+      if (doc.name.indexOf('-') !== -1) doc.name.split('-').forEach(function (n) {names.push(n)});
+      if (doc.name.indexOf('_') !== -1) doc.name.split('_').forEach(function (n) {names.push(n)});
+      names.forEach(function (n) {
+        if (n.length > 1) emit(n.toLowerCase(), doc);
+      });
+    }
+    if (doc['dist-tags'] && doc['dist-tags'].latest && (
+        doc.versions[doc['dist-tags'].latest].keywords || doc.versions[doc['dist-tags'].latest].tags
+        )) {
+      var tags = (doc.versions[doc['dist-tags'].latest].keywords || doc.versions[doc['dist-tags'].latest].tags)
+      tags.forEach(function (tag) {
+        tag.split(' ').forEach(function (t) {
+          if (t.length > 0) emit(t.toLowerCase(), doc);
+        });
+      })
+    }
+    if (doc.description) {
+      doc.description.split(' ').forEach(function (d) {
+        d = d.toLowerCase();
+        while (d.indexOf('.') !== -1) d = d.replace('.', '');
+        while (d.indexOf('\n') !== -1) d = d.replace('\n', '');
+        while (d.indexOf('\r') !== -1) d = d.replace('\n', '');
+        while (d.indexOf('`') !== -1) d = d.replace('`', '');
+        while (d.indexOf('_') !== -1) d = d.replace('_', '');
+        while (d.indexOf('"') !== -1) d = d.replace('"', '');
+        while (d.indexOf('\'') !== -1) d = d.replace('\'', '');
+        while (d.indexOf('(') !== -1) d = d.replace('(', '');
+        while (d.indexOf(')') !== -1) d = d.replace(')', '');
+        while (d.indexOf('[') !== -1) d = d.replace('[', '');
+        while (d.indexOf(']') !== -1) d = d.replace(']', '');
+        while (d.indexOf('{') !== -1) d = d.replace('{', '');
+        while (d.indexOf('}') !== -1) d = d.replace('}', '');
+        while (d.indexOf('*') !== -1) d = d.replace('*', '');
+        while (d.indexOf('%') !== -1) d = d.replace('%', '');
+        while (d.indexOf('+') !== -1) d = d.replace('+', '');
+        if (descriptionBlacklist.indexOf(d) !== -1) d = '';
+        if (d.length > 1) emit(d, doc);
+      })
+    }
+  }
+}
+};
+
 
 ddoc.lists.preBuilt = function (head, req) {
   start({"code": 200, "headers": {"Content-Type": "text/plain"}});

www/app.js

@@ -11,6 +11,7 @@ ddoc =
     , {from:"/api/*", to:'../../*'}
     , {from:"/*", to:'*'}
     ]
+  , lists : {}
   }
 
 
@@ -36,7 +37,7 @@ function packageSearch (doc) {
       if (doc.name.indexOf('-') !== -1) doc.name.split('-').forEach(function (n) {names.push(n)});
       if (doc.name.indexOf('_') !== -1) doc.name.split('_').forEach(function (n) {names.push(n)});
       names.forEach(function (n) {
-        if (n.length > 1) emit(n.toLowerCase(), 1);
+        if (n.length > 1) emit(n.toLowerCase(), doc);
       });
     }
     if (doc['dist-tags'] && doc['dist-tags'].latest && (
@@ -45,7 +46,7 @@ function packageSearch (doc) {
       var tags = (doc.versions[doc['dist-tags'].latest].keywords || doc.versions[doc['dist-tags'].latest].tags)
       tags.forEach(function (tag) {
         tag.split(' ').forEach(function (t) {
-          if (t.length > 0) emit(t.toLowerCase(), 1);
+          if (t.length > 0) emit(t.toLowerCase(), doc);
         });
       })
     }
@@ -69,7 +70,7 @@ function packageSearch (doc) {
         while (d.indexOf('%') !== -1) d = d.replace('%', '');
         while (d.indexOf('+') !== -1) d = d.replace('+', '');
         if (descriptionBlacklist.indexOf(d) !== -1) d = '';
-        if (d.length > 1) emit(d, 1);
+        if (d.length > 1) emit(d, doc);
       })
     }
   }
@@ -122,6 +123,32 @@ ddoc.views =
     }
   }
   ;
+
+ddoc.lists.dependencies_limit = function(head, req) {
+    var deps = [];
+    while(row = getRow()) {
+        deps.push(row);
+    }
+    var sorted = deps.sort(function(a,b) { return req.query.descending !== "true" ? a.value - b.value : b.value - a.value; });    
+    // using list_Limit rather than limit because limit appears to limit the initial view set
+    // assuming there's a supported convention but using this for now
+    var limit = req.query.list_limit && parseInt(req.query.list_limit);
+    send(JSON.stringify({ total_rows: deps.length, rows: limit ? sorted.splice(0, limit) : sorted}));
+};
+
+ddoc.lists.search = function(head, req) {
+    var set = {};
+    var rows = [];
+    while(row = getRow()) {
+        set[row.id] = { key: row.id, count: set[row.id] ? set[row.id].count + 1 : 1, value: row.value };
+    }
+    var keys = Object.keys(set);
+    for(var i=0; i<keys.length; i++) {
+        rows.push(set[keys[i]]);
+    }
+    send(JSON.stringify({ rows: rows} ));
+};
+
 
 // ddoc.validate_doc_update = function (newDoc, oldDoc, userCtx) {
 //   if (newDoc._deleted === true && userCtx.roles.indexOf('_admin') === -1) {

www/attachments/site.js

@@ -127,42 +127,30 @@ app.index = function () {
     $('div#totals').html('<a href="/#/_browse/all">' + (resp.total_rows - 1) +' total packages</a>')
   })
 
-  request({url:'/_view/updated?descending=true&limit='+limit+'&include_docs=true'}, function (err, resp) {
+  request({url:'/_view/updated?descending=true&limit='+limit+'&include_docs=false'}, function (err, resp) {
     resp.rows.forEach(function (row) {
-      docs[row.doc._id] = row.doc;
       $('<div class="top-package"></div>')
-      .append('<div class="top-package-title"><a href="#/'+row.doc._id+'">'+row.doc._id+'</a></div>')
-      .append('<div class="top-package-updated">'+prettyDate(row.doc.time.modified) +'</div>')
+      .append('<div class="top-package-title"><a href="#/'+row.id+'">'+row.id+'</a></div>')
+      .append('<div class="top-package-updated">'+prettyDate(row.key) +'</div>')
       .append('<div class="spacer"></div>')
       .appendTo('div#latest-packages')
     })
   })
 
-  request({url:'/_view/dependencies?group=true'}, function (err, resp) {
+  request({url:'/_list/dependencies_limit/dependencies?group=true&descending=true&list_limit='+limit}, function (err, resp) {
     var results = {};
     resp.rows.forEach(function (row) {
-      if (!results[row.value]) results[row.value] = [];
-      results[row.value].push(row.key);
-    })
-    var keys = Object.keys(results);
-    keys.sort(function(a,b){return a - b;});
-    keys.reverse();
-    for (var i=0;i<limit;i++) {
-      if ($('div.top-package-dep').length == limit) return;
-      results[keys[i]].forEach(function (r) {
-        if ($('div.top-package-dep').length == limit) return;
         $('<div class="top-package"></div>')
-        .append('<div class="top-package-title"><a href="#/'+r+'">'+r+'</a></div>')
-        .append('<div class="top-package-dep">'+keys[i]+'</div>')
+        .append('<div class="top-package-title"><a href="#/'+row.key+'">'+row.key+'</a></div>')
+        .append('<div class="top-package-dep">'+row.value+'</div>')
         .append('<div class="spacer"></div>')
         .appendTo('div#top-dep-packages')
-      });
-    }
+    })
   })
 
   var updateResults = function () {
     currentSearch = $('input#search-input').val().toLowerCase();
-    currentTerms = currentSearch.split(' ');
+    currentTerms = $.trim(currentSearch).split(' ');
     if (lastSearchForPage === currentSearch) return;
     if (currentSearch == '') $('div#top-packages').show();
     else $('div#top-packages').hide();
@@ -264,7 +252,7 @@ app.index = function () {
 
   var handleChange = function () {
     currentSearch = $('input#search-input').val().toLowerCase();
-    currentTerms = currentSearch.split(' ')
+    currentTerms = $.trim(currentSearch).split(' ')
     if (currentSearch === '') {
       $('div#results').html('')
       $('div#top-packages').show();
@@ -274,40 +262,33 @@ app.index = function () {
       , c = currentSearch
       , tlength = terms.length
       ;
+
     terms.forEach(function (term) {
       if (!searchResults[term]) {
         searchResults[term] = 'pending'
         var qs = param(
           { startkey: JSON.stringify(term)
           , endkey: JSON.stringify(term+'ZZZZZZZZZZZZZZZZZZZ')
+          , limit:25
           }
         )
         ;
-        request({url:'/_view/search?'+qs}, function (err, resp) {
+        request({url:'/_list/search/search?'+qs}, function (err, resp) {
           var docids = [];
           searchResults[term] = [];
           resp.rows.forEach(function (row) {
-            searchResults[term].push(row.id);
-            if (docids.indexOf(row.id) === -1 && !docs[row.id]) {
-              docs[row.id] = 'pending';
-              docids.push(row.id);
-            }
+            searchResults[term].push(row.key);
+            row.value.name = row.value.name.toLowerCase();
+            if (row.value.description) row.value.description = row.value.description;
+            docs[row.key] = row.value;
+            updateResults();
           })
           if (docids.length === 0) {
             lastSearchForPage = '';
             updateResults();
             return 
           }
 
-          request({url:'/api/_all_docs?include_docs=true', type:'POST', data:{keys:docids} }, function (err, resp) {
-            resp.rows.forEach(function (row) {
-              row.doc.name = row.doc.name.toLowerCase();
-              if (row.doc.description) row.doc.description = row.doc.description;
-              docs[row.id] = row.doc;
-            })
-            lastSearchForPage = ''
-            updateResults();
-          })
         })
       } else {tlength -= 1}
     })