Add support for NPM packages in search API (#1117)

client/src/catalog-item.html

@@ -96,7 +96,7 @@
     </style>
 
     <template is="dom-if" if="[[data]]">
-      <a id="container" href="/[[data.kind]]/[[data.owner]]/[[data.repo]][[_versionPath(data.version, data.default_version)]]">
+      <a id="container" href="/[[_packagePath(data)]][[_versionPath(data.version, data.default_version)]]">
         <img class="avatar" src="[[data.avatar_url]]&s=[[_dpr(40)]]"   title="[[data.owner]]">
         <div id="header">
           <h2>[[data.repo]]</h2>
@@ -129,6 +129,12 @@ <h2>[[data.repo]]</h2>
         data: Object,
       },
 
+      _packagePath: function(data) {
+        if (data.npmFullPackage)
+          return data.kind + '/' + data.npmFullPackage;
+        return [data.kind, data.owner, data.repo].join('/');
+      },
+
       _versionPath: function(version, latestVersion) {
         return version == latestVersion ? '' : '/' + version;
       },

src/api.py

@@ -94,9 +94,13 @@ def brief_async(library_key, tag=None, assume_latest=False):
         'subscribers': metadata['subscribers'],
         'forks': metadata['forks'],
         'updated_at': metadata['updated_at'],
-        'dependency_count': metadata['dependency_count'],
         'avatar_url': metadata['avatar_url'],
     }
+    if 'dependency_count' in metadata:
+      result['dependency_count'] = metadata['dependency_count']
+    if 'npmFullPackage' in metadata:
+      result['npmFullPackage'] = metadata['npmFullPackage']
+
     if not assume_latest:
       result['latest_version'] = metadata['latest_version']
       result['default_version'] = metadata['default_version']
@@ -137,7 +141,11 @@ def full_async(library_key, tag=None, brief=False, assume_latest=False):
       parts = key.split('/')
       if parts[0] != '@@npm':
         result['npmScope'] = parts[0]
+        result['npmFullPackage'] = key
+      else:
+        result['npmFullPackage'] = parts[1]
       result['npmPackage'] = parts[1]
+
     result['apiKey'] = key
     result['kind'] = library.kind
     result['status'] = library.status

src/manage.py

@@ -481,12 +481,13 @@ def update_package_tags(self):
     tags = versions.keys()
     tags.sort(versiontag.compare)
     # Create a tag map of tag to sha
-    tag_map = dict((tag, versions.get(tag).get('_shasum', '')) for tag in tags)
+    tag_map = dict((tag, versions.get(tag).get('gitHead', '')) for tag in tags
+                   if versiontag.is_valid(tag) and versions.get(tag).get('gitHead'))
 
     if self.library.tags is None or self.library.tags != tags:
       self.library.library_dirty = True
       self.library.tags = tags
-      self.library.tags_map = json.dumps(tag_map)
+      self.library.tag_map = json.dumps(tag_map)
       self.library.tags_updated = datetime.datetime.now()
 
     return tag_map
@@ -850,14 +851,19 @@ def update_collection_dependencies(self, collection_version_key, bower):
 
   def update_search_index(self, owner, repo, version_key, library, bower):
     metadata = json.loads(library.metadata)
+    registry_metadata = json.loads(library.registry_metadata) if library.registry_metadata else None
+    npm_description = registry_metadata.get('description', '') if registry_metadata else ''
+    npm_keywords = registry_metadata.get('keywords', []) if registry_metadata else []
     fields = [
         search.AtomField(name='owner', value=owner),
         search.TextField(name='repo', value=repo),
         search.AtomField(name='kind', value=library.kind),
         search.AtomField(name='version', value=version_key.id()),
         search.TextField(name='github_description', value=metadata.get('description', '')),
         search.TextField(name='bower_description', value=bower.get('description', '')),
+        search.TextField(name='npm_description', value=npm_description),
         search.TextField(name='bower_keywords', value=' '.join(bower.get('keywords', []))),
+        search.TextField(name='npm_keywords', value=' '.join(npm_keywords)),
         search.TextField(name='prefix_matches', value=' '.join(util.generate_prefixes_from_list(
             util.safe_split_strip(metadata.get('description')) + util.safe_split_strip(bower.get('description')) +
             util.safe_split_strip(repo)))),

src/manage_test.py

@@ -789,7 +789,7 @@ def test_ingest_version_pages(self):
 
 class IngestNPMLibraryTest(ManageTestBase):
   def test_ingest_element(self):
-    self.respond_to('https://registry.npmjs.org/@scope%2fpackage', '{"repository": {"url": "git+https://github.com/org/repo.git"}, "license": "BSD-3-Clause", "versions": {"1.0.0": {"_shasum": "lol"}}}')
+    self.respond_to('https://registry.npmjs.org/@scope%2fpackage', '{"repository": {"url": "git+https://github.com/org/repo.git"}, "license": "BSD-3-Clause", "versions": {"1.0.0": {"gitHead": "lol"}}}')
     self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"org"},"name":"repo"}')
     self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
     self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
@@ -828,7 +828,7 @@ def test_ingest_no_package(self):
     self.assertEqual(len(tasks), 0)
 
   def test_ingest_repository_shorthand(self):
-    self.respond_to('https://registry.npmjs.org/package', '{"repository": "org/repo", "license": "BSD-3-Clause", "versions": {"1.0.0": {"_shasum": "lol"}, "0.5.0": {"_shasum": "sha"}}}')
+    self.respond_to('https://registry.npmjs.org/package', '{"repository": "org/repo", "license": "BSD-3-Clause", "versions": {"1.0.0": {"gitHead": "lol"}, "0.5.0": {"gitHead": "sha"}}}')
     self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"org"},"name":"repo"}')
     self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
     self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
@@ -985,5 +985,34 @@ def test_analyzer_index_empty(self):
     behaviors = [field for field in document.fields if field.name == 'behavior']
     self.assertEqual(len(behaviors), 0)
 
+  def test_npm_index(self):
+    metadata = """{
+      "full_name": "full-name"
+    }"""
+    registry_metadata = """{
+      "description": "mydescription",
+      "keywords": ["my-keyword"]
+    }"""
+    library_key = Library(id='@@npm/package', registry_metadata=registry_metadata, metadata=metadata).put()
+    version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
+
+    Content(id='bower', parent=version_key, content="""{"dependencies": {
+      "a": "org/element-1#1.0.0",
+      "b": "org/element-2#1.0.0"
+    }}""").put()
+
+    VersionCache.update(library_key)
+
+    response = self.app.get(util.update_indexes_task('@@npm', 'package'), headers={'X-AppEngine-QueueName': 'default'})
+    self.assertEqual(response.status_int, 200)
+
+    index = search.Index('repo')
+    document = index.get('@@npm/package')
+    self.assertIsNotNone(document)
+    self.assertTrue(len(document.fields) > 0)
+
+    self.assertEqual(document.field('npm_keywords').value, 'my-keyword')
+    self.assertEqual(document.field('npm_description').value, 'mydescription')
+
 if __name__ == '__main__':
   unittest.main()

tests.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import logging
-import optparse
+import argparse
 import os
 import sys
 import unittest
@@ -12,7 +12,7 @@
 SDK_PATH    Path to Google Cloud or Google App Engine SDK installation, usually
             ~/google_cloud_sdk"""
 
-def main(sdk_path):
+def main(sdk_path, test_path, test_pattern):
   # If the sdk path points to a google cloud sdk installation
   # then we should alter it to point to the GAE platform location.
   if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')):
@@ -34,25 +34,33 @@ def main(sdk_path):
   except ImportError:
     print "Note: unable to import appengine_config."
 
-  test_path = os.path.dirname(sys.modules[__name__].__file__)
-
   logging.disable(logging.CRITICAL)
 
   from colour_runner import runner
 
   # Discover and run tests.
-  suite = unittest.loader.TestLoader().discover(test_path, pattern='*_test.py')
+  suite = unittest.loader.TestLoader().discover(test_path, test_pattern)
   result = runner.ColourTextTestRunner(verbosity=2).run(suite)
 
   if not result.wasSuccessful():
     sys.exit(result)
 
 if __name__ == '__main__':
-  parser = optparse.OptionParser(USAGE)
-  options, args = parser.parse_args()
-  if len(args) != 1:
-    print 'Error: Exactly 1 arguments required.'
-    parser.print_help()
-    sys.exit(1)
-  SDK_PATH = args[0]
-  main(SDK_PATH)
+  parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.add_argument(
+      'sdk_path',
+      help='The path to the Google App Engine SDK or the Google Cloud SDK.')
+  parser.add_argument(
+      '--test-path',
+      help='The path to look for tests, defaults to the current directory.',
+      default=os.getcwd())
+  parser.add_argument(
+      '--test-pattern',
+      help='The file pattern for test modules, defaults to *_test.py.',
+      default='*_test.py')
+
+  args = parser.parse_args()
+
+  main(args.sdk_path, args.test_path, args.test_pattern)