diff --git a/client/src/catalog-item.html b/client/src/catalog-item.html index 06e38c2c..fecfb343 100644 --- a/client/src/catalog-item.html +++ b/client/src/catalog-item.html @@ -96,7 +96,7 @@ - + [[data.repo]] @@ -129,6 +129,12 @@ [[data.repo]] data: Object, }, + _packagePath: function(data) { + if (data.npmFullPackage) + return data.kind + '/' + data.npmFullPackage; + return [data.kind, data.owner, data.repo].join('/'); + }, + _versionPath: function(version, latestVersion) { return version == latestVersion ? '' : '/' + version; }, diff --git a/src/api.py b/src/api.py index be27351b..33234f59 100644 --- a/src/api.py +++ b/src/api.py @@ -94,9 +94,13 @@ def brief_async(library_key, tag=None, assume_latest=False): 'subscribers': metadata['subscribers'], 'forks': metadata['forks'], 'updated_at': metadata['updated_at'], - 'dependency_count': metadata['dependency_count'], 'avatar_url': metadata['avatar_url'], } + if 'dependency_count' in metadata: + result['dependency_count'] = metadata['dependency_count'] + if 'npmFullPackage' in metadata: + result['npmFullPackage'] = metadata['npmFullPackage'] + if not assume_latest: result['latest_version'] = metadata['latest_version'] result['default_version'] = metadata['default_version'] @@ -137,7 +141,11 @@ def full_async(library_key, tag=None, brief=False, assume_latest=False): parts = key.split('/') if parts[0] != '@@npm': result['npmScope'] = parts[0] + result['npmFullPackage'] = key + else: + result['npmFullPackage'] = parts[1] result['npmPackage'] = parts[1] + result['apiKey'] = key result['kind'] = library.kind result['status'] = library.status diff --git a/src/manage.py b/src/manage.py index 07ddbd83..adb349dd 100644 --- a/src/manage.py +++ b/src/manage.py @@ -481,12 +481,13 @@ def update_package_tags(self): tags = versions.keys() tags.sort(versiontag.compare) # Create a tag map of tag to sha - tag_map = dict((tag, versions.get(tag).get('_shasum', '')) for tag in tags) + tag_map = dict((tag, versions.get(tag).get('gitHead', '')) for tag in tags + if versiontag.is_valid(tag) and versions.get(tag).get('gitHead')) if self.library.tags is None or self.library.tags != tags: self.library.library_dirty = True self.library.tags = tags - self.library.tags_map = json.dumps(tag_map) + self.library.tag_map = json.dumps(tag_map) self.library.tags_updated = datetime.datetime.now() return tag_map @@ -850,6 +851,9 @@ def update_collection_dependencies(self, collection_version_key, bower): def update_search_index(self, owner, repo, version_key, library, bower): metadata = json.loads(library.metadata) + registry_metadata = json.loads(library.registry_metadata) if library.registry_metadata else None + npm_description = registry_metadata.get('description', '') if registry_metadata else '' + npm_keywords = registry_metadata.get('keywords', []) if registry_metadata else [] fields = [ search.AtomField(name='owner', value=owner), search.TextField(name='repo', value=repo), @@ -857,7 +861,9 @@ def update_search_index(self, owner, repo, version_key, library, bower): search.AtomField(name='version', value=version_key.id()), search.TextField(name='github_description', value=metadata.get('description', '')), search.TextField(name='bower_description', value=bower.get('description', '')), + search.TextField(name='npm_description', value=npm_description), search.TextField(name='bower_keywords', value=' '.join(bower.get('keywords', []))), + search.TextField(name='npm_keywords', value=' '.join(npm_keywords)), search.TextField(name='prefix_matches', value=' '.join(util.generate_prefixes_from_list( util.safe_split_strip(metadata.get('description')) + util.safe_split_strip(bower.get('description')) + util.safe_split_strip(repo)))), diff --git a/src/manage_test.py b/src/manage_test.py index 6de432c7..138848a8 100644 --- a/src/manage_test.py +++ b/src/manage_test.py @@ -789,7 +789,7 @@ def test_ingest_version_pages(self): class IngestNPMLibraryTest(ManageTestBase): def test_ingest_element(self): - self.respond_to('https://registry.npmjs.org/@scope%2fpackage', '{"repository": {"url": "git+https://github.com/org/repo.git"}, "license": "BSD-3-Clause", "versions": {"1.0.0": {"_shasum": "lol"}}}') + self.respond_to('https://registry.npmjs.org/@scope%2fpackage', '{"repository": {"url": "git+https://github.com/org/repo.git"}, "license": "BSD-3-Clause", "versions": {"1.0.0": {"gitHead": "lol"}}}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"org"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') @@ -828,7 +828,7 @@ def test_ingest_no_package(self): self.assertEqual(len(tasks), 0) def test_ingest_repository_shorthand(self): - self.respond_to('https://registry.npmjs.org/package', '{"repository": "org/repo", "license": "BSD-3-Clause", "versions": {"1.0.0": {"_shasum": "lol"}, "0.5.0": {"_shasum": "sha"}}}') + self.respond_to('https://registry.npmjs.org/package', '{"repository": "org/repo", "license": "BSD-3-Clause", "versions": {"1.0.0": {"gitHead": "lol"}, "0.5.0": {"gitHead": "sha"}}}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"org"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') @@ -985,5 +985,34 @@ def test_analyzer_index_empty(self): behaviors = [field for field in document.fields if field.name == 'behavior'] self.assertEqual(len(behaviors), 0) + def test_npm_index(self): + metadata = """{ + "full_name": "full-name" + }""" + registry_metadata = """{ + "description": "mydescription", + "keywords": ["my-keyword"] + }""" + library_key = Library(id='@@npm/package', registry_metadata=registry_metadata, metadata=metadata).put() + version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() + + Content(id='bower', parent=version_key, content="""{"dependencies": { + "a": "org/element-1#1.0.0", + "b": "org/element-2#1.0.0" + }}""").put() + + VersionCache.update(library_key) + + response = self.app.get(util.update_indexes_task('@@npm', 'package'), headers={'X-AppEngine-QueueName': 'default'}) + self.assertEqual(response.status_int, 200) + + index = search.Index('repo') + document = index.get('@@npm/package') + self.assertIsNotNone(document) + self.assertTrue(len(document.fields) > 0) + + self.assertEqual(document.field('npm_keywords').value, 'my-keyword') + self.assertEqual(document.field('npm_description').value, 'mydescription') + if __name__ == '__main__': unittest.main() diff --git a/tests.py b/tests.py index a3dc822f..aa48f152 100644 --- a/tests.py +++ b/tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import logging -import optparse +import argparse import os import sys import unittest @@ -12,7 +12,7 @@ SDK_PATH Path to Google Cloud or Google App Engine SDK installation, usually ~/google_cloud_sdk""" -def main(sdk_path): +def main(sdk_path, test_path, test_pattern): # If the sdk path points to a google cloud sdk installation # then we should alter it to point to the GAE platform location. if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')): @@ -34,25 +34,33 @@ def main(sdk_path): except ImportError: print "Note: unable to import appengine_config." - test_path = os.path.dirname(sys.modules[__name__].__file__) - logging.disable(logging.CRITICAL) from colour_runner import runner # Discover and run tests. - suite = unittest.loader.TestLoader().discover(test_path, pattern='*_test.py') + suite = unittest.loader.TestLoader().discover(test_path, test_pattern) result = runner.ColourTextTestRunner(verbosity=2).run(suite) if not result.wasSuccessful(): sys.exit(result) if __name__ == '__main__': - parser = optparse.OptionParser(USAGE) - options, args = parser.parse_args() - if len(args) != 1: - print 'Error: Exactly 1 arguments required.' - parser.print_help() - sys.exit(1) - SDK_PATH = args[0] - main(SDK_PATH) + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'sdk_path', + help='The path to the Google App Engine SDK or the Google Cloud SDK.') + parser.add_argument( + '--test-path', + help='The path to look for tests, defaults to the current directory.', + default=os.getcwd()) + parser.add_argument( + '--test-pattern', + help='The file pattern for test modules, defaults to *_test.py.', + default='*_test.py') + + args = parser.parse_args() + + main(args.sdk_path, args.test_path, args.test_pattern)