diff --git a/.gitignore b/.gitignore index c75f8c130cf2..e18b0fbc5938 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ sdist develop-eggs .installed.cfg node_modules +htmlcov .wake.json .sass-cache diff --git a/docs/api-reference/index.rst b/docs/api-reference/index.rst index ebc906d28dbd..aecca6bb3e31 100644 --- a/docs/api-reference/index.rst +++ b/docs/api-reference/index.rst @@ -4,6 +4,7 @@ API Reference .. toctree:: :maxdepth: 1 + xml-rpc legacy Application Structure diff --git a/docs/api-reference/xml-rpc.rst b/docs/api-reference/xml-rpc.rst new file mode 100644 index 000000000000..0fddb455bc40 --- /dev/null +++ b/docs/api-reference/xml-rpc.rst @@ -0,0 +1,173 @@ + +PyPI's XML-RPC methods +====================== + +Example usage:: + + >>> import xmlrpclib + >>> import pprint + >>> client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi') + >>> client.package_releases('roundup') + ['1.4.10'] + >>> pprint.pprint(client.release_urls('roundup', '1.4.10')) + [{'comment_text': '', + 'downloads': 3163, + 'filename': 'roundup-1.1.2.tar.gz', + 'has_sig': True, + 'md5_digest': '7c395da56412e263d7600fa7f0afa2e5', + 'packagetype': 'sdist', + 'python_version': 'source', + 'size': 876455, + 'upload_time': , + 'url': 'https://pypi.python.org/packages/source/r/roundup/roundup-1.1.2.tar.gz'}, + {'comment_text': '', + 'downloads': 2067, + 'filename': 'roundup-1.1.2.win32.exe', + 'has_sig': True, + 'md5_digest': '983d565b0b87f83f1b6460e54554a845', + 'packagetype': 'bdist_wininst', + 'python_version': 'any', + 'size': 614270, + 'upload_time': , + 'url': 'https://pypi.python.org/packages/any/r/roundup/roundup-1.1.2.win32.exe'}] + +Changes to Legacy API +--------------------- + +``package_releases`` "show_hidden" flag is now ignored. All versions are +returned. + +``release_data`` "stable_version" is always an empty string. It was never +fully supported anyway. + + +Package Querying +---------------- + +``list_packages()`` + Retrieve a list of the package names registered with the package index. + Returns a list of name strings. + +``package_releases(package_name, show_hidden=False)`` + Retrieve a list of the releases registered for the given package_name, + ordered by version. + + The "show_hidden" flag is now ignored. All versions are returned. + +``package_roles(package_name)`` + Retrieve a list of users and their attributes roles for a given package_name. + Role is either 'Maintainer' or 'Owner'. + +``user_packages(user)`` + Retrieve a list of [role_name, package_name] for a given username. + Role is either 'Maintainer' or 'Owner'. + +``release_downloads(package_name, version)`` + Retrieve a list of files and download count for a given package and release + version. + +``release_urls(package_name, version)`` + Retrieve a list of download URLs for the given package release. + Returns a list of dicts with the following keys: + - url + - packagetype ('sdist', 'bdist', etc) + - filename + - size + - md5_digest + - downloads + - has_sig + - python_version (required version, or 'source', or 'any') + - comment_text + +``release_data(package_name, version)`` + Retrieve metadata describing a specific package release. + Returns a dict with keys for: + - name + - version + - stable_version (always an empty string) + - author + - author_email + - maintainer + - maintainer_email + - home_page + - license + - summary + - description + - keywords + - platform + - download_url + - classifiers (list of classifier strings) + - requires + - requires_dist + - provides + - provides_dist + - requires_external + - requires_python + - obsoletes + - obsoletes_dist + - project_url + - docs_url (URL of the packages.python.org docs if they've been supplied) + If the release does not exist, an empty dictionary is returned. + +``search(spec[, operator])`` + Search the package database using the indicated search spec. + + The spec may include any of the keywords described in the above list (except + 'stable_version' and 'classifiers'), for example: {'description': 'spam'} + will search description fields. Within the spec, a field's value can be a + string or a list of strings (the values within the list are combined with an + OR), for example: {'name': ['foo', 'bar']}. Valid keys for the spec dict are + listed here. Invalid keys are ignored: + - name + - version + - author + - author_email + - maintainer + - maintainer_email + - home_page + - license + - summary + - description + - keywords + - platform + - download_url + + Arguments for different fields are combined using either "and" (the default) + or "or". Example: search({'name': 'foo', 'description': 'bar'}, 'or'). The + results are returned as a list of dicts {'name': package name, 'version': + package release version, 'summary': package release summary} + +``browse(classifiers)`` + Retrieve a list of (name, version) pairs of all releases classified with all + of the given classifiers. 'classifiers' must be a list of Trove classifier + strings. + +``top_packages([number])`` + Retrieve the sorted list of packages ranked by number of downloads. + Optionally limit the list to the number given. + +``updated_releases(since)`` + Retrieve a list of package releases made since the given timestamp. The + releases will be listed in descending release date. + + +Mirroring Support +----------------- + +``changelog(since, with_ids=False)`` + Retrieve a list of four-tuples (name, version, timestamp, action), or + five-tuple including the serial id if ids are requested, since the given + timestamp. All timestamps are UTC values. The argument is a UTC integer + seconds since the epoch. + +``changelog_last_serial()`` + Retrieve the last event's serial id. + +``changelog_since_serial(since_serial)`` + Retrieve a list of five-tuples (name, version, timestamp, action, serial) + since the event identified by the given serial. All timestamps are UTC + values. The argument is a UTC integer seconds since the epoch. + +``list_packages_with_serial()`` + Retrieve a dictionary mapping package names to the last serial for each + package. diff --git a/tests/legacy/test_xmlrpc.py b/tests/legacy/test_xmlrpc.py index da314dcfd45f..590b9e5e16b3 100644 --- a/tests/legacy/test_xmlrpc.py +++ b/tests/legacy/test_xmlrpc.py @@ -14,6 +14,8 @@ from __future__ import absolute_import, division, print_function from __future__ import unicode_literals +import datetime + import pretend import pytest @@ -66,6 +68,20 @@ def test_xmlrpc_handler(monkeypatch): assert Response.calls[0].kwargs == dict(mimetype='text/xml') +def test_xmlrpc_handler_size_limit(monkeypatch): + app = pretend.stub() + + request = pretend.stub( + headers={ + 'Content-Type': 'text/xml', + 'Content-Length': str(10 * 1024 * 1024 + 1) + }, + ) + + with pytest.raises(BadRequest): + xmlrpc.handle_request(app, request) + + def test_xmlrpc_list_packages(): all_projects = [Project("bar"), Project("foo")] @@ -76,11 +92,8 @@ def test_xmlrpc_list_packages(): ), ), ) - request = pretend.stub( - headers={'Content-Type': 'text/xml'} - ) - interface = xmlrpc.Interface(app, request) + interface = xmlrpc.Interface(app, pretend.stub()) result = interface.list_packages() @@ -88,15 +101,317 @@ def test_xmlrpc_list_packages(): assert result == ['bar', 'foo'] -def test_xmlrpc_size(monkeypatch): - app = pretend.stub() +@pytest.mark.parametrize(("num", "result"), [ + (None, [('three', 10000), ('one', 1110), ('two', 22)]), + (2, [('three', 10000), ('one', 1110)]), +]) +def test_xmlrpc_top_packages(num, result): + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_top_projects=pretend.call_recorder(lambda *a: result), + ), + ), + ) - request = pretend.stub( - headers={ - 'Content-Type': 'text/xml', - 'Content-Length': str(10 * 1024 * 1024 + 1) - }, + interface = xmlrpc.Interface(app, pretend.stub()) + + if num: + r = interface.top_packages(num) + assert app.models.packaging.get_top_projects.calls == [ + pretend.call(num) + ] + else: + r = interface.top_packages() + assert app.models.packaging.get_top_projects.calls == [ + pretend.call(None) + ] + + assert r == result + + +def test_xmlrpc_package_releases(): + result = ['1', '2', '3', '4'] + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_project_versions=pretend.call_recorder(lambda *a: result), + ), + ), ) - with pytest.raises(BadRequest): - xmlrpc.handle_request(app, request) + interface = xmlrpc.Interface(app, pretend.stub()) + + assert interface.package_releases('name') == ['1', '2', '3', '4'] + + assert app.models.packaging.get_project_versions.calls == [ + pretend.call('name') + ] + + +@pytest.mark.parametrize("with_ids", [False, True]) +def test_xmlrpc_changelog(with_ids): + now = datetime.datetime.now() + old = datetime.datetime.now() - datetime.timedelta(days=1) + now_plus_1 = datetime.datetime.now() + datetime.timedelta(days=1) + now_plus_2 = datetime.datetime.now() + datetime.timedelta(days=2) + data = [ + dict(name='one', version='1', submitted_date=now, + action='created', id=1), + dict(name='two', version='2', submitted_date=now, + action='new release', id=2), + dict(name='one', version='2', submitted_date=now_plus_1, + action='new release', id=3), + dict(name='one', version='3', submitted_date=now_plus_2, + action='new release', id=4), + ] + result = [ + ('one', '1', now, 'created', 1), + ('two', '2', now, 'new release', 2), + ('one', '2', now_plus_1, 'new release', 3), + ('one', '3', now_plus_2, 'new release', 4), + ] + if not with_ids: + result = [r[:4] for r in result] + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_changelog=pretend.call_recorder(lambda *a: data), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + assert interface.changelog(old, with_ids) == result + + assert app.models.packaging.get_changelog.calls == [ + pretend.call(old) + ] + + +def test_xmlrpc_updated_releases(): + now = datetime.datetime.now() + + result = [ + dict(name='one', version='1', created=now, summary='text'), + dict(name='two', version='2', created=now, summary='text'), + dict(name='two', version='3', created=now, summary='text'), + dict(name='three', version='4', created=now, summary='text')] + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_releases_since=pretend.call_recorder(lambda *a: result), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + old = now = datetime.timedelta(days=1) + assert interface.updated_releases(old) == \ + [('one', '1'), ('two', '2'), ('two', '3'), ('three', '4')] + + assert app.models.packaging.get_releases_since.calls == [ + pretend.call(old) + ] + + +def test_xmlrpc_update_releases(): + now = datetime.datetime.now() + + result = [ + dict(name='one', version='1', created=now, summary='text'), + dict(name='two', version='2', created=now, summary='text'), + dict(name='two', version='3', created=now, summary='text'), + dict(name='three', version='4', created=now, summary='text')] + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_releases_since=pretend.call_recorder(lambda *a: result), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + old = now = datetime.timedelta(days=1) + assert interface.updated_releases(old) == \ + [('one', '1'), ('two', '2'), ('two', '3'), ('three', '4')] + + assert app.models.packaging.get_releases_since.calls == [ + pretend.call(old) + ] + + +def test_xmlrpc_list_packages_with_serial(): + d = dict(one=1, two=2, three=3) + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_projects_with_serial=pretend.call_recorder(lambda: d), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + result = interface.list_packages_with_serial() + + assert app.models.packaging.get_projects_with_serial.calls == [ + pretend.call(), + ] + assert result == d + + +@pytest.mark.parametrize("pgp", [True, False]) +def test_release_urls(pgp, monkeypatch): + downloads = [ + dict( + name="spam", + url='/packages/source/t/spam/spam-1.0.tar.gz', + version="1.0", + filename="spam-1.0.tar.gz", + python_version="source", + packagetype="sdist", + md5_digest="0cc175b9c0f1b6a831c399e269772661", + downloads=10, + size=1234, + pgp_url='/packages/source/t/spam/spam-1.0.tar.gz.sig' + if pgp else None, + comment_text='download for great justice', + ), + dict( + name="spam", + url='/packages/source/t/spam/spam-1.0.zip', + version="1.0", + filename="spam-1.0.zip", + python_version="source", + packagetype="sdist", + md5_digest="0cc175b3c0f1b6a831c399e269772661", + downloads=12, + size=1235, + pgp_url='/packages/source/t/spam/spam-1.0.zip.sig' + if pgp else None, + comment_text=None, + ) + ] + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_downloads=pretend.call_recorder(lambda *a: downloads), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + result = interface.release_urls('spam', '1.0') + + assert app.models.packaging.get_downloads.calls == [ + pretend.call('spam', '1.0'), + ] + assert result == [ + dict( + url='/packages/source/t/spam/spam-1.0.tar.gz', + packagetype="sdist", + filename="spam-1.0.tar.gz", + size=1234, + md5_digest="0cc175b9c0f1b6a831c399e269772661", + downloads=10, + has_sig=pgp, + python_version="source", + comment_text='download for great justice', + ), + dict( + url='/packages/source/t/spam/spam-1.0.zip', + packagetype="sdist", + filename="spam-1.0.zip", + size=1235, + md5_digest="0cc175b3c0f1b6a831c399e269772661", + downloads=12, + has_sig=pgp, + python_version="source", + comment_text=None, + ) + ] + + +def test_release_data(monkeypatch): + resp = dict( + name="spam", + version="1.0", + author="John Doe", + author_email="john.doe@example.com", + maintainer=None, + maintainer_email=None, + home_page="https://example.com/", + license="Apache License v2.0", + summary="A Test Project", + description="A Longer Test Project", + keywords="foo,bar,wat", + platform="All", + download_url="https://example.com/downloads/test-project-1.0.tar.gz", + requires_dist=["requests (>=2.0)"], + provides_dist=["test-project-old"], + project_url={"Repository": "git://git.example.com/"}, + created=datetime.datetime.utcnow(), + ) + # snapshot that info now for comparison later + info = dict(resp) + docs = "https://pythonhosted.org/spam/" + cfiers = ['Section A :: Subsection B :: Aisle 3', 'Section B'] + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_release=pretend.call_recorder(lambda *a: resp), + get_documentation_url=pretend.call_recorder(lambda *a: docs), + get_download_counts=pretend.call_recorder(lambda *a: 10), + get_classifiers=pretend.call_recorder(lambda *a: cfiers), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + result = interface.release_data('spam', '1.0') + + assert app.models.packaging.get_release.calls == [ + pretend.call('spam', '1.0'), + ] + + # modify the model response data according to the expected mutation + info.update( + package_url='http://pypi.python.org/pypi/spam', + release_url='http://pypi.python.org/pypi/spam/1.0', + docs_url=docs, + downloads=10, + classifiers=cfiers, + maintainer='', # converted from None + maintainer_email='', # converted from None + stable_version='', # filled in as no-op + ) + assert result == info + + +def test_release_data_missing(monkeypatch): + def f(*a): + raise IndexError() + + app = pretend.stub( + models=pretend.stub( + packaging=pretend.stub( + get_release=pretend.call_recorder(f), + ), + ), + ) + + interface = xmlrpc.Interface(app, pretend.stub()) + + result = interface.release_data('spam', '1.0') + + assert app.models.packaging.get_release.calls == [ + pretend.call('spam', '1.0'), + ] + + assert result == {} diff --git a/tests/packaging/test_models.py b/tests/packaging/test_models.py index e9d184dd9d0b..fcbc1c6e9667 100644 --- a/tests/packaging/test_models.py +++ b/tests/packaging/test_models.py @@ -188,6 +188,114 @@ def test_get_recently_updated(dbapp): ] +def test_get_releases_since(dbapp): + dbapp.engine.execute(packages.insert().values(name="foo1")) + dbapp.engine.execute(packages.insert().values(name="foo2")) + dbapp.engine.execute(packages.insert().values(name="foo3")) + + now = datetime.datetime.utcnow() + + dbapp.engine.execute(releases.insert().values( + name="foo2", version="1.0", + created=now - datetime.timedelta(seconds=10), + )) + dbapp.engine.execute(releases.insert().values( + name="foo3", version="2.0", + created=now - datetime.timedelta(seconds=9), + )) + dbapp.engine.execute(releases.insert().values( + name="foo1", version="1.0", + created=now - datetime.timedelta(seconds=4), + )) + dbapp.engine.execute(releases.insert().values( + name="foo3", version="1.0", + created=now - datetime.timedelta(seconds=3), + )) + dbapp.engine.execute(releases.insert().values( + name="foo1", version="2.0", created=now, + )) + + since = now - datetime.timedelta(seconds=5) + assert dbapp.models.packaging.get_releases_since(since) == [ + { + "name": "foo1", + "version": "2.0", + "summary": None, + "created": now, + }, + { + "name": "foo3", + "version": "1.0", + "summary": None, + "created": now - datetime.timedelta(seconds=3), + }, + { + "name": "foo1", + "version": "1.0", + "summary": None, + "created": now - datetime.timedelta(seconds=4), + }, + ] + + +def test_get_changelog(dbapp): + now = datetime.datetime.utcnow() + + def create(name, delta): + dbapp.engine.execute(packages.insert().values(name=name)) + dbapp.engine.execute(journals.insert().values(name=name, version=None, + submitted_date=now - delta, action="create", id=create.id)) + create.id += 1 + create.id = 1 + create("foo1", datetime.timedelta(seconds=4)) + create("foo2", datetime.timedelta(seconds=5)) + create("foo3", datetime.timedelta(seconds=10)) + + def release(name, version, delta): + dbapp.engine.execute(releases.insert().values(name=name, + version=version, created=now - delta)) + dbapp.engine.execute(journals.insert().values(id=create.id, name=name, + version=version, submitted_date=now - delta, action="new release")) + create.id += 1 + release("foo2", "1.0", datetime.timedelta(seconds=10)) + release("foo3", "2.0", datetime.timedelta(seconds=9)) + release("foo1", "1.0", datetime.timedelta(seconds=3)) + release("foo3", "1.0", datetime.timedelta(seconds=2)) + release("foo1", "2.0", datetime.timedelta(seconds=1)) + + since = now - datetime.timedelta(seconds=5) + assert dbapp.models.packaging.get_changelog(since) == [ + { + "name": "foo1", + "version": "2.0", + "action": "new release", + "submitted_date": now - datetime.timedelta(seconds=1), + "id": 8, + }, + { + "name": "foo3", + "version": "1.0", + "action": "new release", + "submitted_date": now - datetime.timedelta(seconds=2), + "id": 7, + }, + { + "name": "foo1", + "version": "1.0", + "action": "new release", + "submitted_date": now - datetime.timedelta(seconds=3), + "id": 6, + }, + { + "name": "foo1", + "version": None, + "action": "create", + "submitted_date": now - datetime.timedelta(seconds=4), + "id": 1, + }, + ] + + @pytest.mark.parametrize("projects", [ ["foo", "bar", "zap"], ["fail", "win", "YeS"], @@ -203,6 +311,28 @@ def test_all_projects(projects, dbapp): assert dbapp.models.packaging.all_projects() == all_projects +@pytest.mark.parametrize(("num", "result"), [ + (None, [('three', 10000), ('one', 1110), ('two', 22)]), + (2, [('three', 10000), ('one', 1110)]), +]) +def test_top_projects(num, result, dbapp): + # Insert some data into the database + files = [ + ('one', 10, 'one-1.0.zip'), + ('one', 100, 'one-1.1.zip'), + ('one', 1000, 'one-1.2.zip'), + ('two', 2, 'two-1.0.zip'), + ('two', 20, 'two-1.2.zip'), + ('three', 10000, 'three-1.0.zip'), + ] + for name, downloads, filename in files: + dbapp.engine.execute(release_files.insert().values(name=name, + downloads=downloads, filename=filename)) + + top = dbapp.models.packaging.get_top_projects(num) + assert top == result + + @pytest.mark.parametrize(("name", "normalized"), [ ("foo_bar", "foo-bar"), ("Bar", "bar"), @@ -461,6 +591,18 @@ def test_get_last_serial(name, serial, dbapp): assert dbapp.models.packaging.get_last_serial(name) == serial +def test_get_projects_with_serial(dbapp): + dbapp.engine.execute(journals.insert().values(id=1, name='one')) + dbapp.engine.execute(journals.insert().values(id=2, name='two')) + dbapp.engine.execute(journals.insert().values(id=3, name='three')) + + assert dbapp.models.packaging.get_projects_with_serial() == dict( + one=1, + two=2, + three=3 + ) + + def test_get_project_versions(dbapp): dbapp.engine.execute(packages.insert().values(name="test-project")) dbapp.engine.execute(releases.insert().values( @@ -478,12 +620,14 @@ def test_get_project_versions(dbapp): version="3.0", _pypi_ordering=3, )) + dbapp.engine.execute(releases.insert().values( + name="test-project", + version="4.0", + _pypi_ordering=4, + )) - assert dbapp.models.packaging.get_project_versions("test-project") == [ - "3.0", - "2.0", - "1.0", - ] + assert dbapp.models.packaging.get_project_versions("test-project") == \ + ["4.0", "3.0", "2.0", "1.0"] def test_get_release(dbapp): diff --git a/warehouse/legacy/xmlrpc.py b/warehouse/legacy/xmlrpc.py index e0f3a97641f7..b88f982aae5a 100644 --- a/warehouse/legacy/xmlrpc.py +++ b/warehouse/legacy/xmlrpc.py @@ -47,10 +47,65 @@ def __init__(self, app, request): self.request = request def list_packages(self): - '''Retrieve a list of the package names registered with the package - index. - - Returns a list of name strings. - ''' projects = self.app.models.packaging.all_projects() return [project.name for project in projects] + + def list_packages_with_serial(self): + return self.app.models.packaging.get_projects_with_serial() + + def top_packages(self, num=None): + return self.app.models.packaging.get_top_projects(num) + + def package_releases(self, name, show_hidden=False): + return self.app.models.packaging.get_project_versions(name) + + def updated_releases(self, since): + result = self.app.models.packaging.get_releases_since(since) + return [(row['name'], row['version']) for row in result] + + def changelog(self, since, with_ids=False): + result = self.app.models.packaging.get_changelog(since) + keys = 'name version submitted_date action'.split() + if with_ids: + keys.append('id') + return [tuple(row[key] for key in keys) for row in result] + + def release_urls(self, name, version): + l = [] + for r in self.app.models.packaging.get_downloads(name, version): + l.append(dict( + url=r['url'], + packagetype=r['packagetype'], + filename=r['filename'], + size=r['size'], + md5_digest=r['md5_digest'], + downloads=r['downloads'], + has_sig=r['pgp_url'] is not None, + python_version=r['python_version'], + comment_text=r['comment_text'], + )) + return l + + def release_data(self, name, version): + model = self.app.models.packaging + try: + info = model.get_release(name, version) + except IndexError: + # the CURRENT model code will raise an IndexError on missing + # package but this should be altered + return {} + + info['stable_version'] = '' # legacy; never actually correct + info['classifiers'] = model.get_classifiers(name, version) + info['package_url'] = 'http://pypi.python.org/pypi/%s' % name + info['release_url'] = 'http://pypi.python.org/pypi/%s/%s' % (name, + version) + info['docs_url'] = model.get_documentation_url(name) + info['downloads'] = model.get_download_counts(name) + + # make the data XML-RPC-happy (no explicit null allowed here!) + for k in info: + if info[k] is None: + info[k] = '' + + return info diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 637f43db8c89..b566cb9ef06f 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -66,12 +66,36 @@ def get_recently_updated(self, num=10): with self.engine.connect() as conn: return [dict(r) for r in conn.execute(query, num=num)] + def get_releases_since(self, since): + query = \ + """ SELECT name, version, created, summary + FROM releases + WHERE created > %(since)s + ORDER BY created DESC + """ + + with self.engine.connect() as conn: + return [dict(r) for r in conn.execute(query, since=since)] + def all_projects(self): query = "SELECT name FROM packages ORDER BY lower(name)" with self.engine.connect() as conn: return [Project(r["name"]) for r in conn.execute(query)] + def get_top_projects(self, num=None): + query = \ + """ SELECT name, sum(downloads) + FROM release_files + GROUP BY name + ORDER BY sum(downloads) DESC + """ + if num: + query += "LIMIT %(limit)s" + + with self.engine.connect() as conn: + return [tuple(r) for r in conn.execute(query, limit=num)] + def get_project(self, name): query = \ """ SELECT name @@ -208,6 +232,13 @@ def get_last_serial(self, name=None): with self.engine.connect() as conn: return conn.execute(query, name=name).scalar() + def get_projects_with_serial(self): + # return list of dict(name: max id) + query = "SELECT name, max(id) FROM journals GROUP BY name" + + with self.engine.connect() as conn: + return dict(r for r in conn.execute(query)) + def get_project_versions(self, project): query = \ """ SELECT version @@ -440,3 +471,15 @@ def get_bugtrack_url(self, project): with self.engine.connect() as conn: return conn.execute(query, project=project).scalar() + + # + # Mirroring support + # + def get_changelog(self, since): + query = '''SELECT name, version, submitted_date, action, id + FROM journals + WHERE journals.submitted_date > %(since)s + ORDER BY submitted_date DESC + ''' + with self.engine.connect() as conn: + return [dict(r) for r in conn.execute(query, since=since)]