Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Implement Legacy RSS Feeds #207

Merged
merged 3 commits into from

1 participant

Donald Stufft
Donald Stufft
Owner

Rebases and closes #175 to update it to the latest changes in Warehouse as well as fix some minor issues with it.

Donald Stufft dstufft merged commit a06575b into from
Donald Stufft dstufft deleted the branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 28, 2014
  1. Donald Stufft

    implement legacy RSS feeds

    Richard Jones authored dstufft committed
  2. Donald Stufft
  3. Donald Stufft
This page is out of date. Refresh to see the latest.
2  dev/config.yml
View
@@ -30,6 +30,7 @@ cache:
project_detail: 1
user_profile: 1
legacy_json: 1
+ legacy_rss: 1
varnish:
index: 120
simple: 120
@@ -37,6 +38,7 @@ cache:
project_detail: 120
user_profile: 120
legacy_json: 120
+ legacy_rss: 120
security:
184 tests/legacy/test_pypi.py
View
@@ -19,9 +19,14 @@
import pretend
import pytest
+
+import jinja2
+
from werkzeug.exceptions import NotFound, BadRequest
+from werkzeug.routing import Map
from warehouse.legacy import pypi, xmlrpc
+from warehouse.packaging import urls
@pytest.mark.parametrize("content_type", [None, "text/html", "__empty__"])
@@ -151,3 +156,182 @@ def test_json_missing(monkeypatch, project):
with pytest.raises(NotFound):
pypi.project_json(app, request, project_name='spam')
+
+
+def test_rss(monkeypatch):
+ get_recently_updated = pretend.call_recorder(lambda num=10: [
+ dict(name='spam', version='1.0', summary='hai spam', created='now'),
+ dict(name='ham', version='2.0', summary='hai ham', created='now'),
+ dict(name='spam', version='2.0', summary='hai spam v2', created='now'),
+ ])
+ template = pretend.stub(
+ render=pretend.call_recorder(lambda **ctx: "<xml>dummy</xml>"),
+ )
+ app = pretend.stub(
+ db=pretend.stub(
+ packaging=pretend.stub(
+ get_recently_updated=get_recently_updated,
+ )
+ ),
+ config=pretend.stub(
+ cache=pretend.stub(browser=False, varnish=False),
+ site={"url": "http://test.server/", "name": "PyPI"},
+ ),
+ urls=Map(urls.urls).bind('test.server', '/'),
+ templates=pretend.stub(
+ get_template=pretend.call_recorder(lambda t: template),
+ ),
+ )
+ request = pretend.stub()
+
+ resp = pypi.rss(app, request)
+
+ assert get_recently_updated.calls == [pretend.call(num=40)]
+ assert len(template.render.calls) == 1
+ assert template.render.calls[0].kwargs['releases'] == [
+ {
+ 'url': 'http://test.server/project/spam/1.0/',
+ 'version': u'1.0',
+ 'name': u'spam',
+ 'summary': u'hai spam',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/ham/2.0/',
+ 'version': u'2.0',
+ 'name': u'ham',
+ 'summary': u'hai ham',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/spam/2.0/',
+ 'version': u'2.0',
+ 'name': u'spam',
+ 'summary': u'hai spam v2',
+ 'created': u'now',
+ }]
+ assert resp.data == "<xml>dummy</xml>"
+
+
+def test_packages_rss(monkeypatch):
+ get_recent_projects = pretend.call_recorder(lambda num=10: [
+ dict(name='spam', version='1.0', summary='hai spam', created='now'),
+ dict(name='ham', version='2.0', summary='hai ham', created='now'),
+ dict(name='eggs', version='21.0', summary='hai eggs!', created='now'),
+ ])
+ template = pretend.stub(
+ render=pretend.call_recorder(lambda **ctx: "<xml>dummy</xml>"),
+ )
+ app = pretend.stub(
+ db=pretend.stub(
+ packaging=pretend.stub(
+ get_recent_projects=get_recent_projects,
+ )
+ ),
+ config=pretend.stub(
+ cache=pretend.stub(browser=False, varnish=False),
+ site={"url": "http://test.server/", "name": "PyPI"},
+ ),
+ urls=Map(urls.urls).bind('test.server', '/'),
+ templates=pretend.stub(
+ get_template=pretend.call_recorder(lambda t: template),
+ ),
+ )
+ request = pretend.stub()
+
+ resp = pypi.packages_rss(app, request)
+
+ assert get_recent_projects.calls == [pretend.call(num=40)]
+ assert len(template.render.calls) == 1
+ assert template.render.calls[0].kwargs['releases'] == [
+ {
+ 'url': 'http://test.server/project/spam/',
+ 'version': u'1.0',
+ 'name': u'spam',
+ 'summary': u'hai spam',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/ham/',
+ 'version': u'2.0',
+ 'name': u'ham',
+ 'summary': u'hai ham',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/eggs/',
+ 'version': u'21.0',
+ 'name': u'eggs',
+ 'summary': u'hai eggs!',
+ 'created': u'now',
+ }]
+ assert resp.data == "<xml>dummy</xml>"
+
+
+def test_rss_xml_template(monkeypatch):
+ templates = jinja2.Environment(
+ autoescape=True,
+ auto_reload=False,
+ extensions=[
+ "jinja2.ext.i18n",
+ ],
+ loader=jinja2.PackageLoader("warehouse"),
+ )
+ template = templates.get_template('legacy/rss.xml')
+ content = template.render(
+ site=dict(url='http://test.server/', name="PyPI"),
+ description='package updates',
+ releases=[
+ {
+ 'url': 'http://test.server/project/spam/',
+ 'version': u'1.0',
+ 'name': u'spam',
+ 'summary': u'hai spam',
+ 'created': datetime.date(1970, 1, 1),
+ }, {
+ 'url': 'http://test.server/project/ham/',
+ 'version': u'2.0',
+ 'name': u'ham',
+ 'summary': u'hai ham',
+ 'created': datetime.date(1970, 1, 1),
+ }, {
+ 'url': 'http://test.server/project/eggs/',
+ 'version': u'21.0',
+ 'name': u'eggs',
+ 'summary': u'hai eggs!',
+ 'created': datetime.date(1970, 1, 1),
+ }
+ ],
+ )
+ assert content == '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" \
+"http://my.netscape.com/publish/formats/rss-0.91.dtd">
+<rss version="0.91">
+ <channel>
+ <title>PyPI Recent Package Updates</title>
+ <link>http://test.server/</link>
+ <description>Recent package updates at PyPI</description>
+ <language>en</language>
+ \n\
+ <item>
+ <title>spam 1.0</title>
+ <link>http://test.server/project/spam/</link>
+ <guid>http://test.server/project/spam/</guid>
+ <description>hai spam</description>
+ <pubDate>01 Jan 1970 00:00:00 GMT</pubDate>
+ </item>
+ \n\
+ <item>
+ <title>ham 2.0</title>
+ <link>http://test.server/project/ham/</link>
+ <guid>http://test.server/project/ham/</guid>
+ <description>hai ham</description>
+ <pubDate>01 Jan 1970 00:00:00 GMT</pubDate>
+ </item>
+ \n\
+ <item>
+ <title>eggs 21.0</title>
+ <link>http://test.server/project/eggs/</link>
+ <guid>http://test.server/project/eggs/</guid>
+ <description>hai eggs!</description>
+ <pubDate>01 Jan 1970 00:00:00 GMT</pubDate>
+ </item>
+ \n\
+ </channel>
+</rss>'''
50 tests/packaging/test_db.py
View
@@ -415,6 +415,56 @@ def test_top_projects(num, result, dbapp):
assert top == result
+def test_get_recent_projects(dbapp):
+ def create_package(name, version, ordering, created):
+ dbapp.engine.execute(packages.insert().values(
+ name=name, created=created))
+ dbapp.engine.execute(releases.insert().values(
+ name=name, version=version, _pypi_ordering=ordering,
+ created=created))
+
+ now = datetime.datetime.utcnow()
+ create_package("foo1", "2.0", 2, now)
+ create_package("foo2", "1.0", 1, now - datetime.timedelta(seconds=45))
+ create_package("foo3", "1.0", 1, now - datetime.timedelta(seconds=15))
+ create_package("foo4", "1.0", 1, now - datetime.timedelta(seconds=40))
+ create_package("foo5", "1.0", 1, now - datetime.timedelta(seconds=25))
+ create_package("foo6", "1.0", 1, now - datetime.timedelta(seconds=30))
+ create_package("foo7", "1.0", 1, now - datetime.timedelta(seconds=35))
+
+ dbapp.engine.execute(releases.insert().values(
+ name="foo1", version="1.0", _pypi_ordering=1,
+ created=now - datetime.timedelta(seconds=5),
+ ))
+
+ assert dbapp.db.packaging.get_recent_projects(num=4) == [
+ {
+ "name": "foo1",
+ "version": "2.0",
+ "summary": None,
+ "created": now,
+ },
+ {
+ "name": "foo3",
+ "version": "1.0",
+ "summary": None,
+ "created": now - datetime.timedelta(seconds=15),
+ },
+ {
+ "name": "foo5",
+ "version": "1.0",
+ "summary": None,
+ "created": now - datetime.timedelta(seconds=25),
+ },
+ {
+ "name": "foo6",
+ "version": "1.0",
+ "summary": None,
+ "created": now - datetime.timedelta(seconds=30),
+ },
+ ]
+
+
@pytest.mark.parametrize(("name", "normalized"), [
("foo_bar", "foo-bar"),
("Bar", "bar"),
50 warehouse/legacy/pypi.py
View
@@ -23,7 +23,9 @@
from warehouse.helpers import url_for
from warehouse.http import Response
from warehouse.legacy import xmlrpc
-from warehouse.utils import cache, fastly, is_valid_json_callback_name
+from warehouse.utils import (
+ cache, fastly, is_valid_json_callback_name, render_response,
+)
def pypi(app, request):
@@ -86,3 +88,49 @@ def project_json(app, request, project_name):
response = Response(data, mimetype="application/json")
response.headers['Content-Disposition'] = 'inline'
return response
+
+
+@cache("legacy_rss")
+@fastly("legacy_rss")
+def rss(app, request):
+ """Dump the last N days' updates as an RSS feed.
+ """
+ releases = app.db.packaging.get_recently_updated(num=40)
+ for release in releases:
+ values = dict(project_name=release['name'], version=release['version'])
+ url = app.urls.build('warehouse.packaging.views.project_detail',
+ values, force_external=True)
+ release.update(dict(url=url))
+
+ response = render_response(
+ app, request, "legacy/rss.xml",
+ description='package updates',
+ releases=releases,
+ site=app.config.site,
+ )
+ response.mimetype = 'text/xml; charset=utf-8'
+ # TODO: throw in a last-modified header too?
+ return response
+
+
+@cache("legacy_rss")
+@fastly("legacy_rss")
+def packages_rss(app, request):
+ """Dump the last N days' new projects as an RSS feed.
+ """
+ releases = app.db.packaging.get_recent_projects(num=40)
+ for release in releases:
+ values = dict(project_name=release['name'])
+ url = app.urls.build('warehouse.packaging.views.project_detail',
+ values, force_external=True)
+ release.update(dict(url=url))
+
+ response = render_response(
+ app, request, "legacy/rss.xml",
+ description='new projects',
+ releases=releases,
+ site=app.config.site,
+ )
+ response.mimetype = 'text/xml; charset=utf-8'
+ # TODO: throw in a last-modified header too?
+ return response
23 warehouse/packaging/db.py
View
@@ -54,6 +54,29 @@ class Database(db.Database):
"""
)
+ get_recent_projects = db.rows(
+ # We only consider projects registered in the last 7 days (see
+ # get_recently_updated for reasoning)
+ """ SELECT
+ p.name, r.version, p.created, r.summary
+ FROM releases r, (
+ SELECT packages.name, max_order, packages.created
+ FROM packages
+ JOIN (
+ SELECT name, max(_pypi_ordering) AS max_order
+ FROM releases
+ WHERE created >= now() - interval '7 days'
+ GROUP BY name
+ ) mo ON packages.name = mo.name
+ ) p
+ WHERE p.name = r.name
+ AND p.max_order = r._pypi_ordering
+ AND p.created >= now() - interval '7 days'
+ ORDER BY p.created DESC
+ LIMIT %(num)s
+ """
+ )
+
get_releases_since = db.rows(
""" SELECT name, version, created, summary
FROM releases
34 warehouse/templates/legacy/rss.xml
View
@@ -0,0 +1,34 @@
+{#
+ # Copyright 2013 Donald Stufft
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+-#}
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" "http://my.netscape.com/publish/formats/rss-0.91.dtd">
+<rss version="0.91">
+ <channel>
+ <title>{{ site.name }} Recent {{ description.title() }}</title>
+ <link>{{ site.url }}</link>
+ <description>Recent {{ description }} at {{ site.name }}</description>
+ <language>en</language>
+ {% for release in releases %}
+ <item>
+ <title>{{ release.name }} {{ release.version }}</title>
+ <link>{{ release.url }}</link>
+ <guid>{{ release.url }}</guid>
+ <description>{{ release.summary }}</description>
+ <pubDate>{{ release.created.strftime('%d %b %Y %H:%M:%S GMT') }}</pubDate>
+ </item>
+ {% endfor %}
+ </channel>
+</rss>
Something went wrong with that request. Please try again.