Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Implement Legacy RSS Feeds #207

Merged
merged 3 commits into from

1 participant

@dstufft
Owner

Rebases and closes #175 to update it to the latest changes in Warehouse as well as fix some minor issues with it.

@dstufft dstufft merged commit a06575b into from
@dstufft dstufft deleted the branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 28, 2014
  1. @dstufft

    implement legacy RSS feeds

    Richard Jones authored dstufft committed
  2. @dstufft
  3. @dstufft
This page is out of date. Refresh to see the latest.
View
2  dev/config.yml
@@ -30,6 +30,7 @@ cache:
project_detail: 1
user_profile: 1
legacy_json: 1
+ legacy_rss: 1
varnish:
index: 120
simple: 120
@@ -37,6 +38,7 @@ cache:
project_detail: 120
user_profile: 120
legacy_json: 120
+ legacy_rss: 120
security:
View
184 tests/legacy/test_pypi.py
@@ -19,9 +19,14 @@
import pretend
import pytest
+
+import jinja2
+
from werkzeug.exceptions import NotFound, BadRequest
+from werkzeug.routing import Map
from warehouse.legacy import pypi, xmlrpc
+from warehouse.packaging import urls
@pytest.mark.parametrize("content_type", [None, "text/html", "__empty__"])
@@ -151,3 +156,182 @@ def test_json_missing(monkeypatch, project):
with pytest.raises(NotFound):
pypi.project_json(app, request, project_name='spam')
+
+
+def test_rss(monkeypatch):
+ get_recently_updated = pretend.call_recorder(lambda num=10: [
+ dict(name='spam', version='1.0', summary='hai spam', created='now'),
+ dict(name='ham', version='2.0', summary='hai ham', created='now'),
+ dict(name='spam', version='2.0', summary='hai spam v2', created='now'),
+ ])
+ template = pretend.stub(
+ render=pretend.call_recorder(lambda **ctx: "<xml>dummy</xml>"),
+ )
+ app = pretend.stub(
+ db=pretend.stub(
+ packaging=pretend.stub(
+ get_recently_updated=get_recently_updated,
+ )
+ ),
+ config=pretend.stub(
+ cache=pretend.stub(browser=False, varnish=False),
+ site={"url": "http://test.server/", "name": "PyPI"},
+ ),
+ urls=Map(urls.urls).bind('test.server', '/'),
+ templates=pretend.stub(
+ get_template=pretend.call_recorder(lambda t: template),
+ ),
+ )
+ request = pretend.stub()
+
+ resp = pypi.rss(app, request)
+
+ assert get_recently_updated.calls == [pretend.call(num=40)]
+ assert len(template.render.calls) == 1
+ assert template.render.calls[0].kwargs['releases'] == [
+ {
+ 'url': 'http://test.server/project/spam/1.0/',
+ 'version': u'1.0',
+ 'name': u'spam',
+ 'summary': u'hai spam',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/ham/2.0/',
+ 'version': u'2.0',
+ 'name': u'ham',
+ 'summary': u'hai ham',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/spam/2.0/',
+ 'version': u'2.0',
+ 'name': u'spam',
+ 'summary': u'hai spam v2',
+ 'created': u'now',
+ }]
+ assert resp.data == "<xml>dummy</xml>"
+
+
+def test_packages_rss(monkeypatch):
+ get_recent_projects = pretend.call_recorder(lambda num=10: [
+ dict(name='spam', version='1.0', summary='hai spam', created='now'),
+ dict(name='ham', version='2.0', summary='hai ham', created='now'),
+ dict(name='eggs', version='21.0', summary='hai eggs!', created='now'),
+ ])
+ template = pretend.stub(
+ render=pretend.call_recorder(lambda **ctx: "<xml>dummy</xml>"),
+ )
+ app = pretend.stub(
+ db=pretend.stub(
+ packaging=pretend.stub(
+ get_recent_projects=get_recent_projects,
+ )
+ ),
+ config=pretend.stub(
+ cache=pretend.stub(browser=False, varnish=False),
+ site={"url": "http://test.server/", "name": "PyPI"},
+ ),
+ urls=Map(urls.urls).bind('test.server', '/'),
+ templates=pretend.stub(
+ get_template=pretend.call_recorder(lambda t: template),
+ ),
+ )
+ request = pretend.stub()
+
+ resp = pypi.packages_rss(app, request)
+
+ assert get_recent_projects.calls == [pretend.call(num=40)]
+ assert len(template.render.calls) == 1
+ assert template.render.calls[0].kwargs['releases'] == [
+ {
+ 'url': 'http://test.server/project/spam/',
+ 'version': u'1.0',
+ 'name': u'spam',
+ 'summary': u'hai spam',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/ham/',
+ 'version': u'2.0',
+ 'name': u'ham',
+ 'summary': u'hai ham',
+ 'created': u'now',
+ }, {
+ 'url': 'http://test.server/project/eggs/',
+ 'version': u'21.0',
+ 'name': u'eggs',
+ 'summary': u'hai eggs!',
+ 'created': u'now',
+ }]
+ assert resp.data == "<xml>dummy</xml>"
+
+
+def test_rss_xml_template(monkeypatch):
+ templates = jinja2.Environment(
+ autoescape=True,
+ auto_reload=False,
+ extensions=[
+ "jinja2.ext.i18n",
+ ],
+ loader=jinja2.PackageLoader("warehouse"),
+ )
+ template = templates.get_template('legacy/rss.xml')
+ content = template.render(
+ site=dict(url='http://test.server/', name="PyPI"),
+ description='package updates',
+ releases=[
+ {
+ 'url': 'http://test.server/project/spam/',
+ 'version': u'1.0',
+ 'name': u'spam',
+ 'summary': u'hai spam',
+ 'created': datetime.date(1970, 1, 1),
+ }, {
+ 'url': 'http://test.server/project/ham/',
+ 'version': u'2.0',
+ 'name': u'ham',
+ 'summary': u'hai ham',
+ 'created': datetime.date(1970, 1, 1),
+ }, {
+ 'url': 'http://test.server/project/eggs/',
+ 'version': u'21.0',
+ 'name': u'eggs',
+ 'summary': u'hai eggs!',
+ 'created': datetime.date(1970, 1, 1),
+ }
+ ],
+ )
+ assert content == '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" \
+"http://my.netscape.com/publish/formats/rss-0.91.dtd">
+<rss version="0.91">
+ <channel>
+ <title>PyPI Recent Package Updates</title>
+ <link>http://test.server/</link>
+ <description>Recent package updates at PyPI</description>
+ <language>en</language>
+ \n\
+ <item>
+ <title>spam 1.0</title>
+ <link>http://test.server/project/spam/</link>
+ <guid>http://test.server/project/spam/</guid>
+ <description>hai spam</description>
+ <pubDate>01 Jan 1970 00:00:00 GMT</pubDate>
+ </item>
+ \n\
+ <item>
+ <title>ham 2.0</title>
+ <link>http://test.server/project/ham/</link>
+ <guid>http://test.server/project/ham/</guid>
+ <description>hai ham</description>
+ <pubDate>01 Jan 1970 00:00:00 GMT</pubDate>
+ </item>
+ \n\
+ <item>
+ <title>eggs 21.0</title>
+ <link>http://test.server/project/eggs/</link>
+ <guid>http://test.server/project/eggs/</guid>
+ <description>hai eggs!</description>
+ <pubDate>01 Jan 1970 00:00:00 GMT</pubDate>
+ </item>
+ \n\
+ </channel>
+</rss>'''
View
50 tests/packaging/test_db.py
@@ -415,6 +415,56 @@ def test_top_projects(num, result, dbapp):
assert top == result
+def test_get_recent_projects(dbapp):
+ def create_package(name, version, ordering, created):
+ dbapp.engine.execute(packages.insert().values(
+ name=name, created=created))
+ dbapp.engine.execute(releases.insert().values(
+ name=name, version=version, _pypi_ordering=ordering,
+ created=created))
+
+ now = datetime.datetime.utcnow()
+ create_package("foo1", "2.0", 2, now)
+ create_package("foo2", "1.0", 1, now - datetime.timedelta(seconds=45))
+ create_package("foo3", "1.0", 1, now - datetime.timedelta(seconds=15))
+ create_package("foo4", "1.0", 1, now - datetime.timedelta(seconds=40))
+ create_package("foo5", "1.0", 1, now - datetime.timedelta(seconds=25))
+ create_package("foo6", "1.0", 1, now - datetime.timedelta(seconds=30))
+ create_package("foo7", "1.0", 1, now - datetime.timedelta(seconds=35))
+
+ dbapp.engine.execute(releases.insert().values(
+ name="foo1", version="1.0", _pypi_ordering=1,
+ created=now - datetime.timedelta(seconds=5),
+ ))
+
+ assert dbapp.db.packaging.get_recent_projects(num=4) == [
+ {
+ "name": "foo1",
+ "version": "2.0",
+ "summary": None,
+ "created": now,
+ },
+ {
+ "name": "foo3",
+ "version": "1.0",
+ "summary": None,
+ "created": now - datetime.timedelta(seconds=15),
+ },
+ {
+ "name": "foo5",
+ "version": "1.0",
+ "summary": None,
+ "created": now - datetime.timedelta(seconds=25),
+ },
+ {
+ "name": "foo6",
+ "version": "1.0",
+ "summary": None,
+ "created": now - datetime.timedelta(seconds=30),
+ },
+ ]
+
+
@pytest.mark.parametrize(("name", "normalized"), [
("foo_bar", "foo-bar"),
("Bar", "bar"),
View
50 warehouse/legacy/pypi.py
@@ -23,7 +23,9 @@
from warehouse.helpers import url_for
from warehouse.http import Response
from warehouse.legacy import xmlrpc
-from warehouse.utils import cache, fastly, is_valid_json_callback_name
+from warehouse.utils import (
+ cache, fastly, is_valid_json_callback_name, render_response,
+)
def pypi(app, request):
@@ -86,3 +88,49 @@ def project_json(app, request, project_name):
response = Response(data, mimetype="application/json")
response.headers['Content-Disposition'] = 'inline'
return response
+
+
+@cache("legacy_rss")
+@fastly("legacy_rss")
+def rss(app, request):
+ """Dump the last N days' updates as an RSS feed.
+ """
+ releases = app.db.packaging.get_recently_updated(num=40)
+ for release in releases:
+ values = dict(project_name=release['name'], version=release['version'])
+ url = app.urls.build('warehouse.packaging.views.project_detail',
+ values, force_external=True)
+ release.update(dict(url=url))
+
+ response = render_response(
+ app, request, "legacy/rss.xml",
+ description='package updates',
+ releases=releases,
+ site=app.config.site,
+ )
+ response.mimetype = 'text/xml; charset=utf-8'
+ # TODO: throw in a last-modified header too?
+ return response
+
+
+@cache("legacy_rss")
+@fastly("legacy_rss")
+def packages_rss(app, request):
+ """Dump the last N days' new projects as an RSS feed.
+ """
+ releases = app.db.packaging.get_recent_projects(num=40)
+ for release in releases:
+ values = dict(project_name=release['name'])
+ url = app.urls.build('warehouse.packaging.views.project_detail',
+ values, force_external=True)
+ release.update(dict(url=url))
+
+ response = render_response(
+ app, request, "legacy/rss.xml",
+ description='new projects',
+ releases=releases,
+ site=app.config.site,
+ )
+ response.mimetype = 'text/xml; charset=utf-8'
+ # TODO: throw in a last-modified header too?
+ return response
View
23 warehouse/packaging/db.py
@@ -54,6 +54,29 @@ class Database(db.Database):
"""
)
+ get_recent_projects = db.rows(
+ # We only consider projects registered in the last 7 days (see
+ # get_recently_updated for reasoning)
+ """ SELECT
+ p.name, r.version, p.created, r.summary
+ FROM releases r, (
+ SELECT packages.name, max_order, packages.created
+ FROM packages
+ JOIN (
+ SELECT name, max(_pypi_ordering) AS max_order
+ FROM releases
+ WHERE created >= now() - interval '7 days'
+ GROUP BY name
+ ) mo ON packages.name = mo.name
+ ) p
+ WHERE p.name = r.name
+ AND p.max_order = r._pypi_ordering
+ AND p.created >= now() - interval '7 days'
+ ORDER BY p.created DESC
+ LIMIT %(num)s
+ """
+ )
+
get_releases_since = db.rows(
""" SELECT name, version, created, summary
FROM releases
View
34 warehouse/templates/legacy/rss.xml
@@ -0,0 +1,34 @@
+{#
+ # Copyright 2013 Donald Stufft
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+-#}
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" "http://my.netscape.com/publish/formats/rss-0.91.dtd">
+<rss version="0.91">
+ <channel>
+ <title>{{ site.name }} Recent {{ description.title() }}</title>
+ <link>{{ site.url }}</link>
+ <description>Recent {{ description }} at {{ site.name }}</description>
+ <language>en</language>
+ {% for release in releases %}
+ <item>
+ <title>{{ release.name }} {{ release.version }}</title>
+ <link>{{ release.url }}</link>
+ <guid>{{ release.url }}</guid>
+ <description>{{ release.summary }}</description>
+ <pubDate>{{ release.created.strftime('%d %b %Y %H:%M:%S GMT') }}</pubDate>
+ </item>
+ {% endfor %}
+ </channel>
+</rss>
Something went wrong with that request. Please try again.