Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 97 lines (82 sloc) 2.97 KB
#!/usr/bin/env python
Queries the triplestore for dev8d people and their blogs, figures out the
feed url for the blog and writes out a Planet Venus configuration file.
from socket import setdefaulttimeout
from urllib import urlopen
from urlparse import urlparse
import logging
from rdflib import ConjunctiveGraph, Namespace
from BeautifulSoup import BeautifulSoup
import feedparser
w = Namespace('')
def discover_feeds(url):"looking up feed url for %s" % url)
html = urlopen(url).read()
soup = BeautifulSoup(html)
for link in soup.findAll('link', attrs={'rel': 'alternate'}):
feed_url = link['href']
title = link['title']
# rewrite to feed to be absolute if its relative
if feed_url and not feed_url.startswith('http'):
u = urlparse(url)
feed_url = 'http://%s%s' % (u.netloc, feed_url)
# check that the feed is there and looks ok
if feed_ok(feed_url):
yield title, feed_url
except Exception, e:
logging.error("error when performing feed discovery for %s: %s" %
(url, e))
def feed_ok(url):"checking feed: %s" % url)
feed = feedparser.parse(url)
if len(feed.entries) > 0 and not hasattr(feed.entries[0], 'updated'):
logging.error("feed %s has entries without updated timestamp" % url)
return False
except Exception, e:
logging.error("error when checking feed %s: %s" % (url, e))
return False
return True
def blogs():
g = ConjunctiveGraph('Sleepycat')'store')
for person, blog in g.subject_objects(predicate=w.Blog):
name = g.value(subject=person, predicate=w.Name)
for title, feed_url in discover_feeds(blog):
if title:
title = "%s (%s)" % (name, title)
title = name"found %s <%s>" % (title, feed_url))
yield title, feed_url
def print_config():
print \
name = planet-dev8d
link =
owner_name = Ed Summers
owner_email =
output_theme = theme
cache_directory = cache
output_dir = /var/www/
feed_timeout = 20
items_per_page = 100
log_level = DEBUG
# Subscription configuration
for name, feed in blogs():
if name and feed:
print ("[%s]\nname = %s\n\n" % (feed, name)).encode('utf-8')
if __name__ == '__main__':
format="%(asctime)s - %(levelname)s - %(message)s")"generating planet config")
print_config()"finished generating planet config")