Permalink
Browse files

Providers now subclass molly.provider.Provider, started replacing bat…

…ch decorators with the new celery task decorator.
  • Loading branch information...
1 parent 14c6ef7 commit a6458a844dc6e44a6a8d9137992939e4b785b69a David King committed Mar 16, 2012
@@ -1,7 +1,8 @@
+from molly.conf.provider import Provider
from molly.apps.contact.forms import GenericContactForm
from django.utils.translation import ugettext as _
-class BaseContactProvider(object):
+class BaseContactProvider(Provider):
class NoSuchResult(KeyError):
pass
@@ -88,4 +89,4 @@ class TooManyResults(Exception):
pass
else:
del ldap
- from mit import LDAPContactProvider
+ from mit import LDAPContactProvider
@@ -1,4 +1,6 @@
-class BaseFeedsProvider(object):
+from molly.conf.provider import Provider
+
+class BaseFeedsProvider(Provider):
pass
from rss import RSSFeedsProvider
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
import feedparser
import time
import random
@@ -8,7 +8,7 @@
socket.setdefaulttimeout(5)
from molly.external_media import sanitise_html
-from molly.conf.settings import batch
+from molly.conf.settings import task
from molly.apps.feeds.providers import BaseFeedsProvider
@@ -28,14 +28,16 @@ def struct_to_datetime(s):
class RSSFeedsProvider(BaseFeedsProvider):
verbose_name = 'RSS'
- @batch('%d * * * *' % random.randint(0, 59))
+ @task(run_every=timedelta(seconds=10))
def import_data(self, **metadata):
"""
Pulls RSS feeds
"""
+ print "In import_data"
from molly.apps.feeds.models import Feed
for feed in Feed.objects.filter(provider=self.class_path):
+ print feed.title
logger.info("Importing %s\n" % feed.title)
try:
self.import_feed(feed)
@@ -45,6 +47,8 @@ def import_data(self, **metadata):
return metadata
def import_feed(self, feed):
+ print "In import_feed"
+ print feed.title
from molly.apps.feeds.models import Item
feed_data = feedparser.parse(feed.rss_url)
@@ -1,4 +1,6 @@
-class BaseLibrarySearchProvider(object):
+from molly.conf.provider import Provider
+
+class BaseLibrarySearchProvider(Provider):
"""
Abstract class implementing the interface for a provider for the library app
"""
@@ -23,7 +25,7 @@ def control_number_search(self, control_number):
from z3950 import Z3950
-class BaseMetadataProvider(object):
+class BaseMetadataProvider(Provider):
"""
Abstract class implementing the interface for a provider which fetches
book covers
@@ -37,4 +39,4 @@ def annotate(self, books):
"""
pass
-from google import GoogleBooksProvider
+from google import GoogleBooksProvider
@@ -1,4 +1,7 @@
-class BaseMapsProvider(object):
+from molly.conf.provider import Provider
+
+
+class BaseMapsProvider(Provider):
def import_data(self):
pass
@@ -16,4 +19,4 @@ def augment_metadata(self, entities, **kwargs):
from bbc_tpeg import BBCTPEGPlacesProvider
from tfl import TubeRealtimeProvider
from atcocif import AtcoCifTimetableProvider
-from timetables import TimetableAnnotationProvider
+from timetables import TimetableAnnotationProvider
@@ -1,6 +1,8 @@
-class BasePodcastsProvider(object):
+from molly.conf.provider import Provider
+
+class BasePodcastsProvider(Provider):
pass
from rss import RSSPodcastsProvider
from pp import PodcastProducerPodcastsProvider
-from opml import OPMLPodcastsProvider
+from opml import OPMLPodcastsProvider
@@ -1,13 +1,14 @@
import logging
+from molly.conf.provider import Provider
from django.core.urlresolvers import resolve, Resolver404
from django.http import Http404
from django.views.generic.simple import redirect_to
logger = logging.getLogger(__name__)
-class BaseSearchProvider(object):
+class BaseSearchProvider(Provider):
def perform_search(self, request, query, application=None):
"""
@@ -103,4 +104,4 @@ def _load_query_expansion_terms(self):
return {}
from google_search_appliance import GSASearchProvider
-from application_search import ApplicationSearchProvider
+from application_search import ApplicationSearchProvider
@@ -1,8 +1,9 @@
+from molly.conf.provider import Provider
from datetime import datetime
import dateutil.parser
import feedparser
-class RSSModuleServiceStatusProvider(object):
+class RSSModuleServiceStatusProvider(Provider):
def __init__(self, name, slug, url):
self.name, self.slug, self.url = name, slug, url
@@ -1,4 +1,7 @@
-class BaseTransitLineStatusProvider():
+from molly.conf.provider import Provider
+
+
+class BaseTransitLineStatusProvider(Provider):
def get_status(self):
# Return a dictionary with a key of 'service_name', which is the human
@@ -8,4 +11,4 @@ def get_status(self):
# "disruption_reason"
return {}
-from molly.apps.transport.providers.tfl import TubeStatusProvider
+from molly.apps.transport.providers.tfl import TubeStatusProvider
@@ -12,15 +12,16 @@
from django.contrib.gis.geos import Point
from django.utils.translation import ugettext_lazy as _
-from molly.conf.settings import batch
+from molly.conf.settings import task
+from molly.conf.provider import Provider
from molly.apps.weather.models import (
Weather, OUTLOOK_CHOICES, VISIBILITY_CHOICES, PRESSURE_STATE_CHOICES,
SCALE_CHOICES, PTYPE_OBSERVATION, PTYPE_FORECAST
)
logger = logging.getLogger(__name__)
-class BBCWeatherProvider(object):
+class BBCWeatherProvider(Provider):
"""
Scrapes BBC RSS feeds to obtain weather information
"""
@@ -103,7 +104,7 @@ def _find_choice_match(choices, verbose):
+ r'(Latest Observations|Forecast) for (?P<name>.+)'
)
- @batch('%d-%d/15 * * * *' % (lambda x:(x, x+45))(random.randint(0, 14)))
+ @task(run_every=timedelta(seconds=20))
def import_data(self, **metadata):
"""
Pulls weather data from the BBC
@@ -157,7 +158,11 @@ def import_data(self, **metadata):
weather.location = Point(data['location'], srid=4326)
weather.save()
+ print weather
+ print metadata
+ metadata['n'] = metadata.get('n', 0) + 1
+ print metadata
return metadata
def get_observations_data(self):
@@ -1,4 +1,7 @@
-class BaseGeolocationProvider(object):
+from molly.conf.provider import Provider
+
+
+class BaseGeolocationProvider(Provider):
def reverse_geocode(self, lon, lat):
@@ -9,4 +12,4 @@ def geocode(self, query):
return []
from cloudmade import CloudmadeGeolocationProvider
-from places import PlacesGeolocationProvider
+from places import PlacesGeolocationProvider

0 comments on commit a6458a8

Please sign in to comment.