Permalink
Browse files

Changed all the providers over to celery tasks from batch_processing.

  • Loading branch information...
1 parent 3547577 commit dde92e32827b7c906ec82c1c9c5560b273a68d2b @davbo davbo committed Mar 20, 2012
View
6 molly/apps/feeds/providers/ical.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
import urllib2
import random
import traceback
@@ -9,7 +9,7 @@
socket.setdefaulttimeout(5)
from molly.external_media import sanitise_html
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.apps.feeds.providers import BaseFeedsProvider
@@ -26,7 +26,7 @@ class ICalFeedsProvider(BaseFeedsProvider):
"""
verbose_name = 'iCal'
- @batch('%d * * * *' % random.randint(0, 59))
+ @task(run_every=timedelta(minutes=60))
def import_data(self, **metadata):
"""
Pulls iCal feeds
View
4 molly/apps/feeds/providers/rss.py
@@ -1,8 +1,6 @@
from datetime import datetime, timedelta
import feedparser
import time
-import random
-import traceback
import logging
import socket
socket.setdefaulttimeout(5)
@@ -28,7 +26,7 @@ def struct_to_datetime(s):
class RSSFeedsProvider(BaseFeedsProvider):
verbose_name = 'RSS'
- @task(run_every=timedelta(seconds=10))
+ @task(run_every=timedelta(minutes=60))
def import_data(self, **metadata):
"""
Pulls RSS feeds
View
6 molly/apps/feeds/providers/talks_cam.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
from lxml import etree
import urllib2
import random
@@ -8,7 +8,7 @@
socket.setdefaulttimeout(5)
from molly.external_media import sanitise_html
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.apps.feeds.providers import BaseFeedsProvider
@@ -20,7 +20,7 @@
class TalksCamFeedsProvider(BaseFeedsProvider):
verbose_name = 'TalksCam'
- @batch('%d * * * *' % random.randint(0, 59))
+ @task(run_every=timedelta(minutes=60))
def import_data(self, **metadata):
"""
Pulls TalksCam feeds
View
7 molly/apps/places/providers/acislive.py
@@ -1,13 +1,12 @@
import threading
-from datetime import datetime
+from datetime import datetime, timedelta
from lxml import etree
import re
import logging
from string import ascii_lowercase
import socket
socket.setdefaulttimeout(5)
from urllib2 import urlopen
-import random
from django.db import transaction, reset_queries, connection
from django.http import Http404
@@ -17,7 +16,7 @@
from molly.apps.places import get_entity
from molly.apps.places.providers.naptan import NaptanMapsProvider
from molly.utils.i18n import set_name_in_language
-from molly.conf.settings import batch
+from molly.conf.provider import task
logger = logging.getLogger(__name__)
@@ -254,7 +253,7 @@ def __init__(self, urls=None):
urls = [instance[0] for instance in ACISLiveMapsProvider.ACISLIVE_URLS.items()]
self.urls = urls
- @batch('%d 10 * * sat' % random.randint(0, 59))
+ @task(run_every=timedelta(days=7))
def import_data(self, **metadata):
# Searching can flag up the same results again and again, so store
# which ones we've found
View
4 molly/apps/places/providers/atcocif.py
@@ -15,7 +15,7 @@
from molly.apps.places.models import (Entity, Route, StopOnRoute, Source,
Journey, ScheduledStop)
from molly.apps.places.providers import BaseMapsProvider, NaptanMapsProvider
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.utils.i18n import set_name_in_language
logger = getLogger(__name__)
@@ -37,7 +37,7 @@ def __init__(self, url):
self._cache = EntityCache()
self._entity_type = NaptanMapsProvider(None)._get_entity_types()['BCT'][0]
- @batch('%d 10 * * wed' % random.randint(0, 59))
+ @task(run_every=timedelta(days=7))
def import_data(self, **metadata):
deleted_routes = set(Route.objects.filter(external_ref__startswith=self._url).values_list('external_ref'))
View
5 molly/apps/places/providers/bbc_tpeg.py
@@ -1,5 +1,6 @@
import os, os.path, urllib, random, zipfile, tempfile
from lxml import etree
+from datetime import timedelta
from django.contrib.gis.geos import Point, LineString
from django.conf import settings
@@ -8,7 +9,7 @@
from molly.apps.places.providers import BaseMapsProvider
from molly.apps.places.models import Source, Entity, EntityType, EntityTypeCategory
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.utils.i18n import override, set_name_in_language
class BBCTPEGResolver(etree.Resolver):
@@ -50,7 +51,7 @@ class BBCTPEGPlacesProvider(BaseMapsProvider):
def __init__(self, url=_TPEG_URL):
self._tpeg_url = url
- @batch('%d-59/3 * * * *' % random.randint(0, 2))
+ @task(run_every=timedelta(minutes=3))
def import_data(self, **metadata):
source, entity_type = self._get_source(), self._get_entity_type()
View
7 molly/apps/places/providers/cif.py
@@ -7,8 +7,7 @@
It is dissimilar, but related to, the ATCO-CIF standard
"""
-from datetime import date, time
-import random
+from datetime import date, time, timedelta
from string import capwords
from django.core.exceptions import ObjectDoesNotExist
@@ -18,7 +17,7 @@
from molly.apps.places.models import (Entity, EntityType, EntityTypeCategory,
Source, Route, Journey, ScheduledStop)
from molly.apps.places.providers import BaseMapsProvider
-from molly.conf.settings import batch
+from molly.conf.provider import task
class CifTimetableProvider(BaseMapsProvider):
@@ -188,7 +187,7 @@ def import_from_string(self, cif):
with transaction.commit_on_success():
self._save_stops(self._save_journey(self._save_route()))
- @batch('%d 15 * * mon' % random.randint(0, 59))
+ @task(run_every=timedelta(days=7))
def import_from_file(self, **metadata):
with open(self._filename) as file:
for line in file:
View
5 molly/apps/places/providers/naptan.py
@@ -8,6 +8,7 @@
import csv
from warnings import warn
from collections import defaultdict
+from datetime import timedelta
try:
from cStringIO import StringIO
except:
@@ -24,7 +25,7 @@
from molly.apps.places.providers import BaseMapsProvider
from molly.apps.places.models import EntityType, Entity, EntityGroup, Source, EntityTypeCategory
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.utils.i18n import override, set_name_in_language
class NaptanContentHandler(ContentHandler):
@@ -673,7 +674,7 @@ def __init__(self, method=None, areas=None, username=None, password=None):
areas += ('910',)
self._areas = areas
- @batch('%d 10 * * mon' % random.randint(0, 59))
+ @task(run_every=timedelta(days=7))
def import_data(self, **metadata):
username, password = self._username, self._password
View
5 molly/apps/places/providers/osm.py
@@ -9,6 +9,7 @@
import logging
from xml.sax import saxutils, handler, make_parser
+from datetime import timedelta
from django.db import reset_queries
from django.contrib.gis.geos import Point, LineString, LinearRing
@@ -24,7 +25,7 @@
from molly.utils.misc import AnyMethodRequest
from molly.utils.i18n import override, set_name_in_language
from molly.geolocation import reverse_geocode
-from molly.conf.settings import batch
+from molly.conf.provider import task
logger = logging.getLogger(__name__)
@@ -304,7 +305,7 @@ def to_tuple(tag):
else:
self.identities = {}
- @batch('%d 9 * * mon' % random.randint(0, 59))
+ @task(run_every=timedelta(days=7))
def import_data(self, **metadata):
"Imports places data from OpenStreetMap"
View
5 molly/apps/places/providers/postcodes.py
@@ -8,6 +8,7 @@
import os.path
import re
+from datetime import timedelta
from django.db import transaction, reset_queries
from django.conf import settings
from django.contrib.gis.geos import Point
@@ -18,7 +19,7 @@
from molly.apps.places.models import Entity, EntityType, Source, EntityTypeCategory
from molly.utils.i18n import override, set_name_in_language
-from molly.conf.settings import batch
+from molly.conf.provider import task
class PostcodesMapsProvider(BaseMapsProvider):
def __init__(self, codepoint_path, import_areas=None):
@@ -35,7 +36,7 @@ def _download_codepoint_open(self):
archive_file.write(archive_url.read())
archive_file.close()
- @batch('%d 12 1 1 *' % random.randint(0, 59))
+ @task(run_every=timedelta(days=365))
def import_data(self, **metadata):
entity_type, source = self._get_entity_type(), self._get_source()
View
6 molly/apps/podcasts/providers/opml.py
@@ -1,5 +1,5 @@
from xml.etree import ElementTree as ET
-from datetime import datetime
+from datetime import datetime, timedelta
import urllib
import re
import email
@@ -10,7 +10,7 @@
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_noop as _
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.utils.i18n import set_name_in_language
from molly.apps.podcasts.providers import BasePodcastsProvider
from molly.apps.podcasts.models import Podcast, PodcastItem, PodcastCategory, PodcastEnclosure
@@ -70,7 +70,7 @@ def parse_outline(self, outline):
self.update_podcast(podcast)
- @batch('%d * * * *' % random.randint(0, 59))
+ @task(run_every=timedelta(minutes=60))
def import_data(self, **metadata):
self._category = None
View
6 molly/apps/podcasts/providers/pp.py
@@ -1,13 +1,13 @@
from lxml import etree
-from datetime import datetime
+from datetime import datetime, timedelta
import urllib
import re
import email
import random
from django.conf import settings
-from molly.conf.settings import batch
+from molly.conf.provider import task
from molly.apps.podcasts.providers import BasePodcastsProvider
from molly.apps.podcasts.models import Podcast, PodcastItem, PodcastCategory, PodcastEnclosure
from molly.utils.i18n import set_name_in_language
@@ -18,7 +18,7 @@ class PodcastProducerPodcastsProvider(RSSPodcastsProvider):
def __init__(self, url):
self.url = url
- @batch('%d * * * *' % random.randint(0, 59))
+ @task(run_every=timedelta(minutes=60))
def import_data(self, **metadata):
atom = self.atom
xml = etree.parse(urllib.urlopen(self.url))
View
6 molly/apps/podcasts/providers/rss.py
@@ -1,8 +1,8 @@
import random, urllib, email
from lxml import etree
-from datetime import datetime
+from datetime import datetime, timedelta
-from molly.conf.settings import batch
+from molly.conf.provider import task
import dateutil.parser
from molly.apps.podcasts.providers import BasePodcastsProvider
@@ -36,7 +36,7 @@ def __init__(self, podcasts, medium=None):
def atom(self):
return Namespace('http://www.w3.org/2005/Atom')
- @batch('%d * * * *' % random.randint(0, 59))
+ @task(run_every=timedelta(minutes=60))
def import_data(self, **metadata):
for slug, url in self.podcasts:
podcast, url = Podcast.objects.get_or_create(
View
2 molly/apps/weather/providers/bbc.py
@@ -103,7 +103,7 @@ def _find_choice_match(choices, verbose):
+ r'(Latest Observations|Forecast) for (?P<name>.+)'
)
- @task(run_every=timedelta(seconds=50))
+ @task(run_every=timedelta(minutes=15))
def import_data(self, **metadata):
"""
Pulls weather data from the BBC

0 comments on commit dde92e3

Please sign in to comment.