diff --git a/ummeli/base/management/commands/ummeli_jobs_cron.py b/ummeli/base/management/commands/ummeli_jobs_cron.py index 83d40f53..abc18ffe 100644 --- a/ummeli/base/management/commands/ummeli_jobs_cron.py +++ b/ummeli/base/management/commands/ummeli_jobs_cron.py @@ -5,4 +5,4 @@ class Command(BaseCommand): help = 'update jobs from wegotads' def handle(self, **options): - tasks.run_jobs_update.delay() + tasks.run_jobs_update() diff --git a/ummeli/vlive/jobs/tasks.py b/ummeli/vlive/jobs/tasks.py index 76220aa5..edced09a 100644 --- a/ummeli/vlive/jobs/tasks.py +++ b/ummeli/vlive/jobs/tasks.py @@ -1,7 +1,6 @@ from ummeli.vlive.jobs.parsers import CategoryParser, JobsParser from celery.task import task -from celery.task.sets import TaskSet -from datetime import datetime, timedelta +from datetime import datetime from ummeli.opportunities.models import Job, Province from django.utils.hashcompat import md5_constructor from ummeli.base.utils import category_from_str @@ -16,7 +15,8 @@ def process_jobs(category_tuple, jobs_parser, province): articles = jobs_parser(url=url).parse() for date, source, text in articles: - hash_key = md5_constructor(':'.join([date, source, text])).hexdigest() + hash_key = md5_constructor( + ':'.join([date, source, text])).hexdigest() date_with_year = ('%s-%s' % (date, datetime.now().strftime('%Y'))) date = datetime.strptime(date_with_year, '%d-%m-%Y') @@ -37,27 +37,22 @@ def process_jobs(category_tuple, jobs_parser, province): @task(ignore_result=True) def queue_categories(province, category_parser, jobs_parser): search_id, province_name = province - parser = category_parser(search_id, url = 'http://www.wegotads.co.za/Employment/listings/22001%(path)s?umb=1&search_source=%(id)s') + parser = category_parser( + search_id, url='http://www.wegotads.co.za/Employment/listings/22001%(path)s?umb=1&search_source=%(id)s') urls = parser.parse() - now = datetime.now() - taskset = TaskSet(process_jobs.subtask((url, jobs_parser, province_name), - options = {'eta':now + timedelta(seconds=5 * i)}) - for i, url in enumerate(urls)) - - return taskset.apply_async() + for url in urls: + process_jobs(url, jobs_parser, province_name) @task(ignore_result=True) -def run_jobs_update(category_parser = CategoryParser, jobs_parser = JobsParser): # allow mocking of parsers +# allow mocking of parsers +def run_jobs_update(category_parser=CategoryParser, jobs_parser=JobsParser): provinces = [ (2, 'Gauteng'), (5, 'Western Cape'), (6, 'KwaZulu Natal') ] - now = datetime.now() - taskset = TaskSet(queue_categories.subtask((province, category_parser, jobs_parser), - options = {'eta':now + timedelta(seconds=10 * i)}) - for i, province in enumerate(provinces)) - return taskset.apply_async() + for i, province in enumerate(provinces): + queue_categories(province, category_parser, jobs_parser)