Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 84 lines (73 sloc) 3.02 kb
cacdffb @jterrace Refactored scripts that rerun tasks to a single, generic script. Added r...
jterrace authored
1 import sys
2 import os.path
3 import time
4 from celery.execute import send_task
5 import argparse
6
7 task_names = {'screenshot' : 'celery_tasks.generate_screenshot.generate_screenshot',
8 'generate_optimized' : 'celery_tasks.generate_optimized.generate_optimized',
9 'generate_metadata' : 'celery_tasks.generate_metadata.generate_metadata'}
10
11 def do_task(taskname, path, type, timestamp=None):
12 print 'Issuing %s task for %s type=%s timestamp=%s...' % (taskname, path, type, str(timestamp)),
13 sys.stdout.flush()
14 t = send_task(task_names[taskname], args=[path, type])
15 t.wait(propagate=False)
16 print t.state
17 if t.state == 'FAILED':
18 print 'Printing exception:'
19 print
20 print str(t.result)
21 print
22 time.sleep(1)
23
24 def do_single(task, path, type=None):
25 metadata = get_file_metadata(path)
26 types_to_do = []
27 if type is None:
28 for type in metadata['types']:
29 types_to_do.append(type)
30 else:
31 if type not in metadata['types']:
32 print >> sys.stderr, 'Invalid type', type, 'for path', path
33 return
34 types_to_do.append(type)
35
36 for type in types_to_do:
37 do_task(task, path, type)
38
39 def do_all(task, timestamp=None, type=None):
40 next_start = ""
41 if timestamp is not None:
42 next_start = timestamp
43
44 while next_start is not None:
45 content_items, next_start, prev_start = get_content_by_date(next_start)
46 for item in content_items:
47 path = item['full_path']
48 timestamp = item['full_timestamp']
49 for existing_type in item['metadata']['types'].iterkeys():
50 if type is None or type == existing_type:
51 do_task(task, path, existing_type, timestamp)
52
53 def main():
54 parser = argparse.ArgumentParser(description='Reprocess tasks')
55 parser.add_argument('task', help='task to execute', choices=task_names.keys())
56 subparsers = parser.add_subparsers()
57 all = subparsers.add_parser('all', help='reprocess all')
58 all.add_argument('--type', help='only reprocess this type of all files')
59 all.add_argument('--timestamp', help='start at this timestamp')
60 all.set_defaults(func=do_all)
61 single = subparsers.add_parser('single', help='reprocess a single file')
62 single.set_defaults(func=do_single)
63 single.add_argument('path')
64 single.add_argument('--type', help='only reprocess this type of the file')
65
66 args = parser.parse_args()
67 parsing_result = vars(args)
68 to_execute = parsing_result['func']
69 del parsing_result['func']
70 to_execute(**parsing_result)
71
72 def add_dirs():
73 thisdir = os.path.dirname( os.path.realpath( __file__ ) )
74 upone, tail = os.path.split(thisdir)
75 cdndir = os.path.join(upone, 'sirikata-cdn')
76 celerydir = os.path.join(cdndir, 'celery_tasks')
77 sys.path.append(cdndir)
78 sys.path.append(celerydir)
79
80 if __name__ == '__main__':
81 add_dirs()
82 from content.utils import get_content_by_date, get_file_metadata
83 main()
Something went wrong with that request. Please try again.