-
Notifications
You must be signed in to change notification settings - Fork 6
/
main.py
135 lines (85 loc) · 3.69 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
from time import time
import argparse
import logging
from time import time
from time import sleep
from totalimpactwebapp import db
from totalimpactwebapp import ti_queues
def elapsed(since, round_places=2):
return round(time() - since, round_places)
class UpdateStatus():
seconds_between_chunks = 15
def __init__(self, num_jobs, queue_number):
self.num_jobs_total = num_jobs
self.queue_number = queue_number
self.start_time = time()
self.last_chunk_start_time = time()
self.last_chunk_num_jobs_completed = 0
self.number_of_prints = 0
def print_status_loop(self):
num_jobs_remaining = self.print_status()
while num_jobs_remaining > 0:
num_jobs_remaining = self.print_status()
def print_status(self):
sleep(1) # at top to make sure there's time for the jobs to be saved in redis.
num_jobs_remaining = ti_queues[self.queue_number].count
num_jobs_done = self.num_jobs_total - num_jobs_remaining
print "finished {done} jobs in {elapsed} min. {left} left.".format(
done=num_jobs_done,
elapsed=round(elapsed(self.start_time) / 60, 1),
left=num_jobs_remaining
)
self.number_of_prints += 1
if self.number_of_prints % self.seconds_between_chunks == self.seconds_between_chunks - 1:
num_jobs_finished_this_chunk = num_jobs_done - self.last_chunk_num_jobs_completed
if not num_jobs_finished_this_chunk:
print "No jobs finished this chunk... :/"
else:
chunk_elapsed = elapsed(self.last_chunk_start_time)
jobs_per_hour_this_chunk = num_jobs_finished_this_chunk / float(chunk_elapsed / 3600)
predicted_mins_to_finish = round(
(num_jobs_remaining / float(jobs_per_hour_this_chunk)) * 60,
1
)
print "We're doing {} jobs per hour. At this rate, done in {}min\n".format(
int(jobs_per_hour_this_chunk),
predicted_mins_to_finish
)
self.last_chunk_start_time = time()
self.last_chunk_num_jobs_completed = num_jobs_done
return num_jobs_remaining
def queue_status(queue_number_str):
queue_number = int(queue_number_str)
num_jobs_to_start = ti_queues[queue_number].count
update = UpdateStatus(num_jobs_to_start, queue_number)
update.print_status_loop()
def empty_queue(queue_number_str):
queue_number = int(queue_number_str)
num_jobs = ti_queues[queue_number].count
ti_queues[queue_number].empty()
print "emptied {} jobs on queue #{}....".format(
num_jobs,
queue_number)
def main(fn, optional_args=None):
start = time()
# call function by its name in this module, with all args :)
# http://stackoverflow.com/a/4605/596939
if optional_args:
globals()[fn](*optional_args)
else:
globals()[fn]()
if __name__ == "__main__":
# get args from the command line:
parser = argparse.ArgumentParser(description="Run stuff.")
parser.add_argument('function', type=str, help="what function you want to run")
parser.add_argument('optional_args', nargs='*', help="positional args for the function")
args = vars(parser.parse_args())
function = args["function"]
optional_args = args["optional_args"]
print u"running main.py {function} with these args:{optional_args}\n".format(
function=function, optional_args=optional_args)
global logger
logger = logging.getLogger("ti.main.{function}".format(
function=function))
main(function, optional_args)
db.session.remove()