Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Restructuring imports for simplicity

  • Loading branch information...
commit 4d35ecb06366658e16236de1d2bbd77662a2608f 1 parent 7fdeb9f
Charles Leifer authored August 10, 2012
3  huey/__init__.py
... ...
@@ -0,0 +1,3 @@
  1
+from huey.bin.config import BaseConfiguration
  2
+from huey.decorators import queue_command, periodic_command, crontab
  3
+from huey.queue import Invoker
3  huey/tests/config.py
... ...
@@ -1,7 +1,6 @@
1 1
 import logging
  2
+from huey import BaseConfiguration, Invoker
2 3
 from huey.backends.dummy import DummyQueue, DummyDataStore
3  
-from huey.bin.config import BaseConfiguration
4  
-from huey.queue import Invoker
5 4
 
6 5
 
7 6
 test_queue = DummyQueue('test-queue')
171  huey/tests/consumer.py
@@ -4,13 +4,12 @@
4 4
 import time
5 5
 import unittest
6 6
 
  7
+from huey import queue_command, Invoker, BaseConfiguration
7 8
 from huey.backends.dummy import DummyQueue, DummyDataStore
8  
-from huey.decorators import queue_command
9 9
 from huey.exceptions import QueueException
10  
-from huey.queue import Invoker, QueueCommand, PeriodicQueueCommand
  10
+from huey.queue import QueueCommand, PeriodicQueueCommand
11 11
 from huey.registry import registry
12 12
 from huey.utils import local_to_utc
13  
-from huey.bin.config import BaseConfiguration
14 13
 from huey.bin.huey_consumer import load_config, Consumer, IterableQueue
15 14
 
16 15
 
@@ -60,7 +59,7 @@ class TestLogHandler(logging.Handler):
60 59
     def __init__(self, *args, **kwargs):
61 60
         self.messages = []
62 61
         logging.Handler.__init__(self, *args, **kwargs)
63  
-        
  62
+
64 63
     def emit(self, record):
65 64
         self.messages.append(record.getMessage())
66 65
 
@@ -69,86 +68,86 @@ class SkewConsumerTestCase(unittest.TestCase):
69 68
     def setUp(self):
70 69
         global state
71 70
         state = {}
72  
-        
  71
+
73 72
         self.orig_sleep = time.sleep
74 73
         time.sleep = lambda x: None
75  
-        
  74
+
76 75
         self.consumer = Consumer(test_invoker, DummyConfiguration)
77 76
         self.handler = TestLogHandler()
78 77
         self.consumer.logger.addHandler(self.handler)
79  
-    
  78
+
80 79
     def tearDown(self):
81 80
         self.consumer.shutdown()
82 81
         self.consumer.logger.removeHandler(self.handler)
83 82
         time.sleep = self.orig_sleep
84  
-    
  83
+
85 84
     def test_consumer_loader(self):
86 85
         config = load_config('huey.tests.config.Config')
87 86
         self.assertTrue(isinstance(config.QUEUE, DummyQueue))
88 87
         self.assertEqual(config.QUEUE.name, 'test-queue')
89  
-    
  88
+
90 89
     def spawn(self, func, *args, **kwargs):
91 90
         t = threading.Thread(target=func, args=args, kwargs=kwargs)
92 91
         t.start()
93 92
         return t
94  
-    
  93
+
95 94
     def test_iterable_queue(self):
96 95
         store = []
97 96
         q = IterableQueue()
98  
-        
  97
+
99 98
         def do_queue(queue, result):
100 99
             for message in queue:
101 100
                 result.append(message)
102  
-        
  101
+
103 102
         t = self.spawn(do_queue, q, store)
104 103
         q.put(1)
105 104
         q.put(2)
106 105
         q.put(StopIteration)
107  
-        
  106
+
108 107
         t.join()
109 108
         self.assertFalse(t.is_alive())
110 109
         self.assertEqual(store, [1, 2])
111  
-    
  110
+
112 111
     def test_message_processing(self):
113 112
         self.consumer.start_message_receiver()
114 113
         self.consumer.start_worker_pool()
115  
-        
  114
+
116 115
         self.assertFalse('k' in state)
117  
-        
  116
+
118 117
         res = modify_state('k', 'v')
119 118
         res.get(blocking=True)
120  
-        
  119
+
121 120
         self.assertTrue('k' in state)
122 121
         self.assertEqual(res.get(), 'v')
123  
-    
  122
+
124 123
     def test_worker(self):
125 124
         res = modify_state('x', 'y')
126  
-        
  125
+
127 126
         cmd = test_invoker.dequeue()
128 127
         self.assertEqual(res.get(), None)
129  
-        
  128
+
130 129
         # we will be calling release() after finishing work
131 130
         self.consumer._pool.acquire()
132 131
         self.consumer.worker(cmd)
133  
-        
  132
+
134 133
         self.assertTrue('x' in state)
135 134
         self.assertEqual(res.get(), 'y')
136  
-    
  135
+
137 136
     def test_worker_exception(self):
138 137
         res = blow_up()
139 138
         cmd = test_invoker.dequeue()
140  
-        
  139
+
141 140
         self.consumer._pool.acquire()
142 141
         self.consumer.worker(cmd)
143  
-        
  142
+
144 143
         self.assertEqual(self.handler.messages, [
145 144
             'unhandled exception in worker thread',
146 145
         ])
147  
-    
  146
+
148 147
     def test_retries_and_logging(self):
149 148
         # this will continually fail
150 149
         res = retry_command('blampf')
151  
-        
  150
+
152 151
         cmd = test_invoker.dequeue()
153 152
         self.consumer._pool.acquire()
154 153
         self.consumer.worker(cmd)
@@ -156,7 +155,7 @@ def test_retries_and_logging(self):
156 155
             'unhandled exception in worker thread',
157 156
             're-enqueueing task %s, 2 tries left' % cmd.task_id,
158 157
         ])
159  
-        
  158
+
160 159
         cmd = test_invoker.dequeue()
161 160
         self.assertEqual(cmd.retries, 2)
162 161
         self.consumer._pool.acquire()
@@ -165,7 +164,7 @@ def test_retries_and_logging(self):
165 164
             'unhandled exception in worker thread',
166 165
             're-enqueueing task %s, 1 tries left' % cmd.task_id,
167 166
         ])
168  
-        
  167
+
169 168
         cmd = test_invoker.dequeue()
170 169
         self.assertEqual(cmd.retries, 1)
171 170
         self.consumer._pool.acquire()
@@ -174,7 +173,7 @@ def test_retries_and_logging(self):
174 173
             'unhandled exception in worker thread',
175 174
             're-enqueueing task %s, 0 tries left' % cmd.task_id,
176 175
         ])
177  
-        
  176
+
178 177
         cmd = test_invoker.dequeue()
179 178
         self.assertEqual(cmd.retries, 0)
180 179
         self.consumer._pool.acquire()
@@ -183,14 +182,14 @@ def test_retries_and_logging(self):
183 182
         self.assertEqual(self.handler.messages[-1:], [
184 183
             'unhandled exception in worker thread',
185 184
         ])
186  
-        
  185
+
187 186
         self.assertEqual(test_invoker.dequeue(), None)
188  
-    
  187
+
189 188
     def test_retries_with_success(self):
190 189
         # this will fail once, then succeed
191 190
         res = retry_command('blampf', False)
192 191
         self.assertFalse('blampf' in state)
193  
-        
  192
+
194 193
         cmd = test_invoker.dequeue()
195 194
         self.consumer._pool.acquire()
196 195
         self.consumer.worker(cmd)
@@ -198,98 +197,98 @@ def test_retries_with_success(self):
198 197
             'unhandled exception in worker thread',
199 198
             're-enqueueing task %s, 2 tries left' % cmd.task_id,
200 199
         ])
201  
-        
  200
+
202 201
         cmd = test_invoker.dequeue()
203 202
         self.assertEqual(cmd.retries, 2)
204 203
         self.consumer._pool.acquire()
205 204
         self.consumer.worker(cmd)
206  
-        
  205
+
207 206
         self.assertEqual(state['blampf'], 'fixed')
208  
-        
  207
+
209 208
         self.assertEqual(test_invoker.dequeue(), None)
210  
-    
  209
+
211 210
     def test_pooling(self):
212 211
         # simulate acquiring two worker threads
213 212
         self.consumer._pool.acquire()
214 213
         self.consumer._pool.acquire()
215  
-        
  214
+
216 215
         res = modify_state('x', 'y')
217  
-        
218  
-        # dequeue a *single* message 
  216
+
  217
+        # dequeue a *single* message
219 218
         pt = self.spawn(self.consumer.check_message)
220  
-        
  219
+
221 220
         # work on any messages generated by the processor thread
222 221
         st = self.spawn(self.consumer.worker_pool)
223  
-        
  222
+
224 223
         # our result is not available since all workers are blocked
225 224
         self.assertEqual(res.get(), None)
226 225
         self.assertFalse(self.consumer._pool.acquire(blocking=False))
227  
-        
  226
+
228 227
         # our processor is waiting
229 228
         self.assertTrue(pt.is_alive())
230 229
         self.assertEqual(self.consumer._queue.qsize(), 0)
231  
-        
  230
+
232 231
         # release a worker
233 232
         self.consumer._pool.release()
234  
-        
  233
+
235 234
         # we can get and block now, but will set a timeout of 3 to indicate that
236 235
         # something is wrong
237 236
         self.assertEqual(res.get(blocking=True, timeout=3), 'y')
238  
-        
  237
+
239 238
         # this is done
240 239
         pt.join()
241  
-    
  240
+
242 241
     def test_scheduling(self):
243 242
         dt = datetime.datetime(2011, 1, 1, 0, 0)
244 243
         dt2 = datetime.datetime(2037, 1, 1, 0, 0)
245 244
         r1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
246 245
         r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False)
247  
-        
248  
-        # dequeue a *single* message 
  246
+
  247
+        # dequeue a *single* message
249 248
         pt = self.spawn(self.consumer.check_message)
250  
-        
  249
+
251 250
         # work on any messages generated by the processor thread
252 251
         st = self.spawn(self.consumer.worker_pool)
253  
-        
  252
+
254 253
         pt.join()
255 254
         self.assertTrue('k' in state)
256 255
         self.assertEqual(self.consumer.schedule._schedule, {})
257  
-        
258  
-        # dequeue a *single* message 
  256
+
  257
+        # dequeue a *single* message
259 258
         pt = self.spawn(self.consumer.check_message)
260 259
         pt.join()
261  
-        
  260
+
262 261
         # it got stored in the schedule instead of executing
263 262
         self.assertFalse('k2' in state)
264 263
         self.assertTrue(r2.task_id in self.consumer.schedule._schedule)
265  
-        
  264
+
266 265
         # run through an iteration of the scheduler
267 266
         self.consumer.check_schedule(dt)
268  
-        
  267
+
269 268
         # our command was not enqueued
270 269
         self.assertEqual(len(self.consumer.invoker.queue), 0)
271  
-        
  270
+
272 271
         # try running the scheduler with the time the command should run
273 272
         self.consumer.check_schedule(dt2)
274  
-        
  273
+
275 274
         # it was enqueued
276 275
         self.assertEqual(len(self.consumer.invoker.queue), 1)
277 276
         self.assertEqual(self.consumer.schedule._schedule, {})
278  
-        
  277
+
279 278
         # dequeue and inspect -- it won't be executed because the scheduler will
280 279
         # see that it is scheduled to run in the future and plop it back into the
281 280
         # schedule
282 281
         command = self.consumer.invoker.dequeue()
283 282
         self.assertEqual(command.task_id, r2.task_id)
284 283
         self.assertEqual(command.execute_time, dt2)
285  
-    
  284
+
286 285
     def test_retry_scheduling(self):
287 286
         # this will continually fail
288 287
         res = retry_command_slow('blampf')
289 288
         self.assertEqual(self.consumer.schedule._schedule, {})
290  
-        
  289
+
291 290
         cur_time = datetime.datetime.utcnow()
292  
-        
  291
+
293 292
         cmd = test_invoker.dequeue()
294 293
         self.consumer._pool.acquire()
295 294
         self.consumer.worker(cmd)
@@ -297,93 +296,93 @@ def test_retry_scheduling(self):
297 296
             'unhandled exception in worker thread',
298 297
             're-enqueueing task %s, 2 tries left' % cmd.task_id,
299 298
         ])
300  
-        
  299
+
301 300
         self.assertEqual(self.consumer.schedule._schedule, {
302 301
             cmd.task_id: cmd,
303 302
         })
304 303
         cmd_from_sched = self.consumer.schedule._schedule[cmd.task_id]
305 304
         self.assertEqual(cmd_from_sched.retries, 2)
306 305
         exec_time = cmd.execute_time
307  
-        
  306
+
308 307
         self.assertEqual((exec_time - cur_time).seconds, 10)
309  
-    
  308
+
310 309
     def test_schedule_local_utc(self):
311 310
         dt = datetime.datetime(2011, 1, 1, 0, 0)
312 311
         dt2 = datetime.datetime(2037, 1, 1, 0, 0)
313 312
         r1 = modify_state.schedule(args=('k', 'v'), eta=dt)
314 313
         r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt2)
315  
-        
316  
-        # dequeue a *single* message 
  314
+
  315
+        # dequeue a *single* message
317 316
         pt = self.spawn(self.consumer.check_message)
318  
-        
  317
+
319 318
         # work on any messages generated by the processor thread
320 319
         st = self.spawn(self.consumer.worker_pool)
321  
-        
  320
+
322 321
         pt.join()
323 322
         self.assertTrue('k' in state)
324 323
         self.assertEqual(self.consumer.schedule._schedule, {})
325  
-        
326  
-        # dequeue a *single* message 
  324
+
  325
+        # dequeue a *single* message
327 326
         pt = self.spawn(self.consumer.check_message)
328 327
         pt.join()
329  
-        
  328
+
330 329
         # it got stored in the schedule instead of executing
331 330
         self.assertFalse('k2' in state)
332 331
         self.assertTrue(r2.task_id in self.consumer.schedule._schedule)
333  
-        
  332
+
334 333
         # run through an iteration of the scheduler
335 334
         self.consumer.check_schedule(dt)
336  
-        
  335
+
337 336
         # our command was not enqueued
338 337
         self.assertEqual(len(self.consumer.invoker.queue), 0)
339  
-        
  338
+
340 339
         # try running the scheduler with the time the command should run
341 340
         self.consumer.check_schedule(local_to_utc(dt2))
342  
-        
  341
+
343 342
         # it was enqueued
344 343
         self.assertEqual(len(self.consumer.invoker.queue), 1)
345 344
         self.assertEqual(self.consumer.schedule._schedule, {})
346  
-        
  345
+
347 346
         # dequeue and inspect -- it won't be executed because the scheduler will
348 347
         # see that it is scheduled to run in the future and plop it back into the
349 348
         # schedule
350 349
         command = self.consumer.invoker.dequeue()
351 350
         self.assertEqual(command.task_id, r2.task_id)
352 351
         self.assertEqual(command.execute_time, local_to_utc(dt2))
353  
-    
  352
+
354 353
     def test_schedule_persistence(self):
355 354
         dt = datetime.datetime(2037, 1, 1, 0, 0)
356 355
         dt2 = datetime.datetime(2037, 1, 1, 0, 1)
357 356
         r = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
358 357
         r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False)
359  
-        
  358
+
360 359
         # two messages in the queue
361 360
         self.assertEqual(len(self.consumer.invoker.queue), 2)
362  
-        
  361
+
363 362
         # pull 'em down
364 363
         self.consumer.check_message()
365 364
         self.consumer.check_message()
366  
-        
  365
+
367 366
         self.consumer.save_schedule()
368 367
         self.consumer.schedule._schedule = {}
369  
-        
  368
+
370 369
         self.consumer.load_schedule()
371 370
         self.assertTrue(r.task_id in self.consumer.schedule._schedule)
372 371
         self.assertTrue(r2.task_id in self.consumer.schedule._schedule)
373  
-        
  372
+
374 373
         cmd1 = self.consumer.schedule._schedule[r.task_id]
375 374
         cmd2 = self.consumer.schedule._schedule[r2.task_id]
376  
-        
  375
+
377 376
         self.assertEqual(cmd1.execute_time, dt)
378 377
         self.assertEqual(cmd2.execute_time, dt2)
379  
-        
  378
+
380 379
         # check w/conversion
381 380
         r3 = modify_state.schedule(args=('k3', 'v3'), eta=dt)
382 381
         self.consumer.check_message()
383  
-        
  382
+
384 383
         self.consumer.save_schedule()
385 384
         self.consumer.schedule._schedule = {}
386  
-        
  385
+
387 386
         self.consumer.load_schedule()
388 387
         cmd3 = self.consumer.schedule._schedule[r3.task_id]
389 388
         self.assertEqual(cmd3.execute_time, local_to_utc(dt))
36  huey/tests/crontab.py
... ...
@@ -1,7 +1,7 @@
1 1
 import datetime
2 2
 import unittest
3 3
 
4  
-from huey.decorators import crontab
  4
+from huey import crontab
5 5
 
6 6
 
7 7
 class SkewCrontabTestCase(unittest.TestCase):
@@ -9,52 +9,52 @@ def test_crontab_month(self):
9 9
         # validates the following months, 1, 4, 7, 8, 9
10 10
         valids = [1, 4, 7, 8, 9]
11 11
         validate_m = crontab(month='1,4,*/6,8-9')
12  
-        
  12
+
13 13
         for x in xrange(1, 13):
14 14
             res = validate_m(datetime.datetime(2011, x, 1))
15 15
             self.assertEqual(res, x in valids)
16  
-    
  16
+
17 17
     def test_crontab_day(self):
18 18
         # validates the following days
19 19
         valids = [1, 4, 7, 8, 9, 13, 19, 25, 31]
20 20
         validate_d = crontab(day='*/6,1,4,8-9')
21  
-        
  21
+
22 22
         for x in xrange(1, 32):
23 23
             res = validate_d(datetime.datetime(2011, 1, x))
24 24
             self.assertEqual(res, x in valids)
25  
-    
  25
+
26 26
     def test_crontab_hour(self):
27 27
         # validates the following hours
28 28
         valids = [0, 1, 4, 6, 8, 9, 12, 18]
29 29
         validate_h = crontab(hour='8-9,*/6,1,4')
30  
-        
  30
+
31 31
         for x in xrange(24):
32 32
             res = validate_h(datetime.datetime(2011, 1, 1, x))
33 33
             self.assertEqual(res, x in valids)
34  
-        
  34
+
35 35
         edge = crontab(hour=0)
36 36
         self.assertTrue(edge(datetime.datetime(2011, 1, 1, 0, 0)))
37 37
         self.assertFalse(edge(datetime.datetime(2011, 1, 1, 12, 0)))
38  
-    
  38
+
39 39
     def test_crontab_minute(self):
40 40
         # validates the following minutes
41 41
         valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54]
42 42
         validate_m = crontab(minute='4,8-9,*/6,1')
43  
-        
  43
+
44 44
         for x in xrange(60):
45 45
             res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
46 46
             self.assertEqual(res, x in valids)
47  
-    
  47
+
48 48
     def test_crontab_day_of_week(self):
49 49
         # validates the following days of week
50 50
         # jan, 1, 2011 is a saturday
51 51
         valids = [2, 4, 9, 11, 16, 18, 23, 25, 30]
52 52
         validate_dow = crontab(day_of_week='0,2')
53  
-        
  53
+
54 54
         for x in xrange(1, 32):
55 55
             res = validate_dow(datetime.datetime(2011, 1, x))
56 56
             self.assertEqual(res, x in valids)
57  
-    
  57
+
58 58
     def test_crontab_all_together(self):
59 59
         # jan 1, 2011 is a saturday
60 60
         # may 1, 2011 is a sunday
@@ -65,22 +65,22 @@ def test_crontab_all_together(self):
65 65
             hour='*/4',
66 66
             minute='1-5,10-15,50'
67 67
         )
68  
-        
  68
+
69 69
         self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11)))
70 70
         self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50)))
71 71
         self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1)))
72  
-        
  72
+
73 73
         # fails validation on month
74 74
         self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11)))
75  
-        
  75
+
76 76
         # fails validation on day
77 77
         self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11)))
78  
-        
  78
+
79 79
         # fails validation on day_of_week
80 80
         self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11)))
81  
-        
  81
+
82 82
         # fails validation on hour
83 83
         self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11)))
84  
-        
  84
+
85 85
         # fails validation on minute
86 86
         self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6)))

0 notes on commit 4d35ecb

Please sign in to comment.
Something went wrong with that request. Please try again.