/
test_scheduler.py
242 lines (193 loc) · 10.2 KB
/
test_scheduler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
"""
Copyright 2018, Oath Inc.
Licensed under the terms of the Apache 2.0 license. See LICENSE file in project root for terms.
"""
from __future__ import absolute_import
import unittest
import signal
from celery import app
from mock import create_autospec, patch, MagicMock, Mock
from datetime import timedelta
from celery.beat import Service
from yahoo_panoptes.framework.celery_manager import PanoptesCeleryConfig, PanoptesCeleryPluginScheduler, \
PanoptesScheduleEntry
from yahoo_panoptes.framework.resources import PanoptesContext
from yahoo_panoptes.framework.plugins.scheduler import PanoptesPluginScheduler
from yahoo_panoptes.framework.utilities.tour_of_duty import PanoptesTourOfDuty
from .test_framework import PanoptesTestKeyValueStore, panoptes_mock_kazoo_client, panoptes_mock_redis_strict_client
from .helpers import get_test_conf_file
def _callback(*args):
pass
def _callback_exception():
raise Exception
def _callback_no_args():
pass
def _mock_is_set_true():
return True
class TestPanoptesPluginScheduler(unittest.TestCase):
@patch(u'redis.StrictRedis', panoptes_mock_redis_strict_client)
@patch(u'kazoo.client.KazooClient', panoptes_mock_kazoo_client)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = get_test_conf_file()
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore],
create_message_producer=False, async_message_producer=False,
create_zookeeper_client=True)
self._celery_config = PanoptesCeleryConfig(app_name=u"Polling Plugin Test")
self._scheduler = PanoptesPluginScheduler(
panoptes_context=self._panoptes_context,
plugin_type=u"polling",
plugin_type_display_name=u"Polling",
celery_config=self._celery_config,
lock_timeout=1,
plugin_scheduler_task=_callback
)
def test_start_basic_operations(self):
# Test bad input
with self.assertRaises(AssertionError):
PanoptesPluginScheduler(u"test", u"polling", u"Polling", self._celery_config, 1,
_callback)
with self.assertRaises(AssertionError):
PanoptesPluginScheduler(self._panoptes_context, u"", u"Polling", self._celery_config, 1,
_callback)
with self.assertRaises(AssertionError):
PanoptesPluginScheduler(self._panoptes_context, u"polling", u"", self._celery_config, 1,
_callback)
with self.assertRaises(AssertionError):
PanoptesPluginScheduler(self._panoptes_context, u"polling", u"Polling", u"Test", 1,
_callback)
with self.assertRaises(AssertionError):
PanoptesPluginScheduler(self._panoptes_context, u"polling", u"Polling", self._celery_config, 0,
_callback)
with self.assertRaises(AssertionError):
PanoptesPluginScheduler(self._panoptes_context, u"polling", u"Polling", u"Test", 1,
object)
# Test locking error when starting up the scheduler
mock_lock = MagicMock(side_effect=Exception)
with patch(u'yahoo_panoptes.framework.plugins.scheduler.PanoptesLock', mock_lock):
with self.assertRaises(SystemExit):
self._scheduler.start()
celery_app = self._scheduler.start()
self.assertIsInstance(celery_app, app.base.Celery)
def test_redundant_shutdown_signal(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
self._scheduler.run(celery_beat_service)
temp_is_set = self._scheduler._shutdown_plugin_scheduler.is_set
self._scheduler._shutdown_plugin_scheduler.is_set = _mock_is_set_true
self._scheduler._signal_handler(signal.SIGTERM, None) # pragma: no cover
self._scheduler._shutdown()
self.assertTrue(self._scheduler._t.isAlive())
with self.assertRaises(SystemExit):
self._scheduler._shutdown_plugin_scheduler.is_set = temp_is_set
self._scheduler._signal_handler(signal.SIGTERM, None) # pragma: no cover
def test_shutdown_after_tour_of_duty(self):
mock_tour_of_duty = create_autospec(PanoptesTourOfDuty)
mock_tour_of_duty.completed.return_value = True
mock_tour_of_duty.tasks_completed.return_value = True
mock_tour_of_duty.time_completed.return_value = True
mock_tour_of_duty.memory_growth_completed.return_value = True
with patch(u'yahoo_panoptes.framework.plugins.scheduler.PanoptesTourOfDuty', mock_tour_of_duty):
self._scheduler = PanoptesPluginScheduler(
panoptes_context=self._panoptes_context,
plugin_type=u"polling",
plugin_type_display_name=u"Polling",
celery_config=self._celery_config,
lock_timeout=1,
plugin_scheduler_task=_callback
)
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
self._scheduler.run(celery_beat_service)
def test_celery_beat_error(self):
mock_celery_instance = MagicMock(side_effect=Exception)
with patch(u'yahoo_panoptes.framework.plugins.scheduler.PanoptesCeleryInstance', mock_celery_instance):
celery_app = self._scheduler.start()
self.assertIsNone(celery_app)
class TestPanoptesUniformScheduleEntry(unittest.TestCase):
@patch(u'redis.StrictRedis', panoptes_mock_redis_strict_client)
@patch(u'kazoo.client.KazooClient', panoptes_mock_kazoo_client)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = get_test_conf_file()
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore],
create_message_producer=False, async_message_producer=False,
create_zookeeper_client=True)
self.mock_kv_store = self._panoptes_context.get_kv_store(PanoptesTestKeyValueStore)
@patch('time.time')
def test_not_scheduled_before(self, mock_time):
# Test `Task` Has Never Been Uniformly Scheduled
mock_time.return_value = 60
schedule_entry_not_due = PanoptesScheduleEntry(
name='Test Task', task='',
total_run_count=0, schedule=timedelta(seconds=60),
kv_store=self.mock_kv_store)
self.assertEqual(schedule_entry_not_due.run_at, 72.0)
schedstate = schedule_entry_not_due.is_due()
self.assertEqual(schedstate.is_due, False)
self.assertEqual(schedstate.next, 12.0)
# Task Is Due
mock_time.side_effect = [60, 72, 72]
schedule_entry_due = PanoptesScheduleEntry(
name='Test Task', task='',
total_run_count=0, schedule=timedelta(seconds=60),
kv_store=self.mock_kv_store)
self.assertEqual(schedule_entry_due.run_at, 72.0)
schedstate = schedule_entry_due.is_due()
self.assertEqual(schedstate.is_due, True)
self.assertEqual(schedstate.next, 60.0)
self.assertEqual(self.mock_kv_store.get('plugin_metadata:Test Task:last_uniformly_scheduled'), '72')
# Verify Next Updates The Correct Fields
schedule_entry_due = next(schedule_entry_due)
self.assertEqual(schedule_entry_due.total_run_count, 1)
self.assertEqual(schedule_entry_due.uniformly_scheduled, True)
self.assertIsNotNone(schedule_entry_due.last_run_at)
@patch('time.time')
def test_scheduled_before_and_overdue(self, mock_time):
"""
Verify that a new instance of the scheduler attempts to pick up where the previous
scheduler left off. In this case it is unable as too much time has passed since
the last uniformly scheduled key was set.
"""
mock_time.return_value = 100
schedule_entry = PanoptesScheduleEntry(
name='Test Task', task='',
total_run_count=0, schedule=timedelta(seconds=60),
kv_store=self.mock_kv_store,
last_uniformly_scheduled_at='30')
self.assertEqual(schedule_entry.run_at, 112.0)
schedstate = schedule_entry.is_due()
self.assertEqual(schedstate.is_due, False)
self.assertEqual(schedstate.next, 12.0)
@patch('time.time')
def test_scheduled_before_and_not_overdue(self, mock_time):
"""
Verify that a new instance of the scheduler can pick up where
the previous one left off.
"""
mock_time.return_value = 70
schedule_entry = PanoptesScheduleEntry(
name='Test Task', task='',
total_run_count=0, schedule=timedelta(seconds=60),
kv_store=self.mock_kv_store,
last_uniformly_scheduled_at='60')
self.assertEqual(schedule_entry.run_at, 120.0)
schedstate = schedule_entry.is_due()
self.assertEqual(schedstate.is_due, False)
self.assertEqual(schedstate.next, 50.0)
def test_schedule_entry_unique_identifier_does_not_throw(self):
self.assertEqual(
PanoptesScheduleEntry.schedule_entry_unique_identifier(timedelta(seconds=60), ('Test', 1, 1.0, b'bytes')),
"60.0|Test|1|1.0|b'bytes'"
)
def test_invalid_key_in_redis_is_handled(self):
schedule_entry = PanoptesScheduleEntry(
name='Test Task', task='',
total_run_count=0, schedule=timedelta(seconds=60),
kv_store=self.mock_kv_store,
last_uniformly_scheduled_at='60f')
schedstate = schedule_entry.is_due()
self.assertEqual(schedstate.is_due, False)
self.assertEqual(schedstate.next, 60)