Skip to content
Newer
Older
100644 310 lines (288 sloc) 15.3 KB
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
1 import os
2 import zmq
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
3 from zmq.devices.basedevice import ProcessDevice
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
4 import tornado
5 from tornado.options import define, options
6 import logging
7 import zlib
8 import cPickle as pickle
9 import sys
10 import time
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
11 from threading import Thread
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
12 from multiprocessing import Process, cpu_count
13
8f86110 @JeremyOT added redis, refactored db options to share host/port setting, bug fi…
authored
14 define("database", metavar='mysql|mongodb|redis|none', default="mongodb", help="the database driver to use")
15 define("db_host", default='localhost', help="The host to use for database connections.")
16 define("db_port", default=0, help="The port to use for database connections. Leave this at zero to use the default for the selected database type")
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
17 define("mysql_database", type=str, help="Main MySQL schema name")
18 define("mysql_user", type=str, help="Main MySQL user")
19 define("mysql_password", type=str, help="Main MySQL user password")
20 define("mongodb_database", default="toto_server", help="MongoDB database")
8f86110 @JeremyOT added redis, refactored db options to share host/port setting, bug fi…
authored
21 define("redis_database", default=0, help="Redis DB")
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
22 define("daemon", metavar='start|stop|restart', help="Start, stop or restart this script as a daemon process. Use this setting in conf files, the shorter start, stop, restart aliases as command line arguments. Requires the multiprocessing module.")
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
23 define("processes", default=-1, help="The number of daemon processes to run, pass 0 to run only the load balancer. Negative numbers will run one worker per cpu")
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
24 define("pidfile", default="toto.worker.pid", help="The path to the pidfile for daemon processes will be named <path>.<num>.pid (toto.worker.pid -> toto.worker.0.pid)")
25 define("method_module", default='methods', help="The root module to use for method lookup")
26 define("remote_event_receivers", type=str, help="A comma separated list of remote event address that this event manager should connect to. e.g.: 'tcp://192.168.1.2:8889'", multiple=True)
27 define("event_init_module", default=None, type=str, help="If defined, this module's 'invoke' function will be called with the EventManager instance after the main event handler is registered (e.g.: myevents.setup)")
28 define("start", default=False, help="Alias for daemon=start for command line usage - overrides daemon setting.")
29 define("stop", default=False, help="Alias for daemon=start for command line usage - overrides daemon setting.")
30 define("restart", default=False, help="Alias for daemon=start for command line usage - overrides daemon setting.")
31 define("nodaemon", default=False, help="Alias for daemon='' for command line usage - overrides daemon setting.")
c1f3f4a @JeremyOT simplified cassandraconnection to act as a wrapper around connection …
authored
32 define("startup_function", default=None, type=str, help="An optional function to run on startup - e.g. module.function. The function will be called for each worker process after it is configured and before it starts listening for tasks with the named parameters worker and db_connection.")
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
33 define("debug", default=False, help="Set this to true to prevent Toto from nicely formatting generic errors. With debug=True, errors will print to the command line")
34 define("event_port", default=8999, help="The address to listen to event connections on - due to message queuing, servers use the next higher port as well")
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
35 define("worker_address", default="tcp://*:55555", help="The service will bind to this address with a zmq PULL socket and listen for incoming tasks. Tasks will be load balanced to all workers. If this is set to an empty string, workers will connect directly to worker_socket_address.")
36 define("worker_socket_address", default="ipc:///tmp/workerservice.sock", help="The load balancer will use this address to coordinate tasks between local workers")
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
37 define("control_socket_address", default="ipc:///tmp/workercontrol.sock", help="Workers will subscribe to messages on this socket and listen for control commands. If this is an empty string, the command option will have no effect")
38 define("command", type=str, metavar='status|shutdown', help="Specify a command to send to running workers on the control socket")
04f397d @JeremyOT added serialization and compression customization to workers
authored
39 define("compression_module", type=str, help="The module to use for compressing and decompressing messages. The module must have 'decompress' and 'compress' methods. If not specified, no compression will be used. You can also set worker.compress and worker.decompress in your startup method for increased flexibility")
40 define("serialization_module", type=str, help="The module to use for serializing and deserializing messages. The module must have 'dumps' and 'loads' methods. If not specified, cPickle will be used. You can also set worker.dumps and worker.loads in your startup method for increased flexibility")
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
41
42 #convert p to the absolute path, insert ".i" before the last "." or at the end of the path
43 def pid_path_with_id(p, i):
44 (d, f) = os.path.split(os.path.abspath(p))
45 components = f.rsplit('.', 1)
46 f = '%s.%s' % (components[0], i)
47 if len(components) > 1:
48 f += "." + components[1]
49 return os.path.join(d, f)
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
50
51 class TotoWorkerService():
52
53 def __load_options(self, conf_file=None, **kwargs):
54 for k in kwargs:
55 options[k].set(kwargs[k])
56 if conf_file:
57 tornado.options.parse_config_file(conf_file)
58 tornado.options.parse_command_line()
59 if options.start:
60 options['daemon'].set('start')
61 elif options.stop:
62 options['daemon'].set('stop')
63 elif options.restart:
64 options['daemon'].set('restart')
65 elif options.nodaemon:
66 options['daemon'].set('')
67
68 def __init__(self, conf_file=None, **kwargs):
69 module_options = {'method_module', 'event_init_module'}
70 function_options = {'startup_function'}
71 original_argv, sys.argv = sys.argv, [i for i in sys.argv if i.strip('-').split('=')[0] in module_options]
72 self.__load_options(conf_file, **{i: kwargs[i] for i in kwargs if i in module_options})
73 modules = {getattr(options, i) for i in module_options if getattr(options, i)}
74 for module in modules:
75 __import__(module)
76 function_modules = {getattr(options, i).rsplit('.', 1)[0] for i in function_options if getattr(options, i)}
77 for module in function_modules:
78 __import__(module)
79 sys.argv = original_argv
80 #clear root logger handlers to prevent duplicate logging if user has specified a log file
81 if options.log_file_prefix:
82 root_logger = logging.getLogger()
83 for handler in [h for h in root_logger.handlers]:
84 root_logger.removeHandler(handler)
85 self.__load_options(conf_file, **kwargs)
86 #clear method_module references so we can fully reload with new options
87 for module in modules:
88 for i in (m for m in sys.modules.keys() if m.startswith(module)):
89 del sys.modules[i]
90 for module in function_modules:
91 for i in (m for m in sys.modules.keys() if m.startswith(module)):
92 del sys.modules[i]
93 #prevent the reloaded module from re-defining options
94 define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None
95 self.__event_init = options.event_init_module and __import__(options.event_init_module) or None
96 self.__method_module = options.method_module and __import__(options.method_module) or None
97 tornado.options.define = define
98
99 def __run_server(self):
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
100 balancer = None
101 if options.worker_address:
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
102 balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
103 balancer.bind_in(options.worker_address)
104 balancer.bind_out(options.worker_socket_address)
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
105 balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
106 balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
107 balancer.start()
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
108
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
109 def start_server_process(module, pidfile):
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
110 db_connection = None
111 if options.database == "mongodb":
112 from mongodbconnection import MongoDBConnection
7365f8c @JeremyOT removed excess params from worker db connection
authored
113 db_connection = MongoDBConnection(options.db_host, options.db_port or 27017, options.mongodb_database)
8f86110 @JeremyOT added redis, refactored db options to share host/port setting, bug fi…
authored
114 elif options.database == "redis":
115 from redisconnection import RedisConnection
7365f8c @JeremyOT removed excess params from worker db connection
authored
116 db_connection = RedisConnection(options.db_host, options.db_port or 6379, options.redis_database)
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
117 elif options.database == "mysql":
118 from mysqldbconnection import MySQLdbConnection
7365f8c @JeremyOT removed excess params from worker db connection
authored
119 db_connection = MySQLdbConnection('%s:%s' % (options.db_host, options.db_port or 3306), options.mysql_database, options.mysql_user, options.mysql_password)
8f86110 @JeremyOT added redis, refactored db options to share host/port setting, bug fi…
authored
120 else:
121 from fakeconnection import FakeConnection
122 db_connection = FakeConnection()
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
123
124 if options.remote_event_receivers:
125 from toto.events import EventManager
126 event_manager = EventManager.instance()
127 if options.remote_instances:
128 for address in options.remote_event_receivers.split(','):
129 event_manager.register_server(address)
130 init_module = self.__event_init
131 if init_module:
132 init_module.invoke(event_manager)
04f397d @JeremyOT added serialization and compression customization to workers
authored
133 serialization = options.serialization_module and __import__(options.serialization_module) or pickle
134 compression = options.compression_module and __import__(options.compression_module)
135 worker = TotoWorker(module, options.worker_socket_address, db_connection, compression, serialization, pidfile)
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
136 if options.startup_function:
137 startup_path = options.startup_function.rsplit('.')
138 __import__(startup_path[0]).__dict__[startup_path[1]](worker=worker, db_connection=db_connection)
139 worker.start()
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
140 count = options.processes if options.processes >= 0 else cpu_count()
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
141 processes = []
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
142 worker_pidfiles = options.daemon and [pid_path_with_id(options.pidfile, i) for i in xrange(1, count + 1)] or []
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
143 for i in xrange(count):
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
144 proc = Process(target=start_server_process, args=(self.__method_module, worker_pidfiles and worker_pidfiles[i]))
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
145 proc.daemon = True
146 processes.append(proc)
147 proc.start()
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
148 if count == 0:
149 print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (options.worker_address, options.worker_socket_address)
150 else:
151 print "Starting %s worker process%s. %s." % (count, count > 1 and 'es' or '', options.worker_address and ('Listening on "%s"' % options.worker_address) or ('Connecting to "%s"' % options.worker_socket_address))
152 if options.daemon:
153 i = 1
154 for proc in processes:
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
155 with open(worker_pidfiles[i - 1], 'w') as f:
f1d2e20 @JeremyOT fixed daemon process exit, added ability to run without or only as lo…
authored
156 f.write(str(proc.pid))
157 i += 1
158 if balancer:
159 with open(pid_path_with_id(options.pidfile, i), 'w') as f:
160 f.write(str(balancer.launcher.pid))
6b63d5b @JeremyOT added monitor pub socket to worker queue
authored
161 if balancer:
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
162 balancer.join()
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
163 for proc in processes:
164 proc.join()
165
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
166 def send_worker_command(self, command):
167 if options.control_socket_address:
168 socket = zmq.Context().socket(zmq.PUB)
169 socket.bind(options.control_socket_address)
170 time.sleep(1)
3e54a8f @JeremyOT updated worker command to send unicode
authored
171 socket.send_string('command %s' % command)
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
172 print "Sent command: %s" % options.command
173
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
174 def run(self):
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
175 if options.command:
176 self.send_worker_command(options.command)
177 return
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
178 if options.daemon:
179 import multiprocessing
0b84925 @JeremyOT prevent worker startup if pidfiles present
authored
180 import signal, re
181
182 pattern = pid_path_with_id(options.pidfile, r'\d+').replace('.', r'\.')
183 piddir = os.path.dirname(pattern)
184 existing_pidfiles = [pidfile for pidfile in (os.path.join(piddir, fn) for fn in os.listdir(os.path.dirname(pattern))) if re.match(pattern, pidfile)]
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
185
186 if options.daemon == 'stop' or options.daemon == 'restart':
0b84925 @JeremyOT prevent worker startup if pidfiles present
authored
187 for pidfile in existing_pidfiles:
188 with open(pidfile, 'r') as f:
189 pid = int(f.read())
190 try:
191 os.kill(pid, signal.SIGTERM)
192 except OSError as e:
193 if e.errno != 3:
194 raise
195 print "Stopped server %s" % pid
196 os.remove(pidfile)
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
197
198 if options.daemon == 'start' or options.daemon == 'restart':
199 import sys
0b84925 @JeremyOT prevent worker startup if pidfiles present
authored
200 if existing_pidfiles:
201 print "Not starting, pidfile%s exist%s at %s" % (len(existing_pidfiles) > 1 and 's' or '', len(existing_pidfiles) == 1 and 's' or '', ', '.join(existing_pidfiles))
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
202 return
0b84925 @JeremyOT prevent worker startup if pidfiles present
authored
203 pidfile = pid_path_with_id(options.pidfile, 0)
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
204 #fork and only continue on child process
205 if not os.fork():
206 #detach from controlling terminal
207 os.setsid()
208 #fork again and write pid to pidfile from parent, run server on child
209 pid = os.fork()
210 if pid:
211 with open(pidfile, 'w') as f:
212 f.write(str(pid))
213 else:
214 self.__run_server()
215
216 if options.daemon not in ('start', 'stop', 'restart'):
217 print "Invalid daemon option: " + options.daemon
218
219 else:
220 self.__run_server()
221
222 class TotoWorker():
04f397d @JeremyOT added serialization and compression customization to workers
authored
223 def __init__(self, method_module, socket_address, db_connection, compression=None, serialization=None, pidfile=None):
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
224 self.context = zmq.Context()
225 self.socket_address = socket_address
226 self.method_module = method_module
227 self.db_connection = db_connection
d9572f6 @JeremyOT switched worker to use req instead of push
authored
228 self.db = db_connection and db_connection.db or None
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
229 self.status = 'Initialized'
230 self.running = False
231 self.__pidfile = pidfile
04f397d @JeremyOT added serialization and compression customization to workers
authored
232 self.compress = compression and compression.compress or (lambda x: x)
233 self.decompress = compression and compression.decompress or (lambda x: x)
234 self.loads = serialization and serialization.loads or pickle.loads
235 self.dumps = serialization and serialization.dumps or pickle.dumps
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
236 if options.debug:
237 from traceback import format_exc
238 def log_error(self, e):
aba10fe @JeremyOT fixed worker error logging
authored
239 err_string = format_exc()
240 logging.error(err_string)
241 return err_string
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
242 TotoWorker.log_error = log_error
243
244 def log_error(self, e):
aba10fe @JeremyOT fixed worker error logging
authored
245 err_string = repr(e)
246 logging.error(err_string)
247 return err_string
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
248
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
249 def log_status(self):
250 logging.info('Pid: %s Pidfile: %s status: %s' % (os.getpid(), self.__pidfile, self.status))
251
252 def __monitor_control(self, address=options.control_socket_address):
253 def monitor():
254 socket = self.context.socket(zmq.SUB)
255 socket.setsockopt(zmq.SUBSCRIBE, 'command')
256 socket.connect(address)
257 while self.running:
258 try:
259 command = socket.recv().split(' ', 1)[1]
260 logging.info("Received command: %s" % command)
261 if command == 'shutdown':
262 self.running = False
263 self.context.term()
264 return
265 elif command == 'status':
266 self.log_status()
267 except Exception as e:
268 self.log_error(e)
269 if address:
270 thread = Thread(target=monitor)
271 thread.daemon = True
272 thread.start()
273
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
274 def start(self):
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
275 self.running = True
276 self.__monitor_control()
6b63d5b @JeremyOT added monitor pub socket to worker queue
authored
277 socket = self.context.socket(zmq.REP)
278 socket.connect(self.socket_address)
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
279 pending_reply = False
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
280 while self.running:
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
281 try:
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
282 self.status = 'Listening'
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
283 message = socket.recv_multipart()
284 pending_reply = True
285 message_id = message[0]
04f397d @JeremyOT added serialization and compression customization to workers
authored
286 data = self.loads(self.decompress(message[1]))
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
287 logging.info('Received Task %s: %s' % (message_id, data['method']))
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
288 method = self.method_module
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
289 for i in data['method'].split('.'):
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
290 method = getattr(method, i)
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
291 if hasattr(method.invoke, 'asynchronous'):
292 socket.send_multipart((message_id,))
293 pending_reply = False
294 self.status = 'Working'
295 method.invoke(self, data['parameters'])
296 else:
297 self.status = 'Working'
298 response = method.invoke(self, data['parameters'])
04f397d @JeremyOT added serialization and compression customization to workers
authored
299 socket.send_multipart((message_id, self.compress(self.dumps(response))))
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
300 pending_reply = False
94d625b @JeremyOT renamed remoteworker -> clientsideworker, added worker process and co…
authored
301 except Exception as e:
aba10fe @JeremyOT fixed worker error logging
authored
302 err_string = self.log_error(e)
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
303 if pending_reply:
aba10fe @JeremyOT fixed worker error logging
authored
304 socket.send_multipart((message_id, self.compress(self.dumps(err_string))))
f73d242 @JeremyOT updated worker connection to use ioloop, retry lost requests.
authored
305
f395060 @JeremyOT added join to worker connection, improved worker logging output
authored
306 self.status = 'Finished'
307 self.log_status()
18c33fd @JeremyOT improved ability to hot connect/disconnect workers
authored
308 if self.__pidfile:
309 os.remove(self.__pidfile)
Something went wrong with that request. Please try again.