Skip to content
This repository
  • 4 commits
  • 8 files changed
  • 0 comments
  • 1 contributor
Aug 07, 2012
Alexis Metaireau ametaireau rename server into proxy f1e90c9
Aug 13, 2012
Alexis Metaireau ametaireau Add a new "On the fly" proxy which is able to change its handlers on …
…the fly.

Coupled to a tiny web server, this allows to change how the proxy will behave
from elsewhere. This is useful into other things to run load tests and be sure
that everything behaves the way it's intended.
7502fe1
Alexis Metaireau ametaireau update the documentation about the web interface 2615b81
Alexis Metaireau ametaireau Deal with optional deps 993533d
23 README.rst
Source Rendered
@@ -52,3 +52,26 @@ boolean that tels you if this is the communication to the proxied server or
52 52 from it, name is the name of the callable, settings the settings for *this*
53 53 callable and server the server instance (can be useful to look at the global
54 54 settings for instance, and other utilities)
  55 +
  56 +Controlling vaurien from a web interface
  57 +========================================
  58 +
  59 +Sometimes, it is useful to control how the proxy behaves, on a request to
  60 +request basis. Vaurien provides two proxies, the Random one that we defined
  61 +earlier and the OnTheFly one.
  62 +
  63 +The "on the fly" proxy comes with a simple http server to control itself. It
  64 +has two resources that could be useful to you:
  65 +
  66 + * `/handler [GET, POST]` which allows to either know what is the current
  67 + handler in use (when doing a `GET`) or to set a new one for the next
  68 + calls (`POST`). You can for instance do this with the following curl
  69 + call::
  70 +
  71 + $ curl -d"delay" http://localhost:8080/handler -H "Content-Type: text/plain"
  72 + OK
  73 +
  74 + * `/handlers [GET]` returns a list of handlers that are possible to use::
  75 +
  76 + $ curl http://localhost:8080/handlers -H "Content-Type: application/json"
  77 + {"handlers": ["delay", "errors", "hang", "blackout", "normal"]}
11 vaurien/config.py
@@ -183,6 +183,7 @@ class SettingsDict(dict):
183 183
184 184 * setdefaults: copy any unset settings from another dict
185 185 * getsection: return a dict of settings for just one subsection
  186 + * sections: return a list of sections for the settings
186 187
187 188 """
188 189
@@ -199,6 +200,16 @@ def copy(self):
199 200 new_items[k] = v
200 201 return new_items
201 202
  203 + def sections(self):
  204 + """Return a list of sections for this dict"""
  205 + sections = []
  206 + for key in self.iterkeys():
  207 + if self.separator in key:
  208 + sec, _ = key.rsplit(self.separator, 1)
  209 + if sec not in sections:
  210 + sections.append(sec)
  211 + return sections
  212 +
202 213 def getsection(self, section):
203 214 """Get a dict for just one sub-section of the config.
204 215
23 vaurien/handlers.py
... ... @@ -1,37 +1,40 @@
1 1 import gevent
2 2
3 3
4   -def normal(source, dest, to_backend, name, settings, server):
5   - request = server.get_data(source)
  4 +def normal(source, dest, to_backend, name, settings, proxy):
  5 + request = proxy.get_data(source)
6 6 dest.sendall(request)
7 7
8 8
9   -def delay(source, dest, to_backend, name, settings, server):
  9 +def delay(source, dest, to_backend, name, settings, proxy):
10 10 if to_backend:
11 11 # a bit of delay before calling the backend
12   - gevent.sleep(kwargs['settings'].get('sleep', 1))
  12 + gevent.sleep(settings.get('sleep', 1))
13 13
14   - normal(source, dest, to_backend, name, settings, server)
  14 + normal(source, dest, to_backend, name, settings, proxy)
15 15
16 16
17   -def errors(source, dest, to_backend, name, settings, server):
  17 +def errors(source, dest, to_backend, name, settings, proxy):
18 18 """Throw errors on the socket"""
19 19 if to_backend:
20   - server.get_data(source)
  20 + proxy.get_data(source)
21 21 # XXX find how to handle errors (which errors should we send)
22 22 #
23 23 # depends on the protocol
24 24 dest.sendall("YEAH")
25 25
26 26
27   -def hang(source, dest, to_backend, name, settings, server):
  27 +def hang(source, dest, to_backend, name, settings, proxy):
28 28 """Reads the packets that have been sent."""
29 29 # consume the socket and hang
30   - server.get_data(source)
  30 + proxy.get_data(source)
31 31 while True:
32 32 gevent.sleep(1.)
33 33
34 34
35   -def blackout(source, dest, to_backend, name, settings, server):
  35 +def blackout(source, dest, to_backend, name, settings, proxy):
36 36 """Don't do anything -- the sockets get closed"""
37 37 return
  38 +
  39 +
  40 +handlers = (normal, delay, errors, hang, blackout)
117 vaurien/server.py → vaurien/proxy.py
@@ -5,16 +5,21 @@
5 5 import random
6 6
7 7 from gevent.server import StreamServer
8   -from gevent.socket import create_connection, error
  8 +from gevent.socket import create_connection
9 9
10   -from vaurien.util import import_string, parse_address
11   -from vaurien.handlers import normal
  10 +from vaurien.util import parse_address, get_handlers_from_config
  11 +from vaurien.handlers import handlers as default_handlers, normal
12 12
13 13
14   -class DoWeirdThingsPlease(StreamServer):
  14 +class DefaultProxy(StreamServer):
15 15
16   - def __init__(self, local, distant, protocol=None, settings=None,
17   - statsd=None, logger=None, **kwargs):
  16 + def __init__(self, local, distant, handlers=None, protocol=None,
  17 + settings=None, statsd=None, logger=None, **kwargs):
  18 +
  19 + if handlers is None:
  20 + handlers = {}
  21 + for handler in default_handlers:
  22 + handlers[handler.__name__] = handler
18 23
19 24 logger.info('Starting the mean proxy server')
20 25 logger.info('%s => %s' % (local, distant))
@@ -30,54 +35,19 @@ def __init__(self, local, distant, protocol=None, settings=None,
30 35 self.running = True
31 36 self._statsd = statsd
32 37 self._logger = logger
33   - self.choices = []
34   - self.handlers = {}
35   - self.initialize_choices()
36   -
37   - def initialize_choices(self):
38   - total = 0
39   - behavior = self.settings.getsection('vaurien')['behavior']
40   - choices = {}
41   -
42   - for behavior in behavior.split(','):
43   - choice = behavior.split(':')
44   - if len(choice) != 2:
45   - raise ValueError('You need to use name:percentage')
46   -
47   - percent, handler_name = choice
48   - percent = int(percent)
49   -
50   - # have a look if we have a section named handler:{handler}
51   - settings = self.settings.getsection('handler.%s' % handler_name)
52   - if settings and 'callable' in settings:
53   - handler_location = settings['callable']
54   - else:
55   - handler_location = 'vaurien.handlers.' + handler_name
  38 + self.handlers = handlers
  39 + self.handlers.update(get_handlers_from_config(self.settings, logger))
  40 + self.next_handler = normal
56 41
57   - handler = import_string(handler_location)
58   -
59   - choices[handler_name] = handler, percent
60   - total += percent
61   -
62   - if total > 100:
63   - raise ValueError('The behavior total needs to be 100 or less')
64   - elif total < 100:
65   - missing = 100 - total
66   - if 'normal' in choices:
67   - choices['normal'][1] += missing
68   - else:
69   - choices['normal'] = normal, missing
70   -
71   - for name, (handler, percent) in choices.items():
72   - self.choices.extend(percent * [name])
73   - self.handlers[name] = handler
  42 + def get_next_handler(self):
  43 + return self.next_handler
74 44
75 45 def handle(self, source, address):
76 46 source.setblocking(0)
77 47 dest = create_connection(self.dest)
78 48 dest.setblocking(0)
79   - handler_name = random.choice(self.choices)
80   - handler = self.handlers[handler_name]
  49 + handler = self.get_next_handler()
  50 + handler_name = handler.__name__
81 51 self.statsd_incr(handler_name)
82 52 try:
83 53 back = gevent.spawn(self.weirdify, handler, handler_name, source,
@@ -104,10 +74,10 @@ def weirdify(self, handler, handler_name, source, dest, to_backend):
104 74 """
105 75 self._logger.debug('starting weirdify %s' % to_backend)
106 76 try:
107   - settings = self.settings.getsection('handlers:%s' %
  77 + settings = self.settings.getsection('handlers.%s' %
108 78 handler_name)
109 79 handler(source=source, dest=dest, to_backend=to_backend,
110   - name=handler_name, server=self, settings=settings)
  80 + name=handler_name, proxy=self, settings=settings)
111 81 finally:
112 82 self._logger.debug('exiting weirdify %s' % to_backend)
113 83
@@ -124,3 +94,50 @@ def get_data(self, source, delay=.2):
124 94 pass
125 95
126 96 return data
  97 +
  98 +
  99 +class RandomProxy(DefaultProxy):
  100 +
  101 + def __init__(self, *args, **kwargs):
  102 + super(RandomProxy, self).__init__(*args, **kwargs)
  103 +
  104 + self.choices = []
  105 + self.initialize_choices()
  106 +
  107 + def initialize_choices(self):
  108 + total = 0
  109 + behavior = self.settings.getsection('vaurien')['behavior']
  110 + choices = {}
  111 +
  112 + for behavior in behavior.split(','):
  113 + choice = behavior.split(':')
  114 + if len(choice) != 2:
  115 + raise ValueError('You need to use name:percentage')
  116 +
  117 + percent, handler_name = choice
  118 + percent = int(percent)
  119 +
  120 + choices[handler_name] = self.handlers[handler_name], percent
  121 + total += percent
  122 +
  123 + if total > 100:
  124 + raise ValueError('The behavior total needs to be 100 or less')
  125 + elif total < 100:
  126 + missing = 100 - total
  127 + if 'normal' in choices:
  128 + choices['normal'][1] += missing
  129 + else:
  130 + choices['normal'] = normal, missing
  131 +
  132 + for name, (handler, percent) in choices.items():
  133 + self.choices.extend(percent * [name])
  134 +
  135 + def get_next_handler(self):
  136 + return random.choice(self.choices)
  137 +
  138 +
  139 +class OnTheFlyProxy(DefaultProxy):
  140 +
  141 + def set_next_handler(self, handler):
  142 + self.next_handler = self.handlers[handler]
  143 + self._logger.info('next handler changed to "%s"' % handler)
39 vaurien/run.py
@@ -3,7 +3,7 @@
3 3 import sys
4 4 import logging
5 5
6   -from vaurien.server import DoWeirdThingsPlease
  6 +from vaurien.proxy import OnTheFlyProxy, RandomProxy
7 7 from vaurien.config import load_into_settings, DEFAULT_SETTINGS
8 8 from vaurien import __version__, logger
9 9
@@ -57,6 +57,12 @@ def main():
57 57 parser.add_argument('--config', help='Configuration file', default=None)
58 58 parser.add_argument('--version', action='store_true', default=False,
59 59 help='Displays version and exits.')
  60 + parser.add_argument('--http', action='store_true', default=False,
  61 + help='Start a simple http server to control vaurien')
  62 + parser.add_argument('--http-host', default='localhost',
  63 + help='Host of the http server, if any')
  64 + parser.add_argument('--http-port', default=8080,
  65 + help='Port of the http server, if any')
60 66
61 67 # get the values from the default config
62 68 keys = DEFAULT_SETTINGS.keys()
@@ -111,14 +117,31 @@ def main():
111 117
112 118 statsd = get_statsd_from_settings(settings.getsection('statsd'))
113 119
114   - # creating the server
115   - server = DoWeirdThingsPlease(local=settings['vaurien.local'],
116   - distant=settings['vaurien.distant'],
117   - settings=settings, statsd=statsd,
118   - logger=logger)
119   -
  120 + # creating the proxy
  121 + proxy_args = dict(local=settings['vaurien.local'],
  122 + distant=settings['vaurien.distant'],
  123 + settings=settings, statsd=statsd, logger=logger)
  124 +
  125 + # per default, we want to randomize
  126 + proxy_class = RandomProxy
  127 +
  128 + if args.http:
  129 + # if we are using the http server, then we want to use the OnTheFly
  130 + # proxy
  131 + proxy_class = OnTheFlyProxy
  132 + proxy = proxy_class(**proxy_args)
  133 + from vaurien.webserver import app
  134 + from gevent.wsgi import WSGIServer
  135 +
  136 + setattr(app, 'proxy', proxy)
  137 + # app.run(host=args.http_host, port=args.http_port)
  138 + http_server = WSGIServer((args.http_host, args.http_port), app)
  139 + http_server.start()
  140 + logger.info('Started the HTTP server')
  141 + else:
  142 + proxy = proxy_class(**proxy_args)
120 143 try:
121   - server.serve_forever()
  144 + proxy.serve_forever()
122 145 except KeyboardInterrupt:
123 146 sys.exit(0)
124 147 finally:
23 vaurien/util.py
@@ -95,3 +95,26 @@ def parse_address(address):
95 95 except ValueError:
96 96 sys.exit('Expected HOST:PORT: %r' % address)
97 97 return gethostbyname(hostname), port
  98 +
  99 +
  100 +def get_handlers_from_config(settings, logger=None):
  101 + """Return a dict containing all the handlers that are defined in the
  102 + settings, in addition to all the handlers of vaurien
  103 + """
  104 + handlers = {}
  105 + if logger is None:
  106 + from vaurien import logger
  107 +
  108 + for section in settings.sections():
  109 + if section.startswith('handler.'):
  110 + handler_name = section[len('handler.'):]
  111 +
  112 + # have a look if we have a section named handler:{handler}
  113 + settings = settings.getsection('handler.%s' % handler_name)
  114 + handler_location = settings.get('callable', None)
  115 + if not handler_location:
  116 + logger.warning('callable not found for %s' % handler_name)
  117 + continue
  118 + handler = import_string(handler_location)
  119 + handlers[handler_name] = handler
  120 + return handlers
4 vaurien/web-requirements.txt
... ... @@ -0,0 +1,4 @@
  1 +Flask==0.9
  2 +Jinja2==2.6
  3 +Werkzeug==0.8.3
  4 +blinker==1.2
73 vaurien/webserver.py
... ... @@ -0,0 +1,73 @@
  1 +"""A simple, flask-based webserver able to control how the proxy behaves"""
  2 +import os
  3 +import json
  4 +
  5 +try:
  6 + from flask import (Flask, request, request_started, request_finished,
  7 + make_response)
  8 +except ImportError as e:
  9 + reqs = os.path.join(os.path.abspath(os.path.dirname(__file__)),
  10 + 'web-requirements.txt')
  11 +
  12 + raise ImportError('You need some dependencies to run the web interface. '\
  13 + + 'You can do so by using "pip install -r '
  14 + + '%s"\nInitial error: %s' % (reqs, str(e)))
  15 +
  16 +app = Flask(__name__)
  17 +
  18 +
  19 +@app.route('/handler', methods=['POST', 'GET'])
  20 +def update_renderer():
  21 + if request.method == 'POST':
  22 + handler = request.data
  23 + try:
  24 + app.proxy.set_next_handler(handler)
  25 + except KeyError:
  26 + request.errors.add('headers', 'handler',
  27 + "the '%s' handler does not exist" % handler)
  28 + return "ok"
  29 + else:
  30 + return app.proxy.get_next_handler()
  31 +
  32 +
  33 +@app.route('/handlers')
  34 +def list_handlers():
  35 + """List all the available handlers"""
  36 + resp = make_response(json.dumps({'handlers': app.proxy.handlers.keys()}))
  37 + resp.content_type = 'application/json'
  38 + return resp
  39 +
  40 +
  41 +# utils
  42 +
  43 +class Errors(list):
  44 + """Holds Request errors
  45 + """
  46 + def __init__(self, status=400):
  47 + self.status = status
  48 + super(Errors, self).__init__()
  49 +
  50 + def add(self, location, name=None, description=None):
  51 + """Registers a new error."""
  52 + self.append(dict(
  53 + location=location,
  54 + name=name,
  55 + description=description))
  56 +
  57 +
  58 +def add_errors(sender, **extra):
  59 + """Add errors to the request"""
  60 + setattr(request, 'errors', Errors())
  61 +
  62 +
  63 +def convert_errors(sender, response, **extra):
  64 + """convert errors to json if needed"""
  65 + if len(request.errors) > 0:
  66 + response.data = json.dumps({'status': 'error',
  67 + 'errors': request.errors})
  68 + response.content_type = 'application/json'
  69 + response.status_code = request.errors.status
  70 +
  71 +
  72 +request_started.connect(add_errors, app)
  73 +request_finished.connect(convert_errors, app)

No commit comments for this range

Something went wrong with that request. Please try again.