Skip to content
This repository has been archived by the owner on Feb 11, 2024. It is now read-only.

Commit

Permalink
iss #24: proxied response
Browse files Browse the repository at this point in the history
  • Loading branch information
maizy committed Dec 1, 2014
1 parent 641c413 commit 3e353bc
Show file tree
Hide file tree
Showing 5 changed files with 262 additions and 16 deletions.
27 changes: 27 additions & 0 deletions examples/example.json
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,33 @@
"delay": 15, // seconds
"response": "so long",
"code": 503
},

//proxy some requests to upstream
{
"path": "/fixed_proxy",
"response_proxy": "http://localhost:7777/fixed/path"
},

//proxy request with some delay
{
"path_regexp": "^/slow_proxy/(.*)/(.*)$",
"response_proxy": "http://localhost:7777/otherpath/$1/$2",
"delay": 10 //seconds
},

//proxy with some response headers replaced to config defined
{
"path_regexp": "^/fixed_proxy/(.*)/(.*)$",
"response_proxy": "http://localhost:7777/otherpath/$1/$2",
"headers": {
"Server": "MyServer/0.1"
}
},
{
"path_regexp": "^/fixed_proxy2/(.*)$",
"response_proxy": "http://localhost:7777/otherpath/$1",
"headers_file": "page1.headers"
}
]
}
2 changes: 0 additions & 2 deletions examples/stubs/page1.headers
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ Access-Control-Allow-Credentials: true
Access-Control-Allow-Origin: *
Access-Control-Expose-Headers: ETag, Link, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval
Cache-Control: private, max-age=60, s-maxage=60
Content-Encoding: gzip
Content-Security-Policy: default-src 'none'
Content-Type: application/json; charset=utf-8
Date: Wed, 23 Apr 2014 06: 13: 20 GMT
Expand All @@ -11,7 +10,6 @@ Link: <https: //api.github.com/organizations/12345/repos?per_page=100&page=2>; r
Server: GitHub.com
Status: 200 OK
Strict-Transport-Security: max-age=31536000
Transfer-Encoding: chunked
Vary: Accept, Authorization, Cookie, X-GitHub-OTP
Vary: Accept-Encoding
X-Accepted-OAuth-Scopes: repo
Expand Down
111 changes: 98 additions & 13 deletions zaglushka.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import json
import httplib
import time
from copy import deepcopy
from os import path
from collections import namedtuple

Expand All @@ -14,6 +15,7 @@
from tornado.options import define, options
from tornado.httpserver import HTTPServer
from tornado.httputil import HTTPHeaders
from tornado.httpclient import AsyncHTTPClient, HTTPRequest

logger = logging.getLogger('zaglushka')

Expand Down Expand Up @@ -67,14 +69,7 @@ def __init__(self, raw_config, config_full_path):
rules.append(Rule(matcher, responder))
else:
logger.warn('Unable to build matcher from url spec #{}, skipping'.format(num))
default_response = static_response(
body='',
headers_func=build_static_headers_func({
'X-Zaglushka-Default-Response': 'true',
}),
code=httplib.NOT_FOUND
)
rules.append(Rule(always_match, default_response))
rules.append(Rule(always_match, default_response()))
self.rules = rules
self.stubs_base_path = stubs_base_path

Expand Down Expand Up @@ -159,17 +154,29 @@ def choose_responder(spec, base_stubs_path):
delay = float(spec['delay']) if 'delay' in spec else None
stub_kwargs = {'code': code, 'delay': delay}
headers_func, paths = choose_headers_func(spec, base_stubs_path)
responder = None
if 'response' in spec:
body = spec['response']
if not isinstance(body, basestring):
body = json.dumps(body, ensure_ascii=False, encoding=unicode)
return static_response(body, headers_func, **stub_kwargs), paths
responder = static_response(body, headers_func, **stub_kwargs)
elif 'response_file' in spec:
full_path = path.normpath(path.join(base_stubs_path, spec['response_file']))
paths.add(full_path)
return filebased_response(full_path, headers_func, warn_func=logger.warning, **stub_kwargs), paths
else:
return static_response(b'', headers_func, **stub_kwargs), paths
responder = filebased_response(full_path, headers_func, warn_func=logger.warning, **stub_kwargs)
elif 'response_proxy' in spec and 'path' in spec:
responder = proxied_response(
url=spec['path'], use_regexp=False, proxy_url=spec['response_proxy'],
headers_func=headers_func, warn_func=logger.warn, log_func=logger.debug, **stub_kwargs
)
elif 'response_proxy' in spec and 'path_regexp' in spec:
responder = proxied_response(
url=spec['path_regexp'], use_regexp=True, proxy_url=spec['response_proxy'],
headers_func=headers_func, warn_func=logger.warning, log_func=logger.debug, **stub_kwargs
)
if responder is None:
responder = static_response(b'', headers_func, **stub_kwargs)
return responder, paths


def static_response(body, headers_func, **stub_kwargs):
Expand All @@ -183,6 +190,16 @@ def _body_func(handler, ready_cb):
**stub_kwargs)


def default_response():
return static_response(
body='',
headers_func=build_static_headers_func({
'X-Zaglushka-Default-Response': 'true',
}),
code=httplib.NOT_FOUND
)


def filebased_response(full_path, headers_func, warn_func=None, **stub_kwargs):

def _body_func(handler, ready_cb):
Expand All @@ -200,6 +217,69 @@ def _body_func(handler, ready_cb):
**stub_kwargs)


def _fetch_request(http_client, request, callback):
http_client.fetch(request, callback=callback) # for easier testing


def proxied_response(url, use_regexp, proxy_url, headers_func, warn_func=None, log_func=None, **stub_kwargs):
url_regexp = None
if use_regexp:
try:
url_regexp = re.compile(url)
except re.error as e:
if warn_func is not None:
warn_func('Unable to compile url pattern "{}": {}'.format(url, e))
return default_response()

def _body_func(handler, ready_cb):
request_url = proxy_url
if url_regexp:
match = url_regexp.search(handler.request.uri)
if match is None:
handler.set_header('X-Zaglushka-Failed-Response', 'true')
return ready_cb()
for i, group in enumerate(match.groups(), start=1):
request_url = request_url.replace('${}'.format(i), group)

http_client = handler.application.settings['http_client']
method = handler.request.method
if method in ('HEAD', 'BODY') and handler.request.body == '': # :(
body = None
else:
body = handler.request.body
request = HTTPRequest(request_url, method=method, headers=handler.request.headers,
body=body, follow_redirects=False, allow_nonstandard_methods=True)

def _on_proxied_request_ready(response):
if log_func:
log_func('Request {r.method} {r.url} complete with code={rc}'.format(r=request, rc=response.code))
if response.code == 599: # special tornado status code
handler.set_header('X-Zaglushka-Failed-Response', 'true')
if warn_func is not None:
warn_func('Unable to proxy response to "{u}": {e}'.format(u=request_url, e=response.error))
ready_cb()
return
headers_before = deepcopy(handler.get_headers())
handler.write(response.body)
for header, value in response.headers.iteritems():
handler.add_header(header, value)
# replace with headers from config if any
for header, _ in headers_before.get_all():
handler.clear_header(header)
for value in headers_before.get_list(header):
handler.add_header(header, value)
handler.set_status(response.code)
ready_cb()

if log_func:
log_func('Fetch request {r.method} {r.url}'.format(r=request))
_fetch_request(http_client, request, callback=_on_proxied_request_ready)

return ResponseStub(headers_func=headers_func,
body_func=_body_func,
**stub_kwargs)


def choose_headers_func(spec, base_stubs_path):
paths = set()
if 'headers' in spec:
Expand Down Expand Up @@ -351,6 +431,9 @@ def head(self):
def options(self):
self.send_stub()

def get_headers(self):
return self._headers

def _make_response_with_rule(self, responder):
"""
:type responder: ResponseStub
Expand Down Expand Up @@ -389,10 +472,12 @@ def compute_etag(self):


def build_app(zaglushka_config, debug=False):
http_client = AsyncHTTPClient()
return Application(
handlers=[(r'.*', StubHandler)],
debug=debug,
zaglushka_config=zaglushka_config
zaglushka_config=zaglushka_config,
http_client=http_client
)


Expand Down
3 changes: 2 additions & 1 deletion zaglushka_tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ def get_config_object(self):
def assertResponseBody(self, expected_body, response, msg=''):
msg = ': {}'.format(msg) if msg else ''
expected_body = expected_body.encode('utf-8') if isinstance(expected_body, unicode) else expected_body
self.assertEqual(expected_body, response.body, 'Body not matched{}'.format(msg))
self.assertEqual(expected_body, response.body, 'Body not matched {!r}!={!r} {}'
.format(expected_body, response.body, msg))
real_len = int(response.headers['Content-Length'])
expected_len = len(expected_body)
self.assertEqual(expected_len, real_len,
Expand Down
135 changes: 135 additions & 0 deletions zaglushka_tests/test_proxied_response.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# _*_ coding: utf-8 _*_
import time
import httplib
from os import path
from StringIO import StringIO
from functools import partial

from tornado.httpclient import HTTPResponse

import zaglushka
from zaglushka_tests import ZaglushkaAsyncHTTPTestCase, TEST_RESOURCES_DIR


class ProxiedResponseTestCase(ZaglushkaAsyncHTTPTestCase):

def __init__(self, *args, **kwargs):
super(ProxiedResponseTestCase, self).__init__(*args, **kwargs)
self._orig_fetch = zaglushka._fetch_request

def setUp(self):
self._fetch_called = False
self._fetch_request = None
super(ProxiedResponseTestCase, self).setUp()

def stub_response(self, code, emulated_delay=0.1, **response_args):
self._fetch_called = False
self._fetch_request = None

def _fetch_stub(http_client, request, callback):
response = HTTPResponse(request=request, code=code, **response_args)
self._fetch_request = request
self._fetch_called = True
self.io_loop.add_timeout(time.time() + emulated_delay, partial(callback, response))

zaglushka._fetch_request = _fetch_stub

def assertFetchCalled(self):
self.assertTrue(self._fetch_called)

def assertFetchUrl(self, url):
self.assertIsNotNone(self._fetch_request)
self.assertEqual(self._fetch_request.url, url)

def tearDown(self):
zaglushka._fetch_request = self._orig_fetch
super(ProxiedResponseTestCase, self).tearDown()

def get_zaglushka_config(self):
return {
'urls': [
{
'path': '/fixed_proxy',
'method': 'POST',
'response_proxy': 'http://example.com/path.json',
},
{
'path_regexp': '^/re_proxy/(\d+)/(\w+).json$',
'response_proxy': 'http://re.example.com/resource/$2/$1/$1.js',
},
{
'path_regexp': '^/re_proxy2/(.*)$',
'response_proxy': 'http://re2.example.com/resource/$1.js',
'headers': {
'Overwrite': 'yes',
'Other': ['a', 'b']
}
},
{
'path': '/fixed_proxy2',
'response_proxy': 'http://f2.example.com:8008/resp',
'headers_file': path.join(TEST_RESOURCES_DIR, 'issue11.headers')
},
{
'path': '/delayed_proxy',
'method': 'PUT',
'response_proxy': 'http://example.com/path.json',
'delay': 0.5,
},
]
}

def test_fixed_proxy(self):
expected_headers = {'Host': 'my.example.com'}
self.stub_response(code=200, buffer=StringIO('ok, ggl'), headers=expected_headers)
response = self.fetch('/fixed_proxy', method='POST', body='')
self.assertFetchCalled()
self.assertResponseBody('ok, ggl', response)
self.assertResponseHeaders(expected_headers, response)
self.assertFetchUrl('http://example.com/path.json')

def test_delayed_response(self):
self.stub_response(code=httplib.NOT_FOUND, buffer=StringIO(':('))
start = time.time()
response = self.fetch('/delayed_proxy', method='PUT', body='')
end = time.time()
self.assertFetchCalled()
self.assertResponseBody(':(', response)
self.assertEqual(response.code, httplib.NOT_FOUND)
self.assertGreaterEqual(end - start, 0.5)

def test_regexp_proxy(self):
self.stub_response(code=httplib.OK, buffer=StringIO('yup'))
response = self.fetch('/re_proxy/12345/abcd.json')
self.assertFetchCalled()
self.assertResponseBody('yup', response)
self.assertEqual(response.code, httplib.OK)
self.assertFetchUrl('http://re.example.com/resource/abcd/12345/12345.js')

def test_hardcoded_headers_overwrite(self):
self.stub_response(code=httplib.OK, buffer=StringIO('over'), headers={'Unique': '1234', 'Overwrite': 'no'})
response = self.fetch('/re_proxy2/ab/cd.html')
self.assertFetchCalled()
self.assertResponseBody('over', response)
self.assertResponseHeaders(
{
'Unique': '1234',
'Overwrite': 'yes',
'Other': 'a,b',
},
response)
self.assertFetchUrl('http://re2.example.com/resource/ab/cd.html.js')

def test_filebased_headers_overwrite(self):
self.stub_response(code=httplib.OK, buffer=StringIO(''), headers={'X-GITHUB-REQUEST-ID': 'abc', 'X-ID': '123'})
response = self.fetch('/fixed_proxy2')
self.assertFetchCalled()
self.assertResponseBody('', response)
self.assertResponseHeaders(
{
'Date': 'Wed, 23 Apr 2014 06: 13: 20 GMT',
'X-GitHub-Request-Id': '53950898: 2E4E: 2AD562C: 535759FD',
'X-Id': '123',
},
response)
self.assertFetchUrl('http://f2.example.com:8008/resp')

0 comments on commit 3e353bc

Please sign in to comment.