From f5ef6a29d11eee292d65dc61a39a848a07997dd3 Mon Sep 17 00:00:00 2001 From: Owen Cliffe Date: Thu, 30 Apr 2020 16:02:22 +0100 Subject: [PATCH] Fix issue with header prefixes, remove ambiguous header processing, add test --- README.md | 53 +++++++++++++++-- fdk/context.py | 8 ++- fdk/fixtures.py | 10 +++- fdk/headers.py | 43 +++++++++++--- fdk/tests/funcs.py | 23 ++++++- fdk/tests/test_headers.py | 109 ++++++++++++++++++++++++++++++++++ fdk/tests/test_http_stream.py | 75 ++++++++++++++--------- 7 files changed, 275 insertions(+), 46 deletions(-) create mode 100644 fdk/tests/test_headers.py diff --git a/README.md b/README.md index bc8fe3b..160ceac 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,55 @@ # Function development kit for Python +The python FDK lets you write functions in python 3.6/3.7 +## Simplest possible function + +```python +import io +import logging + +from fdk import response + +def handler(ctx, data: io.BytesIO = None): + logging.getLogger().info("Got incoming request") + return response.Response(ctx, response_data="hello world") +``` -While the FDK contract is HTTP, the intention is for that to be somewhat -abstracted from the user - they write some Function code , this library helps them do that. -## Handling JSON Functions +## Handling HTTP metadata in HTTP Functions +Functions can implement HTTP services when fronted by an HTTP Gateway +When your function is behind an HTTP gateway you can access the inbound HTTP Request via : + + - `ctx.HttpHeaders()` : a map of string -> value | list of values , unlike `ctx.Headers()` this only includes headers + passed by the HTTP gateway (with no functions metadata). + - `ctx.RequestURL()` : the incoming request URL passed by the gateway + - `ctx.Method()` : the HTTP method of the incoming request + +You can set outbound HTTP headers and the HTTP status of the request using `ctx.SetResponseHeaders` or the `Response` + - e.g. `ctx.SetResponseHeaders({"Location","http://example.com/","My-Header2": ["v1","v2"]}, 302)` + - or by passing these to the Response object : +```python + return new Response(ctx, + headers={"Location","http://example.com/","My-Header2": ["v1","v2"]}, + response_data="Page moved", + status_code=302) +``` + +e.g. to redirect users to a different page : +```python +import io +import logging + +from fdk import response + +def handler(ctx, data: io.BytesIO = None): + logging.getLogger().info("Got incoming request for URL %s with headers %s", ctx.RequestURL(), ctx.HTTPHeaders()) + ctx.SetResponseHeaders({"Location": "http://www.example.com"}, 302) + return response.Response(ctx, response_data="Page moved from %s") +``` + + +## Handling JSON in Functions A main loop is supplied that can repeatedly call a user function with a series of requests. In order to utilise this, you can write your `func.py` as follows: @@ -16,7 +60,6 @@ import io from fdk import response - def handler(ctx, data: io.BytesIO=None): name = "World" try: @@ -34,8 +77,8 @@ def handler(ctx, data: io.BytesIO=None): ``` -## Unittest your functions +## Unit testing your functions Starting v0.0.33 FDK-Python provides a testing framework that allows performing unit tests of your function's code. The unit test framework is the [pytest](https://pytest.org/). Coding style remain the same, so, write your tests as you've got used to. diff --git a/fdk/context.py b/fdk/context.py index a086137..27892cf 100644 --- a/fdk/context.py +++ b/fdk/context.py @@ -57,6 +57,7 @@ def __init__(self, app_id, fn_id, call_id, self.__call_id = call_id self.__config = config if config else {} self.__headers = headers if headers else {} + self.__http_headers = {} self.__deadline = deadline self.__content_type = content_type self._request_url = request_url @@ -66,8 +67,10 @@ def __init__(self, app_id, fn_id, call_id, log.log("request headers. gateway: {0} {1}" .format(self.__is_gateway(), headers)) + if self.__is_gateway(): - self.__headers = hs.decap_headers(self.__headers) + self.__headers = hs.decap_headers(headers, True) + self.__http_headers = hs.decap_headers(headers, False) def AppID(self): return self.__app_id @@ -84,6 +87,9 @@ def Config(self): def Headers(self): return self.__headers + def HTTPHeaders(self): + return self.__http_headers + def Format(self): return self.__fn_format diff --git a/fdk/fixtures.py b/fdk/fixtures.py index d37e996..89a3d6c 100644 --- a/fdk/fixtures.py +++ b/fdk/fixtures.py @@ -24,7 +24,6 @@ async def process_response(fn_call_coro): response_data = resp.body() response_status = resp.status() response_headers = resp.context().GetResponseHeaders() - print(response_headers) return response_data, response_status, response_headers @@ -83,10 +82,17 @@ async def setup_fn_call( method=method, request_url=request_url, gateway=gateway ) + return await setup_fn_call_raw(handle_func, content, new_headers) + + +async def setup_fn_call_raw(handle_func, content=None, headers=None): + + if headers is None: + headers = {} # don't decap headers, so we can test them # (just like they come out of fdk) return process_response(runner.handle_request( code(handle_func), constants.HTTPSTREAM, - headers=new_headers, data=content, + headers=headers, data=content, )) diff --git a/fdk/headers.py b/fdk/headers.py index f2fb0a3..3316a50 100644 --- a/fdk/headers.py +++ b/fdk/headers.py @@ -15,29 +15,58 @@ from fdk import constants -def decap_headers(hdsr): +def decap_headers(hdsr, merge=True): ctx_headers = {} if hdsr is not None: for k, v in hdsr.items(): k = k.lower() if k.startswith(constants.FN_HTTP_PREFIX): - ctx_headers[k.lstrip(constants.FN_HTTP_PREFIX)] = v - else: - ctx_headers[k] = v + push_header(ctx_headers, k[len(constants.FN_HTTP_PREFIX):], v) + elif merge: + # http headers override functions headers in context + # this is not ideal but it's the more correct view from the + # consumer perspective than random choice and for things + # like host headers + if k not in ctx_headers: + ctx_headers[k] = v return ctx_headers +def push_header(input_map, key, value): + if key not in input_map: + input_map[key] = value + return + + current_val = input_map[key] + + if isinstance(current_val, list): + if isinstance(value, list): # both lists concat + input_map[key] = current_val + value + else: # copy and append current value + new_val = current_val.copy() + new_val.append(value) + input_map[key] = new_val + else: + if isinstance(value, list): # copy new list value and prepend current + new_value = value.copy() + new_value.insert(0, current_val) + input_map[key] = new_value + else: # both non-lists create a new list + input_map[key] = [current_val, value] + + def encap_headers(headers, status=None): new_headers = {} if headers is not None: for k, v in headers.items(): k = k.lower() + if k.startswith(constants.FN_HTTP_PREFIX): # by default merge + push_header(new_headers, k, v) if (k == constants.CONTENT_TYPE or - k == constants.FN_FDK_VERSION or - k.startswith(constants.FN_HTTP_PREFIX)): + k == constants.FN_FDK_VERSION): # but don't merge these new_headers[k] = v else: - new_headers[constants.FN_HTTP_PREFIX + k] = v + push_header(new_headers, constants.FN_HTTP_PREFIX + k, v) if status is not None: new_headers[constants.FN_HTTP_STATUS] = str(status) diff --git a/fdk/tests/funcs.py b/fdk/tests/funcs.py index 72848ce..f700d0d 100644 --- a/fdk/tests/funcs.py +++ b/fdk/tests/funcs.py @@ -17,7 +17,6 @@ from fdk import response - xml = """ 1979-09-23 @@ -78,7 +77,6 @@ def none_func(ctx, data=None): def timed_sleepr(timeout): - def sleeper(ctx, data=None): time.sleep(timeout) @@ -122,3 +120,24 @@ def access_request_url(ctx, **kwargs): "Request-Method": method, } ) + + +captured_context = None + + +def setup_context_capture(): + global captured_context + captured_context = None + + +def get_captured_context(): + global captured_context + my_context = captured_context + captured_context = None + return my_context + + +def capture_request_ctx(ctx, **kwargs): + global captured_context + captured_context = ctx + return response.Response(ctx, response_data="OK") diff --git a/fdk/tests/test_headers.py b/fdk/tests/test_headers.py new file mode 100644 index 0000000..926677c --- /dev/null +++ b/fdk/tests/test_headers.py @@ -0,0 +1,109 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fdk import headers + + +def test_push_header(): + cases = [ + ({}, "k", "v", {"k": "v"}), + ({}, "k", ["v1", "v2"], {"k": ["v1", "v2"]}), + ({"k": "v1"}, "k", "v2", {"k": ["v1", "v2"]}), + ({"k": ["v1"]}, "k", "v2", {"k": ["v1", "v2"]}), + ({"k": ["v1"]}, "k", ["v2"], {"k": ["v1", "v2"]}), + ({"k": []}, "k", [], {"k": []}), + ({"k": ["v1"]}, "k", [], {"k": ["v1"]}), + ({"k": []}, "k", ["v1"], {"k": ["v1"]}), + ({"k": "v1"}, "k", ["v2", "v3"], {"k": ["v1", "v2", "v3"]}), + ({"k1": "v1"}, "k2", "v2", {"k1": "v1", "k2": "v2"}), + + ] + + for case in cases: + initial = case[0] + working = initial.copy() + key = case[1] + value = case[2] + result = case[3] + headers.push_header(working, key, value) + assert working == result, "Adding %s:%s to %s" \ + % (key, value, initial) + + +def test_encap_no_headers(): + encap = headers.encap_headers({}) + assert not encap, "headers should be empty" + + +def test_encap_simple_headers(): + encap = headers.encap_headers({ + "Test-header": "foo", + "name-Conflict": "h1", + "name-conflict": "h2", + "nAme-conflict": ["h3", "h4"], + "fn-http-h-name-conflict": "h5", + "multi-header": ["bar", "baz"] + }) + assert "fn-http-h-test-header" in encap + assert "fn-http-h-name-conflict" in encap + assert "fn-http-h-multi-header" in encap + + assert encap["fn-http-h-test-header"] == "foo" + assert set(encap["fn-http-h-name-conflict"]) == {"h1", "h2", + "h3", "h4", "h5"} + assert encap["fn-http-h-multi-header"] == ["bar", "baz"] + + +def test_encap_status(): + encap = headers.encap_headers({}, 202) + assert "fn-http-status" in encap + assert encap["fn-http-status"] == "202" + + +def test_encap_status_override(): + encap = headers.encap_headers({"fn-http-status": 412}, 202) + assert "fn-http-status" in encap + assert encap["fn-http-status"] == "202" + + +def test_content_type_version(): + encap = headers.encap_headers({"content-type": "text/plain", + "fn-fdk-version": "1.2.3"}) + + assert encap == {"content-type": "text/plain", "fn-fdk-version": "1.2.3"} + + +def test_decap_headers_merge(): + decap = headers.decap_headers({"fn-http-h-Foo-Header": "v1", + "fn-http-h-merge-header": "v2", + "fn-http-h-merge-Header": ["v3"], + "Foo-Header": "ignored", + "other-header": "bob"}, True) + assert "foo-header" in decap + assert decap["foo-header"] == "v1" + + assert "other-header" in decap + assert decap["other-header"] == "bob" + + assert "merge-header" in decap + assert set(decap["merge-header"]) == {"v2", "v3"} + + +def test_decap_headers_strip(): + decap = headers.decap_headers({"fn-http-h-Foo-Header": "v1", + "fn-http-h-merge-header": ["v2"], + "Foo-Header": "ignored", + "merge-header": "v3", + "other-header": "bad"}, False) + assert decap == {"foo-header": "v1", "merge-header": ["v2"]} diff --git a/fdk/tests/test_http_stream.py b/fdk/tests/test_http_stream.py index 23c7c64..883d502 100644 --- a/fdk/tests/test_http_stream.py +++ b/fdk/tests/test_http_stream.py @@ -43,7 +43,6 @@ async def test_parse_request_without_data(): call = await fixtures.setup_fn_call(funcs.dummy_func) content, status, headers = await call - print(headers) assert 200 == status assert "Hello World" == content @@ -149,7 +148,6 @@ async def test_deadline(): @pytest.mark.asyncio async def test_default_enforced_response_code(): - event_coro = event_handler.event_handle( fixtures.code(funcs.code404)) @@ -161,7 +159,6 @@ async def test_default_enforced_response_code(): @pytest.mark.asyncio async def test_enforced_response_codes_502(): - event_coro = event_handler.event_handle( fixtures.code(funcs.code502)) @@ -173,7 +170,6 @@ async def test_enforced_response_codes_502(): @pytest.mark.asyncio async def test_enforced_response_codes_504(): - event_coro = event_handler.event_handle( fixtures.code(funcs.code504)) @@ -196,39 +192,60 @@ def test_log_frame_header(monkeypatch, capsys): @pytest.mark.asyncio -async def test_request_url_and_method_no_gateway(): - url_path = "/call" - method = "POST" - call = await fixtures.setup_fn_call( - funcs.access_request_url, - request_url=url_path, - method=method, +async def test_request_url_and_method_set_with_gateway(): + headers = { + "fn-http-method": "PUT", + "fn-http-request-url": "/foo-bar?baz", + "fn-http-h-not-aheader": "nothttp" + } + + funcs.setup_context_capture() + + call = await fixtures.setup_fn_call_raw( + funcs.capture_request_ctx, + headers=headers ) content, status, headers = await call + assert content == "OK" - assert "response-request-url" in headers - assert "request-method" in headers + ctx = funcs.get_captured_context() - assert url_path == headers.get("response-request-url") - assert method == headers.get("request-method") + assert ctx.RequestURL() == "/foo-bar?baz", "request URL mismatch, got %s" \ + % ctx.RequestURL() + assert ctx.Method() == "PUT", "method mismatch got %s" % ctx.Method() + assert "fn-http-h-not-aheader" in ctx.Headers() + assert ctx.Headers()["fn-http-h-not-aheader"] == "nothttp" @pytest.mark.asyncio -async def test_request_url_and_method_with_gateway(): - url_path = "/t/app/path" - method = "GET" - call = await fixtures.setup_fn_call( - funcs.access_request_url, - request_url=url_path, - method=method, - gateway=True +async def test_encap_request_headers_gateway(): + headers = { + "fn-intent": "httprequest", + "fn-http-h-my-header": "foo", + "fn-http-h-funny-header": ["baz", "bob"], + "funny-header": "not-this-one", + } + + funcs.setup_context_capture() + call = await fixtures.setup_fn_call_raw( + funcs.capture_request_ctx, + content=None, + headers=headers ) + content, status, headers = await call - assert "response-request-url" not in headers - assert "request-method" not in headers - assert "fn-http-h-response-request-url" in headers - assert "fn-http-h-request-method" in headers + assert content == 'OK' + + input_ctx = funcs.get_captured_context() + + headers = input_ctx.Headers() + + assert "my-header" in headers + assert "funny-header" in headers + + assert headers["my-header"] == "foo" + assert headers["funny-header"] == ["baz", "bob"] - assert url_path == headers.get("fn-http-h-response-request-url") - assert method == headers.get("fn-http-h-request-method") + assert input_ctx.HTTPHeaders() == {"my-header": "foo", + "funny-header": ["baz", "bob"]}