From b9b72ece1c208341e7028f408aeb1b0debc28dc5 Mon Sep 17 00:00:00 2001 From: "michael.yak" Date: Sun, 31 May 2020 23:46:54 +0300 Subject: [PATCH 1/3] SBA should be using port 8080 instead of 8082 --- README.md | 16 ++++++++-------- examples/Advanced/advanced_example_app.py | 2 +- examples/Advanced/docker-compose.yml | 2 +- examples/FastAPI/fastapi_example_app.py | 2 +- examples/Flask/flask_example_app.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2acf6cd..005b0ec 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,10 @@ The examples below show a minimal integration of **FastAPI** and **Flask** appli After installing Flask/FastAPI and Pyctuator, start by launching a local Spring Boot Admin instance: ```sh -docker run --rm --name spring-boot-admin -p 8082:8082 michayaak/spring-boot-admin:2.2.2 +docker run --rm --name spring-boot-admin -p 8080:8080 michayaak/spring-boot-admin:2.2.3-1 ``` -Then go to `http://localhost:8082` to get to the web UI. +Then go to `http://localhost:8080` to get to the web UI. ### Flask The following example is complete and should run as is. @@ -94,7 +94,7 @@ Pyctuator( app_name, "http://host.docker.internal:5000", "http://host.docker.internal:5000/pyctuator", - "http://localhost:8082/instances" + "http://localhost:8080/instances" ) app.run(debug=False, port=5000) @@ -102,7 +102,7 @@ app.run(debug=False, port=5000) The application will automatically register with Spring Boot Admin upon start up. -Log in to the Spring Boot Admin UI at `http://localhost:8082` to interact with the application. +Log in to the Spring Boot Admin UI at `http://localhost:8080` to interact with the application. ### FastAPI The following example is complete and should run as is. @@ -137,7 +137,7 @@ Server(config=(Config(app=app, loop="asyncio"))).run() The application will automatically register with Spring Boot Admin upon start up. -Log in to the Spring Boot Admin UI at `http://localhost:8082` to interact with the application. +Log in to the Spring Boot Admin UI at `http://localhost:8080` to interact with the application. ## Advanced Configuration The following sections are intended for advanced users who want to configure advanced Pyctuator features. @@ -262,7 +262,7 @@ Pyctuator( "Flask Pyctuator", "http://localhost:5000", f"http://localhost:5000/pyctuator", - registration_url=f"http://spring-boot-admin:8082/instances", + registration_url=f"http://spring-boot-admin:8080/instances", registration_auth=auth, ) ``` @@ -274,9 +274,9 @@ To run these examples, you'll need to have Spring Boot Admin running in a local Unless the example includes a docker-compose file, you'll need to start Spring Boot Admin using docker directly: ```sh -docker run -p 8082:8082 michayaak/spring-boot-admin:2.2.2 +docker run -p 8080:8080 michayaak/spring-boot-admin:2.2.3-1 ``` -(the docker image's tag represents the version of Spring Boot Admin, so if you need to use version `2.0.0`, use `michayaak/spring-boot-admin:2.0.0` instead). +(the docker image's tag represents the version of Spring Boot Admin, so if you need to use version `2.0.0`, use `michayaak/spring-boot-admin:2.0.0` instead, note its accepting connections on port 8082). The examples include * [FastAPI Example](examples/FastAPI/README.md) - demonstrates integrating Pyctuator with the FastAPI web framework. diff --git a/examples/Advanced/advanced_example_app.py b/examples/Advanced/advanced_example_app.py index 96b7113..72d3d20 100644 --- a/examples/Advanced/advanced_example_app.py +++ b/examples/Advanced/advanced_example_app.py @@ -50,7 +50,7 @@ "pyctuator_endpoint": f"http://host.docker.internal:8000/pyctuator", # Spring Boot Admin registration URL - "sba_registration_endpoint": f"http://localhost:8082/instances", + "sba_registration_endpoint": f"http://localhost:8080/instances", } } diff --git a/examples/Advanced/docker-compose.yml b/examples/Advanced/docker-compose.yml index 794b414..c46769d 100644 --- a/examples/Advanced/docker-compose.yml +++ b/examples/Advanced/docker-compose.yml @@ -13,4 +13,4 @@ services: sba: image: michayaak/spring-boot-admin:2.2.2 ports: - - 8082:8082 \ No newline at end of file + - 8080:8080 \ No newline at end of file diff --git a/examples/FastAPI/fastapi_example_app.py b/examples/FastAPI/fastapi_example_app.py index 08fbe0d..a170461 100644 --- a/examples/FastAPI/fastapi_example_app.py +++ b/examples/FastAPI/fastapi_example_app.py @@ -35,7 +35,7 @@ def read_root(): "Example FastAPI", f"http://{example_app_public_address}:8000", f"http://{example_app_address_as_seen_from_sba_container}:8000/pyctuator", - f"http://{example_sba_address}:8082/instances", + f"http://{example_sba_address}:8080/instances", app_description=app.description, ) diff --git a/examples/Flask/flask_example_app.py b/examples/Flask/flask_example_app.py index 4bf50fd..3076bc7 100644 --- a/examples/Flask/flask_example_app.py +++ b/examples/Flask/flask_example_app.py @@ -30,7 +30,7 @@ def hello(): "Flask Pyctuator", f"http://{example_app_public_address}:5000", f"http://{example_app_address_as_seen_from_sba_container}:5000/pyctuator", - f"http://{example_sba_address}:8082/instances", + f"http://{example_sba_address}:8080/instances", app_description="Demonstrate Spring Boot Admin Integration with Flask", ) From 22a04b302744a244c1e098c30294cd3320c1199b Mon Sep 17 00:00:00 2001 From: "michael.yak" Date: Mon, 1 Jun 2020 11:24:04 +0300 Subject: [PATCH 2/3] Examples, http-trace allow "Exclude /pyctuator/**", dont request logging In order for the "Http Traces" tab to be able to hide requests sent by Spring Boot Admin to the Pyctuator endpoint, `pyctuator_endpoint_url` must be using the same host and port as `app_url`. The examples initially registered the host's IP for `app_url` instead of using the same host as `pyctuator_endpoint_url` - the idea was that the app's URL shown in SBA will be accessible. Also, both FastAPI and Flask examples were logging every request sent from SBA to Pyctuator. This was annoying so the example now are configuring the loggers not to show access-log. --- README.md | 23 ++++++++++++++--------- examples/Advanced/advanced_example_app.py | 3 +-- examples/FastAPI/fastapi_example_app.py | 21 ++++++++++++++------- examples/Flask/flask_example_app.py | 13 +++++++------ 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 005b0ec..01dc15d 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ def hello(): Pyctuator( app, app_name, - "http://host.docker.internal:5000", - "http://host.docker.internal:5000/pyctuator", - "http://localhost:8080/instances" + app_url="http://host.docker.internal:5000", + pyctuator_endpoint_url="http://host.docker.internal:5000/pyctuator", + registration_url="http://localhost:8080/instances" ) app.run(debug=False, port=5000) @@ -127,9 +127,9 @@ def hello(): Pyctuator( app, "FastAPI Pyctuator", - "http://host.docker.internal:8000", - "http://host.docker.internal:8000/pyctuator", - "http://localhost:8080/instances" + app_url="http://host.docker.internal:8000", + pyctuator_endpoint_url="http://host.docker.internal:8000/pyctuator", + registration_url="http://localhost:8080/instances" ) Server(config=(Config(app=app, loop="asyncio"))).run() @@ -139,6 +139,11 @@ The application will automatically register with Spring Boot Admin upon start up Log in to the Spring Boot Admin UI at `http://localhost:8080` to interact with the application. +### Registration Notes +When registering a service in Spring Boot Admin, note that: +* **Docker** - If the Spring Boot Admin is running in a container while the managed service is running in the docker-host directly, the `app_url` and `pyctuator_endpoint_url` should use `host.docker.internal` as the url's host so Spring Boot Admin will be able to connect to the monitored service. +* **Http Traces** - In order for the "Http Traces" tab to be able to hide requests sent by Spring Boot Admin to the Pyctuator endpoint, `pyctuator_endpoint_url` must be using the same host and port as `app_url`. + ## Advanced Configuration The following sections are intended for advanced users who want to configure advanced Pyctuator features. @@ -260,8 +265,8 @@ auth = BasicAuth(os.getenv("sba-username"), os.getenv("sba-password")) Pyctuator( app, "Flask Pyctuator", - "http://localhost:5000", - f"http://localhost:5000/pyctuator", + app_url="http://localhost:5000", + pyctuator_endpoint_url=f"http://localhost:5000/pyctuator", registration_url=f"http://spring-boot-admin:8080/instances", registration_auth=auth, ) @@ -276,7 +281,7 @@ Unless the example includes a docker-compose file, you'll need to start Spring B ```sh docker run -p 8080:8080 michayaak/spring-boot-admin:2.2.3-1 ``` -(the docker image's tag represents the version of Spring Boot Admin, so if you need to use version `2.0.0`, use `michayaak/spring-boot-admin:2.0.0` instead, note its accepting connections on port 8082). +(the docker image's tag represents the version of Spring Boot Admin, so if you need to use version `2.0.0`, use `michayaak/spring-boot-admin:2.0.0` instead, note it accepts connections on port 8082). The examples include * [FastAPI Example](examples/FastAPI/README.md) - demonstrates integrating Pyctuator with the FastAPI web framework. diff --git a/examples/Advanced/advanced_example_app.py b/examples/Advanced/advanced_example_app.py index 72d3d20..7f74c55 100644 --- a/examples/Advanced/advanced_example_app.py +++ b/examples/Advanced/advanced_example_app.py @@ -1,7 +1,6 @@ import datetime import logging import random -import socket from dataclasses import dataclass from typing import Any, Dict, List from starlette.requests import Request @@ -36,7 +35,7 @@ }, # the URL to use when accessing the application - "public_endpoint": f"http://{socket.gethostbyname(socket.gethostname())}:8000", + "public_endpoint": f"http://host.docker.internal:8000", }, "mysql": { "host": "localhost:3306", diff --git a/examples/FastAPI/fastapi_example_app.py b/examples/FastAPI/fastapi_example_app.py index a170461..ad3f9f2 100644 --- a/examples/FastAPI/fastapi_example_app.py +++ b/examples/FastAPI/fastapi_example_app.py @@ -1,7 +1,6 @@ import datetime import logging import random -import socket from fastapi import FastAPI from uvicorn import Server @@ -26,18 +25,26 @@ def read_root(): return "Hello World!" -example_app_public_address = socket.gethostbyname(socket.gethostname()) -example_app_address_as_seen_from_sba_container = "host.docker.internal" +example_app_address = "host.docker.internal" example_sba_address = "localhost" pyctuator = Pyctuator( app, "Example FastAPI", - f"http://{example_app_public_address}:8000", - f"http://{example_app_address_as_seen_from_sba_container}:8000/pyctuator", - f"http://{example_sba_address}:8080/instances", + app_url=f"http://{example_app_address}:8000", + pyctuator_endpoint_url=f"http://{example_app_address}:8000/pyctuator", + registration_url=f"http://{example_sba_address}:8080/instances", app_description=app.description, ) -server = Server(config=(Config(app=app, loop="asyncio", host="0.0.0.0"))) +# Keep the console clear - configure uvicorn (FastAPI's WSGI web app) not to log the detail of every incoming request +uvicorn_logger = logging.getLogger("uvicorn") +uvicorn_logger.setLevel(logging.WARNING) + +server = Server(config=(Config( + app=app, + loop="asyncio", + host="0.0.0.0", + logger=uvicorn_logger, +))) server.run() diff --git a/examples/Flask/flask_example_app.py b/examples/Flask/flask_example_app.py index 3076bc7..d31f70d 100644 --- a/examples/Flask/flask_example_app.py +++ b/examples/Flask/flask_example_app.py @@ -1,7 +1,6 @@ import datetime import logging import random -import socket from flask import Flask @@ -9,6 +8,9 @@ logging.basicConfig(level=logging.INFO) +# Keep the console clear - configure werkzeug (flask's WSGI web app) not to log the detail of every incoming request +logging.getLogger("werkzeug").setLevel(logging.WARNING) + my_logger = logging.getLogger("example") app = Flask("Flask Example Server") @@ -21,16 +23,15 @@ def hello(): return "Hello World!" -example_app_public_address = socket.gethostbyname(socket.gethostname()) -example_app_address_as_seen_from_sba_container = "host.docker.internal" +example_app_address = "host.docker.internal" example_sba_address = "localhost" Pyctuator( app, "Flask Pyctuator", - f"http://{example_app_public_address}:5000", - f"http://{example_app_address_as_seen_from_sba_container}:5000/pyctuator", - f"http://{example_sba_address}:8080/instances", + app_url=f"http://{example_app_address}:5000", + pyctuator_endpoint_url=f"http://{example_app_address}:5000/pyctuator", + registration_url=f"http://{example_sba_address}:8080/instances", app_description="Demonstrate Spring Boot Admin Integration with Flask", ) From 25e63d7a24260632f2ccb55f24b7ca605de443cc Mon Sep 17 00:00:00 2001 From: "michael.yak" Date: Mon, 1 Jun 2020 11:25:24 +0300 Subject: [PATCH 3/3] Set response content-type `application/vnd.spring-boot.actuator.v2+json` Responses sent by Pyctuator should set content-type to `application/vnd.spring-boot.actuator.v2+json` - the existing mechanism was invoked *after* requests were intercepted so the content-type was missing tom "HTTP Traces" in SBA. --- pyctuator/httptrace/fastapi_http_tracer.py | 48 ------------- pyctuator/httptrace/flask_http_tracer.py | 49 -------------- pyctuator/impl/__init__.py | 1 + pyctuator/impl/fastapi_pyctuator.py | 78 ++++++++++++++-------- pyctuator/impl/flask_pyctuator.py | 69 +++++++++++-------- pyctuator/impl/pyctuator_impl.py | 3 +- tests/test_pyctuator_e2e.py | 41 +++++++++++- 7 files changed, 133 insertions(+), 156 deletions(-) delete mode 100644 pyctuator/httptrace/fastapi_http_tracer.py delete mode 100644 pyctuator/httptrace/flask_http_tracer.py diff --git a/pyctuator/httptrace/fastapi_http_tracer.py b/pyctuator/httptrace/fastapi_http_tracer.py deleted file mode 100644 index 3adc54e..0000000 --- a/pyctuator/httptrace/fastapi_http_tracer.py +++ /dev/null @@ -1,48 +0,0 @@ -from collections import defaultdict -from datetime import datetime -from typing import Mapping, List, Callable - -from starlette.datastructures import Headers -from starlette.requests import Request -from starlette.responses import Response - -from pyctuator.httptrace import TraceRecord, TraceRequest, TraceResponse -from pyctuator.httptrace.http_tracer import HttpTracer - - -class FastApiHttpTracer: - - def __init__(self, http_tracer: HttpTracer) -> None: - self.http_tracer = http_tracer - - async def record_httptrace(self, request: Request, call_next: Callable) -> Response: - request_time = datetime.now() - response: Response = await call_next(request) - response_time = datetime.now() - new_record = self._create_record(request, response, request_time, response_time) - self.http_tracer.add_record(record=new_record) - return response - - def _create_headers_dictionary(self, headers: Headers) -> Mapping[str, List[str]]: - headers_dict: Mapping[str, List[str]] = defaultdict(list) - for (key, value) in headers.items(): - headers_dict[key].append(value) - return headers_dict - - def _create_record( - self, - request: Request, - response: Response, - request_time: datetime, - response_time: datetime, - ) -> TraceRecord: - response_delta_time = response_time - request_time - new_record: TraceRecord = TraceRecord( - request_time, - None, - None, - TraceRequest(request.method, str(request.url), self._create_headers_dictionary(request.headers)), - TraceResponse(response.status_code, self._create_headers_dictionary(response.headers)), - int(response_delta_time.microseconds / 1000), - ) - return new_record diff --git a/pyctuator/httptrace/flask_http_tracer.py b/pyctuator/httptrace/flask_http_tracer.py deleted file mode 100644 index 333dea6..0000000 --- a/pyctuator/httptrace/flask_http_tracer.py +++ /dev/null @@ -1,49 +0,0 @@ -from collections import defaultdict -from datetime import datetime -from typing import Mapping, List - -from flask import Request, Response, after_this_request -from werkzeug.datastructures import Headers - -from pyctuator.httptrace import TraceRecord, TraceRequest, TraceResponse -from pyctuator.httptrace.http_tracer import HttpTracer - - -class FlaskHttpTracer: - - def __init__(self, http_tracer: HttpTracer) -> None: - self.http_tracer = http_tracer - - def record_httptrace_flask(self, request: Request) -> None: - request_time = datetime.now() - - @after_this_request - # pylint: disable=unused-variable - def after_response(response: Response) -> Response: - response_time = datetime.now() - new_record = self.create_record_flask(request, response, request_time, response_time) - self.http_tracer.add_record(record=new_record) - return response - - def create_headers_dictionary_flask(self, headers: Headers) -> Mapping[str, List[str]]: - headers_dict: Mapping[str, List[str]] = defaultdict(list) - for (key, value) in headers.items(): - headers_dict[key].append(value) - return dict(headers_dict) - - def create_record_flask( - self, - request: Request, - response: Response, - request_time: datetime, - response_time: datetime, - ) -> TraceRecord: - response_delta_time = response_time - request_time - return TraceRecord( - request_time, - None, - None, - TraceRequest(request.method, str(request.url), self.create_headers_dictionary_flask(request.headers)), - TraceResponse(response.status_code, self.create_headers_dictionary_flask(response.headers)), - int(response_delta_time.microseconds / 1000), - ) diff --git a/pyctuator/impl/__init__.py b/pyctuator/impl/__init__.py index e69de29..a4311f0 100644 --- a/pyctuator/impl/__init__.py +++ b/pyctuator/impl/__init__.py @@ -0,0 +1 @@ +SBA_V2_CONTENT_TYPE = "application/vnd.spring-boot.actuator.v2+json;charset=UTF-8" diff --git a/pyctuator/impl/fastapi_pyctuator.py b/pyctuator/impl/fastapi_pyctuator.py index 804bed6..484c502 100644 --- a/pyctuator/impl/fastapi_pyctuator.py +++ b/pyctuator/impl/fastapi_pyctuator.py @@ -1,19 +1,24 @@ +from collections import defaultdict +from datetime import datetime from http import HTTPStatus -from typing import Optional, Dict, Callable, Awaitable +from typing import Mapping, List, Callable +from typing import Optional, Dict, Awaitable from fastapi import APIRouter, FastAPI, Header from pydantic import BaseModel +from starlette.datastructures import Headers from starlette.requests import Request from starlette.responses import Response from pyctuator.environment.environment_provider import EnvironmentData from pyctuator.health.health_provider import HealthSummary -from pyctuator.httptrace.fastapi_http_tracer import FastApiHttpTracer +from pyctuator.httptrace import TraceRecord, TraceRequest, TraceResponse from pyctuator.httptrace.http_tracer import Traces -from pyctuator.logging.pyctuator_logging import LoggersData, LoggerLevels -from pyctuator.metrics.metrics_provider import Metric, MetricNames +from pyctuator.impl import SBA_V2_CONTENT_TYPE from pyctuator.impl.pyctuator_impl import PyctuatorImpl, AppInfo from pyctuator.impl.pyctuator_router import PyctuatorRouter, EndpointsData +from pyctuator.logging.pyctuator_logging import LoggersData, LoggerLevels +from pyctuator.metrics.metrics_provider import Metric, MetricNames from pyctuator.threads.thread_dump_provider import ThreadDump @@ -24,6 +29,7 @@ class FastApiLoggerItem(BaseModel): # pylint: disable=too-many-locals class FastApiPyctuator(PyctuatorRouter): + # pylint: disable=unused-variable def __init__( self, app: FastAPI, @@ -32,10 +38,8 @@ def __init__( ) -> None: super().__init__(app, pyctuator_impl) router = APIRouter() - self.fastapi_http_tracer = FastApiHttpTracer(pyctuator_impl.http_tracer) @router.get("/", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_endpoints() -> EndpointsData: return self.get_endpoints_data() @@ -49,7 +53,6 @@ def get_endpoints() -> EndpointsData: @router.options("/logfile", include_in_schema=include_in_openapi_schema) @router.options("/trace", include_in_schema=include_in_openapi_schema) @router.options("/httptrace", include_in_schema=include_in_openapi_schema) - # pylint: disable=unused-variable def options() -> None: """ Spring boot admin, after registration, issues multiple OPTIONS request to the monitored application in order @@ -60,55 +63,45 @@ def options() -> None: """ @router.get("/env", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_environment() -> EnvironmentData: return pyctuator_impl.get_environment() @router.get("/info", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_info() -> AppInfo: return pyctuator_impl.app_info @router.get("/health", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_health() -> HealthSummary: return pyctuator_impl.get_health() @router.get("/metrics", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_metric_names() -> MetricNames: return pyctuator_impl.get_metric_names() @router.get("/metrics/{metric_name}", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_metric_measurement(metric_name: str) -> Metric: return pyctuator_impl.get_metric_measurement(metric_name) # Retrieving All Loggers @router.get("/loggers", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_loggers() -> LoggersData: return pyctuator_impl.logging.get_loggers() @router.post("/loggers/{logger_name}", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def set_logger_level(item: FastApiLoggerItem, logger_name: str) -> Dict: pyctuator_impl.logging.set_logger_level(logger_name, item.configuredLevel) return {} @router.get("/loggers/{logger_name}", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_logger(logger_name: str) -> LoggerLevels: return pyctuator_impl.logging.get_logger(logger_name) @router.get("/dump", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) @router.get("/threaddump", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_thread_dump() -> ThreadDump: return pyctuator_impl.get_thread_dump() @router.get("/logfile", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_logfile(range_header: str = Header(default=None, alias="range")) -> Response: # pylint: disable=redefined-builtin if not range_header: @@ -129,23 +122,50 @@ def get_logfile(range_header: str = Header(default=None, @router.get("/trace", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) @router.get("/httptrace", include_in_schema=include_in_openapi_schema, tags=["pyctuator"]) - # pylint: disable=unused-variable def get_httptrace() -> Traces: return pyctuator_impl.http_tracer.get_httptrace() @app.middleware("http") - # pylint: disable=unused-variable - async def record_httptrace( + async def intercept_requests_and_responses( request: Request, - call_next: Callable[[Request], Awaitable[Response]]) -> Response: - return await self.fastapi_http_tracer.record_httptrace(request, call_next) - - @app.middleware("http") - # pylint: disable=unused-variable - async def add_sba2_support(request: Request, call_next: Callable) -> Response: + call_next: Callable[[Request], Awaitable[Response]] + ) -> Response: + request_time = datetime.now() response: Response = await call_next(request) - if request.url.path.startswith(pyctuator_impl.pyctuator_endpoint_path_prefix): - response.headers["Content-Type"] = pyctuator_impl.sba_v2_content_type + response_time = datetime.now() + + # Set the SBA-V2 content type for responses from Pyctuator + if request.url.path.startswith(self.pyctuator_impl.pyctuator_endpoint_path_prefix): + response.headers["Content-Type"] = SBA_V2_CONTENT_TYPE + + # Record the request and response + new_record = self._create_record(request, response, request_time, response_time) + self.pyctuator_impl.http_tracer.add_record(record=new_record) + return response - app.include_router(router, prefix=(pyctuator_impl.pyctuator_endpoint_path_prefix)) + app.include_router(router, prefix=pyctuator_impl.pyctuator_endpoint_path_prefix) + + def _create_headers_dictionary(self, headers: Headers) -> Mapping[str, List[str]]: + headers_dict: Mapping[str, List[str]] = defaultdict(list) + for (key, value) in headers.items(): + headers_dict[key].append(value) + return headers_dict + + def _create_record( + self, + request: Request, + response: Response, + request_time: datetime, + response_time: datetime, + ) -> TraceRecord: + response_delta_time = response_time - request_time + new_record: TraceRecord = TraceRecord( + request_time, + None, + None, + TraceRequest(request.method, str(request.url), self._create_headers_dictionary(request.headers)), + TraceResponse(response.status_code, self._create_headers_dictionary(response.headers)), + int(response_delta_time.microseconds / 1000), + ) + return new_record diff --git a/pyctuator/impl/flask_pyctuator.py b/pyctuator/impl/flask_pyctuator.py index 0109a4c..3e8eff0 100644 --- a/pyctuator/impl/flask_pyctuator.py +++ b/pyctuator/impl/flask_pyctuator.py @@ -1,13 +1,16 @@ import json +from collections import defaultdict from datetime import datetime, date from http import HTTPStatus -from typing import Dict, Tuple, Any +from typing import Dict, Tuple, Any, Mapping, List -from flask import Flask, Blueprint, request, jsonify +from flask import Flask, Blueprint, request, jsonify, after_this_request from flask import Response, make_response from flask.json import JSONEncoder +from werkzeug.datastructures import Headers -from pyctuator.httptrace.flask_http_tracer import FlaskHttpTracer +from pyctuator.httptrace import TraceRecord, TraceRequest, TraceResponse +from pyctuator.impl import SBA_V2_CONTENT_TYPE from pyctuator.impl.pyctuator_impl import PyctuatorImpl from pyctuator.impl.pyctuator_router import PyctuatorRouter @@ -39,7 +42,7 @@ def default(self, o: Any) -> Any: class FlaskPyctuator(PyctuatorRouter): - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals, unused-variable def __init__( self, app: Flask, @@ -51,76 +54,68 @@ def __init__( path_prefix: str = pyctuator_impl.pyctuator_endpoint_path_prefix flask_blueprint: Blueprint = Blueprint("flask_blueprint", "pyctuator", ) flask_blueprint.json_encoder = CustomJSONEncoder - self.flask_http_tracer = FlaskHttpTracer(pyctuator_impl.http_tracer) @app.before_request - # pylint: disable=unused-variable - def http_tracer_callback() -> None: - self.flask_http_tracer.record_httptrace_flask(request) + def intercept_requests_and_responses() -> None: + request_time = datetime.now() - @flask_blueprint.after_request - # pylint: disable=unused-variable - def add_sba2_support(response: Response) -> Response: - if request.path.startswith(pyctuator_impl.pyctuator_endpoint_path_prefix): - response.headers["Content-Type"] = pyctuator_impl.sba_v2_content_type - return response + @after_this_request + def after_response(response: Response) -> Response: + response_time = datetime.now() + + # Set the SBA-V2 content type for responses from Pyctuator + if request.path.startswith(self.pyctuator_impl.pyctuator_endpoint_path_prefix): + response.headers["Content-Type"] = SBA_V2_CONTENT_TYPE + + # Record the request and response + self.record_request_and_response(response, request_time, response_time) + return response @flask_blueprint.route("/") - # pylint: disable=unused-variable def get_endpoints() -> Any: return jsonify(self.get_endpoints_data()) @flask_blueprint.route("/env") - # pylint: disable=unused-variable def get_environment() -> Any: return jsonify(pyctuator_impl.get_environment()) @flask_blueprint.route("/info") - # pylint: disable=unused-variable def get_info() -> Any: return jsonify(pyctuator_impl.app_info) @flask_blueprint.route("/health") - # pylint: disable=unused-variable def get_health() -> Any: return jsonify(pyctuator_impl.get_health()) @flask_blueprint.route("/metrics") - # pylint: disable=unused-variable def get_metric_names() -> Any: return jsonify(pyctuator_impl.get_metric_names()) @flask_blueprint.route("/metrics/") - # pylint: disable=unused-variable def get_metric_measurement(metric_name: str) -> Any: return jsonify(pyctuator_impl.get_metric_measurement(metric_name)) # Retrieving All Loggers @flask_blueprint.route("/loggers") - # pylint: disable=unused-variable def get_loggers() -> Any: return jsonify(pyctuator_impl.logging.get_loggers()) @flask_blueprint.route("/loggers/", methods=['POST']) - # pylint: disable=unused-variable def set_logger_level(logger_name: str) -> Dict: request_dict = json.loads(request.data) pyctuator_impl.logging.set_logger_level(logger_name, request_dict.get("configuredLevel", None)) return {} @flask_blueprint.route("/loggers/") - # pylint: disable=unused-variable def get_logger(logger_name: str) -> Any: return jsonify(pyctuator_impl.logging.get_logger(logger_name)) @flask_blueprint.route("/threaddump") @flask_blueprint.route("/dump") - # pylint: disable=unused-variable def get_thread_dump() -> Any: return jsonify(pyctuator_impl.get_thread_dump()) @flask_blueprint.route("/logfile") - # pylint: disable=unused-variable def get_logfile() -> Tuple[Response, int]: range_header: str = request.headers.environ.get('HTTP_RANGE') if not range_header: @@ -138,8 +133,30 @@ def get_logfile() -> Tuple[Response, int]: @flask_blueprint.route("/trace") @flask_blueprint.route("/httptrace") - # pylint: disable=unused-variable def get_httptrace() -> Any: return jsonify(pyctuator_impl.http_tracer.get_httptrace()) app.register_blueprint(flask_blueprint, url_prefix=path_prefix) + + def _create_headers_dictionary_flask(self, headers: Headers) -> Mapping[str, List[str]]: + headers_dict: Mapping[str, List[str]] = defaultdict(list) + for (key, value) in headers.items(): + headers_dict[key].append(value) + return dict(headers_dict) + + def record_request_and_response( + self, + response: Response, + request_time: datetime, + response_time: datetime, + ) -> None: + response_delta_time = response_time - request_time + new_record = TraceRecord( + request_time, + None, + None, + TraceRequest(request.method, str(request.url), self._create_headers_dictionary_flask(request.headers)), + TraceResponse(response.status_code, self._create_headers_dictionary_flask(response.headers)), + int(response_delta_time.microseconds / 1000), + ) + self.pyctuator_impl.http_tracer.add_record(record=new_record) diff --git a/pyctuator/impl/pyctuator_impl.py b/pyctuator/impl/pyctuator_impl.py index da5b7f5..1371c88 100644 --- a/pyctuator/impl/pyctuator_impl.py +++ b/pyctuator/impl/pyctuator_impl.py @@ -67,9 +67,8 @@ def __init__( self.thread_dump_provider = ThreadDumpProvider() self.logfile = PyctuatorLogfile(max_size=logfile_max_size, formatter=logfile_formatter) self.http_tracer = HttpTracer() - self.sba_v2_content_type = "application/vnd.spring-boot.actuator.v2+json" - # Determine the endpoint's URL path prefix and make sure it doesn't ends with a "/" + # Determine the endpoint's URL path prefix and make sure it doesn't end with a "/" self.pyctuator_endpoint_path_prefix = urlparse(pyctuator_endpoint_url).path if self.pyctuator_endpoint_path_prefix[-1:] == "/": self.pyctuator_endpoint_path_prefix = self.pyctuator_endpoint_path_prefix[:-1] diff --git a/tests/test_pyctuator_e2e.py b/tests/test_pyctuator_e2e.py index edb9c92..89f5416 100644 --- a/tests/test_pyctuator_e2e.py +++ b/tests/test_pyctuator_e2e.py @@ -4,16 +4,17 @@ import os import random import time -from dataclasses import dataclass +from dataclasses import dataclass, asdict, fields from datetime import datetime, timedelta from http import HTTPStatus -from typing import Generator +from typing import Generator, List import pytest import requests from _pytest.monkeypatch import MonkeyPatch from requests import Response +from pyctuator.impl import SBA_V2_CONTENT_TYPE from tests.conftest import Endpoints, PyctuatorServer, RegistrationRequest, RegistrationTrackerFixture from tests.fast_api_test_server import FastApiPyctuatorServer from tests.flask_test_server import FlaskPyctuatorServer @@ -32,6 +33,42 @@ def pyctuator_server(request) -> Generator: # type: ignore pyctuator_server.stop() +@pytest.mark.usefixtures("boot_admin_server", "pyctuator_server") +@pytest.mark.mark_self_endpoint +def test_response_content_type(endpoints: Endpoints, registration_tracker: RegistrationTrackerFixture) -> None: + # Issue requests to all actuator endpoints and verify the correct content-type is returned + actuator_endpoint_names = [field.name for field in fields(Endpoints) if field.name != "root"] + for actuator_endpoint in actuator_endpoint_names: + actuator_endpoint_url = asdict(endpoints)[actuator_endpoint] + logging.info("Testing content type of %s (%s)", actuator_endpoint, actuator_endpoint_url) + response = requests.get(actuator_endpoint_url) + assert response.status_code == HTTPStatus.OK.value + assert response.headers.get("Content-Type", response.headers.get("content-type")) == SBA_V2_CONTENT_TYPE + + # Issue requests to non-actuator endpoints and verify the correct content-type is returned + assert registration_tracker.registration + for non_actuator_endpoint_url in ["/", "/httptrace_test_url"]: + non_actuator_endpoint_url = registration_tracker.registration.serviceUrl[:-1] + non_actuator_endpoint_url + response = requests.get(non_actuator_endpoint_url) + content_type = response.headers.get("Content-Type", response.headers.get("content-type")) + logging.info("Testing content type, %s from request %s", content_type, non_actuator_endpoint_url) + assert not content_type or content_type.find(SBA_V2_CONTENT_TYPE) == -1 + + # Finally, verify the content-type headers presented by the httptraces are correct + traces = requests.get(endpoints.httptrace).json()["traces"] + for trace in traces: + request_uri = trace["request"]["uri"] + response_headers = trace["response"]["headers"] + content_type_header: List[str] = response_headers.get("content-type", response_headers.get("Content-Type", [])) + + logging.info("Testing httptraces content-type header for request %s, got %s", request_uri, content_type_header) + + if request_uri.find("/pyctuator") > 0: + assert any(SBA_V2_CONTENT_TYPE in ct for ct in content_type_header) + else: + assert all(SBA_V2_CONTENT_TYPE not in ct for ct in content_type_header) + + @pytest.mark.usefixtures("boot_admin_server", "pyctuator_server") @pytest.mark.mark_self_endpoint def test_self_endpoint(endpoints: Endpoints) -> None: