From 0654e467f3863fb02c44bb120673bc5de855edc1 Mon Sep 17 00:00:00 2001 From: HumbertZhang <504490160@qq.com> Date: Fri, 24 Jul 2020 17:01:37 +0800 Subject: [PATCH 1/3] add django collect http parameters --- README.md | 1 + skywalking/config/__init__.py | 2 + skywalking/plugins/sw_django/__init__.py | 13 ++++++- tests/plugin/sw_django/expected.data.yml | 41 ++++++++++++--------- tests/plugin/sw_django/services/consumer.py | 1 + tests/plugin/sw_django/test_django.py | 2 +- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4a54e9ae..7ccb6116 100755 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Environment Variable | Description | Default | `SW_MYSQL_SQL_PARAMETERS_MAX_LENGTH` | The maximum length of the collected parameter, parameters longer than the specified length will be truncated | `512` | | `SW_IGNORE_SUFFIX` | If the operation name of the first span is included in this set, this segment should be ignored. | `.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg` | | `SW_FLASK_COLLECT_HTTP_PARAMS`| This config item controls that whether the Flask plugin should collect the parameters of the request.| `false` | +| `SW_DJANGO_COLLECT_HTTP_PARAMS`| This config item controls that whether the Django plugin should collect the parameters of the request.| `false` | | `SW_HTTP_PARAMS_LENGTH_THRESHOLD`| When `COLLECT_HTTP_PARAMS` is enabled, how many characters to keep and send to the OAP backend, use negative values to keep and send the complete parameters, NB. this config item is added for the sake of performance. | `1024` | diff --git a/skywalking/config/__init__.py b/skywalking/config/__init__.py index 919bd7fd..e6e09203 100644 --- a/skywalking/config/__init__.py +++ b/skywalking/config/__init__.py @@ -34,6 +34,8 @@ flask_collect_http_params = True if os.getenv('SW_FLASK_COLLECT_HTTP_PARAMS') and \ os.getenv('SW_FLASK_COLLECT_HTTP_PARAMS') == 'True' else False # type: bool http_params_length_threshold = int(os.getenv('SW_HTTP_PARAMS_LENGTH_THRESHOLD') or '1024') # type: int +django_collect_http_params = True if os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') and \ + os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') == 'True' else False # type: bool def init( diff --git a/skywalking/plugins/sw_django/__init__.py b/skywalking/plugins/sw_django/__init__.py index 042b16fb..cf3577df 100644 --- a/skywalking/plugins/sw_django/__init__.py +++ b/skywalking/plugins/sw_django/__init__.py @@ -16,7 +16,7 @@ # import logging -from skywalking import Layer, Component +from skywalking import Layer, Component, config from skywalking.trace import tags from skywalking.trace.carrier import Carrier from skywalking.trace.context import get_context @@ -33,6 +33,9 @@ def install(): _get_response = BaseHandler.get_response _handle_uncaught_exception = exception.handle_uncaught_exception + def params_tostring(params): + return "\n".join([k + '=[' + ",".join(params.getlist(k)) + ']' for k, _ in params.items()]) + def _sw_get_response(this: BaseHandler, request): if request is None: resp = _get_response(this, request) @@ -50,7 +53,13 @@ def _sw_get_response(this: BaseHandler, request): span.peer = '%s:%s' % (request.META.get('REMOTE_ADDR'), request.META.get('REMOTE_PORT') or "80") span.tag(Tag(key=tags.HttpMethod, val=request.method)) - span.tag(Tag(key=tags.HttpUrl, val=request.build_absolute_uri())) + span.tag(Tag(key=tags.HttpUrl, val=request.build_absolute_uri().split("?")[0])) + + # you can get request parameters by `request.GET` even though client are using POST or other methods + if config.django_collect_http_params and request.GET: + span.tag(Tag(key=tags.HttpParams, + val=params_tostring(request.GET)[0:config.http_params_length_threshold])) + resp = _get_response(this, request) span.tag(Tag(key=tags.HttpStatus, val=resp.status_code)) if resp.status_code >= 400: diff --git a/tests/plugin/sw_django/expected.data.yml b/tests/plugin/sw_django/expected.data.yml index c946a3c5..7907a8b3 100644 --- a/tests/plugin/sw_django/expected.data.yml +++ b/tests/plugin/sw_django/expected.data.yml @@ -26,6 +26,13 @@ segmentItems: parentSpanId: -1 spanId: 0 spanLayer: Http + startTime: gt 0 + endTime: gt 0 + componentId: 7004 + isError: false + spanType: Entry + peer: not null + skipAnalysis: false tags: - key: http.method value: POST @@ -42,12 +49,6 @@ segmentItems: parentServiceInstance: not null parentService: consumer traceId: not null - startTime: gt 0 - endTime: gt 0 - componentId: 7004 - spanType: Entry - peer: not null - skipAnalysis: false - serviceName: consumer segmentSize: 1 segments: @@ -58,6 +59,13 @@ segmentItems: parentSpanId: 0 spanId: 1 spanLayer: Http + startTime: gt 0 + endTime: gt 0 + componentId: 7002 + isError: false + spanType: Exit + peer: provider:9091 + skipAnalysis: false tags: - key: http.method value: POST @@ -65,27 +73,24 @@ segmentItems: value: http://provider:9091/users - key: status.code value: '200' - startTime: gt 0 - endTime: gt 0 - componentId: 7002 - spanType: Exit - peer: provider:9091 - skipAnalysis: false - operationName: /users operationId: 0 parentSpanId: -1 spanId: 0 spanLayer: Http + startTime: gt 0 + endTime: gt 0 + componentId: 7004 + isError: false + spanType: Entry + peer: not null + skipAnalysis: false tags: - key: http.method value: GET - key: url value: http://0.0.0.0:9090/users + - key: http.params + value: "test=[test1,test2]\ntest2=[test2]" - key: status.code value: '200' - startTime: gt 0 - endTime: gt 0 - componentId: 7004 - spanType: Entry - peer: not null - skipAnalysis: false \ No newline at end of file diff --git a/tests/plugin/sw_django/services/consumer.py b/tests/plugin/sw_django/services/consumer.py index 60c93282..194122c6 100644 --- a/tests/plugin/sw_django/services/consumer.py +++ b/tests/plugin/sw_django/services/consumer.py @@ -27,6 +27,7 @@ config.service_name = "consumer" config.logging_level = "DEBUG" +config.django_collect_http_params = True agent.start() diff --git a/tests/plugin/sw_django/test_django.py b/tests/plugin/sw_django/test_django.py index 81874514..79827f58 100644 --- a/tests/plugin/sw_django/test_django.py +++ b/tests/plugin/sw_django/test_django.py @@ -30,7 +30,7 @@ def setUpClass(cls): cls.compose = DockerCompose(filepath=dirname(inspect.getfile(cls))) cls.compose.start() - cls.compose.wait_for(cls.url(('consumer', '9090'), 'users')) + cls.compose.wait_for(cls.url(('consumer', '9090'), 'users?test=test1&test=test2&test2=test2')) def test_plugin(self): time.sleep(3) From 262ae5ba4666573e34d34be91bef6d5231a7e39a Mon Sep 17 00:00:00 2001 From: HumbertZhang <504490160@qq.com> Date: Fri, 24 Jul 2020 17:24:14 +0800 Subject: [PATCH 2/3] modify expected data in django plugin test --- tests/plugin/sw_django/expected.data.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/plugin/sw_django/expected.data.yml b/tests/plugin/sw_django/expected.data.yml index 7907a8b3..fafebed3 100644 --- a/tests/plugin/sw_django/expected.data.yml +++ b/tests/plugin/sw_django/expected.data.yml @@ -29,7 +29,6 @@ segmentItems: startTime: gt 0 endTime: gt 0 componentId: 7004 - isError: false spanType: Entry peer: not null skipAnalysis: false @@ -62,7 +61,6 @@ segmentItems: startTime: gt 0 endTime: gt 0 componentId: 7002 - isError: false spanType: Exit peer: provider:9091 skipAnalysis: false @@ -81,7 +79,6 @@ segmentItems: startTime: gt 0 endTime: gt 0 componentId: 7004 - isError: false spanType: Entry peer: not null skipAnalysis: false From c43110480809a50b4441be44c6745e93f6cd20c9 Mon Sep 17 00:00:00 2001 From: HumbertZhang <504490160@qq.com> Date: Fri, 24 Jul 2020 17:43:35 +0800 Subject: [PATCH 3/3] simplify install function of django plugin --- skywalking/plugins/sw_django/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/skywalking/plugins/sw_django/__init__.py b/skywalking/plugins/sw_django/__init__.py index cf3577df..0b226217 100644 --- a/skywalking/plugins/sw_django/__init__.py +++ b/skywalking/plugins/sw_django/__init__.py @@ -33,9 +33,6 @@ def install(): _get_response = BaseHandler.get_response _handle_uncaught_exception = exception.handle_uncaught_exception - def params_tostring(params): - return "\n".join([k + '=[' + ",".join(params.getlist(k)) + ']' for k, _ in params.items()]) - def _sw_get_response(this: BaseHandler, request): if request is None: resp = _get_response(this, request) @@ -79,3 +76,7 @@ def _sw_handle_uncaught_exception(request, resolver, exc_info): exception.handle_uncaught_exception = _sw_handle_uncaught_exception except Exception: logger.warning('failed to install plugin %s', __name__) + + +def params_tostring(params): + return "\n".join([k + '=[' + ",".join(params.getlist(k)) + ']' for k, _ in params.items()])