Add http ignore by method

docs/EnvVars.md

@@ -11,7 +11,8 @@ Environment Variable | Description | Default
 | `SW_AGENT_AUTHENTICATION` | The authentication token to verify that the agent is trusted by the backend OAP, as for how to configure the backend, refer to [the yaml](https://github.com/apache/skywalking/blob/4f0f39ffccdc9b41049903cc540b8904f7c9728e/oap-server/server-bootstrap/src/main/resources/application.yml#L155-L158). | unset |
 | `SW_AGENT_LOGGING_LEVEL` | The logging level, could be one of `CRITICAL`, `FATAL`, `ERROR`, `WARN`(`WARNING`), `INFO`, `DEBUG` | `INFO` |
 | `SW_AGENT_DISABLE_PLUGINS` | The name patterns in CSV pattern, plugins whose name matches one of the pattern won't be installed | `''` |
-| `SW_AGENT_MAX_BUFFER_SIZE` | The maximum queue backlog size for sending the segment data to backend, segments beyond this are silently dropped | `'1000'` |
+| `SW_AGENT_MAX_BUFFER_SIZE` | The maximum queue backlog size for sending the segment data to backend, segments beyond this are silently dropped | `'10000'` |
+| `SW_HTTP_IGNORE_METHOD` | Comma-delimited list of http methods to ignore (GET, POST, HEAD, OPTIONS, etc...) | `` |
 | `SW_SQL_PARAMETERS_LENGTH` | The maximum length of the collected parameter, parameters longer than the specified length will be truncated, length 0 turns off parameter tracing | `0` |
 | `SW_PYMONGO_TRACE_PARAMETERS` | Indicates whether to collect the filters of pymongo | `False` |
 | `SW_PYMONGO_PARAMETERS_MAX_LENGTH` | The maximum length of the collected filters, filters longer than the specified length will be truncated |  `512` |

skywalking/config.py

@@ -25,6 +25,7 @@
 QUEUE_TIMEOUT = 1  # type: int
 
 RE_IGNORE_PATH = re.compile('^$')  # type: re.Pattern
+RE_HTTP_IGNORE_METHOD = RE_IGNORE_PATH  # type: re.Pattern
 
 options = None  # here to include 'options' in globals
 options = globals().copy()  # THIS MUST PRECEDE DIRECTLY BEFORE LIST OF CONFIG OPTIONS!
@@ -38,7 +39,7 @@
 authentication = os.getenv('SW_AGENT_AUTHENTICATION')  # type: str
 logging_level = os.getenv('SW_AGENT_LOGGING_LEVEL') or 'INFO'  # type: str
 disable_plugins = (os.getenv('SW_AGENT_DISABLE_PLUGINS') or '').split(',')  # type: List[str]
-max_buffer_size = int(os.getenv('SW_AGENT_MAX_BUFFER_SIZE', '1000'))  # type: int
+max_buffer_size = int(os.getenv('SW_AGENT_MAX_BUFFER_SIZE', '10000'))  # type: int
 sql_parameters_length = int(os.getenv('SW_SQL_PARAMETERS_LENGTH') or '0')  # type: int
 pymongo_trace_parameters = True if os.getenv('SW_PYMONGO_TRACE_PARAMETERS') and \
                                    os.getenv('SW_PYMONGO_TRACE_PARAMETERS') == 'True' else False  # type: bool
@@ -50,6 +51,7 @@
 sanic_collect_http_params = True if os.getenv('SW_SANIC_COLLECT_HTTP_PARAMS') and \
                                     os.getenv('SW_SANIC_COLLECT_HTTP_PARAMS') == 'True' else False  # type: bool
 http_params_length_threshold = int(os.getenv('SW_HTTP_PARAMS_LENGTH_THRESHOLD') or '1024')  # type: int
+http_ignore_method = os.getenv('SW_HTTP_IGNORE_METHOD', '').upper()  # type: str
 django_collect_http_params = True if os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') and \
                                      os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') == 'True' else False  # type: bool
 correlation_element_max_number = int(os.getenv('SW_CORRELATION_ELEMENT_MAX_NUMBER') or '3')  # type: int
@@ -81,6 +83,7 @@ def init(**kwargs):
 def finalize():
     reesc = re.compile(r'([.*+?^=!:${}()|\[\]\\])')
     suffix = r'^.+(?:' + '|'.join(reesc.sub(r'\\\1', s.strip()) for s in ignore_suffix.split(',')) + ')$'
+    method = r'^' + '|'.join(s.strip() for s in http_ignore_method.split(',')) + '$'
     path = '^(?:' + \
            '|'.join(                          # replaces ","
                '(?:(?:[^/]+/)*[^/]+)?'.join(  # replaces "**"
@@ -92,5 +95,10 @@ def finalize():
                ) for p0 in trace_ignore_path.split(',')
            ) + ')$'
 
-    global RE_IGNORE_PATH
+    global RE_IGNORE_PATH, RE_HTTP_IGNORE_METHOD
     RE_IGNORE_PATH = re.compile('%s|%s' % (suffix, path))
+    RE_HTTP_IGNORE_METHOD = re.compile(method, re.IGNORECASE)
+
+
+def ignore_http_method_check(method: str):
+    return RE_HTTP_IGNORE_METHOD.match(method)

skywalking/plugins/sw_aiohttp.py

@@ -15,10 +15,11 @@
 # limitations under the License.
 #
 
-from skywalking import Layer, Component
+from skywalking import Layer, Component, config
 from skywalking.trace import tags
 from skywalking.trace.carrier import Carrier
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
+from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
 
 
@@ -31,9 +32,11 @@ def install():
     async def _sw_request(self: ClientSession, method: str, str_or_url, **kwargs):
         url = URL(str_or_url).with_user(None).with_password(None)
         peer = '%s:%d' % (url.host or '', url.port)
-        context = get_context()
 
-        with context.new_exit_span(op=url.path or "/", peer=peer, component=Component.AioHttp) as span:
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_exit_span(op=url.path or "/", peer=peer, component=Component.AioHttp)
+
+        with span:
             span.layer = Layer.Http
             span.tag(Tag(key=tags.HttpMethod, val=method.upper()))  # pyre-ignore
             span.tag(Tag(key=tags.HttpUrl, val=url))  # pyre-ignore
@@ -62,22 +65,25 @@ async def _sw_request(self: ClientSession, method: str, str_or_url, **kwargs):
     ClientSession._request = _sw_request
 
     async def _sw_handle_request(self, request, start_time: float):
-        context = get_context()
         carrier = Carrier()
+        method = request.method
 
         for item in carrier:
             val = request.headers.get(item.key)
 
             if val is not None:
                 item.val = val
 
-        with context.new_entry_span(op=request.path, carrier=carrier) as span:
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_entry_span(op=request.path, carrier=carrier)
+
+        with span:
             span.layer = Layer.Http
             span.component = Component.AioHttp
             span.peer = '%s:%d' % request._transport_peername if isinstance(request._transport_peername, (list, tuple))\
                 else request._transport_peername
 
-            span.tag(Tag(key=tags.HttpMethod, val=request.method))  # pyre-ignore
+            span.tag(Tag(key=tags.HttpMethod, val=method))  # pyre-ignore
             span.tag(Tag(key=tags.HttpUrl, val=str(request.url)))  # pyre-ignore
 
             resp, reset = await _handle_request(self, request, start_time)

skywalking/plugins/sw_django.py

@@ -18,7 +18,8 @@
 from skywalking import Layer, Component, config
 from skywalking.trace import tags
 from skywalking.trace.carrier import Carrier
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
+from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
 
 version_rule = {
@@ -39,8 +40,9 @@ def _sw_get_response(this, request):
             resp = _get_response(this, request)
             return resp
 
-        context = get_context()
         carrier = Carrier()
+        method = request.method
+
         for item in carrier:
             # Any HTTP headers in the request are converted to META keys by converting all characters to uppercase,
             # replacing any hyphens with underscores and adding an HTTP_ prefix to the name.
@@ -49,12 +51,15 @@ def _sw_get_response(this, request):
             if sw_http_header_key in request.META:
                 item.val = request.META[sw_http_header_key]
 
-        with context.new_entry_span(op=request.path, carrier=carrier) as span:
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_entry_span(op=request.path, carrier=carrier)
+
+        with span:
             span.layer = Layer.Http
             span.component = Component.Django
             span.peer = '%s:%s' % (request.META.get('REMOTE_ADDR'), request.META.get('REMOTE_PORT') or "80")
 
-            span.tag(Tag(key=tags.HttpMethod, val=request.method))
+            span.tag(Tag(key=tags.HttpMethod, val=method))
             span.tag(Tag(key=tags.HttpUrl, val=request.build_absolute_uri().split("?")[0]))
 
             # you can get request parameters by `request.GET` even though client are using POST or other methods

skywalking/plugins/sw_flask.py

@@ -18,7 +18,7 @@
 from skywalking import Layer, Component, config
 from skywalking.trace import tags
 from skywalking.trace.carrier import Carrier
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
 from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
 
@@ -34,18 +34,23 @@ def params_tostring(params):
 
     def _sw_full_dispatch_request(this: Flask):
         import flask
+
         req = flask.request
-        context = get_context()
         carrier = Carrier()
+        method = req.method
 
         for item in carrier:
             if item.key.capitalize() in req.headers:
                 item.val = req.headers[item.key.capitalize()]
-        with context.new_entry_span(op=req.path, carrier=carrier, inherit=Component.General) as span:
+
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_entry_span(op=req.path, carrier=carrier, inherit=Component.General)
+
+        with span:
             span.layer = Layer.Http
             span.component = Component.Flask
             span.peer = '%s:%s' % (req.environ["REMOTE_ADDR"], req.environ["REMOTE_PORT"])
-            span.tag(Tag(key=tags.HttpMethod, val=req.method))
+            span.tag(Tag(key=tags.HttpMethod, val=method))
             span.tag(Tag(key=tags.HttpUrl, val=req.url.split("?")[0]))
             if config.flask_collect_http_params and req.values:
                 span.tag(Tag(key=tags.HttpParams,

skywalking/plugins/sw_http_server.py

@@ -17,10 +17,11 @@
 
 import inspect
 
-from skywalking import Layer, Component
+from skywalking import Layer, Component, config
 from skywalking.trace import tags
 from skywalking.trace.carrier import Carrier
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
+from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
 
 
@@ -55,17 +56,22 @@ def wrap_werkzeug_request_handler(handler):
     _run_wsgi = handler.run_wsgi
 
     def _wrap_run_wsgi():
-        context = get_context()
         carrier = Carrier()
+        method = handler.command
+
         for item in carrier:
             item.val = handler.headers[item.key.capitalize()]
         path = handler.path or '/'
-        with context.new_entry_span(op=path.split("?")[0], carrier=carrier) as span:
+
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_entry_span(op=path.split("?")[0], carrier=carrier)
+
+        with span:
             url = 'http://' + handler.headers["Host"] + path if 'Host' in handler.headers else path
             span.layer = Layer.Http
             span.component = Component.General
             span.peer = '%s:%s' % handler.client_address
-            span.tag(Tag(key=tags.HttpMethod, val=handler.command))
+            span.tag(Tag(key=tags.HttpMethod, val=method))
             span.tag(Tag(key=tags.HttpUrl, val=url))
 
             try:
@@ -103,12 +109,15 @@ def _wrap_do_method(handler, method):
         _do_method = getattr(handler, 'do_' + method)
 
         def _sw_do_method():
-            context = get_context()
             carrier = Carrier()
             for item in carrier:
                 item.val = handler.headers[item.key.capitalize()]
             path = handler.path or '/'
-            with context.new_entry_span(op=path.split("?")[0], carrier=carrier) as span:
+
+            span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+                else get_context().new_entry_span(op=path.split("?")[0], carrier=carrier)
+
+            with span:
                 url = 'http://' + handler.headers["Host"] + path if 'Host' in handler.headers else path
                 span.layer = Layer.Http
                 span.component = Component.General

skywalking/plugins/sw_pyramid.py

@@ -15,32 +15,36 @@
 # limitations under the License.
 #
 
-from skywalking import Layer, Component
+from skywalking import Layer, Component, config
 from skywalking.trace import tags
 from skywalking.trace.carrier import Carrier
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
+from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
 
 
 def install():
     from pyramid.router import Router
 
     def _sw_invoke_request(self, request, *args, **kwargs):
-        context = get_context()
         carrier = Carrier()
+        method = request.method
 
         for item in carrier:
             val = request.headers.get(item.key)
 
             if val is not None:
                 item.val = val
 
-        with context.new_entry_span(op=request.path, carrier=carrier) as span:
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_entry_span(op=request.path, carrier=carrier)
+
+        with span:
             span.layer = Layer.Http
             span.component = Component.Pyramid
             span.peer = request.remote_host or request.remote_addr
 
-            span.tag(Tag(key=tags.HttpMethod, val=request.method))
+            span.tag(Tag(key=tags.HttpMethod, val=method))
             span.tag(Tag(key=tags.HttpUrl, val=str(request.url)))
 
             resp = _invoke_request(self, request, *args, **kwargs)

skywalking/plugins/sw_requests.py

@@ -15,11 +15,11 @@
 # limitations under the License.
 #
 
-from skywalking import Layer, Component
+from skywalking import Layer, Component, config
 from skywalking.trace import tags
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
+from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
-from skywalking import config
 
 
 def install():
@@ -42,9 +42,11 @@ def _sw_request(this: Session, method, url,
                             proxies,
                             hooks, stream, verify, cert, json)
 
-        context = get_context()
-        with context.new_exit_span(op=url_param.path or "/", peer=url_param.netloc,
-                                   component=Component.Requests) as span:
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_exit_span(op=url_param.path or "/", peer=url_param.netloc,
+                                             component=Component.Requests)
+
+        with span:
             carrier = span.inject()
             span.layer = Layer.Http
 

skywalking/plugins/sw_sanic.py

@@ -19,7 +19,7 @@
 from skywalking import Layer, Component, config
 from skywalking.trace import tags
 from skywalking.trace.carrier import Carrier
-from skywalking.trace.context import get_context
+from skywalking.trace.context import get_context, NoopContext
 from skywalking.trace.span import NoopSpan
 from skywalking.trace.tags import Tag
 
@@ -69,17 +69,21 @@ def params_tostring(params):
 
     async def _sw_handle_request(self, request, write_callback, stream_callback):
         req = request
-        context = get_context()
         carrier = Carrier()
+        method = req.method
 
         for item in carrier:
             if item.key.capitalize() in req.headers:
                 item.val = req.headers[item.key.capitalize()]
-        with context.new_entry_span(op=req.path, carrier=carrier) as span:
+
+        span = NoopSpan(NoopContext()) if config.ignore_http_method_check(method) \
+            else get_context().new_entry_span(op=req.path, carrier=carrier)
+
+        with span:
             span.layer = Layer.Http
             span.component = Component.Sanic
             span.peer = '%s:%s' % (req.remote_addr or req.ip, req.port)
-            span.tag(Tag(key=tags.HttpMethod, val=req.method))
+            span.tag(Tag(key=tags.HttpMethod, val=method))
             span.tag(Tag(key=tags.HttpUrl, val=req.url.split("?")[0]))
             if config.sanic_collect_http_params and req.args:
                 span.tag(Tag(key=tags.HttpParams,