From 234e563426bee9e9c302bdb1ce63d887eb6b1a9c Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 4 Nov 2019 09:12:43 -0500 Subject: [PATCH 01/81] test: remove unneeded deps listing --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index daf8a244abc..d22164144ae 100644 --- a/tox.ini +++ b/tox.ini @@ -133,10 +133,6 @@ basepython = py37: python3.7 deps = -# Avoid installing wrapt and msgpack-python, our only packages declared, dependencies, when we are testing the real -# distribution build. - !ddtracerun: wrapt - !msgpack03-!msgpack04-!msgpack05-!ddtracerun: msgpack-python pdbpp pytest>=3 pytest-benchmark From a910a09e081ecbeecebae5c5e196d88c6a16de9c Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 12 Nov 2019 16:08:37 +0100 Subject: [PATCH 02/81] Remove quote enforcement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current quote enforcement is a restrictive and annoying. Furthermore, it arbitrary enforces single quotes whereas tools like black — which are now standard in Python ecosystem — sticks to the Python widely used double quotes. --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index daf8a244abc..faa11f624ce 100644 --- a/tox.ini +++ b/tox.ini @@ -455,10 +455,8 @@ ignore_outcome=true [testenv:flake8] deps= flake8>=3.7,<=3.8 - flake8-quotes==1.0.0 commands=flake8 . basepython=python3.7 -inline-quotes = ' [falcon_autopatch] setenv = From 3609970dd5f914b6908ed5f55933a45bf977b666 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 12 Nov 2019 16:12:12 +0100 Subject: [PATCH 03/81] Add flake8-blind-except Checks that no except statement is used without specifying an exception type. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index faa11f624ce..70d7c2ebbaa 100644 --- a/tox.ini +++ b/tox.ini @@ -455,6 +455,7 @@ ignore_outcome=true [testenv:flake8] deps= flake8>=3.7,<=3.8 + flake8-blind-except commands=flake8 . basepython=python3.7 From 07a3db8905dc4a90ea28b15798be423646089df0 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 12 Nov 2019 16:25:13 +0100 Subject: [PATCH 04/81] flake8: ignore no error We only ignore W503 in favor of W504 to define where we put the line break before binary operators. --- ddtrace/contrib/django/middleware.py | 4 +- ddtrace/contrib/pymongo/parse.py | 2 +- ddtrace/sampler.py | 4 +- docs/conf.py | 24 ++++++------ tests/benchmark.py | 4 +- tests/contrib/cassandra/test.py | 2 +- tests/contrib/consul/test.py | 5 ++- tests/contrib/elasticsearch/test.py | 10 +++-- tests/contrib/tornado/test_config.py | 2 +- .../internal/runtime/test_runtime_metrics.py | 2 +- tests/opentracer/test_tracer.py | 18 ++++----- tests/test_sampler.py | 38 +++++++++---------- tests/utils/tracer.py | 8 ++-- tox.ini | 1 + 14 files changed, 64 insertions(+), 60 deletions(-) diff --git a/ddtrace/contrib/django/middleware.py b/ddtrace/contrib/django/middleware.py index 678a2ba4b9f..dae077409dc 100644 --- a/ddtrace/contrib/django/middleware.py +++ b/ddtrace/contrib/django/middleware.py @@ -42,8 +42,8 @@ def _analytics_enabled(): return ( - (config.analytics_enabled and settings.ANALYTICS_ENABLED is not False) - or settings.ANALYTICS_ENABLED is True + (config.analytics_enabled and settings.ANALYTICS_ENABLED is not False) or + settings.ANALYTICS_ENABLED is True ) and settings.ANALYTICS_SAMPLE_RATE is not None diff --git a/ddtrace/contrib/pymongo/parse.py b/ddtrace/contrib/pymongo/parse.py index 1a576180fd8..a6ed214255e 100644 --- a/ddtrace/contrib/pymongo/parse.py +++ b/ddtrace/contrib/pymongo/parse.py @@ -117,7 +117,7 @@ def parse_msg(msg_bytes): offset += 4 # Parse the msg kind - kind = ord(msg_bytes[offset:offset+1]) + kind = ord(msg_bytes[offset:offset + 1]) offset += 1 # Kinds: https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/#sections diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index a589ef198ed..0eb89d84a94 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -304,8 +304,8 @@ def matches(self, span): return all( self._pattern_matches(prop, pattern) for prop, pattern in [ - (span.service, self.service), - (span.name, self.name), + (span.service, self.service), + (span.name, self.name), ] ) diff --git a/docs/conf.py b/docs/conf.py index 2659074bda2..089dbcf747a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -265,21 +265,21 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples diff --git a/tests/benchmark.py b/tests/benchmark.py index 8558addc9ff..5d7a7385337 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -86,7 +86,7 @@ def func(tracer, level=0): span.set_tag('num', num) if level < 10: - func(tracer, level+1) - func(tracer, level+1) + func(tracer, level + 1) + func(tracer, level + 1) benchmark(func, tracer) diff --git a/tests/contrib/cassandra/test.py b/tests/contrib/cassandra/test.py index f21ee1ae7a8..1fb4c594d6b 100644 --- a/tests/contrib/cassandra/test.py +++ b/tests/contrib/cassandra/test.py @@ -269,7 +269,7 @@ def test_paginated_query(self): assert query.get_tag(cassx.ROW_COUNT) == '1' assert query.get_tag(net.TARGET_HOST) == '127.0.0.1' assert query.get_tag(cassx.PAGINATED) == 'True' - assert query.get_tag(cassx.PAGE_NUMBER) == str(i+1) + assert query.get_tag(cassx.PAGE_NUMBER) == str(i + 1) def test_trace_with_service(self): session, tracer = self._traced_session() diff --git a/tests/contrib/consul/test.py b/tests/contrib/consul/test.py index 28313ea7497..b68f6b85ab4 100644 --- a/tests/contrib/consul/test.py +++ b/tests/contrib/consul/test.py @@ -17,8 +17,9 @@ def setUp(self): super(TestConsulPatch, self).setUp() patch() c = consul.Consul( - host=CONSUL_CONFIG['host'], - port=CONSUL_CONFIG['port']) + host=CONSUL_CONFIG['host'], + port=CONSUL_CONFIG['port'], + ) Pin.override(consul.Consul, service=self.TEST_SERVICE, tracer=self.tracer) Pin.override(consul.Consul.KV, service=self.TEST_SERVICE, tracer=self.tracer) self.c = c diff --git a/tests/contrib/elasticsearch/test.py b/tests/contrib/elasticsearch/test.py index aafb704f14b..95af41a047c 100644 --- a/tests/contrib/elasticsearch/test.py +++ b/tests/contrib/elasticsearch/test.py @@ -45,8 +45,9 @@ def test_elasticsearch(self): tracer = get_dummy_tracer() writer = tracer.writer transport_class = get_traced_transport( - datadog_tracer=tracer, - datadog_service=self.TEST_SERVICE) + datadog_tracer=tracer, + datadog_service=self.TEST_SERVICE, + ) es = elasticsearch.Elasticsearch(transport_class=transport_class, port=ELASTICSEARCH_CONFIG['port']) @@ -154,8 +155,9 @@ def test_elasticsearch_ot(self): ot_tracer = init_tracer('my_svc', tracer) transport_class = get_traced_transport( - datadog_tracer=tracer, - datadog_service=self.TEST_SERVICE) + datadog_tracer=tracer, + datadog_service=self.TEST_SERVICE, + ) es = elasticsearch.Elasticsearch(transport_class=transport_class, port=ELASTICSEARCH_CONFIG['port']) diff --git a/tests/contrib/tornado/test_config.py b/tests/contrib/tornado/test_config.py index 2a57751af86..96b03cbbb84 100644 --- a/tests/contrib/tornado/test_config.py +++ b/tests/contrib/tornado/test_config.py @@ -18,7 +18,7 @@ def get_settings(self): 'agent_hostname': 'dd-agent.service.consul', 'agent_port': 8126, 'settings': { - 'FILTERS': [ + 'FILTERS': [ FilterRequestsOnUrl(r'http://test\.example\.com'), ], }, diff --git a/tests/internal/runtime/test_runtime_metrics.py b/tests/internal/runtime/test_runtime_metrics.py index 6ede5146563..e95a7fb0fd6 100644 --- a/tests/internal/runtime/test_runtime_metrics.py +++ b/tests/internal/runtime/test_runtime_metrics.py @@ -76,7 +76,7 @@ def test_tracer_metrics(self): # Mock socket.socket to hijack the dogstatsd socket with mock.patch('socket.socket'): # configure tracer for runtime metrics - self.tracer._RUNTIME_METRICS_INTERVAL = 1./4 + self.tracer._RUNTIME_METRICS_INTERVAL = 1. / 4 self.tracer.configure(collect_metrics=True) self.tracer.set_tags({'env': 'tests.dog'}) diff --git a/tests/opentracer/test_tracer.py b/tests/opentracer/test_tracer.py index 78a0426fc0c..ddfdf618161 100644 --- a/tests/opentracer/test_tracer.py +++ b/tests/opentracer/test_tracer.py @@ -262,8 +262,8 @@ def test_start_span_manual_child_of(self, ot_tracer, writer): assert spans[2].parent_id is root._dd_span.span_id assert spans[3].parent_id is root._dd_span.span_id assert ( - spans[0].trace_id == spans[1].trace_id - and spans[1].trace_id == spans[2].trace_id + spans[0].trace_id == spans[1].trace_id and + spans[1].trace_id == spans[2].trace_id ) def test_start_span_no_active_span(self, ot_tracer, writer): @@ -285,9 +285,9 @@ def test_start_span_no_active_span(self, ot_tracer, writer): assert spans[2].parent_id is None # and that each span is a new trace assert ( - spans[0].trace_id != spans[1].trace_id - and spans[1].trace_id != spans[2].trace_id - and spans[0].trace_id != spans[2].trace_id + spans[0].trace_id != spans[1].trace_id and + spans[1].trace_id != spans[2].trace_id and + spans[0].trace_id != spans[2].trace_id ) def test_start_active_span_child_finish_after_parent(self, ot_tracer, writer): @@ -367,15 +367,15 @@ def trace_two(): # finally we should ensure that the trace_ids are reasonable # trace_one assert ( - spans[0].trace_id == spans[1].trace_id - and spans[1].trace_id == spans[2].trace_id + spans[0].trace_id == spans[1].trace_id and + spans[1].trace_id == spans[2].trace_id ) # traces should be independent assert spans[2].trace_id != spans[3].trace_id # trace_two assert ( - spans[3].trace_id == spans[4].trace_id - and spans[4].trace_id == spans[5].trace_id + spans[3].trace_id == spans[4].trace_id and + spans[4].trace_id == spans[5].trace_id ) def test_start_active_span(self, ot_tracer, writer): diff --git a/tests/test_sampler.py b/tests/test_sampler.py index b2733bd9b62..4dd4ffd9694 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -268,7 +268,7 @@ def resource_check(resource): ('another.span', re.compile(r'test\.span|another\.span'), True), ('test.span', lambda name: 'span' in name, True), ('test.span', lambda name: 'span' not in name, False), - ('test.span', lambda name: 1/0, False), + ('test.span', lambda name: 1 / 0, False), ] ] ) @@ -282,24 +282,24 @@ def test_sampling_rule_matches_name(span, rule, expected): # DEV: Use sample_rate=1 to ensure SamplingRule._sample always returns True (create_span(service=service), SamplingRule(sample_rate=1, service=pattern), expected) for service, pattern, expected in [ - ('my-service', SamplingRule.NO_RULE, True), - ('my-service', None, False), - (None, None, True), - (None, 'my-service', False), - (None, re.compile(r'my-service'), False), - (None, lambda service: 'service' in service, False), - ('my-service', 'my-service', True), - ('my-service', 'my_service', False), - ('my-service', re.compile(r'^my-'), True), - ('my_service', re.compile(r'^my[_-]'), True), - ('my-service', re.compile(r'^my_'), False), - ('my-service', re.compile(r'my-service'), True), - ('my-service', re.compile(r'my'), True), - ('my-service', re.compile(r'my-service|another-service'), True), - ('another-service', re.compile(r'my-service|another-service'), True), - ('my-service', lambda service: 'service' in service, True), - ('my-service', lambda service: 'service' not in service, False), - ('my-service', lambda service: 1/0, False), + ('my-service', SamplingRule.NO_RULE, True), + ('my-service', None, False), + (None, None, True), + (None, 'my-service', False), + (None, re.compile(r'my-service'), False), + (None, lambda service: 'service' in service, False), + ('my-service', 'my-service', True), + ('my-service', 'my_service', False), + ('my-service', re.compile(r'^my-'), True), + ('my_service', re.compile(r'^my[_-]'), True), + ('my-service', re.compile(r'^my_'), False), + ('my-service', re.compile(r'my-service'), True), + ('my-service', re.compile(r'my'), True), + ('my-service', re.compile(r'my-service|another-service'), True), + ('another-service', re.compile(r'my-service|another-service'), True), + ('my-service', lambda service: 'service' in service, True), + ('my-service', lambda service: 'service' not in service, False), + ('my-service', lambda service: 1 / 0, False), ] ] ) diff --git a/tests/utils/tracer.py b/tests/utils/tracer.py index 180d700cf84..1b15f4aaf05 100644 --- a/tests/utils/tracer.py +++ b/tests/utils/tracer.py @@ -65,10 +65,10 @@ def __init__(self): def _update_writer(self): self.writer = DummyWriter( - hostname=self.writer.api.hostname, - port=self.writer.api.port, - filters=self.writer._filters, - priority_sampler=self.writer._priority_sampler, + hostname=self.writer.api.hostname, + port=self.writer.api.port, + filters=self.writer._filters, + priority_sampler=self.writer._priority_sampler, ) def configure(self, *args, **kwargs): diff --git a/tox.ini b/tox.ini index 70d7c2ebbaa..a1232784d7d 100644 --- a/tox.ini +++ b/tox.ini @@ -758,3 +758,4 @@ exclude= .eggs,*.egg, # We shouldn't lint our vendored dependencies ddtrace/vendor/ +ignore = W504 From f736e8ca50185774fd9d77d66dd149cd642b4b40 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 12 Nov 2019 16:27:43 +0100 Subject: [PATCH 05/81] Add flake8-builtins This makes sure we pick proper variable names. --- ddtrace/__init__.py | 6 +++--- ddtrace/contrib/psycopg/connection.py | 4 ++-- ddtrace/contrib/tornado/stack_context.py | 4 ++-- ddtrace/opentracer/tracer.py | 4 ++-- ddtrace/tracer.py | 4 ++-- ddtrace/utils/time.py | 2 +- docs/conf.py | 2 +- tests/contrib/pymemcache/utils.py | 2 +- tests/internal/runtime/test_container.py | 2 +- tests/opentracer/test_tracer.py | 4 ++-- tests/test_tracer.py | 2 +- tox.ini | 5 ++++- 12 files changed, 22 insertions(+), 19 deletions(-) diff --git a/ddtrace/__init__.py b/ddtrace/__init__.py index a5caccbb1d6..d3c4d60a075 100644 --- a/ddtrace/__init__.py +++ b/ddtrace/__init__.py @@ -33,10 +33,10 @@ _ORIGINAL_EXCEPTHOOK = sys.excepthook -def _excepthook(type, value, traceback): - tracer.global_excepthook(type, value, traceback) +def _excepthook(tp, value, traceback): + tracer.global_excepthook(tp, value, traceback) if _ORIGINAL_EXCEPTHOOK: - return _ORIGINAL_EXCEPTHOOK(type, value, traceback) + return _ORIGINAL_EXCEPTHOOK(tp, value, traceback) def install_excepthook(): diff --git a/ddtrace/contrib/psycopg/connection.py b/ddtrace/contrib/psycopg/connection.py index 703387f9940..2b20cf1efb1 100644 --- a/ddtrace/contrib/psycopg/connection.py +++ b/ddtrace/contrib/psycopg/connection.py @@ -39,7 +39,7 @@ def __init__(self, *args, **kwargs): self._datadog_tags = kwargs.pop('datadog_tags', None) super(TracedCursor, self).__init__(*args, **kwargs) - def execute(self, query, vars=None): + def execute(self, query, vars=None): # noqa: A002 """ just wrap the cursor execution in a span """ if not self._datadog_tracer: return cursor.execute(self, query, vars) @@ -56,7 +56,7 @@ def execute(self, query, vars=None): finally: s.set_metric('db.rowcount', self.rowcount) - def callproc(self, procname, vars=None): + def callproc(self, procname, vars=None): # noqa: A002 """ just wrap the execution in a span """ return cursor.callproc(self, procname, vars) diff --git a/ddtrace/contrib/tornado/stack_context.py b/ddtrace/contrib/tornado/stack_context.py index 74f6a41ca87..367b97e1f14 100644 --- a/ddtrace/contrib/tornado/stack_context.py +++ b/ddtrace/contrib/tornado/stack_context.py @@ -42,7 +42,7 @@ def enter(self): """ pass - def exit(self, type, value, traceback): + def exit(self, type, value, traceback): # noqa: A002 """ Required to preserve the ``StackContext`` protocol. """ @@ -54,7 +54,7 @@ def __enter__(self): _state.contexts = self.new_contexts return self - def __exit__(self, type, value, traceback): + def __exit__(self, type, value, traceback): # noqa: A002 final_contexts = _state.contexts _state.contexts = self.old_contexts diff --git a/ddtrace/opentracer/tracer.py b/ddtrace/opentracer/tracer.py index 482dfcadc54..1a806bf199e 100644 --- a/ddtrace/opentracer/tracer.py +++ b/ddtrace/opentracer/tracer.py @@ -270,7 +270,7 @@ def start_span(self, operation_name=None, child_of=None, references=None, return otspan - def inject(self, span_context, format, carrier): + def inject(self, span_context, format, carrier): # noqa: A002 """Injects a span context into a carrier. :param span_context: span context to inject. @@ -284,7 +284,7 @@ def inject(self, span_context, format, carrier): propagator.inject(span_context, carrier) - def extract(self, format, carrier): + def extract(self, format, carrier): # noqa: A002 """Extracts a span context from a carrier. :param format: format that the carrier is encoded with. diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 59023ef5ec6..d542a766a87 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -133,10 +133,10 @@ def debug_logging(self, value): def __call__(self): return self - def global_excepthook(self, type, value, traceback): + def global_excepthook(self, tp, value, traceback): """The global tracer except hook.""" self._dogstatsd_client.increment('datadog.tracer.uncaught_exceptions', 1, - tags=['class:%s' % type.__name__]) + tags=['class:%s' % tp.__name__]) def get_call_context(self, *args, **kwargs): """ diff --git a/ddtrace/utils/time.py b/ddtrace/utils/time.py index 4789638e26e..014fee9ed8f 100644 --- a/ddtrace/utils/time.py +++ b/ddtrace/utils/time.py @@ -45,7 +45,7 @@ def __enter__(self): self.start() return self - def __exit__(self, type, value, traceback): + def __exit__(self, tp, value, traceback): """Stops the watch.""" self.stop() diff --git a/docs/conf.py b/docs/conf.py index 089dbcf747a..c043374a76d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,7 +59,7 @@ # General information about the project. year = datetime.now().year project = u'ddtrace' -copyright = u'2016-{}, Datadog, Inc.'.format(year) +copyright = u'2016-{}, Datadog, Inc.'.format(year) # noqa: A001 author = u'Datadog, Inc.' # document in order of source diff --git a/tests/contrib/pymemcache/utils.py b/tests/contrib/pymemcache/utils.py index 3d93ffb4f9b..c748ba21891 100644 --- a/tests/contrib/pymemcache/utils.py +++ b/tests/contrib/pymemcache/utils.py @@ -41,7 +41,7 @@ def __init__(self, connect_failure=None): self.connect_failure = connect_failure self.sockets = [] - def socket(self, family, type): + def socket(self, family, type): # noqa: A002 socket = MockSocket([], connect_failure=self.connect_failure) self.sockets.append(socket) return socket diff --git a/tests/internal/runtime/test_container.py b/tests/internal/runtime/test_container.py index 63c2af113c2..ade6bf68289 100644 --- a/tests/internal/runtime/test_container.py +++ b/tests/internal/runtime/test_container.py @@ -9,7 +9,7 @@ # Map expected Py2 exception to Py3 name if PY2: - FileNotFoundError = IOError + FileNotFoundError = IOError # noqa: A001 def get_mock_open(read_data=None): diff --git a/tests/opentracer/test_tracer.py b/tests/opentracer/test_tracer.py index ddfdf618161..a668d01fc72 100644 --- a/tests/opentracer/test_tracer.py +++ b/tests/opentracer/test_tracer.py @@ -314,7 +314,7 @@ def test_start_span_multi_intertwined(self, ot_tracer, writer): event = threading.Event() def trace_one(): - id = 11 + id = 11 # noqa: A001 with ot_tracer.start_active_span(str(id)): id += 1 with ot_tracer.start_active_span(str(id)): @@ -323,7 +323,7 @@ def trace_one(): event.set() def trace_two(): - id = 21 + id = 21 # noqa: A001 event.wait() with ot_tracer.start_active_span(str(id)): id += 1 diff --git a/tests/test_tracer.py b/tests/test_tracer.py index ee9e9f6d602..6edb60f1d5c 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -521,7 +521,7 @@ class Foobar(Exception): called = {} - def original(type, value, traceback): + def original(tp, value, traceback): called['yes'] = True sys.excepthook = original diff --git a/tox.ini b/tox.ini index a1232784d7d..7415ef3ce5a 100644 --- a/tox.ini +++ b/tox.ini @@ -456,6 +456,7 @@ ignore_outcome=true deps= flake8>=3.7,<=3.8 flake8-blind-except + flake8-builtins commands=flake8 . basepython=python3.7 @@ -758,4 +759,6 @@ exclude= .eggs,*.egg, # We shouldn't lint our vendored dependencies ddtrace/vendor/ -ignore = W504 +# Ignore: +# A003: XXX is a python builtin, consider renaming the class attribute +ignore = W504,A003 From d15b79d172b3cd9754d7aea266c00211450a5c69 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 12 Nov 2019 16:28:32 +0100 Subject: [PATCH 06/81] Add flake8-logging-format This makes sure that we use a correct logging format in our log strings. --- ddtrace/api.py | 4 ++-- ddtrace/contrib/cassandra/session.py | 8 ++++---- ddtrace/contrib/django/cache.py | 2 +- ddtrace/contrib/django/middleware.py | 8 ++++---- ddtrace/contrib/django/utils.py | 8 ++++---- ddtrace/contrib/flask/patch.py | 4 ++-- ddtrace/internal/runtime/collector.py | 2 +- ddtrace/internal/runtime/container.py | 4 ++-- ddtrace/internal/runtime/runtime_metrics.py | 2 +- ddtrace/internal/writer.py | 4 ++-- ddtrace/monkey.py | 4 ++-- ddtrace/propagation/http.py | 22 +++++++++------------ ddtrace/sampler.py | 8 ++++---- ddtrace/settings/hooks.py | 4 ++-- ddtrace/span.py | 2 +- ddtrace/tracer.py | 4 ++-- ddtrace/utils/hook.py | 8 ++++---- tests/internal/runtime/test_container.py | 2 +- tests/internal/runtime/test_metrics.py | 9 ++++----- tests/test_api.py | 4 ++-- tests/test_hook.py | 20 ++++++------------- tests/test_sampler.py | 4 ++-- tox.ini | 5 ++++- 23 files changed, 66 insertions(+), 76 deletions(-) diff --git a/ddtrace/api.py b/ddtrace/api.py index 22c502853e7..5654d098b7e 100644 --- a/ddtrace/api.py +++ b/ddtrace/api.py @@ -84,8 +84,8 @@ def get_json(self): return return loads(body) - except (ValueError, TypeError) as err: - log.debug('Unable to parse Datadog Agent JSON response: %s %r', err, body) + except (ValueError, TypeError): + log.debug('Unable to parse Datadog Agent JSON response: %r', body, exc_info=True) def __repr__(self): return '{0}(status={1!r}, body={2!r}, reason={3!r}, msg={4!r})'.format( diff --git a/ddtrace/contrib/cassandra/session.py b/ddtrace/contrib/cassandra/session.py index e6a788be508..35b5eb5fc16 100644 --- a/ddtrace/contrib/cassandra/session.py +++ b/ddtrace/contrib/cassandra/session.py @@ -54,8 +54,8 @@ def _close_span_on_success(result, future): return try: span.set_tags(_extract_result_metas(cassandra.cluster.ResultSet(future, result))) - except Exception as e: - log.debug('an exception occured while setting tags: %s', e) + except Exception: + log.debug('an exception occured while setting tags', exc_info=True) finally: span.finish() delattr(future, CURRENT_SPAN) @@ -78,8 +78,8 @@ def _close_span_on_error(exc, future): span.error = 1 span.set_tag(errors.ERROR_MSG, exc.args[0]) span.set_tag(errors.ERROR_TYPE, exc.__class__.__name__) - except Exception as e: - log.debug('traced_set_final_exception was not able to set the error, failed with error: %s', e) + except Exception: + log.debug('traced_set_final_exception was not able to set the error, failed with error', exc_info=True) finally: span.finish() delattr(future, CURRENT_SPAN) diff --git a/ddtrace/contrib/django/cache.py b/ddtrace/contrib/django/cache.py index 5bb356779e6..0184cc4cd12 100644 --- a/ddtrace/contrib/django/cache.py +++ b/ddtrace/contrib/django/cache.py @@ -78,7 +78,7 @@ def _wrap_method(cls, method_name): # prevent patching each backend's method more than once if hasattr(cls, DATADOG_NAMESPACE.format(method=method_name)): - log.debug('{} already traced'.format(method_name)) + log.debug('%s already traced', method_name) else: method = getattr(cls, method_name) setattr(cls, DATADOG_NAMESPACE.format(method=method_name), method) diff --git a/ddtrace/contrib/django/middleware.py b/ddtrace/contrib/django/middleware.py index dae077409dc..9325569e41f 100644 --- a/ddtrace/contrib/django/middleware.py +++ b/ddtrace/contrib/django/middleware.py @@ -144,8 +144,8 @@ def process_request(self, request): if trace_query_string: span.set_tag(http.QUERY_STRING, request.META['QUERY_STRING']) _set_req_span(request, span) - except Exception as e: - log.debug('error tracing request: %s', e) + except Exception: + log.debug('error tracing request', exc_info=True) def process_view(self, request, view_func, *args, **kwargs): span = _get_req_span(request) @@ -190,8 +190,8 @@ def process_response(self, request, response): span.set_tag(http.STATUS_CODE, response.status_code) span = _set_auth_tags(span, request) span.finish() - except Exception as e: - log.debug('error tracing request: %s', e) + except Exception: + log.debug('error tracing request', exc_info=True) finally: return response diff --git a/ddtrace/contrib/django/utils.py b/ddtrace/contrib/django/utils.py index ced226e83ed..3b4a1acee16 100644 --- a/ddtrace/contrib/django/utils.py +++ b/ddtrace/contrib/django/utils.py @@ -44,8 +44,8 @@ def get_request_uri(request): host = None try: host = request.get_host() # this will include host:port - except Exception as e: - log.debug('Failed to get Django request host: %s', e) + except Exception: + log.debug('Failed to get Django request host', exc_info=True) if not host: try: @@ -58,9 +58,9 @@ def get_request_uri(request): port = str(request.META['SERVER_PORT']) if port != ('443' if request.is_secure() else '80'): host = '{0}:{1}'.format(host, port) - except Exception as e: + except Exception: # This really shouldn't ever happen, but lets guard here just in case - log.debug('Failed to build Django request host: %s', e) + log.debug('Failed to build Django request host', exc_info=True) host = 'unknown' # Build request url from the information available diff --git a/ddtrace/contrib/flask/patch.py b/ddtrace/contrib/flask/patch.py index ebdb5cd9ca1..3b4c07f3008 100644 --- a/ddtrace/contrib/flask/patch.py +++ b/ddtrace/contrib/flask/patch.py @@ -470,8 +470,8 @@ def _traced_request(pin, wrapped, instance, args, kwargs): if not span.get_tag(FLASK_VIEW_ARGS) and request.view_args and config.flask.get('collect_view_args'): for k, v in request.view_args.items(): span.set_tag(u'{}.{}'.format(FLASK_VIEW_ARGS, k), v) - except Exception as e: - log.debug('failed to set tags for "flask.request" span: {}'.format(e)) + except Exception: + log.debug('failed to set tags for "flask.request" span', exc_info=True) with pin.tracer.trace('flask.{}'.format(name), service=pin.service): return wrapped(*args, **kwargs) diff --git a/ddtrace/internal/runtime/collector.py b/ddtrace/internal/runtime/collector.py index 98e7f5afa46..b94cf25c5fe 100644 --- a/ddtrace/internal/runtime/collector.py +++ b/ddtrace/internal/runtime/collector.py @@ -45,7 +45,7 @@ def _load_modules(self): except ImportError: # DEV: disable collector if we cannot load any of the required modules self.enabled = False - log.warning('Could not import module "{}" for {}. Disabling collector.'.format(module, self)) + log.warning('Could not import module "%s" for %s. Disabling collector.', module, self) return None return modules diff --git a/ddtrace/internal/runtime/container.py b/ddtrace/internal/runtime/container.py index 593374f5734..87bf3664092 100644 --- a/ddtrace/internal/runtime/container.py +++ b/ddtrace/internal/runtime/container.py @@ -104,7 +104,7 @@ def get_container_info(pid='self'): info = CGroupInfo.from_line(line) if info and info.container_id: return info - except Exception as err: - log.debug('Failed to parse cgroup file for pid %r: %s', pid, err) + except Exception: + log.debug('Failed to parse cgroup file for pid %r', pid, exc_info=True) return None diff --git a/ddtrace/internal/runtime/runtime_metrics.py b/ddtrace/internal/runtime/runtime_metrics.py index f777bc0c27c..374a943ae0a 100644 --- a/ddtrace/internal/runtime/runtime_metrics.py +++ b/ddtrace/internal/runtime/runtime_metrics.py @@ -71,7 +71,7 @@ def __init__(self, statsd_client, flush_interval=FLUSH_INTERVAL): def flush(self): with self._statsd_client: for key, value in self._runtime_metrics: - log.debug('Writing metric {}:{}'.format(key, value)) + log.debug('Writing metric %s:%s', key, value) self._statsd_client.gauge(key, value) run_periodic = flush diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 6109658679c..9c1fbdff189 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -84,8 +84,8 @@ def flush_queue(self): # filters try: traces = self._apply_filters(traces) - except Exception as err: - log.error('error while filtering traces: {0}'.format(err)) + except Exception: + log.error('error while filtering traces', exc_info=True) return if self._send_stats: diff --git a/ddtrace/monkey.py b/ddtrace/monkey.py index fa734f08075..88b9fe984fa 100644 --- a/ddtrace/monkey.py +++ b/ddtrace/monkey.py @@ -150,10 +150,10 @@ def patch_module(module, raise_errors=True): """ try: return _patch_module(module) - except Exception as exc: + except Exception: if raise_errors: raise - log.debug('failed to patch %s: %s', module, exc) + log.debug('failed to patch %s', module, exc_info=True) return False diff --git a/ddtrace/propagation/http.py b/ddtrace/propagation/http.py index 0bbe4463e0a..f766915567c 100644 --- a/ddtrace/propagation/http.py +++ b/ddtrace/propagation/http.py @@ -135,17 +135,13 @@ def my_controller(url, headers): _dd_origin=origin, ) # If headers are invalid and cannot be parsed, return a new context and log the issue. - except Exception as error: - try: - log.debug( - 'invalid x-datadog-* headers, trace-id: %s, parent-id: %s, priority: %s, origin: %s, error: %s', - headers.get(HTTP_HEADER_TRACE_ID, 0), - headers.get(HTTP_HEADER_PARENT_ID, 0), - headers.get(HTTP_HEADER_SAMPLING_PRIORITY), - headers.get(HTTP_HEADER_ORIGIN, ''), - error, - ) - # We might fail on string formatting errors ; in that case only format the first error - except Exception: - log.debug(error) + except Exception: + log.debug( + 'invalid x-datadog-* headers, trace-id: %s, parent-id: %s, priority: %s, origin: %s', + headers.get(HTTP_HEADER_TRACE_ID, 0), + headers.get(HTTP_HEADER_PARENT_ID, 0), + headers.get(HTTP_HEADER_SAMPLING_PRIORITY), + headers.get(HTTP_HEADER_ORIGIN, ''), + exc_info=True, + ) return Context() diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 0eb89d84a94..160d7e6e901 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -275,8 +275,8 @@ def _pattern_matches(self, prop, pattern): if callable(pattern): try: return bool(pattern(prop)) - except Exception as e: - log.warning('%r pattern %r failed with %r: %s', self, pattern, prop, e) + except Exception: + log.warning('%r pattern %r failed with %r', self, pattern, prop, exc_info=True) # Their function failed to validate, assume it is a False return False @@ -284,9 +284,9 @@ def _pattern_matches(self, prop, pattern): if isinstance(pattern, pattern_type): try: return bool(pattern.match(str(prop))) - except (ValueError, TypeError) as e: + except (ValueError, TypeError): # This is to guard us against the casting to a string (shouldn't happen, but still) - log.warning('%r pattern %r failed with %r: %s', self, pattern, prop, e) + log.warning('%r pattern %r failed with %r', self, pattern, prop, exc_info=True) return False # Exact match on the values diff --git a/ddtrace/settings/hooks.py b/ddtrace/settings/hooks.py index 81b9eeb9adb..ea5a8c4791d 100644 --- a/ddtrace/settings/hooks.py +++ b/ddtrace/settings/hooks.py @@ -111,9 +111,9 @@ def _emit(self, hook, span, *args, **kwargs): for func in self._hooks[hook]: try: func(span, *args, **kwargs) - except Exception as e: + except Exception: # DEV: Use log.debug instead of log.error until we have a throttled logger - log.debug('Failed to run hook {} function {}: {}'.format(hook, func, e)) + log.debug('Failed to run hook %s function %s', hook, func, exc_info=True) def __repr__(self): """Return string representation of this class instance""" diff --git a/ddtrace/span.py b/ddtrace/span.py index 710b12e3374..09b2f68a497 100644 --- a/ddtrace/span.py +++ b/ddtrace/span.py @@ -159,7 +159,7 @@ def set_tag(self, key, value=None): # DEV: `set_metric` will try to cast to `float()` for us self.set_metric(key, value) except (TypeError, ValueError): - log.debug('error setting numeric metric {}:{}'.format(key, value)) + log.debug('error setting numeric metric %s:%s', key, value) return elif key == MANUAL_KEEP_KEY: diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index d542a766a87..7160d15bce7 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -218,7 +218,7 @@ def configure(self, enabled=None, hostname=None, port=None, uds_path=None, https if dogstatsd_url is not None: dogstatsd_kwargs = _parse_dogstatsd_url(dogstatsd_url) - self.log.debug('Connecting to DogStatsd({})'.format(dogstatsd_url)) + self.log.debug('Connecting to DogStatsd(%s)', dogstatsd_url) self._dogstatsd_client = DogStatsd(**dogstatsd_kwargs) if hostname is not None or port is not None or uds_path is not None or https is not None or \ @@ -398,7 +398,7 @@ def _update_dogstatsd_constant_tags(self): '{}:{}'.format(k, v) for k, v in RuntimeTags() ] - self.log.debug('Updating constant tags {}'.format(tags)) + self.log.debug('Updating constant tags %s', tags) self._dogstatsd_client.constant_tags = tags def _start_runtime_worker(self): diff --git a/ddtrace/utils/hook.py b/ddtrace/utils/hook.py index 16284846994..ef143ac629e 100644 --- a/ddtrace/utils/hook.py +++ b/ddtrace/utils/hook.py @@ -54,14 +54,14 @@ def register_post_import_hook(name, hook): hooks = _post_import_hooks.get(name, []) if hook in hooks: - log.debug('hook "{}" already exists on module "{}"'.format(hook, name)) + log.debug('hook "%s" already exists on module "%s"', hook, name) return module = sys.modules.get(name, None) # If the module has been imported already fire the hook and log a debug msg. if module: - log.debug('module "{}" already imported, firing hook'.format(name)) + log.debug('module "%s" already imported, firing hook', name) hook(module) hooks.append(hook) @@ -86,8 +86,8 @@ def notify_module_loaded(module): for hook in hooks: try: hook(module) - except Exception as err: - log.warning('hook "{}" for module "{}" failed: {}'.format(hook, name, err)) + except Exception: + log.warning('hook "%s" for module "%s" failed', hook, name, exc_info=True) class _ImportHookLoader(object): diff --git a/tests/internal/runtime/test_container.py b/tests/internal/runtime/test_container.py index ade6bf68289..2dce42098a2 100644 --- a/tests/internal/runtime/test_container.py +++ b/tests/internal/runtime/test_container.py @@ -299,4 +299,4 @@ def test_get_container_info_exception(mock_log, mock_from_line): mock_open.assert_called_once_with('/proc/self/cgroup', mode='r') # Ensure we logged the exception - mock_log.debug.assert_called_once_with('Failed to parse cgroup file for pid %r: %s', 'self', exception) + mock_log.debug.assert_called_once_with('Failed to parse cgroup file for pid %r', 'self', exc_info=True) diff --git a/tests/internal/runtime/test_metrics.py b/tests/internal/runtime/test_metrics.py index 227713d8004..0b6dd5a4cbc 100644 --- a/tests/internal/runtime/test_metrics.py +++ b/tests/internal/runtime/test_metrics.py @@ -83,11 +83,10 @@ def test_required_module_not_installed(self): collect.assert_not_called() calls = [ - mock.call(( - 'Could not import module "moduleshouldnotexist" for ' - '. ' - 'Disabling collector.' - )) + mock.call( + 'Could not import module "%s" for %s. Disabling collector.', + 'moduleshouldnotexist', vc, + ) ] log_mock.warning.assert_has_calls(calls) diff --git a/tests/test_api.py b/tests/test_api.py index a4734a0542f..51d3a1b570a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -144,11 +144,11 @@ def test_parse_response_json(self, log): ), 'error:unsupported-endpoint': dict( js=None, - log='Unable to parse Datadog Agent JSON response: .*? \'error:unsupported-endpoint\'', + log='Unable to parse Datadog Agent JSON response: \'error:unsupported-endpoint\'', ), 42: dict( # int as key to trigger TypeError js=None, - log='Unable to parse Datadog Agent JSON response: .*? 42', + log='Unable to parse Datadog Agent JSON response: 42', ), '{}': dict(js={}), '[]': dict(js=[]), diff --git a/tests/test_hook.py b/tests/test_hook.py index e9da85b68b8..817916a5bcb 100644 --- a/tests/test_hook.py +++ b/tests/test_hook.py @@ -31,7 +31,7 @@ def test_register_post_import_hook_after_import(self): register_post_import_hook('tests.utils.test_module', test_hook) test_hook.assert_called_once() calls = [ - mock.call('module "tests.utils.test_module" already imported, firing hook') + mock.call('module "%s" already imported, firing hook', "tests.utils.test_module") ] log_mock.debug.assert_has_calls(calls) @@ -80,18 +80,9 @@ def test_register_post_import_hook_duplicate_register(self): register_post_import_hook('tests.utils.test_module', test_hook) import tests.utils.test_module # noqa - # Since the log message will contain the id (non-deterministic) of the hook - # we just check to see if the important parts of the log message are included - # in the message. Those being the name and the module to be hooked. - class Matcher(object): - def __eq__(self, other): - return 'MagicMock' in other and 'already exists on module "tests.utils.test_module"' in other - - calls = [ - mock.call(Matcher()) - ] - self.assertEqual(test_hook.call_count, 1) - log_mock.debug.assert_has_calls(calls) + self.assertEqual(log_mock.debug.mock_calls, [ + mock.call('hook "%s" already exists on module "%s"', test_hook, 'tests.utils.test_module'), + ]) def test_deregister_post_import_hook_no_register(self): """ @@ -174,7 +165,8 @@ def test_hook(module): with mock.patch('ddtrace.utils.hook.log') as log_mock: import tests.utils.test_module # noqa calls = [ - mock.call('hook "{}" for module "tests.utils.test_module" failed: test_hook_failed'.format(test_hook)) + mock.call('hook "%s" for module "%s" failed', + test_hook, 'tests.utils.test_module', exc_info=True) ] log_mock.warning.assert_has_calls(calls) diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 4dd4ffd9694..4a6409372e4 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -384,11 +384,11 @@ def pattern(prop): with mock.patch('ddtrace.sampler.log') as mock_log: assert rule.matches(span) is False mock_log.warning.assert_called_once_with( - '%r pattern %r failed with %r: %s', + '%r pattern %r failed with %r', rule, pattern, 'test.span', - e, + exc_info=True, ) diff --git a/tox.ini b/tox.ini index 7415ef3ce5a..29ad8d063b0 100644 --- a/tox.ini +++ b/tox.ini @@ -457,6 +457,7 @@ deps= flake8>=3.7,<=3.8 flake8-blind-except flake8-builtins + flake8-logging-format commands=flake8 . basepython=python3.7 @@ -761,4 +762,6 @@ exclude= ddtrace/vendor/ # Ignore: # A003: XXX is a python builtin, consider renaming the class attribute -ignore = W504,A003 +# G201 Logging: .exception(...) should be used instead of .error(..., exc_info=True) +ignore = W504,A003,G201 +enable-extensions=G From aaae14927f2a3d08c983957e7fd2e10f3a4152e1 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 12 Nov 2019 18:51:56 +0100 Subject: [PATCH 07/81] Enable flake8-rst-docstrings This fixes various RST directives that we got wrong and log format --- ddtrace/contrib/aiohttp/middlewares.py | 5 +-- ddtrace/contrib/asyncio/helpers.py | 2 +- ddtrace/contrib/django/cache.py | 7 ++-- ddtrace/contrib/django/conf.py | 8 ++--- ddtrace/contrib/flask_cache/tracers.py | 4 +-- ddtrace/contrib/mysql/__init__.py | 2 ++ ddtrace/contrib/mysqldb/__init__.py | 2 ++ ddtrace/contrib/pymongo/__init__.py | 2 ++ ddtrace/contrib/requests/connection.py | 7 ++-- ddtrace/http/headers.py | 6 ++-- ddtrace/internal/runtime/tag_collectors.py | 20 +++++------ ddtrace/opentracer/tracer.py | 12 +++---- ddtrace/payload.py | 2 +- ddtrace/pin.py | 2 +- ddtrace/provider.py | 4 +-- ddtrace/settings/hooks.py | 4 +-- ddtrace/utils/formats.py | 12 ++++--- ddtrace/utils/wrappers.py | 17 ++++------ tests/base/__init__.py | 39 ++++++++++++---------- tests/contrib/cassandra/test.py | 2 +- tests/contrib/django/test_middleware.py | 8 ++--- tests/contrib/patch.py | 36 ++++++++++---------- tests/contrib/sqlalchemy/mixins.py | 9 +++-- tests/test_context.py | 24 ++++++------- tests/utils/span.py | 30 ++++++++--------- tox.ini | 5 +++ 26 files changed, 142 insertions(+), 129 deletions(-) diff --git a/ddtrace/contrib/aiohttp/middlewares.py b/ddtrace/contrib/aiohttp/middlewares.py index 8bd3128bc46..344452e3c6e 100644 --- a/ddtrace/contrib/aiohttp/middlewares.py +++ b/ddtrace/contrib/aiohttp/middlewares.py @@ -20,8 +20,9 @@ def trace_middleware(app, handler): ``aiohttp`` middleware that traces the handler execution. Because handlers are run in different tasks for each request, we attach the Context instance both to the Task and to the Request objects. In this way: - * the Task is used by the internal automatic instrumentation - * the ``Context`` attached to the request can be freely used in the application code + + * the Task is used by the internal automatic instrumentation + * the ``Context`` attached to the request can be freely used in the application code """ @asyncio.coroutine def attach_context(request): diff --git a/ddtrace/contrib/asyncio/helpers.py b/ddtrace/contrib/asyncio/helpers.py index 2a3d0f40e5f..65f00720fb1 100644 --- a/ddtrace/contrib/asyncio/helpers.py +++ b/ddtrace/contrib/asyncio/helpers.py @@ -48,7 +48,7 @@ def run_in_executor(loop, executor, func, *args, tracer=None): coroutine is executed, we may have two different scenarios: * the Context is copied in the new Thread and the trace is sent twice * the coroutine flushes the Context and when the Thread copies the - Context it is already empty (so it will be a root Span) + Context it is already empty (so it will be a root Span) To support both situations, we create a new Context that knows only what was the latest active Span when the new thread was created. In this new thread, diff --git a/ddtrace/contrib/django/cache.py b/ddtrace/contrib/django/cache.py index 0184cc4cd12..ead51a2db83 100644 --- a/ddtrace/contrib/django/cache.py +++ b/ddtrace/contrib/django/cache.py @@ -35,9 +35,10 @@ def patch_cache(tracer): can have different implementations and connectors, this function must handle all possible interactions with the Django cache. What follows is currently traced: - * in-memory cache - * the cache client wrapper that could use any of the common - Django supported cache servers (Redis, Memcached, Database, Custom) + + * in-memory cache + * the cache client wrapper that could use any of the common + Django supported cache servers (Redis, Memcached, Database, Custom) """ # discover used cache backends cache_backends = set([cache['BACKEND'] for cache in django_settings.CACHES.values()]) diff --git a/ddtrace/contrib/django/conf.py b/ddtrace/contrib/django/conf.py index 33d69ea8abd..31dda245343 100644 --- a/ddtrace/contrib/django/conf.py +++ b/ddtrace/contrib/django/conf.py @@ -1,10 +1,10 @@ """ Settings for Datadog tracer are all namespaced in the DATADOG_TRACE setting. -For example your project's `settings.py` file might look like this: +For example your project's `settings.py` file might look like this:: -DATADOG_TRACE = { - 'TRACER': 'myapp.tracer', -} + DATADOG_TRACE = { + 'TRACER': 'myapp.tracer', + } This module provides the `setting` object, that is used to access Datadog settings, checking for user settings first, then falling diff --git a/ddtrace/contrib/flask_cache/tracers.py b/ddtrace/contrib/flask_cache/tracers.py index a83e33c3f4a..210fb4b3724 100644 --- a/ddtrace/contrib/flask_cache/tracers.py +++ b/ddtrace/contrib/flask_cache/tracers.py @@ -33,8 +33,8 @@ def get_traced_cache(ddtracer, service=DEFAULT_SERVICE, meta=None): class TracedCache(Cache): """ Traced cache backend that monitors any operations done by flask_cache. Observed actions are: - * get, set, add, delete, clear - * all many_ operations + * get, set, add, delete, clear + * all ``many_`` operations """ _datadog_tracer = ddtracer _datadog_service = service diff --git a/ddtrace/contrib/mysql/__init__.py b/ddtrace/contrib/mysql/__init__.py index 586e67cae9f..422d7ad9431 100644 --- a/ddtrace/contrib/mysql/__init__.py +++ b/ddtrace/contrib/mysql/__init__.py @@ -1,7 +1,9 @@ """Instrument mysql to report MySQL queries. ``patch_all`` will automatically patch your mysql connection to make it work. + :: + # Make sure to import mysql.connector and not the 'connect' function, # otherwise you won't have access to the patched version from ddtrace import Pin, patch diff --git a/ddtrace/contrib/mysqldb/__init__.py b/ddtrace/contrib/mysqldb/__init__.py index 5050cd1ff21..3219a189c36 100644 --- a/ddtrace/contrib/mysqldb/__init__.py +++ b/ddtrace/contrib/mysqldb/__init__.py @@ -1,7 +1,9 @@ """Instrument mysqlclient / MySQL-python to report MySQL queries. ``patch_all`` will automatically patch your mysql connection to make it work. + :: + # Make sure to import MySQLdb and not the 'connect' function, # otherwise you won't have access to the patched version from ddtrace import Pin, patch diff --git a/ddtrace/contrib/pymongo/__init__.py b/ddtrace/contrib/pymongo/__init__.py index 6e1745dce0d..fd49869d3b1 100644 --- a/ddtrace/contrib/pymongo/__init__.py +++ b/ddtrace/contrib/pymongo/__init__.py @@ -3,7 +3,9 @@ The pymongo integration works by wrapping pymongo's MongoClient to trace network calls. Pymongo 3.0 and greater are the currently supported versions. ``patch_all`` will automatically patch your MongoClient instance to make it work. + :: + # Be sure to import pymongo and not pymongo.MongoClient directly, # otherwise you won't have access to the patched version from ddtrace import Pin, patch diff --git a/ddtrace/contrib/requests/connection.py b/ddtrace/contrib/requests/connection.py index 56e49905393..960e0e4e148 100644 --- a/ddtrace/contrib/requests/connection.py +++ b/ddtrace/contrib/requests/connection.py @@ -16,12 +16,11 @@ def _extract_service_name(session, span, hostname=None): """Extracts the right service name based on the following logic: - `requests` is the default service name - users can change it via `session.service_name = 'clients'` - - if the Span doesn't have a parent, use the set service name - or fallback to the default + - if the Span doesn't have a parent, use the set service name or fallback to the default - if the Span has a parent, use the set service name or the - parent service value if the set service name is the default + parent service value if the set service name is the default - if `split_by_domain` is used, always override users settings - and use the network location as a service name + and use the network location as a service name The priority can be represented as: Updated service name > parent service name > default to `requests`. diff --git a/ddtrace/http/headers.py b/ddtrace/http/headers.py index cd3f07121e4..b680a5c45a0 100644 --- a/ddtrace/http/headers.py +++ b/ddtrace/http/headers.py @@ -70,9 +70,9 @@ def _normalize_tag_name(request_or_response, header_name): """ Given a tag name, e.g. 'Content-Type', returns a corresponding normalized tag name, i.e 'http.request.headers.content_type'. Rules applied actual header name are: - - any letter is converted to lowercase - - any digit is left unchanged - - any block of any length of different ASCII chars is converted to a single underscore '_' + - any letter is converted to lowercase + - any digit is left unchanged + - any block of any length of different ASCII chars is converted to a single underscore '_' :param request_or_response: The context of the headers: request|response :param header_name: The header's name :type header_name: str diff --git a/ddtrace/internal/runtime/tag_collectors.py b/ddtrace/internal/runtime/tag_collectors.py index 12d34417c57..d9c2fb235a9 100644 --- a/ddtrace/internal/runtime/tag_collectors.py +++ b/ddtrace/internal/runtime/tag_collectors.py @@ -31,16 +31,16 @@ class PlatformTagCollector(RuntimeTagCollector): """ Tag collector for the Python interpreter implementation. Tags collected: - - lang_interpreter: - - For CPython this is 'CPython'. - - For Pypy this is 'PyPy'. - - For Jython this is 'Jython'. - - lang_version: - - eg. '2.7.10' - - lang: - - e.g. 'Python' - - tracer_version: - - e.g. '0.29.0' + - ``lang_interpreter``: + + * For CPython this is 'CPython'. + * For Pypy this is ``PyPy`` + * For Jython this is ``Jython`` + + - `lang_version``, eg ``2.7.10`` + - ``lang`` e.g. ``Python`` + - ``tracer_version`` e.g. ``0.29.0`` + """ required_modules = ('platform', 'ddtrace') diff --git a/ddtrace/opentracer/tracer.py b/ddtrace/opentracer/tracer.py index 1a806bf199e..b80402f2c4d 100644 --- a/ddtrace/opentracer/tracer.py +++ b/ddtrace/opentracer/tracer.py @@ -176,13 +176,11 @@ def start_span(self, operation_name=None, child_of=None, references=None, '...', references=[opentracing.child_of(parent_span)]) - Note: the precedence when defining a relationship is the following: - (highest) - 1. *child_of* - 2. *references* - 3. `scope_manager.active` (unless *ignore_active_span* is True) - 4. None - (lowest) + Note: the precedence when defining a relationship is the following, from highest to lowest: + 1. *child_of* + 2. *references* + 3. `scope_manager.active` (unless *ignore_active_span* is True) + 4. None Currently Datadog only supports `child_of` references. diff --git a/ddtrace/payload.py b/ddtrace/payload.py index df5cb29553a..acbede4fbcd 100644 --- a/ddtrace/payload.py +++ b/ddtrace/payload.py @@ -41,7 +41,7 @@ def add_trace(self, trace): Encode and append a trace to this payload :param trace: A trace to append - :type trace: A list of ``ddtrace.span.Span``s + :type trace: A list of :class:`ddtrace.span.Span` """ # No trace or empty trace was given, ignore if not trace: diff --git a/ddtrace/pin.py b/ddtrace/pin.py index fe708698fa6..9e58f7ca099 100644 --- a/ddtrace/pin.py +++ b/ddtrace/pin.py @@ -64,7 +64,7 @@ def _find(*objs): >>> pin = Pin._find(wrapper, instance, conn, app) - :param *objs: The objects to search for a :class:`ddtrace.pin.Pin` on + :param objs: The objects to search for a :class:`ddtrace.pin.Pin` on :type objs: List of objects :rtype: :class:`ddtrace.pin.Pin`, None :returns: The first found :class:`ddtrace.pin.Pin` or `None` is none was found diff --git a/ddtrace/provider.py b/ddtrace/provider.py index 246fbcec52b..1cc1f3b2e04 100644 --- a/ddtrace/provider.py +++ b/ddtrace/provider.py @@ -10,8 +10,8 @@ class BaseContextProvider(six.with_metaclass(abc.ABCMeta)): for a callable class, capable to retrieve the current active ``Context`` instance. Context providers must inherit this class and implement: - * the ``active`` method, that returns the current active ``Context`` - * the ``activate`` method, that sets the current active ``Context`` + * the ``active`` method, that returns the current active ``Context`` + * the ``activate`` method, that sets the current active ``Context`` """ @abc.abstractmethod def _has_active_context(self): diff --git a/ddtrace/settings/hooks.py b/ddtrace/settings/hooks.py index ea5a8c4791d..6713684dd34 100644 --- a/ddtrace/settings/hooks.py +++ b/ddtrace/settings/hooks.py @@ -94,9 +94,9 @@ def _emit(self, hook, span, *args, **kwargs): :type hook: str :param span: The span to call the hook with :type span: :class:`ddtrace.span.Span` - :param *args: Positional arguments to pass to the hook functions + :param args: Positional arguments to pass to the hook functions :type args: list - :param **kwargs: Keyword arguments to pass to the hook functions + :param kwargs: Keyword arguments to pass to the hook functions :type kwargs: dict """ # Return early if no hooks are registered diff --git a/ddtrace/utils/formats.py b/ddtrace/utils/formats.py index 363c9c11638..5a1a4001f8b 100644 --- a/ddtrace/utils/formats.py +++ b/ddtrace/utils/formats.py @@ -7,11 +7,13 @@ def get_env(integration, variable, default=None): """Retrieves environment variables value for the given integration. It must be used for consistency between integrations. The implementation is backward compatible with legacy nomenclature: - * `DATADOG_` is a legacy prefix with lower priority - * `DD_` environment variables have the highest priority - * the environment variable is built concatenating `integration` and `variable` - arguments - * return `default` otherwise + + * `DATADOG_` is a legacy prefix with lower priority + * `DD_` environment variables have the highest priority + * the environment variable is built concatenating `integration` and `variable` + arguments + * return `default` otherwise + """ key = '{}_{}'.format(integration, variable).upper() legacy_env = 'DATADOG_{}'.format(key) diff --git a/ddtrace/utils/wrappers.py b/ddtrace/utils/wrappers.py index a8369c86ee8..8ac7d581b28 100644 --- a/ddtrace/utils/wrappers.py +++ b/ddtrace/utils/wrappers.py @@ -16,20 +16,17 @@ def safe_patch(patchable, key, patch_func, service, meta, tracer): wrapped in the monkey patch == UNBOUND + service and meta) and attach the patched result to patchable at patchable.key - - - if this is the module/class we can rely on methods being unbound, and just have to + - If this is the module/class we can rely on methods being unbound, and just have to update the __dict__ - - - if this is an instance, we have to unbind the current and rebind our + - If this is an instance, we have to unbind the current and rebind our patched method - - - If patchable is an instance and if we've already patched at the module/class level + - If patchable is an instance and if we've already patched at the module/class level then patchable[key] contains an already patched command! - To workaround this, check if patchable or patchable.__class__ are _dogtraced - If is isn't, nothing to worry about, patch the key as usual - But if it is, search for a '__dd_orig_{key}' method on the class, which is - the original unpatched method we wish to trace. + To workaround this, check if patchable or patchable.__class__ are ``_dogtraced`` + If is isn't, nothing to worry about, patch the key as usual + But if it is, search for a '__dd_orig_{key}' method on the class, which is + the original unpatched method we wish to trace. """ def _get_original_method(thing, key): orig = None diff --git a/tests/base/__init__.py b/tests/base/__init__.py index 7c933f113e4..4499ae9f61e 100644 --- a/tests/base/__init__.py +++ b/tests/base/__init__.py @@ -29,9 +29,10 @@ def test_case(self): @contextlib.contextmanager def override_env(env): """ - Temporarily override ``os.environ`` with provided values - >>> with self.override_env(dict(DATADOG_TRACE_DEBUG=True)): - # Your test + Temporarily override ``os.environ`` with provided values:: + + >>> with self.override_env(dict(DATADOG_TRACE_DEBUG=True)): + # Your test """ # Copy the full original environment original = dict(os.environ) @@ -49,9 +50,10 @@ def override_env(env): @contextlib.contextmanager def override_global_config(values): """ - Temporarily override an global configuration - >>> with self.override_global_config(dict(name=value,...)): - # Your test + Temporarily override an global configuration:: + + >>> with self.override_global_config(dict(name=value,...)): + # Your test """ # DEV: Uses dict as interface but internally handled as attributes on Config instance analytics_enabled_original = ddtrace.config.analytics_enabled @@ -69,9 +71,10 @@ def override_global_config(values): @contextlib.contextmanager def override_config(integration, values): """ - Temporarily override an integration configuration value - >>> with self.override_config('flask', dict(service_name='test-service')): - # Your test + Temporarily override an integration configuration value:: + + >>> with self.override_config('flask', dict(service_name='test-service')): + # Your test """ options = getattr(ddtrace.config, integration) @@ -90,9 +93,10 @@ def override_config(integration, values): @contextlib.contextmanager def override_http_config(integration, values): """ - Temporarily override an integration configuration for HTTP value - >>> with self.override_http_config('flask', dict(trace_query_string=True)): - # Your test + Temporarily override an integration configuration for HTTP value:: + + >>> with self.override_http_config('flask', dict(trace_query_string=True)): + # Your test """ options = getattr(ddtrace.config, integration).http @@ -111,11 +115,12 @@ def override_http_config(integration, values): @contextlib.contextmanager def override_sys_modules(modules): """ - Temporarily override ``sys.modules`` with provided dictionary of modules - >>> mock_module = mock.MagicMock() - >>> mock_module.fn.side_effect = lambda: 'test' - >>> with self.override_sys_modules(dict(A=mock_module)): - # Your test + Temporarily override ``sys.modules`` with provided dictionary of modules:: + + >>> mock_module = mock.MagicMock() + >>> mock_module.fn.side_effect = lambda: 'test' + >>> with self.override_sys_modules(dict(A=mock_module)): + # Your test """ original = dict(sys.modules) diff --git a/tests/contrib/cassandra/test.py b/tests/contrib/cassandra/test.py index 1fb4c594d6b..6ab1878f3e7 100644 --- a/tests/contrib/cassandra/test.py +++ b/tests/contrib/cassandra/test.py @@ -80,7 +80,7 @@ def override_config(self, integration, values): """ Temporarily override an integration configuration value >>> with self.override_config('flask', dict(service_name='test-service')): - # Your test + ... # Your test """ options = getattr(config, integration) diff --git a/tests/contrib/django/test_middleware.py b/tests/contrib/django/test_middleware.py index 545aba06b57..35b24ce5e5a 100644 --- a/tests/contrib/django/test_middleware.py +++ b/tests/contrib/django/test_middleware.py @@ -155,10 +155,10 @@ def test_analytics_global_off_integration_on(self): def test_analytics_global_off_integration_on_and_none(self): """ When making a request - When an integration trace search is enabled - Sample rate is set to None - Globally trace search is disabled - We expect the root span to have the appropriate tag + When an integration trace search is enabled + Sample rate is set to None + Globally trace search is disabled + We expect the root span to have the appropriate tag """ with self.override_global_config(dict(analytics_enabled=False)): url = reverse('users-list') diff --git a/tests/contrib/patch.py b/tests/contrib/patch.py index 5ae648f2729..0f688098905 100644 --- a/tests/contrib/patch.py +++ b/tests/contrib/patch.py @@ -204,12 +204,12 @@ def assert_module_patched(self, module): So an appropriate assert_module_patched would look like:: - def assert_module_patched(self, redis): - self.assert_wrapped(redis.StrictRedis.execute_command) - self.assert_wrapped(redis.StrictRedis.pipeline) - self.assert_wrapped(redis.Redis.pipeline) - self.assert_wrapped(redis.client.BasePipeline.execute) - self.assert_wrapped(redis.client.BasePipeline.immediate_execute_command) + def assert_module_patched(self, redis): + self.assert_wrapped(redis.StrictRedis.execute_command) + self.assert_wrapped(redis.StrictRedis.pipeline) + self.assert_wrapped(redis.Redis.pipeline) + self.assert_wrapped(redis.client.BasePipeline.execute) + self.assert_wrapped(redis.client.BasePipeline.immediate_execute_command) :param module: module to check :return: None @@ -229,12 +229,12 @@ def assert_not_module_patched(self, module): So an appropriate assert_not_module_patched would look like:: - def assert_not_module_patched(self, redis): - self.assert_not_wrapped(redis.StrictRedis.execute_command) - self.assert_not_wrapped(redis.StrictRedis.pipeline) - self.assert_not_wrapped(redis.Redis.pipeline) - self.assert_not_wrapped(redis.client.BasePipeline.execute) - self.assert_not_wrapped(redis.client.BasePipeline.immediate_execute_command) + def assert_not_module_patched(self, redis): + self.assert_not_wrapped(redis.StrictRedis.execute_command) + self.assert_not_wrapped(redis.StrictRedis.pipeline) + self.assert_not_wrapped(redis.Redis.pipeline) + self.assert_not_wrapped(redis.client.BasePipeline.execute) + self.assert_not_wrapped(redis.client.BasePipeline.immediate_execute_command) :param module: :return: None @@ -254,12 +254,12 @@ def assert_not_module_double_patched(self, module): So an appropriate assert_not_module_double_patched would look like:: - def assert_not_module_double_patched(self, redis): - self.assert_not_double_wrapped(redis.StrictRedis.execute_command) - self.assert_not_double_wrapped(redis.StrictRedis.pipeline) - self.assert_not_double_wrapped(redis.Redis.pipeline) - self.assert_not_double_wrapped(redis.client.BasePipeline.execute) - self.assert_not_double_wrapped(redis.client.BasePipeline.immediate_execute_command) + def assert_not_module_double_patched(self, redis): + self.assert_not_double_wrapped(redis.StrictRedis.execute_command) + self.assert_not_double_wrapped(redis.StrictRedis.pipeline) + self.assert_not_double_wrapped(redis.Redis.pipeline) + self.assert_not_double_wrapped(redis.client.BasePipeline.execute) + self.assert_not_double_wrapped(redis.client.BasePipeline.immediate_execute_command) :param module: module to check :return: None diff --git a/tests/contrib/sqlalchemy/mixins.py b/tests/contrib/sqlalchemy/mixins.py index 42c94b51798..08a579481e6 100644 --- a/tests/contrib/sqlalchemy/mixins.py +++ b/tests/contrib/sqlalchemy/mixins.py @@ -42,11 +42,10 @@ class SQLAlchemyTestMixin(object): To support a new engine, create a new `TestCase` that inherits from `SQLAlchemyTestMixin` and `TestCase`. Then you must define the following static class variables: - * VENDOR: the database vendor name - * SQL_DB: the `sql.db` tag that we expect (it's the name of the database - available in the `.env` file) - * SERVICE: the service that we expect by default - * ENGINE_ARGS: all arguments required to create the engine + * VENDOR: the database vendor name + * SQL_DB: the `sql.db` tag that we expect (it's the name of the database available in the `.env` file) + * SERVICE: the service that we expect by default + * ENGINE_ARGS: all arguments required to create the engine To check specific tags in each test, you must implement the `check_meta(self, span)` method. diff --git a/tests/test_context.py b/tests/test_context.py index 468c04960c9..3664d440bdb 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -216,9 +216,9 @@ def test_get_report_hostname_default(self, get_hostname): def test_partial_flush(self): """ When calling `Context.get` - When partial flushing is enabled - When we have just enough finished spans to flush - We return the finished spans + When partial flushing is enabled + When we have just enough finished spans to flush + We return the finished spans """ tracer = get_dummy_tracer() ctx = Context() @@ -255,9 +255,9 @@ def test_partial_flush(self): def test_partial_flush_too_many(self): """ When calling `Context.get` - When partial flushing is enabled - When we have more than the minimum number of spans needed to flush - We return the finished spans + When partial flushing is enabled + When we have more than the minimum number of spans needed to flush + We return the finished spans """ tracer = get_dummy_tracer() ctx = Context() @@ -294,9 +294,9 @@ def test_partial_flush_too_many(self): def test_partial_flush_too_few(self): """ When calling `Context.get` - When partial flushing is enabled - When we do not have enough finished spans to flush - We return no spans + When partial flushing is enabled + When we do not have enough finished spans to flush + We return no spans """ tracer = get_dummy_tracer() ctx = Context() @@ -327,9 +327,9 @@ def test_partial_flush_too_few(self): def test_partial_flush_remaining(self): """ When calling `Context.get` - When partial flushing is enabled - When we have some unfinished spans - We keep the unfinished spans around + When partial flushing is enabled + When we have some unfinished spans + We keep the unfinished spans around """ tracer = get_dummy_tracer() ctx = Context() diff --git a/tests/utils/span.py b/tests/utils/span.py index af11fe1aa06..ab4cb97aeeb 100644 --- a/tests/utils/span.py +++ b/tests/utils/span.py @@ -61,15 +61,15 @@ def __eq__(self, other): def matches(self, **kwargs): """ - Helper function to check if this span's properties matches the expected + Helper function to check if this span's properties matches the expected. Example:: span = TestSpan(span) span.matches(name='my.span', resource='GET /') - :param **kwargs: Property/Value pairs to evaluate on this span - :type **kwargs: dict + :param kwargs: Property/Value pairs to evaluate on this span + :type kwargs: dict :returns: True if the arguments passed match, False otherwise :rtype: bool """ @@ -123,8 +123,8 @@ def assert_matches(self, **kwargs): span = TestSpan(span) span.assert_matches(name='my.span') - :param **kwargs: Property/Value pairs to evaluate on this span - :type **kwargs: dict + :param kwargs: Property/Value pairs to evaluate on this span + :type kwargs: dict :raises: AssertionError """ for name, value in kwargs.items(): @@ -297,12 +297,12 @@ def filter_spans(self, *args, **kwargs): """ Helper to filter current spans by provided parameters. - This function will yield all spans whose `TestSpan.matches` function return `True` + This function will yield all spans whose `TestSpan.matches` function return `True`. - :param *args: Positional arguments to pass to :meth:`tests.utils.span.TestSpan.matches` - :type *args: list - :param *kwargs: Keyword arguments to pass to :meth:`tests.utils.span.TestSpan.matches` - :type **kwargs: dict + :param args: Positional arguments to pass to :meth:`tests.utils.span.TestSpan.matches` + :type args: list + :param kwargs: Keyword arguments to pass to :meth:`tests.utils.span.TestSpan.matches` + :type kwargs: dict :returns: generator for the matched :class:`tests.utils.span.TestSpan` :rtype: generator """ @@ -318,12 +318,12 @@ def find_span(self, *args, **kwargs): """ Find a single span matches the provided filter parameters. - This function will find the first span whose `TestSpan.matches` function return `True` + This function will find the first span whose `TestSpan.matches` function return `True`. - :param *args: Positional arguments to pass to :meth:`tests.utils.span.TestSpan.matches` - :type *args: list - :param *kwargs: Keyword arguments to pass to :meth:`tests.utils.span.TestSpan.matches` - :type **kwargs: dict + :param args: Positional arguments to pass to :meth:`tests.utils.span.TestSpan.matches` + :type args: list + :param kwargs: Keyword arguments to pass to :meth:`tests.utils.span.TestSpan.matches` + :type kwargs: dict :returns: The first matching span :rtype: :class:`tests.utils.span.TestSpan` """ diff --git a/tox.ini b/tox.ini index 29ad8d063b0..3ec1d3a0011 100644 --- a/tox.ini +++ b/tox.ini @@ -458,6 +458,9 @@ deps= flake8-blind-except flake8-builtins flake8-logging-format + flake8-rst-docstrings + # needed for some features from flake8-rst-docstrings + pygments commands=flake8 . basepython=python3.7 @@ -765,3 +768,5 @@ exclude= # G201 Logging: .exception(...) should be used instead of .error(..., exc_info=True) ignore = W504,A003,G201 enable-extensions=G +rst-roles = class,meth,obj,ref +rst-directives = py:data From 8cc13aaebfc7bb85c1710176437b4b6d6c1f13e3 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 13 Nov 2019 14:41:55 +0100 Subject: [PATCH 08/81] flake8: enable flake8-docstrings --- ddtrace/contrib/asyncio/helpers.py | 19 ++++++++----------- ddtrace/contrib/boto/patch.py | 8 +++----- ddtrace/contrib/django/middleware.py | 4 +++- ddtrace/contrib/pyramid/__init__.py | 2 +- ddtrace/contrib/tornado/__init__.py | 2 +- ddtrace/filters.py | 4 ++-- ddtrace/utils/formats.py | 6 ++++-- tests/contrib/patch.py | 5 +---- .../contrib/pyramid/test_pyramid_autopatch.py | 3 +-- tox.ini | 4 +++- 10 files changed, 27 insertions(+), 30 deletions(-) diff --git a/ddtrace/contrib/asyncio/helpers.py b/ddtrace/contrib/asyncio/helpers.py index 65f00720fb1..fadd9a58e5b 100644 --- a/ddtrace/contrib/asyncio/helpers.py +++ b/ddtrace/contrib/asyncio/helpers.py @@ -23,11 +23,9 @@ def set_call_context(task, ctx): def ensure_future(coro_or_future, *, loop=None, tracer=None): - """ - Wrapper for the asyncio.ensure_future() function that - sets a context to the newly created Task. If the current - task already has a Context, it will be attached to the - new Task so the Trace list will be preserved. + """Wrapper that sets a context to the newly created Task. + + If the current task already has a Context, it will be attached to the new Task so the Trace list will be preserved. """ tracer = tracer or ddtrace.tracer current_ctx = tracer.get_call_context() @@ -37,12 +35,10 @@ def ensure_future(coro_or_future, *, loop=None, tracer=None): def run_in_executor(loop, executor, func, *args, tracer=None): - """ - Wrapper for the loop.run_in_executor() function that - sets a context to the newly created Thread. If the current - task has a Context, it will be attached as an empty Context - with the current_span activated to inherit the ``trace_id`` - and the ``parent_id``. + """Wrapper function that sets a context to the newly created Thread. + + If the current task has a Context, it will be attached as an empty Context with the current_span activated to + inherit the ``trace_id`` and the ``parent_id``. Because the Executor can run the Thread immediately or after the coroutine is executed, we may have two different scenarios: @@ -53,6 +49,7 @@ def run_in_executor(loop, executor, func, *args, tracer=None): To support both situations, we create a new Context that knows only what was the latest active Span when the new thread was created. In this new thread, we fallback to the thread-local ``Context`` storage. + """ tracer = tracer or ddtrace.tracer ctx = Context() diff --git a/ddtrace/contrib/boto/patch.py b/ddtrace/contrib/boto/patch.py index 62911238dab..0814ac3f849 100644 --- a/ddtrace/contrib/boto/patch.py +++ b/ddtrace/contrib/boto/patch.py @@ -28,15 +28,13 @@ def patch(): - - """ AWSQueryConnection and AWSAuthConnection are two different classes called by - different services for connection. For exemple EC2 uses AWSQueryConnection and - S3 uses AWSAuthConnection - """ if getattr(boto.connection, '_datadog_patch', False): return setattr(boto.connection, '_datadog_patch', True) + # AWSQueryConnection and AWSAuthConnection are two different classes called by + # different services for connection. + # For exemple EC2 uses AWSQueryConnection and S3 uses AWSAuthConnection wrapt.wrap_function_wrapper( 'boto.connection', 'AWSQueryConnection.make_request', patched_query_request ) diff --git a/ddtrace/contrib/django/middleware.py b/ddtrace/contrib/django/middleware.py index 9325569e41f..f1185cc9970 100644 --- a/ddtrace/contrib/django/middleware.py +++ b/ddtrace/contrib/django/middleware.py @@ -49,7 +49,9 @@ def _analytics_enabled(): def get_middleware_insertion_point(): """Returns the attribute name and collection object for the Django middleware. - If middleware cannot be found, returns None for the middleware collection.""" + + If middleware cannot be found, returns None for the middleware collection. + """ middleware = getattr(django_settings, MIDDLEWARE, None) # Prioritise MIDDLEWARE over ..._CLASSES, but only in 1.10 and later. if middleware is not None and django.VERSION >= (1, 10): diff --git a/ddtrace/contrib/pyramid/__init__.py b/ddtrace/contrib/pyramid/__init__.py index cc61eacf9f7..d7a012f42b2 100644 --- a/ddtrace/contrib/pyramid/__init__.py +++ b/ddtrace/contrib/pyramid/__init__.py @@ -1,4 +1,4 @@ -"""To trace requests from a Pyramid application, trace your application +r"""To trace requests from a Pyramid application, trace your application config:: diff --git a/ddtrace/contrib/tornado/__init__.py b/ddtrace/contrib/tornado/__init__.py index e19ba599263..d81bc7d4cf4 100644 --- a/ddtrace/contrib/tornado/__init__.py +++ b/ddtrace/contrib/tornado/__init__.py @@ -1,4 +1,4 @@ -""" +r""" The Tornado integration traces all ``RequestHandler`` defined in a Tornado web application. Auto instrumentation is available using the ``patch`` function that **must be called before** importing the tornado library. diff --git a/ddtrace/filters.py b/ddtrace/filters.py index 9d2ebde015c..37f3ea02cbd 100644 --- a/ddtrace/filters.py +++ b/ddtrace/filters.py @@ -4,7 +4,8 @@ class FilterRequestsOnUrl(object): - """Filter out traces from incoming http requests based on the request's url. + r"""Filter out traces from incoming http requests based on the request's url. + This class takes as argument a list of regular expression patterns representing the urls to be excluded from tracing. A trace will be excluded if its root span contains a ``http.url`` tag and if this tag matches any of @@ -15,7 +16,6 @@ class FilterRequestsOnUrl(object): the urls that should be filtered out. Examples: - To filter out http calls to domain api.example.com:: FilterRequestsOnUrl(r'http://api\\.example\\.com') diff --git a/ddtrace/utils/formats.py b/ddtrace/utils/formats.py index 5a1a4001f8b..5e065ece56c 100644 --- a/ddtrace/utils/formats.py +++ b/ddtrace/utils/formats.py @@ -58,8 +58,10 @@ def deep_getattr(obj, attr_string, default=None): def asbool(value): - """Convert the given String to a boolean object. Accepted - values are `True` and `1`.""" + """Convert the given String to a boolean object. + + Accepted values are `True` and `1`. + """ if value is None: return False diff --git a/tests/contrib/patch.py b/tests/contrib/patch.py index 0f688098905..e9d73617786 100644 --- a/tests/contrib/patch.py +++ b/tests/contrib/patch.py @@ -85,9 +85,7 @@ class PatchTestCase(object): """ @run_in_subprocess class Base(SubprocessTestCase, PatchMixin): - """PatchTestCase provides default test methods to be used for testing - common integration patching logic. - + """Provides default test methods to be used for testing common integration patching logic. Each test method provides a default implementation which will use the provided attributes (described below). If the attributes are not provided a NotImplementedError will be raised for each method that is @@ -99,7 +97,6 @@ class Base(SubprocessTestCase, PatchMixin): __unpatch_func__ unpatch function from the integration. Example: - A simple implementation inheriting this TestCase looks like:: from ddtrace.contrib.redis import unpatch diff --git a/tests/contrib/pyramid/test_pyramid_autopatch.py b/tests/contrib/pyramid/test_pyramid_autopatch.py index 73c2a5f0e40..0e4c8c39bda 100644 --- a/tests/contrib/pyramid/test_pyramid_autopatch.py +++ b/tests/contrib/pyramid/test_pyramid_autopatch.py @@ -44,7 +44,6 @@ def _include_me(config): def test_config_include(): - """ This test makes sure that relative imports still work when the - application is run with ddtrace-run """ + """Makes sure that relative imports still work when the application is run with ddtrace-run.""" config = Configurator() config.include('tests.contrib.pyramid._include_me') diff --git a/tox.ini b/tox.ini index 3ec1d3a0011..a26e64a986a 100644 --- a/tox.ini +++ b/tox.ini @@ -457,6 +457,7 @@ deps= flake8>=3.7,<=3.8 flake8-blind-except flake8-builtins + flake8-docstrings flake8-logging-format flake8-rst-docstrings # needed for some features from flake8-rst-docstrings @@ -766,7 +767,8 @@ exclude= # Ignore: # A003: XXX is a python builtin, consider renaming the class attribute # G201 Logging: .exception(...) should be used instead of .error(..., exc_info=True) -ignore = W504,A003,G201 +# We ignore most of the D errors because there are too many; the goal is to fix them eventually +ignore = W504,A003,G201,D100,D101,D102,D103,D104,D105,D106,D107,D200,D202,D204,D205,D208,D210,D300,D400,D401,D403,D413 enable-extensions=G rst-roles = class,meth,obj,ref rst-directives = py:data From 7bc0f2189ace3d1ecd5c023f561c73b0353811a2 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 15 Nov 2019 08:41:33 -0500 Subject: [PATCH 09/81] internal: use histogram and counters for metrics (#1134) --- ddtrace/internal/writer.py | 31 +++++++++++++---------- tests/internal/test_writer.py | 47 ++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 6109658679c..c64a8009587 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -107,24 +107,24 @@ def flush_queue(self): if self._send_stats: # Statistics about the queue length, size and number of spans self.dogstatsd.increment('datadog.tracer.flushes') - self.dogstatsd.histogram('datadog.tracer.flush.traces', traces_queue_length) - self.dogstatsd.histogram('datadog.tracer.flush.spans', traces_queue_spans) + self._histogram_with_total('datadog.tracer.flush.traces', traces_queue_length) + self._histogram_with_total('datadog.tracer.flush.spans', traces_queue_spans) # Statistics about the filtering - self.dogstatsd.histogram('datadog.tracer.flush.traces_filtered', traces_filtered) + self._histogram_with_total('datadog.tracer.flush.traces_filtered', traces_filtered) # Statistics about API - self.dogstatsd.histogram('datadog.tracer.api.requests', len(traces_responses)) - self.dogstatsd.histogram('datadog.tracer.api.errors', - len(list(t for t in traces_responses - if isinstance(t, Exception)))) + self._histogram_with_total('datadog.tracer.api.requests', len(traces_responses)) + + self._histogram_with_total('datadog.tracer.api.errors', + len(list(t for t in traces_responses if isinstance(t, Exception)))) for status, grouped_responses in itertools.groupby( sorted((t for t in traces_responses if not isinstance(t, Exception)), key=lambda r: r.status), key=lambda r: r.status): - self.dogstatsd.histogram('datadog.tracer.api.responses', - len(list(grouped_responses)), - tags=['status:%d' % status]) + self._histogram_with_total('datadog.tracer.api.responses', + len(list(grouped_responses)), + tags=['status:%d' % status]) # Statistics about the writer thread if hasattr(time, 'thread_time'): @@ -133,6 +133,11 @@ def flush_queue(self): self._last_thread_time = new_thread_time self.dogstatsd.histogram('datadog.tracer.writer.cpu_time', diff) + def _histogram_with_total(self, name, value, tags=None): + """Helper to add metric as a histogram and with a `.total` counter""" + self.dogstatsd.histogram(name, value, tags=tags) + self.dogstatsd.increment('%s.total' % (name, ), value, tags=tags) + def run_periodic(self): if self._send_stats: self.dogstatsd.gauge('datadog.tracer.heartbeat', 1) @@ -146,9 +151,9 @@ def run_periodic(self): # Statistics about the rate at which spans are inserted in the queue dropped, enqueued, enqueued_lengths = self._trace_queue.reset_stats() self.dogstatsd.gauge('datadog.tracer.queue.max_length', self._trace_queue.maxsize) - self.dogstatsd.histogram('datadog.tracer.queue.dropped.traces', dropped) - self.dogstatsd.histogram('datadog.tracer.queue.enqueued.traces', enqueued) - self.dogstatsd.histogram('datadog.tracer.queue.enqueued.spans', enqueued_lengths) + self.dogstatsd.increment('datadog.tracer.queue.dropped.traces', dropped) + self.dogstatsd.increment('datadog.tracer.queue.enqueued.traces', enqueued) + self.dogstatsd.increment('datadog.tracer.queue.enqueued.spans', enqueued_lengths) def on_shutdown(self): try: diff --git a/tests/internal/test_writer.py b/tests/internal/test_writer.py index ba5862bcdfa..69de80c3677 100644 --- a/tests/internal/test_writer.py +++ b/tests/internal/test_writer.py @@ -137,25 +137,29 @@ def test_dogstatsd(self): assert [ mock.call('datadog.tracer.flushes'), + mock.call('datadog.tracer.flush.traces.total', 11, tags=None), + mock.call('datadog.tracer.flush.spans.total', 77, tags=None), + mock.call('datadog.tracer.flush.traces_filtered.total', 0, tags=None), + mock.call('datadog.tracer.api.requests.total', 11, tags=None), + mock.call('datadog.tracer.api.errors.total', 0, tags=None), + mock.call('datadog.tracer.api.responses.total', 11, tags=['status:200']), + mock.call('datadog.tracer.queue.dropped.traces', 0), + mock.call('datadog.tracer.queue.enqueued.traces', 11), + mock.call('datadog.tracer.queue.enqueued.spans', 77), mock.call('datadog.tracer.shutdown'), ] == self.dogstatsd.increment.mock_calls histogram_calls = [ - mock.call('datadog.tracer.flush.traces', 11), - mock.call('datadog.tracer.flush.spans', 77), - mock.call('datadog.tracer.flush.traces_filtered', 0), - mock.call('datadog.tracer.api.requests', 11), - mock.call('datadog.tracer.api.errors', 0), + mock.call('datadog.tracer.flush.traces', 11, tags=None), + mock.call('datadog.tracer.flush.spans', 77, tags=None), + mock.call('datadog.tracer.flush.traces_filtered', 0, tags=None), + mock.call('datadog.tracer.api.requests', 11, tags=None), + mock.call('datadog.tracer.api.errors', 0, tags=None), mock.call('datadog.tracer.api.responses', 11, tags=['status:200']), ] if hasattr(time, 'thread_time'): histogram_calls.append(mock.call('datadog.tracer.writer.cpu_time', mock.ANY)) - histogram_calls += [ - mock.call('datadog.tracer.queue.dropped.traces', 0), - mock.call('datadog.tracer.queue.enqueued.traces', 11), - mock.call('datadog.tracer.queue.enqueued.spans', 77), - ] assert histogram_calls == self.dogstatsd.histogram.mock_calls def test_dogstatsd_failing_api(self): @@ -167,24 +171,27 @@ def test_dogstatsd_failing_api(self): assert [ mock.call('datadog.tracer.flushes'), + mock.call('datadog.tracer.flush.traces.total', 11, tags=None), + mock.call('datadog.tracer.flush.spans.total', 77, tags=None), + mock.call('datadog.tracer.flush.traces_filtered.total', 0, tags=None), + mock.call('datadog.tracer.api.requests.total', 1, tags=None), + mock.call('datadog.tracer.api.errors.total', 1, tags=None), + mock.call('datadog.tracer.queue.dropped.traces', 0), + mock.call('datadog.tracer.queue.enqueued.traces', 11), + mock.call('datadog.tracer.queue.enqueued.spans', 77), mock.call('datadog.tracer.shutdown'), ] == self.dogstatsd.increment.mock_calls histogram_calls = [ - mock.call('datadog.tracer.flush.traces', 11), - mock.call('datadog.tracer.flush.spans', 77), - mock.call('datadog.tracer.flush.traces_filtered', 0), - mock.call('datadog.tracer.api.requests', 1), - mock.call('datadog.tracer.api.errors', 1), + mock.call('datadog.tracer.flush.traces', 11, tags=None), + mock.call('datadog.tracer.flush.spans', 77, tags=None), + mock.call('datadog.tracer.flush.traces_filtered', 0, tags=None), + mock.call('datadog.tracer.api.requests', 1, tags=None), + mock.call('datadog.tracer.api.errors', 1, tags=None), ] if hasattr(time, 'thread_time'): histogram_calls.append(mock.call('datadog.tracer.writer.cpu_time', mock.ANY)) - histogram_calls += [ - mock.call('datadog.tracer.queue.dropped.traces', 0), - mock.call('datadog.tracer.queue.enqueued.traces', 11), - mock.call('datadog.tracer.queue.enqueued.spans', 77), - ] assert histogram_calls == self.dogstatsd.histogram.mock_calls From 2533ee9a78a45da92acf7bab1d4aedfcac7bffae Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 15 Nov 2019 19:00:47 -0500 Subject: [PATCH 10/81] internal: move health metrics enabled to config (#1135) * internal: move health metrics enabled to config * remove unused import --- ddtrace/internal/writer.py | 6 ++-- ddtrace/settings/config.py | 4 +++ tests/base/__init__.py | 3 ++ tests/internal/test_writer.py | 56 ++++++++++++++++++----------------- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index c64a8009587..83d9ccf2b5c 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -6,6 +6,7 @@ from .. import api from .. import _worker from ..internal.logger import get_logger +from ..settings import config from ..vendor import monotonic from ddtrace.vendor.six.moves.queue import Queue, Full, Empty @@ -22,8 +23,6 @@ class AgentWriter(_worker.PeriodicWorkerThread): QUEUE_PROCESSING_INTERVAL = 1 - _ENABLE_STATS = False - def __init__(self, hostname='localhost', port=8126, uds_path=None, https=False, shutdown_timeout=DEFAULT_TIMEOUT, filters=None, priority_sampler=None, @@ -58,13 +57,12 @@ def recreate(self): priority_sampler=self._priority_sampler, dogstatsd=self.dogstatsd, ) - writer._ENABLE_STATS = self._ENABLE_STATS return writer @property def _send_stats(self): """Determine if we're sending stats or not.""" - return self._ENABLE_STATS and self.dogstatsd + return bool(config.health_metrics_enabled and self.dogstatsd) def write(self, spans=None, services=None): if spans: diff --git a/ddtrace/settings/config.py b/ddtrace/settings/config.py index 88ac02ae013..67b14aa76a5 100644 --- a/ddtrace/settings/config.py +++ b/ddtrace/settings/config.py @@ -34,6 +34,10 @@ def __init__(self): get_env('trace', 'report_hostname', default=False) ) + self.health_metrics_enabled = asbool( + get_env('trace', 'health_metrics_enabled', default=False) + ) + def __getattr__(self, name): if name not in self._config: self._config[name] = IntegrationConfig(self, name) diff --git a/tests/base/__init__.py b/tests/base/__init__.py index 7c933f113e4..38eb1cbeb00 100644 --- a/tests/base/__init__.py +++ b/tests/base/__init__.py @@ -56,14 +56,17 @@ def override_global_config(values): # DEV: Uses dict as interface but internally handled as attributes on Config instance analytics_enabled_original = ddtrace.config.analytics_enabled report_hostname_original = ddtrace.config.report_hostname + health_metrics_enabled_original = ddtrace.config.health_metrics_enabled ddtrace.config.analytics_enabled = values.get('analytics_enabled', analytics_enabled_original) ddtrace.config.report_hostname = values.get('report_hostname', report_hostname_original) + ddtrace.config.health_metrics_enabled = values.get('health_metrics_enabled', health_metrics_enabled_original) try: yield finally: ddtrace.config.analytics_enabled = analytics_enabled_original ddtrace.config.report_hostname = report_hostname_original + ddtrace.config.health_metrics_enabled = health_metrics_enabled_original @staticmethod @contextlib.contextmanager diff --git a/tests/internal/test_writer.py b/tests/internal/test_writer.py index 69de80c3677..3f28dae1ec3 100644 --- a/tests/internal/test_writer.py +++ b/tests/internal/test_writer.py @@ -1,5 +1,4 @@ import time -from unittest import TestCase import pytest @@ -8,6 +7,7 @@ from ddtrace.span import Span from ddtrace.api import API from ddtrace.internal.writer import AgentWriter, Q, Empty +from ..base import BaseTestCase class RemoveAllFilter(): @@ -64,34 +64,36 @@ def send_traces(traces): return [Exception('oops')] -class AgentWriterTests(TestCase): +class AgentWriterTests(BaseTestCase): N_TRACES = 11 def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False): - self.dogstatsd = mock.Mock() - worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters) - worker._ENABLE_STATS = enable_stats - worker._STATS_EVERY_INTERVAL = 1 - self.api = api_class() - worker.api = self.api - for i in range(self.N_TRACES): - worker.write([ - Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) - for j in range(7) - ]) - worker.stop() - worker.join() - return worker - - def test_recreate_stats(self): - worker = self.create_worker() - assert worker._ENABLE_STATS is False - new_worker = worker.recreate() - assert new_worker._ENABLE_STATS is False - - worker._ENABLE_STATS = True - new_worker = worker.recreate() - assert new_worker._ENABLE_STATS is True + with self.override_global_config(dict(health_metrics_enabled=enable_stats)): + self.dogstatsd = mock.Mock() + worker = AgentWriter(dogstatsd=self.dogstatsd, filters=filters) + worker._STATS_EVERY_INTERVAL = 1 + self.api = api_class() + worker.api = self.api + for i in range(self.N_TRACES): + worker.write([ + Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) + for j in range(7) + ]) + worker.stop() + worker.join() + return worker + + def test_send_stats(self): + dogstatsd = mock.Mock() + worker = AgentWriter(dogstatsd=dogstatsd) + assert worker._send_stats is False + with self.override_global_config(dict(health_metrics_enabled=True)): + assert worker._send_stats is True + + worker = AgentWriter(dogstatsd=None) + assert worker._send_stats is False + with self.override_global_config(dict(health_metrics_enabled=True)): + assert worker._send_stats is False def test_filters_keep_all(self): filtr = KeepAllFilter() @@ -124,7 +126,7 @@ def test_filters_short_circuit(self): def test_no_dogstats(self): worker = self.create_worker() - assert worker._ENABLE_STATS is False + assert worker._send_stats is False assert [ ] == self.dogstatsd.gauge.mock_calls From e1b0893af2a967bd98912df6f65490d05d70a180 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 18 Nov 2019 13:26:15 -0500 Subject: [PATCH 11/81] ci: add black step to ci (#1137) * ci: add black step to ci * also ignore docs/ --- .circleci/config.yml | 64 ++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 1 + pyproject.toml | 36 +++++++++++++++++++++++++ tox.ini | 6 +++++ 4 files changed, 107 insertions(+) create mode 100644 pyproject.toml diff --git a/.circleci/config.yml b/.circleci/config.yml index de13b1e4510..c1f02b59319 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,17 @@ persist_to_workspace_step: &persist_to_workspace_step jobs: + black: + docker: + - *test_runner + resource_class: *resource_class + steps: + - checkout + - *restore_cache_step + - run: tox -e 'black' --result-json /tmp/black.results + - *persist_to_workspace_step + - *save_cache_step + flake8: docker: - *test_runner @@ -822,165 +833,218 @@ workflows: test: jobs: - build_docs + - black - flake8 - test_build - aiobotocore: requires: - flake8 + - black - aiohttp: requires: - flake8 + - black - aiopg: requires: - flake8 + - black - asyncio: requires: - flake8 + - black - algoliasearch: requires: - flake8 + - black - benchmarks: requires: - flake8 + - black - boto: requires: - flake8 + - black - bottle: requires: - flake8 + - black - cassandra: requires: - flake8 + - black - celery: requires: - flake8 + - black - consul: requires: - flake8 + - black - dbapi: requires: - flake8 + - black - ddtracerun: requires: - flake8 + - black - django: requires: - flake8 + - black - elasticsearch: requires: - flake8 + - black - falcon: requires: - flake8 + - black - flask: requires: - flake8 + - black - futures: requires: - flake8 + - black - gevent: requires: - flake8 + - black - grpc: requires: - flake8 + - black - httplib: requires: - flake8 + - black - integration: requires: - flake8 + - black - internal: requires: - flake8 + - black - jinja2: requires: - flake8 + - black - kombu: requires: - flake8 + - black - mako: requires: - flake8 + - black - molten: requires: - flake8 + - black - mongoengine: requires: - flake8 + - black - mysqlconnector: requires: - flake8 + - black - mysqldb: requires: - flake8 + - black - mysqlpython: requires: - flake8 + - black - opentracer: requires: - flake8 + - black - psycopg: requires: - flake8 + - black - pylibmc: requires: - flake8 + - black - pylons: requires: - flake8 + - black - pymemcache: requires: - flake8 + - black - pymongo: requires: - flake8 + - black - pymysql: requires: - flake8 + - black - pyramid: requires: - flake8 + - black - redis: requires: - flake8 + - black - rediscluster: requires: - flake8 + - black - requests: requires: - flake8 + - black - requestsgevent: requires: - flake8 + - black - sqlalchemy: requires: - flake8 + - black - sqlite3: requires: - flake8 + - black - test_utils: requires: - flake8 + - black - test_logging: requires: - flake8 + - black - tornado: requires: - flake8 + - black - tracer: requires: - flake8 + - black - unit_tests: requires: - flake8 + - black - vertica: requires: - flake8 + - black - wait_all_tests: requires: # Initial jobs - build_docs + - black - flake8 - test_build diff --git a/docker-compose.yml b/docker-compose.yml index a33dbe6629a..17d65aeb260 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,6 +111,7 @@ services: - ./setup.py:/src/setup.py:ro - ./conftest.py:/src/conftest.py:ro - ./tox.ini:/src/tox.ini:ro + - ./pyproject.toml:/src/pyproject.toml:ro - ./.ddtox:/src/.tox - ./scripts:/src/scripts # setuptools_scm needs `.git` to figure out what version we are on diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..fe6974e3afd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[tool.black] +line-length = 120 +target_version = ['py27', 'py34', 'py35', 'py36', 'py37', 'py38'] +exclude = ''' +( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.nox + | \.tox + | \.venv + | _build/ + | buck-out/ + | build/ + | dist/ + | ddtrace/( + [^/]+\.py + | bootstrap/ + | commands/ + | contrib/ + | ext/ + | http/ + | internal/ + | opentracer/ + | propagation/ + | settings/ + | utils/ + | vendor/ + ) + | docs/ + | conftest.py + | setup.py + | tests/ +) +''' diff --git a/tox.ini b/tox.ini index daf8a244abc..22138d3d805 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ # - https://github.com/pypa/virtualenv/issues/596 envlist = flake8 + black wait {py27,py34,py35,py36,py37}-tracer {py27,py34,py35,py36,py37}-internal @@ -452,6 +453,11 @@ deps= # this is somewhat flaky (can fail and still be up) so try the tests anyway ignore_outcome=true +[testenv:black] +deps=black +commands=black --check . +basepython=python3.7 + [testenv:flake8] deps= flake8>=3.7,<=3.8 From 2b0d396fc7f76582e8ffedff48933245a77ebaf2 Mon Sep 17 00:00:00 2001 From: alrex Date: Mon, 18 Nov 2019 12:22:51 -0800 Subject: [PATCH 12/81] Fixing sqlalchemy test failures (#1138) Version 8.0.18 is causing segfaults in sqlalchemy tests. Signed-off-by: Alex Boten --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 22138d3d805..1d8be64ff46 100644 --- a/tox.ini +++ b/tox.ini @@ -287,7 +287,7 @@ deps = mongoengine017: mongoengine>=0.17<0.18 mongoengine018: mongoengine>=0.18<0.19 mongoenginelatest: mongoengine>=0.18 - mysqlconnector: mysql-connector-python + mysqlconnector: mysql-connector-python!=8.0.18 mysqldb12: mysql-python>=1.2,<1.3 mysqlclient13: mysqlclient>=1.3,<1.4 # webob is required for Pylons < 1.0 @@ -444,7 +444,7 @@ basepython=python deps= cassandra-driver psycopg2 - mysql-connector-python + mysql-connector-python!=8.0.18 redis-py-cluster>=1.3.6,<1.4.0 vertica-python>=0.6.0,<0.7.0 kombu>=4.2.0,<4.3.0 From e73da927ad094db830a671590a52f594044e64eb Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Fri, 22 Nov 2019 11:38:33 -0500 Subject: [PATCH 13/81] internal: use debtcollector for keyword argument warnings (#1147) --- ddtrace/tracer.py | 9 +- ddtrace/vendor/__init__.py | 12 + ddtrace/vendor/debtcollector/__init__.py | 45 +++ ddtrace/vendor/debtcollector/_utils.py | 180 ++++++++++++ ddtrace/vendor/debtcollector/moves.py | 197 +++++++++++++ ddtrace/vendor/debtcollector/removals.py | 334 +++++++++++++++++++++++ ddtrace/vendor/debtcollector/renames.py | 45 +++ ddtrace/vendor/debtcollector/updating.py | 68 +++++ tests/test_tracer.py | 10 +- 9 files changed, 893 insertions(+), 7 deletions(-) create mode 100644 ddtrace/vendor/debtcollector/__init__.py create mode 100644 ddtrace/vendor/debtcollector/_utils.py create mode 100644 ddtrace/vendor/debtcollector/moves.py create mode 100644 ddtrace/vendor/debtcollector/removals.py create mode 100644 ddtrace/vendor/debtcollector/renames.py create mode 100644 ddtrace/vendor/debtcollector/updating.py diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 7160d15bce7..85a70911410 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -2,6 +2,7 @@ import logging from os import environ, getpid +from ddtrace.vendor import debtcollector from .constants import FILTERS_KEY, SAMPLE_RATE_METRIC_KEY from .ext import system @@ -14,7 +15,7 @@ from .sampler import AllSampler, DatadogSampler, RateSampler, RateByServiceSampler from .span import Span from .utils.formats import get_env -from .utils.deprecation import deprecated, warn +from .utils.deprecation import deprecated, RemovedInDDTrace10Warning from .vendor.dogstatsd import DogStatsd from . import compat @@ -162,6 +163,10 @@ def context_provider(self): return self._context_provider # TODO: deprecate this method and make sure users create a new tracer if they need different parameters + @debtcollector.removals.removed_kwarg("dogstatsd_host", "Use `dogstatsd_url` instead", + category=RemovedInDDTrace10Warning) + @debtcollector.removals.removed_kwarg("dogstatsd_port", "Use `dogstatsd_url` instead", + category=RemovedInDDTrace10Warning) def configure(self, enabled=None, hostname=None, port=None, uds_path=None, https=None, sampler=None, context_provider=None, wrap_executor=None, priority_sampling=None, settings=None, collect_metrics=None, dogstatsd_host=None, dogstatsd_port=None, @@ -213,8 +218,6 @@ def configure(self, enabled=None, hostname=None, port=None, uds_path=None, https if dogstatsd_host is not None and dogstatsd_url is None: dogstatsd_url = 'udp://{}:{}'.format(dogstatsd_host, dogstatsd_port or self.DEFAULT_DOGSTATSD_PORT) - warn(('tracer.configure(): dogstatsd_host and dogstatsd_port are deprecated. ' - 'Use dogstatsd_url={!r}').format(dogstatsd_url)) if dogstatsd_url is not None: dogstatsd_kwargs = _parse_dogstatsd_url(dogstatsd_url) diff --git a/ddtrace/vendor/__init__.py b/ddtrace/vendor/__init__.py index 2ebb5f926a8..d3d436403d2 100644 --- a/ddtrace/vendor/__init__.py +++ b/ddtrace/vendor/__init__.py @@ -72,6 +72,18 @@ The source `monotonic.py` was added as `monotonic/__init__.py` No other changes were made + +debtcollector +------------- + +Website: https://docs.openstack.org/debtcollector/latest/index.html +Source: https://github.com/openstack/debtcollector +Version: 1.22.0 +License: Apache License 2.0 + +Notes: + Removed dependency on `pbr` and manually set `__version__` + """ # Initialize `ddtrace.vendor.datadog.base.log` logger with our custom rate limited logger diff --git a/ddtrace/vendor/debtcollector/__init__.py b/ddtrace/vendor/debtcollector/__init__.py new file mode 100644 index 00000000000..2fc6fded42e --- /dev/null +++ b/ddtrace/vendor/debtcollector/__init__.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from . import _utils, moves, removals, renames, updating + +__version__ = "1.22.0" + + +def deprecate(prefix, postfix=None, message=None, + version=None, removal_version=None, + stacklevel=3, category=DeprecationWarning): + """Helper to deprecate some thing using generated message format. + + :param prefix: prefix string used as the prefix of the output message + :param postfix: postfix string used as the postfix of the output message + :param message: message string used as ending contents of the deprecate + message + :param version: version string (represents the version this + deprecation was created in) + :param removal_version: version string (represents the version this + deprecation will be removed in); a string of '?' + will denote this will be removed in some future + unknown version + :param stacklevel: stacklevel used in the :func:`warnings.warn` function + to locate where the users code is in the + :func:`warnings.warn` call + :param category: the :mod:`warnings` category to use, defaults to + :py:class:`DeprecationWarning` if not provided + """ + out_message = _utils.generate_message(prefix, postfix=postfix, + version=version, message=message, + removal_version=removal_version) + _utils.deprecation(out_message, stacklevel=stacklevel, + category=category) diff --git a/ddtrace/vendor/debtcollector/_utils.py b/ddtrace/vendor/debtcollector/_utils.py new file mode 100644 index 00000000000..80bada74c4c --- /dev/null +++ b/ddtrace/vendor/debtcollector/_utils.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import inspect +import types +import warnings + +import six + +try: + _TYPE_TYPE = types.TypeType +except AttributeError: + _TYPE_TYPE = type + + +# See: https://docs.python.org/2/library/__builtin__.html#module-__builtin__ +# and see https://docs.python.org/2/reference/executionmodel.html (and likely +# others)... +_BUILTIN_MODULES = ('builtins', '__builtin__', '__builtins__', 'exceptions') +_enabled = True + + +def deprecation(message, stacklevel=None, category=None): + """Warns about some type of deprecation that has been (or will be) made. + + This helper function makes it easier to interact with the warnings module + by standardizing the arguments that the warning function receives so that + it is easier to use. + + This should be used to emit warnings to users (users can easily turn these + warnings off/on, see https://docs.python.org/2/library/warnings.html + as they see fit so that the messages do not fill up the users logs with + warnings that they do not wish to see in production) about functions, + methods, attributes or other code that is deprecated and will be removed + in a future release (this is done using these warnings to avoid breaking + existing users of those functions, methods, code; which a library should + avoid doing by always giving at *least* N + 1 release for users to address + the deprecation warnings). + """ + if not _enabled: + return + if category is None: + category = DeprecationWarning + if stacklevel is None: + warnings.warn(message, category=category) + else: + warnings.warn(message, category=category, stacklevel=stacklevel) + + +def get_qualified_name(obj): + # Prefer the py3.x name (if we can get at it...) + try: + return (True, obj.__qualname__) + except AttributeError: + return (False, obj.__name__) + + +def generate_message(prefix, postfix=None, message=None, + version=None, removal_version=None): + """Helper to generate a common message 'style' for deprecation helpers.""" + message_components = [prefix] + if version: + message_components.append(" in version '%s'" % version) + if removal_version: + if removal_version == "?": + message_components.append(" and will be removed in a future" + " version") + else: + message_components.append(" and will be removed in version '%s'" + % removal_version) + if postfix: + message_components.append(postfix) + if message: + message_components.append(": %s" % message) + return ''.join(message_components) + + +def get_assigned(decorator): + """Helper to fix/workaround https://bugs.python.org/issue3445""" + if six.PY3: + return functools.WRAPPER_ASSIGNMENTS + else: + assigned = [] + for attr_name in functools.WRAPPER_ASSIGNMENTS: + if hasattr(decorator, attr_name): + assigned.append(attr_name) + return tuple(assigned) + + +def get_class_name(obj, fully_qualified=True): + """Get class name for object. + + If object is a type, fully qualified name of the type is returned. + Else, fully qualified name of the type of the object is returned. + For builtin types, just name is returned. + """ + if not isinstance(obj, six.class_types): + obj = type(obj) + try: + built_in = obj.__module__ in _BUILTIN_MODULES + except AttributeError: + pass + else: + if built_in: + return obj.__name__ + + if fully_qualified and hasattr(obj, '__module__'): + return '%s.%s' % (obj.__module__, obj.__name__) + else: + return obj.__name__ + + +def get_method_self(method): + """Gets the ``self`` object attached to this method (or none).""" + if not inspect.ismethod(method): + return None + try: + return six.get_method_self(method) + except AttributeError: + return None + + +def get_callable_name(function): + """Generate a name from callable. + + Tries to do the best to guess fully qualified callable name. + """ + method_self = get_method_self(function) + if method_self is not None: + # This is a bound method. + if isinstance(method_self, six.class_types): + # This is a bound class method. + im_class = method_self + else: + im_class = type(method_self) + try: + parts = (im_class.__module__, function.__qualname__) + except AttributeError: + parts = (im_class.__module__, im_class.__name__, function.__name__) + elif inspect.ismethod(function) or inspect.isfunction(function): + # This could be a function, a static method, a unbound method... + try: + parts = (function.__module__, function.__qualname__) + except AttributeError: + if hasattr(function, 'im_class'): + # This is a unbound method, which exists only in python 2.x + im_class = function.im_class + parts = (im_class.__module__, + im_class.__name__, function.__name__) + else: + parts = (function.__module__, function.__name__) + else: + im_class = type(function) + if im_class is _TYPE_TYPE: + im_class = function + try: + parts = (im_class.__module__, im_class.__qualname__) + except AttributeError: + parts = (im_class.__module__, im_class.__name__) + # When running under sphinx it appears this can be none? if so just + # don't include it... + mod, rest = (parts[0], parts[1:]) + if not mod: + return '.'.join(rest) + else: + return '.'.join(parts) diff --git a/ddtrace/vendor/debtcollector/moves.py b/ddtrace/vendor/debtcollector/moves.py new file mode 100644 index 00000000000..639181aa661 --- /dev/null +++ b/ddtrace/vendor/debtcollector/moves.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import inspect + +from ddtrace.vendor import six +from ddtrace.vendor import wrapt + +from . import _utils + +_KIND_MOVED_PREFIX_TPL = "%s '%s' has moved to '%s'" +_CLASS_MOVED_PREFIX_TPL = "Class '%s' has moved to '%s'" +_MOVED_CALLABLE_POSTFIX = "()" +_FUNC_MOVED_PREFIX_TPL = "Function '%s' has moved to '%s'" + + +def _moved_decorator(kind, new_attribute_name, message=None, + version=None, removal_version=None, stacklevel=3, + attr_postfix=None, category=None): + """Decorates a method/property that was moved to another location.""" + + def decorator(f): + fully_qualified, old_attribute_name = _utils.get_qualified_name(f) + if attr_postfix: + old_attribute_name += attr_postfix + + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + base_name = _utils.get_class_name(wrapped, fully_qualified=False) + if fully_qualified: + old_name = old_attribute_name + else: + old_name = ".".join((base_name, old_attribute_name)) + new_name = ".".join((base_name, new_attribute_name)) + prefix = _KIND_MOVED_PREFIX_TPL % (kind, old_name, new_name) + out_message = _utils.generate_message( + prefix, message=message, + version=version, removal_version=removal_version) + _utils.deprecation(out_message, stacklevel=stacklevel, + category=category) + return wrapped(*args, **kwargs) + + return wrapper(f) + + return decorator + + +def moved_function(new_func, old_func_name, old_module_name, + message=None, version=None, removal_version=None, + stacklevel=3, category=None): + """Deprecates a function that was moved to another location. + + This generates a wrapper around ``new_func`` that will emit a deprecation + warning when called. The warning message will include the new location + to obtain the function from. + """ + new_func_full_name = _utils.get_callable_name(new_func) + new_func_full_name += _MOVED_CALLABLE_POSTFIX + old_func_full_name = ".".join([old_module_name, old_func_name]) + old_func_full_name += _MOVED_CALLABLE_POSTFIX + prefix = _FUNC_MOVED_PREFIX_TPL % (old_func_full_name, new_func_full_name) + out_message = _utils.generate_message(prefix, + message=message, version=version, + removal_version=removal_version) + + @six.wraps(new_func, assigned=_utils.get_assigned(new_func)) + def old_new_func(*args, **kwargs): + _utils.deprecation(out_message, stacklevel=stacklevel, + category=category) + return new_func(*args, **kwargs) + + old_new_func.__name__ = old_func_name + old_new_func.__module__ = old_module_name + return old_new_func + + +class moved_read_only_property(object): + """Descriptor for read-only properties moved to another location. + + This works like the ``@property`` descriptor but can be used instead to + provide the same functionality and also interact with the :mod:`warnings` + module to warn when a property is accessed, so that users of those + properties can know that a previously read-only property at a prior + location/name has moved to another location/name. + + :param old_name: old attribute location/name + :param new_name: new attribute location/name + :param version: version string (represents the version this deprecation + was created in) + :param removal_version: version string (represents the version this + deprecation will be removed in); a string + of '?' will denote this will be removed in + some future unknown version + :param stacklevel: stacklevel used in the :func:`warnings.warn` function + to locate where the users code is when reporting the + deprecation call (the default being 3) + :param category: the :mod:`warnings` category to use, defaults to + :py:class:`DeprecationWarning` if not provided + """ + + def __init__(self, old_name, new_name, + version=None, removal_version=None, + stacklevel=3, category=None): + self._old_name = old_name + self._new_name = new_name + self._message = _utils.generate_message( + "Read-only property '%s' has moved" + " to '%s'" % (self._old_name, self._new_name), + version=version, removal_version=removal_version) + self._stacklevel = stacklevel + self._category = category + + def __get__(self, instance, owner): + _utils.deprecation(self._message, + stacklevel=self._stacklevel, + category=self._category) + # This handles the descriptor being applied on a + # instance or a class and makes both work correctly... + if instance is not None: + real_owner = instance + else: + real_owner = owner + return getattr(real_owner, self._new_name) + + +def moved_method(new_method_name, message=None, + version=None, removal_version=None, stacklevel=3, + category=None): + """Decorates an *instance* method that was moved to another location.""" + if not new_method_name.endswith(_MOVED_CALLABLE_POSTFIX): + new_method_name += _MOVED_CALLABLE_POSTFIX + return _moved_decorator('Method', new_method_name, message=message, + version=version, removal_version=removal_version, + stacklevel=stacklevel, + attr_postfix=_MOVED_CALLABLE_POSTFIX, + category=category) + + +def moved_property(new_attribute_name, message=None, + version=None, removal_version=None, stacklevel=3, + category=None): + """Decorates an *instance* property that was moved to another location.""" + return _moved_decorator('Property', new_attribute_name, message=message, + version=version, removal_version=removal_version, + stacklevel=stacklevel, category=category) + + +def moved_class(new_class, old_class_name, old_module_name, + message=None, version=None, removal_version=None, + stacklevel=3, category=None): + """Deprecates a class that was moved to another location. + + This creates a 'new-old' type that can be used for a + deprecation period that can be inherited from. This will emit warnings + when the old locations class is initialized, telling where the new and + improved location for the old class now is. + """ + + if not inspect.isclass(new_class): + _qual, type_name = _utils.get_qualified_name(type(new_class)) + raise TypeError("Unexpected class type '%s' (expected" + " class type only)" % type_name) + + old_name = ".".join((old_module_name, old_class_name)) + new_name = _utils.get_class_name(new_class) + prefix = _CLASS_MOVED_PREFIX_TPL % (old_name, new_name) + out_message = _utils.generate_message( + prefix, message=message, version=version, + removal_version=removal_version) + + def decorator(f): + + @six.wraps(f, assigned=_utils.get_assigned(f)) + def wrapper(self, *args, **kwargs): + _utils.deprecation(out_message, stacklevel=stacklevel, + category=category) + return f(self, *args, **kwargs) + + return wrapper + + old_class = type(old_class_name, (new_class,), {}) + old_class.__module__ = old_module_name + old_class.__init__ = decorator(old_class.__init__) + return old_class diff --git a/ddtrace/vendor/debtcollector/removals.py b/ddtrace/vendor/debtcollector/removals.py new file mode 100644 index 00000000000..e3ee031177f --- /dev/null +++ b/ddtrace/vendor/debtcollector/removals.py @@ -0,0 +1,334 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import inspect + +from ddtrace.vendor import six +from ddtrace.vendor import wrapt + +from . import _utils + + +def _get_qualified_name(obj): + return _utils.get_qualified_name(obj)[1] + + +def _fetch_first_result(fget, fset, fdel, apply_func, value_not_found=None): + """Fetch first non-none/empty result of applying ``apply_func``.""" + for f in filter(None, (fget, fset, fdel)): + result = apply_func(f) + if result: + return result + return value_not_found + + +class removed_property(object): + """Property descriptor that deprecates a property. + + This works like the ``@property`` descriptor but can be used instead to + provide the same functionality and also interact with the :mod:`warnings` + module to warn when a property is accessed, set and/or deleted. + + :param message: string used as ending contents of the deprecate message + :param version: version string (represents the version this deprecation + was created in) + :param removal_version: version string (represents the version this + deprecation will be removed in); a string + of '?' will denote this will be removed in + some future unknown version + :param stacklevel: stacklevel used in the :func:`warnings.warn` function + to locate where the users code is when reporting the + deprecation call (the default being 3) + :param category: the :mod:`warnings` category to use, defaults to + :py:class:`DeprecationWarning` if not provided + """ + + # Message templates that will be turned into real messages as needed. + _PROPERTY_GONE_TPLS = { + 'set': "Setting the '%s' property is deprecated", + 'get': "Reading the '%s' property is deprecated", + 'delete': "Deleting the '%s' property is deprecated", + } + + def __init__(self, fget=None, fset=None, fdel=None, doc=None, + stacklevel=3, category=DeprecationWarning, + version=None, removal_version=None, message=None): + self.fset = fset + self.fget = fget + self.fdel = fdel + self.stacklevel = stacklevel + self.category = category + self.version = version + self.removal_version = removal_version + self.message = message + if doc is None and inspect.isfunction(fget): + doc = getattr(fget, '__doc__', None) + self._message_cache = {} + self.__doc__ = doc + + def _fetch_message_from_cache(self, kind): + try: + out_message = self._message_cache[kind] + except KeyError: + prefix_tpl = self._PROPERTY_GONE_TPLS[kind] + prefix = prefix_tpl % _fetch_first_result( + self.fget, self.fset, self.fdel, _get_qualified_name, + value_not_found="???") + out_message = _utils.generate_message( + prefix, message=self.message, version=self.version, + removal_version=self.removal_version) + self._message_cache[kind] = out_message + return out_message + + def __call__(self, fget, **kwargs): + self.fget = fget + self.message = kwargs.get('message', self.message) + self.version = kwargs.get('version', self.version) + self.removal_version = kwargs.get('removal_version', + self.removal_version) + self.stacklevel = kwargs.get('stacklevel', self.stacklevel) + self.category = kwargs.get('category', self.category) + self.__doc__ = kwargs.get('doc', + getattr(fget, '__doc__', self.__doc__)) + # Regenerate all the messages... + self._message_cache.clear() + return self + + def __delete__(self, obj): + if self.fdel is None: + raise AttributeError("can't delete attribute") + out_message = self._fetch_message_from_cache('delete') + _utils.deprecation(out_message, stacklevel=self.stacklevel, + category=self.category) + self.fdel(obj) + + def __set__(self, obj, value): + if self.fset is None: + raise AttributeError("can't set attribute") + out_message = self._fetch_message_from_cache('set') + _utils.deprecation(out_message, stacklevel=self.stacklevel, + category=self.category) + self.fset(obj, value) + + def __get__(self, obj, value): + if obj is None: + return self + if self.fget is None: + raise AttributeError("unreadable attribute") + out_message = self._fetch_message_from_cache('get') + _utils.deprecation(out_message, stacklevel=self.stacklevel, + category=self.category) + return self.fget(obj) + + def getter(self, fget): + o = type(self)(fget, self.fset, self.fdel, self.__doc__) + o.message = self.message + o.version = self.version + o.stacklevel = self.stacklevel + o.removal_version = self.removal_version + o.category = self.category + return o + + def setter(self, fset): + o = type(self)(self.fget, fset, self.fdel, self.__doc__) + o.message = self.message + o.version = self.version + o.stacklevel = self.stacklevel + o.removal_version = self.removal_version + o.category = self.category + return o + + def deleter(self, fdel): + o = type(self)(self.fget, self.fset, fdel, self.__doc__) + o.message = self.message + o.version = self.version + o.stacklevel = self.stacklevel + o.removal_version = self.removal_version + o.category = self.category + return o + + +def remove(f=None, message=None, version=None, removal_version=None, + stacklevel=3, category=None): + """Decorates a function, method, or class to emit a deprecation warning + + Due to limitations of the wrapt library (and python) itself, if this + is applied to subclasses of metaclasses then it likely will not work + as expected. More information can be found at bug #1520397 to see if + this situation affects your usage of this *universal* decorator, for + this specific scenario please use :py:func:`.removed_class` instead. + + :param str message: A message to include in the deprecation warning + :param str version: Specify what version the removed function is present in + :param str removal_version: What version the function will be removed. If + '?' is used this implies an undefined future + version + :param int stacklevel: How many entries deep in the call stack before + ignoring + :param type category: warnings message category (this defaults to + ``DeprecationWarning`` when none is provided) + """ + if f is None: + return functools.partial(remove, message=message, + version=version, + removal_version=removal_version, + stacklevel=stacklevel, + category=category) + + @wrapt.decorator + def wrapper(f, instance, args, kwargs): + qualified, f_name = _utils.get_qualified_name(f) + if qualified: + if inspect.isclass(f): + prefix_pre = "Using class" + thing_post = '' + else: + prefix_pre = "Using function/method" + thing_post = '()' + if not qualified: + prefix_pre = "Using function/method" + base_name = None + if instance is None: + # Decorator was used on a class + if inspect.isclass(f): + prefix_pre = "Using class" + thing_post = '' + module_name = _get_qualified_name(inspect.getmodule(f)) + if module_name == '__main__': + f_name = _utils.get_class_name( + f, fully_qualified=False) + else: + f_name = _utils.get_class_name( + f, fully_qualified=True) + # Decorator was a used on a function + else: + thing_post = '()' + module_name = _get_qualified_name(inspect.getmodule(f)) + if module_name != '__main__': + f_name = _utils.get_callable_name(f) + # Decorator was used on a classmethod or instancemethod + else: + thing_post = '()' + base_name = _utils.get_class_name(instance, + fully_qualified=False) + if base_name: + thing_name = ".".join([base_name, f_name]) + else: + thing_name = f_name + else: + thing_name = f_name + if thing_post: + thing_name += thing_post + prefix = prefix_pre + " '%s' is deprecated" % (thing_name) + out_message = _utils.generate_message( + prefix, + version=version, + removal_version=removal_version, + message=message) + _utils.deprecation(out_message, + stacklevel=stacklevel, category=category) + return f(*args, **kwargs) + return wrapper(f) + + +def removed_kwarg(old_name, message=None, + version=None, removal_version=None, stacklevel=3, + category=None): + """Decorates a kwarg accepting function to deprecate a removed kwarg.""" + + prefix = "Using the '%s' argument is deprecated" % old_name + out_message = _utils.generate_message( + prefix, postfix=None, message=message, version=version, + removal_version=removal_version) + + @wrapt.decorator + def wrapper(f, instance, args, kwargs): + if old_name in kwargs: + _utils.deprecation(out_message, + stacklevel=stacklevel, category=category) + return f(*args, **kwargs) + + return wrapper + + +def removed_class(cls_name, replacement=None, message=None, + version=None, removal_version=None, stacklevel=3, + category=None): + """Decorates a class to denote that it will be removed at some point.""" + + def _wrap_it(old_init, out_message): + + @six.wraps(old_init, assigned=_utils.get_assigned(old_init)) + def new_init(self, *args, **kwargs): + _utils.deprecation(out_message, stacklevel=stacklevel, + category=category) + return old_init(self, *args, **kwargs) + + return new_init + + def _check_it(cls): + if not inspect.isclass(cls): + _qual, type_name = _utils.get_qualified_name(type(cls)) + raise TypeError("Unexpected class type '%s' (expected" + " class type only)" % type_name) + + def _cls_decorator(cls): + _check_it(cls) + out_message = _utils.generate_message( + "Using class '%s' (either directly or via inheritance)" + " is deprecated" % cls_name, postfix=None, message=message, + version=version, removal_version=removal_version) + cls.__init__ = _wrap_it(cls.__init__, out_message) + return cls + + return _cls_decorator + + +def removed_module(module, replacement=None, message=None, + version=None, removal_version=None, stacklevel=3, + category=None): + """Helper to be called inside a module to emit a deprecation warning + + :param str replacment: A location (or information about) of any potential + replacement for the removed module (if applicable) + :param str message: A message to include in the deprecation warning + :param str version: Specify what version the removed module is present in + :param str removal_version: What version the module will be removed. If + '?' is used this implies an undefined future + version + :param int stacklevel: How many entries deep in the call stack before + ignoring + :param type category: warnings message category (this defaults to + ``DeprecationWarning`` when none is provided) + """ + if inspect.ismodule(module): + module_name = _get_qualified_name(module) + elif isinstance(module, six.string_types): + module_name = module + else: + _qual, type_name = _utils.get_qualified_name(type(module)) + raise TypeError("Unexpected module type '%s' (expected string or" + " module type only)" % type_name) + prefix = "The '%s' module usage is deprecated" % module_name + if replacement: + postfix = ", please use %s instead" % replacement + else: + postfix = None + out_message = _utils.generate_message(prefix, + postfix=postfix, message=message, + version=version, + removal_version=removal_version) + _utils.deprecation(out_message, + stacklevel=stacklevel, category=category) diff --git a/ddtrace/vendor/debtcollector/renames.py b/ddtrace/vendor/debtcollector/renames.py new file mode 100644 index 00000000000..0a34f72465f --- /dev/null +++ b/ddtrace/vendor/debtcollector/renames.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ddtrace.vendor import wrapt + +from . import _utils + +_KWARG_RENAMED_POSTFIX_TPL = ", please use the '%s' argument instead" +_KWARG_RENAMED_PREFIX_TPL = "Using the '%s' argument is deprecated" + + +def renamed_kwarg(old_name, new_name, message=None, + version=None, removal_version=None, stacklevel=3, + category=None, replace=False): + """Decorates a kwarg accepting function to deprecate a renamed kwarg.""" + + prefix = _KWARG_RENAMED_PREFIX_TPL % old_name + postfix = _KWARG_RENAMED_POSTFIX_TPL % new_name + out_message = _utils.generate_message( + prefix, postfix=postfix, message=message, version=version, + removal_version=removal_version) + + @wrapt.decorator + def decorator(wrapped, instance, args, kwargs): + if old_name in kwargs: + _utils.deprecation(out_message, + stacklevel=stacklevel, category=category) + if replace: + kwargs.setdefault(new_name, kwargs.pop(old_name)) + return wrapped(*args, **kwargs) + + return decorator diff --git a/ddtrace/vendor/debtcollector/updating.py b/ddtrace/vendor/debtcollector/updating.py new file mode 100644 index 00000000000..3055563a151 --- /dev/null +++ b/ddtrace/vendor/debtcollector/updating.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ddtrace.vendor import six +from ddtrace.vendor import wrapt +if six.PY3: + import inspect + Parameter = inspect.Parameter + Signature = inspect.Signature + get_signature = inspect.signature +else: + # Provide an equivalent but use funcsigs instead... + import funcsigs + Parameter = funcsigs.Parameter + Signature = funcsigs.Signature + get_signature = funcsigs.signature + +from . import _utils + +_KWARG_UPDATED_POSTFIX_TPL = (', please update the code to explicitly set %s ' + 'as the value') +_KWARG_UPDATED_PREFIX_TPL = ('The %s argument is changing its default value ' + 'to %s') + + +def updated_kwarg_default_value(name, old_value, new_value, message=None, + version=None, stacklevel=3, + category=FutureWarning): + + """Decorates a kwarg accepting function to change the default value""" + + prefix = _KWARG_UPDATED_PREFIX_TPL % (name, new_value) + postfix = _KWARG_UPDATED_POSTFIX_TPL % old_value + out_message = _utils.generate_message( + prefix, postfix=postfix, message=message, version=version) + + def decorator(f): + sig = get_signature(f) + varnames = list(six.iterkeys(sig.parameters)) + + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + explicit_params = set( + varnames[:len(args)] + list(kwargs.keys()) + ) + allparams = set(varnames) + default_params = set(allparams - explicit_params) + if name in default_params: + _utils.deprecation(out_message, + stacklevel=stacklevel, category=category) + return wrapped(*args, **kwargs) + + return wrapper(f) + + return decorator diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 6edb60f1d5c..e683323cc80 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -457,7 +457,7 @@ def test_configure_dogstatsd_host(self): # verify warnings triggered assert len(w) == 1 assert issubclass(w[-1].category, ddtrace.utils.deprecation.RemovedInDDTrace10Warning) - assert 'Use dogstatsd_url' in str(w[-1].message) + assert 'Use `dogstatsd_url`' in str(w[-1].message) def test_configure_dogstatsd_host_port(self): with warnings.catch_warnings(record=True) as w: @@ -466,9 +466,11 @@ def test_configure_dogstatsd_host_port(self): assert self.tracer._dogstatsd_client.host == 'foo' assert self.tracer._dogstatsd_client.port == 1234 # verify warnings triggered - assert len(w) == 1 - assert issubclass(w[-1].category, ddtrace.utils.deprecation.RemovedInDDTrace10Warning) - assert 'Use dogstatsd_url' in str(w[-1].message) + assert len(w) == 2 + assert issubclass(w[0].category, ddtrace.utils.deprecation.RemovedInDDTrace10Warning) + assert 'Use `dogstatsd_url`' in str(w[0].message) + assert issubclass(w[1].category, ddtrace.utils.deprecation.RemovedInDDTrace10Warning) + assert 'Use `dogstatsd_url`' in str(w[1].message) def test_configure_dogstatsd_url_host_port(self): self.tracer.configure(dogstatsd_url='foo:1234') From a4b45346606ff1c41f6a801d2a70906a3c2761b1 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Mon, 25 Nov 2019 10:07:47 -0500 Subject: [PATCH 14/81] core: deprecate unused app type (#1144) * core: remove unused app type * deprecate app_type --- ddtrace/contrib/aiobotocore/patch.py | 2 +- ddtrace/contrib/aiohttp/patch.py | 2 +- ddtrace/contrib/aiopg/connection.py | 4 +- ddtrace/contrib/algoliasearch/patch.py | 4 +- ddtrace/contrib/boto/patch.py | 4 +- ddtrace/contrib/botocore/patch.py | 2 +- ddtrace/contrib/cassandra/session.py | 2 +- ddtrace/contrib/celery/app.py | 2 - ddtrace/contrib/consul/patch.py | 2 +- ddtrace/contrib/dbapi/__init__.py | 4 +- ddtrace/contrib/elasticsearch/patch.py | 4 +- ddtrace/contrib/flask/patch.py | 5 +-- ddtrace/contrib/httplib/patch.py | 2 +- ddtrace/contrib/kombu/patch.py | 7 +--- ddtrace/contrib/mako/patch.py | 2 +- ddtrace/contrib/molten/patch.py | 6 +-- ddtrace/contrib/mysql/patch.py | 4 +- ddtrace/contrib/mysqldb/patch.py | 4 +- ddtrace/contrib/psycopg/patch.py | 1 - ddtrace/contrib/pymemcache/patch.py | 2 +- ddtrace/contrib/pymongo/client.py | 3 +- ddtrace/contrib/pymysql/patch.py | 4 +- ddtrace/contrib/redis/patch.py | 4 +- ddtrace/contrib/rediscluster/patch.py | 4 +- ddtrace/contrib/requests/patch.py | 2 - ddtrace/contrib/sqlalchemy/engine.py | 3 +- ddtrace/contrib/sqlite3/patch.py | 3 +- ddtrace/contrib/tornado/application.py | 2 +- ddtrace/contrib/vertica/patch.py | 5 +-- ddtrace/ext/__init__.py | 5 --- ddtrace/ext/apps.py | 0 ddtrace/ext/consul.py | 3 -- ddtrace/ext/sql.py | 4 -- ddtrace/pin.py | 14 ++++--- ddtrace/tracer.py | 4 -- tests/contrib/sqlite3/test_sqlite3.py | 3 -- tests/test_integration.py | 53 -------------------------- 37 files changed, 45 insertions(+), 136 deletions(-) delete mode 100644 ddtrace/ext/apps.py diff --git a/ddtrace/contrib/aiobotocore/patch.py b/ddtrace/contrib/aiobotocore/patch.py index 9e5905e17f6..6c376b5ca92 100644 --- a/ddtrace/contrib/aiobotocore/patch.py +++ b/ddtrace/contrib/aiobotocore/patch.py @@ -23,7 +23,7 @@ def patch(): setattr(aiobotocore.client, '_datadog_patch', True) wrapt.wrap_function_wrapper('aiobotocore.client', 'AioBaseClient._make_api_call', _wrapped_api_call) - Pin(service='aws', app='aws', app_type='web').onto(aiobotocore.client.AioBaseClient) + Pin(service='aws', app='aws').onto(aiobotocore.client.AioBaseClient) def unpatch(): diff --git a/ddtrace/contrib/aiohttp/patch.py b/ddtrace/contrib/aiohttp/patch.py index 0a5f0c15c22..81b6deae11a 100644 --- a/ddtrace/contrib/aiohttp/patch.py +++ b/ddtrace/contrib/aiohttp/patch.py @@ -26,7 +26,7 @@ def patch(): _w = wrapt.wrap_function_wrapper _w('aiohttp_jinja2', 'render_template', _trace_render_template) - Pin(app='aiohttp', service=None, app_type='web').onto(aiohttp_jinja2) + Pin(app='aiohttp', service=None).onto(aiohttp_jinja2) def unpatch(): diff --git a/ddtrace/contrib/aiopg/connection.py b/ddtrace/contrib/aiopg/connection.py index da21fd94d47..7e34ba4eb79 100644 --- a/ddtrace/contrib/aiopg/connection.py +++ b/ddtrace/contrib/aiopg/connection.py @@ -5,7 +5,7 @@ from .. import dbapi from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import sql, AppTypes +from ...ext import sql from ...pin import Pin from ...settings import config @@ -77,7 +77,7 @@ class AIOTracedConnection(wrapt.ObjectProxy): def __init__(self, conn, pin=None, cursor_cls=AIOTracedCursor): super(AIOTracedConnection, self).__init__(conn) name = dbapi._get_vendor(conn) - db_pin = pin or Pin(service=name, app=name, app_type=AppTypes.db) + db_pin = pin or Pin(service=name, app=name) db_pin.onto(self) # wrapt requires prefix of `_self` for attributes that are only in the # proxy (since some of our source objects will use `__slots__`) diff --git a/ddtrace/contrib/algoliasearch/patch.py b/ddtrace/contrib/algoliasearch/patch.py index 633e48d1f67..0cc7db0674c 100644 --- a/ddtrace/contrib/algoliasearch/patch.py +++ b/ddtrace/contrib/algoliasearch/patch.py @@ -1,4 +1,3 @@ -from ddtrace.ext import AppTypes from ddtrace.pin import Pin from ddtrace.settings import config from ddtrace.utils.wrappers import unwrap as _u @@ -34,8 +33,7 @@ def patch(): setattr(algoliasearch, '_datadog_patch', True) pin = Pin( - service=config.algoliasearch.service_name, app=APP_NAME, - app_type=AppTypes.db + service=config.algoliasearch.service_name, app=APP_NAME ) if algoliasearch_version < (2, 0) and algoliasearch_version >= (1, 0): diff --git a/ddtrace/contrib/boto/patch.py b/ddtrace/contrib/boto/patch.py index 0814ac3f849..a30b432bde4 100644 --- a/ddtrace/contrib/boto/patch.py +++ b/ddtrace/contrib/boto/patch.py @@ -41,10 +41,10 @@ def patch(): wrapt.wrap_function_wrapper( 'boto.connection', 'AWSAuthConnection.make_request', patched_auth_request ) - Pin(service='aws', app='aws', app_type='web').onto( + Pin(service='aws', app='aws').onto( boto.connection.AWSQueryConnection ) - Pin(service='aws', app='aws', app_type='web').onto( + Pin(service='aws', app='aws').onto( boto.connection.AWSAuthConnection ) diff --git a/ddtrace/contrib/botocore/patch.py b/ddtrace/contrib/botocore/patch.py index 066ba021d2f..65045fabd87 100644 --- a/ddtrace/contrib/botocore/patch.py +++ b/ddtrace/contrib/botocore/patch.py @@ -28,7 +28,7 @@ def patch(): setattr(botocore.client, '_datadog_patch', True) wrapt.wrap_function_wrapper('botocore.client', 'BaseClient._make_api_call', patched_api_call) - Pin(service='aws', app='aws', app_type='web').onto(botocore.client.BaseClient) + Pin(service='aws', app='aws').onto(botocore.client.BaseClient) def unpatch(): diff --git a/ddtrace/contrib/cassandra/session.py b/ddtrace/contrib/cassandra/session.py index 35b5eb5fc16..95a7e3a9462 100644 --- a/ddtrace/contrib/cassandra/session.py +++ b/ddtrace/contrib/cassandra/session.py @@ -32,7 +32,7 @@ def patch(): """ patch will add tracing to the cassandra library. """ setattr(cassandra.cluster.Cluster, 'connect', wrapt.FunctionWrapper(_connect, traced_connect)) - Pin(service=SERVICE, app=SERVICE, app_type='db').onto(cassandra.cluster.Cluster) + Pin(service=SERVICE, app=SERVICE).onto(cassandra.cluster.Cluster) def unpatch(): diff --git a/ddtrace/contrib/celery/app.py b/ddtrace/contrib/celery/app.py index 85eaea7fbd6..0c3a835c7f8 100644 --- a/ddtrace/contrib/celery/app.py +++ b/ddtrace/contrib/celery/app.py @@ -2,7 +2,6 @@ from ddtrace import Pin, config from ddtrace.pin import _DD_PIN_NAME -from ddtrace.ext import AppTypes from .constants import APP from .signals import ( @@ -27,7 +26,6 @@ def patch_app(app, pin=None): pin = pin or Pin( service=config.celery['worker_service_name'], app=APP, - app_type=AppTypes.worker, _config=config.celery, ) pin.onto(app) diff --git a/ddtrace/contrib/consul/patch.py b/ddtrace/contrib/consul/patch.py index 646357c3123..aec3390aeb7 100644 --- a/ddtrace/contrib/consul/patch.py +++ b/ddtrace/contrib/consul/patch.py @@ -17,7 +17,7 @@ def patch(): return setattr(consul, '__datadog_patch', True) - pin = Pin(service=consulx.SERVICE, app=consulx.APP, app_type=consulx.APP_TYPE) + pin = Pin(service=consulx.SERVICE, app=consulx.APP) pin.onto(consul.Consul.KV) for f_name in _KV_FUNCS: diff --git a/ddtrace/contrib/dbapi/__init__.py b/ddtrace/contrib/dbapi/__init__.py index 3d3d2059512..44324824001 100644 --- a/ddtrace/contrib/dbapi/__init__.py +++ b/ddtrace/contrib/dbapi/__init__.py @@ -3,7 +3,7 @@ """ from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import AppTypes, sql +from ...ext import sql from ...internal.logger import get_logger from ...pin import Pin from ...settings import config @@ -155,7 +155,7 @@ def __init__(self, conn, pin=None, cursor_cls=None): super(TracedConnection, self).__init__(conn) name = _get_vendor(conn) self._self_datadog_name = '{}.connection'.format(name) - db_pin = pin or Pin(service=name, app=name, app_type=AppTypes.db) + db_pin = pin or Pin(service=name, app=name) db_pin.onto(self) # wrapt requires prefix of `_self` for attributes that are only in the # proxy (since some of our source objects will use `__slots__`) diff --git a/ddtrace/contrib/elasticsearch/patch.py b/ddtrace/contrib/elasticsearch/patch.py index 046f2f4ad3e..9e8fa980fd0 100644 --- a/ddtrace/contrib/elasticsearch/patch.py +++ b/ddtrace/contrib/elasticsearch/patch.py @@ -6,7 +6,7 @@ from ...compat import urlencode from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import elasticsearch as metadata, http, AppTypes +from ...ext import elasticsearch as metadata, http from ...pin import Pin from ...utils.wrappers import unwrap as _u from ...settings import config @@ -32,7 +32,7 @@ def _patch(elasticsearch): return setattr(elasticsearch, '_datadog_patch', True) _w(elasticsearch.transport, 'Transport.perform_request', _get_perform_request(elasticsearch)) - Pin(service=metadata.SERVICE, app=metadata.APP, app_type=AppTypes.db).onto(elasticsearch.transport.Transport) + Pin(service=metadata.SERVICE, app=metadata.APP).onto(elasticsearch.transport.Transport) def unpatch(): diff --git a/ddtrace/contrib/flask/patch.py b/ddtrace/contrib/flask/patch.py index 3b4c07f3008..0dc9bb2e6e5 100644 --- a/ddtrace/contrib/flask/patch.py +++ b/ddtrace/contrib/flask/patch.py @@ -8,7 +8,6 @@ from ddtrace import config, Pin from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import AppTypes from ...ext import http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator @@ -29,7 +28,6 @@ # DEV: Environment variable 'DATADOG_SERVICE_NAME' used for backwards compatibility service_name=os.environ.get('DATADOG_SERVICE_NAME') or 'flask', app='flask', - app_type=AppTypes.web, collect_view_args=True, distributed_tracing_enabled=True, @@ -70,8 +68,7 @@ def patch(): # Attach service pin to `flask.app.Flask` Pin( service=config.flask['service_name'], - app=config.flask['app'], - app_type=config.flask['app_type'], + app=config.flask['app'] ).onto(flask.Flask) # flask.app.Flask methods that have custom tracing (add metadata, wrap functions, etc) diff --git a/ddtrace/contrib/httplib/patch.py b/ddtrace/contrib/httplib/patch.py index 65c40af2332..f401dcd39e2 100644 --- a/ddtrace/contrib/httplib/patch.py +++ b/ddtrace/contrib/httplib/patch.py @@ -17,7 +17,7 @@ def _wrap_init(func, instance, args, kwargs): - Pin(app='httplib', service=None, app_type=ext_http.TYPE).onto(instance) + Pin(app='httplib', service=None).onto(instance) return func(*args, **kwargs) diff --git a/ddtrace/contrib/kombu/patch.py b/ddtrace/contrib/kombu/patch.py index a4a2062a1ad..9790f20737b 100644 --- a/ddtrace/contrib/kombu/patch.py +++ b/ddtrace/contrib/kombu/patch.py @@ -5,7 +5,6 @@ # project from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...ext import kombu as kombux -from ...ext import AppTypes from ...pin import Pin from ...propagation.http import HTTPPropagator from ...settings import config @@ -48,14 +47,12 @@ def patch(): _w(kombux.TYPE, 'Consumer.receive', traced_receive) Pin( service=config.kombu['service_name'], - app='kombu', - app_type=AppTypes.worker, + app='kombu' ).onto(kombu.messaging.Producer) Pin( service=config.kombu['service_name'], - app='kombu', - app_type=AppTypes.worker, + app='kombu' ).onto(kombu.messaging.Consumer) diff --git a/ddtrace/contrib/mako/patch.py b/ddtrace/contrib/mako/patch.py index ebc179e2d0c..fa574ab82d1 100644 --- a/ddtrace/contrib/mako/patch.py +++ b/ddtrace/contrib/mako/patch.py @@ -15,7 +15,7 @@ def patch(): return setattr(mako, '__datadog_patch', True) - Pin(service='mako', app='mako', app_type=http.TEMPLATE).onto(Template) + Pin(service='mako', app='mako').onto(Template) _w(mako, 'template.Template.render', _wrap_render) _w(mako, 'template.Template.render_unicode', _wrap_render) diff --git a/ddtrace/contrib/molten/patch.py b/ddtrace/contrib/molten/patch.py index be0596944c4..62003cea7b5 100644 --- a/ddtrace/contrib/molten/patch.py +++ b/ddtrace/contrib/molten/patch.py @@ -6,7 +6,7 @@ from ... import Pin, config from ...compat import urlencode from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import AppTypes, http +from ...ext import http from ...propagation.http import HTTPPropagator from ...utils.formats import asbool, get_env from ...utils.importlib import func_name @@ -19,7 +19,6 @@ config._add('molten', dict( service_name=get_env('molten', 'service_name', 'molten'), app='molten', - app_type=AppTypes.web, distributed_tracing=asbool(get_env('molten', 'distributed_tracing', True)), )) @@ -33,8 +32,7 @@ def patch(): pin = Pin( service=config.molten['service_name'], - app=config.molten['app'], - app_type=config.molten['app_type'], + app=config.molten['app'] ) # add pin to module since many classes use __slots__ diff --git a/ddtrace/contrib/mysql/patch.py b/ddtrace/contrib/mysql/patch.py index 30b49e8bc0e..15a0b1aa516 100644 --- a/ddtrace/contrib/mysql/patch.py +++ b/ddtrace/contrib/mysql/patch.py @@ -5,7 +5,7 @@ # project from ddtrace import Pin from ddtrace.contrib.dbapi import TracedConnection -from ...ext import net, db, AppTypes +from ...ext import net, db CONN_ATTR_BY_TAG = { @@ -38,7 +38,7 @@ def _connect(func, instance, args, kwargs): def patch_conn(conn): tags = {t: getattr(conn, a) for t, a in CONN_ATTR_BY_TAG.items() if getattr(conn, a, '') != ''} - pin = Pin(service='mysql', app='mysql', app_type=AppTypes.db, tags=tags) + pin = Pin(service='mysql', app='mysql', tags=tags) # grab the metadata from the conn wrapped = TracedConnection(conn, pin=pin) diff --git a/ddtrace/contrib/mysqldb/patch.py b/ddtrace/contrib/mysqldb/patch.py index ddc72dc7851..ac6e0e1d4e2 100644 --- a/ddtrace/contrib/mysqldb/patch.py +++ b/ddtrace/contrib/mysqldb/patch.py @@ -7,7 +7,7 @@ from ddtrace import Pin from ddtrace.contrib.dbapi import TracedConnection -from ...ext import net, db, AppTypes +from ...ext import net, db from ...utils.wrappers import unwrap as _u KWPOS_BY_TAG = { @@ -55,7 +55,7 @@ def patch_conn(conn, *args, **kwargs): for t, (k, p) in KWPOS_BY_TAG.items() if k in kwargs or len(args) > p} tags[net.TARGET_PORT] = conn.port - pin = Pin(service='mysql', app='mysql', app_type=AppTypes.db, tags=tags) + pin = Pin(service='mysql', app='mysql', tags=tags) # grab the metadata from the conn wrapped = TracedConnection(conn, pin=pin) diff --git a/ddtrace/contrib/psycopg/patch.py b/ddtrace/contrib/psycopg/patch.py index 913a49fd8ee..3aeb68b7134 100644 --- a/ddtrace/contrib/psycopg/patch.py +++ b/ddtrace/contrib/psycopg/patch.py @@ -89,7 +89,6 @@ def patch_conn(conn, traced_conn_cls=Psycopg2TracedConnection): Pin( service='postgres', app='postgres', - app_type='db', tags=tags).onto(c) return c diff --git a/ddtrace/contrib/pymemcache/patch.py b/ddtrace/contrib/pymemcache/patch.py index 5fdad8e1754..6ab0cabc073 100644 --- a/ddtrace/contrib/pymemcache/patch.py +++ b/ddtrace/contrib/pymemcache/patch.py @@ -16,7 +16,7 @@ def patch(): # Create a global pin with default configuration for our pymemcache clients Pin( - app=memcachedx.SERVICE, service=memcachedx.SERVICE, app_type=memcachedx.TYPE + app=memcachedx.SERVICE, service=memcachedx.SERVICE ).onto(pymemcache) diff --git a/ddtrace/contrib/pymongo/client.py b/ddtrace/contrib/pymongo/client.py index 59271901781..25eea416670 100644 --- a/ddtrace/contrib/pymongo/client.py +++ b/ddtrace/contrib/pymongo/client.py @@ -10,7 +10,6 @@ import ddtrace from ...compat import iteritems from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import AppTypes from ...ext import mongo as mongox from ...ext import net as netx from ...internal.logger import get_logger @@ -62,7 +61,7 @@ def __init__(self, client=None, *args, **kwargs): client._topology = TracedTopology(client._topology) # Default Pin - ddtrace.Pin(service=mongox.TYPE, app=mongox.TYPE, app_type=AppTypes.db).onto(self) + ddtrace.Pin(service=mongox.TYPE, app=mongox.TYPE).onto(self) def __setddpin__(self, pin): pin.onto(self._topology) diff --git a/ddtrace/contrib/pymysql/patch.py b/ddtrace/contrib/pymysql/patch.py index 8ea8cf4c3d1..ccb56781f8d 100644 --- a/ddtrace/contrib/pymysql/patch.py +++ b/ddtrace/contrib/pymysql/patch.py @@ -5,7 +5,7 @@ # project from ddtrace import Pin from ddtrace.contrib.dbapi import TracedConnection -from ...ext import net, db, AppTypes +from ...ext import net, db CONN_ATTR_BY_TAG = { net.TARGET_HOST: 'host', @@ -31,7 +31,7 @@ def _connect(func, instance, args, kwargs): def patch_conn(conn): tags = {t: getattr(conn, a, '') for t, a in CONN_ATTR_BY_TAG.items()} - pin = Pin(service='pymysql', app='pymysql', app_type=AppTypes.db, tags=tags) + pin = Pin(service='pymysql', app='pymysql', tags=tags) # grab the metadata from the conn wrapped = TracedConnection(conn, pin=pin) diff --git a/ddtrace/contrib/redis/patch.py b/ddtrace/contrib/redis/patch.py index 66fe8debecb..51361c390e6 100644 --- a/ddtrace/contrib/redis/patch.py +++ b/ddtrace/contrib/redis/patch.py @@ -6,7 +6,7 @@ from ddtrace import config from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import AppTypes, redis as redisx +from ...ext import redis as redisx from ...utils.wrappers import unwrap from .util import format_command_args, _extract_conn_tags @@ -34,7 +34,7 @@ def patch(): _w('redis', 'Redis.pipeline', traced_pipeline) _w('redis.client', 'Pipeline.execute', traced_execute_pipeline) _w('redis.client', 'Pipeline.immediate_execute_command', traced_execute_command) - Pin(service=redisx.DEFAULT_SERVICE, app=redisx.APP, app_type=AppTypes.db).onto(redis.StrictRedis) + Pin(service=redisx.DEFAULT_SERVICE, app=redisx.APP).onto(redis.StrictRedis) def unpatch(): diff --git a/ddtrace/contrib/rediscluster/patch.py b/ddtrace/contrib/rediscluster/patch.py index d73466a17d7..66c2d59c960 100644 --- a/ddtrace/contrib/rediscluster/patch.py +++ b/ddtrace/contrib/rediscluster/patch.py @@ -6,7 +6,7 @@ from ddtrace import config from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import AppTypes, redis as redisx +from ...ext import redis as redisx from ...utils.wrappers import unwrap from ..redis.patch import traced_execute_command, traced_pipeline from ..redis.util import format_command_args @@ -23,7 +23,7 @@ def patch(): _w('rediscluster', 'StrictRedisCluster.execute_command', traced_execute_command) _w('rediscluster', 'StrictRedisCluster.pipeline', traced_pipeline) _w('rediscluster', 'StrictClusterPipeline.execute', traced_execute_pipeline) - Pin(service=redisx.DEFAULT_SERVICE, app=redisx.APP, app_type=AppTypes.db).onto(rediscluster.StrictRedisCluster) + Pin(service=redisx.DEFAULT_SERVICE, app=redisx.APP).onto(rediscluster.StrictRedisCluster) def unpatch(): diff --git a/ddtrace/contrib/requests/patch.py b/ddtrace/contrib/requests/patch.py index 7317ee13355..0072481514c 100644 --- a/ddtrace/contrib/requests/patch.py +++ b/ddtrace/contrib/requests/patch.py @@ -10,7 +10,6 @@ from .legacy import _distributed_tracing, _distributed_tracing_setter from .constants import DEFAULT_SERVICE from .connection import _wrap_send -from ...ext import AppTypes # requests default settings config._add('requests', { @@ -30,7 +29,6 @@ def patch(): Pin( service=config.requests['service_name'], app='requests', - app_type=AppTypes.web, _config=config.requests, ).onto(requests.Session) diff --git a/ddtrace/contrib/sqlalchemy/engine.py b/ddtrace/contrib/sqlalchemy/engine.py index a3fb1af96a2..5f3f3c2c6b1 100644 --- a/ddtrace/contrib/sqlalchemy/engine.py +++ b/ddtrace/contrib/sqlalchemy/engine.py @@ -63,8 +63,7 @@ def __init__(self, tracer, service, engine): Pin( app=self.vendor, tracer=tracer, - service=self.service, - app_type=sqlx.APP_TYPE, + service=self.service ).onto(engine) listen(engine, 'before_cursor_execute', self._before_cur_exec) diff --git a/ddtrace/contrib/sqlite3/patch.py b/ddtrace/contrib/sqlite3/patch.py index c60f8322e6d..3cae546b2f3 100644 --- a/ddtrace/contrib/sqlite3/patch.py +++ b/ddtrace/contrib/sqlite3/patch.py @@ -5,7 +5,6 @@ # project from ...contrib.dbapi import TracedConnection, TracedCursor, FetchTracedCursor -from ...ext import AppTypes from ...pin import Pin from ...settings import config @@ -32,7 +31,7 @@ def traced_connect(func, _, args, kwargs): def patch_conn(conn): wrapped = TracedSQLite(conn) - Pin(service='sqlite', app='sqlite', app_type=AppTypes.db).onto(wrapped) + Pin(service='sqlite', app='sqlite').onto(wrapped) return wrapped diff --git a/ddtrace/contrib/tornado/application.py b/ddtrace/contrib/tornado/application.py index 079c32d697f..a413197d39b 100644 --- a/ddtrace/contrib/tornado/application.py +++ b/ddtrace/contrib/tornado/application.py @@ -53,4 +53,4 @@ def tracer_config(__init__, app, args, kwargs): tracer.set_tags(tags) # configure the PIN object for template rendering - ddtrace.Pin(app='tornado', service=service, app_type='web', tracer=tracer).onto(template) + ddtrace.Pin(app='tornado', service=service, tracer=tracer).onto(template) diff --git a/ddtrace/contrib/vertica/patch.py b/ddtrace/contrib/vertica/patch.py index dfe3aecc2c2..b2a09c394c6 100644 --- a/ddtrace/contrib/vertica/patch.py +++ b/ddtrace/contrib/vertica/patch.py @@ -5,7 +5,7 @@ import ddtrace from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...ext import db as dbx, sql -from ...ext import net, AppTypes +from ...ext import net from ...internal.logger import get_logger from ...pin import Pin from ...settings import config @@ -46,7 +46,6 @@ def cursor_span_end(instance, cursor, _, conf, *args, **kwargs): pin = Pin( service=config.vertica['service_name'], app=APP, - app_type=AppTypes.db, tags=tags, _config=config.vertica['patch']['vertica_python.vertica.cursor.Cursor'], ) @@ -59,7 +58,6 @@ def cursor_span_end(instance, cursor, _, conf, *args, **kwargs): { 'service_name': 'vertica', 'app': 'vertica', - 'app_type': 'db', 'patch': { 'vertica_python.vertica.connection.Connection': { 'routines': { @@ -168,7 +166,6 @@ def init_wrapper(wrapped, instance, args, kwargs): Pin( service=config['service_name'], app=config['app'], - app_type=config['app_type'], tags=config.get('tags', {}), tracer=config.get('tracer', ddtrace.tracer), _config=config['patch'][patch_item], diff --git a/ddtrace/ext/__init__.py b/ddtrace/ext/__init__.py index cabf64ef81f..e69de29bb2d 100644 --- a/ddtrace/ext/__init__.py +++ b/ddtrace/ext/__init__.py @@ -1,5 +0,0 @@ -class AppTypes(object): - web = 'web' - db = 'db' - cache = 'cache' - worker = 'worker' diff --git a/ddtrace/ext/apps.py b/ddtrace/ext/apps.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ddtrace/ext/consul.py b/ddtrace/ext/consul.py index d8653d738db..be17e922361 100644 --- a/ddtrace/ext/consul.py +++ b/ddtrace/ext/consul.py @@ -1,7 +1,4 @@ -from . import AppTypes - APP = 'consul' -APP_TYPE = AppTypes.cache SERVICE = 'consul' CMD = 'consul.command' diff --git a/ddtrace/ext/sql.py b/ddtrace/ext/sql.py index b9b93c371f3..b65c5e87a83 100644 --- a/ddtrace/ext/sql.py +++ b/ddtrace/ext/sql.py @@ -1,9 +1,5 @@ -from ddtrace.ext import AppTypes - - # the type of the spans TYPE = 'sql' -APP_TYPE = AppTypes.db # tags QUERY = 'sql.query' # the query text diff --git a/ddtrace/pin.py b/ddtrace/pin.py index 9e58f7ca099..c64755b491a 100644 --- a/ddtrace/pin.py +++ b/ddtrace/pin.py @@ -1,5 +1,7 @@ import ddtrace +from ddtrace.vendor import debtcollector + from .internal.logger import get_logger from .vendor import wrapt @@ -24,12 +26,12 @@ class Pin(object): >>> pin = Pin.override(conn, service='user-db') >>> conn = sqlite.connect('/tmp/image.db') """ - __slots__ = ['app', 'app_type', 'tags', 'tracer', '_target', '_config', '_initialized'] + __slots__ = ['app', 'tags', 'tracer', '_target', '_config', '_initialized'] + @debtcollector.removals.removed_kwarg("app_type") def __init__(self, service, app=None, app_type=None, tags=None, tracer=None, _config=None): tracer = tracer or ddtrace.tracer self.app = app - self.app_type = app_type self.tags = tags self.tracer = tracer self._target = None @@ -53,8 +55,8 @@ def __setattr__(self, name, value): super(Pin, self).__setattr__(name, value) def __repr__(self): - return 'Pin(service=%s, app=%s, app_type=%s, tags=%s, tracer=%s)' % ( - self.service, self.app, self.app_type, self.tags, self.tracer) + return 'Pin(service=%s, app=%s, tags=%s, tracer=%s)' % ( + self.service, self.app, self.tags, self.tracer) @staticmethod def _find(*objs): @@ -101,6 +103,7 @@ def get_from(obj): return pin @classmethod + @debtcollector.removals.removed_kwarg("app_type") def override(cls, obj, service=None, app=None, app_type=None, tags=None, tracer=None): """Override an object with the given attributes. @@ -121,7 +124,6 @@ def override(cls, obj, service=None, app=None, app_type=None, tags=None, tracer= pin.clone( service=service, app=app, - app_type=app_type, tags=tags, tracer=tracer, ).onto(obj) @@ -158,6 +160,7 @@ def remove_from(self, obj): except AttributeError: log.debug("can't remove pin from object. skipping", exc_info=True) + @debtcollector.removals.removed_kwarg("app_type") def clone(self, service=None, app=None, app_type=None, tags=None, tracer=None): """Return a clone of the pin with the given attributes replaced.""" # do a shallow copy of Pin dicts @@ -175,7 +178,6 @@ def clone(self, service=None, app=None, app_type=None, tags=None, tracer=None): return Pin( service=service or self.service, app=app or self.app, - app_type=app_type or self.app_type, tags=tags, tracer=tracer or self.tracer, # do not clone the Tracer _config=config, diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 85a70911410..d7759a57645 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -536,10 +536,6 @@ def write(self, spans): @deprecated(message='Manually setting service info is no longer necessary', version='1.0.0') def set_service_info(self, *args, **kwargs): """Set the information about the given service. - - :param str service: the internal name of the service (e.g. acme_search, datadog_web) - :param str app: the off the shelf name of the application (e.g. rails, postgres, custom-app) - :param str app_type: the type of the application (e.g. db, web) """ return diff --git a/tests/contrib/sqlite3/test_sqlite3.py b/tests/contrib/sqlite3/test_sqlite3.py index 2a069124be9..45277ccc46b 100644 --- a/tests/contrib/sqlite3/test_sqlite3.py +++ b/tests/contrib/sqlite3/test_sqlite3.py @@ -53,7 +53,6 @@ def test_sqlite(self): db = sqlite3.connect(':memory:') pin = Pin.get_from(db) assert pin - self.assertEqual('db', pin.app_type) pin.clone( service=service, tracer=self.tracer).onto(db) @@ -194,7 +193,6 @@ def test_sqlite_ot(self): db = sqlite3.connect(':memory:') pin = Pin.get_from(db) assert pin - self.assertEqual('db', pin.app_type) pin.clone(tracer=self.tracer).onto(db) cursor = db.execute(q) rows = cursor.fetchall() @@ -213,7 +211,6 @@ def test_sqlite_ot(self): db = sqlite3.connect(':memory:') pin = Pin.get_from(db) assert pin - self.assertEqual('db', pin.app_type) pin.clone(tracer=self.tracer).onto(db) cursor = db.execute(q) rows = cursor.fetchall() diff --git a/tests/test_integration.py b/tests/test_integration.py index 89b85e53843..8752413342a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -277,21 +277,6 @@ def test_send_presampler_headers(self, mocked_http): for k, v in expected_headers.items(): assert v == headers[k] - @mock.patch('ddtrace.api.httplib.HTTPConnection') - def test_send_presampler_headers_not_in_services(self, mocked_http): - # register some services and send them to the trace agent - services = [{ - 'client.service': { - 'app': 'django', - 'app_type': 'web', - }, - }] - - # make a call and retrieve the `conn` Mock object - self.api_msgpack.send_services(services) - request_call = mocked_http.return_value.request - assert request_call.call_count == 0 - def _send_traces_and_check(self, traces, nresponses=1): # test JSON encoder responses = self.api_json.send_traces(traces) @@ -367,44 +352,6 @@ def test_send_multiple_traces_multiple_spans(self): self._send_traces_and_check(traces) - def test_send_single_service(self): - # register some services and send them to the trace agent - services = [{ - 'client.service': { - 'app': 'django', - 'app_type': 'web', - }, - }] - - # test JSON encoder - response = self.api_json.send_services(services) - assert response is None - - # test Msgpack encoder - response = self.api_msgpack.send_services(services) - assert response is None - - def test_send_service_called_multiple_times(self): - # register some services and send them to the trace agent - services = [{ - 'backend': { - 'app': 'django', - 'app_type': 'web', - }, - 'database': { - 'app': 'postgres', - 'app_type': 'db', - }, - }] - - # test JSON encoder - response = self.api_json.send_services(services) - assert response is None - - # test Msgpack encoder - response = self.api_msgpack.send_services(services) - assert response is None - @skipUnless( os.environ.get('TEST_DATADOG_INTEGRATION', False), From b90be8ff3256e643b5f7b1517e5a11b1897370aa Mon Sep 17 00:00:00 2001 From: alrex Date: Tue, 26 Nov 2019 09:36:39 -0800 Subject: [PATCH 15/81] pin multidict dependency for aiobotocore02 tests (#1145) Signed-off-by: Alex Boten --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 38903b86a07..661286a9fe8 100644 --- a/tox.ini +++ b/tox.ini @@ -157,6 +157,7 @@ deps = aiobotocore04: aiobotocore>=0.4,<0.5 aiobotocore03: aiobotocore>=0.3,<0.4 aiobotocore02: aiobotocore>=0.2,<0.3 + aiobotocore02: multidict==4.5.2 aiobotocore{02,03,04}-{py34}: typing aiopg012: aiopg>=0.12,<0.13 aiopg015: aiopg>=0.15,<0.16 From 12e48ebb4fdd408f407cefeff7c8d6c2fc257e99 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Wed, 27 Nov 2019 14:58:54 -0500 Subject: [PATCH 16/81] vendor: fix debtcollector imports (#1152) * vendor: fix debtcollector imports * use relative path for all other vendored libraries --- ddtrace/vendor/debtcollector/_utils.py | 2 +- ddtrace/vendor/debtcollector/moves.py | 4 ++-- ddtrace/vendor/debtcollector/removals.py | 4 ++-- ddtrace/vendor/debtcollector/renames.py | 2 +- ddtrace/vendor/debtcollector/updating.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ddtrace/vendor/debtcollector/_utils.py b/ddtrace/vendor/debtcollector/_utils.py index 80bada74c4c..45691ab14d2 100644 --- a/ddtrace/vendor/debtcollector/_utils.py +++ b/ddtrace/vendor/debtcollector/_utils.py @@ -19,7 +19,7 @@ import types import warnings -import six +from .. import six try: _TYPE_TYPE = types.TypeType diff --git a/ddtrace/vendor/debtcollector/moves.py b/ddtrace/vendor/debtcollector/moves.py index 639181aa661..e0965930bb0 100644 --- a/ddtrace/vendor/debtcollector/moves.py +++ b/ddtrace/vendor/debtcollector/moves.py @@ -16,8 +16,8 @@ import inspect -from ddtrace.vendor import six -from ddtrace.vendor import wrapt +from .. import six +from .. import wrapt from . import _utils diff --git a/ddtrace/vendor/debtcollector/removals.py b/ddtrace/vendor/debtcollector/removals.py index e3ee031177f..0add069e76c 100644 --- a/ddtrace/vendor/debtcollector/removals.py +++ b/ddtrace/vendor/debtcollector/removals.py @@ -15,8 +15,8 @@ import functools import inspect -from ddtrace.vendor import six -from ddtrace.vendor import wrapt +from .. import six +from .. import wrapt from . import _utils diff --git a/ddtrace/vendor/debtcollector/renames.py b/ddtrace/vendor/debtcollector/renames.py index 0a34f72465f..d31853aa6fe 100644 --- a/ddtrace/vendor/debtcollector/renames.py +++ b/ddtrace/vendor/debtcollector/renames.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from ddtrace.vendor import wrapt +from .. import wrapt from . import _utils diff --git a/ddtrace/vendor/debtcollector/updating.py b/ddtrace/vendor/debtcollector/updating.py index 3055563a151..d89eafd206f 100644 --- a/ddtrace/vendor/debtcollector/updating.py +++ b/ddtrace/vendor/debtcollector/updating.py @@ -14,8 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -from ddtrace.vendor import six -from ddtrace.vendor import wrapt +from .. import six +from .. import wrapt if six.PY3: import inspect Parameter = inspect.Parameter From 0f3a62c57e7014814c9c55f819a50512e5b8a92a Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Tue, 3 Dec 2019 08:47:39 -0500 Subject: [PATCH 17/81] core: common span types and revise language tag (#1150) * core: common span types * include enum backport in install_requires * attempt to fix tox issues in ci for celery --- ddtrace/contrib/aiobotocore/patch.py | 4 +- ddtrace/contrib/aiohttp/middlewares.py | 4 +- ddtrace/contrib/aiohttp/template.py | 5 +- ddtrace/contrib/aiopg/connection.py | 5 +- ddtrace/contrib/algoliasearch/patch.py | 3 +- ddtrace/contrib/boto/patch.py | 7 +- ddtrace/contrib/botocore/patch.py | 5 +- ddtrace/contrib/bottle/trace.py | 8 +-- ddtrace/contrib/cassandra/session.py | 4 +- ddtrace/contrib/celery/signals.py | 4 +- ddtrace/contrib/dbapi/__init__.py | 5 +- ddtrace/contrib/django/cache.py | 4 +- ddtrace/contrib/django/middleware.py | 4 +- ddtrace/contrib/django/templates.py | 4 +- ddtrace/contrib/elasticsearch/patch.py | 7 +- ddtrace/contrib/elasticsearch/transport.py | 6 +- ddtrace/contrib/falcon/middleware.py | 4 +- ddtrace/contrib/flask/middleware.py | 7 +- ddtrace/contrib/flask/patch.py | 8 +-- ddtrace/contrib/flask_cache/tracers.py | 4 +- ddtrace/contrib/grpc/client_interceptor.py | 4 +- ddtrace/contrib/grpc/server_interceptor.py | 3 +- ddtrace/contrib/httplib/patch.py | 4 +- ddtrace/contrib/jinja2/patch.py | 8 +-- ddtrace/contrib/kombu/patch.py | 10 +-- ddtrace/contrib/mako/patch.py | 4 +- ddtrace/contrib/molten/patch.py | 4 +- ddtrace/contrib/mongoengine/trace.py | 2 +- ddtrace/contrib/psycopg/connection.py | 7 +- ddtrace/contrib/pylibmc/client.py | 6 +- ddtrace/contrib/pylons/middleware.py | 5 +- ddtrace/contrib/pymemcache/client.py | 4 +- ddtrace/contrib/pymongo/client.py | 11 ++-- ddtrace/contrib/pyramid/trace.py | 8 +-- ddtrace/contrib/redis/patch.py | 7 +- ddtrace/contrib/rediscluster/patch.py | 5 +- ddtrace/contrib/requests/connection.py | 4 +- ddtrace/contrib/sqlalchemy/engine.py | 5 +- ddtrace/contrib/tornado/handlers.py | 4 +- ddtrace/contrib/tornado/template.py | 8 +-- ddtrace/contrib/vertica/patch.py | 16 ++--- ddtrace/ext/__init__.py | 15 +++++ ddtrace/ext/cassandra.py | 4 -- ddtrace/ext/elasticsearch.py | 5 +- ddtrace/ext/http.py | 3 - ddtrace/ext/kombu.py | 3 +- ddtrace/ext/memcached.py | 1 - ddtrace/ext/mongo.py | 3 +- ddtrace/ext/redis.py | 3 - ddtrace/ext/sql.py | 3 - ddtrace/span.py | 4 +- ddtrace/tracer.py | 11 +++- setup.py | 15 ++++- tests/contrib/aiohttp/test_middleware.py | 4 +- tests/contrib/algoliasearch/test.py | 5 +- tests/contrib/boto/test.py | 2 +- tests/contrib/cassandra/test.py | 6 +- tests/contrib/django/test_middleware.py | 4 +- .../test_djangorestframework.py | 4 +- tests/contrib/falcon/test_suite.py | 2 +- tests/contrib/flask/test_request.py | 16 ++--- .../flask_autopatch/test_flask_autopatch.py | 2 +- tests/contrib/flask_cache/test.py | 8 +-- tests/contrib/kombu/test.py | 4 +- tests/contrib/molten/test_molten.py | 1 + tests/contrib/mongoengine/test.py | 4 +- tests/contrib/pylons/test_pylons.py | 2 +- tests/contrib/pymemcache/test_client_mixin.py | 5 +- tests/contrib/pymongo/test.py | 2 +- tests/contrib/pyramid/utils.py | 20 +++--- tests/contrib/requests/test_requests.py | 10 +-- .../tornado/test_executor_decorator.py | 10 +-- .../contrib/tornado/test_tornado_template.py | 6 +- tests/contrib/tornado/test_tornado_web.py | 26 ++++---- tests/contrib/tornado/test_wrap_decorator.py | 12 ++-- tests/test_span.py | 18 ++++- tests/test_tracer.py | 26 ++++++-- tox.ini | 65 +++++++++++++++++++ 78 files changed, 325 insertions(+), 240 deletions(-) diff --git a/ddtrace/contrib/aiobotocore/patch.py b/ddtrace/contrib/aiobotocore/patch.py index 6c376b5ca92..5e732476587 100644 --- a/ddtrace/contrib/aiobotocore/patch.py +++ b/ddtrace/contrib/aiobotocore/patch.py @@ -7,7 +7,7 @@ from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import http, aws +from ...ext import SpanTypes, http, aws from ...compat import PYTHON_VERSION_INFO from ...utils.formats import deep_getattr from ...utils.wrappers import unwrap @@ -79,7 +79,7 @@ def _wrapped_api_call(original_func, instance, args, kwargs): with pin.tracer.trace('{}.command'.format(endpoint_name), service='{}.{}'.format(pin.service, endpoint_name), - span_type=http.TYPE) as span: + span_type=SpanTypes.HTTP) as span: if len(args) > 0: operation = args[0] diff --git a/ddtrace/contrib/aiohttp/middlewares.py b/ddtrace/contrib/aiohttp/middlewares.py index 344452e3c6e..52269b91939 100644 --- a/ddtrace/contrib/aiohttp/middlewares.py +++ b/ddtrace/contrib/aiohttp/middlewares.py @@ -3,7 +3,7 @@ from ..asyncio import context_provider from ...compat import stringify from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...settings import config @@ -43,7 +43,7 @@ def attach_context(request): request_span = tracer.trace( 'aiohttp.request', service=service, - span_type=http.TYPE, + span_type=SpanTypes.WEB, ) # Configure trace search sample rate diff --git a/ddtrace/contrib/aiohttp/template.py b/ddtrace/contrib/aiohttp/template.py index 8dcbef55dfb..2b0c91479b0 100644 --- a/ddtrace/contrib/aiohttp/template.py +++ b/ddtrace/contrib/aiohttp/template.py @@ -2,7 +2,7 @@ from ddtrace import Pin -from ...ext import http +from ...ext import SpanTypes def _trace_render_template(func, module, args, kwargs): @@ -24,7 +24,6 @@ def _trace_render_template(func, module, args, kwargs): template_prefix = getattr(env.loader, 'package_path', '') template_meta = '{}/{}'.format(template_prefix, template_name) - with pin.tracer.trace('aiohttp.template') as span: - span.span_type = http.TEMPLATE + with pin.tracer.trace('aiohttp.template', span_type=SpanTypes.TEMPLATE) as span: span.set_meta('aiohttp.template', template_meta) return func(*args, **kwargs) diff --git a/ddtrace/contrib/aiopg/connection.py b/ddtrace/contrib/aiopg/connection.py index 7e34ba4eb79..f5dc3afb47f 100644 --- a/ddtrace/contrib/aiopg/connection.py +++ b/ddtrace/contrib/aiopg/connection.py @@ -5,7 +5,7 @@ from .. import dbapi from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import sql +from ...ext import SpanTypes, sql from ...pin import Pin from ...settings import config @@ -28,8 +28,7 @@ def _trace_method(self, method, resource, extra_tags, *args, **kwargs): service = pin.service with pin.tracer.trace(self._datadog_name, service=service, - resource=resource) as s: - s.span_type = sql.TYPE + resource=resource, span_type=SpanTypes.SQL) as s: s.set_tag(sql.QUERY, resource) s.set_tags(pin.tags) s.set_tags(extra_tags) diff --git a/ddtrace/contrib/algoliasearch/patch.py b/ddtrace/contrib/algoliasearch/patch.py index 0cc7db0674c..859b6eb0f7d 100644 --- a/ddtrace/contrib/algoliasearch/patch.py +++ b/ddtrace/contrib/algoliasearch/patch.py @@ -7,7 +7,6 @@ SERVICE_NAME = 'algoliasearch' APP_NAME = 'algoliasearch' -SEARCH_SPAN_TYPE = 'algoliasearch.search' try: import algoliasearch @@ -101,7 +100,7 @@ def _patched_search(func, instance, wrapt_args, wrapt_kwargs): if not pin or not pin.enabled(): return func(*wrapt_args, **wrapt_kwargs) - with pin.tracer.trace('algoliasearch.search', service=pin.service, span_type=SEARCH_SPAN_TYPE) as span: + with pin.tracer.trace('algoliasearch.search', service=pin.service) as span: if not span.sampled: return func(*wrapt_args, **wrapt_kwargs) diff --git a/ddtrace/contrib/boto/patch.py b/ddtrace/contrib/boto/patch.py index a30b432bde4..c1de9e9cb5b 100644 --- a/ddtrace/contrib/boto/patch.py +++ b/ddtrace/contrib/boto/patch.py @@ -5,14 +5,13 @@ from ddtrace import config from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import http, aws +from ...ext import SpanTypes, http, aws from ...utils.wrappers import unwrap # Original boto client class _Boto_client = boto.connection.AWSQueryConnection -SPAN_TYPE = 'boto' AWS_QUERY_ARGS_NAME = ('operation_name', 'params', 'path', 'verb') AWS_AUTH_ARGS_NAME = ( 'method', @@ -68,7 +67,7 @@ def patched_query_request(original_func, instance, args, kwargs): with pin.tracer.trace( '{}.command'.format(endpoint_name), service='{}.{}'.format(pin.service, endpoint_name), - span_type=SPAN_TYPE, + span_type=SpanTypes.HTTP, ) as span: operation_name = None @@ -136,7 +135,7 @@ def patched_auth_request(original_func, instance, args, kwargs): with pin.tracer.trace( '{}.command'.format(endpoint_name), service='{}.{}'.format(pin.service, endpoint_name), - span_type=SPAN_TYPE, + span_type=SpanTypes.HTTP, ) as span: if args: diff --git a/ddtrace/contrib/botocore/patch.py b/ddtrace/contrib/botocore/patch.py index 65045fabd87..87ed79ee96b 100644 --- a/ddtrace/contrib/botocore/patch.py +++ b/ddtrace/contrib/botocore/patch.py @@ -9,7 +9,7 @@ # project from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import http, aws +from ...ext import SpanTypes, http, aws from ...utils.formats import deep_getattr from ...utils.wrappers import unwrap @@ -17,7 +17,6 @@ # Original botocore client class _Botocore_client = botocore.client.BaseClient -SPAN_TYPE = 'http' ARGS_NAME = ('action', 'params', 'path', 'verb') TRACED_ARGS = ['params', 'path', 'verb'] @@ -47,7 +46,7 @@ def patched_api_call(original_func, instance, args, kwargs): with pin.tracer.trace('{}.command'.format(endpoint_name), service='{}.{}'.format(pin.service, endpoint_name), - span_type=SPAN_TYPE) as span: + span_type=SpanTypes.HTTP) as span: operation = None if args: diff --git a/ddtrace/contrib/bottle/trace.py b/ddtrace/contrib/bottle/trace.py index 544dacc9c1b..bac17a6ec8b 100644 --- a/ddtrace/contrib/bottle/trace.py +++ b/ddtrace/contrib/bottle/trace.py @@ -6,12 +6,10 @@ # project from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...settings import config -SPAN_TYPE = 'web' - class TracePlugin(object): name = 'trace' @@ -37,7 +35,9 @@ def wrapped(*args, **kwargs): if context.trace_id: self.tracer.context_provider.activate(context) - with self.tracer.trace('bottle.request', service=self.service, resource=resource, span_type=SPAN_TYPE) as s: + with self.tracer.trace( + 'bottle.request', service=self.service, resource=resource, span_type=SpanTypes.WEB + ) as s: # set analytics sample rate with global config enabled s.set_tag( ANALYTICS_SAMPLE_RATE_KEY, diff --git a/ddtrace/contrib/cassandra/session.py b/ddtrace/contrib/cassandra/session.py index 95a7e3a9462..512aba7758d 100644 --- a/ddtrace/contrib/cassandra/session.py +++ b/ddtrace/contrib/cassandra/session.py @@ -9,7 +9,7 @@ # project from ...compat import stringify from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import net, cassandra as cassx, errors +from ...ext import SpanTypes, net, cassandra as cassx, errors from ...internal.logger import get_logger from ...pin import Pin from ...settings import config @@ -181,7 +181,7 @@ def traced_execute_async(func, instance, args, kwargs): def _start_span_and_set_tags(pin, query, session, cluster): service = pin.service tracer = pin.tracer - span = tracer.trace('cassandra.query', service=service, span_type=cassx.TYPE) + span = tracer.trace('cassandra.query', service=service, span_type=SpanTypes.CASSANDRA) _sanitize_query(span, query) span.set_tags(_extract_session_metas(session)) # FIXME[matt] do once? span.set_tags(_extract_cluster_metas(cluster)) diff --git a/ddtrace/contrib/celery/signals.py b/ddtrace/contrib/celery/signals.py index 88f82f7894f..2afcce556ac 100644 --- a/ddtrace/contrib/celery/signals.py +++ b/ddtrace/contrib/celery/signals.py @@ -2,12 +2,12 @@ from celery import registry +from ...ext import SpanTypes from ...internal.logger import get_logger from . import constants as c from .utils import tags_from_context, retrieve_task_id, attach_span, detach_span, retrieve_span log = get_logger(__name__) -SPAN_TYPE = 'worker' def trace_prerun(*args, **kwargs): @@ -28,7 +28,7 @@ def trace_prerun(*args, **kwargs): # propagate the `Span` in the current task Context service = config.celery['worker_service_name'] - span = pin.tracer.trace(c.WORKER_ROOT_SPAN, service=service, resource=task.name, span_type=SPAN_TYPE) + span = pin.tracer.trace(c.WORKER_ROOT_SPAN, service=service, resource=task.name, span_type=SpanTypes.WORKER) attach_span(task, task_id, span) diff --git a/ddtrace/contrib/dbapi/__init__.py b/ddtrace/contrib/dbapi/__init__.py index 44324824001..01b34dab08c 100644 --- a/ddtrace/contrib/dbapi/__init__.py +++ b/ddtrace/contrib/dbapi/__init__.py @@ -3,7 +3,7 @@ """ from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import sql +from ...ext import SpanTypes, sql from ...internal.logger import get_logger from ...pin import Pin from ...settings import config @@ -43,8 +43,7 @@ def _trace_method(self, method, name, resource, extra_tags, *args, **kwargs): if not pin or not pin.enabled(): return method(*args, **kwargs) service = pin.service - with pin.tracer.trace(name, service=service, resource=resource) as s: - s.span_type = sql.TYPE + with pin.tracer.trace(name, service=service, resource=resource, span_type=SpanTypes.SQL) as s: # No reason to tag the query since it is set as the resource by the agent. See: # https://github.com/DataDog/datadog-trace-agent/blob/bda1ebbf170dd8c5879be993bdd4dbae70d10fda/obfuscate/sql.go#L232 s.set_tags(pin.tags) diff --git a/ddtrace/contrib/django/cache.py b/ddtrace/contrib/django/cache.py index ead51a2db83..3113a58b31d 100644 --- a/ddtrace/contrib/django/cache.py +++ b/ddtrace/contrib/django/cache.py @@ -2,6 +2,7 @@ from django.conf import settings as django_settings +from ...ext import SpanTypes from ...internal.logger import get_logger from .conf import settings, import_from_string from .utils import quantize_key_values, _resource_from_cache_prefix @@ -24,7 +25,6 @@ ] # standard tags -TYPE = 'cache' CACHE_BACKEND = 'django.cache.backend' CACHE_COMMAND_KEY = 'django.cache.key' @@ -54,7 +54,7 @@ def _trace_operation(fn, method_name): def wrapped(self, *args, **kwargs): # get the original function method method = getattr(self, DATADOG_NAMESPACE.format(method=method_name)) - with tracer.trace('django.cache', span_type=TYPE, service=cache_service_name) as span: + with tracer.trace('django.cache', span_type=SpanTypes.CACHE, service=cache_service_name) as span: # update the resource name and tag the cache backend span.resource = _resource_from_cache_prefix(method_name, self) cache_backend = '{}.{}'.format(self.__module__, self.__class__.__name__) diff --git a/ddtrace/contrib/django/middleware.py b/ddtrace/contrib/django/middleware.py index f1185cc9970..6e71d200799 100644 --- a/ddtrace/contrib/django/middleware.py +++ b/ddtrace/contrib/django/middleware.py @@ -5,7 +5,7 @@ from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...contrib import func_name -from ...ext import http +from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from ...settings import config @@ -126,7 +126,7 @@ def process_request(self, request): 'django.request', service=settings.DEFAULT_SERVICE, resource='unknown', # will be filled by process view - span_type=http.TYPE, + span_type=SpanTypes.WEB, ) # set analytics sample rate diff --git a/ddtrace/contrib/django/templates.py b/ddtrace/contrib/django/templates.py index c3717e5ec7e..27752b362f7 100644 --- a/ddtrace/contrib/django/templates.py +++ b/ddtrace/contrib/django/templates.py @@ -2,7 +2,7 @@ code to measure django template rendering. """ # project -from ...ext import http +from ...ext import SpanTypes from ...internal.logger import get_logger # 3p @@ -28,7 +28,7 @@ def patch_template(tracer): setattr(Template, RENDER_ATTR, Template.render) def traced_render(self, context): - with tracer.trace('django.template', span_type=http.TEMPLATE) as span: + with tracer.trace('django.template', span_type=SpanTypes.TEMPLATE) as span: try: return Template._datadog_original_render(self, context) finally: diff --git a/ddtrace/contrib/elasticsearch/patch.py b/ddtrace/contrib/elasticsearch/patch.py index 9e8fa980fd0..b8541d71937 100644 --- a/ddtrace/contrib/elasticsearch/patch.py +++ b/ddtrace/contrib/elasticsearch/patch.py @@ -6,7 +6,7 @@ from ...compat import urlencode from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import elasticsearch as metadata, http +from ...ext import SpanTypes, elasticsearch as metadata, http from ...pin import Pin from ...utils.wrappers import unwrap as _u from ...settings import config @@ -32,7 +32,7 @@ def _patch(elasticsearch): return setattr(elasticsearch, '_datadog_patch', True) _w(elasticsearch.transport, 'Transport.perform_request', _get_perform_request(elasticsearch)) - Pin(service=metadata.SERVICE, app=metadata.APP).onto(elasticsearch.transport.Transport) + Pin(service=metadata.SERVICE).onto(elasticsearch.transport.Transport) def unpatch(): @@ -52,7 +52,7 @@ def _perform_request(func, instance, args, kwargs): if not pin or not pin.enabled(): return func(*args, **kwargs) - with pin.tracer.trace('elasticsearch.query') as span: + with pin.tracer.trace('elasticsearch.query', span_type=SpanTypes.ELASTICSEARCH) as span: # Don't instrument if the trace is not sampled if not span.sampled: return func(*args, **kwargs) @@ -62,7 +62,6 @@ def _perform_request(func, instance, args, kwargs): body = kwargs.get('body') span.service = pin.service - span.span_type = metadata.TYPE span.set_tag(metadata.METHOD, method) span.set_tag(metadata.URL, url) span.set_tag(metadata.PARAMS, urlencode(params)) diff --git a/ddtrace/contrib/elasticsearch/transport.py b/ddtrace/contrib/elasticsearch/transport.py index c77f99de390..347f710bd6e 100644 --- a/ddtrace/contrib/elasticsearch/transport.py +++ b/ddtrace/contrib/elasticsearch/transport.py @@ -6,11 +6,10 @@ from ...utils.deprecation import deprecated from ...compat import urlencode -from ...ext import http, elasticsearch as metadata +from ...ext import SpanTypes, http, elasticsearch as metadata from ...settings import config DEFAULT_SERVICE = 'elasticsearch' -SPAN_TYPE = 'elasticsearch' @deprecated(message='Use patching instead (see the docs).', version='1.0.0') @@ -25,14 +24,13 @@ class TracedTransport(elasticsearch.Transport): _datadog_service = datadog_service def perform_request(self, method, url, params=None, body=None): - with self._datadog_tracer.trace('elasticsearch.query') as s: + with self._datadog_tracer.trace('elasticsearch.query', span_type=SpanTypes.ELASTICSEARCH) as s: # Don't instrument if the trace is not sampled if not s.sampled: return super(TracedTransport, self).perform_request( method, url, params=params, body=body) s.service = self._datadog_service - s.span_type = SPAN_TYPE s.set_tag(metadata.METHOD, method) s.set_tag(metadata.URL, url) s.set_tag(metadata.PARAMS, urlencode(params)) diff --git a/ddtrace/contrib/falcon/middleware.py b/ddtrace/contrib/falcon/middleware.py index 81a07a7f31e..e30b0704944 100644 --- a/ddtrace/contrib/falcon/middleware.py +++ b/ddtrace/contrib/falcon/middleware.py @@ -1,6 +1,6 @@ import sys -from ddtrace.ext import http as httpx +from ddtrace.ext import SpanTypes, http as httpx from ddtrace.http import store_request_headers, store_response_headers from ddtrace.propagation.http import HTTPPropagator @@ -30,7 +30,7 @@ def process_request(self, req, resp): span = self.tracer.trace( 'falcon.request', service=self.service, - span_type=httpx.TYPE, + span_type=SpanTypes.WEB, ) # set analytics sample rate with global config enabled diff --git a/ddtrace/contrib/flask/middleware.py b/ddtrace/contrib/flask/middleware.py index 9c46f428b09..fb9a45f88c5 100644 --- a/ddtrace/contrib/flask/middleware.py +++ b/ddtrace/contrib/flask/middleware.py @@ -1,5 +1,5 @@ from ... import compat -from ...ext import http, errors +from ...ext import SpanTypes, http, errors from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from ...utils.deprecation import deprecated @@ -112,7 +112,7 @@ def _start_span(self): g.flask_datadog_span = self.app._tracer.trace( SPAN_NAME, service=self.app._service, - span_type=http.TYPE, + span_type=SpanTypes.WEB, ) except Exception: log.debug('flask: error tracing request', exc_info=True) @@ -189,8 +189,7 @@ def _patch_render(tracer): _render = flask.templating._render def _traced_render(template, context, app): - with tracer.trace('flask.template') as span: - span.span_type = http.TEMPLATE + with tracer.trace('flask.template', span_type=SpanTypes.TEMPLATE) as span: span.set_tag('flask.template', template.name or 'string') return _render(template, context, app) diff --git a/ddtrace/contrib/flask/patch.py b/ddtrace/contrib/flask/patch.py index 0dc9bb2e6e5..243a8cc56bb 100644 --- a/ddtrace/contrib/flask/patch.py +++ b/ddtrace/contrib/flask/patch.py @@ -8,7 +8,7 @@ from ddtrace import config, Pin from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from ...utils.wrappers import unwrap as _u @@ -282,7 +282,7 @@ def traced_wsgi_app(pin, wrapped, instance, args, kwargs): # POST /save # We will override this below in `traced_dispatch_request` when we have a `RequestContext` and possibly a url rule resource = u'{} {}'.format(request.method, request.path) - with pin.tracer.trace('flask.request', service=pin.service, resource=resource, span_type=http.TYPE) as s: + with pin.tracer.trace('flask.request', service=pin.service, resource=resource, span_type=SpanTypes.WEB) as s: # set analytics sample rate with global config enabled sample_rate = config.flask.get_analytics_sample_rate(use_global_config=True) if sample_rate is not None: @@ -396,7 +396,7 @@ def traced_render_template(wrapped, instance, args, kwargs): if not pin or not pin.enabled(): return wrapped(*args, **kwargs) - with pin.tracer.trace('flask.render_template', span_type=http.TEMPLATE): + with pin.tracer.trace('flask.render_template', span_type=SpanTypes.TEMPLATE): return wrapped(*args, **kwargs) @@ -406,7 +406,7 @@ def traced_render_template_string(wrapped, instance, args, kwargs): if not pin or not pin.enabled(): return wrapped(*args, **kwargs) - with pin.tracer.trace('flask.render_template_string', span_type=http.TEMPLATE): + with pin.tracer.trace('flask.render_template_string', span_type=SpanTypes.TEMPLATE): return wrapped(*args, **kwargs) diff --git a/ddtrace/contrib/flask_cache/tracers.py b/ddtrace/contrib/flask_cache/tracers.py index 210fb4b3724..31c7ea9b22c 100644 --- a/ddtrace/contrib/flask_cache/tracers.py +++ b/ddtrace/contrib/flask_cache/tracers.py @@ -8,6 +8,7 @@ # project from .utils import _extract_conn_tags, _resource_from_cache_prefix from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...ext import SpanTypes from ...settings import config # 3rd party @@ -16,7 +17,6 @@ log = logging.Logger(__name__) -TYPE = 'cache' DEFAULT_SERVICE = 'flask-cache' # standard tags @@ -47,7 +47,7 @@ def __trace(self, cmd): # create a new span s = self._datadog_tracer.trace( cmd, - span_type=TYPE, + span_type=SpanTypes.CACHE, service=self._datadog_service ) # set span tags diff --git a/ddtrace/contrib/grpc/client_interceptor.py b/ddtrace/contrib/grpc/client_interceptor.py index a54c21f08ad..5aa763dc0d8 100644 --- a/ddtrace/contrib/grpc/client_interceptor.py +++ b/ddtrace/contrib/grpc/client_interceptor.py @@ -4,7 +4,7 @@ from ddtrace import config from ddtrace.compat import to_unicode -from ddtrace.ext import errors +from ddtrace.ext import SpanTypes, errors from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from ...constants import ANALYTICS_SAMPLE_RATE_KEY @@ -152,7 +152,7 @@ def _intercept_client_call(self, method_kind, client_call_details): span = tracer.trace( 'grpc', - span_type='grpc', + span_type=SpanTypes.GRPC, service=self._pin.service, resource=client_call_details.method, ) diff --git a/ddtrace/contrib/grpc/server_interceptor.py b/ddtrace/contrib/grpc/server_interceptor.py index e8898db24ca..dbce643543a 100644 --- a/ddtrace/contrib/grpc/server_interceptor.py +++ b/ddtrace/contrib/grpc/server_interceptor.py @@ -6,6 +6,7 @@ from ddtrace.compat import to_unicode from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...ext import SpanTypes from ...propagation.http import HTTPPropagator from . import constants from .utils import parse_method_path @@ -64,7 +65,7 @@ def _fn(self, method_kind, behavior, args, kwargs): span = tracer.trace( 'grpc', - span_type='grpc', + span_type=SpanTypes.GRPC, service=self._pin.service, resource=self._handler_call_details.method, ) diff --git a/ddtrace/contrib/httplib/patch.py b/ddtrace/contrib/httplib/patch.py index f401dcd39e2..61297cbd741 100644 --- a/ddtrace/contrib/httplib/patch.py +++ b/ddtrace/contrib/httplib/patch.py @@ -4,7 +4,7 @@ # Project from ...compat import PY2, httplib, parse from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http as ext_http +from ...ext import SpanTypes, http as ext_http from ...http import store_request_headers, store_response_headers from ...internal.logger import get_logger from ...pin import Pin @@ -55,7 +55,7 @@ def _wrap_putrequest(func, instance, args, kwargs): try: # Create a new span and attach to this instance (so we can retrieve/update/close later on the response) - span = pin.tracer.trace(span_name, span_type=ext_http.TYPE) + span = pin.tracer.trace(span_name, span_type=SpanTypes.HTTP) setattr(instance, '_datadog_span', span) method, path = args[:2] diff --git a/ddtrace/contrib/jinja2/patch.py b/ddtrace/contrib/jinja2/patch.py index 38a2cc137c5..988c33db9ea 100644 --- a/ddtrace/contrib/jinja2/patch.py +++ b/ddtrace/contrib/jinja2/patch.py @@ -3,7 +3,7 @@ from ddtrace import config -from ...ext import http +from ...ext import SpanTypes from ...utils.formats import get_env from ...pin import Pin from ...utils.wrappers import unwrap as _u @@ -49,7 +49,7 @@ def _wrap_render(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) template_name = instance.name or DEFAULT_TEMPLATE_NAME - with pin.tracer.trace('jinja2.render', pin.service, span_type=http.TEMPLATE) as span: + with pin.tracer.trace('jinja2.render', pin.service, span_type=SpanTypes.TEMPLATE) as span: try: return wrapped(*args, **kwargs) finally: @@ -67,7 +67,7 @@ def _wrap_compile(wrapped, instance, args, kwargs): else: template_name = kwargs.get('name', DEFAULT_TEMPLATE_NAME) - with pin.tracer.trace('jinja2.compile', pin.service, span_type=http.TEMPLATE) as span: + with pin.tracer.trace('jinja2.compile', pin.service, span_type=SpanTypes.TEMPLATE) as span: try: return wrapped(*args, **kwargs) finally: @@ -81,7 +81,7 @@ def _wrap_load_template(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) template_name = kwargs.get('name', args[0]) - with pin.tracer.trace('jinja2.load', pin.service, span_type=http.TEMPLATE) as span: + with pin.tracer.trace('jinja2.load', pin.service, span_type=SpanTypes.TEMPLATE) as span: template = None try: template = wrapped(*args, **kwargs) diff --git a/ddtrace/contrib/kombu/patch.py b/ddtrace/contrib/kombu/patch.py index 9790f20737b..a82079f3e0c 100644 --- a/ddtrace/contrib/kombu/patch.py +++ b/ddtrace/contrib/kombu/patch.py @@ -4,7 +4,7 @@ # project from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import kombu as kombux +from ...ext import SpanTypes, kombu as kombux from ...pin import Pin from ...propagation.http import HTTPPropagator from ...settings import config @@ -43,8 +43,8 @@ def patch(): # * defines defaults in its kwargs # * potentially overrides kwargs with values from self # * extracts/normalizes things like exchange - _w(kombux.TYPE, 'Producer._publish', traced_publish) - _w(kombux.TYPE, 'Consumer.receive', traced_receive) + _w('kombu', 'Producer._publish', traced_publish) + _w('kombu', 'Consumer.receive', traced_receive) Pin( service=config.kombu['service_name'], app='kombu' @@ -78,7 +78,7 @@ def traced_receive(func, instance, args, kwargs): # only need to active the new context if something was propagated if context.trace_id: pin.tracer.context_provider.activate(context) - with pin.tracer.trace(kombux.RECEIVE_NAME, service=pin.service, span_type='kombu') as s: + with pin.tracer.trace(kombux.RECEIVE_NAME, service=pin.service, span_type=SpanTypes.WORKER) as s: # run the command exchange = message.delivery_info['exchange'] s.resource = exchange @@ -99,7 +99,7 @@ def traced_publish(func, instance, args, kwargs): if not pin or not pin.enabled(): return func(*args, **kwargs) - with pin.tracer.trace(kombux.PUBLISH_NAME, service=pin.service, span_type='kombu') as s: + with pin.tracer.trace(kombux.PUBLISH_NAME, service=pin.service, span_type=SpanTypes.WORKER) as s: exchange_name = get_exchange_from_args(args) s.resource = exchange_name s.set_tag(kombux.EXCHANGE, exchange_name) diff --git a/ddtrace/contrib/mako/patch.py b/ddtrace/contrib/mako/patch.py index fa574ab82d1..5f6da9c2c45 100644 --- a/ddtrace/contrib/mako/patch.py +++ b/ddtrace/contrib/mako/patch.py @@ -1,7 +1,7 @@ import mako from mako.template import Template -from ...ext import http +from ...ext import SpanTypes from ...pin import Pin from ...utils.importlib import func_name from ...utils.wrappers import unwrap as _u @@ -38,7 +38,7 @@ def _wrap_render(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) template_name = instance.filename or DEFAULT_TEMPLATE_NAME - with pin.tracer.trace(func_name(wrapped), pin.service, span_type=http.TEMPLATE) as span: + with pin.tracer.trace(func_name(wrapped), pin.service, span_type=SpanTypes.TEMPLATE) as span: try: template = wrapped(*args, **kwargs) return template diff --git a/ddtrace/contrib/molten/patch.py b/ddtrace/contrib/molten/patch.py index 62003cea7b5..967d06899f5 100644 --- a/ddtrace/contrib/molten/patch.py +++ b/ddtrace/contrib/molten/patch.py @@ -6,7 +6,7 @@ from ... import Pin, config from ...compat import urlencode from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...utils.formats import asbool, get_env from ...utils.importlib import func_name @@ -82,7 +82,7 @@ def patch_app_call(wrapped, instance, args, kwargs): if context.trace_id: pin.tracer.context_provider.activate(context) - with pin.tracer.trace('molten.request', service=pin.service, resource=resource) as span: + with pin.tracer.trace('molten.request', service=pin.service, resource=resource, span_type=SpanTypes.WEB) as span: # set analytics sample rate with global config enabled span.set_tag( ANALYTICS_SAMPLE_RATE_KEY, diff --git a/ddtrace/contrib/mongoengine/trace.py b/ddtrace/contrib/mongoengine/trace.py index 78667789f27..e219bf99300 100644 --- a/ddtrace/contrib/mongoengine/trace.py +++ b/ddtrace/contrib/mongoengine/trace.py @@ -17,7 +17,7 @@ class WrappedConnect(wrapt.ObjectProxy): def __init__(self, connect): super(WrappedConnect, self).__init__(connect) - ddtrace.Pin(service=mongox.TYPE, tracer=ddtrace.tracer).onto(self) + ddtrace.Pin(service=mongox.SERVICE, tracer=ddtrace.tracer).onto(self) def __call__(self, *args, **kwargs): client = self.__wrapped__(*args, **kwargs) diff --git a/ddtrace/contrib/psycopg/connection.py b/ddtrace/contrib/psycopg/connection.py index 2b20cf1efb1..d3e4eb6e955 100644 --- a/ddtrace/contrib/psycopg/connection.py +++ b/ddtrace/contrib/psycopg/connection.py @@ -5,9 +5,7 @@ # stdlib import functools -from ...ext import db -from ...ext import net -from ...ext import sql +from ...ext import SpanTypes, db, net, sql from ...utils.deprecation import deprecated # 3p @@ -44,12 +42,11 @@ def execute(self, query, vars=None): # noqa: A002 if not self._datadog_tracer: return cursor.execute(self, query, vars) - with self._datadog_tracer.trace('postgres.query', service=self._datadog_service) as s: + with self._datadog_tracer.trace('postgres.query', service=self._datadog_service, span_type=SpanTypes.SQL) as s: if not s.sampled: return super(TracedCursor, self).execute(query, vars) s.resource = query - s.span_type = sql.TYPE s.set_tags(self._datadog_tags) try: return super(TracedCursor, self).execute(query, vars) diff --git a/ddtrace/contrib/pylibmc/client.py b/ddtrace/contrib/pylibmc/client.py index 04cc0f7a885..415a0ef9f49 100644 --- a/ddtrace/contrib/pylibmc/client.py +++ b/ddtrace/contrib/pylibmc/client.py @@ -8,8 +8,7 @@ # project import ddtrace from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import memcached -from ...ext import net +from ...ext import SpanTypes, memcached, net from ...internal.logger import get_logger from ...settings import config from .addrs import parse_addresses @@ -136,8 +135,7 @@ def _span(self, cmd_name): 'memcached.cmd', service=pin.service, resource=cmd_name, - # TODO(Benjamin): set a better span type - span_type='cache') + span_type=SpanTypes.CACHE) try: self._tag_span(span) diff --git a/ddtrace/contrib/pylons/middleware.py b/ddtrace/contrib/pylons/middleware.py index 5fe5aa5cf5d..2d2d5df359b 100644 --- a/ddtrace/contrib/pylons/middleware.py +++ b/ddtrace/contrib/pylons/middleware.py @@ -8,7 +8,7 @@ from ...compat import reraise from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from ...settings import config as ddconfig @@ -41,10 +41,9 @@ def __call__(self, environ, start_response): if context.trace_id: self._tracer.context_provider.activate(context) - with self._tracer.trace('pylons.request', service=self._service) as span: + with self._tracer.trace('pylons.request', service=self._service, span_type=SpanTypes.WEB) as span: # Set the service in tracer.trace() as priority sampling requires it to be # set as early as possible when different services share one single agent. - span.span_type = http.TYPE # set analytics sample rate with global config enabled span.set_tag( diff --git a/ddtrace/contrib/pymemcache/client.py b/ddtrace/contrib/pymemcache/client.py index fded1450f2b..891425814b0 100644 --- a/ddtrace/contrib/pymemcache/client.py +++ b/ddtrace/contrib/pymemcache/client.py @@ -15,7 +15,7 @@ # project from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...compat import reraise -from ...ext import net, memcached as memcachedx +from ...ext import SpanTypes, net, memcached as memcachedx from ...internal.logger import get_logger from ...pin import Pin from ...settings import config @@ -141,7 +141,7 @@ def _traced_cmd(self, method_name, *args, **kwargs): memcachedx.CMD, service=p.service, resource=method_name, - span_type=memcachedx.TYPE, + span_type=SpanTypes.CACHE, ) as span: # set analytics sample rate span.set_tag( diff --git a/ddtrace/contrib/pymongo/client.py b/ddtrace/contrib/pymongo/client.py index 25eea416670..3aa5d22320a 100644 --- a/ddtrace/contrib/pymongo/client.py +++ b/ddtrace/contrib/pymongo/client.py @@ -10,8 +10,7 @@ import ddtrace from ...compat import iteritems from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import mongo as mongox -from ...ext import net as netx +from ...ext import SpanTypes, mongo as mongox, net as netx from ...internal.logger import get_logger from ...settings import config from ...utils.deprecation import deprecated @@ -24,7 +23,7 @@ @deprecated(message='Use patching instead (see the docs).', version='1.0.0') -def trace_mongo_client(client, tracer, service=mongox.TYPE): +def trace_mongo_client(client, tracer, service=mongox.SERVICE): traced_client = TracedMongoClient(client) ddtrace.Pin(service=service, tracer=tracer).onto(traced_client) return traced_client @@ -61,7 +60,7 @@ def __init__(self, client=None, *args, **kwargs): client._topology = TracedTopology(client._topology) # Default Pin - ddtrace.Pin(service=mongox.TYPE, app=mongox.TYPE).onto(self) + ddtrace.Pin(service=mongox.SERVICE, app=mongox.SERVICE).onto(self) def __setddpin__(self, pin): pin.onto(self._topology) @@ -103,7 +102,7 @@ def _datadog_trace_operation(self, operation): if not cmd or not pin or not pin.enabled(): return None - span = pin.tracer.trace('pymongo.cmd', span_type=mongox.TYPE, service=pin.service) + span = pin.tracer.trace('pymongo.cmd', span_type=SpanTypes.MONGODB, service=pin.service) span.set_tag(mongox.DB, cmd.db) span.set_tag(mongox.COLLECTION, cmd.coll) span.set_tags(cmd.tags) @@ -222,7 +221,7 @@ def __trace(self, cmd): pin = ddtrace.Pin.get_from(self) s = pin.tracer.trace( 'pymongo.cmd', - span_type=mongox.TYPE, + span_type=SpanTypes.MONGODB, service=pin.service) if cmd.db: diff --git a/ddtrace/contrib/pyramid/trace.py b/ddtrace/contrib/pyramid/trace.py index c6cf08b5946..029352759b2 100644 --- a/ddtrace/contrib/pyramid/trace.py +++ b/ddtrace/contrib/pyramid/trace.py @@ -6,7 +6,7 @@ # project import ddtrace from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from ...settings import config @@ -49,8 +49,7 @@ def trace_render(func, instance, args, kwargs): log.debug('No span found in request, will not be traced') return func(*args, **kwargs) - with span.tracer.trace('pyramid.render') as span: - span.span_type = http.TEMPLATE + with span.tracer.trace('pyramid.render', span_type=SpanTypes.TEMPLATE) as span: return func(*args, **kwargs) @@ -71,7 +70,7 @@ def trace_tween(request): # only need to active the new context if something was propagated if context.trace_id: tracer.context_provider.activate(context) - with tracer.trace('pyramid.request', service=service, resource='404') as span: + with tracer.trace('pyramid.request', service=service, resource='404', span_type=SpanTypes.WEB) as span: # Configure trace search sample rate # DEV: pyramid is special case maintains separate configuration from config api analytics_enabled = settings.get(SETTINGS_ANALYTICS_ENABLED) @@ -100,7 +99,6 @@ def trace_tween(request): span.set_tag(http.STATUS_CODE, 500) raise finally: - span.span_type = http.TYPE # set request tags span.set_tag(http.URL, request.path_url) span.set_tag(http.METHOD, request.method) diff --git a/ddtrace/contrib/redis/patch.py b/ddtrace/contrib/redis/patch.py index 51361c390e6..e1dddd2c9ad 100644 --- a/ddtrace/contrib/redis/patch.py +++ b/ddtrace/contrib/redis/patch.py @@ -6,7 +6,7 @@ from ddtrace import config from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import redis as redisx +from ...ext import SpanTypes, redis as redisx from ...utils.wrappers import unwrap from .util import format_command_args, _extract_conn_tags @@ -62,7 +62,7 @@ def traced_execute_command(func, instance, args, kwargs): if not pin or not pin.enabled(): return func(*args, **kwargs) - with pin.tracer.trace(redisx.CMD, service=pin.service, span_type=redisx.TYPE) as s: + with pin.tracer.trace(redisx.CMD, service=pin.service, span_type=SpanTypes.REDIS) as s: query = format_command_args(args) s.resource = query s.set_tag(redisx.RAWCMD, query) @@ -96,8 +96,7 @@ def traced_execute_pipeline(func, instance, args, kwargs): cmds = [format_command_args(c) for c, _ in instance.command_stack] resource = '\n'.join(cmds) tracer = pin.tracer - with tracer.trace(redisx.CMD, resource=resource, service=pin.service) as s: - s.span_type = redisx.TYPE + with tracer.trace(redisx.CMD, resource=resource, service=pin.service, span_type=SpanTypes.REDIS) as s: s.set_tag(redisx.RAWCMD, resource) s.set_tags(_get_tags(instance)) s.set_metric(redisx.PIPELINE_LEN, len(instance.command_stack)) diff --git a/ddtrace/contrib/rediscluster/patch.py b/ddtrace/contrib/rediscluster/patch.py index 66c2d59c960..b224174dcd1 100644 --- a/ddtrace/contrib/rediscluster/patch.py +++ b/ddtrace/contrib/rediscluster/patch.py @@ -6,7 +6,7 @@ from ddtrace import config from ...constants import ANALYTICS_SAMPLE_RATE_KEY from ...pin import Pin -from ...ext import redis as redisx +from ...ext import SpanTypes, redis as redisx from ...utils.wrappers import unwrap from ..redis.patch import traced_execute_command, traced_pipeline from ..redis.util import format_command_args @@ -46,8 +46,7 @@ def traced_execute_pipeline(func, instance, args, kwargs): cmds = [format_command_args(c.args) for c in instance.command_stack] resource = '\n'.join(cmds) tracer = pin.tracer - with tracer.trace(redisx.CMD, resource=resource, service=pin.service) as s: - s.span_type = redisx.TYPE + with tracer.trace(redisx.CMD, resource=resource, service=pin.service, span_type=SpanTypes.REDIS) as s: s.set_tag(redisx.RAWCMD, resource) s.set_metric(redisx.PIPELINE_LEN, len(instance.command_stack)) diff --git a/ddtrace/contrib/requests/connection.py b/ddtrace/contrib/requests/connection.py index 960e0e4e148..e1536599e80 100644 --- a/ddtrace/contrib/requests/connection.py +++ b/ddtrace/contrib/requests/connection.py @@ -4,7 +4,7 @@ from ...compat import parse from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator from .constants import DEFAULT_SERVICE @@ -67,7 +67,7 @@ def _wrap_send(func, instance, args, kwargs): parsed_uri.fragment )) - with tracer.trace('requests.request', span_type=http.TYPE) as span: + with tracer.trace('requests.request', span_type=SpanTypes.HTTP) as span: # update the span service name before doing any action span.service = _extract_service_name(instance, span, hostname=hostname) diff --git a/ddtrace/contrib/sqlalchemy/engine.py b/ddtrace/contrib/sqlalchemy/engine.py index 5f3f3c2c6b1..06308e8110d 100644 --- a/ddtrace/contrib/sqlalchemy/engine.py +++ b/ddtrace/contrib/sqlalchemy/engine.py @@ -18,8 +18,7 @@ import ddtrace from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import sql as sqlx -from ...ext import net as netx +from ...ext import SpanTypes, sql as sqlx, net as netx from ...pin import Pin from ...settings import config @@ -79,7 +78,7 @@ def _before_cur_exec(self, conn, cursor, statement, *args): span = pin.tracer.trace( self.name, service=pin.service, - span_type=sqlx.TYPE, + span_type=SpanTypes.SQL, resource=statement, ) diff --git a/ddtrace/contrib/tornado/handlers.py b/ddtrace/contrib/tornado/handlers.py index 4b17eeaa940..2699c2e15c2 100644 --- a/ddtrace/contrib/tornado/handlers.py +++ b/ddtrace/contrib/tornado/handlers.py @@ -3,7 +3,7 @@ from .constants import CONFIG_KEY, REQUEST_CONTEXT_KEY, REQUEST_SPAN_KEY from .stack_context import TracerStackContext from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import http +from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...settings import config @@ -35,7 +35,7 @@ def execute(func, handler, args, kwargs): request_span = tracer.trace( 'tornado.request', service=service, - span_type=http.TYPE + span_type=SpanTypes.WEB ) # set analytics sample rate # DEV: tornado is special case maintains separate configuration from config api diff --git a/ddtrace/contrib/tornado/template.py b/ddtrace/contrib/tornado/template.py index 885bbf1bc76..d41bc601aad 100644 --- a/ddtrace/contrib/tornado/template.py +++ b/ddtrace/contrib/tornado/template.py @@ -2,7 +2,7 @@ from ddtrace import Pin -from ...ext import http +from ...ext import SpanTypes def generate(func, renderer, args, kwargs): @@ -24,8 +24,8 @@ def generate(func, renderer, args, kwargs): resource = template_name = renderer.name # trace the original call - with pin.tracer.trace('tornado.template', service=pin.service) as span: - span.span_type = http.TEMPLATE - span.resource = resource + with pin.tracer.trace( + 'tornado.template', service=pin.service, resource=resource, span_type=SpanTypes.TEMPLATE + ) as span: span.set_meta('tornado.template_name', template_name) return func(*args, **kwargs) diff --git a/ddtrace/contrib/vertica/patch.py b/ddtrace/contrib/vertica/patch.py index b2a09c394c6..f893b94b6fd 100644 --- a/ddtrace/contrib/vertica/patch.py +++ b/ddtrace/contrib/vertica/patch.py @@ -4,7 +4,7 @@ import ddtrace from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...ext import db as dbx, sql +from ...ext import SpanTypes, db as dbx from ...ext import net from ...internal.logger import get_logger from ...pin import Pin @@ -71,28 +71,28 @@ def cursor_span_end(instance, cursor, _, conf, *args, **kwargs): 'routines': { 'execute': { 'operation_name': 'vertica.query', - 'span_type': sql.TYPE, + 'span_type': SpanTypes.SQL, 'span_start': execute_span_start, 'span_end': execute_span_end, }, 'copy': { 'operation_name': 'vertica.copy', - 'span_type': sql.TYPE, + 'span_type': SpanTypes.SQL, 'span_start': copy_span_start, }, 'fetchone': { 'operation_name': 'vertica.fetchone', - 'span_type': 'vertica', + 'span_type': SpanTypes.SQL, 'span_end': fetch_span_end, }, 'fetchall': { 'operation_name': 'vertica.fetchall', - 'span_type': 'vertica', + 'span_type': SpanTypes.SQL, 'span_end': fetch_span_end, }, 'nextset': { 'operation_name': 'vertica.nextset', - 'span_type': 'vertica', + 'span_type': SpanTypes.SQL, 'span_end': fetch_span_end, }, }, @@ -198,10 +198,8 @@ def wrapper(wrapped, instance, args, kwargs): operation_name = conf['operation_name'] tracer = pin.tracer - with tracer.trace(operation_name, service=pin.service) as span: + with tracer.trace(operation_name, service=pin.service, span_type=conf.get('span_type')) as span: span.set_tags(pin.tags) - if 'span_type' in conf: - span.span_type = conf['span_type'] if 'span_start' in conf: conf['span_start'](instance, span, conf, *args, **kwargs) diff --git a/ddtrace/ext/__init__.py b/ddtrace/ext/__init__.py index e69de29bb2d..1568dc71126 100644 --- a/ddtrace/ext/__init__.py +++ b/ddtrace/ext/__init__.py @@ -0,0 +1,15 @@ +from enum import Enum + + +class SpanTypes(Enum): + CACHE = "cache" + CASSANDRA = "cassandra" + ELASTICSEARCH = "elasticsearch" + GRPC = "grpc" + HTTP = "http" + MONGODB = "mongodb" + REDIS = "redis" + SQL = "sql" + TEMPLATE = "template" + WEB = "web" + WORKER = "worker" diff --git a/ddtrace/ext/cassandra.py b/ddtrace/ext/cassandra.py index 9dbd836ece1..dd8d03d15ba 100644 --- a/ddtrace/ext/cassandra.py +++ b/ddtrace/ext/cassandra.py @@ -1,7 +1,3 @@ - -# the type of the spans -TYPE = 'cassandra' - # tags CLUSTER = 'cassandra.cluster' KEYSPACE = 'cassandra.keyspace' diff --git a/ddtrace/ext/elasticsearch.py b/ddtrace/ext/elasticsearch.py index e9737cd161f..aedd665fc63 100644 --- a/ddtrace/ext/elasticsearch.py +++ b/ddtrace/ext/elasticsearch.py @@ -1,7 +1,4 @@ -TYPE = 'elasticsearch' -SERVICE = 'elasticsearch' -APP = 'elasticsearch' - +SERVICE = "elasticsearch" # standard tags URL = 'elasticsearch.url' METHOD = 'elasticsearch.method' diff --git a/ddtrace/ext/http.py b/ddtrace/ext/http.py index ab31321df9b..acc8a65aa9f 100644 --- a/ddtrace/ext/http.py +++ b/ddtrace/ext/http.py @@ -7,9 +7,6 @@ span.set_tag(STATUS_CODE, 404) """ -# type of the spans -TYPE = 'http' - # tags URL = 'http.url' METHOD = 'http.method' diff --git a/ddtrace/ext/kombu.py b/ddtrace/ext/kombu.py index 22e4ac6421d..fbf9bfda6c9 100644 --- a/ddtrace/ext/kombu.py +++ b/ddtrace/ext/kombu.py @@ -1,5 +1,4 @@ -# type of the spans -TYPE = 'kombu' +SERVICE = 'kombu' # net extension VHOST = 'out.vhost' diff --git a/ddtrace/ext/memcached.py b/ddtrace/ext/memcached.py index ef3bab4e718..64f03fd7979 100644 --- a/ddtrace/ext/memcached.py +++ b/ddtrace/ext/memcached.py @@ -1,4 +1,3 @@ CMD = 'memcached.command' SERVICE = 'memcached' -TYPE = 'memcached' QUERY = 'memcached.query' diff --git a/ddtrace/ext/mongo.py b/ddtrace/ext/mongo.py index 5a536d3cd57..f145bb60ecc 100644 --- a/ddtrace/ext/mongo.py +++ b/ddtrace/ext/mongo.py @@ -1,5 +1,4 @@ -TYPE = 'mongodb' - +SERVICE = 'mongodb' COLLECTION = 'mongodb.collection' DB = 'mongodb.db' ROWS = 'mongodb.rows' diff --git a/ddtrace/ext/redis.py b/ddtrace/ext/redis.py index fb83d4cc8fb..a512e8a694e 100644 --- a/ddtrace/ext/redis.py +++ b/ddtrace/ext/redis.py @@ -2,9 +2,6 @@ APP = 'redis' DEFAULT_SERVICE = 'redis' -# type of the spans -TYPE = 'redis' - # net extension DB = 'out.redis_db' diff --git a/ddtrace/ext/sql.py b/ddtrace/ext/sql.py index b65c5e87a83..6aec7a60231 100644 --- a/ddtrace/ext/sql.py +++ b/ddtrace/ext/sql.py @@ -1,6 +1,3 @@ -# the type of the spans -TYPE = 'sql' - # tags QUERY = 'sql.query' # the query text ROWS = 'sql.rows' # number of rows returned by a query diff --git a/ddtrace/span.py b/ddtrace/span.py index 09b2f68a497..c4dd490b3b6 100644 --- a/ddtrace/span.py +++ b/ddtrace/span.py @@ -5,7 +5,7 @@ from .compat import StringIO, stringify, iteritems, numeric_types, time_ns from .constants import NUMERIC_TAGS, MANUAL_DROP_KEY, MANUAL_KEEP_KEY -from .ext import errors, priority +from .ext import SpanTypes, errors, priority from .internal.logger import get_logger @@ -74,7 +74,7 @@ def __init__( self.name = name self.service = service self.resource = resource or name - self.span_type = span_type + self.span_type = span_type.value if isinstance(span_type, SpanTypes) else span_type # tags / metatdata self.meta = {} diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index d7759a57645..d74d0c41db0 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -44,6 +44,14 @@ def _parse_dogstatsd_url(url): raise ValueError('Unknown scheme `%s` for DogStatsD URL `{}`'.format(parsed.scheme)) +_INTERNAL_APPLICATION_SPAN_TYPES = [ + "custom", + "template", + "web", + "worker" +] + + class Tracer(object): """ Tracer is used to create, sample and submit spans that measure the @@ -368,7 +376,8 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type context.sampling_priority = AUTO_KEEP if span.sampled else AUTO_REJECT # add tags to root span to correlate trace with runtime metrics - if self._runtime_worker: + # only applied to spans with types that are internal to applications + if self._runtime_worker and span.span_type in _INTERNAL_APPLICATION_SPAN_TYPES: span.set_tag('language', 'python') # add common tags diff --git a/setup.py b/setup.py index cc6ba0e8b3d..07689f1b577 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,17 @@ def run_tests(self): [visualization docs]: https://docs.datadoghq.com/tracing/visualization/ """ +# psutil used to generate runtime metrics for tracer +install_requires = [ + 'psutil>=5.0.0' +] + +# include enum backport +if sys.version_info[:2] < (3, 4): + install_requires.extend([ + 'enum34' + ]) + # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( name='ddtrace', @@ -66,9 +77,7 @@ def run_tests(self): long_description_content_type='text/markdown', license='BSD', packages=find_packages(exclude=['tests*']), - install_requires=[ - 'psutil>=5.0.0', - ], + install_requires=install_requires, extras_require={ # users can include opentracing by having: # install_requires=['ddtrace[opentracing]', ...] diff --git a/tests/contrib/aiohttp/test_middleware.py b/tests/contrib/aiohttp/test_middleware.py index b4f3b10bf31..e49504243d6 100644 --- a/tests/contrib/aiohttp/test_middleware.py +++ b/tests/contrib/aiohttp/test_middleware.py @@ -38,7 +38,7 @@ def test_handler(self): # with the right fields assert 'aiohttp.request' == span.name assert 'aiohttp-web' == span.service - assert 'http' == span.span_type + assert 'web' == span.span_type assert 'GET /' == span.resource assert str(self.client.make_url('/')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') @@ -417,7 +417,7 @@ def _assert_200_parenting(self, traces): # with the right fields assert 'aiohttp.request' == inner_span.name assert 'aiohttp-web' == inner_span.service - assert 'http' == inner_span.span_type + assert 'web' == inner_span.span_type assert 'GET /' == inner_span.resource assert str(self.client.make_url('/')) == inner_span.get_tag(http.URL) assert 'GET' == inner_span.get_tag('http.method') diff --git a/tests/contrib/algoliasearch/test.py b/tests/contrib/algoliasearch/test.py index c717b4e6269..45ecb0db3bc 100644 --- a/tests/contrib/algoliasearch/test.py +++ b/tests/contrib/algoliasearch/test.py @@ -1,6 +1,5 @@ from ddtrace import config, patch_all -from ddtrace.contrib.algoliasearch.patch import (SEARCH_SPAN_TYPE, patch, - unpatch, algoliasearch_version) +from ddtrace.contrib.algoliasearch.patch import (patch, unpatch, algoliasearch_version) from ddtrace.pin import Pin from tests.base import BaseTracerTestCase @@ -74,7 +73,7 @@ def test_algoliasearch(self): span = spans[0] assert span.service == 'algoliasearch' assert span.name == 'algoliasearch.search' - assert span.span_type == SEARCH_SPAN_TYPE + assert span.span_type is None assert span.error == 0 assert span.get_tag('query.args.attributes_to_retrieve') == 'firstname,lastname' # Verify that adding new arguments to the search API will simply be ignored and not cause diff --git a/tests/contrib/boto/test.py b/tests/contrib/boto/test.py index 57ae25d3b68..3cb18fde721 100644 --- a/tests/contrib/boto/test.py +++ b/tests/contrib/boto/test.py @@ -59,7 +59,7 @@ def test_ec2_client(self): self.assertEqual(span.service, 'test-boto-tracing.ec2') self.assertEqual(span.resource, 'ec2.runinstances') self.assertEqual(span.name, 'ec2.command') - self.assertEqual(span.span_type, 'boto') + self.assertEqual(span.span_type, 'http') @mock_ec2 def test_analytics_enabled_with_rate(self): diff --git a/tests/contrib/cassandra/test.py b/tests/contrib/cassandra/test.py index 6ab1878f3e7..f2e3544a71f 100644 --- a/tests/contrib/cassandra/test.py +++ b/tests/contrib/cassandra/test.py @@ -121,7 +121,7 @@ def _test_query_base(self, execute_fn): query = spans[0] assert query.service == self.TEST_SERVICE assert query.resource == self.TEST_QUERY - assert query.span_type == cassx.TYPE + assert query.span_type == 'cassandra' assert query.get_tag(cassx.KEYSPACE) == self.TEST_KEYSPACE assert query.get_tag(net.TARGET_PORT) == self.TEST_PORT @@ -201,7 +201,7 @@ def execute_fn(session, query): assert dd_span.service == self.TEST_SERVICE assert dd_span.resource == self.TEST_QUERY - assert dd_span.span_type == cassx.TYPE + assert dd_span.span_type == 'cassandra' assert dd_span.get_tag(cassx.KEYSPACE) == self.TEST_KEYSPACE assert dd_span.get_tag(net.TARGET_PORT) == self.TEST_PORT @@ -259,7 +259,7 @@ def test_paginated_query(self): query = spans[i] assert query.service == self.TEST_SERVICE assert query.resource == self.TEST_QUERY_PAGINATED - assert query.span_type == cassx.TYPE + assert query.span_type == 'cassandra' assert query.get_tag(cassx.KEYSPACE) == self.TEST_KEYSPACE assert query.get_tag(net.TARGET_PORT) == self.TEST_PORT diff --git a/tests/contrib/django/test_middleware.py b/tests/contrib/django/test_middleware.py index 35b24ce5e5a..e07e0b3b86f 100644 --- a/tests/contrib/django/test_middleware.py +++ b/tests/contrib/django/test_middleware.py @@ -40,7 +40,7 @@ def test_middleware_trace_request(self, query_string=''): assert sp_request.get_tag(http.URL) == 'http://testserver/users/' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' - assert sp_request.span_type == 'http' + assert sp_request.span_type == 'web' assert sp_request.resource == 'tests.contrib.django.app.views.UserList' if config.django.trace_query_string: assert sp_request.get_tag(http.QUERY_STRING) == query_string @@ -455,5 +455,5 @@ def test_middleware_trace_request_404(self): assert sp_request.get_tag(http.URL) == 'http://testserver/unknown-url' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' - assert sp_request.span_type == 'http' + assert sp_request.span_type == 'web' assert sp_request.resource == 'django.views.defaults.page_not_found' diff --git a/tests/contrib/djangorestframework/test_djangorestframework.py b/tests/contrib/djangorestframework/test_djangorestframework.py index 021a4c13659..8cc14067f12 100644 --- a/tests/contrib/djangorestframework/test_djangorestframework.py +++ b/tests/contrib/djangorestframework/test_djangorestframework.py @@ -37,7 +37,7 @@ def test_unpatch(self): assert sp.name == 'django.request' assert sp.resource == 'tests.contrib.djangorestframework.app.views.UserViewSet' assert sp.error == 0 - assert sp.span_type == 'http' + assert sp.span_type == 'web' assert sp.get_tag('http.status_code') == '500' assert sp.get_tag('error.msg') is None @@ -54,7 +54,7 @@ def test_trace_exceptions(self): assert sp.name == 'django.request' assert sp.resource == 'tests.contrib.djangorestframework.app.views.UserViewSet' assert sp.error == 1 - assert sp.span_type == 'http' + assert sp.span_type == 'web' assert sp.get_tag('http.method') == 'GET' assert sp.get_tag('http.status_code') == '500' assert sp.get_tag('error.msg') == 'Authentication credentials were not provided.' diff --git a/tests/contrib/falcon/test_suite.py b/tests/contrib/falcon/test_suite.py index 1e07ef5199d..06ba5a2df2e 100644 --- a/tests/contrib/falcon/test_suite.py +++ b/tests/contrib/falcon/test_suite.py @@ -65,7 +65,7 @@ def test_200(self, query_string=''): else: assert httpx.QUERY_STRING not in span.meta assert span.parent_id is None - assert span.span_type == 'http' + assert span.span_type == 'web' def test_200_qs(self): return self.test_200('foo=bar') diff --git a/tests/contrib/flask/test_request.py b/tests/contrib/flask/test_request.py index aacfd6c4945..03f20dad39f 100644 --- a/tests/contrib/flask/test_request.py +++ b/tests/contrib/flask/test_request.py @@ -55,7 +55,7 @@ def index(): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 0) self.assertIsNone(req_span.parent_id) @@ -293,7 +293,7 @@ def index(): self.assertEqual(req_span.name, 'flask.request') # Note: contains no query string self.assertEqual(req_span.resource, 'GET /') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 0) self.assertIsNone(req_span.parent_id) @@ -360,7 +360,7 @@ def unicode(): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, u'GET /üŋïĉóđē') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 0) self.assertIsNone(req_span.parent_id) @@ -420,7 +420,7 @@ def test_request_404(self): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET 404') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 0) self.assertIsNone(req_span.parent_id) @@ -485,7 +485,7 @@ def not_found(): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /not-found') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 0) self.assertIsNone(req_span.parent_id) @@ -562,7 +562,7 @@ def fivehundred(): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /500') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 1) self.assertIsNone(req_span.parent_id) @@ -650,7 +650,7 @@ def fivehundredone(): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /501') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 1) self.assertIsNone(req_span.parent_id) @@ -762,7 +762,7 @@ def fivehundred(): self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /500') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 1) self.assertIsNone(req_span.parent_id) diff --git a/tests/contrib/flask_autopatch/test_flask_autopatch.py b/tests/contrib/flask_autopatch/test_flask_autopatch.py index 23bca594e0b..0119d6f3708 100644 --- a/tests/contrib/flask_autopatch/test_flask_autopatch.py +++ b/tests/contrib/flask_autopatch/test_flask_autopatch.py @@ -76,7 +76,7 @@ def index(): self.assertEqual(req_span.service, 'test-flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /') - self.assertEqual(req_span.span_type, 'http') + self.assertEqual(req_span.span_type, 'web') self.assertEqual(req_span.error, 0) self.assertIsNone(req_span.parent_id) diff --git a/tests/contrib/flask_cache/test.py b/tests/contrib/flask_cache/test.py index a41bf21ef5c..b8c256f00a3 100644 --- a/tests/contrib/flask_cache/test.py +++ b/tests/contrib/flask_cache/test.py @@ -4,7 +4,7 @@ from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY from ddtrace.ext import net from ddtrace.contrib.flask_cache import get_traced_cache -from ddtrace.contrib.flask_cache.tracers import TYPE, CACHE_BACKEND +from ddtrace.contrib.flask_cache.tracers import CACHE_BACKEND # 3rd party from flask import Flask @@ -176,7 +176,7 @@ def test_default_span_tags(self): # test tags and attributes with self.cache._TracedCache__trace('flask_cache.cmd') as span: self.assertEqual(span.service, self.SERVICE) - self.assertEqual(span.span_type, TYPE) + self.assertEqual(span.span_type, 'cache') self.assertEqual(span.meta[CACHE_BACKEND], 'simple') self.assertTrue(net.TARGET_HOST not in span.meta) self.assertTrue(net.TARGET_PORT not in span.meta) @@ -193,7 +193,7 @@ def test_default_span_tags_for_redis(self): # test tags and attributes with cache._TracedCache__trace('flask_cache.cmd') as span: self.assertEqual(span.service, self.SERVICE) - self.assertEqual(span.span_type, TYPE) + self.assertEqual(span.span_type, 'cache') self.assertEqual(span.meta[CACHE_BACKEND], 'redis') self.assertEqual(span.meta[net.TARGET_HOST], 'localhost') self.assertEqual(span.meta[net.TARGET_PORT], self.TEST_REDIS_PORT) @@ -210,7 +210,7 @@ def test_default_span_tags_memcached(self): # test tags and attributes with cache._TracedCache__trace('flask_cache.cmd') as span: self.assertEqual(span.service, self.SERVICE) - self.assertEqual(span.span_type, TYPE) + self.assertEqual(span.span_type, 'cache') self.assertEqual(span.meta[CACHE_BACKEND], 'memcached') self.assertEqual(span.meta[net.TARGET_HOST], '127.0.0.1') self.assertEqual(span.meta[net.TARGET_PORT], self.TEST_MEMCACHED_PORT) diff --git a/tests/contrib/kombu/test.py b/tests/contrib/kombu/test.py index 94d8487a2ea..82fd8a93ab5 100644 --- a/tests/contrib/kombu/test.py +++ b/tests/contrib/kombu/test.py @@ -69,7 +69,7 @@ def _assert_spans(self): consumer_span = spans[0] self.assertEqual(consumer_span.service, self.TEST_SERVICE) self.assertEqual(consumer_span.name, kombux.PUBLISH_NAME) - self.assertEqual(consumer_span.span_type, 'kombu') + self.assertEqual(consumer_span.span_type, 'worker') self.assertEqual(consumer_span.error, 0) self.assertEqual(consumer_span.get_tag('out.vhost'), '/') self.assertEqual(consumer_span.get_tag('out.host'), '127.0.0.1') @@ -81,7 +81,7 @@ def _assert_spans(self): producer_span = spans[1] self.assertEqual(producer_span.service, self.TEST_SERVICE) self.assertEqual(producer_span.name, kombux.RECEIVE_NAME) - self.assertEqual(producer_span.span_type, 'kombu') + self.assertEqual(producer_span.span_type, 'worker') self.assertEqual(producer_span.error, 0) self.assertEqual(producer_span.get_tag('kombu.exchange'), u'tasks') self.assertEqual(producer_span.get_tag('kombu.routing_key'), u'tasks') diff --git a/tests/contrib/molten/test_molten.py b/tests/contrib/molten/test_molten.py index 47b87c4d5ab..1c3929b8a0b 100644 --- a/tests/contrib/molten/test_molten.py +++ b/tests/contrib/molten/test_molten.py @@ -48,6 +48,7 @@ def test_route_success(self): span = spans[0] self.assertEqual(span.service, 'molten') self.assertEqual(span.name, 'molten.request') + self.assertEqual(span.span_type, 'web') self.assertEqual(span.resource, 'GET /hello/{name}/{age}') self.assertEqual(span.get_tag('http.method'), 'GET') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/hello/Jim/24') diff --git a/tests/contrib/mongoengine/test.py b/tests/contrib/mongoengine/test.py index 11bc7bf4946..fad28a549f2 100644 --- a/tests/contrib/mongoengine/test.py +++ b/tests/contrib/mongoengine/test.py @@ -192,7 +192,7 @@ def test_analytics_without_rate(self): class TestMongoEnginePatchConnectDefault(unittest.TestCase, MongoEngineCore): """Test suite with a global Pin for the connect function with the default configuration""" - TEST_SERVICE = mongox.TYPE + TEST_SERVICE = mongox.SERVICE def setUp(self): patch() @@ -227,7 +227,7 @@ def get_tracer_and_connect(self): class TestMongoEnginePatchClientDefault(unittest.TestCase, MongoEngineCore): """Test suite with a Pin local to a specific client with default configuration""" - TEST_SERVICE = mongox.TYPE + TEST_SERVICE = mongox.SERVICE def setUp(self): patch() diff --git a/tests/contrib/pylons/test_pylons.py b/tests/contrib/pylons/test_pylons.py index 1410ea28ccb..cf9386b6349 100644 --- a/tests/contrib/pylons/test_pylons.py +++ b/tests/contrib/pylons/test_pylons.py @@ -56,7 +56,7 @@ def test_controller_exception(self): assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None - assert span.span_type == 'http' + assert span.span_type == 'web' def test_mw_exc_success(self): """Ensure exceptions can be properly handled by other middleware. diff --git a/tests/contrib/pymemcache/test_client_mixin.py b/tests/contrib/pymemcache/test_client_mixin.py index 4205da246b1..dffa835b233 100644 --- a/tests/contrib/pymemcache/test_client_mixin.py +++ b/tests/contrib/pymemcache/test_client_mixin.py @@ -6,8 +6,7 @@ from ddtrace import Pin from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY from ddtrace.contrib.pymemcache.patch import patch, unpatch -from ddtrace.ext import memcached as memcachedx -from ddtrace.ext import net +from ddtrace.ext import memcached as memcachedx, net from .utils import MockSocket from tests.test_tracer import get_dummy_tracer @@ -38,7 +37,7 @@ def check_spans(self, num_expected, resources_expected, queries_expected): self.assertEqual(span.get_tag(net.TARGET_HOST), TEST_HOST) self.assertEqual(span.get_tag(net.TARGET_PORT), str(TEST_PORT)) self.assertEqual(span.name, memcachedx.CMD) - self.assertEqual(span.span_type, memcachedx.TYPE) + self.assertEqual(span.span_type, 'cache') self.assertEqual(span.service, memcachedx.SERVICE) self.assertEqual(span.get_tag(memcachedx.QUERY), query) self.assertEqual(span.resource, resource) diff --git a/tests/contrib/pymongo/test.py b/tests/contrib/pymongo/test.py index a16f14b6a68..ca455134a62 100644 --- a/tests/contrib/pymongo/test.py +++ b/tests/contrib/pymongo/test.py @@ -353,7 +353,7 @@ def get_tracer_and_client(self): class TestPymongoPatchDefault(unittest.TestCase, PymongoCore): """Test suite for pymongo with the default patched library""" - TEST_SERVICE = mongox.TYPE + TEST_SERVICE = mongox.SERVICE def setUp(self): patch() diff --git a/tests/contrib/pyramid/utils.py b/tests/contrib/pyramid/utils.py index 9dd156bc2b9..effaae0e45b 100644 --- a/tests/contrib/pyramid/utils.py +++ b/tests/contrib/pyramid/utils.py @@ -63,7 +63,7 @@ def test_200(self, query_string=''): assert s.service == 'foobar' assert s.resource == 'GET index' assert s.error == 0 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '200' assert s.meta.get(http.URL) == 'http://localhost/' @@ -152,7 +152,7 @@ def test_404(self): assert s.service == 'foobar' assert s.resource == '404' assert s.error == 0 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '404' assert s.meta.get(http.URL) == 'http://localhost/404' @@ -167,7 +167,7 @@ def test_302(self): assert s.service == 'foobar' assert s.resource == 'GET raise_redirect' assert s.error == 0 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '302' assert s.meta.get(http.URL) == 'http://localhost/redirect' @@ -182,7 +182,7 @@ def test_204(self): assert s.service == 'foobar' assert s.resource == 'GET raise_no_content' assert s.error == 0 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '204' assert s.meta.get(http.URL) == 'http://localhost/nocontent' @@ -200,7 +200,7 @@ def test_exception(self): assert s.service == 'foobar' assert s.resource == 'GET exception' assert s.error == 1 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '500' assert s.meta.get(http.URL) == 'http://localhost/exception' @@ -216,7 +216,7 @@ def test_500(self): assert s.service == 'foobar' assert s.resource == 'GET error' assert s.error == 1 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '500' assert s.meta.get(http.URL) == 'http://localhost/error' @@ -236,7 +236,7 @@ def test_json(self): assert s.service == 'foobar' assert s.resource == 'GET json' assert s.error == 0 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '200' assert s.meta.get(http.URL) == 'http://localhost/json' @@ -260,7 +260,7 @@ def test_renderer(self): assert s.service == 'foobar' assert s.resource == 'GET renderer' assert s.error == 0 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '200' assert s.meta.get(http.URL) == 'http://localhost/renderer' @@ -282,7 +282,7 @@ def test_http_exception_response(self): assert s.service == 'foobar' assert s.resource == '404' assert s.error == 1 - assert s.span_type == 'http' + assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' assert s.meta.get('http.status_code') == '404' assert s.meta.get(http.URL) == 'http://localhost/404/raise_exception' @@ -354,7 +354,7 @@ def test_200_ot(self): assert dd_span.service == 'foobar' assert dd_span.resource == 'GET index' assert dd_span.error == 0 - assert dd_span.span_type == 'http' + assert dd_span.span_type == 'web' assert dd_span.meta.get('http.method') == 'GET' assert dd_span.meta.get('http.status_code') == '200' assert dd_span.meta.get(http.URL) == 'http://localhost/' diff --git a/tests/contrib/requests/test_requests.py b/tests/contrib/requests/test_requests.py index d4455f5d352..4cbceabc2f5 100644 --- a/tests/contrib/requests/test_requests.py +++ b/tests/contrib/requests/test_requests.py @@ -107,7 +107,7 @@ def test_200(self): assert s.get_tag(http.METHOD) == 'GET' assert s.get_tag(http.STATUS_CODE) == '200' assert s.error == 0 - assert s.span_type == http.TYPE + assert s.span_type == 'http' assert http.QUERY_STRING not in s.meta def test_200_send(self): @@ -124,7 +124,7 @@ def test_200_send(self): assert s.get_tag(http.METHOD) == 'GET' assert s.get_tag(http.STATUS_CODE) == '200' assert s.error == 0 - assert s.span_type == http.TYPE + assert s.span_type == 'http' def test_200_query_string(self): # ensure query string is removed before adding url to metadata @@ -140,7 +140,7 @@ def test_200_query_string(self): assert s.get_tag(http.STATUS_CODE) == '200' assert s.get_tag(http.URL) == URL_200 assert s.error == 0 - assert s.span_type == http.TYPE + assert s.span_type == 'http' assert s.get_tag(http.QUERY_STRING) == query_string def test_requests_module_200(self): @@ -156,7 +156,7 @@ def test_requests_module_200(self): assert s.get_tag(http.METHOD) == 'GET' assert s.get_tag(http.STATUS_CODE) == '200' assert s.error == 0 - assert s.span_type == http.TYPE + assert s.span_type == 'http' def test_post_500(self): out = self.session.post(URL_500) @@ -370,7 +370,7 @@ def test_200_ot(self): assert dd_span.get_tag(http.METHOD) == 'GET' assert dd_span.get_tag(http.STATUS_CODE) == '200' assert dd_span.error == 0 - assert dd_span.span_type == http.TYPE + assert dd_span.span_type == 'http' def test_request_and_response_headers(self): # Disabled when not configured diff --git a/tests/contrib/tornado/test_executor_decorator.py b/tests/contrib/tornado/test_executor_decorator.py index 70caf2700bb..e48bac16ae2 100644 --- a/tests/contrib/tornado/test_executor_decorator.py +++ b/tests/contrib/tornado/test_executor_decorator.py @@ -27,7 +27,7 @@ def test_on_executor_handler(self): request_span = traces[1][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -58,7 +58,7 @@ def test_on_executor_submit(self): request_span = traces[1][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorSubmitHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -88,7 +88,7 @@ def test_on_executor_exception_handler(self): request_span = traces[1][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') @@ -125,7 +125,7 @@ def test_on_executor_custom_kwarg(self): request_span = traces[1][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorCustomHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -158,7 +158,7 @@ def test_on_executor_custom_args_kwarg(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorCustomArgsHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') diff --git a/tests/contrib/tornado/test_tornado_template.py b/tests/contrib/tornado/test_tornado_template.py index 87c184a6d41..bbf159dfe1f 100644 --- a/tests/contrib/tornado/test_tornado_template.py +++ b/tests/contrib/tornado/test_tornado_template.py @@ -25,7 +25,7 @@ def test_template_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplateHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -72,7 +72,7 @@ def test_template_partials(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplatePartialHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -127,7 +127,7 @@ def test_template_exception_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplateExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') diff --git a/tests/contrib/tornado/test_tornado_web.py b/tests/contrib/tornado/test_tornado_web.py index ad9ead4826f..f066816cad6 100644 --- a/tests/contrib/tornado/test_tornado_web.py +++ b/tests/contrib/tornado/test_tornado_web.py @@ -30,7 +30,7 @@ def test_success_handler(self, query_string=''): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -60,7 +60,7 @@ def test_nested_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -87,7 +87,7 @@ def test_exception_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') @@ -108,7 +108,7 @@ def test_http_exception_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.HTTPExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '501' == request_span.get_tag('http.status_code') @@ -129,7 +129,7 @@ def test_http_exception_500_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.HTTPException500Handler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') @@ -150,7 +150,7 @@ def test_sync_success_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncSuccessHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -169,7 +169,7 @@ def test_sync_exception_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') @@ -190,7 +190,7 @@ def test_404_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tornado.web.ErrorHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '404' == request_span.get_tag('http.status_code') @@ -211,7 +211,7 @@ def test_redirect_handler(self): redirect_span = traces[0][0] assert 'tornado-web' == redirect_span.service assert 'tornado.request' == redirect_span.name - assert 'http' == redirect_span.span_type + assert 'web' == redirect_span.span_type assert 'tornado.web.RedirectHandler' == redirect_span.resource assert 'GET' == redirect_span.get_tag('http.method') assert '301' == redirect_span.get_tag('http.status_code') @@ -221,7 +221,7 @@ def test_redirect_handler(self): success_span = traces[1][0] assert 'tornado-web' == success_span.service assert 'tornado.request' == success_span.name - assert 'http' == success_span.span_type + assert 'web' == success_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == success_span.resource assert 'GET' == success_span.get_tag('http.method') assert '200' == success_span.get_tag('http.status_code') @@ -241,7 +241,7 @@ def test_static_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tornado.web.StaticFileHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -303,7 +303,7 @@ def test_success_handler_ot(self): assert 'tornado-web' == dd_span.service assert 'tornado.request' == dd_span.name - assert 'http' == dd_span.span_type + assert 'web' == dd_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == dd_span.resource assert 'GET' == dd_span.get_tag('http.method') assert '200' == dd_span.get_tag('http.status_code') @@ -482,7 +482,7 @@ def test_custom_default_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.CustomDefaultHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '400' == request_span.get_tag('http.status_code') diff --git a/tests/contrib/tornado/test_wrap_decorator.py b/tests/contrib/tornado/test_wrap_decorator.py index 7ee18954e85..aea7f453dcd 100644 --- a/tests/contrib/tornado/test_wrap_decorator.py +++ b/tests/contrib/tornado/test_wrap_decorator.py @@ -18,7 +18,7 @@ def test_nested_wrap_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -44,7 +44,7 @@ def test_nested_exception_wrap_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') @@ -74,7 +74,7 @@ def test_sync_nested_wrap_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncNestedWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -100,7 +100,7 @@ def test_sync_nested_exception_wrap_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncNestedExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') @@ -130,7 +130,7 @@ def test_nested_wrap_executor_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '200' == request_span.get_tag('http.status_code') @@ -157,7 +157,7 @@ def test_nested_exception_wrap_executor_handler(self): request_span = traces[0][0] assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name - assert 'http' == request_span.span_type + assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') assert '500' == request_span.get_tag('http.status_code') diff --git a/tests/test_span.py b/tests/test_span.py index f75494bdcde..5f8c3071907 100644 --- a/tests/test_span.py +++ b/tests/test_span.py @@ -6,7 +6,7 @@ from ddtrace.context import Context from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY from ddtrace.span import Span -from ddtrace.ext import errors, priority +from ddtrace.ext import SpanTypes, errors, priority from .base import BaseTracerTestCase @@ -168,6 +168,22 @@ def test_ctx_mgr(self): else: assert 0, 'should have failed' + def test_span_type(self): + s = Span(tracer=None, name='test.span', service='s', resource='r', span_type=SpanTypes.WEB) + s.set_tag('a', '1') + s.set_meta('b', '2') + s.finish() + + d = s.to_dict() + assert d + assert d['span_id'] == s.span_id + assert d['trace_id'] == s.trace_id + assert d['parent_id'] == s.parent_id + assert d['meta'] == {'a': '1', 'b': '2'} + assert d['type'] == 'web' + assert d['error'] == 0 + assert type(d['error']) == int + def test_span_to_dict(self): s = Span(tracer=None, name='test.span', service='s', resource='r') s.span_type = 'foo' diff --git a/tests/test_tracer.py b/tests/test_tracer.py index e683323cc80..b0608045cc4 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -494,16 +494,30 @@ def test_span_no_runtime_tags(self): self.assertIsNone(child.get_tag('language')) - def test_only_root_span_runtime(self): + def test_only_root_span_runtime_internal_span_types(self): self.tracer.configure(collect_metrics=True) - root = self.start_span('root') - context = root.context - child = self.start_span('child', child_of=context) + for span_type in ("custom", "template", "web", "worker"): + root = self.start_span('root', span_type=span_type) + context = root.context + child = self.start_span('child', child_of=context) - self.assertEqual(root.get_tag('language'), 'python') + self.assertEqual(root.get_tag('language'), 'python') - self.assertIsNone(child.get_tag('language')) + self.assertIsNone(child.get_tag('language')) + + def test_only_root_span_runtime_external_span_types(self): + self.tracer.configure(collect_metrics=True) + + for span_type in ("algoliasearch.search", "boto", "cache", "cassandra", "elasticsearch", + "grpc", "kombu", "http", "memcached", "redis", "sql", "vertica"): + root = self.start_span('root', span_type=span_type) + context = root.context + child = self.start_span('child', child_of=context) + + self.assertIsNone(root.get_tag('language')) + + self.assertIsNone(child.get_tag('language')) def test_installed_excepthook(): diff --git a/tox.ini b/tox.ini index 661286a9fe8..cfa1b4a2107 100644 --- a/tox.ini +++ b/tox.ini @@ -147,6 +147,8 @@ deps = # https://github.com/aio-libs/aiohttp/issues/2662 yarl: yarl==0.18.0 yarl10: yarl>=1.0,<1.1 +# backports + py27: enum34 # integrations aiobotocore010: aiobotocore>=0.10,<0.11 aiobotocore09: aiobotocore>=0.9,<0.10 @@ -468,6 +470,69 @@ deps= commands=flake8 . basepython=python3.7 +# do not use develop mode with celery as running multiple python versions within +# same job will cause problem for tests that use ddtrace-run +[celery_contrib] +usedevelop = False +[testenv:celery_contrib-py27-celery31-redis210] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery31-redis210] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery31-redis210] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery31-redis210] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py27-celery40-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py27-celery40-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py27-celery41-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py27-celery41-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery40-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery40-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery41-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery41-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery40-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery40-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery41-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery41-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery40-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery40-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery41-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery41-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py27-celery42-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery42-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery42-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery42-redis210-kombu43] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py27-celery43-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py34-celery43-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py35-celery43-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py36-celery43-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} +[testenv:celery_contrib-py37-celery43-redis320-kombu44] +usedevelop = {[celery_contrib]usedevelop} + [falcon_autopatch] setenv = DATADOG_SERVICE_NAME=my-falcon From 2c1f376fd0e8f190fab4458e586d4ed2999a3085 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Tue, 3 Dec 2019 10:40:22 -0500 Subject: [PATCH 18/81] Start black formatting some files (#1141) * Start black formatting some files * fix flake8 * fix black too --- conftest.py | 4 +-- docker-compose.yml | 1 + docs/conf.py | 66 +++++++++++++++--------------------- pyproject.toml | 3 -- setup.py | 84 ++++++++++++++++++++-------------------------- 5 files changed, 67 insertions(+), 91 deletions(-) diff --git a/conftest.py b/conftest.py index 556608e1dc6..9e3fef991eb 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,7 @@ import pytest -PY_DIR_PATTERN = re.compile(r'^py[23][0-9]$') +PY_DIR_PATTERN = re.compile(r"^py[23][0-9]$") # Determine if the folder should be ignored @@ -48,7 +48,7 @@ def pytest_ignore_collect(path, config): # Directory name match `py[23][0-9]` if PY_DIR_PATTERN.match(dirname): # Split out version numbers into a tuple: `py35` -> `(3, 5)` - min_required = tuple((int(v) for v in dirname.strip('py'))) + min_required = tuple((int(v) for v in dirname.strip("py"))) # If the current Python version does not meet the minimum required, skip this directory if sys.version_info[0:2] < min_required: diff --git a/docker-compose.yml b/docker-compose.yml index 17d65aeb260..2fb599cca1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,6 +111,7 @@ services: - ./setup.py:/src/setup.py:ro - ./conftest.py:/src/conftest.py:ro - ./tox.ini:/src/tox.ini:ro + - ./docs:/src/docs:ro - ./pyproject.toml:/src/pyproject.toml:ro - ./.ddtox:/src/.tox - ./scripts:/src/scripts diff --git a/docs/conf.py b/docs/conf.py index c043374a76d..9357dcd83c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ # append the ddtrace path to syspath -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath("..")) # -- General configuration ------------------------------------------------ @@ -36,34 +36,34 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.extlinks', + "sphinx.ext.autodoc", + "sphinx.ext.extlinks", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. year = datetime.now().year -project = u'ddtrace' -copyright = u'2016-{}, Datadog, Inc.'.format(year) # noqa: A001 -author = u'Datadog, Inc.' +project = u"ddtrace" +copyright = u"2016-{}, Datadog, Inc.".format(year) # noqa: A001 +author = u"Datadog, Inc." # document in order of source -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" # The version info for the project you're documenting, acts as replacement for @@ -94,11 +94,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [ - '_build', - 'Thumbs.db', - '.DS_Store' -] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -120,7 +116,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -137,14 +133,14 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'description': 'Datadog\'s Python tracing client', + "description": "Datadog's Python tracing client", } # Add any paths that contain custom themes here, relative to this directory. @@ -194,14 +190,7 @@ # Custom sidebar templates, maps document names to template names. # -html_sidebars = { - '**': [ - 'about.html', - 'nav.html', - 'relations.html', - 'searchbox.html', - ] -} +html_sidebars = {"**": ["about.html", "nav.html", "relations.html", "searchbox.html"]} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -260,7 +249,7 @@ # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'ddtracedoc' +htmlhelp_basename = "ddtracedoc" # -- Options for LaTeX output --------------------------------------------- @@ -268,15 +257,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -286,8 +272,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'ddtrace.tex', u'ddtrace Documentation', - u'Datadog, Inc', 'manual'), + (master_doc, "ddtrace.tex", u"ddtrace Documentation", u"Datadog, Inc", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -321,10 +306,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'ddtrace', u'ddtrace Documentation', - [author], 1) -] +man_pages = [(master_doc, "ddtrace", u"ddtrace Documentation", [author], 1)] # If true, show URL addresses after external links. # @@ -337,9 +319,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'ddtrace', u'ddtrace Documentation', - author, 'ddtrace', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "ddtrace", + u"ddtrace Documentation", + author, + "ddtrace", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. diff --git a/pyproject.toml b/pyproject.toml index fe6974e3afd..707aa964b0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,9 +28,6 @@ exclude = ''' | utils/ | vendor/ ) - | docs/ - | conftest.py - | setup.py | tests/ ) ''' diff --git a/setup.py b/setup.py index 07689f1b577..4c7052d802c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ class Tox(TestCommand): - user_options = [('tox-args=', 'a', 'Arguments to pass to tox')] + user_options = [("tox-args=", "a", "Arguments to pass to tox")] def initialize_options(self): TestCommand.initialize_options(self) @@ -25,6 +25,7 @@ def run_tests(self): # import here, cause outside the eggs aren't loaded import tox import shlex + args = self.tox_args if args: args = shlex.split(self.tox_args) @@ -56,51 +57,43 @@ def run_tests(self): """ # psutil used to generate runtime metrics for tracer -install_requires = [ - 'psutil>=5.0.0' -] +install_requires = ["psutil>=5.0.0"] # include enum backport if sys.version_info[:2] < (3, 4): - install_requires.extend([ - 'enum34' - ]) + install_requires.extend(["enum34"]) # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( - name='ddtrace', - description='Datadog tracing code', - url='https://github.com/DataDog/dd-trace-py', - author='Datadog, Inc.', - author_email='dev@datadoghq.com', + name="ddtrace", + description="Datadog tracing code", + url="https://github.com/DataDog/dd-trace-py", + author="Datadog, Inc.", + author_email="dev@datadoghq.com", long_description=long_description, - long_description_content_type='text/markdown', - license='BSD', - packages=find_packages(exclude=['tests*']), + long_description_content_type="text/markdown", + license="BSD", + packages=find_packages(exclude=["tests*"]), install_requires=install_requires, extras_require={ # users can include opentracing by having: # install_requires=['ddtrace[opentracing]', ...] - 'opentracing': ['opentracing>=2.0.0'], + "opentracing": ["opentracing>=2.0.0"], }, # plugin tox - tests_require=['tox', 'flake8'], - cmdclass={'test': Tox}, - entry_points={ - 'console_scripts': [ - 'ddtrace-run = ddtrace.commands.ddtrace_run:main' - ] - }, + tests_require=["tox", "flake8"], + cmdclass={"test": Tox}, + entry_points={"console_scripts": ["ddtrace-run = ddtrace.commands.ddtrace_run:main"]}, classifiers=[ - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], use_scm_version=True, - setup_requires=['setuptools_scm'], + setup_requires=["setuptools_scm"], ) @@ -110,8 +103,8 @@ def run_tests(self): # These helpers are useful for attempting build a C-extension and then retrying without it if it fails libraries = [] -if sys.platform == 'win32': - libraries.append('ws2_32') +if sys.platform == "win32": + libraries.append("ws2_32") build_ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError, OSError) else: build_ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) @@ -137,36 +130,33 @@ def build_extension(self, ext): macros = [] -if sys.byteorder == 'big': - macros = [('__BIG_ENDIAN__', '1')] +if sys.byteorder == "big": + macros = [("__BIG_ENDIAN__", "1")] else: - macros = [('__LITTLE_ENDIAN__', '1')] + macros = [("__LITTLE_ENDIAN__", "1")] # Try to build with C extensions first, fallback to only pure-Python if building fails try: kwargs = copy.deepcopy(setup_kwargs) - kwargs['ext_modules'] = [ - Extension( - 'ddtrace.vendor.wrapt._wrappers', - sources=['ddtrace/vendor/wrapt/_wrappers.c'], - ), + kwargs["ext_modules"] = [ + Extension("ddtrace.vendor.wrapt._wrappers", sources=["ddtrace/vendor/wrapt/_wrappers.c"],), Extension( - 'ddtrace.vendor.msgpack._cmsgpack', - sources=['ddtrace/vendor/msgpack/_cmsgpack.cpp'], + "ddtrace.vendor.msgpack._cmsgpack", + sources=["ddtrace/vendor/msgpack/_cmsgpack.cpp"], libraries=libraries, - include_dirs=['ddtrace/vendor/'], + include_dirs=["ddtrace/vendor/"], define_macros=macros, ), ] # DEV: Make sure `cmdclass` exists - kwargs.setdefault('cmdclass', dict()) - kwargs['cmdclass']['build_ext'] = optional_build_ext + kwargs.setdefault("cmdclass", dict()) + kwargs["cmdclass"]["build_ext"] = optional_build_ext setup(**kwargs) except BuildExtFailed: # Set `DDTRACE_BUILD_TRACE=TRUE` in CI to raise any build errors - if os.environ.get('DDTRACE_BUILD_RAISE') == 'TRUE': + if os.environ.get("DDTRACE_BUILD_RAISE") == "TRUE": raise - print('WARNING: Failed to install wrapt/msgpack C-extensions, using pure-Python wrapt/msgpack instead') + print("WARNING: Failed to install wrapt/msgpack C-extensions, using pure-Python wrapt/msgpack instead") setup(**setup_kwargs) From b50462e62e5db968dfe8205fe4dd6a8d2a6e4797 Mon Sep 17 00:00:00 2001 From: Sam Park Date: Wed, 4 Dec 2019 06:27:25 -0800 Subject: [PATCH 19/81] Add support for dogpile.cache (#1123) * Add dogpile support * Docs * Tests * Add to CI * Apply suggestions from code review Co-Authored-By: Tahir H. Butt * Fix tests --- .circleci/config.yml | 16 ++ ddtrace/contrib/dogpile_cache/__init__.py | 48 +++++ ddtrace/contrib/dogpile_cache/lock.py | 37 ++++ ddtrace/contrib/dogpile_cache/patch.py | 37 ++++ ddtrace/contrib/dogpile_cache/region.py | 29 +++ docs/db_integrations.rst | 8 + tests/contrib/dogpile_cache/__init__.py | 0 tests/contrib/dogpile_cache/test_tracing.py | 184 ++++++++++++++++++++ tox.ini | 7 + 9 files changed, 366 insertions(+) create mode 100644 ddtrace/contrib/dogpile_cache/__init__.py create mode 100644 ddtrace/contrib/dogpile_cache/lock.py create mode 100644 ddtrace/contrib/dogpile_cache/patch.py create mode 100644 ddtrace/contrib/dogpile_cache/region.py create mode 100644 tests/contrib/dogpile_cache/__init__.py create mode 100644 tests/contrib/dogpile_cache/test_tracing.py diff --git a/.circleci/config.yml b/.circleci/config.yml index c1f02b59319..2552851da4b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,6 +290,17 @@ jobs: - *persist_to_workspace_step - *save_cache_step + dogpile_cache: + docker: + - *test_runner + resource_class: *resource_class + steps: + - checkout + - *restore_cache_step + - run: scripts/run-tox-scenario '^dogpile_contrib-' + - *persist_to_workspace_step + - *save_cache_step + elasticsearch: docker: - *test_runner @@ -892,6 +903,10 @@ workflows: requires: - flake8 - black + - dogpile_cache: + requires: + - flake8 + - black - elasticsearch: requires: - flake8 @@ -1062,6 +1077,7 @@ workflows: - consul - dbapi - ddtracerun + - dogpile_cache - django - elasticsearch - falcon diff --git a/ddtrace/contrib/dogpile_cache/__init__.py b/ddtrace/contrib/dogpile_cache/__init__.py new file mode 100644 index 00000000000..39eac76c676 --- /dev/null +++ b/ddtrace/contrib/dogpile_cache/__init__.py @@ -0,0 +1,48 @@ +""" +Instrument dogpile.cache__ to report all cached lookups. + +This will add spans around the calls to your cache backend (eg. redis, memory, +etc). The spans will also include the following tags: + +- key/keys: The key(s) dogpile passed to your backend. Note that this will be + the output of the region's ``function_key_generator``, but before any key + mangling is applied (ie. the region's ``key_mangler``). +- region: Name of the region. +- backend: Name of the backend class. +- hit: If the key was found in the cache. +- expired: If the key is expired. This is only relevant if the key was found. + +While cache tracing will generally already have keys in tags, some caching +setups will not have useful tag values - such as when you're using consistent +hashing with memcached - the key(s) will appear as a mangled hash. +:: + + # Patch before importing dogpile.cache + from ddtrace import patch + patch(dogpile_cache=True) + + from dogpile.cache import make_region + + region = make_region().configure( + "dogpile.cache.pylibmc", + expiration_time=3600, + arguments={"url": ["127.0.0.1"]}, + ) + + @region.cache_on_arguments() + def hello(name): + # Some complicated, slow calculation + return "Hello, {}".format(name) + +.. __: https://dogpilecache.sqlalchemy.org/ +""" +from ...utils.importlib import require_modules + + +required_modules = ['dogpile.cache'] + +with require_modules(required_modules) as missing_modules: + if not missing_modules: + from .patch import patch, unpatch + + __all__ = ['patch', 'unpatch'] diff --git a/ddtrace/contrib/dogpile_cache/lock.py b/ddtrace/contrib/dogpile_cache/lock.py new file mode 100644 index 00000000000..e73124655b3 --- /dev/null +++ b/ddtrace/contrib/dogpile_cache/lock.py @@ -0,0 +1,37 @@ +import dogpile + +from ...pin import Pin +from ...utils.formats import asbool + + +def _wrap_lock_ctor(func, instance, args, kwargs): + """ + This seems rather odd. But to track hits, we need to patch the wrapped function that + dogpile passes to the region and locks. Unfortunately it's a closure defined inside + the get_or_create* methods themselves, so we can't easily patch those. + """ + func(*args, **kwargs) + ori_backend_fetcher = instance.value_and_created_fn + + def wrapped_backend_fetcher(): + pin = Pin.get_from(dogpile.cache) + if not pin or not pin.enabled(): + return ori_backend_fetcher() + + hit = False + expired = True + try: + value, createdtime = ori_backend_fetcher() + hit = value is not dogpile.cache.api.NO_VALUE + # dogpile sometimes returns None, but only checks for truthiness. Coalesce + # to minimize APM users' confusion. + expired = instance._is_expired(createdtime) or False + return value, createdtime + finally: + # Keys are checked in random order so the 'final' answer for partial hits + # should really be false (ie. if any are 'negative', then the tag value + # should be). This means ANDing all hit values and ORing all expired values. + span = pin.tracer.current_span() + span.set_tag('hit', asbool(span.get_tag('hit') or 'True') and hit) + span.set_tag('expired', asbool(span.get_tag('expired') or 'False') or expired) + instance.value_and_created_fn = wrapped_backend_fetcher diff --git a/ddtrace/contrib/dogpile_cache/patch.py b/ddtrace/contrib/dogpile_cache/patch.py new file mode 100644 index 00000000000..6525de587be --- /dev/null +++ b/ddtrace/contrib/dogpile_cache/patch.py @@ -0,0 +1,37 @@ +import dogpile + +from ddtrace.pin import Pin, _DD_PIN_NAME, _DD_PIN_PROXY_NAME +from ddtrace.vendor.wrapt import wrap_function_wrapper as _w + +from .lock import _wrap_lock_ctor +from .region import _wrap_get_create, _wrap_get_create_multi + +_get_or_create = dogpile.cache.region.CacheRegion.get_or_create +_get_or_create_multi = dogpile.cache.region.CacheRegion.get_or_create_multi +_lock_ctor = dogpile.lock.Lock.__init__ + + +def patch(): + if getattr(dogpile.cache, '_datadog_patch', False): + return + setattr(dogpile.cache, '_datadog_patch', True) + + _w('dogpile.cache.region', 'CacheRegion.get_or_create', _wrap_get_create) + _w('dogpile.cache.region', 'CacheRegion.get_or_create_multi', _wrap_get_create_multi) + _w('dogpile.lock', 'Lock.__init__', _wrap_lock_ctor) + + Pin(app='dogpile.cache', service='dogpile.cache').onto(dogpile.cache) + + +def unpatch(): + if not getattr(dogpile.cache, '_datadog_patch', False): + return + setattr(dogpile.cache, '_datadog_patch', False) + # This looks silly but the unwrap util doesn't support class instance methods, even + # though wrapt does. This was causing the patches to stack on top of each other + # during testing. + dogpile.cache.region.CacheRegion.get_or_create = _get_or_create + dogpile.cache.region.CacheRegion.get_or_create_multi = _get_or_create_multi + dogpile.lock.Lock.__init__ = _lock_ctor + setattr(dogpile.cache, _DD_PIN_NAME, None) + setattr(dogpile.cache, _DD_PIN_PROXY_NAME, None) diff --git a/ddtrace/contrib/dogpile_cache/region.py b/ddtrace/contrib/dogpile_cache/region.py new file mode 100644 index 00000000000..61d1cdb6183 --- /dev/null +++ b/ddtrace/contrib/dogpile_cache/region.py @@ -0,0 +1,29 @@ +import dogpile + +from ...pin import Pin + + +def _wrap_get_create(func, instance, args, kwargs): + pin = Pin.get_from(dogpile.cache) + if not pin or not pin.enabled(): + return func(*args, **kwargs) + + key = args[0] + with pin.tracer.trace('dogpile.cache', resource='get_or_create', span_type='cache') as span: + span.set_tag('key', key) + span.set_tag('region', instance.name) + span.set_tag('backend', instance.actual_backend.__class__.__name__) + return func(*args, **kwargs) + + +def _wrap_get_create_multi(func, instance, args, kwargs): + pin = Pin.get_from(dogpile.cache) + if not pin or not pin.enabled(): + return func(*args, **kwargs) + + keys = args[0] + with pin.tracer.trace('dogpile.cache', resource='get_or_create_multi', span_type='cache') as span: + span.set_tag('keys', keys) + span.set_tag('region', instance.name) + span.set_tag('backend', instance.actual_backend.__class__.__name__) + return func(*args, **kwargs) diff --git a/docs/db_integrations.rst b/docs/db_integrations.rst index a5c5ddc270b..d48758ca40a 100644 --- a/docs/db_integrations.rst +++ b/docs/db_integrations.rst @@ -25,6 +25,14 @@ Consul .. automodule:: ddtrace.contrib.consul +.. _dogpile.cache: + +dogpile.cache +------------- + +.. automodule:: ddtrace.contrib.dogpile_cache + + .. _elasticsearch: Elasticsearch diff --git a/tests/contrib/dogpile_cache/__init__.py b/tests/contrib/dogpile_cache/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/contrib/dogpile_cache/test_tracing.py b/tests/contrib/dogpile_cache/test_tracing.py new file mode 100644 index 00000000000..79e42592069 --- /dev/null +++ b/tests/contrib/dogpile_cache/test_tracing.py @@ -0,0 +1,184 @@ +import dogpile +import pytest + +from ddtrace import Pin +from ddtrace.contrib.dogpile_cache.patch import patch, unpatch + +from tests.test_tracer import get_dummy_tracer + + +@pytest.fixture +def tracer(): + return get_dummy_tracer() + + +@pytest.fixture +def region(tracer): + patch() + # Setup a simple dogpile cache region for testing. + # The backend is trivial so we can use memory to simplify test setup. + test_region = dogpile.cache.make_region(name='TestRegion') + test_region.configure('dogpile.cache.memory') + Pin.override(dogpile.cache, tracer=tracer) + return test_region + + +@pytest.fixture(autouse=True) +def cleanup(): + yield + unpatch() + + +@pytest.fixture +def single_cache(region): + @region.cache_on_arguments() + def fn(x): + return x * 2 + return fn + + +@pytest.fixture +def multi_cache(region): + @region.cache_multi_on_arguments() + def fn(*x): + return [i * 2 for i in x] + + return fn + + +def test_doesnt_trace_with_no_pin(tracer, single_cache, multi_cache): + # No pin is set + unpatch() + + assert single_cache(1) == 2 + assert tracer.writer.pop_traces() == [] + + assert multi_cache(2, 3) == [4, 6] + assert tracer.writer.pop_traces() == [] + + +def test_doesnt_trace_with_disabled_pin(tracer, single_cache, multi_cache): + tracer.enabled = False + + assert single_cache(1) == 2 + assert tracer.writer.pop_traces() == [] + + assert multi_cache(2, 3) == [4, 6] + assert tracer.writer.pop_traces() == [] + + +def test_traces_get_or_create(tracer, single_cache): + assert single_cache(1) == 2 + traces = tracer.writer.pop_traces() + assert len(traces) == 1 + spans = traces[0] + assert len(spans) == 1 + span = spans[0] + assert span.name == 'dogpile.cache' + assert span.resource == 'get_or_create' + assert span.meta['key'] == 'tests.contrib.dogpile_cache.test_tracing:fn|1' + assert span.meta['hit'] == 'False' + assert span.meta['expired'] == 'True' + assert span.meta['backend'] == 'MemoryBackend' + assert span.meta['region'] == 'TestRegion' + + # Now the results should be cached. + assert single_cache(1) == 2 + traces = tracer.writer.pop_traces() + assert len(traces) == 1 + spans = traces[0] + assert len(spans) == 1 + span = spans[0] + assert span.name == 'dogpile.cache' + assert span.resource == 'get_or_create' + assert span.meta['key'] == 'tests.contrib.dogpile_cache.test_tracing:fn|1' + assert span.meta['hit'] == 'True' + assert span.meta['expired'] == 'False' + assert span.meta['backend'] == 'MemoryBackend' + assert span.meta['region'] == 'TestRegion' + + +def test_traces_get_or_create_multi(tracer, multi_cache): + assert multi_cache(2, 3) == [4, 6] + traces = tracer.writer.pop_traces() + assert len(traces) == 1 + spans = traces[0] + assert len(spans) == 1 + span = spans[0] + assert span.meta['keys'] == ( + "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + + "'tests.contrib.dogpile_cache.test_tracing:fn|3']" + ) + assert span.meta['hit'] == 'False' + assert span.meta['expired'] == 'True' + assert span.meta['backend'] == 'MemoryBackend' + assert span.meta['region'] == 'TestRegion' + + # Partial hit + assert multi_cache(2, 4) == [4, 8] + traces = tracer.writer.pop_traces() + assert len(traces) == 1 + spans = traces[0] + assert len(spans) == 1 + span = spans[0] + assert span.meta['keys'] == ( + "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + + "'tests.contrib.dogpile_cache.test_tracing:fn|4']" + ) + assert span.meta['hit'] == 'False' + assert span.meta['expired'] == 'True' + assert span.meta['backend'] == 'MemoryBackend' + assert span.meta['region'] == 'TestRegion' + + # Full hit + assert multi_cache(2, 4) == [4, 8] + traces = tracer.writer.pop_traces() + assert len(traces) == 1 + spans = traces[0] + assert len(spans) == 1 + span = spans[0] + assert span.meta['keys'] == ( + "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + + "'tests.contrib.dogpile_cache.test_tracing:fn|4']" + ) + assert span.meta['hit'] == 'True' + assert span.meta['expired'] == 'False' + assert span.meta['backend'] == 'MemoryBackend' + assert span.meta['region'] == 'TestRegion' + + +class TestInnerFunctionCalls(object): + def single_cache(self, x): + return x * 2 + + def multi_cache(self, *x): + return [i * 2 for i in x] + + def test_calls_inner_functions_correctly(self, region, mocker): + """ This ensures the get_or_create behavior of dogpile is not altered. """ + spy_single_cache = mocker.spy(self, 'single_cache') + spy_multi_cache = mocker.spy(self, 'multi_cache') + + single_cache = region.cache_on_arguments()(self.single_cache) + multi_cache = region.cache_multi_on_arguments()(self.multi_cache) + + assert 2 == single_cache(1) + spy_single_cache.assert_called_once_with(1) + + # It's now cached - shouldn't need to call the inner function. + spy_single_cache.reset_mock() + assert 2 == single_cache(1) + assert spy_single_cache.call_count == 0 + + assert [6, 8] == multi_cache(3, 4) + spy_multi_cache.assert_called_once_with(3, 4) + + # Partial hit. Only the "new" key should be passed to the inner function. + spy_multi_cache.reset_mock() + assert [6, 10] == multi_cache(3, 5) + spy_multi_cache.assert_called_once_with(5) + + # Full hit. No call to inner function. + spy_multi_cache.reset_mock() + assert [6, 10] == multi_cache(3, 5) + assert spy_single_cache.call_count == 0 diff --git a/tox.ini b/tox.ini index cfa1b4a2107..d622e6a9bed 100644 --- a/tox.ini +++ b/tox.ini @@ -62,6 +62,7 @@ envlist = django_contrib{,_autopatch}-{py34,py35,py36}-django{200}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached django_drf_contrib-{py27,py34,py35,py36}-django{111}-djangorestframework{34,37,38} django_drf_contrib-{py34,py35,py36}-django{200}-djangorestframework{37,38} + dogpile_contrib-{py27,py35,py36,py37}-dogpilecache{06,07,08,latest} elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch{16,17,18,23,24,51,52,53,54,63,64} elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch1{100} elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch2{50} @@ -139,6 +140,7 @@ deps = pytest-benchmark pytest-cov pytest-django + pytest-mock opentracing psutil # test dependencies installed in all envs @@ -210,6 +212,10 @@ deps = djangorestframework34: djangorestframework>=3.4,<3.5 djangorestframework37: djangorestframework>=3.7,<3.8 djangorestframework38: djangorestframework>=3.8,<3.9 + dogpilecache06: dogpile.cache==0.6.* + dogpilecache07: dogpile.cache==0.7.* + dogpilecache08: dogpile.cache==0.8.* + dogpilecachelatest: dogpile.cache elasticsearch16: elasticsearch>=1.6,<1.7 elasticsearch17: elasticsearch>=1.7,<1.8 elasticsearch18: elasticsearch>=1.8,<1.9 @@ -395,6 +401,7 @@ commands = django_contrib: pytest {posargs} tests/contrib/django django_contrib_autopatch: python tests/ddtrace_run.py pytest {posargs} tests/contrib/django django_drf_contrib: pytest {posargs} tests/contrib/djangorestframework + dogpile_contrib: pytest {posargs} tests/contrib/dogpile_cache elasticsearch_contrib: pytest {posargs} tests/contrib/elasticsearch falcon_contrib: pytest {posargs} tests/contrib/falcon/test_middleware.py tests/contrib/falcon/test_distributed_tracing.py falcon_contrib_autopatch: python tests/ddtrace_run.py pytest {posargs} tests/contrib/falcon/test_autopatch.py From f13e9a7206535f34e153d84fec2da6d681eb6668 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Wed, 11 Dec 2019 11:54:13 -0500 Subject: [PATCH 20/81] core: Change DatadogSampler defaults (#1151) --- ddtrace/sampler.py | 23 ++--------- ddtrace/tracer.py | 4 -- tests/test_sampler.py | 89 ++++--------------------------------------- 3 files changed, 11 insertions(+), 105 deletions(-) diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 160d7e6e901..4597db2b243 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -112,14 +112,11 @@ class DatadogSampler(BaseSampler): """ This sampler is currently in ALPHA and it's API may change at any time, use at your own risk. """ - # TODO: Remove '_priority_sampler' when we no longer use the fallback - __slots__ = ('default_sampler', 'rules', '_priority_sampler') + __slots__ = ('default_sampler', 'limiter', 'rules') - DEFAULT_RATE_LIMIT = 100 NO_RATE_LIMIT = -1 - # TODO: Remove _priority_sampler=None when we no longer use the fallback - def __init__(self, rules=None, default_sample_rate=1.0, rate_limit=DEFAULT_RATE_LIMIT, _priority_sampler=None): + def __init__(self, rules=None, default_sample_rate=1.0, rate_limit=NO_RATE_LIMIT): """ Constructor for DatadogSampler sampler @@ -128,7 +125,7 @@ def __init__(self, rules=None, default_sample_rate=1.0, rate_limit=DEFAULT_RATE_ :param default_sample_rate: The default sample rate to apply if no rules matched (default: 1.0) :type default_sample_rate: float 0 <= X <= 1.0 :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules - applied to them, default 100 traces per second + applied to them, default is no rate limit :type rate_limit: :obj:`int` """ # Ensure rules is a list @@ -145,9 +142,6 @@ def __init__(self, rules=None, default_sample_rate=1.0, rate_limit=DEFAULT_RATE_ self.limiter = RateLimiter(rate_limit) self.default_sampler = SamplingRule(sample_rate=default_sample_rate) - # TODO: Remove when we no longer use the fallback - self._priority_sampler = _priority_sampler - def _set_priority(self, span, priority): if span._context: span._context.sampling_priority = priority @@ -173,16 +167,7 @@ def sample(self, span): matching_rule = rule break else: - # No rule matches, fallback to priority sampling if set - if self._priority_sampler: - if self._priority_sampler.sample(span): - self._set_priority(span, AUTO_KEEP) - return True - else: - self._set_priority(span, AUTO_REJECT) - return False - - # No rule matches, no priority sampler, use the default sampler + # No rule matches, use the default sampler matching_rule = self.default_sampler # Sample with the matching sampling rule diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index d74d0c41db0..d05646d5ea6 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -220,10 +220,6 @@ def configure(self, enabled=None, hostname=None, port=None, uds_path=None, https if sampler is not None: self.sampler = sampler - # TODO: Remove when we remove the fallback to priority sampling - if isinstance(self.sampler, DatadogSampler): - self.sampler._priority_sampler = self.priority_sampler - if dogstatsd_host is not None and dogstatsd_url is None: dogstatsd_url = 'udp://{}:{}'.format(dogstatsd_host, dogstatsd_port or self.DEFAULT_DOGSTATSD_PORT) diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 4a6409372e4..fc21eeb431f 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -437,13 +437,13 @@ def test_datadog_sampler_init(): sampler = DatadogSampler() assert sampler.rules == [] assert isinstance(sampler.limiter, RateLimiter) - assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT + assert sampler.limiter.rate_limit == DatadogSampler.NO_RATE_LIMIT # With rules rule = SamplingRule(sample_rate=1) sampler = DatadogSampler(rules=[rule]) assert sampler.rules == [rule] - assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT + assert sampler.limiter.rate_limit == DatadogSampler.NO_RATE_LIMIT # With rate limit sampler = DatadogSampler(rate_limit=10) @@ -624,76 +624,6 @@ def reset(): rules[2].matches.assert_not_called() rules[2].sample.assert_not_called() - # No rules match and priority sampler is defined - # All rules SamplingRule.matches are called - # Priority sampler's `sample` method is called - # Result of priority sampler is returned - # Rate limiter is not called - # TODO: Remove this case when we remove fallback to priority sampling - with reset_mocks(): - span = create_span(tracer=dummy_tracer) - - # Configure mock priority sampler - priority_sampler = RateByServiceSampler() - for rate_sampler in priority_sampler._by_service_samplers.values(): - rate_sampler.set_sample_rate(1) - - spy_sampler = mock.Mock(spec=RateByServiceSampler, wraps=priority_sampler) - sampler._priority_sampler = spy_sampler - - for rule in rules: - rule.matches.return_value = False - rule.sample.return_value = False - - assert sampler.sample(span) is True - assert span._context.sampling_priority is AUTO_KEEP - assert span.sampled is True - mock_is_allowed.assert_not_called() - sampler.default_sampler.sample.assert_not_called() - spy_sampler.sample.assert_called_once_with(span) - assert_sampling_decision_tags(span, agent=1) - - [r.matches.assert_called_once_with(span) for r in rules] - [r.sample.assert_not_called() for r in rules] - - # Reset priority sampler property - sampler._priority_sampler = None - - # No rules match and priority sampler is defined - # All rules SamplingRule.matches are called - # Priority sampler's `sample` method is called - # Result of priority sampler is returned - # Rate limiter is not called - # TODO: Remove this case when we remove fallback to priority sampling - with reset_mocks(): - span = create_span(tracer=dummy_tracer) - - # Configure mock priority sampler - priority_sampler = RateByServiceSampler() - for rate_sampler in priority_sampler._by_service_samplers.values(): - rate_sampler.set_sample_rate(0) - - spy_sampler = mock.Mock(spec=RateByServiceSampler, wraps=priority_sampler) - sampler._priority_sampler = spy_sampler - - for rule in rules: - rule.matches.return_value = False - rule.sample.return_value = False - - assert sampler.sample(span) is False - assert span._context.sampling_priority is AUTO_REJECT - assert span.sampled is False - mock_is_allowed.assert_not_called() - sampler.default_sampler.sample.assert_not_called() - spy_sampler.sample.assert_called_once_with(span) - assert_sampling_decision_tags(span, agent=0) - - [r.matches.assert_called_once_with(span) for r in rules] - [r.sample.assert_not_called() for r in rules] - - # Reset priority sampler property - sampler._priority_sampler = None - def test_datadog_sampler_tracer(dummy_tracer): rule = SamplingRule(sample_rate=1.0, name='test.span') @@ -705,8 +635,7 @@ def test_datadog_sampler_tracer(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -734,8 +663,7 @@ def test_datadog_sampler_tracer_rate_limited(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -762,8 +690,7 @@ def test_datadog_sampler_tracer_rate_0(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -790,8 +717,7 @@ def test_datadog_sampler_tracer_child(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -824,8 +750,7 @@ def test_datadog_sampler_tracer_start_span(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy From cee5629e14a214249748a79adb042f3c48f29758 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Wed, 11 Dec 2019 15:10:16 -0500 Subject: [PATCH 21/81] bottle: fix status code for responses (#1158) * bottle: handle 400 responses * fix tests * cleaner conditionals --- ddtrace/contrib/bottle/trace.py | 22 ++++++++--- tests/contrib/bottle/test.py | 65 +++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/ddtrace/contrib/bottle/trace.py b/ddtrace/contrib/bottle/trace.py index bac17a6ec8b..12c196b32ef 100644 --- a/ddtrace/contrib/bottle/trace.py +++ b/ddtrace/contrib/bottle/trace.py @@ -1,5 +1,5 @@ # 3p -from bottle import response, request, HTTPError +from bottle import response, request, HTTPError, HTTPResponse # stdlib import ddtrace @@ -44,12 +44,16 @@ def wrapped(*args, **kwargs): config.bottle.get_analytics_sample_rate(use_global_config=True) ) - code = 0 + code = None + result = None try: - return callback(*args, **kwargs) - except HTTPError as e: + result = callback(*args, **kwargs) + return result + except (HTTPError, HTTPResponse) as e: # you can interrupt flows using abort(status_code, 'message')... # we need to respect the defined status_code. + # we also need to handle when response is raised as is the + # case with a 4xx status code = e.status_code raise except Exception: @@ -58,7 +62,15 @@ def wrapped(*args, **kwargs): code = 500 raise finally: - response_code = code or response.status_code + if isinstance(result, HTTPResponse): + response_code = result.status_code + elif code: + response_code = code + else: + # bottle local response has not yet been updated so this + # will be default + response_code = response.status_code + if 500 <= response_code < 600: s.error = 1 diff --git a/tests/contrib/bottle/test.py b/tests/contrib/bottle/test.py index 0d085a8b3ca..1de486d61d6 100644 --- a/tests/contrib/bottle/test.py +++ b/tests/contrib/bottle/test.py @@ -83,6 +83,71 @@ def test_query_string_multi_keys_trace(self): with self.override_http_config('bottle', dict(trace_query_string=True)): return self.test_200('foo=bar&foo=baz&x=y') + def test_2xx(self): + @self.app.route('/2xx') + def handled(): + return bottle.HTTPResponse("", status=202) + self._trace_app(self.tracer) + + # make a request + try: + self.app.get('/2xx') + except webtest.AppError: + pass + + spans = self.tracer.writer.pop() + assert len(spans) == 1 + s = spans[0] + assert s.resource == 'GET /2xx' + assert s.get_tag('http.status_code') == '202' + assert s.error == 0 + + def test_400_return(self): + @self.app.route('/400_return') + def handled400(): + return bottle.HTTPResponse(status=400) + self._trace_app(self.tracer) + + # make a request + try: + self.app.get('/400_return') + except webtest.AppError: + pass + + spans = self.tracer.writer.pop() + assert len(spans) == 1 + s = spans[0] + assert s.name == 'bottle.request' + assert s.service == 'bottle-app' + assert s.resource == 'GET /400_return' + assert s.get_tag('http.status_code') == '400' + assert s.get_tag('http.method') == 'GET' + assert s.get_tag(http.URL) == 'http://localhost:80/400_return' + assert s.error == 0 + + def test_400_raise(self): + @self.app.route('/400_raise') + def handled400(): + raise bottle.HTTPResponse(status=400) + self._trace_app(self.tracer) + + # make a request + try: + self.app.get('/400_raise') + except webtest.AppError: + pass + + spans = self.tracer.writer.pop() + assert len(spans) == 1 + s = spans[0] + assert s.name == 'bottle.request' + assert s.service == 'bottle-app' + assert s.resource == 'GET /400_raise' + assert s.get_tag('http.status_code') == '400' + assert s.get_tag('http.method') == 'GET' + assert s.get_tag(http.URL) == 'http://localhost:80/400_raise' + assert s.error == 1 + def test_500(self): @self.app.route('/hi') def hi(): From 2dbc7d8f2cecb9c2653e3d704cf4326e47d3ebe7 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Wed, 18 Dec 2019 15:20:09 -0500 Subject: [PATCH 22/81] core: Deprecate ddtrace.ext.AppTypes (#1162) * core: Deprecate ddtrace.ext.AppTypes * fix flake8 * Remove unused helper --- ddtrace/ext/__init__.py | 22 ++++++++++++++++++++++ ddtrace/ext/consul.py | 4 ++++ ddtrace/ext/sql.py | 5 +++++ ddtrace/utils/__init__.py | 16 ++++++++++++++-- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/ddtrace/ext/__init__.py b/ddtrace/ext/__init__.py index 1568dc71126..d05d30392ae 100644 --- a/ddtrace/ext/__init__.py +++ b/ddtrace/ext/__init__.py @@ -1,5 +1,8 @@ from enum import Enum +from ..vendor.debtcollector import removals +from ..utils import removed_classproperty + class SpanTypes(Enum): CACHE = "cache" @@ -13,3 +16,22 @@ class SpanTypes(Enum): TEMPLATE = "template" WEB = "web" WORKER = "worker" + + +@removals.removed_class("AppTypes") +class AppTypes(object): + @removed_classproperty + def web(cls): + return SpanTypes.WEB + + @removed_classproperty + def db(cls): + return "db" + + @removed_classproperty + def cache(cls): + return SpanTypes.CACHE + + @removed_classproperty + def worker(cls): + return SpanTypes.WORKER diff --git a/ddtrace/ext/consul.py b/ddtrace/ext/consul.py index be17e922361..d1e6f1afaaa 100644 --- a/ddtrace/ext/consul.py +++ b/ddtrace/ext/consul.py @@ -1,4 +1,8 @@ +from . import SpanTypes + APP = 'consul' +# [TODO] Deprecated, remove when we remove AppTypes +APP_TYPE = SpanTypes.CACHE SERVICE = 'consul' CMD = 'consul.command' diff --git a/ddtrace/ext/sql.py b/ddtrace/ext/sql.py index 6aec7a60231..270a71d482f 100644 --- a/ddtrace/ext/sql.py +++ b/ddtrace/ext/sql.py @@ -1,3 +1,8 @@ +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +APP_TYPE = SpanTypes.SQL + # tags QUERY = 'sql.query' # the query text ROWS = 'sql.rows' # number of rows returned by a query diff --git a/ddtrace/utils/__init__.py b/ddtrace/utils/__init__.py index fa8348e58a9..c46c5c7178d 100644 --- a/ddtrace/utils/__init__.py +++ b/ddtrace/utils/__init__.py @@ -1,3 +1,6 @@ +from ..vendor import debtcollector + + # https://stackoverflow.com/a/26853961 def merge_dicts(x, y): """Returns a copy of y merged into x.""" @@ -10,6 +13,15 @@ def get_module_name(module): """Returns a module's name or None if one cannot be found. Relevant PEP: https://www.python.org/dev/peps/pep-0451/ """ - if hasattr(module, '__spec__'): + if hasattr(module, "__spec__"): return module.__spec__.name - return getattr(module, '__name__', None) + return getattr(module, "__name__", None) + + +# Based on: https://stackoverflow.com/a/7864317 +class removed_classproperty(property): + def __get__(self, cls, owner): + debtcollector.deprecate( + "Usage of ddtrace.ext.AppTypes is not longer supported, please use ddtrace.ext.SpanTypes" + ) + return classmethod(self.fget).__get__(None, owner)() From 5ea6a4f1235573f3fc702643eb34c430f633c4a2 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Fri, 20 Dec 2019 11:44:32 -0500 Subject: [PATCH 23/81] core: safely deprecate ext type constants (#1165) --- ddtrace/contrib/elasticsearch/patch.py | 2 +- ddtrace/ext/cassandra.py | 6 ++++++ ddtrace/ext/elasticsearch.py | 8 +++++++- ddtrace/ext/http.py | 5 +++++ ddtrace/ext/kombu.py | 6 ++++++ ddtrace/ext/memcached.py | 5 +++++ ddtrace/ext/mongo.py | 5 +++++ ddtrace/ext/redis.py | 6 ++++++ ddtrace/ext/sql.py | 1 + 9 files changed, 42 insertions(+), 2 deletions(-) diff --git a/ddtrace/contrib/elasticsearch/patch.py b/ddtrace/contrib/elasticsearch/patch.py index b8541d71937..16b9d6e0e0c 100644 --- a/ddtrace/contrib/elasticsearch/patch.py +++ b/ddtrace/contrib/elasticsearch/patch.py @@ -32,7 +32,7 @@ def _patch(elasticsearch): return setattr(elasticsearch, '_datadog_patch', True) _w(elasticsearch.transport, 'Transport.perform_request', _get_perform_request(elasticsearch)) - Pin(service=metadata.SERVICE).onto(elasticsearch.transport.Transport) + Pin(service=metadata.SERVICE, app=metadata.APP).onto(elasticsearch.transport.Transport) def unpatch(): diff --git a/ddtrace/ext/cassandra.py b/ddtrace/ext/cassandra.py index dd8d03d15ba..6b2629a317b 100644 --- a/ddtrace/ext/cassandra.py +++ b/ddtrace/ext/cassandra.py @@ -1,3 +1,9 @@ +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +# the type of the spans +TYPE = SpanTypes.CASSANDRA + # tags CLUSTER = 'cassandra.cluster' KEYSPACE = 'cassandra.keyspace' diff --git a/ddtrace/ext/elasticsearch.py b/ddtrace/ext/elasticsearch.py index aedd665fc63..44d1089176d 100644 --- a/ddtrace/ext/elasticsearch.py +++ b/ddtrace/ext/elasticsearch.py @@ -1,4 +1,10 @@ -SERVICE = "elasticsearch" +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +TYPE = SpanTypes.ELASTICSEARCH +SERVICE = 'elasticsearch' +APP = 'elasticsearch' + # standard tags URL = 'elasticsearch.url' METHOD = 'elasticsearch.method' diff --git a/ddtrace/ext/http.py b/ddtrace/ext/http.py index acc8a65aa9f..3df762247d4 100644 --- a/ddtrace/ext/http.py +++ b/ddtrace/ext/http.py @@ -6,6 +6,11 @@ span.set_tag(URL, '/user/home') span.set_tag(STATUS_CODE, 404) """ +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +# type of the spans +TYPE = SpanTypes.HTTP # tags URL = 'http.url' diff --git a/ddtrace/ext/kombu.py b/ddtrace/ext/kombu.py index fbf9bfda6c9..9eaafeb2e56 100644 --- a/ddtrace/ext/kombu.py +++ b/ddtrace/ext/kombu.py @@ -1,3 +1,9 @@ +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +# type of the spans +TYPE = SpanTypes.WORKER + SERVICE = 'kombu' # net extension diff --git a/ddtrace/ext/memcached.py b/ddtrace/ext/memcached.py index 64f03fd7979..7e71e986142 100644 --- a/ddtrace/ext/memcached.py +++ b/ddtrace/ext/memcached.py @@ -1,3 +1,8 @@ +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +TYPE = SpanTypes.CACHE + CMD = 'memcached.command' SERVICE = 'memcached' QUERY = 'memcached.query' diff --git a/ddtrace/ext/mongo.py b/ddtrace/ext/mongo.py index f145bb60ecc..884764e454c 100644 --- a/ddtrace/ext/mongo.py +++ b/ddtrace/ext/mongo.py @@ -1,3 +1,8 @@ +from . import SpanTypes + +# [TODO] Deprecated, remove when we remove AppTypes +TYPE = SpanTypes.MONGODB + SERVICE = 'mongodb' COLLECTION = 'mongodb.collection' DB = 'mongodb.db' diff --git a/ddtrace/ext/redis.py b/ddtrace/ext/redis.py index a512e8a694e..542175ea66e 100644 --- a/ddtrace/ext/redis.py +++ b/ddtrace/ext/redis.py @@ -1,7 +1,13 @@ +from . import SpanTypes + # defaults APP = 'redis' DEFAULT_SERVICE = 'redis' +# [TODO] Deprecated, remove when we remove AppTypes +# type of the spans +TYPE = SpanTypes.REDIS + # net extension DB = 'out.redis_db' diff --git a/ddtrace/ext/sql.py b/ddtrace/ext/sql.py index 270a71d482f..1d8c1a3a90a 100644 --- a/ddtrace/ext/sql.py +++ b/ddtrace/ext/sql.py @@ -1,6 +1,7 @@ from . import SpanTypes # [TODO] Deprecated, remove when we remove AppTypes +TYPE = SpanTypes.SQL APP_TYPE = SpanTypes.SQL # tags From a5d21865f8b9ff3806c1253a72b91d2f201965e5 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 20 Dec 2019 15:22:08 -0500 Subject: [PATCH 24/81] core: fix new sampler defaults and add env variables to configure (#1166) * core: fix new sampler defaults and add env variables to configure * fix tests --- ddtrace/sampler.py | 12 ++++++++++-- tests/base/__init__.py | 24 +++--------------------- tests/test_sampler.py | 19 +++++++++++++++++-- tests/utils/__init__.py | 23 +++++++++++++++++++++++ 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 4597db2b243..62288ed0803 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -10,6 +10,7 @@ from .ext.priority import AUTO_KEEP, AUTO_REJECT from .internal.logger import get_logger from .internal.rate_limiter import RateLimiter +from .utils.formats import get_env from .vendor import six log = get_logger(__name__) @@ -115,8 +116,10 @@ class DatadogSampler(BaseSampler): __slots__ = ('default_sampler', 'limiter', 'rules') NO_RATE_LIMIT = -1 + DEFAULT_RATE_LIMIT = 100 + DEFAULT_SAMPLE_RATE = 1.0 - def __init__(self, rules=None, default_sample_rate=1.0, rate_limit=NO_RATE_LIMIT): + def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): """ Constructor for DatadogSampler sampler @@ -125,9 +128,14 @@ def __init__(self, rules=None, default_sample_rate=1.0, rate_limit=NO_RATE_LIMIT :param default_sample_rate: The default sample rate to apply if no rules matched (default: 1.0) :type default_sample_rate: float 0 <= X <= 1.0 :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules - applied to them, default is no rate limit + applied to them, (default: ``100``) :type rate_limit: :obj:`int` """ + if default_sample_rate is None: + default_sample_rate = float(get_env('trace', 'sample_rate', default=self.DEFAULT_SAMPLE_RATE)) + if rate_limit is None: + rate_limit = int(get_env('trace', 'rate_limit', default=self.DEFAULT_RATE_LIMIT)) + # Ensure rules is a list if not rules: rules = [] diff --git a/tests/base/__init__.py b/tests/base/__init__.py index fdea76fc986..cd266e8640b 100644 --- a/tests/base/__init__.py +++ b/tests/base/__init__.py @@ -1,10 +1,10 @@ import contextlib -import os import sys import unittest import ddtrace +from ..utils import override_env from ..utils.tracer import DummyTracer from ..utils.span import TestSpanContainer, TestSpan, NO_CHILDREN @@ -25,26 +25,8 @@ def test_case(self): pass """ - @staticmethod - @contextlib.contextmanager - def override_env(env): - """ - Temporarily override ``os.environ`` with provided values:: - - >>> with self.override_env(dict(DATADOG_TRACE_DEBUG=True)): - # Your test - """ - # Copy the full original environment - original = dict(os.environ) - - # Update based on the passed in arguments - os.environ.update(env) - try: - yield - finally: - # Full clear the environment out and reset back to the original - os.environ.clear() - os.environ.update(original) + # Expose `override_env` as `self.override_env` + override_env = staticmethod(override_env) @staticmethod @contextlib.contextmanager diff --git a/tests/test_sampler.py b/tests/test_sampler.py index fc21eeb431f..6849b1d0420 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -15,6 +15,7 @@ from ddtrace.sampler import RateSampler, AllSampler, RateByServiceSampler from ddtrace.span import Span +from .utils import override_env from .test_tracer import get_dummy_tracer @@ -437,17 +438,31 @@ def test_datadog_sampler_init(): sampler = DatadogSampler() assert sampler.rules == [] assert isinstance(sampler.limiter, RateLimiter) - assert sampler.limiter.rate_limit == DatadogSampler.NO_RATE_LIMIT + assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT + assert sampler.default_sampler.sample_rate == DatadogSampler.DEFAULT_SAMPLE_RATE # With rules rule = SamplingRule(sample_rate=1) sampler = DatadogSampler(rules=[rule]) assert sampler.rules == [rule] - assert sampler.limiter.rate_limit == DatadogSampler.NO_RATE_LIMIT + assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT + assert sampler.default_sampler.sample_rate == DatadogSampler.DEFAULT_SAMPLE_RATE # With rate limit sampler = DatadogSampler(rate_limit=10) assert sampler.limiter.rate_limit == 10 + assert sampler.default_sampler.sample_rate == DatadogSampler.DEFAULT_SAMPLE_RATE + + # With default_sample_rate + sampler = DatadogSampler(default_sample_rate=0.5) + assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT + assert sampler.default_sampler.sample_rate == 0.5 + + # From env variables + with override_env(dict(DD_TRACE_SAMPLE_RATE='0.5', DD_TRACE_RATE_LIMIT='10')): + sampler = DatadogSampler() + assert sampler.limiter.rate_limit == 10 + assert sampler.default_sampler.sample_rate == 0.5 # Invalid rules for val in (None, True, False, object(), 1, Exception()): diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e69de29bb2d..1e8c63af8a2 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -0,0 +1,23 @@ +import contextlib +import os + + +@contextlib.contextmanager +def override_env(env): + """ + Temporarily override ``os.environ`` with provided values:: + + >>> with self.override_env(dict(DATADOG_TRACE_DEBUG=True)): + # Your test + """ + # Copy the full original environment + original = dict(os.environ) + + # Update based on the passed in arguments + os.environ.update(env) + try: + yield + finally: + # Full clear the environment out and reset back to the original + os.environ.clear() + os.environ.update(original) From 6b4a62ca3e9d26f0d369f86a6e45914ada3f3593 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 30 Dec 2019 10:34:52 -0500 Subject: [PATCH 25/81] internal: Set numeric tags on Span.metrics instead of Span.meta --- ddtrace/compat.py | 10 +++ ddtrace/contrib/aiobotocore/patch.py | 1 + ddtrace/contrib/flask/middleware.py | 2 +- ddtrace/opentracer/span.py | 7 ++ ddtrace/span.py | 35 ++++++++- tests/commands/ddtrace_run_integration.py | 11 +-- tests/contrib/aiobotocore/py35/test.py | 6 +- tests/contrib/aiobotocore/test.py | 28 +++---- tests/contrib/aiohttp/test_middleware.py | 16 ++-- tests/contrib/aiopg/test.py | 4 +- tests/contrib/boto/test.py | 18 ++--- tests/contrib/botocore/test.py | 20 ++--- tests/contrib/bottle/test.py | 16 ++-- tests/contrib/bottle/test_autopatch.py | 6 +- tests/contrib/bottle/test_distributed.py | 4 +- tests/contrib/cassandra/test.py | 18 ++--- tests/contrib/dbapi/test_unit.py | 8 +- tests/contrib/django/test_middleware.py | 20 ++--- .../test_djangorestframework.py | 4 +- tests/contrib/elasticsearch/test.py | 4 +- tests/contrib/falcon/test_suite.py | 16 ++-- tests/contrib/flask/__init__.py | 10 +-- tests/contrib/flask/test_errorhandler.py | 10 +-- tests/contrib/flask/test_flask_helpers.py | 4 +- tests/contrib/flask/test_hooks.py | 4 +- tests/contrib/flask/test_middleware.py | 22 +++--- tests/contrib/flask/test_request.py | 55 ++------------ tests/contrib/flask/test_signals.py | 6 +- tests/contrib/flask/test_static.py | 4 +- tests/contrib/flask/test_template.py | 4 +- tests/contrib/flask/test_views.py | 28 +------ .../flask_autopatch/test_flask_autopatch.py | 5 +- tests/contrib/flask_cache/test.py | 8 +- .../flask_cache/test_wrapper_safety.py | 4 +- tests/contrib/httplib/test_httplib.py | 74 ++++++------------- tests/contrib/molten/test_molten.py | 6 +- tests/contrib/mysql/test_mysql.py | 12 +-- tests/contrib/mysqldb/test_mysql.py | 16 ++-- tests/contrib/psycopg/test_psycopg.py | 4 +- tests/contrib/pylibmc/test.py | 2 +- tests/contrib/pylons/test_pylons.py | 26 +++---- tests/contrib/pymemcache/test_client_mixin.py | 2 +- tests/contrib/pymongo/test.py | 8 +- tests/contrib/pymysql/test_pymysql.py | 7 +- tests/contrib/pyramid/utils.py | 20 ++--- tests/contrib/redis/test.py | 16 ++-- tests/contrib/requests/test_requests.py | 16 ++-- tests/contrib/sqlalchemy/mixins.py | 2 +- tests/contrib/sqlalchemy/test_mysql.py | 4 +- tests/contrib/sqlalchemy/test_postgres.py | 4 +- tests/contrib/sqlalchemy/test_sqlite.py | 2 +- tests/contrib/sqlite3/test_sqlite3.py | 2 +- .../tornado/test_executor_decorator.py | 10 +-- .../contrib/tornado/test_tornado_template.py | 6 +- tests/contrib/tornado/test_tornado_web.py | 30 ++++---- tests/contrib/tornado/test_wrap_decorator.py | 12 +-- tests/contrib/vertica/test_vertica.py | 8 +- tests/opentracer/test_span.py | 10 +-- tests/opentracer/test_tracer.py | 4 +- tests/test_compat.py | 19 ++++- tests/test_global_config.py | 4 +- tests/test_span.py | 59 +++++++++++++-- tests/test_tracer.py | 4 +- 63 files changed, 419 insertions(+), 388 deletions(-) diff --git a/ddtrace/compat.py b/ddtrace/compat.py index 3b1524c06a1..654f6b68aa7 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -51,6 +51,16 @@ pattern_type = re._pattern_type +def is_integer(obj): + """Helper to determine if the provided ``obj`` is an integer type or not""" + # DEV: We have to make sure it is an integer and not a boolean + # >>> type(True) + # + # >>> isinstance(True, int) + # True + return isinstance(obj, six.integer_types) and not isinstance(obj, bool) + + try: from time import time_ns except ImportError: diff --git a/ddtrace/contrib/aiobotocore/patch.py b/ddtrace/contrib/aiobotocore/patch.py index 5e732476587..ca903f170d1 100644 --- a/ddtrace/contrib/aiobotocore/patch.py +++ b/ddtrace/contrib/aiobotocore/patch.py @@ -48,6 +48,7 @@ def read(self, *args, **kwargs): span.resource = self._self_parent_span.resource span.span_type = self._self_parent_span.span_type span.meta = dict(self._self_parent_span.meta) + span.metrics = dict(self._self_parent_span.metrics) result = yield from self.__wrapped__.read(*args, **kwargs) span.set_tag('Length', len(result)) diff --git a/ddtrace/contrib/flask/middleware.py b/ddtrace/contrib/flask/middleware.py index fb9a45f88c5..5d57e5418e1 100644 --- a/ddtrace/contrib/flask/middleware.py +++ b/ddtrace/contrib/flask/middleware.py @@ -135,7 +135,7 @@ def _finish_span(self, span, exception=None): if not span or not span.sampled: return - code = span.get_tag(http.STATUS_CODE) or 0 + code = span.get_metric(http.STATUS_CODE) or 0 try: code = int(code) except Exception: diff --git a/ddtrace/opentracer/span.py b/ddtrace/opentracer/span.py index 7342c4b4a9d..4cd6d16373c 100644 --- a/ddtrace/opentracer/span.py +++ b/ddtrace/opentracer/span.py @@ -137,6 +137,13 @@ def _get_tag(self, key): """ return self._dd_span.get_tag(key) + def _get_metric(self, key): + """Gets a metric from the span. + + This method retrieves the metric from the underlying datadog span. + """ + return self._dd_span.get_metric(key) + def __enter__(self): return self diff --git a/ddtrace/span.py b/ddtrace/span.py index c4dd490b3b6..5cbca55378b 100644 --- a/ddtrace/span.py +++ b/ddtrace/span.py @@ -3,9 +3,9 @@ import sys import traceback -from .compat import StringIO, stringify, iteritems, numeric_types, time_ns +from .compat import StringIO, stringify, iteritems, numeric_types, time_ns, is_integer from .constants import NUMERIC_TAGS, MANUAL_DROP_KEY, MANUAL_KEEP_KEY -from .ext import SpanTypes, errors, priority +from .ext import SpanTypes, errors, priority, net, http from .internal.logger import get_logger @@ -154,7 +154,31 @@ def set_tag(self, key, value=None): be ignored. """ - if key in NUMERIC_TAGS: + # Determine once up front + is_an_int = is_integer(value) + + # Explicitly try to convert expected integers to `int` + # DEV: Some integrations parse these values from strings, but don't call `int(value)` themselves + INT_TYPES = (net.TARGET_PORT, http.STATUS_CODE) + if key in INT_TYPES and not is_an_int: + try: + value = int(value) + is_an_int = True + except (ValueError, TypeError): + pass + + # Set integers that are less than equal to 2^53 as metrics + if is_an_int and abs(value) <= 2 ** 53: + self.set_metric(key, value) + return + + # All floats should be set as a metric + elif isinstance(value, float): + self.set_metric(key, value) + return + + # Key should explicitly be converted to a float if needed + elif key in NUMERIC_TAGS: try: # DEV: `set_metric` will try to cast to `float()` for us self.set_metric(key, value) @@ -162,6 +186,7 @@ def set_tag(self, key, value=None): log.debug('error setting numeric metric %s:%s', key, value) return + elif key == MANUAL_KEEP_KEY: self.context.sampling_priority = priority.USER_KEEP return @@ -171,6 +196,8 @@ def set_tag(self, key, value=None): try: self.meta[key] = stringify(value) + if key in self.metrics: + del self.metrics[key] except Exception: log.debug('error setting tag %s, ignoring it', key, exc_info=True) @@ -217,6 +244,8 @@ def set_metric(self, key, value): log.debug('ignoring not real metric %s:%s', key, value) return + if key in self.meta: + del self.meta[key] self.metrics[key] = value def set_metrics(self, metrics): diff --git a/tests/commands/ddtrace_run_integration.py b/tests/commands/ddtrace_run_integration.py index b1eef8298dd..875742789ea 100644 --- a/tests/commands/ddtrace_run_integration.py +++ b/tests/commands/ddtrace_run_integration.py @@ -34,14 +34,9 @@ assert span.name == 'redis.command' assert span.span_type == 'redis' assert span.error == 0 - meta = { - 'out.host': u'localhost', - 'out.port': str(REDIS_CONFIG['port']), - 'out.redis_db': u'0', - } - for k, v in meta.items(): - assert span.get_tag(k) == v - + assert span.get_metric('out.port') == REDIS_CONFIG['port'] + assert span.get_metric('out.redis_db') == 0 + assert span.get_tag('out.host') == 'localhost' assert span.get_tag('redis.raw_command').startswith(u'mget 0 1 2 3') assert span.get_tag('redis.raw_command').endswith(u'...') diff --git a/tests/contrib/aiobotocore/py35/test.py b/tests/contrib/aiobotocore/py35/test.py index e597b5f3ca7..a6dd77dcc80 100644 --- a/tests/contrib/aiobotocore/py35/test.py +++ b/tests/contrib/aiobotocore/py35/test.py @@ -45,13 +45,13 @@ async def test_response_context_manager(self): span = traces[0][0] assert span.get_tag('aws.operation') == 'GetObject' - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.service == 'aws.s3' assert span.resource == 's3.getobject' read_span = traces[1][0] assert read_span.get_tag('aws.operation') == 'GetObject' - assert read_span.get_tag('http.status_code') == '200' + assert read_span.get_metric('http.status_code') == 200 assert read_span.service == 'aws.s3' assert read_span.resource == 's3.getobject' assert read_span.name == 's3.command.read' @@ -64,7 +64,7 @@ async def test_response_context_manager(self): span = traces[0][0] assert span.get_tag('aws.operation') == 'GetObject' - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.service == 'aws.s3' assert span.resource == 's3.getobject' assert span.name == 's3.command' diff --git a/tests/contrib/aiobotocore/test.py b/tests/contrib/aiobotocore/test.py index 12d65344e50..13718d2af97 100644 --- a/tests/contrib/aiobotocore/test.py +++ b/tests/contrib/aiobotocore/test.py @@ -36,8 +36,8 @@ def test_traced_client(self): self.assertEqual(span.get_tag('aws.agent'), 'aiobotocore') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(span.get_tag('http.status_code'), '200') - self.assertEqual(span.get_tag('retry_attempts'), '0') + self.assertEqual(span.get_metric('http.status_code'), 200) + self.assertEqual(span.get_metric('retry_attempts'), 0) self.assertEqual(span.service, 'aws.ec2') self.assertEqual(span.resource, 'ec2.describeinstances') self.assertEqual(span.name, 'ec2.command') @@ -70,7 +70,7 @@ def test_s3_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.service, 'aws.s3') self.assertEqual(span.resource, 's3.listbuckets') self.assertEqual(span.name, 's3.command') @@ -87,7 +87,7 @@ def test_s3_put(self): assert spans self.assertEqual(len(spans), 2) self.assertEqual(spans[0].get_tag('aws.operation'), 'CreateBucket') - self.assertEqual(spans[0].get_tag(http.STATUS_CODE), '200') + self.assertEqual(spans[0].get_metric(http.STATUS_CODE), 200) self.assertEqual(spans[0].service, 'aws.s3') self.assertEqual(spans[0].resource, 's3.createbucket') self.assertEqual(spans[1].get_tag('aws.operation'), 'PutObject') @@ -136,14 +136,14 @@ def test_s3_client_read(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.operation'), 'GetObject') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.service, 'aws.s3') self.assertEqual(span.resource, 's3.getobject') if pre_08: read_span = traces[1][0] self.assertEqual(read_span.get_tag('aws.operation'), 'GetObject') - self.assertEqual(read_span.get_tag('http.status_code'), '200') + self.assertEqual(read_span.get_metric('http.status_code'), 200) self.assertEqual(read_span.service, 'aws.s3') self.assertEqual(read_span.resource, 's3.getobject') self.assertEqual(read_span.name, 's3.command.read') @@ -163,7 +163,7 @@ def test_sqs_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListQueues') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.service, 'aws.sqs') self.assertEqual(span.resource, 'sqs.listqueues') @@ -179,7 +179,7 @@ def test_kinesis_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListStreams') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.service, 'aws.kinesis') self.assertEqual(span.resource, 'kinesis.liststreams') @@ -196,7 +196,7 @@ def test_lambda_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListFunctions') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.service, 'aws.lambda') self.assertEqual(span.resource, 'lambda.listfunctions') @@ -212,7 +212,7 @@ def test_kms_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListKeys') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.service, 'aws.kms') self.assertEqual(span.resource, 'kms.listkeys') # checking for protection on STS against security leak @@ -264,8 +264,8 @@ def test_opentraced_client(self): self.assertEqual(dd_span.get_tag('aws.agent'), 'aiobotocore') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(dd_span.get_tag('http.status_code'), '200') - self.assertEqual(dd_span.get_tag('retry_attempts'), '0') + self.assertEqual(dd_span.get_metric('http.status_code'), 200) + self.assertEqual(dd_span.get_metric('retry_attempts'), 0) self.assertEqual(dd_span.service, 'aws.ec2') self.assertEqual(dd_span.resource, 'ec2.describeinstances') self.assertEqual(dd_span.name, 'ec2.command') @@ -305,13 +305,13 @@ def test_opentraced_s3_client(self): self.assertEqual(ot_inner_span2.parent_id, ot_outer_span.span_id) self.assertEqual(dd_span.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(dd_span.get_tag('http.status_code'), '200') + self.assertEqual(dd_span.get_metric('http.status_code'), 200) self.assertEqual(dd_span.service, 'aws.s3') self.assertEqual(dd_span.resource, 's3.listbuckets') self.assertEqual(dd_span.name, 's3.command') self.assertEqual(dd_span2.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(dd_span2.get_tag('http.status_code'), '200') + self.assertEqual(dd_span2.get_metric('http.status_code'), 200) self.assertEqual(dd_span2.service, 'aws.s3') self.assertEqual(dd_span2.resource, 's3.listbuckets') self.assertEqual(dd_span2.name, 's3.command') diff --git a/tests/contrib/aiohttp/test_middleware.py b/tests/contrib/aiohttp/test_middleware.py index e49504243d6..02ad52944a9 100644 --- a/tests/contrib/aiohttp/test_middleware.py +++ b/tests/contrib/aiohttp/test_middleware.py @@ -42,7 +42,7 @@ def test_handler(self): assert 'GET /' == span.resource assert str(self.client.make_url('/')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') - assert '200' == span.get_tag('http.status_code') + assert 200 == span.get_metric('http.status_code') assert 0 == span.error @asyncio.coroutine @@ -64,7 +64,7 @@ def _test_param_handler(self, query_string=''): # with the right fields assert 'GET /echo/{name}' == span.resource assert str(self.client.make_url('/echo/team')) == span.get_tag(http.URL) - assert '200' == span.get_tag('http.status_code') + assert 200 == span.get_metric('http.status_code') if self.app[CONFIG_KEY].get('trace_query_string'): assert query_string == span.get_tag(http.QUERY_STRING) else: @@ -112,7 +112,7 @@ def test_404_handler(self): assert '404' == span.resource assert str(self.client.make_url('/404/not_found')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') - assert '404' == span.get_tag('http.status_code') + assert 404 == span.get_metric('http.status_code') @unittest_run_loop @asyncio.coroutine @@ -128,7 +128,7 @@ def test_server_error(self): assert len(traces[0]) == 1 span = traces[0][0] assert span.get_tag('http.method') == 'GET' - assert span.get_tag('http.status_code') == '500' + assert span.get_metric('http.status_code') == 500 assert span.error == 1 @unittest_run_loop @@ -145,7 +145,7 @@ def test_500_response_code(self): assert len(traces[0]) == 1 span = traces[0][0] assert span.get_tag('http.method') == 'GET' - assert span.get_tag('http.status_code') == '503' + assert span.get_metric('http.status_code') == 503 assert span.error == 1 @unittest_run_loop @@ -168,7 +168,7 @@ def test_coroutine_chaining(self): assert 'GET /chaining/' == root.resource assert str(self.client.make_url('/chaining/')) == root.get_tag(http.URL) assert 'GET' == root.get_tag('http.method') - assert '200' == root.get_tag('http.status_code') + assert 200 == root.get_metric('http.status_code') # span created in the coroutine_chaining handler assert 'aiohttp.coro_1' == handler.name assert root.span_id == handler.parent_id @@ -196,7 +196,7 @@ def test_static_handler(self): assert 'GET /statics' == span.resource assert str(self.client.make_url('/statics/empty.txt')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') - assert '200' == span.get_tag('http.status_code') + assert 200 == span.get_metric('http.status_code') @unittest_run_loop @asyncio.coroutine @@ -421,7 +421,7 @@ def _assert_200_parenting(self, traces): assert 'GET /' == inner_span.resource assert str(self.client.make_url('/')) == inner_span.get_tag(http.URL) assert 'GET' == inner_span.get_tag('http.method') - assert '200' == inner_span.get_tag('http.status_code') + assert 200 == inner_span.get_metric('http.status_code') assert 0 == inner_span.error @unittest_run_loop diff --git a/tests/contrib/aiopg/test.py b/tests/contrib/aiopg/test.py index 62ffa12b6fa..faf7f17a12f 100644 --- a/tests/contrib/aiopg/test.py +++ b/tests/contrib/aiopg/test.py @@ -18,7 +18,7 @@ from tests.contrib.asyncio.utils import AsyncioTestCase, mark_asyncio -TEST_PORT = str(POSTGRES_CONFIG['port']) +TEST_PORT = POSTGRES_CONFIG['port'] class AiopgTestCase(AsyncioTestCase): @@ -118,7 +118,7 @@ def assert_conn_is_traced(self, tracer, db, service): assert span.meta['sql.query'] == q assert span.error == 1 # assert span.meta['out.host'] == 'localhost' - assert span.meta['out.port'] == TEST_PORT + assert span.metrics['out.port'] == TEST_PORT assert span.span_type == 'sql' @mark_asyncio diff --git a/tests/contrib/boto/test.py b/tests/contrib/boto/test.py index 3cb18fde721..201bc92b7f1 100644 --- a/tests/contrib/boto/test.py +++ b/tests/contrib/boto/test.py @@ -41,7 +41,7 @@ def test_ec2_client(self): self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.get_tag(http.METHOD), 'POST') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertIsNone(span.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) @@ -53,7 +53,7 @@ def test_ec2_client(self): self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag('aws.operation'), 'RunInstances') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.get_tag(http.METHOD), 'POST') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.service, 'test-boto-tracing.ec2') @@ -107,7 +107,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.operation'), 'get_all_buckets') @@ -117,7 +117,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.get_tag(http.METHOD), 'PUT') self.assertEqual(span.get_tag('path'), '/') self.assertEqual(span.get_tag('aws.operation'), 'create_bucket') @@ -128,7 +128,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.get_tag(http.METHOD), 'HEAD') self.assertEqual(span.get_tag('aws.operation'), 'head_bucket') self.assertEqual(span.service, 'test-boto-tracing.s3') @@ -161,7 +161,7 @@ def test_s3_put(self): # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket') - self.assertEqual(spans[0].get_tag(http.STATUS_CODE), '200') + self.assertEqual(spans[0].get_metric(http.STATUS_CODE), 200) self.assertEqual(spans[0].service, 'test-boto-tracing.s3') self.assertEqual(spans[0].resource, 's3.put') # get bucket @@ -215,7 +215,7 @@ def test_lambda_client(self): assert spans self.assertEqual(len(spans), 2) span = spans[0] - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.region'), 'us-east-2') self.assertEqual(span.get_tag('aws.operation'), 'list_functions') @@ -285,7 +285,7 @@ def test_ec2_client_ot(self): self.assertEqual(ot_span.resource, 'ot_span') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(dd_span.get_metric(http.STATUS_CODE), 200) self.assertEqual(dd_span.get_tag(http.METHOD), 'POST') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') @@ -301,7 +301,7 @@ def test_ec2_client_ot(self): self.assertEqual(dd_span.parent_id, ot_span.span_id) self.assertEqual(dd_span.get_tag('aws.operation'), 'RunInstances') - self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(dd_span.get_metric(http.STATUS_CODE), 200) self.assertEqual(dd_span.get_tag(http.METHOD), 'POST') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.service, 'test-boto-tracing.ec2') diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index fd6b0833edb..942d137ad6a 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -46,8 +46,8 @@ def test_traced_client(self): self.assertEqual(span.get_tag('aws.agent'), 'botocore') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') - self.assertEqual(span.get_tag('retry_attempts'), '0') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + self.assertEqual(span.get_metric('retry_attempts'), 0) self.assertEqual(span.service, 'test-botocore-tracing.ec2') self.assertEqual(span.resource, 'ec2.describeinstances') self.assertEqual(span.name, 'ec2.command') @@ -82,7 +82,7 @@ def test_s3_client(self): span = spans[0] self.assertEqual(len(spans), 2) self.assertEqual(span.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.service, 'test-botocore-tracing.s3') self.assertEqual(span.resource, 's3.listbuckets') @@ -110,7 +110,7 @@ def test_s3_put(self): span = spans[0] self.assertEqual(len(spans), 2) self.assertEqual(span.get_tag('aws.operation'), 'CreateBucket') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.service, 'test-botocore-tracing.s3') self.assertEqual(span.resource, 's3.createbucket') self.assertEqual(spans[1].get_tag('aws.operation'), 'PutObject') @@ -133,7 +133,7 @@ def test_sqs_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListQueues') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.service, 'test-botocore-tracing.sqs') self.assertEqual(span.resource, 'sqs.listqueues') @@ -150,7 +150,7 @@ def test_kinesis_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListStreams') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.service, 'test-botocore-tracing.kinesis') self.assertEqual(span.resource, 'kinesis.liststreams') @@ -192,7 +192,7 @@ def test_lambda_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListFunctions') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.service, 'test-botocore-tracing.lambda') self.assertEqual(span.resource, 'lambda.listfunctions') @@ -209,7 +209,7 @@ def test_kms_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListKeys') - self.assertEqual(span.get_tag(http.STATUS_CODE), '200') + self.assertEqual(span.get_metric(http.STATUS_CODE), 200) self.assertEqual(span.service, 'test-botocore-tracing.kms') self.assertEqual(span.resource, 'kms.listkeys') @@ -242,8 +242,8 @@ def test_traced_client_ot(self): self.assertEqual(dd_span.get_tag('aws.agent'), 'botocore') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200') - self.assertEqual(dd_span.get_tag('retry_attempts'), '0') + self.assertEqual(dd_span.get_metric(http.STATUS_CODE), 200) + self.assertEqual(dd_span.get_metric('retry_attempts'), 0) self.assertEqual(dd_span.service, 'test-botocore-tracing.ec2') self.assertEqual(dd_span.resource, 'ec2.describeinstances') self.assertEqual(dd_span.name, 'ec2.command') diff --git a/tests/contrib/bottle/test.py b/tests/contrib/bottle/test.py index 1de486d61d6..290984dc359 100644 --- a/tests/contrib/bottle/test.py +++ b/tests/contrib/bottle/test.py @@ -58,7 +58,7 @@ def hi(name): assert s.service == 'bottle-app' assert s.span_type == 'web' assert s.resource == 'GET /hi/' - assert s.get_tag('http.status_code') == '200' + assert s.get_metric('http.status_code') == 200 assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/hi/dougie' if ddtrace.config.bottle.trace_query_string: @@ -99,7 +99,7 @@ def handled(): assert len(spans) == 1 s = spans[0] assert s.resource == 'GET /2xx' - assert s.get_tag('http.status_code') == '202' + assert s.get_metric('http.status_code') == 202 assert s.error == 0 def test_400_return(self): @@ -120,7 +120,7 @@ def handled400(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /400_return' - assert s.get_tag('http.status_code') == '400' + assert s.get_metric('http.status_code') == 400 assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/400_return' assert s.error == 0 @@ -143,7 +143,7 @@ def handled400(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /400_raise' - assert s.get_tag('http.status_code') == '400' + assert s.get_metric('http.status_code') == 400 assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/400_raise' assert s.error == 1 @@ -166,7 +166,7 @@ def hi(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' - assert s.get_tag('http.status_code') == '500' + assert s.get_metric('http.status_code') == 500 assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/hi' assert s.error == 1 @@ -233,7 +233,7 @@ def hi(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' - assert s.get_tag('http.status_code') == '420' + assert s.get_metric('http.status_code') == 420 assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/hi' @@ -254,7 +254,7 @@ def home(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /home/' - assert s.get_tag('http.status_code') == '200' + assert s.get_metric('http.status_code') == 200 assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/home/' @@ -404,7 +404,7 @@ def hi(name): assert dd_span.name == 'bottle.request' assert dd_span.service == 'bottle-app' assert dd_span.resource == 'GET /hi/' - assert dd_span.get_tag('http.status_code') == '200' + assert dd_span.get_metric('http.status_code') == 200 assert dd_span.get_tag('http.method') == 'GET' assert dd_span.get_tag(http.URL) == 'http://localhost:80/hi/dougie' diff --git a/tests/contrib/bottle/test_autopatch.py b/tests/contrib/bottle/test_autopatch.py index 8bd9c48ba35..51ca98fc39a 100644 --- a/tests/contrib/bottle/test_autopatch.py +++ b/tests/contrib/bottle/test_autopatch.py @@ -48,7 +48,7 @@ def hi(name): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi/' - assert s.get_tag('http.status_code') == '200' + assert s.get_metric('http.status_code') == 200 assert s.get_tag('http.method') == 'GET' services = self.tracer.writer.pop_services() @@ -73,7 +73,7 @@ def hi(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' - assert s.get_tag('http.status_code') == '500' + assert s.get_metric('http.status_code') == 500 assert s.get_tag('http.method') == 'GET' def test_bottle_global_tracer(self): @@ -93,5 +93,5 @@ def home(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /home/' - assert s.get_tag('http.status_code') == '200' + assert s.get_metric('http.status_code') == 200 assert s.get_tag('http.method') == 'GET' diff --git a/tests/contrib/bottle/test_distributed.py b/tests/contrib/bottle/test_distributed.py index 86a47af7515..58861999e1f 100644 --- a/tests/contrib/bottle/test_distributed.py +++ b/tests/contrib/bottle/test_distributed.py @@ -56,7 +56,7 @@ def hi(name): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi/' - assert s.get_tag('http.status_code') == '200' + assert s.get_metric('http.status_code') == 200 assert s.get_tag('http.method') == 'GET' # check distributed headers assert 123 == s.trace_id @@ -83,7 +83,7 @@ def hi(name): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi/' - assert s.get_tag('http.status_code') == '200' + assert s.get_metric('http.status_code') == 200 assert s.get_tag('http.method') == 'GET' # check distributed headers assert 123 != s.trace_id diff --git a/tests/contrib/cassandra/test.py b/tests/contrib/cassandra/test.py index f2e3544a71f..e0288366583 100644 --- a/tests/contrib/cassandra/test.py +++ b/tests/contrib/cassandra/test.py @@ -68,7 +68,7 @@ class CassandraBase(object): TEST_QUERY = "SELECT * from test.person WHERE name = 'Cassandra'" TEST_QUERY_PAGINATED = 'SELECT * from test.person' TEST_KEYSPACE = 'test' - TEST_PORT = str(CASSANDRA_CONFIG['port']) + TEST_PORT = CASSANDRA_CONFIG['port'] TEST_SERVICE = 'test-cassandra' def _traced_session(self): @@ -124,8 +124,8 @@ def _test_query_base(self, execute_fn): assert query.span_type == 'cassandra' assert query.get_tag(cassx.KEYSPACE) == self.TEST_KEYSPACE - assert query.get_tag(net.TARGET_PORT) == self.TEST_PORT - assert query.get_tag(cassx.ROW_COUNT) == '1' + assert query.get_metric(net.TARGET_PORT) == self.TEST_PORT + assert query.get_metric(cassx.ROW_COUNT) == 1 assert query.get_tag(cassx.PAGE_NUMBER) is None assert query.get_tag(cassx.PAGINATED) == 'False' assert query.get_tag(net.TARGET_HOST) == '127.0.0.1' @@ -204,8 +204,8 @@ def execute_fn(session, query): assert dd_span.span_type == 'cassandra' assert dd_span.get_tag(cassx.KEYSPACE) == self.TEST_KEYSPACE - assert dd_span.get_tag(net.TARGET_PORT) == self.TEST_PORT - assert dd_span.get_tag(cassx.ROW_COUNT) == '1' + assert dd_span.get_metric(net.TARGET_PORT) == self.TEST_PORT + assert dd_span.get_metric(cassx.ROW_COUNT) == 1 assert dd_span.get_tag(cassx.PAGE_NUMBER) is None assert dd_span.get_tag(cassx.PAGINATED) == 'False' assert dd_span.get_tag(net.TARGET_HOST) == '127.0.0.1' @@ -262,14 +262,14 @@ def test_paginated_query(self): assert query.span_type == 'cassandra' assert query.get_tag(cassx.KEYSPACE) == self.TEST_KEYSPACE - assert query.get_tag(net.TARGET_PORT) == self.TEST_PORT + assert query.get_metric(net.TARGET_PORT) == self.TEST_PORT if i == 3: - assert query.get_tag(cassx.ROW_COUNT) == '0' + assert query.get_metric(cassx.ROW_COUNT) == 0 else: - assert query.get_tag(cassx.ROW_COUNT) == '1' + assert query.get_metric(cassx.ROW_COUNT) == 1 assert query.get_tag(net.TARGET_HOST) == '127.0.0.1' assert query.get_tag(cassx.PAGINATED) == 'True' - assert query.get_tag(cassx.PAGE_NUMBER) == str(i + 1) + assert query.get_metric(cassx.PAGE_NUMBER) == i + 1 def test_trace_with_service(self): session, tracer = self._traced_session() diff --git a/tests/contrib/dbapi/test_unit.py b/tests/contrib/dbapi/test_unit.py index 57de37318fc..18b78f6051c 100644 --- a/tests/contrib/dbapi/test_unit.py +++ b/tests/contrib/dbapi/test_unit.py @@ -171,7 +171,7 @@ def method(): assert span.span_type == 'sql', 'Span has the correct span type' # Row count assert span.get_metric('db.rowcount') == 123, 'Row count is set as a metric' - assert span.get_tag('sql.rows') == '123', 'Row count is set as a tag (for legacy django cursor replacement)' + assert span.get_metric('sql.rows') == 123, 'Row count is set as a tag (for legacy django cursor replacement)' def test_django_traced_cursor_backward_compatibility(self): cursor = self.cursor @@ -190,7 +190,7 @@ def method(): span = tracer.writer.pop()[0] # type: Span # Row count assert span.get_metric('db.rowcount') == 123, 'Row count is set as a metric' - assert span.get_tag('sql.rows') == '123', 'Row count is set as a tag (for legacy django cursor replacement)' + assert span.get_metric('sql.rows') == 123, 'Row count is set as a tag (for legacy django cursor replacement)' def test_cursor_analytics_default(self): cursor = self.cursor @@ -408,7 +408,7 @@ def method(): assert span.span_type == 'sql', 'Span has the correct span type' # Row count assert span.get_metric('db.rowcount') == 123, 'Row count is set as a metric' - assert span.get_tag('sql.rows') == '123', 'Row count is set as a tag (for legacy django cursor replacement)' + assert span.get_metric('sql.rows') == 123, 'Row count is set as a tag (for legacy django cursor replacement)' def test_django_traced_cursor_backward_compatibility(self): cursor = self.cursor @@ -427,7 +427,7 @@ def method(): span = tracer.writer.pop()[0] # type: Span # Row count assert span.get_metric('db.rowcount') == 123, 'Row count is set as a metric' - assert span.get_tag('sql.rows') == '123', 'Row count is set as a tag (for legacy django cursor replacement)' + assert span.get_metric('sql.rows') == 123, 'Row count is set as a tag (for legacy django cursor replacement)' def test_fetch_no_analytics(self): """ Confirm fetch* methods do not have analytics sample rate metric """ diff --git a/tests/contrib/django/test_middleware.py b/tests/contrib/django/test_middleware.py index e07e0b3b86f..39f713eed48 100644 --- a/tests/contrib/django/test_middleware.py +++ b/tests/contrib/django/test_middleware.py @@ -36,7 +36,7 @@ def test_middleware_trace_request(self, query_string=''): sp_database = spans[2] assert sp_database.get_tag('django.db.vendor') == 'sqlite' assert sp_template.get_tag('django.template_name') == 'users_list.html' - assert sp_request.get_tag('http.status_code') == '200' + assert sp_request.get_metric('http.status_code') == 200 assert sp_request.get_tag(http.URL) == 'http://testserver/users/' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' @@ -205,7 +205,7 @@ def test_middleware_trace_errors(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_tag('http.status_code') == '403' + assert span.get_metric('http.status_code') == 403 assert span.get_tag(http.URL) == 'http://testserver/fail-view/' assert span.resource == 'tests.contrib.django.app.views.ForbiddenView' @@ -219,7 +219,7 @@ def test_middleware_trace_function_based_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.get_tag(http.URL) == 'http://testserver/fn-view/' assert span.resource == 'tests.contrib.django.app.views.function_view' @@ -234,7 +234,7 @@ def test_middleware_trace_error_500(self): assert len(spans) == 1 span = spans[0] assert span.error == 1 - assert span.get_tag('http.status_code') == '500' + assert span.get_metric('http.status_code') == 500 assert span.get_tag(http.URL) == 'http://testserver/error-500/' assert span.resource == 'tests.contrib.django.app.views.error_500' assert 'Error 500' in span.get_tag('error.stack') @@ -249,7 +249,7 @@ def test_middleware_trace_callable_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.get_tag(http.URL) == 'http://testserver/feed-view/' assert span.resource == 'tests.contrib.django.app.views.FeedView' @@ -263,7 +263,7 @@ def test_middleware_trace_partial_based_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.get_tag(http.URL) == 'http://testserver/partial-view/' assert span.resource == 'partial' @@ -277,7 +277,7 @@ def test_middleware_trace_lambda_based_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.get_tag(http.URL) == 'http://testserver/lambda-view/' assert span.resource == 'tests.contrib.django.app.views.' @@ -300,7 +300,7 @@ def test_middleware_without_user(self): spans = self.tracer.writer.pop() assert len(spans) == 3 sp_request = spans[0] - assert sp_request.get_tag('http.status_code') == '200' + assert sp_request.get_metric('http.status_code') == 200 assert sp_request.get_tag('django.user.is_authenticated') is None def test_middleware_propagation(self): @@ -425,7 +425,7 @@ def test_middleware_trace_request_ot(self): assert sp_database.get_tag('django.db.vendor') == 'sqlite' assert sp_template.get_tag('django.template_name') == 'users_list.html' - assert sp_request.get_tag('http.status_code') == '200' + assert sp_request.get_metric('http.status_code') == 200 assert sp_request.get_tag(http.URL) == 'http://testserver/users/' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' @@ -451,7 +451,7 @@ def test_middleware_trace_request_404(self): assert sp_template.get_tag('django.template_name') == 'unknown' # Request - assert sp_request.get_tag('http.status_code') == '404' + assert sp_request.get_metric('http.status_code') == 404 assert sp_request.get_tag(http.URL) == 'http://testserver/unknown-url' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' diff --git a/tests/contrib/djangorestframework/test_djangorestframework.py b/tests/contrib/djangorestframework/test_djangorestframework.py index 8cc14067f12..0d2aa41ec6c 100644 --- a/tests/contrib/djangorestframework/test_djangorestframework.py +++ b/tests/contrib/djangorestframework/test_djangorestframework.py @@ -38,7 +38,7 @@ def test_unpatch(self): assert sp.resource == 'tests.contrib.djangorestframework.app.views.UserViewSet' assert sp.error == 0 assert sp.span_type == 'web' - assert sp.get_tag('http.status_code') == '500' + assert sp.get_metric('http.status_code') == 500 assert sp.get_tag('error.msg') is None def test_trace_exceptions(self): @@ -56,6 +56,6 @@ def test_trace_exceptions(self): assert sp.error == 1 assert sp.span_type == 'web' assert sp.get_tag('http.method') == 'GET' - assert sp.get_tag('http.status_code') == '500' + assert sp.get_metric('http.status_code') == 500 assert sp.get_tag('error.msg') == 'Authentication credentials were not provided.' assert 'NotAuthenticated' in sp.get_tag('error.stack') diff --git a/tests/contrib/elasticsearch/test.py b/tests/contrib/elasticsearch/test.py index 95af41a047c..1e36e4a8e34 100644 --- a/tests/contrib/elasticsearch/test.py +++ b/tests/contrib/elasticsearch/test.py @@ -131,7 +131,7 @@ def test_elasticsearch(self): spans = writer.pop() assert spans span = spans[0] - assert span.get_tag(http.STATUS_CODE) == u'404' + assert span.get_metric(http.STATUS_CODE) == 404 # Raise error 400, the index 10 is created twice try: @@ -142,7 +142,7 @@ def test_elasticsearch(self): spans = writer.pop() assert spans span = spans[-1] - assert span.get_tag(http.STATUS_CODE) == u'400' + assert span.get_metric(http.STATUS_CODE) == 400 # Drop the index, checking it won't raise exception on success or failure es.indices.delete(index=self.ES_INDEX, ignore=[400, 404]) diff --git a/tests/contrib/falcon/test_suite.py b/tests/contrib/falcon/test_suite.py index 06ba5a2df2e..20c2298adaf 100644 --- a/tests/contrib/falcon/test_suite.py +++ b/tests/contrib/falcon/test_suite.py @@ -21,7 +21,7 @@ def test_404(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET 404' - assert span.get_tag(httpx.STATUS_CODE) == '404' + assert span.get_metric(httpx.STATUS_CODE) == 404 assert span.get_tag(httpx.URL) == 'http://falconframework.org/fake_endpoint' assert httpx.QUERY_STRING not in span.meta assert span.parent_id is None @@ -41,7 +41,7 @@ def test_exception(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.ResourceException' - assert span.get_tag(httpx.STATUS_CODE) == '500' + assert span.get_metric(httpx.STATUS_CODE) == 500 assert span.get_tag(httpx.URL) == 'http://falconframework.org/exception' assert span.parent_id is None @@ -57,7 +57,7 @@ def test_200(self, query_string=''): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.Resource200' - assert span.get_tag(httpx.STATUS_CODE) == '200' + assert span.get_metric(httpx.STATUS_CODE) == 200 fqs = ('?' + query_string) if query_string else '' assert span.get_tag(httpx.URL) == 'http://falconframework.org/200' + fqs if config.falcon.trace_query_string: @@ -154,7 +154,7 @@ def test_201(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'POST tests.contrib.falcon.app.resources.Resource201' - assert span.get_tag(httpx.STATUS_CODE) == '201' + assert span.get_metric(httpx.STATUS_CODE) == 201 assert span.get_tag(httpx.URL) == 'http://falconframework.org/201' assert span.parent_id is None @@ -170,7 +170,7 @@ def test_500(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.Resource500' - assert span.get_tag(httpx.STATUS_CODE) == '500' + assert span.get_metric(httpx.STATUS_CODE) == 500 assert span.get_tag(httpx.URL) == 'http://falconframework.org/500' assert span.parent_id is None @@ -185,7 +185,7 @@ def test_404_exception(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.ResourceNotFound' - assert span.get_tag(httpx.STATUS_CODE) == '404' + assert span.get_metric(httpx.STATUS_CODE) == 404 assert span.get_tag(httpx.URL) == 'http://falconframework.org/not_found' assert span.parent_id is None @@ -200,7 +200,7 @@ def test_404_exception_no_stacktracer(self): span = traces[0][0] assert span.name == 'falcon.request' assert span.service == self._service - assert span.get_tag(httpx.STATUS_CODE) == '404' + assert span.get_metric(httpx.STATUS_CODE) == 404 assert span.get_tag(errx.ERROR_TYPE) is None assert span.parent_id is None @@ -229,7 +229,7 @@ def test_200_ot(self): assert dd_span.name == 'falcon.request' assert dd_span.service == self._service assert dd_span.resource == 'GET tests.contrib.falcon.app.resources.Resource200' - assert dd_span.get_tag(httpx.STATUS_CODE) == '200' + assert dd_span.get_metric(httpx.STATUS_CODE) == 200 assert dd_span.get_tag(httpx.URL) == 'http://falconframework.org/200' def test_falcon_request_hook(self): diff --git a/tests/contrib/flask/__init__.py b/tests/contrib/flask/__init__.py index 8c9a1524ca2..582f0d44896 100644 --- a/tests/contrib/flask/__init__.py +++ b/tests/contrib/flask/__init__.py @@ -12,7 +12,7 @@ def setUp(self): patch() - self.app = flask.Flask(__name__, template_folder='test_templates/') + self.app = flask.Flask(__name__, template_folder="test_templates/") self.client = self.app.test_client() Pin.override(self.app, tracer=self.tracer) @@ -27,21 +27,21 @@ def get_spans(self): return self.tracer.writer.pop() def assert_is_wrapped(self, obj): - self.assertTrue(isinstance(obj, wrapt.ObjectProxy), '{} is not wrapped'.format(obj)) + self.assertTrue(isinstance(obj, wrapt.ObjectProxy), "{} is not wrapped".format(obj)) def assert_is_not_wrapped(self, obj): - self.assertFalse(isinstance(obj, wrapt.ObjectProxy), '{} is wrapped'.format(obj)) + self.assertFalse(isinstance(obj, wrapt.ObjectProxy), "{} is wrapped".format(obj)) def find_span_by_name(self, spans, name, required=True): """Helper to find the first span with a given name from a list""" span = next((s for s in spans if s.name == name), None) if required: - self.assertIsNotNone(span, 'could not find span with name {}'.format(name)) + self.assertIsNotNone(span, "could not find span with name {}".format(name)) return span def find_span_parent(self, spans, span, required=True): """Helper to search for a span's parent in a given list of spans""" parent = next((s for s in spans if s.span_id == span.parent_id), None) if required: - self.assertIsNotNone(parent, 'could not find parent span {}'.format(span)) + self.assertIsNotNone(parent, "could not find parent span {}".format(span)) return parent diff --git a/tests/contrib/flask/test_errorhandler.py b/tests/contrib/flask/test_errorhandler.py index 57e48f6c25e..66226114224 100644 --- a/tests/contrib/flask/test_errorhandler.py +++ b/tests/contrib/flask/test_errorhandler.py @@ -23,7 +23,7 @@ def test_default_404_handler(self): # flask.request span self.assertEqual(req_span.error, 0) - self.assertEqual(req_span.get_tag('http.status_code'), '404') + self.assertEqual(req_span.get_metric('http.status_code'), 404) self.assertIsNone(req_span.get_tag('flask.endpoint')) self.assertIsNone(req_span.get_tag('flask.url_rule')) @@ -68,7 +68,7 @@ def endpoint_500(): # flask.request span self.assertEqual(req_span.error, 1) - self.assertEqual(req_span.get_tag('http.status_code'), '500') + self.assertEqual(req_span.get_metric('http.status_code'), 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_500') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') @@ -128,7 +128,7 @@ def endpoint_500(): # flask.request span self.assertEqual(req_span.error, 0) - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_500') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') @@ -191,7 +191,7 @@ def endpoint_error(): # flask.request span self.assertEqual(req_span.error, 1) - self.assertEqual(req_span.get_tag('http.status_code'), '500') + self.assertEqual(req_span.get_metric('http.status_code'), 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_error') self.assertEqual(req_span.get_tag('flask.url_rule'), '/error') @@ -258,7 +258,7 @@ def endpoint_error(): # flask.request span self.assertEqual(req_span.error, 0) - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_error') self.assertEqual(req_span.get_tag('flask.url_rule'), '/error') diff --git a/tests/contrib/flask/test_flask_helpers.py b/tests/contrib/flask/test_flask_helpers.py index 976e7f168ef..847dee8396b 100644 --- a/tests/contrib/flask/test_flask_helpers.py +++ b/tests/contrib/flask/test_flask_helpers.py @@ -49,7 +49,7 @@ def test_jsonify(self): self.assertIsNone(spans[0].service) self.assertEqual(spans[0].name, 'flask.jsonify') self.assertEqual(spans[0].resource, 'flask.jsonify') - self.assertEqual(set(['system.pid']), set(spans[0].meta.keys())) + assert spans[0].meta == dict() self.assertEqual(spans[1].name, 'flask.do_teardown_request') self.assertEqual(spans[2].name, 'flask.do_teardown_appcontext') @@ -96,7 +96,7 @@ def test_send_file(self): self.assertEqual(spans[0].service, 'flask') self.assertEqual(spans[0].name, 'flask.send_file') self.assertEqual(spans[0].resource, 'flask.send_file') - self.assertEqual(set(['system.pid']), set(spans[0].meta.keys())) + assert spans[0].meta == dict() self.assertEqual(spans[1].name, 'flask.do_teardown_request') self.assertEqual(spans[2].name, 'flask.do_teardown_appcontext') diff --git a/tests/contrib/flask/test_hooks.py b/tests/contrib/flask/test_hooks.py index beda6c6d4c2..bef73c4a95c 100644 --- a/tests/contrib/flask/test_hooks.py +++ b/tests/contrib/flask/test_hooks.py @@ -81,7 +81,7 @@ def before_request(): self.assertEqual(root.get_tag('flask.endpoint'), 'index') self.assertEqual(root.get_tag('flask.url_rule'), '/') self.assertEqual(root.get_tag('http.method'), 'GET') - self.assertEqual(root.get_tag('http.status_code'), '401') + self.assertEqual(root.get_metric('http.status_code'), 401) self.assertEqual(root.get_tag(http.URL), 'http://localhost/') # Assert hook span @@ -182,7 +182,7 @@ def after_request(response): parent = self.find_span_parent(spans, span) # Assert root span - self.assertEqual(root.get_tag('http.status_code'), '401') + self.assertEqual(root.get_metric('http.status_code'), 401) # Assert hook span self.assertEqual(span.service, 'flask') diff --git a/tests/contrib/flask/test_middleware.py b/tests/contrib/flask/test_middleware.py index 783e8040958..361f8e1cd2b 100644 --- a/tests/contrib/flask/test_middleware.py +++ b/tests/contrib/flask/test_middleware.py @@ -110,7 +110,7 @@ def test_success(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.meta.get(http.STATUS_CODE) == '200' + assert s.metrics.get(http.STATUS_CODE) == 200 assert s.meta.get(http.METHOD) == 'GET' services = self.tracer.writer.pop_services() @@ -137,7 +137,7 @@ def test_template(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.meta.get(http.STATUS_CODE) == '200' + assert s.metrics.get(http.STATUS_CODE) == 200 assert s.meta.get(http.METHOD) == 'GET' t = by_name['flask.template'] @@ -165,7 +165,7 @@ def test_handleme(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.meta.get(http.STATUS_CODE) == '202' + assert s.metrics.get(http.STATUS_CODE) == 202 assert s.meta.get(http.METHOD) == 'GET' def test_template_err(self): @@ -189,7 +189,7 @@ def test_template_err(self): assert s.start >= start assert s.duration <= end - start assert s.error == 1 - assert s.meta.get(http.STATUS_CODE) == '500' + assert s.get_metric(http.STATUS_CODE) == 500 assert s.meta.get(http.METHOD) == 'GET' def test_template_render_err(self): @@ -213,7 +213,7 @@ def test_template_render_err(self): assert s.start >= start assert s.duration <= end - start assert s.error == 1 - assert s.meta.get(http.STATUS_CODE) == '500' + assert s.get_metric(http.STATUS_CODE) == 500 assert s.meta.get(http.METHOD) == 'GET' t = by_name['flask.template'] assert t.get_tag('flask.template') == 'render_err.html' @@ -239,7 +239,7 @@ def test_error(self): assert s.resource == 'error' assert s.start >= start assert s.duration <= end - start - assert s.meta.get(http.STATUS_CODE) == '500' + assert s.get_metric(http.STATUS_CODE) == 500 assert s.meta.get(http.METHOD) == 'GET' def test_fatal(self): @@ -264,7 +264,7 @@ def test_fatal(self): assert s.resource == 'fatal' assert s.start >= start assert s.duration <= end - start - assert s.meta.get(http.STATUS_CODE) == '500' + assert s.get_metric(http.STATUS_CODE) == 500 assert s.meta.get(http.METHOD) == 'GET' assert 'ZeroDivisionError' in s.meta.get(errors.ERROR_TYPE), s.meta assert 'by zero' in s.meta.get(errors.ERROR_MSG) @@ -289,7 +289,7 @@ def test_unicode(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.meta.get(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 assert s.meta.get(http.METHOD) == 'GET' assert s.meta.get(http.URL) == u'http://localhost/üŋïĉóđē' @@ -311,7 +311,7 @@ def test_404(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.meta.get(http.STATUS_CODE) == '404' + assert s.get_metric(http.STATUS_CODE) == 404 assert s.meta.get(http.METHOD) == 'GET' assert s.meta.get(http.URL) == u'http://localhost/404/üŋïĉóđē' @@ -348,7 +348,7 @@ def test_custom_span(self): assert s.service == 'test.flask.service' assert s.resource == 'overridden' assert s.error == 0 - assert s.meta.get(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 assert s.meta.get(http.METHOD) == 'GET' def test_success_200_ot(self): @@ -382,5 +382,5 @@ def test_success_200_ot(self): assert dd_span.start >= start assert dd_span.duration <= end - start assert dd_span.error == 0 - assert dd_span.meta.get(http.STATUS_CODE) == '200' + assert dd_span.get_metric(http.STATUS_CODE) == 200 assert dd_span.meta.get(http.METHOD) == 'GET' diff --git a/tests/contrib/flask/test_request.py b/tests/contrib/flask/test_request.py index 03f20dad39f..863fd828b0a 100644 --- a/tests/contrib/flask/test_request.py +++ b/tests/contrib/flask/test_request.py @@ -60,16 +60,11 @@ def index(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'index') self.assertEqual(req_span.get_tag('flask.url_rule'), '/') self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) assert http.QUERY_STRING not in req_span.meta # Handler span @@ -298,18 +293,13 @@ def index(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'index') # Note: contains no query string self.assertEqual(req_span.get_tag('flask.url_rule'), '/') self.assertEqual(req_span.get_tag('http.method'), 'GET') # Note: contains no query string self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) # Handler span handler_span = spans[4] @@ -365,16 +355,11 @@ def unicode(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'unicode') self.assertEqual(req_span.get_tag('flask.url_rule'), u'/üŋïĉóđē') self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), u'http://localhost/üŋïĉóđē') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) # Handler span handler_span = spans[4] @@ -425,13 +410,9 @@ def test_request_404(self): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/not-found') - self.assertEqual(req_span.get_tag('http.status_code'), '404') + self.assertEqual(req_span.get_metric('http.status_code'), 404) # Dispatch span dispatch_span = spans[3] @@ -490,14 +471,9 @@ def not_found(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.endpoint', 'flask.url_rule', 'flask.version', - 'http.url', 'http.method', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/not-found') - self.assertEqual(req_span.get_tag('http.status_code'), '404') + self.assertEqual(req_span.get_metric('http.status_code'), 404) self.assertEqual(req_span.get_tag('flask.endpoint'), 'not_found') self.assertEqual(req_span.get_tag('flask.url_rule'), '/not-found') @@ -567,14 +543,9 @@ def fivehundred(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/500') - self.assertEqual(req_span.get_tag('http.status_code'), '500') + self.assertEqual(req_span.get_metric('http.status_code'), 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'fivehundred') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') @@ -655,14 +626,9 @@ def fivehundredone(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/501') - self.assertEqual(req_span.get_tag('http.status_code'), '501') + self.assertEqual(req_span.get_metric('http.status_code'), 501) self.assertEqual(req_span.get_tag('flask.endpoint'), 'fivehundredone') self.assertEqual(req_span.get_tag('flask.url_rule'), '/501') @@ -767,14 +733,9 @@ def fivehundred(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), - set(req_span.meta.keys()), - ) self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/500') - self.assertEqual(req_span.get_tag('http.status_code'), '500') + self.assertEqual(req_span.get_metric('http.status_code'), 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'fivehundred') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') diff --git a/tests/contrib/flask/test_signals.py b/tests/contrib/flask/test_signals.py index d2c129060d7..fc6e7b12067 100644 --- a/tests/contrib/flask/test_signals.py +++ b/tests/contrib/flask/test_signals.py @@ -109,7 +109,7 @@ def test_signals(self): self.assertEqual(span.service, 'flask') self.assertEqual(span.name, 'tests.contrib.flask.{}'.format(signal_name)) self.assertEqual(span.resource, 'tests.contrib.flask.{}'.format(signal_name)) - self.assertEqual(set(span.meta.keys()), set(['system.pid', 'flask.signal'])) + self.assertEqual(set(span.meta.keys()), set(['flask.signal'])) self.assertEqual(span.meta['flask.signal'], signal_name) def test_signals_multiple(self): @@ -144,7 +144,7 @@ def test_signals_multiple(self): self.assertEqual(span_a.service, 'flask') self.assertEqual(span_a.name, 'tests.contrib.flask.request_started_a') self.assertEqual(span_a.resource, 'tests.contrib.flask.request_started_a') - self.assertEqual(set(span_a.meta.keys()), set(['system.pid', 'flask.signal'])) + self.assertEqual(set(span_a.meta.keys()), set(['flask.signal'])) self.assertEqual(span_a.meta['flask.signal'], 'request_started') # Assert the span that was created @@ -152,7 +152,7 @@ def test_signals_multiple(self): self.assertEqual(span_b.service, 'flask') self.assertEqual(span_b.name, 'tests.contrib.flask.request_started_b') self.assertEqual(span_b.resource, 'tests.contrib.flask.request_started_b') - self.assertEqual(set(span_b.meta.keys()), set(['system.pid', 'flask.signal'])) + self.assertEqual(set(span_b.meta.keys()), set(['flask.signal'])) self.assertEqual(span_b.meta['flask.signal'], 'request_started') def test_signals_pin_disabled(self): diff --git a/tests/contrib/flask/test_static.py b/tests/contrib/flask/test_static.py index 9841fcfd13e..49bb6a45325 100644 --- a/tests/contrib/flask/test_static.py +++ b/tests/contrib/flask/test_static.py @@ -29,7 +29,7 @@ def test_serve_static_file(self): self.assertEqual(req_span.get_tag('flask.endpoint'), 'static') self.assertEqual(req_span.get_tag('flask.url_rule'), '/static/') self.assertEqual(req_span.get_tag('flask.view_args.filename'), 'test.txt') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/static/test.txt') self.assertEqual(req_span.get_tag('http.method'), 'GET') @@ -70,7 +70,7 @@ def test_serve_static_file_404(self): self.assertEqual(req_span.get_tag('flask.endpoint'), 'static') self.assertEqual(req_span.get_tag('flask.url_rule'), '/static/') self.assertEqual(req_span.get_tag('flask.view_args.filename'), 'unknown-file') - self.assertEqual(req_span.get_tag('http.status_code'), '404') + self.assertEqual(req_span.get_metric('http.status_code'), 404) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/static/unknown-file') self.assertEqual(req_span.get_tag('http.method'), 'GET') diff --git a/tests/contrib/flask/test_template.py b/tests/contrib/flask/test_template.py index fa878143f52..e1593156948 100644 --- a/tests/contrib/flask/test_template.py +++ b/tests/contrib/flask/test_template.py @@ -50,7 +50,7 @@ def test_render_template(self): self.assertIsNone(spans[0].service) self.assertEqual(spans[0].name, 'flask.render_template') self.assertEqual(spans[0].resource, 'test.html') - self.assertEqual(set(spans[0].meta.keys()), set(['system.pid', 'flask.template_name'])) + self.assertEqual(set(spans[0].meta.keys()), set(['flask.template_name'])) self.assertEqual(spans[0].meta['flask.template_name'], 'test.html') self.assertEqual(spans[1].name, 'flask.do_teardown_request') @@ -91,7 +91,7 @@ def test_render_template_string(self): self.assertIsNone(spans[0].service) self.assertEqual(spans[0].name, 'flask.render_template_string') self.assertEqual(spans[0].resource, '') - self.assertEqual(set(spans[0].meta.keys()), set(['system.pid', 'flask.template_name'])) + self.assertEqual(set(spans[0].meta.keys()), set(['flask.template_name'])) self.assertEqual(spans[0].meta['flask.template_name'], '') self.assertEqual(spans[1].name, 'flask.do_teardown_request') diff --git a/tests/contrib/flask/test_views.py b/tests/contrib/flask/test_views.py index 8b551af6b7d..2a2ce8030e3 100644 --- a/tests/contrib/flask/test_views.py +++ b/tests/contrib/flask/test_views.py @@ -36,16 +36,11 @@ def dispatch_request(self, name): # flask.request self.assertEqual(req_span.error, 0) - self.assertEqual( - set(req_span.meta.keys()), - set(['flask.endpoint', 'flask.url_rule', 'flask.version', 'flask.view_args.name', - 'http.method', 'http.status_code', 'http.url', 'system.pid']), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'hello') self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # tests.contrib.flask.test_views.hello @@ -78,16 +73,11 @@ def dispatch_request(self, name): # flask.request self.assertEqual(req_span.error, 1) - self.assertEqual( - set(req_span.meta.keys()), - set(['flask.endpoint', 'flask.url_rule', 'flask.version', 'flask.view_args.name', - 'http.method', 'http.status_code', 'http.url', 'system.pid']), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'hello') self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_tag('http.status_code'), '500') + self.assertEqual(req_span.get_metric('http.status_code'), 500) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # flask.dispatch_request @@ -125,16 +115,11 @@ def get(self, name): # flask.request self.assertEqual(req_span.error, 0) - self.assertEqual( - set(req_span.meta.keys()), - set(['flask.endpoint', 'flask.url_rule', 'flask.version', 'flask.view_args.name', - 'http.method', 'http.status_code', 'http.url', 'system.pid']), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'hello') self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # tests.contrib.flask.test_views.hello @@ -165,16 +150,11 @@ def get(self, name): # flask.request self.assertEqual(req_span.error, 1) - self.assertEqual( - set(req_span.meta.keys()), - set(['flask.endpoint', 'flask.url_rule', 'flask.version', 'flask.view_args.name', - 'http.method', 'http.status_code', 'http.url', 'system.pid']), - ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'hello') self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_tag('http.status_code'), '500') + self.assertEqual(req_span.get_metric('http.status_code'), 500) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # flask.dispatch_request diff --git a/tests/contrib/flask_autopatch/test_flask_autopatch.py b/tests/contrib/flask_autopatch/test_flask_autopatch.py index 0119d6f3708..9ef245df0db 100644 --- a/tests/contrib/flask_autopatch/test_flask_autopatch.py +++ b/tests/contrib/flask_autopatch/test_flask_autopatch.py @@ -82,15 +82,14 @@ def index(): # Request tags self.assertEqual( - set(['system.pid', 'flask.version', 'http.url', 'http.method', - 'flask.endpoint', 'flask.url_rule', 'http.status_code']), + set(['flask.version', 'http.url', 'http.method', 'flask.endpoint', 'flask.url_rule']), set(req_span.meta.keys()), ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'index') self.assertEqual(req_span.get_tag('flask.url_rule'), '/') self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/') - self.assertEqual(req_span.get_tag('http.status_code'), '200') + self.assertEqual(req_span.get_metric('http.status_code'), 200) # Handler span handler_span = spans[4] diff --git a/tests/contrib/flask_cache/test.py b/tests/contrib/flask_cache/test.py index b8c256f00a3..1515cd93f8d 100644 --- a/tests/contrib/flask_cache/test.py +++ b/tests/contrib/flask_cache/test.py @@ -18,8 +18,8 @@ class FlaskCacheTest(BaseTracerTestCase): SERVICE = 'test-flask-cache' - TEST_REDIS_PORT = str(REDIS_CONFIG['port']) - TEST_MEMCACHED_PORT = str(MEMCACHED_CONFIG['port']) + TEST_REDIS_PORT = REDIS_CONFIG['port'] + TEST_MEMCACHED_PORT = MEMCACHED_CONFIG['port'] def setUp(self): super(FlaskCacheTest, self).setUp() @@ -196,7 +196,7 @@ def test_default_span_tags_for_redis(self): self.assertEqual(span.span_type, 'cache') self.assertEqual(span.meta[CACHE_BACKEND], 'redis') self.assertEqual(span.meta[net.TARGET_HOST], 'localhost') - self.assertEqual(span.meta[net.TARGET_PORT], self.TEST_REDIS_PORT) + self.assertEqual(span.metrics[net.TARGET_PORT], self.TEST_REDIS_PORT) def test_default_span_tags_memcached(self): # create the TracedCache instance for a Flask app @@ -213,7 +213,7 @@ def test_default_span_tags_memcached(self): self.assertEqual(span.span_type, 'cache') self.assertEqual(span.meta[CACHE_BACKEND], 'memcached') self.assertEqual(span.meta[net.TARGET_HOST], '127.0.0.1') - self.assertEqual(span.meta[net.TARGET_PORT], self.TEST_MEMCACHED_PORT) + self.assertEqual(span.metrics[net.TARGET_PORT], self.TEST_MEMCACHED_PORT) def test_simple_cache_get_ot(self): """OpenTracing version of test_simple_cache_get.""" diff --git a/tests/contrib/flask_cache/test_wrapper_safety.py b/tests/contrib/flask_cache/test_wrapper_safety.py index d9e72417d53..302e3a2dbf1 100644 --- a/tests/contrib/flask_cache/test_wrapper_safety.py +++ b/tests/contrib/flask_cache/test_wrapper_safety.py @@ -191,7 +191,7 @@ def test_redis_cache_tracing_with_a_wrong_connection(self): assert span.span_type == 'cache' assert span.meta[CACHE_BACKEND] == 'redis' assert span.meta[net.TARGET_HOST] == '127.0.0.1' - assert span.meta[net.TARGET_PORT] == '2230' + assert span.metrics[net.TARGET_PORT] == 2230 assert span.error == 1 def test_memcached_cache_tracing_with_a_wrong_connection(self): @@ -225,7 +225,7 @@ def test_memcached_cache_tracing_with_a_wrong_connection(self): assert span.span_type == 'cache' assert span.meta[CACHE_BACKEND] == 'memcached' assert span.meta[net.TARGET_HOST] == 'localhost' - assert span.meta[net.TARGET_PORT] == '2230' + assert span.metrics[net.TARGET_PORT] == 2230 # the pylibmc backend raises an exception and memcached backend does # not, so don't test anything about the status. diff --git a/tests/contrib/httplib/test_httplib.py b/tests/contrib/httplib/test_httplib.py index 9c75ab60375..80c878aa2d2 100644 --- a/tests/contrib/httplib/test_httplib.py +++ b/tests/contrib/httplib/test_httplib.py @@ -17,7 +17,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase -from ...util import assert_dict_issuperset, override_global_tracer +from ...util import override_global_tracer if PY2: from urllib2 import urlopen, build_opener, Request @@ -152,14 +152,9 @@ def test_httplib_request_get_request(self, query_string=''): self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) - assert_dict_issuperset( - span.meta, - { - 'http.method': 'GET', - 'http.status_code': '200', - 'http.url': URL_200, - } - ) + assert span.get_tag('http.method') == 'GET' + assert span.get_tag('http.url') == URL_200 + assert span.get_metric('http.status_code') == 200 if config.httplib.trace_query_string: assert span.get_tag(http.QUERY_STRING) == query_string else: @@ -194,14 +189,9 @@ def test_httplib_request_get_request_https(self): self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) - assert_dict_issuperset( - span.meta, - { - 'http.method': 'GET', - 'http.status_code': '200', - 'http.url': 'https://httpbin.org/status/200', - } - ) + assert span.get_tag('http.method') == 'GET' + assert span.get_metric('http.status_code') == 200 + assert span.get_tag('http.url') == 'https://httpbin.org/status/200' def test_httplib_request_post_request(self): """ @@ -223,14 +213,9 @@ def test_httplib_request_post_request(self): self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) - assert_dict_issuperset( - span.meta, - { - 'http.method': 'POST', - 'http.status_code': '200', - 'http.url': URL_200, - } - ) + assert span.get_tag('http.method') == 'POST' + assert span.get_metric('http.status_code') == 200 + assert span.get_tag('http.url') == URL_200 def test_httplib_request_get_request_query_string(self): """ @@ -251,15 +236,9 @@ def test_httplib_request_get_request_query_string(self): self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) - assert_dict_issuperset( - span.meta, - { - 'http.method': 'GET', - 'http.status_code': '200', - # check url metadata lacks query string - 'http.url': '{}'.format(URL_200), - } - ) + assert span.get_tag('http.method') == 'GET' + assert span.get_metric('http.status_code') == 200 + assert span.get_tag('http.url') == URL_200 def test_httplib_request_500_request(self): """ @@ -287,7 +266,7 @@ def test_httplib_request_500_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 1) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '500') + self.assertEqual(span.get_metric('http.status_code'), 500) self.assertEqual(span.get_tag('http.url'), URL_500) def test_httplib_request_non_200_request(self): @@ -316,7 +295,7 @@ def test_httplib_request_non_200_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '404') + self.assertEqual(span.get_metric('http.status_code'), 404) self.assertEqual(span.get_tag('http.url'), URL_404) def test_httplib_request_get_request_disabled(self): @@ -400,7 +379,7 @@ def test_urllib_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_urllib_request_https(self): @@ -424,7 +403,7 @@ def test_urllib_request_https(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag('http.url'), 'https://httpbin.org/status/200') def test_urllib_request_object(self): @@ -449,7 +428,7 @@ def test_urllib_request_object(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_urllib_request_opener(self): @@ -473,7 +452,7 @@ def test_urllib_request_opener(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_httplib_request_get_request_ot(self): @@ -502,14 +481,9 @@ def test_httplib_request_get_request_ot(self): self.assertEqual(dd_span.span_type, 'http') self.assertEqual(dd_span.name, self.SPAN_NAME) self.assertEqual(dd_span.error, 0) - assert_dict_issuperset( - dd_span.meta, - { - 'http.method': 'GET', - 'http.status_code': '200', - 'http.url': URL_200, - } - ) + assert dd_span.get_tag('http.method') == 'GET' + assert dd_span.get_metric('http.status_code') == 200 + assert dd_span.get_tag('http.url') == URL_200 def test_analytics_default(self): conn = self.get_http_connection(SOCKET) @@ -581,7 +555,7 @@ def test_urllib_request(self): self.assertEqual(span.name, 'httplib.request') self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_urllib_request_https(self): @@ -605,5 +579,5 @@ def test_urllib_request_https(self): self.assertEqual(span.name, 'httplib.request') self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag('http.url'), 'https://httpbin.org/status/200') diff --git a/tests/contrib/molten/test_molten.py b/tests/contrib/molten/test_molten.py index 1c3929b8a0b..e9b9a6f376a 100644 --- a/tests/contrib/molten/test_molten.py +++ b/tests/contrib/molten/test_molten.py @@ -52,7 +52,7 @@ def test_route_success(self): self.assertEqual(span.resource, 'GET /hello/{name}/{age}') self.assertEqual(span.get_tag('http.method'), 'GET') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/hello/Jim/24') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) assert http.QUERY_STRING not in span.meta # See test_resources below for specifics of this difference @@ -81,7 +81,7 @@ def test_route_success_query_string(self): self.assertEqual(span.resource, 'GET /hello/{name}/{age}') self.assertEqual(span.get_tag('http.method'), 'GET') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/hello/Jim/24') - self.assertEqual(span.get_tag('http.status_code'), '200') + self.assertEqual(span.get_metric('http.status_code'), 200) self.assertEqual(span.get_tag(http.QUERY_STRING), 'foo=bar') def test_analytics_global_on_integration_default(self): @@ -168,7 +168,7 @@ def test_route_failure(self): self.assertEqual(span.resource, 'GET 404') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/goodbye') self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_tag('http.status_code'), '404') + self.assertEqual(span.get_metric('http.status_code'), 404) def test_route_exception(self): def route_error() -> str: diff --git a/tests/contrib/mysql/test_mysql.py b/tests/contrib/mysql/test_mysql.py index 34588f90580..d9872c3f922 100644 --- a/tests/contrib/mysql/test_mysql.py +++ b/tests/contrib/mysql/test_mysql.py @@ -50,9 +50,9 @@ def test_simple_query(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -73,9 +73,9 @@ def test_simple_query_fetchll(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -213,9 +213,9 @@ def test_query_proc(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -250,9 +250,9 @@ def test_simple_query_ot(self): assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' assert dd_span.error == 0 + assert dd_span.get_metric('out.port') == 3306 assert_dict_issuperset(dd_span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -287,9 +287,9 @@ def test_simple_query_ot_fetchall(self): assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' assert dd_span.error == 0 + assert dd_span.get_metric('out.port') == 3306 assert_dict_issuperset(dd_span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -418,9 +418,9 @@ def test_patch_unpatch(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) diff --git a/tests/contrib/mysqldb/test_mysql.py b/tests/contrib/mysqldb/test_mysql.py index 590e6b9285c..15581e1f1df 100644 --- a/tests/contrib/mysqldb/test_mysql.py +++ b/tests/contrib/mysqldb/test_mysql.py @@ -53,9 +53,9 @@ def test_simple_query(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -76,9 +76,9 @@ def test_simple_query_fetchall(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -100,9 +100,9 @@ def test_simple_query_with_positional_args(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -123,9 +123,9 @@ def test_simple_query_with_positional_args_fetchall(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -267,9 +267,9 @@ def test_query_proc(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -301,9 +301,9 @@ def test_simple_query_ot(self): assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' assert dd_span.error == 0 + assert dd_span.get_metric('out.port') == 3306 assert_dict_issuperset(dd_span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -335,9 +335,9 @@ def test_simple_query_ot_fetchall(self): assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' assert dd_span.error == 0 + assert dd_span.get_metric('out.port') == 3306 assert_dict_issuperset(dd_span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) @@ -486,9 +486,9 @@ def test_patch_unpatch(self): assert span.name == 'mysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == 3306 assert_dict_issuperset(span.meta, { 'out.host': u'127.0.0.1', - 'out.port': u'3306', 'db.name': u'test', 'db.user': u'test', }) diff --git a/tests/contrib/psycopg/test_psycopg.py b/tests/contrib/psycopg/test_psycopg.py index a8e7e699ad0..2955b0ec362 100644 --- a/tests/contrib/psycopg/test_psycopg.py +++ b/tests/contrib/psycopg/test_psycopg.py @@ -24,7 +24,7 @@ if PSYCOPG2_VERSION >= (2, 7): from psycopg2.sql import SQL -TEST_PORT = str(POSTGRES_CONFIG['port']) +TEST_PORT = POSTGRES_CONFIG['port'] class PsycopgCore(BaseTracerTestCase): @@ -126,6 +126,8 @@ def assert_conn_is_traced(self, db, service): span_type='sql', meta={ 'out.host': '127.0.0.1', + }, + metrics={ 'out.port': TEST_PORT, }, ), diff --git a/tests/contrib/pylibmc/test.py b/tests/contrib/pylibmc/test.py index 3d0759cdf26..b9444f0e429 100644 --- a/tests/contrib/pylibmc/test.py +++ b/tests/contrib/pylibmc/test.py @@ -184,7 +184,7 @@ def _verify_cache_span(self, s, start, end): assert s.span_type == 'cache' assert s.name == 'memcached.cmd' assert s.get_tag('out.host') == cfg['host'] - assert s.get_tag('out.port') == str(cfg['port']) + assert s.get_metric('out.port') == cfg['port'] def test_analytics_default(self): client, tracer = self.get_client() diff --git a/tests/contrib/pylons/test_pylons.py b/tests/contrib/pylons/test_pylons.py index cf9386b6349..d472addb20e 100644 --- a/tests/contrib/pylons/test_pylons.py +++ b/tests/contrib/pylons/test_pylons.py @@ -51,7 +51,7 @@ def test_controller_exception(self): assert span.resource == 'root.raise_exception' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert http.QUERY_STRING not in span.meta assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None @@ -81,7 +81,7 @@ def test_mw_exc_success(self): assert span.resource == 'None.None' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/' - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None @@ -109,7 +109,7 @@ def test_middleware_exception(self): assert span.resource == 'None.None' assert span.error == 1 assert span.get_tag(http.URL) == 'http://localhost:80/' - assert span.get_tag('http.status_code') == '500' + assert span.get_metric('http.status_code') == 500 assert span.get_tag(errors.ERROR_MSG) == 'Middleware exception' assert span.get_tag(errors.ERROR_TYPE) == 'exceptions.Exception' assert span.get_tag(errors.ERROR_STACK) @@ -131,7 +131,7 @@ def test_exc_success(self): assert span.resource == 'root.raise_exception' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' - assert span.get_tag('http.status_code') == '200' + assert span.get_metric('http.status_code') == 200 assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None @@ -153,7 +153,7 @@ def test_exc_client_failure(self): assert span.resource == 'root.raise_exception' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' - assert span.get_tag('http.status_code') == '404' + assert span.get_metric('http.status_code') == 404 assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None @@ -173,7 +173,7 @@ def test_success_200(self, query_string=''): assert span.service == 'web' assert span.resource == 'root.index' - assert span.meta.get(http.STATUS_CODE) == '200' + assert span.metrics.get(http.STATUS_CODE) == 200 if config.pylons.trace_query_string: assert span.meta.get(http.QUERY_STRING) == query_string else: @@ -263,7 +263,7 @@ def test_template_render(self): assert request.service == 'web' assert request.resource == 'root.render' - assert request.meta.get(http.STATUS_CODE) == '200' + assert request.metrics.get(http.STATUS_CODE) == 200 assert request.error == 0 assert template.service == 'web' @@ -283,7 +283,7 @@ def test_template_render_exception(self): assert request.service == 'web' assert request.resource == 'root.render_exception' - assert request.meta.get(http.STATUS_CODE) == '500' + assert request.metrics.get(http.STATUS_CODE) == 500 assert request.error == 1 assert template.service == 'web' @@ -305,7 +305,7 @@ def test_failure_500(self): assert span.service == 'web' assert span.resource == 'root.raise_exception' assert span.error == 1 - assert span.get_tag('http.status_code') == '500' + assert span.get_metric('http.status_code') == 500 assert span.get_tag('error.msg') == 'Ouch!' assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' assert 'Exception: Ouch!' in span.get_tag('error.stack') @@ -322,7 +322,7 @@ def test_failure_500_with_wrong_code(self): assert span.service == 'web' assert span.resource == 'root.raise_wrong_code' assert span.error == 1 - assert span.get_tag('http.status_code') == '500' + assert span.get_metric('http.status_code') == 500 assert span.meta.get(http.URL) == 'http://localhost:80/raise_wrong_code' assert span.get_tag('error.msg') == 'Ouch!' assert 'Exception: Ouch!' in span.get_tag('error.stack') @@ -339,7 +339,7 @@ def test_failure_500_with_custom_code(self): assert span.service == 'web' assert span.resource == 'root.raise_custom_code' assert span.error == 1 - assert span.get_tag('http.status_code') == '512' + assert span.get_metric('http.status_code') == 512 assert span.meta.get(http.URL) == 'http://localhost:80/raise_custom_code' assert span.get_tag('error.msg') == 'Ouch!' assert 'Exception: Ouch!' in span.get_tag('error.stack') @@ -356,7 +356,7 @@ def test_failure_500_with_code_method(self): assert span.service == 'web' assert span.resource == 'root.raise_code_method' assert span.error == 1 - assert span.get_tag('http.status_code') == '500' + assert span.get_metric('http.status_code') == 500 assert span.meta.get(http.URL) == 'http://localhost:80/raise_code_method' assert span.get_tag('error.msg') == 'Ouch!' @@ -423,6 +423,6 @@ def test_success_200_ot(self): assert dd_span.service == 'web' assert dd_span.resource == 'root.index' - assert dd_span.meta.get(http.STATUS_CODE) == '200' + assert dd_span.metrics.get(http.STATUS_CODE) == 200 assert dd_span.meta.get(http.URL) == 'http://localhost:80/' assert dd_span.error == 0 diff --git a/tests/contrib/pymemcache/test_client_mixin.py b/tests/contrib/pymemcache/test_client_mixin.py index dffa835b233..edd9fe292ac 100644 --- a/tests/contrib/pymemcache/test_client_mixin.py +++ b/tests/contrib/pymemcache/test_client_mixin.py @@ -35,7 +35,7 @@ def check_spans(self, num_expected, resources_expected, queries_expected): for span, resource, query in zip(spans, resources_expected, queries_expected): self.assertEqual(span.get_tag(net.TARGET_HOST), TEST_HOST) - self.assertEqual(span.get_tag(net.TARGET_PORT), str(TEST_PORT)) + self.assertEqual(span.get_metric(net.TARGET_PORT), TEST_PORT) self.assertEqual(span.name, memcachedx.CMD) self.assertEqual(span.span_type, 'cache') self.assertEqual(span.service, memcachedx.SERVICE) diff --git a/tests/contrib/pymongo/test.py b/tests/contrib/pymongo/test.py index ca455134a62..1e69ad1228e 100644 --- a/tests/contrib/pymongo/test.py +++ b/tests/contrib/pymongo/test.py @@ -109,7 +109,7 @@ def test_update(self): assert span.meta.get('mongodb.collection') == 'songs' assert span.meta.get('mongodb.db') == 'testdb' assert span.meta.get('out.host') - assert span.meta.get('out.port') + assert span.metrics.get('out.port') expected_resources = set([ 'drop songs', @@ -158,7 +158,7 @@ def test_delete(self): assert span.meta.get('mongodb.collection') == collection_name assert span.meta.get('mongodb.db') == 'testdb' assert span.meta.get('out.host') - assert span.meta.get('out.port') + assert span.metrics.get('out.port') expected_resources = [ 'drop here.are.songs', @@ -223,7 +223,7 @@ def test_insert_find(self): assert span.meta.get('mongodb.collection') == 'teams' assert span.meta.get('mongodb.db') == 'testdb' assert span.meta.get('out.host'), span.pprint() - assert span.meta.get('out.port'), span.pprint() + assert span.metrics.get('out.port'), span.pprint() assert span.start > start assert span.duration < end - start @@ -292,7 +292,7 @@ def test_update_ot(self): assert span.meta.get('mongodb.collection') == 'songs' assert span.meta.get('mongodb.db') == 'testdb' assert span.meta.get('out.host') - assert span.meta.get('out.port') + assert span.metrics.get('out.port') expected_resources = set([ 'drop songs', diff --git a/tests/contrib/pymysql/test_pymysql.py b/tests/contrib/pymysql/test_pymysql.py index 57637a4caa9..5951c33ce37 100644 --- a/tests/contrib/pymysql/test_pymysql.py +++ b/tests/contrib/pymysql/test_pymysql.py @@ -22,7 +22,6 @@ class PyMySQLCore(object): DB_INFO = { 'out.host': MYSQL_CONFIG.get('host'), - 'out.port': str(MYSQL_CONFIG.get('port')), } if PY2: DB_INFO.update({ @@ -68,6 +67,7 @@ def test_simple_query(self): assert span.name == 'pymysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == MYSQL_CONFIG.get('port') meta = {} meta.update(self.DB_INFO) assert_dict_issuperset(span.meta, meta) @@ -88,6 +88,7 @@ def test_simple_query_fetchall(self): assert span.name == 'pymysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == MYSQL_CONFIG.get('port') meta = {} meta.update(self.DB_INFO) assert_dict_issuperset(span.meta, meta) @@ -230,6 +231,7 @@ def test_query_proc(self): assert span.name == 'pymysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == MYSQL_CONFIG.get('port') meta = {} meta.update(self.DB_INFO) assert_dict_issuperset(span.meta, meta) @@ -260,6 +262,7 @@ def test_simple_query_ot(self): assert dd_span.name == 'pymysql.query' assert dd_span.span_type == 'sql' assert dd_span.error == 0 + assert dd_span.get_metric('out.port') == MYSQL_CONFIG.get('port') meta = {} meta.update(self.DB_INFO) assert_dict_issuperset(dd_span.meta, meta) @@ -291,6 +294,7 @@ def test_simple_query_ot_fetchall(self): assert dd_span.name == 'pymysql.query' assert dd_span.span_type == 'sql' assert dd_span.error == 0 + assert dd_span.get_metric('out.port') == MYSQL_CONFIG.get('port') meta = {} meta.update(self.DB_INFO) assert_dict_issuperset(dd_span.meta, meta) @@ -408,6 +412,7 @@ def test_patch_unpatch(self): assert span.name == 'pymysql.query' assert span.span_type == 'sql' assert span.error == 0 + assert span.get_metric('out.port') == MYSQL_CONFIG.get('port') meta = {} meta.update(self.DB_INFO) diff --git a/tests/contrib/pyramid/utils.py b/tests/contrib/pyramid/utils.py index effaae0e45b..a080a2d3300 100644 --- a/tests/contrib/pyramid/utils.py +++ b/tests/contrib/pyramid/utils.py @@ -65,7 +65,7 @@ def test_200(self, query_string=''): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '200' + assert s.metrics.get('http.status_code') == 200 assert s.meta.get(http.URL) == 'http://localhost/' if config.pyramid.trace_query_string: assert s.meta.get(http.QUERY_STRING) == query_string @@ -154,7 +154,7 @@ def test_404(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '404' + assert s.metrics.get('http.status_code') == 404 assert s.meta.get(http.URL) == 'http://localhost/404' def test_302(self): @@ -169,7 +169,7 @@ def test_302(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '302' + assert s.metrics.get('http.status_code') == 302 assert s.meta.get(http.URL) == 'http://localhost/redirect' def test_204(self): @@ -184,7 +184,7 @@ def test_204(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '204' + assert s.metrics.get('http.status_code') == 204 assert s.meta.get(http.URL) == 'http://localhost/nocontent' def test_exception(self): @@ -202,7 +202,7 @@ def test_exception(self): assert s.error == 1 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '500' + assert s.metrics.get('http.status_code') == 500 assert s.meta.get(http.URL) == 'http://localhost/exception' assert s.meta.get('pyramid.route.name') == 'exception' @@ -218,7 +218,7 @@ def test_500(self): assert s.error == 1 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '500' + assert s.metrics.get('http.status_code') == 500 assert s.meta.get(http.URL) == 'http://localhost/error' assert s.meta.get('pyramid.route.name') == 'error' assert type(s.error) == int @@ -238,7 +238,7 @@ def test_json(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '200' + assert s.metrics.get('http.status_code') == 200 assert s.meta.get(http.URL) == 'http://localhost/json' assert s.meta.get('pyramid.route.name') == 'json' @@ -262,7 +262,7 @@ def test_renderer(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '200' + assert s.metrics.get('http.status_code') == 200 assert s.meta.get(http.URL) == 'http://localhost/renderer' assert s.meta.get('pyramid.route.name') == 'renderer' @@ -284,7 +284,7 @@ def test_http_exception_response(self): assert s.error == 1 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.meta.get('http.status_code') == '404' + assert s.metrics.get('http.status_code') == 404 assert s.meta.get(http.URL) == 'http://localhost/404/raise_exception' def test_insert_tween_if_needed_already_set(self): @@ -356,6 +356,6 @@ def test_200_ot(self): assert dd_span.error == 0 assert dd_span.span_type == 'web' assert dd_span.meta.get('http.method') == 'GET' - assert dd_span.meta.get('http.status_code') == '200' + assert dd_span.metrics.get('http.status_code') == 200 assert dd_span.meta.get(http.URL) == 'http://localhost/' assert dd_span.meta.get('pyramid.route.name') == 'index' diff --git a/tests/contrib/redis/test.py b/tests/contrib/redis/test.py index 637bf7f0694..bf746cd1182 100644 --- a/tests/contrib/redis/test.py +++ b/tests/contrib/redis/test.py @@ -52,11 +52,15 @@ def test_long_command(self): assert span.error == 0 meta = { 'out.host': u'localhost', - 'out.port': str(self.TEST_PORT), - 'out.redis_db': u'0', + } + metrics = { + 'out.port': self.TEST_PORT, + 'out.redis_db': 0, } for k, v in meta.items(): assert span.get_tag(k) == v + for k, v in metrics.items(): + assert span.get_metric(k) == v assert span.get_tag('redis.raw_command').startswith(u'MGET 0 1 2 3') assert span.get_tag('redis.raw_command').endswith(u'...') @@ -71,7 +75,7 @@ def test_basics(self): assert span.name == 'redis.command' assert span.span_type == 'redis' assert span.error == 0 - assert span.get_tag('out.redis_db') == '0' + assert span.get_metric('out.redis_db') == 0 assert span.get_tag('out.host') == 'localhost' assert span.get_tag('redis.raw_command') == u'GET cheese' assert span.get_metric('redis.args_length') == 2 @@ -117,7 +121,7 @@ def test_pipeline_traced(self): assert span.resource == u'SET blah 32\nRPUSH foo éé\nHGETALL xxx' assert span.span_type == 'redis' assert span.error == 0 - assert span.get_tag('out.redis_db') == '0' + assert span.get_metric('out.redis_db') == 0 assert span.get_tag('out.host') == 'localhost' assert span.get_tag('redis.raw_command') == u'SET blah 32\nRPUSH foo éé\nHGETALL xxx' assert span.get_metric('redis.pipeline_length') == 3 @@ -138,7 +142,7 @@ def test_pipeline_immediate(self): assert span.resource == u'SET a 1' assert span.span_type == 'redis' assert span.error == 0 - assert span.get_tag('out.redis_db') == '0' + assert span.get_metric('out.redis_db') == 0 assert span.get_tag('out.host') == 'localhost' def test_meta_override(self): @@ -213,7 +217,7 @@ def test_opentracing(self): assert dd_span.name == 'redis.command' assert dd_span.span_type == 'redis' assert dd_span.error == 0 - assert dd_span.get_tag('out.redis_db') == '0' + assert dd_span.get_metric('out.redis_db') == 0 assert dd_span.get_tag('out.host') == 'localhost' assert dd_span.get_tag('redis.raw_command') == u'GET cheese' assert dd_span.get_metric('redis.args_length') == 2 diff --git a/tests/contrib/requests/test_requests.py b/tests/contrib/requests/test_requests.py index 4cbceabc2f5..5ca6d1aa1c5 100644 --- a/tests/contrib/requests/test_requests.py +++ b/tests/contrib/requests/test_requests.py @@ -73,7 +73,7 @@ def test_args_kwargs(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_tag(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 def test_untraced_request(self): # ensure the unpatch removes tracing @@ -105,7 +105,7 @@ def test_200(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_tag(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 assert s.error == 0 assert s.span_type == 'http' assert http.QUERY_STRING not in s.meta @@ -122,7 +122,7 @@ def test_200_send(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_tag(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 assert s.error == 0 assert s.span_type == 'http' @@ -137,7 +137,7 @@ def test_200_query_string(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_tag(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 assert s.get_tag(http.URL) == URL_200 assert s.error == 0 assert s.span_type == 'http' @@ -154,7 +154,7 @@ def test_requests_module_200(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_tag(http.STATUS_CODE) == '200' + assert s.get_metric(http.STATUS_CODE) == 200 assert s.error == 0 assert s.span_type == 'http' @@ -166,7 +166,7 @@ def test_post_500(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'POST' - assert s.get_tag(http.STATUS_CODE) == '500' + assert s.get_metric(http.STATUS_CODE) == 500 assert s.error == 1 def test_non_existant_url(self): @@ -195,7 +195,7 @@ def test_500(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_tag(http.STATUS_CODE) == '500' + assert s.get_metric(http.STATUS_CODE) == 500 assert s.error == 1 def test_default_service_name(self): @@ -368,7 +368,7 @@ def test_200_ot(self): assert ot_span.service == 'requests_svc' assert dd_span.get_tag(http.METHOD) == 'GET' - assert dd_span.get_tag(http.STATUS_CODE) == '200' + assert dd_span.get_metric(http.STATUS_CODE) == 200 assert dd_span.error == 0 assert dd_span.span_type == 'http' diff --git a/tests/contrib/sqlalchemy/mixins.py b/tests/contrib/sqlalchemy/mixins.py index 08a579481e6..a73a4007172 100644 --- a/tests/contrib/sqlalchemy/mixins.py +++ b/tests/contrib/sqlalchemy/mixins.py @@ -114,7 +114,7 @@ def test_orm_insert(self): assert span.service == self.SERVICE assert 'INSERT INTO players' in span.resource assert span.get_tag('sql.db') == self.SQL_DB - assert span.get_tag('sql.rows') == '1' + assert span.get_metric('sql.rows') == 1 self.check_meta(span) assert span.span_type == 'sql' assert span.error == 0 diff --git a/tests/contrib/sqlalchemy/test_mysql.py b/tests/contrib/sqlalchemy/test_mysql.py index 0ec19e92dc8..2e2037c100d 100644 --- a/tests/contrib/sqlalchemy/test_mysql.py +++ b/tests/contrib/sqlalchemy/test_mysql.py @@ -22,7 +22,7 @@ def tearDown(self): def check_meta(self, span): # check database connection tags self.assertEqual(span.get_tag('out.host'), MYSQL_CONFIG['host']) - self.assertEqual(span.get_tag('out.port'), str(MYSQL_CONFIG['port'])) + self.assertEqual(span.get_metric('out.port'), MYSQL_CONFIG['port']) def test_engine_execute_errors(self): # ensures that SQL errors are reported @@ -40,7 +40,7 @@ def test_engine_execute_errors(self): self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'SELECT * FROM a_wrong_table') self.assertEqual(span.get_tag('sql.db'), self.SQL_DB) - self.assertIsNone(span.get_tag('sql.rows')) + self.assertIsNone(span.get_tag('sql.rows') or span.get_metric('sql.rows')) self.check_meta(span) self.assertEqual(span.span_type, 'sql') self.assertTrue(span.duration > 0) diff --git a/tests/contrib/sqlalchemy/test_postgres.py b/tests/contrib/sqlalchemy/test_postgres.py index 34e78b6f5a3..7832fa4d19a 100644 --- a/tests/contrib/sqlalchemy/test_postgres.py +++ b/tests/contrib/sqlalchemy/test_postgres.py @@ -25,7 +25,7 @@ def tearDown(self): def check_meta(self, span): # check database connection tags self.assertEqual(span.get_tag('out.host'), POSTGRES_CONFIG['host']) - self.assertEqual(span.get_tag('out.port'), str(POSTGRES_CONFIG['port'])) + self.assertEqual(span.get_metric('out.port'), POSTGRES_CONFIG['port']) def test_engine_execute_errors(self): # ensures that SQL errors are reported @@ -43,7 +43,7 @@ def test_engine_execute_errors(self): self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'SELECT * FROM a_wrong_table') self.assertEqual(span.get_tag('sql.db'), self.SQL_DB) - self.assertIsNone(span.get_tag('sql.rows')) + self.assertIsNone(span.get_tag('sql.rows') or span.get_metric('sql.rows')) self.check_meta(span) self.assertEqual(span.span_type, 'sql') self.assertTrue(span.duration > 0) diff --git a/tests/contrib/sqlalchemy/test_sqlite.py b/tests/contrib/sqlalchemy/test_sqlite.py index 1455ecce0cd..b3a78b4e909 100644 --- a/tests/contrib/sqlalchemy/test_sqlite.py +++ b/tests/contrib/sqlalchemy/test_sqlite.py @@ -35,7 +35,7 @@ def test_engine_execute_errors(self): self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'SELECT * FROM a_wrong_table') self.assertEqual(span.get_tag('sql.db'), self.SQL_DB) - self.assertIsNone(span.get_tag('sql.rows')) + self.assertIsNone(span.get_tag('sql.rows') or span.get_metric('sql.rows')) self.assertEqual(span.span_type, 'sql') self.assertTrue(span.duration > 0) # check the error diff --git a/tests/contrib/sqlite3/test_sqlite3.py b/tests/contrib/sqlite3/test_sqlite3.py index 45277ccc46b..e1294bd053a 100644 --- a/tests/contrib/sqlite3/test_sqlite3.py +++ b/tests/contrib/sqlite3/test_sqlite3.py @@ -178,7 +178,7 @@ def test_sqlite_fetchmany_is_traced(self): resource=q, span_type='sql', error=0, - meta={'db.fetch.size': '123'}, + metrics={'db.fetch.size': 123}, ), ) self.assertIsNone(fetchmany_span.get_tag('sql.query')) diff --git a/tests/contrib/tornado/test_executor_decorator.py b/tests/contrib/tornado/test_executor_decorator.py index e48bac16ae2..bc47cf8dd2c 100644 --- a/tests/contrib/tornado/test_executor_decorator.py +++ b/tests/contrib/tornado/test_executor_decorator.py @@ -30,7 +30,7 @@ def test_on_executor_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/executor_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error assert request_span.duration >= 0.05 @@ -61,7 +61,7 @@ def test_on_executor_submit(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorSubmitHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/executor_submit_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error assert request_span.duration >= 0.05 @@ -91,7 +91,7 @@ def test_on_executor_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/executor_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -128,7 +128,7 @@ def test_on_executor_custom_kwarg(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorCustomHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/executor_custom_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error assert request_span.duration >= 0.05 @@ -161,7 +161,7 @@ def test_on_executor_custom_args_kwarg(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorCustomArgsHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/executor_custom_args_handler/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'cannot combine positional and keyword args' == request_span.get_tag('error.msg') diff --git a/tests/contrib/tornado/test_tornado_template.py b/tests/contrib/tornado/test_tornado_template.py index bbf159dfe1f..6cc6e5e96c0 100644 --- a/tests/contrib/tornado/test_tornado_template.py +++ b/tests/contrib/tornado/test_tornado_template.py @@ -28,7 +28,7 @@ def test_template_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplateHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/template/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -75,7 +75,7 @@ def test_template_partials(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplatePartialHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/template_partial/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -130,7 +130,7 @@ def test_template_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplateExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/template_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'ModuleThatDoesNotExist' in request_span.get_tag('error.msg') diff --git a/tests/contrib/tornado/test_tornado_web.py b/tests/contrib/tornado/test_tornado_web.py index f066816cad6..ddcff3201c8 100644 --- a/tests/contrib/tornado/test_tornado_web.py +++ b/tests/contrib/tornado/test_tornado_web.py @@ -33,7 +33,7 @@ def test_success_handler(self, query_string=''): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/success/') == request_span.get_tag(http.URL) if config.tornado.trace_query_string: assert query_string == request_span.get_tag(http.QUERY_STRING) @@ -63,7 +63,7 @@ def test_nested_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/nested/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span @@ -90,7 +90,7 @@ def test_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -111,7 +111,7 @@ def test_http_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.HTTPExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '501' == request_span.get_tag('http.status_code') + assert 501 == request_span.get_metric('http.status_code') assert self.get_url('/http_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'HTTP 501: Not Implemented (unavailable)' == request_span.get_tag('error.msg') @@ -132,7 +132,7 @@ def test_http_exception_500_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.HTTPException500Handler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/http_exception_500/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'HTTP 500: Server Error (server error)' == request_span.get_tag('error.msg') @@ -153,7 +153,7 @@ def test_sync_success_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncSuccessHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/sync_success/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -172,7 +172,7 @@ def test_sync_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/sync_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -193,7 +193,7 @@ def test_404_handler(self): assert 'web' == request_span.span_type assert 'tornado.web.ErrorHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '404' == request_span.get_tag('http.status_code') + assert 404 == request_span.get_metric('http.status_code') assert self.get_url('/does_not_exist/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -214,7 +214,7 @@ def test_redirect_handler(self): assert 'web' == redirect_span.span_type assert 'tornado.web.RedirectHandler' == redirect_span.resource assert 'GET' == redirect_span.get_tag('http.method') - assert '301' == redirect_span.get_tag('http.status_code') + assert 301 == redirect_span.get_metric('http.status_code') assert self.get_url('/redirect/') == redirect_span.get_tag(http.URL) assert 0 == redirect_span.error @@ -224,7 +224,7 @@ def test_redirect_handler(self): assert 'web' == success_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == success_span.resource assert 'GET' == success_span.get_tag('http.method') - assert '200' == success_span.get_tag('http.status_code') + assert 200 == success_span.get_metric('http.status_code') assert self.get_url('/success/') == success_span.get_tag(http.URL) assert 0 == success_span.error @@ -244,7 +244,7 @@ def test_static_handler(self): assert 'web' == request_span.span_type assert 'tornado.web.StaticFileHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/statics/empty.txt') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -266,7 +266,7 @@ def test_propagation(self): # simple sanity check on the span assert 'tornado.request' == request_span.name - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/success/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -306,7 +306,7 @@ def test_success_handler_ot(self): assert 'web' == dd_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == dd_span.resource assert 'GET' == dd_span.get_tag('http.method') - assert '200' == dd_span.get_tag('http.status_code') + assert 200 == dd_span.get_metric('http.status_code') assert self.get_url('/success/') == dd_span.get_tag(http.URL) assert 0 == dd_span.error @@ -448,7 +448,7 @@ def test_no_propagation(self): # simple sanity check on the span assert 'tornado.request' == request_span.name - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/success/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -485,6 +485,6 @@ def test_custom_default_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.CustomDefaultHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '400' == request_span.get_tag('http.status_code') + assert 400 == request_span.get_metric('http.status_code') assert self.get_url('/custom_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error diff --git a/tests/contrib/tornado/test_wrap_decorator.py b/tests/contrib/tornado/test_wrap_decorator.py index aea7f453dcd..9afd854e921 100644 --- a/tests/contrib/tornado/test_wrap_decorator.py +++ b/tests/contrib/tornado/test_wrap_decorator.py @@ -21,7 +21,7 @@ def test_nested_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/nested_wrap/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span @@ -47,7 +47,7 @@ def test_nested_exception_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/nested_exception_wrap/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -77,7 +77,7 @@ def test_sync_nested_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncNestedWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/sync_nested_wrap/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span @@ -103,7 +103,7 @@ def test_sync_nested_exception_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncNestedExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/sync_nested_exception_wrap/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -133,7 +133,7 @@ def test_nested_wrap_executor_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '200' == request_span.get_tag('http.status_code') + assert 200 == request_span.get_metric('http.status_code') assert self.get_url('/executor_wrap_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span in the executor @@ -160,7 +160,7 @@ def test_nested_exception_wrap_executor_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert '500' == request_span.get_tag('http.status_code') + assert 500 == request_span.get_metric('http.status_code') assert self.get_url('/executor_wrap_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') diff --git a/tests/contrib/vertica/test_vertica.py b/tests/contrib/vertica/test_vertica.py index 8e1e5da36d1..753915cb737 100644 --- a/tests/contrib/vertica/test_vertica.py +++ b/tests/contrib/vertica/test_vertica.py @@ -211,7 +211,7 @@ def test_execute_metadata(self): query = "INSERT INTO test_table (a, b) VALUES (1, 'aa');" assert spans[0].resource == query assert spans[0].get_tag('out.host') == '127.0.0.1' - assert spans[0].get_tag('out.port') == '5433' + assert spans[0].get_metric('out.port') == 5433 assert spans[0].get_tag('db.name') == 'docker' assert spans[0].get_tag('db.user') == 'dbadmin' @@ -238,7 +238,7 @@ def test_cursor_override(self): query = "INSERT INTO test_table (a, b) VALUES (1, 'aa');" assert spans[0].resource == query assert spans[0].get_tag('out.host') == '127.0.0.1' - assert spans[0].get_tag('out.port') == '5433' + assert spans[0].get_metric('out.port') == 5433 assert spans[1].resource == 'SELECT * FROM test_table;' @@ -309,7 +309,7 @@ def test_rowcount_oddity(self): assert spans[1].get_metric('db.rowcount') == -1 assert spans[2].name == 'vertica.fetchone' assert spans[2].get_tag('out.host') == '127.0.0.1' - assert spans[2].get_tag('out.port') == '5433' + assert spans[2].get_metric('out.port') == 5433 assert spans[2].get_metric('db.rowcount') == 1 assert spans[3].name == 'vertica.fetchone' assert spans[3].get_metric('db.rowcount') == 2 @@ -380,7 +380,7 @@ def test_opentracing(self): query = "INSERT INTO test_table (a, b) VALUES (1, 'aa');" assert dd_span.resource == query assert dd_span.get_tag('out.host') == '127.0.0.1' - assert dd_span.get_tag('out.port') == '5433' + assert dd_span.get_metric('out.port') == 5433 def test_analytics_default(self): conn, cur = self.test_conn diff --git a/tests/opentracer/test_span.py b/tests/opentracer/test_span.py index fe07f40f683..44282130676 100644 --- a/tests/opentracer/test_span.py +++ b/tests/opentracer/test_span.py @@ -35,7 +35,7 @@ def test_init(self, nop_tracer, nop_span_ctx): def test_tags(self, nop_span): """Set a tag and get it back.""" nop_span.set_tag('test', 23) - assert int(nop_span._get_tag('test')) == 23 + assert nop_span._get_metric('test') == 23 def test_set_baggage(self, nop_span): """Test setting baggage.""" @@ -83,7 +83,7 @@ def test_log_dd_kv(self, nop_span): # ...and that error tags are set with the correct key assert nop_span._get_tag(errors.ERROR_STACK) == stack_trace assert nop_span._get_tag(errors.ERROR_MSG) == 'my error message' - assert nop_span._get_tag(errors.ERROR_TYPE) == '3' + assert nop_span._get_metric(errors.ERROR_TYPE) == 3 def test_operation_name(self, nop_span): """Sanity check for setting the operation name.""" @@ -122,7 +122,7 @@ class TestSpanCompatibility(object): """ def test_set_tag(self, nop_span): nop_span.set_tag('test', 2) - assert nop_span._dd_span.get_tag('test') == str(2) + assert nop_span._get_metric('test') == 2 def test_tag_resource_name(self, nop_span): nop_span.set_tag('resource.name', 'myresource') @@ -145,8 +145,8 @@ def test_tag_peer_hostname(self, nop_span): assert nop_span._dd_span.get_tag('out.host') == 'peername' def test_tag_peer_port(self, nop_span): - nop_span.set_tag('peer.port', '55555') - assert nop_span._dd_span.get_tag('out.port') == '55555' + nop_span.set_tag('peer.port', 55555) + assert nop_span._get_metric('out.port') == 55555 def test_tag_sampling_priority(self, nop_span): nop_span.set_tag('sampling.priority', '2') diff --git a/tests/opentracer/test_tracer.py b/tests/opentracer/test_tracer.py index a668d01fc72..5ca46eb30c4 100644 --- a/tests/opentracer/test_tracer.py +++ b/tests/opentracer/test_tracer.py @@ -86,11 +86,11 @@ def test_global_tags(self): with tracer.start_span('myop') as span: # global tags should be attached to generated all datadog spans assert span._dd_span.get_tag('tag1') == 'value1' - assert span._dd_span.get_tag('tag2') == '2' + assert span._dd_span.get_metric('tag2') == 2 with tracer.start_span('myop2') as span2: assert span2._dd_span.get_tag('tag1') == 'value1' - assert span2._dd_span.get_tag('tag2') == '2' + assert span2._dd_span.get_metric('tag2') == 2 class TestTracer(object): diff --git a/tests/test_compat.py b/tests/test_compat.py index cc1bc09c69f..1bd3ec5fadf 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -6,7 +6,7 @@ import pytest # Project -from ddtrace.compat import to_unicode, PY3, reraise, get_connection_response +from ddtrace.compat import to_unicode, PY3, reraise, get_connection_response, is_integer if PY3: @@ -97,3 +97,20 @@ def test_reraise(self): # this call must be Python 2 and 3 compatible raise reraise(typ, val, tb) assert ex.value.args[0] == 'Ouch!' + + +@pytest.mark.parametrize('obj,expected', [ + (1, True), + (-1, True), + (0, True), + (1.0, False), + (-1.0, False), + (True, False), + (False, False), + (dict(), False), + ([], False), + (tuple(), False), + (object(), False), +]) +def test_is_integer(obj, expected): + assert is_integer(obj) is expected diff --git a/tests/test_global_config.py b/tests/test_global_config.py index 6bfcb1b308f..1b577bc107f 100644 --- a/tests/test_global_config.py +++ b/tests/test_global_config.py @@ -185,7 +185,7 @@ def on_web_request3(span): # Create our span span = self.tracer.start_span('web.request') assert 'web.request' not in span.meta - assert 'web.status' not in span.meta + assert 'web.status' not in span.metrics assert 'web.method' not in span.meta # Emit the span @@ -193,7 +193,7 @@ def on_web_request3(span): # Assert we updated the span as expected assert span.get_tag('web.request') == '/' - assert span.get_tag('web.status') == '200' + assert span.get_metric('web.status') == 200 assert span.get_tag('web.method') == 'GET' def test_settings_hook_failure(self): diff --git a/tests/test_span.py b/tests/test_span.py index 5f8c3071907..426ab8341fd 100644 --- a/tests/test_span.py +++ b/tests/test_span.py @@ -28,12 +28,59 @@ def test_tags(self): s.set_tag('b', 1) s.set_tag('c', '1') d = s.to_dict() - expected = { - 'a': 'a', - 'b': '1', - 'c': '1', + assert d['meta'] == dict(a='a', c='1') + assert d['metrics'] == dict(b=1) + + def test_numeric_tags(self): + s = Span(tracer=None, name='test.span') + s.set_tag('negative', -1) + s.set_tag('zero', 0) + s.set_tag('positive', 1) + s.set_tag('large_int', 2**53) + s.set_tag('really_large_int', (2**53) + 1) + s.set_tag('large_negative_int', -(2**53)) + s.set_tag('really_large_negative_int', -((2**53) + 1)) + s.set_tag('float', 12.3456789) + s.set_tag('negative_float', -12.3456789) + s.set_tag('large_float', 2.0**53) + s.set_tag('really_large_float', (2.0**53) + 1) + + d = s.to_dict() + assert d['meta'] == dict( + really_large_int=str(((2**53) + 1)), + really_large_negative_int=str(-((2**53) + 1)), + ) + assert d['metrics'] == { + 'negative': -1, + 'zero': 0, + 'positive': 1, + 'large_int': 2**53, + 'large_negative_int': -(2**53), + 'float': 12.3456789, + 'negative_float': -12.3456789, + 'large_float': 2.0**53, + 'really_large_float': (2.0**53) + 1, } - assert d['meta'] == expected + + def test_set_tag_bool(self): + s = Span(tracer=None, name='test.span') + s.set_tag('true', True) + s.set_tag('false', False) + + d = s.to_dict() + assert d['meta'] == dict(true='True', false='False') + assert 'metrics' not in d + + def test_set_tag_metric(self): + s = Span(tracer=None, name='test.span') + + s.set_tag('test', 'value') + assert s.meta == dict(test='value') + assert s.metrics == dict() + + s.set_tag('test', 1) + assert s.meta == dict() + assert s.metrics == dict(test=1) def test_set_valid_metrics(self): s = Span(tracer=None, name='test.span') @@ -324,7 +371,7 @@ def test_set_tag_none(self): s = Span(tracer=None, name='root.span', service='s', resource='r') assert s.meta == dict() - s.set_tag('custom.key', 100) + s.set_tag('custom.key', '100') assert s.meta == {'custom.key': '100'} diff --git a/tests/test_tracer.py b/tests/test_tracer.py index b0608045cc4..2ce840a2109 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -99,10 +99,10 @@ def test_tracer_pid(self): pass # Root span should contain the pid of the current process - root_span.assert_meta({system.PID: str(getpid())}, exact=False) + root_span.assert_metrics({system.PID: getpid()}, exact=False) # Child span should not contain a pid tag - child_span.assert_meta(dict(), exact=True) + child_span.assert_metrics(dict(), exact=True) def test_tracer_wrap_default_name(self): @self.tracer.wrap() From abc65119db17cd483ffcda5362b78bba36c0e343 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 8 Jan 2020 11:00:29 -0500 Subject: [PATCH 26/81] core: default to DatadogSampler As well, ensure that unless explicitly configured continue to use RateByServiceSampler for DatadogSampler --- ddtrace/internal/writer.py | 15 +++- ddtrace/sampler.py | 51 +++++++++--- ddtrace/tracer.py | 7 +- tests/test_sampler.py | 158 ++++++++++++++++++++++++++++++------- 4 files changed, 184 insertions(+), 47 deletions(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 39ff2a0a9f9..4b58f2ad637 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -6,6 +6,7 @@ from .. import api from .. import _worker from ..internal.logger import get_logger +from ..sampler import BasePrioritySampler from ..settings import config from ..vendor import monotonic from ddtrace.vendor.six.moves.queue import Queue, Full, Empty @@ -25,13 +26,14 @@ class AgentWriter(_worker.PeriodicWorkerThread): def __init__(self, hostname='localhost', port=8126, uds_path=None, https=False, shutdown_timeout=DEFAULT_TIMEOUT, - filters=None, priority_sampler=None, + filters=None, sampler=None, priority_sampler=None, dogstatsd=None): super(AgentWriter, self).__init__(interval=self.QUEUE_PROCESSING_INTERVAL, exit_timeout=shutdown_timeout, name=self.__class__.__name__) self._trace_queue = Q(maxsize=MAX_TRACES) self._filters = filters + self._sampler = sampler self._priority_sampler = priority_sampler self._last_error_ts = 0 self.dogstatsd = dogstatsd @@ -94,10 +96,17 @@ def flush_queue(self): for response in traces_responses: if isinstance(response, Exception) or response.status >= 400: self._log_error_status(response) - elif self._priority_sampler: + elif self._priority_sampler or isinstance(self._sampler, BasePrioritySampler): result_traces_json = response.get_json() if result_traces_json and 'rate_by_service' in result_traces_json: - self._priority_sampler.set_sample_rate_by_service(result_traces_json['rate_by_service']) + if self._priority_sampler: + self._priority_sampler.update_rate_by_service_sample_rates( + result_traces_json['rate_by_service'], + ) + if isinstance(self._sampler, BasePrioritySampler): + self._sampler.update_rate_by_service_sample_rates( + result_traces_json['rate_by_service'], + ) # Dump statistics # NOTE: Do not use the buffering of dogstatsd as it's not thread-safe diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 62288ed0803..7940ba0624a 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -27,6 +27,12 @@ def sample(self, span): pass +class BasePrioritySampler(six.with_metaclass(abc.ABCMeta)): + @abc.abstractmethod + def update_rate_by_service_sample_rates(self, sample_rates): + pass + + class AllSampler(BaseSampler): """Sampler sampling all the traces""" @@ -60,7 +66,7 @@ def sample(self, span): return ((span.trace_id * KNUTH_FACTOR) % MAX_TRACE_ID) <= self.sampling_id_threshold -class RateByServiceSampler(BaseSampler): +class RateByServiceSampler(BaseSampler, BasePrioritySampler): """Sampler based on a rate, by service Keep (100 * `sample_rate`)% of the traces. @@ -97,7 +103,7 @@ def sample(self, span): span.set_metric(SAMPLING_AGENT_DECISION, sampler.sample_rate) return sampler.sample(span) - def set_sample_rate_by_service(self, rate_by_service): + def update_rate_by_service_sample_rates(self, rate_by_service): new_by_service_samplers = self._get_new_by_service_sampler() for key, sample_rate in iteritems(rate_by_service): new_by_service_samplers[key] = RateSampler(sample_rate) @@ -109,15 +115,15 @@ def set_sample_rate_by_service(self, rate_by_service): RateByServiceSampler._default_key = RateByServiceSampler._key() -class DatadogSampler(BaseSampler): +class DatadogSampler(BaseSampler, BasePrioritySampler): """ This sampler is currently in ALPHA and it's API may change at any time, use at your own risk. """ __slots__ = ('default_sampler', 'limiter', 'rules') NO_RATE_LIMIT = -1 - DEFAULT_RATE_LIMIT = 100 - DEFAULT_SAMPLE_RATE = 1.0 + DEFAULT_RATE_LIMIT = NO_RATE_LIMIT + DEFAULT_SAMPLE_RATE = None def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): """ @@ -128,11 +134,21 @@ def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): :param default_sample_rate: The default sample rate to apply if no rules matched (default: 1.0) :type default_sample_rate: float 0 <= X <= 1.0 :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules - applied to them, (default: ``100``) + applied to them, (default: no rate limit) :type rate_limit: :obj:`int` """ if default_sample_rate is None: - default_sample_rate = float(get_env('trace', 'sample_rate', default=self.DEFAULT_SAMPLE_RATE)) + # If no sample rate was provided explicitly in code, try to load from environment variable + sample_rate = get_env('trace', 'sample_rate', default=self.DEFAULT_SAMPLE_RATE) + + # If no env variable was found, just use the default + if sample_rate is None: + default_sample_rate = self.DEFAULT_SAMPLE_RATE + + # Otherwise, try to convert it to a float + else: + default_sample_rate = float(sample_rate) + if rate_limit is None: rate_limit = int(get_env('trace', 'rate_limit', default=self.DEFAULT_RATE_LIMIT)) @@ -148,7 +164,16 @@ def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): # Configure rate limiter self.limiter = RateLimiter(rate_limit) - self.default_sampler = SamplingRule(sample_rate=default_sample_rate) + + # Default to previous default behavior of RateByServiceSampler + self.default_sampler = RateByServiceSampler() + if default_sample_rate is not None: + self.default_sampler = SamplingRule(sample_rate=default_sample_rate) + + def update_rate_by_service_sample_rates(self, sample_rates): + # Pass through the call to our RateByServiceSampler + if isinstance(self.default_sampler, RateByServiceSampler): + self.default_sampler.update_rate_by_service_sample_rates(sample_rates) def _set_priority(self, span, priority): if span._context: @@ -175,11 +200,15 @@ def sample(self, span): matching_rule = rule break else: - # No rule matches, use the default sampler + # If no rules match, use our defualt sampler + # This may be a RateByServiceSampler matching_rule = self.default_sampler # Sample with the matching sampling rule - span.set_metric(SAMPLING_RULE_DECISION, matching_rule.sample_rate) + # Only set if it isn't the default RateByServiceSampler + # since that gets it's own metric + if isinstance(matching_rule, SamplingRule): + span.set_metric(SAMPLING_RULE_DECISION, matching_rule.sample_rate) if not matching_rule.sample(span): self._set_priority(span, AUTO_REJECT) return False @@ -202,7 +231,7 @@ def sample(self, span): return True -class SamplingRule(object): +class SamplingRule(BaseSampler): """ Definition of a sampling rule used by :class:`DatadogSampler` for applying a sample rate on a span """ diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index d05646d5ea6..97e7ce0c8fa 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -12,7 +12,7 @@ from .internal.writer import AgentWriter from .provider import DefaultContextProvider from .context import Context -from .sampler import AllSampler, DatadogSampler, RateSampler, RateByServiceSampler +from .sampler import BaseSampler, DatadogSampler, RateSampler, RateByServiceSampler from .span import Span from .utils.formats import get_env from .utils.deprecation import deprecated, RemovedInDDTrace10Warning @@ -114,7 +114,7 @@ def __init__(self, url=DEFAULT_AGENT_URL, dogstatsd_url=DEFAULT_DOGSTATSD_URL): port=port, https=https, uds_path=uds_path, - sampler=AllSampler(), + sampler=DatadogSampler(), context_provider=DefaultContextProvider(), dogstatsd_url=dogstatsd_url, ) @@ -229,7 +229,7 @@ def configure(self, enabled=None, hostname=None, port=None, uds_path=None, https self._dogstatsd_client = DogStatsd(**dogstatsd_kwargs) if hostname is not None or port is not None or uds_path is not None or https is not None or \ - filters is not None or priority_sampling is not None: + filters is not None or priority_sampling is not None or sampler is not None: # Preserve hostname and port when overriding filters or priority sampling # This is clumsy and a good reason to get rid of this configure() API if hasattr(self, 'writer') and hasattr(self.writer, 'api'): @@ -247,6 +247,7 @@ def configure(self, enabled=None, hostname=None, port=None, uds_path=None, https uds_path=uds_path, https=https, filters=filters, + sampler=self.sampler, priority_sampler=self.priority_sampler, dogstatsd=self._dogstatsd_client, ) diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 6849b1d0420..218523735f2 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -151,7 +151,7 @@ def test_sample_rate_deviation(self): deviation = abs(samples_with_high_priority - (iterations * sample_rate)) / (iterations * sample_rate) assert deviation < 0.05, 'Deviation too high %f with sample_rate %f' % (deviation, sample_rate) - def test_set_sample_rate_by_service(self): + def test_update_rate_by_service_sample_rates(self): cases = [ { 'service:,env:': 1, @@ -173,7 +173,7 @@ def test_set_sample_rate_by_service(self): tracer.configure(sampler=AllSampler()) priority_sampler = tracer.priority_sampler for case in cases: - priority_sampler.set_sample_rate_by_service(case) + priority_sampler.update_rate_by_service_sample_rates(case) rates = {} for k, v in iteritems(priority_sampler._by_service_samplers): rates[k] = v.sample_rate @@ -182,7 +182,7 @@ def test_set_sample_rate_by_service(self): # works as well as key insertion (and doing this both ways ensures we trigger both cases) cases.reverse() for case in cases: - priority_sampler.set_sample_rate_by_service(case) + priority_sampler.update_rate_by_service_sample_rates(case) rates = {} for k, v in iteritems(priority_sampler._by_service_samplers): rates[k] = v.sample_rate @@ -439,29 +439,31 @@ def test_datadog_sampler_init(): assert sampler.rules == [] assert isinstance(sampler.limiter, RateLimiter) assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT - assert sampler.default_sampler.sample_rate == DatadogSampler.DEFAULT_SAMPLE_RATE + assert isinstance(sampler.default_sampler, RateByServiceSampler) # With rules rule = SamplingRule(sample_rate=1) sampler = DatadogSampler(rules=[rule]) assert sampler.rules == [rule] assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT - assert sampler.default_sampler.sample_rate == DatadogSampler.DEFAULT_SAMPLE_RATE + assert isinstance(sampler.default_sampler, RateByServiceSampler) # With rate limit sampler = DatadogSampler(rate_limit=10) assert sampler.limiter.rate_limit == 10 - assert sampler.default_sampler.sample_rate == DatadogSampler.DEFAULT_SAMPLE_RATE + assert isinstance(sampler.default_sampler, RateByServiceSampler) # With default_sample_rate sampler = DatadogSampler(default_sample_rate=0.5) assert sampler.limiter.rate_limit == DatadogSampler.DEFAULT_RATE_LIMIT + assert isinstance(sampler.default_sampler, SamplingRule) assert sampler.default_sampler.sample_rate == 0.5 # From env variables with override_env(dict(DD_TRACE_SAMPLE_RATE='0.5', DD_TRACE_RATE_LIMIT='10')): sampler = DatadogSampler() assert sampler.limiter.rate_limit == 10 + assert isinstance(sampler.default_sampler, SamplingRule) assert sampler.default_sampler.sample_rate == 0.5 # Invalid rules @@ -477,35 +479,32 @@ def test_datadog_sampler_init(): assert sampler.rules == [rule_1, rule_2, rule_3] -@mock.patch('ddtrace.internal.rate_limiter.RateLimiter.is_allowed') -def test_datadog_sampler_sample_no_rules(mock_is_allowed, dummy_tracer): +@mock.patch('ddtrace.sampler.RateByServiceSampler.sample') +def test_datadog_sampler_sample_no_rules(mock_sample, dummy_tracer): sampler = DatadogSampler() span = create_span(tracer=dummy_tracer) - # Default SamplingRule(sample_rate=1.0) is applied - # No priority sampler configured + # Default RateByServiceSampler() is applied # No rules configured - # RateLimiter is allowed, it is sampled - mock_is_allowed.return_value = True + # No global rate limit + # No rate limit configured + # RateByServiceSampler.sample(span) returns True + mock_sample.return_value = True assert sampler.sample(span) is True assert span._context.sampling_priority is AUTO_KEEP assert span.sampled is True - assert_sampling_decision_tags(span, rule=1.0, limit=1.0) - mock_is_allowed.assert_called_once_with() - mock_is_allowed.reset_mock() span = create_span(tracer=dummy_tracer) - # Default SamplingRule(sample_rate=1.0) is applied - # No priority sampler configured + # Default RateByServiceSampler() is applied # No rules configured - # RateLimit not allowed, it is not sampled - mock_is_allowed.return_value = False + # No global rate limit + # No rate limit configured + # RateByServiceSampler.sample(span) returns False + mock_sample.return_value = False assert sampler.sample(span) is False assert span._context.sampling_priority is AUTO_REJECT assert span.sampled is False - assert_sampling_decision_tags(span, rule=1.0, limit=1.0) - mock_is_allowed.assert_called_once_with() @mock.patch('ddtrace.internal.rate_limiter.RateLimiter.is_allowed') @@ -519,8 +518,6 @@ def test_datadog_sampler_sample_rules(mock_is_allowed, dummy_tracer): mock.Mock(spec=SamplingRule), ] sampler = DatadogSampler(rules=rules) - sampler.default_sampler = mock.Mock(spec=SamplingRule) - sampler.default_sampler.return_value = True # Reset all of our mocks @contextlib.contextmanager @@ -530,8 +527,11 @@ def reset(): for rule in rules: rule.reset_mock() rule.sample_rate = 0.5 - sampler.default_sampler.reset_mock() - sampler.default_sampler.sample_rate = 1.0 + + default_rule = SamplingRule(sample_rate=1.0) + sampler.default_sampler = mock.Mock(spec=SamplingRule, wraps=default_rule) + # Mock has lots of problems with mocking/wrapping over class properties + sampler.default_sampler.sample_rate = default_rule.sample_rate reset() # Reset before, just in case try: @@ -639,6 +639,61 @@ def reset(): rules[2].matches.assert_not_called() rules[2].sample.assert_not_called() + # No rules match and RateByServiceSampler is used + # All rules SamplingRule.matches are called + # Priority sampler's `sample` method is called + # Result of priority sampler is returned + # Rate limiter is not called + with reset_mocks(): + span = create_span(tracer=dummy_tracer) + + # Configure mock priority sampler + priority_sampler = RateByServiceSampler() + sampler.default_sampler = mock.Mock(spec=RateByServiceSampler, wraps=priority_sampler) + + for rule in rules: + rule.matches.return_value = False + rule.sample.return_value = False + + assert sampler.sample(span) is True + assert span._context.sampling_priority is AUTO_KEEP + assert span.sampled is True + mock_is_allowed.assert_called_once() + sampler.default_sampler.sample.assert_called_once_with(span) + assert_sampling_decision_tags(span, agent=1, limit=1) + + [r.matches.assert_called_once_with(span) for r in rules] + [r.sample.assert_not_called() for r in rules] + + # No rules match and priority sampler is defined + # All rules SamplingRule.matches are called + # Priority sampler's `sample` method is called + # Result of priority sampler is returned + # Rate limiter is not called + with reset_mocks(): + span = create_span(tracer=dummy_tracer) + + # Configure mock priority sampler + priority_sampler = RateByServiceSampler() + for rate_sampler in priority_sampler._by_service_samplers.values(): + rate_sampler.set_sample_rate(0) + + sampler.default_sampler = mock.Mock(spec=RateByServiceSampler, wraps=priority_sampler) + + for rule in rules: + rule.matches.return_value = False + rule.sample.return_value = False + + assert sampler.sample(span) is False + assert span._context.sampling_priority is AUTO_REJECT + assert span.sampled is False + mock_is_allowed.assert_not_called() + sampler.default_sampler.sample.assert_called_once_with(span) + assert_sampling_decision_tags(span, agent=0) + + [r.matches.assert_called_once_with(span) for r in rules] + [r.sample.assert_not_called() for r in rules] + def test_datadog_sampler_tracer(dummy_tracer): rule = SamplingRule(sample_rate=1.0, name='test.span') @@ -650,7 +705,8 @@ def test_datadog_sampler_tracer(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - dummy_tracer.configure(sampler=sampler_spy) + # TODO: Remove `priority_sampling=False` when we remove fallback + dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) assert dummy_tracer.sampler is sampler_spy @@ -678,7 +734,8 @@ def test_datadog_sampler_tracer_rate_limited(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - dummy_tracer.configure(sampler=sampler_spy) + # TODO: Remove `priority_sampling=False` when we remove fallback + dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) assert dummy_tracer.sampler is sampler_spy @@ -705,7 +762,8 @@ def test_datadog_sampler_tracer_rate_0(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - dummy_tracer.configure(sampler=sampler_spy) + # TODO: Remove `priority_sampling=False` when we remove fallback + dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) assert dummy_tracer.sampler is sampler_spy @@ -732,7 +790,8 @@ def test_datadog_sampler_tracer_child(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - dummy_tracer.configure(sampler=sampler_spy) + # TODO: Remove `priority_sampling=False` when we remove fallback + dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) assert dummy_tracer.sampler is sampler_spy @@ -765,7 +824,8 @@ def test_datadog_sampler_tracer_start_span(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - dummy_tracer.configure(sampler=sampler_spy) + # TODO: Remove `priority_sampling=False` when we remove fallback + dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) assert dummy_tracer.sampler is sampler_spy @@ -781,3 +841,41 @@ def test_datadog_sampler_tracer_start_span(dummy_tracer): assert span.sampled is True assert span._context.sampling_priority is AUTO_KEEP assert_sampling_decision_tags(span, rule=1.0) + + +def test_datadog_sampler_update_rate_by_service_sample_rates(dummy_tracer): + cases = [ + { + 'service:,env:': 1, + }, + { + 'service:,env:': 1, + 'service:mcnulty,env:dev': 0.33, + 'service:postgres,env:dev': 0.7, + }, + { + 'service:,env:': 1, + 'service:mcnulty,env:dev': 0.25, + 'service:postgres,env:dev': 0.5, + 'service:redis,env:prod': 0.75, + }, + ] + + # By default sampler sets it's default sampler to RateByServiceSampler + sampler = DatadogSampler() + for case in cases: + sampler.update_rate_by_service_sample_rates(case) + rates = {} + for k, v in iteritems(sampler.default_sampler._by_service_samplers): + rates[k] = v.sample_rate + assert case == rates, '%s != %s' % (case, rates) + + # It's important to also test in reverse mode for we want to make sure key deletion + # works as well as key insertion (and doing this both ways ensures we trigger both cases) + cases.reverse() + for case in cases: + sampler.update_rate_by_service_sample_rates(case) + rates = {} + for k, v in iteritems(sampler.default_sampler._by_service_samplers): + rates[k] = v.sample_rate + assert case == rates, '%s != %s' % (case, rates) From 7aabb1758c2917066306d03099bbb16f8124d51e Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 8 Jan 2020 11:04:13 -0500 Subject: [PATCH 27/81] removed unused import --- ddtrace/tracer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 97e7ce0c8fa..a4f3e3d5164 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -12,7 +12,7 @@ from .internal.writer import AgentWriter from .provider import DefaultContextProvider from .context import Context -from .sampler import BaseSampler, DatadogSampler, RateSampler, RateByServiceSampler +from .sampler import DatadogSampler, RateSampler, RateByServiceSampler from .span import Span from .utils.formats import get_env from .utils.deprecation import deprecated, RemovedInDDTrace10Warning From 860f4b41e9efd429008812975ba42b3c23741a4f Mon Sep 17 00:00:00 2001 From: Joseph Kahn Date: Thu, 9 Jan 2020 09:40:16 -0500 Subject: [PATCH 28/81] encode python in the install_requires string This is breaking my build right now when I try to upgrade versions. --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 4c7052d802c..bb4ea5d765d 100644 --- a/setup.py +++ b/setup.py @@ -57,11 +57,8 @@ def run_tests(self): """ # psutil used to generate runtime metrics for tracer -install_requires = ["psutil>=5.0.0"] - -# include enum backport -if sys.version_info[:2] < (3, 4): - install_requires.extend(["enum34"]) +# enum34 is an enum backport for earlier versions of python +install_requires = ["psutil>=5.0.0", "enum34; python_version<='3.4'"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( From 4a49501ee99536714d8bd3903bf837f8eb2323cc Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 9 Jan 2020 16:26:19 -0500 Subject: [PATCH 29/81] not install enum34 for Python 3.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bb4ea5d765d..ffc04feba57 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def run_tests(self): # psutil used to generate runtime metrics for tracer # enum34 is an enum backport for earlier versions of python -install_requires = ["psutil>=5.0.0", "enum34; python_version<='3.4'"] +install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( From c5d21da66786598c4058631165e3295ac101ba6b Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 9 Jan 2020 17:13:19 -0500 Subject: [PATCH 30/81] add funcsigs backport to install --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ffc04feba57..99b43913447 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def run_tests(self): # psutil used to generate runtime metrics for tracer # enum34 is an enum backport for earlier versions of python -install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'"] +# funcsigs backport required for vendored debtcollector +install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'", "funcsigs; python_version<'3.3'"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( From 4022b8ab9f0447f1c17bb909b86c0ca1af2da055 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 9 Jan 2020 18:18:43 -0500 Subject: [PATCH 31/81] copy from debtcollector's requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 99b43913447..4f01bf97095 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ def run_tests(self): # psutil used to generate runtime metrics for tracer # enum34 is an enum backport for earlier versions of python # funcsigs backport required for vendored debtcollector -install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'", "funcsigs; python_version<'3.3'"] +install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( From 6f8be58bc5de5bac4a0c2b43f541928949b9560f Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 9 Jan 2020 18:37:49 -0500 Subject: [PATCH 32/81] tests: Fix botocore py34 tests PyYAML 5.3 dropped support for Python 3.4, we need 5.2 or lower --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index d622e6a9bed..6ce88549dda 100644 --- a/tox.ini +++ b/tox.ini @@ -187,6 +187,7 @@ deps = boto: boto boto: moto<1.0 botocore: botocore + py34-botocore: PyYAML<5.3 botocore: moto>=1.0,<2 bottle11: bottle>=0.11,<0.12 bottle12: bottle>=0.12,<0.13 From 703579643350ddcdf03e41eeaaa44528d6c32f6f Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 13 Jan 2020 09:55:58 -0500 Subject: [PATCH 33/81] default rate limit of 100, always set span.sampled = True --- ddtrace/sampler.py | 21 +++++++++++++-------- ddtrace/tracer.py | 2 ++ tests/test_sampler.py | 32 +++++++++++++++----------------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 7940ba0624a..b44ceff8ff2 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -122,7 +122,7 @@ class DatadogSampler(BaseSampler, BasePrioritySampler): __slots__ = ('default_sampler', 'limiter', 'rules') NO_RATE_LIMIT = -1 - DEFAULT_RATE_LIMIT = NO_RATE_LIMIT + DEFAULT_RATE_LIMIT = 100 DEFAULT_SAMPLE_RATE = None def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): @@ -131,10 +131,10 @@ def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): :param rules: List of :class:`SamplingRule` rules to apply to the root span of every trace, default no rules :type rules: :obj:`list` of :class:`SamplingRule` - :param default_sample_rate: The default sample rate to apply if no rules matched (default: 1.0) + :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``1.0``) :type default_sample_rate: float 0 <= X <= 1.0 :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules - applied to them, (default: no rate limit) + applied to them, (default: ``100``) :type rate_limit: :obj:`int` """ if default_sample_rate is None: @@ -200,15 +200,20 @@ def sample(self, span): matching_rule = rule break else: + # If this is the old sampler, sample and return + if isinstance(self.default_sampler, RateByServiceSampler): + if self.default_sampler.sample(span): + self._set_priority(span, AUTO_KEEP) + return True + else: + self._set_priority(span, AUTO_REJECT) + return False + # If no rules match, use our defualt sampler - # This may be a RateByServiceSampler matching_rule = self.default_sampler # Sample with the matching sampling rule - # Only set if it isn't the default RateByServiceSampler - # since that gets it's own metric - if isinstance(matching_rule, SamplingRule): - span.set_metric(SAMPLING_RULE_DECISION, matching_rule.sample_rate) + span.set_metric(SAMPLING_RULE_DECISION, matching_rule.sample_rate) if not matching_rule.sample(span): self._set_priority(span, AUTO_REJECT) return False diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index a4f3e3d5164..112eafe92d2 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -371,6 +371,8 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type context.sampling_priority = AUTO_REJECT else: context.sampling_priority = AUTO_KEEP if span.sampled else AUTO_REJECT + # We must always mark the span as sampled so it is forwarded to the agent + span.sampled = True # add tags to root span to correlate trace with runtime metrics # only applied to spans with types that are internal to applications diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 218523735f2..ab8e42629aa 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -658,9 +658,9 @@ def reset(): assert sampler.sample(span) is True assert span._context.sampling_priority is AUTO_KEEP assert span.sampled is True - mock_is_allowed.assert_called_once() + mock_is_allowed.assert_not_called() sampler.default_sampler.sample.assert_called_once_with(span) - assert_sampling_decision_tags(span, agent=1, limit=1) + assert_sampling_decision_tags(span, agent=1) [r.matches.assert_called_once_with(span) for r in rules] [r.sample.assert_not_called() for r in rules] @@ -705,8 +705,7 @@ def test_datadog_sampler_tracer(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -717,8 +716,9 @@ def test_datadog_sampler_tracer(dummy_tracer): rule_spy.sample.assert_called_once_with(span) limiter_spy.is_allowed.assert_called_once_with() - # We know it was sampled because we have a sample rate of 1.0 + # It must always mark it as sampled assert span.sampled is True + # We know it was sampled because we have a sample rate of 1.0 assert span._context.sampling_priority is AUTO_KEEP assert_sampling_decision_tags(span, rule=1.0) @@ -734,8 +734,7 @@ def test_datadog_sampler_tracer_rate_limited(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -746,8 +745,8 @@ def test_datadog_sampler_tracer_rate_limited(dummy_tracer): rule_spy.sample.assert_called_once_with(span) limiter_spy.is_allowed.assert_called_once_with() - # We know it was not sampled because of our limiter - assert span.sampled is False + # We must always mark the span as sampled + assert span.sampled is True assert span._context.sampling_priority is AUTO_REJECT assert_sampling_decision_tags(span, rule=1.0, limit=None) @@ -762,8 +761,7 @@ def test_datadog_sampler_tracer_rate_0(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -774,8 +772,9 @@ def test_datadog_sampler_tracer_rate_0(dummy_tracer): rule_spy.sample.assert_called_once_with(span) limiter_spy.is_allowed.assert_not_called() + # It must always mark it as sampled + assert span.sampled is True # We know it was not sampled because we have a sample rate of 0.0 - assert span.sampled is False assert span._context.sampling_priority is AUTO_REJECT assert_sampling_decision_tags(span, rule=0) @@ -790,8 +789,7 @@ def test_datadog_sampler_tracer_child(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -824,8 +822,7 @@ def test_datadog_sampler_tracer_start_span(dummy_tracer): sampler.limiter = limiter_spy sampler_spy = mock.Mock(spec=sampler, wraps=sampler) - # TODO: Remove `priority_sampling=False` when we remove fallback - dummy_tracer.configure(sampler=sampler_spy, priority_sampling=False) + dummy_tracer.configure(sampler=sampler_spy) assert dummy_tracer.sampler is sampler_spy @@ -837,8 +834,9 @@ def test_datadog_sampler_tracer_start_span(dummy_tracer): rule_spy.sample.assert_called_once_with(span) limiter_spy.is_allowed.assert_called_once_with() - # We know it was sampled because we have a sample rate of 1.0 + # It must always mark it as sampled assert span.sampled is True + # We know it was sampled because we have a sample rate of 1.0 assert span._context.sampling_priority is AUTO_KEEP assert_sampling_decision_tags(span, rule=1.0) From 99b86fb6b180832185ae9d0bc258a7039d324bcb Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Tue, 14 Jan 2020 07:23:06 -0500 Subject: [PATCH 34/81] Apply suggestions from code review --- ddtrace/sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index b44ceff8ff2..5a8aed3b66e 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -131,7 +131,7 @@ def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): :param rules: List of :class:`SamplingRule` rules to apply to the root span of every trace, default no rules :type rules: :obj:`list` of :class:`SamplingRule` - :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``1.0``) + :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``None``/Use :class:`RateByServiceSampler` only) :type default_sample_rate: float 0 <= X <= 1.0 :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules applied to them, (default: ``100``) From a87d66d8de5d10055cc89ed1094be61fab35bdfe Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 14 Jan 2020 07:33:32 -0500 Subject: [PATCH 35/81] fix flake8 --- ddtrace/sampler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index 5a8aed3b66e..9b3980fbb57 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -131,7 +131,8 @@ def __init__(self, rules=None, default_sample_rate=None, rate_limit=None): :param rules: List of :class:`SamplingRule` rules to apply to the root span of every trace, default no rules :type rules: :obj:`list` of :class:`SamplingRule` - :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``None``/Use :class:`RateByServiceSampler` only) + :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``None`` / + Use :class:`RateByServiceSampler` only) :type default_sample_rate: float 0 <= X <= 1.0 :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules applied to them, (default: ``100``) From e5c50bb5ef20b34a731120a075c9ad8cddc1dee3 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 14 Jan 2020 14:58:47 +0000 Subject: [PATCH 36/81] Improve tracer.trace docs --- docs/basic_usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/basic_usage.rst b/docs/basic_usage.rst index 069e95c8ead..9c79426730c 100644 --- a/docs/basic_usage.rst +++ b/docs/basic_usage.rst @@ -75,8 +75,8 @@ API details of the decorator can be found here :py:meth:`ddtrace.Tracer.wrap`. Context Manager ^^^^^^^^^^^^^^^ -To trace an arbitrary block of code, you can use the :py:mod:`ddtrace.Span` -context manager:: +To trace an arbitrary block of code, you can use :py:meth:`ddtrace.Tracer.trace` +that returns a :py:mod:`ddtrace.Span` which can be used as a context manager:: # trace some interesting operation with tracer.trace('interesting.operations'): From 0474798c1d69c9309e563a05a9920a3128e6dd63 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 14 Jan 2020 10:32:07 -0500 Subject: [PATCH 37/81] Dual License --- LICENSE => LICENSE.BSD3 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE => LICENSE.BSD3 (100%) diff --git a/LICENSE b/LICENSE.BSD3 similarity index 100% rename from LICENSE rename to LICENSE.BSD3 From b1d95cd4ccd39c730ed3b20f36bac6f43664c244 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 14 Jan 2020 10:32:14 -0500 Subject: [PATCH 38/81] Dual License --- LICENSE | 6 ++ LICENSE.Apache | 200 +++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE | 4 + 3 files changed, 210 insertions(+) create mode 100644 LICENSE create mode 100644 LICENSE.Apache create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000000..5f8fd634a18 --- /dev/null +++ b/LICENSE @@ -0,0 +1,6 @@ +## License + +This work is dual-licensed under Apache 2.0 or BSD3. +You may select, at your option, one of the above-listed licenses. + +`SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause` diff --git a/LICENSE.Apache b/LICENSE.Apache new file mode 100644 index 00000000000..bff56b54315 --- /dev/null +++ b/LICENSE.Apache @@ -0,0 +1,200 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 Datadog, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000000..732c748d439 --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +Datadog dd-trace-py +Copyright 2016-Present Datadog, Inc. + +This product includes software developed at Datadog, Inc. (https://www.datadoghq.com/). From c138cfbc4552ff7ee039a8ee3353c55c8ceb0248 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 15 Jan 2020 15:06:29 -0500 Subject: [PATCH 39/81] core: ensure http.status_code is always set as str on meta --- ddtrace/contrib/flask/middleware.py | 2 +- ddtrace/span.py | 7 ++++- tests/contrib/aiobotocore/py35/test.py | 7 +++-- tests/contrib/aiobotocore/test.py | 25 ++++++++------- tests/contrib/aiohttp/test_middleware.py | 17 +++++----- tests/contrib/boto/test.py | 19 ++++++------ tests/contrib/botocore/test.py | 17 +++++----- tests/contrib/bottle/test.py | 17 +++++----- tests/contrib/bottle/test_autopatch.py | 7 +++-- tests/contrib/bottle/test_distributed.py | 5 +-- tests/contrib/django/test_middleware.py | 21 +++++++------ .../test_djangorestframework.py | 5 +-- tests/contrib/elasticsearch/test.py | 5 +-- tests/contrib/falcon/test_suite.py | 17 +++++----- tests/contrib/flask/test_errorhandler.py | 12 ++++--- tests/contrib/flask/test_hooks.py | 5 +-- tests/contrib/flask/test_middleware.py | 23 +++++++------- tests/contrib/flask/test_request.py | 17 +++++----- tests/contrib/flask/test_static.py | 5 +-- tests/contrib/flask/test_views.py | 9 +++--- .../flask_autopatch/test_flask_autopatch.py | 3 +- tests/contrib/httplib/test_httplib.py | 27 ++++++++-------- tests/contrib/molten/test_molten.py | 7 +++-- tests/contrib/pylons/test_pylons.py | 19 ++++++------ tests/contrib/requests/test_requests.py | 17 +++++----- .../tornado/test_executor_decorator.py | 11 ++++--- .../contrib/tornado/test_tornado_template.py | 7 +++-- tests/contrib/tornado/test_tornado_web.py | 31 ++++++++++--------- tests/contrib/tornado/test_wrap_decorator.py | 13 ++++---- tests/utils/__init__.py | 9 ++++++ 30 files changed, 214 insertions(+), 172 deletions(-) diff --git a/ddtrace/contrib/flask/middleware.py b/ddtrace/contrib/flask/middleware.py index 5d57e5418e1..fb9a45f88c5 100644 --- a/ddtrace/contrib/flask/middleware.py +++ b/ddtrace/contrib/flask/middleware.py @@ -135,7 +135,7 @@ def _finish_span(self, span, exception=None): if not span or not span.sampled: return - code = span.get_metric(http.STATUS_CODE) or 0 + code = span.get_tag(http.STATUS_CODE) or 0 try: code = int(code) except Exception: diff --git a/ddtrace/span.py b/ddtrace/span.py index 5cbca55378b..047082e18de 100644 --- a/ddtrace/span.py +++ b/ddtrace/span.py @@ -153,13 +153,18 @@ def set_tag(self, key, value=None): must be strings (or stringable). If a casting error occurs, it will be ignored. """ + # Special case, force `http.status_code` as a string + # DEV: `http.status_code` *has* to be in `meta` for metrics + # calculated in the trace agent + if key == http.STATUS_CODE: + value = str(value) # Determine once up front is_an_int = is_integer(value) # Explicitly try to convert expected integers to `int` # DEV: Some integrations parse these values from strings, but don't call `int(value)` themselves - INT_TYPES = (net.TARGET_PORT, http.STATUS_CODE) + INT_TYPES = (net.TARGET_PORT, ) if key in INT_TYPES and not is_an_int: try: value = int(value) diff --git a/tests/contrib/aiobotocore/py35/test.py b/tests/contrib/aiobotocore/py35/test.py index a6dd77dcc80..fb12611b083 100644 --- a/tests/contrib/aiobotocore/py35/test.py +++ b/tests/contrib/aiobotocore/py35/test.py @@ -5,6 +5,7 @@ from ..utils import aiobotocore_client from ...asyncio.utils import AsyncioTestCase, mark_asyncio from ....test_tracer import get_dummy_tracer +from ....utils import assert_span_http_status_code class AIOBotocoreTest(AsyncioTestCase): @@ -45,13 +46,13 @@ async def test_response_context_manager(self): span = traces[0][0] assert span.get_tag('aws.operation') == 'GetObject' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.service == 'aws.s3' assert span.resource == 's3.getobject' read_span = traces[1][0] assert read_span.get_tag('aws.operation') == 'GetObject' - assert read_span.get_metric('http.status_code') == 200 + assert_span_http_status_code(read_span, 200) assert read_span.service == 'aws.s3' assert read_span.resource == 's3.getobject' assert read_span.name == 's3.command.read' @@ -64,7 +65,7 @@ async def test_response_context_manager(self): span = traces[0][0] assert span.get_tag('aws.operation') == 'GetObject' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.service == 'aws.s3' assert span.resource == 's3.getobject' assert span.name == 's3.command' diff --git a/tests/contrib/aiobotocore/test.py b/tests/contrib/aiobotocore/test.py index 13718d2af97..ebc4a0e1b20 100644 --- a/tests/contrib/aiobotocore/test.py +++ b/tests/contrib/aiobotocore/test.py @@ -9,6 +9,7 @@ from .utils import aiobotocore_client from ..asyncio.utils import AsyncioTestCase, mark_asyncio from ...test_tracer import get_dummy_tracer +from ...utils import assert_span_http_status_code class AIOBotocoreTest(AsyncioTestCase): @@ -36,7 +37,7 @@ def test_traced_client(self): self.assertEqual(span.get_tag('aws.agent'), 'aiobotocore') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_metric('retry_attempts'), 0) self.assertEqual(span.service, 'aws.ec2') self.assertEqual(span.resource, 'ec2.describeinstances') @@ -70,7 +71,7 @@ def test_s3_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.s3') self.assertEqual(span.resource, 's3.listbuckets') self.assertEqual(span.name, 's3.command') @@ -87,7 +88,7 @@ def test_s3_put(self): assert spans self.assertEqual(len(spans), 2) self.assertEqual(spans[0].get_tag('aws.operation'), 'CreateBucket') - self.assertEqual(spans[0].get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].service, 'aws.s3') self.assertEqual(spans[0].resource, 's3.createbucket') self.assertEqual(spans[1].get_tag('aws.operation'), 'PutObject') @@ -136,14 +137,14 @@ def test_s3_client_read(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.operation'), 'GetObject') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.s3') self.assertEqual(span.resource, 's3.getobject') if pre_08: read_span = traces[1][0] self.assertEqual(read_span.get_tag('aws.operation'), 'GetObject') - self.assertEqual(read_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(read_span, 200) self.assertEqual(read_span.service, 'aws.s3') self.assertEqual(read_span.resource, 's3.getobject') self.assertEqual(read_span.name, 's3.command.read') @@ -163,7 +164,7 @@ def test_sqs_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListQueues') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.sqs') self.assertEqual(span.resource, 'sqs.listqueues') @@ -179,7 +180,7 @@ def test_kinesis_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListStreams') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.kinesis') self.assertEqual(span.resource, 'kinesis.liststreams') @@ -196,7 +197,7 @@ def test_lambda_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListFunctions') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.lambda') self.assertEqual(span.resource, 'lambda.listfunctions') @@ -212,7 +213,7 @@ def test_kms_client(self): span = traces[0][0] self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListKeys') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.kms') self.assertEqual(span.resource, 'kms.listkeys') # checking for protection on STS against security leak @@ -264,7 +265,7 @@ def test_opentraced_client(self): self.assertEqual(dd_span.get_tag('aws.agent'), 'aiobotocore') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(dd_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.get_metric('retry_attempts'), 0) self.assertEqual(dd_span.service, 'aws.ec2') self.assertEqual(dd_span.resource, 'ec2.describeinstances') @@ -305,13 +306,13 @@ def test_opentraced_s3_client(self): self.assertEqual(ot_inner_span2.parent_id, ot_outer_span.span_id) self.assertEqual(dd_span.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(dd_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.service, 'aws.s3') self.assertEqual(dd_span.resource, 's3.listbuckets') self.assertEqual(dd_span.name, 's3.command') self.assertEqual(dd_span2.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(dd_span2.get_metric('http.status_code'), 200) + assert_span_http_status_code(dd_span2, 200) self.assertEqual(dd_span2.service, 'aws.s3') self.assertEqual(dd_span2.resource, 's3.listbuckets') self.assertEqual(dd_span2.name, 's3.command') diff --git a/tests/contrib/aiohttp/test_middleware.py b/tests/contrib/aiohttp/test_middleware.py index 02ad52944a9..77eefc68cac 100644 --- a/tests/contrib/aiohttp/test_middleware.py +++ b/tests/contrib/aiohttp/test_middleware.py @@ -11,6 +11,7 @@ from tests.opentracer.utils import init_tracer from .utils import TraceTestCase from .app.web import setup_app, noop_middleware +from ...utils import assert_span_http_status_code class TestTraceMiddleware(TraceTestCase): @@ -42,7 +43,7 @@ def test_handler(self): assert 'GET /' == span.resource assert str(self.client.make_url('/')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') - assert 200 == span.get_metric('http.status_code') + assert_span_http_status_code(span, 200) assert 0 == span.error @asyncio.coroutine @@ -64,7 +65,7 @@ def _test_param_handler(self, query_string=''): # with the right fields assert 'GET /echo/{name}' == span.resource assert str(self.client.make_url('/echo/team')) == span.get_tag(http.URL) - assert 200 == span.get_metric('http.status_code') + assert_span_http_status_code(span, 200) if self.app[CONFIG_KEY].get('trace_query_string'): assert query_string == span.get_tag(http.QUERY_STRING) else: @@ -112,7 +113,7 @@ def test_404_handler(self): assert '404' == span.resource assert str(self.client.make_url('/404/not_found')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') - assert 404 == span.get_metric('http.status_code') + assert_span_http_status_code(span, 404) @unittest_run_loop @asyncio.coroutine @@ -128,7 +129,7 @@ def test_server_error(self): assert len(traces[0]) == 1 span = traces[0][0] assert span.get_tag('http.method') == 'GET' - assert span.get_metric('http.status_code') == 500 + assert_span_http_status_code(span, 500) assert span.error == 1 @unittest_run_loop @@ -145,7 +146,7 @@ def test_500_response_code(self): assert len(traces[0]) == 1 span = traces[0][0] assert span.get_tag('http.method') == 'GET' - assert span.get_metric('http.status_code') == 503 + assert_span_http_status_code(span, 503) assert span.error == 1 @unittest_run_loop @@ -168,7 +169,7 @@ def test_coroutine_chaining(self): assert 'GET /chaining/' == root.resource assert str(self.client.make_url('/chaining/')) == root.get_tag(http.URL) assert 'GET' == root.get_tag('http.method') - assert 200 == root.get_metric('http.status_code') + assert_span_http_status_code(root, 200) # span created in the coroutine_chaining handler assert 'aiohttp.coro_1' == handler.name assert root.span_id == handler.parent_id @@ -196,7 +197,7 @@ def test_static_handler(self): assert 'GET /statics' == span.resource assert str(self.client.make_url('/statics/empty.txt')) == span.get_tag(http.URL) assert 'GET' == span.get_tag('http.method') - assert 200 == span.get_metric('http.status_code') + assert_span_http_status_code(span, 200) @unittest_run_loop @asyncio.coroutine @@ -421,7 +422,7 @@ def _assert_200_parenting(self, traces): assert 'GET /' == inner_span.resource assert str(self.client.make_url('/')) == inner_span.get_tag(http.URL) assert 'GET' == inner_span.get_tag('http.method') - assert 200 == inner_span.get_metric('http.status_code') + assert_span_http_status_code(inner_span, 200) assert 0 == inner_span.error @unittest_run_loop diff --git a/tests/contrib/boto/test.py b/tests/contrib/boto/test.py index 201bc92b7f1..d785a5021a9 100644 --- a/tests/contrib/boto/test.py +++ b/tests/contrib/boto/test.py @@ -18,6 +18,7 @@ from unittest import skipUnless from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code class BotoTest(BaseTracerTestCase): @@ -41,7 +42,7 @@ def test_ec2_client(self): self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'POST') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertIsNone(span.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) @@ -53,7 +54,7 @@ def test_ec2_client(self): self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag('aws.operation'), 'RunInstances') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'POST') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.service, 'test-boto-tracing.ec2') @@ -107,7 +108,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.operation'), 'get_all_buckets') @@ -117,7 +118,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'PUT') self.assertEqual(span.get_tag('path'), '/') self.assertEqual(span.get_tag('aws.operation'), 'create_bucket') @@ -128,7 +129,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'HEAD') self.assertEqual(span.get_tag('aws.operation'), 'head_bucket') self.assertEqual(span.service, 'test-boto-tracing.s3') @@ -161,7 +162,7 @@ def test_s3_put(self): # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket') - self.assertEqual(spans[0].get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].service, 'test-boto-tracing.s3') self.assertEqual(spans[0].resource, 's3.put') # get bucket @@ -215,7 +216,7 @@ def test_lambda_client(self): assert spans self.assertEqual(len(spans), 2) span = spans[0] - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.region'), 'us-east-2') self.assertEqual(span.get_tag('aws.operation'), 'list_functions') @@ -285,7 +286,7 @@ def test_ec2_client_ot(self): self.assertEqual(ot_span.resource, 'ot_span') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(dd_span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.get_tag(http.METHOD), 'POST') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') @@ -301,7 +302,7 @@ def test_ec2_client_ot(self): self.assertEqual(dd_span.parent_id, ot_span.span_id) self.assertEqual(dd_span.get_tag('aws.operation'), 'RunInstances') - self.assertEqual(dd_span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.get_tag(http.METHOD), 'POST') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.service, 'test-boto-tracing.ec2') diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index 942d137ad6a..02bd75a142d 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -12,6 +12,7 @@ # testing from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code class BotocoreTest(BaseTracerTestCase): @@ -46,7 +47,7 @@ def test_traced_client(self): self.assertEqual(span.get_tag('aws.agent'), 'botocore') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_metric('retry_attempts'), 0) self.assertEqual(span.service, 'test-botocore-tracing.ec2') self.assertEqual(span.resource, 'ec2.describeinstances') @@ -82,7 +83,7 @@ def test_s3_client(self): span = spans[0] self.assertEqual(len(spans), 2) self.assertEqual(span.get_tag('aws.operation'), 'ListBuckets') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.s3') self.assertEqual(span.resource, 's3.listbuckets') @@ -110,7 +111,7 @@ def test_s3_put(self): span = spans[0] self.assertEqual(len(spans), 2) self.assertEqual(span.get_tag('aws.operation'), 'CreateBucket') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.s3') self.assertEqual(span.resource, 's3.createbucket') self.assertEqual(spans[1].get_tag('aws.operation'), 'PutObject') @@ -133,7 +134,7 @@ def test_sqs_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListQueues') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.sqs') self.assertEqual(span.resource, 'sqs.listqueues') @@ -150,7 +151,7 @@ def test_kinesis_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListStreams') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.kinesis') self.assertEqual(span.resource, 'kinesis.liststreams') @@ -192,7 +193,7 @@ def test_lambda_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListFunctions') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.lambda') self.assertEqual(span.resource, 'lambda.listfunctions') @@ -209,7 +210,7 @@ def test_kms_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListKeys') - self.assertEqual(span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.kms') self.assertEqual(span.resource, 'kms.listkeys') @@ -242,7 +243,7 @@ def test_traced_client_ot(self): self.assertEqual(dd_span.get_tag('aws.agent'), 'botocore') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') - self.assertEqual(dd_span.get_metric(http.STATUS_CODE), 200) + assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.get_metric('retry_attempts'), 0) self.assertEqual(dd_span.service, 'test-botocore-tracing.ec2') self.assertEqual(dd_span.resource, 'ec2.describeinstances') diff --git a/tests/contrib/bottle/test.py b/tests/contrib/bottle/test.py index 290984dc359..28af1a808f5 100644 --- a/tests/contrib/bottle/test.py +++ b/tests/contrib/bottle/test.py @@ -4,6 +4,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code from ddtrace import compat from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY @@ -58,7 +59,7 @@ def hi(name): assert s.service == 'bottle-app' assert s.span_type == 'web' assert s.resource == 'GET /hi/' - assert s.get_metric('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/hi/dougie' if ddtrace.config.bottle.trace_query_string: @@ -99,7 +100,7 @@ def handled(): assert len(spans) == 1 s = spans[0] assert s.resource == 'GET /2xx' - assert s.get_metric('http.status_code') == 202 + assert_span_http_status_code(s, 202) assert s.error == 0 def test_400_return(self): @@ -120,7 +121,7 @@ def handled400(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /400_return' - assert s.get_metric('http.status_code') == 400 + assert_span_http_status_code(s, 400) assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/400_return' assert s.error == 0 @@ -143,7 +144,7 @@ def handled400(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /400_raise' - assert s.get_metric('http.status_code') == 400 + assert_span_http_status_code(s, 400) assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/400_raise' assert s.error == 1 @@ -166,7 +167,7 @@ def hi(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' - assert s.get_metric('http.status_code') == 500 + assert_span_http_status_code(s, 500) assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/hi' assert s.error == 1 @@ -233,7 +234,7 @@ def hi(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' - assert s.get_metric('http.status_code') == 420 + assert_span_http_status_code(s, 420) assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/hi' @@ -254,7 +255,7 @@ def home(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /home/' - assert s.get_metric('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.get_tag('http.method') == 'GET' assert s.get_tag(http.URL) == 'http://localhost:80/home/' @@ -404,7 +405,7 @@ def hi(name): assert dd_span.name == 'bottle.request' assert dd_span.service == 'bottle-app' assert dd_span.resource == 'GET /hi/' - assert dd_span.get_metric('http.status_code') == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.get_tag('http.method') == 'GET' assert dd_span.get_tag(http.URL) == 'http://localhost:80/hi/dougie' diff --git a/tests/contrib/bottle/test_autopatch.py b/tests/contrib/bottle/test_autopatch.py index 51ca98fc39a..e4ca346860e 100644 --- a/tests/contrib/bottle/test_autopatch.py +++ b/tests/contrib/bottle/test_autopatch.py @@ -4,6 +4,7 @@ from unittest import TestCase from tests.test_tracer import get_dummy_tracer +from ...utils import assert_span_http_status_code from ddtrace import compat @@ -48,7 +49,7 @@ def hi(name): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi/' - assert s.get_metric('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.get_tag('http.method') == 'GET' services = self.tracer.writer.pop_services() @@ -73,7 +74,7 @@ def hi(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' - assert s.get_metric('http.status_code') == 500 + assert_span_http_status_code(s, 500) assert s.get_tag('http.method') == 'GET' def test_bottle_global_tracer(self): @@ -93,5 +94,5 @@ def home(): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /home/' - assert s.get_metric('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.get_tag('http.method') == 'GET' diff --git a/tests/contrib/bottle/test_distributed.py b/tests/contrib/bottle/test_distributed.py index 58861999e1f..742852c3ab4 100644 --- a/tests/contrib/bottle/test_distributed.py +++ b/tests/contrib/bottle/test_distributed.py @@ -6,6 +6,7 @@ from ddtrace.contrib.bottle import TracePlugin from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code SERVICE = 'bottle-app' @@ -56,7 +57,7 @@ def hi(name): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi/' - assert s.get_metric('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.get_tag('http.method') == 'GET' # check distributed headers assert 123 == s.trace_id @@ -83,7 +84,7 @@ def hi(name): assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi/' - assert s.get_metric('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.get_tag('http.method') == 'GET' # check distributed headers assert 123 != s.trace_id diff --git a/tests/contrib/django/test_middleware.py b/tests/contrib/django/test_middleware.py index 39f713eed48..fb4271a0779 100644 --- a/tests/contrib/django/test_middleware.py +++ b/tests/contrib/django/test_middleware.py @@ -12,6 +12,7 @@ from tests.opentracer.utils import init_tracer from .compat import reverse from .utils import DjangoTraceTestCase, override_ddtrace_settings +from ...utils import assert_span_http_status_code class DjangoMiddlewareTest(DjangoTraceTestCase): @@ -36,7 +37,7 @@ def test_middleware_trace_request(self, query_string=''): sp_database = spans[2] assert sp_database.get_tag('django.db.vendor') == 'sqlite' assert sp_template.get_tag('django.template_name') == 'users_list.html' - assert sp_request.get_metric('http.status_code') == 200 + assert_span_http_status_code(sp_request, 200) assert sp_request.get_tag(http.URL) == 'http://testserver/users/' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' @@ -205,7 +206,7 @@ def test_middleware_trace_errors(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_metric('http.status_code') == 403 + assert_span_http_status_code(span, 403) assert span.get_tag(http.URL) == 'http://testserver/fail-view/' assert span.resource == 'tests.contrib.django.app.views.ForbiddenView' @@ -219,7 +220,7 @@ def test_middleware_trace_function_based_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag(http.URL) == 'http://testserver/fn-view/' assert span.resource == 'tests.contrib.django.app.views.function_view' @@ -234,7 +235,7 @@ def test_middleware_trace_error_500(self): assert len(spans) == 1 span = spans[0] assert span.error == 1 - assert span.get_metric('http.status_code') == 500 + assert_span_http_status_code(span, 500) assert span.get_tag(http.URL) == 'http://testserver/error-500/' assert span.resource == 'tests.contrib.django.app.views.error_500' assert 'Error 500' in span.get_tag('error.stack') @@ -249,7 +250,7 @@ def test_middleware_trace_callable_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag(http.URL) == 'http://testserver/feed-view/' assert span.resource == 'tests.contrib.django.app.views.FeedView' @@ -263,7 +264,7 @@ def test_middleware_trace_partial_based_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag(http.URL) == 'http://testserver/partial-view/' assert span.resource == 'partial' @@ -277,7 +278,7 @@ def test_middleware_trace_lambda_based_view(self): spans = self.tracer.writer.pop() assert len(spans) == 1 span = spans[0] - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag(http.URL) == 'http://testserver/lambda-view/' assert span.resource == 'tests.contrib.django.app.views.' @@ -300,7 +301,7 @@ def test_middleware_without_user(self): spans = self.tracer.writer.pop() assert len(spans) == 3 sp_request = spans[0] - assert sp_request.get_metric('http.status_code') == 200 + assert_span_http_status_code(sp_request, 200) assert sp_request.get_tag('django.user.is_authenticated') is None def test_middleware_propagation(self): @@ -425,7 +426,7 @@ def test_middleware_trace_request_ot(self): assert sp_database.get_tag('django.db.vendor') == 'sqlite' assert sp_template.get_tag('django.template_name') == 'users_list.html' - assert sp_request.get_metric('http.status_code') == 200 + assert_span_http_status_code(sp_request, 200) assert sp_request.get_tag(http.URL) == 'http://testserver/users/' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' @@ -451,7 +452,7 @@ def test_middleware_trace_request_404(self): assert sp_template.get_tag('django.template_name') == 'unknown' # Request - assert sp_request.get_metric('http.status_code') == 404 + assert_span_http_status_code(sp_request, 404) assert sp_request.get_tag(http.URL) == 'http://testserver/unknown-url' assert sp_request.get_tag('django.user.is_authenticated') == 'False' assert sp_request.get_tag('http.method') == 'GET' diff --git a/tests/contrib/djangorestframework/test_djangorestframework.py b/tests/contrib/djangorestframework/test_djangorestframework.py index 0d2aa41ec6c..a91c9f9f010 100644 --- a/tests/contrib/djangorestframework/test_djangorestframework.py +++ b/tests/contrib/djangorestframework/test_djangorestframework.py @@ -3,6 +3,7 @@ from unittest import skipIf from tests.contrib.django.utils import DjangoTraceTestCase +from ...utils import assert_span_http_status_code @skipIf(django.VERSION < (1, 10), 'requires django version >= 1.10') @@ -38,7 +39,7 @@ def test_unpatch(self): assert sp.resource == 'tests.contrib.djangorestframework.app.views.UserViewSet' assert sp.error == 0 assert sp.span_type == 'web' - assert sp.get_metric('http.status_code') == 500 + assert_span_http_status_code(sp, 500) assert sp.get_tag('error.msg') is None def test_trace_exceptions(self): @@ -56,6 +57,6 @@ def test_trace_exceptions(self): assert sp.error == 1 assert sp.span_type == 'web' assert sp.get_tag('http.method') == 'GET' - assert sp.get_metric('http.status_code') == 500 + assert_span_http_status_code(sp, 500) assert sp.get_tag('error.msg') == 'Authentication credentials were not provided.' assert 'NotAuthenticated' in sp.get_tag('error.stack') diff --git a/tests/contrib/elasticsearch/test.py b/tests/contrib/elasticsearch/test.py index 1e36e4a8e34..3d705b3e468 100644 --- a/tests/contrib/elasticsearch/test.py +++ b/tests/contrib/elasticsearch/test.py @@ -14,6 +14,7 @@ from ..config import ELASTICSEARCH_CONFIG from ...test_tracer import get_dummy_tracer from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code class ElasticsearchTest(unittest.TestCase): @@ -131,7 +132,7 @@ def test_elasticsearch(self): spans = writer.pop() assert spans span = spans[0] - assert span.get_metric(http.STATUS_CODE) == 404 + assert_span_http_status_code(span, 404) # Raise error 400, the index 10 is created twice try: @@ -142,7 +143,7 @@ def test_elasticsearch(self): spans = writer.pop() assert spans span = spans[-1] - assert span.get_metric(http.STATUS_CODE) == 400 + assert_span_http_status_code(span, 400) # Drop the index, checking it won't raise exception on success or failure es.indices.delete(index=self.ES_INDEX, ignore=[400, 404]) diff --git a/tests/contrib/falcon/test_suite.py b/tests/contrib/falcon/test_suite.py index 20c2298adaf..6797ce62d32 100644 --- a/tests/contrib/falcon/test_suite.py +++ b/tests/contrib/falcon/test_suite.py @@ -3,6 +3,7 @@ from ddtrace.ext import errors as errx, http as httpx from tests.opentracer.utils import init_tracer +from ...utils import assert_span_http_status_code class FalconTestCase(object): @@ -21,7 +22,7 @@ def test_404(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET 404' - assert span.get_metric(httpx.STATUS_CODE) == 404 + assert_span_http_status_code(span, 404) assert span.get_tag(httpx.URL) == 'http://falconframework.org/fake_endpoint' assert httpx.QUERY_STRING not in span.meta assert span.parent_id is None @@ -41,7 +42,7 @@ def test_exception(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.ResourceException' - assert span.get_metric(httpx.STATUS_CODE) == 500 + assert_span_http_status_code(span, 500) assert span.get_tag(httpx.URL) == 'http://falconframework.org/exception' assert span.parent_id is None @@ -57,7 +58,7 @@ def test_200(self, query_string=''): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.Resource200' - assert span.get_metric(httpx.STATUS_CODE) == 200 + assert_span_http_status_code(span, 200) fqs = ('?' + query_string) if query_string else '' assert span.get_tag(httpx.URL) == 'http://falconframework.org/200' + fqs if config.falcon.trace_query_string: @@ -154,7 +155,7 @@ def test_201(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'POST tests.contrib.falcon.app.resources.Resource201' - assert span.get_metric(httpx.STATUS_CODE) == 201 + assert_span_http_status_code(span, 201) assert span.get_tag(httpx.URL) == 'http://falconframework.org/201' assert span.parent_id is None @@ -170,7 +171,7 @@ def test_500(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.Resource500' - assert span.get_metric(httpx.STATUS_CODE) == 500 + assert_span_http_status_code(span, 500) assert span.get_tag(httpx.URL) == 'http://falconframework.org/500' assert span.parent_id is None @@ -185,7 +186,7 @@ def test_404_exception(self): assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.ResourceNotFound' - assert span.get_metric(httpx.STATUS_CODE) == 404 + assert_span_http_status_code(span, 404) assert span.get_tag(httpx.URL) == 'http://falconframework.org/not_found' assert span.parent_id is None @@ -200,7 +201,7 @@ def test_404_exception_no_stacktracer(self): span = traces[0][0] assert span.name == 'falcon.request' assert span.service == self._service - assert span.get_metric(httpx.STATUS_CODE) == 404 + assert_span_http_status_code(span, 404) assert span.get_tag(errx.ERROR_TYPE) is None assert span.parent_id is None @@ -229,7 +230,7 @@ def test_200_ot(self): assert dd_span.name == 'falcon.request' assert dd_span.service == self._service assert dd_span.resource == 'GET tests.contrib.falcon.app.resources.Resource200' - assert dd_span.get_metric(httpx.STATUS_CODE) == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.get_tag(httpx.URL) == 'http://falconframework.org/200' def test_falcon_request_hook(self): diff --git a/tests/contrib/flask/test_errorhandler.py b/tests/contrib/flask/test_errorhandler.py index 66226114224..bd4d875910b 100644 --- a/tests/contrib/flask/test_errorhandler.py +++ b/tests/contrib/flask/test_errorhandler.py @@ -1,6 +1,7 @@ import flask from . import BaseFlaskTestCase +from ...utils import assert_span_http_status_code class FlaskErrorhandlerTestCase(BaseFlaskTestCase): @@ -23,7 +24,7 @@ def test_default_404_handler(self): # flask.request span self.assertEqual(req_span.error, 0) - self.assertEqual(req_span.get_metric('http.status_code'), 404) + assert_span_http_status_code(req_span, 404) self.assertIsNone(req_span.get_tag('flask.endpoint')) self.assertIsNone(req_span.get_tag('flask.url_rule')) @@ -68,7 +69,8 @@ def endpoint_500(): # flask.request span self.assertEqual(req_span.error, 1) - self.assertEqual(req_span.get_metric('http.status_code'), 500) + + assert_span_http_status_code(req_span, 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_500') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') @@ -128,7 +130,7 @@ def endpoint_500(): # flask.request span self.assertEqual(req_span.error, 0) - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_500') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') @@ -191,7 +193,7 @@ def endpoint_error(): # flask.request span self.assertEqual(req_span.error, 1) - self.assertEqual(req_span.get_metric('http.status_code'), 500) + assert_span_http_status_code(req_span, 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_error') self.assertEqual(req_span.get_tag('flask.url_rule'), '/error') @@ -258,7 +260,7 @@ def endpoint_error(): # flask.request span self.assertEqual(req_span.error, 0) - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) self.assertEqual(req_span.get_tag('flask.endpoint'), 'endpoint_error') self.assertEqual(req_span.get_tag('flask.url_rule'), '/error') diff --git a/tests/contrib/flask/test_hooks.py b/tests/contrib/flask/test_hooks.py index bef73c4a95c..43daaa6f675 100644 --- a/tests/contrib/flask/test_hooks.py +++ b/tests/contrib/flask/test_hooks.py @@ -2,6 +2,7 @@ from ddtrace.ext import http from . import BaseFlaskTestCase +from ...utils import assert_span_http_status_code class FlaskHookTestCase(BaseFlaskTestCase): @@ -81,7 +82,7 @@ def before_request(): self.assertEqual(root.get_tag('flask.endpoint'), 'index') self.assertEqual(root.get_tag('flask.url_rule'), '/') self.assertEqual(root.get_tag('http.method'), 'GET') - self.assertEqual(root.get_metric('http.status_code'), 401) + assert_span_http_status_code(root, 401) self.assertEqual(root.get_tag(http.URL), 'http://localhost/') # Assert hook span @@ -182,7 +183,7 @@ def after_request(response): parent = self.find_span_parent(spans, span) # Assert root span - self.assertEqual(root.get_metric('http.status_code'), 401) + assert_span_http_status_code(root, 401) # Assert hook span self.assertEqual(span.service, 'flask') diff --git a/tests/contrib/flask/test_middleware.py b/tests/contrib/flask/test_middleware.py index 361f8e1cd2b..4261c31678f 100644 --- a/tests/contrib/flask/test_middleware.py +++ b/tests/contrib/flask/test_middleware.py @@ -11,6 +11,7 @@ from tests.opentracer.utils import init_tracer from .web import create_app from ...test_tracer import get_dummy_tracer +from ...utils import assert_span_http_status_code class TestFlask(TestCase): @@ -110,7 +111,7 @@ def test_success(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.metrics.get(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.METHOD) == 'GET' services = self.tracer.writer.pop_services() @@ -137,7 +138,7 @@ def test_template(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.metrics.get(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.METHOD) == 'GET' t = by_name['flask.template'] @@ -165,7 +166,7 @@ def test_handleme(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.metrics.get(http.STATUS_CODE) == 202 + assert_span_http_status_code(s, 202) assert s.meta.get(http.METHOD) == 'GET' def test_template_err(self): @@ -189,7 +190,7 @@ def test_template_err(self): assert s.start >= start assert s.duration <= end - start assert s.error == 1 - assert s.get_metric(http.STATUS_CODE) == 500 + assert_span_http_status_code(s, 500) assert s.meta.get(http.METHOD) == 'GET' def test_template_render_err(self): @@ -213,7 +214,7 @@ def test_template_render_err(self): assert s.start >= start assert s.duration <= end - start assert s.error == 1 - assert s.get_metric(http.STATUS_CODE) == 500 + assert_span_http_status_code(s, 500) assert s.meta.get(http.METHOD) == 'GET' t = by_name['flask.template'] assert t.get_tag('flask.template') == 'render_err.html' @@ -239,7 +240,7 @@ def test_error(self): assert s.resource == 'error' assert s.start >= start assert s.duration <= end - start - assert s.get_metric(http.STATUS_CODE) == 500 + assert_span_http_status_code(s, 500) assert s.meta.get(http.METHOD) == 'GET' def test_fatal(self): @@ -264,7 +265,7 @@ def test_fatal(self): assert s.resource == 'fatal' assert s.start >= start assert s.duration <= end - start - assert s.get_metric(http.STATUS_CODE) == 500 + assert_span_http_status_code(s, 500) assert s.meta.get(http.METHOD) == 'GET' assert 'ZeroDivisionError' in s.meta.get(errors.ERROR_TYPE), s.meta assert 'by zero' in s.meta.get(errors.ERROR_MSG) @@ -289,7 +290,7 @@ def test_unicode(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.METHOD) == 'GET' assert s.meta.get(http.URL) == u'http://localhost/üŋïĉóđē' @@ -311,7 +312,7 @@ def test_404(self): assert s.start >= start assert s.duration <= end - start assert s.error == 0 - assert s.get_metric(http.STATUS_CODE) == 404 + assert_span_http_status_code(s, 404) assert s.meta.get(http.METHOD) == 'GET' assert s.meta.get(http.URL) == u'http://localhost/404/üŋïĉóđē' @@ -348,7 +349,7 @@ def test_custom_span(self): assert s.service == 'test.flask.service' assert s.resource == 'overridden' assert s.error == 0 - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.METHOD) == 'GET' def test_success_200_ot(self): @@ -382,5 +383,5 @@ def test_success_200_ot(self): assert dd_span.start >= start assert dd_span.duration <= end - start assert dd_span.error == 0 - assert dd_span.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.meta.get(http.METHOD) == 'GET' diff --git a/tests/contrib/flask/test_request.py b/tests/contrib/flask/test_request.py index 863fd828b0a..86c0f5e44bf 100644 --- a/tests/contrib/flask/test_request.py +++ b/tests/contrib/flask/test_request.py @@ -7,6 +7,7 @@ from flask import abort from . import BaseFlaskTestCase +from ...utils import assert_span_http_status_code base_exception_name = 'builtins.Exception' @@ -64,7 +65,7 @@ def index(): self.assertEqual(req_span.get_tag('flask.url_rule'), '/') self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) assert http.QUERY_STRING not in req_span.meta # Handler span @@ -299,7 +300,7 @@ def index(): self.assertEqual(req_span.get_tag('http.method'), 'GET') # Note: contains no query string self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) # Handler span handler_span = spans[4] @@ -359,7 +360,7 @@ def unicode(): self.assertEqual(req_span.get_tag('flask.url_rule'), u'/üŋïĉóđē') self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), u'http://localhost/üŋïĉóđē') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) # Handler span handler_span = spans[4] @@ -412,7 +413,7 @@ def test_request_404(self): # Request tags self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/not-found') - self.assertEqual(req_span.get_metric('http.status_code'), 404) + assert_span_http_status_code(req_span, 404) # Dispatch span dispatch_span = spans[3] @@ -473,7 +474,7 @@ def not_found(): # Request tags self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/not-found') - self.assertEqual(req_span.get_metric('http.status_code'), 404) + assert_span_http_status_code(req_span, 404) self.assertEqual(req_span.get_tag('flask.endpoint'), 'not_found') self.assertEqual(req_span.get_tag('flask.url_rule'), '/not-found') @@ -545,7 +546,7 @@ def fivehundred(): # Request tags self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/500') - self.assertEqual(req_span.get_metric('http.status_code'), 500) + assert_span_http_status_code(req_span, 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'fivehundred') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') @@ -628,7 +629,7 @@ def fivehundredone(): # Request tags self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/501') - self.assertEqual(req_span.get_metric('http.status_code'), 501) + assert_span_http_status_code(req_span, 501) self.assertEqual(req_span.get_tag('flask.endpoint'), 'fivehundredone') self.assertEqual(req_span.get_tag('flask.url_rule'), '/501') @@ -735,7 +736,7 @@ def fivehundred(): # Request tags self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/500') - self.assertEqual(req_span.get_metric('http.status_code'), 500) + assert_span_http_status_code(req_span, 500) self.assertEqual(req_span.get_tag('flask.endpoint'), 'fivehundred') self.assertEqual(req_span.get_tag('flask.url_rule'), '/500') diff --git a/tests/contrib/flask/test_static.py b/tests/contrib/flask/test_static.py index 49bb6a45325..9ad8be9053c 100644 --- a/tests/contrib/flask/test_static.py +++ b/tests/contrib/flask/test_static.py @@ -1,6 +1,7 @@ from ddtrace.ext import http from . import BaseFlaskTestCase +from ...utils import assert_span_http_status_code class FlaskStaticFileTestCase(BaseFlaskTestCase): @@ -29,7 +30,7 @@ def test_serve_static_file(self): self.assertEqual(req_span.get_tag('flask.endpoint'), 'static') self.assertEqual(req_span.get_tag('flask.url_rule'), '/static/') self.assertEqual(req_span.get_tag('flask.view_args.filename'), 'test.txt') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/static/test.txt') self.assertEqual(req_span.get_tag('http.method'), 'GET') @@ -70,7 +71,7 @@ def test_serve_static_file_404(self): self.assertEqual(req_span.get_tag('flask.endpoint'), 'static') self.assertEqual(req_span.get_tag('flask.url_rule'), '/static/') self.assertEqual(req_span.get_tag('flask.view_args.filename'), 'unknown-file') - self.assertEqual(req_span.get_metric('http.status_code'), 404) + assert_span_http_status_code(req_span, 404) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/static/unknown-file') self.assertEqual(req_span.get_tag('http.method'), 'GET') diff --git a/tests/contrib/flask/test_views.py b/tests/contrib/flask/test_views.py index 2a2ce8030e3..c45fa4446c3 100644 --- a/tests/contrib/flask/test_views.py +++ b/tests/contrib/flask/test_views.py @@ -4,6 +4,7 @@ from ddtrace.ext import http from . import BaseFlaskTestCase +from ...utils import assert_span_http_status_code base_exception_name = 'builtins.Exception' @@ -40,7 +41,7 @@ def dispatch_request(self, name): self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # tests.contrib.flask.test_views.hello @@ -77,7 +78,7 @@ def dispatch_request(self, name): self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_metric('http.status_code'), 500) + assert_span_http_status_code(req_span, 500) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # flask.dispatch_request @@ -119,7 +120,7 @@ def get(self, name): self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # tests.contrib.flask.test_views.hello @@ -154,7 +155,7 @@ def get(self, name): self.assertEqual(req_span.get_tag('flask.url_rule'), '/hello/') self.assertEqual(req_span.get_tag('flask.view_args.name'), 'flask') self.assertEqual(req_span.get_tag('http.method'), 'GET') - self.assertEqual(req_span.get_metric('http.status_code'), 500) + assert_span_http_status_code(req_span, 500) self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/hello/flask') # flask.dispatch_request diff --git a/tests/contrib/flask_autopatch/test_flask_autopatch.py b/tests/contrib/flask_autopatch/test_flask_autopatch.py index 9ef245df0db..2f433d27e22 100644 --- a/tests/contrib/flask_autopatch/test_flask_autopatch.py +++ b/tests/contrib/flask_autopatch/test_flask_autopatch.py @@ -8,6 +8,7 @@ from ddtrace import Pin from ...test_tracer import get_dummy_tracer +from ...utils import assert_span_http_status_code class FlaskAutopatchTestCase(unittest.TestCase): @@ -89,7 +90,7 @@ def index(): self.assertEqual(req_span.get_tag('flask.url_rule'), '/') self.assertEqual(req_span.get_tag('http.method'), 'GET') self.assertEqual(req_span.get_tag(http.URL), 'http://localhost/') - self.assertEqual(req_span.get_metric('http.status_code'), 200) + assert_span_http_status_code(req_span, 200) # Handler span handler_span = spans[4] diff --git a/tests/contrib/httplib/test_httplib.py b/tests/contrib/httplib/test_httplib.py index 80c878aa2d2..9f0037c27ee 100644 --- a/tests/contrib/httplib/test_httplib.py +++ b/tests/contrib/httplib/test_httplib.py @@ -18,6 +18,7 @@ from ...base import BaseTracerTestCase from ...util import override_global_tracer +from ...utils import assert_span_http_status_code if PY2: from urllib2 import urlopen, build_opener, Request @@ -154,7 +155,7 @@ def test_httplib_request_get_request(self, query_string=''): self.assertEqual(span.error, 0) assert span.get_tag('http.method') == 'GET' assert span.get_tag('http.url') == URL_200 - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) if config.httplib.trace_query_string: assert span.get_tag(http.QUERY_STRING) == query_string else: @@ -190,7 +191,7 @@ def test_httplib_request_get_request_https(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) assert span.get_tag('http.method') == 'GET' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag('http.url') == 'https://httpbin.org/status/200' def test_httplib_request_post_request(self): @@ -214,7 +215,7 @@ def test_httplib_request_post_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) assert span.get_tag('http.method') == 'POST' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag('http.url') == URL_200 def test_httplib_request_get_request_query_string(self): @@ -237,7 +238,7 @@ def test_httplib_request_get_request_query_string(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) assert span.get_tag('http.method') == 'GET' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag('http.url') == URL_200 def test_httplib_request_500_request(self): @@ -266,7 +267,7 @@ def test_httplib_request_500_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 1) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 500) + assert_span_http_status_code(span, 500) self.assertEqual(span.get_tag('http.url'), URL_500) def test_httplib_request_non_200_request(self): @@ -295,7 +296,7 @@ def test_httplib_request_non_200_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 404) + assert_span_http_status_code(span, 404) self.assertEqual(span.get_tag('http.url'), URL_404) def test_httplib_request_get_request_disabled(self): @@ -379,7 +380,7 @@ def test_urllib_request(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_urllib_request_https(self): @@ -403,7 +404,7 @@ def test_urllib_request_https(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag('http.url'), 'https://httpbin.org/status/200') def test_urllib_request_object(self): @@ -428,7 +429,7 @@ def test_urllib_request_object(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_urllib_request_opener(self): @@ -452,7 +453,7 @@ def test_urllib_request_opener(self): self.assertEqual(span.name, self.SPAN_NAME) self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_httplib_request_get_request_ot(self): @@ -482,7 +483,7 @@ def test_httplib_request_get_request_ot(self): self.assertEqual(dd_span.name, self.SPAN_NAME) self.assertEqual(dd_span.error, 0) assert dd_span.get_tag('http.method') == 'GET' - assert dd_span.get_metric('http.status_code') == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.get_tag('http.url') == URL_200 def test_analytics_default(self): @@ -555,7 +556,7 @@ def test_urllib_request(self): self.assertEqual(span.name, 'httplib.request') self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag('http.url'), URL_200) def test_urllib_request_https(self): @@ -579,5 +580,5 @@ def test_urllib_request_https(self): self.assertEqual(span.name, 'httplib.request') self.assertEqual(span.error, 0) self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag('http.url'), 'https://httpbin.org/status/200') diff --git a/tests/contrib/molten/test_molten.py b/tests/contrib/molten/test_molten.py index e9b9a6f376a..8564234bd5e 100644 --- a/tests/contrib/molten/test_molten.py +++ b/tests/contrib/molten/test_molten.py @@ -9,6 +9,7 @@ from ddtrace.contrib.molten.patch import MOLTEN_VERSION from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code # NOTE: Type annotations required by molten otherwise parameters cannot be coerced @@ -52,7 +53,7 @@ def test_route_success(self): self.assertEqual(span.resource, 'GET /hello/{name}/{age}') self.assertEqual(span.get_tag('http.method'), 'GET') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/hello/Jim/24') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) assert http.QUERY_STRING not in span.meta # See test_resources below for specifics of this difference @@ -81,7 +82,7 @@ def test_route_success_query_string(self): self.assertEqual(span.resource, 'GET /hello/{name}/{age}') self.assertEqual(span.get_tag('http.method'), 'GET') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/hello/Jim/24') - self.assertEqual(span.get_metric('http.status_code'), 200) + assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.QUERY_STRING), 'foo=bar') def test_analytics_global_on_integration_default(self): @@ -168,7 +169,7 @@ def test_route_failure(self): self.assertEqual(span.resource, 'GET 404') self.assertEqual(span.get_tag(http.URL), 'http://127.0.0.1:8000/goodbye') self.assertEqual(span.get_tag('http.method'), 'GET') - self.assertEqual(span.get_metric('http.status_code'), 404) + assert_span_http_status_code(span, 404) def test_route_exception(self): def route_error() -> str: diff --git a/tests/contrib/pylons/test_pylons.py b/tests/contrib/pylons/test_pylons.py index d472addb20e..d041c57368e 100644 --- a/tests/contrib/pylons/test_pylons.py +++ b/tests/contrib/pylons/test_pylons.py @@ -12,6 +12,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code class PylonsTestCase(BaseTracerTestCase): @@ -51,7 +52,7 @@ def test_controller_exception(self): assert span.resource == 'root.raise_exception' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert http.QUERY_STRING not in span.meta assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None @@ -81,7 +82,7 @@ def test_mw_exc_success(self): assert span.resource == 'None.None' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None @@ -109,7 +110,7 @@ def test_middleware_exception(self): assert span.resource == 'None.None' assert span.error == 1 assert span.get_tag(http.URL) == 'http://localhost:80/' - assert span.get_metric('http.status_code') == 500 + assert_span_http_status_code(span, 500) assert span.get_tag(errors.ERROR_MSG) == 'Middleware exception' assert span.get_tag(errors.ERROR_TYPE) == 'exceptions.Exception' assert span.get_tag(errors.ERROR_STACK) @@ -131,7 +132,7 @@ def test_exc_success(self): assert span.resource == 'root.raise_exception' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' - assert span.get_metric('http.status_code') == 200 + assert_span_http_status_code(span, 200) assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None @@ -153,7 +154,7 @@ def test_exc_client_failure(self): assert span.resource == 'root.raise_exception' assert span.error == 0 assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' - assert span.get_metric('http.status_code') == 404 + assert_span_http_status_code(span, 404) assert span.get_tag(errors.ERROR_MSG) is None assert span.get_tag(errors.ERROR_TYPE) is None assert span.get_tag(errors.ERROR_STACK) is None @@ -305,7 +306,7 @@ def test_failure_500(self): assert span.service == 'web' assert span.resource == 'root.raise_exception' assert span.error == 1 - assert span.get_metric('http.status_code') == 500 + assert_span_http_status_code(span, 500) assert span.get_tag('error.msg') == 'Ouch!' assert span.get_tag(http.URL) == 'http://localhost:80/raise_exception' assert 'Exception: Ouch!' in span.get_tag('error.stack') @@ -322,7 +323,7 @@ def test_failure_500_with_wrong_code(self): assert span.service == 'web' assert span.resource == 'root.raise_wrong_code' assert span.error == 1 - assert span.get_metric('http.status_code') == 500 + assert_span_http_status_code(span, 500) assert span.meta.get(http.URL) == 'http://localhost:80/raise_wrong_code' assert span.get_tag('error.msg') == 'Ouch!' assert 'Exception: Ouch!' in span.get_tag('error.stack') @@ -339,7 +340,7 @@ def test_failure_500_with_custom_code(self): assert span.service == 'web' assert span.resource == 'root.raise_custom_code' assert span.error == 1 - assert span.get_metric('http.status_code') == 512 + assert_span_http_status_code(span, 512) assert span.meta.get(http.URL) == 'http://localhost:80/raise_custom_code' assert span.get_tag('error.msg') == 'Ouch!' assert 'Exception: Ouch!' in span.get_tag('error.stack') @@ -356,7 +357,7 @@ def test_failure_500_with_code_method(self): assert span.service == 'web' assert span.resource == 'root.raise_code_method' assert span.error == 1 - assert span.get_metric('http.status_code') == 500 + assert_span_http_status_code(span, 500) assert span.meta.get(http.URL) == 'http://localhost:80/raise_code_method' assert span.get_tag('error.msg') == 'Ouch!' diff --git a/tests/contrib/requests/test_requests.py b/tests/contrib/requests/test_requests.py index 5ca6d1aa1c5..97faa06d2f2 100644 --- a/tests/contrib/requests/test_requests.py +++ b/tests/contrib/requests/test_requests.py @@ -13,6 +13,7 @@ from ...base import BaseTracerTestCase from ...util import override_global_tracer +from ...utils import assert_span_http_status_code # socket name comes from https://english.stackexchange.com/a/44048 SOCKET = 'httpbin.org' @@ -73,7 +74,7 @@ def test_args_kwargs(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) def test_untraced_request(self): # ensure the unpatch removes tracing @@ -105,7 +106,7 @@ def test_200(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.error == 0 assert s.span_type == 'http' assert http.QUERY_STRING not in s.meta @@ -122,7 +123,7 @@ def test_200_send(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.error == 0 assert s.span_type == 'http' @@ -137,7 +138,7 @@ def test_200_query_string(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.get_tag(http.URL) == URL_200 assert s.error == 0 assert s.span_type == 'http' @@ -154,7 +155,7 @@ def test_requests_module_200(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(s, 200) assert s.error == 0 assert s.span_type == 'http' @@ -166,7 +167,7 @@ def test_post_500(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'POST' - assert s.get_metric(http.STATUS_CODE) == 500 + assert_span_http_status_code(s, 500) assert s.error == 1 def test_non_existant_url(self): @@ -195,7 +196,7 @@ def test_500(self): assert len(spans) == 1 s = spans[0] assert s.get_tag(http.METHOD) == 'GET' - assert s.get_metric(http.STATUS_CODE) == 500 + assert_span_http_status_code(s, 500) assert s.error == 1 def test_default_service_name(self): @@ -368,7 +369,7 @@ def test_200_ot(self): assert ot_span.service == 'requests_svc' assert dd_span.get_tag(http.METHOD) == 'GET' - assert dd_span.get_metric(http.STATUS_CODE) == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.error == 0 assert dd_span.span_type == 'http' diff --git a/tests/contrib/tornado/test_executor_decorator.py b/tests/contrib/tornado/test_executor_decorator.py index bc47cf8dd2c..f46e3849e48 100644 --- a/tests/contrib/tornado/test_executor_decorator.py +++ b/tests/contrib/tornado/test_executor_decorator.py @@ -6,6 +6,7 @@ from tornado import version_info from .utils import TornadoTestCase +from ...utils import assert_span_http_status_code class TestTornadoExecutor(TornadoTestCase): @@ -30,7 +31,7 @@ def test_on_executor_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/executor_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error assert request_span.duration >= 0.05 @@ -61,7 +62,7 @@ def test_on_executor_submit(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorSubmitHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/executor_submit_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error assert request_span.duration >= 0.05 @@ -91,7 +92,7 @@ def test_on_executor_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/executor_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -128,7 +129,7 @@ def test_on_executor_custom_kwarg(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorCustomHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/executor_custom_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error assert request_span.duration >= 0.05 @@ -161,7 +162,7 @@ def test_on_executor_custom_args_kwarg(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorCustomArgsHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/executor_custom_args_handler/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'cannot combine positional and keyword args' == request_span.get_tag('error.msg') diff --git a/tests/contrib/tornado/test_tornado_template.py b/tests/contrib/tornado/test_tornado_template.py index 6cc6e5e96c0..a496c9145ff 100644 --- a/tests/contrib/tornado/test_tornado_template.py +++ b/tests/contrib/tornado/test_tornado_template.py @@ -3,6 +3,7 @@ import pytest from .utils import TornadoTestCase +from ...utils import assert_span_http_status_code from ddtrace.ext import http @@ -28,7 +29,7 @@ def test_template_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplateHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/template/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -75,7 +76,7 @@ def test_template_partials(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplatePartialHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/template_partial/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -130,7 +131,7 @@ def test_template_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.TemplateExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/template_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'ModuleThatDoesNotExist' in request_span.get_tag('error.msg') diff --git a/tests/contrib/tornado/test_tornado_web.py b/tests/contrib/tornado/test_tornado_web.py index ddcff3201c8..e710da71039 100644 --- a/tests/contrib/tornado/test_tornado_web.py +++ b/tests/contrib/tornado/test_tornado_web.py @@ -8,6 +8,7 @@ import tornado from tests.opentracer.utils import init_tracer +from ...utils import assert_span_http_status_code class TestTornadoWeb(TornadoTestCase): @@ -33,7 +34,7 @@ def test_success_handler(self, query_string=''): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/success/') == request_span.get_tag(http.URL) if config.tornado.trace_query_string: assert query_string == request_span.get_tag(http.QUERY_STRING) @@ -63,7 +64,7 @@ def test_nested_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/nested/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span @@ -90,7 +91,7 @@ def test_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -111,7 +112,7 @@ def test_http_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.HTTPExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 501 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 501) assert self.get_url('/http_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'HTTP 501: Not Implemented (unavailable)' == request_span.get_tag('error.msg') @@ -132,7 +133,7 @@ def test_http_exception_500_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.HTTPException500Handler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/http_exception_500/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'HTTP 500: Server Error (server error)' == request_span.get_tag('error.msg') @@ -153,7 +154,7 @@ def test_sync_success_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncSuccessHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/sync_success/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -172,7 +173,7 @@ def test_sync_exception_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncExceptionHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/sync_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -193,7 +194,7 @@ def test_404_handler(self): assert 'web' == request_span.span_type assert 'tornado.web.ErrorHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 404 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 404) assert self.get_url('/does_not_exist/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -214,7 +215,7 @@ def test_redirect_handler(self): assert 'web' == redirect_span.span_type assert 'tornado.web.RedirectHandler' == redirect_span.resource assert 'GET' == redirect_span.get_tag('http.method') - assert 301 == redirect_span.get_metric('http.status_code') + assert_span_http_status_code(redirect_span, 301) assert self.get_url('/redirect/') == redirect_span.get_tag(http.URL) assert 0 == redirect_span.error @@ -224,7 +225,7 @@ def test_redirect_handler(self): assert 'web' == success_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == success_span.resource assert 'GET' == success_span.get_tag('http.method') - assert 200 == success_span.get_metric('http.status_code') + assert_span_http_status_code(success_span, 200) assert self.get_url('/success/') == success_span.get_tag(http.URL) assert 0 == success_span.error @@ -244,7 +245,7 @@ def test_static_handler(self): assert 'web' == request_span.span_type assert 'tornado.web.StaticFileHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/statics/empty.txt') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -266,7 +267,7 @@ def test_propagation(self): # simple sanity check on the span assert 'tornado.request' == request_span.name - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/success/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -306,7 +307,7 @@ def test_success_handler_ot(self): assert 'web' == dd_span.span_type assert 'tests.contrib.tornado.web.app.SuccessHandler' == dd_span.resource assert 'GET' == dd_span.get_tag('http.method') - assert 200 == dd_span.get_metric('http.status_code') + assert_span_http_status_code(dd_span, 200) assert self.get_url('/success/') == dd_span.get_tag(http.URL) assert 0 == dd_span.error @@ -448,7 +449,7 @@ def test_no_propagation(self): # simple sanity check on the span assert 'tornado.request' == request_span.name - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/success/') == request_span.get_tag(http.URL) assert 0 == request_span.error @@ -485,6 +486,6 @@ def test_custom_default_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.CustomDefaultHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 400 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 400) assert self.get_url('/custom_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error diff --git a/tests/contrib/tornado/test_wrap_decorator.py b/tests/contrib/tornado/test_wrap_decorator.py index 9afd854e921..1c038dfe87a 100644 --- a/tests/contrib/tornado/test_wrap_decorator.py +++ b/tests/contrib/tornado/test_wrap_decorator.py @@ -1,6 +1,7 @@ from ddtrace.ext import http from .utils import TornadoTestCase +from ...utils import assert_span_http_status_code class TestTornadoWebWrapper(TornadoTestCase): @@ -21,7 +22,7 @@ def test_nested_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/nested_wrap/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span @@ -47,7 +48,7 @@ def test_nested_exception_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.NestedExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/nested_exception_wrap/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -77,7 +78,7 @@ def test_sync_nested_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncNestedWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/sync_nested_wrap/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span @@ -103,7 +104,7 @@ def test_sync_nested_exception_wrap_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.SyncNestedExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/sync_nested_exception_wrap/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') @@ -133,7 +134,7 @@ def test_nested_wrap_executor_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 200 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 200) assert self.get_url('/executor_wrap_handler/') == request_span.get_tag(http.URL) assert 0 == request_span.error # check nested span in the executor @@ -160,7 +161,7 @@ def test_nested_exception_wrap_executor_handler(self): assert 'web' == request_span.span_type assert 'tests.contrib.tornado.web.app.ExecutorExceptionWrapHandler' == request_span.resource assert 'GET' == request_span.get_tag('http.method') - assert 500 == request_span.get_metric('http.status_code') + assert_span_http_status_code(request_span, 500) assert self.get_url('/executor_wrap_exception/') == request_span.get_tag(http.URL) assert 1 == request_span.error assert 'Ouch!' == request_span.get_tag('error.msg') diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 1e8c63af8a2..526a52a3c0a 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,15 @@ import contextlib import os +from ddtrace.ext import http + + +def assert_span_http_status_code(span, code): + """Assert on the span's 'http.status_code' tag""" + tag = span.get_tag(http.STATUS_CODE) + code = str(code) + assert tag == code, "%r != %r" % (tag, code) + @contextlib.contextmanager def override_env(env): From 30c4a3f75c216edd02e048119016e2951bd5ba51 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 15 Jan 2020 15:09:34 -0500 Subject: [PATCH 40/81] fix flake8 --- tests/contrib/aiobotocore/test.py | 1 - tests/contrib/botocore/test.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/contrib/aiobotocore/test.py b/tests/contrib/aiobotocore/test.py index ebc4a0e1b20..700d593c8ba 100644 --- a/tests/contrib/aiobotocore/test.py +++ b/tests/contrib/aiobotocore/test.py @@ -3,7 +3,6 @@ from ddtrace.contrib.aiobotocore.patch import patch, unpatch from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY -from ddtrace.ext import http from ddtrace.compat import stringify from .utils import aiobotocore_client diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index 02bd75a142d..e99aa8ce8b1 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -6,7 +6,6 @@ from ddtrace import Pin from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY from ddtrace.contrib.botocore.patch import patch, unpatch -from ddtrace.ext import http from ddtrace.compat import stringify # testing From a923768fc8bfad724cd916d4d4a6eca25b38c580 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 15 Jan 2020 15:16:05 -0500 Subject: [PATCH 41/81] update missed assertions --- tests/contrib/pylons/test_pylons.py | 8 ++++---- tests/contrib/pyramid/utils.py | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/contrib/pylons/test_pylons.py b/tests/contrib/pylons/test_pylons.py index d041c57368e..3bf903e01a0 100644 --- a/tests/contrib/pylons/test_pylons.py +++ b/tests/contrib/pylons/test_pylons.py @@ -174,7 +174,7 @@ def test_success_200(self, query_string=''): assert span.service == 'web' assert span.resource == 'root.index' - assert span.metrics.get(http.STATUS_CODE) == 200 + assert_span_http_status_code(span, 200) if config.pylons.trace_query_string: assert span.meta.get(http.QUERY_STRING) == query_string else: @@ -264,7 +264,7 @@ def test_template_render(self): assert request.service == 'web' assert request.resource == 'root.render' - assert request.metrics.get(http.STATUS_CODE) == 200 + assert_span_http_status_code(request, 200) assert request.error == 0 assert template.service == 'web' @@ -284,7 +284,7 @@ def test_template_render_exception(self): assert request.service == 'web' assert request.resource == 'root.render_exception' - assert request.metrics.get(http.STATUS_CODE) == 500 + assert_span_http_status_code(request, 500) assert request.error == 1 assert template.service == 'web' @@ -424,6 +424,6 @@ def test_success_200_ot(self): assert dd_span.service == 'web' assert dd_span.resource == 'root.index' - assert dd_span.metrics.get(http.STATUS_CODE) == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.meta.get(http.URL) == 'http://localhost:80/' assert dd_span.error == 0 diff --git a/tests/contrib/pyramid/utils.py b/tests/contrib/pyramid/utils.py index a080a2d3300..f3c97e1f546 100644 --- a/tests/contrib/pyramid/utils.py +++ b/tests/contrib/pyramid/utils.py @@ -14,6 +14,7 @@ from ...opentracer.utils import init_tracer from ...base import BaseTracerTestCase +from ...utils import assert_span_http_status_code class PyramidBase(BaseTracerTestCase): @@ -65,7 +66,7 @@ def test_200(self, query_string=''): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.URL) == 'http://localhost/' if config.pyramid.trace_query_string: assert s.meta.get(http.QUERY_STRING) == query_string @@ -154,7 +155,7 @@ def test_404(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 404 + assert_span_http_status_code(s, 404) assert s.meta.get(http.URL) == 'http://localhost/404' def test_302(self): @@ -169,7 +170,7 @@ def test_302(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 302 + assert_span_http_status_code(s, 302) assert s.meta.get(http.URL) == 'http://localhost/redirect' def test_204(self): @@ -184,7 +185,7 @@ def test_204(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 204 + assert_span_http_status_code(s, 204) assert s.meta.get(http.URL) == 'http://localhost/nocontent' def test_exception(self): @@ -202,7 +203,7 @@ def test_exception(self): assert s.error == 1 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 500 + assert_span_http_status_code(s, 500) assert s.meta.get(http.URL) == 'http://localhost/exception' assert s.meta.get('pyramid.route.name') == 'exception' @@ -218,7 +219,7 @@ def test_500(self): assert s.error == 1 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 500 + assert_span_http_status_code(s, 500) assert s.meta.get(http.URL) == 'http://localhost/error' assert s.meta.get('pyramid.route.name') == 'error' assert type(s.error) == int @@ -238,7 +239,7 @@ def test_json(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.URL) == 'http://localhost/json' assert s.meta.get('pyramid.route.name') == 'json' @@ -262,7 +263,7 @@ def test_renderer(self): assert s.error == 0 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 200 + assert_span_http_status_code(s, 200) assert s.meta.get(http.URL) == 'http://localhost/renderer' assert s.meta.get('pyramid.route.name') == 'renderer' @@ -284,7 +285,7 @@ def test_http_exception_response(self): assert s.error == 1 assert s.span_type == 'web' assert s.meta.get('http.method') == 'GET' - assert s.metrics.get('http.status_code') == 404 + assert_span_http_status_code(s, 404) assert s.meta.get(http.URL) == 'http://localhost/404/raise_exception' def test_insert_tween_if_needed_already_set(self): @@ -356,6 +357,6 @@ def test_200_ot(self): assert dd_span.error == 0 assert dd_span.span_type == 'web' assert dd_span.meta.get('http.method') == 'GET' - assert dd_span.metrics.get('http.status_code') == 200 + assert_span_http_status_code(dd_span, 200) assert dd_span.meta.get(http.URL) == 'http://localhost/' assert dd_span.meta.get('pyramid.route.name') == 'index' From 11eea438bf9a69d27925d9ada9424f01f0c47891 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 15 Jan 2020 15:19:44 -0500 Subject: [PATCH 42/81] fix flask test --- tests/contrib/flask_autopatch/test_flask_autopatch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/contrib/flask_autopatch/test_flask_autopatch.py b/tests/contrib/flask_autopatch/test_flask_autopatch.py index 2f433d27e22..1ce50499b93 100644 --- a/tests/contrib/flask_autopatch/test_flask_autopatch.py +++ b/tests/contrib/flask_autopatch/test_flask_autopatch.py @@ -83,7 +83,8 @@ def index(): # Request tags self.assertEqual( - set(['flask.version', 'http.url', 'http.method', 'flask.endpoint', 'flask.url_rule']), + set(['flask.version', 'http.url', 'http.method', 'http.status_code', + 'flask.endpoint', 'flask.url_rule']), set(req_span.meta.keys()), ) self.assertEqual(req_span.get_tag('flask.endpoint'), 'index') From b4aaa29dae75a3c4db748ef48a87163c5cd10b07 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jan 2020 09:28:35 -0500 Subject: [PATCH 43/81] Prefer random.getrandbits on Python 3+, fall back to urandom implementation on 2 (#1183) --- ddtrace/span.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ddtrace/span.py b/ddtrace/span.py index 047082e18de..9c9c2e1fb1e 100644 --- a/ddtrace/span.py +++ b/ddtrace/span.py @@ -12,6 +12,12 @@ log = get_logger(__name__) +if sys.version_info.major < 3: + _getrandbits = random.SystemRandom().getrandbits +else: + _getrandbits = random.getrandbits + + class Span(object): __slots__ = [ @@ -383,9 +389,6 @@ def __repr__(self): ) -_SystemRandom = random.SystemRandom() - - def _new_id(): """Generate a random trace_id or span_id""" - return _SystemRandom.getrandbits(64) + return _getrandbits(64) From 85b941de3dea5944c2049d64df2997842630cfbc Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 3 Feb 2020 08:19:37 -0500 Subject: [PATCH 44/81] tests: fix botocore tests on py3.4 (#1189) --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 6ce88549dda..68ae43010d2 100644 --- a/tox.ini +++ b/tox.ini @@ -188,6 +188,7 @@ deps = boto: moto<1.0 botocore: botocore py34-botocore: PyYAML<5.3 + py34-botocore: jsonpatch<1.25 botocore: moto>=1.0,<2 bottle11: bottle>=0.11,<0.12 bottle12: bottle>=0.12,<0.13 From db35ad1de63d70390cc50b923ede0fcb47bc1ac5 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 3 Feb 2020 08:20:21 -0500 Subject: [PATCH 45/81] black: ddtrace/bootstrap (#1187) --- ddtrace/bootstrap/sitecustomize.py | 83 ++++++++++++++++-------------- pyproject.toml | 1 - 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/ddtrace/bootstrap/sitecustomize.py b/ddtrace/bootstrap/sitecustomize.py index 10b39dd8982..cd3fecc368c 100644 --- a/ddtrace/bootstrap/sitecustomize.py +++ b/ddtrace/bootstrap/sitecustomize.py @@ -12,17 +12,18 @@ from ddtrace.internal.logger import get_logger from ddtrace import constants -logs_injection = asbool(get_env('logs', 'injection')) -DD_LOG_FORMAT = '%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] {}- %(message)s'.format( - '[dd.trace_id=%(dd.trace_id)s dd.span_id=%(dd.span_id)s] ' if logs_injection else '' +logs_injection = asbool(get_env("logs", "injection")) +DD_LOG_FORMAT = "%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] {}- %(message)s".format( + "[dd.trace_id=%(dd.trace_id)s dd.span_id=%(dd.span_id)s] " if logs_injection else "" ) if logs_injection: # immediately patch logging if trace id injected from ddtrace import patch + patch(logging=True) -debug = os.environ.get('DATADOG_TRACE_DEBUG') +debug = os.environ.get("DATADOG_TRACE_DEBUG") # Set here a default logging format for basicConfig @@ -30,7 +31,7 @@ # change the formatter since it applies the formatter to the root handler only # upon initializing it the first time. # See https://github.com/python/cpython/blob/112e4afd582515fcdcc0cde5012a4866e5cfda12/Lib/logging/__init__.py#L1550 -if debug and debug.lower() == 'true': +if debug and debug.lower() == "true": logging.basicConfig(level=logging.DEBUG, format=DD_LOG_FORMAT) else: logging.basicConfig(format=DD_LOG_FORMAT) @@ -38,38 +39,38 @@ log = get_logger(__name__) EXTRA_PATCHED_MODULES = { - 'bottle': True, - 'django': True, - 'falcon': True, - 'flask': True, - 'pylons': True, - 'pyramid': True, + "bottle": True, + "django": True, + "falcon": True, + "flask": True, + "pylons": True, + "pyramid": True, } def update_patched_modules(): - modules_to_patch = os.environ.get('DATADOG_PATCH_MODULES') + modules_to_patch = os.environ.get("DATADOG_PATCH_MODULES") if not modules_to_patch: return - for patch in modules_to_patch.split(','): - if len(patch.split(':')) != 2: - log.debug('skipping malformed patch instruction') + for patch in modules_to_patch.split(","): + if len(patch.split(":")) != 2: + log.debug("skipping malformed patch instruction") continue - module, should_patch = patch.split(':') - if should_patch.lower() not in ['true', 'false']: - log.debug('skipping malformed patch instruction for %s', module) + module, should_patch = patch.split(":") + if should_patch.lower() not in ["true", "false"]: + log.debug("skipping malformed patch instruction for %s", module) continue - EXTRA_PATCHED_MODULES.update({module: should_patch.lower() == 'true'}) + EXTRA_PATCHED_MODULES.update({module: should_patch.lower() == "true"}) def add_global_tags(tracer): tags = {} - for tag in os.environ.get('DD_TRACE_GLOBAL_TAGS', '').split(','): - tag_name, _, tag_value = tag.partition(':') + for tag in os.environ.get("DD_TRACE_GLOBAL_TAGS", "").split(","): + tag_name, _, tag_value = tag.partition(":") if not tag_name or not tag_value: - log.debug('skipping malformed tracer tag') + log.debug("skipping malformed tracer tag") continue tags[tag_name] = tag_value @@ -78,45 +79,47 @@ def add_global_tags(tracer): try: from ddtrace import tracer + patch = True # Respect DATADOG_* environment variables in global tracer configuration # TODO: these variables are deprecated; use utils method and update our documentation # correct prefix should be DD_* - enabled = os.environ.get('DATADOG_TRACE_ENABLED') - hostname = os.environ.get('DD_AGENT_HOST', os.environ.get('DATADOG_TRACE_AGENT_HOSTNAME')) - port = os.environ.get('DATADOG_TRACE_AGENT_PORT') - priority_sampling = os.environ.get('DATADOG_PRIORITY_SAMPLING') + enabled = os.environ.get("DATADOG_TRACE_ENABLED") + hostname = os.environ.get("DD_AGENT_HOST", os.environ.get("DATADOG_TRACE_AGENT_HOSTNAME")) + port = os.environ.get("DATADOG_TRACE_AGENT_PORT") + priority_sampling = os.environ.get("DATADOG_PRIORITY_SAMPLING") opts = {} - if enabled and enabled.lower() == 'false': - opts['enabled'] = False + if enabled and enabled.lower() == "false": + opts["enabled"] = False patch = False if hostname: - opts['hostname'] = hostname + opts["hostname"] = hostname if port: - opts['port'] = int(port) + opts["port"] = int(port) if priority_sampling: - opts['priority_sampling'] = asbool(priority_sampling) + opts["priority_sampling"] = asbool(priority_sampling) - opts['collect_metrics'] = asbool(get_env('runtime_metrics', 'enabled')) + opts["collect_metrics"] = asbool(get_env("runtime_metrics", "enabled")) if opts: tracer.configure(**opts) if logs_injection: - EXTRA_PATCHED_MODULES.update({'logging': True}) + EXTRA_PATCHED_MODULES.update({"logging": True}) if patch: update_patched_modules() from ddtrace import patch_all + patch_all(**EXTRA_PATCHED_MODULES) - if 'DATADOG_ENV' in os.environ: - tracer.set_tags({constants.ENV_KEY: os.environ['DATADOG_ENV']}) + if "DATADOG_ENV" in os.environ: + tracer.set_tags({constants.ENV_KEY: os.environ["DATADOG_ENV"]}) - if 'DD_TRACE_GLOBAL_TAGS' in os.environ: + if "DD_TRACE_GLOBAL_TAGS" in os.environ: add_global_tags(tracer) # Ensure sitecustomize.py is properly called if available in application directories: @@ -130,13 +133,13 @@ def add_global_tags(tracer): path.remove(bootstrap_dir) try: - (f, path, description) = imp.find_module('sitecustomize', path) + (f, path, description) = imp.find_module("sitecustomize", path) except ImportError: pass else: # `sitecustomize.py` found, load it - log.debug('sitecustomize from user found in: %s', path) - imp.load_module('sitecustomize', f, path, description) + log.debug("sitecustomize from user found in: %s", path) + imp.load_module("sitecustomize", f, path, description) # Loading status used in tests to detect if the `sitecustomize` has been # properly loaded without exceptions. This must be the last action in the module @@ -144,4 +147,4 @@ def add_global_tags(tracer): loaded = True except Exception: loaded = False - log.warning('error configuring Datadog tracing', exc_info=True) + log.warning("error configuring Datadog tracing", exc_info=True) diff --git a/pyproject.toml b/pyproject.toml index 707aa964b0e..793f5463922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ exclude = ''' | dist/ | ddtrace/( [^/]+\.py - | bootstrap/ | commands/ | contrib/ | ext/ From bda7a6cb57e3ca53ed1fee89263c640d38c26cc5 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 3 Feb 2020 08:20:57 -0500 Subject: [PATCH 46/81] black: ddtrace/utils/ (#1188) --- ddtrace/utils/attrdict.py | 1 + ddtrace/utils/config.py | 2 +- ddtrace/utils/deprecation.py | 11 ++++++----- ddtrace/utils/formats.py | 26 ++++++++++++-------------- ddtrace/utils/hook.py | 2 ++ ddtrace/utils/importlib.py | 9 +++++---- ddtrace/utils/time.py | 7 +++---- ddtrace/utils/wrappers.py | 15 ++++++++------- pyproject.toml | 1 - 9 files changed, 38 insertions(+), 36 deletions(-) diff --git a/ddtrace/utils/attrdict.py b/ddtrace/utils/attrdict.py index 2ed2689642f..e153e2968c6 100644 --- a/ddtrace/utils/attrdict.py +++ b/ddtrace/utils/attrdict.py @@ -16,6 +16,7 @@ class AttrDict(dict): data = AttrDict(dict(key='value')) print(data.key) """ + def __getattr__(self, key): if key in self: return self[key] diff --git a/ddtrace/utils/config.py b/ddtrace/utils/config.py index 02b6333d191..43221202635 100644 --- a/ddtrace/utils/config.py +++ b/ddtrace/utils/config.py @@ -4,7 +4,7 @@ def get_application_name(): """Attempts to find the application name using system arguments.""" - if hasattr(sys, 'argv') and sys.argv[0]: + if hasattr(sys, "argv") and sys.argv[0]: app_name = os.path.basename(sys.argv[0]) else: app_name = None diff --git a/ddtrace/utils/deprecation.py b/ddtrace/utils/deprecation.py index ea852cc76a4..c2102500cc2 100644 --- a/ddtrace/utils/deprecation.py +++ b/ddtrace/utils/deprecation.py @@ -14,9 +14,7 @@ def format_message(name, message, version): 'fn' is deprecated and will be remove in future versions (1.0). """ return "'{}' is deprecated and will be remove in future versions{}. {}".format( - name, - ' ({})'.format(version) if version else '', - message, + name, " ({})".format(version) if version else "", message, ) @@ -25,7 +23,7 @@ def warn(message, stacklevel=2): warnings.warn(message, RemovedInDDTrace10Warning, stacklevel=stacklevel) -def deprecation(name='', message='', version=None): +def deprecation(name="", message="", version=None): """Function to report a ``DeprecationWarning``. Bear in mind that `DeprecationWarning` are ignored by default so they're not available in user logs. To show them, the application must be launched with a special flag: @@ -39,7 +37,7 @@ def deprecation(name='', message='', version=None): warn(msg, stacklevel=4) -def deprecated(message='', version=None): +def deprecated(message="", version=None): """Decorator function to report a ``DeprecationWarning``. Bear in mind that `DeprecationWarning` are ignored by default so they're not available in user logs. To show them, the application must be launched @@ -50,11 +48,14 @@ def deprecated(message='', version=None): This approach is used by most of the frameworks, including Django (ref: https://docs.djangoproject.com/en/2.0/howto/upgrade-version/#resolving-deprecation-warnings) """ + def decorator(func): @wraps(func) def wrapper(*args, **kwargs): msg = format_message(func.__name__, message, version) warn(msg, stacklevel=3) return func(*args, **kwargs) + return wrapper + return decorator diff --git a/ddtrace/utils/formats.py b/ddtrace/utils/formats.py index 5e065ece56c..7fe13dab553 100644 --- a/ddtrace/utils/formats.py +++ b/ddtrace/utils/formats.py @@ -15,18 +15,16 @@ def get_env(integration, variable, default=None): * return `default` otherwise """ - key = '{}_{}'.format(integration, variable).upper() - legacy_env = 'DATADOG_{}'.format(key) - env = 'DD_{}'.format(key) + key = "{}_{}".format(integration, variable).upper() + legacy_env = "DATADOG_{}".format(key) + env = "DD_{}".format(key) value = os.getenv(env) legacy = os.getenv(legacy_env) if legacy: # Deprecation: `DATADOG_` variables are deprecated deprecation( - name='DATADOG_', - message='Use `DD_` prefix instead', - version='1.0.0', + name="DATADOG_", message="Use `DD_` prefix instead", version="1.0.0", ) value = value or legacy @@ -47,7 +45,7 @@ def deep_getattr(obj, attr_string, default=None): >>> deep_getattr(cass, 'i.dont.exist', default='default') 'default' """ - attrs = attr_string.split('.') + attrs = attr_string.split(".") for attr in attrs: try: obj = getattr(obj, attr) @@ -68,17 +66,17 @@ def asbool(value): if isinstance(value, bool): return value - return value.lower() in ('true', '1') + return value.lower() in ("true", "1") -def flatten_dict(d, sep='.', prefix=''): +def flatten_dict(d, sep=".", prefix=""): """ Returns a normalized dict of depth 1 with keys in order of embedding """ # adapted from https://stackoverflow.com/a/19647596 - return { - prefix + sep + k if prefix else k: v - for kk, vv in d.items() - for k, v in flatten_dict(vv, sep, kk).items() - } if isinstance(d, dict) else {prefix: d} + return ( + {prefix + sep + k if prefix else k: v for kk, vv in d.items() for k, v in flatten_dict(vv, sep, kk).items()} + if isinstance(d, dict) + else {prefix: d} + ) diff --git a/ddtrace/utils/hook.py b/ddtrace/utils/hook.py index ef143ac629e..af1e776ede2 100644 --- a/ddtrace/utils/hook.py +++ b/ddtrace/utils/hook.py @@ -97,6 +97,7 @@ class _ImportHookLoader(object): interest. When a module of interest is imported, then any post import hooks which are registered will be invoked. """ + def load_module(self, fullname): module = sys.modules[fullname] notify_module_loaded(module) @@ -151,6 +152,7 @@ def find_module(self, fullname, path=None): # post import hooks. try: import importlib.util + loader = importlib.util.find_spec(fullname).loader except (ImportError, AttributeError): loader = importlib.find_loader(fullname, path) diff --git a/ddtrace/utils/importlib.py b/ddtrace/utils/importlib.py index 29c1c124793..107b15ff6d6 100644 --- a/ddtrace/utils/importlib.py +++ b/ddtrace/utils/importlib.py @@ -5,6 +5,7 @@ class require_modules(object): """Context manager to check the availability of required modules.""" + def __init__(self, modules): self._missing_modules = [] for module in modules: @@ -22,11 +23,11 @@ def __exit__(self, exc_type, exc_value, traceback): def func_name(f): """Return a human readable version of the function's name.""" - if hasattr(f, '__module__'): - return '%s.%s' % (f.__module__, getattr(f, '__name__', f.__class__.__name__)) - return getattr(f, '__name__', f.__class__.__name__) + if hasattr(f, "__module__"): + return "%s.%s" % (f.__module__, getattr(f, "__name__", f.__class__.__name__)) + return getattr(f, "__name__", f.__class__.__name__) def module_name(instance): """Return the instance module name.""" - return instance.__class__.__module__.split('.')[0] + return instance.__class__.__module__.split(".")[0] diff --git a/ddtrace/utils/time.py b/ddtrace/utils/time.py index 014fee9ed8f..b563b65d411 100644 --- a/ddtrace/utils/time.py +++ b/ddtrace/utils/time.py @@ -15,6 +15,7 @@ class StopWatch(object): .. _monotonic: https://pypi.python.org/pypi/monotonic/ """ + def __init__(self): self._started_at = None self._stopped_at = None @@ -32,8 +33,7 @@ def elapsed(self): """ # NOTE: datetime.timedelta does not support nanoseconds, so keep a float here if self._started_at is None: - raise RuntimeError('Can not get the elapsed time of a stopwatch' - ' if it has not been started/stopped') + raise RuntimeError("Can not get the elapsed time of a stopwatch" " if it has not been started/stopped") if self._stopped_at is None: now = monotonic.monotonic() else: @@ -52,7 +52,6 @@ def __exit__(self, tp, value, traceback): def stop(self): """Stops the watch.""" if self._started_at is None: - raise RuntimeError('Can not stop a stopwatch that has not been' - ' started') + raise RuntimeError("Can not stop a stopwatch that has not been" " started") self._stopped_at = monotonic.monotonic() return self diff --git a/ddtrace/utils/wrappers.py b/ddtrace/utils/wrappers.py index 8ac7d581b28..bfcf69a47e8 100644 --- a/ddtrace/utils/wrappers.py +++ b/ddtrace/utils/wrappers.py @@ -6,11 +6,11 @@ def unwrap(obj, attr): f = getattr(obj, attr, None) - if f and isinstance(f, wrapt.ObjectProxy) and hasattr(f, '__wrapped__'): + if f and isinstance(f, wrapt.ObjectProxy) and hasattr(f, "__wrapped__"): setattr(obj, attr, f.__wrapped__) -@deprecated('`wrapt` library is used instead', version='1.0.0') +@deprecated("`wrapt` library is used instead", version="1.0.0") def safe_patch(patchable, key, patch_func, service, meta, tracer): """ takes patch_func (signature: takes the orig_method that is wrapped in the monkey patch == UNBOUND + service and meta) and @@ -28,15 +28,16 @@ def safe_patch(patchable, key, patch_func, service, meta, tracer): But if it is, search for a '__dd_orig_{key}' method on the class, which is the original unpatched method we wish to trace. """ + def _get_original_method(thing, key): orig = None - if hasattr(thing, '_dogtraced'): + if hasattr(thing, "_dogtraced"): # Search for original method - orig = getattr(thing, '__dd_orig_{}'.format(key), None) + orig = getattr(thing, "__dd_orig_{}".format(key), None) else: orig = getattr(thing, key) # Set it for the next time we attempt to patch `thing` - setattr(thing, '__dd_orig_{}'.format(key), orig) + setattr(thing, "__dd_orig_{}".format(key), orig) return orig @@ -45,7 +46,7 @@ def _get_original_method(thing, key): if not orig: # Should never happen return - elif hasattr(patchable, '__class__'): + elif hasattr(patchable, "__class__"): orig = _get_original_method(patchable.__class__, key) if not orig: # Should never happen @@ -57,5 +58,5 @@ def _get_original_method(thing, key): if inspect.isclass(patchable) or inspect.ismodule(patchable): setattr(patchable, key, dest) - elif hasattr(patchable, '__class__'): + elif hasattr(patchable, "__class__"): setattr(patchable, key, dest.__get__(patchable, patchable.__class__)) diff --git a/pyproject.toml b/pyproject.toml index 793f5463922..0cb4bd95b3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ exclude = ''' | opentracer/ | propagation/ | settings/ - | utils/ | vendor/ ) | tests/ From 7125e740b0303d32dc4e2719c063f3505376094c Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 3 Feb 2020 08:22:16 -0500 Subject: [PATCH 47/81] black: ddtrace/internal/ (#1190) * black: ddtrace/internal/ * fix flake8 --- ddtrace/internal/context_manager.py | 11 +- ddtrace/internal/hostname.py | 1 + ddtrace/internal/logger.py | 13 +-- ddtrace/internal/rate_limiter.py | 32 +++--- ddtrace/internal/runtime/__init__.py | 6 +- ddtrace/internal/runtime/collector.py | 14 +-- ddtrace/internal/runtime/constants.py | 72 +++++-------- ddtrace/internal/runtime/container.py | 35 +++--- ddtrace/internal/runtime/metric_collectors.py | 23 ++-- ddtrace/internal/runtime/runtime_metrics.py | 20 +--- ddtrace/internal/runtime/tag_collectors.py | 14 +-- ddtrace/internal/writer.py | 100 +++++++++--------- pyproject.toml | 1 - 13 files changed, 153 insertions(+), 189 deletions(-) diff --git a/ddtrace/internal/context_manager.py b/ddtrace/internal/context_manager.py index 73b491285a2..b884ed567cf 100644 --- a/ddtrace/internal/context_manager.py +++ b/ddtrace/internal/context_manager.py @@ -9,7 +9,8 @@ try: from contextvars import ContextVar - _DD_CONTEXTVAR = ContextVar('datadog_contextvar', default=None) + + _DD_CONTEXTVAR = ContextVar("datadog_contextvar", default=None) CONTEXTVARS_IS_AVAILABLE = True except ImportError: CONTEXTVARS_IS_AVAILABLE = False @@ -43,6 +44,7 @@ class ThreadLocalContext(BaseContextManager): is required to prevent multiple threads sharing the same ``Context`` in different executions. """ + def __init__(self, reset=True): # always initialize a new thread-local context holder super(ThreadLocalContext, self).__init__(reset=True) @@ -54,14 +56,14 @@ def _has_active_context(self): :returns: Whether an active context exists :rtype: bool """ - ctx = getattr(self._locals, 'context', None) + ctx = getattr(self._locals, "context", None) return ctx is not None def set(self, ctx): - setattr(self._locals, 'context', ctx) + setattr(self._locals, "context", ctx) def get(self): - ctx = getattr(self._locals, 'context', None) + ctx = getattr(self._locals, "context", None) if not ctx: # create a new Context if it's not available ctx = Context() @@ -79,6 +81,7 @@ class ContextVarContextManager(BaseContextManager): 3.7 and above to manage different ``Context`` objects for each thread and async task. """ + def _has_active_context(self): ctx = _DD_CONTEXTVAR.get() return ctx is not None diff --git a/ddtrace/internal/hostname.py b/ddtrace/internal/hostname.py index f5ce2e97298..5e3c941f0b6 100644 --- a/ddtrace/internal/hostname.py +++ b/ddtrace/internal/hostname.py @@ -12,6 +12,7 @@ def wrapper(): _hostname = func() return _hostname + return wrapper diff --git a/ddtrace/internal/logger.py b/ddtrace/internal/logger.py index 6b993053168..04f8ef83f18 100644 --- a/ddtrace/internal/logger.py +++ b/ddtrace/internal/logger.py @@ -43,7 +43,7 @@ def get_logger(name): # without this then we cannot take advantage of the root loggers handlers # https://github.com/python/cpython/blob/7c7839329c2c66d051960ab1df096aed1cc9343e/Lib/logging/__init__.py#L1272-L1294 # noqa # DEV: `_fixupParents` has been around for awhile, but add the `hasattr` guard... just in case. - if hasattr(manager, '_fixupParents'): + if hasattr(manager, "_fixupParents"): manager._fixupParents(logger) # Return out logger @@ -57,10 +57,11 @@ class DDLogger(logging.Logger): This logger class is used to rate limit the output of log messages from within the ``ddtrace`` package. """ - __slots__ = ('buckets', 'rate_limit') + + __slots__ = ("buckets", "rate_limit") # Named tuple used for keeping track of a log lines current time bucket and the number of log lines skipped - LoggingBucket = collections.namedtuple('LoggingBucket', ('bucket', 'skipped')) + LoggingBucket = collections.namedtuple("LoggingBucket", ("bucket", "skipped")) def __init__(self, *args, **kwargs): """Constructor for ``DDLogger``""" @@ -72,7 +73,7 @@ def __init__(self, *args, **kwargs): # Allow 1 log record per name/level/pathname/lineno every 60 seconds by default # Allow configuring via `DD_LOGGING_RATE_LIMIT` # DEV: `DD_LOGGING_RATE_LIMIT=0` means to disable all rate limiting - self.rate_limit = int(get_env('logging', 'rate_limit', default=60)) + self.rate_limit = int(get_env("logging", "rate_limit", default=60)) def handle(self, record): """ @@ -110,8 +111,8 @@ def handle(self, record): if logging_bucket.bucket != current_bucket: # Append count of skipped messages if we have skipped some since our last logging if logging_bucket.skipped: - record.msg = '{}, %s additional messages skipped'.format(record.msg) - record.args = record.args + (logging_bucket.skipped, ) + record.msg = "{}, %s additional messages skipped".format(record.msg) + record.args = record.args + (logging_bucket.skipped,) # Reset our bucket self.buckets[key] = DDLogger.LoggingBucket(current_bucket, 0) diff --git a/ddtrace/internal/rate_limiter.py b/ddtrace/internal/rate_limiter.py index 173a78753b7..8c8ac68eab9 100644 --- a/ddtrace/internal/rate_limiter.py +++ b/ddtrace/internal/rate_limiter.py @@ -8,16 +8,17 @@ class RateLimiter(object): """ A token bucket rate limiter implementation """ + __slots__ = ( - '_lock', - 'current_window', - 'last_update', - 'max_tokens', - 'prev_window_rate', - 'rate_limit', - 'tokens', - 'tokens_allowed', - 'tokens_total', + "_lock", + "current_window", + "last_update", + "max_tokens", + "prev_window_rate", + "rate_limit", + "tokens", + "tokens_allowed", + "tokens_total", ) def __init__(self, rate_limit): @@ -108,10 +109,7 @@ def _replenish(self): self.last_update = now # Update the number of available tokens, but ensure we do not exceed the max - self.tokens = min( - self.max_tokens, - self.tokens + (elapsed * self.rate_limit), - ) + self.tokens = min(self.max_tokens, self.tokens + (elapsed * self.rate_limit),) def _current_window_rate(self): # No tokens have been seen, effectively 100% sample rate @@ -137,12 +135,8 @@ def effective_rate(self): return (self._current_window_rate() + self.prev_window_rate) / 2.0 def __repr__(self): - return '{}(rate_limit={!r}, tokens={!r}, last_update={!r}, effective_rate={!r})'.format( - self.__class__.__name__, - self.rate_limit, - self.tokens, - self.last_update, - self.effective_rate, + return "{}(rate_limit={!r}, tokens={!r}, last_update={!r}, effective_rate={!r})".format( + self.__class__.__name__, self.rate_limit, self.tokens, self.last_update, self.effective_rate, ) __str__ = __repr__ diff --git a/ddtrace/internal/runtime/__init__.py b/ddtrace/internal/runtime/__init__.py index 34b4b34b345..30ba177f21c 100644 --- a/ddtrace/internal/runtime/__init__.py +++ b/ddtrace/internal/runtime/__init__.py @@ -6,7 +6,7 @@ __all__ = [ - 'RuntimeTags', - 'RuntimeMetrics', - 'RuntimeWorker', + "RuntimeTags", + "RuntimeMetrics", + "RuntimeWorker", ] diff --git a/ddtrace/internal/runtime/collector.py b/ddtrace/internal/runtime/collector.py index b94cf25c5fe..c8b8ac0bf82 100644 --- a/ddtrace/internal/runtime/collector.py +++ b/ddtrace/internal/runtime/collector.py @@ -16,6 +16,7 @@ class ValueCollector(object): Functionality is provided for requiring and importing modules which may or may not be installed. """ + enabled = True periodic = False required_modules = [] @@ -67,19 +68,12 @@ def collect(self, keys=None): # filter values for keys if len(keys) > 0 and isinstance(self.value, list): - self.value = [ - (k, v) - for (k, v) in self.value - if k in keys - ] + self.value = [(k, v) for (k, v) in self.value if k in keys] self.value_loaded = True return self.value def __repr__(self): - return '<{}(enabled={},periodic={},required_modules={})>'.format( - self.__class__.__name__, - self.enabled, - self.periodic, - self.required_modules, + return "<{}(enabled={},periodic={},required_modules={})>".format( + self.__class__.__name__, self.enabled, self.periodic, self.required_modules, ) diff --git a/ddtrace/internal/runtime/constants.py b/ddtrace/internal/runtime/constants.py index 23fa30ccb3b..d1a627a9c22 100644 --- a/ddtrace/internal/runtime/constants.py +++ b/ddtrace/internal/runtime/constants.py @@ -1,50 +1,32 @@ -GC_COUNT_GEN0 = 'runtime.python.gc.count.gen0' -GC_COUNT_GEN1 = 'runtime.python.gc.count.gen1' -GC_COUNT_GEN2 = 'runtime.python.gc.count.gen2' - -THREAD_COUNT = 'runtime.python.thread_count' -MEM_RSS = 'runtime.python.mem.rss' -CPU_TIME_SYS = 'runtime.python.cpu.time.sys' -CPU_TIME_USER = 'runtime.python.cpu.time.user' -CPU_PERCENT = 'runtime.python.cpu.percent' -CTX_SWITCH_VOLUNTARY = 'runtime.python.cpu.ctx_switch.voluntary' -CTX_SWITCH_INVOLUNTARY = 'runtime.python.cpu.ctx_switch.involuntary' - -GC_RUNTIME_METRICS = set([ - GC_COUNT_GEN0, - GC_COUNT_GEN1, - GC_COUNT_GEN2, -]) - -PSUTIL_RUNTIME_METRICS = set([ - THREAD_COUNT, - MEM_RSS, - CTX_SWITCH_VOLUNTARY, - CTX_SWITCH_INVOLUNTARY, - CPU_TIME_SYS, - CPU_TIME_USER, - CPU_PERCENT, -]) +GC_COUNT_GEN0 = "runtime.python.gc.count.gen0" +GC_COUNT_GEN1 = "runtime.python.gc.count.gen1" +GC_COUNT_GEN2 = "runtime.python.gc.count.gen2" + +THREAD_COUNT = "runtime.python.thread_count" +MEM_RSS = "runtime.python.mem.rss" +CPU_TIME_SYS = "runtime.python.cpu.time.sys" +CPU_TIME_USER = "runtime.python.cpu.time.user" +CPU_PERCENT = "runtime.python.cpu.percent" +CTX_SWITCH_VOLUNTARY = "runtime.python.cpu.ctx_switch.voluntary" +CTX_SWITCH_INVOLUNTARY = "runtime.python.cpu.ctx_switch.involuntary" + +GC_RUNTIME_METRICS = set([GC_COUNT_GEN0, GC_COUNT_GEN1, GC_COUNT_GEN2]) + +PSUTIL_RUNTIME_METRICS = set( + [THREAD_COUNT, MEM_RSS, CTX_SWITCH_VOLUNTARY, CTX_SWITCH_INVOLUNTARY, CPU_TIME_SYS, CPU_TIME_USER, CPU_PERCENT] +) DEFAULT_RUNTIME_METRICS = GC_RUNTIME_METRICS | PSUTIL_RUNTIME_METRICS -SERVICE = 'service' -ENV = 'env' -LANG_INTERPRETER = 'lang_interpreter' -LANG_VERSION = 'lang_version' -LANG = 'lang' -TRACER_VERSION = 'tracer_version' - -TRACER_TAGS = set([ - SERVICE, - ENV, -]) - -PLATFORM_TAGS = set([ - LANG_INTERPRETER, - LANG_VERSION, - LANG, - TRACER_VERSION, -]) +SERVICE = "service" +ENV = "env" +LANG_INTERPRETER = "lang_interpreter" +LANG_VERSION = "lang_version" +LANG = "lang" +TRACER_VERSION = "tracer_version" + +TRACER_TAGS = set([SERVICE, ENV]) + +PLATFORM_TAGS = set([LANG_INTERPRETER, LANG_VERSION, LANG, TRACER_VERSION]) DEFAULT_RUNTIME_TAGS = TRACER_TAGS | PLATFORM_TAGS diff --git a/ddtrace/internal/runtime/container.py b/ddtrace/internal/runtime/container.py index 87bf3664092..e13c07eeeea 100644 --- a/ddtrace/internal/runtime/container.py +++ b/ddtrace/internal/runtime/container.py @@ -9,14 +9,15 @@ class CGroupInfo(object): """ CGroup class for container information parsed from a group cgroup file """ - __slots__ = ('id', 'groups', 'path', 'container_id', 'controllers', 'pod_id') - UUID_SOURCE_PATTERN = r'[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}' - CONTAINER_SOURCE_PATTERN = r'[0-9a-f]{64}' + __slots__ = ("id", "groups", "path", "container_id", "controllers", "pod_id") - LINE_RE = re.compile(r'^(\d+):([^:]*):(.+)$') - POD_RE = re.compile(r'pod({0})(?:\.slice)?$'.format(UUID_SOURCE_PATTERN)) - CONTAINER_RE = re.compile(r'({0}|{1})(?:\.scope)?$'.format(UUID_SOURCE_PATTERN, CONTAINER_SOURCE_PATTERN)) + UUID_SOURCE_PATTERN = r"[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}" + CONTAINER_SOURCE_PATTERN = r"[0-9a-f]{64}" + + LINE_RE = re.compile(r"^(\d+):([^:]*):(.+)$") + POD_RE = re.compile(r"pod({0})(?:\.slice)?$".format(UUID_SOURCE_PATTERN)) + CONTAINER_RE = re.compile(r"({0}|{1})(?:\.scope)?$".format(UUID_SOURCE_PATTERN, CONTAINER_SOURCE_PATTERN)) def __init__(self, **kwargs): # Initialize all attributes in __slots__ to `None` @@ -48,12 +49,12 @@ def from_line(cls, line): info.id, info.groups, info.path = match.groups() # Parse the controllers from the groups - info.controllers = [c.strip() for c in info.groups.split(',') if c.strip()] + info.controllers = [c.strip() for c in info.groups.split(",") if c.strip()] # Break up the path to grab container_id and pod_id if available # e.g. /docker/ # e.g. /kubepods/test/pod/ - parts = [p for p in info.path.split('/')] + parts = [p for p in info.path.split("/")] # Grab the container id from the path if a valid id is present if len(parts): @@ -73,18 +74,12 @@ def __str__(self): return self.__repr__() def __repr__(self): - return '{}(id={!r}, groups={!r}, path={!r}, container_id={!r}, controllers={!r}, pod_id={!r})'.format( - self.__class__.__name__, - self.id, - self.groups, - self.path, - self.container_id, - self.controllers, - self.pod_id, + return "{}(id={!r}, groups={!r}, path={!r}, container_id={!r}, controllers={!r}, pod_id={!r})".format( + self.__class__.__name__, self.id, self.groups, self.path, self.container_id, self.controllers, self.pod_id, ) -def get_container_info(pid='self'): +def get_container_info(pid="self"): """ Helper to fetch the current container id, if we are running in a container @@ -98,13 +93,13 @@ def get_container_info(pid='self'): :rtype: :class:`CGroupInfo` | None """ try: - cgroup_file = '/proc/{0}/cgroup'.format(pid) - with open(cgroup_file, mode='r') as fp: + cgroup_file = "/proc/{0}/cgroup".format(pid) + with open(cgroup_file, mode="r") as fp: for line in fp: info = CGroupInfo.from_line(line) if info and info.container_id: return info except Exception: - log.debug('Failed to parse cgroup file for pid %r', pid, exc_info=True) + log.debug("Failed to parse cgroup file for pid %r", pid, exc_info=True) return None diff --git a/ddtrace/internal/runtime/metric_collectors.py b/ddtrace/internal/runtime/metric_collectors.py index e1fc9429955..eb140294b6d 100644 --- a/ddtrace/internal/runtime/metric_collectors.py +++ b/ddtrace/internal/runtime/metric_collectors.py @@ -25,10 +25,11 @@ class GCRuntimeMetricCollector(RuntimeMetricCollector): More information at https://docs.python.org/3/library/gc.html """ - required_modules = ['gc'] + + required_modules = ["gc"] def collect_fn(self, keys): - gc = self.modules.get('gc') + gc = self.modules.get("gc") counts = gc.get_count() metrics = [ @@ -47,16 +48,14 @@ class PSUtilRuntimeMetricCollector(RuntimeMetricCollector): See https://psutil.readthedocs.io/en/latest/#psutil.Process.oneshot for more information. """ - required_modules = ['psutil'] + + required_modules = ["psutil"] stored_value = dict( - CPU_TIME_SYS_TOTAL=0, - CPU_TIME_USER_TOTAL=0, - CTX_SWITCH_VOLUNTARY_TOTAL=0, - CTX_SWITCH_INVOLUNTARY_TOTAL=0, + CPU_TIME_SYS_TOTAL=0, CPU_TIME_USER_TOTAL=0, CTX_SWITCH_VOLUNTARY_TOTAL=0, CTX_SWITCH_INVOLUNTARY_TOTAL=0, ) def _on_modules_load(self): - self.proc = self.modules['psutil'].Process(os.getpid()) + self.proc = self.modules["psutil"].Process(os.getpid()) def collect_fn(self, keys): with self.proc.oneshot(): @@ -64,13 +63,13 @@ def collect_fn(self, keys): # TODO[tahir]: better abstraction for metrics based on last value cpu_time_sys_total = self.proc.cpu_times().system cpu_time_user_total = self.proc.cpu_times().user - cpu_time_sys = cpu_time_sys_total - self.stored_value['CPU_TIME_SYS_TOTAL'] - cpu_time_user = cpu_time_user_total - self.stored_value['CPU_TIME_USER_TOTAL'] + cpu_time_sys = cpu_time_sys_total - self.stored_value["CPU_TIME_SYS_TOTAL"] + cpu_time_user = cpu_time_user_total - self.stored_value["CPU_TIME_USER_TOTAL"] ctx_switch_voluntary_total = self.proc.num_ctx_switches().voluntary ctx_switch_involuntary_total = self.proc.num_ctx_switches().involuntary - ctx_switch_voluntary = ctx_switch_voluntary_total - self.stored_value['CTX_SWITCH_VOLUNTARY_TOTAL'] - ctx_switch_involuntary = ctx_switch_involuntary_total - self.stored_value['CTX_SWITCH_INVOLUNTARY_TOTAL'] + ctx_switch_voluntary = ctx_switch_voluntary_total - self.stored_value["CTX_SWITCH_VOLUNTARY_TOTAL"] + ctx_switch_involuntary = ctx_switch_involuntary_total - self.stored_value["CTX_SWITCH_INVOLUNTARY_TOTAL"] self.stored_value = dict( CPU_TIME_SYS_TOTAL=cpu_time_sys_total, diff --git a/ddtrace/internal/runtime/runtime_metrics.py b/ddtrace/internal/runtime/runtime_metrics.py index 374a943ae0a..cbfa8328b8a 100644 --- a/ddtrace/internal/runtime/runtime_metrics.py +++ b/ddtrace/internal/runtime/runtime_metrics.py @@ -26,17 +26,11 @@ def __init__(self, enabled=None): self._collectors = [c() for c in self.COLLECTORS] def __iter__(self): - collected = ( - collector.collect(self._enabled) - for collector in self._collectors - ) + collected = (collector.collect(self._enabled) for collector in self._collectors) return itertools.chain.from_iterable(collected) def __repr__(self): - return '{}(enabled={})'.format( - self.__class__.__name__, - self._enabled, - ) + return "{}(enabled={})".format(self.__class__.__name__, self._enabled,) class RuntimeTags(RuntimeCollectorsIterable): @@ -63,22 +57,18 @@ class RuntimeWorker(_worker.PeriodicWorkerThread): FLUSH_INTERVAL = 10 def __init__(self, statsd_client, flush_interval=FLUSH_INTERVAL): - super(RuntimeWorker, self).__init__(interval=flush_interval, - name=self.__class__.__name__) + super(RuntimeWorker, self).__init__(interval=flush_interval, name=self.__class__.__name__) self._statsd_client = statsd_client self._runtime_metrics = RuntimeMetrics() def flush(self): with self._statsd_client: for key, value in self._runtime_metrics: - log.debug('Writing metric %s:%s', key, value) + log.debug("Writing metric %s:%s", key, value) self._statsd_client.gauge(key, value) run_periodic = flush on_shutdown = flush def __repr__(self): - return '{}(runtime_metrics={})'.format( - self.__class__.__name__, - self._runtime_metrics, - ) + return "{}(runtime_metrics={})".format(self.__class__.__name__, self._runtime_metrics,) diff --git a/ddtrace/internal/runtime/tag_collectors.py b/ddtrace/internal/runtime/tag_collectors.py index d9c2fb235a9..41bad267adc 100644 --- a/ddtrace/internal/runtime/tag_collectors.py +++ b/ddtrace/internal/runtime/tag_collectors.py @@ -17,10 +17,11 @@ class RuntimeTagCollector(ValueCollector): class TracerTagCollector(RuntimeTagCollector): """ Tag collector for the ddtrace Tracer """ - required_modules = ['ddtrace'] + + required_modules = ["ddtrace"] def collect_fn(self, keys): - ddtrace = self.modules.get('ddtrace') + ddtrace = self.modules.get("ddtrace") tags = [(SERVICE, service) for service in ddtrace.tracer._services] if ENV_KEY in ddtrace.tracer.tags: tags.append((ENV_KEY, ddtrace.tracer.tags[ENV_KEY])) @@ -42,13 +43,14 @@ class PlatformTagCollector(RuntimeTagCollector): - ``tracer_version`` e.g. ``0.29.0`` """ - required_modules = ('platform', 'ddtrace') + + required_modules = ("platform", "ddtrace") def collect_fn(self, keys): - platform = self.modules.get('platform') - ddtrace = self.modules.get('ddtrace') + platform = self.modules.get("platform") + ddtrace = self.modules.get("ddtrace") tags = [ - (LANG, 'python'), + (LANG, "python"), (LANG_INTERPRETER, platform.python_implementation()), (LANG_VERSION, platform.python_version()), (TRACER_VERSION, ddtrace.__version__), diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 4b58f2ad637..13250b8370c 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -24,22 +24,31 @@ class AgentWriter(_worker.PeriodicWorkerThread): QUEUE_PROCESSING_INTERVAL = 1 - def __init__(self, hostname='localhost', port=8126, uds_path=None, https=False, - shutdown_timeout=DEFAULT_TIMEOUT, - filters=None, sampler=None, priority_sampler=None, - dogstatsd=None): - super(AgentWriter, self).__init__(interval=self.QUEUE_PROCESSING_INTERVAL, - exit_timeout=shutdown_timeout, - name=self.__class__.__name__) + def __init__( + self, + hostname="localhost", + port=8126, + uds_path=None, + https=False, + shutdown_timeout=DEFAULT_TIMEOUT, + filters=None, + sampler=None, + priority_sampler=None, + dogstatsd=None, + ): + super(AgentWriter, self).__init__( + interval=self.QUEUE_PROCESSING_INTERVAL, exit_timeout=shutdown_timeout, name=self.__class__.__name__ + ) self._trace_queue = Q(maxsize=MAX_TRACES) self._filters = filters self._sampler = sampler self._priority_sampler = priority_sampler self._last_error_ts = 0 self.dogstatsd = dogstatsd - self.api = api.API(hostname, port, uds_path=uds_path, https=https, - priority_sampling=priority_sampler is not None) - if hasattr(time, 'thread_time'): + self.api = api.API( + hostname, port, uds_path=uds_path, https=https, priority_sampling=priority_sampler is not None + ) + if hasattr(time, "thread_time"): self._last_thread_time = time.thread_time() self.start() @@ -85,7 +94,7 @@ def flush_queue(self): try: traces = self._apply_filters(traces) except Exception: - log.error('error while filtering traces', exc_info=True) + log.error("error while filtering traces", exc_info=True) return if self._send_stats: @@ -98,56 +107,55 @@ def flush_queue(self): self._log_error_status(response) elif self._priority_sampler or isinstance(self._sampler, BasePrioritySampler): result_traces_json = response.get_json() - if result_traces_json and 'rate_by_service' in result_traces_json: + if result_traces_json and "rate_by_service" in result_traces_json: if self._priority_sampler: self._priority_sampler.update_rate_by_service_sample_rates( - result_traces_json['rate_by_service'], + result_traces_json["rate_by_service"], ) if isinstance(self._sampler, BasePrioritySampler): - self._sampler.update_rate_by_service_sample_rates( - result_traces_json['rate_by_service'], - ) + self._sampler.update_rate_by_service_sample_rates(result_traces_json["rate_by_service"],) # Dump statistics # NOTE: Do not use the buffering of dogstatsd as it's not thread-safe # https://github.com/DataDog/datadogpy/issues/439 if self._send_stats: # Statistics about the queue length, size and number of spans - self.dogstatsd.increment('datadog.tracer.flushes') - self._histogram_with_total('datadog.tracer.flush.traces', traces_queue_length) - self._histogram_with_total('datadog.tracer.flush.spans', traces_queue_spans) + self.dogstatsd.increment("datadog.tracer.flushes") + self._histogram_with_total("datadog.tracer.flush.traces", traces_queue_length) + self._histogram_with_total("datadog.tracer.flush.spans", traces_queue_spans) # Statistics about the filtering - self._histogram_with_total('datadog.tracer.flush.traces_filtered', traces_filtered) + self._histogram_with_total("datadog.tracer.flush.traces_filtered", traces_filtered) # Statistics about API - self._histogram_with_total('datadog.tracer.api.requests', len(traces_responses)) + self._histogram_with_total("datadog.tracer.api.requests", len(traces_responses)) - self._histogram_with_total('datadog.tracer.api.errors', - len(list(t for t in traces_responses if isinstance(t, Exception)))) + self._histogram_with_total( + "datadog.tracer.api.errors", len(list(t for t in traces_responses if isinstance(t, Exception))) + ) for status, grouped_responses in itertools.groupby( - sorted((t for t in traces_responses if not isinstance(t, Exception)), - key=lambda r: r.status), - key=lambda r: r.status): - self._histogram_with_total('datadog.tracer.api.responses', - len(list(grouped_responses)), - tags=['status:%d' % status]) + sorted((t for t in traces_responses if not isinstance(t, Exception)), key=lambda r: r.status), + key=lambda r: r.status, + ): + self._histogram_with_total( + "datadog.tracer.api.responses", len(list(grouped_responses)), tags=["status:%d" % status] + ) # Statistics about the writer thread - if hasattr(time, 'thread_time'): + if hasattr(time, "thread_time"): new_thread_time = time.thread_time() diff = new_thread_time - self._last_thread_time self._last_thread_time = new_thread_time - self.dogstatsd.histogram('datadog.tracer.writer.cpu_time', diff) + self.dogstatsd.histogram("datadog.tracer.writer.cpu_time", diff) def _histogram_with_total(self, name, value, tags=None): """Helper to add metric as a histogram and with a `.total` counter""" self.dogstatsd.histogram(name, value, tags=tags) - self.dogstatsd.increment('%s.total' % (name, ), value, tags=tags) + self.dogstatsd.increment("%s.total" % (name,), value, tags=tags) def run_periodic(self): if self._send_stats: - self.dogstatsd.gauge('datadog.tracer.heartbeat', 1) + self.dogstatsd.gauge("datadog.tracer.heartbeat", 1) try: self.flush_queue() @@ -157,10 +165,10 @@ def run_periodic(self): # Statistics about the rate at which spans are inserted in the queue dropped, enqueued, enqueued_lengths = self._trace_queue.reset_stats() - self.dogstatsd.gauge('datadog.tracer.queue.max_length', self._trace_queue.maxsize) - self.dogstatsd.increment('datadog.tracer.queue.dropped.traces', dropped) - self.dogstatsd.increment('datadog.tracer.queue.enqueued.traces', enqueued) - self.dogstatsd.increment('datadog.tracer.queue.enqueued.spans', enqueued_lengths) + self.dogstatsd.gauge("datadog.tracer.queue.max_length", self._trace_queue.maxsize) + self.dogstatsd.increment("datadog.tracer.queue.dropped.traces", dropped) + self.dogstatsd.increment("datadog.tracer.queue.enqueued.traces", enqueued) + self.dogstatsd.increment("datadog.tracer.queue.enqueued.spans", enqueued_lengths) def on_shutdown(self): try: @@ -169,7 +177,7 @@ def on_shutdown(self): if not self._send_stats: return - self.dogstatsd.increment('datadog.tracer.shutdown') + self.dogstatsd.increment("datadog.tracer.shutdown") def _log_error_status(self, response): log_level = log.debug @@ -177,10 +185,10 @@ def _log_error_status(self, response): if now > self._last_error_ts + LOG_ERR_INTERVAL: log_level = log.error self._last_error_ts = now - prefix = 'Failed to send traces to Datadog Agent at %s: ' + prefix = "Failed to send traces to Datadog Agent at %s: " if isinstance(response, api.Response): log_level( - prefix + 'HTTP error status %s, reason %s, message %s', + prefix + "HTTP error status %s, reason %s, message %s", self.api, response.status, response.reason, @@ -188,9 +196,7 @@ def _log_error_status(self, response): ) else: log_level( - prefix + '%s', - self.api, - response, + prefix + "%s", self.api, response, ) def _apply_filters(self, traces): @@ -243,7 +249,7 @@ def put(self, item): if qsize != 0: idx = random.randrange(0, qsize) self.queue[idx] = item - log.warning('Writer queue is full has more than %d traces, some traces will be lost', self.maxsize) + log.warning("Writer queue is full has more than %d traces, some traces will be lost", self.maxsize) self.dropped += 1 self._update_stats(item) return @@ -256,7 +262,7 @@ def put(self, item): def _update_stats(self, item): # self.mutex needs to be locked to make sure we don't lose data when resetting self.accepted += 1 - if hasattr(item, '__len__'): + if hasattr(item, "__len__"): item_length = len(item) else: item_length = 1 @@ -268,9 +274,7 @@ def reset_stats(self): :return: The current value of dropped, accepted and accepted_lengths. """ with self.mutex: - dropped, accepted, accepted_lengths = ( - self.dropped, self.accepted, self.accepted_lengths - ) + dropped, accepted, accepted_lengths = (self.dropped, self.accepted, self.accepted_lengths) self.dropped, self.accepted, self.accepted_lengths = 0, 0, 0 return dropped, accepted, accepted_lengths diff --git a/pyproject.toml b/pyproject.toml index 0cb4bd95b3f..64561bce5ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ exclude = ''' | contrib/ | ext/ | http/ - | internal/ | opentracer/ | propagation/ | settings/ From 7594f0103c53dcaf86ec98c8f590c0a9faab7a27 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 3 Feb 2020 08:57:14 -0500 Subject: [PATCH 48/81] internal: Refactor setup.py c-extension building (#1191) * internal: Vendor psutil dependency * fix flake8/black issues * update comment * move vendor's setup code into ddtrace/vendor/*/setup.py * catch any exceptions loading vendor setup * add back required dep * remove unnecessary caching step * re-run setup.py develop before every test run * simplify setup.py * fix black formatting * refactor setup.py extension building --- .circleci/config.yml | 2 - ddtrace/vendor/msgpack/setup.py | 26 ++++++++++ ddtrace/vendor/wrapt/setup.py | 7 +++ setup.py | 87 +++++++++++++++++++++------------ tox.ini | 7 ++- 5 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 ddtrace/vendor/msgpack/setup.py create mode 100644 ddtrace/vendor/wrapt/setup.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 2552851da4b..09150d9cd32 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,7 +64,6 @@ jobs: resource_class: *resource_class steps: - checkout - - *restore_cache_step # Create and activate a Python3.7 virtualenv - run: virtualenv --python python3.7 .venv/build @@ -86,7 +85,6 @@ jobs: # Ensure package long description is valid and will render # https://github.com/pypa/twine/tree/6c4d5ecf2596c72b89b969ccc37b82c160645df8#twine-check - run: .venv/build/bin/twine check dist/* - - *save_cache_step tracer: docker: diff --git a/ddtrace/vendor/msgpack/setup.py b/ddtrace/vendor/msgpack/setup.py new file mode 100644 index 00000000000..addc81cbd99 --- /dev/null +++ b/ddtrace/vendor/msgpack/setup.py @@ -0,0 +1,26 @@ +__all__ = ["get_extensions"] + +from setuptools import Extension +import sys + + +def get_extensions(): + libraries = [] + if sys.platform == "win32": + libraries.append("ws2_32") + + macros = [] + if sys.byteorder == "big": + macros = [("__BIG_ENDIAN__", "1")] + else: + macros = [("__LITTLE_ENDIAN__", "1")] + + ext = Extension( + "ddtrace.vendor.msgpack._cmsgpack", + sources=["ddtrace/vendor/msgpack/_cmsgpack.cpp"], + libraries=libraries, + include_dirs=["ddtrace/vendor/"], + define_macros=macros, + ) + + return [ext] diff --git a/ddtrace/vendor/wrapt/setup.py b/ddtrace/vendor/wrapt/setup.py new file mode 100644 index 00000000000..dae324bd230 --- /dev/null +++ b/ddtrace/vendor/wrapt/setup.py @@ -0,0 +1,7 @@ +__all__ = ["get_extensions"] + +from setuptools import Extension + + +def get_extensions(): + return [Extension("ddtrace.vendor.wrapt._wrappers", sources=["ddtrace/vendor/wrapt/_wrappers.c"],)] diff --git a/setup.py b/setup.py index 4f01bf97095..a21be5dd98a 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,40 @@ from distutils.command.build_ext import build_ext from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError -from setuptools import setup, find_packages, Extension +from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module_from_project_file(mod_name, fname): + """ + Helper used to load a module from a file in this project + + DEV: Loading this way will by-pass loading all parent modules + e.g. importing `ddtrace.vendor.psutil.setup` will load `ddtrace/__init__.py` + which has side effects like loading the tracer + """ + fpath = os.path.join(HERE, fname) + + if sys.version_info >= (3, 5): + import importlib.util + + spec = importlib.util.spec_from_file_location(mod_name, fpath) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + elif sys.version_info >= (3, 3): + from importlib.machinery import SourceFileLoader + + return SourceFileLoader(mod_name, fpath).load_module() + else: + import imp + + return imp.load_source(mod_name, fpath) + + class Tox(TestCommand): user_options = [("tox-args=", "a", "Arguments to pass to tox")] @@ -56,7 +86,6 @@ def run_tests(self): [visualization docs]: https://docs.datadoghq.com/tracing/visualization/ """ -# psutil used to generate runtime metrics for tracer # enum34 is an enum backport for earlier versions of python # funcsigs backport required for vendored debtcollector install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'"] @@ -95,14 +124,7 @@ def run_tests(self): ) -# The following from here to the end of the file is borrowed from wrapt's and msgpack's `setup.py`: -# https://github.com/GrahamDumpleton/wrapt/blob/4ee35415a4b0d570ee6a9b3a14a6931441aeab4b/setup.py -# https://github.com/msgpack/msgpack-python/blob/381c2eff5f8ee0b8669fd6daf1fd1ecaffe7c931/setup.py -# These helpers are useful for attempting build a C-extension and then retrying without it if it fails - -libraries = [] if sys.platform == "win32": - libraries.append("ws2_32") build_ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError, OSError) else: build_ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) @@ -112,49 +134,52 @@ class BuildExtFailed(Exception): pass -# Attempt to build a C-extension, catch and throw a common/custom error if there are any issues +# Attempt to build a C-extension, catch exceptions so failed building skips the extension +# DEV: This is basically what `distutils`'s' `Extension(optional=True)` does class optional_build_ext(build_ext): def run(self): try: build_ext.run(self) - except DistutilsPlatformError: - raise BuildExtFailed() + except DistutilsPlatformError as e: + extensions = [ext.name for ext in self.extensions] + print("WARNING: Failed to build extensions %r, skipping: %s" % (extensions, e)) def build_extension(self, ext): try: build_ext.build_extension(self, ext) - except build_ext_errors: - raise BuildExtFailed() + except build_ext_errors as e: + print("WARNING: Failed to build extension %s, skipping: %s" % (ext.name, e)) -macros = [] -if sys.byteorder == "big": - macros = [("__BIG_ENDIAN__", "1")] -else: - macros = [("__LITTLE_ENDIAN__", "1")] +def get_exts_for(name): + try: + mod = load_module_from_project_file( + "ddtrace.vendor.{}.setup".format(name), "ddtrace/vendor/{}/setup.py".format(name) + ) + return mod.get_extensions() + except Exception as e: + print("WARNING: Failed to load %s extensions, skipping: %s" % (name, e)) + return [] # Try to build with C extensions first, fallback to only pure-Python if building fails try: + all_exts = [] + for extname in ("msgpack", "wrapt"): + exts = get_exts_for(extname) + if exts: + all_exts.extend(exts) + kwargs = copy.deepcopy(setup_kwargs) - kwargs["ext_modules"] = [ - Extension("ddtrace.vendor.wrapt._wrappers", sources=["ddtrace/vendor/wrapt/_wrappers.c"],), - Extension( - "ddtrace.vendor.msgpack._cmsgpack", - sources=["ddtrace/vendor/msgpack/_cmsgpack.cpp"], - libraries=libraries, - include_dirs=["ddtrace/vendor/"], - define_macros=macros, - ), - ] + kwargs["ext_modules"] = all_exts # DEV: Make sure `cmdclass` exists kwargs.setdefault("cmdclass", dict()) kwargs["cmdclass"]["build_ext"] = optional_build_ext setup(**kwargs) -except BuildExtFailed: +except Exception as e: # Set `DDTRACE_BUILD_TRACE=TRUE` in CI to raise any build errors if os.environ.get("DDTRACE_BUILD_RAISE") == "TRUE": raise - print("WARNING: Failed to install wrapt/msgpack C-extensions, using pure-Python wrapt/msgpack instead") + print("WARNING: Failed to install with ddtrace C-extensions, falling back to pure-Python only extensions: %s" % e) setup(**setup_kwargs) diff --git a/tox.ini b/tox.ini index 68ae43010d2..f585d1dc435 100644 --- a/tox.ini +++ b/tox.ini @@ -126,6 +126,11 @@ envlist = benchmarks-{py27,py34,py35,py36,py37} [testenv] +# Always re-run `setup.py develop` to ensure the proper C-extension .so files are created +# DEV: If we don't do this sometimes CircleCI gets messed up and only has the py27 .so +# meaning running on py3.x will fail +# https://stackoverflow.com/questions/57459123/why-do-i-need-to-run-tox-twice-to-test-a-python-package-with-c-extension +commands_pre={envpython} {toxinidir}/setup.py develop usedevelop = True basepython = py27: python2.7 @@ -142,7 +147,6 @@ deps = pytest-django pytest-mock opentracing - psutil # test dependencies installed in all envs mock # force the downgrade as a workaround @@ -839,6 +843,7 @@ exclude= .ddtox,.tox, .git,__pycache__, .eggs,*.egg, + build, # We shouldn't lint our vendored dependencies ddtrace/vendor/ # Ignore: From 1e87c9bdf7769032982349c4ccc0e1c2e6866a16 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 3 Feb 2020 15:46:21 +0100 Subject: [PATCH 49/81] chord(flake8,black): ignore W504 and E231 not respected by black (#1185) black does not respect those error codes and therefore they should be ignored. It does support W503 so that one should be included. This patch therefore black-ifies the files that do not respect W503. --- ddtrace/contrib/django/middleware.py | 68 +++--- ddtrace/contrib/grpc/client_interceptor.py | 51 ++-- ddtrace/contrib/requests/connection.py | 51 ++-- pyproject.toml | 183 ++++++++++++++- tests/contrib/dogpile_cache/test_tracing.py | 76 +++--- tests/contrib/pyramid/utils.py | 247 ++++++++++---------- tests/internal/test_writer.py | 109 +++++---- tests/opentracer/test_tracer.py | 244 +++++++++---------- tests/propagation/test_http.py | 38 ++- tox.ini | 3 +- 10 files changed, 601 insertions(+), 469 deletions(-) diff --git a/ddtrace/contrib/django/middleware.py b/ddtrace/contrib/django/middleware.py index 6e71d200799..8a0539a75bb 100644 --- a/ddtrace/contrib/django/middleware.py +++ b/ddtrace/contrib/django/middleware.py @@ -17,33 +17,33 @@ try: from django.utils.deprecation import MiddlewareMixin + MiddlewareClass = MiddlewareMixin except ImportError: MiddlewareClass = object log = get_logger(__name__) -EXCEPTION_MIDDLEWARE = 'ddtrace.contrib.django.TraceExceptionMiddleware' -TRACE_MIDDLEWARE = 'ddtrace.contrib.django.TraceMiddleware' -MIDDLEWARE = 'MIDDLEWARE' -MIDDLEWARE_CLASSES = 'MIDDLEWARE_CLASSES' +EXCEPTION_MIDDLEWARE = "ddtrace.contrib.django.TraceExceptionMiddleware" +TRACE_MIDDLEWARE = "ddtrace.contrib.django.TraceMiddleware" +MIDDLEWARE = "MIDDLEWARE" +MIDDLEWARE_CLASSES = "MIDDLEWARE_CLASSES" # Default views list available from: # https://github.com/django/django/blob/38e2fdadfd9952e751deed662edf4c496d238f28/django/views/defaults.py # DEV: Django doesn't call `process_view` when falling back to one of these internal error handling views # DEV: We only use these names when `span.resource == 'unknown'` and we have one of these status codes _django_default_views = { - 400: 'django.views.defaults.bad_request', - 403: 'django.views.defaults.permission_denied', - 404: 'django.views.defaults.page_not_found', - 500: 'django.views.defaults.server_error', + 400: "django.views.defaults.bad_request", + 403: "django.views.defaults.permission_denied", + 404: "django.views.defaults.page_not_found", + 500: "django.views.defaults.server_error", } def _analytics_enabled(): return ( - (config.analytics_enabled and settings.ANALYTICS_ENABLED is not False) or - settings.ANALYTICS_ENABLED is True + (config.analytics_enabled and settings.ANALYTICS_ENABLED is not False) or settings.ANALYTICS_ENABLED is True ) and settings.ANALYTICS_SAMPLE_RATE is not None @@ -87,6 +87,7 @@ class InstrumentationMixin(MiddlewareClass): """ Useful mixin base class for tracing middlewares """ + def __init__(self, get_response=None): # disable the middleware if the tracer is not enabled # or if the auto instrumentation is disabled @@ -99,20 +100,22 @@ class TraceExceptionMiddleware(InstrumentationMixin): """ Middleware that traces exceptions raised """ + def process_exception(self, request, exception): try: span = _get_req_span(request) if span: - span.set_tag(http.STATUS_CODE, '500') + span.set_tag(http.STATUS_CODE, "500") span.set_traceback() # will set the exception info except Exception: - log.debug('error processing exception', exc_info=True) + log.debug("error processing exception", exc_info=True) class TraceMiddleware(InstrumentationMixin): """ Middleware that traces Django requests """ + def process_request(self, request): tracer = settings.TRACER if settings.DISTRIBUTED_TRACING: @@ -123,9 +126,9 @@ def process_request(self, request): tracer.context_provider.activate(context) try: span = tracer.trace( - 'django.request', + "django.request", service=settings.DEFAULT_SERVICE, - resource='unknown', # will be filled by process view + resource="unknown", # will be filled by process view span_type=SpanTypes.WEB, ) @@ -133,8 +136,7 @@ def process_request(self, request): # DEV: django is special case maintains separate configuration from config api if _analytics_enabled() and settings.ANALYTICS_SAMPLE_RATE is not None: span.set_tag( - ANALYTICS_SAMPLE_RATE_KEY, - settings.ANALYTICS_SAMPLE_RATE, + ANALYTICS_SAMPLE_RATE_KEY, settings.ANALYTICS_SAMPLE_RATE, ) # Set HTTP Request tags @@ -144,10 +146,10 @@ def process_request(self, request): if trace_query_string is None: trace_query_string = config.django.trace_query_string if trace_query_string: - span.set_tag(http.QUERY_STRING, request.META['QUERY_STRING']) + span.set_tag(http.QUERY_STRING, request.META["QUERY_STRING"]) _set_req_span(request, span) except Exception: - log.debug('error tracing request', exc_info=True) + log.debug("error tracing request", exc_info=True) def process_view(self, request, view_func, *args, **kwargs): span = _get_req_span(request) @@ -166,63 +168,63 @@ def process_response(self, request, response): # If `process_view` was not called, try to determine the correct `span.resource` to set # DEV: `process_view` won't get called if a middle `process_request` returns an HttpResponse # DEV: `process_view` won't get called when internal error handlers are used (e.g. for 404 responses) - if span.resource == 'unknown': + if span.resource == "unknown": try: # Attempt to lookup the view function from the url resolver # https://github.com/django/django/blob/38e2fdadfd9952e751deed662edf4c496d238f28/django/core/handlers/base.py#L104-L113 # noqa urlconf = None - if hasattr(request, 'urlconf'): + if hasattr(request, "urlconf"): urlconf = request.urlconf resolver = get_resolver(urlconf) # Try to resolve the Django view for handling this request - if getattr(request, 'request_match', None): + if getattr(request, "request_match", None): request_match = request.request_match else: # This may raise a `django.urls.exceptions.Resolver404` exception request_match = resolver.resolve(request.path_info) span.resource = func_name(request_match.func) except Exception: - log.debug('error determining request view function', exc_info=True) + log.debug("error determining request view function", exc_info=True) # If the view could not be found, try to set from a static list of # known internal error handler views - span.resource = _django_default_views.get(response.status_code, 'unknown') + span.resource = _django_default_views.get(response.status_code, "unknown") span.set_tag(http.STATUS_CODE, response.status_code) span = _set_auth_tags(span, request) span.finish() except Exception: - log.debug('error tracing request', exc_info=True) + log.debug("error tracing request", exc_info=True) finally: return response def _get_req_span(request): """ Return the datadog span from the given request. """ - return getattr(request, '_datadog_request_span', None) + return getattr(request, "_datadog_request_span", None) def _set_req_span(request, span): """ Set the datadog span on the given request. """ - return setattr(request, '_datadog_request_span', span) + return setattr(request, "_datadog_request_span", span) def _set_auth_tags(span, request): """ Patch any available auth tags from the request onto the span. """ - user = getattr(request, 'user', None) + user = getattr(request, "user", None) if not user: return span - if hasattr(user, 'is_authenticated'): - span.set_tag('django.user.is_authenticated', user_is_authenticated(user)) + if hasattr(user, "is_authenticated"): + span.set_tag("django.user.is_authenticated", user_is_authenticated(user)) - uid = getattr(user, 'pk', None) + uid = getattr(user, "pk", None) if uid: - span.set_tag('django.user.id', uid) + span.set_tag("django.user.id", uid) - uname = getattr(user, 'username', None) + uname = getattr(user, "username", None) if uname: - span.set_tag('django.user.name', uname) + span.set_tag("django.user.name", uname) return span diff --git a/ddtrace/contrib/grpc/client_interceptor.py b/ddtrace/contrib/grpc/client_interceptor.py index 5aa763dc0d8..be1e66efa91 100644 --- a/ddtrace/contrib/grpc/client_interceptor.py +++ b/ddtrace/contrib/grpc/client_interceptor.py @@ -28,9 +28,9 @@ def create_client_interceptor(pin, host, port): def intercept_channel(wrapped, instance, args, kwargs): channel = args[0] interceptors = args[1:] - if isinstance(getattr(channel, '_interceptor', None), _ClientInterceptor): + if isinstance(getattr(channel, "_interceptor", None), _ClientInterceptor): dd_interceptor = channel._interceptor - base_channel = getattr(channel, '_channel', None) + base_channel = getattr(channel, "_channel", None) if base_channel: new_channel = wrapped(channel._channel, *interceptors) return grpc.intercept_channel(new_channel, dd_interceptor) @@ -39,10 +39,9 @@ def intercept_channel(wrapped, instance, args, kwargs): class _ClientCallDetails( - collections.namedtuple( - '_ClientCallDetails', - ('method', 'timeout', 'metadata', 'credentials')), - grpc.ClientCallDetails): + collections.namedtuple("_ClientCallDetails", ("method", "timeout", "metadata", "credentials")), + grpc.ClientCallDetails, +): pass @@ -74,9 +73,9 @@ def _handle_error(span, response_error, status_code): # exception() and traceback() methods if a computation has resulted in an # exception being raised if ( - not callable(getattr(response_error, 'cancelled', None)) and - not callable(getattr(response_error, 'exception', None)) and - not callable(getattr(response_error, 'traceback', None)) + not callable(getattr(response_error, "cancelled", None)) + and not callable(getattr(response_error, "exception", None)) + and not callable(getattr(response_error, "traceback", None)) ): return @@ -129,7 +128,7 @@ def __next__(self): raise except Exception: # DEV: added for safety though should not be reached since wrapped response - log.debug('unexpected non-grpc exception raised, closing open span', exc_info=True) + log.debug("unexpected non-grpc exception raised, closing open span", exc_info=True) self._span.set_traceback() self._span.finish() raise @@ -139,9 +138,11 @@ def next(self): class _ClientInterceptor( - grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor, - grpc.StreamUnaryClientInterceptor, grpc.StreamStreamClientInterceptor): - + grpc.UnaryUnaryClientInterceptor, + grpc.UnaryStreamClientInterceptor, + grpc.StreamUnaryClientInterceptor, + grpc.StreamStreamClientInterceptor, +): def __init__(self, pin, host, port): self._pin = pin self._host = host @@ -151,10 +152,7 @@ def _intercept_client_call(self, method_kind, client_call_details): tracer = self._pin.tracer span = tracer.trace( - 'grpc', - span_type=SpanTypes.GRPC, - service=self._pin.service, - resource=client_call_details.method, + "grpc", span_type=SpanTypes.GRPC, service=self._pin.service, resource=client_call_details.method, ) # tags for method details @@ -189,19 +187,13 @@ def _intercept_client_call(self, method_kind, client_call_details): metadata.extend(headers.items()) client_call_details = _ClientCallDetails( - client_call_details.method, - client_call_details.timeout, - metadata, - client_call_details.credentials, + client_call_details.method, client_call_details.timeout, metadata, client_call_details.credentials, ) return span, client_call_details def intercept_unary_unary(self, continuation, client_call_details, request): - span, client_call_details = self._intercept_client_call( - constants.GRPC_METHOD_KIND_UNARY, - client_call_details, - ) + span, client_call_details = self._intercept_client_call(constants.GRPC_METHOD_KIND_UNARY, client_call_details,) try: response = continuation(client_call_details, request) _handle_response(span, response) @@ -216,8 +208,7 @@ def intercept_unary_unary(self, continuation, client_call_details, request): def intercept_unary_stream(self, continuation, client_call_details, request): span, client_call_details = self._intercept_client_call( - constants.GRPC_METHOD_KIND_SERVER_STREAMING, - client_call_details, + constants.GRPC_METHOD_KIND_SERVER_STREAMING, client_call_details, ) response_iterator = continuation(client_call_details, request) response_iterator = _WrappedResponseCallFuture(response_iterator, span) @@ -225,8 +216,7 @@ def intercept_unary_stream(self, continuation, client_call_details, request): def intercept_stream_unary(self, continuation, client_call_details, request_iterator): span, client_call_details = self._intercept_client_call( - constants.GRPC_METHOD_KIND_CLIENT_STREAMING, - client_call_details, + constants.GRPC_METHOD_KIND_CLIENT_STREAMING, client_call_details, ) try: response = continuation(client_call_details, request_iterator) @@ -242,8 +232,7 @@ def intercept_stream_unary(self, continuation, client_call_details, request_iter def intercept_stream_stream(self, continuation, client_call_details, request_iterator): span, client_call_details = self._intercept_client_call( - constants.GRPC_METHOD_KIND_BIDI_STREAMING, - client_call_details, + constants.GRPC_METHOD_KIND_BIDI_STREAMING, client_call_details, ) response_iterator = continuation(client_call_details, request_iterator) response_iterator = _WrappedResponseCallFuture(response_iterator, span) diff --git a/ddtrace/contrib/requests/connection.py b/ddtrace/contrib/requests/connection.py index e1536599e80..503d4a56c27 100644 --- a/ddtrace/contrib/requests/connection.py +++ b/ddtrace/contrib/requests/connection.py @@ -26,13 +26,11 @@ def _extract_service_name(session, span, hostname=None): Updated service name > parent service name > default to `requests`. """ cfg = config.get_from(session) - if cfg['split_by_domain'] and hostname: + if cfg["split_by_domain"] and hostname: return hostname - service_name = cfg['service_name'] - if (service_name == DEFAULT_SERVICE and - span._parent is not None and - span._parent.service is not None): + service_name = cfg["service_name"] + if service_name == DEFAULT_SERVICE and span._parent is not None and span._parent.service is not None: service_name = span._parent.service return service_name @@ -43,13 +41,13 @@ def _wrap_send(func, instance, args, kwargs): # and is ddtrace.tracer; it's used only inside our tests and can # be easily changed by providing a TracingTestCase that sets common # tracing functionalities. - tracer = getattr(instance, 'datadog_tracer', ddtrace.tracer) + tracer = getattr(instance, "datadog_tracer", ddtrace.tracer) # skip if tracing is not enabled if not tracer.enabled: return func(*args, **kwargs) - request = kwargs.get('request') or args[0] + request = kwargs.get("request") or args[0] if not request: return func(*args, **kwargs) @@ -57,32 +55,31 @@ def _wrap_send(func, instance, args, kwargs): parsed_uri = parse.urlparse(request.url) hostname = parsed_uri.hostname if parsed_uri.port: - hostname = '{}:{}'.format(hostname, parsed_uri.port) - sanitized_url = parse.urlunparse(( - parsed_uri.scheme, - parsed_uri.netloc, - parsed_uri.path, - parsed_uri.params, - None, # drop parsed_uri.query - parsed_uri.fragment - )) - - with tracer.trace('requests.request', span_type=SpanTypes.HTTP) as span: + hostname = "{}:{}".format(hostname, parsed_uri.port) + sanitized_url = parse.urlunparse( + ( + parsed_uri.scheme, + parsed_uri.netloc, + parsed_uri.path, + parsed_uri.params, + None, # drop parsed_uri.query + parsed_uri.fragment, + ) + ) + + with tracer.trace("requests.request", span_type=SpanTypes.HTTP) as span: # update the span service name before doing any action span.service = _extract_service_name(instance, span, hostname=hostname) # Configure trace search sample rate # DEV: analytics enabled on per-session basis cfg = config.get_from(instance) - analytics_enabled = cfg.get('analytics_enabled') + analytics_enabled = cfg.get("analytics_enabled") if analytics_enabled: - span.set_tag( - ANALYTICS_SAMPLE_RATE_KEY, - cfg.get('analytics_sample_rate', True) - ) + span.set_tag(ANALYTICS_SAMPLE_RATE_KEY, cfg.get("analytics_sample_rate", True)) # propagate distributed tracing headers - if cfg.get('distributed_tracing'): + if cfg.get("distributed_tracing"): propagator = HTTPPropagator() propagator.inject(span.context, request.headers) @@ -95,7 +92,7 @@ def _wrap_send(func, instance, args, kwargs): # Storing response headers in the span. Note that response.headers is not a dict, but an iterable # requests custom structure, that we convert to a dict - if hasattr(response, 'headers'): + if hasattr(response, "headers"): store_response_headers(dict(response.headers), span, config.requests) return response finally: @@ -111,7 +108,7 @@ def _wrap_send(func, instance, args, kwargs): # Storing response headers in the span. # Note that response.headers is not a dict, but an iterable # requests custom structure, that we convert to a dict - response_headers = dict(getattr(response, 'headers', {})) + response_headers = dict(getattr(response, "headers", {})) store_response_headers(response_headers, span, config.requests) except Exception: - log.debug('requests: error adding tags', exc_info=True) + log.debug("requests: error adding tags", exc_info=True) diff --git a/pyproject.toml b/pyproject.toml index 64561bce5ea..8790a1e7ba3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,79 @@ exclude = ''' [^/]+\.py | commands/ | contrib/ + ( + aiobotocore + | aiohttp + | aiopg + | algoliasearch + | asyncio + | boto + | botocore + | bottle + | cassandra + | celery + | consul + | dbapi + | django/ + ( + __init__.py + | apps.py + | cache.py + | compat.py + | conf.py + | db.py + | patch.py + | restframework.py + | templates.py + | utils.py + ) + | dogpile_cache + | elasticsearch + | falcon + | flask + | flask_cache + | futures + | gevent + | grpc/ + ( + __init__.py + | constants.py + | patch.py + | server_interceptor.py + | utils.py + ) + | httplib + | jinja2 + | kombu + | logging + | mako + | molten + | mongoengine + | mysql + | mysqldb + | psycopg + | pylibmc + | pylons + | pymemcache + | pymongo + | pymysql + | pyramid + | redis + | rediscluster + | requests/ + ( + __init__.py + | constants.py + | legacy.py + | patch.py + | session.py + ) + | sqlalchemy + | sqlite3 + | tornado + | util.py + | vertica + ) | ext/ | http/ | opentracer/ @@ -26,5 +99,113 @@ exclude = ''' | vendor/ ) | tests/ + ( + base + | benchmark.py + | commands + | contrib/ + ( + aiobotocore + | aiohttp + | aiopg + | algoliasearch + | asyncio + | boto + | botocore + | bottle + | cassandra + | celery + | config.py + | consul + | dbapi + | django + | djangorestframework + | elasticsearch + | falcon + | flask + | flask_autopatch + | flask_cache + | futures + | gevent + | grpc + | httplib + | jinja2 + | kombu + | logging + | mako + | molten + | mongoengine + | mysql + | mysqldb + | patch.py + | psycopg + | pylibmc + | pylons + | pymemcache + | pymongo + | pymysql + | pyramid/ + ( + app/web.py + | __init__.py + | test_pyramid.py + | test_pyramid_autopatch.py + ) + | redis + | rediscluster + | requests + | requests_gevent + | sqlalchemy + | sqlite3 + | test_utils.py + | tornado + | vertica + ) + | ddtrace_run.py + | internal/ + ( + runtime/ + | test_context_manager.py + | test_hostname.py + | test_logger.py + | test_rate_limiter.py + ) + | memory.py + | opentracer/ + ( + conftest.py + | test_dd_compatibility.py + | test_span.py + | test_span_context.py + | test_tracer_asyncio.py + | test_tracer_gevent.py + | test_tracer_tornado.py + | test_utils.py + ) + | propagation/test_utils.py + | subprocesstest.py + | test_api.py + | test_compat.py + | test_context.py + | test_encoders.py + | test_filters.py + | test_global_config.py + | test_helpers.py + | test_hook.py + | test_instance_config.py + | test_integration.py + | test_payload.py + | test_pin.py + | test_sampler.py + | test_span.py + | test_tracer.py + | test_utils.py + | test_worker.py + | unit + | util.py + | utils + | vendor + | wait-for-services.py + ) ) -''' +''' \ No newline at end of file diff --git a/tests/contrib/dogpile_cache/test_tracing.py b/tests/contrib/dogpile_cache/test_tracing.py index 79e42592069..1548bf24e11 100644 --- a/tests/contrib/dogpile_cache/test_tracing.py +++ b/tests/contrib/dogpile_cache/test_tracing.py @@ -17,8 +17,8 @@ def region(tracer): patch() # Setup a simple dogpile cache region for testing. # The backend is trivial so we can use memory to simplify test setup. - test_region = dogpile.cache.make_region(name='TestRegion') - test_region.configure('dogpile.cache.memory') + test_region = dogpile.cache.make_region(name="TestRegion") + test_region.configure("dogpile.cache.memory") Pin.override(dogpile.cache, tracer=tracer) return test_region @@ -34,6 +34,7 @@ def single_cache(region): @region.cache_on_arguments() def fn(x): return x * 2 + return fn @@ -74,13 +75,13 @@ def test_traces_get_or_create(tracer, single_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] - assert span.name == 'dogpile.cache' - assert span.resource == 'get_or_create' - assert span.meta['key'] == 'tests.contrib.dogpile_cache.test_tracing:fn|1' - assert span.meta['hit'] == 'False' - assert span.meta['expired'] == 'True' - assert span.meta['backend'] == 'MemoryBackend' - assert span.meta['region'] == 'TestRegion' + assert span.name == "dogpile.cache" + assert span.resource == "get_or_create" + assert span.meta["key"] == "tests.contrib.dogpile_cache.test_tracing:fn|1" + assert span.meta["hit"] == "False" + assert span.meta["expired"] == "True" + assert span.meta["backend"] == "MemoryBackend" + assert span.meta["region"] == "TestRegion" # Now the results should be cached. assert single_cache(1) == 2 @@ -89,13 +90,13 @@ def test_traces_get_or_create(tracer, single_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] - assert span.name == 'dogpile.cache' - assert span.resource == 'get_or_create' - assert span.meta['key'] == 'tests.contrib.dogpile_cache.test_tracing:fn|1' - assert span.meta['hit'] == 'True' - assert span.meta['expired'] == 'False' - assert span.meta['backend'] == 'MemoryBackend' - assert span.meta['region'] == 'TestRegion' + assert span.name == "dogpile.cache" + assert span.resource == "get_or_create" + assert span.meta["key"] == "tests.contrib.dogpile_cache.test_tracing:fn|1" + assert span.meta["hit"] == "True" + assert span.meta["expired"] == "False" + assert span.meta["backend"] == "MemoryBackend" + assert span.meta["region"] == "TestRegion" def test_traces_get_or_create_multi(tracer, multi_cache): @@ -105,14 +106,13 @@ def test_traces_get_or_create_multi(tracer, multi_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] - assert span.meta['keys'] == ( - "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + - "'tests.contrib.dogpile_cache.test_tracing:fn|3']" + assert span.meta["keys"] == ( + "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + "'tests.contrib.dogpile_cache.test_tracing:fn|3']" ) - assert span.meta['hit'] == 'False' - assert span.meta['expired'] == 'True' - assert span.meta['backend'] == 'MemoryBackend' - assert span.meta['region'] == 'TestRegion' + assert span.meta["hit"] == "False" + assert span.meta["expired"] == "True" + assert span.meta["backend"] == "MemoryBackend" + assert span.meta["region"] == "TestRegion" # Partial hit assert multi_cache(2, 4) == [4, 8] @@ -121,14 +121,13 @@ def test_traces_get_or_create_multi(tracer, multi_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] - assert span.meta['keys'] == ( - "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + - "'tests.contrib.dogpile_cache.test_tracing:fn|4']" + assert span.meta["keys"] == ( + "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + "'tests.contrib.dogpile_cache.test_tracing:fn|4']" ) - assert span.meta['hit'] == 'False' - assert span.meta['expired'] == 'True' - assert span.meta['backend'] == 'MemoryBackend' - assert span.meta['region'] == 'TestRegion' + assert span.meta["hit"] == "False" + assert span.meta["expired"] == "True" + assert span.meta["backend"] == "MemoryBackend" + assert span.meta["region"] == "TestRegion" # Full hit assert multi_cache(2, 4) == [4, 8] @@ -137,14 +136,13 @@ def test_traces_get_or_create_multi(tracer, multi_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] - assert span.meta['keys'] == ( - "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + - "'tests.contrib.dogpile_cache.test_tracing:fn|4']" + assert span.meta["keys"] == ( + "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + "'tests.contrib.dogpile_cache.test_tracing:fn|4']" ) - assert span.meta['hit'] == 'True' - assert span.meta['expired'] == 'False' - assert span.meta['backend'] == 'MemoryBackend' - assert span.meta['region'] == 'TestRegion' + assert span.meta["hit"] == "True" + assert span.meta["expired"] == "False" + assert span.meta["backend"] == "MemoryBackend" + assert span.meta["region"] == "TestRegion" class TestInnerFunctionCalls(object): @@ -156,8 +154,8 @@ def multi_cache(self, *x): def test_calls_inner_functions_correctly(self, region, mocker): """ This ensures the get_or_create behavior of dogpile is not altered. """ - spy_single_cache = mocker.spy(self, 'single_cache') - spy_multi_cache = mocker.spy(self, 'multi_cache') + spy_single_cache = mocker.spy(self, "single_cache") + spy_multi_cache = mocker.spy(self, "multi_cache") single_cache = region.cache_on_arguments()(self.single_cache) multi_cache = region.cache_multi_on_arguments()(self.multi_cache) diff --git a/tests/contrib/pyramid/utils.py b/tests/contrib/pyramid/utils.py index f3c97e1f546..3e9ee9dc81a 100644 --- a/tests/contrib/pyramid/utils.py +++ b/tests/contrib/pyramid/utils.py @@ -19,6 +19,7 @@ class PyramidBase(BaseTracerTestCase): """Base Pyramid test application""" + def setUp(self): super(PyramidBase, self).setUp() self.create_app() @@ -27,7 +28,7 @@ def create_app(self, settings=None): # get default settings or use what is provided settings = settings or self.get_settings() # always set the dummy tracer as a default tracer - settings.update({'datadog_tracer': self.tracer}) + settings.update({"datadog_tracer": self.tracer}) app, renderer = create_app(settings, self.instrument) self.app = webtest.TestApp(app) @@ -42,37 +43,38 @@ def override_settings(self, settings): class PyramidTestCase(PyramidBase): """Pyramid TestCase that includes tests for automatic instrumentation""" + instrument = True def get_settings(self): return { - 'datadog_trace_service': 'foobar', + "datadog_trace_service": "foobar", } - def test_200(self, query_string=''): + def test_200(self, query_string=""): if query_string: - fqs = '?' + query_string + fqs = "?" + query_string else: - fqs = '' - res = self.app.get('/' + fqs, status=200) - assert b'idx' in res.body + fqs = "" + res = self.app.get("/" + fqs, status=200) + assert b"idx" in res.body writer = self.tracer.writer spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == 'GET index' + assert s.service == "foobar" + assert s.resource == "GET index" assert s.error == 0 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 200) - assert s.meta.get(http.URL) == 'http://localhost/' + assert s.meta.get(http.URL) == "http://localhost/" if config.pyramid.trace_query_string: assert s.meta.get(http.QUERY_STRING) == query_string else: assert http.QUERY_STRING not in s.meta - assert s.meta.get('pyramid.route.name') == 'index' + assert s.meta.get("pyramid.route.name") == "index" # ensure services are set correctly services = writer.pop_services() @@ -80,11 +82,11 @@ def test_200(self, query_string=''): assert services == expected def test_200_query_string(self): - return self.test_200('foo=bar') + return self.test_200("foo=bar") def test_200_query_string_trace(self): - with self.override_http_config('pyramid', dict(trace_query_string=True)): - return self.test_200('foo=bar') + with self.override_http_config("pyramid", dict(trace_query_string=True)): + return self.test_200("foo=bar") def test_analytics_global_on_integration_default(self): """ @@ -93,12 +95,10 @@ def test_analytics_global_on_integration_default(self): We expect the root span to have the appropriate tag """ with self.override_global_config(dict(analytics_enabled=True)): - res = self.app.get('/', status=200) - assert b'idx' in res.body + res = self.app.get("/", status=200) + assert b"idx" in res.body - self.assert_structure( - dict(name='pyramid.request', metrics={ANALYTICS_SAMPLE_RATE_KEY: 1.0}), - ) + self.assert_structure(dict(name="pyramid.request", metrics={ANALYTICS_SAMPLE_RATE_KEY: 1.0}),) def test_analytics_global_on_integration_on(self): """ @@ -108,12 +108,10 @@ def test_analytics_global_on_integration_on(self): """ with self.override_global_config(dict(analytics_enabled=True)): self.override_settings(dict(datadog_analytics_enabled=True, datadog_analytics_sample_rate=0.5)) - res = self.app.get('/', status=200) - assert b'idx' in res.body + res = self.app.get("/", status=200) + assert b"idx" in res.body - self.assert_structure( - dict(name='pyramid.request', metrics={ANALYTICS_SAMPLE_RATE_KEY: 0.5}), - ) + self.assert_structure(dict(name="pyramid.request", metrics={ANALYTICS_SAMPLE_RATE_KEY: 0.5}),) def test_analytics_global_off_integration_default(self): """ @@ -122,8 +120,8 @@ def test_analytics_global_off_integration_default(self): We expect the root span to not include tag """ with self.override_global_config(dict(analytics_enabled=False)): - res = self.app.get('/', status=200) - assert b'idx' in res.body + res = self.app.get("/", status=200) + assert b"idx" in res.body root = self.get_root_span() self.assertIsNone(root.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) @@ -136,61 +134,59 @@ def test_analytics_global_off_integration_on(self): """ with self.override_global_config(dict(analytics_enabled=False)): self.override_settings(dict(datadog_analytics_enabled=True, datadog_analytics_sample_rate=0.5)) - res = self.app.get('/', status=200) - assert b'idx' in res.body + res = self.app.get("/", status=200) + assert b"idx" in res.body - self.assert_structure( - dict(name='pyramid.request', metrics={ANALYTICS_SAMPLE_RATE_KEY: 0.5}), - ) + self.assert_structure(dict(name="pyramid.request", metrics={ANALYTICS_SAMPLE_RATE_KEY: 0.5}),) def test_404(self): - self.app.get('/404', status=404) + self.app.get("/404", status=404) writer = self.tracer.writer spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == '404' + assert s.service == "foobar" + assert s.resource == "404" assert s.error == 0 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 404) - assert s.meta.get(http.URL) == 'http://localhost/404' + assert s.meta.get(http.URL) == "http://localhost/404" def test_302(self): - self.app.get('/redirect', status=302) + self.app.get("/redirect", status=302) writer = self.tracer.writer spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == 'GET raise_redirect' + assert s.service == "foobar" + assert s.resource == "GET raise_redirect" assert s.error == 0 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 302) - assert s.meta.get(http.URL) == 'http://localhost/redirect' + assert s.meta.get(http.URL) == "http://localhost/redirect" def test_204(self): - self.app.get('/nocontent', status=204) + self.app.get("/nocontent", status=204) writer = self.tracer.writer spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == 'GET raise_no_content' + assert s.service == "foobar" + assert s.resource == "GET raise_no_content" assert s.error == 0 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 204) - assert s.meta.get(http.URL) == 'http://localhost/nocontent' + assert s.meta.get(http.URL) == "http://localhost/nocontent" def test_exception(self): try: - self.app.get('/exception', status=500) + self.app.get("/exception", status=500) except ZeroDivisionError: pass @@ -198,146 +194,145 @@ def test_exception(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == 'GET exception' + assert s.service == "foobar" + assert s.resource == "GET exception" assert s.error == 1 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 500) - assert s.meta.get(http.URL) == 'http://localhost/exception' - assert s.meta.get('pyramid.route.name') == 'exception' + assert s.meta.get(http.URL) == "http://localhost/exception" + assert s.meta.get("pyramid.route.name") == "exception" def test_500(self): - self.app.get('/error', status=500) + self.app.get("/error", status=500) writer = self.tracer.writer spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == 'GET error' + assert s.service == "foobar" + assert s.resource == "GET error" assert s.error == 1 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 500) - assert s.meta.get(http.URL) == 'http://localhost/error' - assert s.meta.get('pyramid.route.name') == 'error' + assert s.meta.get(http.URL) == "http://localhost/error" + assert s.meta.get("pyramid.route.name") == "error" assert type(s.error) == int def test_json(self): - res = self.app.get('/json', status=200) + res = self.app.get("/json", status=200) parsed = json.loads(compat.to_unicode(res.body)) - assert parsed == {'a': 1} + assert parsed == {"a": 1} writer = self.tracer.writer spans = writer.pop() assert len(spans) == 2 spans_by_name = {s.name: s for s in spans} - s = spans_by_name['pyramid.request'] - assert s.service == 'foobar' - assert s.resource == 'GET json' + s = spans_by_name["pyramid.request"] + assert s.service == "foobar" + assert s.resource == "GET json" assert s.error == 0 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 200) - assert s.meta.get(http.URL) == 'http://localhost/json' - assert s.meta.get('pyramid.route.name') == 'json' + assert s.meta.get(http.URL) == "http://localhost/json" + assert s.meta.get("pyramid.route.name") == "json" - s = spans_by_name['pyramid.render'] - assert s.service == 'foobar' + s = spans_by_name["pyramid.render"] + assert s.service == "foobar" assert s.error == 0 - assert s.span_type == 'template' + assert s.span_type == "template" def test_renderer(self): - self.app.get('/renderer', status=200) - assert self.renderer._received['request'] is not None + self.app.get("/renderer", status=200) + assert self.renderer._received["request"] is not None - self.renderer.assert_(foo='bar') + self.renderer.assert_(foo="bar") writer = self.tracer.writer spans = writer.pop() assert len(spans) == 2 spans_by_name = {s.name: s for s in spans} - s = spans_by_name['pyramid.request'] - assert s.service == 'foobar' - assert s.resource == 'GET renderer' + s = spans_by_name["pyramid.request"] + assert s.service == "foobar" + assert s.resource == "GET renderer" assert s.error == 0 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 200) - assert s.meta.get(http.URL) == 'http://localhost/renderer' - assert s.meta.get('pyramid.route.name') == 'renderer' + assert s.meta.get(http.URL) == "http://localhost/renderer" + assert s.meta.get("pyramid.route.name") == "renderer" - s = spans_by_name['pyramid.render'] - assert s.service == 'foobar' + s = spans_by_name["pyramid.render"] + assert s.service == "foobar" assert s.error == 0 - assert s.span_type == 'template' + assert s.span_type == "template" def test_http_exception_response(self): with pytest.raises(HTTPException): - self.app.get('/404/raise_exception', status=404) + self.app.get("/404/raise_exception", status=404) writer = self.tracer.writer spans = writer.pop() assert len(spans) == 1 s = spans[0] - assert s.service == 'foobar' - assert s.resource == '404' + assert s.service == "foobar" + assert s.resource == "404" assert s.error == 1 - assert s.span_type == 'web' - assert s.meta.get('http.method') == 'GET' + assert s.span_type == "web" + assert s.meta.get("http.method") == "GET" assert_span_http_status_code(s, 404) - assert s.meta.get(http.URL) == 'http://localhost/404/raise_exception' + assert s.meta.get(http.URL) == "http://localhost/404/raise_exception" def test_insert_tween_if_needed_already_set(self): - settings = {'pyramid.tweens': 'ddtrace.contrib.pyramid:trace_tween_factory'} + settings = {"pyramid.tweens": "ddtrace.contrib.pyramid:trace_tween_factory"} insert_tween_if_needed(settings) - assert settings['pyramid.tweens'] == 'ddtrace.contrib.pyramid:trace_tween_factory' + assert settings["pyramid.tweens"] == "ddtrace.contrib.pyramid:trace_tween_factory" def test_insert_tween_if_needed_none(self): - settings = {'pyramid.tweens': ''} + settings = {"pyramid.tweens": ""} insert_tween_if_needed(settings) - assert settings['pyramid.tweens'] == '' + assert settings["pyramid.tweens"] == "" def test_insert_tween_if_needed_excview(self): - settings = {'pyramid.tweens': 'pyramid.tweens.excview_tween_factory'} + settings = {"pyramid.tweens": "pyramid.tweens.excview_tween_factory"} insert_tween_if_needed(settings) assert ( - settings['pyramid.tweens'] == - 'ddtrace.contrib.pyramid:trace_tween_factory\npyramid.tweens.excview_tween_factory' + settings["pyramid.tweens"] + == "ddtrace.contrib.pyramid:trace_tween_factory\npyramid.tweens.excview_tween_factory" ) def test_insert_tween_if_needed_excview_and_other(self): - settings = {'pyramid.tweens': 'a.first.tween\npyramid.tweens.excview_tween_factory\na.last.tween\n'} + settings = {"pyramid.tweens": "a.first.tween\npyramid.tweens.excview_tween_factory\na.last.tween\n"} insert_tween_if_needed(settings) assert ( - settings['pyramid.tweens'] == - 'a.first.tween\n' - 'ddtrace.contrib.pyramid:trace_tween_factory\n' - 'pyramid.tweens.excview_tween_factory\n' - 'a.last.tween\n') + settings["pyramid.tweens"] == "a.first.tween\n" + "ddtrace.contrib.pyramid:trace_tween_factory\n" + "pyramid.tweens.excview_tween_factory\n" + "a.last.tween\n" + ) def test_insert_tween_if_needed_others(self): - settings = {'pyramid.tweens': 'a.random.tween\nand.another.one'} + settings = {"pyramid.tweens": "a.random.tween\nand.another.one"} insert_tween_if_needed(settings) assert ( - settings['pyramid.tweens'] == - 'a.random.tween\nand.another.one\nddtrace.contrib.pyramid:trace_tween_factory' + settings["pyramid.tweens"] == "a.random.tween\nand.another.one\nddtrace.contrib.pyramid:trace_tween_factory" ) def test_include_conflicts(self): # test that includes do not create conflicts - self.override_settings({'pyramid.includes': 'tests.contrib.pyramid.test_pyramid'}) - self.app.get('/404', status=404) + self.override_settings({"pyramid.includes": "tests.contrib.pyramid.test_pyramid"}) + self.app.get("/404", status=404) spans = self.tracer.writer.pop() assert len(spans) == 1 def test_200_ot(self): """OpenTracing version of test_200.""" - ot_tracer = init_tracer('pyramid_svc', self.tracer) + ot_tracer = init_tracer("pyramid_svc", self.tracer) - with ot_tracer.start_active_span('pyramid_get'): - res = self.app.get('/', status=200) - assert b'idx' in res.body + with ot_tracer.start_active_span("pyramid_get"): + res = self.app.get("/", status=200) + assert b"idx" in res.body writer = self.tracer.writer spans = writer.pop() @@ -349,14 +344,14 @@ def test_200_ot(self): assert ot_span.parent_id is None assert dd_span.parent_id == ot_span.span_id - assert ot_span.name == 'pyramid_get' - assert ot_span.service == 'pyramid_svc' + assert ot_span.name == "pyramid_get" + assert ot_span.service == "pyramid_svc" - assert dd_span.service == 'foobar' - assert dd_span.resource == 'GET index' + assert dd_span.service == "foobar" + assert dd_span.resource == "GET index" assert dd_span.error == 0 - assert dd_span.span_type == 'web' - assert dd_span.meta.get('http.method') == 'GET' + assert dd_span.span_type == "web" + assert dd_span.meta.get("http.method") == "GET" assert_span_http_status_code(dd_span, 200) - assert dd_span.meta.get(http.URL) == 'http://localhost/' - assert dd_span.meta.get('pyramid.route.name') == 'index' + assert dd_span.meta.get(http.URL) == "http://localhost/" + assert dd_span.meta.get("pyramid.route.name") == "index" diff --git a/tests/internal/test_writer.py b/tests/internal/test_writer.py index 3f28dae1ec3..4ee0087dfc4 100644 --- a/tests/internal/test_writer.py +++ b/tests/internal/test_writer.py @@ -10,7 +10,7 @@ from ..base import BaseTestCase -class RemoveAllFilter(): +class RemoveAllFilter: def __init__(self): self.filtered_traces = 0 @@ -19,7 +19,7 @@ def process_trace(self, trace): return None -class KeepAllFilter(): +class KeepAllFilter: def __init__(self): self.filtered_traces = 0 @@ -28,7 +28,7 @@ def process_trace(self, trace): return trace -class AddTagFilter(): +class AddTagFilter: def __init__(self, tag_name): self.tag_name = tag_name self.filtered_traces = 0 @@ -36,14 +36,14 @@ def __init__(self, tag_name): def process_trace(self, trace): self.filtered_traces += 1 for span in trace: - span.set_tag(self.tag_name, 'A value') + span.set_tag(self.tag_name, "A value") return trace class DummyAPI(API): def __init__(self): # Call API.__init__ to setup required properties - super(DummyAPI, self).__init__(hostname='localhost', port=8126) + super(DummyAPI, self).__init__(hostname="localhost", port=8126) self.traces = [] @@ -58,10 +58,9 @@ def send_traces(self, traces): class FailingAPI(object): - @staticmethod def send_traces(traces): - return [Exception('oops')] + return [Exception("oops")] class AgentWriterTests(BaseTestCase): @@ -75,10 +74,9 @@ def create_worker(self, filters=None, api_class=DummyAPI, enable_stats=False): self.api = api_class() worker.api = self.api for i in range(self.N_TRACES): - worker.write([ - Span(tracer=None, name='name', trace_id=i, span_id=j, parent_id=j - 1 or None) - for j in range(7) - ]) + worker.write( + [Span(tracer=None, name="name", trace_id=i, span_id=j, parent_id=j - 1 or None) for j in range(7)] + ) worker.stop() worker.join() return worker @@ -108,7 +106,7 @@ def test_filters_remove_all(self): self.assertEqual(filtr.filtered_traces, self.N_TRACES) def test_filters_add_tag(self): - tag_name = 'Tag' + tag_name = "Tag" filtr = AddTagFilter(tag_name) self.create_worker([filtr]) self.assertEqual(len(self.api.traces), self.N_TRACES) @@ -127,72 +125,71 @@ def test_filters_short_circuit(self): def test_no_dogstats(self): worker = self.create_worker() assert worker._send_stats is False - assert [ - ] == self.dogstatsd.gauge.mock_calls + assert [] == self.dogstatsd.gauge.mock_calls def test_dogstatsd(self): self.create_worker(enable_stats=True) assert [ - mock.call('datadog.tracer.heartbeat', 1), - mock.call('datadog.tracer.queue.max_length', 1000), + mock.call("datadog.tracer.heartbeat", 1), + mock.call("datadog.tracer.queue.max_length", 1000), ] == self.dogstatsd.gauge.mock_calls assert [ - mock.call('datadog.tracer.flushes'), - mock.call('datadog.tracer.flush.traces.total', 11, tags=None), - mock.call('datadog.tracer.flush.spans.total', 77, tags=None), - mock.call('datadog.tracer.flush.traces_filtered.total', 0, tags=None), - mock.call('datadog.tracer.api.requests.total', 11, tags=None), - mock.call('datadog.tracer.api.errors.total', 0, tags=None), - mock.call('datadog.tracer.api.responses.total', 11, tags=['status:200']), - mock.call('datadog.tracer.queue.dropped.traces', 0), - mock.call('datadog.tracer.queue.enqueued.traces', 11), - mock.call('datadog.tracer.queue.enqueued.spans', 77), - mock.call('datadog.tracer.shutdown'), + mock.call("datadog.tracer.flushes"), + mock.call("datadog.tracer.flush.traces.total", 11, tags=None), + mock.call("datadog.tracer.flush.spans.total", 77, tags=None), + mock.call("datadog.tracer.flush.traces_filtered.total", 0, tags=None), + mock.call("datadog.tracer.api.requests.total", 11, tags=None), + mock.call("datadog.tracer.api.errors.total", 0, tags=None), + mock.call("datadog.tracer.api.responses.total", 11, tags=["status:200"]), + mock.call("datadog.tracer.queue.dropped.traces", 0), + mock.call("datadog.tracer.queue.enqueued.traces", 11), + mock.call("datadog.tracer.queue.enqueued.spans", 77), + mock.call("datadog.tracer.shutdown"), ] == self.dogstatsd.increment.mock_calls histogram_calls = [ - mock.call('datadog.tracer.flush.traces', 11, tags=None), - mock.call('datadog.tracer.flush.spans', 77, tags=None), - mock.call('datadog.tracer.flush.traces_filtered', 0, tags=None), - mock.call('datadog.tracer.api.requests', 11, tags=None), - mock.call('datadog.tracer.api.errors', 0, tags=None), - mock.call('datadog.tracer.api.responses', 11, tags=['status:200']), + mock.call("datadog.tracer.flush.traces", 11, tags=None), + mock.call("datadog.tracer.flush.spans", 77, tags=None), + mock.call("datadog.tracer.flush.traces_filtered", 0, tags=None), + mock.call("datadog.tracer.api.requests", 11, tags=None), + mock.call("datadog.tracer.api.errors", 0, tags=None), + mock.call("datadog.tracer.api.responses", 11, tags=["status:200"]), ] - if hasattr(time, 'thread_time'): - histogram_calls.append(mock.call('datadog.tracer.writer.cpu_time', mock.ANY)) + if hasattr(time, "thread_time"): + histogram_calls.append(mock.call("datadog.tracer.writer.cpu_time", mock.ANY)) assert histogram_calls == self.dogstatsd.histogram.mock_calls def test_dogstatsd_failing_api(self): self.create_worker(api_class=FailingAPI, enable_stats=True) assert [ - mock.call('datadog.tracer.heartbeat', 1), - mock.call('datadog.tracer.queue.max_length', 1000), + mock.call("datadog.tracer.heartbeat", 1), + mock.call("datadog.tracer.queue.max_length", 1000), ] == self.dogstatsd.gauge.mock_calls assert [ - mock.call('datadog.tracer.flushes'), - mock.call('datadog.tracer.flush.traces.total', 11, tags=None), - mock.call('datadog.tracer.flush.spans.total', 77, tags=None), - mock.call('datadog.tracer.flush.traces_filtered.total', 0, tags=None), - mock.call('datadog.tracer.api.requests.total', 1, tags=None), - mock.call('datadog.tracer.api.errors.total', 1, tags=None), - mock.call('datadog.tracer.queue.dropped.traces', 0), - mock.call('datadog.tracer.queue.enqueued.traces', 11), - mock.call('datadog.tracer.queue.enqueued.spans', 77), - mock.call('datadog.tracer.shutdown'), + mock.call("datadog.tracer.flushes"), + mock.call("datadog.tracer.flush.traces.total", 11, tags=None), + mock.call("datadog.tracer.flush.spans.total", 77, tags=None), + mock.call("datadog.tracer.flush.traces_filtered.total", 0, tags=None), + mock.call("datadog.tracer.api.requests.total", 1, tags=None), + mock.call("datadog.tracer.api.errors.total", 1, tags=None), + mock.call("datadog.tracer.queue.dropped.traces", 0), + mock.call("datadog.tracer.queue.enqueued.traces", 11), + mock.call("datadog.tracer.queue.enqueued.spans", 77), + mock.call("datadog.tracer.shutdown"), ] == self.dogstatsd.increment.mock_calls histogram_calls = [ - mock.call('datadog.tracer.flush.traces', 11, tags=None), - mock.call('datadog.tracer.flush.spans', 77, tags=None), - mock.call('datadog.tracer.flush.traces_filtered', 0, tags=None), - mock.call('datadog.tracer.api.requests', 1, tags=None), - mock.call('datadog.tracer.api.errors', 1, tags=None), + mock.call("datadog.tracer.flush.traces", 11, tags=None), + mock.call("datadog.tracer.flush.spans", 77, tags=None), + mock.call("datadog.tracer.flush.traces_filtered", 0, tags=None), + mock.call("datadog.tracer.api.requests", 1, tags=None), + mock.call("datadog.tracer.api.errors", 1, tags=None), ] - if hasattr(time, 'thread_time'): - histogram_calls.append(mock.call('datadog.tracer.writer.cpu_time', mock.ANY)) + if hasattr(time, "thread_time"): + histogram_calls.append(mock.call("datadog.tracer.writer.cpu_time", mock.ANY)) assert histogram_calls == self.dogstatsd.histogram.mock_calls @@ -203,9 +200,7 @@ def test_queue_full(): q.put(2) q.put([3]) q.put([4, 4]) - assert (list(q.queue) == [[1], 2, [4, 4]] or - list(q.queue) == [[1], [4, 4], [3]] or - list(q.queue) == [[4, 4], 2, [3]]) + assert list(q.queue) == [[1], 2, [4, 4]] or list(q.queue) == [[1], [4, 4], [3]] or list(q.queue) == [[4, 4], 2, [3]] assert q.dropped == 1 assert q.accepted == 4 assert q.accepted_lengths == 5 diff --git a/tests/opentracer/test_tracer.py b/tests/opentracer/test_tracer.py index 5ca46eb30c4..7888e5973f9 100644 --- a/tests/opentracer/test_tracer.py +++ b/tests/opentracer/test_tracer.py @@ -23,80 +23,77 @@ class TestTracerConfig(object): def test_config(self): """Test the configuration of the tracer""" - config = {'enabled': True} - tracer = Tracer(service_name='myservice', config=config) + config = {"enabled": True} + tracer = Tracer(service_name="myservice", config=config) - assert tracer._service_name == 'myservice' + assert tracer._service_name == "myservice" assert tracer._enabled is True def test_no_service_name(self): """A service_name should be generated if one is not provided.""" tracer = Tracer() - assert tracer._service_name == 'pytest' + assert tracer._service_name == "pytest" def test_multiple_tracer_configs(self): """Ensure that a tracer config is a copy of the passed config.""" - config = {'enabled': True} + config = {"enabled": True} - tracer1 = Tracer(service_name='serv1', config=config) - assert tracer1._service_name == 'serv1' + tracer1 = Tracer(service_name="serv1", config=config) + assert tracer1._service_name == "serv1" - config['enabled'] = False - tracer2 = Tracer(service_name='serv2', config=config) + config["enabled"] = False + tracer2 = Tracer(service_name="serv2", config=config) # Ensure tracer1's config was not mutated - assert tracer1._service_name == 'serv1' + assert tracer1._service_name == "serv1" assert tracer1._enabled is True - assert tracer2._service_name == 'serv2' + assert tracer2._service_name == "serv2" assert tracer2._enabled is False def test_invalid_config_key(self): """A config with an invalid key should raise a ConfigException.""" - config = {'enabeld': False} + config = {"enabeld": False} # No debug flag should not raise an error - tracer = Tracer(service_name='mysvc', config=config) + tracer = Tracer(service_name="mysvc", config=config) # With debug flag should raise an error - config['debug'] = True + config["debug"] = True with pytest.raises(ConfigException) as ce_info: tracer = Tracer(config=config) - assert 'enabeld' in str(ce_info) + assert "enabeld" in str(ce_info) assert tracer is not None # Test with multiple incorrect keys - config['setttings'] = {} + config["setttings"] = {} with pytest.raises(ConfigException) as ce_info: - tracer = Tracer(service_name='mysvc', config=config) - assert ['enabeld', 'setttings'] in str(ce_info) + tracer = Tracer(service_name="mysvc", config=config) + assert ["enabeld", "setttings"] in str(ce_info) assert tracer is not None def test_global_tags(self): """Global tags should be passed from the opentracer to the tracer.""" config = { - 'global_tags': { - 'tag1': 'value1', - 'tag2': 2, - }, + "global_tags": {"tag1": "value1", "tag2": 2,}, } - tracer = Tracer(service_name='mysvc', config=config) - with tracer.start_span('myop') as span: + tracer = Tracer(service_name="mysvc", config=config) + with tracer.start_span("myop") as span: # global tags should be attached to generated all datadog spans - assert span._dd_span.get_tag('tag1') == 'value1' - assert span._dd_span.get_metric('tag2') == 2 + assert span._dd_span.get_tag("tag1") == "value1" + assert span._dd_span.get_metric("tag2") == 2 - with tracer.start_span('myop2') as span2: - assert span2._dd_span.get_tag('tag1') == 'value1' - assert span2._dd_span.get_metric('tag2') == 2 + with tracer.start_span("myop2") as span2: + assert span2._dd_span.get_tag("tag1") == "value1" + assert span2._dd_span.get_metric("tag2") == 2 class TestTracer(object): def test_start_span(self, ot_tracer, writer): """Start and finish a span.""" - with ot_tracer.start_span('myop') as span: + with ot_tracer.start_span("myop") as span: pass # span should be finished when the context manager exits @@ -108,16 +105,16 @@ def test_start_span(self, ot_tracer, writer): def test_start_span_references(self, ot_tracer, writer): """Start a span using references.""" - with ot_tracer.start_span('one', references=[child_of()]): + with ot_tracer.start_span("one", references=[child_of()]): pass spans = writer.pop() assert spans[0].parent_id is None - root = ot_tracer.start_active_span('root') + root = ot_tracer.start_active_span("root") # create a child using a parent reference that is not the context parent - with ot_tracer.start_active_span('one'): - with ot_tracer.start_active_span('two', references=[child_of(root.span)]): + with ot_tracer.start_active_span("one"): + with ot_tracer.start_active_span("two", references=[child_of(root.span)]): pass root.close() @@ -127,9 +124,9 @@ def test_start_span_references(self, ot_tracer, writer): def test_start_span_custom_start_time(self, ot_tracer): """Start a span with a custom start time.""" t = 100 - with mock.patch('ddtrace.span.time_ns') as time: + with mock.patch("ddtrace.span.time_ns") as time: time.return_value = 102 * 1e9 - with ot_tracer.start_span('myop', start_time=t) as span: + with ot_tracer.start_span("myop", start_time=t) as span: pass assert span._dd_span.start == t @@ -139,8 +136,8 @@ def test_start_span_with_spancontext(self, ot_tracer, writer): """Start and finish a span using a span context as the child_of reference. """ - with ot_tracer.start_span('myop') as span: - with ot_tracer.start_span('myop', child_of=span.context) as span2: + with ot_tracer.start_span("myop") as span: + with ot_tracer.start_span("myop", child_of=span.context) as span2: pass # span should be finished when the context manager exits @@ -155,36 +152,36 @@ def test_start_span_with_spancontext(self, ot_tracer, writer): def test_start_span_with_tags(self, ot_tracer): """Create a span with initial tags.""" - tags = {'key': 'value', 'key2': 'value2'} - with ot_tracer.start_span('myop', tags=tags) as span: + tags = {"key": "value", "key2": "value2"} + with ot_tracer.start_span("myop", tags=tags) as span: pass - assert span._dd_span.get_tag('key') == 'value' - assert span._dd_span.get_tag('key2') == 'value2' + assert span._dd_span.get_tag("key") == "value" + assert span._dd_span.get_tag("key2") == "value2" def test_start_span_with_resource_name_tag(self, ot_tracer): """Create a span with the tag to set the resource name""" - tags = {'resource.name': 'value', 'key2': 'value2'} - with ot_tracer.start_span('myop', tags=tags) as span: + tags = {"resource.name": "value", "key2": "value2"} + with ot_tracer.start_span("myop", tags=tags) as span: pass # Span resource name should be set to tag value, and should not get set as # a tag on the underlying span. - assert span._dd_span.resource == 'value' - assert span._dd_span.get_tag('resource.name') is None + assert span._dd_span.resource == "value" + assert span._dd_span.get_tag("resource.name") is None # Other tags are set as normal - assert span._dd_span.get_tag('key2') == 'value2' + assert span._dd_span.get_tag("key2") == "value2" def test_start_active_span_multi_child(self, ot_tracer, writer): """Start and finish multiple child spans. This should ensure that child spans can be created 2 levels deep. """ - with ot_tracer.start_active_span('myfirstop') as scope1: + with ot_tracer.start_active_span("myfirstop") as scope1: time.sleep(0.009) - with ot_tracer.start_active_span('mysecondop') as scope2: + with ot_tracer.start_active_span("mysecondop") as scope2: time.sleep(0.007) - with ot_tracer.start_active_span('mythirdop') as scope3: + with ot_tracer.start_active_span("mythirdop") as scope3: time.sleep(0.005) # spans should be finished when the context manager exits @@ -213,11 +210,11 @@ def test_start_active_span_multi_child_siblings(self, ot_tracer, writer): This should test to ensure a parent can have multiple child spans at the same level. """ - with ot_tracer.start_active_span('myfirstop') as scope1: + with ot_tracer.start_active_span("myfirstop") as scope1: time.sleep(0.009) - with ot_tracer.start_active_span('mysecondop') as scope2: + with ot_tracer.start_active_span("mysecondop") as scope2: time.sleep(0.007) - with ot_tracer.start_active_span('mythirdop') as scope3: + with ot_tracer.start_active_span("mythirdop") as scope3: time.sleep(0.005) # spans should be finished when the context manager exits @@ -246,11 +243,11 @@ def test_start_span_manual_child_of(self, ot_tracer, writer): Spans should be created without parents since there will be no call for the active span. """ - root = ot_tracer.start_span('zero') + root = ot_tracer.start_span("zero") - with ot_tracer.start_span('one', child_of=root): - with ot_tracer.start_span('two', child_of=root): - with ot_tracer.start_span('three', child_of=root): + with ot_tracer.start_span("one", child_of=root): + with ot_tracer.start_span("two", child_of=root): + with ot_tracer.start_span("three", child_of=root): pass root.finish() @@ -261,20 +258,17 @@ def test_start_span_manual_child_of(self, ot_tracer, writer): assert spans[1].parent_id is root._dd_span.span_id assert spans[2].parent_id is root._dd_span.span_id assert spans[3].parent_id is root._dd_span.span_id - assert ( - spans[0].trace_id == spans[1].trace_id and - spans[1].trace_id == spans[2].trace_id - ) + assert spans[0].trace_id == spans[1].trace_id and spans[1].trace_id == spans[2].trace_id def test_start_span_no_active_span(self, ot_tracer, writer): """Start spans without using a scope manager. Spans should be created without parents since there will be no call for the active span. """ - with ot_tracer.start_span('one', ignore_active_span=True): - with ot_tracer.start_span('two', ignore_active_span=True): + with ot_tracer.start_span("one", ignore_active_span=True): + with ot_tracer.start_span("two", ignore_active_span=True): pass - with ot_tracer.start_span('three', ignore_active_span=True): + with ot_tracer.start_span("three", ignore_active_span=True): pass spans = writer.pop() @@ -285,15 +279,15 @@ def test_start_span_no_active_span(self, ot_tracer, writer): assert spans[2].parent_id is None # and that each span is a new trace assert ( - spans[0].trace_id != spans[1].trace_id and - spans[1].trace_id != spans[2].trace_id and - spans[0].trace_id != spans[2].trace_id + spans[0].trace_id != spans[1].trace_id + and spans[1].trace_id != spans[2].trace_id + and spans[0].trace_id != spans[2].trace_id ) def test_start_active_span_child_finish_after_parent(self, ot_tracer, writer): """Start a child span and finish it after its parent.""" - span1 = ot_tracer.start_active_span('one').span - span2 = ot_tracer.start_active_span('two').span + span1 = ot_tracer.start_active_span("one").span + span2 = ot_tracer.start_active_span("two").span span1.finish() time.sleep(0.005) span2.finish() @@ -314,7 +308,7 @@ def test_start_span_multi_intertwined(self, ot_tracer, writer): event = threading.Event() def trace_one(): - id = 11 # noqa: A001 + id = 11 # noqa: A001 with ot_tracer.start_active_span(str(id)): id += 1 with ot_tracer.start_active_span(str(id)): @@ -323,7 +317,7 @@ def trace_one(): event.set() def trace_two(): - id = 21 # noqa: A001 + id = 21 # noqa: A001 event.wait() with ot_tracer.start_active_span(str(id)): id += 1 @@ -347,12 +341,12 @@ def trace_two(): # trace_one will finish before trace_two so its spans should be written # before the spans from trace_two, let's confirm this - assert spans[0].name == '11' - assert spans[1].name == '12' - assert spans[2].name == '13' - assert spans[3].name == '21' - assert spans[4].name == '22' - assert spans[5].name == '23' + assert spans[0].name == "11" + assert spans[1].name == "12" + assert spans[2].name == "13" + assert spans[3].name == "21" + assert spans[4].name == "22" + assert spans[5].name == "23" # next let's ensure that each span has the correct parent: # trace_one @@ -366,61 +360,53 @@ def trace_two(): # finally we should ensure that the trace_ids are reasonable # trace_one - assert ( - spans[0].trace_id == spans[1].trace_id and - spans[1].trace_id == spans[2].trace_id - ) + assert spans[0].trace_id == spans[1].trace_id and spans[1].trace_id == spans[2].trace_id # traces should be independent assert spans[2].trace_id != spans[3].trace_id # trace_two - assert ( - spans[3].trace_id == spans[4].trace_id and - spans[4].trace_id == spans[5].trace_id - ) + assert spans[3].trace_id == spans[4].trace_id and spans[4].trace_id == spans[5].trace_id def test_start_active_span(self, ot_tracer, writer): - with ot_tracer.start_active_span('one') as scope: + with ot_tracer.start_active_span("one") as scope: pass - assert scope.span._dd_span.name == 'one' + assert scope.span._dd_span.name == "one" assert scope.span.finished spans = writer.pop() assert spans def test_start_active_span_finish_on_close(self, ot_tracer, writer): - with ot_tracer.start_active_span('one', finish_on_close=False) as scope: + with ot_tracer.start_active_span("one", finish_on_close=False) as scope: pass - assert scope.span._dd_span.name == 'one' + assert scope.span._dd_span.name == "one" assert not scope.span.finished spans = writer.pop() assert not spans def test_start_active_span_nested(self, ot_tracer): """Test the active span of multiple nested calls of start_active_span.""" - with ot_tracer.start_active_span('one') as outer_scope: + with ot_tracer.start_active_span("one") as outer_scope: assert ot_tracer.active_span == outer_scope.span - with ot_tracer.start_active_span('two') as inner_scope: + with ot_tracer.start_active_span("two") as inner_scope: assert ot_tracer.active_span == inner_scope.span - with ot_tracer.start_active_span( - 'three' - ) as innest_scope: # why isn't it innest? innermost so verbose + with ot_tracer.start_active_span("three") as innest_scope: # why isn't it innest? innermost so verbose assert ot_tracer.active_span == innest_scope.span - with ot_tracer.start_active_span('two') as inner_scope: + with ot_tracer.start_active_span("two") as inner_scope: assert ot_tracer.active_span == inner_scope.span assert ot_tracer.active_span == outer_scope.span assert ot_tracer.active_span is None def test_start_active_span_trace(self, ot_tracer, writer): """Test the active span of multiple nested calls of start_active_span.""" - with ot_tracer.start_active_span('one') as outer_scope: - outer_scope.span.set_tag('outer', 2) - with ot_tracer.start_active_span('two') as inner_scope: - inner_scope.span.set_tag('inner', 3) - with ot_tracer.start_active_span('two') as inner_scope: - inner_scope.span.set_tag('inner', 3) - with ot_tracer.start_active_span('three') as innest_scope: - innest_scope.span.set_tag('innerest', 4) + with ot_tracer.start_active_span("one") as outer_scope: + outer_scope.span.set_tag("outer", 2) + with ot_tracer.start_active_span("two") as inner_scope: + inner_scope.span.set_tag("inner", 3) + with ot_tracer.start_active_span("two") as inner_scope: + inner_scope.span.set_tag("inner", 3) + with ot_tracer.start_active_span("three") as innest_scope: + innest_scope.span.set_tag("innerest", 4) spans = writer.pop() @@ -474,9 +460,7 @@ def test_http_headers_base(self, ot_tracer): def test_http_headers_baggage(self, ot_tracer): """extract should undo inject for http headers.""" - span_ctx = SpanContext( - trace_id=123, span_id=456, baggage={'test': 4, 'test2': 'string'} - ) + span_ctx = SpanContext(trace_id=123, span_id=456, baggage={"test": 4, "test2": "string"}) carrier = {} ot_tracer.inject(span_ctx, Format.HTTP_HEADERS, carrier) @@ -497,9 +481,7 @@ def test_empty_propagated_context(self, ot_tracer): def test_text(self, ot_tracer): """extract should undo inject for http headers""" - span_ctx = SpanContext( - trace_id=123, span_id=456, baggage={'test': 4, 'test2': 'string'} - ) + span_ctx = SpanContext(trace_id=123, span_id=456, baggage={"test": 4, "test2": "string"}) carrier = {} ot_tracer.inject(span_ctx, Format.TEXT_MAP, carrier) @@ -512,9 +494,7 @@ def test_text(self, ot_tracer): def test_corrupted_propagated_context(self, ot_tracer): """Corrupted context should raise a SpanContextCorruptedException.""" - span_ctx = SpanContext( - trace_id=123, span_id=456, baggage={'test': 4, 'test2': 'string'} - ) + span_ctx = SpanContext(trace_id=123, span_id=456, baggage={"test": 4, "test2": "string"}) carrier = {} ot_tracer.inject(span_ctx, Format.TEXT_MAP, carrier) @@ -530,12 +510,12 @@ def test_corrupted_propagated_context(self, ot_tracer): def test_immutable_span_context(self, ot_tracer): """Span contexts should be immutable.""" - with ot_tracer.start_span('root') as root: + with ot_tracer.start_span("root") as root: ctx_before = root.context - root.set_baggage_item('test', 2) + root.set_baggage_item("test", 2) assert ctx_before is not root.context - with ot_tracer.start_span('child') as level1: - with ot_tracer.start_span('child') as level2: + with ot_tracer.start_span("child") as level1: + with ot_tracer.start_span("child") as level2: pass assert root.context is not level1.context assert level2.context is not level1.context @@ -543,27 +523,27 @@ def test_immutable_span_context(self, ot_tracer): def test_inherited_baggage(self, ot_tracer): """Baggage should be inherited by child spans.""" - with ot_tracer.start_active_span('root') as root: + with ot_tracer.start_active_span("root") as root: # this should be passed down to the child - root.span.set_baggage_item('root', 1) - root.span.set_baggage_item('root2', 1) - with ot_tracer.start_active_span('child') as level1: - level1.span.set_baggage_item('level1', 1) - with ot_tracer.start_active_span('child') as level2: - level2.span.set_baggage_item('level2', 1) + root.span.set_baggage_item("root", 1) + root.span.set_baggage_item("root2", 1) + with ot_tracer.start_active_span("child") as level1: + level1.span.set_baggage_item("level1", 1) + with ot_tracer.start_active_span("child") as level2: + level2.span.set_baggage_item("level2", 1) # ensure immutability assert level1.span.context is not root.span.context assert level2.span.context is not level1.span.context # level1 should have inherited the baggage of root - assert level1.span.get_baggage_item('root') - assert level1.span.get_baggage_item('root2') + assert level1.span.get_baggage_item("root") + assert level1.span.get_baggage_item("root2") # level2 should have inherited the baggage of both level1 and level2 - assert level2.span.get_baggage_item('root') - assert level2.span.get_baggage_item('root2') - assert level2.span.get_baggage_item('level1') - assert level2.span.get_baggage_item('level2') + assert level2.span.get_baggage_item("root") + assert level2.span.get_baggage_item("root2") + assert level2.span.get_baggage_item("level1") + assert level2.span.get_baggage_item("level2") class TestTracerCompatibility(object): @@ -574,14 +554,14 @@ def test_required_dd_fields(self): by the underlying datadog tracer. """ # a service name is required - tracer = Tracer('service') - with tracer.start_span('my_span') as span: + tracer = Tracer("service") + with tracer.start_span("my_span") as span: assert span._dd_span.service def test_set_global_tracer(): """Sanity check for set_global_tracer""" - my_tracer = Tracer('service') + my_tracer = Tracer("service") set_global_tracer(my_tracer) assert opentracing.tracer is my_tracer diff --git a/tests/propagation/test_http.py b/tests/propagation/test_http.py index 6249c037a4a..267954fd088 100644 --- a/tests/propagation/test_http.py +++ b/tests/propagation/test_http.py @@ -19,61 +19,55 @@ class TestHttpPropagation(TestCase): def test_inject(self): tracer = get_dummy_tracer() - with tracer.trace('global_root_span') as span: + with tracer.trace("global_root_span") as span: span.context.sampling_priority = 2 - span.context._dd_origin = 'synthetics' + span.context._dd_origin = "synthetics" headers = {} propagator = HTTPPropagator() propagator.inject(span.context, headers) assert int(headers[HTTP_HEADER_TRACE_ID]) == span.trace_id assert int(headers[HTTP_HEADER_PARENT_ID]) == span.span_id - assert ( - int(headers[HTTP_HEADER_SAMPLING_PRIORITY]) == - span.context.sampling_priority - ) - assert ( - headers[HTTP_HEADER_ORIGIN] == - span.context._dd_origin - ) + assert int(headers[HTTP_HEADER_SAMPLING_PRIORITY]) == span.context.sampling_priority + assert headers[HTTP_HEADER_ORIGIN] == span.context._dd_origin def test_extract(self): tracer = get_dummy_tracer() headers = { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'x-datadog-sampling-priority': '1', - 'x-datadog-origin': 'synthetics', + "x-datadog-trace-id": "1234", + "x-datadog-parent-id": "5678", + "x-datadog-sampling-priority": "1", + "x-datadog-origin": "synthetics", } propagator = HTTPPropagator() context = propagator.extract(headers) tracer.context_provider.activate(context) - with tracer.trace('local_root_span') as span: + with tracer.trace("local_root_span") as span: assert span.trace_id == 1234 assert span.parent_id == 5678 assert span.context.sampling_priority == 1 - assert span.context._dd_origin == 'synthetics' + assert span.context._dd_origin == "synthetics" def test_WSGI_extract(self): """Ensure we support the WSGI formatted headers as well.""" tracer = get_dummy_tracer() headers = { - 'HTTP_X_DATADOG_TRACE_ID': '1234', - 'HTTP_X_DATADOG_PARENT_ID': '5678', - 'HTTP_X_DATADOG_SAMPLING_PRIORITY': '1', - 'HTTP_X_DATADOG_ORIGIN': 'synthetics', + "HTTP_X_DATADOG_TRACE_ID": "1234", + "HTTP_X_DATADOG_PARENT_ID": "5678", + "HTTP_X_DATADOG_SAMPLING_PRIORITY": "1", + "HTTP_X_DATADOG_ORIGIN": "synthetics", } propagator = HTTPPropagator() context = propagator.extract(headers) tracer.context_provider.activate(context) - with tracer.trace('local_root_span') as span: + with tracer.trace("local_root_span") as span: assert span.trace_id == 1234 assert span.parent_id == 5678 assert span.context.sampling_priority == 1 - assert span.context._dd_origin == 'synthetics' + assert span.context._dd_origin == "synthetics" diff --git a/tox.ini b/tox.ini index f585d1dc435..5e40dd2a22c 100644 --- a/tox.ini +++ b/tox.ini @@ -849,8 +849,9 @@ exclude= # Ignore: # A003: XXX is a python builtin, consider renaming the class attribute # G201 Logging: .exception(...) should be used instead of .error(..., exc_info=True) +# E231,W503: not respected by black # We ignore most of the D errors because there are too many; the goal is to fix them eventually -ignore = W504,A003,G201,D100,D101,D102,D103,D104,D105,D106,D107,D200,D202,D204,D205,D208,D210,D300,D400,D401,D403,D413 +ignore = W503,E231,A003,G201,D100,D101,D102,D103,D104,D105,D106,D107,D200,D202,D204,D205,D208,D210,D300,D400,D401,D403,D413 enable-extensions=G rst-roles = class,meth,obj,ref rst-directives = py:data From 7691e9018e6813552a3ad7b908d8bda9aa33c895 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 7 Feb 2020 13:17:43 +0100 Subject: [PATCH 50/81] fix(writer): start thread as late as possible (#1193) Fixes #1192 --- ddtrace/_worker.py | 4 +++- ddtrace/internal/writer.py | 8 +++++++- tests/test_tracer.py | 10 ---------- tests/test_worker.py | 13 +++++++++++++ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/ddtrace/_worker.py b/ddtrace/_worker.py index ed25db41837..8b2b6db7a6b 100644 --- a/ddtrace/_worker.py +++ b/ddtrace/_worker.py @@ -32,13 +32,14 @@ def __init__(self, interval=_DEFAULT_INTERVAL, exit_timeout=None, name=None, dae self._thread = threading.Thread(target=self._target, name=name) self._thread.daemon = daemon self._stop = threading.Event() + self.started = False self.interval = interval self.exit_timeout = exit_timeout atexit.register(self._atexit) def _atexit(self): self.stop() - if self.exit_timeout is not None: + if self.exit_timeout is not None and self.started: key = 'ctrl-break' if os.name == 'nt' else 'ctrl-c' _LOG.debug( 'Waiting %d seconds for %s to finish. Hit %s to quit.', @@ -50,6 +51,7 @@ def start(self): """Start the periodic worker.""" _LOG.debug('Starting %s thread', self._thread.name) self._thread.start() + self.started = True def stop(self): """Stop the worker.""" diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 13250b8370c..d39c3baa8b6 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -50,7 +50,7 @@ def __init__( ) if hasattr(time, "thread_time"): self._last_thread_time = time.thread_time() - self.start() + self._started = False def recreate(self): """ Create a new instance of :class:`AgentWriter` using the same settings from this instance @@ -76,6 +76,12 @@ def _send_stats(self): return bool(config.health_metrics_enabled and self.dogstatsd) def write(self, spans=None, services=None): + # Start the AgentWriter on first write. + # Starting it earlier might be an issue with gevent, see: + # https://github.com/DataDog/dd-trace-py/issues/1192 + if self._started is False: + self.start() + self._started = True if spans: self._trace_queue.put(spans) diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 2ce840a2109..2da8aa2b0bb 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -629,11 +629,6 @@ def task(t, errors): assert t.writer != original_writer assert t.writer._trace_queue != original_writer._trace_queue - # Stop the background worker so we don't accidetnally flush the - # queue before we can assert on it - t.writer.stop() - t.writer.join() - # Assert the trace got written into the correct queue assert original_writer._trace_queue.qsize() == 0 assert t.writer._trace_queue.qsize() == 1 @@ -656,11 +651,6 @@ def task(t, errors): assert t.writer == original_writer assert t.writer._trace_queue == original_writer._trace_queue - # Stop the background worker so we don't accidentally flush the - # queue before we can assert on it - t.writer.stop() - t.writer.join() - # Assert the trace got written into the correct queue assert original_writer._trace_queue.qsize() == 1 assert t.writer._trace_queue.qsize() == 1 diff --git a/tests/test_worker.py b/tests/test_worker.py index a08d2a2af70..73f682c7fb6 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -56,3 +56,16 @@ def test_restart(): with pytest.raises(RuntimeError): w.start() + + +def test_atexit(): + w = _worker.PeriodicWorkerThread(exit_timeout=1) + w.start() + assert w.started + w._atexit() + + +def test_atexit_not_started(): + w = _worker.PeriodicWorkerThread(exit_timeout=1) + assert not w.started + w._atexit() From 7d92f7ce198eca408bf5fb33ecae06e965c09c77 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 10 Feb 2020 09:11:43 -0500 Subject: [PATCH 51/81] internal: Vendor psutil dependency (#1160) * internal: Vendor psutil dependency * fix flake8/black issues * update comment * move vendor's setup code into ddtrace/vendor/*/setup.py * catch any exceptions loading vendor setup * add back required dep * remove unnecessary caching step * re-run setup.py develop before every test run * simplify setup.py * fix black formatting * fix bad merge --- ddtrace/internal/runtime/metric_collectors.py | 4 +- ddtrace/vendor/__init__.py | 10 + ddtrace/vendor/psutil/__init__.py | 2516 +++++++++++ ddtrace/vendor/psutil/_common.py | 651 +++ ddtrace/vendor/psutil/_compat.py | 332 ++ ddtrace/vendor/psutil/_psaix.py | 554 +++ ddtrace/vendor/psutil/_psbsd.py | 905 ++++ ddtrace/vendor/psutil/_pslinux.py | 2096 ++++++++++ ddtrace/vendor/psutil/_psosx.py | 568 +++ ddtrace/vendor/psutil/_psposix.py | 179 + ddtrace/vendor/psutil/_pssunos.py | 720 ++++ ddtrace/vendor/psutil/_psutil_aix.c | 1137 +++++ ddtrace/vendor/psutil/_psutil_bsd.c | 1093 +++++ ddtrace/vendor/psutil/_psutil_common.c | 132 + ddtrace/vendor/psutil/_psutil_common.h | 31 + ddtrace/vendor/psutil/_psutil_linux.c | 668 +++ ddtrace/vendor/psutil/_psutil_osx.c | 1905 +++++++++ ddtrace/vendor/psutil/_psutil_posix.c | 691 +++ ddtrace/vendor/psutil/_psutil_posix.h | 8 + ddtrace/vendor/psutil/_psutil_sunos.c | 1776 ++++++++ ddtrace/vendor/psutil/_psutil_windows.c | 3723 +++++++++++++++++ ddtrace/vendor/psutil/_pswindows.py | 1127 +++++ ddtrace/vendor/psutil/arch/aix/common.c | 79 + ddtrace/vendor/psutil/arch/aix/common.h | 31 + ddtrace/vendor/psutil/arch/aix/ifaddrs.c | 149 + ddtrace/vendor/psutil/arch/aix/ifaddrs.h | 35 + .../vendor/psutil/arch/aix/net_connections.c | 287 ++ .../vendor/psutil/arch/aix/net_connections.h | 15 + .../psutil/arch/aix/net_kernel_structs.h | 111 + .../vendor/psutil/arch/freebsd/proc_socks.c | 368 ++ .../vendor/psutil/arch/freebsd/proc_socks.h | 9 + ddtrace/vendor/psutil/arch/freebsd/specific.c | 1115 +++++ ddtrace/vendor/psutil/arch/freebsd/specific.h | 34 + .../vendor/psutil/arch/freebsd/sys_socks.c | 362 ++ .../vendor/psutil/arch/freebsd/sys_socks.h | 10 + ddtrace/vendor/psutil/arch/netbsd/socks.c | 447 ++ ddtrace/vendor/psutil/arch/netbsd/socks.h | 10 + ddtrace/vendor/psutil/arch/netbsd/specific.c | 684 +++ ddtrace/vendor/psutil/arch/netbsd/specific.h | 29 + ddtrace/vendor/psutil/arch/openbsd/specific.c | 791 ++++ ddtrace/vendor/psutil/arch/openbsd/specific.h | 27 + ddtrace/vendor/psutil/arch/osx/process_info.c | 382 ++ ddtrace/vendor/psutil/arch/osx/process_info.h | 18 + ddtrace/vendor/psutil/arch/solaris/environ.c | 405 ++ ddtrace/vendor/psutil/arch/solaris/environ.h | 19 + .../vendor/psutil/arch/solaris/v10/ifaddrs.c | 125 + .../vendor/psutil/arch/solaris/v10/ifaddrs.h | 26 + ddtrace/vendor/psutil/arch/windows/global.c | 234 ++ ddtrace/vendor/psutil/arch/windows/global.h | 77 + .../vendor/psutil/arch/windows/inet_ntop.c | 45 + .../vendor/psutil/arch/windows/inet_ntop.h | 17 + ddtrace/vendor/psutil/arch/windows/ntextapi.h | 507 +++ .../psutil/arch/windows/process_handles.c | 513 +++ .../psutil/arch/windows/process_handles.h | 10 + .../vendor/psutil/arch/windows/process_info.c | 1027 +++++ .../vendor/psutil/arch/windows/process_info.h | 31 + ddtrace/vendor/psutil/arch/windows/security.c | 138 + ddtrace/vendor/psutil/arch/windows/security.h | 13 + ddtrace/vendor/psutil/arch/windows/services.c | 485 +++ ddtrace/vendor/psutil/arch/windows/services.h | 17 + ddtrace/vendor/psutil/arch/windows/wmi.c | 115 + ddtrace/vendor/psutil/arch/windows/wmi.h | 12 + ddtrace/vendor/psutil/setup.py | 227 + setup.py | 4 +- 64 files changed, 29862 insertions(+), 4 deletions(-) create mode 100644 ddtrace/vendor/psutil/__init__.py create mode 100644 ddtrace/vendor/psutil/_common.py create mode 100644 ddtrace/vendor/psutil/_compat.py create mode 100644 ddtrace/vendor/psutil/_psaix.py create mode 100644 ddtrace/vendor/psutil/_psbsd.py create mode 100644 ddtrace/vendor/psutil/_pslinux.py create mode 100644 ddtrace/vendor/psutil/_psosx.py create mode 100644 ddtrace/vendor/psutil/_psposix.py create mode 100644 ddtrace/vendor/psutil/_pssunos.py create mode 100644 ddtrace/vendor/psutil/_psutil_aix.c create mode 100644 ddtrace/vendor/psutil/_psutil_bsd.c create mode 100644 ddtrace/vendor/psutil/_psutil_common.c create mode 100644 ddtrace/vendor/psutil/_psutil_common.h create mode 100644 ddtrace/vendor/psutil/_psutil_linux.c create mode 100644 ddtrace/vendor/psutil/_psutil_osx.c create mode 100644 ddtrace/vendor/psutil/_psutil_posix.c create mode 100644 ddtrace/vendor/psutil/_psutil_posix.h create mode 100644 ddtrace/vendor/psutil/_psutil_sunos.c create mode 100644 ddtrace/vendor/psutil/_psutil_windows.c create mode 100644 ddtrace/vendor/psutil/_pswindows.py create mode 100644 ddtrace/vendor/psutil/arch/aix/common.c create mode 100644 ddtrace/vendor/psutil/arch/aix/common.h create mode 100644 ddtrace/vendor/psutil/arch/aix/ifaddrs.c create mode 100644 ddtrace/vendor/psutil/arch/aix/ifaddrs.h create mode 100644 ddtrace/vendor/psutil/arch/aix/net_connections.c create mode 100644 ddtrace/vendor/psutil/arch/aix/net_connections.h create mode 100644 ddtrace/vendor/psutil/arch/aix/net_kernel_structs.h create mode 100644 ddtrace/vendor/psutil/arch/freebsd/proc_socks.c create mode 100644 ddtrace/vendor/psutil/arch/freebsd/proc_socks.h create mode 100644 ddtrace/vendor/psutil/arch/freebsd/specific.c create mode 100644 ddtrace/vendor/psutil/arch/freebsd/specific.h create mode 100644 ddtrace/vendor/psutil/arch/freebsd/sys_socks.c create mode 100644 ddtrace/vendor/psutil/arch/freebsd/sys_socks.h create mode 100644 ddtrace/vendor/psutil/arch/netbsd/socks.c create mode 100644 ddtrace/vendor/psutil/arch/netbsd/socks.h create mode 100644 ddtrace/vendor/psutil/arch/netbsd/specific.c create mode 100644 ddtrace/vendor/psutil/arch/netbsd/specific.h create mode 100644 ddtrace/vendor/psutil/arch/openbsd/specific.c create mode 100644 ddtrace/vendor/psutil/arch/openbsd/specific.h create mode 100644 ddtrace/vendor/psutil/arch/osx/process_info.c create mode 100644 ddtrace/vendor/psutil/arch/osx/process_info.h create mode 100644 ddtrace/vendor/psutil/arch/solaris/environ.c create mode 100644 ddtrace/vendor/psutil/arch/solaris/environ.h create mode 100644 ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.c create mode 100644 ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.h create mode 100644 ddtrace/vendor/psutil/arch/windows/global.c create mode 100644 ddtrace/vendor/psutil/arch/windows/global.h create mode 100644 ddtrace/vendor/psutil/arch/windows/inet_ntop.c create mode 100644 ddtrace/vendor/psutil/arch/windows/inet_ntop.h create mode 100644 ddtrace/vendor/psutil/arch/windows/ntextapi.h create mode 100644 ddtrace/vendor/psutil/arch/windows/process_handles.c create mode 100644 ddtrace/vendor/psutil/arch/windows/process_handles.h create mode 100644 ddtrace/vendor/psutil/arch/windows/process_info.c create mode 100644 ddtrace/vendor/psutil/arch/windows/process_info.h create mode 100644 ddtrace/vendor/psutil/arch/windows/security.c create mode 100644 ddtrace/vendor/psutil/arch/windows/security.h create mode 100644 ddtrace/vendor/psutil/arch/windows/services.c create mode 100644 ddtrace/vendor/psutil/arch/windows/services.h create mode 100644 ddtrace/vendor/psutil/arch/windows/wmi.c create mode 100644 ddtrace/vendor/psutil/arch/windows/wmi.h create mode 100644 ddtrace/vendor/psutil/setup.py diff --git a/ddtrace/internal/runtime/metric_collectors.py b/ddtrace/internal/runtime/metric_collectors.py index eb140294b6d..23ce23df2f3 100644 --- a/ddtrace/internal/runtime/metric_collectors.py +++ b/ddtrace/internal/runtime/metric_collectors.py @@ -49,13 +49,13 @@ class PSUtilRuntimeMetricCollector(RuntimeMetricCollector): for more information. """ - required_modules = ["psutil"] + required_modules = ["ddtrace.vendor.psutil"] stored_value = dict( CPU_TIME_SYS_TOTAL=0, CPU_TIME_USER_TOTAL=0, CTX_SWITCH_VOLUNTARY_TOTAL=0, CTX_SWITCH_INVOLUNTARY_TOTAL=0, ) def _on_modules_load(self): - self.proc = self.modules["psutil"].Process(os.getpid()) + self.proc = self.modules["ddtrace.vendor.psutil"].Process(os.getpid()) def collect_fn(self, keys): with self.proc.oneshot(): diff --git a/ddtrace/vendor/__init__.py b/ddtrace/vendor/__init__.py index d3d436403d2..cbef8ee46fb 100644 --- a/ddtrace/vendor/__init__.py +++ b/ddtrace/vendor/__init__.py @@ -84,6 +84,16 @@ Notes: Removed dependency on `pbr` and manually set `__version__` + +psutil +------ + +Website: https://github.com/giampaolo/psutil +Source: https://github.com/giampaolo/psutil +Version: 5.6.7 +License: BSD 3 + +Notes: """ # Initialize `ddtrace.vendor.datadog.base.log` logger with our custom rate limited logger diff --git a/ddtrace/vendor/psutil/__init__.py b/ddtrace/vendor/psutil/__init__.py new file mode 100644 index 00000000000..b267239e285 --- /dev/null +++ b/ddtrace/vendor/psutil/__init__.py @@ -0,0 +1,2516 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""psutil is a cross-platform library for retrieving information on +running processes and system utilization (CPU, memory, disks, network, +sensors) in Python. Supported platforms: + + - Linux + - Windows + - macOS + - FreeBSD + - OpenBSD + - NetBSD + - Sun Solaris + - AIX + +Works with Python versions from 2.6 to 3.4+. +""" + +from __future__ import division + +import collections +import contextlib +import datetime +import functools +import os +import signal +import subprocess +import sys +import threading +import time +try: + import pwd +except ImportError: + pwd = None + +from . import _common +from ._common import deprecated_method +from ._common import memoize +from ._common import memoize_when_activated +from ._common import wrap_numbers as _wrap_numbers +from ._compat import long +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._compat import PY3 as _PY3 + +from ._common import STATUS_DEAD +from ._common import STATUS_DISK_SLEEP +from ._common import STATUS_IDLE +from ._common import STATUS_LOCKED +from ._common import STATUS_PARKED +from ._common import STATUS_RUNNING +from ._common import STATUS_SLEEPING +from ._common import STATUS_STOPPED +from ._common import STATUS_TRACING_STOP +from ._common import STATUS_WAITING +from ._common import STATUS_WAKING +from ._common import STATUS_ZOMBIE + +from ._common import CONN_CLOSE +from ._common import CONN_CLOSE_WAIT +from ._common import CONN_CLOSING +from ._common import CONN_ESTABLISHED +from ._common import CONN_FIN_WAIT1 +from ._common import CONN_FIN_WAIT2 +from ._common import CONN_LAST_ACK +from ._common import CONN_LISTEN +from ._common import CONN_NONE +from ._common import CONN_SYN_RECV +from ._common import CONN_SYN_SENT +from ._common import CONN_TIME_WAIT +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN + +from ._common import AIX +from ._common import BSD +from ._common import FREEBSD # NOQA +from ._common import LINUX +from ._common import MACOS +from ._common import NETBSD # NOQA +from ._common import OPENBSD # NOQA +from ._common import OSX # deprecated alias +from ._common import POSIX # NOQA +from ._common import SUNOS +from ._common import WINDOWS + +if LINUX: + # This is public API and it will be retrieved from _pslinux.py + # via sys.modules. + PROCFS_PATH = "/proc" + + from . import _pslinux as _psplatform + + from ._pslinux import IOPRIO_CLASS_BE # NOQA + from ._pslinux import IOPRIO_CLASS_IDLE # NOQA + from ._pslinux import IOPRIO_CLASS_NONE # NOQA + from ._pslinux import IOPRIO_CLASS_RT # NOQA + # Linux >= 2.6.36 + if _psplatform.HAS_PRLIMIT: + from ._psutil_linux import RLIM_INFINITY # NOQA + from ._psutil_linux import RLIMIT_AS # NOQA + from ._psutil_linux import RLIMIT_CORE # NOQA + from ._psutil_linux import RLIMIT_CPU # NOQA + from ._psutil_linux import RLIMIT_DATA # NOQA + from ._psutil_linux import RLIMIT_FSIZE # NOQA + from ._psutil_linux import RLIMIT_LOCKS # NOQA + from ._psutil_linux import RLIMIT_MEMLOCK # NOQA + from ._psutil_linux import RLIMIT_NOFILE # NOQA + from ._psutil_linux import RLIMIT_NPROC # NOQA + from ._psutil_linux import RLIMIT_RSS # NOQA + from ._psutil_linux import RLIMIT_STACK # NOQA + # Kinda ugly but considerably faster than using hasattr() and + # setattr() against the module object (we are at import time: + # speed matters). + from . import _psutil_linux + try: + RLIMIT_MSGQUEUE = _psutil_linux.RLIMIT_MSGQUEUE + except AttributeError: + pass + try: + RLIMIT_NICE = _psutil_linux.RLIMIT_NICE + except AttributeError: + pass + try: + RLIMIT_RTPRIO = _psutil_linux.RLIMIT_RTPRIO + except AttributeError: + pass + try: + RLIMIT_RTTIME = _psutil_linux.RLIMIT_RTTIME + except AttributeError: + pass + try: + RLIMIT_SIGPENDING = _psutil_linux.RLIMIT_SIGPENDING + except AttributeError: + pass + +elif WINDOWS: + from . import _pswindows as _psplatform + from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS # NOQA + from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS # NOQA + from ._psutil_windows import HIGH_PRIORITY_CLASS # NOQA + from ._psutil_windows import IDLE_PRIORITY_CLASS # NOQA + from ._psutil_windows import NORMAL_PRIORITY_CLASS # NOQA + from ._psutil_windows import REALTIME_PRIORITY_CLASS # NOQA + from ._pswindows import CONN_DELETE_TCB # NOQA + from ._pswindows import IOPRIO_VERYLOW # NOQA + from ._pswindows import IOPRIO_LOW # NOQA + from ._pswindows import IOPRIO_NORMAL # NOQA + from ._pswindows import IOPRIO_HIGH # NOQA + +elif MACOS: + from . import _psosx as _psplatform + +elif BSD: + from . import _psbsd as _psplatform + +elif SUNOS: + from . import _pssunos as _psplatform + from ._pssunos import CONN_BOUND # NOQA + from ._pssunos import CONN_IDLE # NOQA + + # This is public writable API which is read from _pslinux.py and + # _pssunos.py via sys.modules. + PROCFS_PATH = "/proc" + +elif AIX: + from . import _psaix as _psplatform + + # This is public API and it will be retrieved from _pslinux.py + # via sys.modules. + PROCFS_PATH = "/proc" + +else: # pragma: no cover + raise NotImplementedError('platform %s is not supported' % sys.platform) + + +__all__ = [ + # exceptions + "Error", "NoSuchProcess", "ZombieProcess", "AccessDenied", + "TimeoutExpired", + + # constants + "version_info", "__version__", + + "STATUS_RUNNING", "STATUS_IDLE", "STATUS_SLEEPING", "STATUS_DISK_SLEEP", + "STATUS_STOPPED", "STATUS_TRACING_STOP", "STATUS_ZOMBIE", "STATUS_DEAD", + "STATUS_WAKING", "STATUS_LOCKED", "STATUS_WAITING", "STATUS_LOCKED", + "STATUS_PARKED", + + "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1", + "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", + "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", "CONN_NONE", + + "AF_LINK", + + "NIC_DUPLEX_FULL", "NIC_DUPLEX_HALF", "NIC_DUPLEX_UNKNOWN", + + "POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED", + + "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "MACOS", "OSX", "POSIX", + "SUNOS", "WINDOWS", "AIX", + + # classes + "Process", "Popen", + + # functions + "pid_exists", "pids", "process_iter", "wait_procs", # proc + "virtual_memory", "swap_memory", # memory + "cpu_times", "cpu_percent", "cpu_times_percent", "cpu_count", # cpu + "cpu_stats", # "cpu_freq", "getloadavg" + "net_io_counters", "net_connections", "net_if_addrs", # network + "net_if_stats", + "disk_io_counters", "disk_partitions", "disk_usage", # disk + # "sensors_temperatures", "sensors_battery", "sensors_fans" # sensors + "users", "boot_time", # others +] + + +__all__.extend(_psplatform.__extra__all__) +__author__ = "Giampaolo Rodola'" +__version__ = "5.6.7" +version_info = tuple([int(num) for num in __version__.split('.')]) + +_timer = getattr(time, 'monotonic', time.time) +AF_LINK = _psplatform.AF_LINK +POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED +POWER_TIME_UNKNOWN = _common.POWER_TIME_UNKNOWN +_TOTAL_PHYMEM = None +_LOWEST_PID = None + +# Sanity check in case the user messed up with psutil installation +# or did something weird with sys.path. In this case we might end +# up importing a python module using a C extension module which +# was compiled for a different version of psutil. +# We want to prevent that by failing sooner rather than later. +# See: https://github.com/giampaolo/psutil/issues/564 +if (int(__version__.replace('.', '')) != + getattr(_psplatform.cext, 'version', None)): + msg = "version conflict: %r C extension module was built for another " \ + "version of psutil" % getattr(_psplatform.cext, "__file__") + if hasattr(_psplatform.cext, 'version'): + msg += " (%s instead of %s)" % ( + '.'.join([x for x in str(_psplatform.cext.version)]), __version__) + else: + msg += " (different than %s)" % __version__ + msg += "; you may try to 'pip uninstall psutil', manually remove %s" % ( + getattr(_psplatform.cext, "__file__", + "the existing psutil install directory")) + msg += " or clean the virtual env somehow, then reinstall" + raise ImportError(msg) + + +# ===================================================================== +# --- Exceptions +# ===================================================================== + + +class Error(Exception): + """Base exception class. All other psutil exceptions inherit + from this one. + """ + + def __init__(self, msg=""): + Exception.__init__(self, msg) + self.msg = msg + + def __repr__(self): + ret = "psutil.%s %s" % (self.__class__.__name__, self.msg) + return ret.strip() + + __str__ = __repr__ + + +class NoSuchProcess(Error): + """Exception raised when a process with a certain PID doesn't + or no longer exists. + """ + + def __init__(self, pid, name=None, msg=None): + Error.__init__(self, msg) + self.pid = pid + self.name = name + self.msg = msg + if msg is None: + if name: + details = "(pid=%s, name=%s)" % (self.pid, repr(self.name)) + else: + details = "(pid=%s)" % self.pid + self.msg = "process no longer exists " + details + + +class ZombieProcess(NoSuchProcess): + """Exception raised when querying a zombie process. This is + raised on macOS, BSD and Solaris only, and not always: depending + on the query the OS may be able to succeed anyway. + On Linux all zombie processes are querable (hence this is never + raised). Windows doesn't have zombie processes. + """ + + def __init__(self, pid, name=None, ppid=None, msg=None): + NoSuchProcess.__init__(self, msg) + self.pid = pid + self.ppid = ppid + self.name = name + self.msg = msg + if msg is None: + args = ["pid=%s" % pid] + if name: + args.append("name=%s" % repr(self.name)) + if ppid: + args.append("ppid=%s" % self.ppid) + details = "(%s)" % ", ".join(args) + self.msg = "process still exists but it's a zombie " + details + + +class AccessDenied(Error): + """Exception raised when permission to perform an action is denied.""" + + def __init__(self, pid=None, name=None, msg=None): + Error.__init__(self, msg) + self.pid = pid + self.name = name + self.msg = msg + if msg is None: + if (pid is not None) and (name is not None): + self.msg = "(pid=%s, name=%s)" % (pid, repr(name)) + elif (pid is not None): + self.msg = "(pid=%s)" % self.pid + else: + self.msg = "" + + +class TimeoutExpired(Error): + """Raised on Process.wait(timeout) if timeout expires and process + is still alive. + """ + + def __init__(self, seconds, pid=None, name=None): + Error.__init__(self, "timeout after %s seconds" % seconds) + self.seconds = seconds + self.pid = pid + self.name = name + if (pid is not None) and (name is not None): + self.msg += " (pid=%s, name=%s)" % (pid, repr(name)) + elif (pid is not None): + self.msg += " (pid=%s)" % self.pid + + +# Push exception classes into platform specific module namespace. +_psplatform.NoSuchProcess = NoSuchProcess +_psplatform.ZombieProcess = ZombieProcess +_psplatform.AccessDenied = AccessDenied +_psplatform.TimeoutExpired = TimeoutExpired +if POSIX: + from . import _psposix + _psposix.TimeoutExpired = TimeoutExpired + + +# ===================================================================== +# --- Utils +# ===================================================================== + + +if hasattr(_psplatform, 'ppid_map'): + # Faster version (Windows and Linux). + _ppid_map = _psplatform.ppid_map +else: + def _ppid_map(): + """Return a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + for pid in pids(): + try: + ret[pid] = _psplatform.Process(pid).ppid() + except (NoSuchProcess, ZombieProcess): + pass + return ret + + +def _assert_pid_not_reused(fun): + """Decorator which raises NoSuchProcess in case a process is no + longer running or its PID has been reused. + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + if not self.is_running(): + raise NoSuchProcess(self.pid, self._name) + return fun(self, *args, **kwargs) + return wrapper + + +def _pprint_secs(secs): + """Format seconds in a human readable form.""" + now = time.time() + secs_ago = int(now - secs) + if secs_ago < 60 * 60 * 24: + fmt = "%H:%M:%S" + else: + fmt = "%Y-%m-%d %H:%M:%S" + return datetime.datetime.fromtimestamp(secs).strftime(fmt) + + +# ===================================================================== +# --- Process class +# ===================================================================== + + +class Process(object): + """Represents an OS process with the given PID. + If PID is omitted current process PID (os.getpid()) is used. + Raise NoSuchProcess if PID does not exist. + + Note that most of the methods of this class do not make sure + the PID of the process being queried has been reused over time. + That means you might end up retrieving an information referring + to another process in case the original one this instance + refers to is gone in the meantime. + + The only exceptions for which process identity is pre-emptively + checked and guaranteed are: + + - parent() + - children() + - nice() (set) + - ionice() (set) + - rlimit() (set) + - cpu_affinity (set) + - suspend() + - resume() + - send_signal() + - terminate() + - kill() + + To prevent this problem for all other methods you can: + - use is_running() before querying the process + - if you're continuously iterating over a set of Process + instances use process_iter() which pre-emptively checks + process identity for every yielded instance + """ + + def __init__(self, pid=None): + self._init(pid) + + def _init(self, pid, _ignore_nsp=False): + if pid is None: + pid = os.getpid() + else: + if not _PY3 and not isinstance(pid, (int, long)): + raise TypeError('pid must be an integer (got %r)' % pid) + if pid < 0: + raise ValueError('pid must be a positive integer (got %s)' + % pid) + self._pid = pid + self._name = None + self._exe = None + self._create_time = None + self._gone = False + self._hash = None + self._lock = threading.RLock() + # used for caching on Windows only (on POSIX ppid may change) + self._ppid = None + # platform-specific modules define an _psplatform.Process + # implementation class + self._proc = _psplatform.Process(pid) + self._last_sys_cpu_times = None + self._last_proc_cpu_times = None + # cache creation time for later use in is_running() method + try: + self.create_time() + except AccessDenied: + # We should never get here as AFAIK we're able to get + # process creation time on all platforms even as a + # limited user. + pass + except ZombieProcess: + # Zombies can still be queried by this class (although + # not always) and pids() return them so just go on. + pass + except NoSuchProcess: + if not _ignore_nsp: + msg = 'no process found with pid %s' % pid + raise NoSuchProcess(pid, None, msg) + else: + self._gone = True + # This pair is supposed to indentify a Process instance + # univocally over time (the PID alone is not enough as + # it might refer to a process whose PID has been reused). + # This will be used later in __eq__() and is_running(). + self._ident = (self.pid, self._create_time) + + def __str__(self): + try: + info = collections.OrderedDict() + except AttributeError: + info = {} # Python 2.6 + info["pid"] = self.pid + try: + info["name"] = self.name() + if self._create_time: + info['started'] = _pprint_secs(self._create_time) + except ZombieProcess: + info["status"] = "zombie" + except NoSuchProcess: + info["status"] = "terminated" + except AccessDenied: + pass + return "%s.%s(%s)" % ( + self.__class__.__module__, + self.__class__.__name__, + ", ".join(["%s=%r" % (k, v) for k, v in info.items()])) + + __repr__ = __str__ + + def __eq__(self, other): + # Test for equality with another Process object based + # on PID and creation time. + if not isinstance(other, Process): + return NotImplemented + return self._ident == other._ident + + def __ne__(self, other): + return not self == other + + def __hash__(self): + if self._hash is None: + self._hash = hash(self._ident) + return self._hash + + @property + def pid(self): + """The process PID.""" + return self._pid + + # --- utility methods + + @contextlib.contextmanager + def oneshot(self): + """Utility context manager which considerably speeds up the + retrieval of multiple process information at the same time. + + Internally different process info (e.g. name, ppid, uids, + gids, ...) may be fetched by using the same routine, but + only one information is returned and the others are discarded. + When using this context manager the internal routine is + executed once (in the example below on name()) and the + other info are cached. + + The cache is cleared when exiting the context manager block. + The advice is to use this every time you retrieve more than + one information about the process. If you're lucky, you'll + get a hell of a speedup. + + >>> import psutil + >>> p = psutil.Process() + >>> with p.oneshot(): + ... p.name() # collect multiple info + ... p.cpu_times() # return cached value + ... p.cpu_percent() # return cached value + ... p.create_time() # return cached value + ... + >>> + """ + with self._lock: + if hasattr(self, "_cache"): + # NOOP: this covers the use case where the user enters the + # context twice: + # + # >>> with p.oneshot(): + # ... with p.oneshot(): + # ... + # + # Also, since as_dict() internally uses oneshot() + # I expect that the code below will be a pretty common + # "mistake" that the user will make, so let's guard + # against that: + # + # >>> with p.oneshot(): + # ... p.as_dict() + # ... + yield + else: + try: + # cached in case cpu_percent() is used + self.cpu_times.cache_activate(self) + # cached in case memory_percent() is used + self.memory_info.cache_activate(self) + # cached in case parent() is used + self.ppid.cache_activate(self) + # cached in case username() is used + if POSIX: + self.uids.cache_activate(self) + # specific implementation cache + self._proc.oneshot_enter() + yield + finally: + self.cpu_times.cache_deactivate(self) + self.memory_info.cache_deactivate(self) + self.ppid.cache_deactivate(self) + if POSIX: + self.uids.cache_deactivate(self) + self._proc.oneshot_exit() + + def as_dict(self, attrs=None, ad_value=None): + """Utility method returning process information as a + hashable dictionary. + If *attrs* is specified it must be a list of strings + reflecting available Process class' attribute names + (e.g. ['cpu_times', 'name']) else all public (read + only) attributes are assumed. + *ad_value* is the value which gets assigned in case + AccessDenied or ZombieProcess exception is raised when + retrieving that particular process information. + """ + valid_names = _as_dict_attrnames + if attrs is not None: + if not isinstance(attrs, (list, tuple, set, frozenset)): + raise TypeError("invalid attrs type %s" % type(attrs)) + attrs = set(attrs) + invalid_names = attrs - valid_names + if invalid_names: + raise ValueError("invalid attr name%s %s" % ( + "s" if len(invalid_names) > 1 else "", + ", ".join(map(repr, invalid_names)))) + + retdict = dict() + ls = attrs or valid_names + with self.oneshot(): + for name in ls: + try: + if name == 'pid': + ret = self.pid + else: + meth = getattr(self, name) + ret = meth() + except (AccessDenied, ZombieProcess): + ret = ad_value + except NotImplementedError: + # in case of not implemented functionality (may happen + # on old or exotic systems) we want to crash only if + # the user explicitly asked for that particular attr + if attrs: + raise + continue + retdict[name] = ret + return retdict + + def parent(self): + """Return the parent process as a Process object pre-emptively + checking whether PID has been reused. + If no parent is known return None. + """ + lowest_pid = _LOWEST_PID if _LOWEST_PID is not None else pids()[0] + if self.pid == lowest_pid: + return None + ppid = self.ppid() + if ppid is not None: + ctime = self.create_time() + try: + parent = Process(ppid) + if parent.create_time() <= ctime: + return parent + # ...else ppid has been reused by another process + except NoSuchProcess: + pass + + def parents(self): + """Return the parents of this process as a list of Process + instances. If no parents are known return an empty list. + """ + parents = [] + proc = self.parent() + while proc is not None: + parents.append(proc) + proc = proc.parent() + return parents + + def is_running(self): + """Return whether this process is running. + It also checks if PID has been reused by another process in + which case return False. + """ + if self._gone: + return False + try: + # Checking if PID is alive is not enough as the PID might + # have been reused by another process: we also want to + # verify process identity. + # Process identity / uniqueness over time is guaranteed by + # (PID + creation time) and that is verified in __eq__. + return self == Process(self.pid) + except ZombieProcess: + # We should never get here as it's already handled in + # Process.__init__; here just for extra safety. + return True + except NoSuchProcess: + self._gone = True + return False + + # --- actual API + + @memoize_when_activated + def ppid(self): + """The process parent PID. + On Windows the return value is cached after first call. + """ + # On POSIX we don't want to cache the ppid as it may unexpectedly + # change to 1 (init) in case this process turns into a zombie: + # https://github.com/giampaolo/psutil/issues/321 + # http://stackoverflow.com/questions/356722/ + + # XXX should we check creation time here rather than in + # Process.parent()? + if POSIX: + return self._proc.ppid() + else: # pragma: no cover + self._ppid = self._ppid or self._proc.ppid() + return self._ppid + + def name(self): + """The process name. The return value is cached after first call.""" + # Process name is only cached on Windows as on POSIX it may + # change, see: + # https://github.com/giampaolo/psutil/issues/692 + if WINDOWS and self._name is not None: + return self._name + name = self._proc.name() + if POSIX and len(name) >= 15: + # On UNIX the name gets truncated to the first 15 characters. + # If it matches the first part of the cmdline we return that + # one instead because it's usually more explicative. + # Examples are "gnome-keyring-d" vs. "gnome-keyring-daemon". + try: + cmdline = self.cmdline() + except AccessDenied: + pass + else: + if cmdline: + extended_name = os.path.basename(cmdline[0]) + if extended_name.startswith(name): + name = extended_name + self._name = name + self._proc._name = name + return name + + def exe(self): + """The process executable as an absolute path. + May also be an empty string. + The return value is cached after first call. + """ + def guess_it(fallback): + # try to guess exe from cmdline[0] in absence of a native + # exe representation + cmdline = self.cmdline() + if cmdline and hasattr(os, 'access') and hasattr(os, 'X_OK'): + exe = cmdline[0] # the possible exe + # Attempt to guess only in case of an absolute path. + # It is not safe otherwise as the process might have + # changed cwd. + if (os.path.isabs(exe) and + os.path.isfile(exe) and + os.access(exe, os.X_OK)): + return exe + if isinstance(fallback, AccessDenied): + raise fallback + return fallback + + if self._exe is None: + try: + exe = self._proc.exe() + except AccessDenied as err: + return guess_it(fallback=err) + else: + if not exe: + # underlying implementation can legitimately return an + # empty string; if that's the case we don't want to + # raise AD while guessing from the cmdline + try: + exe = guess_it(fallback=exe) + except AccessDenied: + pass + self._exe = exe + return self._exe + + def cmdline(self): + """The command line this process has been called with.""" + return self._proc.cmdline() + + def status(self): + """The process current status as a STATUS_* constant.""" + try: + return self._proc.status() + except ZombieProcess: + return STATUS_ZOMBIE + + def username(self): + """The name of the user that owns the process. + On UNIX this is calculated by using *real* process uid. + """ + if POSIX: + if pwd is None: + # might happen if python was installed from sources + raise ImportError( + "requires pwd module shipped with standard python") + real_uid = self.uids().real + try: + return pwd.getpwuid(real_uid).pw_name + except KeyError: + # the uid can't be resolved by the system + return str(real_uid) + else: + return self._proc.username() + + def create_time(self): + """The process creation time as a floating point number + expressed in seconds since the epoch, in UTC. + The return value is cached after first call. + """ + if self._create_time is None: + self._create_time = self._proc.create_time() + return self._create_time + + def cwd(self): + """Process current working directory as an absolute path.""" + return self._proc.cwd() + + def nice(self, value=None): + """Get or set process niceness (priority).""" + if value is None: + return self._proc.nice_get() + else: + if not self.is_running(): + raise NoSuchProcess(self.pid, self._name) + self._proc.nice_set(value) + + if POSIX: + + @memoize_when_activated + def uids(self): + """Return process UIDs as a (real, effective, saved) + namedtuple. + """ + return self._proc.uids() + + def gids(self): + """Return process GIDs as a (real, effective, saved) + namedtuple. + """ + return self._proc.gids() + + def terminal(self): + """The terminal associated with this process, if any, + else None. + """ + return self._proc.terminal() + + def num_fds(self): + """Return the number of file descriptors opened by this + process (POSIX only). + """ + return self._proc.num_fds() + + # Linux, BSD, AIX and Windows only + if hasattr(_psplatform.Process, "io_counters"): + + def io_counters(self): + """Return process I/O statistics as a + (read_count, write_count, read_bytes, write_bytes) + namedtuple. + Those are the number of read/write calls performed and the + amount of bytes read and written by the process. + """ + return self._proc.io_counters() + + # Linux and Windows >= Vista only + if hasattr(_psplatform.Process, "ionice_get"): + + def ionice(self, ioclass=None, value=None): + """Get or set process I/O niceness (priority). + + On Linux *ioclass* is one of the IOPRIO_CLASS_* constants. + *value* is a number which goes from 0 to 7. The higher the + value, the lower the I/O priority of the process. + + On Windows only *ioclass* is used and it can be set to 2 + (normal), 1 (low) or 0 (very low). + + Available on Linux and Windows > Vista only. + """ + if ioclass is None: + if value is not None: + raise ValueError("'ioclass' argument must be specified") + return self._proc.ionice_get() + else: + return self._proc.ionice_set(ioclass, value) + + # Linux only + if hasattr(_psplatform.Process, "rlimit"): + + def rlimit(self, resource, limits=None): + """Get or set process resource limits as a (soft, hard) + tuple. + + *resource* is one of the RLIMIT_* constants. + *limits* is supposed to be a (soft, hard) tuple. + + See "man prlimit" for further info. + Available on Linux only. + """ + if limits is None: + return self._proc.rlimit(resource) + else: + return self._proc.rlimit(resource, limits) + + # Windows, Linux and FreeBSD only + if hasattr(_psplatform.Process, "cpu_affinity_get"): + + def cpu_affinity(self, cpus=None): + """Get or set process CPU affinity. + If specified, *cpus* must be a list of CPUs for which you + want to set the affinity (e.g. [0, 1]). + If an empty list is passed, all egible CPUs are assumed + (and set). + (Windows, Linux and BSD only). + """ + if cpus is None: + return list(set(self._proc.cpu_affinity_get())) + else: + if not cpus: + if hasattr(self._proc, "_get_eligible_cpus"): + cpus = self._proc._get_eligible_cpus() + else: + cpus = tuple(range(len(cpu_times(percpu=True)))) + self._proc.cpu_affinity_set(list(set(cpus))) + + # Linux, FreeBSD, SunOS + if hasattr(_psplatform.Process, "cpu_num"): + + def cpu_num(self): + """Return what CPU this process is currently running on. + The returned number should be <= psutil.cpu_count() + and <= len(psutil.cpu_percent(percpu=True)). + It may be used in conjunction with + psutil.cpu_percent(percpu=True) to observe the system + workload distributed across CPUs. + """ + return self._proc.cpu_num() + + # Linux, macOS, Windows, Solaris, AIX + if hasattr(_psplatform.Process, "environ"): + + def environ(self): + """The environment variables of the process as a dict. Note: this + might not reflect changes made after the process started. """ + return self._proc.environ() + + if WINDOWS: + + def num_handles(self): + """Return the number of handles opened by this process + (Windows only). + """ + return self._proc.num_handles() + + def num_ctx_switches(self): + """Return the number of voluntary and involuntary context + switches performed by this process. + """ + return self._proc.num_ctx_switches() + + def num_threads(self): + """Return the number of threads used by this process.""" + return self._proc.num_threads() + + if hasattr(_psplatform.Process, "threads"): + + def threads(self): + """Return threads opened by process as a list of + (id, user_time, system_time) namedtuples representing + thread id and thread CPU times (user/system). + On OpenBSD this method requires root access. + """ + return self._proc.threads() + + @_assert_pid_not_reused + def children(self, recursive=False): + """Return the children of this process as a list of Process + instances, pre-emptively checking whether PID has been reused. + If *recursive* is True return all the parent descendants. + + Example (A == this process): + + A ─┐ + │ + ├─ B (child) ─┐ + │ └─ X (grandchild) ─┐ + │ └─ Y (great grandchild) + ├─ C (child) + └─ D (child) + + >>> import psutil + >>> p = psutil.Process() + >>> p.children() + B, C, D + >>> p.children(recursive=True) + B, X, Y, C, D + + Note that in the example above if process X disappears + process Y won't be listed as the reference to process A + is lost. + """ + ppid_map = _ppid_map() + ret = [] + if not recursive: + for pid, ppid in ppid_map.items(): + if ppid == self.pid: + try: + child = Process(pid) + # if child happens to be older than its parent + # (self) it means child's PID has been reused + if self.create_time() <= child.create_time(): + ret.append(child) + except (NoSuchProcess, ZombieProcess): + pass + else: + # Construct a {pid: [child pids]} dict + reverse_ppid_map = collections.defaultdict(list) + for pid, ppid in ppid_map.items(): + reverse_ppid_map[ppid].append(pid) + # Recursively traverse that dict, starting from self.pid, + # such that we only call Process() on actual children + seen = set() + stack = [self.pid] + while stack: + pid = stack.pop() + if pid in seen: + # Since pids can be reused while the ppid_map is + # constructed, there may be rare instances where + # there's a cycle in the recorded process "tree". + continue + seen.add(pid) + for child_pid in reverse_ppid_map[pid]: + try: + child = Process(child_pid) + # if child happens to be older than its parent + # (self) it means child's PID has been reused + intime = self.create_time() <= child.create_time() + if intime: + ret.append(child) + stack.append(child_pid) + except (NoSuchProcess, ZombieProcess): + pass + return ret + + def cpu_percent(self, interval=None): + """Return a float representing the current process CPU + utilization as a percentage. + + When *interval* is 0.0 or None (default) compares process times + to system CPU times elapsed since last call, returning + immediately (non-blocking). That means that the first time + this is called it will return a meaningful 0.0 value. + + When *interval* is > 0.0 compares process times to system CPU + times elapsed before and after the interval (blocking). + + In this case is recommended for accuracy that this function + be called with at least 0.1 seconds between calls. + + A value > 100.0 can be returned in case of processes running + multiple threads on different CPU cores. + + The returned value is explicitly NOT split evenly between + all available logical CPUs. This means that a busy loop process + running on a system with 2 logical CPUs will be reported as + having 100% CPU utilization instead of 50%. + + Examples: + + >>> import psutil + >>> p = psutil.Process(os.getpid()) + >>> # blocking + >>> p.cpu_percent(interval=1) + 2.0 + >>> # non-blocking (percentage since last call) + >>> p.cpu_percent(interval=None) + 2.9 + >>> + """ + blocking = interval is not None and interval > 0.0 + if interval is not None and interval < 0: + raise ValueError("interval is not positive (got %r)" % interval) + num_cpus = cpu_count() or 1 + + def timer(): + return _timer() * num_cpus + + if blocking: + st1 = timer() + pt1 = self._proc.cpu_times() + time.sleep(interval) + st2 = timer() + pt2 = self._proc.cpu_times() + else: + st1 = self._last_sys_cpu_times + pt1 = self._last_proc_cpu_times + st2 = timer() + pt2 = self._proc.cpu_times() + if st1 is None or pt1 is None: + self._last_sys_cpu_times = st2 + self._last_proc_cpu_times = pt2 + return 0.0 + + delta_proc = (pt2.user - pt1.user) + (pt2.system - pt1.system) + delta_time = st2 - st1 + # reset values for next call in case of interval == None + self._last_sys_cpu_times = st2 + self._last_proc_cpu_times = pt2 + + try: + # This is the utilization split evenly between all CPUs. + # E.g. a busy loop process on a 2-CPU-cores system at this + # point is reported as 50% instead of 100%. + overall_cpus_percent = ((delta_proc / delta_time) * 100) + except ZeroDivisionError: + # interval was too low + return 0.0 + else: + # Note 1: + # in order to emulate "top" we multiply the value for the num + # of CPU cores. This way the busy process will be reported as + # having 100% (or more) usage. + # + # Note 2: + # taskmgr.exe on Windows differs in that it will show 50% + # instead. + # + # Note 3: + # a percentage > 100 is legitimate as it can result from a + # process with multiple threads running on different CPU + # cores (top does the same), see: + # http://stackoverflow.com/questions/1032357 + # https://github.com/giampaolo/psutil/issues/474 + single_cpu_percent = overall_cpus_percent * num_cpus + return round(single_cpu_percent, 1) + + @memoize_when_activated + def cpu_times(self): + """Return a (user, system, children_user, children_system) + namedtuple representing the accumulated process time, in + seconds. + This is similar to os.times() but per-process. + On macOS and Windows children_user and children_system are + always set to 0. + """ + return self._proc.cpu_times() + + @memoize_when_activated + def memory_info(self): + """Return a namedtuple with variable fields depending on the + platform, representing memory information about the process. + + The "portable" fields available on all plaforms are `rss` and `vms`. + + All numbers are expressed in bytes. + """ + return self._proc.memory_info() + + @deprecated_method(replacement="memory_info") + def memory_info_ex(self): + return self.memory_info() + + def memory_full_info(self): + """This method returns the same information as memory_info(), + plus, on some platform (Linux, macOS, Windows), also provides + additional metrics (USS, PSS and swap). + The additional metrics provide a better representation of actual + process memory usage. + + Namely USS is the memory which is unique to a process and which + would be freed if the process was terminated right now. + + It does so by passing through the whole process address. + As such it usually requires higher user privileges than + memory_info() and is considerably slower. + """ + return self._proc.memory_full_info() + + def memory_percent(self, memtype="rss"): + """Compare process memory to total physical system memory and + calculate process memory utilization as a percentage. + *memtype* argument is a string that dictates what type of + process memory you want to compare against (defaults to "rss"). + The list of available strings can be obtained like this: + + >>> psutil.Process().memory_info()._fields + ('rss', 'vms', 'shared', 'text', 'lib', 'data', 'dirty', 'uss', 'pss') + """ + valid_types = list(_psplatform.pfullmem._fields) + if memtype not in valid_types: + raise ValueError("invalid memtype %r; valid types are %r" % ( + memtype, tuple(valid_types))) + fun = self.memory_info if memtype in _psplatform.pmem._fields else \ + self.memory_full_info + metrics = fun() + value = getattr(metrics, memtype) + + # use cached value if available + total_phymem = _TOTAL_PHYMEM or virtual_memory().total + if not total_phymem > 0: + # we should never get here + raise ValueError( + "can't calculate process memory percent because " + "total physical system memory is not positive (%r)" + % total_phymem) + return (value / float(total_phymem)) * 100 + + if hasattr(_psplatform.Process, "memory_maps"): + def memory_maps(self, grouped=True): + """Return process' mapped memory regions as a list of namedtuples + whose fields are variable depending on the platform. + + If *grouped* is True the mapped regions with the same 'path' + are grouped together and the different memory fields are summed. + + If *grouped* is False every mapped region is shown as a single + entity and the namedtuple will also include the mapped region's + address space ('addr') and permission set ('perms'). + """ + it = self._proc.memory_maps() + if grouped: + d = {} + for tupl in it: + path = tupl[2] + nums = tupl[3:] + try: + d[path] = map(lambda x, y: x + y, d[path], nums) + except KeyError: + d[path] = nums + nt = _psplatform.pmmap_grouped + return [nt(path, *d[path]) for path in d] # NOQA + else: + nt = _psplatform.pmmap_ext + return [nt(*x) for x in it] + + def open_files(self): + """Return files opened by process as a list of + (path, fd) namedtuples including the absolute file name + and file descriptor number. + """ + return self._proc.open_files() + + def connections(self, kind='inet'): + """Return socket connections opened by process as a list of + (fd, family, type, laddr, raddr, status) namedtuples. + The *kind* parameter filters for connections that match the + following criteria: + + +------------+----------------------------------------------------+ + | Kind Value | Connections using | + +------------+----------------------------------------------------+ + | inet | IPv4 and IPv6 | + | inet4 | IPv4 | + | inet6 | IPv6 | + | tcp | TCP | + | tcp4 | TCP over IPv4 | + | tcp6 | TCP over IPv6 | + | udp | UDP | + | udp4 | UDP over IPv4 | + | udp6 | UDP over IPv6 | + | unix | UNIX socket (both UDP and TCP protocols) | + | all | the sum of all the possible families and protocols | + +------------+----------------------------------------------------+ + """ + return self._proc.connections(kind) + + # --- signals + + if POSIX: + def _send_signal(self, sig): + assert not self.pid < 0, self.pid + if self.pid == 0: + # see "man 2 kill" + raise ValueError( + "preventing sending signal to process with PID 0 as it " + "would affect every process in the process group of the " + "calling process (os.getpid()) instead of PID 0") + try: + os.kill(self.pid, sig) + except ProcessLookupError: + if OPENBSD and pid_exists(self.pid): + # We do this because os.kill() lies in case of + # zombie processes. + raise ZombieProcess(self.pid, self._name, self._ppid) + else: + self._gone = True + raise NoSuchProcess(self.pid, self._name) + except PermissionError: + raise AccessDenied(self.pid, self._name) + + @_assert_pid_not_reused + def send_signal(self, sig): + """Send a signal *sig* to process pre-emptively checking + whether PID has been reused (see signal module constants) . + On Windows only SIGTERM is valid and is treated as an alias + for kill(). + """ + if POSIX: + self._send_signal(sig) + else: # pragma: no cover + if sig == signal.SIGTERM: + self._proc.kill() + # py >= 2.7 + elif sig in (getattr(signal, "CTRL_C_EVENT", object()), + getattr(signal, "CTRL_BREAK_EVENT", object())): + self._proc.send_signal(sig) + else: + raise ValueError( + "only SIGTERM, CTRL_C_EVENT and CTRL_BREAK_EVENT signals " + "are supported on Windows") + + @_assert_pid_not_reused + def suspend(self): + """Suspend process execution with SIGSTOP pre-emptively checking + whether PID has been reused. + On Windows this has the effect ot suspending all process threads. + """ + if POSIX: + self._send_signal(signal.SIGSTOP) + else: # pragma: no cover + self._proc.suspend() + + @_assert_pid_not_reused + def resume(self): + """Resume process execution with SIGCONT pre-emptively checking + whether PID has been reused. + On Windows this has the effect of resuming all process threads. + """ + if POSIX: + self._send_signal(signal.SIGCONT) + else: # pragma: no cover + self._proc.resume() + + @_assert_pid_not_reused + def terminate(self): + """Terminate the process with SIGTERM pre-emptively checking + whether PID has been reused. + On Windows this is an alias for kill(). + """ + if POSIX: + self._send_signal(signal.SIGTERM) + else: # pragma: no cover + self._proc.kill() + + @_assert_pid_not_reused + def kill(self): + """Kill the current process with SIGKILL pre-emptively checking + whether PID has been reused. + """ + if POSIX: + self._send_signal(signal.SIGKILL) + else: # pragma: no cover + self._proc.kill() + + def wait(self, timeout=None): + """Wait for process to terminate and, if process is a children + of os.getpid(), also return its exit code, else None. + + If the process is already terminated immediately return None + instead of raising NoSuchProcess. + + If *timeout* (in seconds) is specified and process is still + alive raise TimeoutExpired. + + To wait for multiple Process(es) use psutil.wait_procs(). + """ + if timeout is not None and not timeout >= 0: + raise ValueError("timeout must be a positive integer") + return self._proc.wait(timeout) + + +# ===================================================================== +# --- Popen class +# ===================================================================== + + +class Popen(Process): + """A more convenient interface to stdlib subprocess.Popen class. + It starts a sub process and deals with it exactly as when using + subprocess.Popen class but in addition also provides all the + properties and methods of psutil.Process class as a unified + interface: + + >>> import psutil + >>> from subprocess import PIPE + >>> p = psutil.Popen(["python", "-c", "print 'hi'"], stdout=PIPE) + >>> p.name() + 'python' + >>> p.uids() + user(real=1000, effective=1000, saved=1000) + >>> p.username() + 'giampaolo' + >>> p.communicate() + ('hi\n', None) + >>> p.terminate() + >>> p.wait(timeout=2) + 0 + >>> + + For method names common to both classes such as kill(), terminate() + and wait(), psutil.Process implementation takes precedence. + + Unlike subprocess.Popen this class pre-emptively checks whether PID + has been reused on send_signal(), terminate() and kill() so that + you don't accidentally terminate another process, fixing + http://bugs.python.org/issue6973. + + For a complete documentation refer to: + http://docs.python.org/3/library/subprocess.html + """ + + def __init__(self, *args, **kwargs): + # Explicitly avoid to raise NoSuchProcess in case the process + # spawned by subprocess.Popen terminates too quickly, see: + # https://github.com/giampaolo/psutil/issues/193 + self.__subproc = subprocess.Popen(*args, **kwargs) + self._init(self.__subproc.pid, _ignore_nsp=True) + + def __dir__(self): + return sorted(set(dir(Popen) + dir(subprocess.Popen))) + + def __enter__(self): + if hasattr(self.__subproc, '__enter__'): + self.__subproc.__enter__() + return self + + def __exit__(self, *args, **kwargs): + if hasattr(self.__subproc, '__exit__'): + return self.__subproc.__exit__(*args, **kwargs) + else: + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + try: + # Flushing a BufferedWriter may raise an error. + if self.stdin: + self.stdin.close() + finally: + # Wait for the process to terminate, to avoid zombies. + self.wait() + + def __getattribute__(self, name): + try: + return object.__getattribute__(self, name) + except AttributeError: + try: + return object.__getattribute__(self.__subproc, name) + except AttributeError: + raise AttributeError("%s instance has no attribute '%s'" + % (self.__class__.__name__, name)) + + def wait(self, timeout=None): + if self.__subproc.returncode is not None: + return self.__subproc.returncode + ret = super(Popen, self).wait(timeout) + self.__subproc.returncode = ret + return ret + + +# The valid attr names which can be processed by Process.as_dict(). +_as_dict_attrnames = set( + [x for x in dir(Process) if not x.startswith('_') and x not in + ['send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', + 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', + 'memory_info_ex', 'oneshot']]) + + +# ===================================================================== +# --- system processes related functions +# ===================================================================== + + +def pids(): + """Return a list of current running PIDs.""" + global _LOWEST_PID + ret = sorted(_psplatform.pids()) + _LOWEST_PID = ret[0] + return ret + + +def pid_exists(pid): + """Return True if given PID exists in the current process list. + This is faster than doing "pid in psutil.pids()" and + should be preferred. + """ + if pid < 0: + return False + elif pid == 0 and POSIX: + # On POSIX we use os.kill() to determine PID existence. + # According to "man 2 kill" PID 0 has a special meaning + # though: it refers to <> and that is not we want + # to do here. + return pid in pids() + else: + return _psplatform.pid_exists(pid) + + +_pmap = {} +_lock = threading.Lock() + + +def process_iter(attrs=None, ad_value=None): + """Return a generator yielding a Process instance for all + running processes. + + Every new Process instance is only created once and then cached + into an internal table which is updated every time this is used. + + Cached Process instances are checked for identity so that you're + safe in case a PID has been reused by another process, in which + case the cached instance is updated. + + The sorting order in which processes are yielded is based on + their PIDs. + + *attrs* and *ad_value* have the same meaning as in + Process.as_dict(). If *attrs* is specified as_dict() is called + and the resulting dict is stored as a 'info' attribute attached + to returned Process instance. + If *attrs* is an empty list it will retrieve all process info + (slow). + """ + def add(pid): + proc = Process(pid) + if attrs is not None: + proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) + with _lock: + _pmap[proc.pid] = proc + return proc + + def remove(pid): + with _lock: + _pmap.pop(pid, None) + + a = set(pids()) + b = set(_pmap.keys()) + new_pids = a - b + gone_pids = b - a + for pid in gone_pids: + remove(pid) + + with _lock: + ls = sorted(list(_pmap.items()) + + list(dict.fromkeys(new_pids).items())) + + for pid, proc in ls: + try: + if proc is None: # new process + yield add(pid) + else: + # use is_running() to check whether PID has been reused by + # another process in which case yield a new Process instance + if proc.is_running(): + if attrs is not None: + proc.info = proc.as_dict( + attrs=attrs, ad_value=ad_value) + yield proc + else: + yield add(pid) + except NoSuchProcess: + remove(pid) + except AccessDenied: + # Process creation time can't be determined hence there's + # no way to tell whether the pid of the cached process + # has been reused. Just return the cached version. + if proc is None and pid in _pmap: + try: + yield _pmap[pid] + except KeyError: + # If we get here it is likely that 2 threads were + # using process_iter(). + pass + else: + raise + + +def wait_procs(procs, timeout=None, callback=None): + """Convenience function which waits for a list of processes to + terminate. + + Return a (gone, alive) tuple indicating which processes + are gone and which ones are still alive. + + The gone ones will have a new *returncode* attribute indicating + process exit status (may be None). + + *callback* is a function which gets called every time a process + terminates (a Process instance is passed as callback argument). + + Function will return as soon as all processes terminate or when + *timeout* occurs. + Differently from Process.wait() it will not raise TimeoutExpired if + *timeout* occurs. + + Typical use case is: + + - send SIGTERM to a list of processes + - give them some time to terminate + - send SIGKILL to those ones which are still alive + + Example: + + >>> def on_terminate(proc): + ... print("process {} terminated".format(proc)) + ... + >>> for p in procs: + ... p.terminate() + ... + >>> gone, alive = wait_procs(procs, timeout=3, callback=on_terminate) + >>> for p in alive: + ... p.kill() + """ + def check_gone(proc, timeout): + try: + returncode = proc.wait(timeout=timeout) + except TimeoutExpired: + pass + else: + if returncode is not None or not proc.is_running(): + proc.returncode = returncode + gone.add(proc) + if callback is not None: + callback(proc) + + if timeout is not None and not timeout >= 0: + msg = "timeout must be a positive integer, got %s" % timeout + raise ValueError(msg) + gone = set() + alive = set(procs) + if callback is not None and not callable(callback): + raise TypeError("callback %r is not a callable" % callable) + if timeout is not None: + deadline = _timer() + timeout + + while alive: + if timeout is not None and timeout <= 0: + break + for proc in alive: + # Make sure that every complete iteration (all processes) + # will last max 1 sec. + # We do this because we don't want to wait too long on a + # single process: in case it terminates too late other + # processes may disappear in the meantime and their PID + # reused. + max_timeout = 1.0 / len(alive) + if timeout is not None: + timeout = min((deadline - _timer()), max_timeout) + if timeout <= 0: + break + check_gone(proc, timeout) + else: + check_gone(proc, max_timeout) + alive = alive - gone + + if alive: + # Last attempt over processes survived so far. + # timeout == 0 won't make this function wait any further. + for proc in alive: + check_gone(proc, 0) + alive = alive - gone + + return (list(gone), list(alive)) + + +# ===================================================================== +# --- CPU related functions +# ===================================================================== + + +def cpu_count(logical=True): + """Return the number of logical CPUs in the system (same as + os.cpu_count() in Python 3.4). + + If *logical* is False return the number of physical cores only + (e.g. hyper thread CPUs are excluded). + + Return None if undetermined. + + The return value is cached after first call. + If desired cache can be cleared like this: + + >>> psutil.cpu_count.cache_clear() + """ + if logical: + ret = _psplatform.cpu_count_logical() + else: + ret = _psplatform.cpu_count_physical() + if ret is not None and ret < 1: + ret = None + return ret + + +def cpu_times(percpu=False): + """Return system-wide CPU times as a namedtuple. + Every CPU time represents the seconds the CPU has spent in the + given mode. The namedtuple's fields availability varies depending on the + platform: + + - user + - system + - idle + - nice (UNIX) + - iowait (Linux) + - irq (Linux, FreeBSD) + - softirq (Linux) + - steal (Linux >= 2.6.11) + - guest (Linux >= 2.6.24) + - guest_nice (Linux >= 3.2.0) + + When *percpu* is True return a list of namedtuples for each CPU. + First element of the list refers to first CPU, second element + to second CPU and so on. + The order of the list is consistent across calls. + """ + if not percpu: + return _psplatform.cpu_times() + else: + return _psplatform.per_cpu_times() + + +try: + _last_cpu_times = cpu_times() +except Exception: + # Don't want to crash at import time. + _last_cpu_times = None + +try: + _last_per_cpu_times = cpu_times(percpu=True) +except Exception: + # Don't want to crash at import time. + _last_per_cpu_times = None + + +def _cpu_tot_time(times): + """Given a cpu_time() ntuple calculates the total CPU time + (including idle time). + """ + tot = sum(times) + if LINUX: + # On Linux guest times are already accounted in "user" or + # "nice" times, so we subtract them from total. + # Htop does the same. References: + # https://github.com/giampaolo/psutil/pull/940 + # http://unix.stackexchange.com/questions/178045 + # https://github.com/torvalds/linux/blob/ + # 447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/ + # cputime.c#L158 + tot -= getattr(times, "guest", 0) # Linux 2.6.24+ + tot -= getattr(times, "guest_nice", 0) # Linux 3.2.0+ + return tot + + +def _cpu_busy_time(times): + """Given a cpu_time() ntuple calculates the busy CPU time. + We do so by subtracting all idle CPU times. + """ + busy = _cpu_tot_time(times) + busy -= times.idle + # Linux: "iowait" is time during which the CPU does not do anything + # (waits for IO to complete). On Linux IO wait is *not* accounted + # in "idle" time so we subtract it. Htop does the same. + # References: + # https://github.com/torvalds/linux/blob/ + # 447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/cputime.c#L244 + busy -= getattr(times, "iowait", 0) + return busy + + +def _cpu_times_deltas(t1, t2): + assert t1._fields == t2._fields, (t1, t2) + field_deltas = [] + for field in _psplatform.scputimes._fields: + field_delta = getattr(t2, field) - getattr(t1, field) + # CPU times are always supposed to increase over time + # or at least remain the same and that's because time + # cannot go backwards. + # Surprisingly sometimes this might not be the case (at + # least on Windows and Linux), see: + # https://github.com/giampaolo/psutil/issues/392 + # https://github.com/giampaolo/psutil/issues/645 + # https://github.com/giampaolo/psutil/issues/1210 + # Trim negative deltas to zero to ignore decreasing fields. + # top does the same. Reference: + # https://gitlab.com/procps-ng/procps/blob/v3.3.12/top/top.c#L5063 + field_delta = max(0, field_delta) + field_deltas.append(field_delta) + return _psplatform.scputimes(*field_deltas) + + +def cpu_percent(interval=None, percpu=False): + """Return a float representing the current system-wide CPU + utilization as a percentage. + + When *interval* is > 0.0 compares system CPU times elapsed before + and after the interval (blocking). + + When *interval* is 0.0 or None compares system CPU times elapsed + since last call or module import, returning immediately (non + blocking). That means the first time this is called it will + return a meaningless 0.0 value which you should ignore. + In this case is recommended for accuracy that this function be + called with at least 0.1 seconds between calls. + + When *percpu* is True returns a list of floats representing the + utilization as a percentage for each CPU. + First element of the list refers to first CPU, second element + to second CPU and so on. + The order of the list is consistent across calls. + + Examples: + + >>> # blocking, system-wide + >>> psutil.cpu_percent(interval=1) + 2.0 + >>> + >>> # blocking, per-cpu + >>> psutil.cpu_percent(interval=1, percpu=True) + [2.0, 1.0] + >>> + >>> # non-blocking (percentage since last call) + >>> psutil.cpu_percent(interval=None) + 2.9 + >>> + """ + global _last_cpu_times + global _last_per_cpu_times + blocking = interval is not None and interval > 0.0 + if interval is not None and interval < 0: + raise ValueError("interval is not positive (got %r)" % interval) + + def calculate(t1, t2): + times_delta = _cpu_times_deltas(t1, t2) + + all_delta = _cpu_tot_time(times_delta) + busy_delta = _cpu_busy_time(times_delta) + + try: + busy_perc = (busy_delta / all_delta) * 100 + except ZeroDivisionError: + return 0.0 + else: + return round(busy_perc, 1) + + # system-wide usage + if not percpu: + if blocking: + t1 = cpu_times() + time.sleep(interval) + else: + t1 = _last_cpu_times + if t1 is None: + # Something bad happened at import time. We'll + # get a meaningful result on the next call. See: + # https://github.com/giampaolo/psutil/pull/715 + t1 = cpu_times() + _last_cpu_times = cpu_times() + return calculate(t1, _last_cpu_times) + # per-cpu usage + else: + ret = [] + if blocking: + tot1 = cpu_times(percpu=True) + time.sleep(interval) + else: + tot1 = _last_per_cpu_times + if tot1 is None: + # Something bad happened at import time. We'll + # get a meaningful result on the next call. See: + # https://github.com/giampaolo/psutil/pull/715 + tot1 = cpu_times(percpu=True) + _last_per_cpu_times = cpu_times(percpu=True) + for t1, t2 in zip(tot1, _last_per_cpu_times): + ret.append(calculate(t1, t2)) + return ret + + +# Use separate global vars for cpu_times_percent() so that it's +# independent from cpu_percent() and they can both be used within +# the same program. +_last_cpu_times_2 = _last_cpu_times +_last_per_cpu_times_2 = _last_per_cpu_times + + +def cpu_times_percent(interval=None, percpu=False): + """Same as cpu_percent() but provides utilization percentages + for each specific CPU time as is returned by cpu_times(). + For instance, on Linux we'll get: + + >>> cpu_times_percent() + cpupercent(user=4.8, nice=0.0, system=4.8, idle=90.5, iowait=0.0, + irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) + >>> + + *interval* and *percpu* arguments have the same meaning as in + cpu_percent(). + """ + global _last_cpu_times_2 + global _last_per_cpu_times_2 + blocking = interval is not None and interval > 0.0 + if interval is not None and interval < 0: + raise ValueError("interval is not positive (got %r)" % interval) + + def calculate(t1, t2): + nums = [] + times_delta = _cpu_times_deltas(t1, t2) + all_delta = _cpu_tot_time(times_delta) + # "scale" is the value to multiply each delta with to get percentages. + # We use "max" to avoid division by zero (if all_delta is 0, then all + # fields are 0 so percentages will be 0 too. all_delta cannot be a + # fraction because cpu times are integers) + scale = 100.0 / max(1, all_delta) + for field_delta in times_delta: + field_perc = field_delta * scale + field_perc = round(field_perc, 1) + # make sure we don't return negative values or values over 100% + field_perc = min(max(0.0, field_perc), 100.0) + nums.append(field_perc) + return _psplatform.scputimes(*nums) + + # system-wide usage + if not percpu: + if blocking: + t1 = cpu_times() + time.sleep(interval) + else: + t1 = _last_cpu_times_2 + if t1 is None: + # Something bad happened at import time. We'll + # get a meaningful result on the next call. See: + # https://github.com/giampaolo/psutil/pull/715 + t1 = cpu_times() + _last_cpu_times_2 = cpu_times() + return calculate(t1, _last_cpu_times_2) + # per-cpu usage + else: + ret = [] + if blocking: + tot1 = cpu_times(percpu=True) + time.sleep(interval) + else: + tot1 = _last_per_cpu_times_2 + if tot1 is None: + # Something bad happened at import time. We'll + # get a meaningful result on the next call. See: + # https://github.com/giampaolo/psutil/pull/715 + tot1 = cpu_times(percpu=True) + _last_per_cpu_times_2 = cpu_times(percpu=True) + for t1, t2 in zip(tot1, _last_per_cpu_times_2): + ret.append(calculate(t1, t2)) + return ret + + +def cpu_stats(): + """Return CPU statistics.""" + return _psplatform.cpu_stats() + + +if hasattr(_psplatform, "cpu_freq"): + + def cpu_freq(percpu=False): + """Return CPU frequency as a nameduple including current, + min and max frequency expressed in Mhz. + + If *percpu* is True and the system supports per-cpu frequency + retrieval (Linux only) a list of frequencies is returned for + each CPU. If not a list with one element is returned. + """ + ret = _psplatform.cpu_freq() + if percpu: + return ret + else: + num_cpus = float(len(ret)) + if num_cpus == 0: + return None + elif num_cpus == 1: + return ret[0] + else: + currs, mins, maxs = 0.0, 0.0, 0.0 + set_none = False + for cpu in ret: + currs += cpu.current + # On Linux if /proc/cpuinfo is used min/max are set + # to None. + if LINUX and cpu.min is None: + set_none = True + continue + mins += cpu.min + maxs += cpu.max + + current = currs / num_cpus + + if set_none: + min_ = max_ = None + else: + min_ = mins / num_cpus + max_ = maxs / num_cpus + + return _common.scpufreq(current, min_, max_) + + __all__.append("cpu_freq") + + +if hasattr(os, "getloadavg") or hasattr(_psplatform, "getloadavg"): + # Perform this hasattr check once on import time to either use the + # platform based code or proxy straight from the os module. + if hasattr(os, "getloadavg"): + getloadavg = os.getloadavg + else: + getloadavg = _psplatform.getloadavg + + __all__.append("getloadavg") + + +# ===================================================================== +# --- system memory related functions +# ===================================================================== + + +def virtual_memory(): + """Return statistics about system memory usage as a namedtuple + including the following fields, expressed in bytes: + + - total: + total physical memory available. + + - available: + the memory that can be given instantly to processes without the + system going into swap. + This is calculated by summing different memory values depending + on the platform and it is supposed to be used to monitor actual + memory usage in a cross platform fashion. + + - percent: + the percentage usage calculated as (total - available) / total * 100 + + - used: + memory used, calculated differently depending on the platform and + designed for informational purposes only: + macOS: active + wired + BSD: active + wired + cached + Linux: total - free + + - free: + memory not being used at all (zeroed) that is readily available; + note that this doesn't reflect the actual memory available + (use 'available' instead) + + Platform-specific fields: + + - active (UNIX): + memory currently in use or very recently used, and so it is in RAM. + + - inactive (UNIX): + memory that is marked as not used. + + - buffers (BSD, Linux): + cache for things like file system metadata. + + - cached (BSD, macOS): + cache for various things. + + - wired (macOS, BSD): + memory that is marked to always stay in RAM. It is never moved to disk. + + - shared (BSD): + memory that may be simultaneously accessed by multiple processes. + + The sum of 'used' and 'available' does not necessarily equal total. + On Windows 'available' and 'free' are the same. + """ + global _TOTAL_PHYMEM + ret = _psplatform.virtual_memory() + # cached for later use in Process.memory_percent() + _TOTAL_PHYMEM = ret.total + return ret + + +def swap_memory(): + """Return system swap memory statistics as a namedtuple including + the following fields: + + - total: total swap memory in bytes + - used: used swap memory in bytes + - free: free swap memory in bytes + - percent: the percentage usage + - sin: no. of bytes the system has swapped in from disk (cumulative) + - sout: no. of bytes the system has swapped out from disk (cumulative) + + 'sin' and 'sout' on Windows are meaningless and always set to 0. + """ + return _psplatform.swap_memory() + + +# ===================================================================== +# --- disks/paritions related functions +# ===================================================================== + + +def disk_usage(path): + """Return disk usage statistics about the given *path* as a + namedtuple including total, used and free space expressed in bytes + plus the percentage usage. + """ + return _psplatform.disk_usage(path) + + +def disk_partitions(all=False): + """Return mounted partitions as a list of + (device, mountpoint, fstype, opts) namedtuple. + 'opts' field is a raw string separated by commas indicating mount + options which may vary depending on the platform. + + If *all* parameter is False return physical devices only and ignore + all others. + """ + return _psplatform.disk_partitions(all) + + +def disk_io_counters(perdisk=False, nowrap=True): + """Return system disk I/O statistics as a namedtuple including + the following fields: + + - read_count: number of reads + - write_count: number of writes + - read_bytes: number of bytes read + - write_bytes: number of bytes written + - read_time: time spent reading from disk (in ms) + - write_time: time spent writing to disk (in ms) + + Platform specific: + + - busy_time: (Linux, FreeBSD) time spent doing actual I/Os (in ms) + - read_merged_count (Linux): number of merged reads + - write_merged_count (Linux): number of merged writes + + If *perdisk* is True return the same information for every + physical disk installed on the system as a dictionary + with partition names as the keys and the namedtuple + described above as the values. + + If *nowrap* is True it detects and adjust the numbers which overflow + and wrap (restart from 0) and add "old value" to "new value" so that + the returned numbers will always be increasing or remain the same, + but never decrease. + "disk_io_counters.cache_clear()" can be used to invalidate the + cache. + + On recent Windows versions 'diskperf -y' command may need to be + executed first otherwise this function won't find any disk. + """ + kwargs = dict(perdisk=perdisk) if LINUX else {} + rawdict = _psplatform.disk_io_counters(**kwargs) + if not rawdict: + return {} if perdisk else None + if nowrap: + rawdict = _wrap_numbers(rawdict, 'psutil.disk_io_counters') + nt = getattr(_psplatform, "sdiskio", _common.sdiskio) + if perdisk: + for disk, fields in rawdict.items(): + rawdict[disk] = nt(*fields) + return rawdict + else: + return nt(*[sum(x) for x in zip(*rawdict.values())]) + + +disk_io_counters.cache_clear = functools.partial( + _wrap_numbers.cache_clear, 'psutil.disk_io_counters') +disk_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache" + + +# ===================================================================== +# --- network related functions +# ===================================================================== + + +def net_io_counters(pernic=False, nowrap=True): + """Return network I/O statistics as a namedtuple including + the following fields: + + - bytes_sent: number of bytes sent + - bytes_recv: number of bytes received + - packets_sent: number of packets sent + - packets_recv: number of packets received + - errin: total number of errors while receiving + - errout: total number of errors while sending + - dropin: total number of incoming packets which were dropped + - dropout: total number of outgoing packets which were dropped + (always 0 on macOS and BSD) + + If *pernic* is True return the same information for every + network interface installed on the system as a dictionary + with network interface names as the keys and the namedtuple + described above as the values. + + If *nowrap* is True it detects and adjust the numbers which overflow + and wrap (restart from 0) and add "old value" to "new value" so that + the returned numbers will always be increasing or remain the same, + but never decrease. + "disk_io_counters.cache_clear()" can be used to invalidate the + cache. + """ + rawdict = _psplatform.net_io_counters() + if not rawdict: + return {} if pernic else None + if nowrap: + rawdict = _wrap_numbers(rawdict, 'psutil.net_io_counters') + if pernic: + for nic, fields in rawdict.items(): + rawdict[nic] = _common.snetio(*fields) + return rawdict + else: + return _common.snetio(*[sum(x) for x in zip(*rawdict.values())]) + + +net_io_counters.cache_clear = functools.partial( + _wrap_numbers.cache_clear, 'psutil.net_io_counters') +net_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache" + + +def net_connections(kind='inet'): + """Return system-wide socket connections as a list of + (fd, family, type, laddr, raddr, status, pid) namedtuples. + In case of limited privileges 'fd' and 'pid' may be set to -1 + and None respectively. + The *kind* parameter filters for connections that fit the + following criteria: + + +------------+----------------------------------------------------+ + | Kind Value | Connections using | + +------------+----------------------------------------------------+ + | inet | IPv4 and IPv6 | + | inet4 | IPv4 | + | inet6 | IPv6 | + | tcp | TCP | + | tcp4 | TCP over IPv4 | + | tcp6 | TCP over IPv6 | + | udp | UDP | + | udp4 | UDP over IPv4 | + | udp6 | UDP over IPv6 | + | unix | UNIX socket (both UDP and TCP protocols) | + | all | the sum of all the possible families and protocols | + +------------+----------------------------------------------------+ + + On macOS this function requires root privileges. + """ + return _psplatform.net_connections(kind) + + +def net_if_addrs(): + """Return the addresses associated to each NIC (network interface + card) installed on the system as a dictionary whose keys are the + NIC names and value is a list of namedtuples for each address + assigned to the NIC. Each namedtuple includes 5 fields: + + - family: can be either socket.AF_INET, socket.AF_INET6 or + psutil.AF_LINK, which refers to a MAC address. + - address: is the primary address and it is always set. + - netmask: and 'broadcast' and 'ptp' may be None. + - ptp: stands for "point to point" and references the + destination address on a point to point interface + (typically a VPN). + - broadcast: and *ptp* are mutually exclusive. + + Note: you can have more than one address of the same family + associated with each interface. + """ + has_enums = sys.version_info >= (3, 4) + if has_enums: + import socket + rawlist = _psplatform.net_if_addrs() + rawlist.sort(key=lambda x: x[1]) # sort by family + ret = collections.defaultdict(list) + for name, fam, addr, mask, broadcast, ptp in rawlist: + if has_enums: + try: + fam = socket.AddressFamily(fam) + except ValueError: + if WINDOWS and fam == -1: + fam = _psplatform.AF_LINK + elif (hasattr(_psplatform, "AF_LINK") and + _psplatform.AF_LINK == fam): + # Linux defines AF_LINK as an alias for AF_PACKET. + # We re-set the family here so that repr(family) + # will show AF_LINK rather than AF_PACKET + fam = _psplatform.AF_LINK + if fam == _psplatform.AF_LINK: + # The underlying C function may return an incomplete MAC + # address in which case we fill it with null bytes, see: + # https://github.com/giampaolo/psutil/issues/786 + separator = ":" if POSIX else "-" + while addr.count(separator) < 5: + addr += "%s00" % separator + ret[name].append(_common.snicaddr(fam, addr, mask, broadcast, ptp)) + return dict(ret) + + +def net_if_stats(): + """Return information about each NIC (network interface card) + installed on the system as a dictionary whose keys are the + NIC names and value is a namedtuple with the following fields: + + - isup: whether the interface is up (bool) + - duplex: can be either NIC_DUPLEX_FULL, NIC_DUPLEX_HALF or + NIC_DUPLEX_UNKNOWN + - speed: the NIC speed expressed in mega bits (MB); if it can't + be determined (e.g. 'localhost') it will be set to 0. + - mtu: the maximum transmission unit expressed in bytes. + """ + return _psplatform.net_if_stats() + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +# Linux, macOS +if hasattr(_psplatform, "sensors_temperatures"): + + def sensors_temperatures(fahrenheit=False): + """Return hardware temperatures. Each entry is a namedtuple + representing a certain hardware sensor (it may be a CPU, an + hard disk or something else, depending on the OS and its + configuration). + All temperatures are expressed in celsius unless *fahrenheit* + is set to True. + """ + def convert(n): + if n is not None: + return (float(n) * 9 / 5) + 32 if fahrenheit else n + + ret = collections.defaultdict(list) + rawdict = _psplatform.sensors_temperatures() + + for name, values in rawdict.items(): + while values: + label, current, high, critical = values.pop(0) + current = convert(current) + high = convert(high) + critical = convert(critical) + + if high and not critical: + critical = high + elif critical and not high: + high = critical + + ret[name].append( + _common.shwtemp(label, current, high, critical)) + + return dict(ret) + + __all__.append("sensors_temperatures") + + +# Linux, macOS +if hasattr(_psplatform, "sensors_fans"): + + def sensors_fans(): + """Return fans speed. Each entry is a namedtuple + representing a certain hardware sensor. + All speed are expressed in RPM (rounds per minute). + """ + return _psplatform.sensors_fans() + + __all__.append("sensors_fans") + + +# Linux, Windows, FreeBSD, macOS +if hasattr(_psplatform, "sensors_battery"): + + def sensors_battery(): + """Return battery information. If no battery is installed + returns None. + + - percent: battery power left as a percentage. + - secsleft: a rough approximation of how many seconds are left + before the battery runs out of power. May be + POWER_TIME_UNLIMITED or POWER_TIME_UNLIMITED. + - power_plugged: True if the AC power cable is connected. + """ + return _psplatform.sensors_battery() + + __all__.append("sensors_battery") + + +# ===================================================================== +# --- other system related functions +# ===================================================================== + + +def boot_time(): + """Return the system boot time expressed in seconds since the epoch.""" + # Note: we are not caching this because it is subject to + # system clock updates. + return _psplatform.boot_time() + + +def users(): + """Return users currently connected on the system as a list of + namedtuples including the following fields. + + - user: the name of the user + - terminal: the tty or pseudo-tty associated with the user, if any. + - host: the host name associated with the entry, if any. + - started: the creation time as a floating point number expressed in + seconds since the epoch. + """ + return _psplatform.users() + + +# ===================================================================== +# --- Windows services +# ===================================================================== + + +if WINDOWS: + + def win_service_iter(): + """Return a generator yielding a WindowsService instance for all + Windows services installed. + """ + return _psplatform.win_service_iter() + + def win_service_get(name): + """Get a Windows service by *name*. + Raise NoSuchProcess if no service with such name exists. + """ + return _psplatform.win_service_get(name) + + +# ===================================================================== + + +def test(): # pragma: no cover + from ._common import bytes2human + from ._compat import get_terminal_size + + today_day = datetime.date.today() + templ = "%-10s %5s %5s %7s %7s %5s %6s %6s %6s %s" + attrs = ['pid', 'memory_percent', 'name', 'cmdline', 'cpu_times', + 'create_time', 'memory_info', 'status', 'nice', 'username'] + print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE", + "STATUS", "START", "TIME", "CMDLINE")) + for p in process_iter(attrs, ad_value=None): + if p.info['create_time']: + ctime = datetime.datetime.fromtimestamp(p.info['create_time']) + if ctime.date() == today_day: + ctime = ctime.strftime("%H:%M") + else: + ctime = ctime.strftime("%b%d") + else: + ctime = '' + if p.info['cpu_times']: + cputime = time.strftime("%M:%S", + time.localtime(sum(p.info['cpu_times']))) + else: + cputime = '' + + user = p.info['username'] or '' + if not user and POSIX: + try: + user = p.uids()[0] + except Error: + pass + if user and WINDOWS and '\\' in user: + user = user.split('\\')[1] + user = user[:9] + vms = bytes2human(p.info['memory_info'].vms) if \ + p.info['memory_info'] is not None else '' + rss = bytes2human(p.info['memory_info'].rss) if \ + p.info['memory_info'] is not None else '' + memp = round(p.info['memory_percent'], 1) if \ + p.info['memory_percent'] is not None else '' + nice = int(p.info['nice']) if p.info['nice'] else '' + if p.info['cmdline']: + cmdline = ' '.join(p.info['cmdline']) + else: + cmdline = p.info['name'] + status = p.info['status'][:5] if p.info['status'] else '' + + line = templ % ( + user[:10], + p.info['pid'], + memp, + vms, + rss, + nice, + status, + ctime, + cputime, + cmdline) + print(line[:get_terminal_size()[0]]) + + +del memoize, memoize_when_activated, division, deprecated_method +if sys.version_info[0] < 3: + del num, x + +if __name__ == "__main__": + test() diff --git a/ddtrace/vendor/psutil/_common.py b/ddtrace/vendor/psutil/_common.py new file mode 100644 index 00000000000..a0f5e751b83 --- /dev/null +++ b/ddtrace/vendor/psutil/_common.py @@ -0,0 +1,651 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Common objects shared by __init__.py and _ps*.py modules.""" + +# Note: this module is imported by setup.py so it should not import +# psutil or third-party modules. + +from __future__ import division + +import contextlib +import errno +import functools +import os +import socket +import stat +import sys +import threading +import warnings +from collections import defaultdict +from collections import namedtuple +from socket import AF_INET +from socket import SOCK_DGRAM +from socket import SOCK_STREAM +try: + from socket import AF_INET6 +except ImportError: + AF_INET6 = None +try: + from socket import AF_UNIX +except ImportError: + AF_UNIX = None + +if sys.version_info >= (3, 4): + import enum +else: + enum = None + +# can't take it from _common.py as this script is imported by setup.py +PY3 = sys.version_info[0] == 3 + +__all__ = [ + # constants + 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'MACOS', 'OSX', 'POSIX', + 'SUNOS', 'WINDOWS', + 'ENCODING', 'ENCODING_ERRS', 'AF_INET6', + # connection constants + 'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED', + 'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN', + 'CONN_NONE', 'CONN_SYN_RECV', 'CONN_SYN_SENT', 'CONN_TIME_WAIT', + # net constants + 'NIC_DUPLEX_FULL', 'NIC_DUPLEX_HALF', 'NIC_DUPLEX_UNKNOWN', + # process status constants + 'STATUS_DEAD', 'STATUS_DISK_SLEEP', 'STATUS_IDLE', 'STATUS_LOCKED', + 'STATUS_RUNNING', 'STATUS_SLEEPING', 'STATUS_STOPPED', 'STATUS_SUSPENDED', + 'STATUS_TRACING_STOP', 'STATUS_WAITING', 'STATUS_WAKE_KILL', + 'STATUS_WAKING', 'STATUS_ZOMBIE', 'STATUS_PARKED', + # named tuples + 'pconn', 'pcputimes', 'pctxsw', 'pgids', 'pio', 'pionice', 'popenfile', + 'pthread', 'puids', 'sconn', 'scpustats', 'sdiskio', 'sdiskpart', + 'sdiskusage', 'snetio', 'snicaddr', 'snicstats', 'sswap', 'suser', + # utility functions + 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', + 'parse_environ_block', 'path_exists_strict', 'usage_percent', + 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers", + 'bytes2human', 'conn_to_ntuple', +] + + +# =================================================================== +# --- OS constants +# =================================================================== + + +POSIX = os.name == "posix" +WINDOWS = os.name == "nt" +LINUX = sys.platform.startswith("linux") +MACOS = sys.platform.startswith("darwin") +OSX = MACOS # deprecated alias +FREEBSD = sys.platform.startswith("freebsd") +OPENBSD = sys.platform.startswith("openbsd") +NETBSD = sys.platform.startswith("netbsd") +BSD = FREEBSD or OPENBSD or NETBSD +SUNOS = sys.platform.startswith(("sunos", "solaris")) +AIX = sys.platform.startswith("aix") + + +# =================================================================== +# --- API constants +# =================================================================== + + +# Process.status() +STATUS_RUNNING = "running" +STATUS_SLEEPING = "sleeping" +STATUS_DISK_SLEEP = "disk-sleep" +STATUS_STOPPED = "stopped" +STATUS_TRACING_STOP = "tracing-stop" +STATUS_ZOMBIE = "zombie" +STATUS_DEAD = "dead" +STATUS_WAKE_KILL = "wake-kill" +STATUS_WAKING = "waking" +STATUS_IDLE = "idle" # Linux, macOS, FreeBSD +STATUS_LOCKED = "locked" # FreeBSD +STATUS_WAITING = "waiting" # FreeBSD +STATUS_SUSPENDED = "suspended" # NetBSD +STATUS_PARKED = "parked" # Linux + +# Process.connections() and psutil.net_connections() +CONN_ESTABLISHED = "ESTABLISHED" +CONN_SYN_SENT = "SYN_SENT" +CONN_SYN_RECV = "SYN_RECV" +CONN_FIN_WAIT1 = "FIN_WAIT1" +CONN_FIN_WAIT2 = "FIN_WAIT2" +CONN_TIME_WAIT = "TIME_WAIT" +CONN_CLOSE = "CLOSE" +CONN_CLOSE_WAIT = "CLOSE_WAIT" +CONN_LAST_ACK = "LAST_ACK" +CONN_LISTEN = "LISTEN" +CONN_CLOSING = "CLOSING" +CONN_NONE = "NONE" + +# net_if_stats() +if enum is None: + NIC_DUPLEX_FULL = 2 + NIC_DUPLEX_HALF = 1 + NIC_DUPLEX_UNKNOWN = 0 +else: + class NicDuplex(enum.IntEnum): + NIC_DUPLEX_FULL = 2 + NIC_DUPLEX_HALF = 1 + NIC_DUPLEX_UNKNOWN = 0 + + globals().update(NicDuplex.__members__) + +# sensors_battery() +if enum is None: + POWER_TIME_UNKNOWN = -1 + POWER_TIME_UNLIMITED = -2 +else: + class BatteryTime(enum.IntEnum): + POWER_TIME_UNKNOWN = -1 + POWER_TIME_UNLIMITED = -2 + + globals().update(BatteryTime.__members__) + +# --- others + +ENCODING = sys.getfilesystemencoding() +if not PY3: + ENCODING_ERRS = "replace" +else: + try: + ENCODING_ERRS = sys.getfilesystemencodeerrors() # py 3.6 + except AttributeError: + ENCODING_ERRS = "surrogateescape" if POSIX else "replace" + + +# =================================================================== +# --- namedtuples +# =================================================================== + +# --- for system functions + +# psutil.swap_memory() +sswap = namedtuple('sswap', ['total', 'used', 'free', 'percent', 'sin', + 'sout']) +# psutil.disk_usage() +sdiskusage = namedtuple('sdiskusage', ['total', 'used', 'free', 'percent']) +# psutil.disk_io_counters() +sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_time', 'write_time']) +# psutil.disk_partitions() +sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint', 'fstype', 'opts']) +# psutil.net_io_counters() +snetio = namedtuple('snetio', ['bytes_sent', 'bytes_recv', + 'packets_sent', 'packets_recv', + 'errin', 'errout', + 'dropin', 'dropout']) +# psutil.users() +suser = namedtuple('suser', ['name', 'terminal', 'host', 'started', 'pid']) +# psutil.net_connections() +sconn = namedtuple('sconn', ['fd', 'family', 'type', 'laddr', 'raddr', + 'status', 'pid']) +# psutil.net_if_addrs() +snicaddr = namedtuple('snicaddr', + ['family', 'address', 'netmask', 'broadcast', 'ptp']) +# psutil.net_if_stats() +snicstats = namedtuple('snicstats', ['isup', 'duplex', 'speed', 'mtu']) +# psutil.cpu_stats() +scpustats = namedtuple( + 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) +# psutil.cpu_freq() +scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) +# psutil.sensors_temperatures() +shwtemp = namedtuple( + 'shwtemp', ['label', 'current', 'high', 'critical']) +# psutil.sensors_battery() +sbattery = namedtuple('sbattery', ['percent', 'secsleft', 'power_plugged']) +# psutil.sensors_fans() +sfan = namedtuple('sfan', ['label', 'current']) + +# --- for Process methods + +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system']) +# psutil.Process.open_files() +popenfile = namedtuple('popenfile', ['path', 'fd']) +# psutil.Process.threads() +pthread = namedtuple('pthread', ['id', 'user_time', 'system_time']) +# psutil.Process.uids() +puids = namedtuple('puids', ['real', 'effective', 'saved']) +# psutil.Process.gids() +pgids = namedtuple('pgids', ['real', 'effective', 'saved']) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes']) +# psutil.Process.ionice() +pionice = namedtuple('pionice', ['ioclass', 'value']) +# psutil.Process.ctx_switches() +pctxsw = namedtuple('pctxsw', ['voluntary', 'involuntary']) +# psutil.Process.connections() +pconn = namedtuple('pconn', ['fd', 'family', 'type', 'laddr', 'raddr', + 'status']) + +# psutil.connections() and psutil.Process.connections() +addr = namedtuple('addr', ['ip', 'port']) + + +# =================================================================== +# --- Process.connections() 'kind' parameter mapping +# =================================================================== + + +conn_tmap = { + "all": ([AF_INET, AF_INET6, AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]), + "tcp": ([AF_INET, AF_INET6], [SOCK_STREAM]), + "tcp4": ([AF_INET], [SOCK_STREAM]), + "udp": ([AF_INET, AF_INET6], [SOCK_DGRAM]), + "udp4": ([AF_INET], [SOCK_DGRAM]), + "inet": ([AF_INET, AF_INET6], [SOCK_STREAM, SOCK_DGRAM]), + "inet4": ([AF_INET], [SOCK_STREAM, SOCK_DGRAM]), + "inet6": ([AF_INET6], [SOCK_STREAM, SOCK_DGRAM]), +} + +if AF_INET6 is not None: + conn_tmap.update({ + "tcp6": ([AF_INET6], [SOCK_STREAM]), + "udp6": ([AF_INET6], [SOCK_DGRAM]), + }) + +if AF_UNIX is not None: + conn_tmap.update({ + "unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]), + }) + + +# =================================================================== +# --- utils +# =================================================================== + + +def usage_percent(used, total, round_=None): + """Calculate percentage usage of 'used' against 'total'.""" + try: + ret = (float(used) / total) * 100 + except ZeroDivisionError: + return 0.0 + else: + if round_ is not None: + ret = round(ret, round_) + return ret + + +def memoize(fun): + """A simple memoize decorator for functions supporting (hashable) + positional arguments. + It also provides a cache_clear() function for clearing the cache: + + >>> @memoize + ... def foo() + ... return 1 + ... + >>> foo() + 1 + >>> foo.cache_clear() + >>> + """ + @functools.wraps(fun) + def wrapper(*args, **kwargs): + key = (args, frozenset(sorted(kwargs.items()))) + try: + return cache[key] + except KeyError: + ret = cache[key] = fun(*args, **kwargs) + return ret + + def cache_clear(): + """Clear cache.""" + cache.clear() + + cache = {} + wrapper.cache_clear = cache_clear + return wrapper + + +def memoize_when_activated(fun): + """A memoize decorator which is disabled by default. It can be + activated and deactivated on request. + For efficiency reasons it can be used only against class methods + accepting no arguments. + + >>> class Foo: + ... @memoize + ... def foo() + ... print(1) + ... + >>> f = Foo() + >>> # deactivated (default) + >>> foo() + 1 + >>> foo() + 1 + >>> + >>> # activated + >>> foo.cache_activate(self) + >>> foo() + 1 + >>> foo() + >>> foo() + >>> + """ + @functools.wraps(fun) + def wrapper(self): + try: + # case 1: we previously entered oneshot() ctx + ret = self._cache[fun] + except AttributeError: + # case 2: we never entered oneshot() ctx + return fun(self) + except KeyError: + # case 3: we entered oneshot() ctx but there's no cache + # for this entry yet + ret = self._cache[fun] = fun(self) + return ret + + def cache_activate(proc): + """Activate cache. Expects a Process instance. Cache will be + stored as a "_cache" instance attribute.""" + proc._cache = {} + + def cache_deactivate(proc): + """Deactivate and clear cache.""" + try: + del proc._cache + except AttributeError: + pass + + wrapper.cache_activate = cache_activate + wrapper.cache_deactivate = cache_deactivate + return wrapper + + +def isfile_strict(path): + """Same as os.path.isfile() but does not swallow EACCES / EPERM + exceptions, see: + http://mail.python.org/pipermail/python-dev/2012-June/120787.html + """ + try: + st = os.stat(path) + except OSError as err: + if err.errno in (errno.EPERM, errno.EACCES): + raise + return False + else: + return stat.S_ISREG(st.st_mode) + + +def path_exists_strict(path): + """Same as os.path.exists() but does not swallow EACCES / EPERM + exceptions, see: + http://mail.python.org/pipermail/python-dev/2012-June/120787.html + """ + try: + os.stat(path) + except OSError as err: + if err.errno in (errno.EPERM, errno.EACCES): + raise + return False + else: + return True + + +@memoize +def supports_ipv6(): + """Return True if IPv6 is supported on this platform.""" + if not socket.has_ipv6 or AF_INET6 is None: + return False + try: + sock = socket.socket(AF_INET6, socket.SOCK_STREAM) + with contextlib.closing(sock): + sock.bind(("::1", 0)) + return True + except socket.error: + return False + + +def parse_environ_block(data): + """Parse a C environ block of environment variables into a dictionary.""" + # The block is usually raw data from the target process. It might contain + # trailing garbage and lines that do not look like assignments. + ret = {} + pos = 0 + + # localize global variable to speed up access. + WINDOWS_ = WINDOWS + while True: + next_pos = data.find("\0", pos) + # nul byte at the beginning or double nul byte means finish + if next_pos <= pos: + break + # there might not be an equals sign + equal_pos = data.find("=", pos, next_pos) + if equal_pos > pos: + key = data[pos:equal_pos] + value = data[equal_pos + 1:next_pos] + # Windows expects environment variables to be uppercase only + if WINDOWS_: + key = key.upper() + ret[key] = value + pos = next_pos + 1 + + return ret + + +def sockfam_to_enum(num): + """Convert a numeric socket family value to an IntEnum member. + If it's not a known member, return the numeric value itself. + """ + if enum is None: + return num + else: # pragma: no cover + try: + return socket.AddressFamily(num) + except ValueError: + return num + + +def socktype_to_enum(num): + """Convert a numeric socket type value to an IntEnum member. + If it's not a known member, return the numeric value itself. + """ + if enum is None: + return num + else: # pragma: no cover + try: + return socket.SocketKind(num) + except ValueError: + return num + + +def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None): + """Convert a raw connection tuple to a proper ntuple.""" + if fam in (socket.AF_INET, AF_INET6): + if laddr: + laddr = addr(*laddr) + if raddr: + raddr = addr(*raddr) + if type_ == socket.SOCK_STREAM and fam in (AF_INET, AF_INET6): + status = status_map.get(status, CONN_NONE) + else: + status = CONN_NONE # ignore whatever C returned to us + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if pid is None: + return pconn(fd, fam, type_, laddr, raddr, status) + else: + return sconn(fd, fam, type_, laddr, raddr, status, pid) + + +def deprecated_method(replacement): + """A decorator which can be used to mark a method as deprecated + 'replcement' is the method name which will be called instead. + """ + def outer(fun): + msg = "%s() is deprecated and will be removed; use %s() instead" % ( + fun.__name__, replacement) + if fun.__doc__ is None: + fun.__doc__ = msg + + @functools.wraps(fun) + def inner(self, *args, **kwargs): + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + return getattr(self, replacement)(*args, **kwargs) + return inner + return outer + + +class _WrapNumbers: + """Watches numbers so that they don't overflow and wrap + (reset to zero). + """ + + def __init__(self): + self.lock = threading.Lock() + self.cache = {} + self.reminders = {} + self.reminder_keys = {} + + def _add_dict(self, input_dict, name): + assert name not in self.cache + assert name not in self.reminders + assert name not in self.reminder_keys + self.cache[name] = input_dict + self.reminders[name] = defaultdict(int) + self.reminder_keys[name] = defaultdict(set) + + def _remove_dead_reminders(self, input_dict, name): + """In case the number of keys changed between calls (e.g. a + disk disappears) this removes the entry from self.reminders. + """ + old_dict = self.cache[name] + gone_keys = set(old_dict.keys()) - set(input_dict.keys()) + for gone_key in gone_keys: + for remkey in self.reminder_keys[name][gone_key]: + del self.reminders[name][remkey] + del self.reminder_keys[name][gone_key] + + def run(self, input_dict, name): + """Cache dict and sum numbers which overflow and wrap. + Return an updated copy of `input_dict` + """ + if name not in self.cache: + # This was the first call. + self._add_dict(input_dict, name) + return input_dict + + self._remove_dead_reminders(input_dict, name) + + old_dict = self.cache[name] + new_dict = {} + for key in input_dict.keys(): + input_tuple = input_dict[key] + try: + old_tuple = old_dict[key] + except KeyError: + # The input dict has a new key (e.g. a new disk or NIC) + # which didn't exist in the previous call. + new_dict[key] = input_tuple + continue + + bits = [] + for i in range(len(input_tuple)): + input_value = input_tuple[i] + old_value = old_tuple[i] + remkey = (key, i) + if input_value < old_value: + # it wrapped! + self.reminders[name][remkey] += old_value + self.reminder_keys[name][key].add(remkey) + bits.append(input_value + self.reminders[name][remkey]) + + new_dict[key] = tuple(bits) + + self.cache[name] = input_dict + return new_dict + + def cache_clear(self, name=None): + """Clear the internal cache, optionally only for function 'name'.""" + with self.lock: + if name is None: + self.cache.clear() + self.reminders.clear() + self.reminder_keys.clear() + else: + self.cache.pop(name, None) + self.reminders.pop(name, None) + self.reminder_keys.pop(name, None) + + def cache_info(self): + """Return internal cache dicts as a tuple of 3 elements.""" + with self.lock: + return (self.cache, self.reminders, self.reminder_keys) + + +def wrap_numbers(input_dict, name): + """Given an `input_dict` and a function `name`, adjust the numbers + which "wrap" (restart from zero) across different calls by adding + "old value" to "new value" and return an updated dict. + """ + with _wn.lock: + return _wn.run(input_dict, name) + + +_wn = _WrapNumbers() +wrap_numbers.cache_clear = _wn.cache_clear +wrap_numbers.cache_info = _wn.cache_info + + +def open_binary(fname, **kwargs): + return open(fname, "rb", **kwargs) + + +def open_text(fname, **kwargs): + """On Python 3 opens a file in text mode by using fs encoding and + a proper en/decoding errors handler. + On Python 2 this is just an alias for open(name, 'rt'). + """ + if PY3: + # See: + # https://github.com/giampaolo/psutil/issues/675 + # https://github.com/giampaolo/psutil/pull/733 + kwargs.setdefault('encoding', ENCODING) + kwargs.setdefault('errors', ENCODING_ERRS) + return open(fname, "rt", **kwargs) + + +def bytes2human(n, format="%(value).1f%(symbol)s"): + """Used by various scripts. See: + http://goo.gl/zeJZl + + >>> bytes2human(10000) + '9.8K' + >>> bytes2human(100001221) + '95.4M' + """ + symbols = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols[1:]): + prefix[s] = 1 << (i + 1) * 10 + for symbol in reversed(symbols[1:]): + if n >= prefix[symbol]: + value = float(n) / prefix[symbol] + return format % locals() + return format % dict(symbol=symbols[0], value=n) + + +def get_procfs_path(): + """Return updated psutil.PROCFS_PATH constant.""" + return sys.modules['ddtrace.vendor.psutil'].PROCFS_PATH + + +if PY3: + def decode(s): + return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) +else: + def decode(s): + return s diff --git a/ddtrace/vendor/psutil/_compat.py b/ddtrace/vendor/psutil/_compat.py new file mode 100644 index 00000000000..07ab909a84f --- /dev/null +++ b/ddtrace/vendor/psutil/_compat.py @@ -0,0 +1,332 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Module which provides compatibility with older Python versions.""" + +import collections +import errno +import functools +import os +import sys + +__all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", + "lru_cache", "which", "get_terminal_size", + "FileNotFoundError", "PermissionError", "ProcessLookupError", + "InterruptedError", "ChildProcessError", "FileExistsError"] + +PY3 = sys.version_info[0] == 3 + +if PY3: + long = int + xrange = range + unicode = str + basestring = str + + def u(s): + return s + + def b(s): + return s.encode("latin-1") +else: + long = long + xrange = xrange + unicode = unicode + basestring = basestring + + def u(s): + return unicode(s, "unicode_escape") + + def b(s): + return s + + +# --- exceptions + + +if PY3: + FileNotFoundError = FileNotFoundError # NOQA + PermissionError = PermissionError # NOQA + ProcessLookupError = ProcessLookupError # NOQA + InterruptedError = InterruptedError # NOQA + ChildProcessError = ChildProcessError # NOQA + FileExistsError = FileExistsError # NOQA +else: + # https://github.com/PythonCharmers/python-future/blob/exceptions/ + # src/future/types/exceptions/pep3151.py + + def instance_checking_exception(base_exception=Exception): + def wrapped(instance_checker): + class TemporaryClass(base_exception): + + def __init__(self, *args, **kwargs): + if len(args) == 1 and isinstance(args[0], TemporaryClass): + unwrap_me = args[0] + for attr in dir(unwrap_me): + if not attr.startswith('__'): + setattr(self, attr, getattr(unwrap_me, attr)) + else: + super(TemporaryClass, self).__init__(*args, **kwargs) + + class __metaclass__(type): + def __instancecheck__(cls, inst): + return instance_checker(inst) + + def __subclasscheck__(cls, classinfo): + value = sys.exc_info()[1] + return isinstance(value, cls) + + TemporaryClass.__name__ = instance_checker.__name__ + TemporaryClass.__doc__ = instance_checker.__doc__ + return TemporaryClass + + return wrapped + + @instance_checking_exception(EnvironmentError) + def FileNotFoundError(inst): + return getattr(inst, 'errno', object()) == errno.ENOENT + + @instance_checking_exception(EnvironmentError) + def ProcessLookupError(inst): + return getattr(inst, 'errno', object()) == errno.ESRCH + + @instance_checking_exception(EnvironmentError) + def PermissionError(inst): + return getattr(inst, 'errno', object()) in ( + errno.EACCES, errno.EPERM) + + @instance_checking_exception(EnvironmentError) + def InterruptedError(inst): + return getattr(inst, 'errno', object()) == errno.EINTR + + @instance_checking_exception(EnvironmentError) + def ChildProcessError(inst): + return getattr(inst, 'errno', object()) == errno.ECHILD + + @instance_checking_exception(EnvironmentError) + def FileExistsError(inst): + return getattr(inst, 'errno', object()) == errno.EEXIST + + +# --- stdlib additions + + +# py 3.2 functools.lru_cache +# Taken from: http://code.activestate.com/recipes/578078 +# Credit: Raymond Hettinger +try: + from functools import lru_cache +except ImportError: + try: + from threading import RLock + except ImportError: + from dummy_threading import RLock + + _CacheInfo = collections.namedtuple( + "CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + + class _HashedSeq(list): + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + + def _make_key(args, kwds, typed, + kwd_mark=(object(), ), + fasttypes=set((int, str, frozenset, type(None))), + sorted=sorted, tuple=tuple, type=type, len=len): + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + + def lru_cache(maxsize=100, typed=False): + """Least-recently-used cache decorator, see: + http://docs.python.org/3/library/functools.html#functools.lru_cache + """ + def decorating_function(user_function): + cache = dict() + stats = [0, 0] + HITS, MISSES = 0, 1 + make_key = _make_key + cache_get = cache.get + _len = len + lock = RLock() + root = [] + root[:] = [root, root, None, None] + nonlocal_root = [root] + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 + if maxsize == 0: + def wrapper(*args, **kwds): + result = user_function(*args, **kwds) + stats[MISSES] += 1 + return result + elif maxsize is None: + def wrapper(*args, **kwds): + key = make_key(args, kwds, typed) + result = cache_get(key, root) + if result is not root: + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + stats[MISSES] += 1 + return result + else: + def wrapper(*args, **kwds): + if kwds or typed: + key = make_key(args, kwds, typed) + else: + key = args + lock.acquire() + try: + link = cache_get(key) + if link is not None: + root, = nonlocal_root + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + stats[HITS] += 1 + return result + finally: + lock.release() + result = user_function(*args, **kwds) + lock.acquire() + try: + root, = nonlocal_root + if key in cache: + pass + elif _len(cache) >= maxsize: + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + root = nonlocal_root[0] = oldroot[NEXT] + oldkey = root[KEY] + root[KEY] = root[RESULT] = None + del cache[oldkey] + cache[key] = oldroot + else: + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + stats[MISSES] += 1 + finally: + lock.release() + return result + + def cache_info(): + """Report cache statistics""" + lock.acquire() + try: + return _CacheInfo(stats[HITS], stats[MISSES], maxsize, + len(cache)) + finally: + lock.release() + + def cache_clear(): + """Clear the cache and cache statistics""" + lock.acquire() + try: + cache.clear() + root = nonlocal_root[0] + root[:] = [root, root, None, None] + stats[:] = [0, 0] + finally: + lock.release() + + wrapper.__wrapped__ = user_function + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return functools.update_wrapper(wrapper, user_function) + + return decorating_function + + +# python 3.3 +try: + from shutil import which +except ImportError: + def which(cmd, mode=os.F_OK | os.X_OK, path=None): + """Given a command, mode, and a PATH string, return the path which + conforms to the given mode on the PATH, or None if there is no such + file. + + `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result + of os.environ.get("PATH"), or can be overridden with a custom search + path. + """ + def _access_check(fn, mode): + return (os.path.exists(fn) and os.access(fn, mode) and + not os.path.isdir(fn)) + + if os.path.dirname(cmd): + if _access_check(cmd, mode): + return cmd + return None + + if path is None: + path = os.environ.get("PATH", os.defpath) + if not path: + return None + path = path.split(os.pathsep) + + if sys.platform == "win32": + if os.curdir not in path: + path.insert(0, os.curdir) + + pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): + files = [cmd] + else: + files = [cmd + ext for ext in pathext] + else: + files = [cmd] + + seen = set() + for dir in path: + normdir = os.path.normcase(dir) + if normdir not in seen: + seen.add(normdir) + for thefile in files: + name = os.path.join(dir, thefile) + if _access_check(name, mode): + return name + return None + + +# python 3.3 +try: + from shutil import get_terminal_size +except ImportError: + def get_terminal_size(fallback=(80, 24)): + try: + import fcntl + import termios + import struct + except ImportError: + return fallback + else: + try: + # This should work on Linux. + res = struct.unpack( + 'hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234')) + return (res[1], res[0]) + except Exception: + return fallback diff --git a/ddtrace/vendor/psutil/_psaix.py b/ddtrace/vendor/psutil/_psaix.py new file mode 100644 index 00000000000..79e3be15a19 --- /dev/null +++ b/ddtrace/vendor/psutil/_psaix.py @@ -0,0 +1,554 @@ +# Copyright (c) 2009, Giampaolo Rodola' +# Copyright (c) 2017, Arnon Yaari +# All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""AIX platform implementation.""" + +import functools +import glob +import os +import re +import subprocess +import sys +from collections import namedtuple + +from . import _common +from . import _psposix +from . import _psutil_aix as cext +from . import _psutil_posix as cext_posix +from ._common import conn_to_ntuple +from ._common import get_procfs_path +from ._common import memoize_when_activated +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN +from ._common import usage_percent +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._compat import PY3 + + +__extra__all__ = ["PROCFS_PATH"] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +HAS_THREADS = hasattr(cext, "proc_threads") +HAS_NET_IO_COUNTERS = hasattr(cext, "net_io_counters") +HAS_PROC_IO_COUNTERS = hasattr(cext, "proc_io_counters") + +PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') +AF_LINK = cext_posix.AF_LINK + +PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SACTIVE: _common.STATUS_RUNNING, + cext.SSWAP: _common.STATUS_RUNNING, # TODO what status is this? + cext.SSTOP: _common.STATUS_STOPPED, +} + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +proc_info_map = dict( + ppid=0, + rss=1, + vms=2, + create_time=3, + nice=4, + num_threads=5, + status=6, + ttynr=7) + +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms']) +# psutil.Process.memory_full_info() +pfullmem = pmem +# psutil.Process.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + total, avail, free, pinned, inuse = cext.virtual_mem() + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, inuse, free) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + total, free, sin, sout = cext.swap_mem() + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system-wide CPU times as a named tuple""" + ret = cext.per_cpu_times() + return scputimes(*[sum(x) for x in zip(*ret)]) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples""" + ret = cext.per_cpu_times() + return [scputimes(*x) for x in ret] + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # mimic os.cpu_count() behavior + return None + + +def cpu_count_physical(): + cmd = "lsdev -Cc processor" + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if p.returncode != 0: + raise RuntimeError("%r command error\n%s" % (cmd, stderr)) + processors = stdout.strip().splitlines() + return len(processors) or None + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + ctx_switches, interrupts, soft_interrupts, syscalls = cext.cpu_stats() + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls) + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters +disk_usage = _psposix.disk_usage + + +def disk_partitions(all=False): + """Return system disk partitions.""" + # TODO - the filtering logic should be better checked so that + # it tries to reflect 'df' as much as possible + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + # Differently from, say, Linux, we don't have a list of + # common fs types so the best we can do, AFAIK, is to + # filter by filesystem having a total size > 0. + if not disk_usage(mountpoint).total: + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_if_addrs = cext_posix.net_if_addrs + +if HAS_NET_IO_COUNTERS: + net_io_counters = cext.net_io_counters + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + """ + cmap = _common.conn_tmap + if kind not in cmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in cmap]))) + families, types = _common.conn_tmap[kind] + rawlist = cext.net_connections(_pid) + ret = [] + for item in rawlist: + fd, fam, type_, laddr, raddr, status, pid = item + if fam not in families: + continue + if type_ not in types: + continue + nt = conn_to_ntuple(fd, fam, type_, laddr, raddr, status, + TCP_STATUSES, pid=pid if _pid == -1 else None) + ret.append(nt) + return ret + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + duplex_map = {"Full": NIC_DUPLEX_FULL, + "Half": NIC_DUPLEX_HALF} + names = set([x[0] for x in net_if_addrs()]) + ret = {} + for name in names: + isup, mtu = cext.net_if_stats(name) + + # try to get speed and duplex + # TODO: rewrite this in C (entstat forks, so use truss -f to follow. + # looks like it is using an undocumented ioctl?) + duplex = "" + speed = 0 + p = subprocess.Popen(["/usr/bin/entstat", "-d", name], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if p.returncode == 0: + re_result = re.search( + r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout) + if re_result is not None: + speed = int(re_result.group(1)) + duplex = re_result.group(2) + + duplex = duplex_map.get(duplex, NIC_DUPLEX_UNKNOWN) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + localhost = (':0.0', ':0') + for item in rawlist: + user, tty, hostname, tstamp, user_process, pid = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname in localhost: + hostname = 'localhost' + nt = _common.suser(user, tty, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + return [int(x) for x in os.listdir(get_procfs_path()) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix pid.""" + return os.path.exists(os.path.join(get_procfs_path(), str(pid), "psinfo")) + + +def wrap_exceptions(fun): + """Call callable into a try/except clause and translate ENOENT, + EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except (FileNotFoundError, ProcessLookupError): + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + return wrapper + + +class Process(object): + """Wrapper class around underlying C implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def oneshot_enter(self): + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) + + def oneshot_exit(self): + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) + + @wrap_exceptions + @memoize_when_activated + def _proc_basic_info(self): + return cext.proc_basic_info(self.pid, self._procfs_path) + + @wrap_exceptions + @memoize_when_activated + def _proc_cred(self): + return cext.proc_cred(self.pid, self._procfs_path) + + @wrap_exceptions + def name(self): + if self.pid == 0: + return "swapper" + # note: max 16 characters + return cext.proc_name(self.pid, self._procfs_path).rstrip("\x00") + + @wrap_exceptions + def exe(self): + # there is no way to get executable path in AIX other than to guess, + # and guessing is more complex than what's in the wrapping class + cmdline = self.cmdline() + if not cmdline: + return '' + exe = cmdline[0] + if os.path.sep in exe: + # relative or absolute path + if not os.path.isabs(exe): + # if cwd has changed, we're out of luck - this may be wrong! + exe = os.path.abspath(os.path.join(self.cwd(), exe)) + if (os.path.isabs(exe) and + os.path.isfile(exe) and + os.access(exe, os.X_OK)): + return exe + # not found, move to search in PATH using basename only + exe = os.path.basename(exe) + # search for exe name PATH + for path in os.environ["PATH"].split(":"): + possible_exe = os.path.abspath(os.path.join(path, exe)) + if (os.path.isfile(possible_exe) and + os.access(possible_exe, os.X_OK)): + return possible_exe + return '' + + @wrap_exceptions + def cmdline(self): + return cext.proc_args(self.pid) + + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid) + + @wrap_exceptions + def create_time(self): + return self._proc_basic_info()[proc_info_map['create_time']] + + @wrap_exceptions + def num_threads(self): + return self._proc_basic_info()[proc_info_map['num_threads']] + + if HAS_THREADS: + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + # The underlying C implementation retrieves all OS threads + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not retlist: + # will raise NSP if process is gone + os.stat('%s/%s' % (self._procfs_path, self.pid)) + return retlist + + @wrap_exceptions + def connections(self, kind='inet'): + ret = net_connections(kind, _pid=self.pid) + # The underlying C implementation retrieves all OS connections + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not ret: + # will raise NSP if process is gone + os.stat('%s/%s' % (self._procfs_path, self.pid)) + return ret + + @wrap_exceptions + def nice_get(self): + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def ppid(self): + self._ppid = self._proc_basic_info()[proc_info_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + real, effective, saved, _, _, _ = self._proc_cred() + return _common.puids(real, effective, saved) + + @wrap_exceptions + def gids(self): + _, _, _, real, effective, saved = self._proc_cred() + return _common.puids(real, effective, saved) + + @wrap_exceptions + def cpu_times(self): + cpu_times = cext.proc_cpu_times(self.pid, self._procfs_path) + return _common.pcputimes(*cpu_times) + + @wrap_exceptions + def terminal(self): + ttydev = self._proc_basic_info()[proc_info_map['ttynr']] + # convert from 64-bit dev_t to 32-bit dev_t and then map the device + ttydev = (((ttydev & 0x0000FFFF00000000) >> 16) | (ttydev & 0xFFFF)) + # try to match rdev of /dev/pts/* files ttydev + for dev in glob.glob("/dev/**/*"): + if os.stat(dev).st_rdev == ttydev: + return dev + return None + + @wrap_exceptions + def cwd(self): + procfs_path = self._procfs_path + try: + result = os.readlink("%s/%s/cwd" % (procfs_path, self.pid)) + return result.rstrip('/') + except FileNotFoundError: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None + + @wrap_exceptions + def memory_info(self): + ret = self._proc_basic_info() + rss = ret[proc_info_map['rss']] * 1024 + vms = ret[proc_info_map['vms']] * 1024 + return pmem(rss, vms) + + memory_full_info = memory_info + + @wrap_exceptions + def status(self): + code = self._proc_basic_info()[proc_info_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + def open_files(self): + # TODO rewrite without using procfiles (stat /proc/pid/fd/* and then + # find matching name of the inode) + p = subprocess.Popen(["/usr/bin/procfiles", "-n", str(self.pid)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if "no such process" in stderr.lower(): + raise NoSuchProcess(self.pid, self._name) + procfiles = re.findall(r"(\d+): S_IFREG.*\s*.*name:(.*)\n", stdout) + retlist = [] + for fd, path in procfiles: + path = path.strip() + if path.startswith("//"): + path = path[1:] + if path.lower() == "cannot be retrieved": + continue + retlist.append(_common.popenfile(path, int(fd))) + return retlist + + @wrap_exceptions + def num_fds(self): + if self.pid == 0: # no /proc/0/fd + return 0 + return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + + @wrap_exceptions + def num_ctx_switches(self): + return _common.pctxsw( + *cext.proc_num_ctx_switches(self.pid)) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + if HAS_PROC_IO_COUNTERS: + @wrap_exceptions + def io_counters(self): + try: + rc, wc, rb, wb = cext.proc_io_counters(self.pid) + except OSError: + # if process is terminated, proc_io_counters returns OSError + # instead of NSP + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + raise + return _common.pio(rc, wc, rb, wb) diff --git a/ddtrace/vendor/psutil/_psbsd.py b/ddtrace/vendor/psutil/_psbsd.py new file mode 100644 index 00000000000..2f41dc0be9f --- /dev/null +++ b/ddtrace/vendor/psutil/_psbsd.py @@ -0,0 +1,905 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""FreeBSD, OpenBSD and NetBSD platforms implementation.""" + +import contextlib +import errno +import functools +import os +import xml.etree.ElementTree as ET +from collections import namedtuple +from collections import defaultdict + +from . import _common +from . import _psposix +from . import _psutil_bsd as cext +from . import _psutil_posix as cext_posix +from ._common import conn_tmap +from ._common import conn_to_ntuple +from ._common import FREEBSD +from ._common import memoize +from ._common import memoize_when_activated +from ._common import NETBSD +from ._common import OPENBSD +from ._common import usage_percent +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._compat import which + + +__extra__all__ = [] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +if FREEBSD: + PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SRUN: _common.STATUS_RUNNING, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SWAIT: _common.STATUS_WAITING, + cext.SLOCK: _common.STATUS_LOCKED, + } +elif OPENBSD or NETBSD: + PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + # According to /usr/include/sys/proc.h SZOMB is unused. + # test_zombie_process() shows that SDEAD is the right + # equivalent. Also it appears there's no equivalent of + # psutil.STATUS_DEAD. SDEAD really means STATUS_ZOMBIE. + # cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SDEAD: _common.STATUS_ZOMBIE, + cext.SZOMB: _common.STATUS_ZOMBIE, + # From http://www.eecs.harvard.edu/~margo/cs161/videos/proc.h.txt + # OpenBSD has SRUN and SONPROC: SRUN indicates that a process + # is runnable but *not* yet running, i.e. is on a run queue. + # SONPROC indicates that the process is actually executing on + # a CPU, i.e. it is no longer on a run queue. + # As such we'll map SRUN to STATUS_WAKING and SONPROC to + # STATUS_RUNNING + cext.SRUN: _common.STATUS_WAKING, + cext.SONPROC: _common.STATUS_RUNNING, + } +elif NETBSD: + PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SACTIVE: _common.STATUS_RUNNING, + cext.SDYING: _common.STATUS_ZOMBIE, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SDEAD: _common.STATUS_DEAD, + cext.SSUSPENDED: _common.STATUS_SUSPENDED, # unique to NetBSD + } + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +if NETBSD: + PAGESIZE = os.sysconf("SC_PAGESIZE") +else: + PAGESIZE = os.sysconf("SC_PAGE_SIZE") +AF_LINK = cext_posix.AF_LINK + +HAS_PER_CPU_TIMES = hasattr(cext, "per_cpu_times") +HAS_PROC_NUM_THREADS = hasattr(cext, "proc_num_threads") +HAS_PROC_OPEN_FILES = hasattr(cext, 'proc_open_files') +HAS_PROC_NUM_FDS = hasattr(cext, 'proc_num_fds') + +kinfo_proc_map = dict( + ppid=0, + status=1, + real_uid=2, + effective_uid=3, + saved_uid=4, + real_gid=5, + effective_gid=6, + saved_gid=7, + ttynr=8, + create_time=9, + ctx_switches_vol=10, + ctx_switches_unvol=11, + read_io_count=12, + write_io_count=13, + user_time=14, + sys_time=15, + ch_user_time=16, + ch_sys_time=17, + rss=18, + vms=19, + memtext=20, + memdata=21, + memstack=22, + cpunum=23, + name=24, +) + +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.virtual_memory() +svmem = namedtuple( + 'svmem', ['total', 'available', 'percent', 'used', 'free', + 'active', 'inactive', 'buffers', 'cached', 'shared', 'wired']) +# psutil.cpu_times() +scputimes = namedtuple( + 'scputimes', ['user', 'nice', 'system', 'idle', 'irq']) +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms', 'text', 'data', 'stack']) +# psutil.Process.memory_full_info() +pfullmem = pmem +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system']) +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple( + 'pmmap_grouped', 'path rss, private, ref_count, shadow_count') +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr, perms path rss, private, ref_count, shadow_count') +# psutil.disk_io_counters() +if FREEBSD: + sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_time', 'write_time', + 'busy_time']) +else: + sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes']) + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """System virtual memory as a namedtuple.""" + mem = cext.virtual_mem() + total, free, active, inactive, wired, cached, buffers, shared = mem + if NETBSD: + # On NetBSD buffers and shared mem is determined via /proc. + # The C ext set them to 0. + with open('/proc/meminfo', 'rb') as f: + for line in f: + if line.startswith(b'Buffers:'): + buffers = int(line.split()[1]) * 1024 + elif line.startswith(b'MemShared:'): + shared = int(line.split()[1]) * 1024 + avail = inactive + cached + free + used = active + wired + cached + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, used, free, + active, inactive, buffers, cached, shared, wired) + + +def swap_memory(): + """System swap memory as (total, used, free, sin, sout) namedtuple.""" + total, used, free, sin, sout = cext.swap_mem() + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system per-CPU times as a namedtuple""" + user, nice, system, idle, irq = cext.cpu_times() + return scputimes(user, nice, system, idle, irq) + + +if HAS_PER_CPU_TIMES: + def per_cpu_times(): + """Return system CPU times as a namedtuple""" + ret = [] + for cpu_t in cext.per_cpu_times(): + user, nice, system, idle, irq = cpu_t + item = scputimes(user, nice, system, idle, irq) + ret.append(item) + return ret +else: + # XXX + # Ok, this is very dirty. + # On FreeBSD < 8 we cannot gather per-cpu information, see: + # https://github.com/giampaolo/psutil/issues/226 + # If num cpus > 1, on first call we return single cpu times to avoid a + # crash at psutil import time. + # Next calls will fail with NotImplementedError + def per_cpu_times(): + """Return system CPU times as a namedtuple""" + if cpu_count_logical() == 1: + return [cpu_times()] + if per_cpu_times.__called__: + raise NotImplementedError("supported only starting from FreeBSD 8") + per_cpu_times.__called__ = True + return [cpu_times()] + + per_cpu_times.__called__ = False + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + return cext.cpu_count_logical() + + +if OPENBSD or NETBSD: + def cpu_count_physical(): + # OpenBSD and NetBSD do not implement this. + return 1 if cpu_count_logical() == 1 else None +else: + def cpu_count_physical(): + """Return the number of physical CPUs in the system.""" + # From the C module we'll get an XML string similar to this: + # http://manpages.ubuntu.com/manpages/precise/man4/smp.4freebsd.html + # We may get None in case "sysctl kern.sched.topology_spec" + # is not supported on this BSD version, in which case we'll mimic + # os.cpu_count() and return None. + ret = None + s = cext.cpu_count_phys() + if s is not None: + # get rid of padding chars appended at the end of the string + index = s.rfind("") + if index != -1: + s = s[:index + 9] + root = ET.fromstring(s) + try: + ret = len(root.findall('group/children/group/cpu')) or None + finally: + # needed otherwise it will memleak + root.clear() + if not ret: + # If logical CPUs are 1 it's obvious we'll have only 1 + # physical CPU. + if cpu_count_logical() == 1: + return 1 + return ret + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + if FREEBSD: + # Note: the C ext is returning some metrics we are not exposing: + # traps. + ctxsw, intrs, soft_intrs, syscalls, traps = cext.cpu_stats() + elif NETBSD: + # XXX + # Note about intrs: the C extension returns 0. intrs + # can be determined via /proc/stat; it has the same value as + # soft_intrs thought so the kernel is faking it (?). + # + # Note about syscalls: the C extension always sets it to 0 (?). + # + # Note: the C ext is returning some metrics we are not exposing: + # traps, faults and forks. + ctxsw, intrs, soft_intrs, syscalls, traps, faults, forks = \ + cext.cpu_stats() + with open('/proc/stat', 'rb') as f: + for line in f: + if line.startswith(b'intr'): + intrs = int(line.split()[1]) + elif OPENBSD: + # Note: the C ext is returning some metrics we are not exposing: + # traps, faults and forks. + ctxsw, intrs, soft_intrs, syscalls, traps, faults, forks = \ + cext.cpu_stats() + return _common.scpustats(ctxsw, intrs, soft_intrs, syscalls) + + +# ===================================================================== +# --- disks +# ===================================================================== + + +def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples. + 'all' argument is ignored, see: + https://github.com/giampaolo/psutil/issues/906 + """ + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +disk_usage = _psposix.disk_usage +disk_io_counters = cext.disk_io_counters + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_io_counters = cext.net_io_counters +net_if_addrs = cext_posix.net_if_addrs + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + names = net_io_counters().keys() + ret = {} + for name in names: + try: + mtu = cext_posix.net_if_mtu(name) + isup = cext_posix.net_if_flags(name) + duplex, speed = cext_posix.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) + return ret + + +def net_connections(kind): + """System-wide network connections.""" + if OPENBSD: + ret = [] + for pid in pids(): + try: + cons = Process(pid).connections(kind) + except (NoSuchProcess, ZombieProcess): + continue + else: + for conn in cons: + conn = list(conn) + conn.append(pid) + ret.append(_common.sconn(*conn)) + return ret + + if kind not in _common.conn_tmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in conn_tmap]))) + families, types = conn_tmap[kind] + ret = set() + if NETBSD: + rawlist = cext.net_connections(-1) + else: + rawlist = cext.net_connections() + for item in rawlist: + fd, fam, type, laddr, raddr, status, pid = item + # TODO: apply filter at C level + if fam in families and type in types: + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES, pid) + ret.add(nt) + return list(ret) + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +if FREEBSD: + + def sensors_battery(): + """Return battery info.""" + try: + percent, minsleft, power_plugged = cext.sensors_battery() + except NotImplementedError: + # See: https://github.com/giampaolo/psutil/issues/1074 + return None + power_plugged = power_plugged == 1 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft, power_plugged) + + def sensors_temperatures(): + "Return CPU cores temperatures if available, else an empty dict." + ret = defaultdict(list) + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, high = cext.sensors_cpu_temperature(cpu) + if high <= 0: + high = None + name = "Core %s" % cpu + ret["coretemp"].append( + _common.shwtemp(name, current, high, high)) + except NotImplementedError: + pass + + return ret + + def cpu_freq(): + """Return frequency metrics for CPUs. As of Dec 2018 only + CPU 0 appears to be supported by FreeBSD and all other cores + match the frequency of CPU 0. + """ + ret = [] + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, available_freq = cext.cpu_frequency(cpu) + except NotImplementedError: + continue + if available_freq: + try: + min_freq = int(available_freq.split(" ")[-1].split("/")[0]) + except(IndexError, ValueError): + min_freq = None + try: + max_freq = int(available_freq.split(" ")[0].split("/")[0]) + except(IndexError, ValueError): + max_freq = None + ret.append(_common.scpufreq(current, min_freq, max_freq)) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, pid = item + if pid == -1: + assert OPENBSD + pid = None + if tty == '~': + continue # reboot or shutdown + nt = _common.suser(user, tty or None, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +@memoize +def _pid_0_exists(): + try: + Process(0).name() + except NoSuchProcess: + return False + except AccessDenied: + return True + else: + return True + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + ret = cext.pids() + if OPENBSD and (0 not in ret) and _pid_0_exists(): + # On OpenBSD the kernel does not return PID 0 (neither does + # ps) but it's actually querable (Process(0) will succeed). + ret.insert(0, 0) + return ret + + +if OPENBSD or NETBSD: + def pid_exists(pid): + """Return True if pid exists.""" + exists = _psposix.pid_exists(pid) + if not exists: + # We do this because _psposix.pid_exists() lies in case of + # zombie processes. + return pid in pids() + else: + return True +else: + pid_exists = _psposix.pid_exists + + +def is_zombie(pid): + try: + st = cext.proc_oneshot_info(pid)[kinfo_proc_map['status']] + return st == cext.SZOMB + except Exception: + return False + + +def wrap_exceptions(fun): + """Decorator which translates bare OSError exceptions into + NoSuchProcess and AccessDenied. + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except ProcessLookupError: + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except OSError: + if self.pid == 0: + if 0 in pids(): + raise AccessDenied(self.pid, self._name) + else: + raise + raise + return wrapper + + +@contextlib.contextmanager +def wrap_exceptions_procfs(inst): + """Same as above, for routines relying on reading /proc fs.""" + try: + yield + except (ProcessLookupError, FileNotFoundError): + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(inst.pid): + raise NoSuchProcess(inst.pid, inst._name) + else: + raise ZombieProcess(inst.pid, inst._name, inst._ppid) + except PermissionError: + raise AccessDenied(inst.pid, inst._name) + + +class Process(object): + """Wrapper class around underlying C implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_cache"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + cext.proc_name(self.pid) + + @wrap_exceptions + @memoize_when_activated + def oneshot(self): + """Retrieves multiple process info in one shot as a raw tuple.""" + ret = cext.proc_oneshot_info(self.pid) + assert len(ret) == len(kinfo_proc_map) + return ret + + def oneshot_enter(self): + self.oneshot.cache_activate(self) + + def oneshot_exit(self): + self.oneshot.cache_deactivate(self) + + @wrap_exceptions + def name(self): + name = self.oneshot()[kinfo_proc_map['name']] + return name if name is not None else cext.proc_name(self.pid) + + @wrap_exceptions + def exe(self): + if FREEBSD: + return cext.proc_exe(self.pid) + elif NETBSD: + if self.pid == 0: + # /proc/0 dir exists but /proc/0/exe doesn't + return "" + with wrap_exceptions_procfs(self): + return os.readlink("/proc/%s/exe" % self.pid) + else: + # OpenBSD: exe cannot be determined; references: + # https://chromium.googlesource.com/chromium/src/base/+/ + # master/base_paths_posix.cc + # We try our best guess by using which against the first + # cmdline arg (may return None). + cmdline = self.cmdline() + if cmdline: + return which(cmdline[0]) + else: + return "" + + @wrap_exceptions + def cmdline(self): + if OPENBSD and self.pid == 0: + return [] # ...else it crashes + elif NETBSD: + # XXX - most of the times the underlying sysctl() call on Net + # and Open BSD returns a truncated string. + # Also /proc/pid/cmdline behaves the same so it looks + # like this is a kernel bug. + try: + return cext.proc_cmdline(self.pid) + except OSError as err: + if err.errno == errno.EINVAL: + if is_zombie(self.pid): + raise ZombieProcess(self.pid, self._name, self._ppid) + elif not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name, self._ppid) + else: + # XXX: this happens with unicode tests. It means the C + # routine is unable to decode invalid unicode chars. + return [] + else: + raise + else: + return cext.proc_cmdline(self.pid) + + @wrap_exceptions + def terminal(self): + tty_nr = self.oneshot()[kinfo_proc_map['ttynr']] + tmap = _psposix.get_terminal_map() + try: + return tmap[tty_nr] + except KeyError: + return None + + @wrap_exceptions + def ppid(self): + self._ppid = self.oneshot()[kinfo_proc_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + rawtuple = self.oneshot() + return _common.puids( + rawtuple[kinfo_proc_map['real_uid']], + rawtuple[kinfo_proc_map['effective_uid']], + rawtuple[kinfo_proc_map['saved_uid']]) + + @wrap_exceptions + def gids(self): + rawtuple = self.oneshot() + return _common.pgids( + rawtuple[kinfo_proc_map['real_gid']], + rawtuple[kinfo_proc_map['effective_gid']], + rawtuple[kinfo_proc_map['saved_gid']]) + + @wrap_exceptions + def cpu_times(self): + rawtuple = self.oneshot() + return _common.pcputimes( + rawtuple[kinfo_proc_map['user_time']], + rawtuple[kinfo_proc_map['sys_time']], + rawtuple[kinfo_proc_map['ch_user_time']], + rawtuple[kinfo_proc_map['ch_sys_time']]) + + if FREEBSD: + @wrap_exceptions + def cpu_num(self): + return self.oneshot()[kinfo_proc_map['cpunum']] + + @wrap_exceptions + def memory_info(self): + rawtuple = self.oneshot() + return pmem( + rawtuple[kinfo_proc_map['rss']], + rawtuple[kinfo_proc_map['vms']], + rawtuple[kinfo_proc_map['memtext']], + rawtuple[kinfo_proc_map['memdata']], + rawtuple[kinfo_proc_map['memstack']]) + + memory_full_info = memory_info + + @wrap_exceptions + def create_time(self): + return self.oneshot()[kinfo_proc_map['create_time']] + + @wrap_exceptions + def num_threads(self): + if HAS_PROC_NUM_THREADS: + # FreeBSD + return cext.proc_num_threads(self.pid) + else: + return len(self.threads()) + + @wrap_exceptions + def num_ctx_switches(self): + rawtuple = self.oneshot() + return _common.pctxsw( + rawtuple[kinfo_proc_map['ctx_switches_vol']], + rawtuple[kinfo_proc_map['ctx_switches_unvol']]) + + @wrap_exceptions + def threads(self): + # Note: on OpenSBD this (/dev/mem) requires root access. + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + if OPENBSD: + self._assert_alive() + return retlist + + @wrap_exceptions + def connections(self, kind='inet'): + if kind not in conn_tmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in conn_tmap]))) + + if NETBSD: + families, types = conn_tmap[kind] + ret = [] + rawlist = cext.net_connections(self.pid) + for item in rawlist: + fd, fam, type, laddr, raddr, status, pid = item + assert pid == self.pid + if fam in families and type in types: + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) + ret.append(nt) + self._assert_alive() + return list(ret) + + families, types = conn_tmap[kind] + rawlist = cext.proc_connections(self.pid, families, types) + ret = [] + for item in rawlist: + fd, fam, type, laddr, raddr, status = item + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) + ret.append(nt) + + if OPENBSD: + self._assert_alive() + + return ret + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + @wrap_exceptions + def nice_get(self): + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def status(self): + code = self.oneshot()[kinfo_proc_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + @wrap_exceptions + def io_counters(self): + rawtuple = self.oneshot() + return _common.pio( + rawtuple[kinfo_proc_map['read_io_count']], + rawtuple[kinfo_proc_map['write_io_count']], + -1, + -1) + + @wrap_exceptions + def cwd(self): + """Return process current working directory.""" + # sometimes we get an empty string, in which case we turn + # it into None + if OPENBSD and self.pid == 0: + return None # ...else it would raise EINVAL + elif NETBSD or HAS_PROC_OPEN_FILES: + # FreeBSD < 8 does not support functions based on + # kinfo_getfile() and kinfo_getvmmap() + return cext.proc_cwd(self.pid) or None + else: + raise NotImplementedError( + "supported only starting from FreeBSD 8" if + FREEBSD else "") + + nt_mmap_grouped = namedtuple( + 'mmap', 'path rss, private, ref_count, shadow_count') + nt_mmap_ext = namedtuple( + 'mmap', 'addr, perms path rss, private, ref_count, shadow_count') + + def _not_implemented(self): + raise NotImplementedError + + # FreeBSD < 8 does not support functions based on kinfo_getfile() + # and kinfo_getvmmap() + if HAS_PROC_OPEN_FILES: + @wrap_exceptions + def open_files(self): + """Return files opened by process as a list of namedtuples.""" + rawlist = cext.proc_open_files(self.pid) + return [_common.popenfile(path, fd) for path, fd in rawlist] + else: + open_files = _not_implemented + + # FreeBSD < 8 does not support functions based on kinfo_getfile() + # and kinfo_getvmmap() + if HAS_PROC_NUM_FDS: + @wrap_exceptions + def num_fds(self): + """Return the number of file descriptors opened by this process.""" + ret = cext.proc_num_fds(self.pid) + if NETBSD: + self._assert_alive() + return ret + else: + num_fds = _not_implemented + + # --- FreeBSD only APIs + + if FREEBSD: + + @wrap_exceptions + def cpu_affinity_get(self): + return cext.proc_cpu_affinity_get(self.pid) + + @wrap_exceptions + def cpu_affinity_set(self, cpus): + # Pre-emptively check if CPUs are valid because the C + # function has a weird behavior in case of invalid CPUs, + # see: https://github.com/giampaolo/psutil/issues/586 + allcpus = tuple(range(len(per_cpu_times()))) + for cpu in cpus: + if cpu not in allcpus: + raise ValueError("invalid CPU #%i (choose between %s)" + % (cpu, allcpus)) + try: + cext.proc_cpu_affinity_set(self.pid, cpus) + except OSError as err: + # 'man cpuset_setaffinity' about EDEADLK: + # <> + if err.errno in (errno.EINVAL, errno.EDEADLK): + for cpu in cpus: + if cpu not in allcpus: + raise ValueError( + "invalid CPU #%i (choose between %s)" % ( + cpu, allcpus)) + raise + + @wrap_exceptions + def memory_maps(self): + return cext.proc_memory_maps(self.pid) diff --git a/ddtrace/vendor/psutil/_pslinux.py b/ddtrace/vendor/psutil/_pslinux.py new file mode 100644 index 00000000000..a56ead36985 --- /dev/null +++ b/ddtrace/vendor/psutil/_pslinux.py @@ -0,0 +1,2096 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Linux platform implementation.""" + +from __future__ import division + +import base64 +import collections +import errno +import functools +import glob +import os +import re +import socket +import struct +import sys +import traceback +import warnings +from collections import defaultdict +from collections import namedtuple + +from . import _common +from . import _psposix +from . import _psutil_linux as cext +from . import _psutil_posix as cext_posix +from ._common import decode +from ._common import get_procfs_path +from ._common import isfile_strict +from ._common import memoize +from ._common import memoize_when_activated +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN +from ._common import open_binary +from ._common import open_text +from ._common import parse_environ_block +from ._common import path_exists_strict +from ._common import supports_ipv6 +from ._common import usage_percent +from ._compat import b +from ._compat import basestring +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._compat import PY3 + +if sys.version_info >= (3, 4): + import enum +else: + enum = None + + +__extra__all__ = [ + # + 'PROCFS_PATH', + # io prio constants + "IOPRIO_CLASS_NONE", "IOPRIO_CLASS_RT", "IOPRIO_CLASS_BE", + "IOPRIO_CLASS_IDLE", + # connection status constants + "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1", + "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", + "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", ] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +POWER_SUPPLY_PATH = "/sys/class/power_supply" +HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) +HAS_PRLIMIT = hasattr(cext, "linux_prlimit") +HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") +HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") +_DEFAULT = object() + +# RLIMIT_* constants, not guaranteed to be present on all kernels +if HAS_PRLIMIT: + for name in dir(cext): + if name.startswith('RLIM'): + __extra__all__.append(name) + +# Number of clock ticks per second +CLOCK_TICKS = os.sysconf("SC_CLK_TCK") +PAGESIZE = os.sysconf("SC_PAGE_SIZE") +BOOT_TIME = None # set later +# Used when reading "big" files, namely /proc/{pid}/smaps and /proc/net/*. +# On Python 2, using a buffer with open() for such files may result in a +# speedup, see: https://github.com/giampaolo/psutil/issues/708 +BIGFILE_BUFFERING = -1 if PY3 else 8192 +LITTLE_ENDIAN = sys.byteorder == 'little' + +# "man iostat" states that sectors are equivalent with blocks and have +# a size of 512 bytes. Despite this value can be queried at runtime +# via /sys/block/{DISK}/queue/hw_sector_size and results may vary +# between 1k, 2k, or 4k... 512 appears to be a magic constant used +# throughout Linux source code: +# * https://stackoverflow.com/a/38136179/376587 +# * https://lists.gt.net/linux/kernel/2241060 +# * https://github.com/giampaolo/psutil/issues/1305 +# * https://github.com/torvalds/linux/blob/ +# 4f671fe2f9523a1ea206f63fe60a7c7b3a56d5c7/include/linux/bio.h#L99 +# * https://lkml.org/lkml/2015/8/17/234 +DISK_SECTOR_SIZE = 512 + +if enum is None: + AF_LINK = socket.AF_PACKET +else: + AddressFamily = enum.IntEnum('AddressFamily', + {'AF_LINK': int(socket.AF_PACKET)}) + AF_LINK = AddressFamily.AF_LINK + +# ioprio_* constants http://linux.die.net/man/2/ioprio_get +if enum is None: + IOPRIO_CLASS_NONE = 0 + IOPRIO_CLASS_RT = 1 + IOPRIO_CLASS_BE = 2 + IOPRIO_CLASS_IDLE = 3 +else: + class IOPriority(enum.IntEnum): + IOPRIO_CLASS_NONE = 0 + IOPRIO_CLASS_RT = 1 + IOPRIO_CLASS_BE = 2 + IOPRIO_CLASS_IDLE = 3 + + globals().update(IOPriority.__members__) + +# See: +# https://github.com/torvalds/linux/blame/master/fs/proc/array.c +# ...and (TASK_* constants): +# https://github.com/torvalds/linux/blob/master/include/linux/sched.h +PROC_STATUSES = { + "R": _common.STATUS_RUNNING, + "S": _common.STATUS_SLEEPING, + "D": _common.STATUS_DISK_SLEEP, + "T": _common.STATUS_STOPPED, + "t": _common.STATUS_TRACING_STOP, + "Z": _common.STATUS_ZOMBIE, + "X": _common.STATUS_DEAD, + "x": _common.STATUS_DEAD, + "K": _common.STATUS_WAKE_KILL, + "W": _common.STATUS_WAKING, + "I": _common.STATUS_IDLE, + "P": _common.STATUS_PARKED, +} + +# https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h +TCP_STATUSES = { + "01": _common.CONN_ESTABLISHED, + "02": _common.CONN_SYN_SENT, + "03": _common.CONN_SYN_RECV, + "04": _common.CONN_FIN_WAIT1, + "05": _common.CONN_FIN_WAIT2, + "06": _common.CONN_TIME_WAIT, + "07": _common.CONN_CLOSE, + "08": _common.CONN_CLOSE_WAIT, + "09": _common.CONN_LAST_ACK, + "0A": _common.CONN_LISTEN, + "0B": _common.CONN_CLOSING +} + +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.virtual_memory() +svmem = namedtuple( + 'svmem', ['total', 'available', 'percent', 'used', 'free', + 'active', 'inactive', 'buffers', 'cached', 'shared', 'slab']) +# psutil.disk_io_counters() +sdiskio = namedtuple( + 'sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_time', 'write_time', + 'read_merged_count', 'write_merged_count', + 'busy_time']) +# psutil.Process().open_files() +popenfile = namedtuple( + 'popenfile', ['path', 'fd', 'position', 'mode', 'flags']) +# psutil.Process().memory_info() +pmem = namedtuple('pmem', 'rss vms shared text lib data dirty') +# psutil.Process().memory_full_info() +pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', 'pss', 'swap')) +# psutil.Process().memory_maps(grouped=True) +pmmap_grouped = namedtuple( + 'pmmap_grouped', + ['path', 'rss', 'size', 'pss', 'shared_clean', 'shared_dirty', + 'private_clean', 'private_dirty', 'referenced', 'anonymous', 'swap']) +# psutil.Process().memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_chars', 'write_chars']) +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system', + 'iowait']) + + +# ===================================================================== +# --- utils +# ===================================================================== + + +def readlink(path): + """Wrapper around os.readlink().""" + assert isinstance(path, basestring), path + path = os.readlink(path) + # readlink() might return paths containing null bytes ('\x00') + # resulting in "TypeError: must be encoded string without NULL + # bytes, not str" errors when the string is passed to other + # fs-related functions (os.*, open(), ...). + # Apparently everything after '\x00' is garbage (we can have + # ' (deleted)', 'new' and possibly others), see: + # https://github.com/giampaolo/psutil/issues/717 + path = path.split('\x00')[0] + # Certain paths have ' (deleted)' appended. Usually this is + # bogus as the file actually exists. Even if it doesn't we + # don't care. + if path.endswith(' (deleted)') and not path_exists_strict(path): + path = path[:-10] + return path + + +def file_flags_to_mode(flags): + """Convert file's open() flags into a readable string. + Used by Process.open_files(). + """ + modes_map = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'} + mode = modes_map[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)] + if flags & os.O_APPEND: + mode = mode.replace('w', 'a', 1) + mode = mode.replace('w+', 'r+') + # possible values: r, w, a, r+, a+ + return mode + + +def is_storage_device(name): + """Return True if the given name refers to a root device (e.g. + "sda", "nvme0n1") as opposed to a logical partition (e.g. "sda1", + "nvme0n1p1"). If name is a virtual device (e.g. "loop1", "ram") + return True. + """ + # Readapted from iostat source code, see: + # https://github.com/sysstat/sysstat/blob/ + # 97912938cd476645b267280069e83b1c8dc0e1c7/common.c#L208 + # Some devices may have a slash in their name (e.g. cciss/c0d0...). + name = name.replace('/', '!') + including_virtual = True + if including_virtual: + path = "/sys/block/%s" % name + else: + path = "/sys/block/%s/device" % name + return os.access(path, os.F_OK) + + +@memoize +def set_scputimes_ntuple(procfs_path): + """Set a namedtuple of variable fields depending on the CPU times + available on this Linux kernel version which may be: + (user, nice, system, idle, iowait, irq, softirq, [steal, [guest, + [guest_nice]]]) + Used by cpu_times() function. + """ + global scputimes + with open_binary('%s/stat' % procfs_path) as f: + values = f.readline().split()[1:] + fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq'] + vlen = len(values) + if vlen >= 8: + # Linux >= 2.6.11 + fields.append('steal') + if vlen >= 9: + # Linux >= 2.6.24 + fields.append('guest') + if vlen >= 10: + # Linux >= 3.2.0 + fields.append('guest_nice') + scputimes = namedtuple('scputimes', fields) + + +def cat(fname, fallback=_DEFAULT, binary=True): + """Return file content. + fallback: the value returned in case the file does not exist or + cannot be read + binary: whether to open the file in binary or text mode. + """ + try: + with open_binary(fname) if binary else open_text(fname) as f: + return f.read().strip() + except (IOError, OSError): + if fallback is not _DEFAULT: + return fallback + else: + raise + + +try: + set_scputimes_ntuple("/proc") +except Exception: + # Don't want to crash at import time. + traceback.print_exc() + scputimes = namedtuple('scputimes', 'user system idle')(0.0, 0.0, 0.0) + + +# ===================================================================== +# --- system memory +# ===================================================================== + + +def calculate_avail_vmem(mems): + """Fallback for kernels < 3.14 where /proc/meminfo does not provide + "MemAvailable:" column, see: + https://blog.famzah.net/2014/09/24/ + This code reimplements the algorithm outlined here: + https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ + commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + + XXX: on recent kernels this calculation differs by ~1.5% than + "MemAvailable:" as it's calculated slightly differently, see: + https://gitlab.com/procps-ng/procps/issues/42 + https://github.com/famzah/linux-memavailable-procfs/issues/2 + It is still way more realistic than doing (free + cached) though. + """ + # Fallback for very old distros. According to + # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ + # commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + # ...long ago "avail" was calculated as (free + cached). + # We might fallback in such cases: + # "Active(file)" not available: 2.6.28 / Dec 2008 + # "Inactive(file)" not available: 2.6.28 / Dec 2008 + # "SReclaimable:" not available: 2.6.19 / Nov 2006 + # /proc/zoneinfo not available: 2.6.13 / Aug 2005 + free = mems[b'MemFree:'] + fallback = free + mems.get(b"Cached:", 0) + try: + lru_active_file = mems[b'Active(file):'] + lru_inactive_file = mems[b'Inactive(file):'] + slab_reclaimable = mems[b'SReclaimable:'] + except KeyError: + return fallback + try: + f = open_binary('%s/zoneinfo' % get_procfs_path()) + except IOError: + return fallback # kernel 2.6.13 + + watermark_low = 0 + with f: + for line in f: + line = line.strip() + if line.startswith(b'low'): + watermark_low += int(line.split()[1]) + watermark_low *= PAGESIZE + watermark_low = watermark_low + + avail = free - watermark_low + pagecache = lru_active_file + lru_inactive_file + pagecache -= min(pagecache / 2, watermark_low) + avail += pagecache + avail += slab_reclaimable - min(slab_reclaimable / 2.0, watermark_low) + return int(avail) + + +def virtual_memory(): + """Report virtual memory stats. + This implementation matches "free" and "vmstat -s" cmdline + utility values and procps-ng-3.3.12 source was used as a reference + (2016-09-18): + https://gitlab.com/procps-ng/procps/blob/ + 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c + For reference, procps-ng-3.3.10 is the version available on Ubuntu + 16.04. + + Note about "available" memory: up until psutil 4.3 it was + calculated as "avail = (free + buffers + cached)". Now + "MemAvailable:" column (kernel 3.14) from /proc/meminfo is used as + it's more accurate. + That matches "available" column in newer versions of "free". + """ + missing_fields = [] + mems = {} + with open_binary('%s/meminfo' % get_procfs_path()) as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + # /proc doc states that the available fields in /proc/meminfo vary + # by architecture and compile options, but these 3 values are also + # returned by sysinfo(2); as such we assume they are always there. + total = mems[b'MemTotal:'] + free = mems[b'MemFree:'] + try: + buffers = mems[b'Buffers:'] + except KeyError: + # https://github.com/giampaolo/psutil/issues/1010 + buffers = 0 + missing_fields.append('buffers') + try: + cached = mems[b"Cached:"] + except KeyError: + cached = 0 + missing_fields.append('cached') + else: + # "free" cmdline utility sums reclaimable to cached. + # Older versions of procps used to add slab memory instead. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + cached += mems.get(b"SReclaimable:", 0) # since kernel 2.6.19 + + try: + shared = mems[b'Shmem:'] # since kernel 2.6.32 + except KeyError: + try: + shared = mems[b'MemShared:'] # kernels 2.4 + except KeyError: + shared = 0 + missing_fields.append('shared') + + try: + active = mems[b"Active:"] + except KeyError: + active = 0 + missing_fields.append('active') + + try: + inactive = mems[b"Inactive:"] + except KeyError: + try: + inactive = \ + mems[b"Inact_dirty:"] + \ + mems[b"Inact_clean:"] + \ + mems[b"Inact_laundry:"] + except KeyError: + inactive = 0 + missing_fields.append('inactive') + + try: + slab = mems[b"Slab:"] + except KeyError: + slab = 0 + + used = total - free - cached - buffers + if used < 0: + # May be symptomatic of running within a LCX container where such + # values will be dramatically distorted over those of the host. + used = total - free + + # - starting from 4.4.0 we match free's "available" column. + # Before 4.4.0 we calculated it as (free + buffers + cached) + # which matched htop. + # - free and htop available memory differs as per: + # http://askubuntu.com/a/369589 + # http://unix.stackexchange.com/a/65852/168884 + # - MemAvailable has been introduced in kernel 3.14 + try: + avail = mems[b'MemAvailable:'] + except KeyError: + avail = calculate_avail_vmem(mems) + + if avail < 0: + avail = 0 + missing_fields.append('available') + + # If avail is greater than total or our calculation overflows, + # that's symptomatic of running within a LCX container where such + # values will be dramatically distorted over those of the host. + # https://gitlab.com/procps-ng/procps/blob/ + # 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764 + if avail > total: + avail = free + + percent = usage_percent((total - avail), total, round_=1) + + # Warn about missing metrics which are set to 0. + if missing_fields: + msg = "%s memory stats couldn't be determined and %s set to 0" % ( + ", ".join(missing_fields), + "was" if len(missing_fields) == 1 else "were") + warnings.warn(msg, RuntimeWarning) + + return svmem(total, avail, percent, used, free, + active, inactive, buffers, cached, shared, slab) + + +def swap_memory(): + """Return swap memory metrics.""" + mems = {} + with open_binary('%s/meminfo' % get_procfs_path()) as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + # We prefer /proc/meminfo over sysinfo() syscall so that + # psutil.PROCFS_PATH can be used in order to allow retrieval + # for linux containers, see: + # https://github.com/giampaolo/psutil/issues/1015 + try: + total = mems[b'SwapTotal:'] + free = mems[b'SwapFree:'] + except KeyError: + _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() + total *= unit_multiplier + free *= unit_multiplier + + used = total - free + percent = usage_percent(used, total, round_=1) + # get pgin/pgouts + try: + f = open_binary("%s/vmstat" % get_procfs_path()) + except IOError as err: + # see https://github.com/giampaolo/psutil/issues/722 + msg = "'sin' and 'sout' swap memory stats couldn't " \ + "be determined and were set to 0 (%s)" % str(err) + warnings.warn(msg, RuntimeWarning) + sin = sout = 0 + else: + with f: + sin = sout = None + for line in f: + # values are expressed in 4 kilo bytes, we want + # bytes instead + if line.startswith(b'pswpin'): + sin = int(line.split(b' ')[1]) * 4 * 1024 + elif line.startswith(b'pswpout'): + sout = int(line.split(b' ')[1]) * 4 * 1024 + if sin is not None and sout is not None: + break + else: + # we might get here when dealing with exotic Linux + # flavors, see: + # https://github.com/giampaolo/psutil/issues/313 + msg = "'sin' and 'sout' swap memory stats couldn't " \ + "be determined and were set to 0" + warnings.warn(msg, RuntimeWarning) + sin = sout = 0 + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return a named tuple representing the following system-wide + CPU times: + (user, nice, system, idle, iowait, irq, softirq [steal, [guest, + [guest_nice]]]) + Last 3 fields may not be available on all Linux kernel versions. + """ + procfs_path = get_procfs_path() + set_scputimes_ntuple(procfs_path) + with open_binary('%s/stat' % procfs_path) as f: + values = f.readline().split() + fields = values[1:len(scputimes._fields) + 1] + fields = [float(x) / CLOCK_TICKS for x in fields] + return scputimes(*fields) + + +def per_cpu_times(): + """Return a list of namedtuple representing the CPU times + for every CPU available on the system. + """ + procfs_path = get_procfs_path() + set_scputimes_ntuple(procfs_path) + cpus = [] + with open_binary('%s/stat' % procfs_path) as f: + # get rid of the first line which refers to system wide CPU stats + f.readline() + for line in f: + if line.startswith(b'cpu'): + values = line.split() + fields = values[1:len(scputimes._fields) + 1] + fields = [float(x) / CLOCK_TICKS for x in fields] + entry = scputimes(*fields) + cpus.append(entry) + return cpus + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # as a second fallback we try to parse /proc/cpuinfo + num = 0 + with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + for line in f: + if line.lower().startswith(b'processor'): + num += 1 + + # unknown format (e.g. amrel/sparc architectures), see: + # https://github.com/giampaolo/psutil/issues/200 + # try to parse /proc/stat as a last resort + if num == 0: + search = re.compile(r'cpu\d') + with open_text('%s/stat' % get_procfs_path()) as f: + for line in f: + line = line.split(' ')[0] + if search.match(line): + num += 1 + + if num == 0: + # mimic os.cpu_count() + return None + return num + + +def cpu_count_physical(): + """Return the number of physical cores in the system.""" + # Method #1 + core_ids = set() + for path in glob.glob( + "/sys/devices/system/cpu/cpu[0-9]*/topology/core_id"): + with open_binary(path) as f: + core_ids.add(int(f.read())) + result = len(core_ids) + if result != 0: + return result + + # Method #2 + mapping = {} + current_info = {} + with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + for line in f: + line = line.strip().lower() + if not line: + # new section + if (b'physical id' in current_info and + b'cpu cores' in current_info): + mapping[current_info[b'physical id']] = \ + current_info[b'cpu cores'] + current_info = {} + else: + # ongoing section + if (line.startswith(b'physical id') or + line.startswith(b'cpu cores')): + key, value = line.split(b'\t:', 1) + current_info[key] = int(value) + + result = sum(mapping.values()) + return result or None # mimic os.cpu_count() + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + with open_binary('%s/stat' % get_procfs_path()) as f: + ctx_switches = None + interrupts = None + soft_interrupts = None + for line in f: + if line.startswith(b'ctxt'): + ctx_switches = int(line.split()[1]) + elif line.startswith(b'intr'): + interrupts = int(line.split()[1]) + elif line.startswith(b'softirq'): + soft_interrupts = int(line.split()[1]) + if ctx_switches is not None and soft_interrupts is not None \ + and interrupts is not None: + break + syscalls = 0 + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls) + + +if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or \ + os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"): + def cpu_freq(): + """Return frequency metrics for all CPUs. + Contrarily to other OSes, Linux updates these values in + real-time. + """ + def get_path(num): + for p in ("/sys/devices/system/cpu/cpufreq/policy%s" % num, + "/sys/devices/system/cpu/cpu%s/cpufreq" % num): + if os.path.exists(p): + return p + + ret = [] + for n in range(cpu_count_logical()): + path = get_path(n) + if not path: + continue + + pjoin = os.path.join + curr = cat(pjoin(path, "scaling_cur_freq"), fallback=None) + if curr is None: + # Likely an old RedHat, see: + # https://github.com/giampaolo/psutil/issues/1071 + curr = cat(pjoin(path, "cpuinfo_cur_freq"), fallback=None) + if curr is None: + raise NotImplementedError( + "can't find current frequency file") + curr = int(curr) / 1000 + max_ = int(cat(pjoin(path, "scaling_max_freq"))) / 1000 + min_ = int(cat(pjoin(path, "scaling_min_freq"))) / 1000 + ret.append(_common.scpufreq(curr, min_, max_)) + return ret + +elif os.path.exists("/proc/cpuinfo"): + def cpu_freq(): + """Alternate implementation using /proc/cpuinfo. + min and max frequencies are not available and are set to None. + """ + ret = [] + with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + for line in f: + if line.lower().startswith(b'cpu mhz'): + key, value = line.split(b'\t:', 1) + ret.append(_common.scpufreq(float(value), 0., 0.)) + return ret + +else: + def cpu_freq(): + """Dummy implementation when none of the above files are present. + """ + return [] + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_if_addrs = cext_posix.net_if_addrs + + +class _Ipv6UnsupportedError(Exception): + pass + + +class Connections: + """A wrapper on top of /proc/net/* files, retrieving per-process + and system-wide open connections (TCP, UDP, UNIX) similarly to + "netstat -an". + + Note: in case of UNIX sockets we're only able to determine the + local endpoint/path, not the one it's connected to. + According to [1] it would be possible but not easily. + + [1] http://serverfault.com/a/417946 + """ + + def __init__(self): + tcp4 = ("tcp", socket.AF_INET, socket.SOCK_STREAM) + tcp6 = ("tcp6", socket.AF_INET6, socket.SOCK_STREAM) + udp4 = ("udp", socket.AF_INET, socket.SOCK_DGRAM) + udp6 = ("udp6", socket.AF_INET6, socket.SOCK_DGRAM) + unix = ("unix", socket.AF_UNIX, None) + self.tmap = { + "all": (tcp4, tcp6, udp4, udp6, unix), + "tcp": (tcp4, tcp6), + "tcp4": (tcp4,), + "tcp6": (tcp6,), + "udp": (udp4, udp6), + "udp4": (udp4,), + "udp6": (udp6,), + "unix": (unix,), + "inet": (tcp4, tcp6, udp4, udp6), + "inet4": (tcp4, udp4), + "inet6": (tcp6, udp6), + } + self._procfs_path = None + + def get_proc_inodes(self, pid): + inodes = defaultdict(list) + for fd in os.listdir("%s/%s/fd" % (self._procfs_path, pid)): + try: + inode = readlink("%s/%s/fd/%s" % (self._procfs_path, pid, fd)) + except (FileNotFoundError, ProcessLookupError): + # ENOENT == file which is gone in the meantime; + # os.stat('/proc/%s' % self.pid) will be done later + # to force NSP (if it's the case) + continue + except OSError as err: + if err.errno == errno.EINVAL: + # not a link + continue + raise + else: + if inode.startswith('socket:['): + # the process is using a socket + inode = inode[8:][:-1] + inodes[inode].append((pid, int(fd))) + return inodes + + def get_all_inodes(self): + inodes = {} + for pid in pids(): + try: + inodes.update(self.get_proc_inodes(pid)) + except (FileNotFoundError, ProcessLookupError, PermissionError): + # os.listdir() is gonna raise a lot of access denied + # exceptions in case of unprivileged user; that's fine + # as we'll just end up returning a connection with PID + # and fd set to None anyway. + # Both netstat -an and lsof does the same so it's + # unlikely we can do any better. + # ENOENT just means a PID disappeared on us. + continue + return inodes + + @staticmethod + def decode_address(addr, family): + """Accept an "ip:port" address as displayed in /proc/net/* + and convert it into a human readable form, like: + + "0500000A:0016" -> ("10.0.0.5", 22) + "0000000000000000FFFF00000100007F:9E49" -> ("::ffff:127.0.0.1", 40521) + + The IP address portion is a little or big endian four-byte + hexadecimal number; that is, the least significant byte is listed + first, so we need to reverse the order of the bytes to convert it + to an IP address. + The port is represented as a two-byte hexadecimal number. + + Reference: + http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html + """ + ip, port = addr.split(':') + port = int(port, 16) + # this usually refers to a local socket in listen mode with + # no end-points connected + if not port: + return () + if PY3: + ip = ip.encode('ascii') + if family == socket.AF_INET: + # see: https://github.com/giampaolo/psutil/issues/201 + if LITTLE_ENDIAN: + ip = socket.inet_ntop(family, base64.b16decode(ip)[::-1]) + else: + ip = socket.inet_ntop(family, base64.b16decode(ip)) + else: # IPv6 + # old version - let's keep it, just in case... + # ip = ip.decode('hex') + # return socket.inet_ntop(socket.AF_INET6, + # ''.join(ip[i:i+4][::-1] for i in xrange(0, 16, 4))) + ip = base64.b16decode(ip) + try: + # see: https://github.com/giampaolo/psutil/issues/201 + if LITTLE_ENDIAN: + ip = socket.inet_ntop( + socket.AF_INET6, + struct.pack('>4I', *struct.unpack('<4I', ip))) + else: + ip = socket.inet_ntop( + socket.AF_INET6, + struct.pack('<4I', *struct.unpack('<4I', ip))) + except ValueError: + # see: https://github.com/giampaolo/psutil/issues/623 + if not supports_ipv6(): + raise _Ipv6UnsupportedError + else: + raise + return _common.addr(ip, port) + + @staticmethod + def process_inet(file, family, type_, inodes, filter_pid=None): + """Parse /proc/net/tcp* and /proc/net/udp* files.""" + if file.endswith('6') and not os.path.exists(file): + # IPv6 not supported + return + with open_text(file, buffering=BIGFILE_BUFFERING) as f: + f.readline() # skip the first line + for lineno, line in enumerate(f, 1): + try: + _, laddr, raddr, status, _, _, _, _, _, inode = \ + line.split()[:10] + except ValueError: + raise RuntimeError( + "error while parsing %s; malformed line %s %r" % ( + file, lineno, line)) + if inode in inodes: + # # We assume inet sockets are unique, so we error + # # out if there are multiple references to the + # # same inode. We won't do this for UNIX sockets. + # if len(inodes[inode]) > 1 and family != socket.AF_UNIX: + # raise ValueError("ambiguos inode with multiple " + # "PIDs references") + pid, fd = inodes[inode][0] + else: + pid, fd = None, -1 + if filter_pid is not None and filter_pid != pid: + continue + else: + if type_ == socket.SOCK_STREAM: + status = TCP_STATUSES[status] + else: + status = _common.CONN_NONE + try: + laddr = Connections.decode_address(laddr, family) + raddr = Connections.decode_address(raddr, family) + except _Ipv6UnsupportedError: + continue + yield (fd, family, type_, laddr, raddr, status, pid) + + @staticmethod + def process_unix(file, family, inodes, filter_pid=None): + """Parse /proc/net/unix files.""" + with open_text(file, buffering=BIGFILE_BUFFERING) as f: + f.readline() # skip the first line + for line in f: + tokens = line.split() + try: + _, _, _, _, type_, _, inode = tokens[0:7] + except ValueError: + if ' ' not in line: + # see: https://github.com/giampaolo/psutil/issues/766 + continue + raise RuntimeError( + "error while parsing %s; malformed line %r" % ( + file, line)) + if inode in inodes: + # With UNIX sockets we can have a single inode + # referencing many file descriptors. + pairs = inodes[inode] + else: + pairs = [(None, -1)] + for pid, fd in pairs: + if filter_pid is not None and filter_pid != pid: + continue + else: + if len(tokens) == 8: + path = tokens[-1] + else: + path = "" + type_ = _common.socktype_to_enum(int(type_)) + # XXX: determining the remote endpoint of a + # UNIX socket on Linux is not possible, see: + # https://serverfault.com/questions/252723/ + raddr = "" + status = _common.CONN_NONE + yield (fd, family, type_, path, raddr, status, pid) + + def retrieve(self, kind, pid=None): + if kind not in self.tmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in self.tmap]))) + self._procfs_path = get_procfs_path() + if pid is not None: + inodes = self.get_proc_inodes(pid) + if not inodes: + # no connections for this process + return [] + else: + inodes = self.get_all_inodes() + ret = set() + for f, family, type_ in self.tmap[kind]: + if family in (socket.AF_INET, socket.AF_INET6): + ls = self.process_inet( + "%s/net/%s" % (self._procfs_path, f), + family, type_, inodes, filter_pid=pid) + else: + ls = self.process_unix( + "%s/net/%s" % (self._procfs_path, f), + family, inodes, filter_pid=pid) + for fd, family, type_, laddr, raddr, status, bound_pid in ls: + if pid: + conn = _common.pconn(fd, family, type_, laddr, raddr, + status) + else: + conn = _common.sconn(fd, family, type_, laddr, raddr, + status, bound_pid) + ret.add(conn) + return list(ret) + + +_connections = Connections() + + +def net_connections(kind='inet'): + """Return system-wide open connections.""" + return _connections.retrieve(kind) + + +def net_io_counters(): + """Return network I/O statistics for every network interface + installed on the system as a dict of raw tuples. + """ + with open_text("%s/net/dev" % get_procfs_path()) as f: + lines = f.readlines() + retdict = {} + for line in lines[2:]: + colon = line.rfind(':') + assert colon > 0, repr(line) + name = line[:colon].strip() + fields = line[colon + 1:].strip().split() + + # in + (bytes_recv, + packets_recv, + errin, + dropin, + fifoin, # unused + framein, # unused + compressedin, # unused + multicastin, # unused + # out + bytes_sent, + packets_sent, + errout, + dropout, + fifoout, # unused + collisionsout, # unused + carrierout, # unused + compressedout) = map(int, fields) + + retdict[name] = (bytes_sent, bytes_recv, packets_sent, packets_recv, + errin, errout, dropin, dropout) + return retdict + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + duplex_map = {cext.DUPLEX_FULL: NIC_DUPLEX_FULL, + cext.DUPLEX_HALF: NIC_DUPLEX_HALF, + cext.DUPLEX_UNKNOWN: NIC_DUPLEX_UNKNOWN} + names = net_io_counters().keys() + ret = {} + for name in names: + try: + mtu = cext_posix.net_if_mtu(name) + isup = cext_posix.net_if_flags(name) + duplex, speed = cext.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + ret[name] = _common.snicstats(isup, duplex_map[duplex], speed, mtu) + return ret + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_usage = _psposix.disk_usage + + +def disk_io_counters(perdisk=False): + """Return disk I/O statistics for every disk installed on the + system as a dict of raw tuples. + """ + def read_procfs(): + # OK, this is a bit confusing. The format of /proc/diskstats can + # have 3 variations. + # On Linux 2.4 each line has always 15 fields, e.g.: + # "3 0 8 hda 8 8 8 8 8 8 8 8 8 8 8" + # On Linux 2.6+ each line *usually* has 14 fields, and the disk + # name is in another position, like this: + # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8" + # ...unless (Linux 2.6) the line refers to a partition instead + # of a disk, in which case the line has less fields (7): + # "3 1 hda1 8 8 8 8" + # 4.18+ has 4 fields added: + # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8 0 0 0 0" + # See: + # https://www.kernel.org/doc/Documentation/iostats.txt + # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats + with open_text("%s/diskstats" % get_procfs_path()) as f: + lines = f.readlines() + for line in lines: + fields = line.split() + flen = len(fields) + if flen == 15: + # Linux 2.4 + name = fields[3] + reads = int(fields[2]) + (reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields[4:14]) + elif flen == 14 or flen == 18: + # Linux 2.6+, line referring to a disk + name = fields[2] + (reads, reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields[3:14]) + elif flen == 7: + # Linux 2.6+, line referring to a partition + name = fields[2] + reads, rbytes, writes, wbytes = map(int, fields[3:]) + rtime = wtime = reads_merged = writes_merged = busy_time = 0 + else: + raise ValueError("not sure how to interpret line %r" % line) + yield (name, reads, writes, rbytes, wbytes, rtime, wtime, + reads_merged, writes_merged, busy_time) + + def read_sysfs(): + for block in os.listdir('/sys/block'): + for root, _, files in os.walk(os.path.join('/sys/block', block)): + if 'stat' not in files: + continue + with open_text(os.path.join(root, 'stat')) as f: + fields = f.read().strip().split() + name = os.path.basename(root) + (reads, reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields) + yield (name, reads, writes, rbytes, wbytes, rtime, + wtime, reads_merged, writes_merged, busy_time) + + if os.path.exists('%s/diskstats' % get_procfs_path()): + gen = read_procfs() + elif os.path.exists('/sys/block'): + gen = read_sysfs() + else: + raise NotImplementedError( + "%s/diskstats nor /sys/block filesystem are available on this " + "system" % get_procfs_path()) + + retdict = {} + for entry in gen: + (name, reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, + writes_merged, busy_time) = entry + if not perdisk and not is_storage_device(name): + # perdisk=False means we want to calculate totals so we skip + # partitions (e.g. 'sda1', 'nvme0n1p1') and only include + # base disk devices (e.g. 'sda', 'nvme0n1'). Base disks + # include a total of all their partitions + some extra size + # of their own: + # $ cat /proc/diskstats + # 259 0 sda 10485760 ... + # 259 1 sda1 5186039 ... + # 259 1 sda2 5082039 ... + # See: + # https://github.com/giampaolo/psutil/pull/1313 + continue + + rbytes *= DISK_SECTOR_SIZE + wbytes *= DISK_SECTOR_SIZE + retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, + reads_merged, writes_merged, busy_time) + + return retdict + + +def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples.""" + fstypes = set() + procfs_path = get_procfs_path() + with open_text("%s/filesystems" % procfs_path) as f: + for line in f: + line = line.strip() + if not line.startswith("nodev"): + fstypes.add(line.strip()) + else: + # ignore all lines starting with "nodev" except "nodev zfs" + fstype = line.split("\t")[1] + if fstype == "zfs": + fstypes.add("zfs") + + # See: https://github.com/giampaolo/psutil/issues/1307 + if procfs_path == "/proc" and os.path.isfile('/etc/mtab'): + mounts_path = os.path.realpath("/etc/mtab") + else: + mounts_path = os.path.realpath("%s/self/mounts" % procfs_path) + + retlist = [] + partitions = cext.disk_partitions(mounts_path) + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + if device == '' or fstype not in fstypes: + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_temperatures(): + """Return hardware (CPU and others) temperatures as a dict + including hardware name, label, current, max and critical + temperatures. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + - /sys/class/thermal/thermal_zone* is another one but it's more + difficult to parse + """ + ret = collections.defaultdict(list) + basenames = glob.glob('/sys/class/hwmon/hwmon*/temp*_*') + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + # https://github.com/nicolargo/glances/issues/1060 + basenames.extend(glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*')) + basenames = sorted(set([x.split('_')[0] for x in basenames])) + + for base in basenames: + try: + path = base + '_input' + current = float(cat(path)) / 1000.0 + path = os.path.join(os.path.dirname(base), 'name') + unit_name = cat(path, binary=False) + except (IOError, OSError, ValueError) as err: + # A lot of things can go wrong here, so let's just skip the + # whole entry. Sure thing is Linux's /sys/class/hwmon really + # is a stinky broken mess. + # https://github.com/giampaolo/psutil/issues/1009 + # https://github.com/giampaolo/psutil/issues/1101 + # https://github.com/giampaolo/psutil/issues/1129 + # https://github.com/giampaolo/psutil/issues/1245 + # https://github.com/giampaolo/psutil/issues/1323 + warnings.warn("ignoring %r for file %r" % (err, path), + RuntimeWarning) + continue + + high = cat(base + '_max', fallback=None) + critical = cat(base + '_crit', fallback=None) + label = cat(base + '_label', fallback='', binary=False) + + if high is not None: + try: + high = float(high) / 1000.0 + except ValueError: + high = None + if critical is not None: + try: + critical = float(critical) / 1000.0 + except ValueError: + critical = None + + ret[unit_name].append((label, current, high, critical)) + + # Indication that no sensors were detected in /sys/class/hwmon/ + if not basenames: + basenames = glob.glob('/sys/class/thermal/thermal_zone*') + basenames = sorted(set(basenames)) + + for base in basenames: + try: + path = os.path.join(base, 'temp') + current = float(cat(path)) / 1000.0 + path = os.path.join(base, 'type') + unit_name = cat(path, binary=False) + except (IOError, OSError, ValueError) as err: + warnings.warn("ignoring %r for file %r" % (err, path), + RuntimeWarning) + continue + + trip_paths = glob.glob(base + '/trip_point*') + trip_points = set(['_'.join( + os.path.basename(p).split('_')[0:3]) for p in trip_paths]) + critical = None + high = None + for trip_point in trip_points: + path = os.path.join(base, trip_point + "_type") + trip_type = cat(path, fallback='', binary=False) + if trip_type == 'critical': + critical = cat(os.path.join(base, trip_point + "_temp"), + fallback=None) + elif trip_type == 'high': + high = cat(os.path.join(base, trip_point + "_temp"), + fallback=None) + + if high is not None: + try: + high = float(high) / 1000.0 + except ValueError: + high = None + if critical is not None: + try: + critical = float(critical) / 1000.0 + except ValueError: + critical = None + + ret[unit_name].append(('', current, high, critical)) + + return dict(ret) + + +def sensors_fans(): + """Return hardware fans info (for CPU and other peripherals) as a + dict including hardware label and current speed. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + """ + ret = collections.defaultdict(list) + basenames = glob.glob('/sys/class/hwmon/hwmon*/fan*_*') + if not basenames: + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + basenames = glob.glob('/sys/class/hwmon/hwmon*/device/fan*_*') + + basenames = sorted(set([x.split('_')[0] for x in basenames])) + for base in basenames: + try: + current = int(cat(base + '_input')) + except (IOError, OSError) as err: + warnings.warn("ignoring %r" % err, RuntimeWarning) + continue + unit_name = cat(os.path.join(os.path.dirname(base), 'name'), + binary=False) + label = cat(base + '_label', fallback='', binary=False) + ret[unit_name].append(_common.sfan(label, current)) + + return dict(ret) + + +def sensors_battery(): + """Return battery information. + Implementation note: it appears /sys/class/power_supply/BAT0/ + directory structure may vary and provide files with the same + meaning but under different names, see: + https://github.com/giampaolo/psutil/issues/966 + """ + null = object() + + def multi_cat(*paths): + """Attempt to read the content of multiple files which may + not exist. If none of them exist return None. + """ + for path in paths: + ret = cat(path, fallback=null) + if ret != null: + return int(ret) if ret.isdigit() else ret + return None + + bats = [x for x in os.listdir(POWER_SUPPLY_PATH) if x.startswith('BAT')] + if not bats: + return None + # Get the first available battery. Usually this is "BAT0", except + # some rare exceptions: + # https://github.com/giampaolo/psutil/issues/1238 + root = os.path.join(POWER_SUPPLY_PATH, sorted(bats)[0]) + + # Base metrics. + energy_now = multi_cat( + root + "/energy_now", + root + "/charge_now") + power_now = multi_cat( + root + "/power_now", + root + "/current_now") + energy_full = multi_cat( + root + "/energy_full", + root + "/charge_full") + if energy_now is None or power_now is None: + return None + + # Percent. If we have energy_full the percentage will be more + # accurate compared to reading /capacity file (float vs. int). + if energy_full is not None: + try: + percent = 100.0 * energy_now / energy_full + except ZeroDivisionError: + percent = 0.0 + else: + percent = int(cat(root + "/capacity", fallback=-1)) + if percent == -1: + return None + + # Is AC power cable plugged in? + # Note: AC0 is not always available and sometimes (e.g. CentOS7) + # it's called "AC". + power_plugged = None + online = multi_cat( + os.path.join(POWER_SUPPLY_PATH, "AC0/online"), + os.path.join(POWER_SUPPLY_PATH, "AC/online")) + if online is not None: + power_plugged = online == 1 + else: + status = cat(root + "/status", fallback="", binary=False).lower() + if status == "discharging": + power_plugged = False + elif status in ("charging", "full"): + power_plugged = True + + # Seconds left. + # Note to self: we may also calculate the charging ETA as per: + # https://github.com/thialfihar/dotfiles/blob/ + # 013937745fd9050c30146290e8f963d65c0179e6/bin/battery.py#L55 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + else: + try: + secsleft = int(energy_now / power_now * 3600) + except ZeroDivisionError: + secsleft = _common.POWER_TIME_UNKNOWN + + return _common.sbattery(percent, secsleft, power_plugged) + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, user_process, pid = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname in (':0.0', ':0'): + hostname = 'localhost' + nt = _common.suser(user, tty or None, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +def boot_time(): + """Return the system boot time expressed in seconds since the epoch.""" + global BOOT_TIME + path = '%s/stat' % get_procfs_path() + with open_binary(path) as f: + for line in f: + if line.startswith(b'btime'): + ret = float(line.strip().split()[1]) + BOOT_TIME = ret + return ret + raise RuntimeError( + "line 'btime' not found in %s" % path) + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + return [int(x) for x in os.listdir(b(get_procfs_path())) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix PID. Linux TIDs are not + supported (always return False). + """ + if not _psposix.pid_exists(pid): + return False + else: + # Linux's apparently does not distinguish between PIDs and TIDs + # (thread IDs). + # listdir("/proc") won't show any TID (only PIDs) but + # os.stat("/proc/{tid}") will succeed if {tid} exists. + # os.kill() can also be passed a TID. This is quite confusing. + # In here we want to enforce this distinction and support PIDs + # only, see: + # https://github.com/giampaolo/psutil/issues/687 + try: + # Note: already checked that this is faster than using a + # regular expr. Also (a lot) faster than doing + # 'return pid in pids()' + path = "%s/%s/status" % (get_procfs_path(), pid) + with open_binary(path) as f: + for line in f: + if line.startswith(b"Tgid:"): + tgid = int(line.split()[1]) + # If tgid and pid are the same then we're + # dealing with a process PID. + return tgid == pid + raise ValueError("'Tgid' line not found in %s" % path) + except (EnvironmentError, ValueError): + return pid in pids() + + +def ppid_map(): + """Obtain a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + procfs_path = get_procfs_path() + for pid in pids(): + try: + with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: + data = f.read() + except (FileNotFoundError, ProcessLookupError): + # Note: we should be able to access /stat for all processes + # aka it's unlikely we'll bump into EPERM, which is good. + pass + else: + rpar = data.rfind(b')') + dset = data[rpar + 2:].split() + ppid = int(dset[1]) + ret[pid] = ppid + return ret + + +def wrap_exceptions(fun): + """Decorator which translates bare OSError and IOError exceptions + into NoSuchProcess and AccessDenied. + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except ProcessLookupError: + raise NoSuchProcess(self.pid, self._name) + except FileNotFoundError: + if not os.path.exists("%s/%s" % (self._procfs_path, self.pid)): + raise NoSuchProcess(self.pid, self._name) + # Note: zombies will keep existing under /proc until they're + # gone so there's no way to distinguish them in here. + raise + return wrapper + + +class Process(object): + """Linux process implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + os.stat('%s/%s' % (self._procfs_path, self.pid)) + + @wrap_exceptions + @memoize_when_activated + def _parse_stat_file(self): + """Parse /proc/{pid}/stat file and return a dict with various + process info. + Using "man proc" as a reference: where "man proc" refers to + position N always substract 3 (e.g ppid position 4 in + 'man proc' == position 1 in here). + The return value is cached in case oneshot() ctx manager is + in use. + """ + with open_binary("%s/%s/stat" % (self._procfs_path, self.pid)) as f: + data = f.read() + # Process name is between parentheses. It can contain spaces and + # other parentheses. This is taken into account by looking for + # the first occurrence of "(" and the last occurence of ")". + rpar = data.rfind(b')') + name = data[data.find(b'(') + 1:rpar] + fields = data[rpar + 2:].split() + + ret = {} + ret['name'] = name + ret['status'] = fields[0] + ret['ppid'] = fields[1] + ret['ttynr'] = fields[4] + ret['utime'] = fields[11] + ret['stime'] = fields[12] + ret['children_utime'] = fields[13] + ret['children_stime'] = fields[14] + ret['create_time'] = fields[19] + ret['cpu_num'] = fields[36] + ret['blkio_ticks'] = fields[39] # aka 'delayacct_blkio_ticks' + + return ret + + @wrap_exceptions + @memoize_when_activated + def _read_status_file(self): + """Read /proc/{pid}/stat file and return its content. + The return value is cached in case oneshot() ctx manager is + in use. + """ + with open_binary("%s/%s/status" % (self._procfs_path, self.pid)) as f: + return f.read() + + @wrap_exceptions + @memoize_when_activated + def _read_smaps_file(self): + with open_binary("%s/%s/smaps" % (self._procfs_path, self.pid), + buffering=BIGFILE_BUFFERING) as f: + return f.read().strip() + + def oneshot_enter(self): + self._parse_stat_file.cache_activate(self) + self._read_status_file.cache_activate(self) + self._read_smaps_file.cache_activate(self) + + def oneshot_exit(self): + self._parse_stat_file.cache_deactivate(self) + self._read_status_file.cache_deactivate(self) + self._read_smaps_file.cache_deactivate(self) + + @wrap_exceptions + def name(self): + name = self._parse_stat_file()['name'] + if PY3: + name = decode(name) + # XXX - gets changed later and probably needs refactoring + return name + + def exe(self): + try: + return readlink("%s/%s/exe" % (self._procfs_path, self.pid)) + except (FileNotFoundError, ProcessLookupError): + # no such file error; might be raised also if the + # path actually exists for system processes with + # low pids (about 0-20) + if os.path.lexists("%s/%s" % (self._procfs_path, self.pid)): + return "" + else: + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + + @wrap_exceptions + def cmdline(self): + with open_text("%s/%s/cmdline" % (self._procfs_path, self.pid)) as f: + data = f.read() + if not data: + # may happen in case of zombie process + return [] + # 'man proc' states that args are separated by null bytes '\0' + # and last char is supposed to be a null byte. Nevertheless + # some processes may change their cmdline after being started + # (via setproctitle() or similar), they are usually not + # compliant with this rule and use spaces instead. Google + # Chrome process is an example. See: + # https://github.com/giampaolo/psutil/issues/1179 + sep = '\x00' if data.endswith('\x00') else ' ' + if data.endswith(sep): + data = data[:-1] + cmdline = data.split(sep) + # Sometimes last char is a null byte '\0' but the args are + # separated by spaces, see: https://github.com/giampaolo/psutil/ + # issues/1179#issuecomment-552984549 + if sep == '\x00' and len(cmdline) == 1 and ' ' in data: + cmdline = data.split(' ') + return cmdline + + @wrap_exceptions + def environ(self): + with open_text("%s/%s/environ" % (self._procfs_path, self.pid)) as f: + data = f.read() + return parse_environ_block(data) + + @wrap_exceptions + def terminal(self): + tty_nr = int(self._parse_stat_file()['ttynr']) + tmap = _psposix.get_terminal_map() + try: + return tmap[tty_nr] + except KeyError: + return None + + # May not be available on old kernels. + if os.path.exists('/proc/%s/io' % os.getpid()): + @wrap_exceptions + def io_counters(self): + fname = "%s/%s/io" % (self._procfs_path, self.pid) + fields = {} + with open_binary(fname) as f: + for line in f: + # https://github.com/giampaolo/psutil/issues/1004 + line = line.strip() + if line: + try: + name, value = line.split(b': ') + except ValueError: + # https://github.com/giampaolo/psutil/issues/1004 + continue + else: + fields[name] = int(value) + if not fields: + raise RuntimeError("%s file was empty" % fname) + try: + return pio( + fields[b'syscr'], # read syscalls + fields[b'syscw'], # write syscalls + fields[b'read_bytes'], # read bytes + fields[b'write_bytes'], # write bytes + fields[b'rchar'], # read chars + fields[b'wchar'], # write chars + ) + except KeyError as err: + raise ValueError("%r field was not found in %s; found fields " + "are %r" % (err[0], fname, fields)) + + @wrap_exceptions + def cpu_times(self): + values = self._parse_stat_file() + utime = float(values['utime']) / CLOCK_TICKS + stime = float(values['stime']) / CLOCK_TICKS + children_utime = float(values['children_utime']) / CLOCK_TICKS + children_stime = float(values['children_stime']) / CLOCK_TICKS + iowait = float(values['blkio_ticks']) / CLOCK_TICKS + return pcputimes(utime, stime, children_utime, children_stime, iowait) + + @wrap_exceptions + def cpu_num(self): + """What CPU the process is on.""" + return int(self._parse_stat_file()['cpu_num']) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + @wrap_exceptions + def create_time(self): + ctime = float(self._parse_stat_file()['create_time']) + # According to documentation, starttime is in field 21 and the + # unit is jiffies (clock ticks). + # We first divide it for clock ticks and then add uptime returning + # seconds since the epoch, in UTC. + # Also use cached value if available. + bt = BOOT_TIME or boot_time() + return (ctime / CLOCK_TICKS) + bt + + @wrap_exceptions + def memory_info(self): + # ============================================================ + # | FIELD | DESCRIPTION | AKA | TOP | + # ============================================================ + # | rss | resident set size | | RES | + # | vms | total program size | size | VIRT | + # | shared | shared pages (from shared mappings) | | SHR | + # | text | text ('code') | trs | CODE | + # | lib | library (unused in Linux 2.6) | lrs | | + # | data | data + stack | drs | DATA | + # | dirty | dirty pages (unused in Linux 2.6) | dt | | + # ============================================================ + with open_binary("%s/%s/statm" % (self._procfs_path, self.pid)) as f: + vms, rss, shared, text, lib, data, dirty = \ + [int(x) * PAGESIZE for x in f.readline().split()[:7]] + return pmem(rss, vms, shared, text, lib, data, dirty) + + # /proc/pid/smaps does not exist on kernels < 2.6.14 or if + # CONFIG_MMU kernel configuration option is not enabled. + if HAS_SMAPS: + + @wrap_exceptions + def memory_full_info( + self, + # Gets Private_Clean, Private_Dirty, Private_Hugetlb. + _private_re=re.compile(br"\nPrivate.*:\s+(\d+)"), + _pss_re=re.compile(br"\nPss\:\s+(\d+)"), + _swap_re=re.compile(br"\nSwap\:\s+(\d+)")): + basic_mem = self.memory_info() + # Note: using 3 regexes is faster than reading the file + # line by line. + # XXX: on Python 3 the 2 regexes are 30% slower than on + # Python 2 though. Figure out why. + # + # You might be tempted to calculate USS by subtracting + # the "shared" value from the "resident" value in + # /proc//statm. But at least on Linux, statm's "shared" + # value actually counts pages backed by files, which has + # little to do with whether the pages are actually shared. + # /proc/self/smaps on the other hand appears to give us the + # correct information. + smaps_data = self._read_smaps_file() + # Note: smaps file can be empty for certain processes. + # The code below will not crash though and will result to 0. + uss = sum(map(int, _private_re.findall(smaps_data))) * 1024 + pss = sum(map(int, _pss_re.findall(smaps_data))) * 1024 + swap = sum(map(int, _swap_re.findall(smaps_data))) * 1024 + return pfullmem(*basic_mem + (uss, pss, swap)) + + else: + memory_full_info = memory_info + + if HAS_SMAPS: + + @wrap_exceptions + def memory_maps(self): + """Return process's mapped memory regions as a list of named + tuples. Fields are explained in 'man proc'; here is an updated + (Apr 2012) version: http://goo.gl/fmebo + + /proc/{PID}/smaps does not exist on kernels < 2.6.14 or if + CONFIG_MMU kernel configuration option is not enabled. + """ + def get_blocks(lines, current_block): + data = {} + for line in lines: + fields = line.split(None, 5) + if not fields[0].endswith(b':'): + # new block section + yield (current_block.pop(), data) + current_block.append(line) + else: + try: + data[fields[0]] = int(fields[1]) * 1024 + except ValueError: + if fields[0].startswith(b'VmFlags:'): + # see issue #369 + continue + else: + raise ValueError("don't know how to inte" + "rpret line %r" % line) + yield (current_block.pop(), data) + + data = self._read_smaps_file() + # Note: smaps file can be empty for certain processes. + if not data: + return [] + lines = data.split(b'\n') + ls = [] + first_line = lines.pop(0) + current_block = [first_line] + for header, data in get_blocks(lines, current_block): + hfields = header.split(None, 5) + try: + addr, perms, offset, dev, inode, path = hfields + except ValueError: + addr, perms, offset, dev, inode, path = \ + hfields + [''] + if not path: + path = '[anon]' + else: + if PY3: + path = decode(path) + path = path.strip() + if (path.endswith(' (deleted)') and not + path_exists_strict(path)): + path = path[:-10] + ls.append(( + decode(addr), decode(perms), path, + data[b'Rss:'], + data.get(b'Size:', 0), + data.get(b'Pss:', 0), + data.get(b'Shared_Clean:', 0), + data.get(b'Shared_Dirty:', 0), + data.get(b'Private_Clean:', 0), + data.get(b'Private_Dirty:', 0), + data.get(b'Referenced:', 0), + data.get(b'Anonymous:', 0), + data.get(b'Swap:', 0) + )) + return ls + + @wrap_exceptions + def cwd(self): + try: + return readlink("%s/%s/cwd" % (self._procfs_path, self.pid)) + except (FileNotFoundError, ProcessLookupError): + # https://github.com/giampaolo/psutil/issues/986 + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + + @wrap_exceptions + def num_ctx_switches(self, + _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)')): + data = self._read_status_file() + ctxsw = _ctxsw_re.findall(data) + if not ctxsw: + raise NotImplementedError( + "'voluntary_ctxt_switches' and 'nonvoluntary_ctxt_switches'" + "lines were not found in %s/%s/status; the kernel is " + "probably older than 2.6.23" % ( + self._procfs_path, self.pid)) + else: + return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) + + @wrap_exceptions + def num_threads(self, _num_threads_re=re.compile(br'Threads:\t(\d+)')): + # Note: on Python 3 using a re is faster than iterating over file + # line by line. On Python 2 is the exact opposite, and iterating + # over a file on Python 3 is slower than on Python 2. + data = self._read_status_file() + return int(_num_threads_re.findall(data)[0]) + + @wrap_exceptions + def threads(self): + thread_ids = os.listdir("%s/%s/task" % (self._procfs_path, self.pid)) + thread_ids.sort() + retlist = [] + hit_enoent = False + for thread_id in thread_ids: + fname = "%s/%s/task/%s/stat" % ( + self._procfs_path, self.pid, thread_id) + try: + with open_binary(fname) as f: + st = f.read().strip() + except FileNotFoundError: + # no such file or directory; it means thread + # disappeared on us + hit_enoent = True + continue + # ignore the first two values ("pid (exe)") + st = st[st.find(b')') + 2:] + values = st.split(b' ') + utime = float(values[11]) / CLOCK_TICKS + stime = float(values[12]) / CLOCK_TICKS + ntuple = _common.pthread(int(thread_id), utime, stime) + retlist.append(ntuple) + if hit_enoent: + self._assert_alive() + return retlist + + @wrap_exceptions + def nice_get(self): + # with open_text('%s/%s/stat' % (self._procfs_path, self.pid)) as f: + # data = f.read() + # return int(data.split()[18]) + + # Use C implementation + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + # starting from CentOS 6. + if HAS_CPU_AFFINITY: + + @wrap_exceptions + def cpu_affinity_get(self): + return cext.proc_cpu_affinity_get(self.pid) + + def _get_eligible_cpus( + self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")): + # See: https://github.com/giampaolo/psutil/issues/956 + data = self._read_status_file() + match = _re.findall(data) + if match: + return list(range(int(match[0][0]), int(match[0][1]) + 1)) + else: + return list(range(len(per_cpu_times()))) + + @wrap_exceptions + def cpu_affinity_set(self, cpus): + try: + cext.proc_cpu_affinity_set(self.pid, cpus) + except (OSError, ValueError) as err: + if isinstance(err, ValueError) or err.errno == errno.EINVAL: + eligible_cpus = self._get_eligible_cpus() + all_cpus = tuple(range(len(per_cpu_times()))) + for cpu in cpus: + if cpu not in all_cpus: + raise ValueError( + "invalid CPU number %r; choose between %s" % ( + cpu, eligible_cpus)) + if cpu not in eligible_cpus: + raise ValueError( + "CPU number %r is not eligible; choose " + "between %s" % (cpu, eligible_cpus)) + raise + + # only starting from kernel 2.6.13 + if HAS_PROC_IO_PRIORITY: + + @wrap_exceptions + def ionice_get(self): + ioclass, value = cext.proc_ioprio_get(self.pid) + if enum is not None: + ioclass = IOPriority(ioclass) + return _common.pionice(ioclass, value) + + @wrap_exceptions + def ionice_set(self, ioclass, value): + if value is None: + value = 0 + if value and ioclass in (IOPRIO_CLASS_IDLE, IOPRIO_CLASS_NONE): + raise ValueError("%r ioclass accepts no value" % ioclass) + if value < 0 or value > 7: + raise ValueError("value not in 0-7 range") + return cext.proc_ioprio_set(self.pid, ioclass, value) + + if HAS_PRLIMIT: + + @wrap_exceptions + def rlimit(self, resource, limits=None): + # If pid is 0 prlimit() applies to the calling process and + # we don't want that. We should never get here though as + # PID 0 is not supported on Linux. + if self.pid == 0: + raise ValueError("can't use prlimit() against PID 0 process") + try: + if limits is None: + # get + return cext.linux_prlimit(self.pid, resource) + else: + # set + if len(limits) != 2: + raise ValueError( + "second argument must be a (soft, hard) tuple, " + "got %s" % repr(limits)) + soft, hard = limits + cext.linux_prlimit(self.pid, resource, soft, hard) + except OSError as err: + if err.errno == errno.ENOSYS and pid_exists(self.pid): + # I saw this happening on Travis: + # https://travis-ci.org/giampaolo/psutil/jobs/51368273 + raise ZombieProcess(self.pid, self._name, self._ppid) + else: + raise + + @wrap_exceptions + def status(self): + letter = self._parse_stat_file()['status'] + if PY3: + letter = letter.decode() + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(letter, '?') + + @wrap_exceptions + def open_files(self): + retlist = [] + files = os.listdir("%s/%s/fd" % (self._procfs_path, self.pid)) + hit_enoent = False + for fd in files: + file = "%s/%s/fd/%s" % (self._procfs_path, self.pid, fd) + try: + path = readlink(file) + except (FileNotFoundError, ProcessLookupError): + # ENOENT == file which is gone in the meantime + hit_enoent = True + continue + except OSError as err: + if err.errno == errno.EINVAL: + # not a link + continue + raise + else: + # If path is not an absolute there's no way to tell + # whether it's a regular file or not, so we skip it. + # A regular file is always supposed to be have an + # absolute path though. + if path.startswith('/') and isfile_strict(path): + # Get file position and flags. + file = "%s/%s/fdinfo/%s" % ( + self._procfs_path, self.pid, fd) + try: + with open_binary(file) as f: + pos = int(f.readline().split()[1]) + flags = int(f.readline().split()[1], 8) + except FileNotFoundError: + # fd gone in the meantime; process may + # still be alive + hit_enoent = True + else: + mode = file_flags_to_mode(flags) + ntuple = popenfile( + path, int(fd), int(pos), mode, flags) + retlist.append(ntuple) + if hit_enoent: + self._assert_alive() + return retlist + + @wrap_exceptions + def connections(self, kind='inet'): + ret = _connections.retrieve(kind, self.pid) + self._assert_alive() + return ret + + @wrap_exceptions + def num_fds(self): + return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + + @wrap_exceptions + def ppid(self): + return int(self._parse_stat_file()['ppid']) + + @wrap_exceptions + def uids(self, _uids_re=re.compile(br'Uid:\t(\d+)\t(\d+)\t(\d+)')): + data = self._read_status_file() + real, effective, saved = _uids_re.findall(data)[0] + return _common.puids(int(real), int(effective), int(saved)) + + @wrap_exceptions + def gids(self, _gids_re=re.compile(br'Gid:\t(\d+)\t(\d+)\t(\d+)')): + data = self._read_status_file() + real, effective, saved = _gids_re.findall(data)[0] + return _common.pgids(int(real), int(effective), int(saved)) diff --git a/ddtrace/vendor/psutil/_psosx.py b/ddtrace/vendor/psutil/_psosx.py new file mode 100644 index 00000000000..7f28447bb18 --- /dev/null +++ b/ddtrace/vendor/psutil/_psosx.py @@ -0,0 +1,568 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""macOS platform implementation.""" + +import contextlib +import errno +import functools +import os +from collections import namedtuple + +from . import _common +from . import _psposix +from . import _psutil_osx as cext +from . import _psutil_posix as cext_posix +from ._common import conn_tmap +from ._common import conn_to_ntuple +from ._common import isfile_strict +from ._common import memoize_when_activated +from ._common import parse_environ_block +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._common import usage_percent + + +__extra__all__ = [] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +PAGESIZE = os.sysconf("SC_PAGE_SIZE") +AF_LINK = cext_posix.AF_LINK + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SRUN: _common.STATUS_RUNNING, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SZOMB: _common.STATUS_ZOMBIE, +} + +kinfo_proc_map = dict( + ppid=0, + ruid=1, + euid=2, + suid=3, + rgid=4, + egid=5, + sgid=6, + ttynr=7, + ctime=8, + status=9, + name=10, +) + +pidtaskinfo_map = dict( + cpuutime=0, + cpustime=1, + rss=2, + vms=3, + pfaults=4, + pageins=5, + numthreads=6, + volctxsw=7, +) + +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'nice', 'system', 'idle']) +# psutil.virtual_memory() +svmem = namedtuple( + 'svmem', ['total', 'available', 'percent', 'used', 'free', + 'active', 'inactive', 'wired']) +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms', 'pfaults', 'pageins']) +# psutil.Process.memory_full_info() +pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """System virtual memory as a namedtuple.""" + total, active, inactive, wired, free, speculative = cext.virtual_mem() + # This is how Zabbix calculate avail and used mem: + # https://github.com/zabbix/zabbix/blob/trunk/src/libs/zbxsysinfo/ + # osx/memory.c + # Also see: https://github.com/giampaolo/psutil/issues/1277 + avail = inactive + free + used = active + wired + # This is NOT how Zabbix calculates free mem but it matches "free" + # cmdline utility. + free -= speculative + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, used, free, + active, inactive, wired) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + total, used, free, sin, sout = cext.swap_mem() + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system CPU times as a namedtuple.""" + user, nice, system, idle = cext.cpu_times() + return scputimes(user, nice, system, idle) + + +def per_cpu_times(): + """Return system CPU times as a named tuple""" + ret = [] + for cpu_t in cext.per_cpu_times(): + user, nice, system, idle = cpu_t + item = scputimes(user, nice, system, idle) + ret.append(item) + return ret + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + return cext.cpu_count_logical() + + +def cpu_count_physical(): + """Return the number of physical CPUs in the system.""" + return cext.cpu_count_phys() + + +def cpu_stats(): + ctx_switches, interrupts, soft_interrupts, syscalls, traps = \ + cext.cpu_stats() + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls) + + +def cpu_freq(): + """Return CPU frequency. + On macOS per-cpu frequency is not supported. + Also, the returned frequency never changes, see: + https://arstechnica.com/civis/viewtopic.php?f=19&t=465002 + """ + curr, min_, max_ = cext.cpu_freq() + return [_common.scpufreq(curr, min_, max_)] + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_usage = _psposix.disk_usage +disk_io_counters = cext.disk_io_counters + + +def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples.""" + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + if not os.path.isabs(device) or not os.path.exists(device): + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + """Return battery information.""" + try: + percent, minsleft, power_plugged = cext.sensors_battery() + except NotImplementedError: + # no power source - return None according to interface + return None + power_plugged = power_plugged == 1 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft, power_plugged) + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_io_counters = cext.net_io_counters +net_if_addrs = cext_posix.net_if_addrs + + +def net_connections(kind='inet'): + """System-wide network connections.""" + # Note: on macOS this will fail with AccessDenied unless + # the process is owned by root. + ret = [] + for pid in pids(): + try: + cons = Process(pid).connections(kind) + except NoSuchProcess: + continue + else: + if cons: + for c in cons: + c = list(c) + [pid] + ret.append(_common.sconn(*c)) + return ret + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + names = net_io_counters().keys() + ret = {} + for name in names: + try: + mtu = cext_posix.net_if_mtu(name) + isup = cext_posix.net_if_flags(name) + duplex, speed = cext_posix.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, pid = item + if tty == '~': + continue # reboot or shutdown + if not tstamp: + continue + nt = _common.suser(user, tty or None, hostname or None, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + ls = cext.pids() + if 0 not in ls: + # On certain macOS versions pids() C doesn't return PID 0 but + # "ps" does and the process is querable via sysctl(): + # https://travis-ci.org/giampaolo/psutil/jobs/309619941 + try: + Process(0).create_time() + ls.insert(0, 0) + except NoSuchProcess: + pass + except AccessDenied: + ls.insert(0, 0) + return ls + + +pid_exists = _psposix.pid_exists + + +def wrap_exceptions(fun): + """Decorator which translates bare OSError exceptions into + NoSuchProcess and AccessDenied. + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except ProcessLookupError: + raise NoSuchProcess(self.pid, self._name) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except cext.ZombieProcessError: + raise ZombieProcess(self.pid, self._name, self._ppid) + return wrapper + + +@contextlib.contextmanager +def catch_zombie(proc): + """There are some poor C APIs which incorrectly raise ESRCH when + the process is still alive or it's a zombie, or even RuntimeError + (those who don't set errno). This is here in order to solve: + https://github.com/giampaolo/psutil/issues/1044 + """ + try: + yield + except (OSError, RuntimeError) as err: + if isinstance(err, RuntimeError) or err.errno == errno.ESRCH: + try: + # status() is not supposed to lie and correctly detect + # zombies so if it raises ESRCH it's true. + status = proc.status() + except NoSuchProcess: + raise err + else: + if status == _common.STATUS_ZOMBIE: + raise ZombieProcess(proc.pid, proc._name, proc._ppid) + else: + raise AccessDenied(proc.pid, proc._name) + else: + raise + + +class Process(object): + """Wrapper class around underlying C implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_cache"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + + @wrap_exceptions + @memoize_when_activated + def _get_kinfo_proc(self): + # Note: should work with all PIDs without permission issues. + ret = cext.proc_kinfo_oneshot(self.pid) + assert len(ret) == len(kinfo_proc_map) + return ret + + @wrap_exceptions + @memoize_when_activated + def _get_pidtaskinfo(self): + # Note: should work for PIDs owned by user only. + with catch_zombie(self): + ret = cext.proc_pidtaskinfo_oneshot(self.pid) + assert len(ret) == len(pidtaskinfo_map) + return ret + + def oneshot_enter(self): + self._get_kinfo_proc.cache_activate(self) + self._get_pidtaskinfo.cache_activate(self) + + def oneshot_exit(self): + self._get_kinfo_proc.cache_deactivate(self) + self._get_pidtaskinfo.cache_deactivate(self) + + @wrap_exceptions + def name(self): + name = self._get_kinfo_proc()[kinfo_proc_map['name']] + return name if name is not None else cext.proc_name(self.pid) + + @wrap_exceptions + def exe(self): + with catch_zombie(self): + return cext.proc_exe(self.pid) + + @wrap_exceptions + def cmdline(self): + with catch_zombie(self): + return cext.proc_cmdline(self.pid) + + @wrap_exceptions + def environ(self): + with catch_zombie(self): + return parse_environ_block(cext.proc_environ(self.pid)) + + @wrap_exceptions + def ppid(self): + self._ppid = self._get_kinfo_proc()[kinfo_proc_map['ppid']] + return self._ppid + + @wrap_exceptions + def cwd(self): + with catch_zombie(self): + return cext.proc_cwd(self.pid) + + @wrap_exceptions + def uids(self): + rawtuple = self._get_kinfo_proc() + return _common.puids( + rawtuple[kinfo_proc_map['ruid']], + rawtuple[kinfo_proc_map['euid']], + rawtuple[kinfo_proc_map['suid']]) + + @wrap_exceptions + def gids(self): + rawtuple = self._get_kinfo_proc() + return _common.puids( + rawtuple[kinfo_proc_map['rgid']], + rawtuple[kinfo_proc_map['egid']], + rawtuple[kinfo_proc_map['sgid']]) + + @wrap_exceptions + def terminal(self): + tty_nr = self._get_kinfo_proc()[kinfo_proc_map['ttynr']] + tmap = _psposix.get_terminal_map() + try: + return tmap[tty_nr] + except KeyError: + return None + + @wrap_exceptions + def memory_info(self): + rawtuple = self._get_pidtaskinfo() + return pmem( + rawtuple[pidtaskinfo_map['rss']], + rawtuple[pidtaskinfo_map['vms']], + rawtuple[pidtaskinfo_map['pfaults']], + rawtuple[pidtaskinfo_map['pageins']], + ) + + @wrap_exceptions + def memory_full_info(self): + basic_mem = self.memory_info() + uss = cext.proc_memory_uss(self.pid) + return pfullmem(*basic_mem + (uss, )) + + @wrap_exceptions + def cpu_times(self): + rawtuple = self._get_pidtaskinfo() + return _common.pcputimes( + rawtuple[pidtaskinfo_map['cpuutime']], + rawtuple[pidtaskinfo_map['cpustime']], + # children user / system times are not retrievable (set to 0) + 0.0, 0.0) + + @wrap_exceptions + def create_time(self): + return self._get_kinfo_proc()[kinfo_proc_map['ctime']] + + @wrap_exceptions + def num_ctx_switches(self): + # Unvoluntary value seems not to be available; + # getrusage() numbers seems to confirm this theory. + # We set it to 0. + vol = self._get_pidtaskinfo()[pidtaskinfo_map['volctxsw']] + return _common.pctxsw(vol, 0) + + @wrap_exceptions + def num_threads(self): + return self._get_pidtaskinfo()[pidtaskinfo_map['numthreads']] + + @wrap_exceptions + def open_files(self): + if self.pid == 0: + return [] + files = [] + with catch_zombie(self): + rawlist = cext.proc_open_files(self.pid) + for path, fd in rawlist: + if isfile_strict(path): + ntuple = _common.popenfile(path, fd) + files.append(ntuple) + return files + + @wrap_exceptions + def connections(self, kind='inet'): + if kind not in conn_tmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in conn_tmap]))) + families, types = conn_tmap[kind] + with catch_zombie(self): + rawlist = cext.proc_connections(self.pid, families, types) + ret = [] + for item in rawlist: + fd, fam, type, laddr, raddr, status = item + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) + ret.append(nt) + return ret + + @wrap_exceptions + def num_fds(self): + if self.pid == 0: + return 0 + with catch_zombie(self): + return cext.proc_num_fds(self.pid) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + @wrap_exceptions + def nice_get(self): + with catch_zombie(self): + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + with catch_zombie(self): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def status(self): + code = self._get_kinfo_proc()[kinfo_proc_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + return retlist diff --git a/ddtrace/vendor/psutil/_psposix.py b/ddtrace/vendor/psutil/_psposix.py new file mode 100644 index 00000000000..24570224fe9 --- /dev/null +++ b/ddtrace/vendor/psutil/_psposix.py @@ -0,0 +1,179 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Routines common to all posix systems.""" + +import glob +import os +import sys +import time + +from ._common import memoize +from ._common import sdiskusage +from ._common import usage_percent +from ._compat import ChildProcessError +from ._compat import FileNotFoundError +from ._compat import InterruptedError +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._compat import PY3 +from ._compat import unicode + + +__all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map'] + + +# This object gets set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +TimeoutExpired = None + + +def pid_exists(pid): + """Check whether pid exists in the current process table.""" + if pid == 0: + # According to "man 2 kill" PID 0 has a special meaning: + # it refers to <> so we don't want to go any further. + # If we get here it means this UNIX platform *does* have + # a process with id 0. + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + # EPERM clearly means there's a process to deny access to + return True + # According to "man 2 kill" possible error values are + # (EINVAL, EPERM, ESRCH) + else: + return True + + +def wait_pid(pid, timeout=None, proc_name=None): + """Wait for process with pid 'pid' to terminate and return its + exit status code as an integer. + + If pid is not a children of os.getpid() (current process) just + waits until the process disappears and return None. + + If pid does not exist at all return None immediately. + + Raise TimeoutExpired on timeout expired. + """ + def check_timeout(delay): + if timeout is not None: + if timer() >= stop_at: + raise TimeoutExpired(timeout, pid=pid, name=proc_name) + time.sleep(delay) + return min(delay * 2, 0.04) + + timer = getattr(time, 'monotonic', time.time) + if timeout is not None: + def waitcall(): + return os.waitpid(pid, os.WNOHANG) + stop_at = timer() + timeout + else: + def waitcall(): + return os.waitpid(pid, 0) + + delay = 0.0001 + while True: + try: + retpid, status = waitcall() + except InterruptedError: + delay = check_timeout(delay) + except ChildProcessError: + # This has two meanings: + # - pid is not a child of os.getpid() in which case + # we keep polling until it's gone + # - pid never existed in the first place + # In both cases we'll eventually return None as we + # can't determine its exit status code. + while True: + if pid_exists(pid): + delay = check_timeout(delay) + else: + return + else: + if retpid == 0: + # WNOHANG was used, pid is still running + delay = check_timeout(delay) + continue + # process exited due to a signal; return the integer of + # that signal + if os.WIFSIGNALED(status): + return -os.WTERMSIG(status) + # process exited using exit(2) system call; return the + # integer exit(2) system call has been called with + elif os.WIFEXITED(status): + return os.WEXITSTATUS(status) + else: + # should never happen + raise ValueError("unknown process exit status %r" % status) + + +def disk_usage(path): + """Return disk usage associated with path. + Note: UNIX usually reserves 5% disk space which is not accessible + by user. In this function "total" and "used" values reflect the + total and used disk space whereas "free" and "percent" represent + the "free" and "used percent" user disk space. + """ + if PY3: + st = os.statvfs(path) + else: + # os.statvfs() does not support unicode on Python 2: + # - https://github.com/giampaolo/psutil/issues/416 + # - http://bugs.python.org/issue18695 + try: + st = os.statvfs(path) + except UnicodeEncodeError: + if isinstance(path, unicode): + try: + path = path.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + pass + st = os.statvfs(path) + else: + raise + + # Total space which is only available to root (unless changed + # at system level). + total = (st.f_blocks * st.f_frsize) + # Remaining free space usable by root. + avail_to_root = (st.f_bfree * st.f_frsize) + # Remaining free space usable by user. + avail_to_user = (st.f_bavail * st.f_frsize) + # Total space being used in general. + used = (total - avail_to_root) + # Total space which is available to user (same as 'total' but + # for the user). + total_user = used + avail_to_user + # User usage percent compared to the total amount of space + # the user can use. This number would be higher if compared + # to root's because the user has less space (usually -5%). + usage_percent_user = usage_percent(used, total_user, round_=1) + + # NB: the percentage is -5% than what shown by df due to + # reserved blocks that we are currently not considering: + # https://github.com/giampaolo/psutil/issues/829#issuecomment-223750462 + return sdiskusage( + total=total, used=used, free=avail_to_user, percent=usage_percent_user) + + +@memoize +def get_terminal_map(): + """Get a map of device-id -> path as a dict. + Used by Process.terminal() + """ + ret = {} + ls = glob.glob('/dev/tty*') + glob.glob('/dev/pts/*') + for name in ls: + assert name not in ret, name + try: + ret[os.stat(name).st_rdev] = name + except FileNotFoundError: + pass + return ret diff --git a/ddtrace/vendor/psutil/_pssunos.py b/ddtrace/vendor/psutil/_pssunos.py new file mode 100644 index 00000000000..2aa2a86615d --- /dev/null +++ b/ddtrace/vendor/psutil/_pssunos.py @@ -0,0 +1,720 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sun OS Solaris platform implementation.""" + +import errno +import functools +import os +import socket +import subprocess +import sys +from collections import namedtuple +from socket import AF_INET + +from . import _common +from . import _psposix +from . import _psutil_posix as cext_posix +from . import _psutil_sunos as cext +from ._common import AF_INET6 +from ._common import get_procfs_path +from ._common import isfile_strict +from ._common import memoize_when_activated +from ._common import sockfam_to_enum +from ._common import socktype_to_enum +from ._common import usage_percent +from ._compat import b +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError +from ._compat import PY3 + + +__extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') +AF_LINK = cext_posix.AF_LINK +IS_64_BIT = sys.maxsize > 2**32 + +CONN_IDLE = "IDLE" +CONN_BOUND = "BOUND" + +PROC_STATUSES = { + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SRUN: _common.STATUS_RUNNING, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SIDL: _common.STATUS_IDLE, + cext.SONPROC: _common.STATUS_RUNNING, # same as run + cext.SWAIT: _common.STATUS_WAITING, +} + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, + cext.TCPS_IDLE: CONN_IDLE, # sunos specific + cext.TCPS_BOUND: CONN_BOUND, # sunos specific +} + +proc_info_map = dict( + ppid=0, + rss=1, + vms=2, + create_time=3, + nice=4, + num_threads=5, + status=6, + ttynr=7, + uid=8, + euid=9, + gid=10, + egid=11) + +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) +# psutil.cpu_times(percpu=True) +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system']) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms']) +pfullmem = pmem +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple('pmmap_grouped', + ['path', 'rss', 'anonymous', 'locked']) +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """Report virtual memory metrics.""" + # we could have done this with kstat, but IMHO this is good enough + total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE + # note: there's no difference on Solaris + free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE + used = total - free + percent = usage_percent(used, total, round_=1) + return svmem(total, avail, percent, used, free) + + +def swap_memory(): + """Report swap memory metrics.""" + sin, sout = cext.swap_mem() + # XXX + # we are supposed to get total/free by doing so: + # http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/ + # usr/src/cmd/swap/swap.c + # ...nevertheless I can't manage to obtain the same numbers as 'swap' + # cmdline utility, so let's parse its output (sigh!) + p = subprocess.Popen(['/usr/bin/env', 'PATH=/usr/sbin:/sbin:%s' % + os.environ['PATH'], 'swap', '-l'], + stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout = stdout.decode(sys.stdout.encoding) + if p.returncode != 0: + raise RuntimeError("'swap -l' failed (retcode=%s)" % p.returncode) + + lines = stdout.strip().split('\n')[1:] + if not lines: + raise RuntimeError('no swap device(s) configured') + total = free = 0 + for line in lines: + line = line.split() + t, f = line[-2:] + total += int(int(t) * 512) + free += int(int(f) * 512) + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, + sin * PAGE_SIZE, sout * PAGE_SIZE) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system-wide CPU times as a named tuple""" + ret = cext.per_cpu_times() + return scputimes(*[sum(x) for x in zip(*ret)]) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples""" + ret = cext.per_cpu_times() + return [scputimes(*x) for x in ret] + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # mimic os.cpu_count() behavior + return None + + +def cpu_count_physical(): + """Return the number of physical CPUs in the system.""" + return cext.cpu_count_phys() + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + ctx_switches, interrupts, syscalls, traps = cext.cpu_stats() + soft_interrupts = 0 + return _common.scpustats(ctx_switches, interrupts, soft_interrupts, + syscalls) + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters +disk_usage = _psposix.disk_usage + + +def disk_partitions(all=False): + """Return system disk partitions.""" + # TODO - the filtering logic should be better checked so that + # it tries to reflect 'df' as much as possible + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + # Differently from, say, Linux, we don't have a list of + # common fs types so the best we can do, AFAIK, is to + # filter by filesystem having a total size > 0. + if not disk_usage(mountpoint).total: + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_io_counters = cext.net_io_counters +net_if_addrs = cext_posix.net_if_addrs + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + Only INET sockets are returned (UNIX are not). + """ + cmap = _common.conn_tmap.copy() + if _pid == -1: + cmap.pop('unix', 0) + if kind not in cmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in cmap]))) + families, types = _common.conn_tmap[kind] + rawlist = cext.net_connections(_pid) + ret = set() + for item in rawlist: + fd, fam, type_, laddr, raddr, status, pid = item + if fam not in families: + continue + if type_ not in types: + continue + # TODO: refactor and use _common.conn_to_ntuple. + if fam in (AF_INET, AF_INET6): + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) + status = TCP_STATUSES[status] + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if _pid == -1: + nt = _common.sconn(fd, fam, type_, laddr, raddr, status, pid) + else: + nt = _common.pconn(fd, fam, type_, laddr, raddr, status) + ret.add(nt) + return list(ret) + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + ret = cext.net_if_stats() + for name, items in ret.items(): + isup, duplex, speed, mtu = items + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + localhost = (':0.0', ':0') + for item in rawlist: + user, tty, hostname, tstamp, user_process, pid = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname in localhost: + hostname = 'localhost' + nt = _common.suser(user, tty, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + return [int(x) for x in os.listdir(b(get_procfs_path())) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix pid.""" + return _psposix.pid_exists(pid) + + +def wrap_exceptions(fun): + """Call callable into a try/except clause and translate ENOENT, + EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except (FileNotFoundError, ProcessLookupError): + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except OSError: + if self.pid == 0: + if 0 in pids(): + raise AccessDenied(self.pid, self._name) + else: + raise + raise + return wrapper + + +class Process(object): + """Wrapper class around underlying C implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + os.stat('%s/%s' % (self._procfs_path, self.pid)) + + def oneshot_enter(self): + self._proc_name_and_args.cache_activate(self) + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) + + def oneshot_exit(self): + self._proc_name_and_args.cache_deactivate(self) + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) + + @wrap_exceptions + @memoize_when_activated + def _proc_name_and_args(self): + return cext.proc_name_and_args(self.pid, self._procfs_path) + + @wrap_exceptions + @memoize_when_activated + def _proc_basic_info(self): + ret = cext.proc_basic_info(self.pid, self._procfs_path) + assert len(ret) == len(proc_info_map) + return ret + + @wrap_exceptions + @memoize_when_activated + def _proc_cred(self): + return cext.proc_cred(self.pid, self._procfs_path) + + @wrap_exceptions + def name(self): + # note: max len == 15 + return self._proc_name_and_args()[0] + + @wrap_exceptions + def exe(self): + try: + return os.readlink( + "%s/%s/path/a.out" % (self._procfs_path, self.pid)) + except OSError: + pass # continue and guess the exe name from the cmdline + # Will be guessed later from cmdline but we want to explicitly + # invoke cmdline here in order to get an AccessDenied + # exception if the user has not enough privileges. + self.cmdline() + return "" + + @wrap_exceptions + def cmdline(self): + return self._proc_name_and_args()[1].split(' ') + + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid, self._procfs_path) + + @wrap_exceptions + def create_time(self): + return self._proc_basic_info()[proc_info_map['create_time']] + + @wrap_exceptions + def num_threads(self): + return self._proc_basic_info()[proc_info_map['num_threads']] + + @wrap_exceptions + def nice_get(self): + # Note #1: getpriority(3) doesn't work for realtime processes. + # Psinfo is what ps uses, see: + # https://github.com/giampaolo/psutil/issues/1194 + return self._proc_basic_info()[proc_info_map['nice']] + + @wrap_exceptions + def nice_set(self, value): + if self.pid in (2, 3): + # Special case PIDs: internally setpriority(3) return ESRCH + # (no such process), no matter what. + # The process actually exists though, as it has a name, + # creation time, etc. + raise AccessDenied(self.pid, self._name) + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def ppid(self): + self._ppid = self._proc_basic_info()[proc_info_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + try: + real, effective, saved, _, _, _ = self._proc_cred() + except AccessDenied: + real = self._proc_basic_info()[proc_info_map['uid']] + effective = self._proc_basic_info()[proc_info_map['euid']] + saved = None + return _common.puids(real, effective, saved) + + @wrap_exceptions + def gids(self): + try: + _, _, _, real, effective, saved = self._proc_cred() + except AccessDenied: + real = self._proc_basic_info()[proc_info_map['gid']] + effective = self._proc_basic_info()[proc_info_map['egid']] + saved = None + return _common.puids(real, effective, saved) + + @wrap_exceptions + def cpu_times(self): + try: + times = cext.proc_cpu_times(self.pid, self._procfs_path) + except OSError as err: + if err.errno == errno.EOVERFLOW and not IS_64_BIT: + # We may get here if we attempt to query a 64bit process + # with a 32bit python. + # Error originates from read() and also tools like "cat" + # fail in the same way (!). + # Since there simply is no way to determine CPU times we + # return 0.0 as a fallback. See: + # https://github.com/giampaolo/psutil/issues/857 + times = (0.0, 0.0, 0.0, 0.0) + else: + raise + return _common.pcputimes(*times) + + @wrap_exceptions + def cpu_num(self): + return cext.proc_cpu_num(self.pid, self._procfs_path) + + @wrap_exceptions + def terminal(self): + procfs_path = self._procfs_path + hit_enoent = False + tty = wrap_exceptions( + self._proc_basic_info()[proc_info_map['ttynr']]) + if tty != cext.PRNODEV: + for x in (0, 1, 2, 255): + try: + return os.readlink( + '%s/%d/path/%d' % (procfs_path, self.pid, x)) + except FileNotFoundError: + hit_enoent = True + continue + if hit_enoent: + self._assert_alive() + + @wrap_exceptions + def cwd(self): + # /proc/PID/path/cwd may not be resolved by readlink() even if + # it exists (ls shows it). If that's the case and the process + # is still alive return None (we can return None also on BSD). + # Reference: http://goo.gl/55XgO + procfs_path = self._procfs_path + try: + return os.readlink("%s/%s/path/cwd" % (procfs_path, self.pid)) + except FileNotFoundError: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None + + @wrap_exceptions + def memory_info(self): + ret = self._proc_basic_info() + rss = ret[proc_info_map['rss']] * 1024 + vms = ret[proc_info_map['vms']] * 1024 + return pmem(rss, vms) + + memory_full_info = memory_info + + @wrap_exceptions + def status(self): + code = self._proc_basic_info()[proc_info_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + @wrap_exceptions + def threads(self): + procfs_path = self._procfs_path + ret = [] + tids = os.listdir('%s/%d/lwp' % (procfs_path, self.pid)) + hit_enoent = False + for tid in tids: + tid = int(tid) + try: + utime, stime = cext.query_process_thread( + self.pid, tid, procfs_path) + except EnvironmentError as err: + if err.errno == errno.EOVERFLOW and not IS_64_BIT: + # We may get here if we attempt to query a 64bit process + # with a 32bit python. + # Error originates from read() and also tools like "cat" + # fail in the same way (!). + # Since there simply is no way to determine CPU times we + # return 0.0 as a fallback. See: + # https://github.com/giampaolo/psutil/issues/857 + continue + # ENOENT == thread gone in meantime + if err.errno == errno.ENOENT: + hit_enoent = True + continue + raise + else: + nt = _common.pthread(tid, utime, stime) + ret.append(nt) + if hit_enoent: + self._assert_alive() + return ret + + @wrap_exceptions + def open_files(self): + retlist = [] + hit_enoent = False + procfs_path = self._procfs_path + pathdir = '%s/%d/path' % (procfs_path, self.pid) + for fd in os.listdir('%s/%d/fd' % (procfs_path, self.pid)): + path = os.path.join(pathdir, fd) + if os.path.islink(path): + try: + file = os.readlink(path) + except FileNotFoundError: + hit_enoent = True + continue + else: + if isfile_strict(file): + retlist.append(_common.popenfile(file, int(fd))) + if hit_enoent: + self._assert_alive() + return retlist + + def _get_unix_sockets(self, pid): + """Get UNIX sockets used by process by parsing 'pfiles' output.""" + # TODO: rewrite this in C (...but the damn netstat source code + # does not include this part! Argh!!) + cmd = "pfiles %s" % pid + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if p.returncode != 0: + if 'permission denied' in stderr.lower(): + raise AccessDenied(self.pid, self._name) + if 'no such process' in stderr.lower(): + raise NoSuchProcess(self.pid, self._name) + raise RuntimeError("%r command error\n%s" % (cmd, stderr)) + + lines = stdout.split('\n')[2:] + for i, line in enumerate(lines): + line = line.lstrip() + if line.startswith('sockname: AF_UNIX'): + path = line.split(' ', 2)[2] + type = lines[i - 2].strip() + if type == 'SOCK_STREAM': + type = socket.SOCK_STREAM + elif type == 'SOCK_DGRAM': + type = socket.SOCK_DGRAM + else: + type = -1 + yield (-1, socket.AF_UNIX, type, path, "", _common.CONN_NONE) + + @wrap_exceptions + def connections(self, kind='inet'): + ret = net_connections(kind, _pid=self.pid) + # The underlying C implementation retrieves all OS connections + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not ret: + # will raise NSP if process is gone + os.stat('%s/%s' % (self._procfs_path, self.pid)) + + # UNIX sockets + if kind in ('all', 'unix'): + ret.extend([_common.pconn(*conn) for conn in + self._get_unix_sockets(self.pid)]) + return ret + + nt_mmap_grouped = namedtuple('mmap', 'path rss anon locked') + nt_mmap_ext = namedtuple('mmap', 'addr perms path rss anon locked') + + @wrap_exceptions + def memory_maps(self): + def toaddr(start, end): + return '%s-%s' % (hex(start)[2:].strip('L'), + hex(end)[2:].strip('L')) + + procfs_path = self._procfs_path + retlist = [] + try: + rawlist = cext.proc_memory_maps(self.pid, procfs_path) + except OSError as err: + if err.errno == errno.EOVERFLOW and not IS_64_BIT: + # We may get here if we attempt to query a 64bit process + # with a 32bit python. + # Error originates from read() and also tools like "cat" + # fail in the same way (!). + # Since there simply is no way to determine CPU times we + # return 0.0 as a fallback. See: + # https://github.com/giampaolo/psutil/issues/857 + return [] + else: + raise + hit_enoent = False + for item in rawlist: + addr, addrsize, perm, name, rss, anon, locked = item + addr = toaddr(addr, addrsize) + if not name.startswith('['): + try: + name = os.readlink( + '%s/%s/path/%s' % (procfs_path, self.pid, name)) + except OSError as err: + if err.errno == errno.ENOENT: + # sometimes the link may not be resolved by + # readlink() even if it exists (ls shows it). + # If that's the case we just return the + # unresolved link path. + # This seems an incosistency with /proc similar + # to: http://goo.gl/55XgO + name = '%s/%s/path/%s' % (procfs_path, self.pid, name) + hit_enoent = True + else: + raise + retlist.append((addr, perm, name, rss, anon, locked)) + if hit_enoent: + self._assert_alive() + return retlist + + @wrap_exceptions + def num_fds(self): + return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + + @wrap_exceptions + def num_ctx_switches(self): + return _common.pctxsw( + *cext.proc_num_ctx_switches(self.pid, self._procfs_path)) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) diff --git a/ddtrace/vendor/psutil/_psutil_aix.c b/ddtrace/vendor/psutil/_psutil_aix.c new file mode 100644 index 00000000000..9f58f606b0c --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_aix.c @@ -0,0 +1,1137 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola' + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* + * AIX support is experimental at this time. + * The following functions and methods are unsupported on the AIX platform: + * - psutil.Process.memory_maps + * + * Known limitations: + * - psutil.Process.io_counters read count is always 0 + * - psutil.Process.io_counters may not be available on older AIX versions + * - psutil.Process.threads may not be available on older AIX versions + # - psutil.net_io_counters may not be available on older AIX versions + * - reading basic process info may fail or return incorrect values when + * process is starting (see IBM APAR IV58499 - fixed in newer AIX versions) + * - sockets and pipes may not be counted in num_fds (fixed in newer AIX + * versions) + * + * Useful resources: + * - proc filesystem: http://www-01.ibm.com/support/knowledgecenter/ + * ssw_aix_72/com.ibm.aix.files/proc.htm + * - libperfstat: http://www-01.ibm.com/support/knowledgecenter/ + * ssw_aix_72/com.ibm.aix.files/libperfstat.h.htm + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arch/aix/ifaddrs.h" +#include "arch/aix/net_connections.h" +#include "arch/aix/common.h" +#include "_psutil_common.h" +#include "_psutil_posix.h" + + +#define TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) + +/* + * Read a file content and fills a C structure with it. + */ +int +psutil_file_to_struct(char *path, void *fstruct, size_t size) { + int fd; + size_t nbytes; + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + return 0; + } + nbytes = read(fd, fstruct, size); + if (nbytes <= 0) { + close(fd); + PyErr_SetFromErrno(PyExc_OSError); + return 0; + } + if (nbytes != size) { + close(fd); + PyErr_SetString(PyExc_RuntimeError, "structure size mismatch"); + return 0; + } + close(fd); + return nbytes; +} + + +/* + * Return process ppid, rss, vms, ctime, nice, nthreads, status and tty + * as a Python tuple. + */ +static PyObject * +psutil_proc_basic_info(PyObject *self, PyObject *args) { + int pid; + char path[100]; + psinfo_t info; + pstatus_t status; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + + if (info.pr_nlwp == 0 && info.pr_lwp.pr_lwpid == 0) { + // From the /proc docs: "If the process is a zombie, the pr_nlwp + // and pr_lwp.pr_lwpid flags are zero." + status.pr_stat = SZOMB; + } else if (info.pr_flag & SEXIT) { + // "exiting" processes don't have /proc//status + // There are other "exiting" processes that 'ps' shows as "active" + status.pr_stat = SACTIVE; + } else { + sprintf(path, "%s/%i/status", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&status, sizeof(status))) + return NULL; + } + + return Py_BuildValue("KKKdiiiK", + (unsigned long long) info.pr_ppid, // parent pid + (unsigned long long) info.pr_rssize, // rss + (unsigned long long) info.pr_size, // vms + TV2DOUBLE(info.pr_start), // create time + (int) info.pr_lwp.pr_nice, // nice + (int) info.pr_nlwp, // no. of threads + (int) status.pr_stat, // status code + (unsigned long long)info.pr_ttydev // tty nr + ); +} + + +/* + * Return process name as a Python string. + */ +static PyObject * +psutil_proc_name(PyObject *self, PyObject *args) { + int pid; + char path[100]; + psinfo_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + + return PyUnicode_DecodeFSDefaultAndSize(info.pr_fname, PRFNSZ); +} + + +/* + * Return process command line arguments as a Python list + */ +static PyObject * +psutil_proc_args(PyObject *self, PyObject *args) { + int pid; + PyObject *py_retlist = PyList_New(0); + PyObject *py_arg = NULL; + struct procsinfo procbuf; + long arg_max; + char *argbuf = NULL; + char *curarg = NULL; + int ret; + + if (py_retlist == NULL) + return NULL; + if (!PyArg_ParseTuple(args, "i", &pid)) + goto error; + arg_max = sysconf(_SC_ARG_MAX); + argbuf = malloc(arg_max); + if (argbuf == NULL) { + PyErr_NoMemory(); + goto error; + } + + procbuf.pi_pid = pid; + ret = getargs(&procbuf, sizeof(procbuf), argbuf, ARG_MAX); + if (ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + curarg = argbuf; + /* getargs will always append an extra NULL to end the arg list, + * even if the buffer is not big enough (even though it is supposed + * to be) so the following 'while' is safe */ + while (*curarg != '\0') { + py_arg = PyUnicode_DecodeFSDefault(curarg); + if (!py_arg) + goto error; + if (PyList_Append(py_retlist, py_arg)) + goto error; + Py_DECREF(py_arg); + curarg = strchr(curarg, '\0') + 1; + } + + free(argbuf); + + return py_retlist; + +error: + if (argbuf != NULL) + free(argbuf); + Py_XDECREF(py_retlist); + Py_XDECREF(py_arg); + return NULL; +} + + +/* + * Return process environment variables as a Python dict + */ +static PyObject * +psutil_proc_environ(PyObject *self, PyObject *args) { + int pid; + PyObject *py_retdict = PyDict_New(); + PyObject *py_key = NULL; + PyObject *py_val = NULL; + struct procsinfo procbuf; + long env_max; + char *envbuf = NULL; + char *curvar = NULL; + char *separator = NULL; + int ret; + + if (py_retdict == NULL) + return NULL; + if (!PyArg_ParseTuple(args, "i", &pid)) + goto error; + env_max = sysconf(_SC_ARG_MAX); + envbuf = malloc(env_max); + if (envbuf == NULL) { + PyErr_NoMemory(); + goto error; + } + + procbuf.pi_pid = pid; + ret = getevars(&procbuf, sizeof(procbuf), envbuf, ARG_MAX); + if (ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + curvar = envbuf; + /* getevars will always append an extra NULL to end the arg list, + * even if the buffer is not big enough (even though it is supposed + * to be) so the following 'while' is safe */ + while (*curvar != '\0') { + separator = strchr(curvar, '='); + if (separator != NULL) { + py_key = PyUnicode_DecodeFSDefaultAndSize( + curvar, + (Py_ssize_t)(separator - curvar) + ); + if (!py_key) + goto error; + py_val = PyUnicode_DecodeFSDefault(separator + 1); + if (!py_val) + goto error; + if (PyDict_SetItem(py_retdict, py_key, py_val)) + goto error; + Py_CLEAR(py_key); + Py_CLEAR(py_val); + } + curvar = strchr(curvar, '\0') + 1; + } + + free(envbuf); + + return py_retdict; + +error: + if (envbuf != NULL) + free(envbuf); + Py_XDECREF(py_retdict); + Py_XDECREF(py_key); + Py_XDECREF(py_val); + return NULL; +} + + +#ifdef CURR_VERSION_THREAD + +/* + * Retrieves all threads used by process returning a list of tuples + * including thread id, user time and system time. + */ +static PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + long pid; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + perfstat_thread_t *threadt = NULL; + perfstat_id_t id; + int i, rc, thread_count; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + /* Get the count of threads */ + thread_count = perfstat_thread(NULL, NULL, sizeof(perfstat_thread_t), 0); + if (thread_count <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* Allocate enough memory */ + threadt = (perfstat_thread_t *)calloc(thread_count, + sizeof(perfstat_thread_t)); + if (threadt == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, ""); + rc = perfstat_thread(&id, threadt, sizeof(perfstat_thread_t), + thread_count); + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < thread_count; i++) { + if (threadt[i].pid != pid) + continue; + + py_tuple = Py_BuildValue("Idd", + threadt[i].tid, + threadt[i].ucpu_time, + threadt[i].scpu_time); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + free(threadt); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (threadt != NULL) + free(threadt); + return NULL; +} + +#endif + + +#ifdef CURR_VERSION_PROCESS + +static PyObject * +psutil_proc_io_counters(PyObject *self, PyObject *args) { + long pid; + int rc; + perfstat_process_t procinfo; + perfstat_id_t id; + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + snprintf(id.name, sizeof(id.name), "%ld", pid); + rc = perfstat_process(&id, &procinfo, sizeof(perfstat_process_t), 1); + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("(KKKK)", + procinfo.inOps, // XXX always 0 + procinfo.outOps, + procinfo.inBytes, // XXX always 0 + procinfo.outBytes); +} + +#endif + + +/* + * Return process user and system CPU times as a Python tuple. + */ +static PyObject * +psutil_proc_cpu_times(PyObject *self, PyObject *args) { + int pid; + char path[100]; + pstatus_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/status", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + // results are more precise than os.times() + return Py_BuildValue("dddd", + TV2DOUBLE(info.pr_utime), + TV2DOUBLE(info.pr_stime), + TV2DOUBLE(info.pr_cutime), + TV2DOUBLE(info.pr_cstime)); +} + + +/* + * Return process uids/gids as a Python tuple. + */ +static PyObject * +psutil_proc_cred(PyObject *self, PyObject *args) { + int pid; + char path[100]; + prcred_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/cred", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + return Py_BuildValue("iiiiii", + info.pr_ruid, info.pr_euid, info.pr_suid, + info.pr_rgid, info.pr_egid, info.pr_sgid); +} + + +/* + * Return process voluntary and involuntary context switches as a Python tuple. + */ +static PyObject * +psutil_proc_num_ctx_switches(PyObject *self, PyObject *args) { + PyObject *py_tuple = NULL; + pid32_t requested_pid; + pid32_t pid = 0; + int np = 0; + struct procentry64 *processes = (struct procentry64 *)NULL; + struct procentry64 *p; + + if (! PyArg_ParseTuple(args, "i", &requested_pid)) + return NULL; + + processes = psutil_read_process_table(&np); + if (!processes) + return NULL; + + /* Loop through processes */ + for (p = processes; np > 0; np--, p++) { + pid = p->pi_pid; + if (requested_pid != pid) + continue; + py_tuple = Py_BuildValue("LL", + (long long) p->pi_ru.ru_nvcsw, /* voluntary context switches */ + (long long) p->pi_ru.ru_nivcsw); /* involuntary */ + free(processes); + return py_tuple; + } + + /* finished iteration without finding requested pid */ + free(processes); + return NoSuchProcess(""); +} + + +/* + * Return users currently connected on the system. + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + struct utmpx *ut; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; + PyObject *py_user_proc = NULL; + + if (py_retlist == NULL) + return NULL; + + setutxent(); + while (NULL != (ut = getutxent())) { + if (ut->ut_type == USER_PROCESS) + py_user_proc = Py_True; + else + py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut->ut_tv.tv_sec, // tstamp + py_user_proc, // (bool) user process + ut->ut_pid // process id + ); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_username); + Py_CLEAR(py_tty); + Py_CLEAR(py_hostname); + Py_CLEAR(py_tuple); + } + endutxent(); + + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (ut != NULL) + endutxent(); + return NULL; +} + + +/* + * Return disk mounted partitions as a list of tuples including device, + * mount point and filesystem type. + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + FILE *file = NULL; + struct mntent * mt = NULL; + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + file = setmntent(MNTTAB, "rb"); + if (file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + mt = getmntent(file); + while (mt != NULL) { + py_dev = PyUnicode_DecodeFSDefault(mt->mnt_fsname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(mt->mnt_dir); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue( + "(OOss)", + py_dev, // device + py_mountp, // mount point + mt->mnt_type, // fs type + mt->mnt_opts); // options + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_dev); + Py_CLEAR(py_mountp); + Py_CLEAR(py_tuple); + mt = getmntent(file); + } + endmntent(file); + return py_retlist; + +error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (file != NULL) + endmntent(file); + return NULL; +} + + +#if defined(CURR_VERSION_NETINTERFACE) && CURR_VERSION_NETINTERFACE >= 3 + +/* + * Return a list of tuples for network I/O statistics. + */ +static PyObject * +psutil_net_io_counters(PyObject *self, PyObject *args) { + perfstat_netinterface_t *statp = NULL; + int tot, i; + perfstat_id_t first; + + PyObject *py_retdict = PyDict_New(); + PyObject *py_ifc_info = NULL; + + if (py_retdict == NULL) + return NULL; + + /* check how many perfstat_netinterface_t structures are available */ + tot = perfstat_netinterface( + NULL, NULL, sizeof(perfstat_netinterface_t), 0); + if (tot == 0) { + // no network interfaces - return empty dict + return py_retdict; + } + if (tot < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + statp = (perfstat_netinterface_t *) + malloc(tot * sizeof(perfstat_netinterface_t)); + if (statp == NULL) { + PyErr_NoMemory(); + goto error; + } + strcpy(first.name, FIRST_NETINTERFACE); + tot = perfstat_netinterface(&first, statp, + sizeof(perfstat_netinterface_t), tot); + if (tot < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < tot; i++) { + py_ifc_info = Py_BuildValue("(KKKKKKKK)", + statp[i].obytes, /* number of bytes sent on interface */ + statp[i].ibytes, /* number of bytes received on interface */ + statp[i].opackets, /* number of packets sent on interface */ + statp[i].ipackets, /* number of packets received on interface */ + statp[i].ierrors, /* number of input errors on interface */ + statp[i].oerrors, /* number of output errors on interface */ + statp[i].if_iqdrops, /* Dropped on input, this interface */ + statp[i].xmitdrops /* number of packets not transmitted */ + ); + if (!py_ifc_info) + goto error; + if (PyDict_SetItemString(py_retdict, statp[i].name, py_ifc_info)) + goto error; + Py_DECREF(py_ifc_info); + } + + free(statp); + return py_retdict; + +error: + if (statp != NULL) + free(statp); + Py_XDECREF(py_ifc_info); + Py_DECREF(py_retdict); + return NULL; +} + +#endif + + +static PyObject* +psutil_net_if_stats(PyObject* self, PyObject* args) { + char *nic_name; + int sock = 0; + int ret; + int mtu; + struct ifreq ifr; + PyObject *py_is_up = NULL; + PyObject *py_retlist = NULL; + + if (! PyArg_ParseTuple(args, "s", &nic_name)) + return NULL; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) + goto error; + + strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + + // is up? + ret = ioctl(sock, SIOCGIFFLAGS, &ifr); + if (ret == -1) + goto error; + if ((ifr.ifr_flags & IFF_UP) != 0) + py_is_up = Py_True; + else + py_is_up = Py_False; + Py_INCREF(py_is_up); + + // MTU + ret = ioctl(sock, SIOCGIFMTU, &ifr); + if (ret == -1) + goto error; + mtu = ifr.ifr_mtu; + + close(sock); + py_retlist = Py_BuildValue("[Oi]", py_is_up, mtu); + if (!py_retlist) + goto error; + Py_DECREF(py_is_up); + return py_retlist; + +error: + Py_XDECREF(py_is_up); + if (sock != 0) + close(sock); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} + + +static PyObject * +psutil_boot_time(PyObject *self, PyObject *args) { + float boot_time = 0.0; + struct utmpx *ut; + + setutxent(); + while (NULL != (ut = getutxent())) { + if (ut->ut_type == BOOT_TIME) { + boot_time = (float)ut->ut_tv.tv_sec; + break; + } + } + endutxent(); + if (boot_time == 0.0) { + /* could not find BOOT_TIME in getutxent loop */ + PyErr_SetString(PyExc_RuntimeError, "can't determine boot time"); + return NULL; + } + return Py_BuildValue("f", boot_time); +} + + +/* + * Return a Python list of tuple representing per-cpu times + */ +static PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + int ncpu, rc, i; + long ticks; + perfstat_cpu_t *cpu = NULL; + perfstat_id_t id; + PyObject *py_retlist = PyList_New(0); + PyObject *py_cputime = NULL; + + if (py_retlist == NULL) + return NULL; + + /* get the number of ticks per second */ + ticks = sysconf(_SC_CLK_TCK); + if (ticks < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* get the number of cpus in ncpu */ + ncpu = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0); + if (ncpu <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* allocate enough memory to hold the ncpu structures */ + cpu = (perfstat_cpu_t *) malloc(ncpu * sizeof(perfstat_cpu_t)); + if (cpu == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, ""); + rc = perfstat_cpu(&id, cpu, sizeof(perfstat_cpu_t), ncpu); + + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < ncpu; i++) { + py_cputime = Py_BuildValue( + "(dddd)", + (double)cpu[i].user / ticks, + (double)cpu[i].sys / ticks, + (double)cpu[i].idle / ticks, + (double)cpu[i].wait / ticks); + if (!py_cputime) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_DECREF(py_cputime); + } + free(cpu); + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + if (cpu != NULL) + free(cpu); + return NULL; +} + + +/* + * Return disk IO statistics. + */ +static PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + PyObject *py_retdict = PyDict_New(); + PyObject *py_disk_info = NULL; + perfstat_disk_t *diskt = NULL; + perfstat_id_t id; + int i, rc, disk_count; + + if (py_retdict == NULL) + return NULL; + + /* Get the count of disks */ + disk_count = perfstat_disk(NULL, NULL, sizeof(perfstat_disk_t), 0); + if (disk_count <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* Allocate enough memory */ + diskt = (perfstat_disk_t *)calloc(disk_count, + sizeof(perfstat_disk_t)); + if (diskt == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, FIRST_DISK); + rc = perfstat_disk(&id, diskt, sizeof(perfstat_disk_t), + disk_count); + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < disk_count; i++) { + py_disk_info = Py_BuildValue( + "KKKKKK", + diskt[i].__rxfers, + diskt[i].xfers - diskt[i].__rxfers, + diskt[i].rblks * diskt[i].bsize, + diskt[i].wblks * diskt[i].bsize, + diskt[i].rserv / 1000 / 1000, // from nano to milli secs + diskt[i].wserv / 1000 / 1000 // from nano to milli secs + ); + if (py_disk_info == NULL) + goto error; + if (PyDict_SetItemString(py_retdict, diskt[i].name, + py_disk_info)) + goto error; + Py_DECREF(py_disk_info); + } + free(diskt); + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + if (diskt != NULL) + free(diskt); + return NULL; +} + + +/* + * Return virtual memory usage statistics. + */ +static PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + int rc; + int pagesize = getpagesize(); + perfstat_memory_total_t memory; + + rc = perfstat_memory_total( + NULL, &memory, sizeof(perfstat_memory_total_t), 1); + if (rc <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("KKKKK", + (unsigned long long) memory.real_total * pagesize, + (unsigned long long) memory.real_avail * pagesize, + (unsigned long long) memory.real_free * pagesize, + (unsigned long long) memory.real_pinned * pagesize, + (unsigned long long) memory.real_inuse * pagesize + ); +} + + +/* + * Return stats about swap memory. + */ +static PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { + int rc; + int pagesize = getpagesize(); + perfstat_memory_total_t memory; + + rc = perfstat_memory_total( + NULL, &memory, sizeof(perfstat_memory_total_t), 1); + if (rc <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("KKKK", + (unsigned long long) memory.pgsp_total * pagesize, + (unsigned long long) memory.pgsp_free * pagesize, + (unsigned long long) memory.pgins * pagesize, + (unsigned long long) memory.pgouts * pagesize + ); +} + + +/* + * Return CPU statistics. + */ +static PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + int ncpu, rc, i; + // perfstat_cpu_total_t doesn't have invol/vol cswitch, only pswitch + // which is apparently something else. We have to sum over all cpus + perfstat_cpu_t *cpu = NULL; + perfstat_id_t id; + u_longlong_t cswitches = 0; + u_longlong_t devintrs = 0; + u_longlong_t softintrs = 0; + u_longlong_t syscall = 0; + + /* get the number of cpus in ncpu */ + ncpu = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0); + if (ncpu <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* allocate enough memory to hold the ncpu structures */ + cpu = (perfstat_cpu_t *) malloc(ncpu * sizeof(perfstat_cpu_t)); + if (cpu == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, ""); + rc = perfstat_cpu(&id, cpu, sizeof(perfstat_cpu_t), ncpu); + + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < ncpu; i++) { + cswitches += cpu[i].invol_cswitch + cpu[i].vol_cswitch; + devintrs += cpu[i].devintrs; + softintrs += cpu[i].softintrs; + syscall += cpu[i].syscall; + } + + free(cpu); + + return Py_BuildValue( + "KKKK", + cswitches, + devintrs, + softintrs, + syscall + ); + +error: + if (cpu != NULL) + free(cpu); + return NULL; +} + + +/* + * define the psutil C module methods and initialize the module. + */ +static PyMethodDef +PsutilMethods[] = +{ + // --- process-related functions + {"proc_basic_info", psutil_proc_basic_info, METH_VARARGS, + "Return process ppid, rss, vms, ctime, nice, nthreads, status and tty"}, + {"proc_name", psutil_proc_name, METH_VARARGS, + "Return process name."}, + {"proc_args", psutil_proc_args, METH_VARARGS, + "Return process command line arguments."}, + {"proc_environ", psutil_proc_environ, METH_VARARGS, + "Return process environment variables."}, + {"proc_cpu_times", psutil_proc_cpu_times, METH_VARARGS, + "Return process user and system CPU times."}, + {"proc_cred", psutil_proc_cred, METH_VARARGS, + "Return process uids/gids."}, +#ifdef CURR_VERSION_THREAD + {"proc_threads", psutil_proc_threads, METH_VARARGS, + "Return process threads"}, +#endif +#ifdef CURR_VERSION_PROCESS + {"proc_io_counters", psutil_proc_io_counters, METH_VARARGS, + "Get process I/O counters."}, +#endif + {"proc_num_ctx_switches", psutil_proc_num_ctx_switches, METH_VARARGS, + "Get process I/O counters."}, + + // --- system-related functions + {"users", psutil_users, METH_VARARGS, + "Return currently connected users."}, + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return disk partitions."}, + {"boot_time", psutil_boot_time, METH_VARARGS, + "Return system boot time in seconds since the EPOCH."}, + {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, + "Return system per-cpu times as a list of tuples"}, + {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS, + "Return a Python dict of tuples for disk I/O statistics."}, + {"virtual_mem", psutil_virtual_mem, METH_VARARGS, + "Return system virtual memory usage statistics"}, + {"swap_mem", psutil_swap_mem, METH_VARARGS, + "Return stats about swap memory, in bytes"}, +#if defined(CURR_VERSION_NETINTERFACE) && CURR_VERSION_NETINTERFACE >= 3 + {"net_io_counters", psutil_net_io_counters, METH_VARARGS, + "Return a Python dict of tuples for network I/O statistics."}, +#endif + {"net_connections", psutil_net_connections, METH_VARARGS, + "Return system-wide connections"}, + {"net_if_stats", psutil_net_if_stats, METH_VARARGS, + "Return NIC stats (isup, mtu)"}, + {"cpu_stats", psutil_cpu_stats, METH_VARARGS, + "Return CPU statistics"}, + + // --- others + {"set_testing", psutil_set_testing, METH_NOARGS, + "Set psutil in testing mode"}, + + {NULL, NULL, 0, NULL} +}; + + +struct module_state { + PyObject *error; +}; + +#if PY_MAJOR_VERSION >= 3 +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else +#define GETSTATE(m) (&_state) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if PY_MAJOR_VERSION >= 3 + +static int +psutil_aix_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +static int +psutil_aix_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "psutil_aix", + NULL, + sizeof(struct module_state), + PsutilMethods, + NULL, + psutil_aix_traverse, + psutil_aix_clear, + NULL +}; + +#define INITERROR return NULL + +PyMODINIT_FUNC PyInit__psutil_aix(void) + +#else +#define INITERROR return + +void init_psutil_aix(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("_psutil_aix", PsutilMethods); +#endif + PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); + + PyModule_AddIntConstant(module, "SIDL", SIDL); + PyModule_AddIntConstant(module, "SZOMB", SZOMB); + PyModule_AddIntConstant(module, "SACTIVE", SACTIVE); + PyModule_AddIntConstant(module, "SSWAP", SSWAP); + PyModule_AddIntConstant(module, "SSTOP", SSTOP); + + PyModule_AddIntConstant(module, "TCPS_CLOSED", TCPS_CLOSED); + PyModule_AddIntConstant(module, "TCPS_CLOSING", TCPS_CLOSING); + PyModule_AddIntConstant(module, "TCPS_CLOSE_WAIT", TCPS_CLOSE_WAIT); + PyModule_AddIntConstant(module, "TCPS_LISTEN", TCPS_LISTEN); + PyModule_AddIntConstant(module, "TCPS_ESTABLISHED", TCPS_ESTABLISHED); + PyModule_AddIntConstant(module, "TCPS_SYN_SENT", TCPS_SYN_SENT); + PyModule_AddIntConstant(module, "TCPS_SYN_RCVD", TCPS_SYN_RECEIVED); + PyModule_AddIntConstant(module, "TCPS_FIN_WAIT_1", TCPS_FIN_WAIT_1); + PyModule_AddIntConstant(module, "TCPS_FIN_WAIT_2", TCPS_FIN_WAIT_2); + PyModule_AddIntConstant(module, "TCPS_LAST_ACK", TCPS_LAST_ACK); + PyModule_AddIntConstant(module, "TCPS_TIME_WAIT", TCPS_TIME_WAIT); + PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + + psutil_setup(); + + if (module == NULL) + INITERROR; +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} + +#ifdef __cplusplus +} +#endif diff --git a/ddtrace/vendor/psutil/_psutil_bsd.c b/ddtrace/vendor/psutil/_psutil_bsd.c new file mode 100644 index 00000000000..723d6198c35 --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_bsd.c @@ -0,0 +1,1093 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Landry Breuil (OpenBSD). + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Platform-specific module methods for FreeBSD and OpenBSD. + + * OpenBSD references: + * - OpenBSD source code: https://github.com/openbsd/src + * + * OpenBSD / NetBSD: missing APIs compared to FreeBSD implementation: + * - psutil.net_connections() + * - psutil.Process.get/set_cpu_affinity() (not supported natively) + * - psutil.Process.memory_maps() + */ + +#if defined(PSUTIL_NETBSD) + #define _KMEMUSER +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(__NetBSD__) +#include +#endif +#include +#include +#include +#include +#include // for struct xsocket +#include +#include +// for xinpcb struct +#include +#include +#include +#include +#include +#include +#include +#include // for struct xtcpcb +#include // for TCP connection states +#include // for inet_ntop() + +#include + +#include // net io counters +#include +#include + +#include // process open files/connections +#include + +#include "_psutil_common.h" +#include "_psutil_posix.h" + +#ifdef PSUTIL_FREEBSD + #include "arch/freebsd/specific.h" + #include "arch/freebsd/sys_socks.h" + #include "arch/freebsd/proc_socks.h" + + #include + #include // get io counters + #include // process open files, shared libs (kinfo_getvmmap) + #if __FreeBSD_version < 900000 + #include // system users + #else + #include + #endif +#elif PSUTIL_OPENBSD + #include "arch/openbsd/specific.h" + + #include + #include // for VREG + #define _KERNEL // for DTYPE_VNODE + #include + #undef _KERNEL + #include // for CPUSTATES & CP_* +#elif PSUTIL_NETBSD + #include "arch/netbsd/specific.h" + #include "arch/netbsd/socks.h" + + #include + #include // for VREG + #include // for CPUSTATES & CP_* + #ifndef DTYPE_VNODE + #define DTYPE_VNODE 1 + #endif +#endif + + + +// convert a timeval struct to a double +#define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) + +#ifdef PSUTIL_FREEBSD + // convert a bintime struct to milliseconds + #define PSUTIL_BT2MSEC(bt) (bt.sec * 1000 + (((uint64_t) 1000000000 * \ + (uint32_t) (bt.frac >> 32) ) >> 32 ) / 1000000) +#endif + +#if defined(PSUTIL_OPENBSD) || defined (PSUTIL_NETBSD) + #define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) +#endif + + +/* + * Return a Python list of all the PIDs running on the system. + */ +static PyObject * +psutil_pids(PyObject *self, PyObject *args) { + kinfo_proc *proclist = NULL; + kinfo_proc *orig_address = NULL; + size_t num_processes; + size_t idx; + PyObject *py_retlist = PyList_New(0); + PyObject *py_pid = NULL; + + if (py_retlist == NULL) + return NULL; + + // TODO: RuntimeError is inappropriate here; we could return the + // original error instead. + if (psutil_get_proc_list(&proclist, &num_processes) != 0) { + if (errno != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + else { + PyErr_SetString(PyExc_RuntimeError, + "failed to retrieve process list"); + } + goto error; + } + + if (num_processes > 0) { + orig_address = proclist; // save so we can free it after we're done + for (idx = 0; idx < num_processes; idx++) { +#ifdef PSUTIL_FREEBSD + py_pid = Py_BuildValue("i", proclist->ki_pid); +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) + py_pid = Py_BuildValue("i", proclist->p_pid); +#endif + if (!py_pid) + goto error; + if (PyList_Append(py_retlist, py_pid)) + goto error; + Py_CLEAR(py_pid); + proclist++; + } + free(orig_address); + } + + return py_retlist; + +error: + Py_XDECREF(py_pid); + Py_DECREF(py_retlist); + if (orig_address != NULL) + free(orig_address); + return NULL; +} + + +/* + * Return a Python float indicating the system boot time expressed in + * seconds since the epoch. + */ +static PyObject * +psutil_boot_time(PyObject *self, PyObject *args) { + // fetch sysctl "kern.boottime" + static int request[2] = { CTL_KERN, KERN_BOOTTIME }; + struct timeval boottime; + size_t len = sizeof(boottime); + + if (sysctl(request, 2, &boottime, &len, NULL, 0) == -1) + return PyErr_SetFromErrno(PyExc_OSError); + return Py_BuildValue("d", (double)boottime.tv_sec); +} + + +/* + * Collect different info about a process in one shot and return + * them as a big Python tuple. + */ +static PyObject * +psutil_proc_oneshot_info(PyObject *self, PyObject *args) { + long pid; + long rss; + long vms; + long memtext; + long memdata; + long memstack; + int oncpu; + kinfo_proc kp; + long pagesize = sysconf(_SC_PAGESIZE); + char str[1000]; + PyObject *py_name; + PyObject *py_retlist; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kp) == -1) + return NULL; + + // Process +#ifdef PSUTIL_FREEBSD + sprintf(str, "%s", kp.ki_comm); +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) + sprintf(str, "%s", kp.p_comm); +#endif + py_name = PyUnicode_DecodeFSDefault(str); + if (! py_name) { + // Likely a decoding error. We don't want to fail the whole + // operation. The python module may retry with proc_name(). + PyErr_Clear(); + py_name = Py_None; + } + // Py_INCREF(py_name); + + // Calculate memory. +#ifdef PSUTIL_FREEBSD + rss = (long)kp.ki_rssize * pagesize; + vms = (long)kp.ki_size; + memtext = (long)kp.ki_tsize * pagesize; + memdata = (long)kp.ki_dsize * pagesize; + memstack = (long)kp.ki_ssize * pagesize; +#else + rss = (long)kp.p_vm_rssize * pagesize; + #ifdef PSUTIL_OPENBSD + // VMS, this is how ps determines it on OpenBSD: + // https://github.com/openbsd/src/blob/ + // 588f7f8c69786211f2d16865c552afb91b1c7cba/bin/ps/print.c#L505 + vms = (long)(kp.p_vm_dsize + kp.p_vm_ssize + kp.p_vm_tsize) * pagesize; + #elif PSUTIL_NETBSD + // VMS, this is how top determines it on NetBSD: + // https://github.com/IIJ-NetBSD/netbsd-src/blob/master/external/ + // bsd/top/dist/machine/m_netbsd.c + vms = (long)kp.p_vm_msize * pagesize; + #endif + memtext = (long)kp.p_vm_tsize * pagesize; + memdata = (long)kp.p_vm_dsize * pagesize; + memstack = (long)kp.p_vm_ssize * pagesize; +#endif + +#ifdef PSUTIL_FREEBSD + // what CPU we're on; top was used as an example: + // https://svnweb.freebsd.org/base/head/usr.bin/top/machine.c? + // view=markup&pathrev=273835 + // XXX - note: for "intr" PID this is -1. + if (kp.ki_stat == SRUN && kp.ki_oncpu != NOCPU) + oncpu = kp.ki_oncpu; + else + oncpu = kp.ki_lastcpu; +#else + // On Net/OpenBSD we have kp.p_cpuid but it appears it's always + // set to KI_NOCPU. Even if it's not, ki_lastcpu does not exist + // so there's no way to determine where "sleeping" processes + // were. Not supported. + oncpu = -1; +#endif + + // Return a single big tuple with all process info. + py_retlist = Py_BuildValue( + "(lillllllidllllddddlllllbO)", +#ifdef PSUTIL_FREEBSD + // + (long)kp.ki_ppid, // (long) ppid + (int)kp.ki_stat, // (int) status + // UIDs + (long)kp.ki_ruid, // (long) real uid + (long)kp.ki_uid, // (long) effective uid + (long)kp.ki_svuid, // (long) saved uid + // GIDs + (long)kp.ki_rgid, // (long) real gid + (long)kp.ki_groups[0], // (long) effective gid + (long)kp.ki_svuid, // (long) saved gid + // + kp.ki_tdev, // (int) tty nr + PSUTIL_TV2DOUBLE(kp.ki_start), // (double) create time + // ctx switches + kp.ki_rusage.ru_nvcsw, // (long) ctx switches (voluntary) + kp.ki_rusage.ru_nivcsw, // (long) ctx switches (unvoluntary) + // IO count + kp.ki_rusage.ru_inblock, // (long) read io count + kp.ki_rusage.ru_oublock, // (long) write io count + // CPU times: convert from micro seconds to seconds. + PSUTIL_TV2DOUBLE(kp.ki_rusage.ru_utime), // (double) user time + PSUTIL_TV2DOUBLE(kp.ki_rusage.ru_stime), // (double) sys time + PSUTIL_TV2DOUBLE(kp.ki_rusage_ch.ru_utime), // (double) children utime + PSUTIL_TV2DOUBLE(kp.ki_rusage_ch.ru_stime), // (double) children stime + // memory + rss, // (long) rss + vms, // (long) vms + memtext, // (long) mem text + memdata, // (long) mem data + memstack, // (long) mem stack + // others + oncpu, // (int) the CPU we are on +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) + // + (long)kp.p_ppid, // (long) ppid + (int)kp.p_stat, // (int) status + // UIDs + (long)kp.p_ruid, // (long) real uid + (long)kp.p_uid, // (long) effective uid + (long)kp.p_svuid, // (long) saved uid + // GIDs + (long)kp.p_rgid, // (long) real gid + (long)kp.p_groups[0], // (long) effective gid + (long)kp.p_svuid, // (long) saved gid + // + kp.p_tdev, // (int) tty nr + PSUTIL_KPT2DOUBLE(kp.p_ustart), // (double) create time + // ctx switches + kp.p_uru_nvcsw, // (long) ctx switches (voluntary) + kp.p_uru_nivcsw, // (long) ctx switches (unvoluntary) + // IO count + kp.p_uru_inblock, // (long) read io count + kp.p_uru_oublock, // (long) write io count + // CPU times: convert from micro seconds to seconds. + PSUTIL_KPT2DOUBLE(kp.p_uutime), // (double) user time + PSUTIL_KPT2DOUBLE(kp.p_ustime), // (double) sys time + // OpenBSD and NetBSD provide children user + system times summed + // together (no distinction). + kp.p_uctime_sec + kp.p_uctime_usec / 1000000.0, // (double) ch utime + kp.p_uctime_sec + kp.p_uctime_usec / 1000000.0, // (double) ch stime + // memory + rss, // (long) rss + vms, // (long) vms + memtext, // (long) mem text + memdata, // (long) mem data + memstack, // (long) mem stack + // others + oncpu, // (int) the CPU we are on +#endif + py_name // (pystr) name + ); + + Py_DECREF(py_name); + return py_retlist; +} + + +/* + * Return process name from kinfo_proc as a Python string. + */ +static PyObject * +psutil_proc_name(PyObject *self, PyObject *args) { + long pid; + kinfo_proc kp; + char str[1000]; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kp) == -1) + return NULL; + +#ifdef PSUTIL_FREEBSD + sprintf(str, "%s", kp.ki_comm); +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) + sprintf(str, "%s", kp.p_comm); +#endif + return PyUnicode_DecodeFSDefault(str); +} + + +/* + * Return process cmdline as a Python list of cmdline arguments. + */ +static PyObject * +psutil_proc_cmdline(PyObject *self, PyObject *args) { + long pid; + PyObject *py_retlist = NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + py_retlist = psutil_get_cmdline(pid); + if (py_retlist == NULL) + return NULL; + return Py_BuildValue("N", py_retlist); +} + + +/* + * Return the number of logical CPUs in the system. + * XXX this could be shared with macOS + */ +static PyObject * +psutil_cpu_count_logical(PyObject *self, PyObject *args) { + int mib[2]; + int ncpu; + size_t len; + + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + len = sizeof(ncpu); + + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", ncpu); +} + + +/* + * Return a Python tuple representing user, kernel and idle CPU times + */ +static PyObject * +psutil_cpu_times(PyObject *self, PyObject *args) { +#ifdef PSUTIL_NETBSD + u_int64_t cpu_time[CPUSTATES]; +#else + long cpu_time[CPUSTATES]; +#endif + size_t size = sizeof(cpu_time); + int ret; + +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) + ret = sysctlbyname("kern.cp_time", &cpu_time, &size, NULL, 0); +#elif PSUTIL_OPENBSD + int mib[] = {CTL_KERN, KERN_CPTIME}; + ret = sysctl(mib, 2, &cpu_time, &size, NULL, 0); +#endif + if (ret == -1) + return PyErr_SetFromErrno(PyExc_OSError); + return Py_BuildValue("(ddddd)", + (double)cpu_time[CP_USER] / CLOCKS_PER_SEC, + (double)cpu_time[CP_NICE] / CLOCKS_PER_SEC, + (double)cpu_time[CP_SYS] / CLOCKS_PER_SEC, + (double)cpu_time[CP_IDLE] / CLOCKS_PER_SEC, + (double)cpu_time[CP_INTR] / CLOCKS_PER_SEC + ); +} + + + /* + * Return files opened by process as a list of (path, fd) tuples. + * TODO: this is broken as it may report empty paths. 'procstat' + * utility has the same problem see: + * https://github.com/giampaolo/psutil/issues/595 + */ +#if (defined(__FreeBSD_version) && __FreeBSD_version >= 800000) || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) +static PyObject * +psutil_proc_open_files(PyObject *self, PyObject *args) { + long pid; + int i; + int cnt; + int regular; + int fd; + char *path; + struct kinfo_file *freep = NULL; + struct kinfo_file *kif; + kinfo_proc kipp; + PyObject *py_tuple = NULL; + PyObject *py_path = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + if (psutil_kinfo_proc(pid, &kipp) == -1) + goto error; + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + goto error; + } + + for (i = 0; i < cnt; i++) { + kif = &freep[i]; + +#ifdef PSUTIL_FREEBSD + regular = (kif->kf_type == KF_TYPE_VNODE) && \ + (kif->kf_vnode_type == KF_VTYPE_VREG); + fd = kif->kf_fd; + path = kif->kf_path; +#elif PSUTIL_OPENBSD + regular = (kif->f_type == DTYPE_VNODE) && (kif->v_type == VREG); + fd = kif->fd_fd; + // XXX - it appears path is not exposed in the kinfo_file struct. + path = ""; +#elif PSUTIL_NETBSD + regular = (kif->ki_ftype == DTYPE_VNODE) && (kif->ki_vtype == VREG); + fd = kif->ki_fd; + // XXX - it appears path is not exposed in the kinfo_file struct. + path = ""; +#endif + if (regular == 1) { + py_path = PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; + py_tuple = Py_BuildValue("(Oi)", py_path, fd); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_path); + Py_CLEAR(py_tuple); + } + } + free(freep); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (freep != NULL) + free(freep); + return NULL; +} +#endif + + +/* + * Return a list of tuples including device, mount point and fs type + * for all partitions mounted on the system. + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + int num; + int i; + long len; + uint64_t flags; + char opts[200]; +#ifdef PSUTIL_NETBSD + struct statvfs *fs = NULL; +#else + struct statfs *fs = NULL; +#endif + PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; + PyObject *py_tuple = NULL; + + if (py_retlist == NULL) + return NULL; + + // get the number of mount points + Py_BEGIN_ALLOW_THREADS +#ifdef PSUTIL_NETBSD + num = getvfsstat(NULL, 0, MNT_NOWAIT); +#else + num = getfsstat(NULL, 0, MNT_NOWAIT); +#endif + Py_END_ALLOW_THREADS + if (num == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + len = sizeof(*fs) * num; + fs = malloc(len); + if (fs == NULL) { + PyErr_NoMemory(); + goto error; + } + + Py_BEGIN_ALLOW_THREADS +#ifdef PSUTIL_NETBSD + num = getvfsstat(fs, len, MNT_NOWAIT); +#else + num = getfsstat(fs, len, MNT_NOWAIT); +#endif + Py_END_ALLOW_THREADS + if (num == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < num; i++) { + py_tuple = NULL; + opts[0] = 0; +#ifdef PSUTIL_NETBSD + flags = fs[i].f_flag; +#else + flags = fs[i].f_flags; +#endif + + // see sys/mount.h + if (flags & MNT_RDONLY) + strlcat(opts, "ro", sizeof(opts)); + else + strlcat(opts, "rw", sizeof(opts)); + if (flags & MNT_SYNCHRONOUS) + strlcat(opts, ",sync", sizeof(opts)); + if (flags & MNT_NOEXEC) + strlcat(opts, ",noexec", sizeof(opts)); + if (flags & MNT_NOSUID) + strlcat(opts, ",nosuid", sizeof(opts)); + if (flags & MNT_ASYNC) + strlcat(opts, ",async", sizeof(opts)); + if (flags & MNT_NOATIME) + strlcat(opts, ",noatime", sizeof(opts)); + if (flags & MNT_SOFTDEP) + strlcat(opts, ",softdep", sizeof(opts)); +#ifdef PSUTIL_FREEBSD + if (flags & MNT_UNION) + strlcat(opts, ",union", sizeof(opts)); + if (flags & MNT_SUIDDIR) + strlcat(opts, ",suiddir", sizeof(opts)); + if (flags & MNT_SOFTDEP) + strlcat(opts, ",softdep", sizeof(opts)); + if (flags & MNT_NOSYMFOLLOW) + strlcat(opts, ",nosymfollow", sizeof(opts)); + if (flags & MNT_GJOURNAL) + strlcat(opts, ",gjournal", sizeof(opts)); + if (flags & MNT_MULTILABEL) + strlcat(opts, ",multilabel", sizeof(opts)); + if (flags & MNT_ACLS) + strlcat(opts, ",acls", sizeof(opts)); + if (flags & MNT_NOCLUSTERR) + strlcat(opts, ",noclusterr", sizeof(opts)); + if (flags & MNT_NOCLUSTERW) + strlcat(opts, ",noclusterw", sizeof(opts)); + if (flags & MNT_NFS4ACLS) + strlcat(opts, ",nfs4acls", sizeof(opts)); +#elif PSUTIL_NETBSD + if (flags & MNT_NODEV) + strlcat(opts, ",nodev", sizeof(opts)); + if (flags & MNT_UNION) + strlcat(opts, ",union", sizeof(opts)); + if (flags & MNT_NOCOREDUMP) + strlcat(opts, ",nocoredump", sizeof(opts)); +#ifdef MNT_RELATIME + if (flags & MNT_RELATIME) + strlcat(opts, ",relatime", sizeof(opts)); +#endif + if (flags & MNT_IGNORE) + strlcat(opts, ",ignore", sizeof(opts)); +#ifdef MNT_DISCARD + if (flags & MNT_DISCARD) + strlcat(opts, ",discard", sizeof(opts)); +#endif +#ifdef MNT_EXTATTR + if (flags & MNT_EXTATTR) + strlcat(opts, ",extattr", sizeof(opts)); +#endif + if (flags & MNT_LOG) + strlcat(opts, ",log", sizeof(opts)); + if (flags & MNT_SYMPERM) + strlcat(opts, ",symperm", sizeof(opts)); + if (flags & MNT_NODEVMTIME) + strlcat(opts, ",nodevmtime", sizeof(opts)); +#endif + py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue("(OOss)", + py_dev, // device + py_mountp, // mount point + fs[i].f_fstypename, // fs type + opts); // options + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_dev); + Py_CLEAR(py_mountp); + Py_CLEAR(py_tuple); + } + + free(fs); + return py_retlist; + +error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (fs != NULL) + free(fs); + return NULL; +} + + +/* + * Return a Python list of named tuples with overall network I/O information + */ +static PyObject * +psutil_net_io_counters(PyObject *self, PyObject *args) { + char *buf = NULL, *lim, *next; + struct if_msghdr *ifm; + int mib[6]; + size_t len; + PyObject *py_retdict = PyDict_New(); + PyObject *py_ifc_info = NULL; + if (py_retdict == NULL) + return NULL; + + mib[0] = CTL_NET; // networking subsystem + mib[1] = PF_ROUTE; // type of information + mib[2] = 0; // protocol (IPPROTO_xxx) + mib[3] = 0; // address family + mib[4] = NET_RT_IFLIST; // operation + mib[5] = 0; + + if (sysctl(mib, 6, NULL, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + buf = malloc(len); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } + + if (sysctl(mib, 6, buf, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + lim = buf + len; + + for (next = buf; next < lim; ) { + py_ifc_info = NULL; + ifm = (struct if_msghdr *)next; + next += ifm->ifm_msglen; + + if (ifm->ifm_type == RTM_IFINFO) { + struct if_msghdr *if2m = (struct if_msghdr *)ifm; + struct sockaddr_dl *sdl = (struct sockaddr_dl *)(if2m + 1); + char ifc_name[32]; + + strncpy(ifc_name, sdl->sdl_data, sdl->sdl_nlen); + ifc_name[sdl->sdl_nlen] = 0; + // XXX: ignore usbus interfaces: + // http://lists.freebsd.org/pipermail/freebsd-current/ + // 2011-October/028752.html + // 'ifconfig -a' doesn't show them, nor do we. + if (strncmp(ifc_name, "usbus", 5) == 0) + continue; + + py_ifc_info = Py_BuildValue("(kkkkkkki)", + if2m->ifm_data.ifi_obytes, + if2m->ifm_data.ifi_ibytes, + if2m->ifm_data.ifi_opackets, + if2m->ifm_data.ifi_ipackets, + if2m->ifm_data.ifi_ierrors, + if2m->ifm_data.ifi_oerrors, + if2m->ifm_data.ifi_iqdrops, +#ifdef _IFI_OQDROPS + if2m->ifm_data.ifi_oqdrops +#else + 0 +#endif + ); + if (!py_ifc_info) + goto error; + if (PyDict_SetItemString(py_retdict, ifc_name, py_ifc_info)) + goto error; + Py_CLEAR(py_ifc_info); + } + else { + continue; + } + } + + free(buf); + return py_retdict; + +error: + Py_XDECREF(py_ifc_info); + Py_DECREF(py_retdict); + if (buf != NULL) + free(buf); + return NULL; +} + + +/* + * Return currently connected users as a list of tuples. + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + PyObject *py_retlist = PyList_New(0); + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; + PyObject *py_tuple = NULL; + + if (py_retlist == NULL) + return NULL; + +#if (defined(__FreeBSD_version) && (__FreeBSD_version < 900000)) || PSUTIL_OPENBSD + struct utmp ut; + FILE *fp; + + Py_BEGIN_ALLOW_THREADS + fp = fopen(_PATH_UTMP, "r"); + Py_END_ALLOW_THREADS + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, _PATH_UTMP); + goto error; + } + + while (fread(&ut, sizeof(ut), 1, fp) == 1) { + if (*ut.ut_name == '\0') + continue; + py_username = PyUnicode_DecodeFSDefault(ut.ut_name); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut.ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut.ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut.ut_time, // start time +#ifdef PSUTIL_OPENBSD + -1 // process id (set to None later) +#else + ut.ut_pid // process id +#endif + ); + if (!py_tuple) { + fclose(fp); + goto error; + } + if (PyList_Append(py_retlist, py_tuple)) { + fclose(fp); + goto error; + } + Py_CLEAR(py_username); + Py_CLEAR(py_tty); + Py_CLEAR(py_hostname); + Py_CLEAR(py_tuple); + } + + fclose(fp); +#else + struct utmpx *utx; + setutxent(); + while ((utx = getutxent()) != NULL) { + if (utx->ut_type != USER_PROCESS) + continue; + py_username = PyUnicode_DecodeFSDefault(utx->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(utx->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(utx->ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)utx->ut_tv.tv_sec, // start time +#ifdef PSUTIL_OPENBSD + -1 // process id (set to None later) +#else + utx->ut_pid // process id +#endif + ); + + if (!py_tuple) { + endutxent(); + goto error; + } + if (PyList_Append(py_retlist, py_tuple)) { + endutxent(); + goto error; + } + Py_CLEAR(py_username); + Py_CLEAR(py_tty); + Py_CLEAR(py_hostname); + Py_CLEAR(py_tuple); + } + + endutxent(); +#endif + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + return NULL; +} + + +/* + * define the psutil C module methods and initialize the module. + */ +static PyMethodDef mod_methods[] = { + // --- per-process functions + + {"proc_oneshot_info", psutil_proc_oneshot_info, METH_VARARGS, + "Return multiple info about a process"}, + {"proc_name", psutil_proc_name, METH_VARARGS, + "Return process name"}, + {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS, + "Return process cmdline as a list of cmdline arguments"}, + {"proc_threads", psutil_proc_threads, METH_VARARGS, + "Return process threads"}, +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_OPENBSD) + {"proc_connections", psutil_proc_connections, METH_VARARGS, + "Return connections opened by process"}, +#endif + {"proc_cwd", psutil_proc_cwd, METH_VARARGS, + "Return process current working directory."}, +#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) + {"proc_num_fds", psutil_proc_num_fds, METH_VARARGS, + "Return the number of file descriptors opened by this process"}, + {"proc_open_files", psutil_proc_open_files, METH_VARARGS, + "Return files opened by process as a list of (path, fd) tuples"}, +#endif +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) + {"proc_num_threads", psutil_proc_num_threads, METH_VARARGS, + "Return number of threads used by process"}, +#endif +#if defined(PSUTIL_FREEBSD) + {"proc_exe", psutil_proc_exe, METH_VARARGS, + "Return process pathname executable"}, + {"proc_memory_maps", psutil_proc_memory_maps, METH_VARARGS, + "Return a list of tuples for every process's memory map"}, + {"proc_cpu_affinity_get", psutil_proc_cpu_affinity_get, METH_VARARGS, + "Return process CPU affinity."}, + {"proc_cpu_affinity_set", psutil_proc_cpu_affinity_set, METH_VARARGS, + "Set process CPU affinity."}, + {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, + "Return an XML string to determine the number physical CPUs."}, +#endif + + // --- system-related functions + + {"pids", psutil_pids, METH_VARARGS, + "Returns a list of PIDs currently running on the system"}, + {"cpu_count_logical", psutil_cpu_count_logical, METH_VARARGS, + "Return number of logical CPUs on the system"}, + {"virtual_mem", psutil_virtual_mem, METH_VARARGS, + "Return system virtual memory usage statistics"}, + {"swap_mem", psutil_swap_mem, METH_VARARGS, + "Return swap mem stats"}, + {"cpu_times", psutil_cpu_times, METH_VARARGS, + "Return system cpu times as a tuple (user, system, nice, idle, irc)"}, + {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, + "Return system per-cpu times as a list of tuples"}, + {"boot_time", psutil_boot_time, METH_VARARGS, + "Return the system boot time expressed in seconds since the epoch."}, + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return a list of tuples including device, mount point and " + "fs type for all partitions mounted on the system."}, + {"net_io_counters", psutil_net_io_counters, METH_VARARGS, + "Return dict of tuples of networks I/O information."}, + {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS, + "Return a Python dict of tuples for disk I/O information"}, + {"users", psutil_users, METH_VARARGS, + "Return currently connected users as a list of tuples"}, + {"cpu_stats", psutil_cpu_stats, METH_VARARGS, + "Return CPU statistics"}, +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) + {"net_connections", psutil_net_connections, METH_VARARGS, + "Return system-wide open connections."}, +#endif +#if defined(PSUTIL_FREEBSD) + {"sensors_battery", psutil_sensors_battery, METH_VARARGS, + "Return battery information."}, + {"sensors_cpu_temperature", psutil_sensors_cpu_temperature, METH_VARARGS, + "Return temperature information for a given CPU core number."}, + {"cpu_frequency", psutil_cpu_freq, METH_VARARGS, + "Return frequency of a given CPU"}, +#endif + + // --- others + {"set_testing", psutil_set_testing, METH_NOARGS, + "Set psutil in testing mode"}, + + {NULL, NULL, 0, NULL} +}; + +#if PY_MAJOR_VERSION >= 3 + #define INITERR return NULL + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_bsd", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL + }; + + PyObject *PyInit__psutil_bsd(void) +#else /* PY_MAJOR_VERSION */ + #define INITERR return + + void init_psutil_bsd(void) +#endif /* PY_MAJOR_VERSION */ +{ + PyObject *v; +#if PY_MAJOR_VERSION >= 3 + PyObject *mod = PyModule_Create(&moduledef); +#else + PyObject *mod = Py_InitModule("_psutil_bsd", mod_methods); +#endif + if (mod == NULL) + INITERR; + + if (PyModule_AddIntConstant(mod, "version", PSUTIL_VERSION)) INITERR; + // process status constants + +#ifdef PSUTIL_FREEBSD + if (PyModule_AddIntConstant(mod, "SIDL", SIDL)) INITERR; + if (PyModule_AddIntConstant(mod, "SRUN", SRUN)) INITERR; + if (PyModule_AddIntConstant(mod, "SSLEEP", SSLEEP)) INITERR; + if (PyModule_AddIntConstant(mod, "SSTOP", SSTOP)) INITERR; + if (PyModule_AddIntConstant(mod, "SZOMB", SZOMB)) INITERR; + if (PyModule_AddIntConstant(mod, "SWAIT", SWAIT)) INITERR; + if (PyModule_AddIntConstant(mod, "SLOCK", SLOCK)) INITERR; +#elif PSUTIL_OPENBSD + if (PyModule_AddIntConstant(mod, "SIDL", SIDL)) INITERR; + if (PyModule_AddIntConstant(mod, "SRUN", SRUN)) INITERR; + if (PyModule_AddIntConstant(mod, "SSLEEP", SSLEEP)) INITERR; + if (PyModule_AddIntConstant(mod, "SSTOP", SSTOP)) INITERR; + if (PyModule_AddIntConstant(mod, "SZOMB", SZOMB)) INITERR; // unused + if (PyModule_AddIntConstant(mod, "SDEAD", SDEAD)) INITERR; + if (PyModule_AddIntConstant(mod, "SONPROC", SONPROC)) INITERR; +#elif defined(PSUTIL_NETBSD) + if (PyModule_AddIntConstant(mod, "SIDL", LSIDL)) INITERR; + if (PyModule_AddIntConstant(mod, "SRUN", LSRUN)) INITERR; + if (PyModule_AddIntConstant(mod, "SSLEEP", LSSLEEP)) INITERR; + if (PyModule_AddIntConstant(mod, "SSTOP", LSSTOP)) INITERR; + if (PyModule_AddIntConstant(mod, "SZOMB", LSZOMB)) INITERR; + if (PyModule_AddIntConstant(mod, "SDEAD", LSDEAD)) INITERR; + if (PyModule_AddIntConstant(mod, "SONPROC", LSONPROC)) INITERR; + // unique to NetBSD + if (PyModule_AddIntConstant(mod, "SSUSPENDED", LSSUSPENDED)) INITERR; +#endif + + // connection status constants + if (PyModule_AddIntConstant(mod, "TCPS_CLOSED", TCPS_CLOSED)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_CLOSING", TCPS_CLOSING)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_CLOSE_WAIT", TCPS_CLOSE_WAIT)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_LISTEN", TCPS_LISTEN)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_ESTABLISHED", TCPS_ESTABLISHED)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_SYN_SENT", TCPS_SYN_SENT)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_SYN_RECEIVED", TCPS_SYN_RECEIVED)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_FIN_WAIT_1", TCPS_FIN_WAIT_1)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_FIN_WAIT_2", TCPS_FIN_WAIT_2)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_LAST_ACK", TCPS_LAST_ACK)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_TIME_WAIT", TCPS_TIME_WAIT)) + INITERR; + // PSUTIL_CONN_NONE + if (PyModule_AddIntConstant(mod, "PSUTIL_CONN_NONE", 128)) INITERR; + + psutil_setup(); + + if (mod == NULL) + INITERR; +#if PY_MAJOR_VERSION >= 3 + return mod; +#endif +} diff --git a/ddtrace/vendor/psutil/_psutil_common.c b/ddtrace/vendor/psutil/_psutil_common.c new file mode 100644 index 00000000000..c6e37bc22ed --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_common.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Routines common to all platforms. + */ + +#include +#ifdef _WIN32 +#include +#endif + +#include "_psutil_common.h" + +// Global vars. +int PSUTIL_DEBUG = 0; +int PSUTIL_TESTING = 0; + + +/* + * Backport of unicode FS APIs from Python 3. + * On Python 2 we just return a plain byte string + * which is never supposed to raise decoding errors. + * See: https://github.com/giampaolo/psutil/issues/1040 + */ +#if PY_MAJOR_VERSION < 3 +PyObject * +PyUnicode_DecodeFSDefault(char *s) { + return PyString_FromString(s); +} + + +PyObject * +PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { + return PyString_FromStringAndSize(s, size); +} +#endif + + +/* + * Set OSError(errno=ESRCH, strerror="No such process") Python exception. + * If msg != "" the exception message will change in accordance. + */ +PyObject * +NoSuchProcess(const char *msg) { + PyObject *exc; + exc = PyObject_CallFunction( + PyExc_OSError, "(is)", ESRCH, strlen(msg) ? msg : strerror(ESRCH)); + PyErr_SetObject(PyExc_OSError, exc); + Py_XDECREF(exc); + return NULL; +} + + +/* + * Same as PyErr_SetFromErrno(0) but adds the syscall to the exception + * message. + */ +PyObject * +PyErr_SetFromOSErrnoWithSyscall(const char *syscall) { + char fullmsg[1024]; + +#ifdef _WIN32 + sprintf(fullmsg, "(originated from %s)", syscall); + PyErr_SetFromWindowsErrWithFilename(GetLastError(), fullmsg); +#else + PyObject *exc; + sprintf(fullmsg, "%s (originated from %s)", strerror(errno), syscall); + exc = PyObject_CallFunction(PyExc_OSError, "(is)", errno, fullmsg); + PyErr_SetObject(PyExc_OSError, exc); + Py_XDECREF(exc); +#endif + return NULL; +} + + +/* + * Set OSError(errno=EACCES, strerror="Permission denied") Python exception. + * If msg != "" the exception message will change in accordance. + */ +PyObject * +AccessDenied(const char *msg) { + PyObject *exc; + exc = PyObject_CallFunction( + PyExc_OSError, "(is)", EACCES, strlen(msg) ? msg : strerror(EACCES)); + PyErr_SetObject(PyExc_OSError, exc); + Py_XDECREF(exc); + return NULL; +} + + +/* + * Enable testing mode. This has the same effect as setting PSUTIL_TESTING + * env var. This dual method exists because updating os.environ on + * Windows has no effect. Called on unit tests setup. + */ +PyObject * +psutil_set_testing(PyObject *self, PyObject *args) { + PSUTIL_TESTING = 1; + Py_INCREF(Py_None); + return Py_None; +} + + +/* + * Print a debug message on stderr. No-op if PSUTIL_DEBUG env var is not set. + */ +void +psutil_debug(const char* format, ...) { + va_list argptr; + if (PSUTIL_DEBUG) { + va_start(argptr, format); + fprintf(stderr, "psutil-debug> "); + vfprintf(stderr, format, argptr); + fprintf(stderr, "\n"); + va_end(argptr); + } +} + + +/* + * Called on module import on all platforms. + */ +int +psutil_setup(void) { + if (getenv("PSUTIL_DEBUG") != NULL) + PSUTIL_DEBUG = 1; + if (getenv("PSUTIL_TESTING") != NULL) + PSUTIL_TESTING = 1; + return 0; +} diff --git a/ddtrace/vendor/psutil/_psutil_common.h b/ddtrace/vendor/psutil/_psutil_common.h new file mode 100644 index 00000000000..7f58ad17382 --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_common.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef PSUTIL_PSUTIL_COMMON_H +#define PSUTIL_PSUTIL_COMMON_H + +#include + +extern int PSUTIL_TESTING; +extern int PSUTIL_DEBUG; + +// a signaler for connections without an actual status +static const int PSUTIL_CONN_NONE = 128; + +#if PY_MAJOR_VERSION < 3 +PyObject* PyUnicode_DecodeFSDefault(char *s); +PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); +#endif + +PyObject* AccessDenied(const char *msg); +PyObject* NoSuchProcess(const char *msg); +PyObject* PyErr_SetFromOSErrnoWithSyscall(const char *syscall); + +PyObject* psutil_set_testing(PyObject *self, PyObject *args); +void psutil_debug(const char* format, ...); +int psutil_setup(void); + +#endif // PSUTIL_PSUTIL_COMMON_H diff --git a/ddtrace/vendor/psutil/_psutil_linux.c b/ddtrace/vendor/psutil/_psutil_linux.c new file mode 100644 index 00000000000..0d16eb42761 --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_linux.c @@ -0,0 +1,668 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Linux-specific functions. + */ + +#ifndef _GNU_SOURCE + #define _GNU_SOURCE 1 +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// see: https://github.com/giampaolo/psutil/issues/659 +#ifdef PSUTIL_ETHTOOL_MISSING_TYPES + #include + typedef __u64 u64; + typedef __u32 u32; + typedef __u16 u16; + typedef __u8 u8; +#endif +/* Avoid redefinition of struct sysinfo with musl libc */ +#define _LINUX_SYSINFO_H +#include + +/* The minimum number of CPUs allocated in a cpu_set_t */ +static const int NCPUS_START = sizeof(unsigned long) * CHAR_BIT; + +// Linux >= 2.6.13 +#define PSUTIL_HAVE_IOPRIO defined(__NR_ioprio_get) && defined(__NR_ioprio_set) + +// Linux >= 2.6.36 (supposedly) and glibc >= 13 +#define PSUTIL_HAVE_PRLIMIT \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)) && \ + (__GLIBC__ >= 2 && __GLIBC_MINOR__ >= 13) && \ + defined(__NR_prlimit64) + +#if PSUTIL_HAVE_PRLIMIT + #define _FILE_OFFSET_BITS 64 + #include + #include +#endif + +// Should exist starting from CentOS 6 (year 2011). +#ifdef CPU_ALLOC + #define PSUTIL_HAVE_CPU_AFFINITY +#endif + +#include "_psutil_common.h" +#include "_psutil_posix.h" + +// May happen on old RedHat versions, see: +// https://github.com/giampaolo/psutil/issues/607 +#ifndef DUPLEX_UNKNOWN + #define DUPLEX_UNKNOWN 0xff +#endif + + +#if PSUTIL_HAVE_IOPRIO +enum { + IOPRIO_WHO_PROCESS = 1, +}; + +static inline int +ioprio_get(int which, int who) { + return syscall(__NR_ioprio_get, which, who); +} + +static inline int +ioprio_set(int which, int who, int ioprio) { + return syscall(__NR_ioprio_set, which, who, ioprio); +} + +#define IOPRIO_CLASS_SHIFT 13 +#define IOPRIO_PRIO_MASK ((1UL << IOPRIO_CLASS_SHIFT) - 1) + +#define IOPRIO_PRIO_CLASS(mask) ((mask) >> IOPRIO_CLASS_SHIFT) +#define IOPRIO_PRIO_DATA(mask) ((mask) & IOPRIO_PRIO_MASK) +#define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data) + + +/* + * Return a (ioclass, iodata) Python tuple representing process I/O priority. + */ +static PyObject * +psutil_proc_ioprio_get(PyObject *self, PyObject *args) { + long pid; + int ioprio, ioclass, iodata; + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + ioprio = ioprio_get(IOPRIO_WHO_PROCESS, pid); + if (ioprio == -1) + return PyErr_SetFromErrno(PyExc_OSError); + ioclass = IOPRIO_PRIO_CLASS(ioprio); + iodata = IOPRIO_PRIO_DATA(ioprio); + return Py_BuildValue("ii", ioclass, iodata); +} + + +/* + * A wrapper around ioprio_set(); sets process I/O priority. + * ioclass can be either IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE + * or 0. iodata goes from 0 to 7 depending on ioclass specified. + */ +static PyObject * +psutil_proc_ioprio_set(PyObject *self, PyObject *args) { + long pid; + int ioprio, ioclass, iodata; + int retval; + + if (! PyArg_ParseTuple(args, "lii", &pid, &ioclass, &iodata)) + return NULL; + ioprio = IOPRIO_PRIO_VALUE(ioclass, iodata); + retval = ioprio_set(IOPRIO_WHO_PROCESS, pid, ioprio); + if (retval == -1) + return PyErr_SetFromErrno(PyExc_OSError); + Py_RETURN_NONE; +} +#endif + + +#if PSUTIL_HAVE_PRLIMIT +/* + * A wrapper around prlimit(2); sets process resource limits. + * This can be used for both get and set, in which case extra + * 'soft' and 'hard' args must be provided. + */ +static PyObject * +psutil_linux_prlimit(PyObject *self, PyObject *args) { + long pid; + int ret, resource; + struct rlimit old, new; + struct rlimit *newp = NULL; + PyObject *py_soft = NULL; + PyObject *py_hard = NULL; + + if (! PyArg_ParseTuple(args, "li|OO", &pid, &resource, &py_soft, &py_hard)) + return NULL; + + // get + if (py_soft == NULL && py_hard == NULL) { + ret = prlimit(pid, resource, NULL, &old); + if (ret == -1) + return PyErr_SetFromErrno(PyExc_OSError); +#if defined(PSUTIL_HAVE_LONG_LONG) + if (sizeof(old.rlim_cur) > sizeof(long)) { + return Py_BuildValue("LL", + (PY_LONG_LONG)old.rlim_cur, + (PY_LONG_LONG)old.rlim_max); + } +#endif + return Py_BuildValue("ll", (long)old.rlim_cur, (long)old.rlim_max); + } + + // set + else { +#if defined(PSUTIL_HAVE_LARGEFILE_SUPPORT) + new.rlim_cur = PyLong_AsLongLong(py_soft); + if (new.rlim_cur == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; + new.rlim_max = PyLong_AsLongLong(py_hard); + if (new.rlim_max == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; +#else + new.rlim_cur = PyLong_AsLong(py_soft); + if (new.rlim_cur == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; + new.rlim_max = PyLong_AsLong(py_hard); + if (new.rlim_max == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; +#endif + newp = &new; + ret = prlimit(pid, resource, newp, &old); + if (ret == -1) + return PyErr_SetFromErrno(PyExc_OSError); + Py_RETURN_NONE; + } +} +#endif + + +/* + * Return disk mounted partitions as a list of tuples including device, + * mount point and filesystem type + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + FILE *file = NULL; + struct mntent *entry; + const char *mtab_path; + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + if (!PyArg_ParseTuple(args, "s", &mtab_path)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + file = setmntent(mtab_path, "r"); + Py_END_ALLOW_THREADS + if ((file == 0) || (file == NULL)) { + psutil_debug("setmntent() failed"); + PyErr_SetFromErrnoWithFilename(PyExc_OSError, mtab_path); + goto error; + } + + while ((entry = getmntent(file))) { + if (entry == NULL) { + PyErr_Format(PyExc_RuntimeError, "getmntent() syscall failed"); + goto error; + } + py_dev = PyUnicode_DecodeFSDefault(entry->mnt_fsname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(entry->mnt_dir); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue("(OOss)", + py_dev, // device + py_mountp, // mount point + entry->mnt_type, // fs type + entry->mnt_opts); // options + if (! py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_dev); + Py_CLEAR(py_mountp); + Py_CLEAR(py_tuple); + } + endmntent(file); + return py_retlist; + +error: + if (file != NULL) + endmntent(file); + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + return NULL; +} + + +/* + * A wrapper around sysinfo(), return system memory usage statistics. + */ +static PyObject * +psutil_linux_sysinfo(PyObject *self, PyObject *args) { + struct sysinfo info; + + if (sysinfo(&info) != 0) + return PyErr_SetFromErrno(PyExc_OSError); + // note: boot time might also be determined from here + return Py_BuildValue( + "(kkkkkkI)", + info.totalram, // total + info.freeram, // free + info.bufferram, // buffer + info.sharedram, // shared + info.totalswap, // swap tot + info.freeswap, // swap free + info.mem_unit // multiplier + ); +} + + +/* + * Return process CPU affinity as a Python list + */ +#ifdef PSUTIL_HAVE_CPU_AFFINITY + +static PyObject * +psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { + int cpu, ncpus, count, cpucount_s; + long pid; + size_t setsize; + cpu_set_t *mask = NULL; + PyObject *py_list = NULL; + + if (!PyArg_ParseTuple(args, "l", &pid)) + return NULL; + ncpus = NCPUS_START; + while (1) { + setsize = CPU_ALLOC_SIZE(ncpus); + mask = CPU_ALLOC(ncpus); + if (mask == NULL) { + psutil_debug("CPU_ALLOC() failed"); + return PyErr_NoMemory(); + } + if (sched_getaffinity(pid, setsize, mask) == 0) + break; + CPU_FREE(mask); + if (errno != EINVAL) + return PyErr_SetFromErrno(PyExc_OSError); + if (ncpus > INT_MAX / 2) { + PyErr_SetString(PyExc_OverflowError, "could not allocate " + "a large enough CPU set"); + return NULL; + } + ncpus = ncpus * 2; + } + + py_list = PyList_New(0); + if (py_list == NULL) + goto error; + + cpucount_s = CPU_COUNT_S(setsize, mask); + for (cpu = 0, count = cpucount_s; count; cpu++) { + if (CPU_ISSET_S(cpu, setsize, mask)) { +#if PY_MAJOR_VERSION >= 3 + PyObject *cpu_num = PyLong_FromLong(cpu); +#else + PyObject *cpu_num = PyInt_FromLong(cpu); +#endif + if (cpu_num == NULL) + goto error; + if (PyList_Append(py_list, cpu_num)) { + Py_DECREF(cpu_num); + goto error; + } + Py_DECREF(cpu_num); + --count; + } + } + CPU_FREE(mask); + return py_list; + +error: + if (mask) + CPU_FREE(mask); + Py_XDECREF(py_list); + return NULL; +} + + +/* + * Set process CPU affinity; expects a bitmask + */ +static PyObject * +psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { + cpu_set_t cpu_set; + size_t len; + long pid; + int i, seq_len; + PyObject *py_cpu_set; + PyObject *py_cpu_seq = NULL; + + if (!PyArg_ParseTuple(args, "lO", &pid, &py_cpu_set)) + return NULL; + + if (!PySequence_Check(py_cpu_set)) { + PyErr_Format(PyExc_TypeError, "sequence argument expected, got %s", + Py_TYPE(py_cpu_set)->tp_name); + goto error; + } + + py_cpu_seq = PySequence_Fast(py_cpu_set, "expected a sequence or integer"); + if (!py_cpu_seq) + goto error; + seq_len = PySequence_Fast_GET_SIZE(py_cpu_seq); + CPU_ZERO(&cpu_set); + for (i = 0; i < seq_len; i++) { + PyObject *item = PySequence_Fast_GET_ITEM(py_cpu_seq, i); +#if PY_MAJOR_VERSION >= 3 + long value = PyLong_AsLong(item); +#else + long value = PyInt_AsLong(item); +#endif + if ((value == -1) || PyErr_Occurred()) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, "invalid CPU value"); + goto error; + } + CPU_SET(value, &cpu_set); + } + + len = sizeof(cpu_set); + if (sched_setaffinity(pid, len, &cpu_set)) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + Py_DECREF(py_cpu_seq); + Py_RETURN_NONE; + +error: + if (py_cpu_seq != NULL) + Py_DECREF(py_cpu_seq); + return NULL; +} +#endif /* PSUTIL_HAVE_CPU_AFFINITY */ + + +/* + * Return currently connected users as a list of tuples. + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + struct utmp *ut; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; + PyObject *py_user_proc = NULL; + + if (py_retlist == NULL) + return NULL; + setutent(); + while (NULL != (ut = getutent())) { + py_tuple = NULL; + py_user_proc = NULL; + if (ut->ut_type == USER_PROCESS) + py_user_proc = Py_True; + else + py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut->ut_tv.tv_sec, // tstamp + py_user_proc, // (bool) user process + ut->ut_pid // process id + ); + if (! py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_username); + Py_CLEAR(py_tty); + Py_CLEAR(py_hostname); + Py_CLEAR(py_tuple); + } + endutent(); + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + endutent(); + return NULL; +} + + +/* + * Return stats about a particular network + * interface. References: + * https://github.com/dpaleino/wicd/blob/master/wicd/backends/be-ioctl.py + * http://www.i-scream.org/libstatgrab/ + */ +static PyObject* +psutil_net_if_duplex_speed(PyObject* self, PyObject* args) { + char *nic_name; + int sock = 0; + int ret; + int duplex; + int speed; + struct ifreq ifr; + struct ethtool_cmd ethcmd; + PyObject *py_retlist = NULL; + + if (! PyArg_ParseTuple(args, "s", &nic_name)) + return NULL; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) + return PyErr_SetFromOSErrnoWithSyscall("socket()"); + strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + + // duplex and speed + memset(ðcmd, 0, sizeof ethcmd); + ethcmd.cmd = ETHTOOL_GSET; + ifr.ifr_data = (void *)ðcmd; + ret = ioctl(sock, SIOCETHTOOL, &ifr); + + if (ret != -1) { + duplex = ethcmd.duplex; + speed = ethcmd.speed; + } + else { + if ((errno == EOPNOTSUPP) || (errno == EINVAL)) { + // EOPNOTSUPP may occur in case of wi-fi cards. + // For EINVAL see: + // https://github.com/giampaolo/psutil/issues/797 + // #issuecomment-202999532 + duplex = DUPLEX_UNKNOWN; + speed = 0; + } + else { + PyErr_SetFromOSErrnoWithSyscall("ioctl(SIOCETHTOOL)"); + goto error; + } + } + + py_retlist = Py_BuildValue("[ii]", duplex, speed); + if (!py_retlist) + goto error; + close(sock); + return py_retlist; + +error: + if (sock != -1) + close(sock); + return NULL; +} + + +/* + * Module init. + */ + +static PyMethodDef mod_methods[] = { + // --- per-process functions + +#ifdef PSUTIL_HAVE_IOPRIO + {"proc_ioprio_get", psutil_proc_ioprio_get, METH_VARARGS, + "Get process I/O priority"}, + {"proc_ioprio_set", psutil_proc_ioprio_set, METH_VARARGS, + "Set process I/O priority"}, +#endif +#ifdef PSUTIL_HAVE_CPU_AFFINITY + {"proc_cpu_affinity_get", psutil_proc_cpu_affinity_get, METH_VARARGS, + "Return process CPU affinity as a Python long (the bitmask)."}, + {"proc_cpu_affinity_set", psutil_proc_cpu_affinity_set, METH_VARARGS, + "Set process CPU affinity; expects a bitmask."}, +#endif + + // --- system related functions + + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return disk mounted partitions as a list of tuples including " + "device, mount point and filesystem type"}, + {"users", psutil_users, METH_VARARGS, + "Return currently connected users as a list of tuples"}, + {"net_if_duplex_speed", psutil_net_if_duplex_speed, METH_VARARGS, + "Return duplex and speed info about a NIC"}, + + // --- linux specific + + {"linux_sysinfo", psutil_linux_sysinfo, METH_VARARGS, + "A wrapper around sysinfo(), return system memory usage statistics"}, +#if PSUTIL_HAVE_PRLIMIT + {"linux_prlimit", psutil_linux_prlimit, METH_VARARGS, + "Get or set process resource limits."}, +#endif + // --- others + {"set_testing", psutil_set_testing, METH_NOARGS, + "Set psutil in testing mode"}, + + {NULL, NULL, 0, NULL} +}; + + +#if PY_MAJOR_VERSION >= 3 + #define INITERR return NULL + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_linux", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL + }; + + PyObject *PyInit__psutil_linux(void) +#else /* PY_MAJOR_VERSION */ + #define INITERR return + + void init_psutil_linux(void) +#endif /* PY_MAJOR_VERSION */ +{ + PyObject *v; +#if PY_MAJOR_VERSION >= 3 + PyObject *mod = PyModule_Create(&moduledef); +#else + PyObject *mod = Py_InitModule("_psutil_linux", mod_methods); +#endif + if (mod == NULL) + INITERR; + + if (PyModule_AddIntConstant(mod, "version", PSUTIL_VERSION)) INITERR; +#if PSUTIL_HAVE_PRLIMIT + if (PyModule_AddIntConstant(mod, "RLIMIT_AS", RLIMIT_AS)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_CORE", RLIMIT_CORE)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_CPU", RLIMIT_CPU)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_DATA", RLIMIT_DATA)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_FSIZE", RLIMIT_FSIZE)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_LOCKS", RLIMIT_LOCKS)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_MEMLOCK", RLIMIT_MEMLOCK)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_NOFILE", RLIMIT_NOFILE)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_NPROC", RLIMIT_NPROC)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_RSS", RLIMIT_RSS)) INITERR; + if (PyModule_AddIntConstant(mod, "RLIMIT_STACK", RLIMIT_STACK)) INITERR; + +#if defined(HAVE_LONG_LONG) + if (sizeof(RLIM_INFINITY) > sizeof(long)) { + v = PyLong_FromLongLong((PY_LONG_LONG) RLIM_INFINITY); + } else +#endif + { + v = PyLong_FromLong((long) RLIM_INFINITY); + } + if (v) { + PyModule_AddObject(mod, "RLIM_INFINITY", v); + } + +#ifdef RLIMIT_MSGQUEUE + if (PyModule_AddIntConstant(mod, "RLIMIT_MSGQUEUE", RLIMIT_MSGQUEUE)) INITERR; +#endif +#ifdef RLIMIT_NICE + if (PyModule_AddIntConstant(mod, "RLIMIT_NICE", RLIMIT_NICE)) INITERR; +#endif +#ifdef RLIMIT_RTPRIO + if (PyModule_AddIntConstant(mod, "RLIMIT_RTPRIO", RLIMIT_RTPRIO)) INITERR; +#endif +#ifdef RLIMIT_RTTIME + if (PyModule_AddIntConstant(mod, "RLIMIT_RTTIME", RLIMIT_RTTIME)) INITERR; +#endif +#ifdef RLIMIT_SIGPENDING + if (PyModule_AddIntConstant(mod, "RLIMIT_SIGPENDING", RLIMIT_SIGPENDING)) + INITERR; +#endif +#endif + if (PyModule_AddIntConstant(mod, "DUPLEX_HALF", DUPLEX_HALF)) INITERR; + if (PyModule_AddIntConstant(mod, "DUPLEX_FULL", DUPLEX_FULL)) INITERR; + if (PyModule_AddIntConstant(mod, "DUPLEX_UNKNOWN", DUPLEX_UNKNOWN)) INITERR; + + if (mod == NULL) + INITERR; +#if PY_MAJOR_VERSION >= 3 + return mod; +#endif +} diff --git a/ddtrace/vendor/psutil/_psutil_osx.c b/ddtrace/vendor/psutil/_psutil_osx.c new file mode 100644 index 00000000000..76ec0ee850a --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_osx.c @@ -0,0 +1,1905 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * macOS platform-specific module methods. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "_psutil_common.h" +#include "_psutil_posix.h" +#include "arch/osx/process_info.h" + + +#define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) + +static PyObject *ZombieProcessError; + + +/* + * A wrapper around host_statistics() invoked with HOST_VM_INFO. + */ +int +psutil_sys_vminfo(vm_statistics_data_t *vmstat) { + kern_return_t ret; + mach_msg_type_number_t count = sizeof(*vmstat) / sizeof(integer_t); + mach_port_t mport = mach_host_self(); + + ret = host_statistics(mport, HOST_VM_INFO, (host_info_t)vmstat, &count); + if (ret != KERN_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "host_statistics(HOST_VM_INFO) syscall failed: %s", + mach_error_string(ret)); + return 0; + } + mach_port_deallocate(mach_task_self(), mport); + return 1; +} + + +/* + * A wrapper around task_for_pid() which sucks big time: + * - it's not documented + * - errno is set only sometimes + * - sometimes errno is ENOENT (?!?) + * - for PIDs != getpid() or PIDs which are not members of the procmod + * it requires root + * As such we can only guess what the heck went wrong and fail either + * with NoSuchProcess, ZombieProcessError or giveup with AccessDenied. + * Here's some history: + * https://github.com/giampaolo/psutil/issues/1181 + * https://github.com/giampaolo/psutil/issues/1209 + * https://github.com/giampaolo/psutil/issues/1291#issuecomment-396062519 + */ +int +psutil_task_for_pid(long pid, mach_port_t *task) +{ + // See: https://github.com/giampaolo/psutil/issues/1181 + kern_return_t err = KERN_SUCCESS; + + err = task_for_pid(mach_task_self(), (pid_t)pid, task); + if (err != KERN_SUCCESS) { + if (psutil_pid_exists(pid) == 0) + NoSuchProcess("task_for_pid() failed"); + else if (psutil_is_zombie(pid) == 1) + PyErr_SetString(ZombieProcessError, "task_for_pid() failed"); + else { + psutil_debug( + "task_for_pid() failed (pid=%ld, err=%i, errno=%i, msg='%s'); " + "setting AccessDenied()", + pid, err, errno, mach_error_string(err)); + AccessDenied("task_for_pid() failed"); + } + return 1; + } + return 0; +} + + +/* + * Return a Python list of all the PIDs running on the system. + */ +static PyObject * +psutil_pids(PyObject *self, PyObject *args) { + kinfo_proc *proclist = NULL; + kinfo_proc *orig_address = NULL; + size_t num_processes; + size_t idx; + PyObject *py_pid = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + if (psutil_get_proc_list(&proclist, &num_processes) != 0) + goto error; + + // save the address of proclist so we can free it later + orig_address = proclist; + for (idx = 0; idx < num_processes; idx++) { + py_pid = Py_BuildValue("i", proclist->kp_proc.p_pid); + if (! py_pid) + goto error; + if (PyList_Append(py_retlist, py_pid)) + goto error; + Py_CLEAR(py_pid); + proclist++; + } + free(orig_address); + + return py_retlist; + +error: + Py_XDECREF(py_pid); + Py_DECREF(py_retlist); + if (orig_address != NULL) + free(orig_address); + return NULL; +} + + +/* + * Return multiple process info as a Python tuple in one shot by + * using sysctl() and filling up a kinfo_proc struct. + * It should be possible to do this for all processes without + * incurring into permission (EPERM) errors. + * This will also succeed for zombie processes returning correct + * information. + */ +static PyObject * +psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args) { + long pid; + struct kinfo_proc kp; + PyObject *py_name; + PyObject *py_retlist; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_get_kinfo_proc(pid, &kp) == -1) + return NULL; + + py_name = PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); + if (! py_name) { + // Likely a decoding error. We don't want to fail the whole + // operation. The python module may retry with proc_name(). + PyErr_Clear(); + py_name = Py_None; + } + + py_retlist = Py_BuildValue( + "lllllllidiO", + (long)kp.kp_eproc.e_ppid, // (long) ppid + (long)kp.kp_eproc.e_pcred.p_ruid, // (long) real uid + (long)kp.kp_eproc.e_ucred.cr_uid, // (long) effective uid + (long)kp.kp_eproc.e_pcred.p_svuid, // (long) saved uid + (long)kp.kp_eproc.e_pcred.p_rgid, // (long) real gid + (long)kp.kp_eproc.e_ucred.cr_groups[0], // (long) effective gid + (long)kp.kp_eproc.e_pcred.p_svgid, // (long) saved gid + kp.kp_eproc.e_tdev, // (int) tty nr + PSUTIL_TV2DOUBLE(kp.kp_proc.p_starttime), // (double) create time + (int)kp.kp_proc.p_stat, // (int) status + py_name // (pystr) name + ); + + if (py_retlist != NULL) { + // XXX shall we decref() also in case of Py_BuildValue() error? + Py_DECREF(py_name); + } + return py_retlist; +} + + +/* + * Return multiple process info as a Python tuple in one shot by + * using proc_pidinfo(PROC_PIDTASKINFO) and filling a proc_taskinfo + * struct. + * Contrarily from proc_kinfo above this function will fail with + * EACCES for PIDs owned by another user and with ESRCH for zombie + * processes. + */ +static PyObject * +psutil_proc_pidtaskinfo_oneshot(PyObject *self, PyObject *args) { + long pid; + struct proc_taskinfo pti; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &pti, sizeof(pti)) <= 0) + return NULL; + + return Py_BuildValue( + "(ddKKkkkk)", + (float)pti.pti_total_user / 1000000000.0, // (float) cpu user time + (float)pti.pti_total_system / 1000000000.0, // (float) cpu sys time + // Note about memory: determining other mem stats on macOS is a mess: + // http://www.opensource.apple.com/source/top/top-67/libtop.c?txt + // I just give up. + // struct proc_regioninfo pri; + // psutil_proc_pidinfo(pid, PROC_PIDREGIONINFO, 0, &pri, sizeof(pri)) + pti.pti_resident_size, // (uns long long) rss + pti.pti_virtual_size, // (uns long long) vms + pti.pti_faults, // (uns long) number of page faults (pages) + pti.pti_pageins, // (uns long) number of actual pageins (pages) + pti.pti_threadnum, // (uns long) num threads + // Unvoluntary value seems not to be available; + // pti.pti_csw probably refers to the sum of the two; + // getrusage() numbers seems to confirm this theory. + pti.pti_csw // (uns long) voluntary ctx switches + ); +} + + +/* + * Return process name from kinfo_proc as a Python string. + */ +static PyObject * +psutil_proc_name(PyObject *self, PyObject *args) { + long pid; + struct kinfo_proc kp; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_get_kinfo_proc(pid, &kp) == -1) + return NULL; + return PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); +} + + +/* + * Return process current working directory. + * Raises NSP in case of zombie process. + */ +static PyObject * +psutil_proc_cwd(PyObject *self, PyObject *args) { + long pid; + struct proc_vnodepathinfo pathinfo; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + if (psutil_proc_pidinfo( + pid, PROC_PIDVNODEPATHINFO, 0, &pathinfo, sizeof(pathinfo)) <= 0) + { + return NULL; + } + + return PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); +} + + +/* + * Return path of the process executable. + */ +static PyObject * +psutil_proc_exe(PyObject *self, PyObject *args) { + long pid; + char buf[PATH_MAX]; + int ret; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + errno = 0; + ret = proc_pidpath((pid_t)pid, &buf, sizeof(buf)); + if (ret == 0) { + if (pid == 0) + AccessDenied(""); + else + psutil_raise_for_pid(pid, "proc_pidpath()"); + return NULL; + } + return PyUnicode_DecodeFSDefault(buf); +} + + +/* + * Return process cmdline as a Python list of cmdline arguments. + */ +static PyObject * +psutil_proc_cmdline(PyObject *self, PyObject *args) { + long pid; + PyObject *py_retlist = NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + // get the commandline, defined in arch/osx/process_info.c + py_retlist = psutil_get_cmdline(pid); + return py_retlist; +} + + +/* + * Return process environment as a Python string. + */ +static PyObject * +psutil_proc_environ(PyObject *self, PyObject *args) { + long pid; + PyObject *py_retdict = NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + // get the environment block, defined in arch/osx/process_info.c + py_retdict = psutil_get_environ(pid); + return py_retdict; +} + + +/* + * Return the number of logical CPUs in the system. + * XXX this could be shared with BSD. + */ +static PyObject * +psutil_cpu_count_logical(PyObject *self, PyObject *args) { + /* + int mib[2]; + int ncpu; + size_t len; + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + len = sizeof(ncpu); + + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", ncpu); + */ + int num; + size_t size = sizeof(int); + + if (sysctlbyname("hw.logicalcpu", &num, &size, NULL, 2)) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", num); +} + + +/* + * Return the number of physical CPUs in the system. + */ +static PyObject * +psutil_cpu_count_phys(PyObject *self, PyObject *args) { + int num; + size_t size = sizeof(int); + + if (sysctlbyname("hw.physicalcpu", &num, &size, NULL, 0)) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", num); +} + + +/* + * Indicates if the given virtual address on the given architecture is in the + * shared VM region. + */ +static bool +psutil_in_shared_region(mach_vm_address_t addr, cpu_type_t type) { + mach_vm_address_t base; + mach_vm_address_t size; + + switch (type) { + case CPU_TYPE_ARM: + base = SHARED_REGION_BASE_ARM; + size = SHARED_REGION_SIZE_ARM; + break; + case CPU_TYPE_I386: + base = SHARED_REGION_BASE_I386; + size = SHARED_REGION_SIZE_I386; + break; + case CPU_TYPE_X86_64: + base = SHARED_REGION_BASE_X86_64; + size = SHARED_REGION_SIZE_X86_64; + break; + default: + return false; + } + + return base <= addr && addr < (base + size); +} + + +/* + * Returns the USS (unique set size) of the process. Reference: + * https://dxr.mozilla.org/mozilla-central/source/xpcom/base/ + * nsMemoryReporterManager.cpp + */ +static PyObject * +psutil_proc_memory_uss(PyObject *self, PyObject *args) { + long pid; + size_t len; + cpu_type_t cpu_type; + size_t private_pages = 0; + mach_vm_size_t size = 0; + mach_msg_type_number_t info_count = VM_REGION_TOP_INFO_COUNT; + kern_return_t kr; + vm_size_t page_size; + mach_vm_address_t addr = MACH_VM_MIN_ADDRESS; + mach_port_t task = MACH_PORT_NULL; + vm_region_top_info_data_t info; + mach_port_t object_name; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + if (psutil_task_for_pid(pid, &task) != 0) + return NULL; + + len = sizeof(cpu_type); + if (sysctlbyname("sysctl.proc_cputype", &cpu_type, &len, NULL, 0) != 0) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('sysctl.proc_cputype')"); + } + + // Roughly based on libtop_update_vm_regions in + // http://www.opensource.apple.com/source/top/top-100.1.2/libtop.c + for (addr = 0; ; addr += size) { + kr = mach_vm_region( + task, &addr, &size, VM_REGION_TOP_INFO, (vm_region_info_t)&info, + &info_count, &object_name); + if (kr == KERN_INVALID_ADDRESS) { + // Done iterating VM regions. + break; + } + else if (kr != KERN_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "mach_vm_region(VM_REGION_TOP_INFO) syscall failed"); + return NULL; + } + + if (psutil_in_shared_region(addr, cpu_type) && + info.share_mode != SM_PRIVATE) { + continue; + } + + switch (info.share_mode) { +#ifdef SM_LARGE_PAGE + case SM_LARGE_PAGE: + // NB: Large pages are not shareable and always resident. +#endif + case SM_PRIVATE: + private_pages += info.private_pages_resident; + private_pages += info.shared_pages_resident; + break; + case SM_COW: + private_pages += info.private_pages_resident; + if (info.ref_count == 1) { + // Treat copy-on-write pages as private if they only + // have one reference. + private_pages += info.shared_pages_resident; + } + break; + case SM_SHARED: + default: + break; + } + } + + mach_port_deallocate(mach_task_self(), task); + + if (host_page_size(mach_host_self(), &page_size) != KERN_SUCCESS) + page_size = PAGE_SIZE; + + return Py_BuildValue("K", private_pages * page_size); +} + + +/* + * Return system virtual memory stats. + * See: + * https://opensource.apple.com/source/system_cmds/system_cmds-790/ + * vm_stat.tproj/vm_stat.c.auto.html + */ +static PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + int mib[2]; + uint64_t total; + size_t len = sizeof(total); + vm_statistics_data_t vm; + int pagesize = getpagesize(); + // physical mem + mib[0] = CTL_HW; + mib[1] = HW_MEMSIZE; + + // This is also available as sysctlbyname("hw.memsize"). + if (sysctl(mib, 2, &total, &len, NULL, 0)) { + if (errno != 0) + PyErr_SetFromErrno(PyExc_OSError); + else + PyErr_Format( + PyExc_RuntimeError, "sysctl(HW_MEMSIZE) syscall failed"); + return NULL; + } + + // vm + if (!psutil_sys_vminfo(&vm)) + return NULL; + + return Py_BuildValue( + "KKKKKK", + total, + (unsigned long long) vm.active_count * pagesize, // active + (unsigned long long) vm.inactive_count * pagesize, // inactive + (unsigned long long) vm.wire_count * pagesize, // wired + (unsigned long long) vm.free_count * pagesize, // free + (unsigned long long) vm.speculative_count * pagesize // speculative + ); +} + + +/* + * Return stats about swap memory. + */ +static PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { + int mib[2]; + size_t size; + struct xsw_usage totals; + vm_statistics_data_t vmstat; + int pagesize = getpagesize(); + + mib[0] = CTL_VM; + mib[1] = VM_SWAPUSAGE; + size = sizeof(totals); + if (sysctl(mib, 2, &totals, &size, NULL, 0) == -1) { + if (errno != 0) + PyErr_SetFromErrno(PyExc_OSError); + else + PyErr_Format( + PyExc_RuntimeError, "sysctl(VM_SWAPUSAGE) syscall failed"); + return NULL; + } + if (!psutil_sys_vminfo(&vmstat)) + return NULL; + + return Py_BuildValue( + "LLLKK", + totals.xsu_total, + totals.xsu_used, + totals.xsu_avail, + (unsigned long long)vmstat.pageins * pagesize, + (unsigned long long)vmstat.pageouts * pagesize); +} + + +/* + * Return a Python tuple representing user, kernel and idle CPU times + */ +static PyObject * +psutil_cpu_times(PyObject *self, PyObject *args) { + mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT; + kern_return_t error; + host_cpu_load_info_data_t r_load; + + mach_port_t host_port = mach_host_self(); + error = host_statistics(host_port, HOST_CPU_LOAD_INFO, + (host_info_t)&r_load, &count); + if (error != KERN_SUCCESS) { + return PyErr_Format( + PyExc_RuntimeError, + "host_statistics(HOST_CPU_LOAD_INFO) syscall failed: %s", + mach_error_string(error)); + } + mach_port_deallocate(mach_task_self(), host_port); + + return Py_BuildValue( + "(dddd)", + (double)r_load.cpu_ticks[CPU_STATE_USER] / CLK_TCK, + (double)r_load.cpu_ticks[CPU_STATE_NICE] / CLK_TCK, + (double)r_load.cpu_ticks[CPU_STATE_SYSTEM] / CLK_TCK, + (double)r_load.cpu_ticks[CPU_STATE_IDLE] / CLK_TCK + ); +} + + +/* + * Return a Python list of tuple representing per-cpu times + */ +static PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + natural_t cpu_count; + natural_t i; + processor_info_array_t info_array; + mach_msg_type_number_t info_count; + kern_return_t error; + processor_cpu_load_info_data_t *cpu_load_info = NULL; + int ret; + PyObject *py_retlist = PyList_New(0); + PyObject *py_cputime = NULL; + + if (py_retlist == NULL) + return NULL; + + mach_port_t host_port = mach_host_self(); + error = host_processor_info(host_port, PROCESSOR_CPU_LOAD_INFO, + &cpu_count, &info_array, &info_count); + if (error != KERN_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "host_processor_info(PROCESSOR_CPU_LOAD_INFO) syscall failed: %s", + mach_error_string(error)); + goto error; + } + mach_port_deallocate(mach_task_self(), host_port); + + cpu_load_info = (processor_cpu_load_info_data_t *) info_array; + + for (i = 0; i < cpu_count; i++) { + py_cputime = Py_BuildValue( + "(dddd)", + (double)cpu_load_info[i].cpu_ticks[CPU_STATE_USER] / CLK_TCK, + (double)cpu_load_info[i].cpu_ticks[CPU_STATE_NICE] / CLK_TCK, + (double)cpu_load_info[i].cpu_ticks[CPU_STATE_SYSTEM] / CLK_TCK, + (double)cpu_load_info[i].cpu_ticks[CPU_STATE_IDLE] / CLK_TCK + ); + if (!py_cputime) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_CLEAR(py_cputime); + } + + ret = vm_deallocate(mach_task_self(), (vm_address_t)info_array, + info_count * sizeof(int)); + if (ret != KERN_SUCCESS) + PyErr_WarnEx(PyExc_RuntimeWarning, "vm_deallocate() failed", 2); + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + if (cpu_load_info != NULL) { + ret = vm_deallocate(mach_task_self(), (vm_address_t)info_array, + info_count * sizeof(int)); + if (ret != KERN_SUCCESS) + PyErr_WarnEx(PyExc_RuntimeWarning, "vm_deallocate() failed", 2); + } + return NULL; +} + + +/* + * Retrieve CPU frequency. + */ +static PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + int64_t curr; + int64_t min; + int64_t max; + size_t size = sizeof(int64_t); + + if (sysctlbyname("hw.cpufrequency", &curr, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('hw.cpufrequency')"); + } + if (sysctlbyname("hw.cpufrequency_min", &min, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('hw.cpufrequency_min')"); + } + if (sysctlbyname("hw.cpufrequency_max", &max, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('hw.cpufrequency_max')"); + } + + return Py_BuildValue( + "KKK", + curr / 1000 / 1000, + min / 1000 / 1000, + max / 1000 / 1000); +} + + +/* + * Return a Python float indicating the system boot time expressed in + * seconds since the epoch. + */ +static PyObject * +psutil_boot_time(PyObject *self, PyObject *args) { + // fetch sysctl "kern.boottime" + static int request[2] = { CTL_KERN, KERN_BOOTTIME }; + struct timeval result; + size_t result_len = sizeof result; + time_t boot_time = 0; + + if (sysctl(request, 2, &result, &result_len, NULL, 0) == -1) + return PyErr_SetFromErrno(PyExc_OSError); + boot_time = result.tv_sec; + return Py_BuildValue("f", (float)boot_time); +} + + +/* + * Return a list of tuples including device, mount point and fs type + * for all partitions mounted on the system. + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + int num; + int i; + int len; + uint64_t flags; + char opts[400]; + struct statfs *fs = NULL; + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + // get the number of mount points + Py_BEGIN_ALLOW_THREADS + num = getfsstat(NULL, 0, MNT_NOWAIT); + Py_END_ALLOW_THREADS + if (num == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + len = sizeof(*fs) * num; + fs = malloc(len); + if (fs == NULL) { + PyErr_NoMemory(); + goto error; + } + + Py_BEGIN_ALLOW_THREADS + num = getfsstat(fs, len, MNT_NOWAIT); + Py_END_ALLOW_THREADS + if (num == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < num; i++) { + opts[0] = 0; + flags = fs[i].f_flags; + + // see sys/mount.h + if (flags & MNT_RDONLY) + strlcat(opts, "ro", sizeof(opts)); + else + strlcat(opts, "rw", sizeof(opts)); + if (flags & MNT_SYNCHRONOUS) + strlcat(opts, ",sync", sizeof(opts)); + if (flags & MNT_NOEXEC) + strlcat(opts, ",noexec", sizeof(opts)); + if (flags & MNT_NOSUID) + strlcat(opts, ",nosuid", sizeof(opts)); + if (flags & MNT_UNION) + strlcat(opts, ",union", sizeof(opts)); + if (flags & MNT_ASYNC) + strlcat(opts, ",async", sizeof(opts)); + if (flags & MNT_EXPORTED) + strlcat(opts, ",exported", sizeof(opts)); + if (flags & MNT_QUARANTINE) + strlcat(opts, ",quarantine", sizeof(opts)); + if (flags & MNT_LOCAL) + strlcat(opts, ",local", sizeof(opts)); + if (flags & MNT_QUOTA) + strlcat(opts, ",quota", sizeof(opts)); + if (flags & MNT_ROOTFS) + strlcat(opts, ",rootfs", sizeof(opts)); + if (flags & MNT_DOVOLFS) + strlcat(opts, ",dovolfs", sizeof(opts)); + if (flags & MNT_DONTBROWSE) + strlcat(opts, ",dontbrowse", sizeof(opts)); + if (flags & MNT_IGNORE_OWNERSHIP) + strlcat(opts, ",ignore-ownership", sizeof(opts)); + if (flags & MNT_AUTOMOUNTED) + strlcat(opts, ",automounted", sizeof(opts)); + if (flags & MNT_JOURNALED) + strlcat(opts, ",journaled", sizeof(opts)); + if (flags & MNT_NOUSERXATTR) + strlcat(opts, ",nouserxattr", sizeof(opts)); + if (flags & MNT_DEFWRITE) + strlcat(opts, ",defwrite", sizeof(opts)); + if (flags & MNT_MULTILABEL) + strlcat(opts, ",multilabel", sizeof(opts)); + if (flags & MNT_NOATIME) + strlcat(opts, ",noatime", sizeof(opts)); + if (flags & MNT_UPDATE) + strlcat(opts, ",update", sizeof(opts)); + if (flags & MNT_RELOAD) + strlcat(opts, ",reload", sizeof(opts)); + if (flags & MNT_FORCE) + strlcat(opts, ",force", sizeof(opts)); + if (flags & MNT_CMDFLAGS) + strlcat(opts, ",cmdflags", sizeof(opts)); + + py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue( + "(OOss)", + py_dev, // device + py_mountp, // mount point + fs[i].f_fstypename, // fs type + opts); // options + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_dev); + Py_CLEAR(py_mountp); + Py_CLEAR(py_tuple); + } + + free(fs); + return py_retlist; + +error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (fs != NULL) + free(fs); + return NULL; +} + + +/* + * Return process threads + */ +static PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + long pid; + int err, ret; + kern_return_t kr; + unsigned int info_count = TASK_BASIC_INFO_COUNT; + mach_port_t task = MACH_PORT_NULL; + struct task_basic_info tasks_info; + thread_act_port_array_t thread_list = NULL; + thread_info_data_t thinfo_basic; + thread_basic_info_t basic_info_th; + mach_msg_type_number_t thread_count, thread_info_count, j; + + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + if (psutil_task_for_pid(pid, &task) != 0) + goto error; + + info_count = TASK_BASIC_INFO_COUNT; + err = task_info(task, TASK_BASIC_INFO, (task_info_t)&tasks_info, + &info_count); + if (err != KERN_SUCCESS) { + // errcode 4 is "invalid argument" (access denied) + if (err == 4) { + AccessDenied(""); + } + else { + // otherwise throw a runtime error with appropriate error code + PyErr_Format(PyExc_RuntimeError, + "task_info(TASK_BASIC_INFO) syscall failed"); + } + goto error; + } + + err = task_threads(task, &thread_list, &thread_count); + if (err != KERN_SUCCESS) { + PyErr_Format(PyExc_RuntimeError, "task_threads() syscall failed"); + goto error; + } + + for (j = 0; j < thread_count; j++) { + thread_info_count = THREAD_INFO_MAX; + kr = thread_info(thread_list[j], THREAD_BASIC_INFO, + (thread_info_t)thinfo_basic, &thread_info_count); + if (kr != KERN_SUCCESS) { + PyErr_Format(PyExc_RuntimeError, + "thread_info(THREAD_BASIC_INFO) syscall failed"); + goto error; + } + + basic_info_th = (thread_basic_info_t)thinfo_basic; + py_tuple = Py_BuildValue( + "Iff", + j + 1, + basic_info_th->user_time.seconds + \ + (float)basic_info_th->user_time.microseconds / 1000000.0, + basic_info_th->system_time.seconds + \ + (float)basic_info_th->system_time.microseconds / 1000000.0 + ); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + + ret = vm_deallocate(task, (vm_address_t)thread_list, + thread_count * sizeof(int)); + if (ret != KERN_SUCCESS) + PyErr_WarnEx(PyExc_RuntimeWarning, "vm_deallocate() failed", 2); + + mach_port_deallocate(mach_task_self(), task); + + return py_retlist; + +error: + if (task != MACH_PORT_NULL) + mach_port_deallocate(mach_task_self(), task); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (thread_list != NULL) { + ret = vm_deallocate(task, (vm_address_t)thread_list, + thread_count * sizeof(int)); + if (ret != KERN_SUCCESS) + PyErr_WarnEx(PyExc_RuntimeWarning, "vm_deallocate() failed", 2); + } + return NULL; +} + + +/* + * Return process open files as a Python tuple. + * References: + * - lsof source code: http://goo.gl/SYW79 and http://goo.gl/m78fd + * - /usr/include/sys/proc_info.h + */ +static PyObject * +psutil_proc_open_files(PyObject *self, PyObject *args) { + long pid; + int pidinfo_result; + int iterations; + int i; + unsigned long nb; + + struct proc_fdinfo *fds_pointer = NULL; + struct proc_fdinfo *fdp_pointer; + struct vnode_fdinfowithpath vi; + + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_path = NULL; + + if (py_retlist == NULL) + return NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + pidinfo_result = psutil_proc_pidinfo(pid, PROC_PIDLISTFDS, 0, NULL, 0); + if (pidinfo_result <= 0) + goto error; + + fds_pointer = malloc(pidinfo_result); + if (fds_pointer == NULL) { + PyErr_NoMemory(); + goto error; + } + pidinfo_result = psutil_proc_pidinfo( + pid, PROC_PIDLISTFDS, 0, fds_pointer, pidinfo_result); + if (pidinfo_result <= 0) + goto error; + + iterations = (pidinfo_result / PROC_PIDLISTFD_SIZE); + + for (i = 0; i < iterations; i++) { + fdp_pointer = &fds_pointer[i]; + + if (fdp_pointer->proc_fdtype == PROX_FDTYPE_VNODE) { + errno = 0; + nb = proc_pidfdinfo((pid_t)pid, + fdp_pointer->proc_fd, + PROC_PIDFDVNODEPATHINFO, + &vi, + sizeof(vi)); + + // --- errors checking + if ((nb <= 0) || nb < sizeof(vi)) { + if ((errno == ENOENT) || (errno == EBADF)) { + // no such file or directory or bad file descriptor; + // let's assume the file has been closed or removed + continue; + } + else { + psutil_raise_for_pid( + pid, "proc_pidinfo(PROC_PIDFDVNODEPATHINFO)"); + goto error; + } + } + // --- /errors checking + + // --- construct python list + py_path = PyUnicode_DecodeFSDefault(vi.pvip.vip_path); + if (! py_path) + goto error; + py_tuple = Py_BuildValue( + "(Oi)", + py_path, + (int)fdp_pointer->proc_fd); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + Py_CLEAR(py_path); + // --- /construct python list + } + } + + free(fds_pointer); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_path); + Py_DECREF(py_retlist); + if (fds_pointer != NULL) + free(fds_pointer); + return NULL; // exception has already been set earlier +} + + +/* + * Return process TCP and UDP connections as a list of tuples. + * Raises NSP in case of zombie process. + * References: + * - lsof source code: http://goo.gl/SYW79 and http://goo.gl/wNrC0 + * - /usr/include/sys/proc_info.h + */ +static PyObject * +psutil_proc_connections(PyObject *self, PyObject *args) { + long pid; + int pidinfo_result; + int iterations; + int i; + unsigned long nb; + + struct proc_fdinfo *fds_pointer = NULL; + struct proc_fdinfo *fdp_pointer; + struct socket_fdinfo si; + + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + PyObject *py_af_filter = NULL; + PyObject *py_type_filter = NULL; + + if (py_retlist == NULL) + return NULL; + + if (! PyArg_ParseTuple(args, "lOO", &pid, &py_af_filter, &py_type_filter)) + goto error; + + if (!PySequence_Check(py_af_filter) || !PySequence_Check(py_type_filter)) { + PyErr_SetString(PyExc_TypeError, "arg 2 or 3 is not a sequence"); + goto error; + } + + if (pid == 0) + return py_retlist; + pidinfo_result = psutil_proc_pidinfo(pid, PROC_PIDLISTFDS, 0, NULL, 0); + if (pidinfo_result <= 0) + goto error; + + fds_pointer = malloc(pidinfo_result); + if (fds_pointer == NULL) { + PyErr_NoMemory(); + goto error; + } + + pidinfo_result = psutil_proc_pidinfo( + pid, PROC_PIDLISTFDS, 0, fds_pointer, pidinfo_result); + if (pidinfo_result <= 0) + goto error; + + iterations = (pidinfo_result / PROC_PIDLISTFD_SIZE); + for (i = 0; i < iterations; i++) { + py_tuple = NULL; + py_laddr = NULL; + py_raddr = NULL; + fdp_pointer = &fds_pointer[i]; + + if (fdp_pointer->proc_fdtype == PROX_FDTYPE_SOCKET) { + errno = 0; + nb = proc_pidfdinfo((pid_t)pid, fdp_pointer->proc_fd, + PROC_PIDFDSOCKETINFO, &si, sizeof(si)); + + // --- errors checking + if ((nb <= 0) || (nb < sizeof(si))) { + if (errno == EBADF) { + // let's assume socket has been closed + continue; + } + else { + psutil_raise_for_pid( + pid, "proc_pidinfo(PROC_PIDFDSOCKETINFO)"); + goto error; + } + } + // --- /errors checking + + // + int fd, family, type, lport, rport, state; + char lip[200], rip[200]; + int inseq; + PyObject *py_family; + PyObject *py_type; + + fd = (int)fdp_pointer->proc_fd; + family = si.psi.soi_family; + type = si.psi.soi_type; + + // apply filters + py_family = PyLong_FromLong((long)family); + inseq = PySequence_Contains(py_af_filter, py_family); + Py_DECREF(py_family); + if (inseq == 0) + continue; + py_type = PyLong_FromLong((long)type); + inseq = PySequence_Contains(py_type_filter, py_type); + Py_DECREF(py_type); + if (inseq == 0) + continue; + + if (errno != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + if ((family == AF_INET) || (family == AF_INET6)) { + if (family == AF_INET) { + inet_ntop(AF_INET, + &si.psi.soi_proto.pri_tcp.tcpsi_ini. \ + insi_laddr.ina_46.i46a_addr4, + lip, + sizeof(lip)); + inet_ntop(AF_INET, + &si.psi.soi_proto.pri_tcp.tcpsi_ini.insi_faddr. \ + ina_46.i46a_addr4, + rip, + sizeof(rip)); + } + else { + inet_ntop(AF_INET6, + &si.psi.soi_proto.pri_tcp.tcpsi_ini. \ + insi_laddr.ina_6, + lip, sizeof(lip)); + inet_ntop(AF_INET6, + &si.psi.soi_proto.pri_tcp.tcpsi_ini. \ + insi_faddr.ina_6, + rip, sizeof(rip)); + } + + // check for inet_ntop failures + if (errno != 0) { + PyErr_SetFromOSErrnoWithSyscall("inet_ntop()"); + goto error; + } + + lport = ntohs(si.psi.soi_proto.pri_tcp.tcpsi_ini.insi_lport); + rport = ntohs(si.psi.soi_proto.pri_tcp.tcpsi_ini.insi_fport); + if (type == SOCK_STREAM) + state = (int)si.psi.soi_proto.pri_tcp.tcpsi_state; + else + state = PSUTIL_CONN_NONE; + + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", rip, rport); + else + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + + // construct the python list + py_tuple = Py_BuildValue( + "(iiiNNi)", fd, family, type, py_laddr, py_raddr, state); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + else if (family == AF_UNIX) { + py_laddr = PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); + if (!py_laddr) + goto error; + py_raddr = PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); + if (!py_raddr) + goto error; + // construct the python list + py_tuple = Py_BuildValue( + "(iiiOOi)", + fd, family, type, + py_laddr, + py_raddr, + PSUTIL_CONN_NONE); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + Py_CLEAR(py_laddr); + Py_CLEAR(py_raddr); + } + } + } + + free(fds_pointer); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + Py_DECREF(py_retlist); + if (fds_pointer != NULL) + free(fds_pointer); + return NULL; +} + + +/* + * Return number of file descriptors opened by process. + * Raises NSP in case of zombie process. + */ +static PyObject * +psutil_proc_num_fds(PyObject *self, PyObject *args) { + long pid; + int pidinfo_result; + int num; + struct proc_fdinfo *fds_pointer; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + pidinfo_result = proc_pidinfo((pid_t)pid, PROC_PIDLISTFDS, 0, NULL, 0); + if (pidinfo_result <= 0) + return PyErr_SetFromErrno(PyExc_OSError); + + fds_pointer = malloc(pidinfo_result); + if (fds_pointer == NULL) + return PyErr_NoMemory(); + pidinfo_result = proc_pidinfo((pid_t)pid, PROC_PIDLISTFDS, 0, fds_pointer, + pidinfo_result); + if (pidinfo_result <= 0) { + free(fds_pointer); + return PyErr_SetFromErrno(PyExc_OSError); + } + + num = (pidinfo_result / PROC_PIDLISTFD_SIZE); + free(fds_pointer); + return Py_BuildValue("i", num); +} + + +/* + * Return a Python list of named tuples with overall network I/O information + */ +static PyObject * +psutil_net_io_counters(PyObject *self, PyObject *args) { + char *buf = NULL, *lim, *next; + struct if_msghdr *ifm; + int mib[6]; + mib[0] = CTL_NET; // networking subsystem + mib[1] = PF_ROUTE; // type of information + mib[2] = 0; // protocol (IPPROTO_xxx) + mib[3] = 0; // address family + mib[4] = NET_RT_IFLIST2; // operation + mib[5] = 0; + size_t len; + PyObject *py_ifc_info = NULL; + PyObject *py_retdict = PyDict_New(); + + if (py_retdict == NULL) + return NULL; + + if (sysctl(mib, 6, NULL, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + buf = malloc(len); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } + + if (sysctl(mib, 6, buf, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + lim = buf + len; + + for (next = buf; next < lim; ) { + ifm = (struct if_msghdr *)next; + next += ifm->ifm_msglen; + + if (ifm->ifm_type == RTM_IFINFO2) { + py_ifc_info = NULL; + struct if_msghdr2 *if2m = (struct if_msghdr2 *)ifm; + struct sockaddr_dl *sdl = (struct sockaddr_dl *)(if2m + 1); + char ifc_name[32]; + + strncpy(ifc_name, sdl->sdl_data, sdl->sdl_nlen); + ifc_name[sdl->sdl_nlen] = 0; + + py_ifc_info = Py_BuildValue( + "(KKKKKKKi)", + if2m->ifm_data.ifi_obytes, + if2m->ifm_data.ifi_ibytes, + if2m->ifm_data.ifi_opackets, + if2m->ifm_data.ifi_ipackets, + if2m->ifm_data.ifi_ierrors, + if2m->ifm_data.ifi_oerrors, + if2m->ifm_data.ifi_iqdrops, + 0); // dropout not supported + + if (!py_ifc_info) + goto error; + if (PyDict_SetItemString(py_retdict, ifc_name, py_ifc_info)) + goto error; + Py_CLEAR(py_ifc_info); + } + else { + continue; + } + } + + free(buf); + return py_retdict; + +error: + Py_XDECREF(py_ifc_info); + Py_DECREF(py_retdict); + if (buf != NULL) + free(buf); + return NULL; +} + + +/* + * Return a Python dict of tuples for disk I/O information + */ +static PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + CFDictionaryRef parent_dict; + CFDictionaryRef props_dict; + CFDictionaryRef stats_dict; + io_registry_entry_t parent; + io_registry_entry_t disk; + io_iterator_t disk_list; + PyObject *py_disk_info = NULL; + PyObject *py_retdict = PyDict_New(); + + if (py_retdict == NULL) + return NULL; + + // Get list of disks + if (IOServiceGetMatchingServices(kIOMasterPortDefault, + IOServiceMatching(kIOMediaClass), + &disk_list) != kIOReturnSuccess) { + PyErr_SetString( + PyExc_RuntimeError, "unable to get the list of disks."); + goto error; + } + + // Iterate over disks + while ((disk = IOIteratorNext(disk_list)) != 0) { + py_disk_info = NULL; + parent_dict = NULL; + props_dict = NULL; + stats_dict = NULL; + + if (IORegistryEntryGetParentEntry(disk, kIOServicePlane, &parent) + != kIOReturnSuccess) { + PyErr_SetString(PyExc_RuntimeError, + "unable to get the disk's parent."); + IOObjectRelease(disk); + goto error; + } + + if (IOObjectConformsTo(parent, "IOBlockStorageDriver")) { + if (IORegistryEntryCreateCFProperties( + disk, + (CFMutableDictionaryRef *) &parent_dict, + kCFAllocatorDefault, + kNilOptions + ) != kIOReturnSuccess) + { + PyErr_SetString(PyExc_RuntimeError, + "unable to get the parent's properties."); + IOObjectRelease(disk); + IOObjectRelease(parent); + goto error; + } + + if (IORegistryEntryCreateCFProperties( + parent, + (CFMutableDictionaryRef *) &props_dict, + kCFAllocatorDefault, + kNilOptions + ) != kIOReturnSuccess) + { + PyErr_SetString(PyExc_RuntimeError, + "unable to get the disk properties."); + CFRelease(props_dict); + IOObjectRelease(disk); + IOObjectRelease(parent); + goto error; + } + + const int kMaxDiskNameSize = 64; + CFStringRef disk_name_ref = (CFStringRef)CFDictionaryGetValue( + parent_dict, CFSTR(kIOBSDNameKey)); + char disk_name[kMaxDiskNameSize]; + + CFStringGetCString(disk_name_ref, + disk_name, + kMaxDiskNameSize, + CFStringGetSystemEncoding()); + + stats_dict = (CFDictionaryRef)CFDictionaryGetValue( + props_dict, CFSTR(kIOBlockStorageDriverStatisticsKey)); + + if (stats_dict == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "Unable to get disk stats."); + goto error; + } + + CFNumberRef number; + int64_t reads = 0; + int64_t writes = 0; + int64_t read_bytes = 0; + int64_t write_bytes = 0; + int64_t read_time = 0; + int64_t write_time = 0; + + // Get disk reads/writes + if ((number = (CFNumberRef)CFDictionaryGetValue( + stats_dict, + CFSTR(kIOBlockStorageDriverStatisticsReadsKey)))) + { + CFNumberGetValue(number, kCFNumberSInt64Type, &reads); + } + if ((number = (CFNumberRef)CFDictionaryGetValue( + stats_dict, + CFSTR(kIOBlockStorageDriverStatisticsWritesKey)))) + { + CFNumberGetValue(number, kCFNumberSInt64Type, &writes); + } + + // Get disk bytes read/written + if ((number = (CFNumberRef)CFDictionaryGetValue( + stats_dict, + CFSTR(kIOBlockStorageDriverStatisticsBytesReadKey)))) + { + CFNumberGetValue(number, kCFNumberSInt64Type, &read_bytes); + } + if ((number = (CFNumberRef)CFDictionaryGetValue( + stats_dict, + CFSTR(kIOBlockStorageDriverStatisticsBytesWrittenKey)))) + { + CFNumberGetValue(number, kCFNumberSInt64Type, &write_bytes); + } + + // Get disk time spent reading/writing (nanoseconds) + if ((number = (CFNumberRef)CFDictionaryGetValue( + stats_dict, + CFSTR(kIOBlockStorageDriverStatisticsTotalReadTimeKey)))) + { + CFNumberGetValue(number, kCFNumberSInt64Type, &read_time); + } + if ((number = (CFNumberRef)CFDictionaryGetValue( + stats_dict, + CFSTR(kIOBlockStorageDriverStatisticsTotalWriteTimeKey)))) + { + CFNumberGetValue(number, kCFNumberSInt64Type, &write_time); + } + + // Read/Write time on macOS comes back in nanoseconds and in psutil + // we've standardized on milliseconds so do the conversion. + py_disk_info = Py_BuildValue( + "(KKKKKK)", + reads, + writes, + read_bytes, + write_bytes, + read_time / 1000 / 1000, + write_time / 1000 / 1000); + if (!py_disk_info) + goto error; + if (PyDict_SetItemString(py_retdict, disk_name, py_disk_info)) + goto error; + Py_CLEAR(py_disk_info); + + CFRelease(parent_dict); + IOObjectRelease(parent); + CFRelease(props_dict); + IOObjectRelease(disk); + } + } + + IOObjectRelease (disk_list); + + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + return NULL; +} + + +/* + * Return currently connected users as a list of tuples. + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + struct utmpx *utx; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + while ((utx = getutxent()) != NULL) { + if (utx->ut_type != USER_PROCESS) + continue; + py_username = PyUnicode_DecodeFSDefault(utx->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(utx->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(utx->ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)utx->ut_tv.tv_sec, // start time + utx->ut_pid // process id + ); + if (!py_tuple) { + endutxent(); + goto error; + } + if (PyList_Append(py_retlist, py_tuple)) { + endutxent(); + goto error; + } + Py_CLEAR(py_username); + Py_CLEAR(py_tty); + Py_CLEAR(py_hostname); + Py_CLEAR(py_tuple); + } + + endutxent(); + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + return NULL; +} + + +/* + * Return CPU statistics. + */ +static PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + struct vmmeter vmstat; + kern_return_t ret; + mach_msg_type_number_t count = sizeof(vmstat) / sizeof(integer_t); + mach_port_t mport = mach_host_self(); + + ret = host_statistics(mport, HOST_VM_INFO, (host_info_t)&vmstat, &count); + if (ret != KERN_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "host_statistics(HOST_VM_INFO) failed: %s", + mach_error_string(ret)); + return NULL; + } + mach_port_deallocate(mach_task_self(), mport); + + return Py_BuildValue( + "IIIII", + vmstat.v_swtch, // ctx switches + vmstat.v_intr, // interrupts + vmstat.v_soft, // software interrupts + vmstat.v_syscall, // syscalls + vmstat.v_trap // traps + ); +} + + +/* + * Return battery information. + */ +static PyObject * +psutil_sensors_battery(PyObject *self, PyObject *args) { + PyObject *py_tuple = NULL; + CFTypeRef power_info = NULL; + CFArrayRef power_sources_list = NULL; + CFDictionaryRef power_sources_information = NULL; + CFNumberRef capacity_ref = NULL; + CFNumberRef time_to_empty_ref = NULL; + CFStringRef ps_state_ref = NULL; + uint32_t capacity; /* units are percent */ + int time_to_empty; /* units are minutes */ + int is_power_plugged; + + power_info = IOPSCopyPowerSourcesInfo(); + + if (!power_info) { + PyErr_SetString(PyExc_RuntimeError, + "IOPSCopyPowerSourcesInfo() syscall failed"); + goto error; + } + + power_sources_list = IOPSCopyPowerSourcesList(power_info); + if (!power_sources_list) { + PyErr_SetString(PyExc_RuntimeError, + "IOPSCopyPowerSourcesList() syscall failed"); + goto error; + } + + /* Should only get one source. But in practice, check for > 0 sources */ + if (!CFArrayGetCount(power_sources_list)) { + PyErr_SetString(PyExc_NotImplementedError, "no battery"); + goto error; + } + + power_sources_information = IOPSGetPowerSourceDescription( + power_info, CFArrayGetValueAtIndex(power_sources_list, 0)); + + capacity_ref = (CFNumberRef) CFDictionaryGetValue( + power_sources_information, CFSTR(kIOPSCurrentCapacityKey)); + if (!CFNumberGetValue(capacity_ref, kCFNumberSInt32Type, &capacity)) { + PyErr_SetString(PyExc_RuntimeError, + "No battery capacity infomration in power sources info"); + goto error; + } + + ps_state_ref = (CFStringRef) CFDictionaryGetValue( + power_sources_information, CFSTR(kIOPSPowerSourceStateKey)); + is_power_plugged = CFStringCompare( + ps_state_ref, CFSTR(kIOPSACPowerValue), 0) + == kCFCompareEqualTo; + + time_to_empty_ref = (CFNumberRef) CFDictionaryGetValue( + power_sources_information, CFSTR(kIOPSTimeToEmptyKey)); + if (!CFNumberGetValue(time_to_empty_ref, + kCFNumberIntType, &time_to_empty)) { + /* This value is recommended for non-Apple power sources, so it's not + * an error if it doesn't exist. We'll return -1 for "unknown" */ + /* A value of -1 indicates "Still Calculating the Time" also for + * apple power source */ + time_to_empty = -1; + } + + py_tuple = Py_BuildValue("Iii", + capacity, time_to_empty, is_power_plugged); + if (!py_tuple) { + goto error; + } + + CFRelease(power_info); + CFRelease(power_sources_list); + /* Caller should NOT release power_sources_information */ + + return py_tuple; + +error: + if (power_info) + CFRelease(power_info); + if (power_sources_list) + CFRelease(power_sources_list); + Py_XDECREF(py_tuple); + return NULL; +} + + +/* + * define the psutil C module methods and initialize the module. + */ +static PyMethodDef mod_methods[] = { + // --- per-process functions + + {"proc_kinfo_oneshot", psutil_proc_kinfo_oneshot, METH_VARARGS, + "Return multiple process info."}, + {"proc_pidtaskinfo_oneshot", psutil_proc_pidtaskinfo_oneshot, METH_VARARGS, + "Return multiple process info."}, + {"proc_name", psutil_proc_name, METH_VARARGS, + "Return process name"}, + {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS, + "Return process cmdline as a list of cmdline arguments"}, + {"proc_environ", psutil_proc_environ, METH_VARARGS, + "Return process environment data"}, + {"proc_exe", psutil_proc_exe, METH_VARARGS, + "Return path of the process executable"}, + {"proc_cwd", psutil_proc_cwd, METH_VARARGS, + "Return process current working directory."}, + {"proc_memory_uss", psutil_proc_memory_uss, METH_VARARGS, + "Return process USS memory"}, + {"proc_threads", psutil_proc_threads, METH_VARARGS, + "Return process threads as a list of tuples"}, + {"proc_open_files", psutil_proc_open_files, METH_VARARGS, + "Return files opened by process as a list of tuples"}, + {"proc_num_fds", psutil_proc_num_fds, METH_VARARGS, + "Return the number of fds opened by process."}, + {"proc_connections", psutil_proc_connections, METH_VARARGS, + "Get process TCP and UDP connections as a list of tuples"}, + + // --- system-related functions + + {"pids", psutil_pids, METH_VARARGS, + "Returns a list of PIDs currently running on the system"}, + {"cpu_count_logical", psutil_cpu_count_logical, METH_VARARGS, + "Return number of logical CPUs on the system"}, + {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, + "Return number of physical CPUs on the system"}, + {"virtual_mem", psutil_virtual_mem, METH_VARARGS, + "Return system virtual memory stats"}, + {"swap_mem", psutil_swap_mem, METH_VARARGS, + "Return stats about swap memory, in bytes"}, + {"cpu_times", psutil_cpu_times, METH_VARARGS, + "Return system cpu times as a tuple (user, system, nice, idle, irc)"}, + {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, + "Return system per-cpu times as a list of tuples"}, + {"cpu_freq", psutil_cpu_freq, METH_VARARGS, + "Return cpu current frequency"}, + {"boot_time", psutil_boot_time, METH_VARARGS, + "Return the system boot time expressed in seconds since the epoch."}, + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return a list of tuples including device, mount point and " + "fs type for all partitions mounted on the system."}, + {"net_io_counters", psutil_net_io_counters, METH_VARARGS, + "Return dict of tuples of networks I/O information."}, + {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS, + "Return dict of tuples of disks I/O information."}, + {"users", psutil_users, METH_VARARGS, + "Return currently connected users as a list of tuples"}, + {"cpu_stats", psutil_cpu_stats, METH_VARARGS, + "Return CPU statistics"}, + {"sensors_battery", psutil_sensors_battery, METH_VARARGS, + "Return battery information."}, + + // --- others + {"set_testing", psutil_set_testing, METH_NOARGS, + "Set psutil in testing mode"}, + + {NULL, NULL, 0, NULL} +}; + + +#if PY_MAJOR_VERSION >= 3 + #define INITERR return NULL + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_osx", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL + }; + + PyObject *PyInit__psutil_osx(void) +#else /* PY_MAJOR_VERSION */ + #define INITERR return + + void init_psutil_osx(void) +#endif /* PY_MAJOR_VERSION */ +{ + PyObject *v; +#if PY_MAJOR_VERSION >= 3 + PyObject *mod = PyModule_Create(&moduledef); +#else + PyObject *mod = Py_InitModule("_psutil_osx", mod_methods); +#endif + if (mod == NULL) + INITERR; + + if (psutil_setup() != 0) + INITERR; + + if (PyModule_AddIntConstant(mod, "version", PSUTIL_VERSION)) + INITERR; + // process status constants, defined in: + // http://fxr.watson.org/fxr/source/bsd/sys/proc.h?v=xnu-792.6.70#L149 + if (PyModule_AddIntConstant(mod, "SIDL", SIDL)) + INITERR; + if (PyModule_AddIntConstant(mod, "SRUN", SRUN)) + INITERR; + if (PyModule_AddIntConstant(mod, "SSLEEP", SSLEEP)) + INITERR; + if (PyModule_AddIntConstant(mod, "SSTOP", SSTOP)) + INITERR; + if (PyModule_AddIntConstant(mod, "SZOMB", SZOMB)) + INITERR; + // connection status constants + if (PyModule_AddIntConstant(mod, "TCPS_CLOSED", TCPS_CLOSED)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_CLOSING", TCPS_CLOSING)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_CLOSE_WAIT", TCPS_CLOSE_WAIT)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_LISTEN", TCPS_LISTEN)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_ESTABLISHED", TCPS_ESTABLISHED)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_SYN_SENT", TCPS_SYN_SENT)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_SYN_RECEIVED", TCPS_SYN_RECEIVED)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_FIN_WAIT_1", TCPS_FIN_WAIT_1)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_FIN_WAIT_2", TCPS_FIN_WAIT_2)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_LAST_ACK", TCPS_LAST_ACK)) + INITERR; + if (PyModule_AddIntConstant(mod, "TCPS_TIME_WAIT", TCPS_TIME_WAIT)) + INITERR; + if (PyModule_AddIntConstant(mod, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE)) + INITERR; + + // Exception. + ZombieProcessError = PyErr_NewException( + "_psutil_osx.ZombieProcessError", NULL, NULL); + if (ZombieProcessError == NULL) + INITERR; + Py_INCREF(ZombieProcessError); + if (PyModule_AddObject(mod, "ZombieProcessError", ZombieProcessError)) { + Py_DECREF(ZombieProcessError); + INITERR; + } + + if (mod == NULL) + INITERR; +#if PY_MAJOR_VERSION >= 3 + return mod; +#endif +} diff --git a/ddtrace/vendor/psutil/_psutil_posix.c b/ddtrace/vendor/psutil/_psutil_posix.c new file mode 100644 index 00000000000..aa600849176 --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_posix.c @@ -0,0 +1,691 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Functions specific to all POSIX compliant platforms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef PSUTIL_SUNOS10 + #include "arch/solaris/v10/ifaddrs.h" +#elif PSUTIL_AIX + #include "arch/aix/ifaddrs.h" +#else + #include +#endif + +#if defined(PSUTIL_LINUX) + #include + #include + #include +#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) + #include + #include + #include + #include + #include + #include +#elif defined(PSUTIL_SUNOS) + #include + #include +#elif defined(PSUTIL_AIX) + #include +#endif + +#include "_psutil_common.h" + +/* + * Check if PID exists. Return values: + * 1: exists + * 0: does not exist + * -1: error (Python exception is set) + */ +int +psutil_pid_exists(long pid) { + int ret; + + // No negative PID exists, plus -1 is an alias for sending signal + // too all processes except system ones. Not what we want. + if (pid < 0) + return 0; + + // As per "man 2 kill" PID 0 is an alias for sending the signal to + // every process in the process group of the calling process. + // Not what we want. Some platforms have PID 0, some do not. + // We decide that at runtime. + if (pid == 0) { +#if defined(PSUTIL_LINUX) || defined(PSUTIL_FREEBSD) + return 0; +#else + return 1; +#endif + } + +#if defined(PSUTIL_OSX) + ret = kill((pid_t)pid , 0); +#else + ret = kill(pid , 0); +#endif + + if (ret == 0) + return 1; + else { + if (errno == ESRCH) { + // ESRCH == No such process + return 0; + } + else if (errno == EPERM) { + // EPERM clearly indicates there's a process to deny + // access to. + return 1; + } + else { + // According to "man 2 kill" possible error values are + // (EINVAL, EPERM, ESRCH) therefore we should never get + // here. If we do let's be explicit in considering this + // an error. + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + } +} + + +/* + * Utility used for those syscalls which do not return a meaningful + * error that we can translate into an exception which makes sense. + * As such, we'll have to guess. + * On UNIX, if errno is set, we return that one (OSError). + * Else, if PID does not exist we assume the syscall failed because + * of that so we raise NoSuchProcess. + * If none of this is true we giveup and raise RuntimeError(msg). + * This will always set a Python exception and return NULL. + */ +int +psutil_raise_for_pid(long pid, char *syscall_name) { + // Set exception to AccessDenied if pid exists else NoSuchProcess. + if (errno != 0) { + // Unlikely we get here. + PyErr_SetFromErrno(PyExc_OSError); + return 0; + } + else if (psutil_pid_exists(pid) == 0) { + psutil_debug("%s syscall failed and PID %i no longer exists; " + "assume NoSuchProcess", syscall_name, pid); + NoSuchProcess(""); + } + else { + PyErr_Format(PyExc_RuntimeError, "%s syscall failed", syscall_name); + } + return 0; +} + + +/* + * Given a PID return process priority as a Python integer. + */ +static PyObject * +psutil_posix_getpriority(PyObject *self, PyObject *args) { + long pid; + int priority; + errno = 0; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + +#ifdef PSUTIL_OSX + priority = getpriority(PRIO_PROCESS, (id_t)pid); +#else + priority = getpriority(PRIO_PROCESS, pid); +#endif + if (errno != 0) + return PyErr_SetFromErrno(PyExc_OSError); + return Py_BuildValue("i", priority); +} + + +/* + * Given a PID and a value change process priority. + */ +static PyObject * +psutil_posix_setpriority(PyObject *self, PyObject *args) { + long pid; + int priority; + int retval; + + if (! PyArg_ParseTuple(args, "li", &pid, &priority)) + return NULL; + +#ifdef PSUTIL_OSX + retval = setpriority(PRIO_PROCESS, (id_t)pid, priority); +#else + retval = setpriority(PRIO_PROCESS, pid, priority); +#endif + if (retval == -1) + return PyErr_SetFromErrno(PyExc_OSError); + Py_RETURN_NONE; +} + + +/* + * Translate a sockaddr struct into a Python string. + * Return None if address family is not AF_INET* or AF_PACKET. + */ +static PyObject * +psutil_convert_ipaddr(struct sockaddr *addr, int family) { + char buf[NI_MAXHOST]; + int err; + int addrlen; + size_t n; + size_t len; + const char *data; + char *ptr; + + if (addr == NULL) { + Py_INCREF(Py_None); + return Py_None; + } + else if (family == AF_INET || family == AF_INET6) { + if (family == AF_INET) + addrlen = sizeof(struct sockaddr_in); + else + addrlen = sizeof(struct sockaddr_in6); + err = getnameinfo(addr, addrlen, buf, sizeof(buf), NULL, 0, + NI_NUMERICHOST); + if (err != 0) { + // XXX we get here on FreeBSD when processing 'lo' / AF_INET6 + // broadcast. Not sure what to do other than returning None. + // ifconfig does not show anything BTW. + //PyErr_Format(PyExc_RuntimeError, gai_strerror(err)); + //return NULL; + Py_INCREF(Py_None); + return Py_None; + } + else { + return Py_BuildValue("s", buf); + } + } +#ifdef PSUTIL_LINUX + else if (family == AF_PACKET) { + struct sockaddr_ll *lladdr = (struct sockaddr_ll *)addr; + len = lladdr->sll_halen; + data = (const char *)lladdr->sll_addr; + } +#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) + else if (addr->sa_family == AF_LINK) { + // Note: prior to Python 3.4 socket module does not expose + // AF_LINK so we'll do. + struct sockaddr_dl *dladdr = (struct sockaddr_dl *)addr; + len = dladdr->sdl_alen; + data = LLADDR(dladdr); + } +#endif + else { + // unknown family + Py_INCREF(Py_None); + return Py_None; + } + + // AF_PACKET or AF_LINK + if (len > 0) { + ptr = buf; + for (n = 0; n < len; ++n) { + sprintf(ptr, "%02x:", data[n] & 0xff); + ptr += 3; + } + *--ptr = '\0'; + return Py_BuildValue("s", buf); + } + else { + Py_INCREF(Py_None); + return Py_None; + } +} + + +/* + * Return NICs information a-la ifconfig as a list of tuples. + * TODO: on Solaris we won't get any MAC address. + */ +static PyObject* +psutil_net_if_addrs(PyObject* self, PyObject* args) { + struct ifaddrs *ifaddr, *ifa; + int family; + + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_address = NULL; + PyObject *py_netmask = NULL; + PyObject *py_broadcast = NULL; + PyObject *py_ptp = NULL; + + if (py_retlist == NULL) + return NULL; + if (getifaddrs(&ifaddr) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (!ifa->ifa_addr) + continue; + family = ifa->ifa_addr->sa_family; + py_address = psutil_convert_ipaddr(ifa->ifa_addr, family); + // If the primary address can't be determined just skip it. + // I've never seen this happen on Linux but I did on FreeBSD. + if (py_address == Py_None) + continue; + if (py_address == NULL) + goto error; + py_netmask = psutil_convert_ipaddr(ifa->ifa_netmask, family); + if (py_netmask == NULL) + goto error; + + if (ifa->ifa_flags & IFF_BROADCAST) { + py_broadcast = psutil_convert_ipaddr(ifa->ifa_broadaddr, family); + Py_INCREF(Py_None); + py_ptp = Py_None; + } + else if (ifa->ifa_flags & IFF_POINTOPOINT) { + py_ptp = psutil_convert_ipaddr(ifa->ifa_dstaddr, family); + Py_INCREF(Py_None); + py_broadcast = Py_None; + } + else { + Py_INCREF(Py_None); + Py_INCREF(Py_None); + py_broadcast = Py_None; + py_ptp = Py_None; + } + + if ((py_broadcast == NULL) || (py_ptp == NULL)) + goto error; + py_tuple = Py_BuildValue( + "(siOOOO)", + ifa->ifa_name, + family, + py_address, + py_netmask, + py_broadcast, + py_ptp + ); + + if (! py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + Py_CLEAR(py_address); + Py_CLEAR(py_netmask); + Py_CLEAR(py_broadcast); + Py_CLEAR(py_ptp); + } + + freeifaddrs(ifaddr); + return py_retlist; + +error: + if (ifaddr != NULL) + freeifaddrs(ifaddr); + Py_DECREF(py_retlist); + Py_XDECREF(py_tuple); + Py_XDECREF(py_address); + Py_XDECREF(py_netmask); + Py_XDECREF(py_broadcast); + Py_XDECREF(py_ptp); + return NULL; +} + + +/* + * Return NIC MTU. References: + * http://www.i-scream.org/libstatgrab/ + */ +static PyObject * +psutil_net_if_mtu(PyObject *self, PyObject *args) { + char *nic_name; + int sock = -1; + int ret; +#ifdef PSUTIL_SUNOS10 + struct lifreq lifr; +#else + struct ifreq ifr; +#endif + + if (! PyArg_ParseTuple(args, "s", &nic_name)) + return NULL; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) + goto error; + +#ifdef PSUTIL_SUNOS10 + strncpy(lifr.lifr_name, nic_name, sizeof(lifr.lifr_name)); + ret = ioctl(sock, SIOCGIFMTU, &lifr); +#else + strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + ret = ioctl(sock, SIOCGIFMTU, &ifr); +#endif + if (ret == -1) + goto error; + close(sock); + +#ifdef PSUTIL_SUNOS10 + return Py_BuildValue("i", lifr.lifr_mtu); +#else + return Py_BuildValue("i", ifr.ifr_mtu); +#endif + +error: + if (sock != -1) + close(sock); + return PyErr_SetFromErrno(PyExc_OSError); +} + + +/* + * Inspect NIC flags, returns a bool indicating whether the NIC is + * running. References: + * http://www.i-scream.org/libstatgrab/ + */ +static PyObject * +psutil_net_if_flags(PyObject *self, PyObject *args) { + char *nic_name; + int sock = -1; + int ret; + struct ifreq ifr; + + if (! PyArg_ParseTuple(args, "s", &nic_name)) + return NULL; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) + goto error; + + strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + ret = ioctl(sock, SIOCGIFFLAGS, &ifr); + if (ret == -1) + goto error; + + close(sock); + if ((ifr.ifr_flags & IFF_UP) != 0) + return Py_BuildValue("O", Py_True); + else + return Py_BuildValue("O", Py_False); + +error: + if (sock != -1) + close(sock); + return PyErr_SetFromErrno(PyExc_OSError); +} + + +/* + * net_if_stats() macOS/BSD implementation. + */ +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) + +int psutil_get_nic_speed(int ifm_active) { + // Determine NIC speed. Taken from: + // http://www.i-scream.org/libstatgrab/ + // Assuming only ETHER devices + switch(IFM_TYPE(ifm_active)) { + case IFM_ETHER: + switch(IFM_SUBTYPE(ifm_active)) { +#if defined(IFM_HPNA_1) && ((!defined(IFM_10G_LR)) \ + || (IFM_10G_LR != IFM_HPNA_1)) + // HomePNA 1.0 (1Mb/s) + case(IFM_HPNA_1): + return 1; +#endif + // 10 Mbit + case(IFM_10_T): // 10BaseT - RJ45 + case(IFM_10_2): // 10Base2 - Thinnet + case(IFM_10_5): // 10Base5 - AUI + case(IFM_10_STP): // 10BaseT over shielded TP + case(IFM_10_FL): // 10baseFL - Fiber + return 10; + // 100 Mbit + case(IFM_100_TX): // 100BaseTX - RJ45 + case(IFM_100_FX): // 100BaseFX - Fiber + case(IFM_100_T4): // 100BaseT4 - 4 pair cat 3 + case(IFM_100_VG): // 100VG-AnyLAN + case(IFM_100_T2): // 100BaseT2 + return 100; + // 1000 Mbit + case(IFM_1000_SX): // 1000BaseSX - multi-mode fiber + case(IFM_1000_LX): // 1000baseLX - single-mode fiber + case(IFM_1000_CX): // 1000baseCX - 150ohm STP +#if defined(IFM_1000_TX) && !defined(PSUTIL_OPENBSD) + // FreeBSD 4 and others (but NOT OpenBSD) -> #define IFM_1000_T in net/if_media.h + case(IFM_1000_TX): +#endif +#ifdef IFM_1000_FX + case(IFM_1000_FX): +#endif +#ifdef IFM_1000_T + case(IFM_1000_T): +#endif + return 1000; +#if defined(IFM_10G_SR) || defined(IFM_10G_LR) || defined(IFM_10G_CX4) \ + || defined(IFM_10G_T) +#ifdef IFM_10G_SR + case(IFM_10G_SR): +#endif +#ifdef IFM_10G_LR + case(IFM_10G_LR): +#endif +#ifdef IFM_10G_CX4 + case(IFM_10G_CX4): +#endif +#ifdef IFM_10G_TWINAX + case(IFM_10G_TWINAX): +#endif +#ifdef IFM_10G_TWINAX_LONG + case(IFM_10G_TWINAX_LONG): +#endif +#ifdef IFM_10G_T + case(IFM_10G_T): +#endif + return 10000; +#endif +#if defined(IFM_2500_SX) +#ifdef IFM_2500_SX + case(IFM_2500_SX): +#endif + return 2500; +#endif // any 2.5GBit stuff... + // We don't know what it is + default: + return 0; + } + break; + +#ifdef IFM_TOKEN + case IFM_TOKEN: + switch(IFM_SUBTYPE(ifm_active)) { + case IFM_TOK_STP4: // Shielded twisted pair 4m - DB9 + case IFM_TOK_UTP4: // Unshielded twisted pair 4m - RJ45 + return 4; + case IFM_TOK_STP16: // Shielded twisted pair 16m - DB9 + case IFM_TOK_UTP16: // Unshielded twisted pair 16m - RJ45 + return 16; +#if defined(IFM_TOK_STP100) || defined(IFM_TOK_UTP100) +#ifdef IFM_TOK_STP100 + case IFM_TOK_STP100: // Shielded twisted pair 100m - DB9 +#endif +#ifdef IFM_TOK_UTP100 + case IFM_TOK_UTP100: // Unshielded twisted pair 100m - RJ45 +#endif + return 100; +#endif + // We don't know what it is + default: + return 0; + } + break; +#endif + +#ifdef IFM_FDDI + case IFM_FDDI: + switch(IFM_SUBTYPE(ifm_active)) { + // We don't know what it is + default: + return 0; + } + break; +#endif + case IFM_IEEE80211: + switch(IFM_SUBTYPE(ifm_active)) { + case IFM_IEEE80211_FH1: // Frequency Hopping 1Mbps + case IFM_IEEE80211_DS1: // Direct Sequence 1Mbps + return 1; + case IFM_IEEE80211_FH2: // Frequency Hopping 2Mbps + case IFM_IEEE80211_DS2: // Direct Sequence 2Mbps + return 2; + case IFM_IEEE80211_DS5: // Direct Sequence 5Mbps + return 5; + case IFM_IEEE80211_DS11: // Direct Sequence 11Mbps + return 11; + case IFM_IEEE80211_DS22: // Direct Sequence 22Mbps + return 22; + // We don't know what it is + default: + return 0; + } + break; + + default: + return 0; + } +} + + +/* + * Return stats about a particular network interface. + * References: + * http://www.i-scream.org/libstatgrab/ + */ +static PyObject * +psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { + char *nic_name; + int sock = -1; + int ret; + int duplex; + int speed; + struct ifreq ifr; + struct ifmediareq ifmed; + + if (! PyArg_ParseTuple(args, "s", &nic_name)) + return NULL; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) + return PyErr_SetFromErrno(PyExc_OSError); + strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + + // speed / duplex + memset(&ifmed, 0, sizeof(struct ifmediareq)); + strlcpy(ifmed.ifm_name, nic_name, sizeof(ifmed.ifm_name)); + ret = ioctl(sock, SIOCGIFMEDIA, (caddr_t)&ifmed); + if (ret == -1) { + speed = 0; + duplex = 0; + } + else { + speed = psutil_get_nic_speed(ifmed.ifm_active); + if ((ifmed.ifm_active | IFM_FDX) == ifmed.ifm_active) + duplex = 2; + else if ((ifmed.ifm_active | IFM_HDX) == ifmed.ifm_active) + duplex = 1; + else + duplex = 0; + } + + close(sock); + return Py_BuildValue("[ii]", duplex, speed); +} +#endif // net_if_stats() macOS/BSD implementation + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * define the psutil C module methods and initialize the module. + */ +static PyMethodDef mod_methods[] = { + {"getpriority", psutil_posix_getpriority, METH_VARARGS, + "Return process priority"}, + {"setpriority", psutil_posix_setpriority, METH_VARARGS, + "Set process priority"}, + {"net_if_addrs", psutil_net_if_addrs, METH_VARARGS, + "Retrieve NICs information"}, + {"net_if_mtu", psutil_net_if_mtu, METH_VARARGS, + "Retrieve NIC MTU"}, + {"net_if_flags", psutil_net_if_flags, METH_VARARGS, + "Retrieve NIC flags"}, +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) + {"net_if_duplex_speed", psutil_net_if_duplex_speed, METH_VARARGS, + "Return NIC stats."}, +#endif + {NULL, NULL, 0, NULL} +}; + + +#if PY_MAJOR_VERSION >= 3 + #define INITERR return NULL + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_psutil_posix", + NULL, + -1, + mod_methods, + NULL, + NULL, + NULL, + NULL + }; + + PyObject *PyInit__psutil_posix(void) +#else /* PY_MAJOR_VERSION */ + #define INITERR return + + void init_psutil_posix(void) +#endif /* PY_MAJOR_VERSION */ +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *mod = PyModule_Create(&moduledef); +#else + PyObject *mod = Py_InitModule("_psutil_posix", mod_methods); +#endif + if (mod == NULL) + INITERR; + +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) || defined(PSUTIL_SUNOS) || defined(PSUTIL_AIX) + if (PyModule_AddIntConstant(mod, "AF_LINK", AF_LINK)) INITERR; +#endif + + if (mod == NULL) + INITERR; +#if PY_MAJOR_VERSION >= 3 + return mod; +#endif +} + +#ifdef __cplusplus +} +#endif diff --git a/ddtrace/vendor/psutil/_psutil_posix.h b/ddtrace/vendor/psutil/_psutil_posix.h new file mode 100644 index 00000000000..fe25b366950 --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_posix.h @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +int psutil_pid_exists(long pid); +void psutil_raise_for_pid(long pid, char *msg); diff --git a/ddtrace/vendor/psutil/_psutil_sunos.c b/ddtrace/vendor/psutil/_psutil_sunos.c new file mode 100644 index 00000000000..31d6f364fbc --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_sunos.c @@ -0,0 +1,1776 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Functions specific to Sun OS Solaris platforms. + * + * Thanks to Justin Venus who originally wrote a consistent part of + * this in Cython which I later on translated in C. + */ + +/* fix compilation issue on SunOS 5.10, see: + * https://github.com/giampaolo/psutil/issues/421 + * https://github.com/giampaolo/psutil/issues/1077 + * http://us-east.manta.joyent.com/jmc/public/opensolaris/ARChive/PSARC/2010/111/materials/s10ceval.txt + * + * Because LEGACY_MIB_SIZE defined in the same file there is no way to make autoconfiguration =\ +*/ + +#define NEW_MIB_COMPLIANT 1 +#define _STRUCTURED_PROC 1 + +#include + +#if !defined(_LP64) && _FILE_OFFSET_BITS == 64 +# undef _FILE_OFFSET_BITS +# undef _LARGEFILE64_SOURCE +#endif + +#include +#include +#include +#include +#include +#include // for MNTTAB +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // fabs() + +#include "_psutil_common.h" +#include "_psutil_posix.h" + +#include "arch/solaris/environ.h" + +#define PSUTIL_TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) + + +/* + * Read a file content and fills a C structure with it. + */ +static int +psutil_file_to_struct(char *path, void *fstruct, size_t size) { + int fd; + ssize_t nbytes; + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + return 0; + } + nbytes = read(fd, fstruct, size); + if (nbytes == -1) { + close(fd); + PyErr_SetFromErrno(PyExc_OSError); + return 0; + } + if (nbytes != (ssize_t) size) { + close(fd); + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + return 0; + } + close(fd); + return nbytes; +} + + +/* + * Return process ppid, rss, vms, ctime, nice, nthreads, status and tty + * as a Python tuple. + */ +static PyObject * +psutil_proc_basic_info(PyObject *self, PyObject *args) { + int pid; + char path[1000]; + psinfo_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + return Py_BuildValue( + "ikkdiiikiiii", + info.pr_ppid, // parent pid + info.pr_rssize, // rss + info.pr_size, // vms + PSUTIL_TV2DOUBLE(info.pr_start), // create time + info.pr_lwp.pr_nice, // nice + info.pr_nlwp, // no. of threads + info.pr_lwp.pr_state, // status code + info.pr_ttydev, // tty nr + (int)info.pr_uid, // real user id + (int)info.pr_euid, // effective user id + (int)info.pr_gid, // real group id + (int)info.pr_egid // effective group id + ); +} + +/* + * Join array of C strings to C string with delemiter dm. + * Omit empty records. + */ +static int +cstrings_array_to_string(char **joined, char ** array, size_t count, char dm) { + int i; + size_t total_length = 0; + size_t item_length = 0; + char *result = NULL; + char *last = NULL; + + if (!array || !joined) + return 0; + + for (i=0; i 0) { + py_args = PyUnicode_DecodeFSDefault(argv_plain); + free(argv_plain); + } else if (joined < 0) { + goto error; + } + + psutil_free_cstrings_array(argv, argc); + } + } + + /* If we can't read process memory or can't decode the result + * then return args from /proc. */ + if (!py_args) { + PyErr_Clear(); + py_args = PyUnicode_DecodeFSDefault(info.pr_psargs); + } + + /* Both methods has been failed. */ + if (!py_args) + goto error; + + py_retlist = Py_BuildValue("OO", py_name, py_args); + if (!py_retlist) + goto error; + + Py_DECREF(py_name); + Py_DECREF(py_args); + return py_retlist; + +error: + Py_XDECREF(py_name); + Py_XDECREF(py_args); + Py_XDECREF(py_retlist); + return NULL; +} + + +/* + * Return process environ block. + */ +static PyObject * +psutil_proc_environ(PyObject *self, PyObject *args) { + int pid; + char path[1000]; + psinfo_t info; + const char *procfs_path; + char **env = NULL; + ssize_t env_count = -1; + char *dm; + int i = 0; + PyObject *py_envname = NULL; + PyObject *py_envval = NULL; + PyObject *py_retdict = PyDict_New(); + + if (! py_retdict) + return PyErr_NoMemory(); + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + goto error; + + if (! info.pr_envp) { + AccessDenied(""); + goto error; + } + + env = psutil_read_raw_env(info, procfs_path, &env_count); + if (! env && env_count != 0) + goto error; + + for (i=0; i= 0) + psutil_free_cstrings_array(env, env_count); + + Py_XDECREF(py_envname); + Py_XDECREF(py_envval); + Py_XDECREF(py_retdict); + return NULL; +} + + +/* + * Return process user and system CPU times as a Python tuple. + */ +static PyObject * +psutil_proc_cpu_times(PyObject *self, PyObject *args) { + int pid; + char path[1000]; + pstatus_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/status", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + // results are more precise than os.times() + return Py_BuildValue( + "(dddd)", + PSUTIL_TV2DOUBLE(info.pr_utime), + PSUTIL_TV2DOUBLE(info.pr_stime), + PSUTIL_TV2DOUBLE(info.pr_cutime), + PSUTIL_TV2DOUBLE(info.pr_cstime) + ); +} + + +/* + * Return what CPU the process is running on. + */ +static PyObject * +psutil_proc_cpu_num(PyObject *self, PyObject *args) { + int fd = NULL; + int pid; + char path[1000]; + struct prheader header; + struct lwpsinfo *lwp = NULL; + int nent; + int size; + int proc_num; + ssize_t nbytes; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/lpsinfo", procfs_path, pid); + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + return NULL; + } + + // read header + nbytes = pread(fd, &header, sizeof(header), 0); + if (nbytes == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (nbytes != sizeof(header)) { + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + goto error; + } + + // malloc + nent = header.pr_nent; + size = header.pr_entsize * nent; + lwp = malloc(size); + if (lwp == NULL) { + PyErr_NoMemory(); + goto error; + } + + // read the rest + nbytes = pread(fd, lwp, size, sizeof(header)); + if (nbytes == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (nbytes != size) { + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + goto error; + } + + // done + proc_num = lwp->pr_onpro; + close(fd); + free(lwp); + return Py_BuildValue("i", proc_num); + +error: + if (fd != -1) + close(fd); + free(lwp); + return NULL; +} + + +/* + * Return process uids/gids as a Python tuple. + */ +static PyObject * +psutil_proc_cred(PyObject *self, PyObject *args) { + int pid; + char path[1000]; + prcred_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/cred", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + return Py_BuildValue("iiiiii", + info.pr_ruid, info.pr_euid, info.pr_suid, + info.pr_rgid, info.pr_egid, info.pr_sgid); +} + + +/* + * Return process voluntary and involuntary context switches as a Python tuple. + */ +static PyObject * +psutil_proc_num_ctx_switches(PyObject *self, PyObject *args) { + int pid; + char path[1000]; + prusage_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/usage", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + return Py_BuildValue("kk", info.pr_vctx, info.pr_ictx); +} + + +/* + * Process IO counters. + * + * Commented out and left here as a reminder. Apparently we cannot + * retrieve process IO stats because: + * - 'pr_ioch' is a sum of chars read and written, with no distinction + * - 'pr_inblk' and 'pr_oublk', which should be the number of bytes + * read and written, hardly increase and according to: + * http://www.brendangregg.com/Solaris/paper_diskubyp1.pdf + * ...they should be meaningless anyway. + * +static PyObject* +proc_io_counters(PyObject* self, PyObject* args) { + int pid; + char path[1000]; + prusage_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/usage", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + + // On Solaris we only have 'pr_ioch' which accounts for bytes read + // *and* written. + // 'pr_inblk' and 'pr_oublk' should be expressed in blocks of + // 8KB according to: + // http://www.brendangregg.com/Solaris/paper_diskubyp1.pdf (pag. 8) + return Py_BuildValue("kkkk", + info.pr_ioch, + info.pr_ioch, + info.pr_inblk, + info.pr_oublk); +} + */ + + +/* + * Return information about a given process thread. + */ +static PyObject * +psutil_proc_query_thread(PyObject *self, PyObject *args) { + int pid, tid; + char path[1000]; + lwpstatus_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "iis", &pid, &tid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/lwp/%i/lwpstatus", procfs_path, pid, tid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + return Py_BuildValue("dd", + PSUTIL_TV2DOUBLE(info.pr_utime), + PSUTIL_TV2DOUBLE(info.pr_stime)); +} + + +/* + * Return information about system virtual memory. + */ +static PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { +// XXX (arghhh!) +// total/free swap mem: commented out as for some reason I can't +// manage to get the same results shown by "swap -l", despite the +// code below is exactly the same as: +// http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/ +// cmd/swap/swap.c +// We're going to parse "swap -l" output from Python (sigh!) + +/* + struct swaptable *st; + struct swapent *swapent; + int i; + struct stat64 statbuf; + char *path; + char fullpath[MAXPATHLEN+1]; + int num; + + if ((num = swapctl(SC_GETNSWP, NULL)) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + if (num == 0) { + PyErr_SetString(PyExc_RuntimeError, "no swap devices configured"); + return NULL; + } + if ((st = malloc(num * sizeof(swapent_t) + sizeof (int))) == NULL) { + PyErr_SetString(PyExc_RuntimeError, "malloc failed"); + return NULL; + } + if ((path = malloc(num * MAXPATHLEN)) == NULL) { + PyErr_SetString(PyExc_RuntimeError, "malloc failed"); + return NULL; + } + swapent = st->swt_ent; + for (i = 0; i < num; i++, swapent++) { + swapent->ste_path = path; + path += MAXPATHLEN; + } + st->swt_n = num; + if ((num = swapctl(SC_LIST, st)) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + swapent = st->swt_ent; + long t = 0, f = 0; + for (i = 0; i < num; i++, swapent++) { + int diskblks_per_page =(int)(sysconf(_SC_PAGESIZE) >> DEV_BSHIFT); + t += (long)swapent->ste_pages; + f += (long)swapent->ste_free; + } + + free(st); + return Py_BuildValue("(kk)", t, f); +*/ + + kstat_ctl_t *kc; + kstat_t *k; + cpu_stat_t *cpu; + int cpu_count = 0; + int flag = 0; + uint_t sin = 0; + uint_t sout = 0; + + kc = kstat_open(); + if (kc == NULL) + return PyErr_SetFromErrno(PyExc_OSError);; + + k = kc->kc_chain; + while (k != NULL) { + if ((strncmp(k->ks_name, "cpu_stat", 8) == 0) && \ + (kstat_read(kc, k, NULL) != -1) ) + { + flag = 1; + cpu = (cpu_stat_t *) k->ks_data; + sin += cpu->cpu_vminfo.pgswapin; // num pages swapped in + sout += cpu->cpu_vminfo.pgswapout; // num pages swapped out + } + cpu_count += 1; + k = k->ks_next; + } + kstat_close(kc); + if (!flag) { + PyErr_SetString(PyExc_RuntimeError, "no swap device was found"); + return NULL; + } + return Py_BuildValue("(II)", sin, sout); +} + + +/* + * Return users currently connected on the system. + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + struct utmpx *ut; + PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; + PyObject *py_user_proc = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + setutxent(); + while (NULL != (ut = getutxent())) { + if (ut->ut_type == USER_PROCESS) + py_user_proc = Py_True; + else + py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut->ut_tv.tv_sec, // tstamp + py_user_proc, // (bool) user process + ut->ut_pid // process id + ); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_username); + Py_CLEAR(py_tty); + Py_CLEAR(py_hostname); + Py_CLEAR(py_tuple); + } + endutxent(); + + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + endutxent(); + return NULL; +} + + +/* + * Return disk mounted partitions as a list of tuples including device, + * mount point and filesystem type. + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + FILE *file; + struct mnttab mt; + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + file = fopen(MNTTAB, "rb"); + if (file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + while (getmntent(file, &mt) == 0) { + py_dev = PyUnicode_DecodeFSDefault(mt.mnt_special); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(mt.mnt_mountp); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue( + "(OOss)", + py_dev, // device + py_mountp, // mount point + mt.mnt_fstype, // fs type + mt.mnt_mntopts); // options + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_dev); + Py_CLEAR(py_mountp); + Py_CLEAR(py_tuple); + } + fclose(file); + return py_retlist; + +error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (file != NULL) + fclose(file); + return NULL; +} + + +/* + * Return system-wide CPU times. + */ +static PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + kstat_ctl_t *kc; + kstat_t *ksp; + cpu_stat_t cs; + PyObject *py_retlist = PyList_New(0); + PyObject *py_cputime = NULL; + + if (py_retlist == NULL) + return NULL; + + kc = kstat_open(); + if (kc == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (ksp = kc->kc_chain; ksp != NULL; ksp = ksp->ks_next) { + if (strcmp(ksp->ks_module, "cpu_stat") == 0) { + if (kstat_read(kc, ksp, &cs) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + py_cputime = Py_BuildValue("ffff", + (float)cs.cpu_sysinfo.cpu[CPU_USER], + (float)cs.cpu_sysinfo.cpu[CPU_KERNEL], + (float)cs.cpu_sysinfo.cpu[CPU_IDLE], + (float)cs.cpu_sysinfo.cpu[CPU_WAIT]); + if (py_cputime == NULL) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_CLEAR(py_cputime); + } + } + + kstat_close(kc); + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + if (kc != NULL) + kstat_close(kc); + return NULL; +} + + +/* + * Return disk IO statistics. + */ +static PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + kstat_ctl_t *kc; + kstat_t *ksp; + kstat_io_t kio; + PyObject *py_retdict = PyDict_New(); + PyObject *py_disk_info = NULL; + + if (py_retdict == NULL) + return NULL; + kc = kstat_open(); + if (kc == NULL) { + PyErr_SetFromErrno(PyExc_OSError);; + goto error; + } + ksp = kc->kc_chain; + while (ksp != NULL) { + if (ksp->ks_type == KSTAT_TYPE_IO) { + if (strcmp(ksp->ks_class, "disk") == 0) { + if (kstat_read(kc, ksp, &kio) == -1) { + kstat_close(kc); + return PyErr_SetFromErrno(PyExc_OSError);; + } + py_disk_info = Py_BuildValue( + "(IIKKLL)", + kio.reads, + kio.writes, + kio.nread, + kio.nwritten, + kio.rtime / 1000 / 1000, // from nano to milli secs + kio.wtime / 1000 / 1000 // from nano to milli secs + ); + if (!py_disk_info) + goto error; + if (PyDict_SetItemString(py_retdict, ksp->ks_name, + py_disk_info)) + goto error; + Py_CLEAR(py_disk_info); + } + } + ksp = ksp->ks_next; + } + kstat_close(kc); + + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + if (kc != NULL) + kstat_close(kc); + return NULL; +} + + +/* + * Return process memory mappings. + */ +static PyObject * +psutil_proc_memory_maps(PyObject *self, PyObject *args) { + int pid; + int fd = -1; + char path[1000]; + char perms[10]; + const char *name; + struct stat st; + pstatus_t status; + + prxmap_t *xmap = NULL, *p; + off_t size; + size_t nread; + int nmap; + uintptr_t pr_addr_sz; + uintptr_t stk_base_sz, brk_base_sz; + const char *procfs_path; + + PyObject *py_tuple = NULL; + PyObject *py_path = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + goto error; + + sprintf(path, "%s/%i/status", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&status, sizeof(status))) + goto error; + + sprintf(path, "%s/%i/xmap", procfs_path, pid); + if (stat(path, &st) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + size = st.st_size; + + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + xmap = (prxmap_t *)malloc(size); + if (xmap == NULL) { + PyErr_NoMemory(); + goto error; + } + + nread = pread(fd, xmap, size, 0); + nmap = nread / sizeof(prxmap_t); + p = xmap; + + while (nmap) { + nmap -= 1; + if (p == NULL) { + p += 1; + continue; + } + + perms[0] = '\0'; + pr_addr_sz = p->pr_vaddr + p->pr_size; + + // perms + sprintf(perms, "%c%c%c%c", p->pr_mflags & MA_READ ? 'r' : '-', + p->pr_mflags & MA_WRITE ? 'w' : '-', + p->pr_mflags & MA_EXEC ? 'x' : '-', + p->pr_mflags & MA_SHARED ? 's' : '-'); + + // name + if (strlen(p->pr_mapname) > 0) { + name = p->pr_mapname; + } + else { + if ((p->pr_mflags & MA_ISM) || (p->pr_mflags & MA_SHM)) { + name = "[shmid]"; + } + else { + stk_base_sz = status.pr_stkbase + status.pr_stksize; + brk_base_sz = status.pr_brkbase + status.pr_brksize; + + if ((pr_addr_sz > status.pr_stkbase) && + (p->pr_vaddr < stk_base_sz)) { + name = "[stack]"; + } + else if ((p->pr_mflags & MA_ANON) && \ + (pr_addr_sz > status.pr_brkbase) && \ + (p->pr_vaddr < brk_base_sz)) { + name = "[heap]"; + } + else { + name = "[anon]"; + } + } + } + + py_path = PyUnicode_DecodeFSDefault(name); + if (! py_path) + goto error; + py_tuple = Py_BuildValue( + "kksOkkk", + (unsigned long)p->pr_vaddr, + (unsigned long)pr_addr_sz, + perms, + py_path, + (unsigned long)p->pr_rss * p->pr_pagesize, + (unsigned long)p->pr_anon * p->pr_pagesize, + (unsigned long)p->pr_locked * p->pr_pagesize); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_path); + Py_CLEAR(py_tuple); + + // increment pointer + p += 1; + } + + close(fd); + free(xmap); + return py_retlist; + +error: + if (fd != -1) + close(fd); + Py_XDECREF(py_tuple); + Py_XDECREF(py_path); + Py_DECREF(py_retlist); + if (xmap != NULL) + free(xmap); + return NULL; +} + + +/* + * Return a list of tuples for network I/O statistics. + */ +static PyObject * +psutil_net_io_counters(PyObject *self, PyObject *args) { + kstat_ctl_t *kc = NULL; + kstat_t *ksp; + kstat_named_t *rbytes, *wbytes, *rpkts, *wpkts, *ierrs, *oerrs; + int ret; + int sock = -1; + struct lifreq ifr; + + PyObject *py_retdict = PyDict_New(); + PyObject *py_ifc_info = NULL; + + if (py_retdict == NULL) + return NULL; + kc = kstat_open(); + if (kc == NULL) + goto error; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + ksp = kc->kc_chain; + while (ksp != NULL) { + if (ksp->ks_type != KSTAT_TYPE_NAMED) + goto next; + if (strcmp(ksp->ks_class, "net") != 0) + goto next; + // skip 'lo' (localhost) because it doesn't have the statistics we need + // and it makes kstat_data_lookup() fail + if (strcmp(ksp->ks_module, "lo") == 0) + goto next; + + // check if this is a network interface by sending a ioctl + strncpy(ifr.lifr_name, ksp->ks_name, sizeof(ifr.lifr_name)); + ret = ioctl(sock, SIOCGLIFFLAGS, &ifr); + if (ret == -1) + goto next; + + if (kstat_read(kc, ksp, NULL) == -1) { + errno = 0; + goto next; + } + + rbytes = (kstat_named_t *)kstat_data_lookup(ksp, "rbytes"); + wbytes = (kstat_named_t *)kstat_data_lookup(ksp, "obytes"); + rpkts = (kstat_named_t *)kstat_data_lookup(ksp, "ipackets"); + wpkts = (kstat_named_t *)kstat_data_lookup(ksp, "opackets"); + ierrs = (kstat_named_t *)kstat_data_lookup(ksp, "ierrors"); + oerrs = (kstat_named_t *)kstat_data_lookup(ksp, "oerrors"); + + if ((rbytes == NULL) || (wbytes == NULL) || (rpkts == NULL) || + (wpkts == NULL) || (ierrs == NULL) || (oerrs == NULL)) + { + PyErr_SetString(PyExc_RuntimeError, "kstat_data_lookup() failed"); + goto error; + } + + if (rbytes->data_type == KSTAT_DATA_UINT64) + { + py_ifc_info = Py_BuildValue("(KKKKIIii)", + wbytes->value.ui64, + rbytes->value.ui64, + wpkts->value.ui64, + rpkts->value.ui64, + ierrs->value.ui32, + oerrs->value.ui32, + 0, // dropin not supported + 0 // dropout not supported + ); + } + else + { + py_ifc_info = Py_BuildValue("(IIIIIIii)", + wbytes->value.ui32, + rbytes->value.ui32, + wpkts->value.ui32, + rpkts->value.ui32, + ierrs->value.ui32, + oerrs->value.ui32, + 0, // dropin not supported + 0 // dropout not supported + ); + } + if (!py_ifc_info) + goto error; + if (PyDict_SetItemString(py_retdict, ksp->ks_name, py_ifc_info)) + goto error; + Py_CLEAR(py_ifc_info); + goto next; + +next: + ksp = ksp->ks_next; + } + + kstat_close(kc); + close(sock); + return py_retdict; + +error: + Py_XDECREF(py_ifc_info); + Py_DECREF(py_retdict); + if (kc != NULL) + kstat_close(kc); + if (sock != -1) { + close(sock); + } + return NULL; +} + + +/* + * Return TCP and UDP connections opened by process. + * UNIX sockets are excluded. + * + * Thanks to: + * https://github.com/DavidGriffith/finx/blob/master/ + * nxsensor-3.5.0-1/src/sysdeps/solaris.c + * ...and: + * https://hg.java.net/hg/solaris~on-src/file/tip/usr/src/cmd/ + * cmd-inet/usr.bin/netstat/netstat.c + */ +static PyObject * +psutil_net_connections(PyObject *self, PyObject *args) { + long pid; + int sd = 0; + mib2_tcpConnEntry_t tp; + mib2_udpEntry_t ude; +#if defined(AF_INET6) + mib2_tcp6ConnEntry_t tp6; + mib2_udp6Entry_t ude6; +#endif + char buf[512]; + int i, flags, getcode, num_ent, state, ret; + char lip[INET6_ADDRSTRLEN], rip[INET6_ADDRSTRLEN]; + int lport, rport; + int processed_pid; + int databuf_init = 0; + struct strbuf ctlbuf, databuf; + struct T_optmgmt_req tor = {0}; + struct T_optmgmt_ack toa = {0}; + struct T_error_ack tea = {0}; + struct opthdr mibhdr = {0}; + + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + sd = open("/dev/arp", O_RDWR); + if (sd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "/dev/arp"); + goto error; + } + + ret = ioctl(sd, I_PUSH, "tcp"); + if (ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + ret = ioctl(sd, I_PUSH, "udp"); + if (ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + // + // OK, this mess is basically copied and pasted from nxsensor project + // which copied and pasted it from netstat source code, mibget() + // function. Also see: + // http://stackoverflow.com/questions/8723598/ + tor.PRIM_type = T_SVR4_OPTMGMT_REQ; + tor.OPT_offset = sizeof (struct T_optmgmt_req); + tor.OPT_length = sizeof (struct opthdr); + tor.MGMT_flags = T_CURRENT; + mibhdr.level = MIB2_IP; + mibhdr.name = 0; + +#ifdef NEW_MIB_COMPLIANT + mibhdr.len = 1; +#else + mibhdr.len = 0; +#endif + memcpy(buf, &tor, sizeof tor); + memcpy(buf + tor.OPT_offset, &mibhdr, sizeof mibhdr); + + ctlbuf.buf = buf; + ctlbuf.len = tor.OPT_offset + tor.OPT_length; + flags = 0; // request to be sent in non-priority + + if (putmsg(sd, &ctlbuf, (struct strbuf *)0, flags) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + ctlbuf.maxlen = sizeof (buf); + for (;;) { + flags = 0; + getcode = getmsg(sd, &ctlbuf, (struct strbuf *)0, &flags); + memcpy(&toa, buf, sizeof toa); + memcpy(&tea, buf, sizeof tea); + + if (getcode != MOREDATA || + ctlbuf.len < (int)sizeof (struct T_optmgmt_ack) || + toa.PRIM_type != T_OPTMGMT_ACK || + toa.MGMT_flags != T_SUCCESS) + { + break; + } + if (ctlbuf.len >= (int)sizeof (struct T_error_ack) && + tea.PRIM_type == T_ERROR_ACK) + { + PyErr_SetString(PyExc_RuntimeError, "ERROR_ACK"); + goto error; + } + if (getcode == 0 && + ctlbuf.len >= (int)sizeof (struct T_optmgmt_ack) && + toa.PRIM_type == T_OPTMGMT_ACK && + toa.MGMT_flags == T_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "ERROR_T_OPTMGMT_ACK"); + goto error; + } + + memset(&mibhdr, 0x0, sizeof(mibhdr)); + memcpy(&mibhdr, buf + toa.OPT_offset, toa.OPT_length); + + databuf.maxlen = mibhdr.len; + databuf.len = 0; + databuf.buf = (char *)malloc((int)mibhdr.len); + if (!databuf.buf) { + PyErr_NoMemory(); + goto error; + } + databuf_init = 1; + + flags = 0; + getcode = getmsg(sd, (struct strbuf *)0, &databuf, &flags); + if (getcode < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + // TCPv4 + if (mibhdr.level == MIB2_TCP && mibhdr.name == MIB2_TCP_13) { + num_ent = mibhdr.len / sizeof(mib2_tcpConnEntry_t); + for (i = 0; i < num_ent; i++) { + memcpy(&tp, databuf.buf + i * sizeof tp, sizeof tp); +#ifdef NEW_MIB_COMPLIANT + processed_pid = tp.tcpConnCreationProcess; +#else + processed_pid = 0; +#endif + if (pid != -1 && processed_pid != pid) + continue; + // construct local/remote addresses + inet_ntop(AF_INET, &tp.tcpConnLocalAddress, lip, sizeof(lip)); + inet_ntop(AF_INET, &tp.tcpConnRemAddress, rip, sizeof(rip)); + lport = tp.tcpConnLocalPort; + rport = tp.tcpConnRemPort; + + // contruct python tuple/list + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", rip, rport); + else { + py_raddr = Py_BuildValue("()"); + } + if (!py_raddr) + goto error; + state = tp.tcpConnEntryInfo.ce_state; + + // add item + py_tuple = Py_BuildValue("(iiiNNiI)", -1, AF_INET, SOCK_STREAM, + py_laddr, py_raddr, state, + processed_pid); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + } +#if defined(AF_INET6) + // TCPv6 + else if (mibhdr.level == MIB2_TCP6 && mibhdr.name == MIB2_TCP6_CONN) + { + num_ent = mibhdr.len / sizeof(mib2_tcp6ConnEntry_t); + + for (i = 0; i < num_ent; i++) { + memcpy(&tp6, databuf.buf + i * sizeof tp6, sizeof tp6); +#ifdef NEW_MIB_COMPLIANT + processed_pid = tp6.tcp6ConnCreationProcess; +#else + processed_pid = 0; +#endif + if (pid != -1 && processed_pid != pid) + continue; + // construct local/remote addresses + inet_ntop(AF_INET6, &tp6.tcp6ConnLocalAddress, lip, sizeof(lip)); + inet_ntop(AF_INET6, &tp6.tcp6ConnRemAddress, rip, sizeof(rip)); + lport = tp6.tcp6ConnLocalPort; + rport = tp6.tcp6ConnRemPort; + + // contruct python tuple/list + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", rip, rport); + else + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + state = tp6.tcp6ConnEntryInfo.ce_state; + + // add item + py_tuple = Py_BuildValue("(iiiNNiI)", -1, AF_INET6, SOCK_STREAM, + py_laddr, py_raddr, state, processed_pid); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + } +#endif + // UDPv4 + else if (mibhdr.level == MIB2_UDP || mibhdr.level == MIB2_UDP_ENTRY) { + num_ent = mibhdr.len / sizeof(mib2_udpEntry_t); + assert(num_ent * sizeof(mib2_udpEntry_t) == mibhdr.len); + for (i = 0; i < num_ent; i++) { + memcpy(&ude, databuf.buf + i * sizeof ude, sizeof ude); +#ifdef NEW_MIB_COMPLIANT + processed_pid = ude.udpCreationProcess; +#else + processed_pid = 0; +#endif + if (pid != -1 && processed_pid != pid) + continue; + // XXX Very ugly hack! It seems we get here only the first + // time we bump into a UDPv4 socket. PID is a very high + // number (clearly impossible) and the address does not + // belong to any valid interface. Not sure what else + // to do other than skipping. + if (processed_pid > 131072) + continue; + inet_ntop(AF_INET, &ude.udpLocalAddress, lip, sizeof(lip)); + lport = ude.udpLocalPort; + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + py_tuple = Py_BuildValue("(iiiNNiI)", -1, AF_INET, SOCK_DGRAM, + py_laddr, py_raddr, PSUTIL_CONN_NONE, + processed_pid); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + } +#if defined(AF_INET6) + // UDPv6 + else if (mibhdr.level == MIB2_UDP6 || + mibhdr.level == MIB2_UDP6_ENTRY) + { + num_ent = mibhdr.len / sizeof(mib2_udp6Entry_t); + for (i = 0; i < num_ent; i++) { + memcpy(&ude6, databuf.buf + i * sizeof ude6, sizeof ude6); +#ifdef NEW_MIB_COMPLIANT + processed_pid = ude6.udp6CreationProcess; +#else + processed_pid = 0; +#endif + if (pid != -1 && processed_pid != pid) + continue; + inet_ntop(AF_INET6, &ude6.udp6LocalAddress, lip, sizeof(lip)); + lport = ude6.udp6LocalPort; + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + py_tuple = Py_BuildValue("(iiiNNiI)", -1, AF_INET6, SOCK_DGRAM, + py_laddr, py_raddr, PSUTIL_CONN_NONE, + processed_pid); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + } +#endif + free(databuf.buf); + } + + close(sd); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + Py_DECREF(py_retlist); + if (databuf_init == 1) + free(databuf.buf); + if (sd != 0) + close(sd); + return NULL; +} + + +static PyObject * +psutil_boot_time(PyObject *self, PyObject *args) { + float boot_time = 0.0; + struct utmpx *ut; + + setutxent(); + while (NULL != (ut = getutxent())) { + if (ut->ut_type == BOOT_TIME) { + boot_time = (float)ut->ut_tv.tv_sec; + break; + } + } + endutxent(); + if (fabs(boot_time) < 0.000001) { + /* could not find BOOT_TIME in getutxent loop */ + PyErr_SetString(PyExc_RuntimeError, "can't determine boot time"); + return NULL; + } + return Py_BuildValue("f", boot_time); +} + + +/* + * Return the number of physical CPU cores on the system. + */ +static PyObject * +psutil_cpu_count_phys(PyObject *self, PyObject *args) { + kstat_ctl_t *kc; + kstat_t *ksp; + int ncpus = 0; + + kc = kstat_open(); + if (kc == NULL) + goto error; + ksp = kstat_lookup(kc, "cpu_info", -1, NULL); + if (ksp == NULL) + goto error; + + for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) { + if (strcmp(ksp->ks_module, "cpu_info") != 0) + continue; + if (kstat_read(kc, ksp, NULL) == -1) + goto error; + ncpus += 1; + } + + kstat_close(kc); + if (ncpus > 0) + return Py_BuildValue("i", ncpus); + else + goto error; + +error: + // mimic os.cpu_count() + if (kc != NULL) + kstat_close(kc); + Py_RETURN_NONE; +} + + +/* + * Return stats about a particular network + * interface. References: + * https://github.com/dpaleino/wicd/blob/master/wicd/backends/be-ioctl.py + * http://www.i-scream.org/libstatgrab/ + */ +static PyObject* +psutil_net_if_stats(PyObject* self, PyObject* args) { + kstat_ctl_t *kc = NULL; + kstat_t *ksp; + kstat_named_t *knp; + int ret; + int sock = -1; + int duplex; + int speed; + + PyObject *py_retdict = PyDict_New(); + PyObject *py_ifc_info = NULL; + PyObject *py_is_up = NULL; + + if (py_retdict == NULL) + return NULL; + kc = kstat_open(); + if (kc == NULL) + goto error; + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) { + if (strcmp(ksp->ks_class, "net") == 0) { + struct lifreq ifr; + + kstat_read(kc, ksp, NULL); + if (ksp->ks_type != KSTAT_TYPE_NAMED) + continue; + if (strcmp(ksp->ks_class, "net") != 0) + continue; + + strncpy(ifr.lifr_name, ksp->ks_name, sizeof(ifr.lifr_name)); + ret = ioctl(sock, SIOCGLIFFLAGS, &ifr); + if (ret == -1) + continue; // not a network interface + + // is up? + if ((ifr.lifr_flags & IFF_UP) != 0) { + if ((knp = kstat_data_lookup(ksp, "link_up")) != NULL) { + if (knp->value.ui32 != 0u) + py_is_up = Py_True; + else + py_is_up = Py_False; + } + else { + py_is_up = Py_True; + } + } + else { + py_is_up = Py_False; + } + Py_INCREF(py_is_up); + + // duplex + duplex = 0; // unknown + if ((knp = kstat_data_lookup(ksp, "link_duplex")) != NULL) { + if (knp->value.ui32 == 1) + duplex = 1; // half + else if (knp->value.ui32 == 2) + duplex = 2; // full + } + + // speed + if ((knp = kstat_data_lookup(ksp, "ifspeed")) != NULL) + // expressed in bits per sec, we want mega bits per sec + speed = (int)knp->value.ui64 / 1000000; + else + speed = 0; + + // mtu + ret = ioctl(sock, SIOCGLIFMTU, &ifr); + if (ret == -1) + goto error; + + py_ifc_info = Py_BuildValue("(Oiii)", py_is_up, duplex, speed, + ifr.lifr_mtu); + if (!py_ifc_info) + goto error; + if (PyDict_SetItemString(py_retdict, ksp->ks_name, py_ifc_info)) + goto error; + Py_CLEAR(py_ifc_info); + } + } + + close(sock); + kstat_close(kc); + return py_retdict; + +error: + Py_XDECREF(py_is_up); + Py_XDECREF(py_ifc_info); + Py_DECREF(py_retdict); + if (sock != -1) + close(sock); + if (kc != NULL) + kstat_close(kc); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} + + +/* + * Return CPU statistics. + */ +static PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + kstat_ctl_t *kc; + kstat_t *ksp; + cpu_stat_t cs; + unsigned int ctx_switches = 0; + unsigned int interrupts = 0; + unsigned int traps = 0; + unsigned int syscalls = 0; + + kc = kstat_open(); + if (kc == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (ksp = kc->kc_chain; ksp != NULL; ksp = ksp->ks_next) { + if (strcmp(ksp->ks_module, "cpu_stat") == 0) { + if (kstat_read(kc, ksp, &cs) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + // voluntary + involuntary + ctx_switches += cs.cpu_sysinfo.pswitch + cs.cpu_sysinfo.inv_swtch; + interrupts += cs.cpu_sysinfo.intr; + traps += cs.cpu_sysinfo.trap; + syscalls += cs.cpu_sysinfo.syscall; + } + } + + kstat_close(kc); + return Py_BuildValue( + "IIII", ctx_switches, interrupts, syscalls, traps); + +error: + if (kc != NULL) + kstat_close(kc); + return NULL; +} + + +/* + * define the psutil C module methods and initialize the module. + */ +static PyMethodDef +PsutilMethods[] = { + // --- process-related functions + {"proc_basic_info", psutil_proc_basic_info, METH_VARARGS, + "Return process ppid, rss, vms, ctime, nice, nthreads, status and tty"}, + {"proc_name_and_args", psutil_proc_name_and_args, METH_VARARGS, + "Return process name and args."}, + {"proc_environ", psutil_proc_environ, METH_VARARGS, + "Return process environment."}, + {"proc_cpu_times", psutil_proc_cpu_times, METH_VARARGS, + "Return process user and system CPU times."}, + {"proc_cred", psutil_proc_cred, METH_VARARGS, + "Return process uids/gids."}, + {"query_process_thread", psutil_proc_query_thread, METH_VARARGS, + "Return info about a process thread"}, + {"proc_memory_maps", psutil_proc_memory_maps, METH_VARARGS, + "Return process memory mappings"}, + {"proc_num_ctx_switches", psutil_proc_num_ctx_switches, METH_VARARGS, + "Return the number of context switches performed by process"}, + {"proc_cpu_num", psutil_proc_cpu_num, METH_VARARGS, + "Return what CPU the process is on"}, + + // --- system-related functions + {"swap_mem", psutil_swap_mem, METH_VARARGS, + "Return information about system swap memory."}, + {"users", psutil_users, METH_VARARGS, + "Return currently connected users."}, + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return disk partitions."}, + {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, + "Return system per-CPU times."}, + {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS, + "Return a Python dict of tuples for disk I/O statistics."}, + {"net_io_counters", psutil_net_io_counters, METH_VARARGS, + "Return a Python dict of tuples for network I/O statistics."}, + {"boot_time", psutil_boot_time, METH_VARARGS, + "Return system boot time in seconds since the EPOCH."}, + {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, + "Return the number of physical CPUs on the system."}, + {"net_connections", psutil_net_connections, METH_VARARGS, + "Return TCP and UDP syste-wide open connections."}, + {"net_if_stats", psutil_net_if_stats, METH_VARARGS, + "Return NIC stats (isup, duplex, speed, mtu)"}, + {"cpu_stats", psutil_cpu_stats, METH_VARARGS, + "Return CPU statistics"}, + + // --- others + {"set_testing", psutil_set_testing, METH_NOARGS, + "Set psutil in testing mode"}, + + {NULL, NULL, 0, NULL} +}; + + +struct module_state { + PyObject *error; +}; + +#if PY_MAJOR_VERSION >= 3 +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else +#define GETSTATE(m) (&_state) +#endif + +#if PY_MAJOR_VERSION >= 3 + +static int +psutil_sunos_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +static int +psutil_sunos_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "psutil_sunos", + NULL, + sizeof(struct module_state), + PsutilMethods, + NULL, + psutil_sunos_traverse, + psutil_sunos_clear, + NULL +}; + +#define INITERROR return NULL + +PyMODINIT_FUNC PyInit__psutil_sunos(void) + +#else +#define INITERROR return + +void init_psutil_sunos(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("_psutil_sunos", PsutilMethods); +#endif + if (module == NULL) + INITERROR; + + if (psutil_setup() != 0) + INITERROR; + + PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); + + PyModule_AddIntConstant(module, "SSLEEP", SSLEEP); + PyModule_AddIntConstant(module, "SRUN", SRUN); + PyModule_AddIntConstant(module, "SZOMB", SZOMB); + PyModule_AddIntConstant(module, "SSTOP", SSTOP); + PyModule_AddIntConstant(module, "SIDL", SIDL); + PyModule_AddIntConstant(module, "SONPROC", SONPROC); + PyModule_AddIntConstant(module, "SWAIT", SWAIT); + + PyModule_AddIntConstant(module, "PRNODEV", PRNODEV); // for process tty + + PyModule_AddIntConstant(module, "TCPS_CLOSED", TCPS_CLOSED); + PyModule_AddIntConstant(module, "TCPS_CLOSING", TCPS_CLOSING); + PyModule_AddIntConstant(module, "TCPS_CLOSE_WAIT", TCPS_CLOSE_WAIT); + PyModule_AddIntConstant(module, "TCPS_LISTEN", TCPS_LISTEN); + PyModule_AddIntConstant(module, "TCPS_ESTABLISHED", TCPS_ESTABLISHED); + PyModule_AddIntConstant(module, "TCPS_SYN_SENT", TCPS_SYN_SENT); + PyModule_AddIntConstant(module, "TCPS_SYN_RCVD", TCPS_SYN_RCVD); + PyModule_AddIntConstant(module, "TCPS_FIN_WAIT_1", TCPS_FIN_WAIT_1); + PyModule_AddIntConstant(module, "TCPS_FIN_WAIT_2", TCPS_FIN_WAIT_2); + PyModule_AddIntConstant(module, "TCPS_LAST_ACK", TCPS_LAST_ACK); + PyModule_AddIntConstant(module, "TCPS_TIME_WAIT", TCPS_TIME_WAIT); + // sunos specific + PyModule_AddIntConstant(module, "TCPS_IDLE", TCPS_IDLE); + // sunos specific + PyModule_AddIntConstant(module, "TCPS_BOUND", TCPS_BOUND); + PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + + if (module == NULL) + INITERROR; +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} diff --git a/ddtrace/vendor/psutil/_psutil_windows.c b/ddtrace/vendor/psutil/_psutil_windows.c new file mode 100644 index 00000000000..f35d0076d29 --- /dev/null +++ b/ddtrace/vendor/psutil/_psutil_windows.c @@ -0,0 +1,3723 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Windows platform-specific module methods for _psutil_windows. + * + * List of undocumented Windows NT APIs which are used in here and in + * other modules: + * - NtQuerySystemInformation + * - NtQueryInformationProcess + * - NtQueryObject + * - NtSuspendProcess + * - NtResumeProcess + */ + +// Fixes clash between winsock2.h and windows.h +#define WIN32_LEAN_AND_MEAN + +#include +#include +#include +#include +#include // disk_io_counters() +#include +#include +#include // users() +#include // cpu_freq() +#if (_WIN32_WINNT >= 0x0600) // Windows >= Vista +#include // net_connections() +#endif + +// Link with Iphlpapi.lib +#pragma comment(lib, "IPHLPAPI.lib") + +#include "arch/windows/ntextapi.h" +#include "arch/windows/global.h" +#include "arch/windows/security.h" +#include "arch/windows/process_info.h" +#include "arch/windows/process_handles.h" +#include "arch/windows/inet_ntop.h" +#include "arch/windows/services.h" +#include "arch/windows/wmi.h" +#include "_psutil_common.h" + + +/* + * ============================================================================ + * Utilities + * ============================================================================ + */ + +#define MALLOC(x) HeapAlloc(GetProcessHeap(), 0, (x)) +#define FREE(x) HeapFree(GetProcessHeap(), 0, (x)) +#define LO_T 1e-7 +#define HI_T 429.4967296 +#define BYTESWAP_USHORT(x) ((((USHORT)(x) << 8) | ((USHORT)(x) >> 8)) & 0xffff) +#ifndef AF_INET6 +#define AF_INET6 23 +#endif + + +PIP_ADAPTER_ADDRESSES +psutil_get_nic_addresses() { + // allocate a 15 KB buffer to start with + int outBufLen = 15000; + DWORD dwRetVal = 0; + ULONG attempts = 0; + PIP_ADAPTER_ADDRESSES pAddresses = NULL; + + do { + pAddresses = (IP_ADAPTER_ADDRESSES *) malloc(outBufLen); + if (pAddresses == NULL) { + PyErr_NoMemory(); + return NULL; + } + + dwRetVal = GetAdaptersAddresses(AF_UNSPEC, 0, NULL, pAddresses, + &outBufLen); + if (dwRetVal == ERROR_BUFFER_OVERFLOW) { + free(pAddresses); + pAddresses = NULL; + } + else { + break; + } + + attempts++; + } while ((dwRetVal == ERROR_BUFFER_OVERFLOW) && (attempts < 3)); + + if (dwRetVal != NO_ERROR) { + PyErr_SetString( + PyExc_RuntimeError, "GetAdaptersAddresses() syscall failed."); + return NULL; + } + + return pAddresses; +} + + +/* + * Return the number of logical, active CPUs. Return 0 if undetermined. + * See discussion at: https://bugs.python.org/issue33166#msg314631 + */ +unsigned int +psutil_get_num_cpus(int fail_on_err) { + unsigned int ncpus = 0; + + // Minimum requirement: Windows 7 + if (psutil_GetActiveProcessorCount != NULL) { + ncpus = psutil_GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); + if ((ncpus == 0) && (fail_on_err == 1)) { + PyErr_SetFromWindowsErr(0); + } + } + else { + psutil_debug("GetActiveProcessorCount() not available; " + "using GetSystemInfo()"); + ncpus = (unsigned int)PSUTIL_SYSTEM_INFO.dwNumberOfProcessors; + if ((ncpus <= 0) && (fail_on_err == 1)) { + PyErr_SetString( + PyExc_RuntimeError, + "GetSystemInfo() failed to retrieve CPU count"); + } + } + return ncpus; +} + + +/* + * ============================================================================ + * Public Python API + * ============================================================================ + */ + +// Raised by Process.wait(). +static PyObject *TimeoutExpired; +static PyObject *TimeoutAbandoned; + +/* + * Return a Python float representing the system uptime expressed in seconds + * since the epoch. + */ +static PyObject * +psutil_boot_time(PyObject *self, PyObject *args) { + ULONGLONG uptime; + time_t pt; + FILETIME fileTime; + ULONGLONG ll; + + GetSystemTimeAsFileTime(&fileTime); + /* + HUGE thanks to: + http://johnstewien.spaces.live.com/blog/cns!E6885DB5CEBABBC8!831.entry + + This function converts the FILETIME structure to the 32 bit + Unix time structure. + The time_t is a 32-bit value for the number of seconds since + January 1, 1970. A FILETIME is a 64-bit for the number of + 100-nanosecond periods since January 1, 1601. Convert by + subtracting the number of 100-nanosecond period between 01-01-1970 + and 01-01-1601, from time_t the divide by 1e+7 to get to the same + base granularity. + */ + ll = (((ULONGLONG) + (fileTime.dwHighDateTime)) << 32) + fileTime.dwLowDateTime; + pt = (time_t)((ll - 116444736000000000ull) / 10000000ull); + + if (psutil_GetTickCount64 != NULL) { + // Windows >= Vista + uptime = psutil_GetTickCount64() / 1000ull; + } + else { + // Windows XP. + // GetTickCount() time will wrap around to zero if the + // system is run continuously for 49.7 days. + uptime = (ULONGLONG)GetTickCount() / 1000ull; + } + return Py_BuildValue("K", pt - uptime); +} + + +/* + * Return 1 if PID exists in the current process list, else 0. + */ +static PyObject * +psutil_pid_exists(PyObject *self, PyObject *args) { + long pid; + int status; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + status = psutil_pid_is_running(pid); + if (-1 == status) + return NULL; // exception raised in psutil_pid_is_running() + return PyBool_FromLong(status); +} + + +/* + * Return a Python list of all the PIDs running on the system. + */ +static PyObject * +psutil_pids(PyObject *self, PyObject *args) { + DWORD *proclist = NULL; + DWORD numberOfReturnedPIDs; + DWORD i; + PyObject *py_pid = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + proclist = psutil_get_pids(&numberOfReturnedPIDs); + if (proclist == NULL) + goto error; + + for (i = 0; i < numberOfReturnedPIDs; i++) { + py_pid = Py_BuildValue("I", proclist[i]); + if (!py_pid) + goto error; + if (PyList_Append(py_retlist, py_pid)) + goto error; + Py_CLEAR(py_pid); + } + + // free C array allocated for PIDs + free(proclist); + return py_retlist; + +error: + Py_XDECREF(py_pid); + Py_DECREF(py_retlist); + if (proclist != NULL) + free(proclist); + return NULL; +} + + +/* + * Kill a process given its PID. + */ +static PyObject * +psutil_proc_kill(PyObject *self, PyObject *args) { + HANDLE hProcess; + long pid; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (pid == 0) + return AccessDenied(""); + + hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pid); + if (hProcess == NULL) { + if (GetLastError() == ERROR_INVALID_PARAMETER) { + // see https://github.com/giampaolo/psutil/issues/24 + psutil_debug("OpenProcess -> ERROR_INVALID_PARAMETER turned " + "into NoSuchProcess"); + NoSuchProcess(""); + } + else { + PyErr_SetFromWindowsErr(0); + } + return NULL; + } + + if (! TerminateProcess(hProcess, SIGTERM)) { + // ERROR_ACCESS_DENIED may happen if the process already died. See: + // https://github.com/giampaolo/psutil/issues/1099 + if (GetLastError() != ERROR_ACCESS_DENIED) { + PyErr_SetFromOSErrnoWithSyscall("TerminateProcess"); + return NULL; + } + } + + CloseHandle(hProcess); + Py_RETURN_NONE; +} + + +/* + * Wait for process to terminate and return its exit code. + */ +static PyObject * +psutil_proc_wait(PyObject *self, PyObject *args) { + HANDLE hProcess; + DWORD ExitCode; + DWORD retVal; + long pid; + long timeout; + + if (! PyArg_ParseTuple(args, "ll", &pid, &timeout)) + return NULL; + if (pid == 0) + return AccessDenied(""); + + hProcess = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_INFORMATION, + FALSE, pid); + if (hProcess == NULL) { + if (GetLastError() == ERROR_INVALID_PARAMETER) { + // no such process; we do not want to raise NSP but + // return None instead. + Py_RETURN_NONE; + } + else + return PyErr_SetFromWindowsErr(0); + } + + // wait until the process has terminated + Py_BEGIN_ALLOW_THREADS + retVal = WaitForSingleObject(hProcess, timeout); + Py_END_ALLOW_THREADS + + // handle return code + if (retVal == WAIT_FAILED) { + PyErr_SetFromOSErrnoWithSyscall("WaitForSingleObject"); + CloseHandle(hProcess); + return NULL; + } + if (retVal == WAIT_TIMEOUT) { + PyErr_SetString(TimeoutExpired, + "WaitForSingleObject() returned WAIT_TIMEOUT"); + CloseHandle(hProcess); + return NULL; + } + if (retVal == WAIT_ABANDONED) { + psutil_debug("WaitForSingleObject() -> WAIT_ABANDONED"); + PyErr_SetString(TimeoutAbandoned, + "WaitForSingleObject() returned WAIT_ABANDONED"); + CloseHandle(hProcess); + return NULL; + } + + // WaitForSingleObject() returned WAIT_OBJECT_0. It means the + // process is gone so we can get its process exit code. The PID + // may still stick around though but we'll handle that from Python. + if (GetExitCodeProcess(hProcess, &ExitCode) == 0) { + PyErr_SetFromOSErrnoWithSyscall("GetExitCodeProcess"); + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); + +#if PY_MAJOR_VERSION >= 3 + return PyLong_FromLong((long) ExitCode); +#else + return PyInt_FromLong((long) ExitCode); +#endif +} + + +/* + * Return a Python tuple (user_time, kernel_time) + */ +static PyObject * +psutil_proc_cpu_times(PyObject *self, PyObject *args) { + long pid; + HANDLE hProcess; + FILETIME ftCreate, ftExit, ftKernel, ftUser; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + + if (hProcess == NULL) + return NULL; + if (! GetProcessTimes(hProcess, &ftCreate, &ftExit, &ftKernel, &ftUser)) { + if (GetLastError() == ERROR_ACCESS_DENIED) { + // usually means the process has died so we throw a NoSuchProcess + // here + NoSuchProcess(""); + } + else { + PyErr_SetFromWindowsErr(0); + } + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); + + /* + * User and kernel times are represented as a FILETIME structure + * which contains a 64-bit value representing the number of + * 100-nanosecond intervals since January 1, 1601 (UTC): + * http://msdn.microsoft.com/en-us/library/ms724284(VS.85).aspx + * To convert it into a float representing the seconds that the + * process has executed in user/kernel mode I borrowed the code + * below from Python's Modules/posixmodule.c + */ + return Py_BuildValue( + "(dd)", + (double)(ftUser.dwHighDateTime * 429.4967296 + \ + ftUser.dwLowDateTime * 1e-7), + (double)(ftKernel.dwHighDateTime * 429.4967296 + \ + ftKernel.dwLowDateTime * 1e-7) + ); +} + + +/* + * Return a Python float indicating the process create time expressed in + * seconds since the epoch. + */ +static PyObject * +psutil_proc_create_time(PyObject *self, PyObject *args) { + long pid; + long long unix_time; + HANDLE hProcess; + FILETIME ftCreate, ftExit, ftKernel, ftUser; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + // special case for PIDs 0 and 4, return system boot time + if (0 == pid || 4 == pid) + return psutil_boot_time(NULL, NULL); + + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) + return NULL; + if (! GetProcessTimes(hProcess, &ftCreate, &ftExit, &ftKernel, &ftUser)) { + if (GetLastError() == ERROR_ACCESS_DENIED) { + // usually means the process has died so we throw a + // NoSuchProcess here + NoSuchProcess(""); + } + else { + PyErr_SetFromWindowsErr(0); + } + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); + + /* + // Make sure the process is not gone as OpenProcess alone seems to be + // unreliable in doing so (it seems a previous call to p.wait() makes + // it unreliable). + // This check is important as creation time is used to make sure the + // process is still running. + ret = GetExitCodeProcess(hProcess, &exitCode); + CloseHandle(hProcess); + if (ret != 0) { + if (exitCode != STILL_ACTIVE) + return NoSuchProcess(""); + } + else { + // Ignore access denied as it means the process is still alive. + // For all other errors, we want an exception. + if (GetLastError() != ERROR_ACCESS_DENIED) + return PyErr_SetFromWindowsErr(0); + } + */ + + // Convert the FILETIME structure to a Unix time. + // It's the best I could find by googling and borrowing code here + // and there. The time returned has a precision of 1 second. + unix_time = ((LONGLONG)ftCreate.dwHighDateTime) << 32; + unix_time += ftCreate.dwLowDateTime - 116444736000000000LL; + unix_time /= 10000000; + return Py_BuildValue("d", (double)unix_time); +} + + +/* + * Return the number of active, logical CPUs. + */ +static PyObject * +psutil_cpu_count_logical(PyObject *self, PyObject *args) { + unsigned int ncpus; + + ncpus = psutil_get_num_cpus(0); + if (ncpus != 0) + return Py_BuildValue("I", ncpus); + else + Py_RETURN_NONE; // mimick os.cpu_count() +} + + +/* + * Return the number of physical CPU cores (hyper-thread CPUs count + * is excluded). + */ +static PyObject * +psutil_cpu_count_phys(PyObject *self, PyObject *args) { + DWORD rc; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX buffer = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ptr = NULL; + DWORD length = 0; + DWORD offset = 0; + DWORD ncpus = 0; + DWORD prev_processor_info_size = 0; + + // GetLogicalProcessorInformationEx() is available from Windows 7 + // onward. Differently from GetLogicalProcessorInformation() + // it supports process groups, meaning this is able to report more + // than 64 CPUs. See: + // https://bugs.python.org/issue33166 + if (psutil_GetLogicalProcessorInformationEx == NULL) { + psutil_debug("Win < 7; cpu_count_phys() forced to None"); + Py_RETURN_NONE; + } + + while (1) { + rc = psutil_GetLogicalProcessorInformationEx( + RelationAll, buffer, &length); + if (rc == FALSE) { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + if (buffer) { + free(buffer); + } + buffer = \ + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)malloc(length); + if (NULL == buffer) { + PyErr_NoMemory(); + return NULL; + } + } + else { + psutil_debug("GetLogicalProcessorInformationEx() returned ", + GetLastError()); + goto return_none; + } + } + else { + break; + } + } + + ptr = buffer; + while (offset < length) { + // Advance ptr by the size of the previous + // SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX struct. + ptr = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*)\ + (((char*)ptr) + prev_processor_info_size); + + if (ptr->Relationship == RelationProcessorCore) { + ncpus += 1; + } + + // When offset == length, we've reached the last processor + // info struct in the buffer. + offset += ptr->Size; + prev_processor_info_size = ptr->Size; + } + + free(buffer); + if (ncpus != 0) { + return Py_BuildValue("I", ncpus); + } + else { + psutil_debug("GetLogicalProcessorInformationEx() count was 0"); + Py_RETURN_NONE; // mimick os.cpu_count() + } + +return_none: + if (buffer != NULL) + free(buffer); + Py_RETURN_NONE; +} + + +/* + * Return process cmdline as a Python list of cmdline arguments. + */ +static PyObject * +psutil_proc_cmdline(PyObject *self, PyObject *args, PyObject *kwdict) { + long pid; + int pid_return; + int use_peb; + PyObject *py_usepeb = Py_True; + static char *keywords[] = {"pid", "use_peb", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "i|O", + keywords, &pid, &py_usepeb)) { + return NULL; + } + if ((pid == 0) || (pid == 4)) + return Py_BuildValue("[]"); + + pid_return = psutil_pid_is_running(pid); + if (pid_return == 0) + return NoSuchProcess(""); + if (pid_return == -1) + return NULL; + + use_peb = (py_usepeb == Py_True) ? 1 : 0; + return psutil_get_cmdline(pid, use_peb); +} + + +/* + * Return process cmdline as a Python list of cmdline arguments. + */ +static PyObject * +psutil_proc_environ(PyObject *self, PyObject *args) { + long pid; + int pid_return; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if ((pid == 0) || (pid == 4)) + return Py_BuildValue("s", ""); + + pid_return = psutil_pid_is_running(pid); + if (pid_return == 0) + return NoSuchProcess(""); + if (pid_return == -1) + return NULL; + + return psutil_get_environ(pid); +} + + +/* + * Return process executable path. + */ +static PyObject * +psutil_proc_exe(PyObject *self, PyObject *args) { + long pid; + HANDLE hProcess; + wchar_t exe[MAX_PATH]; +#if (_WIN32_WINNT >= 0x0600) // >= Vista + unsigned int size = sizeof(exe); +#endif + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (NULL == hProcess) + return NULL; + + // Here we differentiate between XP and Vista+ because + // QueryFullProcessImageNameW is better than GetProcessImageFileNameW + // (avoid using QueryDosDevice on the returned path), see: + // https://github.com/giampaolo/psutil/issues/1394 +#if (_WIN32_WINNT >= 0x0600) // Windows >= Vista + memset(exe, 0, MAX_PATH); + if (QueryFullProcessImageNameW(hProcess, 0, exe, &size) == 0) { + PyErr_SetFromOSErrnoWithSyscall("QueryFullProcessImageNameW"); + CloseHandle(hProcess); + return NULL; + } +#else // Windows XP + if (GetProcessImageFileNameW(hProcess, exe, MAX_PATH) == 0) { + // see: https://github.com/giampaolo/psutil/issues/1394 + if (GetLastError() == 0) + PyErr_SetFromWindowsErr(ERROR_ACCESS_DENIED); + else + PyErr_SetFromOSErrnoWithSyscall("GetProcessImageFileNameW"); + CloseHandle(hProcess); + return NULL; + } +#endif + CloseHandle(hProcess); + return PyUnicode_FromWideChar(exe, wcslen(exe)); +} + + +/* + * Return process base name. + * Note: psutil_proc_exe() is attempted first because it's faster + * but it raise AccessDenied for processes owned by other users + * in which case we fall back on using this. + */ +static PyObject * +psutil_proc_name(PyObject *self, PyObject *args) { + long pid; + int ok; + PROCESSENTRY32W pentry; + HANDLE hSnapShot; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + hSnapShot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, pid); + if (hSnapShot == INVALID_HANDLE_VALUE) + return PyErr_SetFromOSErrnoWithSyscall("CreateToolhelp32Snapshot"); + pentry.dwSize = sizeof(PROCESSENTRY32W); + ok = Process32FirstW(hSnapShot, &pentry); + if (! ok) { + PyErr_SetFromOSErrnoWithSyscall("Process32FirstW"); + CloseHandle(hSnapShot); + return NULL; + } + while (ok) { + if (pentry.th32ProcessID == pid) { + CloseHandle(hSnapShot); + return PyUnicode_FromWideChar( + pentry.szExeFile, wcslen(pentry.szExeFile)); + } + ok = Process32NextW(hSnapShot, &pentry); + } + + CloseHandle(hSnapShot); + NoSuchProcess(""); + return NULL; +} + + +/* + * Return process memory information as a Python tuple. + */ +static PyObject * +psutil_proc_memory_info(PyObject *self, PyObject *args) { + HANDLE hProcess; + DWORD pid; +#if (_WIN32_WINNT >= 0x0501) // Windows XP with SP2 + PROCESS_MEMORY_COUNTERS_EX cnt; +#else + PROCESS_MEMORY_COUNTERS cnt; +#endif + SIZE_T private = 0; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (NULL == hProcess) + return NULL; + + if (! GetProcessMemoryInfo(hProcess, (PPROCESS_MEMORY_COUNTERS)&cnt, + sizeof(cnt))) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + +#if (_WIN32_WINNT >= 0x0501) // Windows XP with SP2 + private = cnt.PrivateUsage; +#endif + + CloseHandle(hProcess); + + // PROCESS_MEMORY_COUNTERS values are defined as SIZE_T which on 64bits + // is an (unsigned long long) and on 32bits is an (unsigned int). + // "_WIN64" is defined if we're running a 64bit Python interpreter not + // exclusively if the *system* is 64bit. +#if defined(_WIN64) + return Py_BuildValue( + "(kKKKKKKKKK)", + cnt.PageFaultCount, // unsigned long + (unsigned long long)cnt.PeakWorkingSetSize, + (unsigned long long)cnt.WorkingSetSize, + (unsigned long long)cnt.QuotaPeakPagedPoolUsage, + (unsigned long long)cnt.QuotaPagedPoolUsage, + (unsigned long long)cnt.QuotaPeakNonPagedPoolUsage, + (unsigned long long)cnt.QuotaNonPagedPoolUsage, + (unsigned long long)cnt.PagefileUsage, + (unsigned long long)cnt.PeakPagefileUsage, + (unsigned long long)private); +#else + return Py_BuildValue( + "(kIIIIIIIII)", + cnt.PageFaultCount, // unsigned long + (unsigned int)cnt.PeakWorkingSetSize, + (unsigned int)cnt.WorkingSetSize, + (unsigned int)cnt.QuotaPeakPagedPoolUsage, + (unsigned int)cnt.QuotaPagedPoolUsage, + (unsigned int)cnt.QuotaPeakNonPagedPoolUsage, + (unsigned int)cnt.QuotaNonPagedPoolUsage, + (unsigned int)cnt.PagefileUsage, + (unsigned int)cnt.PeakPagefileUsage, + (unsigned int)private); +#endif +} + + +static int +psutil_GetProcWsetInformation( + DWORD pid, + HANDLE hProcess, + PMEMORY_WORKING_SET_INFORMATION *wSetInfo) +{ + NTSTATUS status; + PVOID buffer; + SIZE_T bufferSize; + + bufferSize = 0x8000; + buffer = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, bufferSize); + + while ((status = psutil_NtQueryVirtualMemory( + hProcess, + NULL, + MemoryWorkingSetInformation, + buffer, + bufferSize, + NULL)) == STATUS_INFO_LENGTH_MISMATCH) + { + HeapFree(GetProcessHeap(), 0, buffer); + bufferSize *= 2; + psutil_debug("NtQueryVirtualMemory increase bufsize %zd", bufferSize); + // Fail if we're resizing the buffer to something very large. + if (bufferSize > 256 * 1024 * 1024) { + PyErr_SetString(PyExc_RuntimeError, + "NtQueryVirtualMemory bufsize is too large"); + return 1; + } + buffer = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, bufferSize); + } + + if (!NT_SUCCESS(status)) { + if (status == STATUS_ACCESS_DENIED) { + AccessDenied("originated from NtQueryVirtualMemory"); + } + else if (psutil_pid_is_running(pid) == 0) { + NoSuchProcess(""); + } + else { + PyErr_Clear(); + psutil_SetFromNTStatusErr( + status, "NtQueryVirtualMemory(MemoryWorkingSetInformation)"); + } + HeapFree(GetProcessHeap(), 0, buffer); + return 1; + } + + *wSetInfo = (PMEMORY_WORKING_SET_INFORMATION)buffer; + return 0; +} + + +/* + * Returns the USS of the process. + * Reference: + * https://dxr.mozilla.org/mozilla-central/source/xpcom/base/ + * nsMemoryReporterManager.cpp + */ +static PyObject * +psutil_proc_memory_uss(PyObject *self, PyObject *args) { + DWORD pid; + HANDLE hProcess; + PSUTIL_PROCESS_WS_COUNTERS wsCounters; + PMEMORY_WORKING_SET_INFORMATION wsInfo; + ULONG_PTR i; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) + return NULL; + + if (psutil_GetProcWsetInformation(pid, hProcess, &wsInfo) != 0) { + CloseHandle(hProcess); + return NULL; + } + memset(&wsCounters, 0, sizeof(PSUTIL_PROCESS_WS_COUNTERS)); + + for (i = 0; i < wsInfo->NumberOfEntries; i++) { + // This is what ProcessHacker does. + /* + wsCounters.NumberOfPages++; + if (wsInfo->WorkingSetInfo[i].ShareCount > 1) + wsCounters.NumberOfSharedPages++; + if (wsInfo->WorkingSetInfo[i].ShareCount == 0) + wsCounters.NumberOfPrivatePages++; + if (wsInfo->WorkingSetInfo[i].Shared) + wsCounters.NumberOfShareablePages++; + */ + + // This is what we do: count shared pages that only one process + // is using as private (USS). + if (!wsInfo->WorkingSetInfo[i].Shared || + wsInfo->WorkingSetInfo[i].ShareCount <= 1) { + wsCounters.NumberOfPrivatePages++; + } + } + + HeapFree(GetProcessHeap(), 0, wsInfo); + CloseHandle(hProcess); + + return Py_BuildValue("I", wsCounters.NumberOfPrivatePages); +} + + +/* + * Return a Python integer indicating the total amount of physical memory + * in bytes. + */ +static PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + + if (! GlobalMemoryStatusEx(&memInfo)) + return PyErr_SetFromWindowsErr(0); + return Py_BuildValue("(LLLLLL)", + memInfo.ullTotalPhys, // total + memInfo.ullAvailPhys, // avail + memInfo.ullTotalPageFile, // total page file + memInfo.ullAvailPageFile, // avail page file + memInfo.ullTotalVirtual, // total virtual + memInfo.ullAvailVirtual); // avail virtual +} + +/* + * Retrieves system CPU timing information as a (user, system, idle) + * tuple. On a multiprocessor system, the values returned are the + * sum of the designated times across all processors. + */ +static PyObject * +psutil_cpu_times(PyObject *self, PyObject *args) { + double idle, kernel, user, system; + FILETIME idle_time, kernel_time, user_time; + + if (!GetSystemTimes(&idle_time, &kernel_time, &user_time)) + return PyErr_SetFromWindowsErr(0); + + idle = (double)((HI_T * idle_time.dwHighDateTime) + \ + (LO_T * idle_time.dwLowDateTime)); + user = (double)((HI_T * user_time.dwHighDateTime) + \ + (LO_T * user_time.dwLowDateTime)); + kernel = (double)((HI_T * kernel_time.dwHighDateTime) + \ + (LO_T * kernel_time.dwLowDateTime)); + + // Kernel time includes idle time. + // We return only busy kernel time subtracting idle time from + // kernel time. + system = (kernel - idle); + return Py_BuildValue("(ddd)", user, system, idle); +} + + +/* + * Same as above but for all system CPUs. + */ +static PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + double idle, kernel, systemt, user, interrupt, dpc; + NTSTATUS status; + _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *sppi = NULL; + UINT i; + unsigned int ncpus; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + // retrieves number of processors + ncpus = psutil_get_num_cpus(1); + if (ncpus == 0) + goto error; + + // allocates an array of _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION + // structures, one per processor + sppi = (_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *) \ + malloc(ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION)); + if (sppi == NULL) { + PyErr_NoMemory(); + goto error; + } + + // gets cpu time informations + status = psutil_NtQuerySystemInformation( + SystemProcessorPerformanceInformation, + sppi, + ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION), + NULL); + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, + "NtQuerySystemInformation(SystemProcessorPerformanceInformation)" + ); + goto error; + } + + // computes system global times summing each + // processor value + idle = user = kernel = interrupt = dpc = 0; + for (i = 0; i < ncpus; i++) { + py_tuple = NULL; + user = (double)((HI_T * sppi[i].UserTime.HighPart) + + (LO_T * sppi[i].UserTime.LowPart)); + idle = (double)((HI_T * sppi[i].IdleTime.HighPart) + + (LO_T * sppi[i].IdleTime.LowPart)); + kernel = (double)((HI_T * sppi[i].KernelTime.HighPart) + + (LO_T * sppi[i].KernelTime.LowPart)); + interrupt = (double)((HI_T * sppi[i].InterruptTime.HighPart) + + (LO_T * sppi[i].InterruptTime.LowPart)); + dpc = (double)((HI_T * sppi[i].DpcTime.HighPart) + + (LO_T * sppi[i].DpcTime.LowPart)); + + // kernel time includes idle time on windows + // we return only busy kernel time subtracting + // idle time from kernel time + systemt = kernel - idle; + py_tuple = Py_BuildValue( + "(ddddd)", + user, + systemt, + idle, + interrupt, + dpc + ); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + } + + free(sppi); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (sppi) + free(sppi); + return NULL; +} + + +/* + * Return process current working directory as a Python string. + */ +static PyObject * +psutil_proc_cwd(PyObject *self, PyObject *args) { + long pid; + int pid_return; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + pid_return = psutil_pid_is_running(pid); + if (pid_return == 0) + return NoSuchProcess(""); + if (pid_return == -1) + return NULL; + + return psutil_get_cwd(pid); +} + + +/* + * Resume or suspends a process + */ +static PyObject * +psutil_proc_suspend_or_resume(PyObject *self, PyObject *args) { + long pid; + NTSTATUS status; + HANDLE hProcess; + PyObject* suspend; + + if (! PyArg_ParseTuple(args, "lO", &pid, &suspend)) + return NULL; + + hProcess = psutil_handle_from_pid(pid, PROCESS_SUSPEND_RESUME); + if (hProcess == NULL) + return NULL; + + if (PyObject_IsTrue(suspend)) + status = psutil_NtSuspendProcess(hProcess); + else + status = psutil_NtResumeProcess(hProcess); + + if (! NT_SUCCESS(status)) { + CloseHandle(hProcess); + return psutil_SetFromNTStatusErr(status, "NtSuspend|ResumeProcess"); + } + + CloseHandle(hProcess); + Py_RETURN_NONE; +} + + +static PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + HANDLE hThread; + THREADENTRY32 te32 = {0}; + long pid; + int pid_return; + int rc; + FILETIME ftDummy, ftKernel, ftUser; + HANDLE hThreadSnap = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + if (pid == 0) { + // raise AD instead of returning 0 as procexp is able to + // retrieve useful information somehow + AccessDenied(""); + goto error; + } + + pid_return = psutil_pid_is_running(pid); + if (pid_return == 0) { + NoSuchProcess(""); + goto error; + } + if (pid_return == -1) + goto error; + + hThreadSnap = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (hThreadSnap == INVALID_HANDLE_VALUE) { + PyErr_SetFromOSErrnoWithSyscall("CreateToolhelp32Snapshot"); + goto error; + } + + // Fill in the size of the structure before using it + te32.dwSize = sizeof(THREADENTRY32); + + if (! Thread32First(hThreadSnap, &te32)) { + PyErr_SetFromOSErrnoWithSyscall("Thread32First"); + goto error; + } + + // Walk the thread snapshot to find all threads of the process. + // If the thread belongs to the process, increase the counter. + do { + if (te32.th32OwnerProcessID == pid) { + py_tuple = NULL; + hThread = NULL; + hThread = OpenThread(THREAD_QUERY_INFORMATION, + FALSE, te32.th32ThreadID); + if (hThread == NULL) { + // thread has disappeared on us + continue; + } + + rc = GetThreadTimes(hThread, &ftDummy, &ftDummy, &ftKernel, + &ftUser); + if (rc == 0) { + PyErr_SetFromOSErrnoWithSyscall("GetThreadTimes"); + goto error; + } + + /* + * User and kernel times are represented as a FILETIME structure + * which contains a 64-bit value representing the number of + * 100-nanosecond intervals since January 1, 1601 (UTC): + * http://msdn.microsoft.com/en-us/library/ms724284(VS.85).aspx + * To convert it into a float representing the seconds that the + * process has executed in user/kernel mode I borrowed the code + * below from Python's Modules/posixmodule.c + */ + py_tuple = Py_BuildValue( + "kdd", + te32.th32ThreadID, + (double)(ftUser.dwHighDateTime * 429.4967296 + \ + ftUser.dwLowDateTime * 1e-7), + (double)(ftKernel.dwHighDateTime * 429.4967296 + \ + ftKernel.dwLowDateTime * 1e-7)); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + + CloseHandle(hThread); + } + } while (Thread32Next(hThreadSnap, &te32)); + + CloseHandle(hThreadSnap); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (hThread != NULL) + CloseHandle(hThread); + if (hThreadSnap != NULL) + CloseHandle(hThreadSnap); + return NULL; +} + + +static PyObject * +psutil_proc_open_files(PyObject *self, PyObject *args) { + long pid; + HANDLE processHandle; + DWORD access = PROCESS_DUP_HANDLE | PROCESS_QUERY_INFORMATION; + PyObject *py_retlist; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + processHandle = psutil_handle_from_pid(pid, access); + if (processHandle == NULL) + return NULL; + + py_retlist = psutil_get_open_files(pid, processHandle); + if (py_retlist == NULL) { + PyErr_SetFromWindowsErr(0); + CloseHandle(processHandle); + return NULL; + } + + CloseHandle(processHandle); + return py_retlist; +} + + +/* + Accept a filename's drive in native format like "\Device\HarddiskVolume1\" + and return the corresponding drive letter (e.g. "C:\\"). + If no match is found return an empty string. +*/ +static PyObject * +psutil_win32_QueryDosDevice(PyObject *self, PyObject *args) { + LPCTSTR lpDevicePath; + TCHAR d = TEXT('A'); + TCHAR szBuff[5]; + + if (!PyArg_ParseTuple(args, "s", &lpDevicePath)) + return NULL; + + while (d <= TEXT('Z')) { + TCHAR szDeviceName[3] = {d, TEXT(':'), TEXT('\0')}; + TCHAR szTarget[512] = {0}; + if (QueryDosDevice(szDeviceName, szTarget, 511) != 0) { + if (_tcscmp(lpDevicePath, szTarget) == 0) { + _stprintf_s(szBuff, _countof(szBuff), TEXT("%c:"), d); + return Py_BuildValue("s", szBuff); + } + } + d++; + } + return Py_BuildValue("s", ""); +} + + +/* + * Return process username as a "DOMAIN//USERNAME" string. + */ +static PyObject * +psutil_proc_username(PyObject *self, PyObject *args) { + long pid; + HANDLE processHandle = NULL; + HANDLE tokenHandle = NULL; + PTOKEN_USER user = NULL; + ULONG bufferSize; + WCHAR *name = NULL; + WCHAR *domainName = NULL; + ULONG nameSize; + ULONG domainNameSize; + SID_NAME_USE nameUse; + PyObject *py_username = NULL; + PyObject *py_domain = NULL; + PyObject *py_tuple = NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + processHandle = psutil_handle_from_pid( + pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (processHandle == NULL) + return NULL; + + if (!OpenProcessToken(processHandle, TOKEN_QUERY, &tokenHandle)) { + PyErr_SetFromOSErrnoWithSyscall("OpenProcessToken"); + goto error; + } + + CloseHandle(processHandle); + processHandle = NULL; + + // Get the user SID. + bufferSize = 0x100; + while (1) { + user = malloc(bufferSize); + if (user == NULL) { + PyErr_NoMemory(); + goto error; + } + if (!GetTokenInformation(tokenHandle, TokenUser, user, bufferSize, + &bufferSize)) + { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(user); + continue; + } + else { + PyErr_SetFromOSErrnoWithSyscall("GetTokenInformation"); + goto error; + } + } + break; + } + + CloseHandle(tokenHandle); + tokenHandle = NULL; + + // resolve the SID to a name + nameSize = 0x100; + domainNameSize = 0x100; + while (1) { + name = malloc(nameSize * sizeof(WCHAR)); + if (name == NULL) { + PyErr_NoMemory(); + goto error; + } + domainName = malloc(domainNameSize * sizeof(WCHAR)); + if (domainName == NULL) { + PyErr_NoMemory(); + goto error; + } + if (!LookupAccountSidW(NULL, user->User.Sid, name, &nameSize, + domainName, &domainNameSize, &nameUse)) + { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(name); + free(domainName); + continue; + } + else { + PyErr_SetFromOSErrnoWithSyscall("LookupAccountSidW"); + goto error; + } + } + break; + } + + py_domain = PyUnicode_FromWideChar(domainName, wcslen(domainName)); + if (! py_domain) + goto error; + py_username = PyUnicode_FromWideChar(name, wcslen(name)); + if (! py_username) + goto error; + py_tuple = Py_BuildValue("OO", py_domain, py_username); + if (! py_tuple) + goto error; + Py_DECREF(py_domain); + Py_DECREF(py_username); + + free(name); + free(domainName); + free(user); + + return py_tuple; + +error: + if (processHandle != NULL) + CloseHandle(processHandle); + if (tokenHandle != NULL) + CloseHandle(tokenHandle); + if (name != NULL) + free(name); + if (domainName != NULL) + free(domainName); + if (user != NULL) + free(user); + Py_XDECREF(py_domain); + Py_XDECREF(py_username); + Py_XDECREF(py_tuple); + return NULL; +} + + +// https://msdn.microsoft.com/library/aa365928.aspx +// TODO properly handle return code +static DWORD __GetExtendedTcpTable(_GetExtendedTcpTable call, + ULONG address_family, + PVOID * data, DWORD * size) +{ + // Due to other processes being active on the machine, it's possible + // that the size of the table increases between the moment where we + // query the size and the moment where we query the data. Therefore, it's + // important to call this in a loop to retry if that happens. + // See https://github.com/giampaolo/psutil/pull/1335 concerning 0xC0000001 error + // and https://github.com/giampaolo/psutil/issues/1294 + DWORD error = ERROR_INSUFFICIENT_BUFFER; + *size = 0; + *data = NULL; + error = call(NULL, size, FALSE, address_family, + TCP_TABLE_OWNER_PID_ALL, 0); + while (error == ERROR_INSUFFICIENT_BUFFER || error == 0xC0000001) + { + *data = malloc(*size); + if (*data == NULL) { + error = ERROR_NOT_ENOUGH_MEMORY; + continue; + } + error = call(*data, size, FALSE, address_family, + TCP_TABLE_OWNER_PID_ALL, 0); + if (error != NO_ERROR) { + free(*data); + *data = NULL; + } + } + return error; +} + + +// https://msdn.microsoft.com/library/aa365930.aspx +// TODO properly check return value +static DWORD __GetExtendedUdpTable(_GetExtendedUdpTable call, + ULONG address_family, + PVOID * data, DWORD * size) +{ + // Due to other processes being active on the machine, it's possible + // that the size of the table increases between the moment where we + // query the size and the moment where we query the data. Therefore, it's + // important to call this in a loop to retry if that happens. + // See https://github.com/giampaolo/psutil/pull/1335 concerning 0xC0000001 error + // and https://github.com/giampaolo/psutil/issues/1294 + DWORD error = ERROR_INSUFFICIENT_BUFFER; + *size = 0; + *data = NULL; + error = call(NULL, size, FALSE, address_family, + UDP_TABLE_OWNER_PID, 0); + while (error == ERROR_INSUFFICIENT_BUFFER || error == 0xC0000001) + { + *data = malloc(*size); + if (*data == NULL) { + error = ERROR_NOT_ENOUGH_MEMORY; + continue; + } + error = call(*data, size, FALSE, address_family, + UDP_TABLE_OWNER_PID, 0); + if (error != NO_ERROR) { + free(*data); + *data = NULL; + } + } + + if (error == ERROR_NOT_ENOUGH_MEMORY) { + PyErr_NoMemory(); + return 1; + } + if (error != NO_ERROR) { + PyErr_SetFromWindowsErr(error); + return 1; + } + return 0; +} + + +#define psutil_conn_decref_objs() \ + Py_DECREF(_AF_INET); \ + Py_DECREF(_AF_INET6);\ + Py_DECREF(_SOCK_STREAM);\ + Py_DECREF(_SOCK_DGRAM); + + +/* + * Return a list of network connections opened by a process + */ +static PyObject * +psutil_net_connections(PyObject *self, PyObject *args) { + static long null_address[4] = { 0, 0, 0, 0 }; + unsigned long pid; + int pid_return; + PVOID table = NULL; + DWORD tableSize; + DWORD error; + PMIB_TCPTABLE_OWNER_PID tcp4Table; + PMIB_UDPTABLE_OWNER_PID udp4Table; + PMIB_TCP6TABLE_OWNER_PID tcp6Table; + PMIB_UDP6TABLE_OWNER_PID udp6Table; + ULONG i; + CHAR addressBufferLocal[65]; + CHAR addressBufferRemote[65]; + + PyObject *py_retlist; + PyObject *py_conn_tuple = NULL; + PyObject *py_af_filter = NULL; + PyObject *py_type_filter = NULL; + PyObject *py_addr_tuple_local = NULL; + PyObject *py_addr_tuple_remote = NULL; + PyObject *_AF_INET = PyLong_FromLong((long)AF_INET); + PyObject *_AF_INET6 = PyLong_FromLong((long)AF_INET6); + PyObject *_SOCK_STREAM = PyLong_FromLong((long)SOCK_STREAM); + PyObject *_SOCK_DGRAM = PyLong_FromLong((long)SOCK_DGRAM); + + // Import some functions. + if (! PyArg_ParseTuple(args, "lOO", &pid, &py_af_filter, &py_type_filter)) + goto error; + + if (!PySequence_Check(py_af_filter) || !PySequence_Check(py_type_filter)) { + psutil_conn_decref_objs(); + PyErr_SetString(PyExc_TypeError, "arg 2 or 3 is not a sequence"); + return NULL; + } + + if (pid != -1) { + pid_return = psutil_pid_is_running(pid); + if (pid_return == 0) { + psutil_conn_decref_objs(); + return NoSuchProcess(""); + } + else if (pid_return == -1) { + psutil_conn_decref_objs(); + return NULL; + } + } + + py_retlist = PyList_New(0); + if (py_retlist == NULL) { + psutil_conn_decref_objs(); + return NULL; + } + + // TCP IPv4 + + if ((PySequence_Contains(py_af_filter, _AF_INET) == 1) && + (PySequence_Contains(py_type_filter, _SOCK_STREAM) == 1)) + { + table = NULL; + py_conn_tuple = NULL; + py_addr_tuple_local = NULL; + py_addr_tuple_remote = NULL; + tableSize = 0; + + error = __GetExtendedTcpTable(psutil_GetExtendedTcpTable, + AF_INET, &table, &tableSize); + if (error != 0) + goto error; + tcp4Table = table; + for (i = 0; i < tcp4Table->dwNumEntries; i++) { + if (pid != -1) { + if (tcp4Table->table[i].dwOwningPid != pid) { + continue; + } + } + + if (tcp4Table->table[i].dwLocalAddr != 0 || + tcp4Table->table[i].dwLocalPort != 0) + { + struct in_addr addr; + + addr.S_un.S_addr = tcp4Table->table[i].dwLocalAddr; + psutil_rtlIpv4AddressToStringA(&addr, addressBufferLocal); + py_addr_tuple_local = Py_BuildValue( + "(si)", + addressBufferLocal, + BYTESWAP_USHORT(tcp4Table->table[i].dwLocalPort)); + } + else { + py_addr_tuple_local = PyTuple_New(0); + } + + if (py_addr_tuple_local == NULL) + goto error; + + // On Windows <= XP, remote addr is filled even if socket + // is in LISTEN mode in which case we just ignore it. + if ((tcp4Table->table[i].dwRemoteAddr != 0 || + tcp4Table->table[i].dwRemotePort != 0) && + (tcp4Table->table[i].dwState != MIB_TCP_STATE_LISTEN)) + { + struct in_addr addr; + + addr.S_un.S_addr = tcp4Table->table[i].dwRemoteAddr; + psutil_rtlIpv4AddressToStringA(&addr, addressBufferRemote); + py_addr_tuple_remote = Py_BuildValue( + "(si)", + addressBufferRemote, + BYTESWAP_USHORT(tcp4Table->table[i].dwRemotePort)); + } + else + { + py_addr_tuple_remote = PyTuple_New(0); + } + + if (py_addr_tuple_remote == NULL) + goto error; + + py_conn_tuple = Py_BuildValue( + "(iiiNNiI)", + -1, + AF_INET, + SOCK_STREAM, + py_addr_tuple_local, + py_addr_tuple_remote, + tcp4Table->table[i].dwState, + tcp4Table->table[i].dwOwningPid); + if (!py_conn_tuple) + goto error; + if (PyList_Append(py_retlist, py_conn_tuple)) + goto error; + Py_CLEAR(py_conn_tuple); + } + + free(table); + table = NULL; + tableSize = 0; + } + + // TCP IPv6 + if ((PySequence_Contains(py_af_filter, _AF_INET6) == 1) && + (PySequence_Contains(py_type_filter, _SOCK_STREAM) == 1) && + (psutil_rtlIpv6AddressToStringA != NULL)) + { + table = NULL; + py_conn_tuple = NULL; + py_addr_tuple_local = NULL; + py_addr_tuple_remote = NULL; + tableSize = 0; + + error = __GetExtendedTcpTable(psutil_GetExtendedTcpTable, + AF_INET6, &table, &tableSize); + if (error != 0) + goto error; + tcp6Table = table; + for (i = 0; i < tcp6Table->dwNumEntries; i++) + { + if (pid != -1) { + if (tcp6Table->table[i].dwOwningPid != pid) { + continue; + } + } + + if (memcmp(tcp6Table->table[i].ucLocalAddr, null_address, 16) + != 0 || tcp6Table->table[i].dwLocalPort != 0) + { + struct in6_addr addr; + + memcpy(&addr, tcp6Table->table[i].ucLocalAddr, 16); + psutil_rtlIpv6AddressToStringA(&addr, addressBufferLocal); + py_addr_tuple_local = Py_BuildValue( + "(si)", + addressBufferLocal, + BYTESWAP_USHORT(tcp6Table->table[i].dwLocalPort)); + } + else { + py_addr_tuple_local = PyTuple_New(0); + } + + if (py_addr_tuple_local == NULL) + goto error; + + // On Windows <= XP, remote addr is filled even if socket + // is in LISTEN mode in which case we just ignore it. + if ((memcmp(tcp6Table->table[i].ucRemoteAddr, null_address, 16) + != 0 || + tcp6Table->table[i].dwRemotePort != 0) && + (tcp6Table->table[i].dwState != MIB_TCP_STATE_LISTEN)) + { + struct in6_addr addr; + + memcpy(&addr, tcp6Table->table[i].ucRemoteAddr, 16); + psutil_rtlIpv6AddressToStringA(&addr, addressBufferRemote); + py_addr_tuple_remote = Py_BuildValue( + "(si)", + addressBufferRemote, + BYTESWAP_USHORT(tcp6Table->table[i].dwRemotePort)); + } + else { + py_addr_tuple_remote = PyTuple_New(0); + } + + if (py_addr_tuple_remote == NULL) + goto error; + + py_conn_tuple = Py_BuildValue( + "(iiiNNiI)", + -1, + AF_INET6, + SOCK_STREAM, + py_addr_tuple_local, + py_addr_tuple_remote, + tcp6Table->table[i].dwState, + tcp6Table->table[i].dwOwningPid); + if (!py_conn_tuple) + goto error; + if (PyList_Append(py_retlist, py_conn_tuple)) + goto error; + Py_CLEAR(py_conn_tuple); + } + + free(table); + table = NULL; + tableSize = 0; + } + + // UDP IPv4 + + if ((PySequence_Contains(py_af_filter, _AF_INET) == 1) && + (PySequence_Contains(py_type_filter, _SOCK_DGRAM) == 1)) + { + table = NULL; + py_conn_tuple = NULL; + py_addr_tuple_local = NULL; + py_addr_tuple_remote = NULL; + tableSize = 0; + error = __GetExtendedUdpTable(psutil_GetExtendedUdpTable, + AF_INET, &table, &tableSize); + if (error != 0) + goto error; + udp4Table = table; + for (i = 0; i < udp4Table->dwNumEntries; i++) + { + if (pid != -1) { + if (udp4Table->table[i].dwOwningPid != pid) { + continue; + } + } + + if (udp4Table->table[i].dwLocalAddr != 0 || + udp4Table->table[i].dwLocalPort != 0) + { + struct in_addr addr; + + addr.S_un.S_addr = udp4Table->table[i].dwLocalAddr; + psutil_rtlIpv4AddressToStringA(&addr, addressBufferLocal); + py_addr_tuple_local = Py_BuildValue( + "(si)", + addressBufferLocal, + BYTESWAP_USHORT(udp4Table->table[i].dwLocalPort)); + } + else { + py_addr_tuple_local = PyTuple_New(0); + } + + if (py_addr_tuple_local == NULL) + goto error; + + py_conn_tuple = Py_BuildValue( + "(iiiNNiI)", + -1, + AF_INET, + SOCK_DGRAM, + py_addr_tuple_local, + PyTuple_New(0), + PSUTIL_CONN_NONE, + udp4Table->table[i].dwOwningPid); + if (!py_conn_tuple) + goto error; + if (PyList_Append(py_retlist, py_conn_tuple)) + goto error; + Py_CLEAR(py_conn_tuple); + } + + free(table); + table = NULL; + tableSize = 0; + } + + // UDP IPv6 + + if ((PySequence_Contains(py_af_filter, _AF_INET6) == 1) && + (PySequence_Contains(py_type_filter, _SOCK_DGRAM) == 1) && + (psutil_rtlIpv6AddressToStringA != NULL)) + { + table = NULL; + py_conn_tuple = NULL; + py_addr_tuple_local = NULL; + py_addr_tuple_remote = NULL; + tableSize = 0; + error = __GetExtendedUdpTable(psutil_GetExtendedUdpTable, + AF_INET6, &table, &tableSize); + if (error != 0) + goto error; + udp6Table = table; + for (i = 0; i < udp6Table->dwNumEntries; i++) { + if (pid != -1) { + if (udp6Table->table[i].dwOwningPid != pid) { + continue; + } + } + + if (memcmp(udp6Table->table[i].ucLocalAddr, null_address, 16) + != 0 || udp6Table->table[i].dwLocalPort != 0) + { + struct in6_addr addr; + + memcpy(&addr, udp6Table->table[i].ucLocalAddr, 16); + psutil_rtlIpv6AddressToStringA(&addr, addressBufferLocal); + py_addr_tuple_local = Py_BuildValue( + "(si)", + addressBufferLocal, + BYTESWAP_USHORT(udp6Table->table[i].dwLocalPort)); + } + else { + py_addr_tuple_local = PyTuple_New(0); + } + + if (py_addr_tuple_local == NULL) + goto error; + + py_conn_tuple = Py_BuildValue( + "(iiiNNiI)", + -1, + AF_INET6, + SOCK_DGRAM, + py_addr_tuple_local, + PyTuple_New(0), + PSUTIL_CONN_NONE, + udp6Table->table[i].dwOwningPid); + if (!py_conn_tuple) + goto error; + if (PyList_Append(py_retlist, py_conn_tuple)) + goto error; + Py_CLEAR(py_conn_tuple); + } + + free(table); + table = NULL; + tableSize = 0; + } + + psutil_conn_decref_objs(); + return py_retlist; + +error: + psutil_conn_decref_objs(); + Py_XDECREF(py_conn_tuple); + Py_XDECREF(py_addr_tuple_local); + Py_XDECREF(py_addr_tuple_remote); + Py_DECREF(py_retlist); + if (table != NULL) + free(table); + return NULL; +} + + +/* + * Get process priority as a Python integer. + */ +static PyObject * +psutil_proc_priority_get(PyObject *self, PyObject *args) { + long pid; + DWORD priority; + HANDLE hProcess; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) + return NULL; + + priority = GetPriorityClass(hProcess); + if (priority == 0) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + CloseHandle(hProcess); + return Py_BuildValue("i", priority); +} + + +/* + * Set process priority. + */ +static PyObject * +psutil_proc_priority_set(PyObject *self, PyObject *args) { + long pid; + int priority; + int retval; + HANDLE hProcess; + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; + + if (! PyArg_ParseTuple(args, "li", &pid, &priority)) + return NULL; + hProcess = psutil_handle_from_pid(pid, access); + if (hProcess == NULL) + return NULL; + + retval = SetPriorityClass(hProcess, priority); + if (retval == 0) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); + Py_RETURN_NONE; +} + + +#if (_WIN32_WINNT >= 0x0600) // Windows Vista +/* + * Get process IO priority as a Python integer. + */ +static PyObject * +psutil_proc_io_priority_get(PyObject *self, PyObject *args) { + long pid; + HANDLE hProcess; + DWORD IoPriority; + NTSTATUS status; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) + return NULL; + + status = psutil_NtQueryInformationProcess( + hProcess, + ProcessIoPriority, + &IoPriority, + sizeof(DWORD), + NULL + ); + + CloseHandle(hProcess); + if (! NT_SUCCESS(status)) + return psutil_SetFromNTStatusErr(status, "NtQueryInformationProcess"); + return Py_BuildValue("i", IoPriority); +} + + +/* + * Set process IO priority. + */ +static PyObject * +psutil_proc_io_priority_set(PyObject *self, PyObject *args) { + long pid; + DWORD prio; + HANDLE hProcess; + NTSTATUS status; + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; + + if (! PyArg_ParseTuple(args, "li", &pid, &prio)) + return NULL; + + hProcess = psutil_handle_from_pid(pid, access); + if (hProcess == NULL) + return NULL; + + status = psutil_NtSetInformationProcess( + hProcess, + ProcessIoPriority, + (PVOID)&prio, + sizeof(DWORD) + ); + + CloseHandle(hProcess); + if (! NT_SUCCESS(status)) + return psutil_SetFromNTStatusErr(status, "NtSetInformationProcess"); + Py_RETURN_NONE; +} +#endif + + +/* + * Return a Python tuple referencing process I/O counters. + */ +static PyObject * +psutil_proc_io_counters(PyObject *self, PyObject *args) { + DWORD pid; + HANDLE hProcess; + IO_COUNTERS IoCounters; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (NULL == hProcess) + return NULL; + + if (! GetProcessIoCounters(hProcess, &IoCounters)) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); + return Py_BuildValue("(KKKKKK)", + IoCounters.ReadOperationCount, + IoCounters.WriteOperationCount, + IoCounters.ReadTransferCount, + IoCounters.WriteTransferCount, + IoCounters.OtherOperationCount, + IoCounters.OtherTransferCount); +} + + +/* + * Return process CPU affinity as a bitmask + */ +static PyObject * +psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { + DWORD pid; + HANDLE hProcess; + DWORD_PTR proc_mask; + DWORD_PTR system_mask; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) { + return NULL; + } + if (GetProcessAffinityMask(hProcess, &proc_mask, &system_mask) == 0) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); +#ifdef _WIN64 + return Py_BuildValue("K", (unsigned long long)proc_mask); +#else + return Py_BuildValue("k", (unsigned long)proc_mask); +#endif +} + + +/* + * Set process CPU affinity + */ +static PyObject * +psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { + DWORD pid; + HANDLE hProcess; + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; + DWORD_PTR mask; + +#ifdef _WIN64 + if (! PyArg_ParseTuple(args, "lK", &pid, &mask)) +#else + if (! PyArg_ParseTuple(args, "lk", &pid, &mask)) +#endif + { + return NULL; + } + hProcess = psutil_handle_from_pid(pid, access); + if (hProcess == NULL) + return NULL; + + if (SetProcessAffinityMask(hProcess, mask) == 0) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + + CloseHandle(hProcess); + Py_RETURN_NONE; +} + + +/* + * Return True if all process threads are in waiting/suspended state. + */ +static PyObject * +psutil_proc_is_suspended(PyObject *self, PyObject *args) { + DWORD pid; + ULONG i; + PSYSTEM_PROCESS_INFORMATION process; + PVOID buffer; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (! psutil_get_proc_info(pid, &process, &buffer)) + return NULL; + for (i = 0; i < process->NumberOfThreads; i++) { + if (process->Threads[i].ThreadState != Waiting || + process->Threads[i].WaitReason != Suspended) + { + free(buffer); + Py_RETURN_FALSE; + } + } + free(buffer); + Py_RETURN_TRUE; +} + + +/* + * Return path's disk total and free as a Python tuple. + */ +static PyObject * +psutil_disk_usage(PyObject *self, PyObject *args) { + BOOL retval; + ULARGE_INTEGER _, total, free; + char *path; + + if (PyArg_ParseTuple(args, "u", &path)) { + Py_BEGIN_ALLOW_THREADS + retval = GetDiskFreeSpaceExW((LPCWSTR)path, &_, &total, &free); + Py_END_ALLOW_THREADS + goto return_; + } + + // on Python 2 we also want to accept plain strings other + // than Unicode +#if PY_MAJOR_VERSION <= 2 + PyErr_Clear(); // drop the argument parsing error + if (PyArg_ParseTuple(args, "s", &path)) { + Py_BEGIN_ALLOW_THREADS + retval = GetDiskFreeSpaceEx(path, &_, &total, &free); + Py_END_ALLOW_THREADS + goto return_; + } +#endif + + return NULL; + +return_: + if (retval == 0) + return PyErr_SetFromWindowsErrWithFilename(0, path); + else + return Py_BuildValue("(LL)", total.QuadPart, free.QuadPart); +} + + +/* + * Return a Python list of named tuples with overall network I/O information + */ +static PyObject * +psutil_net_io_counters(PyObject *self, PyObject *args) { + DWORD dwRetVal = 0; + +#if (_WIN32_WINNT >= 0x0600) // Windows Vista and above + MIB_IF_ROW2 *pIfRow = NULL; +#else // Windows XP + MIB_IFROW *pIfRow = NULL; +#endif + + PIP_ADAPTER_ADDRESSES pAddresses = NULL; + PIP_ADAPTER_ADDRESSES pCurrAddresses = NULL; + PyObject *py_retdict = PyDict_New(); + PyObject *py_nic_info = NULL; + PyObject *py_nic_name = NULL; + + if (py_retdict == NULL) + return NULL; + pAddresses = psutil_get_nic_addresses(); + if (pAddresses == NULL) + goto error; + pCurrAddresses = pAddresses; + + while (pCurrAddresses) { + py_nic_name = NULL; + py_nic_info = NULL; + +#if (_WIN32_WINNT >= 0x0600) // Windows Vista and above + pIfRow = (MIB_IF_ROW2 *) malloc(sizeof(MIB_IF_ROW2)); +#else // Windows XP + pIfRow = (MIB_IFROW *) malloc(sizeof(MIB_IFROW)); +#endif + + if (pIfRow == NULL) { + PyErr_NoMemory(); + goto error; + } + +#if (_WIN32_WINNT >= 0x0600) // Windows Vista and above + SecureZeroMemory((PVOID)pIfRow, sizeof(MIB_IF_ROW2)); + pIfRow->InterfaceIndex = pCurrAddresses->IfIndex; + dwRetVal = GetIfEntry2(pIfRow); +#else // Windows XP + pIfRow->dwIndex = pCurrAddresses->IfIndex; + dwRetVal = GetIfEntry(pIfRow); +#endif + + if (dwRetVal != NO_ERROR) { + PyErr_SetString(PyExc_RuntimeError, + "GetIfEntry() or GetIfEntry2() syscalls failed."); + goto error; + } + +#if (_WIN32_WINNT >= 0x0600) // Windows Vista and above + py_nic_info = Py_BuildValue("(KKKKKKKK)", + pIfRow->OutOctets, + pIfRow->InOctets, + (pIfRow->OutUcastPkts + pIfRow->OutNUcastPkts), + (pIfRow->InUcastPkts + pIfRow->InNUcastPkts), + pIfRow->InErrors, + pIfRow->OutErrors, + pIfRow->InDiscards, + pIfRow->OutDiscards); +#else // Windows XP + py_nic_info = Py_BuildValue("(kkkkkkkk)", + pIfRow->dwOutOctets, + pIfRow->dwInOctets, + (pIfRow->dwOutUcastPkts + pIfRow->dwOutNUcastPkts), + (pIfRow->dwInUcastPkts + pIfRow->dwInNUcastPkts), + pIfRow->dwInErrors, + pIfRow->dwOutErrors, + pIfRow->dwInDiscards, + pIfRow->dwOutDiscards); +#endif + + if (!py_nic_info) + goto error; + + py_nic_name = PyUnicode_FromWideChar( + pCurrAddresses->FriendlyName, + wcslen(pCurrAddresses->FriendlyName)); + + if (py_nic_name == NULL) + goto error; + if (PyDict_SetItem(py_retdict, py_nic_name, py_nic_info)) + goto error; + Py_CLEAR(py_nic_name); + Py_CLEAR(py_nic_info); + + free(pIfRow); + pCurrAddresses = pCurrAddresses->Next; + } + + free(pAddresses); + return py_retdict; + +error: + Py_XDECREF(py_nic_name); + Py_XDECREF(py_nic_info); + Py_DECREF(py_retdict); + if (pAddresses != NULL) + free(pAddresses); + if (pIfRow != NULL) + free(pIfRow); + return NULL; +} + + +/* + * Return a Python dict of tuples for disk I/O information. This may + * require running "diskperf -y" command first. + */ +static PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + DISK_PERFORMANCE diskPerformance; + DWORD dwSize; + HANDLE hDevice = NULL; + char szDevice[MAX_PATH]; + char szDeviceDisplay[MAX_PATH]; + int devNum; + int i; + DWORD ioctrlSize; + BOOL ret; + PyObject *py_retdict = PyDict_New(); + PyObject *py_tuple = NULL; + + if (py_retdict == NULL) + return NULL; + // Apparently there's no way to figure out how many times we have + // to iterate in order to find valid drives. + // Let's assume 32, which is higher than 26, the number of letters + // in the alphabet (from A:\ to Z:\). + for (devNum = 0; devNum <= 32; ++devNum) { + py_tuple = NULL; + sprintf_s(szDevice, MAX_PATH, "\\\\.\\PhysicalDrive%d", devNum); + hDevice = CreateFile(szDevice, 0, FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, OPEN_EXISTING, 0, NULL); + if (hDevice == INVALID_HANDLE_VALUE) + continue; + + // DeviceIoControl() sucks! + i = 0; + ioctrlSize = sizeof(diskPerformance); + while (1) { + i += 1; + ret = DeviceIoControl( + hDevice, IOCTL_DISK_PERFORMANCE, NULL, 0, &diskPerformance, + ioctrlSize, &dwSize, NULL); + if (ret != 0) + break; // OK! + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + // Retry with a bigger buffer (+ limit for retries). + if (i <= 1024) { + ioctrlSize *= 2; + continue; + } + } + else if (GetLastError() == ERROR_INVALID_FUNCTION) { + // This happens on AppVeyor: + // https://ci.appveyor.com/project/giampaolo/psutil/build/ + // 1364/job/ascpdi271b06jle3 + // Assume it means we're dealing with some exotic disk + // and go on. + psutil_debug("DeviceIoControl -> ERROR_INVALID_FUNCTION; " + "ignore PhysicalDrive%i", devNum); + goto next; + } + else if (GetLastError() == ERROR_NOT_SUPPORTED) { + // Again, let's assume we're dealing with some exotic disk. + psutil_debug("DeviceIoControl -> ERROR_NOT_SUPPORTED; " + "ignore PhysicalDrive%i", devNum); + goto next; + } + // XXX: it seems we should also catch ERROR_INVALID_PARAMETER: + // https://sites.ualberta.ca/dept/aict/uts/software/openbsd/ + // ports/4.1/i386/openafs/w-openafs-1.4.14-transarc/ + // openafs-1.4.14/src/usd/usd_nt.c + + // XXX: we can also bump into ERROR_MORE_DATA in which case + // (quoting doc) we're supposed to retry with a bigger buffer + // and specify a new "starting point", whatever it means. + PyErr_SetFromWindowsErr(0); + goto error; + } + + sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%i", devNum); + py_tuple = Py_BuildValue( + "(IILLKK)", + diskPerformance.ReadCount, + diskPerformance.WriteCount, + diskPerformance.BytesRead, + diskPerformance.BytesWritten, + // convert to ms: + // https://github.com/giampaolo/psutil/issues/1012 + (unsigned long long) + (diskPerformance.ReadTime.QuadPart) / 10000000, + (unsigned long long) + (diskPerformance.WriteTime.QuadPart) / 10000000); + if (!py_tuple) + goto error; + if (PyDict_SetItemString(py_retdict, szDeviceDisplay, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + +next: + CloseHandle(hDevice); + } + + return py_retdict; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retdict); + if (hDevice != NULL) + CloseHandle(hDevice); + return NULL; +} + + +static char *psutil_get_drive_type(int type) { + switch (type) { + case DRIVE_FIXED: + return "fixed"; + case DRIVE_CDROM: + return "cdrom"; + case DRIVE_REMOVABLE: + return "removable"; + case DRIVE_UNKNOWN: + return "unknown"; + case DRIVE_NO_ROOT_DIR: + return "unmounted"; + case DRIVE_REMOTE: + return "remote"; + case DRIVE_RAMDISK: + return "ramdisk"; + default: + return "?"; + } +} + + +#ifndef _ARRAYSIZE +#define _ARRAYSIZE(a) (sizeof(a)/sizeof(a[0])) +#endif + + +/* + * Return disk partitions as a list of tuples such as + * (drive_letter, drive_letter, type, "") + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + DWORD num_bytes; + char drive_strings[255]; + char *drive_letter = drive_strings; + char mp_buf[MAX_PATH]; + char mp_path[MAX_PATH]; + int all; + int type; + int ret; + unsigned int old_mode = 0; + char opts[20]; + HANDLE mp_h; + BOOL mp_flag= TRUE; + LPTSTR fs_type[MAX_PATH + 1] = { 0 }; + DWORD pflags = 0; + PyObject *py_all; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + + if (py_retlist == NULL) { + return NULL; + } + + // avoid to visualize a message box in case something goes wrong + // see https://github.com/giampaolo/psutil/issues/264 + old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); + + if (! PyArg_ParseTuple(args, "O", &py_all)) + goto error; + all = PyObject_IsTrue(py_all); + + Py_BEGIN_ALLOW_THREADS + num_bytes = GetLogicalDriveStrings(254, drive_letter); + Py_END_ALLOW_THREADS + + if (num_bytes == 0) { + PyErr_SetFromWindowsErr(0); + goto error; + } + + while (*drive_letter != 0) { + py_tuple = NULL; + opts[0] = 0; + fs_type[0] = 0; + + Py_BEGIN_ALLOW_THREADS + type = GetDriveType(drive_letter); + Py_END_ALLOW_THREADS + + // by default we only show hard drives and cd-roms + if (all == 0) { + if ((type == DRIVE_UNKNOWN) || + (type == DRIVE_NO_ROOT_DIR) || + (type == DRIVE_REMOTE) || + (type == DRIVE_RAMDISK)) { + goto next; + } + // floppy disk: skip it by default as it introduces a + // considerable slowdown. + if ((type == DRIVE_REMOVABLE) && + (strcmp(drive_letter, "A:\\") == 0)) { + goto next; + } + } + + ret = GetVolumeInformation( + (LPCTSTR)drive_letter, NULL, _ARRAYSIZE(drive_letter), + NULL, NULL, &pflags, (LPTSTR)fs_type, _ARRAYSIZE(fs_type)); + if (ret == 0) { + // We might get here in case of a floppy hard drive, in + // which case the error is (21, "device not ready"). + // Let's pretend it didn't happen as we already have + // the drive name and type ('removable'). + strcat_s(opts, _countof(opts), ""); + SetLastError(0); + } + else { + if (pflags & FILE_READ_ONLY_VOLUME) + strcat_s(opts, _countof(opts), "ro"); + else + strcat_s(opts, _countof(opts), "rw"); + if (pflags & FILE_VOLUME_IS_COMPRESSED) + strcat_s(opts, _countof(opts), ",compressed"); + + // Check for mount points on this volume and add/get info + // (checks first to know if we can even have mount points) + if (pflags & FILE_SUPPORTS_REPARSE_POINTS) { + + mp_h = FindFirstVolumeMountPoint(drive_letter, mp_buf, MAX_PATH); + if (mp_h != INVALID_HANDLE_VALUE) { + while (mp_flag) { + + // Append full mount path with drive letter + strcpy_s(mp_path, _countof(mp_path), drive_letter); + strcat_s(mp_path, _countof(mp_path), mp_buf); + + py_tuple = Py_BuildValue( + "(ssss)", + drive_letter, + mp_path, + fs_type, // Typically NTFS + opts); + + if (!py_tuple || PyList_Append(py_retlist, py_tuple) == -1) { + FindVolumeMountPointClose(mp_h); + goto error; + } + + Py_CLEAR(py_tuple); + + // Continue looking for more mount points + mp_flag = FindNextVolumeMountPoint(mp_h, mp_buf, MAX_PATH); + } + FindVolumeMountPointClose(mp_h); + } + + } + } + + if (strlen(opts) > 0) + strcat_s(opts, _countof(opts), ","); + strcat_s(opts, _countof(opts), psutil_get_drive_type(type)); + + py_tuple = Py_BuildValue( + "(ssss)", + drive_letter, + drive_letter, + fs_type, // either FAT, FAT32, NTFS, HPFS, CDFS, UDF or NWFS + opts); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + goto next; + +next: + drive_letter = strchr(drive_letter, 0) + 1; + } + + SetErrorMode(old_mode); + return py_retlist; + +error: + SetErrorMode(old_mode); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + return NULL; +} + +/* + * Return a Python dict of tuples for disk I/O information + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + HANDLE hServer = WTS_CURRENT_SERVER_HANDLE; + WCHAR *buffer_user = NULL; + LPTSTR buffer_addr = NULL; + PWTS_SESSION_INFO sessions = NULL; + DWORD count; + DWORD i; + DWORD sessionId; + DWORD bytes; + PWTS_CLIENT_ADDRESS address; + char address_str[50]; + long long unix_time; + WINSTATION_INFO station_info; + ULONG returnLen; + PyObject *py_tuple = NULL; + PyObject *py_address = NULL; + PyObject *py_username = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + if (WTSEnumerateSessions(hServer, 0, 1, &sessions, &count) == 0) { + PyErr_SetFromOSErrnoWithSyscall("WTSEnumerateSessions"); + goto error; + } + + for (i = 0; i < count; i++) { + py_address = NULL; + py_tuple = NULL; + sessionId = sessions[i].SessionId; + if (buffer_user != NULL) + WTSFreeMemory(buffer_user); + if (buffer_addr != NULL) + WTSFreeMemory(buffer_addr); + + buffer_user = NULL; + buffer_addr = NULL; + + // username + bytes = 0; + if (WTSQuerySessionInformationW(hServer, sessionId, WTSUserName, + &buffer_user, &bytes) == 0) { + PyErr_SetFromOSErrnoWithSyscall("WTSQuerySessionInformationW"); + goto error; + } + if (bytes <= 2) + continue; + + // address + bytes = 0; + if (WTSQuerySessionInformation(hServer, sessionId, WTSClientAddress, + &buffer_addr, &bytes) == 0) { + PyErr_SetFromOSErrnoWithSyscall("WTSQuerySessionInformation"); + goto error; + } + + address = (PWTS_CLIENT_ADDRESS)buffer_addr; + if (address->AddressFamily == 0) { // AF_INET + sprintf_s(address_str, + _countof(address_str), + "%u.%u.%u.%u", + address->Address[0], + address->Address[1], + address->Address[2], + address->Address[3]); + py_address = Py_BuildValue("s", address_str); + if (!py_address) + goto error; + } + else { + py_address = Py_None; + } + + // login time + if (! psutil_WinStationQueryInformationW( + hServer, + sessionId, + WinStationInformation, + &station_info, + sizeof(station_info), + &returnLen)) + { + PyErr_SetFromOSErrnoWithSyscall("WinStationQueryInformationW"); + goto error; + } + + unix_time = ((LONGLONG)station_info.ConnectTime.dwHighDateTime) << 32; + unix_time += \ + station_info.ConnectTime.dwLowDateTime - 116444736000000000LL; + unix_time /= 10000000; + + py_username = PyUnicode_FromWideChar(buffer_user, wcslen(buffer_user)); + if (py_username == NULL) + goto error; + py_tuple = Py_BuildValue("OOd", + py_username, + py_address, + (double)unix_time); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_username); + Py_CLEAR(py_address); + Py_CLEAR(py_tuple); + } + + WTSFreeMemory(sessions); + WTSFreeMemory(buffer_user); + WTSFreeMemory(buffer_addr); + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tuple); + Py_XDECREF(py_address); + Py_DECREF(py_retlist); + + if (sessions != NULL) + WTSFreeMemory(sessions); + if (buffer_user != NULL) + WTSFreeMemory(buffer_user); + if (buffer_addr != NULL) + WTSFreeMemory(buffer_addr); + return NULL; +} + + +/* + * Return the number of handles opened by process. + */ +static PyObject * +psutil_proc_num_handles(PyObject *self, PyObject *args) { + DWORD pid; + HANDLE hProcess; + DWORD handleCount; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (NULL == hProcess) + return NULL; + if (! GetProcessHandleCount(hProcess, &handleCount)) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + CloseHandle(hProcess); + return Py_BuildValue("k", handleCount); +} + + +/* + * Get various process information by using NtQuerySystemInformation. + * We use this as a fallback when faster functions fail with access + * denied. This is slower because it iterates over all processes. + * Returned tuple includes the following process info: + * + * - num_threads() + * - ctx_switches() + * - num_handles() (fallback) + * - cpu_times() (fallback) + * - create_time() (fallback) + * - io_counters() (fallback) + * - memory_info() (fallback) + */ +static PyObject * +psutil_proc_info(PyObject *self, PyObject *args) { + DWORD pid; + PSYSTEM_PROCESS_INFORMATION process; + PVOID buffer; + ULONG i; + ULONG ctx_switches = 0; + double user_time; + double kernel_time; + long long create_time; + SIZE_T mem_private; + PyObject *py_retlist; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (! psutil_get_proc_info(pid, &process, &buffer)) + return NULL; + + for (i = 0; i < process->NumberOfThreads; i++) + ctx_switches += process->Threads[i].ContextSwitches; + user_time = (double)process->UserTime.HighPart * HI_T + \ + (double)process->UserTime.LowPart * LO_T; + kernel_time = (double)process->KernelTime.HighPart * HI_T + \ + (double)process->KernelTime.LowPart * LO_T; + + // Convert the LARGE_INTEGER union to a Unix time. + // It's the best I could find by googling and borrowing code here + // and there. The time returned has a precision of 1 second. + if (0 == pid || 4 == pid) { + // the python module will translate this into BOOT_TIME later + create_time = 0; + } + else { + create_time = ((LONGLONG)process->CreateTime.HighPart) << 32; + create_time += process->CreateTime.LowPart - 116444736000000000LL; + create_time /= 10000000; + } + +#if (_WIN32_WINNT >= 0x0501) // Windows XP with SP2 + mem_private = process->PrivatePageCount; +#else + mem_private = 0; +#endif + + py_retlist = Py_BuildValue( +#if defined(_WIN64) + "kkdddiKKKKKK" "kKKKKKKKKK", +#else + "kkdddiKKKKKK" "kIIIIIIIII", +#endif + process->HandleCount, // num handles + ctx_switches, // num ctx switches + user_time, // cpu user time + kernel_time, // cpu kernel time + (double)create_time, // create time + (int)process->NumberOfThreads, // num threads + // IO counters + process->ReadOperationCount.QuadPart, // io rcount + process->WriteOperationCount.QuadPart, // io wcount + process->ReadTransferCount.QuadPart, // io rbytes + process->WriteTransferCount.QuadPart, // io wbytes + process->OtherOperationCount.QuadPart, // io others count + process->OtherTransferCount.QuadPart, // io others bytes + // memory + process->PageFaultCount, // num page faults + process->PeakWorkingSetSize, // peak wset + process->WorkingSetSize, // wset + process->QuotaPeakPagedPoolUsage, // peak paged pool + process->QuotaPagedPoolUsage, // paged pool + process->QuotaPeakNonPagedPoolUsage, // peak non paged pool + process->QuotaNonPagedPoolUsage, // non paged pool + process->PagefileUsage, // pagefile + process->PeakPagefileUsage, // peak pagefile + mem_private // private + ); + + free(buffer); + return py_retlist; +} + + +static char *get_region_protection_string(ULONG protection) { + switch (protection & 0xff) { + case PAGE_NOACCESS: + return ""; + case PAGE_READONLY: + return "r"; + case PAGE_READWRITE: + return "rw"; + case PAGE_WRITECOPY: + return "wc"; + case PAGE_EXECUTE: + return "x"; + case PAGE_EXECUTE_READ: + return "xr"; + case PAGE_EXECUTE_READWRITE: + return "xrw"; + case PAGE_EXECUTE_WRITECOPY: + return "xwc"; + default: + return "?"; + } +} + + +/* + * Return a list of process's memory mappings. + */ +static PyObject * +psutil_proc_memory_maps(PyObject *self, PyObject *args) { + MEMORY_BASIC_INFORMATION basicInfo; + DWORD pid; + HANDLE hProcess = NULL; + PVOID baseAddress; + ULONGLONG previousAllocationBase; + WCHAR mappedFileName[MAX_PATH]; + LPVOID maxAddr; + // required by GetMappedFileNameW + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_VM_READ; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_str = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + hProcess = psutil_handle_from_pid(pid, access); + if (NULL == hProcess) + goto error; + + maxAddr = PSUTIL_SYSTEM_INFO.lpMaximumApplicationAddress; + baseAddress = NULL; + + while (VirtualQueryEx(hProcess, baseAddress, &basicInfo, + sizeof(MEMORY_BASIC_INFORMATION))) + { + py_tuple = NULL; + if (baseAddress > maxAddr) + break; + if (GetMappedFileNameW(hProcess, baseAddress, mappedFileName, + sizeof(mappedFileName))) + { + py_str = PyUnicode_FromWideChar(mappedFileName, + wcslen(mappedFileName)); + if (py_str == NULL) + goto error; +#ifdef _WIN64 + py_tuple = Py_BuildValue( + "(KsOI)", + (unsigned long long)baseAddress, +#else + py_tuple = Py_BuildValue( + "(ksOI)", + (unsigned long)baseAddress, +#endif + get_region_protection_string(basicInfo.Protect), + py_str, + basicInfo.RegionSize); + + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + Py_CLEAR(py_str); + } + previousAllocationBase = (ULONGLONG)basicInfo.AllocationBase; + baseAddress = (PCHAR)baseAddress + basicInfo.RegionSize; + } + + CloseHandle(hProcess); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_str); + Py_DECREF(py_retlist); + if (hProcess != NULL) + CloseHandle(hProcess); + return NULL; +} + + +/* + * Return a {pid:ppid, ...} dict for all running processes. + */ +static PyObject * +psutil_ppid_map(PyObject *self, PyObject *args) { + PyObject *py_pid = NULL; + PyObject *py_ppid = NULL; + PyObject *py_retdict = PyDict_New(); + HANDLE handle = NULL; + PROCESSENTRY32 pe = {0}; + pe.dwSize = sizeof(PROCESSENTRY32); + + if (py_retdict == NULL) + return NULL; + handle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + Py_DECREF(py_retdict); + return NULL; + } + + if (Process32First(handle, &pe)) { + do { + py_pid = Py_BuildValue("I", pe.th32ProcessID); + if (py_pid == NULL) + goto error; + py_ppid = Py_BuildValue("I", pe.th32ParentProcessID); + if (py_ppid == NULL) + goto error; + if (PyDict_SetItem(py_retdict, py_pid, py_ppid)) + goto error; + Py_CLEAR(py_pid); + Py_CLEAR(py_ppid); + } while (Process32Next(handle, &pe)); + } + + CloseHandle(handle); + return py_retdict; + +error: + Py_XDECREF(py_pid); + Py_XDECREF(py_ppid); + Py_DECREF(py_retdict); + CloseHandle(handle); + return NULL; +} + + +/* + * Return NICs addresses. + */ + +static PyObject * +psutil_net_if_addrs(PyObject *self, PyObject *args) { + unsigned int i = 0; + ULONG family; + PCTSTR intRet; + PCTSTR netmaskIntRet; + char *ptr; + char buff_addr[1024]; + char buff_macaddr[1024]; + char buff_netmask[1024]; + DWORD dwRetVal = 0; +#if (_WIN32_WINNT >= 0x0600) // Windows Vista and above + ULONG converted_netmask; + UINT netmask_bits; + struct in_addr in_netmask; +#endif + PIP_ADAPTER_ADDRESSES pAddresses = NULL; + PIP_ADAPTER_ADDRESSES pCurrAddresses = NULL; + PIP_ADAPTER_UNICAST_ADDRESS pUnicast = NULL; + + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_address = NULL; + PyObject *py_mac_address = NULL; + PyObject *py_nic_name = NULL; + PyObject *py_netmask = NULL; + + if (py_retlist == NULL) + return NULL; + + pAddresses = psutil_get_nic_addresses(); + if (pAddresses == NULL) + goto error; + pCurrAddresses = pAddresses; + + while (pCurrAddresses) { + pUnicast = pCurrAddresses->FirstUnicastAddress; + + netmaskIntRet = NULL; + py_nic_name = NULL; + py_nic_name = PyUnicode_FromWideChar( + pCurrAddresses->FriendlyName, + wcslen(pCurrAddresses->FriendlyName)); + if (py_nic_name == NULL) + goto error; + + // MAC address + if (pCurrAddresses->PhysicalAddressLength != 0) { + ptr = buff_macaddr; + *ptr = '\0'; + for (i = 0; i < (int) pCurrAddresses->PhysicalAddressLength; i++) { + if (i == (pCurrAddresses->PhysicalAddressLength - 1)) { + sprintf_s(ptr, _countof(buff_macaddr), "%.2X\n", + (int)pCurrAddresses->PhysicalAddress[i]); + } + else { + sprintf_s(ptr, _countof(buff_macaddr), "%.2X-", + (int)pCurrAddresses->PhysicalAddress[i]); + } + ptr += 3; + } + *--ptr = '\0'; + + py_mac_address = Py_BuildValue("s", buff_macaddr); + if (py_mac_address == NULL) + goto error; + + Py_INCREF(Py_None); + Py_INCREF(Py_None); + Py_INCREF(Py_None); + py_tuple = Py_BuildValue( + "(OiOOOO)", + py_nic_name, + -1, // this will be converted later to AF_LINK + py_mac_address, + Py_None, // netmask (not supported) + Py_None, // broadcast (not supported) + Py_None // ptp (not supported on Windows) + ); + if (! py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + Py_CLEAR(py_mac_address); + } + + // find out the IP address associated with the NIC + if (pUnicast != NULL) { + for (i = 0; pUnicast != NULL; i++) { + family = pUnicast->Address.lpSockaddr->sa_family; + if (family == AF_INET) { + struct sockaddr_in *sa_in = (struct sockaddr_in *) + pUnicast->Address.lpSockaddr; + intRet = inet_ntop(AF_INET, &(sa_in->sin_addr), buff_addr, + sizeof(buff_addr)); + if (!intRet) + goto error; +#if (_WIN32_WINNT >= 0x0600) // Windows Vista and above + netmask_bits = pUnicast->OnLinkPrefixLength; + dwRetVal = ConvertLengthToIpv4Mask(netmask_bits, &converted_netmask); + if (dwRetVal == NO_ERROR) { + in_netmask.s_addr = converted_netmask; + netmaskIntRet = inet_ntop( + AF_INET, &in_netmask, buff_netmask, + sizeof(buff_netmask)); + if (!netmaskIntRet) + goto error; + } +#endif + } + else if (family == AF_INET6) { + struct sockaddr_in6 *sa_in6 = (struct sockaddr_in6 *) + pUnicast->Address.lpSockaddr; + intRet = inet_ntop(AF_INET6, &(sa_in6->sin6_addr), + buff_addr, sizeof(buff_addr)); + if (!intRet) + goto error; + } + else { + // we should never get here + pUnicast = pUnicast->Next; + continue; + } + +#if PY_MAJOR_VERSION >= 3 + py_address = PyUnicode_FromString(buff_addr); +#else + py_address = PyString_FromString(buff_addr); +#endif + if (py_address == NULL) + goto error; + + if (netmaskIntRet != NULL) { +#if PY_MAJOR_VERSION >= 3 + py_netmask = PyUnicode_FromString(buff_netmask); +#else + py_netmask = PyString_FromString(buff_netmask); +#endif + } else { + Py_INCREF(Py_None); + py_netmask = Py_None; + } + + Py_INCREF(Py_None); + Py_INCREF(Py_None); + py_tuple = Py_BuildValue( + "(OiOOOO)", + py_nic_name, + family, + py_address, + py_netmask, + Py_None, // broadcast (not supported) + Py_None // ptp (not supported on Windows) + ); + + if (! py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_CLEAR(py_tuple); + Py_CLEAR(py_address); + Py_CLEAR(py_netmask); + + pUnicast = pUnicast->Next; + } + } + Py_CLEAR(py_nic_name); + pCurrAddresses = pCurrAddresses->Next; + } + + free(pAddresses); + return py_retlist; + +error: + if (pAddresses) + free(pAddresses); + Py_DECREF(py_retlist); + Py_XDECREF(py_tuple); + Py_XDECREF(py_address); + Py_XDECREF(py_nic_name); + Py_XDECREF(py_netmask); + return NULL; +} + + +/* + * Provides stats about NIC interfaces installed on the system. + * TODO: get 'duplex' (currently it's hard coded to '2', aka + 'full duplex') + */ +static PyObject * +psutil_net_if_stats(PyObject *self, PyObject *args) { + int i; + DWORD dwSize = 0; + DWORD dwRetVal = 0; + MIB_IFTABLE *pIfTable; + MIB_IFROW *pIfRow; + PIP_ADAPTER_ADDRESSES pAddresses = NULL; + PIP_ADAPTER_ADDRESSES pCurrAddresses = NULL; + char descr[MAX_PATH]; + int ifname_found; + + PyObject *py_nic_name = NULL; + PyObject *py_retdict = PyDict_New(); + PyObject *py_ifc_info = NULL; + PyObject *py_is_up = NULL; + + if (py_retdict == NULL) + return NULL; + + pAddresses = psutil_get_nic_addresses(); + if (pAddresses == NULL) + goto error; + + pIfTable = (MIB_IFTABLE *) malloc(sizeof (MIB_IFTABLE)); + if (pIfTable == NULL) { + PyErr_NoMemory(); + goto error; + } + dwSize = sizeof(MIB_IFTABLE); + if (GetIfTable(pIfTable, &dwSize, FALSE) == ERROR_INSUFFICIENT_BUFFER) { + free(pIfTable); + pIfTable = (MIB_IFTABLE *) malloc(dwSize); + if (pIfTable == NULL) { + PyErr_NoMemory(); + goto error; + } + } + // Make a second call to GetIfTable to get the actual + // data we want. + if ((dwRetVal = GetIfTable(pIfTable, &dwSize, FALSE)) != NO_ERROR) { + PyErr_SetString(PyExc_RuntimeError, "GetIfTable() syscall failed"); + goto error; + } + + for (i = 0; i < (int) pIfTable->dwNumEntries; i++) { + pIfRow = (MIB_IFROW *) & pIfTable->table[i]; + + // GetIfTable is not able to give us NIC with "friendly names" + // so we determine them via GetAdapterAddresses() which + // provides friendly names *and* descriptions and find the + // ones that match. + ifname_found = 0; + pCurrAddresses = pAddresses; + while (pCurrAddresses) { + sprintf_s(descr, MAX_PATH, "%wS", pCurrAddresses->Description); + if (lstrcmp(descr, pIfRow->bDescr) == 0) { + py_nic_name = PyUnicode_FromWideChar( + pCurrAddresses->FriendlyName, + wcslen(pCurrAddresses->FriendlyName)); + if (py_nic_name == NULL) + goto error; + ifname_found = 1; + break; + } + pCurrAddresses = pCurrAddresses->Next; + } + if (ifname_found == 0) { + // Name not found means GetAdapterAddresses() doesn't list + // this NIC, only GetIfTable, meaning it's not really a NIC + // interface so we skip it. + continue; + } + + // is up? + if((pIfRow->dwOperStatus == MIB_IF_OPER_STATUS_CONNECTED || + pIfRow->dwOperStatus == MIB_IF_OPER_STATUS_OPERATIONAL) && + pIfRow->dwAdminStatus == 1 ) { + py_is_up = Py_True; + } + else { + py_is_up = Py_False; + } + Py_INCREF(py_is_up); + + py_ifc_info = Py_BuildValue( + "(Oikk)", + py_is_up, + 2, // there's no way to know duplex so let's assume 'full' + pIfRow->dwSpeed / 1000000, // expressed in bytes, we want Mb + pIfRow->dwMtu + ); + if (!py_ifc_info) + goto error; + if (PyDict_SetItem(py_retdict, py_nic_name, py_ifc_info)) + goto error; + Py_CLEAR(py_nic_name); + Py_CLEAR(py_ifc_info); + } + + free(pIfTable); + free(pAddresses); + return py_retdict; + +error: + Py_XDECREF(py_is_up); + Py_XDECREF(py_ifc_info); + Py_XDECREF(py_nic_name); + Py_DECREF(py_retdict); + if (pIfTable != NULL) + free(pIfTable); + if (pAddresses != NULL) + free(pAddresses); + return NULL; +} + + +/* + * Return CPU statistics. + */ +static PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + NTSTATUS status; + _SYSTEM_PERFORMANCE_INFORMATION *spi = NULL; + _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *sppi = NULL; + _SYSTEM_INTERRUPT_INFORMATION *InterruptInformation = NULL; + unsigned int ncpus; + UINT i; + ULONG64 dpcs = 0; + ULONG interrupts = 0; + + // retrieves number of processors + ncpus = psutil_get_num_cpus(1); + if (ncpus == 0) + goto error; + + // get syscalls / ctx switches + spi = (_SYSTEM_PERFORMANCE_INFORMATION *) \ + malloc(ncpus * sizeof(_SYSTEM_PERFORMANCE_INFORMATION)); + if (spi == NULL) { + PyErr_NoMemory(); + goto error; + } + status = psutil_NtQuerySystemInformation( + SystemPerformanceInformation, + spi, + ncpus * sizeof(_SYSTEM_PERFORMANCE_INFORMATION), + NULL); + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQuerySystemInformation(SystemPerformanceInformation)"); + goto error; + } + + // get DPCs + InterruptInformation = \ + malloc(sizeof(_SYSTEM_INTERRUPT_INFORMATION) * ncpus); + if (InterruptInformation == NULL) { + PyErr_NoMemory(); + goto error; + } + + status = psutil_NtQuerySystemInformation( + SystemInterruptInformation, + InterruptInformation, + ncpus * sizeof(SYSTEM_INTERRUPT_INFORMATION), + NULL); + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQuerySystemInformation(SystemInterruptInformation)"); + goto error; + } + for (i = 0; i < ncpus; i++) { + dpcs += InterruptInformation[i].DpcCount; + } + + // get interrupts + sppi = (_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *) \ + malloc(ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION)); + if (sppi == NULL) { + PyErr_NoMemory(); + goto error; + } + + status = psutil_NtQuerySystemInformation( + SystemProcessorPerformanceInformation, + sppi, + ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION), + NULL); + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, + "NtQuerySystemInformation(SystemProcessorPerformanceInformation)"); + goto error; + } + + for (i = 0; i < ncpus; i++) { + interrupts += sppi[i].InterruptCount; + } + + // done + free(spi); + free(InterruptInformation); + free(sppi); + return Py_BuildValue( + "kkkk", + spi->ContextSwitches, + interrupts, + (unsigned long)dpcs, + spi->SystemCalls + ); + +error: + if (spi) + free(spi); + if (InterruptInformation) + free(InterruptInformation); + if (sppi) + free(sppi); + return NULL; +} + + +/* + * Return CPU frequency. + */ +static PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + PROCESSOR_POWER_INFORMATION *ppi; + NTSTATUS ret; + ULONG size; + LPBYTE pBuffer = NULL; + ULONG current; + ULONG max; + unsigned int ncpus; + + // Get the number of CPUs. + ncpus = psutil_get_num_cpus(1); + if (ncpus == 0) + return NULL; + + // Allocate size. + size = ncpus * sizeof(PROCESSOR_POWER_INFORMATION); + pBuffer = (BYTE*)LocalAlloc(LPTR, size); + if (! pBuffer) + return PyErr_SetFromWindowsErr(0); + + // Syscall. + ret = CallNtPowerInformation( + ProcessorInformation, NULL, 0, pBuffer, size); + if (ret != 0) { + PyErr_SetString(PyExc_RuntimeError, + "CallNtPowerInformation syscall failed"); + goto error; + } + + // Results. + ppi = (PROCESSOR_POWER_INFORMATION *)pBuffer; + max = ppi->MaxMhz; + current = ppi->CurrentMhz; + LocalFree(pBuffer); + + return Py_BuildValue("kk", current, max); + +error: + if (pBuffer != NULL) + LocalFree(pBuffer); + return NULL; +} + + +/* + * Return battery usage stats. + */ +static PyObject * +psutil_sensors_battery(PyObject *self, PyObject *args) { + SYSTEM_POWER_STATUS sps; + + if (GetSystemPowerStatus(&sps) == 0) + return PyErr_SetFromWindowsErr(0); + return Py_BuildValue( + "iiiI", + sps.ACLineStatus, // whether AC is connected: 0=no, 1=yes, 255=unknown + // status flag: + // 1, 2, 4 = high, low, critical + // 8 = charging + // 128 = no battery + sps.BatteryFlag, + sps.BatteryLifePercent, // percent + sps.BatteryLifeTime // remaining secs + ); +} + + +/* + * System memory page size as an int. + */ +static PyObject * +psutil_getpagesize(PyObject *self, PyObject *args) { + // XXX: we may want to use GetNativeSystemInfo to differentiate + // page size for WoW64 processes (but am not sure). + return Py_BuildValue("I", PSUTIL_SYSTEM_INFO.dwPageSize); +} + + +// ------------------------ Python init --------------------------- + +static PyMethodDef +PsutilMethods[] = { + // --- per-process functions + {"proc_cmdline", (PyCFunction)(void(*)(void))psutil_proc_cmdline, + METH_VARARGS | METH_KEYWORDS, + "Return process cmdline as a list of cmdline arguments"}, + {"proc_environ", psutil_proc_environ, METH_VARARGS, + "Return process environment data"}, + {"proc_exe", psutil_proc_exe, METH_VARARGS, + "Return path of the process executable"}, + {"proc_name", psutil_proc_name, METH_VARARGS, + "Return process name"}, + {"proc_kill", psutil_proc_kill, METH_VARARGS, + "Kill the process identified by the given PID"}, + {"proc_cpu_times", psutil_proc_cpu_times, METH_VARARGS, + "Return tuple of user/kern time for the given PID"}, + {"proc_create_time", psutil_proc_create_time, METH_VARARGS, + "Return a float indicating the process create time expressed in " + "seconds since the epoch"}, + {"proc_memory_info", psutil_proc_memory_info, METH_VARARGS, + "Return a tuple of process memory information"}, + {"proc_memory_uss", psutil_proc_memory_uss, METH_VARARGS, + "Return the USS of the process"}, + {"proc_cwd", psutil_proc_cwd, METH_VARARGS, + "Return process current working directory"}, + {"proc_suspend_or_resume", psutil_proc_suspend_or_resume, METH_VARARGS, + "Suspend or resume a process"}, + {"proc_open_files", psutil_proc_open_files, METH_VARARGS, + "Return files opened by process"}, + {"proc_username", psutil_proc_username, METH_VARARGS, + "Return the username of a process"}, + {"proc_threads", psutil_proc_threads, METH_VARARGS, + "Return process threads information as a list of tuple"}, + {"proc_wait", psutil_proc_wait, METH_VARARGS, + "Wait for process to terminate and return its exit code."}, + {"proc_priority_get", psutil_proc_priority_get, METH_VARARGS, + "Return process priority."}, + {"proc_priority_set", psutil_proc_priority_set, METH_VARARGS, + "Set process priority."}, +#if (_WIN32_WINNT >= 0x0600) // Windows Vista + {"proc_io_priority_get", psutil_proc_io_priority_get, METH_VARARGS, + "Return process IO priority."}, + {"proc_io_priority_set", psutil_proc_io_priority_set, METH_VARARGS, + "Set process IO priority."}, +#endif + {"proc_cpu_affinity_get", psutil_proc_cpu_affinity_get, METH_VARARGS, + "Return process CPU affinity as a bitmask."}, + {"proc_cpu_affinity_set", psutil_proc_cpu_affinity_set, METH_VARARGS, + "Set process CPU affinity."}, + {"proc_io_counters", psutil_proc_io_counters, METH_VARARGS, + "Get process I/O counters."}, + {"proc_is_suspended", psutil_proc_is_suspended, METH_VARARGS, + "Return True if one of the process threads is in a suspended state"}, + {"proc_num_handles", psutil_proc_num_handles, METH_VARARGS, + "Return the number of handles opened by process."}, + {"proc_memory_maps", psutil_proc_memory_maps, METH_VARARGS, + "Return a list of process's memory mappings"}, + + // --- alternative pinfo interface + {"proc_info", psutil_proc_info, METH_VARARGS, + "Various process information"}, + + // --- system-related functions + {"pids", psutil_pids, METH_VARARGS, + "Returns a list of PIDs currently running on the system"}, + {"ppid_map", psutil_ppid_map, METH_VARARGS, + "Return a {pid:ppid, ...} dict for all running processes"}, + {"pid_exists", psutil_pid_exists, METH_VARARGS, + "Determine if the process exists in the current process list."}, + {"cpu_count_logical", psutil_cpu_count_logical, METH_VARARGS, + "Returns the number of logical CPUs on the system"}, + {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, + "Returns the number of physical CPUs on the system"}, + {"boot_time", psutil_boot_time, METH_VARARGS, + "Return the system boot time expressed in seconds since the epoch."}, + {"virtual_mem", psutil_virtual_mem, METH_VARARGS, + "Return the total amount of physical memory, in bytes"}, + {"cpu_times", psutil_cpu_times, METH_VARARGS, + "Return system cpu times as a list"}, + {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, + "Return system per-cpu times as a list of tuples"}, + {"disk_usage", psutil_disk_usage, METH_VARARGS, + "Return path's disk total and free as a Python tuple."}, + {"net_io_counters", psutil_net_io_counters, METH_VARARGS, + "Return dict of tuples of networks I/O information."}, + {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS, + "Return dict of tuples of disks I/O information."}, + {"users", psutil_users, METH_VARARGS, + "Return a list of currently connected users."}, + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return disk partitions."}, + {"net_connections", psutil_net_connections, METH_VARARGS, + "Return system-wide connections"}, + {"net_if_addrs", psutil_net_if_addrs, METH_VARARGS, + "Return NICs addresses."}, + {"net_if_stats", psutil_net_if_stats, METH_VARARGS, + "Return NICs stats."}, + {"cpu_stats", psutil_cpu_stats, METH_VARARGS, + "Return NICs stats."}, + {"cpu_freq", psutil_cpu_freq, METH_VARARGS, + "Return CPU frequency."}, +#if (_WIN32_WINNT >= 0x0600) // Windows Vista + {"init_loadavg_counter", (PyCFunction)psutil_init_loadavg_counter, + METH_VARARGS, + "Initializes the emulated load average calculator."}, + {"getloadavg", (PyCFunction)psutil_get_loadavg, METH_VARARGS, + "Returns the emulated POSIX-like load average."}, +#endif + {"sensors_battery", psutil_sensors_battery, METH_VARARGS, + "Return battery metrics usage."}, + {"getpagesize", psutil_getpagesize, METH_VARARGS, + "Return system memory page size."}, + + // --- windows services + {"winservice_enumerate", psutil_winservice_enumerate, METH_VARARGS, + "List all services"}, + {"winservice_query_config", psutil_winservice_query_config, METH_VARARGS, + "Return service config"}, + {"winservice_query_status", psutil_winservice_query_status, METH_VARARGS, + "Return service config"}, + {"winservice_query_descr", psutil_winservice_query_descr, METH_VARARGS, + "Return the description of a service"}, + {"winservice_start", psutil_winservice_start, METH_VARARGS, + "Start a service"}, + {"winservice_stop", psutil_winservice_stop, METH_VARARGS, + "Stop a service"}, + + // --- windows API bindings + {"win32_QueryDosDevice", psutil_win32_QueryDosDevice, METH_VARARGS, + "QueryDosDevice binding"}, + + // --- others + {"set_testing", psutil_set_testing, METH_NOARGS, + "Set psutil in testing mode"}, + + {NULL, NULL, 0, NULL} +}; + + +struct module_state { + PyObject *error; +}; + +#if PY_MAJOR_VERSION >= 3 +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else +#define GETSTATE(m) (&_state) +static struct module_state _state; +#endif + +#if PY_MAJOR_VERSION >= 3 + +static int psutil_windows_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +static int psutil_windows_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "psutil_windows", + NULL, + sizeof(struct module_state), + PsutilMethods, + NULL, + psutil_windows_traverse, + psutil_windows_clear, + NULL +}; + +#define INITERROR return NULL + +PyMODINIT_FUNC PyInit__psutil_windows(void) + +#else +#define INITERROR return +void init_psutil_windows(void) +#endif +{ + struct module_state *st = NULL; +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("_psutil_windows", PsutilMethods); +#endif + if (module == NULL) + INITERROR; + + if (psutil_setup() != 0) + INITERROR; + if (psutil_load_globals() != 0) + INITERROR; + if (psutil_set_se_debug() != 0) + INITERROR; + + st = GETSTATE(module); + st->error = PyErr_NewException("_psutil_windows.Error", NULL, NULL); + if (st->error == NULL) { + Py_DECREF(module); + INITERROR; + } + + // Exceptions. + TimeoutExpired = PyErr_NewException( + "_psutil_windows.TimeoutExpired", NULL, NULL); + Py_INCREF(TimeoutExpired); + PyModule_AddObject(module, "TimeoutExpired", TimeoutExpired); + + TimeoutAbandoned = PyErr_NewException( + "_psutil_windows.TimeoutAbandoned", NULL, NULL); + Py_INCREF(TimeoutAbandoned); + PyModule_AddObject(module, "TimeoutAbandoned", TimeoutAbandoned); + + // version constant + PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); + + // process status constants + // http://msdn.microsoft.com/en-us/library/ms683211(v=vs.85).aspx + PyModule_AddIntConstant( + module, "ABOVE_NORMAL_PRIORITY_CLASS", ABOVE_NORMAL_PRIORITY_CLASS); + PyModule_AddIntConstant( + module, "BELOW_NORMAL_PRIORITY_CLASS", BELOW_NORMAL_PRIORITY_CLASS); + PyModule_AddIntConstant( + module, "HIGH_PRIORITY_CLASS", HIGH_PRIORITY_CLASS); + PyModule_AddIntConstant( + module, "IDLE_PRIORITY_CLASS", IDLE_PRIORITY_CLASS); + PyModule_AddIntConstant( + module, "NORMAL_PRIORITY_CLASS", NORMAL_PRIORITY_CLASS); + PyModule_AddIntConstant( + module, "REALTIME_PRIORITY_CLASS", REALTIME_PRIORITY_CLASS); + + // connection status constants + // http://msdn.microsoft.com/en-us/library/cc669305.aspx + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_CLOSED", MIB_TCP_STATE_CLOSED); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_CLOSING", MIB_TCP_STATE_CLOSING); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_CLOSE_WAIT", MIB_TCP_STATE_CLOSE_WAIT); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_LISTEN", MIB_TCP_STATE_LISTEN); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_ESTAB", MIB_TCP_STATE_ESTAB); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_SYN_SENT", MIB_TCP_STATE_SYN_SENT); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_SYN_RCVD", MIB_TCP_STATE_SYN_RCVD); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_FIN_WAIT1", MIB_TCP_STATE_FIN_WAIT1); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_FIN_WAIT2", MIB_TCP_STATE_FIN_WAIT2); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_LAST_ACK", MIB_TCP_STATE_LAST_ACK); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_TIME_WAIT", MIB_TCP_STATE_TIME_WAIT); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_TIME_WAIT", MIB_TCP_STATE_TIME_WAIT); + PyModule_AddIntConstant( + module, "MIB_TCP_STATE_DELETE_TCB", MIB_TCP_STATE_DELETE_TCB); + PyModule_AddIntConstant( + module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + + // service status constants + /* + PyModule_AddIntConstant( + module, "SERVICE_CONTINUE_PENDING", SERVICE_CONTINUE_PENDING); + PyModule_AddIntConstant( + module, "SERVICE_PAUSE_PENDING", SERVICE_PAUSE_PENDING); + PyModule_AddIntConstant( + module, "SERVICE_PAUSED", SERVICE_PAUSED); + PyModule_AddIntConstant( + module, "SERVICE_RUNNING", SERVICE_RUNNING); + PyModule_AddIntConstant( + module, "SERVICE_START_PENDING", SERVICE_START_PENDING); + PyModule_AddIntConstant( + module, "SERVICE_STOP_PENDING", SERVICE_STOP_PENDING); + PyModule_AddIntConstant( + module, "SERVICE_STOPPED", SERVICE_STOPPED); + */ + + // ...for internal use in _psutil_windows.py + PyModule_AddIntConstant( + module, "INFINITE", INFINITE); + PyModule_AddIntConstant( + module, "ERROR_ACCESS_DENIED", ERROR_ACCESS_DENIED); + PyModule_AddIntConstant( + module, "ERROR_INVALID_NAME", ERROR_INVALID_NAME); + PyModule_AddIntConstant( + module, "ERROR_SERVICE_DOES_NOT_EXIST", ERROR_SERVICE_DOES_NOT_EXIST); + PyModule_AddIntConstant( + module, "ERROR_PRIVILEGE_NOT_HELD", ERROR_PRIVILEGE_NOT_HELD); + PyModule_AddIntConstant( + module, "WINVER", PSUTIL_WINVER); + PyModule_AddIntConstant( + module, "WINDOWS_XP", PSUTIL_WINDOWS_XP); + PyModule_AddIntConstant( + module, "WINDOWS_SERVER_2003", PSUTIL_WINDOWS_SERVER_2003); + PyModule_AddIntConstant( + module, "WINDOWS_VISTA", PSUTIL_WINDOWS_VISTA); + PyModule_AddIntConstant( + module, "WINDOWS_7", PSUTIL_WINDOWS_7); + PyModule_AddIntConstant( + module, "WINDOWS_8", PSUTIL_WINDOWS_8); + PyModule_AddIntConstant( + module, "WINDOWS_8_1", PSUTIL_WINDOWS_8_1); + PyModule_AddIntConstant( + module, "WINDOWS_10", PSUTIL_WINDOWS_10); + +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} diff --git a/ddtrace/vendor/psutil/_pswindows.py b/ddtrace/vendor/psutil/_pswindows.py new file mode 100644 index 00000000000..636b0af905c --- /dev/null +++ b/ddtrace/vendor/psutil/_pswindows.py @@ -0,0 +1,1127 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Windows platform implementation.""" + +import contextlib +import errno +import functools +import os +import sys +import time +from collections import namedtuple + +from . import _common +try: + from . import _psutil_windows as cext +except ImportError as err: + if str(err).lower().startswith("dll load failed") and \ + sys.getwindowsversion()[0] < 6: + # We may get here if: + # 1) we are on an old Windows version + # 2) psutil was installed via pip + wheel + # See: https://github.com/giampaolo/psutil/issues/811 + # It must be noted that psutil can still (kind of) work + # on outdated systems if compiled / installed from sources, + # but if we get here it means this this was a wheel (or exe). + msg = "this Windows version is too old (< Windows Vista); " + msg += "psutil 3.4.2 is the latest version which supports Windows " + msg += "2000, XP and 2003 server" + raise RuntimeError(msg) + else: + raise + +from ._common import conn_tmap +from ._common import conn_to_ntuple +from ._common import ENCODING +from ._common import ENCODING_ERRS +from ._common import isfile_strict +from ._common import memoize +from ._common import memoize_when_activated +from ._common import parse_environ_block +from ._common import usage_percent +from ._compat import long +from ._compat import lru_cache +from ._compat import PY3 +from ._compat import unicode +from ._compat import xrange +from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS +from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS +from ._psutil_windows import HIGH_PRIORITY_CLASS +from ._psutil_windows import IDLE_PRIORITY_CLASS +from ._psutil_windows import NORMAL_PRIORITY_CLASS +from ._psutil_windows import REALTIME_PRIORITY_CLASS + +if sys.version_info >= (3, 4): + import enum +else: + enum = None + +# process priority constants, import from __init__.py: +# http://msdn.microsoft.com/en-us/library/ms686219(v=vs.85).aspx +__extra__all__ = [ + "win_service_iter", "win_service_get", + # Process priority + "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", + "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", "NORMAL_PRIORITY_CLASS", + "REALTIME_PRIORITY_CLASS", + # IO priority + "IOPRIO_VERYLOW", "IOPRIO_LOW", "IOPRIO_NORMAL", "IOPRIO_HIGH", + # others + "CONN_DELETE_TCB", "AF_LINK", +] + + +# ===================================================================== +# --- globals +# ===================================================================== + +CONN_DELETE_TCB = "DELETE_TCB" +HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_io_priority_get") +HAS_GETLOADAVG = hasattr(cext, "getloadavg") +ERROR_PARTIAL_COPY = 299 + + +if enum is None: + AF_LINK = -1 +else: + AddressFamily = enum.IntEnum('AddressFamily', {'AF_LINK': -1}) + AF_LINK = AddressFamily.AF_LINK + +TCP_STATUSES = { + cext.MIB_TCP_STATE_ESTAB: _common.CONN_ESTABLISHED, + cext.MIB_TCP_STATE_SYN_SENT: _common.CONN_SYN_SENT, + cext.MIB_TCP_STATE_SYN_RCVD: _common.CONN_SYN_RECV, + cext.MIB_TCP_STATE_FIN_WAIT1: _common.CONN_FIN_WAIT1, + cext.MIB_TCP_STATE_FIN_WAIT2: _common.CONN_FIN_WAIT2, + cext.MIB_TCP_STATE_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.MIB_TCP_STATE_CLOSED: _common.CONN_CLOSE, + cext.MIB_TCP_STATE_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.MIB_TCP_STATE_LAST_ACK: _common.CONN_LAST_ACK, + cext.MIB_TCP_STATE_LISTEN: _common.CONN_LISTEN, + cext.MIB_TCP_STATE_CLOSING: _common.CONN_CLOSING, + cext.MIB_TCP_STATE_DELETE_TCB: CONN_DELETE_TCB, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +if enum is not None: + class Priority(enum.IntEnum): + ABOVE_NORMAL_PRIORITY_CLASS = ABOVE_NORMAL_PRIORITY_CLASS + BELOW_NORMAL_PRIORITY_CLASS = BELOW_NORMAL_PRIORITY_CLASS + HIGH_PRIORITY_CLASS = HIGH_PRIORITY_CLASS + IDLE_PRIORITY_CLASS = IDLE_PRIORITY_CLASS + NORMAL_PRIORITY_CLASS = NORMAL_PRIORITY_CLASS + REALTIME_PRIORITY_CLASS = REALTIME_PRIORITY_CLASS + + globals().update(Priority.__members__) + +if enum is None: + IOPRIO_VERYLOW = 0 + IOPRIO_LOW = 1 + IOPRIO_NORMAL = 2 + IOPRIO_HIGH = 3 +else: + class IOPriority(enum.IntEnum): + IOPRIO_VERYLOW = 0 + IOPRIO_LOW = 1 + IOPRIO_NORMAL = 2 + IOPRIO_HIGH = 3 + globals().update(IOPriority.__members__) + +pinfo_map = dict( + num_handles=0, + ctx_switches=1, + user_time=2, + kernel_time=3, + create_time=4, + num_threads=5, + io_rcount=6, + io_wcount=7, + io_rbytes=8, + io_wbytes=9, + io_count_others=10, + io_bytes_others=11, + num_page_faults=12, + peak_wset=13, + wset=14, + peak_paged_pool=15, + paged_pool=16, + peak_non_paged_pool=17, + non_paged_pool=18, + pagefile=19, + peak_pagefile=20, + mem_private=21, +) + +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + +# More values at: https://stackoverflow.com/a/20804735/376587 +WIN_10 = (10, 0) +WIN_8 = (6, 2) +WIN_7 = (6, 1) +WIN_SERVER_2008 = (6, 0) +WIN_VISTA = (6, 0) +WIN_SERVER_2003 = (5, 2) +WIN_XP = (5, 1) + + +@lru_cache() +def get_winver(): + """Usage: + >>> if get_winver() <= WIN_VISTA: + ... ... + """ + wv = sys.getwindowsversion() + return (wv.major, wv.minor) + + +IS_WIN_XP = get_winver() < WIN_VISTA + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.cpu_times() +scputimes = namedtuple('scputimes', + ['user', 'system', 'idle', 'interrupt', 'dpc']) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_info() +pmem = namedtuple( + 'pmem', ['rss', 'vms', + 'num_page_faults', 'peak_wset', 'wset', 'peak_paged_pool', + 'paged_pool', 'peak_nonpaged_pool', 'nonpaged_pool', + 'pagefile', 'peak_pagefile', 'private']) +# psutil.Process.memory_full_info() +pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple('pmmap_grouped', ['path', 'rss']) +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'other_count', 'other_bytes']) + + +# ===================================================================== +# --- utils +# ===================================================================== + + +@lru_cache(maxsize=512) +def convert_dos_path(s): + r"""Convert paths using native DOS format like: + "\Device\HarddiskVolume1\Windows\systemew\file.txt" + into: + "C:\Windows\systemew\file.txt" + """ + rawdrive = '\\'.join(s.split('\\')[:3]) + driveletter = cext.win32_QueryDosDevice(rawdrive) + return os.path.join(driveletter, s[len(rawdrive):]) + + +def py2_strencode(s): + """Encode a unicode string to a byte string by using the default fs + encoding + "replace" error handler. + """ + if PY3: + return s + else: + if isinstance(s, str): + return s + else: + return s.encode(ENCODING, ENCODING_ERRS) + + +@memoize +def getpagesize(): + return cext.getpagesize() + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """System virtual memory as a namedtuple.""" + mem = cext.virtual_mem() + totphys, availphys, totpagef, availpagef, totvirt, freevirt = mem + # + total = totphys + avail = availphys + free = availphys + used = total - avail + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, used, free) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + mem = cext.virtual_mem() + total = mem[2] + free = mem[3] + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, 0, 0) + + +# ===================================================================== +# --- disk +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters + + +def disk_usage(path): + """Return disk usage associated with path.""" + if PY3 and isinstance(path, bytes): + # XXX: do we want to use "strict"? Probably yes, in order + # to fail immediately. After all we are accepting input here... + path = path.decode(ENCODING, errors="strict") + total, free = cext.disk_usage(path) + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sdiskusage(total, used, free, percent) + + +def disk_partitions(all): + """Return disk partitions.""" + rawlist = cext.disk_partitions(all) + return [_common.sdiskpart(*x) for x in rawlist] + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system CPU times as a named tuple.""" + user, system, idle = cext.cpu_times() + # Internally, GetSystemTimes() is used, and it doesn't return + # interrupt and dpc times. cext.per_cpu_times() does, so we + # rely on it to get those only. + percpu_summed = scputimes(*[sum(n) for n in zip(*cext.per_cpu_times())]) + return scputimes(user, system, idle, + percpu_summed.interrupt, percpu_summed.dpc) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples.""" + ret = [] + for user, system, idle, interrupt, dpc in cext.per_cpu_times(): + item = scputimes(user, system, idle, interrupt, dpc) + ret.append(item) + return ret + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + return cext.cpu_count_logical() + + +def cpu_count_physical(): + """Return the number of physical CPU cores in the system.""" + return cext.cpu_count_phys() + + +def cpu_stats(): + """Return CPU statistics.""" + ctx_switches, interrupts, dpcs, syscalls = cext.cpu_stats() + soft_interrupts = 0 + return _common.scpustats(ctx_switches, interrupts, soft_interrupts, + syscalls) + + +def cpu_freq(): + """Return CPU frequency. + On Windows per-cpu frequency is not supported. + """ + curr, max_ = cext.cpu_freq() + min_ = 0.0 + return [_common.scpufreq(float(curr), min_, float(max_))] + + +if HAS_GETLOADAVG: + _loadavg_inititialized = False + + def getloadavg(): + """Return the number of processes in the system run queue averaged + over the last 1, 5, and 15 minutes respectively as a tuple""" + global _loadavg_inititialized + + if not _loadavg_inititialized: + cext.init_loadavg_counter() + _loadavg_inititialized = True + + # Drop to 2 decimal points which is what Linux does + raw_loads = cext.getloadavg() + return tuple([round(load, 2) for load in raw_loads]) + + +# ===================================================================== +# --- network +# ===================================================================== + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + """ + if kind not in conn_tmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in conn_tmap]))) + families, types = conn_tmap[kind] + rawlist = cext.net_connections(_pid, families, types) + ret = set() + for item in rawlist: + fd, fam, type, laddr, raddr, status, pid = item + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, TCP_STATUSES, + pid=pid if _pid == -1 else None) + ret.add(nt) + return list(ret) + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + ret = {} + rawdict = cext.net_if_stats() + for name, items in rawdict.items(): + if not PY3: + assert isinstance(name, unicode), type(name) + name = py2_strencode(name) + isup, duplex, speed, mtu = items + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) + return ret + + +def net_io_counters(): + """Return network I/O statistics for every network interface + installed on the system as a dict of raw tuples. + """ + ret = cext.net_io_counters() + return dict([(py2_strencode(k), v) for k, v in ret.items()]) + + +def net_if_addrs(): + """Return the addresses associated to each NIC.""" + ret = [] + for items in cext.net_if_addrs(): + items = list(items) + items[0] = py2_strencode(items[0]) + ret.append(items) + return ret + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + """Return battery information.""" + # For constants meaning see: + # https://msdn.microsoft.com/en-us/library/windows/desktop/ + # aa373232(v=vs.85).aspx + acline_status, flags, percent, secsleft = cext.sensors_battery() + power_plugged = acline_status == 1 + no_battery = bool(flags & 128) + charging = bool(flags & 8) + + if no_battery: + return None + if power_plugged or charging: + secsleft = _common.POWER_TIME_UNLIMITED + elif secsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + + return _common.sbattery(percent, secsleft, power_plugged) + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +_last_btime = 0 + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + # This dirty hack is to adjust the precision of the returned + # value which may have a 1 second fluctuation, see: + # https://github.com/giampaolo/psutil/issues/1007 + global _last_btime + ret = float(cext.boot_time()) + if abs(ret - _last_btime) <= 1: + return _last_btime + else: + _last_btime = ret + return ret + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, hostname, tstamp = item + user = py2_strencode(user) + nt = _common.suser(user, None, hostname, tstamp, None) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- Windows services +# ===================================================================== + + +def win_service_iter(): + """Yields a list of WindowsService instances.""" + for name, display_name in cext.winservice_enumerate(): + yield WindowsService(py2_strencode(name), py2_strencode(display_name)) + + +def win_service_get(name): + """Open a Windows service and return it as a WindowsService instance.""" + service = WindowsService(name, None) + service._display_name = service._query_config()['display_name'] + return service + + +class WindowsService(object): + """Represents an installed Windows service.""" + + def __init__(self, name, display_name): + self._name = name + self._display_name = display_name + + def __str__(self): + details = "(name=%r, display_name=%r)" % ( + self._name, self._display_name) + return "%s%s" % (self.__class__.__name__, details) + + def __repr__(self): + return "<%s at %s>" % (self.__str__(), id(self)) + + def __eq__(self, other): + # Test for equality with another WindosService object based + # on name. + if not isinstance(other, WindowsService): + return NotImplemented + return self._name == other._name + + def __ne__(self, other): + return not self == other + + def _query_config(self): + with self._wrap_exceptions(): + display_name, binpath, username, start_type = \ + cext.winservice_query_config(self._name) + # XXX - update _self.display_name? + return dict( + display_name=py2_strencode(display_name), + binpath=py2_strencode(binpath), + username=py2_strencode(username), + start_type=py2_strencode(start_type)) + + def _query_status(self): + with self._wrap_exceptions(): + status, pid = cext.winservice_query_status(self._name) + if pid == 0: + pid = None + return dict(status=status, pid=pid) + + @contextlib.contextmanager + def _wrap_exceptions(self): + """Ctx manager which translates bare OSError and WindowsError + exceptions into NoSuchProcess and AccessDenied. + """ + try: + yield + except OSError as err: + if is_permission_err(err): + raise AccessDenied( + pid=None, name=self._name, + msg="service %r is not querable (not enough privileges)" % + self._name) + elif err.winerror in (cext.ERROR_INVALID_NAME, + cext.ERROR_SERVICE_DOES_NOT_EXIST): + raise NoSuchProcess( + pid=None, name=self._name, + msg="service %r does not exist)" % self._name) + else: + raise + + # config query + + def name(self): + """The service name. This string is how a service is referenced + and can be passed to win_service_get() to get a new + WindowsService instance. + """ + return self._name + + def display_name(self): + """The service display name. The value is cached when this class + is instantiated. + """ + return self._display_name + + def binpath(self): + """The fully qualified path to the service binary/exe file as + a string, including command line arguments. + """ + return self._query_config()['binpath'] + + def username(self): + """The name of the user that owns this service.""" + return self._query_config()['username'] + + def start_type(self): + """A string which can either be "automatic", "manual" or + "disabled". + """ + return self._query_config()['start_type'] + + # status query + + def pid(self): + """The process PID, if any, else None. This can be passed + to Process class to control the service's process. + """ + return self._query_status()['pid'] + + def status(self): + """Service status as a string.""" + return self._query_status()['status'] + + def description(self): + """Service long description.""" + return py2_strencode(cext.winservice_query_descr(self.name())) + + # utils + + def as_dict(self): + """Utility method retrieving all the information above as a + dictionary. + """ + d = self._query_config() + d.update(self._query_status()) + d['name'] = self.name() + d['display_name'] = self.display_name() + d['description'] = self.description() + return d + + # actions + # XXX: the necessary C bindings for start() and stop() are + # implemented but for now I prefer not to expose them. + # I may change my mind in the future. Reasons: + # - they require Administrator privileges + # - can't implement a timeout for stop() (unless by using a thread, + # which sucks) + # - would require adding ServiceAlreadyStarted and + # ServiceAlreadyStopped exceptions, adding two new APIs. + # - we might also want to have modify(), which would basically mean + # rewriting win32serviceutil.ChangeServiceConfig, which involves a + # lot of stuff (and API constants which would pollute the API), see: + # http://pyxr.sourceforge.net/PyXR/c/python24/lib/site-packages/ + # win32/lib/win32serviceutil.py.html#0175 + # - psutil is typically about "read only" monitoring stuff; + # win_service_* APIs should only be used to retrieve a service and + # check whether it's running + + # def start(self, timeout=None): + # with self._wrap_exceptions(): + # cext.winservice_start(self.name()) + # if timeout: + # giveup_at = time.time() + timeout + # while True: + # if self.status() == "running": + # return + # else: + # if time.time() > giveup_at: + # raise TimeoutExpired(timeout) + # else: + # time.sleep(.1) + + # def stop(self): + # # Note: timeout is not implemented because it's just not + # # possible, see: + # # http://stackoverflow.com/questions/11973228/ + # with self._wrap_exceptions(): + # return cext.winservice_stop(self.name()) + + +# ===================================================================== +# --- processes +# ===================================================================== + + +pids = cext.pids +pid_exists = cext.pid_exists +ppid_map = cext.ppid_map # used internally by Process.children() + + +def is_permission_err(exc): + """Return True if this is a permission error.""" + assert isinstance(exc, OSError), exc + # On Python 2 OSError doesn't always have 'winerror'. Sometimes + # it does, in which case the original exception was WindowsError + # (which is a subclass of OSError). + return exc.errno in (errno.EPERM, errno.EACCES) or \ + getattr(exc, "winerror", -1) in (cext.ERROR_ACCESS_DENIED, + cext.ERROR_PRIVILEGE_NOT_HELD) + + +def convert_oserror(exc, pid=None, name=None): + """Convert OSError into NoSuchProcess or AccessDenied.""" + assert isinstance(exc, OSError), exc + if is_permission_err(exc): + return AccessDenied(pid=pid, name=name) + if exc.errno == errno.ESRCH: + return NoSuchProcess(pid=pid, name=name) + raise exc + + +def wrap_exceptions(fun): + """Decorator which converts OSError into NoSuchProcess or AccessDenied.""" + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except OSError as err: + raise convert_oserror(err, pid=self.pid, name=self._name) + return wrapper + + +def retry_error_partial_copy(fun): + """Workaround for https://github.com/giampaolo/psutil/issues/875. + See: https://stackoverflow.com/questions/4457745#4457745 + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + delay = 0.0001 + times = 33 + for x in range(times): # retries for roughly 1 second + try: + return fun(self, *args, **kwargs) + except WindowsError as _: + err = _ + if err.winerror == ERROR_PARTIAL_COPY: + time.sleep(delay) + delay = min(delay * 2, 0.04) + continue + else: + raise + else: + msg = "%s retried %s times, converted to AccessDenied as it's " \ + "still returning %r" % (fun, times, err) + raise AccessDenied(pid=self.pid, name=self._name, msg=msg) + return wrapper + + +class Process(object): + """Wrapper class around underlying C implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_cache"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + + # --- oneshot() stuff + + def oneshot_enter(self): + self.oneshot_info.cache_activate(self) + + def oneshot_exit(self): + self.oneshot_info.cache_deactivate(self) + + @wrap_exceptions + @memoize_when_activated + def oneshot_info(self): + """Return multiple information about this process as a + raw tuple. + """ + ret = cext.proc_info(self.pid) + assert len(ret) == len(pinfo_map) + return ret + + @wrap_exceptions + def name(self): + """Return process name, which on Windows is always the final + part of the executable. + """ + # This is how PIDs 0 and 4 are always represented in taskmgr + # and process-hacker. + if self.pid == 0: + return "System Idle Process" + elif self.pid == 4: + return "System" + else: + try: + # Note: this will fail with AD for most PIDs owned + # by another user but it's faster. + return py2_strencode(os.path.basename(self.exe())) + except AccessDenied: + return py2_strencode(cext.proc_name(self.pid)) + + @wrap_exceptions + def exe(self): + # Dual implementation, see: + # https://github.com/giampaolo/psutil/pull/1413 + if not IS_WIN_XP: + exe = cext.proc_exe(self.pid) + else: + if self.pid in (0, 4): + # https://github.com/giampaolo/psutil/issues/414 + # https://github.com/giampaolo/psutil/issues/528 + raise AccessDenied(self.pid, self._name) + exe = cext.proc_exe(self.pid) + exe = convert_dos_path(exe) + return py2_strencode(exe) + + @wrap_exceptions + @retry_error_partial_copy + def cmdline(self): + if cext.WINVER >= cext.WINDOWS_8_1: + # PEB method detects cmdline changes but requires more + # privileges: https://github.com/giampaolo/psutil/pull/1398 + try: + ret = cext.proc_cmdline(self.pid, use_peb=True) + except OSError as err: + if is_permission_err(err): + ret = cext.proc_cmdline(self.pid, use_peb=False) + else: + raise + else: + ret = cext.proc_cmdline(self.pid, use_peb=True) + if PY3: + return ret + else: + return [py2_strencode(s) for s in ret] + + @wrap_exceptions + @retry_error_partial_copy + def environ(self): + ustr = cext.proc_environ(self.pid) + if ustr and not PY3: + assert isinstance(ustr, unicode), type(ustr) + return parse_environ_block(py2_strencode(ustr)) + + def ppid(self): + try: + return ppid_map()[self.pid] + except KeyError: + raise NoSuchProcess(self.pid, self._name) + + def _get_raw_meminfo(self): + try: + return cext.proc_memory_info(self.pid) + except OSError as err: + if is_permission_err(err): + # TODO: the C ext can probably be refactored in order + # to get this from cext.proc_info() + info = self.oneshot_info() + return ( + info[pinfo_map['num_page_faults']], + info[pinfo_map['peak_wset']], + info[pinfo_map['wset']], + info[pinfo_map['peak_paged_pool']], + info[pinfo_map['paged_pool']], + info[pinfo_map['peak_non_paged_pool']], + info[pinfo_map['non_paged_pool']], + info[pinfo_map['pagefile']], + info[pinfo_map['peak_pagefile']], + info[pinfo_map['mem_private']], + ) + raise + + @wrap_exceptions + def memory_info(self): + # on Windows RSS == WorkingSetSize and VSM == PagefileUsage. + # Underlying C function returns fields of PROCESS_MEMORY_COUNTERS + # struct. + t = self._get_raw_meminfo() + rss = t[2] # wset + vms = t[7] # pagefile + return pmem(*(rss, vms, ) + t) + + @wrap_exceptions + def memory_full_info(self): + basic_mem = self.memory_info() + uss = cext.proc_memory_uss(self.pid) + uss *= getpagesize() + return pfullmem(*basic_mem + (uss, )) + + def memory_maps(self): + try: + raw = cext.proc_memory_maps(self.pid) + except OSError as err: + # XXX - can't use wrap_exceptions decorator as we're + # returning a generator; probably needs refactoring. + raise convert_oserror(err, self.pid, self._name) + else: + for addr, perm, path, rss in raw: + path = convert_dos_path(path) + if not PY3: + assert isinstance(path, unicode), type(path) + path = py2_strencode(path) + addr = hex(addr) + yield (addr, perm, path, rss) + + @wrap_exceptions + def kill(self): + return cext.proc_kill(self.pid) + + @wrap_exceptions + def send_signal(self, sig): + os.kill(self.pid, sig) + + @wrap_exceptions + def wait(self, timeout=None): + if timeout is None: + cext_timeout = cext.INFINITE + else: + # WaitForSingleObject() expects time in milliseconds. + cext_timeout = int(timeout * 1000) + + timer = getattr(time, 'monotonic', time.time) + stop_at = timer() + timeout if timeout is not None else None + + try: + # Exit code is supposed to come from GetExitCodeProcess(). + # May also be None if OpenProcess() failed with + # ERROR_INVALID_PARAMETER, meaning PID is already gone. + exit_code = cext.proc_wait(self.pid, cext_timeout) + except cext.TimeoutExpired: + # WaitForSingleObject() returned WAIT_TIMEOUT. Just raise. + raise TimeoutExpired(timeout, self.pid, self._name) + except cext.TimeoutAbandoned: + # WaitForSingleObject() returned WAIT_ABANDONED, see: + # https://github.com/giampaolo/psutil/issues/1224 + # We'll just rely on the internal polling and return None + # when the PID disappears. Subprocess module does the same + # (return None): + # https://github.com/python/cpython/blob/ + # be50a7b627d0aa37e08fa8e2d5568891f19903ce/ + # Lib/subprocess.py#L1193-L1194 + exit_code = None + + # At this point WaitForSingleObject() returned WAIT_OBJECT_0, + # meaning the process is gone. Stupidly there are cases where + # its PID may still stick around so we do a further internal + # polling. + delay = 0.0001 + while True: + if not pid_exists(self.pid): + return exit_code + if stop_at and timer() >= stop_at: + raise TimeoutExpired(timeout, pid=self.pid, name=self._name) + time.sleep(delay) + delay = min(delay * 2, 0.04) # incremental delay + + @wrap_exceptions + def username(self): + if self.pid in (0, 4): + return 'NT AUTHORITY\\SYSTEM' + domain, user = cext.proc_username(self.pid) + return py2_strencode(domain) + '\\' + py2_strencode(user) + + @wrap_exceptions + def create_time(self): + # special case for kernel process PIDs; return system boot time + if self.pid in (0, 4): + return boot_time() + try: + return cext.proc_create_time(self.pid) + except OSError as err: + if is_permission_err(err): + return self.oneshot_info()[pinfo_map['create_time']] + raise + + @wrap_exceptions + def num_threads(self): + return self.oneshot_info()[pinfo_map['num_threads']] + + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + return retlist + + @wrap_exceptions + def cpu_times(self): + try: + user, system = cext.proc_cpu_times(self.pid) + except OSError as err: + if not is_permission_err(err): + raise + info = self.oneshot_info() + user = info[pinfo_map['user_time']] + system = info[pinfo_map['kernel_time']] + # Children user/system times are not retrievable (set to 0). + return _common.pcputimes(user, system, 0.0, 0.0) + + @wrap_exceptions + def suspend(self): + cext.proc_suspend_or_resume(self.pid, True) + + @wrap_exceptions + def resume(self): + cext.proc_suspend_or_resume(self.pid, False) + + @wrap_exceptions + @retry_error_partial_copy + def cwd(self): + if self.pid in (0, 4): + raise AccessDenied(self.pid, self._name) + # return a normalized pathname since the native C function appends + # "\\" at the and of the path + path = cext.proc_cwd(self.pid) + return py2_strencode(os.path.normpath(path)) + + @wrap_exceptions + def open_files(self): + if self.pid in (0, 4): + return [] + ret = set() + # Filenames come in in native format like: + # "\Device\HarddiskVolume1\Windows\systemew\file.txt" + # Convert the first part in the corresponding drive letter + # (e.g. "C:\") by using Windows's QueryDosDevice() + raw_file_names = cext.proc_open_files(self.pid) + for _file in raw_file_names: + _file = convert_dos_path(_file) + if isfile_strict(_file): + if not PY3: + _file = py2_strencode(_file) + ntuple = _common.popenfile(_file, -1) + ret.add(ntuple) + return list(ret) + + @wrap_exceptions + def connections(self, kind='inet'): + return net_connections(kind, _pid=self.pid) + + @wrap_exceptions + def nice_get(self): + value = cext.proc_priority_get(self.pid) + if enum is not None: + value = Priority(value) + return value + + @wrap_exceptions + def nice_set(self, value): + return cext.proc_priority_set(self.pid, value) + + # available on Windows >= Vista + if HAS_PROC_IO_PRIORITY: + @wrap_exceptions + def ionice_get(self): + ret = cext.proc_io_priority_get(self.pid) + if enum is not None: + ret = IOPriority(ret) + return ret + + @wrap_exceptions + def ionice_set(self, ioclass, value): + if value: + raise TypeError("value argument not accepted on Windows") + if ioclass not in (IOPRIO_VERYLOW, IOPRIO_LOW, IOPRIO_NORMAL, + IOPRIO_HIGH): + raise ValueError("%s is not a valid priority" % ioclass) + cext.proc_io_priority_set(self.pid, ioclass) + + @wrap_exceptions + def io_counters(self): + try: + ret = cext.proc_io_counters(self.pid) + except OSError as err: + if not is_permission_err(err): + raise + info = self.oneshot_info() + ret = ( + info[pinfo_map['io_rcount']], + info[pinfo_map['io_wcount']], + info[pinfo_map['io_rbytes']], + info[pinfo_map['io_wbytes']], + info[pinfo_map['io_count_others']], + info[pinfo_map['io_bytes_others']], + ) + return pio(*ret) + + @wrap_exceptions + def status(self): + suspended = cext.proc_is_suspended(self.pid) + if suspended: + return _common.STATUS_STOPPED + else: + return _common.STATUS_RUNNING + + @wrap_exceptions + def cpu_affinity_get(self): + def from_bitmask(x): + return [i for i in xrange(64) if (1 << i) & x] + bitmask = cext.proc_cpu_affinity_get(self.pid) + return from_bitmask(bitmask) + + @wrap_exceptions + def cpu_affinity_set(self, value): + def to_bitmask(l): + if not l: + raise ValueError("invalid argument %r" % l) + out = 0 + for b in l: + out |= 2 ** b + return out + + # SetProcessAffinityMask() states that ERROR_INVALID_PARAMETER + # is returned for an invalid CPU but this seems not to be true, + # therefore we check CPUs validy beforehand. + allcpus = list(range(len(per_cpu_times()))) + for cpu in value: + if cpu not in allcpus: + if not isinstance(cpu, (int, long)): + raise TypeError( + "invalid CPU %r; an integer is required" % cpu) + else: + raise ValueError("invalid CPU %r" % cpu) + + bitmask = to_bitmask(value) + cext.proc_cpu_affinity_set(self.pid, bitmask) + + @wrap_exceptions + def num_handles(self): + try: + return cext.proc_num_handles(self.pid) + except OSError as err: + if is_permission_err(err): + return self.oneshot_info()[pinfo_map['num_handles']] + raise + + @wrap_exceptions + def num_ctx_switches(self): + ctx_switches = self.oneshot_info()[pinfo_map['ctx_switches']] + # only voluntary ctx switches are supported + return _common.pctxsw(ctx_switches, 0) diff --git a/ddtrace/vendor/psutil/arch/aix/common.c b/ddtrace/vendor/psutil/arch/aix/common.c new file mode 100644 index 00000000000..6115a15db51 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/common.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include "common.h" + +/* psutil_kread() - read from kernel memory */ +int +psutil_kread( + int Kd, /* kernel memory file descriptor */ + KA_T addr, /* kernel memory address */ + char *buf, /* buffer to receive data */ + size_t len) { /* length to read */ + int br; + + if (lseek64(Kd, (off64_t)addr, L_SET) == (off64_t)-1) { + PyErr_SetFromErrno(PyExc_OSError); + return 1; + } + br = read(Kd, buf, len); + if (br == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return 1; + } + if (br != len) { + PyErr_SetString(PyExc_RuntimeError, + "size mismatch when reading kernel memory fd"); + return 1; + } + return 0; +} + +struct procentry64 * +psutil_read_process_table(int * num) { + size_t msz; + pid32_t pid = 0; + struct procentry64 *processes = (struct procentry64 *)NULL; + struct procentry64 *p; + int Np = 0; /* number of processes allocated in 'processes' */ + int np = 0; /* number of processes read into 'processes' */ + int i; /* number of processes read in current iteration */ + + msz = (size_t)(PROCSIZE * PROCINFO_INCR); + processes = (struct procentry64 *)malloc(msz); + if (!processes) { + PyErr_NoMemory(); + return NULL; + } + Np = PROCINFO_INCR; + p = processes; + while ((i = getprocs64(p, PROCSIZE, (struct fdsinfo64 *)NULL, 0, &pid, + PROCINFO_INCR)) + == PROCINFO_INCR) { + np += PROCINFO_INCR; + if (np >= Np) { + msz = (size_t)(PROCSIZE * (Np + PROCINFO_INCR)); + processes = (struct procentry64 *)realloc((char *)processes, msz); + if (!processes) { + PyErr_NoMemory(); + return NULL; + } + Np += PROCINFO_INCR; + } + p = (struct procentry64 *)((char *)processes + (np * PROCSIZE)); + } + + /* add the number of processes read in the last iteration */ + if (i > 0) + np += i; + + *num = np; + return processes; +} \ No newline at end of file diff --git a/ddtrace/vendor/psutil/arch/aix/common.h b/ddtrace/vendor/psutil/arch/aix/common.h new file mode 100644 index 00000000000..b677d8c29ee --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/common.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef __PSUTIL_AIX_COMMON_H__ +#define __PSUTIL_AIX_COMMON_H__ + +#include + +#define PROCINFO_INCR (256) +#define PROCSIZE (sizeof(struct procentry64)) +#define FDSINFOSIZE (sizeof(struct fdsinfo64)) +#define KMEM "/dev/kmem" + +typedef u_longlong_t KA_T; + +/* psutil_kread() - read from kernel memory */ +int psutil_kread(int Kd, /* kernel memory file descriptor */ + KA_T addr, /* kernel memory address */ + char *buf, /* buffer to receive data */ + size_t len); /* length to read */ + +struct procentry64 * +psutil_read_process_table( + int * num /* out - number of processes read */ +); + +#endif /* __PSUTIL_AIX_COMMON_H__ */ diff --git a/ddtrace/vendor/psutil/arch/aix/ifaddrs.c b/ddtrace/vendor/psutil/arch/aix/ifaddrs.c new file mode 100644 index 00000000000..1a819365ab8 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/ifaddrs.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/*! Based on code from + https://lists.samba.org/archive/samba-technical/2009-February/063079.html +!*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ifaddrs.h" + +#define MAX(x,y) ((x)>(y)?(x):(y)) +#define SIZE(p) MAX((p).sa_len,sizeof(p)) + + +static struct sockaddr * +sa_dup(struct sockaddr *sa1) +{ + struct sockaddr *sa2; + size_t sz = sa1->sa_len; + sa2 = (struct sockaddr *) calloc(1, sz); + if (sa2 == NULL) + return NULL; + memcpy(sa2, sa1, sz); + return sa2; +} + + +void freeifaddrs(struct ifaddrs *ifp) +{ + if (NULL == ifp) return; + free(ifp->ifa_name); + free(ifp->ifa_addr); + free(ifp->ifa_netmask); + free(ifp->ifa_dstaddr); + freeifaddrs(ifp->ifa_next); + free(ifp); +} + + +int getifaddrs(struct ifaddrs **ifap) +{ + int sd, ifsize; + char *ccp, *ecp; + struct ifconf ifc; + struct ifreq *ifr; + struct ifaddrs *cifa = NULL; /* current */ + struct ifaddrs *pifa = NULL; /* previous */ + const size_t IFREQSZ = sizeof(struct ifreq); + int fam; + + *ifap = NULL; + + sd = socket(AF_INET, SOCK_DGRAM, 0); + if (sd == -1) + goto error; + + /* find how much memory to allocate for the SIOCGIFCONF call */ + if (ioctl(sd, SIOCGSIZIFCONF, (caddr_t)&ifsize) < 0) + goto error; + + ifc.ifc_req = (struct ifreq *) calloc(1, ifsize); + if (ifc.ifc_req == NULL) + goto error; + ifc.ifc_len = ifsize; + + if (ioctl(sd, SIOCGIFCONF, &ifc) < 0) + goto error; + + ccp = (char *)ifc.ifc_req; + ecp = ccp + ifsize; + + while (ccp < ecp) { + + ifr = (struct ifreq *) ccp; + ifsize = sizeof(ifr->ifr_name) + SIZE(ifr->ifr_addr); + fam = ifr->ifr_addr.sa_family; + + if (fam == AF_INET || fam == AF_INET6) { + cifa = (struct ifaddrs *) calloc(1, sizeof(struct ifaddrs)); + if (cifa == NULL) + goto error; + cifa->ifa_next = NULL; + + if (pifa == NULL) *ifap = cifa; /* first one */ + else pifa->ifa_next = cifa; + + cifa->ifa_name = strdup(ifr->ifr_name); + if (cifa->ifa_name == NULL) + goto error; + cifa->ifa_flags = 0; + cifa->ifa_dstaddr = NULL; + + cifa->ifa_addr = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_addr == NULL) + goto error; + + if (fam == AF_INET) { + if (ioctl(sd, SIOCGIFNETMASK, ifr, IFREQSZ) < 0) + goto error; + cifa->ifa_netmask = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_netmask == NULL) + goto error; + } + + if (0 == ioctl(sd, SIOCGIFFLAGS, ifr)) /* optional */ + cifa->ifa_flags = ifr->ifr_flags; + + if (fam == AF_INET) { + if (ioctl(sd, SIOCGIFDSTADDR, ifr, IFREQSZ) < 0) { + if (0 == ioctl(sd, SIOCGIFBRDADDR, ifr, IFREQSZ)) { + cifa->ifa_dstaddr = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_dstaddr == NULL) + goto error; + } + } + else { + cifa->ifa_dstaddr = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_dstaddr == NULL) + goto error; + } + } + pifa = cifa; + } + + ccp += ifsize; + } + free(ifc.ifc_req); + close(sd); + return 0; +error: + if (ifc.ifc_req != NULL) + free(ifc.ifc_req); + if (sd != -1) + close(sd); + freeifaddrs(*ifap); + return (-1); +} \ No newline at end of file diff --git a/ddtrace/vendor/psutil/arch/aix/ifaddrs.h b/ddtrace/vendor/psutil/arch/aix/ifaddrs.h new file mode 100644 index 00000000000..3920c1ccca7 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/ifaddrs.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/*! Based on code from + https://lists.samba.org/archive/samba-technical/2009-February/063079.html +!*/ + + +#ifndef GENERIC_AIX_IFADDRS_H +#define GENERIC_AIX_IFADDRS_H + +#include +#include + +#undef ifa_dstaddr +#undef ifa_broadaddr +#define ifa_broadaddr ifa_dstaddr + +struct ifaddrs { + struct ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + struct sockaddr *ifa_addr; + struct sockaddr *ifa_netmask; + struct sockaddr *ifa_dstaddr; +}; + +extern int getifaddrs(struct ifaddrs **); +extern void freeifaddrs(struct ifaddrs *); + +#endif \ No newline at end of file diff --git a/ddtrace/vendor/psutil/arch/aix/net_connections.c b/ddtrace/vendor/psutil/arch/aix/net_connections.c new file mode 100644 index 00000000000..69b43892012 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/net_connections.c @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* Baded on code from lsof: + * http://www.ibm.com/developerworks/aix/library/au-lsof.html + * - dialects/aix/dproc.c:gather_proc_info + * - lib/prfp.c:process_file + * - dialects/aix/dsock.c:process_socket + * - dialects/aix/dproc.c:get_kernel_access +*/ + +#include +#include +#include +#define _KERNEL +#include +#undef _KERNEL +#include +#include +#include +#include +#include +#include + +#include "../../_psutil_common.h" +#include "net_kernel_structs.h" +#include "net_connections.h" +#include "common.h" + +#define NO_SOCKET (PyObject *)(-1) + +static int +read_unp_addr( + int Kd, + KA_T unp_addr, + char *buf, + size_t buflen +) { + struct sockaddr_un *ua = (struct sockaddr_un *)NULL; + struct sockaddr_un un; + struct mbuf64 mb; + int uo; + + if (psutil_kread(Kd, unp_addr, (char *)&mb, sizeof(mb))) { + return 1; + } + + uo = (int)(mb.m_hdr.mh_data - unp_addr); + if ((uo + sizeof(struct sockaddr)) <= sizeof(mb)) + ua = (struct sockaddr_un *)((char *)&mb + uo); + else { + if (psutil_kread(Kd, (KA_T)mb.m_hdr.mh_data, + (char *)&un, sizeof(un))) { + return 1; + } + ua = &un; + } + if (ua && ua->sun_path[0]) { + if (mb.m_len > sizeof(struct sockaddr_un)) + mb.m_len = sizeof(struct sockaddr_un); + *((char *)ua + mb.m_len - 1) = '\0'; + snprintf(buf, buflen, "%s", ua->sun_path); + } + return 0; +} + +static PyObject * +process_file(int Kd, pid32_t pid, int fd, KA_T fp) { + struct file64 f; + struct socket64 s; + struct protosw64 p; + struct domain d; + struct inpcb64 inp; + int fam; + struct tcpcb64 t; + int state = PSUTIL_CONN_NONE; + unsigned char *laddr = (unsigned char *)NULL; + unsigned char *raddr = (unsigned char *)NULL; + int rport, lport; + char laddr_str[INET6_ADDRSTRLEN]; + char raddr_str[INET6_ADDRSTRLEN]; + struct unpcb64 unp; + char unix_laddr_str[PATH_MAX] = { 0 }; + char unix_raddr_str[PATH_MAX] = { 0 }; + + /* Read file structure */ + if (psutil_kread(Kd, fp, (char *)&f, sizeof(f))) { + return NULL; + } + if (!f.f_count || f.f_type != DTYPE_SOCKET) { + return NO_SOCKET; + } + + if (psutil_kread(Kd, (KA_T) f.f_data, (char *) &s, sizeof(s))) { + return NULL; + } + + if (!s.so_type) { + return NO_SOCKET; + } + + if (!s.so_proto) { + PyErr_SetString(PyExc_RuntimeError, "invalid socket protocol handle"); + return NULL; + } + if (psutil_kread(Kd, (KA_T)s.so_proto, (char *)&p, sizeof(p))) { + return NULL; + } + + if (!p.pr_domain) { + PyErr_SetString(PyExc_RuntimeError, "invalid socket protocol domain"); + return NULL; + } + if (psutil_kread(Kd, (KA_T)p.pr_domain, (char *)&d, sizeof(d))) { + return NULL; + } + + fam = d.dom_family; + if (fam == AF_INET || fam == AF_INET6) { + /* Read protocol control block */ + if (!s.so_pcb) { + PyErr_SetString(PyExc_RuntimeError, "invalid socket PCB"); + return NULL; + } + if (psutil_kread(Kd, (KA_T) s.so_pcb, (char *) &inp, sizeof(inp))) { + return NULL; + } + + if (p.pr_protocol == IPPROTO_TCP) { + /* If this is a TCP socket, read its control block */ + if (inp.inp_ppcb + && !psutil_kread(Kd, (KA_T)inp.inp_ppcb, + (char *)&t, sizeof(t))) + state = t.t_state; + } + + if (fam == AF_INET6) { + laddr = (unsigned char *)&inp.inp_laddr6; + if (!IN6_IS_ADDR_UNSPECIFIED(&inp.inp_faddr6)) { + raddr = (unsigned char *)&inp.inp_faddr6; + rport = (int)ntohs(inp.inp_fport); + } + } + if (fam == AF_INET) { + laddr = (unsigned char *)&inp.inp_laddr; + if (inp.inp_faddr.s_addr != INADDR_ANY || inp.inp_fport != 0) { + raddr = (unsigned char *)&inp.inp_faddr; + rport = (int)ntohs(inp.inp_fport); + } + } + lport = (int)ntohs(inp.inp_lport); + + inet_ntop(fam, laddr, laddr_str, sizeof(laddr_str)); + + if (raddr != NULL) { + inet_ntop(fam, raddr, raddr_str, sizeof(raddr_str)); + return Py_BuildValue("(iii(si)(si)ii)", fd, fam, + s.so_type, laddr_str, lport, raddr_str, + rport, state, pid); + } + else { + return Py_BuildValue("(iii(si)()ii)", fd, fam, + s.so_type, laddr_str, lport, state, + pid); + } + } + + + if (fam == AF_UNIX) { + if (psutil_kread(Kd, (KA_T) s.so_pcb, (char *)&unp, sizeof(unp))) { + return NULL; + } + if ((KA_T) f.f_data != (KA_T) unp.unp_socket) { + PyErr_SetString(PyExc_RuntimeError, "unp_socket mismatch"); + return NULL; + } + + if (unp.unp_addr) { + if (read_unp_addr(Kd, unp.unp_addr, unix_laddr_str, + sizeof(unix_laddr_str))) { + return NULL; + } + } + + if (unp.unp_conn) { + if (psutil_kread(Kd, (KA_T) unp.unp_conn, (char *)&unp, + sizeof(unp))) { + return NULL; + } + if (read_unp_addr(Kd, unp.unp_addr, unix_raddr_str, + sizeof(unix_raddr_str))) { + return NULL; + } + } + + return Py_BuildValue("(iiissii)", fd, d.dom_family, + s.so_type, unix_laddr_str, unix_raddr_str, PSUTIL_CONN_NONE, + pid); + } + return NO_SOCKET; +} + +PyObject * +psutil_net_connections(PyObject *self, PyObject *args) { + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + KA_T fp; + int Kd = -1; + int i, np; + struct procentry64 *p; + struct fdsinfo64 *fds = (struct fdsinfo64 *)NULL; + pid32_t requested_pid; + pid32_t pid; + struct procentry64 *processes = (struct procentry64 *)NULL; + /* the process table */ + + if (py_retlist == NULL) + goto error; + if (! PyArg_ParseTuple(args, "i", &requested_pid)) + goto error; + + Kd = open(KMEM, O_RDONLY, 0); + if (Kd < 0) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, KMEM); + goto error; + } + + processes = psutil_read_process_table(&np); + if (!processes) + goto error; + + /* Loop through processes */ + for (p = processes; np > 0; np--, p++) { + pid = p->pi_pid; + if (requested_pid != -1 && requested_pid != pid) + continue; + if (p->pi_state == 0 || p->pi_state == SZOMB) + continue; + + if (!fds) { + fds = (struct fdsinfo64 *)malloc((size_t)FDSINFOSIZE); + if (!fds) { + PyErr_NoMemory(); + goto error; + } + } + if (getprocs64((struct procentry64 *)NULL, PROCSIZE, fds, FDSINFOSIZE, + &pid, 1) + != 1) + continue; + + /* loop over file descriptors */ + for (i = 0; i < p->pi_maxofile; i++) { + fp = (KA_T)fds->pi_ufd[i].fp; + if (fp) { + py_tuple = process_file(Kd, p->pi_pid, i, fp); + if (py_tuple == NULL) + goto error; + if (py_tuple != NO_SOCKET) { + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + } + } + } + close(Kd); + free(processes); + if (fds != NULL) + free(fds); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (Kd > 0) + close(Kd); + if (processes != NULL) + free(processes); + if (fds != NULL) + free(fds); + return NULL; +} diff --git a/ddtrace/vendor/psutil/arch/aix/net_connections.h b/ddtrace/vendor/psutil/arch/aix/net_connections.h new file mode 100644 index 00000000000..222bcaf354e --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/net_connections.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef __NET_CONNECTIONS_H__ +#define __NET_CONNECTIONS_H__ + +#include + +PyObject* psutil_net_connections(PyObject *self, PyObject *args); + +#endif /* __NET_CONNECTIONS_H__ */ \ No newline at end of file diff --git a/ddtrace/vendor/psutil/arch/aix/net_kernel_structs.h b/ddtrace/vendor/psutil/arch/aix/net_kernel_structs.h new file mode 100644 index 00000000000..4e7a088c143 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/aix/net_kernel_structs.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* The kernel is always 64 bit but Python is usually compiled as a 32 bit + * process. We're reading the kernel memory to get the network connections, + * so we need the structs we read to be defined with 64 bit "pointers". + * Here are the partial definitions of the structs we use, taken from the + * header files, with data type sizes converted to their 64 bit counterparts, + * and unused data truncated. */ + +#ifdef __64BIT__ +/* In case we're in a 64 bit process after all */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define file64 file +#define socket64 socket +#define protosw64 protosw +#define inpcb64 inpcb +#define tcpcb64 tcpcb +#define unpcb64 unpcb +#define mbuf64 mbuf +#else /* __64BIT__ */ + struct file64 { + int f_flag; + int f_count; + int f_options; + int f_type; + u_longlong_t f_data; + }; + + struct socket64 { + short so_type; /* generic type, see socket.h */ + short so_options; /* from socket call, see socket.h */ + ushort so_linger; /* time to linger while closing */ + short so_state; /* internal state flags SS_*, below */ + u_longlong_t so_pcb; /* protocol control block */ + u_longlong_t so_proto; /* protocol handle */ + }; + + struct protosw64 { + short pr_type; /* socket type used for */ + u_longlong_t pr_domain; /* domain protocol a member of */ + short pr_protocol; /* protocol number */ + short pr_flags; /* see below */ + }; + + struct inpcb64 { + u_longlong_t inp_next,inp_prev; + /* pointers to other pcb's */ + u_longlong_t inp_head; /* pointer back to chain of inpcb's + for this protocol */ + u_int32_t inp_iflowinfo; /* input flow label */ + u_short inp_fport; /* foreign port */ + u_int16_t inp_fatype; /* foreign address type */ + union in_addr_6 inp_faddr_6; /* foreign host table entry */ + u_int32_t inp_oflowinfo; /* output flow label */ + u_short inp_lport; /* local port */ + u_int16_t inp_latype; /* local address type */ + union in_addr_6 inp_laddr_6; /* local host table entry */ + u_longlong_t inp_socket; /* back pointer to socket */ + u_longlong_t inp_ppcb; /* pointer to per-protocol pcb */ + u_longlong_t space_rt; + struct sockaddr_in6 spare_dst; + u_longlong_t inp_ifa; /* interface address to use */ + int inp_flags; /* generic IP/datagram flags */ +}; + +struct tcpcb64 { + u_longlong_t seg__next; + u_longlong_t seg__prev; + short t_state; /* state of this connection */ +}; + +struct unpcb64 { + u_longlong_t unp_socket; /* pointer back to socket */ + u_longlong_t unp_vnode; /* if associated with file */ + ino_t unp_vno; /* fake vnode number */ + u_longlong_t unp_conn; /* control block of connected socket */ + u_longlong_t unp_refs; /* referencing socket linked list */ + u_longlong_t unp_nextref; /* link in unp_refs list */ + u_longlong_t unp_addr; /* bound address of socket */ +}; + +struct m_hdr64 +{ + u_longlong_t mh_next; /* next buffer in chain */ + u_longlong_t mh_nextpkt; /* next chain in queue/record */ + long mh_len; /* amount of data in this mbuf */ + u_longlong_t mh_data; /* location of data */ +}; + +struct mbuf64 +{ + struct m_hdr64 m_hdr; +}; + +#define m_len m_hdr.mh_len + +#endif /* __64BIT__ */ \ No newline at end of file diff --git a/ddtrace/vendor/psutil/arch/freebsd/proc_socks.c b/ddtrace/vendor/psutil/arch/freebsd/proc_socks.c new file mode 100644 index 00000000000..a458a01e531 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/freebsd/proc_socks.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Retrieves per-process open socket connections. + */ + +#include +#include +#include // for struct xsocket +#include +#include +#include // for xinpcb struct +#include +#include // for struct xtcpcb +#include // for inet_ntop() +#include + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + + +// The tcplist fetching and walking is borrowed from netstat/inet.c. +static char * +psutil_fetch_tcplist(void) { + char *buf; + size_t len; + + for (;;) { + if (sysctlbyname("net.inet.tcp.pcblist", NULL, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + buf = malloc(len); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + if (sysctlbyname("net.inet.tcp.pcblist", buf, &len, NULL, 0) < 0) { + free(buf); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + return buf; + } +} + + +static int +psutil_sockaddr_port(int family, struct sockaddr_storage *ss) { + struct sockaddr_in6 *sin6; + struct sockaddr_in *sin; + + if (family == AF_INET) { + sin = (struct sockaddr_in *)ss; + return (sin->sin_port); + } + else { + sin6 = (struct sockaddr_in6 *)ss; + return (sin6->sin6_port); + } +} + + +static void * +psutil_sockaddr_addr(int family, struct sockaddr_storage *ss) { + struct sockaddr_in6 *sin6; + struct sockaddr_in *sin; + + if (family == AF_INET) { + sin = (struct sockaddr_in *)ss; + return (&sin->sin_addr); + } + else { + sin6 = (struct sockaddr_in6 *)ss; + return (&sin6->sin6_addr); + } +} + + +static socklen_t +psutil_sockaddr_addrlen(int family) { + if (family == AF_INET) + return (sizeof(struct in_addr)); + else + return (sizeof(struct in6_addr)); +} + + +static int +psutil_sockaddr_matches(int family, int port, void *pcb_addr, + struct sockaddr_storage *ss) { + if (psutil_sockaddr_port(family, ss) != port) + return (0); + return (memcmp(psutil_sockaddr_addr(family, ss), pcb_addr, + psutil_sockaddr_addrlen(family)) == 0); +} + + +#if __FreeBSD_version >= 1200026 +static struct xtcpcb * +psutil_search_tcplist(char *buf, struct kinfo_file *kif) { + struct xtcpcb *tp; + struct xinpcb *inp; +#else +static struct tcpcb * +psutil_search_tcplist(char *buf, struct kinfo_file *kif) { + struct tcpcb *tp; + struct inpcb *inp; +#endif + struct xinpgen *xig, *oxig; + struct xsocket *so; + + oxig = xig = (struct xinpgen *)buf; + for (xig = (struct xinpgen *)((char *)xig + xig->xig_len); + xig->xig_len > sizeof(struct xinpgen); + xig = (struct xinpgen *)((char *)xig + xig->xig_len)) { + +#if __FreeBSD_version >= 1200026 + tp = (struct xtcpcb *)xig; + inp = &tp->xt_inp; + so = &inp->xi_socket; +#else + tp = &((struct xtcpcb *)xig)->xt_tp; + inp = &((struct xtcpcb *)xig)->xt_inp; + so = &((struct xtcpcb *)xig)->xt_socket; +#endif + + if (so->so_type != kif->kf_sock_type || + so->xso_family != kif->kf_sock_domain || + so->xso_protocol != kif->kf_sock_protocol) + continue; + + if (kif->kf_sock_domain == AF_INET) { + if (!psutil_sockaddr_matches( + AF_INET, inp->inp_lport, &inp->inp_laddr, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_local)) +#else + &kif->kf_un.kf_sock.kf_sa_local)) +#endif + continue; + if (!psutil_sockaddr_matches( + AF_INET, inp->inp_fport, &inp->inp_faddr, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_peer)) +#else + &kif->kf_un.kf_sock.kf_sa_peer)) +#endif + continue; + } else { + if (!psutil_sockaddr_matches( + AF_INET6, inp->inp_lport, &inp->in6p_laddr, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_local)) +#else + &kif->kf_un.kf_sock.kf_sa_local)) +#endif + continue; + if (!psutil_sockaddr_matches( + AF_INET6, inp->inp_fport, &inp->in6p_faddr, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_peer)) +#else + &kif->kf_un.kf_sock.kf_sa_peer)) +#endif + continue; + } + + return (tp); + } + return NULL; +} + + + +PyObject * +psutil_proc_connections(PyObject *self, PyObject *args) { + // Return connections opened by process. + long pid; + int i; + int cnt; + struct kinfo_file *freep = NULL; + struct kinfo_file *kif; + char *tcplist = NULL; +#if __FreeBSD_version >= 1200026 + struct xtcpcb *tcp; +#else + struct tcpcb *tcp; +#endif + + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + PyObject *py_af_filter = NULL; + PyObject *py_type_filter = NULL; + PyObject *py_family = NULL; + PyObject *py_type = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "lOO", &pid, &py_af_filter, &py_type_filter)) + goto error; + if (!PySequence_Check(py_af_filter) || !PySequence_Check(py_type_filter)) { + PyErr_SetString(PyExc_TypeError, "arg 2 or 3 is not a sequence"); + goto error; + } + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + goto error; + } + + tcplist = psutil_fetch_tcplist(); + if (tcplist == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < cnt; i++) { + int lport, rport, state; + char lip[200], rip[200]; + char path[PATH_MAX]; + int inseq; + py_tuple = NULL; + py_laddr = NULL; + py_raddr = NULL; + + kif = &freep[i]; + if (kif->kf_type == KF_TYPE_SOCKET) { + // apply filters + py_family = PyLong_FromLong((long)kif->kf_sock_domain); + inseq = PySequence_Contains(py_af_filter, py_family); + Py_DECREF(py_family); + if (inseq == 0) + continue; + py_type = PyLong_FromLong((long)kif->kf_sock_type); + inseq = PySequence_Contains(py_type_filter, py_type); + Py_DECREF(py_type); + if (inseq == 0) + continue; + // IPv4 / IPv6 socket + if ((kif->kf_sock_domain == AF_INET) || + (kif->kf_sock_domain == AF_INET6)) { + // fill status + state = PSUTIL_CONN_NONE; + if (kif->kf_sock_type == SOCK_STREAM) { + tcp = psutil_search_tcplist(tcplist, kif); + if (tcp != NULL) + state = (int)tcp->t_state; + } + + // build addr and port + inet_ntop( + kif->kf_sock_domain, + psutil_sockaddr_addr(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_local), +#else + &kif->kf_un.kf_sock.kf_sa_local), +#endif + lip, + sizeof(lip)); + inet_ntop( + kif->kf_sock_domain, + psutil_sockaddr_addr(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_peer), +#else + &kif->kf_un.kf_sock.kf_sa_peer), +#endif + rip, + sizeof(rip)); + lport = htons(psutil_sockaddr_port(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_local)); +#else + &kif->kf_un.kf_sock.kf_sa_local)); +#endif + rport = htons(psutil_sockaddr_port(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 + &kif->kf_sa_peer)); +#else + &kif->kf_un.kf_sock.kf_sa_peer)); +#endif + + // construct python tuple/list + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", rip, rport); + else + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + py_tuple = Py_BuildValue( + "(iiiNNi)", + kif->kf_fd, + kif->kf_sock_domain, + kif->kf_sock_type, + py_laddr, + py_raddr, + state + ); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + // UNIX socket. + // Note: remote path cannot be determined. + else if (kif->kf_sock_domain == AF_UNIX) { + struct sockaddr_un *sun; + +#if __FreeBSD_version < 1200031 + sun = (struct sockaddr_un *)&kif->kf_sa_local; +#else + sun = (struct sockaddr_un *)&kif->kf_un.kf_sock.kf_sa_local; +#endif + snprintf( + path, sizeof(path), "%.*s", + (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), + sun->sun_path); + + py_laddr = PyUnicode_DecodeFSDefault(path); + if (! py_laddr) + goto error; + + py_tuple = Py_BuildValue( + "(iiiOsi)", + kif->kf_fd, + kif->kf_sock_domain, + kif->kf_sock_type, + py_laddr, + "", // raddr can't be determined + PSUTIL_CONN_NONE + ); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + Py_DECREF(py_laddr); + } + } + } + free(freep); + free(tcplist); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + Py_DECREF(py_retlist); + if (freep != NULL) + free(freep); + if (tcplist != NULL) + free(tcplist); + return NULL; +} diff --git a/ddtrace/vendor/psutil/arch/freebsd/proc_socks.h b/ddtrace/vendor/psutil/arch/freebsd/proc_socks.h new file mode 100644 index 00000000000..a7996b10749 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/freebsd/proc_socks.h @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +PyObject* psutil_proc_connections(PyObject* self, PyObject* args); diff --git a/ddtrace/vendor/psutil/arch/freebsd/specific.c b/ddtrace/vendor/psutil/arch/freebsd/specific.c new file mode 100644 index 00000000000..26b802422db --- /dev/null +++ b/ddtrace/vendor/psutil/arch/freebsd/specific.c @@ -0,0 +1,1115 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Helper functions specific to FreeBSD. + * Used by _psutil_bsd module methods. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // needed for vmtotal struct +#include // for swap mem +#include // process open files, shared libs (kinfo_getvmmap), cwd +#include + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + +#define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) +#define PSUTIL_BT2MSEC(bt) (bt.sec * 1000 + (((uint64_t) 1000000000 * (uint32_t) \ + (bt.frac >> 32) ) >> 32 ) / 1000000) +#define DECIKELVIN_2_CELCIUS(t) (t - 2731) / 10 +#ifndef _PATH_DEVNULL +#define _PATH_DEVNULL "/dev/null" +#endif + + +// ============================================================================ +// Utility functions +// ============================================================================ + + +int +psutil_kinfo_proc(const pid_t pid, struct kinfo_proc *proc) { + // Fills a kinfo_proc struct based on process pid. + int mib[4]; + size_t size; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = pid; + + size = sizeof(struct kinfo_proc); + if (sysctl((int *)mib, 4, proc, &size, NULL, 0) == -1) { + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_PID)"); + return -1; + } + + // sysctl stores 0 in the size if we can't find the process information. + if (size == 0) { + NoSuchProcess(""); + return -1; + } + return 0; +} + + +// remove spaces from string +static void psutil_remove_spaces(char *str) { + char *p1 = str; + char *p2 = str; + do + while (*p2 == ' ') + p2++; + while ((*p1++ = *p2++)); +} + + +// ============================================================================ +// APIS +// ============================================================================ + +int +psutil_get_proc_list(struct kinfo_proc **procList, size_t *procCount) { + // Returns a list of all BSD processes on the system. This routine + // allocates the list and puts it in *procList and a count of the + // number of entries in *procCount. You are responsible for freeing + // this list (use "free" from System framework). + // On success, the function returns 0. + // On error, the function returns a BSD errno value. + int err; + struct kinfo_proc *result; + int done; + int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_PROC, 0 }; + size_t length; + + assert( procList != NULL); + assert(*procList == NULL); + assert(procCount != NULL); + + *procCount = 0; + + /* + * We start by calling sysctl with result == NULL and length == 0. + * That will succeed, and set length to the appropriate length. + * We then allocate a buffer of that size and call sysctl again + * with that buffer. If that succeeds, we're done. If that fails + * with ENOMEM, we have to throw away our buffer and loop. Note + * that the loop causes use to call sysctl with NULL again; this + * is necessary because the ENOMEM failure case sets length to + * the amount of data returned, not the amount of data that + * could have been returned. + */ + result = NULL; + done = 0; + do { + assert(result == NULL); + // Call sysctl with a NULL buffer. + length = 0; + err = sysctl((int *)name, (sizeof(name) / sizeof(*name)) - 1, + NULL, &length, NULL, 0); + if (err == -1) + err = errno; + + // Allocate an appropriately sized buffer based on the results + // from the previous call. + if (err == 0) { + result = malloc(length); + if (result == NULL) + err = ENOMEM; + } + + // Call sysctl again with the new buffer. If we get an ENOMEM + // error, toss away our buffer and start again. + if (err == 0) { + err = sysctl((int *) name, (sizeof(name) / sizeof(*name)) - 1, + result, &length, NULL, 0); + if (err == -1) + err = errno; + if (err == 0) { + done = 1; + } + else if (err == ENOMEM) { + assert(result != NULL); + free(result); + result = NULL; + err = 0; + } + } + } while (err == 0 && ! done); + + // Clean up and establish post conditions. + if (err != 0 && result != NULL) { + free(result); + result = NULL; + } + + *procList = result; + *procCount = length / sizeof(struct kinfo_proc); + + assert((err == 0) == (*procList != NULL)); + return err; +} + + +/* + * XXX no longer used; it probably makese sense to remove it. + * Borrowed from psi Python System Information project + * + * Get command arguments and environment variables. + * + * Based on code from ps. + * + * Returns: + * 0 for success; + * -1 for failure (Exception raised); + * 1 for insufficient privileges. + */ +static char +*psutil_get_cmd_args(long pid, size_t *argsize) { + int mib[4]; + int argmax; + size_t size = sizeof(argmax); + char *procargs = NULL; + + // Get the maximum process arguments size. + mib[0] = CTL_KERN; + mib[1] = KERN_ARGMAX; + + size = sizeof(argmax); + if (sysctl(mib, 2, &argmax, &size, NULL, 0) == -1) + return NULL; + + // Allocate space for the arguments. + procargs = (char *)malloc(argmax); + if (procargs == NULL) { + PyErr_NoMemory(); + return NULL; + } + + // Make a sysctl() call to get the raw argument space of the process. + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_ARGS; + mib[3] = pid; + + size = argmax; + if (sysctl(mib, 4, procargs, &size, NULL, 0) == -1) { + free(procargs); + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_ARGS)"); + return NULL; + } + + // return string and set the length of arguments + *argsize = size; + return procargs; +} + + +// returns the command line as a python list object +PyObject * +psutil_get_cmdline(long pid) { + char *argstr = NULL; + size_t pos = 0; + size_t argsize = 0; + PyObject *py_retlist = Py_BuildValue("[]"); + PyObject *py_arg = NULL; + + if (pid < 0) + return py_retlist; + argstr = psutil_get_cmd_args(pid, &argsize); + if (argstr == NULL) + goto error; + + // args are returned as a flattened string with \0 separators between + // arguments add each string to the list then step forward to the next + // separator + if (argsize > 0) { + while (pos < argsize) { + py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); + if (!py_arg) + goto error; + if (PyList_Append(py_retlist, py_arg)) + goto error; + Py_DECREF(py_arg); + pos = pos + strlen(&argstr[pos]) + 1; + } + } + + free(argstr); + return py_retlist; + +error: + Py_XDECREF(py_arg); + Py_DECREF(py_retlist); + if (argstr != NULL) + free(argstr); + return NULL; +} + + +/* + * Return process pathname executable. + * Thanks to Robert N. M. Watson: + * http://fxr.googlebit.com/source/usr.bin/procstat/procstat_bin.c?v=8-CURRENT + */ +PyObject * +psutil_proc_exe(PyObject *self, PyObject *args) { + long pid; + char pathname[PATH_MAX]; + int error; + int mib[4]; + int ret; + size_t size; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PATHNAME; + mib[3] = pid; + + size = sizeof(pathname); + error = sysctl(mib, 4, pathname, &size, NULL, 0); + if (error == -1) { + // see: https://github.com/giampaolo/psutil/issues/907 + if (errno == ENOENT) { + return PyUnicode_DecodeFSDefault(""); + } + else { + return \ + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_PATHNAME)"); + } + } + if (size == 0 || strlen(pathname) == 0) { + ret = psutil_pid_exists(pid); + if (ret == -1) + return NULL; + else if (ret == 0) + return NoSuchProcess(""); + else + strcpy(pathname, ""); + } + + return PyUnicode_DecodeFSDefault(pathname); +} + + +PyObject * +psutil_proc_num_threads(PyObject *self, PyObject *args) { + // Return number of threads used by process as a Python integer. + long pid; + struct kinfo_proc kp; + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kp) == -1) + return NULL; + return Py_BuildValue("l", (long)kp.ki_numthreads); +} + + +PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + // Retrieves all threads used by process returning a list of tuples + // including thread id, user time and system time. + // Thanks to Robert N. M. Watson: + // http://code.metager.de/source/xref/freebsd/usr.bin/procstat/ + // procstat_threads.c + long pid; + int mib[4]; + struct kinfo_proc *kip = NULL; + struct kinfo_proc *kipp = NULL; + int error; + unsigned int i; + size_t size; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + // we need to re-query for thread information, so don't use *kipp + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID | KERN_PROC_INC_THREAD; + mib[3] = pid; + + size = 0; + error = sysctl(mib, 4, NULL, &size, NULL, 0); + if (error == -1) { + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_INC_THREAD)"); + goto error; + } + if (size == 0) { + NoSuchProcess(""); + goto error; + } + + kip = malloc(size); + if (kip == NULL) { + PyErr_NoMemory(); + goto error; + } + + error = sysctl(mib, 4, kip, &size, NULL, 0); + if (error == -1) { + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_INC_THREAD)"); + goto error; + } + if (size == 0) { + NoSuchProcess(""); + goto error; + } + + for (i = 0; i < size / sizeof(*kipp); i++) { + kipp = &kip[i]; + py_tuple = Py_BuildValue("Idd", + kipp->ki_tid, + PSUTIL_TV2DOUBLE(kipp->ki_rusage.ru_utime), + PSUTIL_TV2DOUBLE(kipp->ki_rusage.ru_stime)); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + free(kip); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (kip != NULL) + free(kip); + return NULL; +} + + +PyObject * +psutil_cpu_count_phys(PyObject *self, PyObject *args) { + // Return an XML string from which we'll determine the number of + // physical CPU cores in the system. + void *topology = NULL; + size_t size = 0; + PyObject *py_str; + + if (sysctlbyname("kern.sched.topology_spec", NULL, &size, NULL, 0)) + goto error; + + topology = malloc(size); + if (!topology) { + PyErr_NoMemory(); + return NULL; + } + + if (sysctlbyname("kern.sched.topology_spec", topology, &size, NULL, 0)) + goto error; + + py_str = Py_BuildValue("s", topology); + free(topology); + return py_str; + +error: + if (topology != NULL) + free(topology); + Py_RETURN_NONE; +} + + +/* + * Return virtual memory usage statistics. + */ +PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + unsigned long total; + unsigned int active, inactive, wired, cached, free; + size_t size = sizeof(total); + struct vmtotal vm; + int mib[] = {CTL_VM, VM_METER}; + long pagesize = getpagesize(); +#if __FreeBSD_version > 702101 + long buffers; +#else + int buffers; +#endif + size_t buffers_size = sizeof(buffers); + + if (sysctlbyname("hw.physmem", &total, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall("sysctlbyname('hw.physmem')"); + } + if (sysctlbyname("vm.stats.vm.v_active_count", &active, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_active_count')"); + } + if (sysctlbyname("vm.stats.vm.v_inactive_count", &inactive, &size, NULL, 0)) + { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_inactive_count')"); + } + if (sysctlbyname("vm.stats.vm.v_wire_count", &wired, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_wire_count')"); + } + // https://github.com/giampaolo/psutil/issues/997 + if (sysctlbyname("vm.stats.vm.v_cache_count", &cached, &size, NULL, 0)) { + cached = 0; + } + if (sysctlbyname("vm.stats.vm.v_free_count", &free, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_free_count')"); + } + if (sysctlbyname("vfs.bufspace", &buffers, &buffers_size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall("sysctlbyname('vfs.bufspace')"); + } + + size = sizeof(vm); + if (sysctl(mib, 2, &vm, &size, NULL, 0) != 0) { + return PyErr_SetFromOSErrnoWithSyscall("sysctl(CTL_VM | VM_METER)"); + } + + return Py_BuildValue("KKKKKKKK", + (unsigned long long) total, + (unsigned long long) free * pagesize, + (unsigned long long) active * pagesize, + (unsigned long long) inactive * pagesize, + (unsigned long long) wired * pagesize, + (unsigned long long) cached * pagesize, + (unsigned long long) buffers, + (unsigned long long) (vm.t_vmshr + vm.t_rmshr) * pagesize // shared + ); +} + + +PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { + // Return swap memory stats (see 'swapinfo' cmdline tool) + kvm_t *kd; + struct kvm_swap kvmsw[1]; + unsigned int swapin, swapout, nodein, nodeout; + size_t size = sizeof(unsigned int); + int pagesize; + + kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open failed"); + if (kd == NULL) { + PyErr_SetString(PyExc_RuntimeError, "kvm_open() syscall failed"); + return NULL; + } + + if (kvm_getswapinfo(kd, kvmsw, 1, 0) < 0) { + kvm_close(kd); + PyErr_SetString(PyExc_RuntimeError, + "kvm_getswapinfo() syscall failed"); + return NULL; + } + + kvm_close(kd); + + if (sysctlbyname("vm.stats.vm.v_swapin", &swapin, &size, NULL, 0) == -1) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_swapin)'"); + } + if (sysctlbyname("vm.stats.vm.v_swapout", &swapout, &size, NULL, 0) == -1){ + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_swapout)'"); + } + if (sysctlbyname("vm.stats.vm.v_vnodein", &nodein, &size, NULL, 0) == -1) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_vnodein)'"); + } + if (sysctlbyname("vm.stats.vm.v_vnodeout", &nodeout, &size, NULL, 0) == -1) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.vm.v_vnodeout)'"); + } + + pagesize = getpagesize(); + if (pagesize <= 0) { + PyErr_SetString(PyExc_ValueError, "invalid getpagesize()"); + return NULL; + } + + return Py_BuildValue( + "(KKKII)", + (unsigned long long)kvmsw[0].ksw_total * pagesize, // total + (unsigned long long)kvmsw[0].ksw_used * pagesize, // used + (unsigned long long)kvmsw[0].ksw_total * pagesize - // free + kvmsw[0].ksw_used * pagesize, + swapin + swapout, // swap in + nodein + nodeout // swap out + ); +} + + +#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 +PyObject * +psutil_proc_cwd(PyObject *self, PyObject *args) { + long pid; + struct kinfo_file *freep = NULL; + struct kinfo_file *kif; + struct kinfo_proc kipp; + PyObject *py_path = NULL; + + int i, cnt; + + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + if (psutil_kinfo_proc(pid, &kipp) == -1) + goto error; + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + goto error; + } + + for (i = 0; i < cnt; i++) { + kif = &freep[i]; + if (kif->kf_fd == KF_FD_TYPE_CWD) { + py_path = PyUnicode_DecodeFSDefault(kif->kf_path); + if (!py_path) + goto error; + break; + } + } + /* + * For lower pids it seems we can't retrieve any information + * (lsof can't do that it either). Since this happens even + * as root we return an empty string instead of AccessDenied. + */ + if (py_path == NULL) + py_path = PyUnicode_DecodeFSDefault(""); + free(freep); + return py_path; + +error: + Py_XDECREF(py_path); + if (freep != NULL) + free(freep); + return NULL; +} +#endif + + +#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 +PyObject * +psutil_proc_num_fds(PyObject *self, PyObject *args) { + long pid; + int cnt; + + struct kinfo_file *freep; + struct kinfo_proc kipp; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kipp) == -1) + return NULL; + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + return NULL; + } + free(freep); + + return Py_BuildValue("i", cnt); +} +#endif + + +PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + static int maxcpus; + int mib[2]; + int ncpu; + size_t len; + size_t size; + int i; + PyObject *py_retlist = PyList_New(0); + PyObject *py_cputime = NULL; + + if (py_retlist == NULL) + return NULL; + + // retrieve maxcpus value + size = sizeof(maxcpus); + if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) { + Py_DECREF(py_retlist); + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('kern.smp.maxcpus')"); + } + long cpu_time[maxcpus][CPUSTATES]; + + // retrieve the number of cpus + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + len = sizeof(ncpu); + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) { + PyErr_SetFromOSErrnoWithSyscall("sysctl(HW_NCPU)"); + goto error; + } + + // per-cpu info + size = sizeof(cpu_time); + if (sysctlbyname("kern.cp_times", &cpu_time, &size, NULL, 0) == -1) { + PyErr_SetFromOSErrnoWithSyscall("sysctlbyname('kern.smp.maxcpus')"); + goto error; + } + + for (i = 0; i < ncpu; i++) { + py_cputime = Py_BuildValue( + "(ddddd)", + (double)cpu_time[i][CP_USER] / CLOCKS_PER_SEC, + (double)cpu_time[i][CP_NICE] / CLOCKS_PER_SEC, + (double)cpu_time[i][CP_SYS] / CLOCKS_PER_SEC, + (double)cpu_time[i][CP_IDLE] / CLOCKS_PER_SEC, + (double)cpu_time[i][CP_INTR] / CLOCKS_PER_SEC); + if (!py_cputime) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_DECREF(py_cputime); + } + + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + return NULL; +} + + +PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + int i; + struct statinfo stats; + + PyObject *py_retdict = PyDict_New(); + PyObject *py_disk_info = NULL; + + if (py_retdict == NULL) + return NULL; + if (devstat_checkversion(NULL) < 0) { + PyErr_Format(PyExc_RuntimeError, + "devstat_checkversion() syscall failed"); + goto error; + } + + stats.dinfo = (struct devinfo *)malloc(sizeof(struct devinfo)); + if (stats.dinfo == NULL) { + PyErr_NoMemory(); + goto error; + } + bzero(stats.dinfo, sizeof(struct devinfo)); + + if (devstat_getdevs(NULL, &stats) == -1) { + PyErr_Format(PyExc_RuntimeError, "devstat_getdevs() syscall failed"); + goto error; + } + + for (i = 0; i < stats.dinfo->numdevs; i++) { + py_disk_info = NULL; + struct devstat current; + char disk_name[128]; + current = stats.dinfo->devices[i]; + snprintf(disk_name, sizeof(disk_name), "%s%d", + current.device_name, + current.unit_number); + + py_disk_info = Py_BuildValue( + "(KKKKLLL)", + current.operations[DEVSTAT_READ], // no reads + current.operations[DEVSTAT_WRITE], // no writes + current.bytes[DEVSTAT_READ], // bytes read + current.bytes[DEVSTAT_WRITE], // bytes written + (long long) PSUTIL_BT2MSEC(current.duration[DEVSTAT_READ]), // r time + (long long) PSUTIL_BT2MSEC(current.duration[DEVSTAT_WRITE]), // w time + (long long) PSUTIL_BT2MSEC(current.busy_time) // busy time + ); // finished transactions + if (!py_disk_info) + goto error; + if (PyDict_SetItemString(py_retdict, disk_name, py_disk_info)) + goto error; + Py_DECREF(py_disk_info); + } + + if (stats.dinfo->mem_ptr) + free(stats.dinfo->mem_ptr); + free(stats.dinfo); + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + if (stats.dinfo != NULL) + free(stats.dinfo); + return NULL; +} + + +PyObject * +psutil_proc_memory_maps(PyObject *self, PyObject *args) { + // Return a list of tuples for every process memory maps. + //'procstat' cmdline utility has been used as an example. + long pid; + int ptrwidth; + int i, cnt; + char addr[1000]; + char perms[4]; + char *path; + struct kinfo_proc kp; + struct kinfo_vmentry *freep = NULL; + struct kinfo_vmentry *kve; + ptrwidth = 2 * sizeof(void *); + PyObject *py_tuple = NULL; + PyObject *py_path = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + if (psutil_kinfo_proc(pid, &kp) == -1) + goto error; + + errno = 0; + freep = kinfo_getvmmap(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getvmmap()"); + goto error; + } + for (i = 0; i < cnt; i++) { + py_tuple = NULL; + kve = &freep[i]; + addr[0] = '\0'; + perms[0] = '\0'; + sprintf(addr, "%#*jx-%#*jx", ptrwidth, (uintmax_t)kve->kve_start, + ptrwidth, (uintmax_t)kve->kve_end); + psutil_remove_spaces(addr); + strlcat(perms, kve->kve_protection & KVME_PROT_READ ? "r" : "-", + sizeof(perms)); + strlcat(perms, kve->kve_protection & KVME_PROT_WRITE ? "w" : "-", + sizeof(perms)); + strlcat(perms, kve->kve_protection & KVME_PROT_EXEC ? "x" : "-", + sizeof(perms)); + + if (strlen(kve->kve_path) == 0) { + switch (kve->kve_type) { + case KVME_TYPE_NONE: + path = "[none]"; + break; + case KVME_TYPE_DEFAULT: + path = "[default]"; + break; + case KVME_TYPE_VNODE: + path = "[vnode]"; + break; + case KVME_TYPE_SWAP: + path = "[swap]"; + break; + case KVME_TYPE_DEVICE: + path = "[device]"; + break; + case KVME_TYPE_PHYS: + path = "[phys]"; + break; + case KVME_TYPE_DEAD: + path = "[dead]"; + break; + case KVME_TYPE_SG: + path = "[sg]"; + break; + case KVME_TYPE_UNKNOWN: + path = "[unknown]"; + break; + default: + path = "[?]"; + break; + } + } + else { + path = kve->kve_path; + } + + py_path = PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; + py_tuple = Py_BuildValue("ssOiiii", + addr, // "start-end" address + perms, // "rwx" permissions + py_path, // path + kve->kve_resident, // rss + kve->kve_private_resident, // private + kve->kve_ref_count, // ref count + kve->kve_shadow_count); // shadow count + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_path); + Py_DECREF(py_tuple); + } + free(freep); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_path); + Py_DECREF(py_retlist); + if (freep != NULL) + free(freep); + return NULL; +} + + +PyObject* +psutil_proc_cpu_affinity_get(PyObject* self, PyObject* args) { + // Get process CPU affinity. + // Reference: + // http://sources.freebsd.org/RELENG_9/src/usr.bin/cpuset/cpuset.c + long pid; + int ret; + int i; + cpuset_t mask; + PyObject* py_retlist; + PyObject* py_cpu_num; + + if (!PyArg_ParseTuple(args, "i", &pid)) + return NULL; + ret = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid, + sizeof(mask), &mask); + if (ret != 0) + return PyErr_SetFromErrno(PyExc_OSError); + + py_retlist = PyList_New(0); + if (py_retlist == NULL) + return NULL; + + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, &mask)) { + py_cpu_num = Py_BuildValue("i", i); + if (py_cpu_num == NULL) + goto error; + if (PyList_Append(py_retlist, py_cpu_num)) + goto error; + } + } + + return py_retlist; + +error: + Py_XDECREF(py_cpu_num); + Py_DECREF(py_retlist); + return NULL; +} + + +PyObject * +psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { + // Set process CPU affinity. + // Reference: + // http://sources.freebsd.org/RELENG_9/src/usr.bin/cpuset/cpuset.c + long pid; + int i; + int seq_len; + int ret; + cpuset_t cpu_set; + PyObject *py_cpu_set; + PyObject *py_cpu_seq = NULL; + + if (!PyArg_ParseTuple(args, "lO", &pid, &py_cpu_set)) + return NULL; + + py_cpu_seq = PySequence_Fast(py_cpu_set, "expected a sequence or integer"); + if (!py_cpu_seq) + return NULL; + seq_len = PySequence_Fast_GET_SIZE(py_cpu_seq); + + // calculate the mask + CPU_ZERO(&cpu_set); + for (i = 0; i < seq_len; i++) { + PyObject *item = PySequence_Fast_GET_ITEM(py_cpu_seq, i); +#if PY_MAJOR_VERSION >= 3 + long value = PyLong_AsLong(item); +#else + long value = PyInt_AsLong(item); +#endif + if (value == -1 || PyErr_Occurred()) + goto error; + CPU_SET(value, &cpu_set); + } + + // set affinity + ret = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid, + sizeof(cpu_set), &cpu_set); + if (ret != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + Py_DECREF(py_cpu_seq); + Py_RETURN_NONE; + +error: + if (py_cpu_seq != NULL) + Py_DECREF(py_cpu_seq); + return NULL; +} + + +PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + unsigned int v_soft; + unsigned int v_intr; + unsigned int v_syscall; + unsigned int v_trap; + unsigned int v_swtch; + size_t size = sizeof(v_soft); + + if (sysctlbyname("vm.stats.sys.v_soft", &v_soft, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_soft')"); + } + if (sysctlbyname("vm.stats.sys.v_intr", &v_intr, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_intr')"); + } + if (sysctlbyname("vm.stats.sys.v_syscall", &v_syscall, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_syscall')"); + } + if (sysctlbyname("vm.stats.sys.v_trap", &v_trap, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_trap')"); + } + if (sysctlbyname("vm.stats.sys.v_swtch", &v_swtch, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_swtch')"); + } + + return Py_BuildValue( + "IIIII", + v_swtch, // ctx switches + v_intr, // interrupts + v_soft, // software interrupts + v_syscall, // syscalls + v_trap // traps + ); +} + + +/* + * Return battery information. + */ +PyObject * +psutil_sensors_battery(PyObject *self, PyObject *args) { + int percent; + int minsleft; + int power_plugged; + size_t size = sizeof(percent); + + if (sysctlbyname("hw.acpi.battery.life", &percent, &size, NULL, 0)) + goto error; + if (sysctlbyname("hw.acpi.battery.time", &minsleft, &size, NULL, 0)) + goto error; + if (sysctlbyname("hw.acpi.acline", &power_plugged, &size, NULL, 0)) + goto error; + return Py_BuildValue("iii", percent, minsleft, power_plugged); + +error: + // see: https://github.com/giampaolo/psutil/issues/1074 + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "no battery"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} + + +/* + * Return temperature information for a given CPU core number. + */ +PyObject * +psutil_sensors_cpu_temperature(PyObject *self, PyObject *args) { + int current; + int tjmax; + int core; + char sensor[26]; + size_t size = sizeof(current); + + if (! PyArg_ParseTuple(args, "i", &core)) + return NULL; + sprintf(sensor, "dev.cpu.%d.temperature", core); + if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) + goto error; + current = DECIKELVIN_2_CELCIUS(current); + + // Return -273 in case of faliure. + sprintf(sensor, "dev.cpu.%d.coretemp.tjmax", core); + if (sysctlbyname(sensor, &tjmax, &size, NULL, 0)) + tjmax = 0; + tjmax = DECIKELVIN_2_CELCIUS(tjmax); + + return Py_BuildValue("ii", current, tjmax); + +error: + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "no temperature sensors"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} + + +/* + * Return frequency information of a given CPU. + * As of Dec 2018 only CPU 0 appears to be supported and all other + * cores match the frequency of CPU 0. + */ +PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + int current; + int core; + char sensor[26]; + char available_freq_levels[1000]; + size_t size = sizeof(current); + + if (! PyArg_ParseTuple(args, "i", &core)) + return NULL; + // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ + sprintf(sensor, "dev.cpu.%d.freq", core); + if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) + goto error; + + size = sizeof(available_freq_levels); + // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ + // In case of failure, an empty string is returned. + sprintf(sensor, "dev.cpu.%d.freq_levels", core); + sysctlbyname(sensor, &available_freq_levels, &size, NULL, 0); + + return Py_BuildValue("is", current, available_freq_levels); + +error: + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "unable to read frequency"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} diff --git a/ddtrace/vendor/psutil/arch/freebsd/specific.h b/ddtrace/vendor/psutil/arch/freebsd/specific.h new file mode 100644 index 00000000000..875c8166465 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/freebsd/specific.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +typedef struct kinfo_proc kinfo_proc; + +int psutil_get_proc_list(struct kinfo_proc **procList, size_t *procCount); +int psutil_kinfo_proc(const pid_t pid, struct kinfo_proc *proc); + +// +PyObject* psutil_cpu_count_phys(PyObject* self, PyObject* args); +PyObject* psutil_disk_io_counters(PyObject* self, PyObject* args); +PyObject* psutil_get_cmdline(long pid); +PyObject* psutil_per_cpu_times(PyObject* self, PyObject* args); +PyObject* psutil_proc_cpu_affinity_get(PyObject* self, PyObject* args); +PyObject* psutil_proc_cpu_affinity_set(PyObject* self, PyObject* args); +PyObject* psutil_proc_cwd(PyObject* self, PyObject* args); +PyObject* psutil_proc_exe(PyObject* self, PyObject* args); +PyObject* psutil_proc_memory_maps(PyObject* self, PyObject* args); +PyObject* psutil_proc_num_fds(PyObject* self, PyObject* args); +PyObject* psutil_proc_num_threads(PyObject* self, PyObject* args); +PyObject* psutil_proc_threads(PyObject* self, PyObject* args); +PyObject* psutil_swap_mem(PyObject* self, PyObject* args); +PyObject* psutil_virtual_mem(PyObject* self, PyObject* args); +PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); +#if defined(PSUTIL_FREEBSD) +PyObject* psutil_sensors_battery(PyObject* self, PyObject* args); +PyObject* psutil_sensors_cpu_temperature(PyObject* self, PyObject* args); +PyObject* psutil_cpu_freq(PyObject* self, PyObject* args); +#endif diff --git a/ddtrace/vendor/psutil/arch/freebsd/sys_socks.c b/ddtrace/vendor/psutil/arch/freebsd/sys_socks.c new file mode 100644 index 00000000000..e0e2046be6d --- /dev/null +++ b/ddtrace/vendor/psutil/arch/freebsd/sys_socks.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Retrieves system-wide open socket connections. This is based off of + * sockstat utility source code: + * https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c + */ + +#include +#include +#include +#include // for struct xsocket +#include +#include +#include +#include // for xinpcb struct +#include +#include +#include // for struct xtcpcb +#include // for inet_ntop() + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + +static struct xfile *psutil_xfiles; +static int psutil_nxfiles; + + +int +psutil_populate_xfiles() { + size_t len; + + if ((psutil_xfiles = malloc(len = sizeof *psutil_xfiles)) == NULL) { + PyErr_NoMemory(); + return 0; + } + while (sysctlbyname("kern.file", psutil_xfiles, &len, 0, 0) == -1) { + if (errno != ENOMEM) { + PyErr_SetFromErrno(0); + return 0; + } + len *= 2; + if ((psutil_xfiles = realloc(psutil_xfiles, len)) == NULL) { + PyErr_NoMemory(); + return 0; + } + } + if (len > 0 && psutil_xfiles->xf_size != sizeof *psutil_xfiles) { + PyErr_Format(PyExc_RuntimeError, "struct xfile size mismatch"); + return 0; + } + psutil_nxfiles = len / sizeof *psutil_xfiles; + return 1; +} + + +struct xfile * +psutil_get_file_from_sock(void *sock) { + struct xfile *xf; + int n; + + for (xf = psutil_xfiles, n = 0; n < psutil_nxfiles; ++n, ++xf) { + if (xf->xf_data == sock) + return xf; + } + return NULL; +} + + +// Reference: +// https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c +int psutil_gather_inet(int proto, PyObject *py_retlist) { + struct xinpgen *xig, *exig; + struct xinpcb *xip; + struct xtcpcb *xtp; +#if __FreeBSD_version >= 1200026 + struct xinpcb *inp; +#else + struct inpcb *inp; +#endif + struct xsocket *so; + const char *varname = NULL; + size_t len, bufsize; + void *buf; + int retry; + int type; + + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + + switch (proto) { + case IPPROTO_TCP: + varname = "net.inet.tcp.pcblist"; + type = SOCK_STREAM; + break; + case IPPROTO_UDP: + varname = "net.inet.udp.pcblist"; + type = SOCK_DGRAM; + break; + } + + buf = NULL; + bufsize = 8192; + retry = 5; + do { + for (;;) { + buf = realloc(buf, bufsize); + if (buf == NULL) + continue; // XXX + len = bufsize; + if (sysctlbyname(varname, buf, &len, NULL, 0) == 0) + break; + if (errno != ENOMEM) { + PyErr_SetFromErrno(0); + goto error; + } + bufsize *= 2; + } + xig = (struct xinpgen *)buf; + exig = (struct xinpgen *)(void *)((char *)buf + len - sizeof *exig); + if (xig->xig_len != sizeof *xig || exig->xig_len != sizeof *exig) { + PyErr_Format(PyExc_RuntimeError, "struct xinpgen size mismatch"); + goto error; + } + } while (xig->xig_gen != exig->xig_gen && retry--); + + for (;;) { + struct xfile *xf; + int lport, rport, status, family; + + xig = (struct xinpgen *)(void *)((char *)xig + xig->xig_len); + if (xig >= exig) + break; + + switch (proto) { + case IPPROTO_TCP: + xtp = (struct xtcpcb *)xig; + if (xtp->xt_len != sizeof *xtp) { + PyErr_Format(PyExc_RuntimeError, + "struct xtcpcb size mismatch"); + goto error; + } + inp = &xtp->xt_inp; +#if __FreeBSD_version >= 1200026 + so = &inp->xi_socket; + status = xtp->t_state; +#else + so = &xtp->xt_socket; + status = xtp->xt_tp.t_state; +#endif + break; + case IPPROTO_UDP: + xip = (struct xinpcb *)xig; + if (xip->xi_len != sizeof *xip) { + PyErr_Format(PyExc_RuntimeError, + "struct xinpcb size mismatch"); + goto error; + } +#if __FreeBSD_version >= 1200026 + inp = xip; +#else + inp = &xip->xi_inp; +#endif + so = &xip->xi_socket; + status = PSUTIL_CONN_NONE; + break; + default: + PyErr_Format(PyExc_RuntimeError, "invalid proto"); + goto error; + } + + char lip[200], rip[200]; + + xf = psutil_get_file_from_sock(so->xso_so); + if (xf == NULL) + continue; + lport = ntohs(inp->inp_lport); + rport = ntohs(inp->inp_fport); + + if (inp->inp_vflag & INP_IPV4) { + family = AF_INET; + inet_ntop(AF_INET, &inp->inp_laddr.s_addr, lip, sizeof(lip)); + inet_ntop(AF_INET, &inp->inp_faddr.s_addr, rip, sizeof(rip)); + } + else if (inp->inp_vflag & INP_IPV6) { + family = AF_INET6; + inet_ntop(AF_INET6, &inp->in6p_laddr.s6_addr, lip, sizeof(lip)); + inet_ntop(AF_INET6, &inp->in6p_faddr.s6_addr, rip, sizeof(rip)); + } + + // construct python tuple/list + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", rip, rport); + else + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + py_tuple = Py_BuildValue( + "(iiiNNii)", + xf->xf_fd, // fd + family, // family + type, // type + py_laddr, // laddr + py_raddr, // raddr + status, // status + xf->xf_pid); // pid + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + + free(buf); + return 1; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + free(buf); + return 0; +} + + +int psutil_gather_unix(int proto, PyObject *py_retlist) { + struct xunpgen *xug, *exug; + struct xunpcb *xup; + const char *varname = NULL; + const char *protoname = NULL; + size_t len; + size_t bufsize; + void *buf; + int retry; + struct sockaddr_un *sun; + char path[PATH_MAX]; + + PyObject *py_tuple = NULL; + PyObject *py_lpath = NULL; + + switch (proto) { + case SOCK_STREAM: + varname = "net.local.stream.pcblist"; + protoname = "stream"; + break; + case SOCK_DGRAM: + varname = "net.local.dgram.pcblist"; + protoname = "dgram"; + break; + } + + buf = NULL; + bufsize = 8192; + retry = 5; + + do { + for (;;) { + buf = realloc(buf, bufsize); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } + len = bufsize; + if (sysctlbyname(varname, buf, &len, NULL, 0) == 0) + break; + if (errno != ENOMEM) { + PyErr_SetFromErrno(0); + goto error; + } + bufsize *= 2; + } + xug = (struct xunpgen *)buf; + exug = (struct xunpgen *)(void *) + ((char *)buf + len - sizeof *exug); + if (xug->xug_len != sizeof *xug || exug->xug_len != sizeof *exug) { + PyErr_Format(PyExc_RuntimeError, "struct xinpgen size mismatch"); + goto error; + } + } while (xug->xug_gen != exug->xug_gen && retry--); + + for (;;) { + struct xfile *xf; + + xug = (struct xunpgen *)(void *)((char *)xug + xug->xug_len); + if (xug >= exug) + break; + xup = (struct xunpcb *)xug; + if (xup->xu_len != sizeof *xup) + goto error; + + xf = psutil_get_file_from_sock(xup->xu_socket.xso_so); + if (xf == NULL) + continue; + + sun = (struct sockaddr_un *)&xup->xu_addr; + snprintf(path, sizeof(path), "%.*s", + (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), + sun->sun_path); + py_lpath = PyUnicode_DecodeFSDefault(path); + if (! py_lpath) + goto error; + + py_tuple = Py_BuildValue("(iiiOsii)", + xf->xf_fd, // fd + AF_UNIX, // family + proto, // type + py_lpath, // lpath + "", // rath + PSUTIL_CONN_NONE, // status + xf->xf_pid); // pid + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_lpath); + Py_DECREF(py_tuple); + } + + free(buf); + return 1; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_lpath); + free(buf); + return 0; +} + + +PyObject* +psutil_net_connections(PyObject* self, PyObject* args) { + // Return system-wide open connections. + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (psutil_populate_xfiles() != 1) + goto error; + if (psutil_gather_inet(IPPROTO_TCP, py_retlist) == 0) + goto error; + if (psutil_gather_inet(IPPROTO_UDP, py_retlist) == 0) + goto error; + if (psutil_gather_unix(SOCK_STREAM, py_retlist) == 0) + goto error; + if (psutil_gather_unix(SOCK_DGRAM, py_retlist) == 0) + goto error; + + free(psutil_xfiles); + return py_retlist; + +error: + Py_DECREF(py_retlist); + free(psutil_xfiles); + return NULL; +} diff --git a/ddtrace/vendor/psutil/arch/freebsd/sys_socks.h b/ddtrace/vendor/psutil/arch/freebsd/sys_socks.h new file mode 100644 index 00000000000..75247926556 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/freebsd/sys_socks.h @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +PyObject* psutil_net_connections(PyObject* self, PyObject* args); diff --git a/ddtrace/vendor/psutil/arch/netbsd/socks.c b/ddtrace/vendor/psutil/arch/netbsd/socks.c new file mode 100644 index 00000000000..f370f094667 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/netbsd/socks.c @@ -0,0 +1,447 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. + * Copyright (c) 2015, Ryo ONODERA. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + + +// address family filter +enum af_filter { + INET, + INET4, + INET6, + TCP, + TCP4, + TCP6, + UDP, + UDP4, + UDP6, + UNIX, + ALL, +}; + +// kinfo_file results +struct kif { + SLIST_ENTRY(kif) kifs; + struct kinfo_file *kif; +}; + +// kinfo_file results list +SLIST_HEAD(kifhead, kif) kihead = SLIST_HEAD_INITIALIZER(kihead); + + +// kinfo_pcb results +struct kpcb { + SLIST_ENTRY(kpcb) kpcbs; + struct kinfo_pcb *kpcb; +}; + +// kinfo_pcb results list +SLIST_HEAD(kpcbhead, kpcb) kpcbhead = SLIST_HEAD_INITIALIZER(kpcbhead); + +static void psutil_kiflist_init(void); +static void psutil_kiflist_clear(void); +static void psutil_kpcblist_init(void); +static void psutil_kpcblist_clear(void); +static int psutil_get_files(void); +static int psutil_get_sockets(const char *name); +static int psutil_get_info(int aff); + + +// Initialize kinfo_file results list. +static void +psutil_kiflist_init(void) { + SLIST_INIT(&kihead); + return; +} + + +// Clear kinfo_file results list. +static void +psutil_kiflist_clear(void) { + while (!SLIST_EMPTY(&kihead)) { + SLIST_REMOVE_HEAD(&kihead, kifs); + } + + return; +} + + +// Initialize kinof_pcb result list. +static void +psutil_kpcblist_init(void) { + SLIST_INIT(&kpcbhead); + return; +} + + +// Clear kinof_pcb result list. +static void +psutil_kpcblist_clear(void) { + while (!SLIST_EMPTY(&kpcbhead)) { + SLIST_REMOVE_HEAD(&kpcbhead, kpcbs); + } + + return; +} + + +// Get all open files including socket. +static int +psutil_get_files(void) { + size_t len; + size_t j; + int mib[6]; + char *buf; + off_t offset; + + mib[0] = CTL_KERN; + mib[1] = KERN_FILE2; + mib[2] = KERN_FILE_BYFILE; + mib[3] = 0; + mib[4] = sizeof(struct kinfo_file); + mib[5] = 0; + + if (sysctl(mib, 6, NULL, &len, NULL, 0) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + offset = len % sizeof(off_t); + mib[5] = len / sizeof(struct kinfo_file); + + if ((buf = malloc(len + offset)) == NULL) { + PyErr_NoMemory(); + return -1; + } + + if (sysctl(mib, 6, buf + offset, &len, NULL, 0) == -1) { + free(buf); + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + len /= sizeof(struct kinfo_file); + struct kinfo_file *ki = (struct kinfo_file *)(buf + offset); + + for (j = 0; j < len; j++) { + struct kif *kif = malloc(sizeof(struct kif)); + kif->kif = &ki[j]; + SLIST_INSERT_HEAD(&kihead, kif, kifs); + } + + /* + // debug + struct kif *k; + SLIST_FOREACH(k, &kihead, kifs) { + printf("%d\n", k->kif->ki_pid); + } + */ + + return 0; +} + + +// Get open sockets. +static int +psutil_get_sockets(const char *name) { + size_t namelen; + int mib[8]; + struct kinfo_pcb *pcb; + size_t len; + size_t j; + + memset(mib, 0, sizeof(mib)); + + if (sysctlnametomib(name, mib, &namelen) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (sysctl(mib, __arraycount(mib), NULL, &len, NULL, 0) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if ((pcb = malloc(len)) == NULL) { + PyErr_NoMemory(); + return -1; + } + memset(pcb, 0, len); + + mib[6] = sizeof(*pcb); + mib[7] = len / sizeof(*pcb); + + if (sysctl(mib, __arraycount(mib), pcb, &len, NULL, 0) == -1) { + free(pcb); + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + len /= sizeof(struct kinfo_pcb); + struct kinfo_pcb *kp = (struct kinfo_pcb *)pcb; + + for (j = 0; j < len; j++) { + struct kpcb *kpcb = malloc(sizeof(struct kpcb)); + kpcb->kpcb = &kp[j]; + SLIST_INSERT_HEAD(&kpcbhead, kpcb, kpcbs); + } + + /* + // debug + struct kif *k; + struct kpcb *k; + SLIST_FOREACH(k, &kpcbhead, kpcbs) { + printf("ki_type: %d\n", k->kpcb->ki_type); + printf("ki_family: %d\n", k->kpcb->ki_family); + } + */ + + return 0; +} + + +// Collect open file and connections. +static int +psutil_get_info(int aff) { + switch (aff) { + case INET: + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + break; + case INET4: + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + break; + case INET6: + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + break; + case TCP: + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + break; + case TCP4: + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + break; + case TCP6: + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + break; + case UDP: + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + break; + case UDP4: + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + break; + case UDP6: + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + break; + case UNIX: + if (psutil_get_sockets("net.local.stream.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.seqpacket.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.dgram.pcblist") != 0) + return -1; + break; + case ALL: + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.stream.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.seqpacket.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.dgram.pcblist") != 0) + return -1; + break; + } + + return 0; +} + + +/* + * Return system-wide connections (unless a pid != -1 is passed). + */ +PyObject * +psutil_net_connections(PyObject *self, PyObject *args) { + char laddr[PATH_MAX]; + char raddr[PATH_MAX]; + int32_t lport; + int32_t rport; + int32_t status; + pid_t pid; + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + psutil_kiflist_init(); + psutil_kpcblist_init(); + if (psutil_get_files() != 0) + goto error; + if (psutil_get_info(ALL) != 0) + goto error; + + struct kif *k; + SLIST_FOREACH(k, &kihead, kifs) { + struct kpcb *kp; + if ((pid != -1) && (k->kif->ki_pid != (unsigned int)pid)) + continue; + SLIST_FOREACH(kp, &kpcbhead, kpcbs) { + if (k->kif->ki_fdata != kp->kpcb->ki_sockaddr) + continue; + + // IPv4 or IPv6 + if ((kp->kpcb->ki_family == AF_INET) || + (kp->kpcb->ki_family == AF_INET6)) { + + if (kp->kpcb->ki_family == AF_INET) { + // IPv4 + struct sockaddr_in *sin_src = + (struct sockaddr_in *)&kp->kpcb->ki_src; + struct sockaddr_in *sin_dst = + (struct sockaddr_in *)&kp->kpcb->ki_dst; + // source addr and port + inet_ntop(AF_INET, &sin_src->sin_addr, laddr, + sizeof(laddr)); + lport = ntohs(sin_src->sin_port); + // remote addr and port + inet_ntop(AF_INET, &sin_dst->sin_addr, raddr, + sizeof(raddr)); + rport = ntohs(sin_dst->sin_port); + } + else { + // IPv6 + struct sockaddr_in6 *sin6_src = + (struct sockaddr_in6 *)&kp->kpcb->ki_src; + struct sockaddr_in6 *sin6_dst = + (struct sockaddr_in6 *)&kp->kpcb->ki_dst; + // local addr and port + inet_ntop(AF_INET6, &sin6_src->sin6_addr, laddr, + sizeof(laddr)); + lport = ntohs(sin6_src->sin6_port); + // remote addr and port + inet_ntop(AF_INET6, &sin6_dst->sin6_addr, raddr, + sizeof(raddr)); + rport = ntohs(sin6_dst->sin6_port); + } + + // status + if (kp->kpcb->ki_type == SOCK_STREAM) + status = kp->kpcb->ki_tstate; + else + status = PSUTIL_CONN_NONE; + + // build addr tuple + py_laddr = Py_BuildValue("(si)", laddr, lport); + if (! py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", raddr, rport); + else + py_raddr = Py_BuildValue("()"); + if (! py_raddr) + goto error; + } + else if (kp->kpcb->ki_family == AF_UNIX) { + // UNIX sockets + struct sockaddr_un *sun_src = + (struct sockaddr_un *)&kp->kpcb->ki_src; + struct sockaddr_un *sun_dst = + (struct sockaddr_un *)&kp->kpcb->ki_dst; + strcpy(laddr, sun_src->sun_path); + strcpy(raddr, sun_dst->sun_path); + status = PSUTIL_CONN_NONE; + py_laddr = PyUnicode_DecodeFSDefault(laddr); + if (! py_laddr) + goto error; + py_raddr = PyUnicode_DecodeFSDefault(raddr); + if (! py_raddr) + goto error; + } + else { + continue; + } + + // append tuple to list + py_tuple = Py_BuildValue( + "(iiiOOii)", + k->kif->ki_fd, + kp->kpcb->ki_family, + kp->kpcb->ki_type, + py_laddr, + py_raddr, + status, + k->kif->ki_pid); + if (! py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_laddr); + Py_DECREF(py_raddr); + Py_DECREF(py_tuple); + } + } + + psutil_kiflist_clear(); + psutil_kpcblist_clear(); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + return 0; +} diff --git a/ddtrace/vendor/psutil/arch/netbsd/socks.h b/ddtrace/vendor/psutil/arch/netbsd/socks.h new file mode 100644 index 00000000000..9e6a97c0a8c --- /dev/null +++ b/ddtrace/vendor/psutil/arch/netbsd/socks.h @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. + * Copyright (c) 2015, Ryo ONODERA. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +PyObject *psutil_proc_connections(PyObject *, PyObject *); +PyObject *psutil_net_connections(PyObject *, PyObject *); diff --git a/ddtrace/vendor/psutil/arch/netbsd/specific.c b/ddtrace/vendor/psutil/arch/netbsd/specific.c new file mode 100644 index 00000000000..25adffcd3bf --- /dev/null +++ b/ddtrace/vendor/psutil/arch/netbsd/specific.c @@ -0,0 +1,684 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Landry Breuil. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Platform-specific module methods for NetBSD. + */ + +#if defined(PSUTIL_NETBSD) + #define _KMEMUSER +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for swap_mem +#include +#include +// connection stuff +#include // for NI_MAXHOST +#include +#include // for CPUSTATES & CP_* +#define _KERNEL // for DTYPE_* +#include +#undef _KERNEL +#include // struct diskstats +#include +#include + +#include "specific.h" +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + +#define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) +#define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) + + +// ============================================================================ +// Utility functions +// ============================================================================ + + +int +psutil_kinfo_proc(pid_t pid, kinfo_proc *proc) { + // Fills a kinfo_proc struct based on process pid. + int ret; + int mib[6]; + size_t size = sizeof(kinfo_proc); + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC2; + mib[2] = KERN_PROC_PID; + mib[3] = pid; + mib[4] = size; + mib[5] = 1; + + ret = sysctl((int*)mib, 6, proc, &size, NULL, 0); + if (ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + // sysctl stores 0 in the size if we can't find the process information. + if (size == 0) { + NoSuchProcess(""); + return -1; + } + return 0; +} + + +struct kinfo_file * +kinfo_getfile(pid_t pid, int* cnt) { + // Mimic's FreeBSD kinfo_file call, taking a pid and a ptr to an + // int as arg and returns an array with cnt struct kinfo_file. + int mib[6]; + size_t len; + struct kinfo_file* kf; + mib[0] = CTL_KERN; + mib[1] = KERN_FILE2; + mib[2] = KERN_FILE_BYPID; + mib[3] = (int) pid; + mib[4] = sizeof(struct kinfo_file); + mib[5] = 0; + + // get the size of what would be returned + if (sysctl(mib, 6, NULL, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + if ((kf = malloc(len)) == NULL) { + PyErr_NoMemory(); + return NULL; + } + mib[5] = (int)(len / sizeof(struct kinfo_file)); + if (sysctl(mib, 6, kf, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + *cnt = (int)(len / sizeof(struct kinfo_file)); + return kf; +} + +PyObject * +psutil_proc_cwd(PyObject *self, PyObject *args) { + long pid; + + char path[MAXPATHLEN]; + size_t pathlen = sizeof path; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + +#ifdef KERN_PROC_CWD + int name[] = { CTL_KERN, KERN_PROC_ARGS, pid, KERN_PROC_CWD}; + if (sysctl(name, 4, path, &pathlen, NULL, 0) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } +#else + char *buf; + if (asprintf(&buf, "/proc/%d/cwd", (int)pid) < 0) { + PyErr_NoMemory(); + return NULL; + } + + ssize_t len = readlink(buf, path, sizeof(path) - 1); + free(buf); + if (len == -1) { + if (errno == ENOENT) + NoSuchProcess(""); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + path[len] = '\0'; +#endif + + return PyUnicode_DecodeFSDefault(path); +} + + +// XXX: This is no longer used as per +// https://github.com/giampaolo/psutil/pull/557#issuecomment-171912820 +// Current implementation uses /proc instead. +// Left here just in case. +/* +PyObject * +psutil_proc_exe(PyObject *self, PyObject *args) { +#if __NetBSD_Version__ >= 799000000 + pid_t pid; + char pathname[MAXPATHLEN]; + int error; + int mib[4]; + int ret; + size_t size; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (pid == 0) { + // else returns ENOENT + return Py_BuildValue("s", ""); + } + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC_ARGS; + mib[2] = pid; + mib[3] = KERN_PROC_PATHNAME; + + size = sizeof(pathname); + error = sysctl(mib, 4, NULL, &size, NULL, 0); + if (error == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + error = sysctl(mib, 4, pathname, &size, NULL, 0); + if (error == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + if (size == 0 || strlen(pathname) == 0) { + ret = psutil_pid_exists(pid); + if (ret == -1) + return NULL; + else if (ret == 0) + return NoSuchProcess(""); + else + strcpy(pathname, ""); + } + + return PyUnicode_DecodeFSDefault(pathname); +#else + return Py_BuildValue("s", ""); +#endif +} +*/ + +PyObject * +psutil_proc_num_threads(PyObject *self, PyObject *args) { + // Return number of threads used by process as a Python integer. + long pid; + kinfo_proc kp; + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kp) == -1) + return NULL; + return Py_BuildValue("l", (long)kp.p_nlwps); +} + +PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + pid_t pid; + int mib[5]; + int i, nlwps; + ssize_t st; + size_t size; + struct kinfo_lwp *kl = NULL; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + mib[0] = CTL_KERN; + mib[1] = KERN_LWP; + mib[2] = pid; + mib[3] = sizeof(struct kinfo_lwp); + mib[4] = 0; + + st = sysctl(mib, 5, NULL, &size, NULL, 0); + if (st == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (size == 0) { + NoSuchProcess(""); + goto error; + } + + mib[4] = size / sizeof(size_t); + kl = malloc(size); + if (kl == NULL) { + PyErr_NoMemory(); + goto error; + } + + st = sysctl(mib, 5, kl, &size, NULL, 0); + if (st == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (size == 0) { + NoSuchProcess(""); + goto error; + } + + nlwps = (int)(size / sizeof(struct kinfo_lwp)); + for (i = 0; i < nlwps; i++) { + py_tuple = Py_BuildValue("idd", + (&kl[i])->l_lid, + PSUTIL_KPT2DOUBLE((&kl[i])->l_rtime), + PSUTIL_KPT2DOUBLE((&kl[i])->l_rtime)); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + free(kl); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (kl != NULL) + free(kl); + return NULL; +} + + +// ============================================================================ +// APIS +// ============================================================================ + +int +psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { + // Returns a list of all BSD processes on the system. This routine + // allocates the list and puts it in *procList and a count of the + // number of entries in *procCount. You are responsible for freeing + // this list (use "free" from System framework). + // On success, the function returns 0. + // On error, the function returns a BSD errno value. + kinfo_proc *result; + // Declaring name as const requires us to cast it when passing it to + // sysctl because the prototype doesn't include the const modifier. + char errbuf[_POSIX2_LINE_MAX]; + int cnt; + kvm_t *kd; + + assert( procList != NULL); + assert(*procList == NULL); + assert(procCount != NULL); + + kd = kvm_openfiles(NULL, NULL, NULL, KVM_NO_FILES, errbuf); + + if (kd == NULL) { + PyErr_Format( + PyExc_RuntimeError, "kvm_openfiles() syscall failed: %s", errbuf); + return errno; + } + + result = kvm_getproc2(kd, KERN_PROC_ALL, 0, sizeof(kinfo_proc), &cnt); + if (result == NULL) { + PyErr_Format(PyExc_RuntimeError, "kvm_getproc2() syscall failed"); + kvm_close(kd); + return errno; + } + + *procCount = (size_t)cnt; + + size_t mlen = cnt * sizeof(kinfo_proc); + + if ((*procList = malloc(mlen)) == NULL) { + PyErr_NoMemory(); + kvm_close(kd); + return errno; + } + + memcpy(*procList, result, mlen); + assert(*procList != NULL); + kvm_close(kd); + + return 0; +} + + +char * +psutil_get_cmd_args(pid_t pid, size_t *argsize) { + int mib[4]; + int st; + size_t len; + char *procargs; + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC_ARGS; + mib[2] = pid; + mib[3] = KERN_PROC_ARGV; + len = 0; + + st = sysctl(mib, __arraycount(mib), NULL, &len, NULL, 0); + if (st == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + procargs = (char *)malloc(len); + if (procargs == NULL) { + PyErr_NoMemory(); + return NULL; + } + st = sysctl(mib, __arraycount(mib), procargs, &len, NULL, 0); + if (st == -1) { + free(procargs); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + *argsize = len; + return procargs; +} + + +// Return the command line as a python list object. +// XXX - most of the times sysctl() returns a truncated string. +// Also /proc/pid/cmdline behaves the same so it looks like this +// is a kernel bug. +PyObject * +psutil_get_cmdline(pid_t pid) { + char *argstr = NULL; + size_t pos = 0; + size_t argsize = 0; + PyObject *py_arg = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (pid == 0) + return py_retlist; + + argstr = psutil_get_cmd_args(pid, &argsize); + if (argstr == NULL) + goto error; + + // args are returned as a flattened string with \0 separators between + // arguments add each string to the list then step forward to the next + // separator + if (argsize > 0) { + while (pos < argsize) { + py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); + if (!py_arg) + goto error; + if (PyList_Append(py_retlist, py_arg)) + goto error; + Py_DECREF(py_arg); + pos = pos + strlen(&argstr[pos]) + 1; + } + } + + free(argstr); + return py_retlist; + +error: + Py_XDECREF(py_arg); + Py_DECREF(py_retlist); + if (argstr != NULL) + free(argstr); + return NULL; +} + + +/* + * Virtual memory stats, taken from: + * https://github.com/satterly/zabbix-stats/blob/master/src/libs/zbxsysinfo/ + * netbsd/memory.c + */ +PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + size_t size; + struct uvmexp_sysctl uv; + int mib[] = {CTL_VM, VM_UVMEXP2}; + long pagesize = getpagesize(); + + size = sizeof(uv); + if (sysctl(mib, 2, &uv, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("KKKKKKKK", + (unsigned long long) uv.npages << uv.pageshift, // total + (unsigned long long) uv.free << uv.pageshift, // free + (unsigned long long) uv.active << uv.pageshift, // active + (unsigned long long) uv.inactive << uv.pageshift, // inactive + (unsigned long long) uv.wired << uv.pageshift, // wired + (unsigned long long) uv.filepages + uv.execpages * pagesize, // cached + // These are determined from /proc/meminfo in Python. + (unsigned long long) 0, // buffers + (unsigned long long) 0 // shared + ); +} + + +PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { + uint64_t swap_total, swap_free; + struct swapent *swdev; + int nswap, i; + + nswap = swapctl(SWAP_NSWAP, 0, 0); + if (nswap == 0) { + // This means there's no swap partition. + return Py_BuildValue("(iiiii)", 0, 0, 0, 0, 0); + } + + swdev = calloc(nswap, sizeof(*swdev)); + if (swdev == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + if (swapctl(SWAP_STATS, swdev, nswap) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + // Total things up. + swap_total = swap_free = 0; + for (i = 0; i < nswap; i++) { + if (swdev[i].se_flags & SWF_ENABLE) { + swap_total += swdev[i].se_nblks * DEV_BSIZE; + swap_free += (swdev[i].se_nblks - swdev[i].se_inuse) * DEV_BSIZE; + } + } + free(swdev); + + // Get swap in/out + unsigned int total; + size_t size = sizeof(total); + struct uvmexp_sysctl uv; + int mib[] = {CTL_VM, VM_UVMEXP2}; + long pagesize = getpagesize(); + size = sizeof(uv); + if (sysctl(mib, 2, &uv, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + return Py_BuildValue("(LLLll)", + swap_total, + (swap_total - swap_free), + swap_free, + (long) uv.pgswapin * pagesize, // swap in + (long) uv.pgswapout * pagesize); // swap out + +error: + free(swdev); + return NULL; +} + + +PyObject * +psutil_proc_num_fds(PyObject *self, PyObject *args) { + long pid; + int cnt; + + struct kinfo_file *freep; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + return NULL; + } + free(freep); + + return Py_BuildValue("i", cnt); +} + + +PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + // XXX: why static? + int mib[3]; + int ncpu; + size_t len; + size_t size; + int i; + PyObject *py_cputime = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + // retrieve the number of cpus + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + len = sizeof(ncpu); + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + uint64_t cpu_time[CPUSTATES]; + + for (i = 0; i < ncpu; i++) { + // per-cpu info + mib[0] = CTL_KERN; + mib[1] = KERN_CP_TIME; + mib[2] = i; + size = sizeof(cpu_time); + if (sysctl(mib, 3, &cpu_time, &size, NULL, 0) == -1) { + warn("failed to get kern.cptime2"); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + py_cputime = Py_BuildValue( + "(ddddd)", + (double)cpu_time[CP_USER] / CLOCKS_PER_SEC, + (double)cpu_time[CP_NICE] / CLOCKS_PER_SEC, + (double)cpu_time[CP_SYS] / CLOCKS_PER_SEC, + (double)cpu_time[CP_IDLE] / CLOCKS_PER_SEC, + (double)cpu_time[CP_INTR] / CLOCKS_PER_SEC); + if (!py_cputime) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_DECREF(py_cputime); + } + + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + return NULL; +} + + +PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + int i, dk_ndrive, mib[3]; + size_t len; + struct io_sysctl *stats = NULL; + PyObject *py_disk_info = NULL; + PyObject *py_retdict = PyDict_New(); + + if (py_retdict == NULL) + return NULL; + mib[0] = CTL_HW; + mib[1] = HW_IOSTATS; + mib[2] = sizeof(struct io_sysctl); + len = 0; + if (sysctl(mib, 3, NULL, &len, NULL, 0) < 0) { + warn("can't get HW_IOSTATS"); + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + dk_ndrive = (int)(len / sizeof(struct io_sysctl)); + + stats = malloc(len); + if (stats == NULL) { + PyErr_NoMemory(); + goto error; + } + if (sysctl(mib, 3, stats, &len, NULL, 0) < 0 ) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < dk_ndrive; i++) { + py_disk_info = Py_BuildValue( + "(KKKK)", + stats[i].rxfer, + stats[i].wxfer, + stats[i].rbytes, + stats[i].wbytes + ); + if (!py_disk_info) + goto error; + if (PyDict_SetItemString(py_retdict, stats[i].name, py_disk_info)) + goto error; + Py_DECREF(py_disk_info); + } + + free(stats); + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + if (stats != NULL) + free(stats); + return NULL; +} + + +PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + size_t size; + struct uvmexp_sysctl uv; + int uvmexp_mib[] = {CTL_VM, VM_UVMEXP2}; + + size = sizeof(uv); + if (sysctl(uvmexp_mib, 2, &uv, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue( + "IIIIIII", + uv.swtch, // ctx switches + uv.intrs, // interrupts - XXX always 0, will be determined via /proc + uv.softs, // soft interrupts + uv.syscalls, // syscalls - XXX always 0 + uv.traps, // traps + uv.faults, // faults + uv.forks // forks + ); +} diff --git a/ddtrace/vendor/psutil/arch/netbsd/specific.h b/ddtrace/vendor/psutil/arch/netbsd/specific.h new file mode 100644 index 00000000000..391ed164a4c --- /dev/null +++ b/ddtrace/vendor/psutil/arch/netbsd/specific.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Landry Breuil. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +typedef struct kinfo_proc2 kinfo_proc; + +int psutil_kinfo_proc(pid_t pid, kinfo_proc *proc); +struct kinfo_file * kinfo_getfile(pid_t pid, int* cnt); +int psutil_get_proc_list(kinfo_proc **procList, size_t *procCount); +char *psutil_get_cmd_args(pid_t pid, size_t *argsize); + +// +PyObject *psutil_get_cmdline(pid_t pid); +PyObject *psutil_proc_threads(PyObject *self, PyObject *args); +PyObject *psutil_virtual_mem(PyObject *self, PyObject *args); +PyObject *psutil_swap_mem(PyObject *self, PyObject *args); +PyObject *psutil_proc_num_fds(PyObject *self, PyObject *args); +PyObject *psutil_proc_connections(PyObject *self, PyObject *args); +PyObject *psutil_per_cpu_times(PyObject *self, PyObject *args); +PyObject* psutil_disk_io_counters(PyObject* self, PyObject* args); +PyObject* psutil_proc_exe(PyObject* self, PyObject* args); +PyObject* psutil_proc_num_threads(PyObject* self, PyObject* args); +PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); +PyObject *psutil_proc_cwd(PyObject *self, PyObject *args); diff --git a/ddtrace/vendor/psutil/arch/openbsd/specific.c b/ddtrace/vendor/psutil/arch/openbsd/specific.c new file mode 100644 index 00000000000..33ebdeecbaf --- /dev/null +++ b/ddtrace/vendor/psutil/arch/openbsd/specific.c @@ -0,0 +1,791 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Landry Breuil. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Platform-specific module methods for OpenBSD. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for VFS_* +#include // for swap_mem +#include // for vmtotal struct +#include +#include +// connection stuff +#include // for NI_MAXHOST +#include +#include // for CPUSTATES & CP_* +#define _KERNEL // for DTYPE_* +#include +#undef _KERNEL +#include // struct diskstats +#include // for inet_ntoa() +#include // for warn() & err() + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + +#define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) +// #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) + + +// ============================================================================ +// Utility functions +// ============================================================================ + +int +psutil_kinfo_proc(pid_t pid, struct kinfo_proc *proc) { + // Fills a kinfo_proc struct based on process pid. + int ret; + int mib[6]; + size_t size = sizeof(struct kinfo_proc); + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = pid; + mib[4] = size; + mib[5] = 1; + + ret = sysctl((int*)mib, 6, proc, &size, NULL, 0); + if (ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + // sysctl stores 0 in the size if we can't find the process information. + if (size == 0) { + NoSuchProcess(""); + return -1; + } + return 0; +} + + +struct kinfo_file * +kinfo_getfile(long pid, int* cnt) { + // Mimic's FreeBSD kinfo_file call, taking a pid and a ptr to an + // int as arg and returns an array with cnt struct kinfo_file. + int mib[6]; + size_t len; + struct kinfo_file* kf; + mib[0] = CTL_KERN; + mib[1] = KERN_FILE; + mib[2] = KERN_FILE_BYPID; + mib[3] = (int) pid; + mib[4] = sizeof(struct kinfo_file); + mib[5] = 0; + + /* get the size of what would be returned */ + if (sysctl(mib, 6, NULL, &len, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + if ((kf = malloc(len)) == NULL) { + PyErr_NoMemory(); + return NULL; + } + mib[5] = (int)(len / sizeof(struct kinfo_file)); + if (sysctl(mib, 6, kf, &len, NULL, 0) < 0) { + free(kf); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + *cnt = (int)(len / sizeof(struct kinfo_file)); + return kf; +} + + +// ============================================================================ +// APIS +// ============================================================================ + +int +psutil_get_proc_list(struct kinfo_proc **procList, size_t *procCount) { + // Returns a list of all BSD processes on the system. This routine + // allocates the list and puts it in *procList and a count of the + // number of entries in *procCount. You are responsible for freeing + // this list (use "free" from System framework). + // On success, the function returns 0. + // On error, the function returns a BSD errno value. + struct kinfo_proc *result; + // Declaring name as const requires us to cast it when passing it to + // sysctl because the prototype doesn't include the const modifier. + char errbuf[_POSIX2_LINE_MAX]; + int cnt; + kvm_t *kd; + + assert(procList != NULL); + assert(*procList == NULL); + assert(procCount != NULL); + + kd = kvm_openfiles(NULL, NULL, NULL, KVM_NO_FILES, errbuf); + + if (kd == NULL) { + return errno; + } + + result = kvm_getprocs(kd, KERN_PROC_ALL, 0, sizeof(struct kinfo_proc), &cnt); + if (result == NULL) { + kvm_close(kd); + err(1, NULL); + return errno; + } + + *procCount = (size_t)cnt; + + size_t mlen = cnt * sizeof(struct kinfo_proc); + + if ((*procList = malloc(mlen)) == NULL) { + kvm_close(kd); + err(1, NULL); + return errno; + } + + memcpy(*procList, result, mlen); + assert(*procList != NULL); + kvm_close(kd); + + return 0; +} + + +char ** +_psutil_get_argv(long pid) { + static char **argv; + int argv_mib[] = {CTL_KERN, KERN_PROC_ARGS, pid, KERN_PROC_ARGV}; + size_t argv_size = 128; + // Loop and reallocate until we have enough space to fit argv. + for (;; argv_size *= 2) { + if (argv_size >= 8192) { + PyErr_SetString(PyExc_RuntimeError, + "can't allocate enough space for KERN_PROC_ARGV"); + return NULL; + } + if ((argv = realloc(argv, argv_size)) == NULL) + continue; + if (sysctl(argv_mib, 4, argv, &argv_size, NULL, 0) == 0) + return argv; + if (errno == ENOMEM) + continue; + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } +} + + +// returns the command line as a python list object +PyObject * +psutil_get_cmdline(long pid) { + static char **argv; + char **p; + PyObject *py_arg = NULL; + PyObject *py_retlist = Py_BuildValue("[]"); + + if (!py_retlist) + return NULL; + if (pid < 0) + return py_retlist; + + if ((argv = _psutil_get_argv(pid)) == NULL) + goto error; + + for (p = argv; *p != NULL; p++) { + py_arg = PyUnicode_DecodeFSDefault(*p); + if (!py_arg) + goto error; + if (PyList_Append(py_retlist, py_arg)) + goto error; + Py_DECREF(py_arg); + } + return py_retlist; + +error: + Py_XDECREF(py_arg); + Py_DECREF(py_retlist); + return NULL; +} + + +PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + // OpenBSD reference: + // https://github.com/janmojzis/pstree/blob/master/proc_kvm.c + // Note: this requires root access, else it will fail trying + // to access /dev/kmem. + long pid; + kvm_t *kd = NULL; + int nentries, i; + char errbuf[4096]; + struct kinfo_proc *kp; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + kd = kvm_openfiles(0, 0, 0, O_RDONLY, errbuf); + if (! kd) { + if (strstr(errbuf, "Permission denied") != NULL) + AccessDenied(""); + else + PyErr_Format(PyExc_RuntimeError, "kvm_openfiles() syscall failed"); + goto error; + } + + kp = kvm_getprocs( + kd, KERN_PROC_PID | KERN_PROC_SHOW_THREADS | KERN_PROC_KTHREAD, pid, + sizeof(*kp), &nentries); + if (! kp) { + if (strstr(errbuf, "Permission denied") != NULL) + AccessDenied(""); + else + PyErr_Format(PyExc_RuntimeError, "kvm_getprocs() syscall failed"); + goto error; + } + + for (i = 0; i < nentries; i++) { + if (kp[i].p_tid < 0) + continue; + if (kp[i].p_pid == pid) { + py_tuple = Py_BuildValue( + "Idd", + kp[i].p_tid, + PSUTIL_KPT2DOUBLE(kp[i].p_uutime), + PSUTIL_KPT2DOUBLE(kp[i].p_ustime)); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + } + + kvm_close(kd); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (kd != NULL) + kvm_close(kd); + return NULL; +} + + +PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + int64_t total_physmem; + int uvmexp_mib[] = {CTL_VM, VM_UVMEXP}; + int bcstats_mib[] = {CTL_VFS, VFS_GENERIC, VFS_BCACHESTAT}; + int physmem_mib[] = {CTL_HW, HW_PHYSMEM64}; + int vmmeter_mib[] = {CTL_VM, VM_METER}; + size_t size; + struct uvmexp uvmexp; + struct bcachestats bcstats; + struct vmtotal vmdata; + long pagesize = getpagesize(); + + size = sizeof(total_physmem); + if (sysctl(physmem_mib, 2, &total_physmem, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + size = sizeof(uvmexp); + if (sysctl(uvmexp_mib, 2, &uvmexp, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + size = sizeof(bcstats); + if (sysctl(bcstats_mib, 3, &bcstats, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + size = sizeof(vmdata); + if (sysctl(vmmeter_mib, 2, &vmdata, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("KKKKKKKK", + // Note: many programs calculate total memory as + // "uvmexp.npages * pagesize" but this is incorrect and does not + // match "sysctl | grep hw.physmem". + (unsigned long long) total_physmem, + (unsigned long long) uvmexp.free * pagesize, + (unsigned long long) uvmexp.active * pagesize, + (unsigned long long) uvmexp.inactive * pagesize, + (unsigned long long) uvmexp.wired * pagesize, + // this is how "top" determines it + (unsigned long long) bcstats.numbufpages * pagesize, // cached + (unsigned long long) 0, // buffers + (unsigned long long) vmdata.t_vmshr + vmdata.t_rmshr // shared + ); +} + + +PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { + uint64_t swap_total, swap_free; + struct swapent *swdev; + int nswap, i; + + if ((nswap = swapctl(SWAP_NSWAP, 0, 0)) == 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + if ((swdev = calloc(nswap, sizeof(*swdev))) == NULL) { + PyErr_NoMemory(); + return NULL; + } + + if (swapctl(SWAP_STATS, swdev, nswap) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + // Total things up. + swap_total = swap_free = 0; + for (i = 0; i < nswap; i++) { + if (swdev[i].se_flags & SWF_ENABLE) { + swap_free += (swdev[i].se_nblks - swdev[i].se_inuse); + swap_total += swdev[i].se_nblks; + } + } + + free(swdev); + return Py_BuildValue("(LLLII)", + swap_total * DEV_BSIZE, + (swap_total - swap_free) * DEV_BSIZE, + swap_free * DEV_BSIZE, + // swap in / swap out is not supported as the + // swapent struct does not provide any info + // about it. + 0, 0); + +error: + free(swdev); + return NULL; +} + + +PyObject * +psutil_proc_num_fds(PyObject *self, PyObject *args) { + long pid; + int cnt; + + struct kinfo_file *freep; + struct kinfo_proc kipp; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kipp) == -1) + return NULL; + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + return NULL; + } + free(freep); + + return Py_BuildValue("i", cnt); +} + + +PyObject * +psutil_proc_cwd(PyObject *self, PyObject *args) { + // Reference: + // https://github.com/openbsd/src/blob/ + // 588f7f8c69786211f2d16865c552afb91b1c7cba/bin/ps/print.c#L191 + long pid; + struct kinfo_proc kp; + char path[MAXPATHLEN]; + size_t pathlen = sizeof path; + + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + if (psutil_kinfo_proc(pid, &kp) == -1) + return NULL; + + int name[] = { CTL_KERN, KERN_PROC_CWD, pid }; + if (sysctl(name, 3, path, &pathlen, NULL, 0) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + return PyUnicode_DecodeFSDefault(path); +} + + +// see sys/kern/kern_sysctl.c lines 1100 and +// usr.bin/fstat/fstat.c print_inet_details() +static char * +psutil_convert_ipv4(int family, uint32_t addr[4]) { + struct in_addr a; + memcpy(&a, addr, sizeof(a)); + return inet_ntoa(a); +} + + +static char * +psutil_inet6_addrstr(struct in6_addr *p) +{ + struct sockaddr_in6 sin6; + static char hbuf[NI_MAXHOST]; + const int niflags = NI_NUMERICHOST; + + memset(&sin6, 0, sizeof(sin6)); + sin6.sin6_family = AF_INET6; + sin6.sin6_len = sizeof(struct sockaddr_in6); + sin6.sin6_addr = *p; + if (IN6_IS_ADDR_LINKLOCAL(p) && + *(u_int16_t *)&sin6.sin6_addr.s6_addr[2] != 0) { + sin6.sin6_scope_id = + ntohs(*(u_int16_t *)&sin6.sin6_addr.s6_addr[2]); + sin6.sin6_addr.s6_addr[2] = sin6.sin6_addr.s6_addr[3] = 0; + } + + if (getnameinfo((struct sockaddr *)&sin6, sin6.sin6_len, + hbuf, sizeof(hbuf), NULL, 0, niflags)) + return "invalid"; + + return hbuf; +} + + +/* + * List process connections. + * Note: there is no net_connections() on OpenBSD. The Python + * implementation will iterate over all processes and use this + * function. + * Note: local and remote paths cannot be determined for UNIX sockets. + */ +PyObject * +psutil_proc_connections(PyObject *self, PyObject *args) { + long pid; + int i; + int cnt; + struct kinfo_file *freep = NULL; + struct kinfo_file *kif; + char *tcplist = NULL; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + PyObject *py_af_filter = NULL; + PyObject *py_type_filter = NULL; + PyObject *py_family = NULL; + PyObject *_type = NULL; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "lOO", &pid, &py_af_filter, &py_type_filter)) + goto error; + if (!PySequence_Check(py_af_filter) || !PySequence_Check(py_type_filter)) { + PyErr_SetString(PyExc_TypeError, "arg 2 or 3 is not a sequence"); + goto error; + } + + errno = 0; + freep = kinfo_getfile(pid, &cnt); + if (freep == NULL) { + psutil_raise_for_pid(pid, "kinfo_getfile()"); + goto error; + } + + for (i = 0; i < cnt; i++) { + int state; + int lport; + int rport; + char addrbuf[NI_MAXHOST + 2]; + int inseq; + struct in6_addr laddr6; + py_tuple = NULL; + py_laddr = NULL; + py_raddr = NULL; + + kif = &freep[i]; + if (kif->f_type == DTYPE_SOCKET) { + // apply filters + py_family = PyLong_FromLong((long)kif->so_family); + inseq = PySequence_Contains(py_af_filter, py_family); + Py_DECREF(py_family); + if (inseq == 0) + continue; + _type = PyLong_FromLong((long)kif->so_type); + inseq = PySequence_Contains(py_type_filter, _type); + Py_DECREF(_type); + if (inseq == 0) + continue; + + // IPv4 / IPv6 socket + if ((kif->so_family == AF_INET) || (kif->so_family == AF_INET6)) { + // fill status + if (kif->so_type == SOCK_STREAM) + state = kif->t_state; + else + state = PSUTIL_CONN_NONE; + + // ports + lport = ntohs(kif->inp_lport); + rport = ntohs(kif->inp_fport); + + // local address, IPv4 + if (kif->so_family == AF_INET) { + py_laddr = Py_BuildValue( + "(si)", + psutil_convert_ipv4(kif->so_family, kif->inp_laddru), + lport); + if (!py_laddr) + goto error; + } + else { + // local address, IPv6 + memcpy(&laddr6, kif->inp_laddru, sizeof(laddr6)); + snprintf(addrbuf, sizeof(addrbuf), "%s", + psutil_inet6_addrstr(&laddr6)); + py_laddr = Py_BuildValue("(si)", addrbuf, lport); + if (!py_laddr) + goto error; + } + + if (rport != 0) { + // remote address, IPv4 + if (kif->so_family == AF_INET) { + py_raddr = Py_BuildValue( + "(si)", + psutil_convert_ipv4( + kif->so_family, kif->inp_faddru), + rport); + } + else { + // remote address, IPv6 + memcpy(&laddr6, kif->inp_faddru, sizeof(laddr6)); + snprintf(addrbuf, sizeof(addrbuf), "%s", + psutil_inet6_addrstr(&laddr6)); + py_raddr = Py_BuildValue("(si)", addrbuf, rport); + if (!py_raddr) + goto error; + } + } + else { + py_raddr = Py_BuildValue("()"); + } + + if (!py_raddr) + goto error; + py_tuple = Py_BuildValue( + "(iiiNNi)", + kif->fd_fd, + kif->so_family, + kif->so_type, + py_laddr, + py_raddr, + state); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + // UNIX socket. + // XXX: local addr is supposed to be in "unp_path" but it + // always empty; also "fstat" command is not able to show + // UNIX socket paths. + else if (kif->so_family == AF_UNIX) { + py_tuple = Py_BuildValue( + "(iiissi)", + kif->fd_fd, + kif->so_family, + kif->so_type, + "", // laddr (kif->unp_path is empty) + "", // raddr + PSUTIL_CONN_NONE); + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + Py_INCREF(Py_None); + } + } + } + free(freep); + free(tcplist); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + Py_DECREF(py_retlist); + if (freep != NULL) + free(freep); + if (tcplist != NULL) + free(tcplist); + return NULL; +} + + +PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + int mib[3]; + int ncpu; + size_t len; + size_t size; + int i; + PyObject *py_retlist = PyList_New(0); + PyObject *py_cputime = NULL; + + if (py_retlist == NULL) + return NULL; + + + // retrieve the number of cpus + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + len = sizeof(ncpu); + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + uint64_t cpu_time[CPUSTATES]; + + for (i = 0; i < ncpu; i++) { + // per-cpu info + mib[0] = CTL_KERN; + mib[1] = KERN_CPTIME2; + mib[2] = i; + size = sizeof(cpu_time); + if (sysctl(mib, 3, &cpu_time, &size, NULL, 0) == -1) { + warn("failed to get kern.cptime2"); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + py_cputime = Py_BuildValue( + "(ddddd)", + (double)cpu_time[CP_USER] / CLOCKS_PER_SEC, + (double)cpu_time[CP_NICE] / CLOCKS_PER_SEC, + (double)cpu_time[CP_SYS] / CLOCKS_PER_SEC, + (double)cpu_time[CP_IDLE] / CLOCKS_PER_SEC, + (double)cpu_time[CP_INTR] / CLOCKS_PER_SEC); + if (!py_cputime) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_DECREF(py_cputime); + } + + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + return NULL; +} + + +PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + int i, dk_ndrive, mib[3]; + size_t len; + struct diskstats *stats = NULL; + + PyObject *py_retdict = PyDict_New(); + PyObject *py_disk_info = NULL; + if (py_retdict == NULL) + return NULL; + + mib[0] = CTL_HW; + mib[1] = HW_DISKSTATS; + len = 0; + if (sysctl(mib, 2, NULL, &len, NULL, 0) < 0) { + warn("can't get hw.diskstats size"); + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + dk_ndrive = (int)(len / sizeof(struct diskstats)); + + stats = malloc(len); + if (stats == NULL) { + warn("can't malloc"); + PyErr_NoMemory(); + goto error; + } + if (sysctl(mib, 2, stats, &len, NULL, 0) < 0 ) { + warn("could not read hw.diskstats"); + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < dk_ndrive; i++) { + py_disk_info = Py_BuildValue( + "(KKKK)", + stats[i].ds_rxfer, // num reads + stats[i].ds_wxfer, // num writes + stats[i].ds_rbytes, // read bytes + stats[i].ds_wbytes // write bytes + ); + if (!py_disk_info) + goto error; + if (PyDict_SetItemString(py_retdict, stats[i].ds_name, py_disk_info)) + goto error; + Py_DECREF(py_disk_info); + } + + free(stats); + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + if (stats != NULL) + free(stats); + return NULL; +} + + +PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + size_t size; + struct uvmexp uv; + int uvmexp_mib[] = {CTL_VM, VM_UVMEXP}; + + size = sizeof(uv); + if (sysctl(uvmexp_mib, 2, &uv, &size, NULL, 0) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue( + "IIIIIII", + uv.swtch, // ctx switches + uv.intrs, // interrupts - XXX always 0, will be determined via /proc + uv.softs, // soft interrupts + uv.syscalls, // syscalls - XXX always 0 + uv.traps, // traps + uv.faults, // faults + uv.forks // forks + ); +} diff --git a/ddtrace/vendor/psutil/arch/openbsd/specific.h b/ddtrace/vendor/psutil/arch/openbsd/specific.h new file mode 100644 index 00000000000..4f870268d61 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/openbsd/specific.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Landry Breuil. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +typedef struct kinfo_proc kinfo_proc; + +int psutil_kinfo_proc(pid_t pid, struct kinfo_proc *proc); +struct kinfo_file * kinfo_getfile(long pid, int* cnt); +int psutil_get_proc_list(struct kinfo_proc **procList, size_t *procCount); +char **_psutil_get_argv(long pid); +PyObject * psutil_get_cmdline(long pid); + +// +PyObject *psutil_proc_threads(PyObject *self, PyObject *args); +PyObject *psutil_virtual_mem(PyObject *self, PyObject *args); +PyObject *psutil_swap_mem(PyObject *self, PyObject *args); +PyObject *psutil_proc_num_fds(PyObject *self, PyObject *args); +PyObject *psutil_proc_cwd(PyObject *self, PyObject *args); +PyObject *psutil_proc_connections(PyObject *self, PyObject *args); +PyObject *psutil_per_cpu_times(PyObject *self, PyObject *args); +PyObject* psutil_disk_io_counters(PyObject* self, PyObject* args); +PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); diff --git a/ddtrace/vendor/psutil/arch/osx/process_info.c b/ddtrace/vendor/psutil/arch/osx/process_info.c new file mode 100644 index 00000000000..d21c048edb3 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/osx/process_info.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Helper functions related to fetching process information. + * Used by _psutil_osx module methods. + */ + + +#include +#include +#include +#include // for INT_MAX +#include +#include +#include +#include +#include +#include + +#include "process_info.h" +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + +/* + * Returns a list of all BSD processes on the system. This routine + * allocates the list and puts it in *procList and a count of the + * number of entries in *procCount. You are responsible for freeing + * this list (use "free" from System framework). + * On success, the function returns 0. + * On error, the function returns a BSD errno value. + */ +int +psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { + int mib3[3] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL }; + size_t size, size2; + void *ptr; + int err; + int lim = 8; // some limit + + assert( procList != NULL); + assert(*procList == NULL); + assert(procCount != NULL); + + *procCount = 0; + + /* + * We start by calling sysctl with ptr == NULL and size == 0. + * That will succeed, and set size to the appropriate length. + * We then allocate a buffer of at least that size and call + * sysctl with that buffer. If that succeeds, we're done. + * If that call fails with ENOMEM, we throw the buffer away + * and try again. + * Note that the loop calls sysctl with NULL again. This is + * is necessary because the ENOMEM failure case sets size to + * the amount of data returned, not the amount of data that + * could have been returned. + */ + while (lim-- > 0) { + size = 0; + if (sysctl((int *)mib3, 3, NULL, &size, NULL, 0) == -1) { + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_ALL)"); + return 1; + } + size2 = size + (size >> 3); // add some + if (size2 > size) { + ptr = malloc(size2); + if (ptr == NULL) + ptr = malloc(size); + else + size = size2; + } + else { + ptr = malloc(size); + } + if (ptr == NULL) { + PyErr_NoMemory(); + return 1; + } + + if (sysctl((int *)mib3, 3, ptr, &size, NULL, 0) == -1) { + err = errno; + free(ptr); + if (err != ENOMEM) { + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_ALL)"); + return 1; + } + } + else { + *procList = (kinfo_proc *)ptr; + *procCount = size / sizeof(kinfo_proc); + if (procCount <= 0) { + PyErr_Format(PyExc_RuntimeError, "no PIDs found"); + return 1; + } + return 0; // success + } + } + + PyErr_Format(PyExc_RuntimeError, "couldn't collect PIDs list"); + return 1; +} + + +// Read the maximum argument size for processes +int +psutil_get_argmax() { + int argmax; + int mib[] = { CTL_KERN, KERN_ARGMAX }; + size_t size = sizeof(argmax); + + if (sysctl(mib, 2, &argmax, &size, NULL, 0) == 0) + return argmax; + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_ARGMAX)"); + return 0; +} + + +// Return 1 if pid refers to a zombie process else 0. +int +psutil_is_zombie(long pid) { + struct kinfo_proc kp; + + if (psutil_get_kinfo_proc(pid, &kp) == -1) + return 0; + return (kp.kp_proc.p_stat == SZOMB) ? 1 : 0; +} + + + +// return process args as a python list +PyObject * +psutil_get_cmdline(long pid) { + int mib[3]; + int nargs; + size_t len; + char *procargs = NULL; + char *arg_ptr; + char *arg_end; + char *curr_arg; + size_t argmax; + + PyObject *py_arg = NULL; + PyObject *py_retlist = NULL; + + // special case for PID 0 (kernel_task) where cmdline cannot be fetched + if (pid == 0) + return Py_BuildValue("[]"); + + // read argmax and allocate memory for argument space. + argmax = psutil_get_argmax(); + if (! argmax) + goto error; + + procargs = (char *)malloc(argmax); + if (NULL == procargs) { + PyErr_NoMemory(); + goto error; + } + + // read argument space + mib[0] = CTL_KERN; + mib[1] = KERN_PROCARGS2; + mib[2] = (pid_t)pid; + if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { + // In case of zombie process we'll get EINVAL. We translate it + // to NSP and _psosx.py will translate it to ZP. + if ((errno == EINVAL) && (psutil_pid_exists(pid))) + NoSuchProcess(""); + else + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + arg_end = &procargs[argmax]; + // copy the number of arguments to nargs + memcpy(&nargs, procargs, sizeof(nargs)); + + arg_ptr = procargs + sizeof(nargs); + len = strlen(arg_ptr); + arg_ptr += len + 1; + + if (arg_ptr == arg_end) { + free(procargs); + return Py_BuildValue("[]"); + } + + // skip ahead to the first argument + for (; arg_ptr < arg_end; arg_ptr++) { + if (*arg_ptr != '\0') + break; + } + + // iterate through arguments + curr_arg = arg_ptr; + py_retlist = Py_BuildValue("[]"); + if (!py_retlist) + goto error; + while (arg_ptr < arg_end && nargs > 0) { + if (*arg_ptr++ == '\0') { + py_arg = PyUnicode_DecodeFSDefault(curr_arg); + if (! py_arg) + goto error; + if (PyList_Append(py_retlist, py_arg)) + goto error; + Py_DECREF(py_arg); + // iterate to next arg and decrement # of args + curr_arg = arg_ptr; + nargs--; + } + } + + free(procargs); + return py_retlist; + +error: + Py_XDECREF(py_arg); + Py_XDECREF(py_retlist); + if (procargs != NULL) + free(procargs); + return NULL; +} + + +// return process environment as a python string +PyObject * +psutil_get_environ(long pid) { + int mib[3]; + int nargs; + char *procargs = NULL; + char *procenv = NULL; + char *arg_ptr; + char *arg_end; + char *env_start; + size_t argmax; + PyObject *py_ret = NULL; + + // special case for PID 0 (kernel_task) where cmdline cannot be fetched + if (pid == 0) + goto empty; + + // read argmax and allocate memory for argument space. + argmax = psutil_get_argmax(); + if (! argmax) + goto error; + + procargs = (char *)malloc(argmax); + if (NULL == procargs) { + PyErr_NoMemory(); + goto error; + } + + // read argument space + mib[0] = CTL_KERN; + mib[1] = KERN_PROCARGS2; + mib[2] = (pid_t)pid; + if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { + // In case of zombie process we'll get EINVAL. We translate it + // to NSP and _psosx.py will translate it to ZP. + if ((errno == EINVAL) && (psutil_pid_exists(pid))) + NoSuchProcess(""); + else + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + arg_end = &procargs[argmax]; + // copy the number of arguments to nargs + memcpy(&nargs, procargs, sizeof(nargs)); + + // skip executable path + arg_ptr = procargs + sizeof(nargs); + arg_ptr = memchr(arg_ptr, '\0', arg_end - arg_ptr); + + if (arg_ptr == NULL || arg_ptr == arg_end) + goto empty; + + // skip ahead to the first argument + for (; arg_ptr < arg_end; arg_ptr++) { + if (*arg_ptr != '\0') + break; + } + + // iterate through arguments + while (arg_ptr < arg_end && nargs > 0) { + if (*arg_ptr++ == '\0') + nargs--; + } + + // build an environment variable block + env_start = arg_ptr; + + procenv = calloc(1, arg_end - arg_ptr); + if (procenv == NULL) { + PyErr_NoMemory(); + goto error; + } + + while (*arg_ptr != '\0' && arg_ptr < arg_end) { + char *s = memchr(arg_ptr + 1, '\0', arg_end - arg_ptr); + + if (s == NULL) + break; + + memcpy(procenv + (arg_ptr - env_start), arg_ptr, s - arg_ptr); + + arg_ptr = s + 1; + } + + py_ret = PyUnicode_DecodeFSDefaultAndSize( + procenv, arg_ptr - env_start + 1); + if (!py_ret) { + // XXX: don't want to free() this as per: + // https://github.com/giampaolo/psutil/issues/926 + // It sucks but not sure what else to do. + procargs = NULL; + goto error; + } + + free(procargs); + free(procenv); + + return py_ret; + +empty: + if (procargs != NULL) + free(procargs); + return Py_BuildValue("s", ""); + +error: + Py_XDECREF(py_ret); + if (procargs != NULL) + free(procargs); + if (procenv != NULL) + free(procargs); + return NULL; +} + + +int +psutil_get_kinfo_proc(long pid, struct kinfo_proc *kp) { + int mib[4]; + size_t len; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = (pid_t)pid; + + // fetch the info with sysctl() + len = sizeof(struct kinfo_proc); + + // now read the data from sysctl + if (sysctl(mib, 4, kp, &len, NULL, 0) == -1) { + // raise an exception and throw errno as the error + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + // sysctl succeeds but len is zero, happens when process has gone away + if (len == 0) { + NoSuchProcess(""); + return -1; + } + return 0; +} + + +/* + * A wrapper around proc_pidinfo(). + * Returns 0 on failure (and Python exception gets already set). + */ +int +psutil_proc_pidinfo(long pid, int flavor, uint64_t arg, void *pti, int size) { + errno = 0; + int ret = proc_pidinfo((int)pid, flavor, arg, pti, size); + if ((ret <= 0) || ((unsigned long)ret < sizeof(pti))) { + psutil_raise_for_pid(pid, "proc_pidinfo()"); + return 0; + } + return ret; +} diff --git a/ddtrace/vendor/psutil/arch/osx/process_info.h b/ddtrace/vendor/psutil/arch/osx/process_info.h new file mode 100644 index 00000000000..bd7ffa89caf --- /dev/null +++ b/ddtrace/vendor/psutil/arch/osx/process_info.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +typedef struct kinfo_proc kinfo_proc; + +int psutil_get_argmax(void); +int psutil_is_zombie(long pid); +int psutil_get_kinfo_proc(long pid, struct kinfo_proc *kp); +int psutil_get_proc_list(kinfo_proc **procList, size_t *procCount); +int psutil_proc_pidinfo( + long pid, int flavor, uint64_t arg, void *pti, int size); +PyObject* psutil_get_cmdline(long pid); +PyObject* psutil_get_environ(long pid); diff --git a/ddtrace/vendor/psutil/arch/solaris/environ.c b/ddtrace/vendor/psutil/arch/solaris/environ.c new file mode 100644 index 00000000000..1af4c129344 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/solaris/environ.c @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Oleksii Shevchuk. + * All rights reserved. Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + * + * Functions specific for Process.environ(). + */ + +#define NEW_MIB_COMPLIANT 1 +#define _STRUCTURED_PROC 1 + +#include + +#if !defined(_LP64) && _FILE_OFFSET_BITS == 64 +# undef _FILE_OFFSET_BITS +# undef _LARGEFILE64_SOURCE +#endif + +#include +#include +#include +#include + +#include "environ.h" + + +#define STRING_SEARCH_BUF_SIZE 512 + + +/* + * Open address space of specified process and return file descriptor. + * @param pid a pid of process. + * @param procfs_path a path to mounted procfs filesystem. + * @return file descriptor or -1 in case of error. + */ +static int +open_address_space(pid_t pid, const char *procfs_path) { + int fd; + char proc_path[PATH_MAX]; + + snprintf(proc_path, PATH_MAX, "%s/%i/as", procfs_path, pid); + fd = open(proc_path, O_RDONLY); + if (fd < 0) + PyErr_SetFromErrno(PyExc_OSError); + + return fd; +} + + +/* + * Read chunk of data by offset to specified buffer of the same size. + * @param fd a file descriptor. + * @param offset an required offset in file. + * @param buf a buffer where to store result. + * @param buf_size a size of buffer where data will be stored. + * @return amount of bytes stored to the buffer or -1 in case of + * error. + */ +static int +read_offt(int fd, off_t offset, char *buf, size_t buf_size) { + size_t to_read = buf_size; + size_t stored = 0; + int r; + + while (to_read) { + r = pread(fd, buf + stored, to_read, offset + stored); + if (r < 0) + goto error; + else if (r == 0) + break; + + to_read -= r; + stored += r; + } + + return stored; + + error: + PyErr_SetFromErrno(PyExc_OSError); + return -1; +} + + +/* + * Read null-terminated string from file descriptor starting from + * specified offset. + * @param fd a file descriptor of opened address space. + * @param offset an offset in specified file descriptor. + * @return allocated null-terminated string or NULL in case of error. +*/ +static char * +read_cstring_offt(int fd, off_t offset) { + int r; + int i = 0; + off_t end = offset; + size_t len; + char buf[STRING_SEARCH_BUF_SIZE]; + char *result = NULL; + + if (lseek(fd, offset, SEEK_SET) == (off_t)-1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + // Search end of string + for (;;) { + r = read(fd, buf, sizeof(buf)); + if (r == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + else if (r == 0) { + break; + } + else { + for (i=0; i= 0 && count) + *count = env_count; + + if (env_count > 0) + result = read_cstrings_block( + as, info.pr_envp, ptr_size, env_count); + + close(as); + return result; +} + + +/* + * Free array of cstrings. + * @param array an array of cstrings returned by psutil_read_raw_env, + * psutil_read_raw_args or any other function. + * @param count a count of strings in the passed array + */ +void +psutil_free_cstrings_array(char **array, size_t count) { + int i; + + if (!array) + return; + for (i=0; i +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ifaddrs.h" + +#define MAX(x,y) ((x)>(y)?(x):(y)) +#define SIZE(p) MAX((p).ss_len,sizeof(p)) + + +static struct sockaddr * +sa_dup (struct sockaddr_storage *sa1) +{ + struct sockaddr *sa2; + size_t sz = sizeof(struct sockaddr_storage); + sa2 = (struct sockaddr *) calloc(1,sz); + memcpy(sa2,sa1,sz); + return(sa2); +} + + +void freeifaddrs (struct ifaddrs *ifp) +{ + if (NULL == ifp) return; + free(ifp->ifa_name); + free(ifp->ifa_addr); + free(ifp->ifa_netmask); + free(ifp->ifa_dstaddr); + freeifaddrs(ifp->ifa_next); + free(ifp); +} + + +int getifaddrs (struct ifaddrs **ifap) +{ + int sd = -1; + char *ccp, *ecp; + struct lifconf ifc; + struct lifreq *ifr; + struct lifnum lifn; + struct ifaddrs *cifa = NULL; /* current */ + struct ifaddrs *pifa = NULL; /* previous */ + const size_t IFREQSZ = sizeof(struct lifreq); + + sd = socket(AF_INET, SOCK_STREAM, 0); + if (sd < 0) + goto error; + + ifc.lifc_buf = NULL; + *ifap = NULL; + /* find how much memory to allocate for the SIOCGLIFCONF call */ + lifn.lifn_family = AF_UNSPEC; + lifn.lifn_flags = 0; + if (ioctl(sd, SIOCGLIFNUM, &lifn) < 0) + goto error; + + /* Sun and Apple code likes to pad the interface count here in case interfaces + * are coming up between calls */ + lifn.lifn_count += 4; + + ifc.lifc_family = AF_UNSPEC; + ifc.lifc_len = lifn.lifn_count * sizeof(struct lifreq); + ifc.lifc_buf = calloc(1, ifc.lifc_len); + if (ioctl(sd, SIOCGLIFCONF, &ifc) < 0) + goto error; + + ccp = (char *)ifc.lifc_req; + ecp = ccp + ifc.lifc_len; + + while (ccp < ecp) { + + ifr = (struct lifreq *) ccp; + cifa = (struct ifaddrs *) calloc(1, sizeof(struct ifaddrs)); + cifa->ifa_next = NULL; + cifa->ifa_name = strdup(ifr->lifr_name); + + if (pifa == NULL) *ifap = cifa; /* first one */ + else pifa->ifa_next = cifa; + + if (ioctl(sd, SIOCGLIFADDR, ifr, IFREQSZ) < 0) + goto error; + cifa->ifa_addr = sa_dup(&ifr->lifr_addr); + + if (ioctl(sd, SIOCGLIFNETMASK, ifr, IFREQSZ) < 0) + goto error; + cifa->ifa_netmask = sa_dup(&ifr->lifr_addr); + + cifa->ifa_flags = 0; + cifa->ifa_dstaddr = NULL; + + if (0 == ioctl(sd, SIOCGLIFFLAGS, ifr)) /* optional */ + cifa->ifa_flags = ifr->lifr_flags; + + if (ioctl(sd, SIOCGLIFDSTADDR, ifr, IFREQSZ) < 0) { + if (0 == ioctl(sd, SIOCGLIFBRDADDR, ifr, IFREQSZ)) + cifa->ifa_dstaddr = sa_dup(&ifr->lifr_addr); + } + else cifa->ifa_dstaddr = sa_dup(&ifr->lifr_addr); + + pifa = cifa; + ccp += IFREQSZ; + } + free(ifc.lifc_buf); + close(sd); + return 0; +error: + if (ifc.lifc_buf != NULL) + free(ifc.lifc_buf); + if (sd != -1) + close(sd); + freeifaddrs(*ifap); + return (-1); +} diff --git a/ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.h b/ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.h new file mode 100644 index 00000000000..0953a9b99a0 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.h @@ -0,0 +1,26 @@ +/* Reference: https://lists.samba.org/archive/samba-technical/2009-February/063079.html */ + + +#ifndef __IFADDRS_H__ +#define __IFADDRS_H__ + +#include +#include + +#undef ifa_dstaddr +#undef ifa_broadaddr +#define ifa_broadaddr ifa_dstaddr + +struct ifaddrs { + struct ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + struct sockaddr *ifa_addr; + struct sockaddr *ifa_netmask; + struct sockaddr *ifa_dstaddr; +}; + +extern int getifaddrs(struct ifaddrs **); +extern void freeifaddrs(struct ifaddrs *); + +#endif diff --git a/ddtrace/vendor/psutil/arch/windows/global.c b/ddtrace/vendor/psutil/arch/windows/global.c new file mode 100644 index 00000000000..4d8526e31d6 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/global.c @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * This code is executed on import. It loads private/undocumented + * Windows APIs and sets Windows version constants so that they are + * available globally. + */ + +#include +#include +#include "ntextapi.h" +#include "global.h" + + +// Needed to make these globally visible. +int PSUTIL_WINVER; +SYSTEM_INFO PSUTIL_SYSTEM_INFO; + +#define NT_FACILITY_MASK 0xfff +#define NT_FACILITY_SHIFT 16 +#define NT_FACILITY(Status) \ + ((((ULONG)(Status)) >> NT_FACILITY_SHIFT) & NT_FACILITY_MASK) +#define NT_NTWIN32(status) (NT_FACILITY(Status) == FACILITY_WIN32) +#define WIN32_FROM_NTSTATUS(Status) (((ULONG)(Status)) & 0xffff) + + +// A wrapper around GetModuleHandle and GetProcAddress. +PVOID +psutil_GetProcAddress(LPCSTR libname, LPCSTR procname) { + HMODULE mod; + FARPROC addr; + + if ((mod = GetModuleHandleA(libname)) == NULL) { + PyErr_SetFromWindowsErrWithFilename(0, libname); + return NULL; + } + if ((addr = GetProcAddress(mod, procname)) == NULL) { + PyErr_SetFromWindowsErrWithFilename(0, procname); + return NULL; + } + return addr; +} + + +// A wrapper around LoadLibrary and GetProcAddress. +PVOID +psutil_GetProcAddressFromLib(LPCSTR libname, LPCSTR procname) { + HMODULE mod; + FARPROC addr; + + Py_BEGIN_ALLOW_THREADS + mod = LoadLibraryA(libname); + Py_END_ALLOW_THREADS + if (mod == NULL) { + PyErr_SetFromWindowsErrWithFilename(0, libname); + return NULL; + } + if ((addr = GetProcAddress(mod, procname)) == NULL) { + PyErr_SetFromWindowsErrWithFilename(0, procname); + FreeLibrary(mod); + return NULL; + } + // Causes crash. + // FreeLibrary(mod); + return addr; +} + + +/* + * Convert a NTSTATUS value to a Win32 error code and set the proper + * Python exception. + */ +PVOID +psutil_SetFromNTStatusErr(NTSTATUS Status, const char *syscall) { + ULONG err; + char fullmsg[1024]; + + if (NT_NTWIN32(Status)) + err = WIN32_FROM_NTSTATUS(Status); + else + err = psutil_RtlNtStatusToDosErrorNoTeb(Status); + // if (GetLastError() != 0) + // err = GetLastError(); + sprintf(fullmsg, "(originated from %s)", syscall); + return PyErr_SetFromWindowsErrWithFilename(err, fullmsg); +} + + +static int +psutil_loadlibs() { + /* + * Mandatory. + */ + psutil_NtQuerySystemInformation = psutil_GetProcAddressFromLib( + "ntdll.dll", "NtQuerySystemInformation"); + if (psutil_NtQuerySystemInformation == NULL) + return 1; + + psutil_NtQueryInformationProcess = psutil_GetProcAddress( + "ntdll.dll", "NtQueryInformationProcess"); + if (! psutil_NtQueryInformationProcess) + return 1; + + psutil_NtSetInformationProcess = psutil_GetProcAddress( + "ntdll.dll", "NtSetInformationProcess"); + if (! psutil_NtSetInformationProcess) + return 1; + + psutil_WinStationQueryInformationW = psutil_GetProcAddressFromLib( + "winsta.dll", "WinStationQueryInformationW"); + if (! psutil_WinStationQueryInformationW) + return 1; + + psutil_NtQueryObject = psutil_GetProcAddressFromLib( + "ntdll.dll", "NtQueryObject"); + if (! psutil_NtQueryObject) + return 1; + + psutil_rtlIpv4AddressToStringA = psutil_GetProcAddressFromLib( + "ntdll.dll", "RtlIpv4AddressToStringA"); + if (! psutil_rtlIpv4AddressToStringA) + return 1; + + // minimum requirement: Win XP SP3 + psutil_GetExtendedTcpTable = psutil_GetProcAddressFromLib( + "iphlpapi.dll", "GetExtendedTcpTable"); + if (! psutil_GetExtendedTcpTable) + return 1; + + // minimum requirement: Win XP SP3 + psutil_GetExtendedUdpTable = psutil_GetProcAddressFromLib( + "iphlpapi.dll", "GetExtendedUdpTable"); + if (! psutil_GetExtendedUdpTable) + return 1; + + psutil_RtlGetVersion = psutil_GetProcAddressFromLib( + "ntdll.dll", "RtlGetVersion"); + if (! psutil_RtlGetVersion) + return 1; + + psutil_NtSuspendProcess = psutil_GetProcAddressFromLib( + "ntdll", "NtSuspendProcess"); + if (! psutil_NtSuspendProcess) + return 1; + + psutil_NtResumeProcess = psutil_GetProcAddressFromLib( + "ntdll", "NtResumeProcess"); + if (! psutil_NtResumeProcess) + return 1; + + psutil_NtQueryVirtualMemory = psutil_GetProcAddressFromLib( + "ntdll", "NtQueryVirtualMemory"); + if (! psutil_NtQueryVirtualMemory) + return 1; + + psutil_RtlNtStatusToDosErrorNoTeb = psutil_GetProcAddressFromLib( + "ntdll", "RtlNtStatusToDosErrorNoTeb"); + if (! psutil_RtlNtStatusToDosErrorNoTeb) + return 1; + + /* + * Optional. + */ + // not available on Wine + psutil_rtlIpv6AddressToStringA = psutil_GetProcAddressFromLib( + "ntdll.dll", "RtlIpv6AddressToStringA"); + + // minimum requirement: Win Vista + psutil_GetTickCount64 = psutil_GetProcAddress( + "kernel32", "GetTickCount64"); + + // minimum requirement: Win 7 + psutil_GetActiveProcessorCount = psutil_GetProcAddress( + "kernel32", "GetActiveProcessorCount"); + + // minumum requirement: Win 7 + psutil_GetLogicalProcessorInformationEx = psutil_GetProcAddressFromLib( + "kernel32", "GetLogicalProcessorInformationEx"); + + PyErr_Clear(); + return 0; +} + + +static int +psutil_set_winver() { + RTL_OSVERSIONINFOEXW versionInfo; + ULONG maj; + ULONG min; + + versionInfo.dwOSVersionInfoSize = sizeof(RTL_OSVERSIONINFOEXW); + memset(&versionInfo, 0, sizeof(RTL_OSVERSIONINFOEXW)); + psutil_RtlGetVersion((PRTL_OSVERSIONINFOW)&versionInfo); + maj = versionInfo.dwMajorVersion; + min = versionInfo.dwMinorVersion; + if (maj == 5 && min == 1) + PSUTIL_WINVER = PSUTIL_WINDOWS_XP; + else if (maj == 5 && min == 2) + PSUTIL_WINVER = PSUTIL_WINDOWS_SERVER_2003; + else if (maj == 6 && min == 0) + PSUTIL_WINVER = PSUTIL_WINDOWS_VISTA; // or Server 2008 + else if (maj == 6 && min == 1) + PSUTIL_WINVER = PSUTIL_WINDOWS_7; + else if (maj == 6 && min == 2) + PSUTIL_WINVER = PSUTIL_WINDOWS_8; + else if (maj == 6 && min == 3) + PSUTIL_WINVER = PSUTIL_WINDOWS_8_1; + else if (maj == 10 && min == 0) + PSUTIL_WINVER = PSUTIL_WINDOWS_10; + else + PSUTIL_WINVER = PSUTIL_WINDOWS_NEW; + return 0; +} + + +static int +psutil_load_sysinfo() { + GetSystemInfo(&PSUTIL_SYSTEM_INFO); + return 0; +} + + +int +psutil_load_globals() { + if (psutil_loadlibs() != 0) + return 1; + if (psutil_set_winver() != 0) + return 1; + if (psutil_load_sysinfo() != 0) + return 1; + return 0; +} diff --git a/ddtrace/vendor/psutil/arch/windows/global.h b/ddtrace/vendor/psutil/arch/windows/global.h new file mode 100644 index 00000000000..10ae640595d --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/global.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + + * List of constants and objects that are globally available. + */ + +#include +#include "ntextapi.h" + +extern int PSUTIL_WINVER; +extern SYSTEM_INFO PSUTIL_SYSTEM_INFO; +#define PSUTIL_WINDOWS_XP 51 +#define PSUTIL_WINDOWS_SERVER_2003 52 +#define PSUTIL_WINDOWS_VISTA 60 +#define PSUTIL_WINDOWS_7 61 +#define PSUTIL_WINDOWS_8 62 +#define PSUTIL_WINDOWS_8_1 63 +#define PSUTIL_WINDOWS_10 100 +#define PSUTIL_WINDOWS_NEW MAXLONG + +int psutil_load_globals(); +PVOID psutil_GetProcAddress(LPCSTR libname, LPCSTR procname); +PVOID psutil_GetProcAddressFromLib(LPCSTR libname, LPCSTR procname); +PVOID psutil_SetFromNTStatusErr(NTSTATUS Status, const char *syscall); + +_NtQuerySystemInformation \ + psutil_NtQuerySystemInformation; + +_NtQueryInformationProcess \ + psutil_NtQueryInformationProcess; + +_NtSetInformationProcess + psutil_NtSetInformationProcess; + +_WinStationQueryInformationW \ + psutil_WinStationQueryInformationW; + +_RtlIpv4AddressToStringA \ + psutil_rtlIpv4AddressToStringA; + +_RtlIpv6AddressToStringA \ + psutil_rtlIpv6AddressToStringA; + +_GetExtendedTcpTable \ + psutil_GetExtendedTcpTable; + +_GetExtendedUdpTable \ + psutil_GetExtendedUdpTable; + +_GetActiveProcessorCount \ + psutil_GetActiveProcessorCount; + +_GetTickCount64 \ + psutil_GetTickCount64; + +_NtQueryObject \ + psutil_NtQueryObject; + +_GetLogicalProcessorInformationEx \ + psutil_GetLogicalProcessorInformationEx; + +_RtlGetVersion \ + psutil_RtlGetVersion; + +_NtSuspendProcess \ + psutil_NtSuspendProcess; + +_NtResumeProcess \ + psutil_NtResumeProcess; + +_NtQueryVirtualMemory \ + psutil_NtQueryVirtualMemory; + +_RtlNtStatusToDosErrorNoTeb \ + psutil_RtlNtStatusToDosErrorNoTeb; diff --git a/ddtrace/vendor/psutil/arch/windows/inet_ntop.c b/ddtrace/vendor/psutil/arch/windows/inet_ntop.c new file mode 100644 index 00000000000..3db507b9ba0 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/inet_ntop.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Jeff Tang. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include "inet_ntop.h" + +// From: https://memset.wordpress.com/2010/10/09/inet_ntop-for-win32/ +PCSTR WSAAPI +inet_ntop(INT family, PVOID pAddr, PSTR stringBuf, size_t strBufSize) { + DWORD dwAddressLength = 0; + struct sockaddr_storage srcaddr; + struct sockaddr_in *srcaddr4 = (struct sockaddr_in*) &srcaddr; + struct sockaddr_in6 *srcaddr6 = (struct sockaddr_in6*) &srcaddr; + + memset(&srcaddr, 0, sizeof(struct sockaddr_storage)); + srcaddr.ss_family = family; + + if (family == AF_INET) { + dwAddressLength = sizeof(struct sockaddr_in); + memcpy(&(srcaddr4->sin_addr), pAddr, sizeof(struct in_addr)); + } + else if (family == AF_INET6) { + dwAddressLength = sizeof(struct sockaddr_in6); + memcpy(&(srcaddr6->sin6_addr), pAddr, sizeof(struct in6_addr)); + } + else { + PyErr_SetString(PyExc_ValueError, "invalid family"); + return NULL; + } + + if (WSAAddressToStringA( + (LPSOCKADDR) &srcaddr, + dwAddressLength, + 0, + stringBuf, + (LPDWORD) &strBufSize) != 0) + { + PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); + return NULL; + } + return stringBuf; +} diff --git a/ddtrace/vendor/psutil/arch/windows/inet_ntop.h b/ddtrace/vendor/psutil/arch/windows/inet_ntop.h new file mode 100644 index 00000000000..2d86c26cf18 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/inet_ntop.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Jeff Tang. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +// because of WSAAddressToStringA +#define _WINSOCK_DEPRECATED_NO_WARNINGS +#include + +PCSTR WSAAPI +inet_ntop( + __in INT Family, + __in const VOID * pAddr, + __out_ecount(StringBufSize) PSTR pStringBuf, + __in size_t StringBufSize +); diff --git a/ddtrace/vendor/psutil/arch/windows/ntextapi.h b/ddtrace/vendor/psutil/arch/windows/ntextapi.h new file mode 100644 index 00000000000..b6f23d99676 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/ntextapi.h @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ +#if !defined(__NTEXTAPI_H__) +#define __NTEXTAPI_H__ +#include +#include + +typedef LONG NTSTATUS; + +#define STATUS_INFO_LENGTH_MISMATCH 0xc0000004 +#define STATUS_BUFFER_TOO_SMALL 0xC0000023L +#define SystemExtendedHandleInformation 64 +#define MemoryWorkingSetInformation 0x1 +#define STATUS_ACCESS_DENIED ((NTSTATUS)0xC0000022L) + +/* + * ================================================================ + * Enums. + * ================================================================ + */ + +typedef enum _PROCESSINFOCLASS2 { + _ProcessBasicInformation, + ProcessQuotaLimits, + ProcessIoCounters, + ProcessVmCounters, + ProcessTimes, + ProcessBasePriority, + ProcessRaisePriority, + _ProcessDebugPort, + ProcessExceptionPort, + ProcessAccessToken, + ProcessLdtInformation, + ProcessLdtSize, + ProcessDefaultHardErrorMode, + ProcessIoPortHandlers, + ProcessPooledUsageAndLimits, + ProcessWorkingSetWatch, + ProcessUserModeIOPL, + ProcessEnableAlignmentFaultFixup, + ProcessPriorityClass, + ProcessWx86Information, + ProcessHandleCount, + ProcessAffinityMask, + ProcessPriorityBoost, + ProcessDeviceMap, + ProcessSessionInformation, + ProcessForegroundInformation, + _ProcessWow64Information, + /* added after XP+ */ + _ProcessImageFileName, + ProcessLUIDDeviceMapsEnabled, + _ProcessBreakOnTermination, + ProcessDebugObjectHandle, + ProcessDebugFlags, + ProcessHandleTracing, + ProcessIoPriority, + ProcessExecuteFlags, + ProcessResourceManagement, + ProcessCookie, + ProcessImageInformation, + MaxProcessInfoClass +} PROCESSINFOCLASS2; + +#define PROCESSINFOCLASS PROCESSINFOCLASS2 +#define ProcessBasicInformation _ProcessBasicInformation +#define ProcessWow64Information _ProcessWow64Information +#define ProcessDebugPort _ProcessDebugPort +#define ProcessImageFileName _ProcessImageFileName +#define ProcessBreakOnTermination _ProcessBreakOnTermination + +/* + * ================================================================ + * Structs. + * ================================================================ + */ + +typedef struct { + LARGE_INTEGER IdleTime; + LARGE_INTEGER KernelTime; + LARGE_INTEGER UserTime; + LARGE_INTEGER DpcTime; + LARGE_INTEGER InterruptTime; + ULONG InterruptCount; +} _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION; + +typedef struct { + LARGE_INTEGER IdleProcessTime; + LARGE_INTEGER IoReadTransferCount; + LARGE_INTEGER IoWriteTransferCount; + LARGE_INTEGER IoOtherTransferCount; + ULONG IoReadOperationCount; + ULONG IoWriteOperationCount; + ULONG IoOtherOperationCount; + ULONG AvailablePages; + ULONG CommittedPages; + ULONG CommitLimit; + ULONG PeakCommitment; + ULONG PageFaultCount; + ULONG CopyOnWriteCount; + ULONG TransitionCount; + ULONG CacheTransitionCount; + ULONG DemandZeroCount; + ULONG PageReadCount; + ULONG PageReadIoCount; + ULONG CacheReadCount; + ULONG CacheIoCount; + ULONG DirtyPagesWriteCount; + ULONG DirtyWriteIoCount; + ULONG MappedPagesWriteCount; + ULONG MappedWriteIoCount; + ULONG PagedPoolPages; + ULONG NonPagedPoolPages; + ULONG PagedPoolAllocs; + ULONG PagedPoolFrees; + ULONG NonPagedPoolAllocs; + ULONG NonPagedPoolFrees; + ULONG FreeSystemPtes; + ULONG ResidentSystemCodePage; + ULONG TotalSystemDriverPages; + ULONG TotalSystemCodePages; + ULONG NonPagedPoolLookasideHits; + ULONG PagedPoolLookasideHits; + ULONG AvailablePagedPoolPages; + ULONG ResidentSystemCachePage; + ULONG ResidentPagedPoolPage; + ULONG ResidentSystemDriverPage; + ULONG CcFastReadNoWait; + ULONG CcFastReadWait; + ULONG CcFastReadResourceMiss; + ULONG CcFastReadNotPossible; + ULONG CcFastMdlReadNoWait; + ULONG CcFastMdlReadWait; + ULONG CcFastMdlReadResourceMiss; + ULONG CcFastMdlReadNotPossible; + ULONG CcMapDataNoWait; + ULONG CcMapDataWait; + ULONG CcMapDataNoWaitMiss; + ULONG CcMapDataWaitMiss; + ULONG CcPinMappedDataCount; + ULONG CcPinReadNoWait; + ULONG CcPinReadWait; + ULONG CcPinReadNoWaitMiss; + ULONG CcPinReadWaitMiss; + ULONG CcCopyReadNoWait; + ULONG CcCopyReadWait; + ULONG CcCopyReadNoWaitMiss; + ULONG CcCopyReadWaitMiss; + ULONG CcMdlReadNoWait; + ULONG CcMdlReadWait; + ULONG CcMdlReadNoWaitMiss; + ULONG CcMdlReadWaitMiss; + ULONG CcReadAheadIos; + ULONG CcLazyWriteIos; + ULONG CcLazyWritePages; + ULONG CcDataFlushes; + ULONG CcDataPages; + ULONG ContextSwitches; + ULONG FirstLevelTbFills; + ULONG SecondLevelTbFills; + ULONG SystemCalls; +} _SYSTEM_PERFORMANCE_INFORMATION; + +typedef struct { + ULONG ContextSwitches; + ULONG DpcCount; + ULONG DpcRate; + ULONG TimeIncrement; + ULONG DpcBypassCount; + ULONG ApcBypassCount; +} _SYSTEM_INTERRUPT_INFORMATION; + +typedef enum _KTHREAD_STATE { + Initialized, + Ready, + Running, + Standby, + Terminated, + Waiting, + Transition, + DeferredReady, + GateWait, + MaximumThreadState +} KTHREAD_STATE, *PKTHREAD_STATE; + +typedef enum _KWAIT_REASON { + Executive, + FreePage, + PageIn, + PoolAllocation, + DelayExecution, + Suspended, + UserRequest, + WrExecutive, + WrFreePage, + WrPageIn, + WrPoolAllocation, + WrDelayExecution, + WrSuspended, + WrUserRequest, + WrEventPair, + WrQueue, + WrLpcReceive, + WrLpcReply, + WrVirtualMemory, + WrPageOut, + WrRendezvous, + WrKeyedEvent, + WrTerminated, + WrProcessInSwap, + WrCpuRateControl, + WrCalloutStack, + WrKernel, + WrResource, + WrPushLock, + WrMutex, + WrQuantumEnd, + WrDispatchInt, + WrPreempted, + WrYieldExecution, + WrFastMutex, + WrGuardedMutex, + WrRundown, + WrAlertByThreadId, + WrDeferredPreempt, + MaximumWaitReason +} KWAIT_REASON, *PKWAIT_REASON; + +typedef struct _SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX { + PVOID Object; + HANDLE UniqueProcessId; + HANDLE HandleValue; + ULONG GrantedAccess; + USHORT CreatorBackTraceIndex; + USHORT ObjectTypeIndex; + ULONG HandleAttributes; + ULONG Reserved; +} SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX, *PSYSTEM_HANDLE_TABLE_ENTRY_INFO_EX; + +typedef struct _SYSTEM_HANDLE_INFORMATION_EX { + ULONG_PTR NumberOfHandles; + ULONG_PTR Reserved; + SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX Handles[1]; +} SYSTEM_HANDLE_INFORMATION_EX, *PSYSTEM_HANDLE_INFORMATION_EX; + +typedef struct _CLIENT_ID2 { + HANDLE UniqueProcess; + HANDLE UniqueThread; +} CLIENT_ID2, *PCLIENT_ID2; + +#define CLIENT_ID CLIENT_ID2 +#define PCLIENT_ID PCLIENT_ID2 + +typedef struct _SYSTEM_THREAD_INFORMATION2 { + LARGE_INTEGER KernelTime; + LARGE_INTEGER UserTime; + LARGE_INTEGER CreateTime; + ULONG WaitTime; + PVOID StartAddress; + CLIENT_ID ClientId; + LONG Priority; + LONG BasePriority; + ULONG ContextSwitches; + ULONG ThreadState; + KWAIT_REASON WaitReason; +} SYSTEM_THREAD_INFORMATION2, *PSYSTEM_THREAD_INFORMATION2; + +#define SYSTEM_THREAD_INFORMATION SYSTEM_THREAD_INFORMATION2 +#define PSYSTEM_THREAD_INFORMATION PSYSTEM_THREAD_INFORMATION2 + +typedef struct _TEB *PTEB; + +typedef struct _SYSTEM_EXTENDED_THREAD_INFORMATION { + SYSTEM_THREAD_INFORMATION ThreadInfo; + PVOID StackBase; + PVOID StackLimit; + PVOID Win32StartAddress; + PTEB TebBase; + ULONG_PTR Reserved2; + ULONG_PTR Reserved3; + ULONG_PTR Reserved4; +} SYSTEM_EXTENDED_THREAD_INFORMATION, *PSYSTEM_EXTENDED_THREAD_INFORMATION; + +typedef struct _SYSTEM_PROCESS_INFORMATION2 { + ULONG NextEntryOffset; + ULONG NumberOfThreads; + LARGE_INTEGER SpareLi1; + LARGE_INTEGER SpareLi2; + LARGE_INTEGER SpareLi3; + LARGE_INTEGER CreateTime; + LARGE_INTEGER UserTime; + LARGE_INTEGER KernelTime; + UNICODE_STRING ImageName; + LONG BasePriority; + HANDLE UniqueProcessId; + HANDLE InheritedFromUniqueProcessId; + ULONG HandleCount; + ULONG SessionId; + ULONG_PTR PageDirectoryBase; + SIZE_T PeakVirtualSize; + SIZE_T VirtualSize; + DWORD PageFaultCount; + SIZE_T PeakWorkingSetSize; + SIZE_T WorkingSetSize; + SIZE_T QuotaPeakPagedPoolUsage; + SIZE_T QuotaPagedPoolUsage; + SIZE_T QuotaPeakNonPagedPoolUsage; + SIZE_T QuotaNonPagedPoolUsage; + SIZE_T PagefileUsage; + SIZE_T PeakPagefileUsage; + SIZE_T PrivatePageCount; + LARGE_INTEGER ReadOperationCount; + LARGE_INTEGER WriteOperationCount; + LARGE_INTEGER OtherOperationCount; + LARGE_INTEGER ReadTransferCount; + LARGE_INTEGER WriteTransferCount; + LARGE_INTEGER OtherTransferCount; + SYSTEM_THREAD_INFORMATION Threads[1]; +} SYSTEM_PROCESS_INFORMATION2, *PSYSTEM_PROCESS_INFORMATION2; + +#define SYSTEM_PROCESS_INFORMATION SYSTEM_PROCESS_INFORMATION2 +#define PSYSTEM_PROCESS_INFORMATION PSYSTEM_PROCESS_INFORMATION2 + +typedef struct _PROCESSOR_POWER_INFORMATION { + ULONG Number; + ULONG MaxMhz; + ULONG CurrentMhz; + ULONG MhzLimit; + ULONG MaxIdleState; + ULONG CurrentIdleState; +} PROCESSOR_POWER_INFORMATION, *PPROCESSOR_POWER_INFORMATION; + +#ifndef __IPHLPAPI_H__ +typedef struct in6_addr { + union { + UCHAR Byte[16]; + USHORT Word[8]; + } u; +} IN6_ADDR, *PIN6_ADDR, FAR *LPIN6_ADDR; +#endif + +// http://msdn.microsoft.com/en-us/library/aa813741(VS.85).aspx +typedef struct { + BYTE Reserved1[16]; + PVOID Reserved2[5]; + UNICODE_STRING CurrentDirectoryPath; + PVOID CurrentDirectoryHandle; + UNICODE_STRING DllPath; + UNICODE_STRING ImagePathName; + UNICODE_STRING CommandLine; + LPCWSTR env; +} RTL_USER_PROCESS_PARAMETERS_, *PRTL_USER_PROCESS_PARAMETERS_; + +typedef struct _WINSTATION_INFO { + BYTE Reserved1[72]; + ULONG SessionId; + BYTE Reserved2[4]; + FILETIME ConnectTime; + FILETIME DisconnectTime; + FILETIME LastInputTime; + FILETIME LoginTime; + BYTE Reserved3[1096]; + FILETIME CurrentTime; +} WINSTATION_INFO, *PWINSTATION_INFO; + +#if (_WIN32_WINNT < 0x0601) // Windows < 7 (Vista and XP) +typedef struct _SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX { + LOGICAL_PROCESSOR_RELATIONSHIP Relationship; + DWORD Size; + _ANONYMOUS_UNION + union { + PROCESSOR_RELATIONSHIP Processor; + NUMA_NODE_RELATIONSHIP NumaNode; + CACHE_RELATIONSHIP Cache; + GROUP_RELATIONSHIP Group; + } DUMMYUNIONNAME; +} SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX, *PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX; +#endif + +typedef struct _MEMORY_WORKING_SET_BLOCK { + ULONG_PTR Protection : 5; + ULONG_PTR ShareCount : 3; + ULONG_PTR Shared : 1; + ULONG_PTR Node : 3; +#ifdef _WIN64 + ULONG_PTR VirtualPage : 52; +#else + ULONG VirtualPage : 20; +#endif +} MEMORY_WORKING_SET_BLOCK, *PMEMORY_WORKING_SET_BLOCK; + +typedef struct _MEMORY_WORKING_SET_INFORMATION { + ULONG_PTR NumberOfEntries; + MEMORY_WORKING_SET_BLOCK WorkingSetInfo[1]; +} MEMORY_WORKING_SET_INFORMATION, *PMEMORY_WORKING_SET_INFORMATION; + +typedef struct _PSUTIL_PROCESS_WS_COUNTERS { + SIZE_T NumberOfPages; + SIZE_T NumberOfPrivatePages; + SIZE_T NumberOfSharedPages; + SIZE_T NumberOfShareablePages; +} PSUTIL_PROCESS_WS_COUNTERS, *PPSUTIL_PROCESS_WS_COUNTERS; + +/* + * ================================================================ + * Type defs for modules loaded at runtime. + * ================================================================ + */ + +typedef BOOL (WINAPI *_GetLogicalProcessorInformationEx)( + LOGICAL_PROCESSOR_RELATIONSHIP relationship, + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer, + PDWORD ReturnLength); + +typedef BOOLEAN (WINAPI * _WinStationQueryInformationW)( + HANDLE ServerHandle, + ULONG SessionId, + WINSTATIONINFOCLASS WinStationInformationClass, + PVOID pWinStationInformation, + ULONG WinStationInformationLength, + PULONG pReturnLength); + +typedef NTSTATUS (NTAPI *_NtQueryInformationProcess)( + HANDLE ProcessHandle, + DWORD ProcessInformationClass, + PVOID ProcessInformation, + DWORD ProcessInformationLength, + PDWORD ReturnLength); + +typedef NTSTATUS (NTAPI *_NtQuerySystemInformation)( + ULONG SystemInformationClass, + PVOID SystemInformation, + ULONG SystemInformationLength, + PULONG ReturnLength); + +typedef NTSTATUS (NTAPI *_NtSetInformationProcess)( + HANDLE ProcessHandle, + DWORD ProcessInformationClass, + PVOID ProcessInformation, + DWORD ProcessInformationLength); + +typedef PSTR (NTAPI * _RtlIpv4AddressToStringA)( + struct in_addr *Addr, + PSTR S); + +typedef PSTR (NTAPI * _RtlIpv6AddressToStringA)( + struct in6_addr *Addr, + PSTR P); + +typedef DWORD (WINAPI * _GetExtendedTcpTable)( + PVOID pTcpTable, + PDWORD pdwSize, + BOOL bOrder, + ULONG ulAf, + TCP_TABLE_CLASS TableClass, + ULONG Reserved); + +typedef DWORD (WINAPI * _GetExtendedUdpTable)( + PVOID pUdpTable, + PDWORD pdwSize, + BOOL bOrder, + ULONG ulAf, + UDP_TABLE_CLASS TableClass, + ULONG Reserved); + +typedef DWORD (CALLBACK *_GetActiveProcessorCount)( + WORD GroupNumber); + +typedef ULONGLONG (CALLBACK *_GetTickCount64)( + void); + +typedef NTSTATUS (NTAPI *_NtQueryObject)( + HANDLE Handle, + OBJECT_INFORMATION_CLASS ObjectInformationClass, + PVOID ObjectInformation, + ULONG ObjectInformationLength, + PULONG ReturnLength); + +typedef NTSTATUS (WINAPI *_RtlGetVersion) ( + PRTL_OSVERSIONINFOW lpVersionInformation +); + +typedef NTSTATUS (WINAPI *_NtResumeProcess) ( + HANDLE hProcess +); + +typedef NTSTATUS (WINAPI *_NtSuspendProcess) ( + HANDLE hProcess +); + +typedef NTSTATUS (NTAPI *_NtQueryVirtualMemory) ( + HANDLE ProcessHandle, + PVOID BaseAddress, + int MemoryInformationClass, + PVOID MemoryInformation, + SIZE_T MemoryInformationLength, + PSIZE_T ReturnLength +); + +typedef ULONG (WINAPI *_RtlNtStatusToDosErrorNoTeb) ( + NTSTATUS status +); + +#endif // __NTEXTAPI_H__ diff --git a/ddtrace/vendor/psutil/arch/windows/process_handles.c b/ddtrace/vendor/psutil/arch/windows/process_handles.c new file mode 100644 index 00000000000..5966669e280 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/process_handles.c @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + */ + +#include +#include +#include +#include "ntextapi.h" +#include "global.h" +#include "process_handles.h" +#include "process_info.h" +#include "../../_psutil_common.h" + +CRITICAL_SECTION g_cs; +BOOL g_initialized = FALSE; +NTSTATUS g_status; +HANDLE g_hFile = NULL; +HANDLE g_hEvtStart = NULL; +HANDLE g_hEvtFinish = NULL; +HANDLE g_hThread = NULL; +PUNICODE_STRING g_pNameBuffer = NULL; +ULONG g_dwSize = 0; +ULONG g_dwLength = 0; + + +#define ObjectNameInformation 1 +#define NTQO_TIMEOUT 100 + + +static VOID +psutil_get_open_files_init(BOOL threaded) { + if (g_initialized == TRUE) + return; + + // Create events for signalling work between threads + if (threaded == TRUE) { + g_hEvtStart = CreateEvent(NULL, FALSE, FALSE, NULL); + g_hEvtFinish = CreateEvent(NULL, FALSE, FALSE, NULL); + InitializeCriticalSection(&g_cs); + } + + g_initialized = TRUE; +} + + +static DWORD WINAPI +psutil_wait_thread(LPVOID lpvParam) { + // Loop infinitely waiting for work + while (TRUE) { + WaitForSingleObject(g_hEvtStart, INFINITE); + + // TODO: return code not checked + g_status = psutil_NtQueryObject( + g_hFile, + ObjectNameInformation, + g_pNameBuffer, + g_dwSize, + &g_dwLength); + SetEvent(g_hEvtFinish); + } +} + + +static DWORD +psutil_create_thread() { + DWORD dwWait = 0; + + if (g_hThread == NULL) + g_hThread = CreateThread( + NULL, + 0, + psutil_wait_thread, + NULL, + 0, + NULL); + if (g_hThread == NULL) + return GetLastError(); + + // Signal the worker thread to start + SetEvent(g_hEvtStart); + + // Wait for the worker thread to finish + dwWait = WaitForSingleObject(g_hEvtFinish, NTQO_TIMEOUT); + + // If the thread hangs, kill it and cleanup + if (dwWait == WAIT_TIMEOUT) { + SuspendThread(g_hThread); + TerminateThread(g_hThread, 1); + WaitForSingleObject(g_hThread, INFINITE); + CloseHandle(g_hThread); + + g_hThread = NULL; + } + + return dwWait; +} + + +static PyObject * +psutil_get_open_files_ntqueryobject(long dwPid, HANDLE hProcess) { + NTSTATUS status; + PSYSTEM_HANDLE_INFORMATION_EX pHandleInfo = NULL; + DWORD dwInfoSize = 0x10000; + DWORD dwRet = 0; + PSYSTEM_HANDLE_TABLE_ENTRY_INFO_EX hHandle = NULL; + DWORD i = 0; + BOOLEAN error = FALSE; + DWORD dwWait = 0; + PyObject* py_retlist = NULL; + PyObject* py_path = NULL; + + if (g_initialized == FALSE) + psutil_get_open_files_init(TRUE); + + // Due to the use of global variables, ensure only 1 call + // to psutil_get_open_files() is running + EnterCriticalSection(&g_cs); + + if (g_hEvtStart == NULL || + g_hEvtFinish == NULL) + + { + PyErr_SetFromWindowsErr(0); + error = TRUE; + goto cleanup; + } + + // Py_BuildValue raises an exception if NULL is returned + py_retlist = PyList_New(0); + if (py_retlist == NULL) { + error = TRUE; + goto cleanup; + } + + do { + if (pHandleInfo != NULL) { + HeapFree(GetProcessHeap(), 0, pHandleInfo); + pHandleInfo = NULL; + } + + // NtQuerySystemInformation won't give us the correct buffer size, + // so we guess by doubling the buffer size. + dwInfoSize *= 2; + pHandleInfo = HeapAlloc(GetProcessHeap(), + HEAP_ZERO_MEMORY, + dwInfoSize); + + if (pHandleInfo == NULL) { + PyErr_NoMemory(); + error = TRUE; + goto cleanup; + } + } while ((status = psutil_NtQuerySystemInformation( + SystemExtendedHandleInformation, + pHandleInfo, + dwInfoSize, + &dwRet)) == STATUS_INFO_LENGTH_MISMATCH); + + // NtQuerySystemInformation stopped giving us STATUS_INFO_LENGTH_MISMATCH + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQuerySystemInformation(SystemExtendedHandleInformation)"); + error = TRUE; + goto cleanup; + } + + for (i = 0; i < pHandleInfo->NumberOfHandles; i++) { + hHandle = &pHandleInfo->Handles[i]; + + // Check if this hHandle belongs to the PID the user specified. + if (hHandle->UniqueProcessId != (ULONG_PTR)dwPid) + goto loop_cleanup; + + if (!DuplicateHandle(hProcess, + (HANDLE)hHandle->HandleValue, + GetCurrentProcess(), + &g_hFile, + 0, + TRUE, + DUPLICATE_SAME_ACCESS)) + { + /* + printf("[%d] DuplicateHandle (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + goto loop_cleanup; + } + + // Guess buffer size is MAX_PATH + 1 + g_dwLength = (MAX_PATH+1) * sizeof(WCHAR); + + do { + // Release any previously allocated buffer + if (g_pNameBuffer != NULL) { + HeapFree(GetProcessHeap(), 0, g_pNameBuffer); + g_pNameBuffer = NULL; + g_dwSize = 0; + } + + // NtQueryObject puts the required buffer size in g_dwLength + // WinXP edge case puts g_dwLength == 0, just skip this handle + if (g_dwLength == 0) + goto loop_cleanup; + + g_dwSize = g_dwLength; + if (g_dwSize > 0) { + g_pNameBuffer = HeapAlloc(GetProcessHeap(), + HEAP_ZERO_MEMORY, + g_dwSize); + + if (g_pNameBuffer == NULL) + goto loop_cleanup; + } + + dwWait = psutil_create_thread(); + + // If the call does not return, skip this handle + if (dwWait != WAIT_OBJECT_0) + goto loop_cleanup; + + } while (g_status == STATUS_INFO_LENGTH_MISMATCH); + + // NtQueryObject stopped returning STATUS_INFO_LENGTH_MISMATCH + if (!NT_SUCCESS(g_status)) + goto loop_cleanup; + + // Convert to PyUnicode and append it to the return list + if (g_pNameBuffer->Length > 0) { + /* + printf("[%d] Filename (%#x) %#d bytes: %S\n", + dwPid, + hHandle->HandleValue, + g_pNameBuffer->Length, + g_pNameBuffer->Buffer); + */ + + py_path = PyUnicode_FromWideChar(g_pNameBuffer->Buffer, + g_pNameBuffer->Length/2); + if (py_path == NULL) { + /* + printf("[%d] PyUnicode_FromWideChar (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + error = TRUE; + goto loop_cleanup; + } + + if (PyList_Append(py_retlist, py_path)) { + /* + printf("[%d] PyList_Append (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + error = TRUE; + goto loop_cleanup; + } + } + +loop_cleanup: + Py_XDECREF(py_path); + py_path = NULL; + + if (g_pNameBuffer != NULL) + HeapFree(GetProcessHeap(), 0, g_pNameBuffer); + g_pNameBuffer = NULL; + g_dwSize = 0; + g_dwLength = 0; + + if (g_hFile != NULL) + CloseHandle(g_hFile); + g_hFile = NULL; +} + +cleanup: + if (g_pNameBuffer != NULL) + HeapFree(GetProcessHeap(), 0, g_pNameBuffer); + g_pNameBuffer = NULL; + g_dwSize = 0; + g_dwLength = 0; + + if (g_hFile != NULL) + CloseHandle(g_hFile); + g_hFile = NULL; + + if (pHandleInfo != NULL) + HeapFree(GetProcessHeap(), 0, pHandleInfo); + pHandleInfo = NULL; + + if (error) { + Py_XDECREF(py_retlist); + py_retlist = NULL; + } + + LeaveCriticalSection(&g_cs); + + return py_retlist; +} + + +static PyObject * +psutil_get_open_files_getmappedfilename(long dwPid, HANDLE hProcess) { + NTSTATUS status; + PSYSTEM_HANDLE_INFORMATION_EX pHandleInfo = NULL; + DWORD dwInfoSize = 0x10000; + DWORD dwRet = 0; + PSYSTEM_HANDLE_TABLE_ENTRY_INFO_EX hHandle = NULL; + HANDLE hFile = NULL; + HANDLE hMap = NULL; + DWORD i = 0; + BOOLEAN error = FALSE; + PyObject* py_retlist = NULL; + PyObject* py_path = NULL; + ULONG dwSize = 0; + LPVOID pMem = NULL; + wchar_t pszFilename[MAX_PATH+1]; + + if (g_initialized == FALSE) + psutil_get_open_files_init(FALSE); + + // Py_BuildValue raises an exception if NULL is returned + py_retlist = PyList_New(0); + if (py_retlist == NULL) { + error = TRUE; + goto cleanup; + } + + do { + if (pHandleInfo != NULL) { + HeapFree(GetProcessHeap(), 0, pHandleInfo); + pHandleInfo = NULL; + } + + // NtQuerySystemInformation won't give us the correct buffer size, + // so we guess by doubling the buffer size. + dwInfoSize *= 2; + pHandleInfo = HeapAlloc(GetProcessHeap(), + HEAP_ZERO_MEMORY, + dwInfoSize); + + if (pHandleInfo == NULL) { + PyErr_NoMemory(); + error = TRUE; + goto cleanup; + } + } while ((status = psutil_NtQuerySystemInformation( + SystemExtendedHandleInformation, + pHandleInfo, + dwInfoSize, + &dwRet)) == STATUS_INFO_LENGTH_MISMATCH); + + // NtQuerySystemInformation stopped giving us STATUS_INFO_LENGTH_MISMATCH + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQuerySystemInformation(SystemExtendedHandleInformation)"); + error = TRUE; + goto cleanup; + } + + for (i = 0; i < pHandleInfo->NumberOfHandles; i++) { + hHandle = &pHandleInfo->Handles[i]; + + // Check if this hHandle belongs to the PID the user specified. + if (hHandle->UniqueProcessId != (ULONG_PTR)dwPid) + goto loop_cleanup; + + if (!DuplicateHandle(hProcess, + (HANDLE)hHandle->HandleValue, + GetCurrentProcess(), + &hFile, + 0, + TRUE, + DUPLICATE_SAME_ACCESS)) + { + /* + printf("[%d] DuplicateHandle (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + goto loop_cleanup; + } + + hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL); + if (hMap == NULL) { + /* + printf("[%d] CreateFileMapping (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + goto loop_cleanup; + } + + pMem = MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 1); + + if (pMem == NULL) { + /* + printf("[%d] MapViewOfFile (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + goto loop_cleanup; + } + + dwSize = GetMappedFileName( + GetCurrentProcess(), pMem, (LPSTR)pszFilename, MAX_PATH); + if (dwSize == 0) { + /* + printf("[%d] GetMappedFileName (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + goto loop_cleanup; + } + + pszFilename[dwSize] = '\0'; + /* + printf("[%d] Filename (%#x) %#d bytes: %S\n", + dwPid, + hHandle->HandleValue, + dwSize, + pszFilename); + */ + + py_path = PyUnicode_FromWideChar(pszFilename, dwSize); + if (py_path == NULL) { + /* + printf("[%d] PyUnicode_FromStringAndSize (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + error = TRUE; + goto loop_cleanup; + } + + if (PyList_Append(py_retlist, py_path)) { + /* + printf("[%d] PyList_Append (%#x): %#x \n", + dwPid, + hHandle->HandleValue, + GetLastError()); + */ + error = TRUE; + goto loop_cleanup; + } + +loop_cleanup: + Py_XDECREF(py_path); + py_path = NULL; + + if (pMem != NULL) + UnmapViewOfFile(pMem); + pMem = NULL; + + if (hMap != NULL) + CloseHandle(hMap); + hMap = NULL; + + if (hFile != NULL) + CloseHandle(hFile); + hFile = NULL; + + dwSize = 0; + } + +cleanup: + if (pMem != NULL) + UnmapViewOfFile(pMem); + pMem = NULL; + + if (hMap != NULL) + CloseHandle(hMap); + hMap = NULL; + + if (hFile != NULL) + CloseHandle(hFile); + hFile = NULL; + + if (pHandleInfo != NULL) + HeapFree(GetProcessHeap(), 0, pHandleInfo); + pHandleInfo = NULL; + + if (error) { + Py_XDECREF(py_retlist); + py_retlist = NULL; + } + + return py_retlist; +} + + +/* + * The public function. + */ +PyObject * +psutil_get_open_files(long dwPid, HANDLE hProcess) { + // Threaded version only works for Vista+ + if (PSUTIL_WINVER >= PSUTIL_WINDOWS_VISTA) + return psutil_get_open_files_ntqueryobject(dwPid, hProcess); + else + return psutil_get_open_files_getmappedfilename(dwPid, hProcess); +} diff --git a/ddtrace/vendor/psutil/arch/windows/process_handles.h b/ddtrace/vendor/psutil/arch/windows/process_handles.h new file mode 100644 index 00000000000..342ce8fd26a --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/process_handles.h @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include + +PyObject* psutil_get_open_files(long pid, HANDLE processHandle); diff --git a/ddtrace/vendor/psutil/arch/windows/process_info.c b/ddtrace/vendor/psutil/arch/windows/process_info.c new file mode 100644 index 00000000000..5ea5f765c0c --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/process_info.c @@ -0,0 +1,1027 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Helper functions related to fetching process information. Used by + * _psutil_windows module methods. + */ + +#include +#include +#include +#include + +#include "ntextapi.h" +#include "global.h" +#include "security.h" +#include "process_info.h" +#include "../../_psutil_common.h" + + +// ==================================================================== +// Helper structures to access the memory correctly. +// Some of these might also be defined in the winternl.h header file +// but unfortunately not in a usable way. +// ==================================================================== + +// https://msdn.microsoft.com/en-us/library/aa813706(v=vs.85).aspx +#ifdef _WIN64 +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[21]; + PVOID LoaderData; + PRTL_USER_PROCESS_PARAMETERS_ ProcessParameters; + /* More fields ... */ +} PEB_; +#else +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[1]; + PVOID Reserved3[2]; + PVOID Ldr; + PRTL_USER_PROCESS_PARAMETERS_ ProcessParameters; + /* More fields ... */ +} PEB_; +#endif + +#ifdef _WIN64 +/* When we are a 64 bit process accessing a 32 bit (WoW64) process we need to + use the 32 bit structure layout. */ +typedef struct { + USHORT Length; + USHORT MaxLength; + DWORD Buffer; +} UNICODE_STRING32; + +typedef struct { + BYTE Reserved1[16]; + DWORD Reserved2[5]; + UNICODE_STRING32 CurrentDirectoryPath; + DWORD CurrentDirectoryHandle; + UNICODE_STRING32 DllPath; + UNICODE_STRING32 ImagePathName; + UNICODE_STRING32 CommandLine; + DWORD env; +} RTL_USER_PROCESS_PARAMETERS32; + +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[1]; + DWORD Reserved3[2]; + DWORD Ldr; + DWORD ProcessParameters; + /* More fields ... */ +} PEB32; +#else +/* When we are a 32 bit (WoW64) process accessing a 64 bit process we need to + use the 64 bit structure layout and a special function to read its memory. + */ +typedef NTSTATUS (NTAPI *_NtWow64ReadVirtualMemory64)( + HANDLE ProcessHandle, + PVOID64 BaseAddress, + PVOID Buffer, + ULONG64 Size, + PULONG64 NumberOfBytesRead); + +typedef struct { + PVOID Reserved1[2]; + PVOID64 PebBaseAddress; + PVOID Reserved2[4]; + PVOID UniqueProcessId[2]; + PVOID Reserved3[2]; +} PROCESS_BASIC_INFORMATION64; + +typedef struct { + USHORT Length; + USHORT MaxLength; + PVOID64 Buffer; +} UNICODE_STRING64; + +typedef struct { + BYTE Reserved1[16]; + PVOID64 Reserved2[5]; + UNICODE_STRING64 CurrentDirectoryPath; + PVOID64 CurrentDirectoryHandle; + UNICODE_STRING64 DllPath; + UNICODE_STRING64 ImagePathName; + UNICODE_STRING64 CommandLine; + PVOID64 env; +} RTL_USER_PROCESS_PARAMETERS64; + +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[21]; + PVOID64 LoaderData; + PVOID64 ProcessParameters; + /* More fields ... */ +} PEB64; +#endif + + +#define PSUTIL_FIRST_PROCESS(Processes) ( \ + (PSYSTEM_PROCESS_INFORMATION)(Processes)) +#define PSUTIL_NEXT_PROCESS(Process) ( \ + ((PSYSTEM_PROCESS_INFORMATION)(Process))->NextEntryOffset ? \ + (PSYSTEM_PROCESS_INFORMATION)((PCHAR)(Process) + \ + ((PSYSTEM_PROCESS_INFORMATION)(Process))->NextEntryOffset) : NULL) + + +// ==================================================================== +// Process and PIDs utiilties. +// ==================================================================== + + +#define STATUS_BUFFER_OVERFLOW ((NTSTATUS)0x80000005L) + +/* + * Return 1 if PID exists, 0 if not, -1 on error. + */ +int +psutil_pid_in_pids(DWORD pid) { + DWORD *proclist = NULL; + DWORD numberOfReturnedPIDs; + DWORD i; + + proclist = psutil_get_pids(&numberOfReturnedPIDs); + if (proclist == NULL) + return -1; + for (i = 0; i < numberOfReturnedPIDs; i++) { + if (proclist[i] == pid) { + free(proclist); + return 1; + } + } + free(proclist); + return 0; +} + + +/* + * Given a process HANDLE checks whether it's actually running. + * Returns: + * - 1: running + * - 0: not running + * - -1: WindowsError + * - -2: AssertionError + */ +int +psutil_is_phandle_running(HANDLE hProcess, DWORD pid) { + DWORD processExitCode = 0; + + if (hProcess == NULL) { + if (GetLastError() == ERROR_INVALID_PARAMETER) { + // Yeah, this is the actual error code in case of + // "no such process". + if (! psutil_assert_pid_not_exists( + pid, "iphr: OpenProcess() -> ERROR_INVALID_PARAMETER")) { + return -2; + } + return 0; + } + return -1; + } + + if (GetExitCodeProcess(hProcess, &processExitCode)) { + // XXX - maybe STILL_ACTIVE is not fully reliable as per: + // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 + if (processExitCode == STILL_ACTIVE) { + if (! psutil_assert_pid_exists( + pid, "iphr: GetExitCodeProcess() -> STILL_ACTIVE")) { + return -2; + } + return 1; + } + else { + // We can't be sure so we look into pids. + if (psutil_pid_in_pids(pid) == 1) { + return 1; + } + else { + CloseHandle(hProcess); + return 0; + } + } + } + + CloseHandle(hProcess); + if (! psutil_assert_pid_not_exists( pid, "iphr: exit fun")) { + return -2; + } + return -1; +} + + +/* + * Given a process HANDLE checks whether it's actually running and if + * it does return it, else return NULL with the proper Python exception + * set. + */ +HANDLE +psutil_check_phandle(HANDLE hProcess, DWORD pid) { + int ret = psutil_is_phandle_running(hProcess, pid); + if (ret == 1) { + return hProcess; + } + else if (ret == 0) { + return NoSuchProcess(""); + } + else if (ret == -1) { + if (GetLastError() == ERROR_ACCESS_DENIED) + return PyErr_SetFromWindowsErr(0); + else + return PyErr_SetFromOSErrnoWithSyscall("OpenProcess"); + } + else { + return NULL; + } +} + + +/* + * A wrapper around OpenProcess setting NSP exception if process + * no longer exists. + * "pid" is the process pid, "dwDesiredAccess" is the first argument + * exptected by OpenProcess. + * Return a process handle or NULL. + */ +HANDLE +psutil_handle_from_pid(DWORD pid, DWORD access) { + HANDLE hProcess; + + if (pid == 0) { + // otherwise we'd get NoSuchProcess + return AccessDenied(""); + } + // needed for GetExitCodeProcess + access |= PROCESS_QUERY_LIMITED_INFORMATION; + hProcess = OpenProcess(access, FALSE, pid); + return psutil_check_phandle(hProcess, pid); +} + + +DWORD * +psutil_get_pids(DWORD *numberOfReturnedPIDs) { + // Win32 SDK says the only way to know if our process array + // wasn't large enough is to check the returned size and make + // sure that it doesn't match the size of the array. + // If it does we allocate a larger array and try again + + // Stores the actual array + DWORD *procArray = NULL; + DWORD procArrayByteSz; + int procArraySz = 0; + + // Stores the byte size of the returned array from enumprocesses + DWORD enumReturnSz = 0; + + do { + procArraySz += 1024; + if (procArray != NULL) + free(procArray); + procArrayByteSz = procArraySz * sizeof(DWORD); + procArray = malloc(procArrayByteSz); + if (procArray == NULL) { + PyErr_NoMemory(); + return NULL; + } + if (! EnumProcesses(procArray, procArrayByteSz, &enumReturnSz)) { + free(procArray); + PyErr_SetFromWindowsErr(0); + return NULL; + } + } while (enumReturnSz == procArraySz * sizeof(DWORD)); + + // The number of elements is the returned size / size of each element + *numberOfReturnedPIDs = enumReturnSz / sizeof(DWORD); + + return procArray; +} + + +int +psutil_assert_pid_exists(DWORD pid, char *err) { + if (PSUTIL_TESTING) { + if (psutil_pid_in_pids(pid) == 0) { + PyErr_SetString(PyExc_AssertionError, err); + return 0; + } + } + return 1; +} + + +int +psutil_assert_pid_not_exists(DWORD pid, char *err) { + if (PSUTIL_TESTING) { + if (psutil_pid_in_pids(pid) == 1) { + PyErr_SetString(PyExc_AssertionError, err); + return 0; + } + } + return 1; +} + + +/* +/* Check for PID existance by using OpenProcess() + GetExitCodeProcess. +/* Returns: + * 1: pid exists + * 0: it doesn't + * -1: error + */ +int +psutil_pid_is_running(DWORD pid) { + HANDLE hProcess; + DWORD exitCode; + DWORD err; + + // Special case for PID 0 System Idle Process + if (pid == 0) + return 1; + if (pid < 0) + return 0; + hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (NULL == hProcess) { + err = GetLastError(); + // Yeah, this is the actual error code in case of "no such process". + if (err == ERROR_INVALID_PARAMETER) { + if (! psutil_assert_pid_not_exists( + pid, "pir: OpenProcess() -> INVALID_PARAMETER")) { + return -1; + } + return 0; + } + // Access denied obviously means there's a process to deny access to. + else if (err == ERROR_ACCESS_DENIED) { + if (! psutil_assert_pid_exists( + pid, "pir: OpenProcess() ACCESS_DENIED")) { + return -1; + } + return 1; + } + // Be strict and raise an exception; the caller is supposed + // to take -1 into account. + else { + PyErr_SetFromOSErrnoWithSyscall("OpenProcess(PROCESS_VM_READ)"); + return -1; + } + } + + if (GetExitCodeProcess(hProcess, &exitCode)) { + CloseHandle(hProcess); + // XXX - maybe STILL_ACTIVE is not fully reliable as per: + // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 + if (exitCode == STILL_ACTIVE) { + if (! psutil_assert_pid_exists( + pid, "pir: GetExitCodeProcess() -> STILL_ACTIVE")) { + return -1; + } + return 1; + } + // We can't be sure so we look into pids. + else { + return psutil_pid_in_pids(pid); + } + } + else { + err = GetLastError(); + CloseHandle(hProcess); + // Same as for OpenProcess, assume access denied means there's + // a process to deny access to. + if (err == ERROR_ACCESS_DENIED) { + if (! psutil_assert_pid_exists( + pid, "pir: GetExitCodeProcess() -> ERROR_ACCESS_DENIED")) { + return -1; + } + return 1; + } + else { + PyErr_SetFromOSErrnoWithSyscall("GetExitCodeProcess"); + return -1; + } + } +} + + +/* Given a pointer into a process's memory, figure out how much data can be + * read from it. */ +static int +psutil_get_process_region_size(HANDLE hProcess, LPCVOID src, SIZE_T *psize) { + MEMORY_BASIC_INFORMATION info; + + if (!VirtualQueryEx(hProcess, src, &info, sizeof(info))) { + PyErr_SetFromOSErrnoWithSyscall("VirtualQueryEx"); + return -1; + } + + *psize = info.RegionSize - ((char*)src - (char*)info.BaseAddress); + return 0; +} + + +enum psutil_process_data_kind { + KIND_CMDLINE, + KIND_CWD, + KIND_ENVIRON, +}; + +/* Get data from the process with the given pid. The data is returned in the + pdata output member as a nul terminated string which must be freed on + success. + + On success 0 is returned. On error the output parameter is not touched, -1 + is returned, and an appropriate Python exception is set. */ +static int +psutil_get_process_data(long pid, + enum psutil_process_data_kind kind, + WCHAR **pdata, + SIZE_T *psize) { + /* This function is quite complex because there are several cases to be + considered: + + Two cases are really simple: we (i.e. the python interpreter) and the + target process are both 32 bit or both 64 bit. In that case the memory + layout of the structures matches up and all is well. + + When we are 64 bit and the target process is 32 bit we need to use + custom 32 bit versions of the structures. + + When we are 32 bit and the target process is 64 bit we need to use + custom 64 bit version of the structures. Also we need to use separate + Wow64 functions to get the information. + + A few helper structs are defined above so that the compiler can handle + calculating the correct offsets. + + Additional help also came from the following sources: + + https://github.com/kohsuke/winp and + http://wj32.org/wp/2009/01/24/howto-get-the-command-line-of-processes/ + http://stackoverflow.com/a/14012919 + http://www.drdobbs.com/embracing-64-bit-windows/184401966 + */ + _NtQueryInformationProcess NtQueryInformationProcess = NULL; +#ifndef _WIN64 + static _NtQueryInformationProcess NtWow64QueryInformationProcess64 = NULL; + static _NtWow64ReadVirtualMemory64 NtWow64ReadVirtualMemory64 = NULL; +#endif + HANDLE hProcess = NULL; + LPCVOID src; + SIZE_T size; + WCHAR *buffer = NULL; +#ifdef _WIN64 + LPVOID ppeb32 = NULL; +#else + PVOID64 src64; + BOOL weAreWow64; + BOOL theyAreWow64; +#endif + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_VM_READ; + NTSTATUS status; + + hProcess = psutil_handle_from_pid(pid, access); + if (hProcess == NULL) + return -1; + +#ifdef _WIN64 + /* 64 bit case. Check if the target is a 32 bit process running in WoW64 + * mode. */ + status = psutil_NtQueryInformationProcess( + hProcess, + ProcessWow64Information, + &ppeb32, + sizeof(LPVOID), + NULL); + + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQueryInformationProcess(ProcessWow64Information)"); + goto error; + } + + if (ppeb32 != NULL) { + /* We are 64 bit. Target process is 32 bit running in WoW64 mode. */ + PEB32 peb32; + RTL_USER_PROCESS_PARAMETERS32 procParameters32; + + // read PEB + if (!ReadProcessMemory(hProcess, ppeb32, &peb32, sizeof(peb32), NULL)) { + // May fail with ERROR_PARTIAL_COPY, see: + // https://github.com/giampaolo/psutil/issues/875 + PyErr_SetFromWindowsErr(0); + goto error; + } + + // read process parameters + if (!ReadProcessMemory(hProcess, + UlongToPtr(peb32.ProcessParameters), + &procParameters32, + sizeof(procParameters32), + NULL)) + { + // May fail with ERROR_PARTIAL_COPY, see: + // https://github.com/giampaolo/psutil/issues/875 + PyErr_SetFromWindowsErr(0); + goto error; + } + + switch (kind) { + case KIND_CMDLINE: + src = UlongToPtr(procParameters32.CommandLine.Buffer), + size = procParameters32.CommandLine.Length; + break; + case KIND_CWD: + src = UlongToPtr(procParameters32.CurrentDirectoryPath.Buffer); + size = procParameters32.CurrentDirectoryPath.Length; + break; + case KIND_ENVIRON: + src = UlongToPtr(procParameters32.env); + break; + } + } else +#else + /* 32 bit case. Check if the target is also 32 bit. */ + if (!IsWow64Process(GetCurrentProcess(), &weAreWow64) || + !IsWow64Process(hProcess, &theyAreWow64)) { + PyErr_SetFromOSErrnoWithSyscall("IsWow64Process"); + goto error; + } + + if (weAreWow64 && !theyAreWow64) { + /* We are 32 bit running in WoW64 mode. Target process is 64 bit. */ + PROCESS_BASIC_INFORMATION64 pbi64; + PEB64 peb64; + RTL_USER_PROCESS_PARAMETERS64 procParameters64; + + if (NtWow64QueryInformationProcess64 == NULL) { + NtWow64QueryInformationProcess64 = \ + psutil_GetProcAddressFromLib( + "ntdll.dll", "NtWow64QueryInformationProcess64"); + if (NtWow64QueryInformationProcess64 == NULL) { + PyErr_Clear(); + AccessDenied("can't query 64-bit process in 32-bit-WoW mode"); + goto error; + } + } + if (NtWow64ReadVirtualMemory64 == NULL) { + NtWow64ReadVirtualMemory64 = \ + psutil_GetProcAddressFromLib( + "ntdll.dll", "NtWow64ReadVirtualMemory64"); + if (NtWow64ReadVirtualMemory64 == NULL) { + PyErr_Clear(); + AccessDenied("can't query 64-bit process in 32-bit-WoW mode"); + goto error; + } + } + + status = NtWow64QueryInformationProcess64( + hProcess, + ProcessBasicInformation, + &pbi64, + sizeof(pbi64), + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, + "NtWow64QueryInformationProcess64(ProcessBasicInformation)" + ); + goto error; + } + + // read peb + status = NtWow64ReadVirtualMemory64( + hProcess, + pbi64.PebBaseAddress, + &peb64, + sizeof(peb64), + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64"); + goto error; + } + + // read process parameters + status = NtWow64ReadVirtualMemory64( + hProcess, + peb64.ProcessParameters, + &procParameters64, + sizeof(procParameters64), + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, + "NtWow64ReadVirtualMemory64(ProcessParameters)" + ); + goto error; + } + + switch (kind) { + case KIND_CMDLINE: + src64 = procParameters64.CommandLine.Buffer; + size = procParameters64.CommandLine.Length; + break; + case KIND_CWD: + src64 = procParameters64.CurrentDirectoryPath.Buffer, + size = procParameters64.CurrentDirectoryPath.Length; + break; + case KIND_ENVIRON: + src64 = procParameters64.env; + break; + } + } else +#endif + /* Target process is of the same bitness as us. */ + { + PROCESS_BASIC_INFORMATION pbi; + PEB_ peb; + RTL_USER_PROCESS_PARAMETERS_ procParameters; + + status = psutil_NtQueryInformationProcess( + hProcess, + ProcessBasicInformation, + &pbi, + sizeof(pbi), + NULL); + + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQueryInformationProcess(ProcessBasicInformation)"); + goto error; + } + + + // read peb + if (!ReadProcessMemory(hProcess, + pbi.PebBaseAddress, + &peb, + sizeof(peb), + NULL)) + { + // May fail with ERROR_PARTIAL_COPY, see: + // https://github.com/giampaolo/psutil/issues/875 + PyErr_SetFromWindowsErr(0); + goto error; + } + + // read process parameters + if (!ReadProcessMemory(hProcess, + peb.ProcessParameters, + &procParameters, + sizeof(procParameters), + NULL)) + { + // May fail with ERROR_PARTIAL_COPY, see: + // https://github.com/giampaolo/psutil/issues/875 + PyErr_SetFromWindowsErr(0); + goto error; + } + + switch (kind) { + case KIND_CMDLINE: + src = procParameters.CommandLine.Buffer; + size = procParameters.CommandLine.Length; + break; + case KIND_CWD: + src = procParameters.CurrentDirectoryPath.Buffer; + size = procParameters.CurrentDirectoryPath.Length; + break; + case KIND_ENVIRON: + src = procParameters.env; + break; + } + } + + if (kind == KIND_ENVIRON) { +#ifndef _WIN64 + if (weAreWow64 && !theyAreWow64) { + AccessDenied("can't query 64-bit process in 32-bit-WoW mode"); + goto error; + } + else +#endif + if (psutil_get_process_region_size(hProcess, src, &size) != 0) + goto error; + } + + buffer = calloc(size + 2, 1); + if (buffer == NULL) { + PyErr_NoMemory(); + goto error; + } + +#ifndef _WIN64 + if (weAreWow64 && !theyAreWow64) { + status = NtWow64ReadVirtualMemory64( + hProcess, + src64, + buffer, + size, + NULL); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64"); + goto error; + } + } else +#endif + if (!ReadProcessMemory(hProcess, src, buffer, size, NULL)) { + // May fail with ERROR_PARTIAL_COPY, see: + // https://github.com/giampaolo/psutil/issues/875 + PyErr_SetFromWindowsErr(0); + goto error; + } + + CloseHandle(hProcess); + + *pdata = buffer; + *psize = size; + + return 0; + +error: + if (hProcess != NULL) + CloseHandle(hProcess); + if (buffer != NULL) + free(buffer); + return -1; +} + + +/* + * Get process cmdline by using NtQueryInformationProcess. This is a + * method alternative to PEB which is less likely to result in + * AccessDenied. Requires Windows 8.1+. + */ +static int +psutil_cmdline_query_proc(long pid, WCHAR **pdata, SIZE_T *psize) { + HANDLE hProcess; + ULONG bufLen = 0; + NTSTATUS status; + char * buffer = NULL; + WCHAR * bufWchar = NULL; + PUNICODE_STRING tmp = NULL; + size_t size; + int ProcessCommandLineInformation = 60; + + if (PSUTIL_WINVER < PSUTIL_WINDOWS_8_1) { + PyErr_SetString( + PyExc_RuntimeError, "requires Windows 8.1+"); + goto error; + } + + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) + goto error; + + // get the right buf size + status = psutil_NtQueryInformationProcess( + hProcess, + ProcessCommandLineInformation, + NULL, + 0, + &bufLen); + + // 0xC0000225 == STATUS_NOT_FOUND, see: + // https://github.com/giampaolo/psutil/issues/1501 + if (status == 0xC0000225) { + AccessDenied("NtQueryInformationProcess(ProcessBasicInformation) -> " + "STATUS_NOT_FOUND translated into PermissionError"); + goto error; + } + + if (status != STATUS_BUFFER_OVERFLOW && \ + status != STATUS_BUFFER_TOO_SMALL && \ + status != STATUS_INFO_LENGTH_MISMATCH) { + psutil_SetFromNTStatusErr( + status, "NtQueryInformationProcess(ProcessBasicInformation)"); + goto error; + } + + // allocate memory + buffer = calloc(bufLen, 1); + if (buffer == NULL) { + PyErr_NoMemory(); + goto error; + } + + // get the cmdline + status = psutil_NtQueryInformationProcess( + hProcess, + ProcessCommandLineInformation, + buffer, + bufLen, + &bufLen + ); + if (!NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQueryInformationProcess(ProcessCommandLineInformation)"); + goto error; + } + + // build the string + tmp = (PUNICODE_STRING)buffer; + size = wcslen(tmp->Buffer) + 1; + bufWchar = (WCHAR *)calloc(size, sizeof(WCHAR)); + if (bufWchar == NULL) { + PyErr_NoMemory(); + goto error; + } + wcscpy_s(bufWchar, size, tmp->Buffer); + *pdata = bufWchar; + *psize = size * sizeof(WCHAR); + free(buffer); + CloseHandle(hProcess); + return 0; + +error: + if (buffer != NULL) + free(buffer); + if (hProcess != NULL) + CloseHandle(hProcess); + return -1; +} + + +/* + * Return a Python list representing the arguments for the process + * with given pid or NULL on error. + */ +PyObject * +psutil_get_cmdline(long pid, int use_peb) { + PyObject *ret = NULL; + WCHAR *data = NULL; + SIZE_T size; + PyObject *py_retlist = NULL; + PyObject *py_unicode = NULL; + LPWSTR *szArglist = NULL; + int nArgs, i; + int func_ret; + + /* + Reading the PEB to get the cmdline seem to be the best method if + somebody has tampered with the parameters after creating the process. + For instance, create a process as suspended, patch the command line + in its PEB and unfreeze it. It requires more privileges than + NtQueryInformationProcess though (the fallback): + - https://github.com/giampaolo/psutil/pull/1398 + - https://blog.xpnsec.com/how-to-argue-like-cobalt-strike/ + */ + if (use_peb == 1) + func_ret = psutil_get_process_data(pid, KIND_CMDLINE, &data, &size); + else + func_ret = psutil_cmdline_query_proc(pid, &data, &size); + if (func_ret != 0) + goto out; + + // attempt to parse the command line using Win32 API + szArglist = CommandLineToArgvW(data, &nArgs); + if (szArglist == NULL) { + PyErr_SetFromOSErrnoWithSyscall("CommandLineToArgvW"); + goto out; + } + + // arglist parsed as array of UNICODE_STRING, so convert each to + // Python string object and add to arg list + py_retlist = PyList_New(nArgs); + if (py_retlist == NULL) + goto out; + for (i = 0; i < nArgs; i++) { + py_unicode = PyUnicode_FromWideChar(szArglist[i], + wcslen(szArglist[i])); + if (py_unicode == NULL) + goto out; + PyList_SET_ITEM(py_retlist, i, py_unicode); + py_unicode = NULL; + } + ret = py_retlist; + py_retlist = NULL; + +out: + if (szArglist != NULL) + LocalFree(szArglist); + if (data != NULL) + free(data); + Py_XDECREF(py_unicode); + Py_XDECREF(py_retlist); + return ret; +} + + +PyObject * +psutil_get_cwd(long pid) { + PyObject *ret = NULL; + WCHAR *data = NULL; + SIZE_T size; + + if (psutil_get_process_data(pid, KIND_CWD, &data, &size) != 0) + goto out; + + // convert wchar array to a Python unicode string + ret = PyUnicode_FromWideChar(data, wcslen(data)); + +out: + if (data != NULL) + free(data); + + return ret; +} + + +/* + * returns a Python string containing the environment variable data for the + * process with given pid or NULL on error. + */ +PyObject * +psutil_get_environ(long pid) { + PyObject *ret = NULL; + WCHAR *data = NULL; + SIZE_T size; + + if (psutil_get_process_data(pid, KIND_ENVIRON, &data, &size) != 0) + goto out; + + // convert wchar array to a Python unicode string + ret = PyUnicode_FromWideChar(data, size / 2); + +out: + if (data != NULL) + free(data); + return ret; +} + + +/* + * Given a process PID and a PSYSTEM_PROCESS_INFORMATION structure + * fills the structure with various process information by using + * NtQuerySystemInformation. + * We use this as a fallback when faster functions fail with access + * denied. This is slower because it iterates over all processes. + * On success return 1, else 0 with Python exception already set. + */ +int +psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, + PVOID *retBuffer) { + static ULONG initialBufferSize = 0x4000; + NTSTATUS status; + PVOID buffer; + ULONG bufferSize; + PSYSTEM_PROCESS_INFORMATION process; + + bufferSize = initialBufferSize; + buffer = malloc(bufferSize); + if (buffer == NULL) { + PyErr_NoMemory(); + goto error; + } + + while (TRUE) { + status = psutil_NtQuerySystemInformation( + SystemProcessInformation, + buffer, + bufferSize, + &bufferSize); + if (status == STATUS_BUFFER_TOO_SMALL || + status == STATUS_INFO_LENGTH_MISMATCH) + { + free(buffer); + buffer = malloc(bufferSize); + if (buffer == NULL) { + PyErr_NoMemory(); + goto error; + } + } + else { + break; + } + } + + if (! NT_SUCCESS(status)) { + psutil_SetFromNTStatusErr( + status, "NtQuerySystemInformation(SystemProcessInformation)"); + goto error; + } + + if (bufferSize <= 0x20000) + initialBufferSize = bufferSize; + + process = PSUTIL_FIRST_PROCESS(buffer); + do { + if (process->UniqueProcessId == (HANDLE)pid) { + *retProcess = process; + *retBuffer = buffer; + return 1; + } + } while ( (process = PSUTIL_NEXT_PROCESS(process)) ); + + NoSuchProcess(""); + goto error; + +error: + if (buffer != NULL) + free(buffer); + return 0; +} diff --git a/ddtrace/vendor/psutil/arch/windows/process_info.h b/ddtrace/vendor/psutil/arch/windows/process_info.h new file mode 100644 index 00000000000..4278c4df9e0 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/process_info.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#if !defined(__PROCESS_INFO_H) +#define __PROCESS_INFO_H + +#include +#include +#include "security.h" +#include "ntextapi.h" + +#define HANDLE_TO_PYNUM(handle) PyLong_FromUnsignedLong((unsigned long) handle) +#define PYNUM_TO_HANDLE(obj) ((HANDLE)PyLong_AsUnsignedLong(obj)) + +DWORD* psutil_get_pids(DWORD *numberOfReturnedPIDs); +HANDLE psutil_handle_from_pid(DWORD pid, DWORD dwDesiredAccess); +int psutil_pid_is_running(DWORD pid); +int psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, + PVOID *retBuffer); + +int psutil_assert_pid_exists(DWORD pid, char *err); +int psutil_assert_pid_not_exists(DWORD pid, char *err); + +PyObject* psutil_get_cmdline(long pid, int use_peb); +PyObject* psutil_get_cwd(long pid); +PyObject* psutil_get_environ(long pid); + +#endif diff --git a/ddtrace/vendor/psutil/arch/windows/security.c b/ddtrace/vendor/psutil/arch/windows/security.c new file mode 100644 index 00000000000..4e2c7435b22 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/security.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Security related functions for Windows platform (Set privileges such as + * SE DEBUG). + */ + +#include +#include +#include "../../_psutil_common.h" + + +static BOOL +psutil_set_privilege(HANDLE hToken, LPCTSTR Privilege, BOOL bEnablePrivilege) { + TOKEN_PRIVILEGES tp; + LUID luid; + TOKEN_PRIVILEGES tpPrevious; + DWORD cbPrevious = sizeof(TOKEN_PRIVILEGES); + + if (! LookupPrivilegeValue(NULL, Privilege, &luid)) { + PyErr_SetFromOSErrnoWithSyscall("LookupPrivilegeValue"); + return 1; + } + + // first pass. get current privilege setting + tp.PrivilegeCount = 1; + tp.Privileges[0].Luid = luid; + tp.Privileges[0].Attributes = 0; + + if (! AdjustTokenPrivileges( + hToken, + FALSE, + &tp, + sizeof(TOKEN_PRIVILEGES), + &tpPrevious, + &cbPrevious)) + { + PyErr_SetFromOSErrnoWithSyscall("AdjustTokenPrivileges"); + return 1; + } + + // Second pass. Set privilege based on previous setting. + tpPrevious.PrivilegeCount = 1; + tpPrevious.Privileges[0].Luid = luid; + + if (bEnablePrivilege) + tpPrevious.Privileges[0].Attributes |= (SE_PRIVILEGE_ENABLED); + else + tpPrevious.Privileges[0].Attributes ^= + (SE_PRIVILEGE_ENABLED & tpPrevious.Privileges[0].Attributes); + + if (! AdjustTokenPrivileges( + hToken, + FALSE, + &tpPrevious, + cbPrevious, + NULL, + NULL)) + { + PyErr_SetFromOSErrnoWithSyscall("AdjustTokenPrivileges"); + return 1; + } + + return 0; +} + + +static HANDLE +psutil_get_thisproc_token() { + HANDLE hToken = NULL; + HANDLE me = GetCurrentProcess(); + + if (! OpenProcessToken( + me, TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) + { + if (GetLastError() == ERROR_NO_TOKEN) + { + if (! ImpersonateSelf(SecurityImpersonation)) { + PyErr_SetFromOSErrnoWithSyscall("ImpersonateSelf"); + return NULL; + } + if (! OpenProcessToken( + me, TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) + { + PyErr_SetFromOSErrnoWithSyscall("OpenProcessToken"); + return NULL; + } + } + else { + PyErr_SetFromOSErrnoWithSyscall("OpenProcessToken"); + return NULL; + } + } + + return hToken; +} + + +static void +psutil_print_err() { + char *msg = "psutil module couldn't set SE DEBUG mode for this process; " \ + "please file an issue against psutil bug tracker"; + psutil_debug(msg); + if (GetLastError() != ERROR_ACCESS_DENIED) + PyErr_WarnEx(PyExc_RuntimeWarning, msg, 1); + PyErr_Clear(); +} + + +/* + * Set this process in SE DEBUG mode so that we have more chances of + * querying processes owned by other users, including many owned by + * Administrator and Local System. + * https://docs.microsoft.com/windows-hardware/drivers/debugger/debug-privilege + * This is executed on module import and we don't crash on error. + */ +int +psutil_set_se_debug() { + HANDLE hToken; + int err = 1; + + if ((hToken = psutil_get_thisproc_token()) == NULL) { + // "return 1;" to get an exception + psutil_print_err(); + return 0; + } + + if (psutil_set_privilege(hToken, SE_DEBUG_NAME, TRUE) != 0) { + // "return 1;" to get an exception + psutil_print_err(); + } + + RevertToSelf(); + CloseHandle(hToken); + return 0; +} diff --git a/ddtrace/vendor/psutil/arch/windows/security.h b/ddtrace/vendor/psutil/arch/windows/security.h new file mode 100644 index 00000000000..8d4ddb00d41 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/security.h @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Security related functions for Windows platform (Set privileges such as + * SeDebug), as well as security helper functions. + */ + +#include + +int psutil_set_se_debug(); + diff --git a/ddtrace/vendor/psutil/arch/windows/services.c b/ddtrace/vendor/psutil/arch/windows/services.c new file mode 100644 index 00000000000..92458494b4a --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/services.c @@ -0,0 +1,485 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include +#include + +#include "services.h" +#include "../../_psutil_common.h" + +// ================================================================== +// utils +// ================================================================== + +SC_HANDLE +psutil_get_service_handler(char *service_name, DWORD scm_access, DWORD access) +{ + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; + SC_HANDLE sc = NULL; + SC_HANDLE hService = NULL; + + sc = OpenSCManager(NULL, NULL, scm_access); + if (sc == NULL) { + PyErr_SetFromOSErrnoWithSyscall("OpenSCManager"); + return NULL; + } + hService = OpenService(sc, service_name, access); + if (hService == NULL) { + PyErr_SetFromOSErrnoWithSyscall("OpenService"); + CloseServiceHandle(sc); + return NULL; + } + CloseServiceHandle(sc); + return hService; +} + + +// XXX - expose these as constants? +static const char * +get_startup_string(DWORD startup) { + switch (startup) { + case SERVICE_AUTO_START: + return "automatic"; + case SERVICE_DEMAND_START: + return "manual"; + case SERVICE_DISABLED: + return "disabled"; +/* + // drivers only (since we use EnumServicesStatusEx() with + // SERVICE_WIN32) + case SERVICE_BOOT_START: + return "boot-start"; + case SERVICE_SYSTEM_START: + return "system-start"; +*/ + default: + return "unknown"; + } +} + + +// XXX - expose these as constants? +static const char * +get_state_string(DWORD state) { + switch (state) { + case SERVICE_RUNNING: + return "running"; + case SERVICE_PAUSED: + return "paused"; + case SERVICE_START_PENDING: + return "start_pending"; + case SERVICE_PAUSE_PENDING: + return "pause_pending"; + case SERVICE_CONTINUE_PENDING: + return "continue_pending"; + case SERVICE_STOP_PENDING: + return "stop_pending"; + case SERVICE_STOPPED: + return "stopped"; + default: + return "unknown"; + } +} + + +// ================================================================== +// APIs +// ================================================================== + +/* + * Enumerate all services. + */ +PyObject * +psutil_winservice_enumerate(PyObject *self, PyObject *args) { + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; + BOOL ok; + SC_HANDLE sc = NULL; + DWORD bytesNeeded = 0; + DWORD srvCount; + DWORD resumeHandle = 0; + DWORD dwBytes = 0; + DWORD i; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_name = NULL; + PyObject *py_display_name = NULL; + + if (py_retlist == NULL) + return NULL; + + sc = OpenSCManager(NULL, NULL, SC_MANAGER_ENUMERATE_SERVICE); + if (sc == NULL) { + PyErr_SetFromOSErrnoWithSyscall("OpenSCManager"); + return NULL; + } + + for (;;) { + ok = EnumServicesStatusExW( + sc, + SC_ENUM_PROCESS_INFO, + SERVICE_WIN32, // XXX - extend this to include drivers etc.? + SERVICE_STATE_ALL, + (LPBYTE)lpService, + dwBytes, + &bytesNeeded, + &srvCount, + &resumeHandle, + NULL); + if (ok || (GetLastError() != ERROR_MORE_DATA)) + break; + if (lpService) + free(lpService); + dwBytes = bytesNeeded; + lpService = (ENUM_SERVICE_STATUS_PROCESSW*)malloc(dwBytes); + } + + for (i = 0; i < srvCount; i++) { + // Get unicode name / display name. + py_name = NULL; + py_name = PyUnicode_FromWideChar( + lpService[i].lpServiceName, wcslen(lpService[i].lpServiceName)); + if (py_name == NULL) + goto error; + + py_display_name = NULL; + py_display_name = PyUnicode_FromWideChar( + lpService[i].lpDisplayName, wcslen(lpService[i].lpDisplayName)); + if (py_display_name == NULL) + goto error; + + // Construct the result. + py_tuple = Py_BuildValue("(OO)", py_name, py_display_name); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_display_name); + Py_DECREF(py_name); + Py_DECREF(py_tuple); + } + + // Free resources. + CloseServiceHandle(sc); + free(lpService); + return py_retlist; + +error: + Py_DECREF(py_name); + Py_XDECREF(py_display_name); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (sc != NULL) + CloseServiceHandle(sc); + if (lpService != NULL) + free(lpService); + return NULL; +} + + +/* + * Get service config information. Returns: + * - display_name + * - binpath + * - username + * - startup_type + */ +PyObject * +psutil_winservice_query_config(PyObject *self, PyObject *args) { + char *service_name; + SC_HANDLE hService = NULL; + BOOL ok; + DWORD bytesNeeded = 0; + DWORD resumeHandle = 0; + DWORD dwBytes = 0; + QUERY_SERVICE_CONFIGW *qsc = NULL; + PyObject *py_tuple = NULL; + PyObject *py_unicode_display_name = NULL; + PyObject *py_unicode_binpath = NULL; + PyObject *py_unicode_username = NULL; + + if (!PyArg_ParseTuple(args, "s", &service_name)) + return NULL; + hService = psutil_get_service_handler( + service_name, SC_MANAGER_ENUMERATE_SERVICE, SERVICE_QUERY_CONFIG); + if (hService == NULL) + goto error; + + // First call to QueryServiceConfigW() is necessary to get the + // right size. + bytesNeeded = 0; + QueryServiceConfigW(hService, NULL, 0, &bytesNeeded); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + PyErr_SetFromOSErrnoWithSyscall("QueryServiceConfigW"); + goto error; + } + qsc = (QUERY_SERVICE_CONFIGW *)malloc(bytesNeeded); + ok = QueryServiceConfigW(hService, qsc, bytesNeeded, &bytesNeeded); + if (ok == 0) { + PyErr_SetFromOSErrnoWithSyscall("QueryServiceConfigW"); + goto error; + } + + // Get unicode display name. + py_unicode_display_name = PyUnicode_FromWideChar( + qsc->lpDisplayName, wcslen(qsc->lpDisplayName)); + if (py_unicode_display_name == NULL) + goto error; + + // Get unicode bin path. + py_unicode_binpath = PyUnicode_FromWideChar( + qsc->lpBinaryPathName, wcslen(qsc->lpBinaryPathName)); + if (py_unicode_binpath == NULL) + goto error; + + // Get unicode username. + py_unicode_username = PyUnicode_FromWideChar( + qsc->lpServiceStartName, wcslen(qsc->lpServiceStartName)); + if (py_unicode_username == NULL) + goto error; + + // Construct result tuple. + py_tuple = Py_BuildValue( + "(OOOs)", + py_unicode_display_name, + py_unicode_binpath, + py_unicode_username, + get_startup_string(qsc->dwStartType) // startup + ); + if (py_tuple == NULL) + goto error; + + // Free resources. + Py_DECREF(py_unicode_display_name); + Py_DECREF(py_unicode_binpath); + Py_DECREF(py_unicode_username); + free(qsc); + CloseServiceHandle(hService); + return py_tuple; + +error: + Py_XDECREF(py_unicode_display_name); + Py_XDECREF(py_unicode_binpath); + Py_XDECREF(py_unicode_username); + Py_XDECREF(py_tuple); + if (hService != NULL) + CloseServiceHandle(hService); + if (qsc != NULL) + free(qsc); + return NULL; +} + + +/* + * Get service status information. Returns: + * - status + * - pid + */ +PyObject * +psutil_winservice_query_status(PyObject *self, PyObject *args) { + char *service_name; + SC_HANDLE hService = NULL; + BOOL ok; + DWORD bytesNeeded = 0; + DWORD resumeHandle = 0; + DWORD dwBytes = 0; + SERVICE_STATUS_PROCESS *ssp = NULL; + PyObject *py_tuple = NULL; + + if (!PyArg_ParseTuple(args, "s", &service_name)) + return NULL; + hService = psutil_get_service_handler( + service_name, SC_MANAGER_ENUMERATE_SERVICE, SERVICE_QUERY_STATUS); + if (hService == NULL) + goto error; + + // First call to QueryServiceStatusEx() is necessary to get the + // right size. + QueryServiceStatusEx(hService, SC_STATUS_PROCESS_INFO, NULL, 0, + &bytesNeeded); + if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { + // Also services.msc fails in the same manner, so we return an + // empty string. + CloseServiceHandle(hService); + return Py_BuildValue("s", ""); + } + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + PyErr_SetFromOSErrnoWithSyscall("QueryServiceStatusEx"); + goto error; + } + ssp = (SERVICE_STATUS_PROCESS *)HeapAlloc( + GetProcessHeap(), 0, bytesNeeded); + if (ssp == NULL) { + PyErr_NoMemory(); + goto error; + } + + // Actual call. + ok = QueryServiceStatusEx(hService, SC_STATUS_PROCESS_INFO, (LPBYTE)ssp, + bytesNeeded, &bytesNeeded); + if (ok == 0) { + PyErr_SetFromOSErrnoWithSyscall("QueryServiceStatusEx"); + goto error; + } + + py_tuple = Py_BuildValue( + "(sk)", + get_state_string(ssp->dwCurrentState), + ssp->dwProcessId + ); + if (py_tuple == NULL) + goto error; + + CloseServiceHandle(hService); + HeapFree(GetProcessHeap(), 0, ssp); + return py_tuple; + +error: + Py_XDECREF(py_tuple); + if (hService != NULL) + CloseServiceHandle(hService); + if (ssp != NULL) + HeapFree(GetProcessHeap(), 0, ssp); + return NULL; +} + + +/* + * Get service description. + */ +PyObject * +psutil_winservice_query_descr(PyObject *self, PyObject *args) { + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; + BOOL ok; + DWORD bytesNeeded = 0; + DWORD resumeHandle = 0; + DWORD dwBytes = 0; + SC_HANDLE hService = NULL; + SERVICE_DESCRIPTIONW *scd = NULL; + char *service_name; + PyObject *py_retstr = NULL; + + if (!PyArg_ParseTuple(args, "s", &service_name)) + return NULL; + hService = psutil_get_service_handler( + service_name, SC_MANAGER_ENUMERATE_SERVICE, SERVICE_QUERY_CONFIG); + if (hService == NULL) + goto error; + + // This first call to QueryServiceConfig2W() is necessary in order + // to get the right size. + bytesNeeded = 0; + QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, + &bytesNeeded); + if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { + // Also services.msc fails in the same manner, so we return an + // empty string. + CloseServiceHandle(hService); + return Py_BuildValue("s", ""); + } + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + PyErr_SetFromOSErrnoWithSyscall("QueryServiceConfig2W"); + goto error; + } + + scd = (SERVICE_DESCRIPTIONW *)malloc(bytesNeeded); + ok = QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, + (LPBYTE)scd, bytesNeeded, &bytesNeeded); + if (ok == 0) { + PyErr_SetFromOSErrnoWithSyscall("QueryServiceConfig2W"); + goto error; + } + + if (scd->lpDescription == NULL) { + py_retstr = Py_BuildValue("s", ""); + } + else { + py_retstr = PyUnicode_FromWideChar( + scd->lpDescription, wcslen(scd->lpDescription)); + } + if (!py_retstr) + goto error; + + free(scd); + CloseServiceHandle(hService); + return py_retstr; + +error: + if (hService != NULL) + CloseServiceHandle(hService); + if (lpService != NULL) + free(lpService); + return NULL; +} + + +/* + * Start service. + * XXX - note: this is exposed but not used. + */ +PyObject * +psutil_winservice_start(PyObject *self, PyObject *args) { + char *service_name; + BOOL ok; + SC_HANDLE hService = NULL; + + if (!PyArg_ParseTuple(args, "s", &service_name)) + return NULL; + hService = psutil_get_service_handler( + service_name, SC_MANAGER_ALL_ACCESS, SERVICE_START); + if (hService == NULL) { + goto error; + } + ok = StartService(hService, 0, NULL); + if (ok == 0) { + PyErr_SetFromOSErrnoWithSyscall("StartService"); + goto error; + } + + Py_RETURN_NONE; + +error: + if (hService != NULL) + CloseServiceHandle(hService); + return NULL; +} + + +/* + * Stop service. + * XXX - note: this is exposed but not used. + */ +PyObject * +psutil_winservice_stop(PyObject *self, PyObject *args) { + char *service_name; + BOOL ok; + SC_HANDLE hService = NULL; + SERVICE_STATUS ssp; + + if (!PyArg_ParseTuple(args, "s", &service_name)) + return NULL; + hService = psutil_get_service_handler( + service_name, SC_MANAGER_ALL_ACCESS, SERVICE_STOP); + if (hService == NULL) + goto error; + + // Note: this can hang for 30 secs. + Py_BEGIN_ALLOW_THREADS + ok = ControlService(hService, SERVICE_CONTROL_STOP, &ssp); + Py_END_ALLOW_THREADS + if (ok == 0) { + PyErr_SetFromOSErrnoWithSyscall("ControlService"); + goto error; + } + + CloseServiceHandle(hService); + Py_RETURN_NONE; + +error: + if (hService != NULL) + CloseServiceHandle(hService); + return NULL; +} diff --git a/ddtrace/vendor/psutil/arch/windows/services.h b/ddtrace/vendor/psutil/arch/windows/services.h new file mode 100644 index 00000000000..286ed232c90 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/services.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include + +SC_HANDLE psutil_get_service_handle( +char service_name, DWORD scm_access, DWORD access); +PyObject *psutil_winservice_enumerate(PyObject *self, PyObject *args); +PyObject *psutil_winservice_query_config(PyObject *self, PyObject *args); +PyObject *psutil_winservice_query_status(PyObject *self, PyObject *args); +PyObject *psutil_winservice_query_descr(PyObject *self, PyObject *args); +PyObject *psutil_winservice_start(PyObject *self, PyObject *args); +PyObject *psutil_winservice_stop(PyObject *self, PyObject *args); diff --git a/ddtrace/vendor/psutil/arch/windows/wmi.c b/ddtrace/vendor/psutil/arch/windows/wmi.c new file mode 100644 index 00000000000..f43d790c035 --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/wmi.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Functions related to the Windows Management Instrumentation API. + */ + +#include +#include +#include + +#include "../../_psutil_common.h" + + +// We use an exponentially weighted moving average, just like Unix systems do +// https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation +// +// These constants serve as the damping factor and are calculated with +// 1 / exp(sampling interval in seconds / window size in seconds) +// +// This formula comes from linux's include/linux/sched/loadavg.h +// https://github.com/torvalds/linux/blob/345671ea0f9258f410eb057b9ced9cefbbe5dc78/include/linux/sched/loadavg.h#L20-L23 +#define LOADAVG_FACTOR_1F 0.9200444146293232478931553241 +#define LOADAVG_FACTOR_5F 0.9834714538216174894737477501 +#define LOADAVG_FACTOR_15F 0.9944598480048967508795473394 +// The time interval in seconds between taking load counts, same as Linux +#define SAMPLING_INTERVAL 5 + +double load_avg_1m = 0; +double load_avg_5m = 0; +double load_avg_15m = 0; + + +VOID CALLBACK LoadAvgCallback(PVOID hCounter) { + PDH_FMT_COUNTERVALUE displayValue; + double currentLoad; + PDH_STATUS err; + + err = PdhGetFormattedCounterValue( + (PDH_HCOUNTER)hCounter, PDH_FMT_DOUBLE, 0, &displayValue); + // Skip updating the load if we can't get the value successfully + if (err != ERROR_SUCCESS) { + return; + } + currentLoad = displayValue.doubleValue; + + load_avg_1m = load_avg_1m * LOADAVG_FACTOR_1F + currentLoad * \ + (1.0 - LOADAVG_FACTOR_1F); + load_avg_5m = load_avg_5m * LOADAVG_FACTOR_5F + currentLoad * \ + (1.0 - LOADAVG_FACTOR_5F); + load_avg_15m = load_avg_15m * LOADAVG_FACTOR_15F + currentLoad * \ + (1.0 - LOADAVG_FACTOR_15F); +} + + +PyObject * +psutil_init_loadavg_counter(PyObject *self, PyObject *args) { + WCHAR *szCounterPath = L"\\System\\Processor Queue Length"; + PDH_STATUS s; + BOOL ret; + HQUERY hQuery; + HCOUNTER hCounter; + HANDLE event; + HANDLE waitHandle; + + if ((PdhOpenQueryW(NULL, 0, &hQuery)) != ERROR_SUCCESS) + goto error; + + s = PdhAddEnglishCounterW(hQuery, szCounterPath, 0, &hCounter); + if (s != ERROR_SUCCESS) + goto error; + + event = CreateEventW(NULL, FALSE, FALSE, L"LoadUpdateEvent"); + if (event == NULL) { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + s = PdhCollectQueryDataEx(hQuery, SAMPLING_INTERVAL, event); + if (s != ERROR_SUCCESS) + goto error; + + ret = RegisterWaitForSingleObject( + &waitHandle, + event, + (WAITORTIMERCALLBACK)LoadAvgCallback, + (PVOID) + hCounter, + INFINITE, + WT_EXECUTEDEFAULT); + + if (ret == 0) { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + Py_RETURN_NONE; + +error: + PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); + return NULL; +} + + +/* + * Gets the emulated 1 minute, 5 minute and 15 minute load averages + * (processor queue length) for the system. + * `init_loadavg_counter` must be called before this function to engage the + * mechanism that records load values. + */ +PyObject * +psutil_get_loadavg(PyObject *self, PyObject *args) { + return Py_BuildValue("(ddd)", load_avg_1m, load_avg_5m, load_avg_15m); +} diff --git a/ddtrace/vendor/psutil/arch/windows/wmi.h b/ddtrace/vendor/psutil/arch/windows/wmi.h new file mode 100644 index 00000000000..0210f2d699e --- /dev/null +++ b/ddtrace/vendor/psutil/arch/windows/wmi.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + + + +#include + +PyObject* psutil_init_loadavg_counter(); +PyObject* psutil_get_loadavg(); diff --git a/ddtrace/vendor/psutil/setup.py b/ddtrace/vendor/psutil/setup.py new file mode 100644 index 00000000000..21de235487c --- /dev/null +++ b/ddtrace/vendor/psutil/setup.py @@ -0,0 +1,227 @@ +__all__ = ["get_extensions"] + +import contextlib +import io +import os +import platform +from setuptools import Extension +import shutil +import sys +import tempfile + +POSIX = os.name == "posix" +WINDOWS = os.name == "nt" +LINUX = sys.platform.startswith("linux") +MACOS = sys.platform.startswith("darwin") +OSX = MACOS # deprecated alias +FREEBSD = sys.platform.startswith("freebsd") +OPENBSD = sys.platform.startswith("openbsd") +NETBSD = sys.platform.startswith("netbsd") +BSD = FREEBSD or OPENBSD or NETBSD +SUNOS = sys.platform.startswith(("sunos", "solaris")) +AIX = sys.platform.startswith("aix") + + +@contextlib.contextmanager +def silenced_output(stream_name): + class DummyFile(io.BytesIO): + # see: https://github.com/giampaolo/psutil/issues/678 + errors = "ignore" + + def write(self, s): + pass + + orig = getattr(sys, stream_name) + try: + setattr(sys, stream_name, DummyFile()) + yield + finally: + setattr(sys, stream_name, orig) + + +def get_extensions(): + macros = [("PSUTIL_VERSION", 567)] + if POSIX: + macros.append(("PSUTIL_POSIX", 1)) + if BSD: + macros.append(("PSUTIL_BSD", 1)) + + sources = ["ddtrace/vendor/psutil/_psutil_common.c"] + if POSIX: + sources.append("ddtrace/vendor/psutil/_psutil_posix.c") + + if WINDOWS: + + def get_winver(): + win_maj, win_min = sys.getwindowsversion()[0:2] + return "0x0%s" % ((win_maj * 100) + win_min) + + if sys.getwindowsversion()[0] < 6: + msg = "this Windows version is too old (< Windows Vista); " + msg += "psutil 3.4.2 is the latest version which supports Windows " + msg += "2000, XP and 2003 server" + raise RuntimeError(msg) + + macros.append(("PSUTIL_WINDOWS", 1)) + macros.extend( + [ + # be nice to mingw, see: + # http://www.mingw.org/wiki/Use_more_recent_defined_functions + ("_WIN32_WINNT", get_winver()), + ("_AVAIL_WINVER_", get_winver()), + ("_CRT_SECURE_NO_WARNINGS", None), + # see: https://github.com/giampaolo/psutil/issues/348 + ("PSAPI_VERSION", 1), + ] + ) + + sources += [ + "ddtrace/vendor/psutil/_psutil_windows.c", + "ddtrace/vendor/psutil/arch/windows/process_info.c", + "ddtrace/vendor/psutil/arch/windows/process_handles.c", + "ddtrace/vendor/psutil/arch/windows/security.c", + "ddtrace/vendor/psutil/arch/windows/inet_ntop.c", + "ddtrace/vendor/psutil/arch/windows/services.c", + "ddtrace/vendor/psutil/arch/windows/global.c", + "ddtrace/vendor/psutil/arch/windows/wmi.c", + ] + ext = Extension( + "ddtrace.vendor.psutil._psutil_windows", + sources=sources, + define_macros=macros, + libraries=[ + "psapi", + "kernel32", + "advapi32", + "shell32", + "netapi32", + "wtsapi32", + "ws2_32", + "PowrProf", + "pdh", + ], + # extra_compile_args=["/Z7"], + # extra_link_args=["/DEBUG"] + ) + + elif MACOS: + macros.append(("PSUTIL_OSX", 1)) + sources += [ + "ddtrace/vendor/psutil/_psutil_osx.c", + "ddtrace/vendor/psutil/arch/osx/process_info.c", + ] + ext = Extension( + "ddtrace.vendor.psutil._psutil_osx", + sources=sources, + define_macros=macros, + extra_link_args=["-framework", "CoreFoundation", "-framework", "IOKit"], + ) + + elif FREEBSD: + macros.append(("PSUTIL_FREEBSD", 1)) + sources += [ + "ddtrace/vendor/psutil/_psutil_bsd.c", + "ddtrace/vendor/psutil/arch/freebsd/specific.c", + "ddtrace/vendor/psutil/arch/freebsd/sys_socks.c", + "ddtrace/vendor/psutil/arch/freebsd/proc_socks.c", + ] + ext = Extension( + "ddtrace.vendor.psutil._psutil_bsd", sources=sources, define_macros=macros, libraries=["devstat"], + ) + + elif OPENBSD: + macros.append(("PSUTIL_OPENBSD", 1)) + ext = Extension( + "ddtrace.vendor.psutil._psutil_bsd", + sources=sources + ["ddtrace/vendor/psutil/_psutil_bsd.c", "ddtrace/vendor/psutil/arch/openbsd/specific.c"], + define_macros=macros, + libraries=["kvm"], + ) + + elif NETBSD: + macros.append(("PSUTIL_NETBSD", 1)) + sources += [ + "ddtrace/vendor/psutil/_psutil_bsd.c", + "ddtrace/vendor/psutil/arch/netbsd/specific.c", + "ddtrace/vendor/psutil/arch/netbsd/socks.c", + ] + ext = Extension("ddtrace.vendor.psutil._psutil_bsd", sources=sources, define_macros=macros, libraries=["kvm"],) + + elif LINUX: + + def get_ethtool_macro(): + # see: https://github.com/giampaolo/ddtrace/vendor/psutil/issues/659 + from distutils.unixccompiler import UnixCCompiler + from distutils.errors import CompileError + + with tempfile.NamedTemporaryFile(suffix=".c", delete=False, mode="wt") as f: + f.write("#include ") + + output_dir = tempfile.mkdtemp() + try: + compiler = UnixCCompiler() + # https://github.com/giampaolo/ddtrace/vendor/psutil/pull/1568 + if os.getenv("CC"): + compiler.set_executable("compiler_so", os.getenv("CC")) + with silenced_output("stderr"): + with silenced_output("stdout"): + compiler.compile([f.name], output_dir=output_dir) + except CompileError: + return ("PSUTIL_ETHTOOL_MISSING_TYPES", 1) + else: + return None + finally: + os.remove(f.name) + shutil.rmtree(output_dir) + + macros.append(("PSUTIL_LINUX", 1)) + ETHTOOL_MACRO = get_ethtool_macro() + if ETHTOOL_MACRO is not None: + macros.append(ETHTOOL_MACRO) + ext = Extension( + "ddtrace.vendor.psutil._psutil_linux", + sources=sources + ["ddtrace/vendor/psutil/_psutil_linux.c"], + define_macros=macros, + ) + + elif SUNOS: + macros.append(("PSUTIL_SUNOS", 1)) + sources += [ + "ddtrace/vendor/psutil/_psutil_sunos.c", + "ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.c", + "ddtrace/vendor/psutil/arch/solaris/environ.c", + ] + ext = Extension( + "ddtrace.vendor.psutil._psutil_sunos", + sources=sources, + define_macros=macros, + libraries=["kstat", "nsl", "socket"], + ) + + elif AIX: + macros.append(("PSUTIL_AIX", 1)) + sources += [ + "ddtrace/vendor/psutil/_psutil_aix.c", + "ddtrace/vendor/psutil/arch/aix/net_connections.c", + "ddtrace/vendor/psutil/arch/aix/common.c", + "ddtrace/vendor/psutil/arch/aix/ifaddrs.c", + ] + ext = Extension( + "ddtrace.vendor.psutil._psutil_aix", sources=sources, libraries=["perfstat"], define_macros=macros, + ) + else: + raise RuntimeError("platform %s is not supported" % sys.platform) + + if POSIX: + posix_extension = Extension("ddtrace.vendor.psutil._psutil_posix", define_macros=macros, sources=sources) + if SUNOS: + posix_extension.libraries.append("socket") + if platform.release() == "5.10": + posix_extension.sources.append("ddtrace/vendor/psutil/arch/solaris/v10/ifaddrs.c") + posix_extension.define_macros.append(("PSUTIL_SUNOS10", 1)) + elif AIX: + posix_extension.sources.append("ddtrace/vendor/psutil/arch/aix/ifaddrs.c") + + return [ext, posix_extension] + else: + return [ext] diff --git a/setup.py b/setup.py index a21be5dd98a..8d6693102f6 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def run_tests(self): # enum34 is an enum backport for earlier versions of python # funcsigs backport required for vendored debtcollector -install_requires = ["psutil>=5.0.0", "enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'"] +install_requires = ["enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( @@ -165,7 +165,7 @@ def get_exts_for(name): # Try to build with C extensions first, fallback to only pure-Python if building fails try: all_exts = [] - for extname in ("msgpack", "wrapt"): + for extname in ("msgpack", "wrapt", "psutil"): exts = get_exts_for(extname) if exts: all_exts.extend(exts) From 140b953e31e085553cf6e50e2a6a68488ed40040 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Tue, 11 Feb 2020 08:26:30 -0500 Subject: [PATCH 52/81] tests: Run django_contrib and django_drf_contrib tests separately (#1197) This should help with concurrency in CircleCI and help speed up the slow Django builds --- .circleci/config.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09150d9cd32..f8831db78b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -336,7 +336,25 @@ jobs: steps: - checkout - *restore_cache_step - - run: scripts/run-tox-scenario '^django_' + - run: scripts/run-tox-scenario '^django_contrib' + - *persist_to_workspace_step + - *save_cache_step + + djangorestframework: + docker: + - *test_runner + - image: redis:4.0-alpine + - image: memcached:1.5-alpine + - image: datadog/docker-dd-agent + env: + - DD_APM_ENABLED=true + - DD_BIND_HOST=0.0.0.0 + - DD_API_KEY=invalid_key_but_this_is_fine + resource_class: *resource_class + steps: + - checkout + - *restore_cache_step + - run: scripts/run-tox-scenario '^django_drf_contrib' - *persist_to_workspace_step - *save_cache_step @@ -901,6 +919,10 @@ workflows: requires: - flake8 - black + - djangorestframework: + requires: + - flake8 + - black - dogpile_cache: requires: - flake8 @@ -1077,6 +1099,7 @@ workflows: - ddtracerun - dogpile_cache - django + - djangorestframework - elasticsearch - falcon - flask From f0746c39c4ef9c52b9a7590f7daf0e6c7c613dc4 Mon Sep 17 00:00:00 2001 From: Kyle Verhoog Date: Tue, 11 Feb 2020 09:12:26 -0500 Subject: [PATCH 53/81] [django] New Django Integration (#1161) * [django] Setup pytest-django * fix djangorestframework tests * Move existing django tests (#997) * [django] Remove autopatching (#998) * Move existing django tests * [django] Remove autopatching * add back missing 'apply_django_patches' changes * fix merge conflict * [django] new patching approach * [django] trace all middleware __call__, process_request, process_response * [django] add documentation * [django] add other middleware hooks * [django] updated request data * [django] add tests * [django] update tox.ini * [django] add instrumentation for rendering and views * [django] add db and cache instrumentation back * [django] add idempotency checks, tests * [django] add django v1 tests * [django] add patch test case * [django] fix view instrumentation * [django] pull duplicate app code into shared modules * [django] clarify middleware patching * [django] finally get working db test * [django] use pytest test cases * [django] get caches working * [django] add cache client tests * [django] begin adding legacy configuration support * [django] add migration guide, integration documentation * [django] check if DRF installed before attempting to patch * [django] add service name config tests * [django] add back database service name prefix * [django] enable by default * [django] add template tests * [django] remove old code * [django] add analytics config tests * [django] error handling tests * [django] port over rest of middleware tests * [django] fix trace_query_string * [django] delete old tests * [django] python2 compatibility * [django] get DRF working and tested * [django] update django, drf matrix support * [django] housekeeping * [django] lint * [django] update tox matrix * [django] actually use vendored wrapt * [django] add tests for view methods; change log levels * [django] linting * [django] convert to SpanTypes, use debtcollector * [django] run black * [django] rm more old code * [django] add black-compatibility flake ignore * [django] docstring tweaks; cleanup * [django] ignore rst whitespace flake rule * [django] add config to disable caches and databases * [django] add urlpattern tests * [django] remove static.static instrumentation * Add back missing TraceMiddleware * manually call patch() when using old settings without calling patch * add migration from settings * add migration docs * fix service_name * fix django db * fix unpatching * black format * fix black/flake8 * fix black * black check django contrib * update docs * skip unpatch in test * remove unused import * allow wider range of env names * just use _SERVICE_NAME for now * fix migration tests * fix db patching * fix drf tests * avoid info logs Co-authored-by: Brett Langdon Co-authored-by: Tahir H. Butt --- ddtrace/contrib/dbapi/__init__.py | 2 +- ddtrace/contrib/django/__init__.py | 265 ++-- ddtrace/contrib/django/apps.py | 29 +- ddtrace/contrib/django/cache.py | 111 -- ddtrace/contrib/django/compat.py | 8 +- ddtrace/contrib/django/conf.py | 124 +- ddtrace/contrib/django/db.py | 76 -- ddtrace/contrib/django/middleware.py | 227 +--- ddtrace/contrib/django/patch.py | 626 ++++++++- ddtrace/contrib/django/restframework.py | 51 +- ddtrace/contrib/django/templates.py | 48 - ddtrace/contrib/django/utils.py | 70 +- ddtrace/monkey.py | 2 +- ddtrace/utils/wrappers.py | 7 + pyproject.toml | 17 +- tests/contrib/django/app/middlewares.py | 36 - tests/contrib/django/app/settings.py | 132 -- tests/contrib/django/app/settings_untraced.py | 106 -- tests/contrib/django/app/views.py | 74 - tests/contrib/django/compat.py | 2 +- tests/contrib/django/conftest.py | 50 +- .../django/{app => django1_app}/__init__.py | 0 tests/contrib/django/django1_app/settings.py | 88 ++ tests/contrib/django/django1_app/urls.py | 19 + tests/contrib/django/django_app/__init__.py | 0 tests/contrib/django/django_app/extra_urls.py | 11 + tests/contrib/django/django_app/settings.py | 84 ++ tests/contrib/django/django_app/urls.py | 32 + tests/contrib/django/middleware.py | 78 ++ tests/contrib/django/runtests.py | 18 - .../{app => }/templates/cached_list.html | 0 .../{app => }/templates/users_list.html | 0 tests/contrib/django/test_autopatching.py | 98 -- tests/contrib/django/test_cache_backends.py | 246 ---- tests/contrib/django/test_cache_client.py | 364 ----- tests/contrib/django/test_cache_views.py | 93 -- tests/contrib/django/test_cache_wrapper.py | 133 -- tests/contrib/django/test_connection.py | 72 - tests/contrib/django/test_django.py | 1204 +++++++++++++++++ tests/contrib/django/test_django_migration.py | 43 + tests/contrib/django/test_django_patch.py | 45 + tests/contrib/django/test_instrumentation.py | 35 - tests/contrib/django/test_middleware.py | 460 ------- tests/contrib/django/test_templates.py | 46 - tests/contrib/django/test_tracing_disabled.py | 42 - tests/contrib/django/test_utils.py | 17 - tests/contrib/django/utils.py | 87 -- tests/contrib/django/views.py | 78 ++ .../djangorestframework/app/exceptions.py | 2 +- .../djangorestframework/app/settings.py | 118 +- .../contrib/djangorestframework/app/views.py | 11 +- tests/contrib/djangorestframework/conftest.py | 13 +- .../test_djangorestframework.py | 85 +- tests/contrib/patch.py | 89 +- tox.ini | 100 +- 55 files changed, 2880 insertions(+), 2994 deletions(-) delete mode 100644 ddtrace/contrib/django/cache.py delete mode 100644 ddtrace/contrib/django/db.py delete mode 100644 ddtrace/contrib/django/templates.py delete mode 100644 tests/contrib/django/app/middlewares.py delete mode 100644 tests/contrib/django/app/settings.py delete mode 100644 tests/contrib/django/app/settings_untraced.py delete mode 100644 tests/contrib/django/app/views.py rename tests/contrib/django/{app => django1_app}/__init__.py (100%) create mode 100644 tests/contrib/django/django1_app/settings.py create mode 100644 tests/contrib/django/django1_app/urls.py create mode 100644 tests/contrib/django/django_app/__init__.py create mode 100644 tests/contrib/django/django_app/extra_urls.py create mode 100644 tests/contrib/django/django_app/settings.py create mode 100644 tests/contrib/django/django_app/urls.py create mode 100644 tests/contrib/django/middleware.py delete mode 100755 tests/contrib/django/runtests.py rename tests/contrib/django/{app => }/templates/cached_list.html (100%) rename tests/contrib/django/{app => }/templates/users_list.html (100%) delete mode 100644 tests/contrib/django/test_autopatching.py delete mode 100644 tests/contrib/django/test_cache_backends.py delete mode 100644 tests/contrib/django/test_cache_client.py delete mode 100644 tests/contrib/django/test_cache_views.py delete mode 100644 tests/contrib/django/test_cache_wrapper.py delete mode 100644 tests/contrib/django/test_connection.py create mode 100644 tests/contrib/django/test_django.py create mode 100644 tests/contrib/django/test_django_migration.py create mode 100644 tests/contrib/django/test_django_patch.py delete mode 100644 tests/contrib/django/test_instrumentation.py delete mode 100644 tests/contrib/django/test_middleware.py delete mode 100644 tests/contrib/django/test_templates.py delete mode 100644 tests/contrib/django/test_tracing_disabled.py delete mode 100644 tests/contrib/django/test_utils.py delete mode 100644 tests/contrib/django/utils.py create mode 100644 tests/contrib/django/views.py diff --git a/ddtrace/contrib/dbapi/__init__.py b/ddtrace/contrib/dbapi/__init__.py index 01b34dab08c..e8c577c7741 100644 --- a/ddtrace/contrib/dbapi/__init__.py +++ b/ddtrace/contrib/dbapi/__init__.py @@ -19,7 +19,7 @@ class TracedCursor(wrapt.ObjectProxy): - """ TracedCursor wraps a psql cursor and traces it's queries. """ + """ TracedCursor wraps a psql cursor and traces its queries. """ def __init__(self, cursor, pin): super(TracedCursor, self).__init__(cursor) diff --git a/ddtrace/contrib/django/__init__.py b/ddtrace/contrib/django/__init__.py index f3405dd4f2a..9a38d888c09 100644 --- a/ddtrace/contrib/django/__init__.py +++ b/ddtrace/contrib/django/__init__.py @@ -1,36 +1,7 @@ """ -The Django integration will trace users requests, template renderers, database and cache -calls. +The Django__ integration traces requests, views, template renderers, database +and cache calls in a Django application. -**Note:** by default the tracer is **disabled** (will not send spans) when -the Django setting ``DEBUG`` is ``True``. This can be overridden by explicitly enabling -the tracer with ``DATADOG_TRACE['ENABLED'] = True``, as described below. - -To enable the Django integration, add the application to your installed -apps, as follows:: - - INSTALLED_APPS = [ - # your Django apps... - - # the order is not important - 'ddtrace.contrib.django', - ] - -The configuration for this integration is namespaced under the ``DATADOG_TRACE`` -Django setting. For example, your ``settings.py`` may contain:: - - DATADOG_TRACE = { - 'DEFAULT_SERVICE': 'my-django-app', - 'TAGS': {'env': 'production'}, - } - -If you need to access to Datadog settings, you can:: - - from ddtrace.contrib.django.conf import settings - - tracer = settings.TRACER - tracer.trace("something") - # your code ... To have Django capture the tracer logs, ensure the ``LOGGING`` variable in ``settings.py`` looks similar to:: @@ -45,57 +16,199 @@ } -The available settings are: - -* ``DEFAULT_SERVICE`` (default: ``'django'``): set the service name used by the - tracer. Usually this configuration must be updated with a meaningful name. -* ``DEFAULT_DATABASE_PREFIX`` (default: ``''``): set a prefix value to database services, - so that your service is listed such as `prefix-defaultdb`. -* ``DEFAULT_CACHE_SERVICE`` (default: ``''``): set the django cache service name used - by the tracer. Change this name if you want to see django cache spans as a cache application. -* ``TAGS`` (default: ``{}``): set global tags that should be applied to all - spans. -* ``TRACER`` (default: ``ddtrace.tracer``): set the default tracer - instance that is used to trace Django internals. By default the ``ddtrace`` - tracer is used. -* ``ENABLED`` (default: ``not django_settings.DEBUG``): defines if the tracer is - enabled or not. If set to false, the code is still instrumented but no spans - are sent to the trace agent. This setting cannot be changed at runtime - and a restart is required. By default the tracer is disabled when in ``DEBUG`` - mode, enabled otherwise. -* ``DISTRIBUTED_TRACING`` (default: ``True``): defines if the tracer should - use incoming X-DATADOG-* HTTP headers to extend a trace created remotely. It is - required for distributed tracing if this application is called remotely from another - instrumented application. - We suggest to enable it only for internal services where headers are under your control. -* ``ANALYTICS_ENABLED`` (default: ``None``): enables APM events in Trace Search & Analytics. -* ``AGENT_HOSTNAME`` (default: ``localhost``): define the hostname of the trace agent. -* ``AGENT_PORT`` (default: ``8126``): define the port of the trace agent. -* ``AUTO_INSTRUMENT`` (default: ``True``): if set to false the code will not be - instrumented (even if ``INSTRUMENT_DATABASE``, ``INSTRUMENT_CACHE`` or - ``INSTRUMENT_TEMPLATE`` are set to ``True``), while the tracer may be active - for your internal usage. This could be useful if you want to use the Django - integration, but you want to trace only particular functions or views. If set - to False, the request middleware will be disabled even if present. -* ``INSTRUMENT_DATABASE`` (default: ``True``): if set to ``False`` database will not - be instrumented. Only configurable when ``AUTO_INSTRUMENT`` is set to ``True``. -* ``INSTRUMENT_CACHE`` (default: ``True``): if set to ``False`` cache will not - be instrumented. Only configurable when ``AUTO_INSTRUMENT`` is set to ``True``. -* ``INSTRUMENT_TEMPLATE`` (default: ``True``): if set to ``False`` template - rendering will not be instrumented. Only configurable when ``AUTO_INSTRUMENT`` - is set to ``True``. -""" +Configuration +~~~~~~~~~~~~~ +.. py:data:: ddtrace.config.django['distributed_tracing_enabled'] + + Whether or not to parse distributed tracing headers from requests received by your Django app. + + Default: ``True`` + +.. py:data:: ddtrace.config.django['analytics_enabled'] + + Whether to generate APM events for Django in Trace Search & Analytics. + + Can also be enabled with the ``DD_DJANGO_ANALYTICS_ENABLED`` environment variable. + + Default: ``None`` + +.. py:data:: ddtrace.config.django['service_name'] + + The service name reported for your Django app. + + Can also be configured via the ``DD_SERVICE_NAME`` environment variable. + + Default: ``'django'`` + +.. py:data:: ddtrace.config.django['cache_service_name'] + + The service name reported for your Django app cache layer. + + Can also be configured via the ``DD_DJANGO_CACHE_SERVICE_NAME`` environment variable. + + Default: ``'django'`` + +.. py:data:: ddtrace.config.django['database_service_name_prefix'] + + A string to be prepended to the service name reported for your Django app database layer. + + Can also be configured via the ``DD_DJANGO_DATABASE_SERVICE_NAME_PREFIX`` environment variable. + + The database service name is the name of the database appended with 'db'. + + Default: ``''`` + +.. py:data:: ddtrace.config.django['instrument_databases'] + + Whether or not to instrument databases. + + Default: ``True`` + +.. py:data:: ddtrace.config.django['instrument_caches'] + + Whether or not to instrument caches. + + Default: ``True`` + +.. py:data:: ddtrace.config.django['trace_query_string'] + + Whether or not to include the query string as a tag. + + Default: ``False`` + + +Example:: + + from ddtrace import config + + # Enable distributed tracing + config.django['distributed_tracing_enabled'] = True + + # Override service name + config.django['service_name'] = 'custom-service-name' + + +Migration from ddtrace<=0.33.0 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The Django integration provides automatic migration from enabling tracing using +a middleware to the method consistent with our integrations. Application +developers are encouraged to convert their configuration of the tracer to the +later. + +1. Remove ``'ddtrace.config.django'`` from ``INSTALLED_APPS`` in + ``settings.py``. + +2. Replace ``DATADOG_TRACE`` configuration in ``settings.py`` according to the + table below. + +3. Remove ``TraceMiddleware`` or ``TraceExceptionMiddleware`` if used in + ``settings.py``. + +The mapping from old configuration settings to new ones. + ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``DATADOG_TRACE`` | Configuration | ++=============================+=========================================================================================================================+ +| ``AGENT_HOSTNAME`` | ``DD_AGENT_HOST`` environment variable or ``tracer.configure(hostname=)`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``AGENT_PORT`` | ``DD_TRACE_AGENT_PORT`` environment variable or ``tracer.configure(port=)`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``AUTO_INSTRUMENT`` | N/A Instrumentation is automatic | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``INSTRUMENT_CACHE`` | N/A Instrumentation is automatic | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``INSTRUMENT_DATABASE`` | ``config.django['instrument_databases']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``INSTRUMENT_TEMPLATE`` | ``config.django['instrument_caches']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``DEFAULT_DATABASE_PREFIX`` | ``config.django['database_service_name_prefix']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``DEFAULT_SERVICE`` | ``DD_SERVICE_NAME`` environment variable or ``config.django['service_name']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``DEFAULT_CACHE_SERVICE`` | ``config.django['cache_service_name']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``ENABLED`` | ``tracer.configure(enabled=)`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``DISTRIBUTED_TRACING`` | ``config.django['distributed_tracing_enabled']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``ANALYTICS_ENABLED`` | ``config.django['analytics_enabled']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``ANALYTICS_SAMPLE_RATE`` | ``config.django['analytics_sample_rate']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``TRACE_QUERY_STRING`` | ``config.django['trace_query_string']`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``TAGS`` | ``DD_TRACE_GLOBAL_TAGS`` environment variable or ``tracer.set_tags()`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ +| ``TRACER`` | N/A - if a particular tracer is required for the Django integration use ``Pin.override(Pin.get_from(django), tracer=)`` | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------+ + +Examples +-------- +Before:: + + # settings.py + INSTALLED_APPS = [ + # your Django apps... + 'ddtrace.contrib.django', + ] + + DATADOG_TRACE = { + 'AGENT_HOSTNAME': 'localhost', + 'AGENT_PORT': 8126, + 'AUTO_INSTRUMENT': True, + 'INSTRUMENT_CACHE': True, + 'INSTRUMENT_DATABASE': True, + 'INSTRUMENT_TEMPLATE': True, + 'DEFAULT_SERVICE': 'my-django-app', + 'DEFAULT_CACHE_SERVICE': 'my-cache', + 'DEFAULT_DATABASE_PREFIX': 'my-', + 'ENABLED': True, + 'DISTRIBUTED_TRACING': True, + 'ANALYTICS_ENABLED': True, + 'ANALYTICS_SAMPLE_RATE': 0.5, + 'TRACE_QUERY_STRING': None, + 'TAGS': {'env': 'production'}, + 'TRACER': 'my.custom.tracer', + } + +After:: + + # settings.py + INSTALLED_APPS = [ + # your Django apps... + ] + + from ddtrace import config, tracer + tracer.configure(hostname='localhost', port=8126, enabled=True) + config.django['service_name'] = 'my-django-app' + config.django['cache_service_name'] = 'my-cache' + config.django['django_service_name_prefix'] = 'my-' + config.django['instrument_databases'] = True + config.django['instrument_caches'] = True + config.django['trace_query_string'] = True + config.django['analytics_enabled'] = True + config.django['analytics_sample_rate'] = 0.5 + tracer.set_tags({'env': 'production'}) + + import my.custom.tracer + from ddtrace import Pin + import django + Pin.override(Pin.get_from(django), tracer=my.custom.tracer) + +.. __: https://www.djangoproject.com/ +""" # noqa: E501 from ...utils.importlib import require_modules -required_modules = ['django'] +required_modules = ["django"] with require_modules(required_modules) as missing_modules: if not missing_modules: - from .middleware import TraceMiddleware, TraceExceptionMiddleware - from .patch import patch - __all__ = ['TraceMiddleware', 'TraceExceptionMiddleware', 'patch'] + from .middleware import TraceMiddleware + from .patch import patch, unpatch + + __all__ = ["patch", "unpatch", "TraceMiddleware"] # define the Django app configuration -default_app_config = 'ddtrace.contrib.django.apps.TracerConfig' +default_app_config = "ddtrace.contrib.django.apps.TracerConfig" diff --git a/ddtrace/contrib/django/apps.py b/ddtrace/contrib/django/apps.py index 7cec2eededc..d26eb429cb9 100644 --- a/ddtrace/contrib/django/apps.py +++ b/ddtrace/contrib/django/apps.py @@ -1,19 +1,26 @@ -# 3rd party -from django.apps import AppConfig, apps +from django.apps import AppConfig -# project -from .patch import apply_django_patches +from ddtrace.vendor.debtcollector import removals +from ...internal.logger import get_logger +from .patch import patch +log = get_logger(__name__) + +@removals.removed_class( + "TracerConfig", + message="""Adding ddtrace.contrib.django to settings.py is no longer required, instead do: + import ddtrace + ddtrace.patch_all() + """, +) class TracerConfig(AppConfig): - name = 'ddtrace.contrib.django' - label = 'datadog_django' + name = "ddtrace.contrib.django" + label = "datadog_django" def ready(self): """ - Ready is called as soon as the registry is fully populated. - Tracing capabilities must be enabled in this function so that - all Django internals are properly configured. + Ready formerly was used to initiate patching of Django and DRF. """ - rest_framework_is_installed = apps.is_installed('rest_framework') - apply_django_patches(patch_rest_framework=rest_framework_is_installed) + # Make sure our app is configured + patch() diff --git a/ddtrace/contrib/django/cache.py b/ddtrace/contrib/django/cache.py deleted file mode 100644 index 3113a58b31d..00000000000 --- a/ddtrace/contrib/django/cache.py +++ /dev/null @@ -1,111 +0,0 @@ -from functools import wraps - -from django.conf import settings as django_settings - -from ...ext import SpanTypes -from ...internal.logger import get_logger -from .conf import settings, import_from_string -from .utils import quantize_key_values, _resource_from_cache_prefix - - -log = get_logger(__name__) - -# code instrumentation -DATADOG_NAMESPACE = '__datadog_original_{method}' -TRACED_METHODS = [ - 'get', - 'set', - 'add', - 'delete', - 'incr', - 'decr', - 'get_many', - 'set_many', - 'delete_many', -] - -# standard tags -CACHE_BACKEND = 'django.cache.backend' -CACHE_COMMAND_KEY = 'django.cache.key' - - -def patch_cache(tracer): - """ - Function that patches the inner cache system. Because the cache backend - can have different implementations and connectors, this function must - handle all possible interactions with the Django cache. What follows - is currently traced: - - * in-memory cache - * the cache client wrapper that could use any of the common - Django supported cache servers (Redis, Memcached, Database, Custom) - """ - # discover used cache backends - cache_backends = set([cache['BACKEND'] for cache in django_settings.CACHES.values()]) - - def _trace_operation(fn, method_name): - """ - Return a wrapped function that traces a cache operation - """ - cache_service_name = settings.DEFAULT_CACHE_SERVICE \ - if settings.DEFAULT_CACHE_SERVICE else settings.DEFAULT_SERVICE - - @wraps(fn) - def wrapped(self, *args, **kwargs): - # get the original function method - method = getattr(self, DATADOG_NAMESPACE.format(method=method_name)) - with tracer.trace('django.cache', span_type=SpanTypes.CACHE, service=cache_service_name) as span: - # update the resource name and tag the cache backend - span.resource = _resource_from_cache_prefix(method_name, self) - cache_backend = '{}.{}'.format(self.__module__, self.__class__.__name__) - span.set_tag(CACHE_BACKEND, cache_backend) - - if args: - keys = quantize_key_values(args[0]) - span.set_tag(CACHE_COMMAND_KEY, keys) - - return method(*args, **kwargs) - return wrapped - - def _wrap_method(cls, method_name): - """ - For the given class, wraps the method name with a traced operation - so that the original method is executed, while the span is properly - created - """ - # check if the backend owns the given bounded method - if not hasattr(cls, method_name): - return - - # prevent patching each backend's method more than once - if hasattr(cls, DATADOG_NAMESPACE.format(method=method_name)): - log.debug('%s already traced', method_name) - else: - method = getattr(cls, method_name) - setattr(cls, DATADOG_NAMESPACE.format(method=method_name), method) - setattr(cls, method_name, _trace_operation(method, method_name)) - - # trace all backends - for cache_module in cache_backends: - cache = import_from_string(cache_module, cache_module) - - for method in TRACED_METHODS: - _wrap_method(cache, method) - - -def unpatch_method(cls, method_name): - method = getattr(cls, DATADOG_NAMESPACE.format(method=method_name), None) - if method is None: - log.debug('nothing to do, the class is not patched') - return - setattr(cls, method_name, method) - delattr(cls, DATADOG_NAMESPACE.format(method=method_name)) - - -def unpatch_cache(): - cache_backends = set([cache['BACKEND'] for cache in django_settings.CACHES.values()]) - for cache_module in cache_backends: - cache = import_from_string(cache_module, cache_module) - - for method in TRACED_METHODS: - unpatch_method(cache, method) diff --git a/ddtrace/contrib/django/compat.py b/ddtrace/contrib/django/compat.py index f686b7f117f..bfe39ffabd3 100644 --- a/ddtrace/contrib/django/compat.py +++ b/ddtrace/contrib/django/compat.py @@ -1,6 +1,5 @@ import django - if django.VERSION >= (1, 10, 1): from django.urls import get_resolver @@ -8,6 +7,8 @@ def user_is_authenticated(user): # Explicit comparison due to the following bug # https://code.djangoproject.com/ticket/26988 return user.is_authenticated == True # noqa E712 + + else: from django.conf import settings from django.core import urlresolvers @@ -16,12 +17,15 @@ def user_is_authenticated(user): return user.is_authenticated() if django.VERSION >= (1, 9, 0): + def get_resolver(urlconf=None): urlconf = urlconf or settings.ROOT_URLCONF urlresolvers.set_urlconf(urlconf) return urlresolvers.get_resolver(urlconf) + else: + def get_resolver(urlconf=None): urlconf = urlconf or settings.ROOT_URLCONF urlresolvers.set_urlconf(urlconf) - return urlresolvers.RegexURLResolver(r'^/', urlconf) + return urlresolvers.RegexURLResolver(r"^/", urlconf) diff --git a/ddtrace/contrib/django/conf.py b/ddtrace/contrib/django/conf.py index 31dda245343..487e793d65b 100644 --- a/ddtrace/contrib/django/conf.py +++ b/ddtrace/contrib/django/conf.py @@ -24,28 +24,26 @@ # List of available settings with their defaults DEFAULTS = { - 'AGENT_HOSTNAME': 'localhost', - 'AGENT_PORT': 8126, - 'AUTO_INSTRUMENT': True, - 'INSTRUMENT_CACHE': True, - 'INSTRUMENT_DATABASE': True, - 'INSTRUMENT_TEMPLATE': True, - 'DEFAULT_DATABASE_PREFIX': '', - 'DEFAULT_SERVICE': 'django', - 'DEFAULT_CACHE_SERVICE': '', - 'ENABLED': True, - 'DISTRIBUTED_TRACING': True, - 'ANALYTICS_ENABLED': None, - 'ANALYTICS_SAMPLE_RATE': True, - 'TRACE_QUERY_STRING': None, - 'TAGS': {}, - 'TRACER': 'ddtrace.tracer', + "AGENT_HOSTNAME": "localhost", + "AGENT_PORT": 8126, + "AUTO_INSTRUMENT": True, + "INSTRUMENT_CACHE": True, + "INSTRUMENT_DATABASE": True, + "INSTRUMENT_TEMPLATE": True, + "DEFAULT_DATABASE_PREFIX": "", + "DEFAULT_SERVICE": "django", + "DEFAULT_CACHE_SERVICE": "", + "ENABLED": True, + "DISTRIBUTED_TRACING": True, + "ANALYTICS_ENABLED": None, + "ANALYTICS_SAMPLE_RATE": True, + "TRACE_QUERY_STRING": None, + "TAGS": {}, + "TRACER": "ddtrace.tracer", } # List of settings that may be in string import notation. -IMPORT_STRINGS = ( - 'TRACER', -) +IMPORT_STRINGS = ("TRACER",) # List of settings that have been removed REMOVED_SETTINGS = () @@ -57,17 +55,12 @@ def import_from_string(val, setting_name): """ try: # Nod to tastypie's use of importlib. - parts = val.split('.') - module_path, class_name = '.'.join(parts[:-1]), parts[-1] + parts = val.split(".") + module_path, class_name = ".".join(parts[:-1]), parts[-1] module = importlib.import_module(module_path) return getattr(module, class_name) except (ImportError, AttributeError) as e: - msg = 'Could not import "{}" for setting "{}". {}: {}.'.format( - val, - setting_name, - e.__class__.__name__, - e, - ) + msg = 'Could not import "{}" for setting "{}". {}: {}.'.format(val, setting_name, e.__class__.__name__, e,) raise ImportError(msg) @@ -84,41 +77,44 @@ class DatadogSettings(object): Any setting with string import paths will be automatically resolved and return the class, rather than the string literal. """ + def __init__(self, user_settings=None, defaults=None, import_strings=None): if user_settings: self._user_settings = self.__check_user_settings(user_settings) self.defaults = defaults or DEFAULTS - if os.environ.get('DATADOG_ENV'): - self.defaults['TAGS'].update({'env': os.environ.get('DATADOG_ENV')}) - if os.environ.get('DATADOG_SERVICE_NAME'): - self.defaults['DEFAULT_SERVICE'] = os.environ.get('DATADOG_SERVICE_NAME') + if os.environ.get("DATADOG_ENV"): + self.defaults["TAGS"].update({"env": os.environ.get("DATADOG_ENV")}) - host = os.environ.get('DD_AGENT_HOST', os.environ.get('DATADOG_TRACE_AGENT_HOSTNAME')) + for env_name in ("DATADOG_SERVICE_NAME", "DD_SERVICE_NAME"): + if os.environ.get(env_name): + self.defaults["DEFAULT_SERVICE"] = os.environ[env_name] + + host = os.environ.get("DD_AGENT_HOST", os.environ.get("DATADOG_TRACE_AGENT_HOSTNAME")) if host: - self.defaults['AGENT_HOSTNAME'] = host + self.defaults["AGENT_HOSTNAME"] = host - port = os.environ.get('DD_TRACE_AGENT_PORT', os.environ.get('DATADOG_TRACE_AGENT_PORT')) + port = os.environ.get("DD_TRACE_AGENT_PORT", os.environ.get("DATADOG_TRACE_AGENT_PORT")) if port: # if the agent port is a string, the underlying library that creates the socket # stops working try: port = int(port) except ValueError: - log.warning('DD_TRACE_AGENT_PORT is not an integer value; default to 8126') + log.warning("DD_TRACE_AGENT_PORT is not an integer value; default to 8126") else: - self.defaults['AGENT_PORT'] = port + self.defaults["AGENT_PORT"] = port self.import_strings = import_strings or IMPORT_STRINGS @property def user_settings(self): - if not hasattr(self, '_user_settings'): - self._user_settings = getattr(django_settings, 'DATADOG_TRACE', {}) + if not hasattr(self, "_user_settings"): + self._user_settings = getattr(django_settings, "DATADOG_TRACE", {}) # TODO[manu]: prevents docs import errors; provide a better implementation - if 'ENABLED' not in self._user_settings: - self._user_settings['ENABLED'] = not django_settings.DEBUG + if "ENABLED" not in self._user_settings: + self._user_settings["ENABLED"] = not django_settings.DEBUG return self._user_settings def __getattr__(self, attr): @@ -141,12 +137,10 @@ def __getattr__(self, attr): return val def __check_user_settings(self, user_settings): - SETTINGS_DOC = 'http://pypi.datadoghq.com/trace/docs/#module-ddtrace.contrib.django' + SETTINGS_DOC = "http://pypi.datadoghq.com/trace/docs/#module-ddtrace.contrib.django" for setting in REMOVED_SETTINGS: if setting in user_settings: - raise RuntimeError( - 'The "{}" setting has been removed, check "{}".'.format(setting, SETTINGS_DOC) - ) + raise RuntimeError('The "{}" setting has been removed, check "{}".'.format(setting, SETTINGS_DOC)) return user_settings @@ -158,6 +152,44 @@ def reload_settings(*args, **kwargs): Triggers a reload when Django emits the reloading signal """ global settings - setting, value = kwargs['setting'], kwargs['value'] - if setting == 'DATADOG_TRACE': + setting, value = kwargs["setting"], kwargs["value"] + if setting == "DATADOG_TRACE": settings = DatadogSettings(value, DEFAULTS, IMPORT_STRINGS) + + +# Migration from installing and configuring ddtrace as a Django middleware app + +_MAPPING_SETTINGS_CONFIG = dict( + DEFAULT_SERVICE="service_name", + DEFAULT_CACHE_SERVICE="cache_service_name", + DEFAULT_DATABASE_PREFIX="database_service_name_prefix", + DISTRIBUTED_TRACING="distributed_tracing_enabled", + INSTRUMENT_DATABASES="instrument_databases", + INSTRUMENT_CACHE="instrument_caches", + ANALYTICS_ENABLED="analytics_enabled", + ANALYTICS_SAMPLE_RATE="analytics_sample_rate", + TRACE_QUERY_STRING="trace_query_string", +) + + +def configure_from_settings(pin, config, settings): + # set config options from settings using mapping + for (setting_key, config_key) in _MAPPING_SETTINGS_CONFIG.items(): + if setting_key in settings: + setattr(config, config_key, settings[setting_key]) + + # re-configure tracer + if "TRACE" in settings: + pin.tracer = settings["TRACER"] + + if "TAGS" in settings: + pin.tracer.set_tags(settings["TAGS"]) + + if "ENABLED" in settings: + pin.tracer.enabled = settings["ENABLED"] + + if "AGENT_HOSTNAME" in settings: + pin.tracer.writer.api.hostname = settings["AGENT_HOSTNAME"] + + if "AGENT_PORT" in settings: + pin.tracer.writer.api.port = settings["AGENT_PORT"] diff --git a/ddtrace/contrib/django/db.py b/ddtrace/contrib/django/db.py deleted file mode 100644 index f5b6c804e18..00000000000 --- a/ddtrace/contrib/django/db.py +++ /dev/null @@ -1,76 +0,0 @@ -from django.db import connections - -# project -from ...ext import sql as sqlx -from ...internal.logger import get_logger -from ...pin import Pin - -from .conf import settings -from ..dbapi import TracedCursor as DbApiTracedCursor - -log = get_logger(__name__) - -CURSOR_ATTR = '_datadog_original_cursor' -ALL_CONNS_ATTR = '_datadog_original_connections_all' - - -def patch_db(tracer): - if hasattr(connections, ALL_CONNS_ATTR): - log.debug('db already patched') - return - setattr(connections, ALL_CONNS_ATTR, connections.all) - - def all_connections(self): - conns = getattr(self, ALL_CONNS_ATTR)() - for conn in conns: - patch_conn(tracer, conn) - return conns - - connections.all = all_connections.__get__(connections, type(connections)) - - -def unpatch_db(): - for c in connections.all(): - unpatch_conn(c) - - all_connections = getattr(connections, ALL_CONNS_ATTR, None) - if all_connections is None: - log.debug('nothing to do, the db is not patched') - return - connections.all = all_connections - delattr(connections, ALL_CONNS_ATTR) - - -def patch_conn(tracer, conn): - if hasattr(conn, CURSOR_ATTR): - return - - setattr(conn, CURSOR_ATTR, conn.cursor) - - def cursor(): - database_prefix = ( - '{}-'.format(settings.DEFAULT_DATABASE_PREFIX) - if settings.DEFAULT_DATABASE_PREFIX else '' - ) - alias = getattr(conn, 'alias', 'default') - service = '{}{}{}'.format(database_prefix, alias, 'db') - vendor = getattr(conn, 'vendor', 'db') - prefix = sqlx.normalize_vendor(vendor) - tags = { - 'django.db.vendor': vendor, - 'django.db.alias': alias, - } - - pin = Pin(service, tags=tags, tracer=tracer, app=prefix) - return DbApiTracedCursor(conn._datadog_original_cursor(), pin) - - conn.cursor = cursor - - -def unpatch_conn(conn): - cursor = getattr(conn, CURSOR_ATTR, None) - if cursor is None: - log.debug('nothing to do, the connection is not patched') - return - conn.cursor = cursor - delattr(conn, CURSOR_ATTR) diff --git a/ddtrace/contrib/django/middleware.py b/ddtrace/contrib/django/middleware.py index 8a0539a75bb..5fc01a60f3d 100644 --- a/ddtrace/contrib/django/middleware.py +++ b/ddtrace/contrib/django/middleware.py @@ -1,19 +1,4 @@ -# project -from .conf import settings -from .compat import user_is_authenticated, get_resolver -from .utils import get_request_uri - -from ...constants import ANALYTICS_SAMPLE_RATE_KEY -from ...contrib import func_name -from ...ext import SpanTypes, http -from ...internal.logger import get_logger -from ...propagation.http import HTTPPropagator -from ...settings import config - -# 3p -from django.core.exceptions import MiddlewareNotUsed -from django.conf import settings as django_settings -import django +from ...vendor import debtcollector try: from django.utils.deprecation import MiddlewareMixin @@ -22,209 +7,9 @@ except ImportError: MiddlewareClass = object -log = get_logger(__name__) - -EXCEPTION_MIDDLEWARE = "ddtrace.contrib.django.TraceExceptionMiddleware" -TRACE_MIDDLEWARE = "ddtrace.contrib.django.TraceMiddleware" -MIDDLEWARE = "MIDDLEWARE" -MIDDLEWARE_CLASSES = "MIDDLEWARE_CLASSES" - -# Default views list available from: -# https://github.com/django/django/blob/38e2fdadfd9952e751deed662edf4c496d238f28/django/views/defaults.py -# DEV: Django doesn't call `process_view` when falling back to one of these internal error handling views -# DEV: We only use these names when `span.resource == 'unknown'` and we have one of these status codes -_django_default_views = { - 400: "django.views.defaults.bad_request", - 403: "django.views.defaults.permission_denied", - 404: "django.views.defaults.page_not_found", - 500: "django.views.defaults.server_error", -} - - -def _analytics_enabled(): - return ( - (config.analytics_enabled and settings.ANALYTICS_ENABLED is not False) or settings.ANALYTICS_ENABLED is True - ) and settings.ANALYTICS_SAMPLE_RATE is not None - - -def get_middleware_insertion_point(): - """Returns the attribute name and collection object for the Django middleware. - - If middleware cannot be found, returns None for the middleware collection. - """ - middleware = getattr(django_settings, MIDDLEWARE, None) - # Prioritise MIDDLEWARE over ..._CLASSES, but only in 1.10 and later. - if middleware is not None and django.VERSION >= (1, 10): - return MIDDLEWARE, middleware - return MIDDLEWARE_CLASSES, getattr(django_settings, MIDDLEWARE_CLASSES, None) - - -def insert_trace_middleware(): - middleware_attribute, middleware = get_middleware_insertion_point() - if middleware is not None and TRACE_MIDDLEWARE not in set(middleware): - setattr(django_settings, middleware_attribute, type(middleware)((TRACE_MIDDLEWARE,)) + middleware) - - -def remove_trace_middleware(): - _, middleware = get_middleware_insertion_point() - if middleware and TRACE_MIDDLEWARE in set(middleware): - middleware.remove(TRACE_MIDDLEWARE) - - -def insert_exception_middleware(): - middleware_attribute, middleware = get_middleware_insertion_point() - if middleware is not None and EXCEPTION_MIDDLEWARE not in set(middleware): - setattr(django_settings, middleware_attribute, middleware + type(middleware)((EXCEPTION_MIDDLEWARE,))) - - -def remove_exception_middleware(): - _, middleware = get_middleware_insertion_point() - if middleware and EXCEPTION_MIDDLEWARE in set(middleware): - middleware.remove(EXCEPTION_MIDDLEWARE) - - -class InstrumentationMixin(MiddlewareClass): - """ - Useful mixin base class for tracing middlewares - """ - - def __init__(self, get_response=None): - # disable the middleware if the tracer is not enabled - # or if the auto instrumentation is disabled - self.get_response = get_response - if not settings.AUTO_INSTRUMENT: - raise MiddlewareNotUsed - - -class TraceExceptionMiddleware(InstrumentationMixin): - """ - Middleware that traces exceptions raised - """ - - def process_exception(self, request, exception): - try: - span = _get_req_span(request) - if span: - span.set_tag(http.STATUS_CODE, "500") - span.set_traceback() # will set the exception info - except Exception: - log.debug("error processing exception", exc_info=True) - - -class TraceMiddleware(InstrumentationMixin): - """ - Middleware that traces Django requests - """ - - def process_request(self, request): - tracer = settings.TRACER - if settings.DISTRIBUTED_TRACING: - propagator = HTTPPropagator() - context = propagator.extract(request.META) - # Only need to active the new context if something was propagated - if context.trace_id: - tracer.context_provider.activate(context) - try: - span = tracer.trace( - "django.request", - service=settings.DEFAULT_SERVICE, - resource="unknown", # will be filled by process view - span_type=SpanTypes.WEB, - ) - - # set analytics sample rate - # DEV: django is special case maintains separate configuration from config api - if _analytics_enabled() and settings.ANALYTICS_SAMPLE_RATE is not None: - span.set_tag( - ANALYTICS_SAMPLE_RATE_KEY, settings.ANALYTICS_SAMPLE_RATE, - ) - - # Set HTTP Request tags - span.set_tag(http.METHOD, request.method) - span.set_tag(http.URL, get_request_uri(request)) - trace_query_string = settings.TRACE_QUERY_STRING - if trace_query_string is None: - trace_query_string = config.django.trace_query_string - if trace_query_string: - span.set_tag(http.QUERY_STRING, request.META["QUERY_STRING"]) - _set_req_span(request, span) - except Exception: - log.debug("error tracing request", exc_info=True) - - def process_view(self, request, view_func, *args, **kwargs): - span = _get_req_span(request) - if span: - span.resource = func_name(view_func) - - def process_response(self, request, response): - try: - span = _get_req_span(request) - if span: - if response.status_code < 500 and span.error: - # remove any existing stack trace since it must have been - # handled appropriately - span._remove_exc_info() - - # If `process_view` was not called, try to determine the correct `span.resource` to set - # DEV: `process_view` won't get called if a middle `process_request` returns an HttpResponse - # DEV: `process_view` won't get called when internal error handlers are used (e.g. for 404 responses) - if span.resource == "unknown": - try: - # Attempt to lookup the view function from the url resolver - # https://github.com/django/django/blob/38e2fdadfd9952e751deed662edf4c496d238f28/django/core/handlers/base.py#L104-L113 # noqa - urlconf = None - if hasattr(request, "urlconf"): - urlconf = request.urlconf - resolver = get_resolver(urlconf) - - # Try to resolve the Django view for handling this request - if getattr(request, "request_match", None): - request_match = request.request_match - else: - # This may raise a `django.urls.exceptions.Resolver404` exception - request_match = resolver.resolve(request.path_info) - span.resource = func_name(request_match.func) - except Exception: - log.debug("error determining request view function", exc_info=True) - - # If the view could not be found, try to set from a static list of - # known internal error handler views - span.resource = _django_default_views.get(response.status_code, "unknown") - - span.set_tag(http.STATUS_CODE, response.status_code) - span = _set_auth_tags(span, request) - span.finish() - except Exception: - log.debug("error tracing request", exc_info=True) - finally: - return response - - -def _get_req_span(request): - """ Return the datadog span from the given request. """ - return getattr(request, "_datadog_request_span", None) - - -def _set_req_span(request, span): - """ Set the datadog span on the given request. """ - return setattr(request, "_datadog_request_span", span) - - -def _set_auth_tags(span, request): - """ Patch any available auth tags from the request onto the span. """ - user = getattr(request, "user", None) - if not user: - return span - - if hasattr(user, "is_authenticated"): - span.set_tag("django.user.is_authenticated", user_is_authenticated(user)) - - uid = getattr(user, "pk", None) - if uid: - span.set_tag("django.user.id", uid) - - uname = getattr(user, "username", None) - if uname: - span.set_tag("django.user.name", uname) - return span +@debtcollector.removals.removed_class( + "TraceMiddlware", message="Usage of TraceMiddleware is not longer needed, please remove from your settings.py" +) +class TraceMiddleware(MiddlewareClass): + pass diff --git a/ddtrace/contrib/django/patch.py b/ddtrace/contrib/django/patch.py index 9dd6a54d32e..4e5534642c7 100644 --- a/ddtrace/contrib/django/patch.py +++ b/ddtrace/contrib/django/patch.py @@ -1,94 +1,584 @@ -# 3rd party -from ddtrace.vendor import wrapt -import django -from django.db import connections +""" +The Django patching works as follows: -# project -from .db import patch_db -from .conf import settings -from .cache import patch_cache -from .templates import patch_template -from .middleware import insert_exception_middleware, insert_trace_middleware +Django internals are instrumented via normal `patch()`. -from ...internal.logger import get_logger +`django.apps.registry.Apps.populate` is patched to add instrumentation for any +specific Django apps like Django Rest Framework (DRF). +""" +import os +import sys -log = get_logger(__name__) +from inspect import isclass, isfunction +from ddtrace import config, Pin +from ddtrace.vendor import debtcollector, wrapt +from ddtrace.compat import parse +from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY +from ddtrace.contrib import func_name, dbapi +from ddtrace.ext import http, sql as sqlx, SpanTypes +from ddtrace.internal.logger import get_logger +from ddtrace.propagation.http import HTTPPropagator +from ddtrace.utils.formats import get_env +from ddtrace.utils.wrappers import unwrap, iswrapped -def patch(): - """Patch the instrumented methods - """ - if getattr(django, '_datadog_patch', False): - return - setattr(django, '_datadog_patch', True) +from .compat import get_resolver, user_is_authenticated +from . import utils, conf - _w = wrapt.wrap_function_wrapper - _w('django', 'setup', traced_setup) +wrap = wrapt.wrap_function_wrapper +log = get_logger(__name__) -def traced_setup(wrapped, instance, args, kwargs): - from django.conf import settings +config._add( + "django", + dict( + service_name=os.environ.get("DD_SERVICE_NAME") or os.environ.get("DATADOG_SERVICE_NAME") or "django", + cache_service_name=get_env("django", "cache_service_name") or "django", + database_service_name_prefix=get_env("django", "database_service_name_prefix", default=""), + distributed_tracing_enabled=True, + instrument_databases=True, + instrument_caches=True, + analytics_enabled=None, # None allows the value to be overridden by the global config + analytics_sample_rate=None, + trace_query_string=None, # Default to global config + ), +) - if 'ddtrace.contrib.django' not in settings.INSTALLED_APPS: - if isinstance(settings.INSTALLED_APPS, tuple): - # INSTALLED_APPS is a tuple < 1.9 - settings.INSTALLED_APPS = settings.INSTALLED_APPS + ('ddtrace.contrib.django', ) - else: - settings.INSTALLED_APPS.append('ddtrace.contrib.django') +propagator = HTTPPropagator() - wrapped(*args, **kwargs) +def with_traced_module(func): + """Helper for providing tracing essentials (module and pin) for tracing + wrappers. -def apply_django_patches(patch_rest_framework): - """ - Ready is called as soon as the registry is fully populated. - In order for all Django internals are properly configured, this - must be called after the app is finished starting + This helper enables tracing wrappers to dynamically be disabled when the + corresponding pin is disabled. + + Usage:: + + @with_traced_module + def my_traced_wrapper(django, pin, func, instance, args, kwargs): + # Do tracing stuff + pass + + def patch(): + import django + wrap(django.somefunc, my_traced_wrapper(django)) """ - tracer = settings.TRACER - if settings.TAGS: - tracer.set_tags(settings.TAGS) + def with_mod(mod): + def wrapper(wrapped, instance, args, kwargs): + pin = Pin._find(instance, mod) + if pin and not pin.enabled(): + return wrapped(*args, **kwargs) + elif not pin: + log.debug("Pin not found for traced method %r", wrapped) + return wrapped(*args, **kwargs) + return func(mod, pin, wrapped, instance, args, kwargs) - # configure the tracer instance - # TODO[manu]: we may use configure() but because it creates a new - # AgentWriter, it breaks all tests. The configure() behavior must - # be changed to use it in this integration - tracer.enabled = settings.ENABLED - tracer.writer.api.hostname = settings.AGENT_HOSTNAME - tracer.writer.api.port = settings.AGENT_PORT + return wrapper - if settings.AUTO_INSTRUMENT: - # trace Django internals - insert_trace_middleware() - insert_exception_middleware() + return with_mod - if settings.INSTRUMENT_TEMPLATE: - try: - patch_template(tracer) - except Exception: - log.exception('error patching Django template rendering') - if settings.INSTRUMENT_DATABASE: +def patch_conn(django, conn): + def cursor(django, pin, func, instance, args, kwargs): + database_prefix = config.django.database_service_name_prefix + alias = getattr(conn, "alias", "default") + service = "{}{}{}".format(database_prefix, alias, "db") + vendor = getattr(conn, "vendor", "db") + prefix = sqlx.normalize_vendor(vendor) + tags = { + "django.db.vendor": vendor, + "django.db.alias": alias, + } + pin = Pin(service, tags=tags, tracer=pin.tracer, app=prefix) + return dbapi.TracedCursor(func(*args, **kwargs), pin) + + if not isinstance(conn.cursor, wrapt.ObjectProxy): + conn.cursor = wrapt.FunctionWrapper(conn.cursor, with_traced_module(cursor)(django)) + + +def instrument_dbs(django): + def all_connections(wrapped, instance, args, kwargs): + conns = wrapped(*args, **kwargs) + for conn in conns: try: - patch_db(tracer) - # This is the trigger to patch individual connections. - # By patching these here, all processes including - # management commands are also traced. - connections.all() + patch_conn(django, conn) except Exception: - log.exception('error patching Django database connections') + log.debug("Error instrumenting database connection %r", conn, exc_info=True) + return conns + + if not isinstance(django.db.connections.all, wrapt.ObjectProxy): + django.db.connections.all = wrapt.FunctionWrapper(django.db.connections.all, all_connections) - if settings.INSTRUMENT_CACHE: + if hasattr(django.db, "connection") and not isinstance(django.db.connection.cursor, wrapt.ObjectProxy): + patch_conn(django, django.db.connection) + + +def _set_request_tags(span, request): + span.set_tag("django.request.class", func_name(request)) + span.set_tag(http.METHOD, request.method) + + user = getattr(request, "user", None) + if user is not None: + if hasattr(user, "is_authenticated"): + span.set_tag("django.user.is_authenticated", user_is_authenticated(user)) + + uid = getattr(user, "pk", None) + if uid: + span.set_tag("django.user.id", uid) + + username = getattr(user, "username", None) + if username: + span.set_tag("django.user.name", username) + + +@with_traced_module +def traced_cache(django, pin, func, instance, args, kwargs): + if not config.django.instrument_caches: + return func(*args, **kwargs) + + # get the original function method + with pin.tracer.trace("django.cache", span_type=SpanTypes.CACHE, service=config.django.cache_service_name) as span: + # update the resource name and tag the cache backend + span.resource = utils.resource_from_cache_prefix(func_name(func), instance) + cache_backend = "{}.{}".format(instance.__module__, instance.__class__.__name__) + span.set_tag("django.cache.backend", cache_backend) + + if args: + keys = utils.quantize_key_values(args[0]) + span.set_tag("django.cache.key", keys) + + return func(*args, **kwargs) + + +def instrument_caches(django): + cache_backends = set([cache["BACKEND"] for cache in django.conf.settings.CACHES.values()]) + for cache_path in cache_backends: + split = cache_path.split(".") + cache_module = ".".join(split[:-1]) + cache_cls = split[-1] + for method in ["get", "set", "add", "delete", "incr", "decr", "get_many", "set_many", "delete_many"]: try: - patch_cache(tracer) + cls = django.utils.module_loading.import_string(cache_path) + # DEV: this can be removed when we add an idempotent `wrap` + if not iswrapped(cls, method): + wrap(cache_module, "{0}.{1}".format(cache_cls, method), traced_cache(django)) except Exception: - log.exception('error patching Django cache') + log.debug("Error instrumenting cache %r", cache_path, exc_info=True) + + +@with_traced_module +def traced_populate(django, pin, func, instance, args, kwargs): + """django.apps.registry.Apps.populate is the method used to populate all the apps. + + It is used as a hook to install instrumentation for 3rd party apps (like DRF). + + `populate()` works in 3 phases: + + - Phase 1: Initializes the app configs and imports the app modules. + - Phase 2: Imports models modules for each app. + - Phase 3: runs ready() of each app config. + + If all 3 phases successfully run then `instance.ready` will be `True`. + """ + + # populate() can be called multiple times, we don't want to instrument more than once + if instance.ready: + log.debug("Django instrumentation already installed, skipping.") + return func(*args, **kwargs) + + ret = func(*args, **kwargs) + + if not instance.ready: + log.debug("populate() failed skipping instrumentation.") + return ret + + settings = django.conf.settings + + if hasattr(settings, "DATADOG_TRACE"): + debtcollector.deprecate(("Using DATADOG_TRACE Django settings are no longer supported. ")) + conf.configure_from_settings(pin, config.django, settings.DATADOG_TRACE) + + # Instrument databases + if config.django.instrument_databases: + try: + instrument_dbs(django) + except Exception: + log.debug("Error instrumenting Django database connections", exc_info=True) + + # Instrument caches + if config.django.instrument_caches: + try: + instrument_caches(django) + except Exception: + log.debug("Error instrumenting Django caches", exc_info=True) + + # Instrument Django Rest Framework if it's installed + INSTALLED_APPS = getattr(settings, "INSTALLED_APPS", []) + + if "rest_framework" in INSTALLED_APPS: + try: + from .restframework import patch_restframework + + patch_restframework(django) + except Exception: + log.debug("Error patching rest_framework", exc_info=True) + + return ret + + +def traced_func(django, name, resource=None): + """Returns a function to trace Django functions.""" + + def wrapped(django, pin, func, instance, args, kwargs): + with pin.tracer.trace(name, resource=resource): + return func(*args, **kwargs) + + return with_traced_module(wrapped)(django) + + +def traced_process_exception(django, name, resource=None): + def wrapped(django, pin, func, instance, args, kwargs): + with pin.tracer.trace(name, resource=resource) as span: + resp = func(*args, **kwargs) + + # If the response code is erroneous then grab the traceback + # and set an error. + if hasattr(resp, "status_code") and 500 <= resp.status_code < 600: + span.set_traceback() + return resp + + return with_traced_module(wrapped)(django) + + +@with_traced_module +def traced_load_middleware(django, pin, func, instance, args, kwargs): + """Patches django.core.handlers.base.BaseHandler.load_middleware to instrument all middlewares.""" + settings_middleware = [] + # Gather all the middleware + if getattr(django.conf.settings, "MIDDLEWARE", None): + settings_middleware += django.conf.settings.MIDDLEWARE + if getattr(django.conf.settings, "MIDDLEWARE_CLASSES", None): + settings_middleware += django.conf.settings.MIDDLEWARE_CLASSES + + # Iterate over each middleware provided in settings.py + # Each middleware can either be a function or a class + for mw_path in settings_middleware: + mw = django.utils.module_loading.import_string(mw_path) + + # Instrument function-based middleware + if isfunction(mw) and not iswrapped(mw): + split = mw_path.split(".") + if len(split) < 2: + continue + base = ".".join(split[:-1]) + attr = split[-1] + + # Function-based middleware is a factory which returns a handler function for requests. + # So instead of tracing the factory, we want to trace its returned value. + # We wrap the factory to return a traced version of the handler function. + def wrapped_factory(func, instance, args, kwargs): + # r is the middleware handler function returned from the factory + r = func(*args, **kwargs) + return wrapt.FunctionWrapper(r, traced_func(django, "django.middleware", resource=mw_path)) + + wrap(base, attr, wrapped_factory) + + # Instrument class-based middleware + elif isclass(mw): + for hook in [ + "process_request", + "process_response", + "process_view", + "process_template_response", + "__call__", + ]: + if hasattr(mw, hook) and not iswrapped(mw, hook): + wrap(mw, hook, traced_func(django, "django.middleware", resource=mw_path + ".{0}".format(hook))) + # Do a little extra for `process_exception` + if hasattr(mw, "process_exception") and not iswrapped(mw, "process_exception"): + res = mw_path + ".{0}".format("process_exception") + wrap(mw, "process_exception", traced_process_exception(django, "django.middleware", resource=res)) - # Instrument rest_framework app to trace custom exception handling. - if patch_rest_framework: + return func(*args, **kwargs) + + +@with_traced_module +def traced_get_response(django, pin, func, instance, args, kwargs): + """Trace django.core.handlers.base.BaseHandler.get_response() (or other implementations). + + This is the main entry point for requests. + + Django requests are handled by a Handler.get_response method (inherited from base.BaseHandler). + This method invokes the middleware chain and returns the response generated by the chain. + """ + + request = kwargs.get("request", args[0]) + if request is None: + return func(*args, **kwargs) + + try: + request_headers = request.META + + if config.django.distributed_tracing_enabled: + context = propagator.extract(request_headers) + if context.trace_id: + pin.tracer.context_provider.activate(context) + + # Determine the resolver and resource name for this request + resolver = get_resolver(getattr(request, "urlconf", None)) + + if django.VERSION < (1, 10, 0): + error_type_404 = django.core.urlresolvers.Resolver404 + else: + error_type_404 = django.urls.exceptions.Resolver404 + + route = None + resolver_match = None + resource = request.method + try: + # Resolve the requested url + resolver_match = resolver.resolve(request.path_info) + + # Determine the resource name to use + # In Django >= 2.2.0 we can access the original route or regex pattern + if django.VERSION >= (2, 2, 0): + route = utils.get_django_2_route(resolver, resolver_match) + if route: + resource = "{0} {1}".format(request.method, route) + else: + resource = request.method + # Older versions just use the view/handler name, e.g. `views.MyView.handler` + else: + # TODO: Validate if `resolver.pattern.regex.pattern` is available or not + callback, callback_args, callback_kwargs = resolver_match + resource = "{0} {1}".format(request.method, func_name(callback)) + + except error_type_404: + # Normalize all 404 requests into a single resource name + # DEV: This is for potential cardinality issues + resource = "{0} 404".format(request.method) + except Exception: + log.debug( + "Failed to resolve request path %r with path info %r", + request, + getattr(request, "path_info", "not-set"), + exc_info=True, + ) + except Exception: + log.debug("Failed to trace django request %r", args, exc_info=True) + return func(*args, **kwargs) + else: + with pin.tracer.trace( + "django.request", resource=resource, service=config.django["service_name"], span_type=SpanTypes.HTTP + ) as span: + analytics_sr = config.django.get_analytics_sample_rate(use_global_config=True) + if analytics_sr is not None: + span.set_tag(ANALYTICS_SAMPLE_RATE_KEY, analytics_sr) + + if config.django.http.trace_query_string: + span.set_tag(http.QUERY_STRING, request_headers["QUERY_STRING"]) + + # Not a 404 request + if resolver_match: + span.set_tag("django.view", resolver_match.view_name) + utils.set_tag_array(span, "django.namespace", resolver_match.namespaces) + + # Django >= 2.0.0 + if hasattr(resolver_match, "app_names"): + utils.set_tag_array(span, "django.app", resolver_match.app_names) + + if route: + span.set_tag("http.route", route) + + # Set HTTP Request tags + # Build `http.url` tag value from request info + # DEV: We are explicitly omitting query strings since they may contain sensitive information + span.set_tag( + http.URL, + parse.urlunparse( + parse.ParseResult( + scheme=request.scheme, + netloc=request.get_host(), # this will include `host:port` + path=request.path, + params="", + query="", + fragment="", + ) + ), + ) + + response = func(*args, **kwargs) + + # Note: this call must be done after the function call because + # some attributes (like `user`) are added to the request through + # the middleware chain + _set_request_tags(span, request) + + if response: + span.set_tag(http.STATUS_CODE, response.status_code) + if 500 <= response.status_code < 600: + span.error = 1 + span.set_tag("django.response.class", func_name(response)) + if hasattr(response, "template_name"): + utils.set_tag_array(span, "django.response.template", response.template_name) + return response + + +@with_traced_module +def traced_template_render(django, pin, wrapped, instance, args, kwargs): + """Instrument django.template.base.Template.render for tracing template rendering.""" + template_name = getattr(instance, "name", None) + if template_name: + resource = template_name + else: + resource = "{0}.{1}".format(func_name(instance), wrapped.__name__) + + with pin.tracer.trace("django.template.render", resource=resource, span_type=http.TEMPLATE) as span: + if template_name: + span.set_tag("django.template.name", template_name) + engine = getattr(instance, "engine", None) + if engine: + span.set_tag("django.template.engine.class", func_name(engine)) + + return wrapped(*args, **kwargs) + + +def instrument_view(django, view): + """Helper to wrap Django views.""" + + # All views should be callable, double check before doing anything + if not callable(view) or isinstance(view, wrapt.ObjectProxy): + return view + + # Patch view HTTP methods and lifecycle methods + http_method_names = getattr(view, "http_method_names", ("get", "delete", "post", "options", "head")) + lifecycle_methods = ("setup", "dispatch", "http_method_not_allowed") + for name in list(http_method_names) + list(lifecycle_methods): + try: + func = getattr(view, name, None) + if not func or isinstance(func, wrapt.ObjectProxy): + continue + + resource = "{0}.{1}".format(func_name(view), name) + op_name = "django.view.{0}".format(name) + wrap(view, name, traced_func(django, name=op_name, resource=resource)) + except Exception: + log.debug("Failed to instrument Django view %r function %s", view, name, exc_info=True) + + # Patch response methods + response_cls = getattr(view, "response_class", None) + if response_cls: + methods = ("render",) + for name in methods: try: - from .restframework import patch_restframework - patch_restframework(tracer) + func = getattr(response_cls, name, None) + # Do not wrap if the method does not exist or is already wrapped + if not func or isinstance(func, wrapt.ObjectProxy): + continue + + resource = "{0}.{1}".format(func_name(response_cls), name) + op_name = "django.response.{0}".format(name) + wrap(response_cls, name, traced_func(django, name=op_name, resource=resource)) except Exception: - log.exception('error patching rest_framework app') + log.debug("Failed to instrument Django response %r function %s", response_cls, name, exc_info=True) + + # Return a wrapped version of this view + return wrapt.FunctionWrapper(view, traced_func(django, "django.view", resource=func_name(view))) + + +@with_traced_module +def traced_urls_path(django, pin, wrapped, instance, args, kwargs): + """Wrapper for url path helpers to ensure all views registered as urls are traced.""" + try: + if "view" in kwargs: + kwargs["view"] = instrument_view(django, kwargs["view"]) + elif len(args) >= 2: + args = list(args) + args[1] = instrument_view(django, args[1]) + args = tuple(args) + except Exception: + log.debug("Failed to instrument Django url path %r %r", args, kwargs, exc_info=True) + return wrapped(*args, **kwargs) + + +@with_traced_module +def traced_as_view(django, pin, func, instance, args, kwargs): + """ + Wrapper for django's View.as_view class method + """ + try: + instrument_view(django, instance) + except Exception: + log.debug("Failed to instrument Django view %r", instance, exc_info=True) + view = func(*args, **kwargs) + return wrapt.FunctionWrapper(view, traced_func(django, "django.view", resource=func_name(view))) + + +def _patch(django): + Pin(service=config.django["service_name"]).onto(django) + wrap(django, "apps.registry.Apps.populate", traced_populate(django)) + + # DEV: this check will be replaced with import hooks in the future + if "django.core.handlers.base" not in sys.modules: + import django.core.handlers.base + wrap(django, "core.handlers.base.BaseHandler.load_middleware", traced_load_middleware(django)) + wrap(django, "core.handlers.base.BaseHandler.get_response", traced_get_response(django)) + + # DEV: this check will be replaced with import hooks in the future + if "django.template.base" not in sys.modules: + import django.template.base + wrap(django, "template.base.Template.render", traced_template_render(django)) + + # DEV: this check will be replaced with import hooks in the future + if "django.conf.urls.static" not in sys.modules: + import django.conf.urls.static + wrap(django, "conf.urls.url", traced_urls_path(django)) + if django.VERSION >= (2, 0, 0): + wrap(django, "urls.path", traced_urls_path(django)) + wrap(django, "urls.re_path", traced_urls_path(django)) + + # DEV: this check will be replaced with import hooks in the future + if "django.views.generic.base" not in sys.modules: + import django.views.generic.base + wrap(django, "views.generic.base.View.as_view", traced_as_view(django)) + + +def patch(): + # DEV: this import will eventually be replaced with the module given from an import hook + import django + + if getattr(django, "_datadog_patch", False): + return + _patch(django) + + setattr(django, "_datadog_patch", True) + + +def _unpatch(django): + unwrap(django.apps.registry.Apps, "populate") + unwrap(django.core.handlers.base.BaseHandler, "load_middleware") + unwrap(django.core.handlers.base.BaseHandler, "get_response") + unwrap(django.template.base.Template, "render") + unwrap(django.conf.urls.static, "static") + unwrap(django.conf.urls, "url") + if django.VERSION >= (2, 0, 0): + unwrap(django.urls, "path") + unwrap(django.urls, "re_path") + unwrap(django.views.generic.base.View, "as_view") + for conn in django.db.connections.all(): + unwrap(conn, "cursor") + unwrap(django.db.connections, "all") + + +def unpatch(): + import django + + if not getattr(django, "_datadog_patch", False): + return + + _unpatch(django) + + setattr(django, "_datadog_patch", False) diff --git a/ddtrace/contrib/django/restframework.py b/ddtrace/contrib/django/restframework.py index 1970111e0d1..f864c0343a1 100644 --- a/ddtrace/contrib/django/restframework.py +++ b/ddtrace/contrib/django/restframework.py @@ -1,42 +1,33 @@ from ddtrace.vendor.wrapt import wrap_function_wrapper as wrap -from rest_framework.views import APIView +import rest_framework.views -from ...utils.wrappers import unwrap +from ...utils.wrappers import iswrapped +from .patch import with_traced_module -def patch_restframework(tracer): - """ Patches rest_framework app. - - To trace exceptions occuring during view processing we currently use a TraceExceptionMiddleware. - However the rest_framework handles exceptions before they come to our middleware. - So we need to manually patch the rest_framework exception handler - to set the exception stack trace in the current span. - +@with_traced_module +def _traced_handle_exception(django, pin, wrapped, instance, args, kwargs): + """Sets the error message, error type and exception stack trace to the current span + before calling the original exception handler. """ + span = pin.tracer.current_span() - def _traced_handle_exception(wrapped, instance, args, kwargs): - """ Sets the error message, error type and exception stack trace to the current span - before calling the original exception handler. - """ - span = tracer.current_span() - if span is not None: - span.set_traceback() + if span is not None: + span.set_traceback() - return wrapped(*args, **kwargs) + return wrapped(*args, **kwargs) - # do not patch if already patched - if getattr(APIView, '_datadog_patch', False): - return - else: - setattr(APIView, '_datadog_patch', True) - # trace the handle_exception method - wrap('rest_framework.views', 'APIView.handle_exception', _traced_handle_exception) +def patch_restframework(django): + """Patches rest_framework app. + To trace exceptions occurring during view processing we currently use a TraceExceptionMiddleware. + However the rest_framework handles exceptions before they come to our middleware. + So we need to manually patch the rest_framework exception handler + to set the exception stack trace in the current span. + """ -def unpatch_restframework(): - """ Unpatches rest_framework app.""" - if getattr(APIView, '_datadog_patch', False): - setattr(APIView, '_datadog_patch', False) - unwrap(APIView, 'handle_exception') + # trace the handle_exception method + if not iswrapped(rest_framework.views.APIView, "handle_exception"): + wrap("rest_framework.views", "APIView.handle_exception", _traced_handle_exception(django)) diff --git a/ddtrace/contrib/django/templates.py b/ddtrace/contrib/django/templates.py deleted file mode 100644 index 27752b362f7..00000000000 --- a/ddtrace/contrib/django/templates.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -code to measure django template rendering. -""" -# project -from ...ext import SpanTypes -from ...internal.logger import get_logger - -# 3p -from django.template import Template - -log = get_logger(__name__) - -RENDER_ATTR = '_datadog_original_render' - - -def patch_template(tracer): - """ will patch django's template rendering function to include timing - and trace information. - """ - - # FIXME[matt] we're patching the template class here. ideally we'd only - # patch so we can use multiple tracers at once, but i suspect this is fine - # in practice. - if getattr(Template, RENDER_ATTR, None): - log.debug('already patched') - return - - setattr(Template, RENDER_ATTR, Template.render) - - def traced_render(self, context): - with tracer.trace('django.template', span_type=SpanTypes.TEMPLATE) as span: - try: - return Template._datadog_original_render(self, context) - finally: - template_name = self.name or getattr(context, 'template_name', None) or 'unknown' - span.resource = template_name - span.set_tag('django.template_name', template_name) - - Template.render = traced_render - - -def unpatch_template(): - render = getattr(Template, RENDER_ATTR, None) - if render is None: - log.debug('nothing to do Template is already patched') - return - Template.render = render - delattr(Template, RENDER_ATTR) diff --git a/ddtrace/contrib/django/utils.py b/ddtrace/contrib/django/utils.py index 3b4a1acee16..c6e271ecbb7 100644 --- a/ddtrace/contrib/django/utils.py +++ b/ddtrace/contrib/django/utils.py @@ -1,15 +1,15 @@ -from ...compat import parse from ...internal.logger import get_logger + log = get_logger(__name__) -def _resource_from_cache_prefix(resource, cache): +def resource_from_cache_prefix(resource, cache): """ Combine the resource name with the cache prefix (if any) """ - if getattr(cache, 'key_prefix', None): - name = '{} {}'.format(resource, cache.key_prefix) + if getattr(cache, "key_prefix", None): + name = "{} {}".format(resource, cache.key_prefix) else: name = resource @@ -23,7 +23,7 @@ def quantize_key_values(key): with values is used, we removes the values from the span meta attributes. For example:: - >>> quantize_key_values({'key', 'value'}) + >>> quantize_key_values({'key': 'value'}) # returns ['key'] """ if isinstance(key, dict): @@ -32,44 +32,28 @@ def quantize_key_values(key): return key -def get_request_uri(request): - """ - Helper to rebuild the original request url +def get_django_2_route(resolver, resolver_match): + # Try to use `resolver_match.route` if available + # Otherwise, look for `resolver.pattern.regex.pattern` + route = resolver_match.route + if not route: + # DEV: Use all these `getattr`s to protect against changes between versions + pattern = getattr(resolver, "pattern", None) + if pattern: + regex = getattr(pattern, "regex", None) + if regex: + route = getattr(regex, "pattern", "") - query string or fragments are not included. - """ - # DEV: We do this instead of `request.build_absolute_uri()` since - # an exception can get raised, we want to always build a url - # regardless of any exceptions raised from `request.get_host()` - host = None - try: - host = request.get_host() # this will include host:port - except Exception: - log.debug('Failed to get Django request host', exc_info=True) + return route - if not host: - try: - # Try to build host how Django would have - # https://github.com/django/django/blob/e8d0d2a5efc8012dcc8bf1809dec065ebde64c81/django/http/request.py#L85-L102 - if 'HTTP_HOST' in request.META: - host = request.META['HTTP_HOST'] - else: - host = request.META['SERVER_NAME'] - port = str(request.META['SERVER_PORT']) - if port != ('443' if request.is_secure() else '80'): - host = '{0}:{1}'.format(host, port) - except Exception: - # This really shouldn't ever happen, but lets guard here just in case - log.debug('Failed to build Django request host', exc_info=True) - host = 'unknown' - # Build request url from the information available - # DEV: We are explicitly omitting query strings since they may contain sensitive information - return parse.urlunparse(parse.ParseResult( - scheme=request.scheme, - netloc=host, - path=request.path, - params='', - query='', - fragment='', - )) +def set_tag_array(span, prefix, value): + """Helper to set a span tag as a single value or an array""" + if not value: + return + + if len(value) == 1: + span.set_tag(prefix, value[0]) + else: + for i, v in enumerate(value, start=0): + span.set_tag("{0}.{1}".format(prefix, i), v) diff --git a/ddtrace/monkey.py b/ddtrace/monkey.py index 88b9fe984fa..bc005bfa22d 100644 --- a/ddtrace/monkey.py +++ b/ddtrace/monkey.py @@ -26,6 +26,7 @@ 'cassandra': True, 'celery': True, 'consul': True, + 'django': True, 'elasticsearch': True, 'algoliasearch': True, 'futures': False, # experimental propagation @@ -55,7 +56,6 @@ 'kombu': False, # Ignore some web framework integrations that might be configured explicitly in code - 'django': False, 'falcon': False, 'pylons': False, 'pyramid': False, diff --git a/ddtrace/utils/wrappers.py b/ddtrace/utils/wrappers.py index bfcf69a47e8..38b4f5a049a 100644 --- a/ddtrace/utils/wrappers.py +++ b/ddtrace/utils/wrappers.py @@ -4,6 +4,13 @@ from .deprecation import deprecated +def iswrapped(obj, attr=None): + """Returns whether an attribute is wrapped or not.""" + if attr is not None: + obj = getattr(obj, attr, None) + return hasattr(obj, "__wrapped__") and isinstance(obj, wrapt.ObjectProxy) + + def unwrap(obj, attr): f = getattr(obj, attr, None) if f and isinstance(f, wrapt.ObjectProxy) and hasattr(f, "__wrapped__"): diff --git a/pyproject.toml b/pyproject.toml index 8790a1e7ba3..ba35d7871c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,19 +31,6 @@ exclude = ''' | celery | consul | dbapi - | django/ - ( - __init__.py - | apps.py - | cache.py - | compat.py - | conf.py - | db.py - | patch.py - | restframework.py - | templates.py - | utils.py - ) | dogpile_cache | elasticsearch | falcon @@ -118,8 +105,6 @@ exclude = ''' | config.py | consul | dbapi - | django - | djangorestframework | elasticsearch | falcon | flask @@ -208,4 +193,4 @@ exclude = ''' | wait-for-services.py ) ) -''' \ No newline at end of file +''' diff --git a/tests/contrib/django/app/middlewares.py b/tests/contrib/django/app/middlewares.py deleted file mode 100644 index c3169972915..00000000000 --- a/tests/contrib/django/app/middlewares.py +++ /dev/null @@ -1,36 +0,0 @@ -from django.http import HttpResponse - -try: - from django.utils.deprecation import MiddlewareMixin - MiddlewareClass = MiddlewareMixin -except ImportError: - MiddlewareClass = object - - -class CatchExceptionMiddleware(MiddlewareClass): - def process_exception(self, request, exception): - return HttpResponse(status=500) - - -class HandleErrorMiddlewareSuccess(MiddlewareClass): - """ Converts an HttpError (that may be returned from an exception handler) - generated by a view or previous middleware and returns a 200 - HttpResponse. - """ - def process_response(self, request, response): - if response.status_code == 500: - return HttpResponse(status=200) - - return response - - -class HandleErrorMiddlewareClientError(MiddlewareClass): - """ Converts an HttpError (that may be returned from an exception handler) - generated by a view or previous middleware and returns a 404 - HttpResponse. - """ - def process_response(self, request, response): - if response.status_code == 500: - return HttpResponse(status=404) - - return response diff --git a/tests/contrib/django/app/settings.py b/tests/contrib/django/app/settings.py deleted file mode 100644 index b17e604e493..00000000000 --- a/tests/contrib/django/app/settings.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Settings configuration for the Django web framework. Update this -configuration if you need to change the default behavior of -Django during tests -""" -import os -import django - - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:' - } -} - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', - }, - 'redis': { - 'BACKEND': 'django_redis.cache.RedisCache', - 'LOCATION': 'redis://127.0.0.1:6379/1', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', - }, - 'python_memcached': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '127.0.0.1:11211', - }, - 'django_pylibmc': { - 'BACKEND': 'django_pylibmc.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', - 'BINARY': True, - 'OPTIONS': { - 'tcp_nodelay': True, - 'ketama': True - } - }, -} - -SITE_ID = 1 -SECRET_KEY = 'not_very_secret_in_tests' -USE_I18N = True -USE_L10N = True -STATIC_URL = '/static/' -ROOT_URLCONF = 'tests.contrib.django.app.views' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(BASE_DIR, 'app', 'templates'), - ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -if (1, 10) <= django.VERSION < (2, 0): - MIDDLEWARE = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - - 'tests.contrib.django.app.middlewares.CatchExceptionMiddleware', - ] - -# Django 2.0 has different defaults -elif django.VERSION >= (2, 0): - MIDDLEWARE = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - - 'tests.contrib.django.app.middlewares.CatchExceptionMiddleware', - ] - -# Pre 1.10 style -else: - MIDDLEWARE_CLASSES = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - - 'tests.contrib.django.app.middlewares.CatchExceptionMiddleware', - ] - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - - # tracer app - 'ddtrace.contrib.django', -] - -DATADOG_TRACE = { - # tracer with a DummyWriter - 'TRACER': 'tests.contrib.django.utils.tracer', - 'ENABLED': True, - 'TAGS': { - 'env': 'test', - }, -} diff --git a/tests/contrib/django/app/settings_untraced.py b/tests/contrib/django/app/settings_untraced.py deleted file mode 100644 index eb9f878b85d..00000000000 --- a/tests/contrib/django/app/settings_untraced.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Settings configuration for the Django web framework. Update this -configuration if you need to change the default behavior of -Django during tests -""" -import os - - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:' - } -} - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', - }, - 'redis': { - 'BACKEND': 'django_redis.cache.RedisCache', - 'LOCATION': 'redis://127.0.0.1:6379/1', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', - }, - 'python_memcached': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '127.0.0.1:11211', - }, - 'django_pylibmc': { - 'BACKEND': 'django_pylibmc.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', - 'BINARY': True, - 'OPTIONS': { - 'tcp_nodelay': True, - 'ketama': True - } - }, -} - -SITE_ID = 1 -SECRET_KEY = 'not_very_secret_in_tests' -USE_I18N = True -USE_L10N = True -STATIC_URL = '/static/' -ROOT_URLCONF = 'tests.contrib.django.app.views' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(BASE_DIR, 'app', 'templates'), - ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -# 1.10+ style -MIDDLEWARE = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -] - -# Pre 1.10 style -MIDDLEWARE_CLASSES = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -] - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', -] - -DATADOG_TRACE = { - # tracer with a DummyWriter - 'TRACER': 'tests.contrib.django.utils.tracer', - 'ENABLED': True, -} diff --git a/tests/contrib/django/app/views.py b/tests/contrib/django/app/views.py deleted file mode 100644 index 8e570a536b7..00000000000 --- a/tests/contrib/django/app/views.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Class based views used for Django tests. -""" - -from functools import partial - -from django.http import HttpResponse -from django.conf.urls import url - -from django.views.generic import ListView, TemplateView -from django.views.decorators.cache import cache_page - -from django.contrib.auth.models import User -from django.contrib.syndication.views import Feed - - -class UserList(ListView): - model = User - template_name = 'users_list.html' - - -class TemplateCachedUserList(ListView): - model = User - template_name = 'cached_list.html' - - -class ForbiddenView(TemplateView): - def get(self, request, *args, **kwargs): - return HttpResponse(status=403) - - -def function_view(request): - return HttpResponse(status=200) - - -def error_500(request): - raise Exception('Error 500') - - -class FeedView(Feed): - """ - A callable view that is part of the Django framework - """ - title = 'Police beat site news' - link = '/sitenews/' - description = 'Updates on changes and additions to police beat central.' - - def items(self): - return [] - - def item_title(self, item): - return 'empty' - - def item_description(self, item): - return 'empty' - - -partial_view = partial(function_view) - -# disabling flake8 test below, yes, declaring a func like this is bad, we know -lambda_view = lambda request: function_view(request) # NOQA - -# use this url patterns for tests -urlpatterns = [ - url(r'^users/$', UserList.as_view(), name='users-list'), - url(r'^cached-template/$', TemplateCachedUserList.as_view(), name='cached-template-list'), - url(r'^cached-users/$', cache_page(60)(UserList.as_view()), name='cached-users-list'), - url(r'^fail-view/$', ForbiddenView.as_view(), name='forbidden-view'), - url(r'^fn-view/$', function_view, name='fn-view'), - url(r'^feed-view/$', FeedView(), name='feed-view'), - url(r'^partial-view/$', partial_view, name='partial-view'), - url(r'^lambda-view/$', lambda_view, name='lambda-view'), - url(r'^error-500/$', error_500, name='error-500'), -] diff --git a/tests/contrib/django/compat.py b/tests/contrib/django/compat.py index c591277ff72..37ff0f8239a 100644 --- a/tests/contrib/django/compat.py +++ b/tests/contrib/django/compat.py @@ -1,4 +1,4 @@ -__all__ = ['reverse'] +__all__ = ["reverse"] try: from django.core.urlresolvers import reverse diff --git a/tests/contrib/django/conftest.py b/tests/contrib/django/conftest.py index 37511609f7f..f69a73f7239 100644 --- a/tests/contrib/django/conftest.py +++ b/tests/contrib/django/conftest.py @@ -1,16 +1,56 @@ import os import django from django.conf import settings +import pytest + +from ddtrace import Pin +from ddtrace.contrib.django import patch + +from ...utils.span import TracerSpanContainer +from ...utils.tracer import DummyTracer + # We manually designate which settings we will be using in an environment variable # This is similar to what occurs in the `manage.py` -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.contrib.django.app.settings') +if django.VERSION >= (2, 0, 0): + app_name = "django_app" +else: + app_name = "django1_app" +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.contrib.django.{0}.settings".format(app_name)) # `pytest` automatically calls this function once when tests are run. def pytest_configure(): settings.DEBUG = False - if django.VERSION < (1, 7, 0): - settings.configure() - else: - django.setup() + patch() + django.setup() + + +@pytest.fixture(autouse=True) +def patch_django(tracer): + # Patch Django and override tracer to be our test tracer + pin = Pin.get_from(django) + original_tracer = pin.tracer + Pin.override(django, tracer=tracer) + + # Yield to our test + yield + + # Reset the tracer pinned to Django and unpatch + # DEV: unable to properly unpatch and reload django app with each test + # unpatch() + Pin.override(django, tracer=original_tracer) + + +@pytest.fixture +def tracer(): + tracer = DummyTracer() + yield tracer + tracer.writer.pop() + + +@pytest.fixture +def test_spans(tracer): + container = TracerSpanContainer(tracer) + yield container + container.reset() diff --git a/tests/contrib/django/app/__init__.py b/tests/contrib/django/django1_app/__init__.py similarity index 100% rename from tests/contrib/django/app/__init__.py rename to tests/contrib/django/django1_app/__init__.py diff --git a/tests/contrib/django/django1_app/settings.py b/tests/contrib/django/django1_app/settings.py new file mode 100644 index 00000000000..c64e1eee0c2 --- /dev/null +++ b/tests/contrib/django/django1_app/settings.py @@ -0,0 +1,88 @@ +import os + +ALLOWED_HOSTS = [ + "testserver", +] + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}} + +CACHES = { + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "unique-snowflake",}, + "redis": {"BACKEND": "django_redis.cache.RedisCache", "LOCATION": "redis://127.0.0.1:6379/1",}, + "pylibmc": {"BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", "LOCATION": "127.0.0.1:11211",}, + "python_memcached": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": "127.0.0.1:11211", + }, + "django_pylibmc": { + "BACKEND": "django_pylibmc.memcached.PyLibMCCache", + "LOCATION": "127.0.0.1:11211", + "BINARY": True, + "OPTIONS": {"tcp_nodelay": True, "ketama": True}, + }, +} + +SITE_ID = 1 +SECRET_KEY = "not_very_secret_in_tests" +USE_I18N = True +USE_L10N = True +STATIC_URL = "/static/" +ROOT_URLCONF = "tests.contrib.django.django1_app.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates"),], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +MIDDLEWARE_CLASSES = [ + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.auth.middleware.SessionAuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", +] + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", +] + +# Allows for testing django instrumentation before migration to tracer config api +if os.environ.get("TEST_DATADOG_DJANGO_MIGRATION"): + INSTALLED_APPS.append("ddtrace.contrib.django") + DATADOG_TRACE = { + "AGENT_HOSTNAME": "host-test", + "AGENT_PORT": 1234, + "AUTO_INSTRUMENT": True, + "INSTRUMENT_CACHE": True, + "INSTRUMENT_DATABASE": True, + "INSTRUMENT_TEMPLATE": True, + "DEFAULT_DATABASE_PREFIX": "db-test-", + "DEFAULT_SERVICE": "django-test", + "DEFAULT_CACHE_SERVICE": "cache-test", + "ENABLED": True, + "DISTRIBUTED_TRACING": True, + "ANALYTICS_ENABLED": True, + "ANALYTICS_SAMPLE_RATE": True, + "TRACE_QUERY_STRING": True, + "TAGS": {"env": "env-test"}, + "TRACER": "ddtrace.tracer", + } diff --git a/tests/contrib/django/django1_app/urls.py b/tests/contrib/django/django1_app/urls.py new file mode 100644 index 00000000000..984e5a702ec --- /dev/null +++ b/tests/contrib/django/django1_app/urls.py @@ -0,0 +1,19 @@ +from django.conf.urls import url +from django.views.decorators.cache import cache_page + +from .. import views + + +urlpatterns = [ + url(r"^$", views.index), + url(r"^simple/$", views.BasicView.as_view()), + url(r"^users/$", views.UserList.as_view(), name="users-list"), + url(r"^cached-template/$", views.TemplateCachedUserList.as_view(), name="cached-template-list"), + url(r"^cached-users/$", cache_page(60)(views.UserList.as_view()), name="cached-users-list"), + url(r"^fail-view/$", views.ForbiddenView.as_view(), name="forbidden-view"), + url(r"^fn-view/$", views.function_view, name="fn-view"), + url(r"^feed-view/$", views.FeedView(), name="feed-view"), + url(r"^partial-view/$", views.partial_view, name="partial-view"), + url(r"^lambda-view/$", views.lambda_view, name="lambda-view"), + url(r"^error-500/$", views.error_500, name="error-500"), +] diff --git a/tests/contrib/django/django_app/__init__.py b/tests/contrib/django/django_app/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/contrib/django/django_app/extra_urls.py b/tests/contrib/django/django_app/extra_urls.py new file mode 100644 index 00000000000..33c2e61ae43 --- /dev/null +++ b/tests/contrib/django/django_app/extra_urls.py @@ -0,0 +1,11 @@ +from django.conf.urls import url +from django.http import HttpResponse + + +def include_view(request): + return HttpResponse(status=200) + + +urlpatterns = [ + url("test/", include_view), +] diff --git a/tests/contrib/django/django_app/settings.py b/tests/contrib/django/django_app/settings.py new file mode 100644 index 00000000000..501ba618bea --- /dev/null +++ b/tests/contrib/django/django_app/settings.py @@ -0,0 +1,84 @@ +import os + +ALLOWED_HOSTS = [ + "testserver", +] + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}} + +CACHES = { + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "unique-snowflake",}, + "redis": {"BACKEND": "django_redis.cache.RedisCache", "LOCATION": "redis://127.0.0.1:6379/1",}, + "pylibmc": {"BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", "LOCATION": "127.0.0.1:11211",}, + "python_memcached": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": "127.0.0.1:11211", + }, +} + +SITE_ID = 1 +SECRET_KEY = "not_very_secret_in_tests" +USE_I18N = True +USE_L10N = True +STATIC_URL = "/static/" +ROOT_URLCONF = "tests.contrib.django.django_app.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates"),], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +MIDDLEWARE = [ + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + "tests.contrib.django.middleware.ClsMiddleware", + "tests.contrib.django.middleware.fn_middleware", + "tests.contrib.django.middleware.EverythingMiddleware", +] + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", +] + +# Allows for testing django instrumentation before migration to tracer config api +if os.environ.get("TEST_DATADOG_DJANGO_MIGRATION"): + INSTALLED_APPS.append("ddtrace.contrib.django") + DATADOG_TRACE = { + "AGENT_HOSTNAME": "host-test", + "AGENT_PORT": 1234, + "AUTO_INSTRUMENT": True, + "INSTRUMENT_CACHE": True, + "INSTRUMENT_DATABASE": True, + "INSTRUMENT_TEMPLATE": True, + "DEFAULT_DATABASE_PREFIX": "db-test-", + "DEFAULT_SERVICE": "django-test", + "DEFAULT_CACHE_SERVICE": "cache-test", + "ENABLED": True, + "DISTRIBUTED_TRACING": True, + "ANALYTICS_ENABLED": True, + "ANALYTICS_SAMPLE_RATE": True, + "TRACE_QUERY_STRING": True, + "TAGS": {"env": "env-test"}, + "TRACER": "ddtrace.tracer", + } diff --git a/tests/contrib/django/django_app/urls.py b/tests/contrib/django/django_app/urls.py new file mode 100644 index 00000000000..e2f5a08f2e6 --- /dev/null +++ b/tests/contrib/django/django_app/urls.py @@ -0,0 +1,32 @@ +from django.conf.urls import url +from django.http import HttpResponse +from django.views.decorators.cache import cache_page +from django.urls import include, path, re_path + +from .. import views + + +def repath_view(request): + return HttpResponse(status=200) + + +def path_view(request): + return HttpResponse(status=200) + + +urlpatterns = [ + url(r"^$", views.index), + url(r"^simple/$", views.BasicView.as_view()), + url(r"^users/$", views.UserList.as_view(), name="users-list"), + url(r"^cached-template/$", views.TemplateCachedUserList.as_view(), name="cached-template-list"), + url(r"^cached-users/$", cache_page(60)(views.UserList.as_view()), name="cached-users-list"), + url(r"^fail-view/$", views.ForbiddenView.as_view(), name="forbidden-view"), + url(r"^fn-view/$", views.function_view, name="fn-view"), + url(r"^feed-view/$", views.FeedView(), name="feed-view"), + url(r"^partial-view/$", views.partial_view, name="partial-view"), + url(r"^lambda-view/$", views.lambda_view, name="lambda-view"), + url(r"^error-500/$", views.error_500, name="error-500"), + re_path(r"re-path.*/", repath_view), + path("path/", path_view), + path("include/", include("tests.contrib.django.django_app.extra_urls")), +] diff --git a/tests/contrib/django/middleware.py b/tests/contrib/django/middleware.py new file mode 100644 index 00000000000..9ffc588c96f --- /dev/null +++ b/tests/contrib/django/middleware.py @@ -0,0 +1,78 @@ +from django.http import HttpResponse + +try: + from django.utils.deprecation import MiddlewareMixin + + MiddlewareClass = MiddlewareMixin +except ImportError: + MiddlewareClass = object + + +class CatchExceptionMiddleware(MiddlewareClass): + def process_exception(self, request, exception): + return HttpResponse(status=500) + + +class HandleErrorMiddlewareSuccess(MiddlewareClass): + def process_exception(self, request, exception): + return HttpResponse(status=200) + + +class HandleErrorMiddlewareClientError(MiddlewareClass): + """ Converts an HttpError (that may be returned from an exception handler) + generated by a view or previous middleware and returns a 404 + HttpResponse. + """ + + def process_response(self, request, response): + if response.status_code == 500: + return HttpResponse(status=404) + + return response + + +def fn_middleware(get_response): + """Function factory middleware.""" + + def mw(request): + response = get_response(request) + return response + + return mw + + +class ClsMiddleware: + """Class middleware.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + return response + + +class EverythingMiddleware: + """Middleware using all possible middleware hooks.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + return response + + def process_response(self, req, resp): + return resp + + def process_request(self, req): + return req + + def process_exception(self, request, exception): + pass + + def process_view(self, request, view_func, view_args, view_kwargs): + pass + + def process_template_response(self, req, resp): + return resp diff --git a/tests/contrib/django/runtests.py b/tests/contrib/django/runtests.py deleted file mode 100755 index 0ece0b6956a..00000000000 --- a/tests/contrib/django/runtests.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -import os -import sys - - -if __name__ == '__main__': - # define django defaults - app_to_test = 'tests/contrib/django' - - # append the project root to the PYTHONPATH: - # this is required because we don't want to put the current file - # in the project_root - current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - project_root = os.path.join(current_dir, '..', '..') - sys.path.append(project_root) - - from django.core.management import execute_from_command_line - execute_from_command_line([sys.argv[0], 'test', app_to_test]) diff --git a/tests/contrib/django/app/templates/cached_list.html b/tests/contrib/django/templates/cached_list.html similarity index 100% rename from tests/contrib/django/app/templates/cached_list.html rename to tests/contrib/django/templates/cached_list.html diff --git a/tests/contrib/django/app/templates/users_list.html b/tests/contrib/django/templates/users_list.html similarity index 100% rename from tests/contrib/django/app/templates/users_list.html rename to tests/contrib/django/templates/users_list.html diff --git a/tests/contrib/django/test_autopatching.py b/tests/contrib/django/test_autopatching.py deleted file mode 100644 index 270bdd2573b..00000000000 --- a/tests/contrib/django/test_autopatching.py +++ /dev/null @@ -1,98 +0,0 @@ -import django - -from ddtrace.monkey import patch -from .utils import DjangoTraceTestCase -from django.conf import settings -from unittest import skipIf - - -class DjangoAutopatchTest(DjangoTraceTestCase): - def setUp(self): - super(DjangoAutopatchTest, self).setUp() - patch(django=True) - django.setup() - - @skipIf(django.VERSION >= (1, 10), 'skip if version above 1.10') - def test_autopatching_middleware_classes(self): - assert django._datadog_patch - assert 'ddtrace.contrib.django' in settings.INSTALLED_APPS - assert settings.MIDDLEWARE_CLASSES[0] == 'ddtrace.contrib.django.TraceMiddleware' - assert settings.MIDDLEWARE_CLASSES[-1] == 'ddtrace.contrib.django.TraceExceptionMiddleware' - - @skipIf(django.VERSION >= (1, 10), 'skip if version above 1.10') - def test_autopatching_twice_middleware_classes(self): - assert django._datadog_patch - # Call django.setup() twice and ensure we don't add a duplicate tracer - django.setup() - - found_app = settings.INSTALLED_APPS.count('ddtrace.contrib.django') - assert found_app == 1 - - assert settings.MIDDLEWARE_CLASSES[0] == 'ddtrace.contrib.django.TraceMiddleware' - assert settings.MIDDLEWARE_CLASSES[-1] == 'ddtrace.contrib.django.TraceExceptionMiddleware' - - found_mw = settings.MIDDLEWARE_CLASSES.count('ddtrace.contrib.django.TraceMiddleware') - assert found_mw == 1 - found_mw = settings.MIDDLEWARE_CLASSES.count('ddtrace.contrib.django.TraceExceptionMiddleware') - assert found_mw == 1 - - @skipIf(django.VERSION < (1, 10), 'skip if version is below 1.10') - def test_autopatching_middleware(self): - assert django._datadog_patch - assert 'ddtrace.contrib.django' in settings.INSTALLED_APPS - assert settings.MIDDLEWARE[0] == 'ddtrace.contrib.django.TraceMiddleware' - # MIDDLEWARE_CLASSES gets created internally in django 1.10 & 1.11 but doesn't - # exist at all in 2.0. - assert not getattr(settings, 'MIDDLEWARE_CLASSES', None) or \ - 'ddtrace.contrib.django.TraceMiddleware' \ - not in settings.MIDDLEWARE_CLASSES - assert settings.MIDDLEWARE[-1] == 'ddtrace.contrib.django.TraceExceptionMiddleware' - assert not getattr(settings, 'MIDDLEWARE_CLASSES', None) or \ - 'ddtrace.contrib.django.TraceExceptionMiddleware' \ - not in settings.MIDDLEWARE_CLASSES - - @skipIf(django.VERSION < (1, 10), 'skip if version is below 1.10') - def test_autopatching_twice_middleware(self): - assert django._datadog_patch - # Call django.setup() twice and ensure we don't add a duplicate tracer - django.setup() - - found_app = settings.INSTALLED_APPS.count('ddtrace.contrib.django') - assert found_app == 1 - - assert settings.MIDDLEWARE[0] == 'ddtrace.contrib.django.TraceMiddleware' - # MIDDLEWARE_CLASSES gets created internally in django 1.10 & 1.11 but doesn't - # exist at all in 2.0. - assert not getattr(settings, 'MIDDLEWARE_CLASSES', None) or \ - 'ddtrace.contrib.django.TraceMiddleware' \ - not in settings.MIDDLEWARE_CLASSES - assert settings.MIDDLEWARE[-1] == 'ddtrace.contrib.django.TraceExceptionMiddleware' - assert not getattr(settings, 'MIDDLEWARE_CLASSES', None) or \ - 'ddtrace.contrib.django.TraceExceptionMiddleware' \ - not in settings.MIDDLEWARE_CLASSES - - found_mw = settings.MIDDLEWARE.count('ddtrace.contrib.django.TraceMiddleware') - assert found_mw == 1 - - found_mw = settings.MIDDLEWARE.count('ddtrace.contrib.django.TraceExceptionMiddleware') - assert found_mw == 1 - - -class DjangoAutopatchCustomMiddlewareTest(DjangoTraceTestCase): - @skipIf(django.VERSION < (1, 10), 'skip if version is below 1.10') - def test_autopatching_empty_middleware(self): - with self.settings(MIDDLEWARE=[]): - patch(django=True) - django.setup() - assert django._datadog_patch - assert 'ddtrace.contrib.django' in settings.INSTALLED_APPS - assert settings.MIDDLEWARE[0] == 'ddtrace.contrib.django.TraceMiddleware' - # MIDDLEWARE_CLASSES gets created internally in django 1.10 & 1.11 but doesn't - # exist at all in 2.0. - assert not getattr(settings, 'MIDDLEWARE_CLASSES', None) or \ - 'ddtrace.contrib.django.TraceMiddleware' \ - not in settings.MIDDLEWARE_CLASSES - assert settings.MIDDLEWARE[-1] == 'ddtrace.contrib.django.TraceExceptionMiddleware' - assert not getattr(settings, 'MIDDLEWARE_CLASSES', None) or \ - 'ddtrace.contrib.django.TraceExceptionMiddleware' \ - not in settings.MIDDLEWARE_CLASSES diff --git a/tests/contrib/django/test_cache_backends.py b/tests/contrib/django/test_cache_backends.py deleted file mode 100644 index c0cf3490904..00000000000 --- a/tests/contrib/django/test_cache_backends.py +++ /dev/null @@ -1,246 +0,0 @@ -import time - -# 3rd party -from django.core.cache import caches - -# testing -from .utils import DjangoTraceTestCase -from ...util import assert_dict_issuperset - - -class DjangoCacheRedisTest(DjangoTraceTestCase): - """ - Ensures that the cache system is properly traced in - different cache backend - """ - def test_cache_redis_get(self): - # get the redis cache - cache = caches['redis'] - - # (trace) the cache miss - start = time.time() - cache.get('missing_key') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django_redis.cache.RedisCache', - 'django.cache.key': 'missing_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_redis_get_many(self): - # get the redis cache - cache = caches['redis'] - - # (trace) the cache miss - start = time.time() - cache.get_many(['missing_key', 'another_key']) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get_many' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django_redis.cache.RedisCache', - 'django.cache.key': str(['missing_key', 'another_key']), - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_pylibmc_get(self): - # get the redis cache - cache = caches['pylibmc'] - - # (trace) the cache miss - start = time.time() - cache.get('missing_key') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.memcached.PyLibMCCache', - 'django.cache.key': 'missing_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_pylibmc_get_many(self): - # get the redis cache - cache = caches['pylibmc'] - - # (trace) the cache miss - start = time.time() - cache.get_many(['missing_key', 'another_key']) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get_many' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.memcached.PyLibMCCache', - 'django.cache.key': str(['missing_key', 'another_key']), - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_memcached_get(self): - # get the redis cache - cache = caches['python_memcached'] - - # (trace) the cache miss - start = time.time() - cache.get('missing_key') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.memcached.MemcachedCache', - 'django.cache.key': 'missing_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_memcached_get_many(self): - # get the redis cache - cache = caches['python_memcached'] - - # (trace) the cache miss - start = time.time() - cache.get_many(['missing_key', 'another_key']) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get_many' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.memcached.MemcachedCache', - 'django.cache.key': str(['missing_key', 'another_key']), - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_django_pylibmc_get(self): - # get the redis cache - cache = caches['django_pylibmc'] - - # (trace) the cache miss - start = time.time() - cache.get('missing_key') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django_pylibmc.memcached.PyLibMCCache', - 'django.cache.key': 'missing_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_django_pylibmc_get_many(self): - # get the redis cache - cache = caches['django_pylibmc'] - - # (trace) the cache miss - start = time.time() - cache.get_many(['missing_key', 'another_key']) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get_many' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django_pylibmc.memcached.PyLibMCCache', - 'django.cache.key': str(['missing_key', 'another_key']), - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end diff --git a/tests/contrib/django/test_cache_client.py b/tests/contrib/django/test_cache_client.py deleted file mode 100644 index 89725073088..00000000000 --- a/tests/contrib/django/test_cache_client.py +++ /dev/null @@ -1,364 +0,0 @@ -import time - -# 3rd party -from django.core.cache import caches - -# testing -from .utils import DjangoTraceTestCase, override_ddtrace_settings -from ...util import assert_dict_issuperset - - -class DjangoCacheWrapperTest(DjangoTraceTestCase): - """ - Ensures that the cache system is properly traced - """ - def test_cache_get(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.get('missing_key') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'get' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'missing_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - @override_ddtrace_settings(DEFAULT_CACHE_SERVICE='foo') - def test_cache_service_can_be_overriden(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - cache.get('missing_key') - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'foo' - - @override_ddtrace_settings(INSTRUMENT_CACHE=False) - def test_cache_disabled(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - cache.get('missing_key') - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 0 - - def test_cache_set(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.set('a_new_key', 50) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'set' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'a_new_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_add(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.add('a_new_key', 50) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'add' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'a_new_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_delete(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.delete('an_existing_key') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - span = spans[0] - assert span.service == 'django' - assert span.resource == 'delete' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'an_existing_key', - 'env': 'test', - } - - assert_dict_issuperset(span.meta, expected_meta) - assert start < span.start < span.start + span.duration < end - - def test_cache_incr(self): - # get the default cache, set the value and reset the spans - cache = caches['default'] - cache.set('value', 0) - self.tracer.writer.spans = [] - - # (trace) the cache miss - start = time.time() - cache.incr('value') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 2 - - span_incr = spans[0] - span_get = spans[1] - - # LocMemCache doesn't provide an atomic operation - assert span_get.service == 'django' - assert span_get.resource == 'get' - assert span_get.name == 'django.cache' - assert span_get.span_type == 'cache' - assert span_get.error == 0 - assert span_incr.service == 'django' - assert span_incr.resource == 'incr' - assert span_incr.name == 'django.cache' - assert span_incr.span_type == 'cache' - assert span_incr.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'value', - 'env': 'test', - } - - assert_dict_issuperset(span_get.meta, expected_meta) - assert_dict_issuperset(span_incr.meta, expected_meta) - assert start < span_incr.start < span_incr.start + span_incr.duration < end - - def test_cache_decr(self): - # get the default cache, set the value and reset the spans - cache = caches['default'] - cache.set('value', 0) - self.tracer.writer.spans = [] - - # (trace) the cache miss - start = time.time() - cache.decr('value') - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 3 - - span_decr = spans[0] - span_incr = spans[1] - span_get = spans[2] - - # LocMemCache doesn't provide an atomic operation - assert span_get.service == 'django' - assert span_get.resource == 'get' - assert span_get.name == 'django.cache' - assert span_get.span_type == 'cache' - assert span_get.error == 0 - assert span_incr.service == 'django' - assert span_incr.resource == 'incr' - assert span_incr.name == 'django.cache' - assert span_incr.span_type == 'cache' - assert span_incr.error == 0 - assert span_decr.service == 'django' - assert span_decr.resource == 'decr' - assert span_decr.name == 'django.cache' - assert span_decr.span_type == 'cache' - assert span_decr.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'value', - 'env': 'test', - } - - assert_dict_issuperset(span_get.meta, expected_meta) - assert_dict_issuperset(span_incr.meta, expected_meta) - assert_dict_issuperset(span_decr.meta, expected_meta) - assert start < span_decr.start < span_decr.start + span_decr.duration < end - - def test_cache_get_many(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.get_many(['missing_key', 'another_key']) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 3 - - span_get_many = spans[0] - span_get_first = spans[1] - span_get_second = spans[2] - - # LocMemCache doesn't provide an atomic operation - assert span_get_first.service == 'django' - assert span_get_first.resource == 'get' - assert span_get_first.name == 'django.cache' - assert span_get_first.span_type == 'cache' - assert span_get_first.error == 0 - assert span_get_second.service == 'django' - assert span_get_second.resource == 'get' - assert span_get_second.name == 'django.cache' - assert span_get_second.span_type == 'cache' - assert span_get_second.error == 0 - assert span_get_many.service == 'django' - assert span_get_many.resource == 'get_many' - assert span_get_many.name == 'django.cache' - assert span_get_many.span_type == 'cache' - assert span_get_many.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': str(['missing_key', 'another_key']), - 'env': 'test', - } - - assert_dict_issuperset(span_get_many.meta, expected_meta) - assert start < span_get_many.start < span_get_many.start + span_get_many.duration < end - - def test_cache_set_many(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.set_many({'first_key': 1, 'second_key': 2}) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 3 - - span_set_many = spans[0] - span_set_first = spans[1] - span_set_second = spans[2] - - # LocMemCache doesn't provide an atomic operation - assert span_set_first.service == 'django' - assert span_set_first.resource == 'set' - assert span_set_first.name == 'django.cache' - assert span_set_first.span_type == 'cache' - assert span_set_first.error == 0 - assert span_set_second.service == 'django' - assert span_set_second.resource == 'set' - assert span_set_second.name == 'django.cache' - assert span_set_second.span_type == 'cache' - assert span_set_second.error == 0 - assert span_set_many.service == 'django' - assert span_set_many.resource == 'set_many' - assert span_set_many.name == 'django.cache' - assert span_set_many.span_type == 'cache' - assert span_set_many.error == 0 - - assert span_set_many.meta['django.cache.backend'] == 'django.core.cache.backends.locmem.LocMemCache' - assert 'first_key' in span_set_many.meta['django.cache.key'] - assert 'second_key' in span_set_many.meta['django.cache.key'] - assert start < span_set_many.start < span_set_many.start + span_set_many.duration < end - - def test_cache_delete_many(self): - # get the default cache - cache = caches['default'] - - # (trace) the cache miss - start = time.time() - cache.delete_many(['missing_key', 'another_key']) - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 3 - - span_delete_many = spans[0] - span_delete_first = spans[1] - span_delete_second = spans[2] - - # LocMemCache doesn't provide an atomic operation - assert span_delete_first.service == 'django' - assert span_delete_first.resource == 'delete' - assert span_delete_first.name == 'django.cache' - assert span_delete_first.span_type == 'cache' - assert span_delete_first.error == 0 - assert span_delete_second.service == 'django' - assert span_delete_second.resource == 'delete' - assert span_delete_second.name == 'django.cache' - assert span_delete_second.span_type == 'cache' - assert span_delete_second.error == 0 - assert span_delete_many.service == 'django' - assert span_delete_many.resource == 'delete_many' - assert span_delete_many.name == 'django.cache' - assert span_delete_many.span_type == 'cache' - assert span_delete_many.error == 0 - - assert span_delete_many.meta['django.cache.backend'] == 'django.core.cache.backends.locmem.LocMemCache' - assert 'missing_key' in span_delete_many.meta['django.cache.key'] - assert 'another_key' in span_delete_many.meta['django.cache.key'] - assert start < span_delete_many.start < span_delete_many.start + span_delete_many.duration < end diff --git a/tests/contrib/django/test_cache_views.py b/tests/contrib/django/test_cache_views.py deleted file mode 100644 index a26394809e5..00000000000 --- a/tests/contrib/django/test_cache_views.py +++ /dev/null @@ -1,93 +0,0 @@ -# testing -from .compat import reverse -from .utils import DjangoTraceTestCase - - -class DjangoCacheViewTest(DjangoTraceTestCase): - """ - Ensures that the cache system is properly traced - """ - def test_cached_view(self): - # make the first request so that the view is cached - url = reverse('cached-users-list') - response = self.client.get(url) - assert response.status_code == 200 - - # check the first call for a non-cached view - spans = self.tracer.writer.pop() - assert len(spans) == 6 - # the cache miss - assert spans[1].resource == 'get' - # store the result in the cache - assert spans[4].resource == 'set' - assert spans[5].resource == 'set' - - # check if the cache hit is traced - response = self.client.get(url) - spans = self.tracer.writer.pop() - assert len(spans) == 3 - - span_header = spans[1] - span_view = spans[2] - assert span_view.service == 'django' - assert span_view.resource == 'get' - assert span_view.name == 'django.cache' - assert span_view.span_type == 'cache' - assert span_view.error == 0 - assert span_header.service == 'django' - assert span_header.resource == 'get' - assert span_header.name == 'django.cache' - assert span_header.span_type == 'cache' - assert span_header.error == 0 - - expected_meta_view = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': ( - 'views.decorators.cache.cache_page..' - 'GET.03cdc1cc4aab71b038a6764e5fcabb82.d41d8cd98f00b204e9800998ecf8427e.en-us' - ), - 'env': 'test', - } - - expected_meta_header = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'views.decorators.cache.cache_header..03cdc1cc4aab71b038a6764e5fcabb82.en-us', - 'env': 'test', - } - - assert span_view.meta == expected_meta_view - assert span_header.meta == expected_meta_header - - def test_cached_template(self): - # make the first request so that the view is cached - url = reverse('cached-template-list') - response = self.client.get(url) - assert response.status_code == 200 - - # check the first call for a non-cached view - spans = self.tracer.writer.pop() - assert len(spans) == 5 - # the cache miss - assert spans[2].resource == 'get' - # store the result in the cache - assert spans[4].resource == 'set' - - # check if the cache hit is traced - response = self.client.get(url) - spans = self.tracer.writer.pop() - assert len(spans) == 3 - - span_template_cache = spans[2] - assert span_template_cache.service == 'django' - assert span_template_cache.resource == 'get' - assert span_template_cache.name == 'django.cache' - assert span_template_cache.span_type == 'cache' - assert span_template_cache.error == 0 - - expected_meta = { - 'django.cache.backend': 'django.core.cache.backends.locmem.LocMemCache', - 'django.cache.key': 'template.cache.users_list.d41d8cd98f00b204e9800998ecf8427e', - 'env': 'test', - } - - assert span_template_cache.meta == expected_meta diff --git a/tests/contrib/django/test_cache_wrapper.py b/tests/contrib/django/test_cache_wrapper.py deleted file mode 100644 index 11c314c0093..00000000000 --- a/tests/contrib/django/test_cache_wrapper.py +++ /dev/null @@ -1,133 +0,0 @@ -# 3rd party -from django.core.cache import caches -import pytest - -# testing -from .utils import DjangoTraceTestCase - - -class DjangoCacheTest(DjangoTraceTestCase): - """ - Ensures that the tracing doesn't break the Django - cache framework - """ - def test_wrapper_get_and_set(self): - # get the default cache - cache = caches['default'] - - value = cache.get('missing_key') - assert value is None - - cache.set('a_key', 50) - value = cache.get('a_key') - assert value == 50 - - def test_wrapper_add(self): - # get the default cache - cache = caches['default'] - - cache.add('a_key', 50) - value = cache.get('a_key') - assert value == 50 - - # add should not update a key if it's present - cache.add('a_key', 40) - value = cache.get('a_key') - assert value == 50 - - def test_wrapper_delete(self): - # get the default cache - cache = caches['default'] - - cache.set('a_key', 50) - cache.delete('a_key') - value = cache.get('a_key') - assert value is None - - def test_wrapper_incr_safety(self): - # get the default cache - cache = caches['default'] - - # it should fail not because of our wrapper - with pytest.raises(ValueError) as ex: - cache.incr('missing_key') - - # the error is not caused by our tracer - assert ex.value.args[0] == "Key 'missing_key' not found" - # an error trace must be sent - spans = self.tracer.writer.pop() - assert len(spans) == 2 - span = spans[0] - assert span.resource == 'incr' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 1 - - def test_wrapper_incr(self): - # get the default cache - cache = caches['default'] - - cache.set('value', 0) - value = cache.incr('value') - assert value == 1 - value = cache.get('value') - assert value == 1 - - def test_wrapper_decr_safety(self): - # get the default cache - cache = caches['default'] - - # it should fail not because of our wrapper - with pytest.raises(ValueError) as ex: - cache.decr('missing_key') - - # the error is not caused by our tracer - assert ex.value.args[0] == "Key 'missing_key' not found" - # an error trace must be sent - spans = self.tracer.writer.pop() - assert len(spans) == 3 - span = spans[0] - assert span.resource == 'decr' - assert span.name == 'django.cache' - assert span.span_type == 'cache' - assert span.error == 1 - - def test_wrapper_decr(self): - # get the default cache - cache = caches['default'] - - cache.set('value', 0) - value = cache.decr('value') - assert value == -1 - value = cache.get('value') - assert value == -1 - - def test_wrapper_get_many(self): - # get the default cache - cache = caches['default'] - - cache.set('a_key', 50) - cache.set('another_key', 60) - - values = cache.get_many(['a_key', 'another_key']) - assert isinstance(values, dict) - assert values['a_key'] == 50 - assert values['another_key'] == 60 - - def test_wrapper_set_many(self): - # get the default cache - cache = caches['default'] - - cache.set_many({'a_key': 50, 'another_key': 60}) - assert cache.get('a_key') == 50 - assert cache.get('another_key') == 60 - - def test_wrapper_delete_many(self): - # get the default cache - cache = caches['default'] - - cache.set('a_key', 50) - cache.set('another_key', 60) - cache.delete_many(['a_key', 'another_key']) - assert cache.get('a_key') is None - assert cache.get('another_key') is None diff --git a/tests/contrib/django/test_connection.py b/tests/contrib/django/test_connection.py deleted file mode 100644 index a18ffc222cc..00000000000 --- a/tests/contrib/django/test_connection.py +++ /dev/null @@ -1,72 +0,0 @@ -import mock -import time - -# 3rd party -from django.contrib.auth.models import User - -from ddtrace.contrib.django.conf import settings -from ddtrace.contrib.django.patch import apply_django_patches, connections - -# testing -from .utils import DjangoTraceTestCase, override_ddtrace_settings - - -class DjangoConnectionTest(DjangoTraceTestCase): - """ - Ensures that database connections are properly traced - """ - def test_connection(self): - # trace a simple query - start = time.time() - users = User.objects.count() - assert users == 0 - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert spans, spans - assert len(spans) == 1 - - span = spans[0] - assert span.name == 'sqlite.query' - assert span.service == 'defaultdb' - assert span.span_type == 'sql' - assert span.get_tag('django.db.vendor') == 'sqlite' - assert span.get_tag('django.db.alias') == 'default' - assert start < span.start < span.start + span.duration < end - - def test_django_db_query_in_resource_not_in_tags(self): - User.objects.count() - spans = self.tracer.writer.pop() - assert spans[0].name == 'sqlite.query' - assert spans[0].resource == 'SELECT COUNT(*) AS "__count" FROM "auth_user"' - assert spans[0].get_tag('sql.query') is None - - @override_ddtrace_settings(INSTRUMENT_DATABASE=False) - def test_connection_disabled(self): - # trace a simple query - users = User.objects.count() - assert users == 0 - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 0 - - def test_should_append_database_prefix(self): - # trace a simple query and check if the prefix is correctly - # loaded from Django settings - settings.DEFAULT_DATABASE_PREFIX = 'my_prefix_db' - User.objects.count() - - traces = self.tracer.writer.pop_traces() - assert len(traces) == 1 - assert len(traces[0]) == 1 - span = traces[0][0] - assert span.service == 'my_prefix_db-defaultdb' - - def test_apply_django_patches_calls_connections_all(self): - with mock.patch.object(connections, 'all') as mock_connections: - apply_django_patches(patch_rest_framework=False) - - assert mock_connections.call_count == 1 - assert mock_connections.mock_calls == [mock.call()] diff --git a/tests/contrib/django/test_django.py b/tests/contrib/django/test_django.py new file mode 100644 index 00000000000..6862e9efb0f --- /dev/null +++ b/tests/contrib/django/test_django.py @@ -0,0 +1,1204 @@ +import django +from django.test import modify_settings +import os +import pytest + +from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY, SAMPLING_PRIORITY_KEY +from ddtrace.ext import http, errors +from ddtrace.ext.priority import USER_KEEP +from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID, HTTP_HEADER_PARENT_ID, HTTP_HEADER_SAMPLING_PRIORITY +from ddtrace.propagation.utils import get_wsgi_header + +from tests.base import BaseTestCase +from tests.opentracer.utils import init_tracer +from ...util import assert_dict_issuperset + +pytestmark = pytest.mark.skipif("TEST_DATADOG_DJANGO_MIGRATION" in os.environ, reason="test only without migration") + + +@pytest.mark.skipif(django.VERSION < (2, 0, 0), reason="") +def test_django_v2XX_request_root_span(client, test_spans): + """ + When making a request to a Django app + We properly create the `django.request` root span + """ + resp = client.get("/") + assert resp.status_code == 200 + assert resp.content == b"Hello, test app." + + spans = test_spans.get_spans() + # Assert the correct number of traces and spans + assert len(spans) == 26 + + # Assert the structure of the root `django.request` span + root = test_spans.get_root_span() + + if django.VERSION >= (2, 2, 0): + resource = "GET ^$" + else: + resource = "GET tests.contrib.django.views.index" + + meta = { + "django.request.class": "django.core.handlers.wsgi.WSGIRequest", + "django.response.class": "django.http.response.HttpResponse", + "django.user.is_authenticated": "False", + "django.view": "tests.contrib.django.views.index", + "http.method": "GET", + "http.status_code": "200", + "http.url": "http://testserver/", + } + if django.VERSION >= (2, 2, 0): + meta["http.route"] = "^$" + + assert http.QUERY_STRING not in root.meta + root.assert_matches( + name="django.request", + service="django", + resource=resource, + parent_id=None, + span_type="http", + error=0, + meta=meta, + ) + + +@pytest.mark.skipif(django.VERSION >= (2, 0, 0), reason="") +def test_v1XX_middleware(client, test_spans): + resp = client.get("/") + assert resp.status_code == 200 + assert resp.content == b"Hello, test app." + + # Assert the correct number of traces and spans + if django.VERSION < (1, 11, 0): + test_spans.assert_span_count(15) + else: + test_spans.assert_span_count(16) + + # Get all the `django.middleware` spans in this trace + middleware_spans = list(test_spans.filter_spans(name="django.middleware")) + if django.VERSION < (1, 11, 0): + assert len(middleware_spans) == 13 + else: + assert len(middleware_spans) == 14 + + root = test_spans.get_root_span() + root.assert_matches(name="django.request") + + # Assert common span structure + for span in middleware_spans: + span.assert_matches( + name="django.middleware", + service="django", + error=0, + span_type=None, + parent_id=root.span_id, # They are all children of the root django.request + ) + + # DEV: Order matters here, we want all `process_request` before `process_view`, before `process_response` + expected_resources = [ + "django.contrib.sessions.middleware.SessionMiddleware.process_request", + "django.middleware.common.CommonMiddleware.process_request", + "django.middleware.csrf.CsrfViewMiddleware.process_request", # Not in < 1.11.0 + "django.contrib.auth.middleware.AuthenticationMiddleware.process_request", + "django.contrib.auth.middleware.SessionAuthenticationMiddleware.process_request", + "django.contrib.messages.middleware.MessageMiddleware.process_request", + "django.middleware.security.SecurityMiddleware.process_request", + "django.middleware.csrf.CsrfViewMiddleware.process_view", + "django.middleware.security.SecurityMiddleware.process_response", + "django.middleware.clickjacking.XFrameOptionsMiddleware.process_response", + "django.contrib.messages.middleware.MessageMiddleware.process_response", + "django.middleware.csrf.CsrfViewMiddleware.process_response", + "django.middleware.common.CommonMiddleware.process_response", + "django.contrib.sessions.middleware.SessionMiddleware.process_response", + ] + if django.VERSION < (1, 11, 0): + expected_resources.remove("django.middleware.csrf.CsrfViewMiddleware.process_request") + middleware_spans = sorted(middleware_spans, key=lambda s: s.start) + span_resources = [s.resource for s in middleware_spans] + assert span_resources == expected_resources + + +""" +Middleware tests +""" + + +@pytest.mark.skipif(django.VERSION < (2, 0, 0), reason="") +def test_v2XX_middleware(client, test_spans): + """ + When making a request to a Django app + We properly create the `django.middleware` spans + """ + resp = client.get("/") + assert resp.status_code == 200 + assert resp.content == b"Hello, test app." + + # Assert the correct number of traces and spans + test_spans.assert_span_count(26) + + # Get all the `django.middleware` spans in this trace + middleware_spans = list(test_spans.filter_spans(name="django.middleware")) + assert len(middleware_spans) == 24 + + # Assert common span structure + for span in middleware_spans: + span.assert_matches( + name="django.middleware", service="django", error=0, span_type=None, + ) + + span_resources = { + "django.contrib.auth.middleware.AuthenticationMiddleware.__call__", + "django.contrib.auth.middleware.AuthenticationMiddleware.process_request", + "django.contrib.messages.middleware.MessageMiddleware.__call__", + "django.contrib.messages.middleware.MessageMiddleware.process_request", + "django.contrib.messages.middleware.MessageMiddleware.process_response", + "django.contrib.sessions.middleware.SessionMiddleware.__call__", + "django.contrib.sessions.middleware.SessionMiddleware.process_request", + "django.contrib.sessions.middleware.SessionMiddleware.process_response", + "django.middleware.clickjacking.XFrameOptionsMiddleware.__call__", + "django.middleware.clickjacking.XFrameOptionsMiddleware.process_response", + "django.middleware.common.CommonMiddleware.__call__", + "django.middleware.common.CommonMiddleware.process_request", + "django.middleware.common.CommonMiddleware.process_response", + "django.middleware.csrf.CsrfViewMiddleware.__call__", + "django.middleware.csrf.CsrfViewMiddleware.process_request", + "django.middleware.csrf.CsrfViewMiddleware.process_response", + "django.middleware.csrf.CsrfViewMiddleware.process_view", + "django.middleware.security.SecurityMiddleware.__call__", + "django.middleware.security.SecurityMiddleware.process_request", + "django.middleware.security.SecurityMiddleware.process_response", + "tests.contrib.django.middleware.ClsMiddleware.__call__", + "tests.contrib.django.middleware.EverythingMiddleware", + "tests.contrib.django.middleware.EverythingMiddleware.__call__", + "tests.contrib.django.middleware.EverythingMiddleware.process_view", + } + assert set([s.resource for s in middleware_spans]) == span_resources + + # Get middleware spans in reverse order of start time + middleware_spans = sorted(middleware_spans, key=lambda s: s.start, reverse=True) + + # Assert the first middleware span's parent is the root span (django.request) + root_span = test_spans.get_root_span() + assert root_span.name == "django.request" + first_middleware = middleware_spans[-1] + assert first_middleware.parent_id == root_span.span_id + + +def test_django_request_not_found(client, test_spans): + """ + When making a request to a Django app + When the endpoint doesn't exist + We create a 404 span + """ + resp = client.get("/unknown/endpoint") + assert resp.status_code == 404 + + if django.VERSION >= (3, 0, 0): + content = ( + b'\n\n\n\n Not Found\n' + b"\n\n

Not Found

The requested resource was not found " + b"on this server.

\n\n\n" + ) + elif django.VERSION >= (1, 11, 0): + content = b"

Not Found

The requested resource was not found on this server.

" + else: + content = b"

Not Found

The requested URL /unknown/endpoint was not found on this server.

" + assert resp.content == content + + # Assert the correct number of traces and spans + if django.VERSION >= (2, 0, 0): + span_count = 27 + elif django.VERSION >= (1, 11, 0): + span_count = 18 + else: + span_count = 16 + test_spans.assert_span_count(span_count) + + # Assert the structure of the root `django.request` span + root = test_spans.get_root_span() + root.assert_matches( + name="django.request", + service="django", + resource="GET 404", + parent_id=None, + span_type="http", + error=0, + meta={ + "django.request.class": "django.core.handlers.wsgi.WSGIRequest", + "django.response.class": "django.http.response.HttpResponseNotFound", + "http.method": "GET", + "http.status_code": "404", + "http.url": "http://testserver/unknown/endpoint", + }, + ) + + # Assert template render + render_spans = list(test_spans.filter_spans(name="django.template.render")) + assert len(render_spans) == 1 + + render_span = render_spans[0] + render_span.assert_matches( + name="django.template.render", + resource="django.template.base.Template.render", + meta={"django.template.engine.class": "django.template.engine.Engine",}, + ) + + +def test_middleware_trace_error_500(client, test_spans): + # ensures exceptions generated by views are traced + with modify_settings( + **( + dict(MIDDLEWARE={"append": "tests.contrib.django.middleware.CatchExceptionMiddleware"}) + if django.VERSION >= (2, 0, 0) + else dict(MIDDLEWARE_CLASSES={"append": "tests.contrib.django.middleware.CatchExceptionMiddleware"}) + ) + ): + assert client.get("/error-500/").status_code == 500 + + error_spans = list(test_spans.filter_spans(error=1)) + # There should be 3 spans flagged as errors + # 1. The view span which wraps the original error + # 2. The root span which should just be flagged as an error (no exception info) + # 3. The middleware span that catches the exception and converts it to a 500 + assert len(error_spans) == 3 + + # Test the root span + span = test_spans.get_root_span() + assert span.error == 1 + assert span.get_tag("http.status_code") == "500" + assert span.get_tag(http.URL) == "http://testserver/error-500/" + if django.VERSION >= (2, 2, 0): + assert span.resource == "GET ^error-500/$" + else: + assert span.resource == "GET tests.contrib.django.views.error_500" + assert span.get_tag(errors.ERROR_MSG) is None + assert span.get_tag(errors.ERROR_TYPE) is None + assert span.get_tag(errors.ERROR_STACK) is None + + # Test the view span (where the exception is generated) + view_span = list(test_spans.filter_spans(name="django.view")) + assert len(view_span) == 1 + view_span = view_span[0] + assert view_span.error == 1 + # Make sure the message is somewhere in the stack trace + assert "Error 500" in view_span.get_tag(errors.ERROR_STACK) + + # Test the catch exception middleware + res = "tests.contrib.django.middleware.CatchExceptionMiddleware.process_exception" + mw_span = list(test_spans.filter_spans(resource=res))[0] + assert mw_span.error == 1 + # Make sure the message is somewhere in the stack trace + assert "Error 500" in view_span.get_tag(errors.ERROR_STACK) + assert mw_span.get_tag(errors.ERROR_MSG) is not None + assert mw_span.get_tag(errors.ERROR_TYPE) is not None + assert mw_span.get_tag(errors.ERROR_STACK) is not None + + +def test_middleware_handled_view_exception_success(client, test_spans): + """ + When an exception is raised in a view and then handled + Only the culprit span contains error properties + """ + with modify_settings( + **( + dict(MIDDLEWARE={"append": "tests.contrib.django.middleware.HandleErrorMiddlewareSuccess"}) + if django.VERSION >= (2, 0, 0) + else dict(MIDDLEWARE_CLASSES={"append": "tests.contrib.django.middleware.HandleErrorMiddlewareSuccess"}) + ) + ): + assert client.get("/error-500/").status_code == 200 + + error_spans = list(test_spans.filter_spans(error=1)) + # There should be 1 span flagged as erroneous: + # - The view span which wraps the original error + assert len(error_spans) == 1 + + # Test the root span + root_span = test_spans.get_root_span() + assert root_span.error == 0 + assert root_span.get_tag(errors.ERROR_STACK) is None + assert root_span.get_tag(errors.ERROR_MSG) is None + assert root_span.get_tag(errors.ERROR_TYPE) is None + + # Test the view span (where the exception is generated) + view_span = list(test_spans.filter_spans(name="django.view")) + assert len(view_span) == 1 + view_span = view_span[0] + assert view_span.error == 1 + # Make sure the message is somewhere in the stack trace + assert "Error 500" in view_span.get_tag("error.stack") + + +""" +View tests +""" + + +def test_request_view(client, test_spans): + """ + When making a request to a Django app + A `django.view` span is produced + """ + resp = client.get("/") + assert resp.status_code == 200 + assert resp.content == b"Hello, test app." + + view_spans = list(test_spans.filter_spans(name="django.view")) + assert len(view_spans) == 1 + + # Assert span properties + view_span = view_spans[0] + view_span.assert_matches( + name="django.view", service="django", resource="tests.contrib.django.views.index", error=0, + ) + + +def test_lambda_based_view(client, test_spans): + # ensures that the internals are properly traced when using a function view + assert client.get("/lambda-view/").status_code == 200 + + span = test_spans.get_root_span() + assert span.get_tag("http.status_code") == "200" + assert span.get_tag(http.URL) == "http://testserver/lambda-view/" + if django.VERSION >= (2, 2, 0): + assert span.resource == "GET ^lambda-view/$" + else: + assert span.resource == "GET tests.contrib.django.views." + + +def test_middleware_trace_function_based_view(client, test_spans): + # ensures that the internals are properly traced when using a function views + assert client.get("/fn-view/").status_code == 200 + + span = test_spans.get_root_span() + assert span.get_tag("http.status_code") == "200" + assert span.get_tag(http.URL) == "http://testserver/fn-view/" + if django.VERSION >= (2, 2, 0): + assert span.resource == "GET ^fn-view/$" + else: + assert span.resource == "GET tests.contrib.django.views.function_view" + + +def test_middleware_trace_callable_view(client, test_spans): + # ensures that the internals are properly traced when using callable views + assert client.get("/feed-view/").status_code == 200 + + span = test_spans.get_root_span() + assert span.get_tag("http.status_code") == "200" + assert span.get_tag(http.URL) == "http://testserver/feed-view/" + if django.VERSION >= (2, 2, 0): + assert span.resource == "GET ^feed-view/$" + else: + assert span.resource == "GET tests.contrib.django.views.FeedView" + + +def test_middleware_trace_errors(client, test_spans): + # ensures that the internals are properly traced + assert client.get("/fail-view/").status_code == 403 + + span = test_spans.get_root_span() + assert span.get_tag("http.status_code") == "403" + assert span.get_tag(http.URL) == "http://testserver/fail-view/" + if django.VERSION >= (2, 2, 0): + assert span.resource == "GET ^fail-view/$" + else: + assert span.resource == "GET tests.contrib.django.views.ForbiddenView" + + +def test_middleware_trace_partial_based_view(client, test_spans): + # ensures that the internals are properly traced when using a function views + assert client.get("/partial-view/").status_code == 200 + + span = test_spans.get_root_span() + assert span.get_tag("http.status_code") == "200" + assert span.get_tag(http.URL) == "http://testserver/partial-view/" + if django.VERSION >= (2, 2, 0): + assert span.resource == "GET ^partial-view/$" + else: + assert span.resource == "GET partial" + + +def test_simple_view_get(client, test_spans): + # The `get` method of a view should be traced + assert client.get("/simple/").status_code == 200 + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.dispatch"))) == 1 + spans = list(test_spans.filter_spans(name="django.view.get")) + assert len(spans) == 1 + span = spans[0] + span.assert_matches( + resource="tests.contrib.django.views.BasicView.get", error=0, + ) + + +def test_simple_view_post(client, test_spans): + # The `post` method of a view should be traced + assert client.post("/simple/").status_code == 200 + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.dispatch"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.post"))) == 1 + spans = list(test_spans.filter_spans(name="django.view.post")) + assert len(spans) == 1 + span = spans[0] + span.assert_matches( + resource="tests.contrib.django.views.BasicView.post", error=0, + ) + + +def test_simple_view_delete(client, test_spans): + # The `delete` method of a view should be traced + assert client.delete("/simple/").status_code == 200 + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.dispatch"))) == 1 + spans = list(test_spans.filter_spans(name="django.view.delete")) + assert len(spans) == 1 + span = spans[0] + span.assert_matches( + resource="tests.contrib.django.views.BasicView.delete", error=0, + ) + + +def test_simple_view_options(client, test_spans): + # The `options` method of a view should be traced + assert client.options("/simple/").status_code == 200 + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.dispatch"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.options"))) == 1 + spans = list(test_spans.filter_spans(name="django.view.options")) + assert len(spans) == 1 + span = spans[0] + span.assert_matches( + resource="tests.contrib.django.views.BasicView.options", error=0, + ) + + +def test_simple_view_head(client, test_spans): + # The `head` method of a view should be traced + assert client.head("/simple/").status_code == 200 + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + assert len(list(test_spans.filter_spans(name="django.view.dispatch"))) == 1 + spans = list(test_spans.filter_spans(name="django.view.head")) + assert len(spans) == 1 + span = spans[0] + span.assert_matches( + resource="tests.contrib.django.views.BasicView.head", error=0, + ) + + +""" +Database tests +""" + + +@pytest.mark.django_db +def test_connection(client, test_spans): + """ + When database queries are made from Django + The queries are traced + """ + from django.contrib.auth.models import User + + users = User.objects.count() + assert users == 0 + + test_spans.assert_span_count(1) + spans = test_spans.get_spans() + + span = spans[0] + assert span.name == "sqlite.query" + assert span.service == "defaultdb" + assert span.span_type == "sql" + assert span.get_tag("django.db.vendor") == "sqlite" + assert span.get_tag("django.db.alias") == "default" + + +""" +Caching tests +""" + + +def test_cache_get(test_spans): + # get the default cache + cache = django.core.cache.caches["default"] + + cache.get("missing_key") + + spans = test_spans.get_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.service == "django" + assert span.resource == "django.core.cache.backends.locmem.get" + assert span.name == "django.cache" + assert span.span_type == "cache" + assert span.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "missing_key", + } + + assert_dict_issuperset(span.meta, expected_meta) + + +def test_cache_set(test_spans): + # get the default cache + cache = django.core.cache.caches["default"] + + # (trace) the cache miss + cache.set("a_new_key", 50) + + spans = test_spans.get_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.service == "django" + assert span.resource == "django.core.cache.backends.locmem.set" + assert span.name == "django.cache" + assert span.span_type == "cache" + assert span.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "a_new_key", + } + + assert_dict_issuperset(span.meta, expected_meta) + + +def test_cache_delete(test_spans): + # get the default cache + cache = django.core.cache.caches["default"] + + cache.delete("an_existing_key") + + spans = test_spans.get_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.service == "django" + assert span.resource == "django.core.cache.backends.locmem.delete" + assert span.name == "django.cache" + assert span.span_type == "cache" + assert span.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "an_existing_key", + } + + assert_dict_issuperset(span.meta, expected_meta) + + +@pytest.mark.skipif(django.VERSION >= (2, 1, 0), reason="") +def test_cache_incr_1XX(test_spans): + # get the default cache, set the value and reset the spans + cache = django.core.cache.caches["default"] + cache.set("value", 0) + test_spans.tracer.writer.spans = [] + + cache.incr("value") + + spans = test_spans.get_spans() + assert len(spans) == 2 + + span_incr = spans[0] + span_get = spans[1] + + # LocMemCache doesn't provide an atomic operation + assert span_get.service == "django" + assert span_get.resource == "django.core.cache.backends.locmem.get" + assert span_get.name == "django.cache" + assert span_get.span_type == "cache" + assert span_get.error == 0 + assert span_incr.service == "django" + assert span_incr.resource == "django.core.cache.backends.locmem.incr" + assert span_incr.name == "django.cache" + assert span_incr.span_type == "cache" + assert span_incr.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "value", + } + + assert_dict_issuperset(span_get.meta, expected_meta) + assert_dict_issuperset(span_incr.meta, expected_meta) + + +@pytest.mark.skipif(django.VERSION < (2, 1, 0), reason="") +def test_cache_incr_2XX(test_spans): + # get the default cache, set the value and reset the spans + cache = django.core.cache.caches["default"] + cache.set("value", 0) + test_spans.tracer.writer.spans = [] + + cache.incr("value") + + spans = test_spans.get_spans() + assert len(spans) == 1 + + span_incr = spans[0] + + # LocMemCache doesn't provide an atomic operation + assert span_incr.service == "django" + assert span_incr.resource == "django.core.cache.backends.locmem.incr" + assert span_incr.name == "django.cache" + assert span_incr.span_type == "cache" + assert span_incr.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "value", + } + + assert_dict_issuperset(span_incr.meta, expected_meta) + + +@pytest.mark.skipif(django.VERSION >= (2, 1, 0), reason="") +def test_cache_decr_1XX(test_spans): + # get the default cache, set the value and reset the spans + cache = django.core.cache.caches["default"] + cache.set("value", 0) + test_spans.tracer.writer.spans = [] + + cache.decr("value") + + spans = test_spans.get_spans() + assert len(spans) == 3 + + span_decr = spans[0] + span_incr = spans[1] + span_get = spans[2] + + # LocMemCache doesn't provide an atomic operation + assert span_get.service == "django" + assert span_get.resource == "django.core.cache.backends.locmem.get" + assert span_get.name == "django.cache" + assert span_get.span_type == "cache" + assert span_get.error == 0 + assert span_incr.service == "django" + assert span_incr.resource == "django.core.cache.backends.locmem.incr" + assert span_incr.name == "django.cache" + assert span_incr.span_type == "cache" + assert span_incr.error == 0 + assert span_decr.service == "django" + assert span_decr.resource == "django.core.cache.backends.base.decr" + assert span_decr.name == "django.cache" + assert span_decr.span_type == "cache" + assert span_decr.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "value", + } + + assert_dict_issuperset(span_get.meta, expected_meta) + assert_dict_issuperset(span_incr.meta, expected_meta) + assert_dict_issuperset(span_decr.meta, expected_meta) + + +@pytest.mark.skipif(django.VERSION < (2, 1, 0), reason="") +def test_cache_decr_2XX(test_spans): + # get the default cache, set the value and reset the spans + cache = django.core.cache.caches["default"] + cache.set("value", 0) + test_spans.tracer.writer.spans = [] + + cache.decr("value") + + spans = test_spans.get_spans() + assert len(spans) == 2 + + span_decr = spans[0] + span_incr = spans[1] + + # LocMemCache doesn't provide an atomic operation + assert span_incr.service == "django" + assert span_incr.resource == "django.core.cache.backends.locmem.incr" + assert span_incr.name == "django.cache" + assert span_incr.span_type == "cache" + assert span_incr.error == 0 + assert span_decr.service == "django" + assert span_decr.resource == "django.core.cache.backends.base.decr" + assert span_decr.name == "django.cache" + assert span_decr.span_type == "cache" + assert span_decr.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "value", + } + + assert_dict_issuperset(span_incr.meta, expected_meta) + assert_dict_issuperset(span_decr.meta, expected_meta) + + +def test_cache_get_many(test_spans): + # get the default cache + cache = django.core.cache.caches["default"] + + cache.get_many(["missing_key", "another_key"]) + + spans = test_spans.get_spans() + assert len(spans) == 3 + + span_get_many = spans[0] + span_get_first = spans[1] + span_get_second = spans[2] + + # LocMemCache doesn't provide an atomic operation + assert span_get_first.service == "django" + assert span_get_first.resource == "django.core.cache.backends.locmem.get" + assert span_get_first.name == "django.cache" + assert span_get_first.span_type == "cache" + assert span_get_first.error == 0 + assert span_get_second.service == "django" + assert span_get_second.resource == "django.core.cache.backends.locmem.get" + assert span_get_second.name == "django.cache" + assert span_get_second.span_type == "cache" + assert span_get_second.error == 0 + assert span_get_many.service == "django" + assert span_get_many.resource == "django.core.cache.backends.base.get_many" + assert span_get_many.name == "django.cache" + assert span_get_many.span_type == "cache" + assert span_get_many.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": str(["missing_key", "another_key"]), + } + + assert_dict_issuperset(span_get_many.meta, expected_meta) + + +def test_cache_set_many(test_spans): + # get the default cache + cache = django.core.cache.caches["default"] + + cache.set_many({"first_key": 1, "second_key": 2}) + + spans = test_spans.get_spans() + assert len(spans) == 3 + + span_set_many = spans[0] + span_set_first = spans[1] + span_set_second = spans[2] + + # LocMemCache doesn't provide an atomic operation + assert span_set_first.service == "django" + assert span_set_first.resource == "django.core.cache.backends.locmem.set" + assert span_set_first.name == "django.cache" + assert span_set_first.span_type == "cache" + assert span_set_first.error == 0 + assert span_set_second.service == "django" + assert span_set_second.resource == "django.core.cache.backends.locmem.set" + assert span_set_second.name == "django.cache" + assert span_set_second.span_type == "cache" + assert span_set_second.error == 0 + assert span_set_many.service == "django" + assert span_set_many.resource == "django.core.cache.backends.base.set_many" + assert span_set_many.name == "django.cache" + assert span_set_many.span_type == "cache" + assert span_set_many.error == 0 + + assert span_set_many.meta["django.cache.backend"] == "django.core.cache.backends.locmem.LocMemCache" + assert "first_key" in span_set_many.meta["django.cache.key"] + assert "second_key" in span_set_many.meta["django.cache.key"] + + +def test_cache_delete_many(test_spans): + # get the default cache + cache = django.core.cache.caches["default"] + + cache.delete_many(["missing_key", "another_key"]) + + spans = test_spans.get_spans() + assert len(spans) == 3 + + span_delete_many = spans[0] + span_delete_first = spans[1] + span_delete_second = spans[2] + + # LocMemCache doesn't provide an atomic operation + assert span_delete_first.service == "django" + assert span_delete_first.resource == "django.core.cache.backends.locmem.delete" + assert span_delete_first.name == "django.cache" + assert span_delete_first.span_type == "cache" + assert span_delete_first.error == 0 + assert span_delete_second.service == "django" + assert span_delete_second.resource == "django.core.cache.backends.locmem.delete" + assert span_delete_second.name == "django.cache" + assert span_delete_second.span_type == "cache" + assert span_delete_second.error == 0 + assert span_delete_many.service == "django" + assert span_delete_many.resource == "django.core.cache.backends.base.delete_many" + assert span_delete_many.name == "django.cache" + assert span_delete_many.span_type == "cache" + assert span_delete_many.error == 0 + + assert span_delete_many.meta["django.cache.backend"] == "django.core.cache.backends.locmem.LocMemCache" + assert "missing_key" in span_delete_many.meta["django.cache.key"] + assert "another_key" in span_delete_many.meta["django.cache.key"] + + +@pytest.mark.django_db +def test_cached_view(client, test_spans): + # make the first request so that the view is cached + response = client.get("/cached-users/") + assert response.status_code == 200 + + # check the first call for a non-cached view + spans = list(test_spans.filter_spans(name="django.cache")) + assert len(spans) == 3 + # the cache miss + assert spans[0].resource == "django.core.cache.backends.locmem.get" + # store the result in the cache + assert spans[1].resource == "django.core.cache.backends.locmem.set" + assert spans[2].resource == "django.core.cache.backends.locmem.set" + + # check if the cache hit is traced + response = client.get("/cached-users/") + assert response.status_code == 200 + spans = list(test_spans.filter_spans(name="django.cache")) + # There should be two more spans now + assert len(spans) == 5 + + span_header = spans[3] + span_view = spans[4] + assert span_view.service == "django" + assert span_view.resource == "django.core.cache.backends.locmem.get" + assert span_view.name == "django.cache" + assert span_view.span_type == "cache" + assert span_view.error == 0 + assert span_header.service == "django" + assert span_header.resource == "django.core.cache.backends.locmem.get" + assert span_header.name == "django.cache" + assert span_header.span_type == "cache" + assert span_header.error == 0 + + expected_meta_view = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": ( + "views.decorators.cache.cache_page.." + "GET.03cdc1cc4aab71b038a6764e5fcabb82.d41d8cd98f00b204e9800998ecf8427e.en-us" + ), + } + + expected_meta_header = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "views.decorators.cache.cache_header..03cdc1cc4aab71b038a6764e5fcabb82.en-us", + } + + assert span_view.meta == expected_meta_view + assert span_header.meta == expected_meta_header + + +@pytest.mark.django_db +def test_cached_template(client, test_spans): + # make the first request so that the view is cached + response = client.get("/cached-template/") + assert response.status_code == 200 + + # check the first call for a non-cached view + spans = list(test_spans.filter_spans(name="django.cache")) + assert len(spans) == 2 + # the cache miss + assert spans[0].resource == "django.core.cache.backends.locmem.get" + # store the result in the cache + assert spans[1].resource == "django.core.cache.backends.locmem.set" + + # check if the cache hit is traced + response = client.get("/cached-template/") + assert response.status_code == 200 + spans = list(test_spans.filter_spans(name="django.cache")) + # Should have 1 more span + assert len(spans) == 3 + + span_template_cache = spans[2] + assert span_template_cache.service == "django" + assert span_template_cache.resource == "django.core.cache.backends.locmem.get" + assert span_template_cache.name == "django.cache" + assert span_template_cache.span_type == "cache" + assert span_template_cache.error == 0 + + expected_meta = { + "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache", + "django.cache.key": "template.cache.users_list.d41d8cd98f00b204e9800998ecf8427e", + } + + assert span_template_cache.meta == expected_meta + + +""" +Configuration tests +""" + + +def test_service_can_be_overridden(client, test_spans): + with BaseTestCase.override_config("django", dict(service_name="test-service")): + response = client.get("/") + assert response.status_code == 200 + + spans = test_spans.get_spans() + assert len(spans) > 0 + + span = spans[0] + assert span.service == "test-service" + + +@pytest.mark.django_db +def test_database_service_prefix_can_be_overridden(test_spans): + with BaseTestCase.override_config("django", dict(database_service_name_prefix="my-")): + from django.contrib.auth.models import User + + User.objects.count() + + spans = test_spans.get_spans() + assert len(spans) > 0 + + span = spans[0] + assert span.service == "my-defaultdb" + + +def test_cache_service_can_be_overridden(test_spans): + cache = django.core.cache.caches["default"] + + with BaseTestCase.override_config("django", dict(cache_service_name="test-cache-service")): + cache.get("missing_key") + + spans = test_spans.get_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.service == "test-cache-service" + + +def test_django_request_distributed(client, test_spans): + """ + When making a request to a Django app + With distributed tracing headers + The django.request span properly inherits from the distributed trace + """ + headers = { + get_wsgi_header(HTTP_HEADER_TRACE_ID): "12345", + get_wsgi_header(HTTP_HEADER_PARENT_ID): "78910", + get_wsgi_header(HTTP_HEADER_SAMPLING_PRIORITY): USER_KEEP, + } + resp = client.get("/", **headers) + assert resp.status_code == 200 + assert resp.content == b"Hello, test app." + + # Assert that the trace properly inherits from the distributed headers + # DEV: Do not use `test_spans.get_root_span()` since that expects `parent_id is None` + root = test_spans.find_span(name="django.request") + root.assert_matches( + name="django.request", trace_id=12345, parent_id=78910, metrics={SAMPLING_PRIORITY_KEY: USER_KEEP,}, + ) + + +def test_django_request_distributed_disabled(client, test_spans): + """ + When making a request to a Django app + With distributed tracing headers + When distributed tracing is disabled + The django.request span doesn't inherit from the distributed trace + """ + headers = { + get_wsgi_header(HTTP_HEADER_TRACE_ID): "12345", + get_wsgi_header(HTTP_HEADER_PARENT_ID): "78910", + get_wsgi_header(HTTP_HEADER_SAMPLING_PRIORITY): USER_KEEP, + } + with BaseTestCase.override_config("django", dict(distributed_tracing_enabled=False)): + resp = client.get("/", **headers) + assert resp.status_code == 200 + assert resp.content == b"Hello, test app." + + # Assert the trace doesn't inherit from the distributed trace + root = test_spans.find_span(name="django.request") + assert root.trace_id != 12345 + assert root.parent_id is None + + +@pytest.mark.django_db +def test_analytics_global_off_integration_default(client, test_spans): + """ + When making a request + When an integration trace search is not set and sample rate is set and globally trace search is disabled + We expect the root span to not include tag + """ + with BaseTestCase.override_global_config(dict(analytics_enabled=False)): + assert client.get("/users/").status_code == 200 + + req_span = test_spans.get_root_span() + assert req_span.name == "django.request" + assert req_span.get_metric(ANALYTICS_SAMPLE_RATE_KEY) is None + + +@pytest.mark.django_db +def test_analytics_global_on_integration_default(client, test_spans): + """ + When making a request + When an integration trace search is not event sample rate is not set and globally trace search is enabled + We expect the root span to have the appropriate tag + """ + with BaseTestCase.override_global_config(dict(analytics_enabled=True)): + assert client.get("/users/").status_code == 200 + + req_span = test_spans.get_root_span() + assert req_span.name == "django.request" + assert req_span.get_metric(ANALYTICS_SAMPLE_RATE_KEY) == 1.0 + + +@pytest.mark.django_db +def test_analytics_global_off_integration_on(client, test_spans): + """ + When making a request + When an integration trace search is enabled and sample rate is set and globally trace search is disabled + We expect the root span to have the appropriate tag + """ + with BaseTestCase.override_global_config(dict(analytics_enabled=False)): + with BaseTestCase.override_config("django", dict(analytics_enabled=True, analytics_sample_rate=0.5)): + assert client.get("/users/").status_code == 200 + + sp_request = test_spans.get_root_span() + assert sp_request.name == "django.request" + assert sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY) == 0.5 + + +@pytest.mark.django_db +def test_analytics_global_off_integration_on_and_none(client, test_spans): + """ + When making a request + When an integration trace search is enabled + Sample rate is set to None + Globally trace search is disabled + We expect the root span to have the appropriate tag + """ + with BaseTestCase.override_global_config(dict(analytics_enabled=False)): + with BaseTestCase.override_config("django", dict(analytics_enabled=False, analytics_sample_rate=1.0)): + assert client.get("/users/").status_code == 200 + + sp_request = test_spans.get_root_span() + assert sp_request.name == "django.request" + assert sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY) is None + + +def test_trace_query_string_integration_enabled(client, test_spans): + with BaseTestCase.override_http_config("django", dict(trace_query_string=True)): + assert client.get("/?key1=value1&key2=value2").status_code == 200 + + sp_request = test_spans.get_root_span() + assert sp_request.name == "django.request" + assert sp_request.get_tag(http.QUERY_STRING) == "key1=value1&key2=value2" + + +def test_disabled_caches(client, test_spans): + with BaseTestCase.override_config("django", dict(instrument_caches=False)): + cache = django.core.cache.caches["default"] + cache.get("missing_key") + + spans = test_spans.get_spans() + assert len(spans) == 0 + + +""" +Template tests +""" + + +def test_template(test_spans): + # prepare a base template using the default engine + template = django.template.Template("Hello {{name}}!") + ctx = django.template.Context({"name": "Django"}) + + assert template.render(ctx) == "Hello Django!" + + spans = test_spans.get_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.span_type == "template" + assert span.name == "django.template.render" + + template.name = "my-template" + assert template.render(ctx) == "Hello Django!" + spans = test_spans.get_spans() + assert len(spans) == 2 + + span = spans[1] + assert span.get_tag("django.template.name") == "my-template" + + +""" +OpenTracing tests +""" + + +@pytest.mark.django_db +def test_middleware_trace_request_ot(client, test_spans, tracer): + """OpenTracing version of test_middleware_trace_request.""" + ot_tracer = init_tracer("my_svc", tracer) + + # ensures that the internals are properly traced + with ot_tracer.start_active_span("ot_span"): + assert client.get("/users/").status_code == 200 + + # check for spans + spans = test_spans.get_spans() + ot_span = spans[0] + sp_request = spans[1] + + # confirm parenting + assert ot_span.parent_id is None + assert sp_request.parent_id == ot_span.span_id + + assert ot_span.resource == "ot_span" + assert ot_span.service == "my_svc" + + assert sp_request.get_tag("http.status_code") == "200" + assert sp_request.get_tag(http.URL) == "http://testserver/users/" + assert sp_request.get_tag("django.user.is_authenticated") == "False" + assert sp_request.get_tag("http.method") == "GET" + + +""" +urlpatterns tests +There are a variety of ways a user can point to their views. +""" + + +@pytest.mark.skipif(django.VERSION < (2, 0, 0), reason="path only exists in >=2.0.0") +def test_urlpatterns_path(client, test_spans): + """ + When a view is specified using `django.urls.path` + The view is traced + """ + assert client.get("/path/").status_code == 200 + + # Ensure the view was traced + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + + +@pytest.mark.skipif(django.VERSION < (2, 0, 0), reason="include only exists in >=2.0.0") +def test_urlpatterns_include(client, test_spans): + """ + When a view is specified using `django.urls.include` + The view is traced + """ + assert client.get("/include/test/").status_code == 200 + + # Ensure the view was traced + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 + + +@pytest.mark.skipif(django.VERSION < (2, 0, 0), reason="repath only exists in >=2.0.0") +def test_urlpatterns_repath(client, test_spans): + """ + When a view is specified using `django.urls.repath` + The view is traced + """ + assert client.get("/re-path123/").status_code == 200 + + # Ensure the view was traced + assert len(list(test_spans.filter_spans(name="django.view"))) == 1 diff --git a/tests/contrib/django/test_django_migration.py b/tests/contrib/django/test_django_migration.py new file mode 100644 index 00000000000..9eb040430ee --- /dev/null +++ b/tests/contrib/django/test_django_migration.py @@ -0,0 +1,43 @@ +import django +import os +import pytest + +from ddtrace import config, Pin +from ddtrace.contrib.django.conf import configure_from_settings + +from tests.base import BaseTestCase + + +pytestmark = pytest.mark.skipif( + "TEST_DATADOG_DJANGO_MIGRATION" not in os.environ, reason="test only relevant for migration" +) + +""" +migration tests +""" + + +def test_configure_from_settings(tracer): + pin = Pin.get_from(django) + + with BaseTestCase.override_config("django", dict()): + assert "ddtrace.contrib.django" in django.conf.settings.INSTALLED_APPS + assert hasattr(django.conf.settings, "DATADOG_TRACE") + + configure_from_settings(pin, config.django, django.conf.settings.DATADOG_TRACE) + + assert config.django.service_name == "django-test" + assert config.django.cache_service_name == "cache-test" + assert config.django.database_service_name_prefix == "db-test-" + assert config.django.distributed_tracing_enabled is True + assert config.django.instrument_databases is True + assert config.django.instrument_caches is True + assert config.django.analytics_enabled is True + assert config.django.analytics_sample_rate is True + # TODO: uncomment when figured out why setting this is not working + # assert config.django.trace_query_string is True + + assert pin.tracer.enabled is True + assert pin.tracer.tags["env"] == "env-test" + assert pin.tracer.writer.api.hostname == "host-test" + assert pin.tracer.writer.api.port == 1234 diff --git a/tests/contrib/django/test_django_patch.py b/tests/contrib/django/test_django_patch.py new file mode 100644 index 00000000000..26a96beef8e --- /dev/null +++ b/tests/contrib/django/test_django_patch.py @@ -0,0 +1,45 @@ +import os +import pytest + +from ddtrace.contrib.django import patch +from tests.contrib.patch import PatchTestCase + + +pytestmark = pytest.mark.skipif("TEST_DATADOG_DJANGO_MIGRATION" in os.environ, reason="test only without migration") + + +class TestDjangoPatch(PatchTestCase.Base): + __integration_name__ = "django" + __module_name__ = "django" + __patch_func__ = patch + __unpatch_func__ = None + + def assert_module_patched(self, django): + self.assert_wrapped(django.apps.registry.Apps.populate) + self.assert_wrapped(django.core.handlers.base.BaseHandler.load_middleware) + self.assert_wrapped(django.core.handlers.base.BaseHandler.get_response) + self.assert_wrapped(django.template.base.Template.render) + if django.VERSION >= (2, 0, 0): + self.assert_wrapped(django.urls.path) + self.assert_wrapped(django.urls.re_path) + self.assert_wrapped(django.views.generic.base.View.as_view) + + def assert_not_module_patched(self, django): + self.assert_not_wrapped(django.apps.registry.Apps.populate) + self.assert_not_wrapped(django.core.handlers.base.BaseHandler.load_middleware) + self.assert_not_wrapped(django.core.handlers.base.BaseHandler.get_response) + self.assert_not_wrapped(django.template.base.Template.render) + if django.VERSION >= (2, 0, 0): + self.assert_not_wrapped(django.urls.path) + self.assert_not_wrapped(django.urls.re_path) + self.assert_not_wrapped(django.views.generic.base.View.as_view) + + def assert_not_module_double_patched(self, django): + self.assert_not_double_wrapped(django.apps.registry.Apps.populate) + self.assert_not_double_wrapped(django.core.handlers.base.BaseHandler.load_middleware) + self.assert_not_double_wrapped(django.core.handlers.base.BaseHandler.get_response) + self.assert_not_double_wrapped(django.template.base.Template.render) + if django.VERSION >= (2, 0, 0): + self.assert_not_double_wrapped(django.urls.path) + self.assert_not_double_wrapped(django.urls.re_path) + self.assert_not_double_wrapped(django.views.generic.base.View.as_view) diff --git a/tests/contrib/django/test_instrumentation.py b/tests/contrib/django/test_instrumentation.py deleted file mode 100644 index a8960efcb50..00000000000 --- a/tests/contrib/django/test_instrumentation.py +++ /dev/null @@ -1,35 +0,0 @@ -# project -from ddtrace.contrib.django.conf import DatadogSettings - -# testing -from .utils import DjangoTraceTestCase - - -class DjangoInstrumentationTest(DjangoTraceTestCase): - """ - Ensures that Django is correctly configured according to - users settings - """ - def test_tracer_flags(self): - assert self.tracer.enabled - assert self.tracer.writer.api.hostname == 'localhost' - assert self.tracer.writer.api.port == 8126 - assert self.tracer.tags == {'env': 'test'} - - def test_environment_vars(self): - # Django defaults can be overridden by env vars, ensuring that - # environment strings are properly converted - with self.override_env(dict( - DATADOG_TRACE_AGENT_HOSTNAME='agent.consul.local', - DATADOG_TRACE_AGENT_PORT='58126' - )): - settings = DatadogSettings() - assert settings.AGENT_HOSTNAME == 'agent.consul.local' - assert settings.AGENT_PORT == 58126 - - def test_environment_var_wrong_port(self): - # ensures that a wrong Agent Port doesn't crash the system - # and defaults to 8126 - with self.override_env(dict(DATADOG_TRACE_AGENT_PORT='something')): - settings = DatadogSettings() - assert settings.AGENT_PORT == 8126 diff --git a/tests/contrib/django/test_middleware.py b/tests/contrib/django/test_middleware.py deleted file mode 100644 index fb4271a0779..00000000000 --- a/tests/contrib/django/test_middleware.py +++ /dev/null @@ -1,460 +0,0 @@ -# 3rd party -from django.test import modify_settings -from django.db import connections - -# project -from ddtrace import config -from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY, SAMPLING_PRIORITY_KEY -from ddtrace.contrib.django.db import unpatch_conn -from ddtrace.ext import errors, http - -# testing -from tests.opentracer.utils import init_tracer -from .compat import reverse -from .utils import DjangoTraceTestCase, override_ddtrace_settings -from ...utils import assert_span_http_status_code - - -class DjangoMiddlewareTest(DjangoTraceTestCase): - """ - Ensures that the middleware traces all Django internals - """ - def test_middleware_trace_request(self, query_string=''): - # ensures that the internals are properly traced - url = reverse('users-list') - if query_string: - fqs = '?' + query_string - else: - fqs = '' - response = self.client.get(url + fqs) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - sp_template = spans[1] - sp_database = spans[2] - assert sp_database.get_tag('django.db.vendor') == 'sqlite' - assert sp_template.get_tag('django.template_name') == 'users_list.html' - assert_span_http_status_code(sp_request, 200) - assert sp_request.get_tag(http.URL) == 'http://testserver/users/' - assert sp_request.get_tag('django.user.is_authenticated') == 'False' - assert sp_request.get_tag('http.method') == 'GET' - assert sp_request.span_type == 'web' - assert sp_request.resource == 'tests.contrib.django.app.views.UserList' - if config.django.trace_query_string: - assert sp_request.get_tag(http.QUERY_STRING) == query_string - else: - assert http.QUERY_STRING not in sp_request.meta - - def test_middleware_trace_request_qs(self): - return self.test_middleware_trace_request('foo=bar') - - def test_middleware_trace_request_multi_qs(self): - return self.test_middleware_trace_request('foo=bar&foo=baz&x=y') - - def test_middleware_trace_request_no_qs_trace(self): - with self.override_global_config(dict(trace_query_string=True)): - return self.test_middleware_trace_request() - - def test_middleware_trace_request_qs_trace(self): - with self.override_global_config(dict(trace_query_string=True)): - return self.test_middleware_trace_request('foo=bar') - - def test_middleware_trace_request_multi_qs_trace(self): - with self.override_global_config(dict(trace_query_string=True)): - return self.test_middleware_trace_request('foo=bar&foo=baz&x=y') - - def test_analytics_global_on_integration_default(self): - """ - When making a request - When an integration trace search is not event sample rate is not set and globally trace search is enabled - We expect the root span to have the appropriate tag - """ - with self.override_global_config(dict(analytics_enabled=True)): - url = reverse('users-list') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - sp_template = spans[1] - sp_database = spans[2] - self.assertEqual(sp_request.name, 'django.request') - self.assertEqual(sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY), 1.0) - self.assertIsNone(sp_template.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - self.assertIsNone(sp_database.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - - @override_ddtrace_settings(ANALYTICS_ENABLED=True, ANALYTICS_SAMPLE_RATE=0.5) - def test_analytics_global_on_integration_on(self): - """ - When making a request - When an integration trace search is enabled and sample rate is set and globally trace search is enabled - We expect the root span to have the appropriate tag - """ - with self.override_global_config(dict(analytics_enabled=True)): - url = reverse('users-list') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - sp_template = spans[1] - sp_database = spans[2] - self.assertEqual(sp_request.name, 'django.request') - self.assertEqual(sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY), 0.5) - self.assertIsNone(sp_template.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - self.assertIsNone(sp_database.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - - def test_analytics_global_off_integration_default(self): - """ - When making a request - When an integration trace search is not set and sample rate is set and globally trace search is disabled - We expect the root span to not include tag - """ - with self.override_global_config(dict(analytics_enabled=False)): - url = reverse('users-list') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - sp_template = spans[1] - sp_database = spans[2] - self.assertEqual(sp_request.name, 'django.request') - self.assertIsNone(sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - self.assertIsNone(sp_template.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - self.assertIsNone(sp_database.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - - @override_ddtrace_settings(ANALYTICS_ENABLED=True, ANALYTICS_SAMPLE_RATE=0.5) - def test_analytics_global_off_integration_on(self): - """ - When making a request - When an integration trace search is enabled and sample rate is set and globally trace search is disabled - We expect the root span to have the appropriate tag - """ - with self.override_global_config(dict(analytics_enabled=False)): - url = reverse('users-list') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - sp_template = spans[1] - sp_database = spans[2] - self.assertEqual(sp_request.name, 'django.request') - self.assertEqual(sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY), 0.5) - self.assertIsNone(sp_template.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - self.assertIsNone(sp_database.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) - - @override_ddtrace_settings(ANALYTICS_ENABLED=True, ANALYTICS_SAMPLE_RATE=None) - def test_analytics_global_off_integration_on_and_none(self): - """ - When making a request - When an integration trace search is enabled - Sample rate is set to None - Globally trace search is disabled - We expect the root span to have the appropriate tag - """ - with self.override_global_config(dict(analytics_enabled=False)): - url = reverse('users-list') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - sp_template = spans[1] - sp_database = spans[2] - self.assertEqual(sp_request.name, 'django.request') - assert sp_request.get_metric(ANALYTICS_SAMPLE_RATE_KEY) is None - assert sp_template.get_metric(ANALYTICS_SAMPLE_RATE_KEY) is None - assert sp_database.get_metric(ANALYTICS_SAMPLE_RATE_KEY) is None - - def test_database_patch(self): - # We want to test that a connection-recreation event causes connections - # to get repatched. However since django tests are a atomic transaction - # we can't change the connection. Instead we test that the connection - # does get repatched if it's not patched. - for conn in connections.all(): - unpatch_conn(conn) - # ensures that the internals are properly traced - url = reverse('users-list') - response = self.client.get(url) - assert response.status_code == 200 - - # We would be missing span #3, the database span, if the connection - # wasn't patched. - spans = self.tracer.writer.pop() - assert len(spans) == 3 - assert spans[0].name == 'django.request' - assert spans[1].name == 'django.template' - assert spans[2].name == 'sqlite.query' - - def test_middleware_trace_errors(self): - # ensures that the internals are properly traced - url = reverse('forbidden-view') - response = self.client.get(url) - assert response.status_code == 403 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - span = spans[0] - assert_span_http_status_code(span, 403) - assert span.get_tag(http.URL) == 'http://testserver/fail-view/' - assert span.resource == 'tests.contrib.django.app.views.ForbiddenView' - - def test_middleware_trace_function_based_view(self): - # ensures that the internals are properly traced when using a function views - url = reverse('fn-view') - response = self.client.get(url) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - span = spans[0] - assert_span_http_status_code(span, 200) - assert span.get_tag(http.URL) == 'http://testserver/fn-view/' - assert span.resource == 'tests.contrib.django.app.views.function_view' - - def test_middleware_trace_error_500(self): - # ensures we trace exceptions generated by views - url = reverse('error-500') - response = self.client.get(url) - assert response.status_code == 500 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - span = spans[0] - assert span.error == 1 - assert_span_http_status_code(span, 500) - assert span.get_tag(http.URL) == 'http://testserver/error-500/' - assert span.resource == 'tests.contrib.django.app.views.error_500' - assert 'Error 500' in span.get_tag('error.stack') - - def test_middleware_trace_callable_view(self): - # ensures that the internals are properly traced when using callable views - url = reverse('feed-view') - response = self.client.get(url) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - span = spans[0] - assert_span_http_status_code(span, 200) - assert span.get_tag(http.URL) == 'http://testserver/feed-view/' - assert span.resource == 'tests.contrib.django.app.views.FeedView' - - def test_middleware_trace_partial_based_view(self): - # ensures that the internals are properly traced when using a function views - url = reverse('partial-view') - response = self.client.get(url) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - span = spans[0] - assert_span_http_status_code(span, 200) - assert span.get_tag(http.URL) == 'http://testserver/partial-view/' - assert span.resource == 'partial' - - def test_middleware_trace_lambda_based_view(self): - # ensures that the internals are properly traced when using a function views - url = reverse('lambda-view') - response = self.client.get(url) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - span = spans[0] - assert_span_http_status_code(span, 200) - assert span.get_tag(http.URL) == 'http://testserver/lambda-view/' - assert span.resource == 'tests.contrib.django.app.views.' - - @modify_settings( - MIDDLEWARE={ - 'remove': 'django.contrib.auth.middleware.AuthenticationMiddleware', - }, - MIDDLEWARE_CLASSES={ - 'remove': 'django.contrib.auth.middleware.AuthenticationMiddleware', - }, - ) - def test_middleware_without_user(self): - # remove the AuthenticationMiddleware so that the ``request`` - # object doesn't have the ``user`` field - url = reverse('users-list') - response = self.client.get(url) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - assert_span_http_status_code(sp_request, 200) - assert sp_request.get_tag('django.user.is_authenticated') is None - - def test_middleware_propagation(self): - # ensures that we properly propagate http context - url = reverse('users-list') - headers = { - 'x-datadog-trace-id': '100', - 'x-datadog-parent-id': '42', - 'x-datadog-sampling-priority': '2', - } - response = self.client.get(url, **headers) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - - # Check for proper propagated attributes - assert sp_request.trace_id == 100 - assert sp_request.parent_id == 42 - assert sp_request.get_metric(SAMPLING_PRIORITY_KEY) == 2 - - @override_ddtrace_settings(DISTRIBUTED_TRACING=False) - def test_middleware_no_propagation(self): - # ensures that we properly propagate http context - url = reverse('users-list') - headers = { - 'x-datadog-trace-id': '100', - 'x-datadog-parent-id': '42', - 'x-datadog-sampling-priority': '2', - } - response = self.client.get(url, **headers) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 3 - sp_request = spans[0] - - # Check that propagation didn't happen - assert sp_request.trace_id != 100 - assert sp_request.parent_id != 42 - assert sp_request.get_metric(SAMPLING_PRIORITY_KEY) != 2 - - @modify_settings( - MIDDLEWARE={ - 'append': 'tests.contrib.django.app.middlewares.HandleErrorMiddlewareSuccess', - }, - MIDDLEWARE_CLASSES={ - 'append': 'tests.contrib.django.app.middlewares.HandleErrorMiddlewareSuccess', - }, - ) - def test_middleware_handled_view_exception_success(self): - """ Test when an exception is raised in a view and then handled, that - the resulting span does not possess error properties. - """ - url = reverse('error-500') - response = self.client.get(url) - assert response.status_code == 200 - - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - sp_request = spans[0] - - assert sp_request.error == 0 - assert sp_request.get_tag(errors.ERROR_STACK) is None - assert sp_request.get_tag(errors.ERROR_MSG) is None - assert sp_request.get_tag(errors.ERROR_TYPE) is None - - @modify_settings( - MIDDLEWARE={ - 'append': 'tests.contrib.django.app.middlewares.HandleErrorMiddlewareClientError', - }, - MIDDLEWARE_CLASSES={ - 'append': 'tests.contrib.django.app.middlewares.HandleErrorMiddlewareClientError', - }, - ) - def test_middleware_handled_view_exception_client_error(self): - """ Test the case that when an exception is raised in a view and then - handled, that the resulting span does not possess error properties. - """ - url = reverse('error-500') - response = self.client.get(url) - assert response.status_code == 404 - - spans = self.tracer.writer.pop() - assert len(spans) == 1 - - sp_request = spans[0] - - assert sp_request.error == 0 - assert sp_request.get_tag(errors.ERROR_STACK) is None - assert sp_request.get_tag(errors.ERROR_MSG) is None - assert sp_request.get_tag(errors.ERROR_TYPE) is None - - def test_middleware_trace_request_ot(self): - """OpenTracing version of test_middleware_trace_request.""" - ot_tracer = init_tracer('my_svc', self.tracer) - - # ensures that the internals are properly traced - url = reverse('users-list') - with ot_tracer.start_active_span('ot_span'): - response = self.client.get(url) - assert response.status_code == 200 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 4 - ot_span = spans[0] - sp_request = spans[1] - sp_template = spans[2] - sp_database = spans[3] - - # confirm parenting - assert ot_span.parent_id is None - assert sp_request.parent_id == ot_span.span_id - - assert ot_span.resource == 'ot_span' - assert ot_span.service == 'my_svc' - - assert sp_database.get_tag('django.db.vendor') == 'sqlite' - assert sp_template.get_tag('django.template_name') == 'users_list.html' - assert_span_http_status_code(sp_request, 200) - assert sp_request.get_tag(http.URL) == 'http://testserver/users/' - assert sp_request.get_tag('django.user.is_authenticated') == 'False' - assert sp_request.get_tag('http.method') == 'GET' - - def test_middleware_trace_request_404(self): - """ - When making a request to an unknown url in django - when we do not have a 404 view handler set - we set a resource name for the default view handler - """ - response = self.client.get('/unknown-url') - assert response.status_code == 404 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 2 - sp_request = spans[0] - sp_template = spans[1] - - # Template - # DEV: The template name is `unknown` because unless they define a `404.html` - # django generates the template from a string, which will not have a `Template.name` set - assert sp_template.get_tag('django.template_name') == 'unknown' - - # Request - assert_span_http_status_code(sp_request, 404) - assert sp_request.get_tag(http.URL) == 'http://testserver/unknown-url' - assert sp_request.get_tag('django.user.is_authenticated') == 'False' - assert sp_request.get_tag('http.method') == 'GET' - assert sp_request.span_type == 'web' - assert sp_request.resource == 'django.views.defaults.page_not_found' diff --git a/tests/contrib/django/test_templates.py b/tests/contrib/django/test_templates.py deleted file mode 100644 index 8e8b6ea021e..00000000000 --- a/tests/contrib/django/test_templates.py +++ /dev/null @@ -1,46 +0,0 @@ -import time - -# 3rd party -from django.template import Context, Template - -# testing -from .utils import DjangoTraceTestCase, override_ddtrace_settings - - -class DjangoTemplateTest(DjangoTraceTestCase): - """ - Ensures that the template system is properly traced - """ - def test_template(self): - # prepare a base template using the default engine - template = Template('Hello {{name}}!') - ctx = Context({'name': 'Django'}) - - # (trace) the template rendering - start = time.time() - assert template.render(ctx) == 'Hello Django!' - end = time.time() - - # tests - spans = self.tracer.writer.pop() - assert spans, spans - assert len(spans) == 1 - - span = spans[0] - assert span.span_type == 'template' - assert span.name == 'django.template' - assert span.get_tag('django.template_name') == 'unknown' - assert start < span.start < span.start + span.duration < end - - @override_ddtrace_settings(INSTRUMENT_TEMPLATE=False) - def test_template_disabled(self): - # prepare a base template using the default engine - template = Template('Hello {{name}}!') - ctx = Context({'name': 'Django'}) - - # (trace) the template rendering - assert template.render(ctx) == 'Hello Django!' - - # tests - spans = self.tracer.writer.pop() - assert len(spans) == 0 diff --git a/tests/contrib/django/test_tracing_disabled.py b/tests/contrib/django/test_tracing_disabled.py deleted file mode 100644 index 61605f2dea1..00000000000 --- a/tests/contrib/django/test_tracing_disabled.py +++ /dev/null @@ -1,42 +0,0 @@ -# 3rd party -from django.apps import apps -from django.test import TestCase - -# project -from ddtrace.tracer import Tracer -from ddtrace.contrib.django.conf import settings - -# testing -from ...test_tracer import DummyWriter - - -class DjangoTracingDisabledTest(TestCase): - def setUp(self): - # backup previous conf - self.backupEnabled = settings.ENABLED - self.backupTracer = settings.TRACER - - # Use a new tracer to be sure that a new service - # would be sent to the the writer - self.tracer = Tracer() - self.tracer.writer = DummyWriter() - - # Restart app with tracing disabled - settings.ENABLED = False - self.app = apps.get_app_config('datadog_django') - self.app.ready() - - def tearDown(self): - # Reset the original settings - settings.ENABLED = self.backupEnabled - settings.TRACER = self.backupTracer - self.app.ready() - - def test_no_service_info_is_written(self): - services = self.tracer.writer.pop_services() - assert len(services) == 0 - - def test_no_trace_is_written(self): - settings.TRACER.trace('client.testing').finish() - traces = self.tracer.writer.pop_traces() - assert len(traces) == 0 diff --git a/tests/contrib/django/test_utils.py b/tests/contrib/django/test_utils.py deleted file mode 100644 index 2d7cf074058..00000000000 --- a/tests/contrib/django/test_utils.py +++ /dev/null @@ -1,17 +0,0 @@ -# 3d party -from django.test import TestCase - -# project -from ddtrace.contrib.django.utils import quantize_key_values - - -class DjangoUtilsTest(TestCase): - def test_quantize_key_values(self): - """ - Ensure that the utility functions properly convert a dictionary object - """ - key = {'second_key': 2, 'first_key': 1} - result = quantize_key_values(key) - assert len(result) == 2 - assert 'first_key' in result - assert 'second_key' in result diff --git a/tests/contrib/django/utils.py b/tests/contrib/django/utils.py deleted file mode 100644 index 9cd2c420f38..00000000000 --- a/tests/contrib/django/utils.py +++ /dev/null @@ -1,87 +0,0 @@ -from functools import wraps - -# 3rd party -from django.apps import apps -from django.test import TestCase - -# project -from ddtrace.tracer import Tracer -from ddtrace.contrib.django.conf import settings -from ddtrace.contrib.django.db import patch_db, unpatch_db -from ddtrace.contrib.django.cache import unpatch_cache -from ddtrace.contrib.django.templates import unpatch_template -from ddtrace.contrib.django.middleware import remove_exception_middleware, remove_trace_middleware - -# testing -from ...base import BaseTestCase -from ...test_tracer import DummyWriter - - -# testing tracer -tracer = Tracer() -tracer.writer = DummyWriter() - - -class DjangoTraceTestCase(BaseTestCase, TestCase): - """ - Base class that provides an internal tracer according to given - Datadog settings. This class ensures that the tracer spans are - properly reset after each run. The tracer is available in - the ``self.tracer`` attribute. - """ - def setUp(self): - # assign the default tracer - self.tracer = settings.TRACER - # empty the tracer spans from previous operations - # such as database creation queries - self.tracer.writer.spans = [] - self.tracer.writer.pop_traces() - # gets unpatched for some tests - patch_db(self.tracer) - - def tearDown(self): - # empty the tracer spans from test operations - self.tracer.writer.spans = [] - self.tracer.writer.pop_traces() - - -class override_ddtrace_settings(object): - def __init__(self, *args, **kwargs): - self.items = list(kwargs.items()) - - def unpatch_all(self): - unpatch_cache() - unpatch_db() - unpatch_template() - remove_trace_middleware() - remove_exception_middleware() - - def __enter__(self): - self.enable() - - def __exit__(self, exc_type, exc_value, traceback): - self.disable() - - def enable(self): - self.backup = {} - for name, value in self.items: - self.backup[name] = getattr(settings, name) - setattr(settings, name, value) - self.unpatch_all() - app = apps.get_app_config('datadog_django') - app.ready() - - def disable(self): - for name, value in self.items: - setattr(settings, name, self.backup[name]) - self.unpatch_all() - remove_exception_middleware() - app = apps.get_app_config('datadog_django') - app.ready() - - def __call__(self, func): - @wraps(func) - def inner(*args, **kwargs): - with(self): - return func(*args, **kwargs) - return inner diff --git a/tests/contrib/django/views.py b/tests/contrib/django/views.py new file mode 100644 index 00000000000..c523628fda9 --- /dev/null +++ b/tests/contrib/django/views.py @@ -0,0 +1,78 @@ +""" +Class based views used for Django tests. +""" + +from functools import partial + +from django.http import HttpResponse + +from django.views.generic import ListView, TemplateView, View + +from django.contrib.auth.models import User +from django.contrib.syndication.views import Feed + + +class UserList(ListView): + model = User + template_name = "users_list.html" + + +class TemplateCachedUserList(ListView): + model = User + template_name = "cached_list.html" + + +class BasicView(View): + def get(self, request): + return HttpResponse("") + + def post(self, request): + return HttpResponse("") + + def delete(self, request): + return HttpResponse("") + + def head(self, request): + return HttpResponse("") + + +class ForbiddenView(TemplateView): + def get(self, request, *args, **kwargs): + return HttpResponse(status=403) + + +def function_view(request): + return HttpResponse(status=200) + + +def error_500(request): + raise Exception("Error 500") + + +class FeedView(Feed): + """ + A callable view that is part of the Django framework + """ + + title = "Police beat site news" + link = "/sitenews/" + description = "Updates on changes and additions to police beat central." + + def items(self): + return [] + + def item_title(self, item): + return "empty" + + def item_description(self, item): + return "empty" + + +partial_view = partial(function_view) + +# disabling flake8 test below, yes, declaring a func like this is bad, we know +lambda_view = lambda request: function_view(request) # NOQA + + +def index(request): + return HttpResponse("Hello, test app.") diff --git a/tests/contrib/djangorestframework/app/exceptions.py b/tests/contrib/djangorestframework/app/exceptions.py index 0f4fce70e48..c98152a7e71 100644 --- a/tests/contrib/djangorestframework/app/exceptions.py +++ b/tests/contrib/djangorestframework/app/exceptions.py @@ -8,6 +8,6 @@ def custom_exception_handler(exc, context): # We overwrite the response status code to 500 if response is not None: - return Response({'detail': str(exc)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + return Response({"detail": str(exc)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return response diff --git a/tests/contrib/djangorestframework/app/settings.py b/tests/contrib/djangorestframework/app/settings.py index ac24bd45fda..a98d4518e3d 100644 --- a/tests/contrib/djangorestframework/app/settings.py +++ b/tests/contrib/djangorestframework/app/settings.py @@ -9,33 +9,26 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:' - } -} +DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}} SITE_ID = 1 -SECRET_KEY = 'not_very_secret_in_tests' +SECRET_KEY = "not_very_secret_in_tests" USE_I18N = True USE_L10N = True -STATIC_URL = '/static/' -ROOT_URLCONF = 'tests.contrib.djangorestframework.app.views' +STATIC_URL = "/static/" +ROOT_URLCONF = "tests.contrib.djangorestframework.app.views" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(BASE_DIR, 'app', 'templates'), - ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "app", "templates"),], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, @@ -43,73 +36,54 @@ if (1, 10) <= django.VERSION < (2, 0): MIDDLEWARE = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - - 'tests.contrib.django.app.middlewares.CatchExceptionMiddleware', + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.auth.middleware.SessionAuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + "tests.contrib.django.middleware.CatchExceptionMiddleware", ] # Django 2.0 has different defaults elif django.VERSION >= (2, 0): MIDDLEWARE = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - - 'tests.contrib.django.app.middlewares.CatchExceptionMiddleware', + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + "tests.contrib.django.middleware.CatchExceptionMiddleware", ] # Pre 1.10 style else: MIDDLEWARE_CLASSES = [ - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - - 'tests.contrib.django.app.middlewares.CatchExceptionMiddleware', + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.auth.middleware.SessionAuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + "tests.contrib.django.middleware.CatchExceptionMiddleware", ] INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - - # tracer app - 'ddtrace.contrib.django', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", # djangorestframework - 'rest_framework' + "rest_framework", ] -DATADOG_TRACE = { - # tracer with a DummyWriter - 'TRACER': 'tests.contrib.django.utils.tracer', - 'ENABLED': True, - 'TAGS': { - 'env': 'test', - }, -} - REST_FRAMEWORK = { - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.IsAdminUser', - ], - - 'EXCEPTION_HANDLER': 'tests.contrib.djangorestframework.app.exceptions.custom_exception_handler' + "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAdminUser",], + "EXCEPTION_HANDLER": "tests.contrib.djangorestframework.app.exceptions.custom_exception_handler", } diff --git a/tests/contrib/djangorestframework/app/views.py b/tests/contrib/djangorestframework/app/views.py index 88179c67716..68e98a34710 100644 --- a/tests/contrib/djangorestframework/app/views.py +++ b/tests/contrib/djangorestframework/app/views.py @@ -7,23 +7,24 @@ class UserSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = User - fields = ('url', 'username', 'email', 'groups') + fields = ("url", "username", "email", "groups") class UserViewSet(viewsets.ModelViewSet): """ API endpoint that allows users to be viewed or edited. """ - queryset = User.objects.all().order_by('-date_joined') + + queryset = User.objects.all().order_by("-date_joined") serializer_class = UserSerializer router = routers.DefaultRouter() -router.register(r'users', UserViewSet) +router.register(r"users", UserViewSet) # Wire up our API using automatic URL routing. # Additionally, we include login URLs for the browsable API. urlpatterns = [ - url(r'^', include(router.urls)), - url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), + url(r"^", include(router.urls)), + url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), ] diff --git a/tests/contrib/djangorestframework/conftest.py b/tests/contrib/djangorestframework/conftest.py index a30ef07cfdf..8584db9df2a 100644 --- a/tests/contrib/djangorestframework/conftest.py +++ b/tests/contrib/djangorestframework/conftest.py @@ -1,16 +1,19 @@ +__all__ = ["pytest_configure", "test_spans", "tracer", "patch_django"] + import os import django from django.conf import settings +from ddtrace.contrib.django import patch +from ..django.conftest import test_spans, tracer, patch_django + # We manually designate which settings we will be using in an environment variable # This is similar to what occurs in the `manage.py` -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.contrib.djangorestframework.app.settings') +os.environ["DJANGO_SETTINGS_MODULE"] = "tests.contrib.djangorestframework.app.settings" # `pytest` automatically calls this function once when tests are run. def pytest_configure(): settings.DEBUG = False - if django.VERSION < (1, 7, 0): - settings.configure() - else: - django.setup() + patch() + django.setup() diff --git a/tests/contrib/djangorestframework/test_djangorestframework.py b/tests/contrib/djangorestframework/test_djangorestframework.py index a91c9f9f010..cd6d8926b13 100644 --- a/tests/contrib/djangorestframework/test_djangorestframework.py +++ b/tests/contrib/djangorestframework/test_djangorestframework.py @@ -1,62 +1,33 @@ import django -from django.apps import apps -from unittest import skipIf +import pytest -from tests.contrib.django.utils import DjangoTraceTestCase from ...utils import assert_span_http_status_code -@skipIf(django.VERSION < (1, 10), 'requires django version >= 1.10') -class RestFrameworkTest(DjangoTraceTestCase): - def setUp(self): - super(RestFrameworkTest, self).setUp() - - # would raise an exception - from rest_framework.views import APIView - from ddtrace.contrib.django.restframework import unpatch_restframework - - self.APIView = APIView - self.unpatch_restframework = unpatch_restframework - - def test_setup(self): - assert apps.is_installed('rest_framework') - assert hasattr(self.APIView, '_datadog_patch') - - def test_unpatch(self): - self.unpatch_restframework() - assert not getattr(self.APIView, '_datadog_patch') - - response = self.client.get('/users/') - - # Our custom exception handler is setting the status code to 500 - assert response.status_code == 500 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - sp = spans[0] - assert sp.name == 'django.request' - assert sp.resource == 'tests.contrib.djangorestframework.app.views.UserViewSet' - assert sp.error == 0 - assert sp.span_type == 'web' - assert_span_http_status_code(sp, 500) - assert sp.get_tag('error.msg') is None - - def test_trace_exceptions(self): - response = self.client.get('/users/') - - # Our custom exception handler is setting the status code to 500 - assert response.status_code == 500 - - # check for spans - spans = self.tracer.writer.pop() - assert len(spans) == 1 - sp = spans[0] - assert sp.name == 'django.request' - assert sp.resource == 'tests.contrib.djangorestframework.app.views.UserViewSet' - assert sp.error == 1 - assert sp.span_type == 'web' - assert sp.get_tag('http.method') == 'GET' - assert_span_http_status_code(sp, 500) - assert sp.get_tag('error.msg') == 'Authentication credentials were not provided.' - assert 'NotAuthenticated' in sp.get_tag('error.stack') +@pytest.mark.skipif(django.VERSION < (1, 10), reason="requires django version >= 1.10") +@pytest.mark.django_db +def test_trace_exceptions(client, test_spans): # noqa flake8 complains about shadowing test_spans + response = client.get("/users/") + + # Our custom exception handler is setting the status code to 500 + assert response.status_code == 500 + + sp = test_spans.get_root_span() + assert sp.name == "django.request" + if django.VERSION >= (2, 2, 0): + assert sp.resource == "GET ^users/$" + else: + assert sp.resource == "GET tests.contrib.djangorestframework.app.views.UserViewSet" + assert sp.error == 1 + assert sp.span_type == "http" + assert_span_http_status_code(sp, 500) + assert sp.get_tag("http.method") == "GET" + + # the DRF integration should set the traceback on the django.view span + # (as it's the current span when the exception info is set) + view_spans = list(test_spans.filter_spans(name="django.view")) + assert len(view_spans) == 1 + err_span = view_spans[0] + assert err_span.error == 1 + assert err_span.get_tag("error.msg") == "Authentication credentials were not provided." + assert "NotAuthenticated" in err_span.get_tag("error.stack") diff --git a/tests/contrib/patch.py b/tests/contrib/patch.py index e9d73617786..538cdb3a455 100644 --- a/tests/contrib/patch.py +++ b/tests/contrib/patch.py @@ -65,11 +65,14 @@ def raise_if_no_attrs(f): required_attrs = [ '__module_name__', '__integration_name__', - '__unpatch_func__', + '__patch_func__', ] @functools.wraps(f) def checked_method(self, *args, **kwargs): + if getattr(self, '__unpatch_func__') is None: + return + for attr in required_attrs: if not getattr(self, attr): raise NotImplementedError(f.__doc__) @@ -77,6 +80,19 @@ def checked_method(self, *args, **kwargs): return checked_method +def noop_if_no_unpatch(f): + """ + A helper for PatchTestCase test methods that will no-op the test if the + __unpatch_func__ attribute is None + """ + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + if getattr(self, '__unpatch_func__') is None: + return + return f(self, *args, **kwargs) + return wrapper + + class PatchTestCase(object): """ unittest or other test runners will pick up the base test case as a testcase @@ -94,16 +110,18 @@ class Base(SubprocessTestCase, PatchMixin): Attributes: __integration_name__ the name of the integration. __module_name__ module which the integration patches. + __patch_func__ patch function from the integration. __unpatch_func__ unpatch function from the integration. Example: A simple implementation inheriting this TestCase looks like:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.redis import patch, unpatch class RedisPatchTestCase(PatchTestCase.Base): __integration_name__ = 'redis' __module_name__ 'redis' + __patch_func__ = patch __unpatch_func__ = unpatch def assert_module_patched(self, redis): @@ -126,11 +144,12 @@ def test_patch_import(self): """ __integration_name__ = None __module_name__ = None + __patch_func__ = None __unpatch_func__ = None def __init__(self, *args, **kwargs): - # DEV: Python will wrap a function when assigning to a class as an - # attribute. So we cannot call self.__unpatch_func__() as the `self` + # DEV: Python will wrap a function when assigning it to a class as an + # attribute. So we cannot call self.__unpatch_func__() because the `self` # reference will be passed as an argument. # So we need to unwrap the function and then wrap it in a function # that will absorb the unpatch function. @@ -140,11 +159,15 @@ def __init__(self, *args, **kwargs): def unpatch(): unpatch_func() self.__unpatch_func__ = unpatch - super(PatchTestCase.Base, self).__init__(*args, **kwargs) - def patch(self, *args, **kwargs): - from ddtrace import patch - return patch(*args, **kwargs) + # Same for __patch_func__() + if self.__patch_func__: + patch_func = self.__patch_func__.__func__ + + def patch(): + patch_func() + self.__patch_func__ = patch + super(PatchTestCase.Base, self).__init__(*args, **kwargs) def _gen_test_attrs(self, ops): """ @@ -281,7 +304,7 @@ def test_import_patch(self): self.assert_not_module_imported(self.__module_name__) module = importlib.import_module(self.__module_name__) self.assert_not_module_patched(module) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) @raise_if_no_attrs @@ -299,7 +322,7 @@ def test_patch_import(self): """ self.assert_not_module_imported(self.__module_name__) module = importlib.import_module(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) @raise_if_no_attrs @@ -318,10 +341,10 @@ def test_import_patch_patch(self): self.assert_not_module_double_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__module_name__: True}) + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) - self.patch(**{self.__module_name__: True}) + self.__patch_func__() self.assert_not_module_double_patched(module) @raise_if_no_attrs @@ -340,10 +363,10 @@ def test_patch_import_patch(self): self.assert_not_module_double_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__module_name__: True}) + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) - self.patch(**{self.__module_name__: True}) + self.__patch_func__() self.assert_not_module_double_patched(module) @raise_if_no_attrs @@ -361,8 +384,8 @@ def test_patch_patch_import(self): self.assert_not_double_wrapped(redis.StrictRedis.execute_command) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__module_name__: True}) - self.patch(**{self.__module_name__: True}) + self.__patch_func__() + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) self.assert_not_module_double_patched(module) @@ -386,11 +409,12 @@ def test_import_patch_unpatch_patch(self): """ self.assert_not_module_imported(self.__module_name__) module = importlib.import_module(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) self.__unpatch_func__() self.assert_not_module_patched(module) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() + self.__patch_func__() self.assert_module_patched(module) @raise_if_no_attrs @@ -411,12 +435,12 @@ def test_patch_import_unpatch_patch(self): self.assert_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) self.__unpatch_func__() self.assert_not_module_patched(module) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) @raise_if_no_attrs @@ -437,11 +461,11 @@ def test_patch_unpatch_import_patch(self): self.assert_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.__unpatch_func__() module = importlib.import_module(self.__module_name__) self.assert_not_module_patched(module) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) @raise_if_no_attrs @@ -462,9 +486,9 @@ def test_patch_unpatch_patch_import(self): self.assert_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.__unpatch_func__() - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) @@ -483,7 +507,8 @@ def test_unpatch_patch_import(self): """ self.assert_not_module_imported(self.__module_name__) self.__unpatch_func__() - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) @@ -503,7 +528,7 @@ def test_patch_unpatch_import(self): self.assert_not_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.__unpatch_func__() module = importlib.import_module(self.__module_name__) self.assert_not_module_patched(module) @@ -526,7 +551,7 @@ def test_import_unpatch_patch(self): module = importlib.import_module(self.__module_name__) self.__unpatch_func__() self.assert_not_module_patched(module) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) @raise_if_no_attrs @@ -546,7 +571,7 @@ def test_import_patch_unpatch(self): self.assert_not_module_imported(self.__module_name__) module = importlib.import_module(self.__module_name__) self.assert_not_module_patched(module) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) self.__unpatch_func__() self.assert_not_module_patched(module) @@ -566,7 +591,7 @@ def test_patch_import_unpatch(self): self.assert_not_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() module = importlib.import_module(self.__module_name__) self.assert_module_patched(module) self.__unpatch_func__() @@ -591,7 +616,7 @@ def test_import_patch_unpatch_unpatch(self): """ self.assert_not_module_imported(self.__module_name__) module = importlib.import_module(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.assert_module_patched(module) self.__unpatch_func__() self.assert_not_module_patched(module) @@ -615,7 +640,7 @@ def test_patch_unpatch_import_unpatch(self): self.assert_not_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.__unpatch_func__() module = importlib.import_module(self.__module_name__) self.assert_not_module_patched(module) @@ -638,7 +663,7 @@ def test_patch_unpatch_unpatch_import(self): self.assert_not_module_patched(redis) """ self.assert_not_module_imported(self.__module_name__) - self.patch(**{self.__integration_name__: True}) + self.__patch_func__() self.__unpatch_func__() self.__unpatch_func__() module = importlib.import_module(self.__module_name__) diff --git a/tox.ini b/tox.ini index 5e40dd2a22c..e18b9b228eb 100644 --- a/tox.ini +++ b/tox.ini @@ -58,10 +58,21 @@ envlist = celery_contrib-{py27,py34,py35,py36,py37}-celery43-redis320-kombu44 consul_contrib-py{27,34,35,36,37}-consul{07,10,11} dbapi_contrib-{py27,py34,py35,py36} - django_contrib{,_autopatch}-{py27,py34,py35,py36}-django{18,111}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached - django_contrib{,_autopatch}-{py34,py35,py36}-django{200}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached - django_drf_contrib-{py27,py34,py35,py36}-django{111}-djangorestframework{34,37,38} - django_drf_contrib-{py34,py35,py36}-django{200}-djangorestframework{37,38} +# Django Python version support +# 1.11 2.7, 3.4, 3.5, 3.6, 3.7 (added in 1.11.17) +# 2.0 3.4, 3.5, 3.6, 3.7 +# 2.1 3.5, 3.6, 3.7 +# 2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8) +# 3.0 3.6, 3.7, 3.8 +# 3.1 3.6, 3.7, 3.8 +# Source: https://docs.djangoproject.com/en/dev/faq/install/#what-python-version-can-i-use-with-django + django_contrib{,_migration}-{py27,py34,py35,py36}-django{18,111}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached + django_contrib{,_migration}-{py35}-django{20,21,22}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached + django_contrib{,_migration}-{py36,py37}-django{20,21,22,30,latest}-djangopylibmc06-djangoredis45-pylibmc-redis{210}-memcached + django_drf_contrib-{py27,py34,py35,py36}-django{111}-djangorestframework{34,37} + django_drf_contrib-{py35}-django{22}-djangorestframework{38,310,latest} + django_drf_contrib-{py36,py37}-django{22}-djangorestframework{38,310,latest} + django_drf_contrib-{py36,py37}-django{30}-djangorestframework{310,latest} dogpile_contrib-{py27,py35,py36,py37}-dogpilecache{06,07,08,latest} elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch{16,17,18,23,24,51,52,53,54,63,64} elasticsearch_contrib-{py27,py34,py35,py36}-elasticsearch1{100} @@ -212,12 +223,20 @@ deps = ddtracerun: redis django18: django>=1.8,<1.9 django111: django>=1.11,<1.12 - django200: django>=2.0,<2.1 + django20: django>=2.0,<2.1 + django21: django>=2.1,<2.2 + django22: django>=2.2,<2.3 + django30: django>=3.0,<3.1 + djangolatest: django>=3.0 djangopylibmc06: django-pylibmc>=0.6,<0.7 djangoredis45: django-redis>=4.5,<4.6 djangorestframework34: djangorestframework>=3.4,<3.5 djangorestframework37: djangorestframework>=3.7,<3.8 djangorestframework38: djangorestframework>=3.8,<3.9 + djangorestframework39: djangorestframework>=3.9,<3.10 + djangorestframework310: djangorestframework>=3.10,<3.11 + djangorestframework311: djangorestframework>=3.11,<3.12 + djangorestframeworklatest: djangorestframework>=3.11 dogpilecache06: dogpile.cache==0.6.* dogpilecache07: dogpile.cache==0.7.* dogpilecache08: dogpile.cache==0.8.* @@ -405,7 +424,7 @@ commands = consul_contrib: pytest {posargs} tests/contrib/consul dbapi_contrib: pytest {posargs} tests/contrib/dbapi django_contrib: pytest {posargs} tests/contrib/django - django_contrib_autopatch: python tests/ddtrace_run.py pytest {posargs} tests/contrib/django + django_contrib_migration: pytest {posargs} tests/contrib/django django_drf_contrib: pytest {posargs} tests/contrib/djangorestframework dogpile_contrib: pytest {posargs} tests/contrib/dogpile_cache elasticsearch_contrib: pytest {posargs} tests/contrib/elasticsearch @@ -829,6 +848,73 @@ setenv = setenv = {[bottle_autopatch]setenv} +# DEV: envs to test migration of django instrumentation to ddtrace config api +[django_contrib_migration] +setenv = + TEST_DATADOG_DJANGO_MIGRATION = 1 +[testenv:django_contrib_migration-py27-django18-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py27-django111-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py34-django18-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py34-django111-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py35-django18-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py35-django111-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-django18-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-django111-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py35-django20-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py35-django21-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py35-django22-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-django20-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-django21-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-django22-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-django30-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py36-djangolatest-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py37-django20-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py37-django21-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py37-django22-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py37-django30-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} +[testenv:django_contrib_migration-py37-djangolatest-djangopylibmc06-djangoredis45-pylibmc-redis210-memcached] +setenv = + {[django_contrib_migration]setenv} # DEV: We use `conftest.py` as a local pytest plugin to configure hooks for collection [pytest] @@ -851,7 +937,7 @@ exclude= # G201 Logging: .exception(...) should be used instead of .error(..., exc_info=True) # E231,W503: not respected by black # We ignore most of the D errors because there are too many; the goal is to fix them eventually -ignore = W503,E231,A003,G201,D100,D101,D102,D103,D104,D105,D106,D107,D200,D202,D204,D205,D208,D210,D300,D400,D401,D403,D413 +ignore = W503,E231,A003,G201,D100,D101,D102,D103,D104,D105,D106,D107,D200,D202,D204,D205,D208,D210,D300,D400,D401,D403,D413,RST301 enable-extensions=G rst-roles = class,meth,obj,ref rst-directives = py:data From 8158e25251ac6acc2d3cd471e0ec4b1e4d445a29 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 12 Feb 2020 17:05:05 +0100 Subject: [PATCH 54/81] fix(ci): limit Werkzeug to <1 for old Flask version The API changed in Werkzeug >= 1 and is not compatible with older Flask version. Capping Werkzeug makes tests work again. --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index e18b9b228eb..caf697140e6 100644 --- a/tox.ini +++ b/tox.ini @@ -266,8 +266,11 @@ deps = falcon13: falcon>=1.3,<1.4 falcon14: falcon>=1.4,<1.5 flask09: flask>=0.9,<0.10 + flask09: Werkzeug<1 flask010: flask>=0.10,<0.11 + flask010: Werkzeug<1 flask011: flask>=0.11,<0.12 + flask011: Werkzeug<1 flask012: flask>=0.12,<0.13 flask10: flask>=1.0,<1.1 flaskcache012: flask_cache>=0.12,<0.13 From d69475c4ead21470f0c7a2bca4c883b4aa401ebb Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 12 Feb 2020 20:33:00 +0100 Subject: [PATCH 55/81] fix(tox): flake8 does not need to install (#1200) This makes running flake8 in tox faster. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index caf697140e6..2aa05e56397 100644 --- a/tox.ini +++ b/tox.ini @@ -493,6 +493,8 @@ commands=black --check . basepython=python3.7 [testenv:flake8] +commands_pre= +skip_install=true deps= flake8>=3.7,<=3.8 flake8-blind-except From e1d9eab8529c1818b0fa83b6492b80a8d61eaad3 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Wed, 12 Feb 2020 16:56:31 -0500 Subject: [PATCH 56/81] core: unvendor msgpack (#1199) * core: unvendor msgpack * fix deps * remove msgpack version check for use_bin_type * fix flake8 * remove checks for msgpack c-extension * remove old comment Co-authored-by: Brett Langdon --- .circleci/config.yml | 4 +- ddtrace/encoding.py | 3 +- ddtrace/vendor/__init__.py | 13 - ddtrace/vendor/msgpack/__init__.py | 65 - ddtrace/vendor/msgpack/_cmsgpack.cpp | 15777 --------------------- ddtrace/vendor/msgpack/_cmsgpack.pyx | 4 - ddtrace/vendor/msgpack/_packer.pyx | 362 - ddtrace/vendor/msgpack/_unpacker.pyx | 569 - ddtrace/vendor/msgpack/_version.py | 1 - ddtrace/vendor/msgpack/buff_converter.h | 28 - ddtrace/vendor/msgpack/exceptions.py | 48 - ddtrace/vendor/msgpack/fallback.py | 1027 -- ddtrace/vendor/msgpack/pack.h | 119 - ddtrace/vendor/msgpack/pack_template.h | 778 - ddtrace/vendor/msgpack/setup.py | 26 - ddtrace/vendor/msgpack/sysdep.h | 194 - ddtrace/vendor/msgpack/unpack.h | 287 - ddtrace/vendor/msgpack/unpack_define.h | 95 - ddtrace/vendor/msgpack/unpack_template.h | 454 - setup.py | 5 +- tests/test_encoders.py | 2 +- tests/test_integration.py | 2 +- tox.ini | 7 +- 23 files changed, 13 insertions(+), 19857 deletions(-) delete mode 100644 ddtrace/vendor/msgpack/__init__.py delete mode 100644 ddtrace/vendor/msgpack/_cmsgpack.cpp delete mode 100644 ddtrace/vendor/msgpack/_cmsgpack.pyx delete mode 100644 ddtrace/vendor/msgpack/_packer.pyx delete mode 100644 ddtrace/vendor/msgpack/_unpacker.pyx delete mode 100644 ddtrace/vendor/msgpack/_version.py delete mode 100644 ddtrace/vendor/msgpack/buff_converter.h delete mode 100644 ddtrace/vendor/msgpack/exceptions.py delete mode 100644 ddtrace/vendor/msgpack/fallback.py delete mode 100644 ddtrace/vendor/msgpack/pack.h delete mode 100644 ddtrace/vendor/msgpack/pack_template.h delete mode 100644 ddtrace/vendor/msgpack/setup.py delete mode 100644 ddtrace/vendor/msgpack/sysdep.h delete mode 100644 ddtrace/vendor/msgpack/unpack.h delete mode 100644 ddtrace/vendor/msgpack/unpack_define.h delete mode 100644 ddtrace/vendor/msgpack/unpack_template.h diff --git a/.circleci/config.yml b/.circleci/config.yml index f8831db78b7..d54ecf172c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,7 @@ jobs: steps: - checkout - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-tracer$' + - run: scripts/run-tox-scenario '^py..-tracer' - *persist_to_workspace_step - *save_cache_step @@ -133,7 +133,7 @@ jobs: steps: - checkout - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-integration$' + - run: scripts/run-tox-scenario '^py..-integration' - *persist_to_workspace_step - *save_cache_step diff --git a/ddtrace/encoding.py b/ddtrace/encoding.py index f650966b557..c6159645aff 100644 --- a/ddtrace/encoding.py +++ b/ddtrace/encoding.py @@ -4,10 +4,9 @@ from .internal.logger import get_logger -# Try to import msgpack, fallback to just JSON if something went wrong # DEV: We are ok with the pure Python fallback for msgpack if the C-extension failed to install try: - from ddtrace.vendor import msgpack + import msgpack # DEV: `use_bin_type` only exists since `0.4.0`, but we vendor a more recent version MSGPACK_PARAMS = {'use_bin_type': True} MSGPACK_ENCODING = True diff --git a/ddtrace/vendor/__init__.py b/ddtrace/vendor/__init__.py index cbef8ee46fb..009f120a571 100644 --- a/ddtrace/vendor/__init__.py +++ b/ddtrace/vendor/__init__.py @@ -8,19 +8,6 @@ Dependencies ============ -msgpack -------- - -Website: https://msgpack.org/ -Source: https://github.com/msgpack/msgpack-python -Version: 0.6.1 -License: Apache License, Version 2.0 - -Notes: - If you need to update any `*.pyx` files, be sure to run `cython --cplus msgpack/_cmsgpack.pyx` to regenerate `_cmsgpack.cpp` - - `_packer.pyx` and `_unpacker.pyx` were updated to import from `ddtrace.vendor.msgpack` - six --- diff --git a/ddtrace/vendor/msgpack/__init__.py b/ddtrace/vendor/msgpack/__init__.py deleted file mode 100644 index 4ad9c1a5e13..00000000000 --- a/ddtrace/vendor/msgpack/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -# coding: utf-8 -from ._version import version -from .exceptions import * - -from collections import namedtuple - - -class ExtType(namedtuple('ExtType', 'code data')): - """ExtType represents ext type in msgpack.""" - def __new__(cls, code, data): - if not isinstance(code, int): - raise TypeError("code must be int") - if not isinstance(data, bytes): - raise TypeError("data must be bytes") - if not 0 <= code <= 127: - raise ValueError("code must be 0~127") - return super(ExtType, cls).__new__(cls, code, data) - - -import os -if os.environ.get('MSGPACK_PUREPYTHON'): - from .fallback import Packer, unpackb, Unpacker -else: - try: - from ._cmsgpack import Packer, unpackb, Unpacker - except ImportError: - from .fallback import Packer, unpackb, Unpacker - - -def pack(o, stream, **kwargs): - """ - Pack object `o` and write it to `stream` - - See :class:`Packer` for options. - """ - packer = Packer(**kwargs) - stream.write(packer.pack(o)) - - -def packb(o, **kwargs): - """ - Pack object `o` and return packed bytes - - See :class:`Packer` for options. - """ - return Packer(**kwargs).pack(o) - - -def unpack(stream, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ExtraData` when `stream` contains extra bytes. - See :class:`Unpacker` for options. - """ - data = stream.read() - return unpackb(data, **kwargs) - - -# alias for compatibility to simplejson/marshal/pickle. -load = unpack -loads = unpackb - -dump = pack -dumps = packb diff --git a/ddtrace/vendor/msgpack/_cmsgpack.cpp b/ddtrace/vendor/msgpack/_cmsgpack.cpp deleted file mode 100644 index c5506e4b1b5..00000000000 --- a/ddtrace/vendor/msgpack/_cmsgpack.cpp +++ /dev/null @@ -1,15777 +0,0 @@ -/* Generated by Cython 0.29.6 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) - #error Cython requires Python 2.6+ or Python 3.3+. -#else -#define CYTHON_ABI "0_29_6" -#define CYTHON_HEX_VERSION 0x001D06F0 -#define CYTHON_FUTURE_DIVISION 1 -#include -#ifndef offsetof - #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#endif -#ifndef DL_IMPORT - #define DL_IMPORT(t) t -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#define __PYX_COMMA , -#ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x02070000 - #define HAVE_LONG_LONG - #endif -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef Py_HUGE_VAL - #define Py_HUGE_VAL HUGE_VAL -#endif -#ifdef PYPY_VERSION - #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #undef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_PYTYPE_LOOKUP - #define CYTHON_USE_PYTYPE_LOOKUP 0 - #if PY_VERSION_HEX < 0x03050000 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #elif !defined(CYTHON_USE_ASYNC_SLOTS) - #define CYTHON_USE_ASYNC_SLOTS 1 - #endif - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #undef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #undef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 1 - #undef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 0 - #undef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 0 - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 - #undef CYTHON_PEP489_MULTI_PHASE_INIT - #define CYTHON_PEP489_MULTI_PHASE_INIT 0 - #undef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE 0 - #undef CYTHON_USE_DICT_VERSIONS - #define CYTHON_USE_DICT_VERSIONS 0 - #undef CYTHON_USE_EXC_INFO_STACK - #define CYTHON_USE_EXC_INFO_STACK 0 -#elif defined(PYSTON_VERSION) - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #undef CYTHON_USE_PYTYPE_LOOKUP - #define CYTHON_USE_PYTYPE_LOOKUP 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 - #undef CYTHON_PEP489_MULTI_PHASE_INIT - #define CYTHON_PEP489_MULTI_PHASE_INIT 0 - #undef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE 0 - #undef CYTHON_USE_DICT_VERSIONS - #define CYTHON_USE_DICT_VERSIONS 0 - #undef CYTHON_USE_EXC_INFO_STACK - #define CYTHON_USE_EXC_INFO_STACK 0 -#else - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 1 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYTYPE_LOOKUP - #define CYTHON_USE_PYTYPE_LOOKUP 0 - #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) - #define CYTHON_USE_PYTYPE_LOOKUP 1 - #endif - #if PY_MAJOR_VERSION < 3 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #elif !defined(CYTHON_USE_ASYNC_SLOTS) - #define CYTHON_USE_ASYNC_SLOTS 1 - #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) - #define CYTHON_USE_PYLONG_INTERNALS 1 - #endif - #ifndef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 1 - #endif - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #if PY_VERSION_HEX < 0x030300F0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #elif !defined(CYTHON_USE_UNICODE_WRITER) - #define CYTHON_USE_UNICODE_WRITER 1 - #endif - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #ifndef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 1 - #endif - #ifndef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 1 - #endif - #ifndef CYTHON_PEP489_MULTI_PHASE_INIT - #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000) - #endif - #ifndef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) - #endif - #ifndef CYTHON_USE_DICT_VERSIONS - #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX >= 0x030600B1) - #endif - #ifndef CYTHON_USE_EXC_INFO_STACK - #define CYTHON_USE_EXC_INFO_STACK (PY_VERSION_HEX >= 0x030700A3) - #endif -#endif -#if !defined(CYTHON_FAST_PYCCALL) -#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) -#endif -#if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" - #undef SHIFT - #undef BASE - #undef MASK - #ifdef SIZEOF_VOID_P - enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; - #endif -#endif -#ifndef __has_attribute - #define __has_attribute(x) 0 -#endif -#ifndef __has_cpp_attribute - #define __has_cpp_attribute(x) 0 -#endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_MAYBE_UNUSED_VAR -# if defined(__cplusplus) - template void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } -# else -# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) -#ifdef _MSC_VER - #ifndef _MSC_STDINT_H_ - #if _MSC_VER < 1300 - typedef unsigned char uint8_t; - typedef unsigned int uint32_t; - #else - typedef unsigned __int8 uint8_t; - typedef unsigned __int32 uint32_t; - #endif - #endif -#else - #include -#endif -#ifndef CYTHON_FALLTHROUGH - #if defined(__cplusplus) && __cplusplus >= 201103L - #if __has_cpp_attribute(fallthrough) - #define CYTHON_FALLTHROUGH [[fallthrough]] - #elif __has_cpp_attribute(clang::fallthrough) - #define CYTHON_FALLTHROUGH [[clang::fallthrough]] - #elif __has_cpp_attribute(gnu::fallthrough) - #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] - #endif - #endif - #ifndef CYTHON_FALLTHROUGH - #if __has_attribute(fallthrough) - #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) - #else - #define CYTHON_FALLTHROUGH - #endif - #endif - #if defined(__clang__ ) && defined(__apple_build_version__) - #if __apple_build_version__ < 7000000 - #undef CYTHON_FALLTHROUGH - #define CYTHON_FALLTHROUGH - #endif - #endif -#endif - -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." -#endif -#ifndef CYTHON_INLINE - #if defined(__clang__) - #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) - #else - #define CYTHON_INLINE inline - #endif -#endif -template -void __Pyx_call_destructor(T& x) { - x.~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - T *operator&() { return ptr; } - operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; } - template bool operator !=(U other) { return *ptr != other; } - private: - T *ptr; -}; - -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) - #define Py_OptimizeFlag 0 -#endif -#define __PYX_BUILD_PY_SSIZE_T "n" -#define CYTHON_FORMAT_SSIZE_T "z" -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyClass_Type -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyType_Type -#endif -#ifndef Py_TPFLAGS_CHECKTYPES - #define Py_TPFLAGS_CHECKTYPES 0 -#endif -#ifndef Py_TPFLAGS_HAVE_INDEX - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#ifndef Py_TPFLAGS_HAVE_NEWBUFFER - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#ifndef Py_TPFLAGS_HAVE_FINALIZE - #define Py_TPFLAGS_HAVE_FINALIZE 0 -#endif -#ifndef METH_STACKLESS - #define METH_STACKLESS 0 -#endif -#if PY_VERSION_HEX <= 0x030700A3 || !defined(METH_FASTCALL) - #ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - #endif - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); - typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames); -#else - #define __Pyx_PyCFunctionFast _PyCFunctionFast - #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords -#endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))))) -#else -#define __Pyx_PyFastCFunction_Check(func) 0 -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030400A1 - #define PyMem_RawMalloc(n) PyMem_Malloc(n) - #define PyMem_RawRealloc(p, n) PyMem_Realloc(p, n) - #define PyMem_RawFree(p) PyMem_Free(p) -#endif -#if CYTHON_COMPILING_IN_PYSTON - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) -#endif -#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 - #define __Pyx_PyThreadState_Current PyThreadState_GET() -#elif PY_VERSION_HEX >= 0x03060000 - #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() -#elif PY_VERSION_HEX >= 0x03000000 - #define __Pyx_PyThreadState_Current PyThreadState_GET() -#else - #define __Pyx_PyThreadState_Current _PyThreadState_Current -#endif -#if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) -#include "pythread.h" -#define Py_tss_NEEDS_INIT 0 -typedef int Py_tss_t; -static CYTHON_INLINE int PyThread_tss_create(Py_tss_t *key) { - *key = PyThread_create_key(); - return 0; -} -static CYTHON_INLINE Py_tss_t * PyThread_tss_alloc(void) { - Py_tss_t *key = (Py_tss_t *)PyObject_Malloc(sizeof(Py_tss_t)); - *key = Py_tss_NEEDS_INIT; - return key; -} -static CYTHON_INLINE void PyThread_tss_free(Py_tss_t *key) { - PyObject_Free(key); -} -static CYTHON_INLINE int PyThread_tss_is_created(Py_tss_t *key) { - return *key != Py_tss_NEEDS_INIT; -} -static CYTHON_INLINE void PyThread_tss_delete(Py_tss_t *key) { - PyThread_delete_key(*key); - *key = Py_tss_NEEDS_INIT; -} -static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { - return PyThread_set_key_value(*key, value); -} -static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { - return PyThread_get_key_value(*key); -} -#endif -#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) -#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) -#else -#define __Pyx_PyDict_NewPresized(n) PyDict_New() -#endif -#if PY_MAJOR_VERSION >= 3 || CYTHON_FUTURE_DIVISION - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS -#define __Pyx_PyDict_GetItemStr(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) -#else -#define __Pyx_PyDict_GetItemStr(dict, name) PyDict_GetItem(dict, name) -#endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) - #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) - #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) -#else - #define CYTHON_PEP393_ENABLED 0 - #define PyUnicode_1BYTE_KIND 1 - #define PyUnicode_2BYTE_KIND 2 - #define PyUnicode_4BYTE_KIND 4 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) - #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) -#endif -#if CYTHON_COMPILING_IN_PYPY - #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) -#else - #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ - PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyString_Check(b) && !PyString_CheckExact(b)))) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) -#else - #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) -#endif -#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) - #define PyObject_ASCII(o) PyObject_Repr(o) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyStringObject PyUnicodeObject - #define PyString_Type PyUnicode_Type - #define PyString_Check PyUnicode_Check - #define PyString_CheckExact PyUnicode_CheckExact - #define PyObject_Unicode PyObject_Str -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) - #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) -#else - #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) - #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) -#endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) -#endif -#if CYTHON_ASSUME_SAFE_MACROS - #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) -#else - #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyIntObject PyLongObject - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define PyNumber_Int PyNumber_Long -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBoolObject PyLongObject -#endif -#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY - #ifndef PyUnicode_InternFromString - #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) - #endif -#endif -#if PY_VERSION_HEX < 0x030200A4 - typedef long Py_hash_t; - #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong -#else - #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : (Py_INCREF(func), func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif -#if CYTHON_USE_ASYNC_SLOTS - #if PY_VERSION_HEX >= 0x030500B1 - #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods - #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) - #else - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL -#endif -#ifndef __Pyx_PyAsyncMethodsStruct - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } __Pyx_PyAsyncMethodsStruct; -#endif - -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES -#endif -#include -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) -#define __Pyx_truncl trunc -#else -#define __Pyx_truncl truncl -#endif - - -#define __PYX_ERR(f_index, lineno, Ln_error) \ -{ \ - __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ -} - -#ifndef __PYX_EXTERN_C - #ifdef __cplusplus - #define __PYX_EXTERN_C extern "C" - #else - #define __PYX_EXTERN_C extern - #endif -#endif - -#define __PYX_HAVE__msgpack___cmsgpack -#define __PYX_HAVE_API__msgpack___cmsgpack -/* Early includes */ -#include -#include -#include "pythread.h" -#include "pack.h" -#include "buff_converter.h" -#include -#include -#include "unpack.h" -#ifdef _OPENMP -#include -#endif /* _OPENMP */ - -#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) -#define CYTHON_WITHOUT_ASSERTIONS -#endif - -typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; - const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; - -#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 1 -#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT (PY_MAJOR_VERSION >= 3 && __PYX_DEFAULT_STRING_ENCODING_IS_UTF8) -#define __PYX_DEFAULT_STRING_ENCODING "ascii" -#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString -#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#define __Pyx_uchar_cast(c) ((unsigned char)c) -#define __Pyx_long_cast(x) ((long)x) -#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ - (sizeof(type) < sizeof(Py_ssize_t)) ||\ - (sizeof(type) > sizeof(Py_ssize_t) &&\ - likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX) &&\ - (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ - v == (type)PY_SSIZE_T_MIN))) ||\ - (sizeof(type) == sizeof(Py_ssize_t) &&\ - (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX))) ) -static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { - return (size_t) i < (size_t) limit; -} -#if defined (__cplusplus) && __cplusplus >= 201103L - #include - #define __Pyx_sst_abs(value) std::abs(value) -#elif SIZEOF_INT >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) abs(value) -#elif SIZEOF_LONG >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) - #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define __Pyx_sst_abs(value) llabs(value) -#elif defined (__GNUC__) - #define __Pyx_sst_abs(value) __builtin_llabs(value) -#else - #define __Pyx_sst_abs(value) ((value<0) ? -value : value) -#endif -static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); -#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) -#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#else - #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize -#endif -#define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) -#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) -#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) -#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) -#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { - const Py_UNICODE *u_end = u; - while (*u_end++) ; - return (size_t)(u_end - u - 1); -} -#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) -#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode -#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode -#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) -#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); -#define __Pyx_PySequence_Tuple(obj)\ - (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_ASSUME_SAFE_MACROS -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif -#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) -#if PY_MAJOR_VERSION >= 3 -#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) -#else -#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) -#endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII -static int __Pyx_sys_getdefaultencoding_not_ascii; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - PyObject* ascii_chars_u = NULL; - PyObject* ascii_chars_b = NULL; - const char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - if (strcmp(default_encoding_c, "ascii") == 0) { - __Pyx_sys_getdefaultencoding_not_ascii = 0; - } else { - char ascii_chars[128]; - int c; - for (c = 0; c < 128; c++) { - ascii_chars[c] = c; - } - __Pyx_sys_getdefaultencoding_not_ascii = 1; - ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); - if (!ascii_chars_u) goto bad; - ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); - if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { - PyErr_Format( - PyExc_ValueError, - "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", - default_encoding_c); - goto bad; - } - Py_DECREF(ascii_chars_u); - Py_DECREF(ascii_chars_b); - } - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - Py_XDECREF(ascii_chars_u); - Py_XDECREF(ascii_chars_b); - return -1; -} -#endif -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) -#else -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -static char* __PYX_DEFAULT_STRING_ENCODING; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c) + 1); - if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; - strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - return -1; -} -#endif -#endif - - -/* Test for GCC > 2.95 */ -#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) - #define likely(x) __builtin_expect(!!(x), 1) - #define unlikely(x) __builtin_expect(!!(x), 0) -#else /* !__GNUC__ or GCC < 2.95 */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif /* __GNUC__ */ -static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } - -static PyObject *__pyx_m = NULL; -static PyObject *__pyx_d; -static PyObject *__pyx_b; -static PyObject *__pyx_cython_runtime = NULL; -static PyObject *__pyx_empty_tuple; -static PyObject *__pyx_empty_bytes; -static PyObject *__pyx_empty_unicode; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; - - -static const char *__pyx_f[] = { - "msgpack/_packer.pyx", - "msgpack/_unpacker.pyx", - "stringsource", - "msgpack/_cmsgpack.pyx", - "type.pxd", - "bool.pxd", - "complex.pxd", -}; - -/* "msgpack/_unpacker.pyx":13 - * from libc.string cimport * - * from libc.limits cimport * - * ctypedef unsigned long long uint64_t # <<<<<<<<<<<<<< - * - * from ddtrace.vendor.msgpack.exceptions import ( - */ -typedef unsigned PY_LONG_LONG __pyx_t_7msgpack_9_cmsgpack_uint64_t; - -/*--- Type declarations ---*/ -struct __pyx_obj_7msgpack_9_cmsgpack_Packer; -struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker; -struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack; -struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack; - -/* "msgpack/_packer.pyx":148 - * self.pk.buf = NULL - * - * cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: # <<<<<<<<<<<<<< - * cdef long long llval - * cdef unsigned long long ullval - */ -struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack { - int __pyx_n; - int nest_limit; -}; - -/* "msgpack/_unpacker.pyx":477 - * self.file_like = None - * - * cdef object _unpack(self, execute_fn execute, bint iter=0): # <<<<<<<<<<<<<< - * cdef int ret - * cdef object obj - */ -struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack { - int __pyx_n; - int iter; -}; - -/* "msgpack/_packer.pyx":54 - * - * - * cdef class Packer(object): # <<<<<<<<<<<<<< - * """ - * MessagePack Packer - */ -struct __pyx_obj_7msgpack_9_cmsgpack_Packer { - PyObject_HEAD - struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *__pyx_vtab; - struct msgpack_packer pk; - PyObject *_default; - PyObject *_bencoding; - PyObject *_berrors; - char const *encoding; - char const *unicode_errors; - int strict_types; - PyBoolObject *use_float; - int autoreset; -}; - - -/* "msgpack/_unpacker.pyx":229 - * - * - * cdef class Unpacker(object): # <<<<<<<<<<<<<< - * """Streaming unpacker. - * - */ -struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker { - PyObject_HEAD - struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *__pyx_vtab; - unpack_context ctx; - char *buf; - Py_ssize_t buf_size; - Py_ssize_t buf_head; - Py_ssize_t buf_tail; - PyObject *file_like; - PyObject *file_like_read; - Py_ssize_t read_size; - PyObject *object_hook; - PyObject *object_pairs_hook; - PyObject *list_hook; - PyObject *ext_hook; - PyObject *encoding; - PyObject *unicode_errors; - Py_ssize_t max_buffer_size; - __pyx_t_7msgpack_9_cmsgpack_uint64_t stream_offset; -}; - - - -/* "msgpack/_packer.pyx":54 - * - * - * cdef class Packer(object): # <<<<<<<<<<<<<< - * """ - * MessagePack Packer - */ - -struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer { - int (*_pack)(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *, PyObject *, struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack *__pyx_optional_args); - PyObject *(*pack)(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *, PyObject *, int __pyx_skip_dispatch); -}; -static struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *__pyx_vtabptr_7msgpack_9_cmsgpack_Packer; - - -/* "msgpack/_unpacker.pyx":229 - * - * - * cdef class Unpacker(object): # <<<<<<<<<<<<<< - * """Streaming unpacker. - * - */ - -struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker { - PyObject *(*append_buffer)(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *, void *, Py_ssize_t); - PyObject *(*read_from_file)(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *); - PyObject *(*_unpack)(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *, execute_fn, struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack *__pyx_optional_args); -}; -static struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *__pyx_vtabptr_7msgpack_9_cmsgpack_Unpacker; - -/* --- Runtime support code (head) --- */ -/* Refnanny.proto */ -#ifndef CYTHON_REFNANNY - #define CYTHON_REFNANNY 0 -#endif -#if CYTHON_REFNANNY - typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); - void (*FinishContext)(void**); - } __Pyx_RefNannyAPIStruct; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); - #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; -#ifdef WITH_THREAD - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - if (acquire_gil) {\ - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - PyGILState_Release(__pyx_gilstate_save);\ - } else {\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - } -#else - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) -#endif - #define __Pyx_RefNannyFinishContext()\ - __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) -#else - #define __Pyx_RefNannyDeclarations - #define __Pyx_RefNannySetupContext(name, acquire_gil) - #define __Pyx_RefNannyFinishContext() - #define __Pyx_INCREF(r) Py_INCREF(r) - #define __Pyx_DECREF(r) Py_DECREF(r) - #define __Pyx_GOTREF(r) - #define __Pyx_GIVEREF(r) - #define __Pyx_XINCREF(r) Py_XINCREF(r) - #define __Pyx_XDECREF(r) Py_XDECREF(r) - #define __Pyx_XGOTREF(r) - #define __Pyx_XGIVEREF(r) -#endif -#define __Pyx_XDECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_XDECREF(tmp);\ - } while (0) -#define __Pyx_DECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_DECREF(tmp);\ - } while (0) -#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) -#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) - -/* PyObjectGetAttrStr.proto */ -#if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) -#endif - -/* GetBuiltinName.proto */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name); - -/* RaiseArgTupleInvalid.proto */ -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); - -/* KeywordStringCheck.proto */ -static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; -#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#define __Pyx_PyErr_Occurred() PyErr_Occurred() -#endif - -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#if CYTHON_COMPILING_IN_CPYTHON -#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) -#else -#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) -#endif -#else -#define __Pyx_PyErr_Clear() PyErr_Clear() -#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) -#endif - -/* RaiseException.proto */ -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - -/* RaiseDoubleKeywords.proto */ -static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); - -/* ParseKeywords.proto */ -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ - const char* function_name); - -/* ExtTypeTest.proto */ -static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); - -/* GetTopmostException.proto */ -#if CYTHON_USE_EXC_INFO_STACK -static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); -#endif - -/* SaveResetException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -#else -#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) -#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) -#endif - -/* PyErrExceptionMatches.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) -static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); -#else -#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) -#endif - -/* GetException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#define __Pyx_BUILD_ASSERT_EXPR(cond)\ - (sizeof(char [1 - 2*!(cond)]) - 1) -#ifndef Py_MEMBER_SIZE -#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#endif - static size_t __pyx_pyframe_localsplus_offset = 0; - #include "frameobject.h" - #define __Pxy_PyFrame_Initialize_Offsets()\ - ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ - (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) - #define __Pyx_PyFrame_GetLocalsplus(frame)\ - (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) -#endif - -/* PyObjectCall2Args.proto */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - -/* SwapException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); -#endif - -/* IterFinish.proto */ -static CYTHON_INLINE int __Pyx_IterFinish(void); - -/* PyObjectCallNoArg.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) -#endif - -/* PyObjectGetMethod.proto */ -static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); - -/* PyObjectCallMethod0.proto */ -static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); - -/* RaiseNeedMoreValuesToUnpack.proto */ -static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -/* RaiseTooManyValuesToUnpack.proto */ -static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); - -/* UnpackItemEndCheck.proto */ -static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); - -/* RaiseNoneIterError.proto */ -static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void); - -/* UnpackTupleError.proto */ -static void __Pyx_UnpackTupleError(PyObject *, Py_ssize_t index); - -/* UnpackTuple2.proto */ -#define __Pyx_unpack_tuple2(tuple, value1, value2, is_tuple, has_known_size, decref_tuple)\ - (likely(is_tuple || PyTuple_Check(tuple)) ?\ - (likely(has_known_size || PyTuple_GET_SIZE(tuple) == 2) ?\ - __Pyx_unpack_tuple2_exact(tuple, value1, value2, decref_tuple) :\ - (__Pyx_UnpackTupleError(tuple, 2), -1)) :\ - __Pyx_unpack_tuple2_generic(tuple, value1, value2, has_known_size, decref_tuple)) -static CYTHON_INLINE int __Pyx_unpack_tuple2_exact( - PyObject* tuple, PyObject** value1, PyObject** value2, int decref_tuple); -static int __Pyx_unpack_tuple2_generic( - PyObject* tuple, PyObject** value1, PyObject** value2, int has_known_size, int decref_tuple); - -/* dict_iter.proto */ -static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* dict, int is_dict, PyObject* method_name, - Py_ssize_t* p_orig_length, int* p_is_dict); -static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t orig_length, Py_ssize_t* ppos, - PyObject** pkey, PyObject** pvalue, PyObject** pitem, int is_dict); - -/* PyDictVersioning.proto */ -#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS -#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) -#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) -#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ - (version_var) = __PYX_GET_DICT_VERSION(dict);\ - (cache_var) = (value); -#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ - (VAR) = __pyx_dict_cached_value;\ - } else {\ - (VAR) = __pyx_dict_cached_value = (LOOKUP);\ - __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ - }\ -} -static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); -static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); -static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); -#else -#define __PYX_GET_DICT_VERSION(dict) (0) -#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) -#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); -#endif - -/* GetModuleGlobalName.proto */ -#if CYTHON_USE_DICT_VERSIONS -#define __Pyx_GetModuleGlobalName(var, name) {\ - static PY_UINT64_T __pyx_dict_version = 0;\ - static PyObject *__pyx_dict_cached_value = NULL;\ - (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ - (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ - __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ - PY_UINT64_T __pyx_dict_version;\ - PyObject *__pyx_dict_cached_value;\ - (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ -} -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); -#else -#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) -#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); -#endif - -/* ReRaiseException.proto */ -static CYTHON_INLINE void __Pyx_ReraiseException(void); - -/* None.proto */ -static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t, Py_ssize_t); - -/* BuildPyUnicode.proto */ -static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, char* chars, int clength, - int prepend_sign, char padding_char); - -/* CIntToPyUnicode.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char); - -/* PyObject_GenericGetAttrNoDict.proto */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); -#else -#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr -#endif - -/* PyObject_GenericGetAttr.proto */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name); -#else -#define __Pyx_PyObject_GenericGetAttr PyObject_GenericGetAttr -#endif - -/* SetVTable.proto */ -static int __Pyx_SetVtable(PyObject *dict, void *vtable); - -/* SetupReduce.proto */ -static int __Pyx_setup_reduce(PyObject* type_obj); - -/* TypeImport.proto */ -#ifndef __PYX_HAVE_RT_ImportType_proto -#define __PYX_HAVE_RT_ImportType_proto -enum __Pyx_ImportType_CheckSize { - __Pyx_ImportType_CheckSize_Error = 0, - __Pyx_ImportType_CheckSize_Warn = 1, - __Pyx_ImportType_CheckSize_Ignore = 2 -}; -static PyTypeObject *__Pyx_ImportType(PyObject* module, const char *module_name, const char *class_name, size_t size, enum __Pyx_ImportType_CheckSize check_size); -#endif - -/* Import.proto */ -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); - -/* ImportFrom.proto */ -static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); - -/* CLineInTraceback.proto */ -#ifdef CYTHON_CLINE_IN_TRACEBACK -#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) -#else -static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); -#endif - -/* CodeObjectCache.proto */ -typedef struct { - PyCodeObject* code_object; - int code_line; -} __Pyx_CodeObjectCacheEntry; -struct __Pyx_CodeObjectCache { - int count; - int max_count; - __Pyx_CodeObjectCacheEntry* entries; -}; -static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); -static PyCodeObject *__pyx_find_code_object(int code_line); -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); - -/* AddTraceback.proto */ -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_PY_LONG_LONG(unsigned PY_LONG_LONG value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_As_PY_LONG_LONG(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_As_unsigned_PY_LONG_LONG(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); - -/* FastTypeChecks.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) -static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); -static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); -static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); -#else -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) -#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) -#endif -#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) - -/* CheckBinaryVersion.proto */ -static int __Pyx_check_binary_version(void); - -/* InitStrings.proto */ -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); - -static int __pyx_f_7msgpack_9_cmsgpack_6Packer__pack(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o, struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack *__pyx_optional_args); /* proto*/ -static PyObject *__pyx_f_7msgpack_9_cmsgpack_6Packer_pack(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_skip_dispatch); /* proto*/ -static PyObject *__pyx_f_7msgpack_9_cmsgpack_8Unpacker_append_buffer(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, void *__pyx_v__buf, Py_ssize_t __pyx_v__buf_len); /* proto*/ -static PyObject *__pyx_f_7msgpack_9_cmsgpack_8Unpacker_read_from_file(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto*/ -static PyObject *__pyx_f_7msgpack_9_cmsgpack_8Unpacker__unpack(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, execute_fn __pyx_v_execute, struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack *__pyx_optional_args); /* proto*/ - -/* Module declarations from 'cpython.version' */ - -/* Module declarations from '__builtin__' */ - -/* Module declarations from 'cpython.type' */ -static PyTypeObject *__pyx_ptype_7cpython_4type_type = 0; - -/* Module declarations from 'libc.string' */ - -/* Module declarations from 'libc.stdio' */ - -/* Module declarations from 'cpython.object' */ - -/* Module declarations from 'cpython.ref' */ - -/* Module declarations from 'cpython.exc' */ - -/* Module declarations from 'cpython.module' */ - -/* Module declarations from 'cpython.mem' */ - -/* Module declarations from 'cpython.tuple' */ - -/* Module declarations from 'cpython.list' */ - -/* Module declarations from 'cpython.sequence' */ - -/* Module declarations from 'cpython.mapping' */ - -/* Module declarations from 'cpython.iterator' */ - -/* Module declarations from 'cpython.number' */ - -/* Module declarations from 'cpython.int' */ - -/* Module declarations from '__builtin__' */ - -/* Module declarations from 'cpython.bool' */ -static PyTypeObject *__pyx_ptype_7cpython_4bool_bool = 0; - -/* Module declarations from 'cpython.long' */ - -/* Module declarations from 'cpython.float' */ - -/* Module declarations from '__builtin__' */ - -/* Module declarations from 'cpython.complex' */ -static PyTypeObject *__pyx_ptype_7cpython_7complex_complex = 0; - -/* Module declarations from 'cpython.string' */ - -/* Module declarations from 'cpython.unicode' */ - -/* Module declarations from 'cpython.dict' */ - -/* Module declarations from 'cpython.instance' */ - -/* Module declarations from 'cpython.function' */ - -/* Module declarations from 'cpython.method' */ - -/* Module declarations from 'cpython.weakref' */ - -/* Module declarations from 'cpython.getargs' */ - -/* Module declarations from 'cpython.pythread' */ - -/* Module declarations from 'cpython.pystate' */ - -/* Module declarations from 'cpython.cobject' */ - -/* Module declarations from 'cpython.oldbuffer' */ - -/* Module declarations from 'cpython.set' */ - -/* Module declarations from 'cpython.buffer' */ - -/* Module declarations from 'cpython.bytes' */ - -/* Module declarations from 'cpython.pycapsule' */ - -/* Module declarations from 'cpython' */ - -/* Module declarations from 'cpython.bytearray' */ - -/* Module declarations from 'libc.stdlib' */ - -/* Module declarations from 'libc.limits' */ - -/* Module declarations from 'msgpack._cmsgpack' */ -static PyTypeObject *__pyx_ptype_7msgpack_9_cmsgpack_Packer = 0; -static PyTypeObject *__pyx_ptype_7msgpack_9_cmsgpack_Unpacker = 0; -static int __pyx_v_7msgpack_9_cmsgpack_DEFAULT_RECURSE_LIMIT; -static PY_LONG_LONG __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT; -static CYTHON_INLINE int __pyx_f_7msgpack_9_cmsgpack_PyBytesLike_Check(PyObject *); /*proto*/ -static CYTHON_INLINE int __pyx_f_7msgpack_9_cmsgpack_PyBytesLike_CheckExact(PyObject *); /*proto*/ -static CYTHON_INLINE PyObject *__pyx_f_7msgpack_9_cmsgpack_init_ctx(unpack_context *, PyObject *, PyObject *, PyObject *, PyObject *, int, int, int, char const *, char const *, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t); /*proto*/ -static CYTHON_INLINE int __pyx_f_7msgpack_9_cmsgpack_get_data_from_buffer(PyObject *, Py_buffer *, char **, Py_ssize_t *, int *); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack._cmsgpack" -extern int __pyx_module_is_main_msgpack___cmsgpack; -int __pyx_module_is_main_msgpack___cmsgpack = 0; - -/* Implementation of 'msgpack._cmsgpack' */ -static PyObject *__pyx_builtin_MemoryError; -static PyObject *__pyx_builtin_DeprecationWarning; -static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_builtin_ValueError; -static PyObject *__pyx_builtin_OverflowError; -static PyObject *__pyx_builtin_RuntimeError; -static PyObject *__pyx_builtin_NotImplementedError; -static PyObject *__pyx_builtin_BufferError; -static PyObject *__pyx_builtin_RuntimeWarning; -static PyObject *__pyx_builtin_AssertionError; -static PyObject *__pyx_builtin_StopIteration; -static const char __pyx_k_d[] = "d"; -static const char __pyx_k_buf[] = "buf"; -static const char __pyx_k_ctx[] = "ctx"; -static const char __pyx_k_obj[] = "obj"; -static const char __pyx_k_off[] = "off"; -static const char __pyx_k_raw[] = "raw"; -static const char __pyx_k_ret[] = "ret"; -static const char __pyx_k_cenc[] = "cenc"; -static const char __pyx_k_cerr[] = "cerr"; -static const char __pyx_k_code[] = "code"; -static const char __pyx_k_data[] = "data"; -static const char __pyx_k_main[] = "__main__"; -static const char __pyx_k_name[] = "__name__"; -static const char __pyx_k_pack[] = "pack"; -static const char __pyx_k_read[] = "read"; -static const char __pyx_k_test[] = "__test__"; -static const char __pyx_k_view[] = "view"; -static const char __pyx_k_items[] = "items"; -static const char __pyx_k_Packer[] = "Packer"; -static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_kwargs[] = "kwargs"; -static const char __pyx_k_packed[] = "packed"; -static const char __pyx_k_reduce[] = "__reduce__"; -static const char __pyx_k_stream[] = "stream"; -static const char __pyx_k_unpack[] = "unpack"; -static const char __pyx_k_ExtType[] = "ExtType"; -static const char __pyx_k_buf_len[] = "buf_len"; -static const char __pyx_k_default[] = "default"; -static const char __pyx_k_unpackb[] = "unpackb"; -static const char __pyx_k_Unpacker[] = "Unpacker"; -static const char __pyx_k_encoding[] = "encoding"; -static const char __pyx_k_ext_hook[] = "ext_hook"; -static const char __pyx_k_getstate[] = "__getstate__"; -static const char __pyx_k_setstate[] = "__setstate__"; -static const char __pyx_k_typecode[] = "typecode"; -static const char __pyx_k_use_list[] = "use_list"; -static const char __pyx_k_ExtraData[] = "ExtraData"; -static const char __pyx_k_OutOfData[] = "OutOfData"; -static const char __pyx_k_TypeError[] = "TypeError"; -static const char __pyx_k_autoreset[] = "autoreset"; -static const char __pyx_k_file_like[] = "file_like"; -static const char __pyx_k_list_hook[] = "list_hook"; -static const char __pyx_k_read_size[] = "read_size"; -static const char __pyx_k_reduce_ex[] = "__reduce_ex__"; -static const char __pyx_k_BufferFull[] = "BufferFull"; -static const char __pyx_k_StackError[] = "StackError"; -static const char __pyx_k_ValueError[] = "ValueError"; -static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; -static const char __pyx_k_BufferError[] = "BufferError"; -static const char __pyx_k_FormatError[] = "FormatError"; -static const char __pyx_k_MemoryError[] = "MemoryError"; -static const char __pyx_k_max_bin_len[] = "max_bin_len"; -static const char __pyx_k_max_ext_len[] = "max_ext_len"; -static const char __pyx_k_max_map_len[] = "max_map_len"; -static const char __pyx_k_max_str_len[] = "max_str_len"; -static const char __pyx_k_object_hook[] = "object_hook"; -static const char __pyx_k_RuntimeError[] = "RuntimeError"; -static const char __pyx_k_new_protocol[] = "new_protocol"; -static const char __pyx_k_strict_types[] = "strict_types"; -static const char __pyx_k_use_bin_type[] = "use_bin_type"; -static const char __pyx_k_OverflowError[] = "OverflowError"; -static const char __pyx_k_StopIteration[] = "StopIteration"; -static const char __pyx_k_max_array_len[] = "max_array_len"; -static const char __pyx_k_reduce_cython[] = "__reduce_cython__"; -static const char __pyx_k_AssertionError[] = "AssertionError"; -static const char __pyx_k_RuntimeWarning[] = "RuntimeWarning"; -static const char __pyx_k_internal_error[] = "internal error"; -static const char __pyx_k_strict_map_key[] = "strict_map_key"; -static const char __pyx_k_unicode_errors[] = "unicode_errors"; -static const char __pyx_k_max_buffer_size[] = "max_buffer_size"; -static const char __pyx_k_setstate_cython[] = "__setstate_cython__"; -static const char __pyx_k_use_single_float[] = "use_single_float"; -static const char __pyx_k_dict_is_too_large[] = "dict is too large"; -static const char __pyx_k_list_is_too_large[] = "list is too large"; -static const char __pyx_k_msgpack__cmsgpack[] = "msgpack._cmsgpack"; -static const char __pyx_k_object_pairs_hook[] = "object_pairs_hook"; -static const char __pyx_k_DeprecationWarning[] = "DeprecationWarning"; -static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; -static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; -static const char __pyx_k_Unpack_failed_error[] = "Unpack failed: error = "; -static const char __pyx_k_EXT_data_is_too_large[] = "EXT data is too large"; -static const char __pyx_k_msgpack__unpacker_pyx[] = "msgpack/_unpacker.pyx"; -static const char __pyx_k_No_more_data_to_unpack[] = "No more data to unpack."; -static const char __pyx_k_ddtrace_vendor_msgpack[] = "ddtrace.vendor.msgpack"; -static const char __pyx_k_memoryview_is_too_large[] = "memoryview is too large"; -static const char __pyx_k_could_not_get_memoryview[] = "could not get memoryview"; -static const char __pyx_k_recursion_limit_exceeded[] = "recursion limit exceeded."; -static const char __pyx_k_Integer_value_out_of_range[] = "Integer value out of range"; -static const char __pyx_k_default_must_be_a_callable[] = "default must be a callable."; -static const char __pyx_k_default_read_extended_type[] = "default_read_extended_type"; -static const char __pyx_k_ext_hook_must_be_a_callable[] = "ext_hook must be a callable."; -static const char __pyx_k_unicode_string_is_too_large[] = "unicode string is too large"; -static const char __pyx_k_list_hook_must_be_a_callable[] = "list_hook must be a callable."; -static const char __pyx_k_Unpack_failed_incomplete_input[] = "Unpack failed: incomplete input"; -static const char __pyx_k_object_hook_must_be_a_callable[] = "object_hook must be a callable."; -static const char __pyx_k_file_like_read_must_be_a_callab[] = "`file_like.read` must be a callable."; -static const char __pyx_k_unpacker_feed_is_not_be_able_to[] = "unpacker.feed() is not be able to use with `file_like`."; -static const char __pyx_k_Cannot_decode_extended_type_with[] = "Cannot decode extended type with typecode=%d"; -static const char __pyx_k_Unable_to_allocate_internal_buff[] = "Unable to allocate internal buffer."; -static const char __pyx_k_Unable_to_enlarge_internal_buffe[] = "Unable to enlarge internal buffer."; -static const char __pyx_k_cannot_unpack_from_multi_byte_ob[] = "cannot unpack from multi-byte object"; -static const char __pyx_k_could_not_get_buffer_for_memoryv[] = "could not get buffer for memoryview"; -static const char __pyx_k_ddtrace_vendor_msgpack_exception[] = "ddtrace.vendor.msgpack.exceptions"; -static const char __pyx_k_no_default___reduce___due_to_non[] = "no default __reduce__ due to non-trivial __cinit__"; -static const char __pyx_k_object_pairs_hook_and_object_hoo[] = "object_pairs_hook and object_hook are mutually exclusive."; -static const char __pyx_k_object_pairs_hook_must_be_a_call[] = "object_pairs_hook must be a callable."; -static const char __pyx_k_read_size_should_be_less_or_equa[] = "read_size should be less or equal to max_buffer_size"; -static const char __pyx_k_using_old_buffer_interface_to_un[] = "using old buffer interface to unpack %s; this leads to unpacking errors if slicing is used and will be removed in a future version"; -static PyObject *__pyx_n_s_AssertionError; -static PyObject *__pyx_n_s_BufferError; -static PyObject *__pyx_n_s_BufferFull; -static PyObject *__pyx_kp_u_Cannot_decode_extended_type_with; -static PyObject *__pyx_n_s_DeprecationWarning; -static PyObject *__pyx_kp_u_EXT_data_is_too_large; -static PyObject *__pyx_n_s_ExtType; -static PyObject *__pyx_n_s_ExtraData; -static PyObject *__pyx_n_s_FormatError; -static PyObject *__pyx_kp_u_Integer_value_out_of_range; -static PyObject *__pyx_n_s_MemoryError; -static PyObject *__pyx_kp_u_No_more_data_to_unpack; -static PyObject *__pyx_n_s_NotImplementedError; -static PyObject *__pyx_n_s_OutOfData; -static PyObject *__pyx_n_s_OverflowError; -static PyObject *__pyx_n_s_Packer; -static PyObject *__pyx_n_s_RuntimeError; -static PyObject *__pyx_n_s_RuntimeWarning; -static PyObject *__pyx_n_s_StackError; -static PyObject *__pyx_n_s_StopIteration; -static PyObject *__pyx_n_s_TypeError; -static PyObject *__pyx_kp_u_Unable_to_allocate_internal_buff; -static PyObject *__pyx_kp_u_Unable_to_enlarge_internal_buffe; -static PyObject *__pyx_kp_u_Unpack_failed_error; -static PyObject *__pyx_kp_u_Unpack_failed_incomplete_input; -static PyObject *__pyx_n_s_Unpacker; -static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s_autoreset; -static PyObject *__pyx_n_s_buf; -static PyObject *__pyx_n_s_buf_len; -static PyObject *__pyx_kp_u_cannot_unpack_from_multi_byte_ob; -static PyObject *__pyx_n_s_cenc; -static PyObject *__pyx_n_s_cerr; -static PyObject *__pyx_n_s_cline_in_traceback; -static PyObject *__pyx_n_s_code; -static PyObject *__pyx_kp_u_could_not_get_buffer_for_memoryv; -static PyObject *__pyx_kp_u_could_not_get_memoryview; -static PyObject *__pyx_n_s_ctx; -static PyObject *__pyx_n_u_d; -static PyObject *__pyx_n_s_data; -static PyObject *__pyx_n_s_ddtrace_vendor_msgpack; -static PyObject *__pyx_n_s_ddtrace_vendor_msgpack_exception; -static PyObject *__pyx_n_s_default; -static PyObject *__pyx_kp_u_default_must_be_a_callable; -static PyObject *__pyx_n_s_default_read_extended_type; -static PyObject *__pyx_kp_u_dict_is_too_large; -static PyObject *__pyx_n_s_encoding; -static PyObject *__pyx_n_s_ext_hook; -static PyObject *__pyx_kp_u_ext_hook_must_be_a_callable; -static PyObject *__pyx_n_s_file_like; -static PyObject *__pyx_kp_u_file_like_read_must_be_a_callab; -static PyObject *__pyx_n_s_getstate; -static PyObject *__pyx_n_s_import; -static PyObject *__pyx_kp_u_internal_error; -static PyObject *__pyx_n_s_items; -static PyObject *__pyx_n_s_kwargs; -static PyObject *__pyx_n_s_list_hook; -static PyObject *__pyx_kp_u_list_hook_must_be_a_callable; -static PyObject *__pyx_kp_u_list_is_too_large; -static PyObject *__pyx_n_s_main; -static PyObject *__pyx_n_s_max_array_len; -static PyObject *__pyx_n_s_max_bin_len; -static PyObject *__pyx_n_s_max_buffer_size; -static PyObject *__pyx_n_s_max_ext_len; -static PyObject *__pyx_n_s_max_map_len; -static PyObject *__pyx_n_s_max_str_len; -static PyObject *__pyx_kp_u_memoryview_is_too_large; -static PyObject *__pyx_n_s_msgpack__cmsgpack; -static PyObject *__pyx_kp_s_msgpack__unpacker_pyx; -static PyObject *__pyx_n_s_name; -static PyObject *__pyx_n_s_new_protocol; -static PyObject *__pyx_kp_s_no_default___reduce___due_to_non; -static PyObject *__pyx_n_s_obj; -static PyObject *__pyx_n_s_object_hook; -static PyObject *__pyx_kp_u_object_hook_must_be_a_callable; -static PyObject *__pyx_n_s_object_pairs_hook; -static PyObject *__pyx_kp_u_object_pairs_hook_and_object_hoo; -static PyObject *__pyx_kp_u_object_pairs_hook_must_be_a_call; -static PyObject *__pyx_n_s_off; -static PyObject *__pyx_n_s_pack; -static PyObject *__pyx_n_s_packed; -static PyObject *__pyx_n_s_pyx_vtable; -static PyObject *__pyx_n_s_raw; -static PyObject *__pyx_n_s_read; -static PyObject *__pyx_n_s_read_size; -static PyObject *__pyx_kp_u_read_size_should_be_less_or_equa; -static PyObject *__pyx_kp_u_recursion_limit_exceeded; -static PyObject *__pyx_n_s_reduce; -static PyObject *__pyx_n_s_reduce_cython; -static PyObject *__pyx_n_s_reduce_ex; -static PyObject *__pyx_n_s_ret; -static PyObject *__pyx_n_s_setstate; -static PyObject *__pyx_n_s_setstate_cython; -static PyObject *__pyx_n_s_stream; -static PyObject *__pyx_n_s_strict_map_key; -static PyObject *__pyx_n_s_strict_types; -static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_typecode; -static PyObject *__pyx_n_s_unicode_errors; -static PyObject *__pyx_kp_u_unicode_string_is_too_large; -static PyObject *__pyx_n_s_unpack; -static PyObject *__pyx_n_s_unpackb; -static PyObject *__pyx_kp_u_unpacker_feed_is_not_be_able_to; -static PyObject *__pyx_n_s_use_bin_type; -static PyObject *__pyx_n_s_use_list; -static PyObject *__pyx_n_s_use_single_float; -static PyObject *__pyx_kp_u_using_old_buffer_interface_to_un; -static PyObject *__pyx_n_s_view; -static int __pyx_pf_7msgpack_9_cmsgpack_6Packer___cinit__(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self); /* proto */ -static int __pyx_pf_7msgpack_9_cmsgpack_6Packer_2__init__(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_default, PyObject *__pyx_v_encoding, PyObject *__pyx_v_unicode_errors, int __pyx_v_use_single_float, int __pyx_v_autoreset, int __pyx_v_use_bin_type, int __pyx_v_strict_types); /* proto */ -static void __pyx_pf_7msgpack_9_cmsgpack_6Packer_4__dealloc__(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_6pack(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_obj); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_8pack_ext_type(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_typecode, PyObject *__pyx_v_data); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_10pack_array_header(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PY_LONG_LONG __pyx_v_size); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_12pack_map_header(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PY_LONG_LONG __pyx_v_size); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_14pack_map_pairs(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_pairs); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_16reset(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_18bytes(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_20getbuffer(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_22__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_24__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_default_read_extended_type(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_typecode, CYTHON_UNUSED PyObject *__pyx_v_data); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_2unpackb(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_packed, PyObject *__pyx_v_object_hook, PyObject *__pyx_v_list_hook, int __pyx_v_use_list, int __pyx_v_raw, int __pyx_v_strict_map_key, PyObject *__pyx_v_encoding, PyObject *__pyx_v_unicode_errors, PyObject *__pyx_v_object_pairs_hook, PyObject *__pyx_v_ext_hook, Py_ssize_t __pyx_v_max_str_len, Py_ssize_t __pyx_v_max_bin_len, Py_ssize_t __pyx_v_max_array_len, Py_ssize_t __pyx_v_max_map_len, Py_ssize_t __pyx_v_max_ext_len); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_4unpack(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_stream, PyObject *__pyx_v_kwargs); /* proto */ -static int __pyx_pf_7msgpack_9_cmsgpack_8Unpacker___cinit__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static void __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_2__dealloc__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static int __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_4__init__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, PyObject *__pyx_v_file_like, Py_ssize_t __pyx_v_read_size, int __pyx_v_use_list, int __pyx_v_raw, int __pyx_v_strict_map_key, PyObject *__pyx_v_object_hook, PyObject *__pyx_v_object_pairs_hook, PyObject *__pyx_v_list_hook, PyObject *__pyx_v_encoding, PyObject *__pyx_v_unicode_errors, Py_ssize_t __pyx_v_max_buffer_size, PyObject *__pyx_v_ext_hook, Py_ssize_t __pyx_v_max_str_len, Py_ssize_t __pyx_v_max_bin_len, Py_ssize_t __pyx_v_max_array_len, Py_ssize_t __pyx_v_max_map_len, Py_ssize_t __pyx_v_max_ext_len); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_6feed(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, PyObject *__pyx_v_next_bytes); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_8read_bytes(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, Py_ssize_t __pyx_v_nbytes); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_10unpack(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_12skip(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_14read_array_header(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_16read_map_header(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_18tell(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_20__iter__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_22__next__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_24__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_26__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ -static PyObject *__pyx_tp_new_7msgpack_9_cmsgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_7msgpack_9_cmsgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_int_0; -static int __pyx_k__3; -static PyObject *__pyx_k__22; -static PyObject *__pyx_k__24; -static PyObject *__pyx_tuple_; -static PyObject *__pyx_tuple__2; -static PyObject *__pyx_tuple__4; -static PyObject *__pyx_tuple__5; -static PyObject *__pyx_tuple__6; -static PyObject *__pyx_tuple__7; -static PyObject *__pyx_tuple__8; -static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__10; -static PyObject *__pyx_tuple__11; -static PyObject *__pyx_tuple__12; -static PyObject *__pyx_tuple__13; -static PyObject *__pyx_tuple__14; -static PyObject *__pyx_tuple__15; -static PyObject *__pyx_tuple__16; -static PyObject *__pyx_tuple__17; -static PyObject *__pyx_tuple__18; -static PyObject *__pyx_tuple__19; -static PyObject *__pyx_tuple__20; -static PyObject *__pyx_tuple__21; -static PyObject *__pyx_tuple__23; -static PyObject *__pyx_tuple__25; -static PyObject *__pyx_tuple__26; -static PyObject *__pyx_tuple__27; -static PyObject *__pyx_tuple__28; -static PyObject *__pyx_tuple__29; -static PyObject *__pyx_tuple__30; -static PyObject *__pyx_tuple__31; -static PyObject *__pyx_tuple__32; -static PyObject *__pyx_tuple__34; -static PyObject *__pyx_tuple__36; -static PyObject *__pyx_codeobj__33; -static PyObject *__pyx_codeobj__35; -static PyObject *__pyx_codeobj__37; -/* Late includes */ - -/* "msgpack/_packer.pyx":46 - * - * - * cdef inline int PyBytesLike_Check(object o): # <<<<<<<<<<<<<< - * return PyBytes_Check(o) or PyByteArray_Check(o) - * - */ - -static CYTHON_INLINE int __pyx_f_7msgpack_9_cmsgpack_PyBytesLike_Check(PyObject *__pyx_v_o) { - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - __Pyx_RefNannySetupContext("PyBytesLike_Check", 0); - - /* "msgpack/_packer.pyx":47 - * - * cdef inline int PyBytesLike_Check(object o): - * return PyBytes_Check(o) or PyByteArray_Check(o) # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_2 = PyBytes_Check(__pyx_v_o); - if (!__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L3_bool_binop_done; - } - __pyx_t_2 = PyByteArray_Check(__pyx_v_o); - __pyx_t_1 = __pyx_t_2; - __pyx_L3_bool_binop_done:; - __pyx_r = __pyx_t_1; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":46 - * - * - * cdef inline int PyBytesLike_Check(object o): # <<<<<<<<<<<<<< - * return PyBytes_Check(o) or PyByteArray_Check(o) - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":50 - * - * - * cdef inline int PyBytesLike_CheckExact(object o): # <<<<<<<<<<<<<< - * return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) - * - */ - -static CYTHON_INLINE int __pyx_f_7msgpack_9_cmsgpack_PyBytesLike_CheckExact(PyObject *__pyx_v_o) { - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - __Pyx_RefNannySetupContext("PyBytesLike_CheckExact", 0); - - /* "msgpack/_packer.pyx":51 - * - * cdef inline int PyBytesLike_CheckExact(object o): - * return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_2 = PyBytes_CheckExact(__pyx_v_o); - if (!__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L3_bool_binop_done; - } - __pyx_t_2 = PyByteArray_CheckExact(__pyx_v_o); - __pyx_t_1 = __pyx_t_2; - __pyx_L3_bool_binop_done:; - __pyx_r = __pyx_t_1; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":50 - * - * - * cdef inline int PyBytesLike_CheckExact(object o): # <<<<<<<<<<<<<< - * return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":107 - * cdef bint autoreset - * - * def __cinit__(self): # <<<<<<<<<<<<<< - * cdef int buf_size = 1024*1024 - * self.pk.buf = PyMem_Malloc(buf_size) - */ - -/* Python wrapper */ -static int __pyx_pw_7msgpack_9_cmsgpack_6Packer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_7msgpack_9_cmsgpack_6Packer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} - if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__cinit__", 0))) return -1; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer___cinit__(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_7msgpack_9_cmsgpack_6Packer___cinit__(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self) { - int __pyx_v_buf_size; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "msgpack/_packer.pyx":108 - * - * def __cinit__(self): - * cdef int buf_size = 1024*1024 # <<<<<<<<<<<<<< - * self.pk.buf = PyMem_Malloc(buf_size) - * if self.pk.buf == NULL: - */ - __pyx_v_buf_size = 0x100000; - - /* "msgpack/_packer.pyx":109 - * def __cinit__(self): - * cdef int buf_size = 1024*1024 - * self.pk.buf = PyMem_Malloc(buf_size) # <<<<<<<<<<<<<< - * if self.pk.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") - */ - __pyx_v_self->pk.buf = ((char *)PyMem_Malloc(__pyx_v_buf_size)); - - /* "msgpack/_packer.pyx":110 - * cdef int buf_size = 1024*1024 - * self.pk.buf = PyMem_Malloc(buf_size) - * if self.pk.buf == NULL: # <<<<<<<<<<<<<< - * raise MemoryError("Unable to allocate internal buffer.") - * self.pk.buf_size = buf_size - */ - __pyx_t_1 = ((__pyx_v_self->pk.buf == NULL) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":111 - * self.pk.buf = PyMem_Malloc(buf_size) - * if self.pk.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") # <<<<<<<<<<<<<< - * self.pk.buf_size = buf_size - * self.pk.length = 0 - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(0, 111, __pyx_L1_error) - - /* "msgpack/_packer.pyx":110 - * cdef int buf_size = 1024*1024 - * self.pk.buf = PyMem_Malloc(buf_size) - * if self.pk.buf == NULL: # <<<<<<<<<<<<<< - * raise MemoryError("Unable to allocate internal buffer.") - * self.pk.buf_size = buf_size - */ - } - - /* "msgpack/_packer.pyx":112 - * if self.pk.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") - * self.pk.buf_size = buf_size # <<<<<<<<<<<<<< - * self.pk.length = 0 - * - */ - __pyx_v_self->pk.buf_size = __pyx_v_buf_size; - - /* "msgpack/_packer.pyx":113 - * raise MemoryError("Unable to allocate internal buffer.") - * self.pk.buf_size = buf_size - * self.pk.length = 0 # <<<<<<<<<<<<<< - * - * def __init__(self, default=None, encoding=None, unicode_errors=None, - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":107 - * cdef bint autoreset - * - * def __cinit__(self): # <<<<<<<<<<<<<< - * cdef int buf_size = 1024*1024 - * self.pk.buf = PyMem_Malloc(buf_size) - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":115 - * self.pk.length = 0 - * - * def __init__(self, default=None, encoding=None, unicode_errors=None, # <<<<<<<<<<<<<< - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - * bint strict_types=False): - */ - -/* Python wrapper */ -static int __pyx_pw_7msgpack_9_cmsgpack_6Packer_3__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_7msgpack_9_cmsgpack_6Packer_3__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_default = 0; - PyObject *__pyx_v_encoding = 0; - PyObject *__pyx_v_unicode_errors = 0; - int __pyx_v_use_single_float; - int __pyx_v_autoreset; - int __pyx_v_use_bin_type; - int __pyx_v_strict_types; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_default,&__pyx_n_s_encoding,&__pyx_n_s_unicode_errors,&__pyx_n_s_use_single_float,&__pyx_n_s_autoreset,&__pyx_n_s_use_bin_type,&__pyx_n_s_strict_types,0}; - PyObject* values[7] = {0,0,0,0,0,0,0}; - values[0] = ((PyObject *)Py_None); - values[1] = ((PyObject *)Py_None); - values[2] = ((PyObject *)Py_None); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - CYTHON_FALLTHROUGH; - case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - CYTHON_FALLTHROUGH; - case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - CYTHON_FALLTHROUGH; - case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - CYTHON_FALLTHROUGH; - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_default); - if (value) { values[0] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 1: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_encoding); - if (value) { values[1] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 2: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_unicode_errors); - if (value) { values[2] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 3: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_use_single_float); - if (value) { values[3] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 4: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_autoreset); - if (value) { values[4] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 5: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_use_bin_type); - if (value) { values[5] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 6: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_strict_types); - if (value) { values[6] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(0, 115, __pyx_L3_error) - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - CYTHON_FALLTHROUGH; - case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - CYTHON_FALLTHROUGH; - case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - CYTHON_FALLTHROUGH; - case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - CYTHON_FALLTHROUGH; - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_default = values[0]; - __pyx_v_encoding = values[1]; - __pyx_v_unicode_errors = values[2]; - if (values[3]) { - __pyx_v_use_single_float = __Pyx_PyObject_IsTrue(values[3]); if (unlikely((__pyx_v_use_single_float == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 116, __pyx_L3_error) - } else { - - /* "msgpack/_packer.pyx":116 - * - * def __init__(self, default=None, encoding=None, unicode_errors=None, - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, # <<<<<<<<<<<<<< - * bint strict_types=False): - * if encoding is not None: - */ - __pyx_v_use_single_float = ((int)0); - } - if (values[4]) { - __pyx_v_autoreset = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_autoreset == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 116, __pyx_L3_error) - } else { - __pyx_v_autoreset = ((int)1); - } - if (values[5]) { - __pyx_v_use_bin_type = __Pyx_PyObject_IsTrue(values[5]); if (unlikely((__pyx_v_use_bin_type == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 116, __pyx_L3_error) - } else { - __pyx_v_use_bin_type = ((int)0); - } - if (values[6]) { - __pyx_v_strict_types = __Pyx_PyObject_IsTrue(values[6]); if (unlikely((__pyx_v_strict_types == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 117, __pyx_L3_error) - } else { - - /* "msgpack/_packer.pyx":117 - * def __init__(self, default=None, encoding=None, unicode_errors=None, - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - * bint strict_types=False): # <<<<<<<<<<<<<< - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) - */ - __pyx_v_strict_types = ((int)0); - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 0, 7, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 115, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_2__init__(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), __pyx_v_default, __pyx_v_encoding, __pyx_v_unicode_errors, __pyx_v_use_single_float, __pyx_v_autoreset, __pyx_v_use_bin_type, __pyx_v_strict_types); - - /* "msgpack/_packer.pyx":115 - * self.pk.length = 0 - * - * def __init__(self, default=None, encoding=None, unicode_errors=None, # <<<<<<<<<<<<<< - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - * bint strict_types=False): - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_7msgpack_9_cmsgpack_6Packer_2__init__(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_default, PyObject *__pyx_v_encoding, PyObject *__pyx_v_unicode_errors, int __pyx_v_use_single_float, int __pyx_v_autoreset, int __pyx_v_use_bin_type, int __pyx_v_strict_types) { - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - char const *__pyx_t_5; - char const *__pyx_t_6; - __Pyx_RefNannySetupContext("__init__", 0); - - /* "msgpack/_packer.pyx":118 - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - * bint strict_types=False): - * if encoding is not None: # <<<<<<<<<<<<<< - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) - * self.use_float = use_single_float - */ - __pyx_t_1 = (__pyx_v_encoding != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "msgpack/_packer.pyx":119 - * bint strict_types=False): - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) # <<<<<<<<<<<<<< - * self.use_float = use_single_float - * self.strict_types = strict_types - */ - __pyx_t_3 = PyErr_WarnEx(__pyx_builtin_DeprecationWarning, ((char *)"encoding is deprecated."), 1); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(0, 119, __pyx_L1_error) - - /* "msgpack/_packer.pyx":118 - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - * bint strict_types=False): - * if encoding is not None: # <<<<<<<<<<<<<< - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) - * self.use_float = use_single_float - */ - } - - /* "msgpack/_packer.pyx":120 - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) - * self.use_float = use_single_float # <<<<<<<<<<<<<< - * self.strict_types = strict_types - * self.autoreset = autoreset - */ - __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_use_single_float); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 120, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - if (!(likely(__Pyx_TypeTest(__pyx_t_4, __pyx_ptype_7cpython_4bool_bool)))) __PYX_ERR(0, 120, __pyx_L1_error) - __Pyx_GIVEREF(__pyx_t_4); - __Pyx_GOTREF(__pyx_v_self->use_float); - __Pyx_DECREF(((PyObject *)__pyx_v_self->use_float)); - __pyx_v_self->use_float = ((PyBoolObject *)__pyx_t_4); - __pyx_t_4 = 0; - - /* "msgpack/_packer.pyx":121 - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) - * self.use_float = use_single_float - * self.strict_types = strict_types # <<<<<<<<<<<<<< - * self.autoreset = autoreset - * self.pk.use_bin_type = use_bin_type - */ - __pyx_v_self->strict_types = __pyx_v_strict_types; - - /* "msgpack/_packer.pyx":122 - * self.use_float = use_single_float - * self.strict_types = strict_types - * self.autoreset = autoreset # <<<<<<<<<<<<<< - * self.pk.use_bin_type = use_bin_type - * if default is not None: - */ - __pyx_v_self->autoreset = __pyx_v_autoreset; - - /* "msgpack/_packer.pyx":123 - * self.strict_types = strict_types - * self.autoreset = autoreset - * self.pk.use_bin_type = use_bin_type # <<<<<<<<<<<<<< - * if default is not None: - * if not PyCallable_Check(default): - */ - __pyx_v_self->pk.use_bin_type = __pyx_v_use_bin_type; - - /* "msgpack/_packer.pyx":124 - * self.autoreset = autoreset - * self.pk.use_bin_type = use_bin_type - * if default is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(default): - * raise TypeError("default must be a callable.") - */ - __pyx_t_2 = (__pyx_v_default != Py_None); - __pyx_t_1 = (__pyx_t_2 != 0); - if (__pyx_t_1) { - - /* "msgpack/_packer.pyx":125 - * self.pk.use_bin_type = use_bin_type - * if default is not None: - * if not PyCallable_Check(default): # <<<<<<<<<<<<<< - * raise TypeError("default must be a callable.") - * self._default = default - */ - __pyx_t_1 = ((!(PyCallable_Check(__pyx_v_default) != 0)) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":126 - * if default is not None: - * if not PyCallable_Check(default): - * raise TypeError("default must be a callable.") # <<<<<<<<<<<<<< - * self._default = default - * - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 126, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(0, 126, __pyx_L1_error) - - /* "msgpack/_packer.pyx":125 - * self.pk.use_bin_type = use_bin_type - * if default is not None: - * if not PyCallable_Check(default): # <<<<<<<<<<<<<< - * raise TypeError("default must be a callable.") - * self._default = default - */ - } - - /* "msgpack/_packer.pyx":124 - * self.autoreset = autoreset - * self.pk.use_bin_type = use_bin_type - * if default is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(default): - * raise TypeError("default must be a callable.") - */ - } - - /* "msgpack/_packer.pyx":127 - * if not PyCallable_Check(default): - * raise TypeError("default must be a callable.") - * self._default = default # <<<<<<<<<<<<<< - * - * self._bencoding = encoding - */ - __Pyx_INCREF(__pyx_v_default); - __Pyx_GIVEREF(__pyx_v_default); - __Pyx_GOTREF(__pyx_v_self->_default); - __Pyx_DECREF(__pyx_v_self->_default); - __pyx_v_self->_default = __pyx_v_default; - - /* "msgpack/_packer.pyx":129 - * self._default = default - * - * self._bencoding = encoding # <<<<<<<<<<<<<< - * if encoding is None: - * if PY_MAJOR_VERSION < 3: - */ - __Pyx_INCREF(__pyx_v_encoding); - __Pyx_GIVEREF(__pyx_v_encoding); - __Pyx_GOTREF(__pyx_v_self->_bencoding); - __Pyx_DECREF(__pyx_v_self->_bencoding); - __pyx_v_self->_bencoding = __pyx_v_encoding; - - /* "msgpack/_packer.pyx":130 - * - * self._bencoding = encoding - * if encoding is None: # <<<<<<<<<<<<<< - * if PY_MAJOR_VERSION < 3: - * self.encoding = 'utf-8' - */ - __pyx_t_1 = (__pyx_v_encoding == Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "msgpack/_packer.pyx":131 - * self._bencoding = encoding - * if encoding is None: - * if PY_MAJOR_VERSION < 3: # <<<<<<<<<<<<<< - * self.encoding = 'utf-8' - * else: - */ - __pyx_t_2 = ((PY_MAJOR_VERSION < 3) != 0); - if (__pyx_t_2) { - - /* "msgpack/_packer.pyx":132 - * if encoding is None: - * if PY_MAJOR_VERSION < 3: - * self.encoding = 'utf-8' # <<<<<<<<<<<<<< - * else: - * self.encoding = NULL - */ - __pyx_v_self->encoding = ((char const *)"utf-8"); - - /* "msgpack/_packer.pyx":131 - * self._bencoding = encoding - * if encoding is None: - * if PY_MAJOR_VERSION < 3: # <<<<<<<<<<<<<< - * self.encoding = 'utf-8' - * else: - */ - goto __pyx_L7; - } - - /* "msgpack/_packer.pyx":134 - * self.encoding = 'utf-8' - * else: - * self.encoding = NULL # <<<<<<<<<<<<<< - * else: - * self.encoding = self._bencoding - */ - /*else*/ { - __pyx_v_self->encoding = NULL; - } - __pyx_L7:; - - /* "msgpack/_packer.pyx":130 - * - * self._bencoding = encoding - * if encoding is None: # <<<<<<<<<<<<<< - * if PY_MAJOR_VERSION < 3: - * self.encoding = 'utf-8' - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":136 - * self.encoding = NULL - * else: - * self.encoding = self._bencoding # <<<<<<<<<<<<<< - * - * self._berrors = unicode_errors - */ - /*else*/ { - __pyx_t_5 = __Pyx_PyObject_AsString(__pyx_v_self->_bencoding); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) __PYX_ERR(0, 136, __pyx_L1_error) - __pyx_v_self->encoding = __pyx_t_5; - } - __pyx_L6:; - - /* "msgpack/_packer.pyx":138 - * self.encoding = self._bencoding - * - * self._berrors = unicode_errors # <<<<<<<<<<<<<< - * if unicode_errors is None: - * self.unicode_errors = NULL - */ - __Pyx_INCREF(__pyx_v_unicode_errors); - __Pyx_GIVEREF(__pyx_v_unicode_errors); - __Pyx_GOTREF(__pyx_v_self->_berrors); - __Pyx_DECREF(__pyx_v_self->_berrors); - __pyx_v_self->_berrors = __pyx_v_unicode_errors; - - /* "msgpack/_packer.pyx":139 - * - * self._berrors = unicode_errors - * if unicode_errors is None: # <<<<<<<<<<<<<< - * self.unicode_errors = NULL - * else: - */ - __pyx_t_2 = (__pyx_v_unicode_errors == Py_None); - __pyx_t_1 = (__pyx_t_2 != 0); - if (__pyx_t_1) { - - /* "msgpack/_packer.pyx":140 - * self._berrors = unicode_errors - * if unicode_errors is None: - * self.unicode_errors = NULL # <<<<<<<<<<<<<< - * else: - * self.unicode_errors = self._berrors - */ - __pyx_v_self->unicode_errors = NULL; - - /* "msgpack/_packer.pyx":139 - * - * self._berrors = unicode_errors - * if unicode_errors is None: # <<<<<<<<<<<<<< - * self.unicode_errors = NULL - * else: - */ - goto __pyx_L8; - } - - /* "msgpack/_packer.pyx":142 - * self.unicode_errors = NULL - * else: - * self.unicode_errors = self._berrors # <<<<<<<<<<<<<< - * - * def __dealloc__(self): - */ - /*else*/ { - __pyx_t_6 = __Pyx_PyObject_AsString(__pyx_v_self->_berrors); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) __PYX_ERR(0, 142, __pyx_L1_error) - __pyx_v_self->unicode_errors = __pyx_t_6; - } - __pyx_L8:; - - /* "msgpack/_packer.pyx":115 - * self.pk.length = 0 - * - * def __init__(self, default=None, encoding=None, unicode_errors=None, # <<<<<<<<<<<<<< - * bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - * bint strict_types=False): - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":144 - * self.unicode_errors = self._berrors - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * PyMem_Free(self.pk.buf) - * self.pk.buf = NULL - */ - -/* Python wrapper */ -static void __pyx_pw_7msgpack_9_cmsgpack_6Packer_5__dealloc__(PyObject *__pyx_v_self); /*proto*/ -static void __pyx_pw_7msgpack_9_cmsgpack_6Packer_5__dealloc__(PyObject *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); - __pyx_pf_7msgpack_9_cmsgpack_6Packer_4__dealloc__(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -static void __pyx_pf_7msgpack_9_cmsgpack_6Packer_4__dealloc__(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__", 0); - - /* "msgpack/_packer.pyx":145 - * - * def __dealloc__(self): - * PyMem_Free(self.pk.buf) # <<<<<<<<<<<<<< - * self.pk.buf = NULL - * - */ - PyMem_Free(__pyx_v_self->pk.buf); - - /* "msgpack/_packer.pyx":146 - * def __dealloc__(self): - * PyMem_Free(self.pk.buf) - * self.pk.buf = NULL # <<<<<<<<<<<<<< - * - * cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: - */ - __pyx_v_self->pk.buf = NULL; - - /* "msgpack/_packer.pyx":144 - * self.unicode_errors = self._berrors - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * PyMem_Free(self.pk.buf) - * self.pk.buf = NULL - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -/* "msgpack/_packer.pyx":148 - * self.pk.buf = NULL - * - * cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: # <<<<<<<<<<<<<< - * cdef long long llval - * cdef unsigned long long ullval - */ - -static int __pyx_f_7msgpack_9_cmsgpack_6Packer__pack(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o, struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack *__pyx_optional_args) { - int __pyx_v_nest_limit = __pyx_k__3; - PY_LONG_LONG __pyx_v_llval; - unsigned PY_LONG_LONG __pyx_v_ullval; - long __pyx_v_longval; - float __pyx_v_fval; - double __pyx_v_dval; - char *__pyx_v_rawval; - int __pyx_v_ret; - PyObject *__pyx_v_d = 0; - Py_ssize_t __pyx_v_L; - int __pyx_v_default_used; - int __pyx_v_strict_types; - Py_buffer __pyx_v_view; - CYTHON_UNUSED PyObject *__pyx_v_oe = NULL; - PyObject *__pyx_v_k = NULL; - PyObject *__pyx_v_v = NULL; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - unsigned PY_LONG_LONG __pyx_t_7; - PY_LONG_LONG __pyx_t_8; - int __pyx_t_9; - PyObject *__pyx_t_10 = NULL; - PyObject *__pyx_t_11 = NULL; - int __pyx_t_12; - PyObject *__pyx_t_13 = NULL; - PyObject *__pyx_t_14 = NULL; - PyObject *__pyx_t_15 = NULL; - int __pyx_t_16; - char const *__pyx_t_17; - PyObject *__pyx_t_18 = NULL; - PyObject *__pyx_t_19 = NULL; - PyObject *__pyx_t_20 = NULL; - PyObject *__pyx_t_21 = NULL; - PyObject *__pyx_t_22 = NULL; - PyObject *__pyx_t_23 = NULL; - long __pyx_t_24; - float __pyx_t_25; - double __pyx_t_26; - Py_ssize_t __pyx_t_27; - PyObject *__pyx_t_28; - char *__pyx_t_29; - Py_ssize_t __pyx_t_30; - struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack __pyx_t_31; - PyObject *(*__pyx_t_32)(PyObject *); - __Pyx_RefNannySetupContext("_pack", 0); - if (__pyx_optional_args) { - if (__pyx_optional_args->__pyx_n > 0) { - __pyx_v_nest_limit = __pyx_optional_args->nest_limit; - } - } - __Pyx_INCREF(__pyx_v_o); - - /* "msgpack/_packer.pyx":158 - * cdef dict d - * cdef Py_ssize_t L - * cdef int default_used = 0 # <<<<<<<<<<<<<< - * cdef bint strict_types = self.strict_types - * cdef Py_buffer view - */ - __pyx_v_default_used = 0; - - /* "msgpack/_packer.pyx":159 - * cdef Py_ssize_t L - * cdef int default_used = 0 - * cdef bint strict_types = self.strict_types # <<<<<<<<<<<<<< - * cdef Py_buffer view - * - */ - __pyx_t_1 = __pyx_v_self->strict_types; - __pyx_v_strict_types = __pyx_t_1; - - /* "msgpack/_packer.pyx":162 - * cdef Py_buffer view - * - * if nest_limit < 0: # <<<<<<<<<<<<<< - * raise ValueError("recursion limit exceeded.") - * - */ - __pyx_t_1 = ((__pyx_v_nest_limit < 0) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":163 - * - * if nest_limit < 0: - * raise ValueError("recursion limit exceeded.") # <<<<<<<<<<<<<< - * - * while True: - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 163, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(0, 163, __pyx_L1_error) - - /* "msgpack/_packer.pyx":162 - * cdef Py_buffer view - * - * if nest_limit < 0: # <<<<<<<<<<<<<< - * raise ValueError("recursion limit exceeded.") - * - */ - } - - /* "msgpack/_packer.pyx":165 - * raise ValueError("recursion limit exceeded.") - * - * while True: # <<<<<<<<<<<<<< - * if o is None: - * ret = msgpack_pack_nil(&self.pk) - */ - while (1) { - - /* "msgpack/_packer.pyx":166 - * - * while True: - * if o is None: # <<<<<<<<<<<<<< - * ret = msgpack_pack_nil(&self.pk) - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): - */ - __pyx_t_1 = (__pyx_v_o == Py_None); - __pyx_t_3 = (__pyx_t_1 != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":167 - * while True: - * if o is None: - * ret = msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): - * if o: - */ - __pyx_v_ret = msgpack_pack_nil((&__pyx_v_self->pk)); - - /* "msgpack/_packer.pyx":166 - * - * while True: - * if o is None: # <<<<<<<<<<<<<< - * ret = msgpack_pack_nil(&self.pk) - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":168 - * if o is None: - * ret = msgpack_pack_nil(&self.pk) - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): # <<<<<<<<<<<<<< - * if o: - * ret = msgpack_pack_true(&self.pk) - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_3 = PyBool_Check(__pyx_v_o); - } else { - __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_o, __pyx_ptype_7cpython_4bool_bool); - __pyx_t_3 = __pyx_t_1; - } - __pyx_t_1 = (__pyx_t_3 != 0); - if (__pyx_t_1) { - - /* "msgpack/_packer.pyx":169 - * ret = msgpack_pack_nil(&self.pk) - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): - * if o: # <<<<<<<<<<<<<< - * ret = msgpack_pack_true(&self.pk) - * else: - */ - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_o); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(0, 169, __pyx_L1_error) - if (__pyx_t_1) { - - /* "msgpack/_packer.pyx":170 - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): - * if o: - * ret = msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< - * else: - * ret = msgpack_pack_false(&self.pk) - */ - __pyx_v_ret = msgpack_pack_true((&__pyx_v_self->pk)); - - /* "msgpack/_packer.pyx":169 - * ret = msgpack_pack_nil(&self.pk) - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): - * if o: # <<<<<<<<<<<<<< - * ret = msgpack_pack_true(&self.pk) - * else: - */ - goto __pyx_L7; - } - - /* "msgpack/_packer.pyx":172 - * ret = msgpack_pack_true(&self.pk) - * else: - * ret = msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< - * elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): - * # PyInt_Check(long) is True for Python 3. - */ - /*else*/ { - __pyx_v_ret = msgpack_pack_false((&__pyx_v_self->pk)); - } - __pyx_L7:; - - /* "msgpack/_packer.pyx":168 - * if o is None: - * ret = msgpack_pack_nil(&self.pk) - * elif PyBool_Check(o) if strict_types else isinstance(o, bool): # <<<<<<<<<<<<<< - * if o: - * ret = msgpack_pack_true(&self.pk) - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":173 - * else: - * ret = msgpack_pack_false(&self.pk) - * elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): # <<<<<<<<<<<<<< - * # PyInt_Check(long) is True for Python 3. - * # So we should test long before int. - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_1 = PyLong_CheckExact(__pyx_v_o); - } else { - __pyx_t_1 = PyLong_Check(__pyx_v_o); - } - __pyx_t_3 = (__pyx_t_1 != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":176 - * # PyInt_Check(long) is True for Python 3. - * # So we should test long before int. - * try: # <<<<<<<<<<<<<< - * if o > 0: - * ullval = o - */ - { - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ExceptionSave(&__pyx_t_4, &__pyx_t_5, &__pyx_t_6); - __Pyx_XGOTREF(__pyx_t_4); - __Pyx_XGOTREF(__pyx_t_5); - __Pyx_XGOTREF(__pyx_t_6); - /*try:*/ { - - /* "msgpack/_packer.pyx":177 - * # So we should test long before int. - * try: - * if o > 0: # <<<<<<<<<<<<<< - * ullval = o - * ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - */ - __pyx_t_2 = PyObject_RichCompare(__pyx_v_o, __pyx_int_0, Py_GT); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 177, __pyx_L8_error) - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 177, __pyx_L8_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":178 - * try: - * if o > 0: - * ullval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - * else: - */ - __pyx_t_7 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_v_o); if (unlikely((__pyx_t_7 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) __PYX_ERR(0, 178, __pyx_L8_error) - __pyx_v_ullval = __pyx_t_7; - - /* "msgpack/_packer.pyx":179 - * if o > 0: - * ullval = o - * ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) # <<<<<<<<<<<<<< - * else: - * llval = o - */ - __pyx_v_ret = msgpack_pack_unsigned_long_long((&__pyx_v_self->pk), __pyx_v_ullval); - - /* "msgpack/_packer.pyx":177 - * # So we should test long before int. - * try: - * if o > 0: # <<<<<<<<<<<<<< - * ullval = o - * ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - */ - goto __pyx_L16; - } - - /* "msgpack/_packer.pyx":181 - * ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - * else: - * llval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_long_long(&self.pk, llval) - * except OverflowError as oe: - */ - /*else*/ { - __pyx_t_8 = __Pyx_PyInt_As_PY_LONG_LONG(__pyx_v_o); if (unlikely((__pyx_t_8 == (PY_LONG_LONG)-1) && PyErr_Occurred())) __PYX_ERR(0, 181, __pyx_L8_error) - __pyx_v_llval = __pyx_t_8; - - /* "msgpack/_packer.pyx":182 - * else: - * llval = o - * ret = msgpack_pack_long_long(&self.pk, llval) # <<<<<<<<<<<<<< - * except OverflowError as oe: - * if not default_used and self._default is not None: - */ - __pyx_v_ret = msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_llval); - } - __pyx_L16:; - - /* "msgpack/_packer.pyx":176 - * # PyInt_Check(long) is True for Python 3. - * # So we should test long before int. - * try: # <<<<<<<<<<<<<< - * if o > 0: - * ullval = o - */ - } - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - goto __pyx_L15_try_end; - __pyx_L8_error:; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_packer.pyx":183 - * llval = o - * ret = msgpack_pack_long_long(&self.pk, llval) - * except OverflowError as oe: # <<<<<<<<<<<<<< - * if not default_used and self._default is not None: - * o = self._default(o) - */ - __pyx_t_9 = __Pyx_PyErr_ExceptionMatches(__pyx_builtin_OverflowError); - if (__pyx_t_9) { - __Pyx_AddTraceback("msgpack._cmsgpack.Packer._pack", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_10, &__pyx_t_11) < 0) __PYX_ERR(0, 183, __pyx_L10_except_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_t_10); - __Pyx_GOTREF(__pyx_t_11); - __Pyx_INCREF(__pyx_t_10); - __pyx_v_oe = __pyx_t_10; - /*try:*/ { - - /* "msgpack/_packer.pyx":184 - * ret = msgpack_pack_long_long(&self.pk, llval) - * except OverflowError as oe: - * if not default_used and self._default is not None: # <<<<<<<<<<<<<< - * o = self._default(o) - * default_used = True - */ - __pyx_t_1 = ((!(__pyx_v_default_used != 0)) != 0); - if (__pyx_t_1) { - } else { - __pyx_t_3 = __pyx_t_1; - goto __pyx_L25_bool_binop_done; - } - __pyx_t_1 = (__pyx_v_self->_default != Py_None); - __pyx_t_12 = (__pyx_t_1 != 0); - __pyx_t_3 = __pyx_t_12; - __pyx_L25_bool_binop_done:; - if (likely(__pyx_t_3)) { - - /* "msgpack/_packer.pyx":185 - * except OverflowError as oe: - * if not default_used and self._default is not None: - * o = self._default(o) # <<<<<<<<<<<<<< - * default_used = True - * continue - */ - __Pyx_INCREF(__pyx_v_self->_default); - __pyx_t_14 = __pyx_v_self->_default; __pyx_t_15 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_14))) { - __pyx_t_15 = PyMethod_GET_SELF(__pyx_t_14); - if (likely(__pyx_t_15)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_14); - __Pyx_INCREF(__pyx_t_15); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_14, function); - } - } - __pyx_t_13 = (__pyx_t_15) ? __Pyx_PyObject_Call2Args(__pyx_t_14, __pyx_t_15, __pyx_v_o) : __Pyx_PyObject_CallOneArg(__pyx_t_14, __pyx_v_o); - __Pyx_XDECREF(__pyx_t_15); __pyx_t_15 = 0; - if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 185, __pyx_L22_error) - __Pyx_GOTREF(__pyx_t_13); - __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; - __Pyx_DECREF_SET(__pyx_v_o, __pyx_t_13); - __pyx_t_13 = 0; - - /* "msgpack/_packer.pyx":186 - * if not default_used and self._default is not None: - * o = self._default(o) - * default_used = True # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_v_default_used = 1; - - /* "msgpack/_packer.pyx":187 - * o = self._default(o) - * default_used = True - * continue # <<<<<<<<<<<<<< - * else: - * raise OverflowError("Integer value out of range") - */ - goto __pyx_L19_continue; - - /* "msgpack/_packer.pyx":184 - * ret = msgpack_pack_long_long(&self.pk, llval) - * except OverflowError as oe: - * if not default_used and self._default is not None: # <<<<<<<<<<<<<< - * o = self._default(o) - * default_used = True - */ - } - - /* "msgpack/_packer.pyx":189 - * continue - * else: - * raise OverflowError("Integer value out of range") # <<<<<<<<<<<<<< - * elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - * longval = o - */ - /*else*/ { - __pyx_t_13 = __Pyx_PyObject_Call(__pyx_builtin_OverflowError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 189, __pyx_L22_error) - __Pyx_GOTREF(__pyx_t_13); - __Pyx_Raise(__pyx_t_13, 0, 0, 0); - __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; - __PYX_ERR(0, 189, __pyx_L22_error) - } - } - - /* "msgpack/_packer.pyx":183 - * llval = o - * ret = msgpack_pack_long_long(&self.pk, llval) - * except OverflowError as oe: # <<<<<<<<<<<<<< - * if not default_used and self._default is not None: - * o = self._default(o) - */ - /*finally:*/ { - __pyx_L22_error:; - /*exception exit:*/{ - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __pyx_t_18 = 0; __pyx_t_19 = 0; __pyx_t_20 = 0; __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; - __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0; - __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; - __Pyx_XDECREF(__pyx_t_15); __pyx_t_15 = 0; - if (PY_MAJOR_VERSION >= 3) __Pyx_ExceptionSwap(&__pyx_t_21, &__pyx_t_22, &__pyx_t_23); - if ((PY_MAJOR_VERSION < 3) || unlikely(__Pyx_GetException(&__pyx_t_18, &__pyx_t_19, &__pyx_t_20) < 0)) __Pyx_ErrFetch(&__pyx_t_18, &__pyx_t_19, &__pyx_t_20); - __Pyx_XGOTREF(__pyx_t_18); - __Pyx_XGOTREF(__pyx_t_19); - __Pyx_XGOTREF(__pyx_t_20); - __Pyx_XGOTREF(__pyx_t_21); - __Pyx_XGOTREF(__pyx_t_22); - __Pyx_XGOTREF(__pyx_t_23); - __pyx_t_9 = __pyx_lineno; __pyx_t_16 = __pyx_clineno; __pyx_t_17 = __pyx_filename; - { - __Pyx_DECREF(__pyx_v_oe); - __pyx_v_oe = NULL; - } - if (PY_MAJOR_VERSION >= 3) { - __Pyx_XGIVEREF(__pyx_t_21); - __Pyx_XGIVEREF(__pyx_t_22); - __Pyx_XGIVEREF(__pyx_t_23); - __Pyx_ExceptionReset(__pyx_t_21, __pyx_t_22, __pyx_t_23); - } - __Pyx_XGIVEREF(__pyx_t_18); - __Pyx_XGIVEREF(__pyx_t_19); - __Pyx_XGIVEREF(__pyx_t_20); - __Pyx_ErrRestore(__pyx_t_18, __pyx_t_19, __pyx_t_20); - __pyx_t_18 = 0; __pyx_t_19 = 0; __pyx_t_20 = 0; __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; - __pyx_lineno = __pyx_t_9; __pyx_clineno = __pyx_t_16; __pyx_filename = __pyx_t_17; - goto __pyx_L10_except_error; - } - __pyx_L19_continue: { - __Pyx_DECREF(__pyx_v_oe); - __pyx_v_oe = NULL; - goto __pyx_L18_except_continue; - } - } - __pyx_L18_except_continue:; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - goto __pyx_L14_try_continue; - } - goto __pyx_L10_except_error; - __pyx_L10_except_error:; - - /* "msgpack/_packer.pyx":176 - * # PyInt_Check(long) is True for Python 3. - * # So we should test long before int. - * try: # <<<<<<<<<<<<<< - * if o > 0: - * ullval = o - */ - __Pyx_XGIVEREF(__pyx_t_4); - __Pyx_XGIVEREF(__pyx_t_5); - __Pyx_XGIVEREF(__pyx_t_6); - __Pyx_ExceptionReset(__pyx_t_4, __pyx_t_5, __pyx_t_6); - goto __pyx_L1_error; - __pyx_L14_try_continue:; - __Pyx_XGIVEREF(__pyx_t_4); - __Pyx_XGIVEREF(__pyx_t_5); - __Pyx_XGIVEREF(__pyx_t_6); - __Pyx_ExceptionReset(__pyx_t_4, __pyx_t_5, __pyx_t_6); - goto __pyx_L4_continue; - __pyx_L15_try_end:; - } - - /* "msgpack/_packer.pyx":173 - * else: - * ret = msgpack_pack_false(&self.pk) - * elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): # <<<<<<<<<<<<<< - * # PyInt_Check(long) is True for Python 3. - * # So we should test long before int. - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":190 - * else: - * raise OverflowError("Integer value out of range") - * elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): # <<<<<<<<<<<<<< - * longval = o - * ret = msgpack_pack_long(&self.pk, longval) - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_3 = PyInt_CheckExact(__pyx_v_o); - } else { - __pyx_t_3 = PyInt_Check(__pyx_v_o); - } - __pyx_t_12 = (__pyx_t_3 != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":191 - * raise OverflowError("Integer value out of range") - * elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - * longval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_long(&self.pk, longval) - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - */ - __pyx_t_24 = __Pyx_PyInt_As_long(__pyx_v_o); if (unlikely((__pyx_t_24 == (long)-1) && PyErr_Occurred())) __PYX_ERR(0, 191, __pyx_L1_error) - __pyx_v_longval = __pyx_t_24; - - /* "msgpack/_packer.pyx":192 - * elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - * longval = o - * ret = msgpack_pack_long(&self.pk, longval) # <<<<<<<<<<<<<< - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - * if self.use_float: - */ - __pyx_v_ret = msgpack_pack_long((&__pyx_v_self->pk), __pyx_v_longval); - - /* "msgpack/_packer.pyx":190 - * else: - * raise OverflowError("Integer value out of range") - * elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): # <<<<<<<<<<<<<< - * longval = o - * ret = msgpack_pack_long(&self.pk, longval) - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":193 - * longval = o - * ret = msgpack_pack_long(&self.pk, longval) - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): # <<<<<<<<<<<<<< - * if self.use_float: - * fval = o - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_12 = PyFloat_CheckExact(__pyx_v_o); - } else { - __pyx_t_12 = PyFloat_Check(__pyx_v_o); - } - __pyx_t_3 = (__pyx_t_12 != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":194 - * ret = msgpack_pack_long(&self.pk, longval) - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - * if self.use_float: # <<<<<<<<<<<<<< - * fval = o - * ret = msgpack_pack_float(&self.pk, fval) - */ - __pyx_t_3 = __Pyx_PyObject_IsTrue(((PyObject *)__pyx_v_self->use_float)); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 194, __pyx_L1_error) - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":195 - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - * if self.use_float: - * fval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_float(&self.pk, fval) - * else: - */ - __pyx_t_25 = __pyx_PyFloat_AsFloat(__pyx_v_o); if (unlikely((__pyx_t_25 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 195, __pyx_L1_error) - __pyx_v_fval = __pyx_t_25; - - /* "msgpack/_packer.pyx":196 - * if self.use_float: - * fval = o - * ret = msgpack_pack_float(&self.pk, fval) # <<<<<<<<<<<<<< - * else: - * dval = o - */ - __pyx_v_ret = msgpack_pack_float((&__pyx_v_self->pk), __pyx_v_fval); - - /* "msgpack/_packer.pyx":194 - * ret = msgpack_pack_long(&self.pk, longval) - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - * if self.use_float: # <<<<<<<<<<<<<< - * fval = o - * ret = msgpack_pack_float(&self.pk, fval) - */ - goto __pyx_L31; - } - - /* "msgpack/_packer.pyx":198 - * ret = msgpack_pack_float(&self.pk, fval) - * else: - * dval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_double(&self.pk, dval) - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - */ - /*else*/ { - __pyx_t_26 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely((__pyx_t_26 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 198, __pyx_L1_error) - __pyx_v_dval = __pyx_t_26; - - /* "msgpack/_packer.pyx":199 - * else: - * dval = o - * ret = msgpack_pack_double(&self.pk, dval) # <<<<<<<<<<<<<< - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - * L = len(o) - */ - __pyx_v_ret = msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_dval); - } - __pyx_L31:; - - /* "msgpack/_packer.pyx":193 - * longval = o - * ret = msgpack_pack_long(&self.pk, longval) - * elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): # <<<<<<<<<<<<<< - * if self.use_float: - * fval = o - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":200 - * dval = o - * ret = msgpack_pack_double(&self.pk, dval) - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_16 = __pyx_f_7msgpack_9_cmsgpack_PyBytesLike_CheckExact(__pyx_v_o); - } else { - __pyx_t_16 = __pyx_f_7msgpack_9_cmsgpack_PyBytesLike_Check(__pyx_v_o); - } - __pyx_t_3 = (__pyx_t_16 != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":201 - * ret = msgpack_pack_double(&self.pk, dval) - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - * L = len(o) # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - */ - __pyx_t_27 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_27 == ((Py_ssize_t)-1))) __PYX_ERR(0, 201, __pyx_L1_error) - __pyx_v_L = __pyx_t_27; - - /* "msgpack/_packer.pyx":202 - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - * rawval = o - */ - __pyx_t_3 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":203 - * L = len(o) - * if L > ITEM_LIMIT: - * PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) # <<<<<<<<<<<<<< - * rawval = o - * ret = msgpack_pack_bin(&self.pk, L) - */ - __pyx_t_28 = PyErr_Format(__pyx_builtin_ValueError, ((char *)"%.200s object is too large"), Py_TYPE(__pyx_v_o)->tp_name); if (unlikely(__pyx_t_28 == ((PyObject *)NULL))) __PYX_ERR(0, 203, __pyx_L1_error) - - /* "msgpack/_packer.pyx":202 - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - * rawval = o - */ - } - - /* "msgpack/_packer.pyx":204 - * if L > ITEM_LIMIT: - * PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - * rawval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: - */ - __pyx_t_29 = __Pyx_PyObject_AsWritableString(__pyx_v_o); if (unlikely((!__pyx_t_29) && PyErr_Occurred())) __PYX_ERR(0, 204, __pyx_L1_error) - __pyx_v_rawval = __pyx_t_29; - - /* "msgpack/_packer.pyx":205 - * PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - * rawval = o - * ret = msgpack_pack_bin(&self.pk, L) # <<<<<<<<<<<<<< - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - */ - __pyx_v_ret = msgpack_pack_bin((&__pyx_v_self->pk), __pyx_v_L); - - /* "msgpack/_packer.pyx":206 - * rawval = o - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - */ - __pyx_t_3 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":207 - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) # <<<<<<<<<<<<<< - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - * if self.encoding == NULL and self.unicode_errors == NULL: - */ - __pyx_v_ret = msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_v_L); - - /* "msgpack/_packer.pyx":206 - * rawval = o - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - */ - } - - /* "msgpack/_packer.pyx":200 - * dval = o - * ret = msgpack_pack_double(&self.pk, dval) - * elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":208 - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): # <<<<<<<<<<<<<< - * if self.encoding == NULL and self.unicode_errors == NULL: - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_3 = PyUnicode_CheckExact(__pyx_v_o); - } else { - __pyx_t_3 = PyUnicode_Check(__pyx_v_o); - } - __pyx_t_12 = (__pyx_t_3 != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":209 - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - * if self.encoding == NULL and self.unicode_errors == NULL: # <<<<<<<<<<<<<< - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - * if ret == -2: - */ - __pyx_t_3 = ((__pyx_v_self->encoding == NULL) != 0); - if (__pyx_t_3) { - } else { - __pyx_t_12 = __pyx_t_3; - goto __pyx_L35_bool_binop_done; - } - __pyx_t_3 = ((__pyx_v_self->unicode_errors == NULL) != 0); - __pyx_t_12 = __pyx_t_3; - __pyx_L35_bool_binop_done:; - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":210 - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - * if self.encoding == NULL and self.unicode_errors == NULL: - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); # <<<<<<<<<<<<<< - * if ret == -2: - * raise ValueError("unicode string is too large") - */ - __pyx_v_ret = msgpack_pack_unicode((&__pyx_v_self->pk), __pyx_v_o, __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT); - - /* "msgpack/_packer.pyx":211 - * if self.encoding == NULL and self.unicode_errors == NULL: - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - * if ret == -2: # <<<<<<<<<<<<<< - * raise ValueError("unicode string is too large") - * else: - */ - __pyx_t_12 = ((__pyx_v_ret == -2L) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":212 - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - * if ret == -2: - * raise ValueError("unicode string is too large") # <<<<<<<<<<<<<< - * else: - * o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - */ - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 212, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_Raise(__pyx_t_11, 0, 0, 0); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __PYX_ERR(0, 212, __pyx_L1_error) - - /* "msgpack/_packer.pyx":211 - * if self.encoding == NULL and self.unicode_errors == NULL: - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - * if ret == -2: # <<<<<<<<<<<<<< - * raise ValueError("unicode string is too large") - * else: - */ - } - - /* "msgpack/_packer.pyx":209 - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - * if self.encoding == NULL and self.unicode_errors == NULL: # <<<<<<<<<<<<<< - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - * if ret == -2: - */ - goto __pyx_L34; - } - - /* "msgpack/_packer.pyx":214 - * raise ValueError("unicode string is too large") - * else: - * o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - /*else*/ { - __pyx_t_11 = PyUnicode_AsEncodedString(__pyx_v_o, __pyx_v_self->encoding, __pyx_v_self->unicode_errors); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 214, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_DECREF_SET(__pyx_v_o, __pyx_t_11); - __pyx_t_11 = 0; - - /* "msgpack/_packer.pyx":215 - * else: - * o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - * L = len(o) # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * raise ValueError("unicode string is too large") - */ - __pyx_t_27 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_27 == ((Py_ssize_t)-1))) __PYX_ERR(0, 215, __pyx_L1_error) - __pyx_v_L = __pyx_t_27; - - /* "msgpack/_packer.pyx":216 - * o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("unicode string is too large") - * ret = msgpack_pack_raw(&self.pk, L) - */ - __pyx_t_12 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":217 - * L = len(o) - * if L > ITEM_LIMIT: - * raise ValueError("unicode string is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw(&self.pk, L) - * if ret == 0: - */ - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 217, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_Raise(__pyx_t_11, 0, 0, 0); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __PYX_ERR(0, 217, __pyx_L1_error) - - /* "msgpack/_packer.pyx":216 - * o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("unicode string is too large") - * ret = msgpack_pack_raw(&self.pk, L) - */ - } - - /* "msgpack/_packer.pyx":218 - * if L > ITEM_LIMIT: - * raise ValueError("unicode string is too large") - * ret = msgpack_pack_raw(&self.pk, L) # <<<<<<<<<<<<<< - * if ret == 0: - * rawval = o - */ - __pyx_v_ret = msgpack_pack_raw((&__pyx_v_self->pk), __pyx_v_L); - - /* "msgpack/_packer.pyx":219 - * raise ValueError("unicode string is too large") - * ret = msgpack_pack_raw(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * rawval = o - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - */ - __pyx_t_12 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":220 - * ret = msgpack_pack_raw(&self.pk, L) - * if ret == 0: - * rawval = o # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyDict_CheckExact(o): - */ - __pyx_t_29 = __Pyx_PyObject_AsWritableString(__pyx_v_o); if (unlikely((!__pyx_t_29) && PyErr_Occurred())) __PYX_ERR(0, 220, __pyx_L1_error) - __pyx_v_rawval = __pyx_t_29; - - /* "msgpack/_packer.pyx":221 - * if ret == 0: - * rawval = o - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) # <<<<<<<<<<<<<< - * elif PyDict_CheckExact(o): - * d = o - */ - __pyx_v_ret = msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_v_L); - - /* "msgpack/_packer.pyx":219 - * raise ValueError("unicode string is too large") - * ret = msgpack_pack_raw(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * rawval = o - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - */ - } - } - __pyx_L34:; - - /* "msgpack/_packer.pyx":208 - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): # <<<<<<<<<<<<<< - * if self.encoding == NULL and self.unicode_errors == NULL: - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":222 - * rawval = o - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyDict_CheckExact(o): # <<<<<<<<<<<<<< - * d = o - * L = len(d) - */ - __pyx_t_12 = (PyDict_CheckExact(__pyx_v_o) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":223 - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyDict_CheckExact(o): - * d = o # <<<<<<<<<<<<<< - * L = len(d) - * if L > ITEM_LIMIT: - */ - __pyx_t_11 = __pyx_v_o; - __Pyx_INCREF(__pyx_t_11); - __pyx_v_d = ((PyObject*)__pyx_t_11); - __pyx_t_11 = 0; - - /* "msgpack/_packer.pyx":224 - * elif PyDict_CheckExact(o): - * d = o - * L = len(d) # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") - */ - if (unlikely(__pyx_v_d == Py_None)) { - PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(0, 224, __pyx_L1_error) - } - __pyx_t_27 = PyDict_Size(__pyx_v_d); if (unlikely(__pyx_t_27 == ((Py_ssize_t)-1))) __PYX_ERR(0, 224, __pyx_L1_error) - __pyx_v_L = __pyx_t_27; - - /* "msgpack/_packer.pyx":225 - * d = o - * L = len(d) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - */ - __pyx_t_12 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":226 - * L = len(d) - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: - */ - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 226, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_Raise(__pyx_t_11, 0, 0, 0); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __PYX_ERR(0, 226, __pyx_L1_error) - - /* "msgpack/_packer.pyx":225 - * d = o - * L = len(d) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - */ - } - - /* "msgpack/_packer.pyx":227 - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) # <<<<<<<<<<<<<< - * if ret == 0: - * for k, v in d.items(): - */ - __pyx_v_ret = msgpack_pack_map((&__pyx_v_self->pk), __pyx_v_L); - - /* "msgpack/_packer.pyx":228 - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * for k, v in d.items(): - * ret = self._pack(k, nest_limit-1) - */ - __pyx_t_12 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":229 - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: - * for k, v in d.items(): # <<<<<<<<<<<<<< - * ret = self._pack(k, nest_limit-1) - * if ret != 0: break - */ - __pyx_t_27 = 0; - if (unlikely(__pyx_v_d == Py_None)) { - PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "items"); - __PYX_ERR(0, 229, __pyx_L1_error) - } - __pyx_t_10 = __Pyx_dict_iterator(__pyx_v_d, 1, __pyx_n_s_items, (&__pyx_t_30), (&__pyx_t_16)); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 229, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_XDECREF(__pyx_t_11); - __pyx_t_11 = __pyx_t_10; - __pyx_t_10 = 0; - while (1) { - __pyx_t_9 = __Pyx_dict_iter_next(__pyx_t_11, __pyx_t_30, &__pyx_t_27, &__pyx_t_10, &__pyx_t_2, NULL, __pyx_t_16); - if (unlikely(__pyx_t_9 == 0)) break; - if (unlikely(__pyx_t_9 == -1)) __PYX_ERR(0, 229, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_GOTREF(__pyx_t_2); - __Pyx_XDECREF_SET(__pyx_v_k, __pyx_t_10); - __pyx_t_10 = 0; - __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_2); - __pyx_t_2 = 0; - - /* "msgpack/_packer.pyx":230 - * if ret == 0: - * for k, v in d.items(): - * ret = self._pack(k, nest_limit-1) # <<<<<<<<<<<<<< - * if ret != 0: break - * ret = self._pack(v, nest_limit-1) - */ - __pyx_t_31.__pyx_n = 1; - __pyx_t_31.nest_limit = (__pyx_v_nest_limit - 1); - __pyx_t_9 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_k, &__pyx_t_31); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 230, __pyx_L1_error) - __pyx_v_ret = __pyx_t_9; - - /* "msgpack/_packer.pyx":231 - * for k, v in d.items(): - * ret = self._pack(k, nest_limit-1) - * if ret != 0: break # <<<<<<<<<<<<<< - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - */ - __pyx_t_12 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_12) { - goto __pyx_L43_break; - } - - /* "msgpack/_packer.pyx":232 - * ret = self._pack(k, nest_limit-1) - * if ret != 0: break - * ret = self._pack(v, nest_limit-1) # <<<<<<<<<<<<<< - * if ret != 0: break - * elif not strict_types and PyDict_Check(o): - */ - __pyx_t_31.__pyx_n = 1; - __pyx_t_31.nest_limit = (__pyx_v_nest_limit - 1); - __pyx_t_9 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_v, &__pyx_t_31); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 232, __pyx_L1_error) - __pyx_v_ret = __pyx_t_9; - - /* "msgpack/_packer.pyx":233 - * if ret != 0: break - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break # <<<<<<<<<<<<<< - * elif not strict_types and PyDict_Check(o): - * L = len(o) - */ - __pyx_t_12 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_12) { - goto __pyx_L43_break; - } - } - __pyx_L43_break:; - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - - /* "msgpack/_packer.pyx":228 - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * for k, v in d.items(): - * ret = self._pack(k, nest_limit-1) - */ - } - - /* "msgpack/_packer.pyx":222 - * rawval = o - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyDict_CheckExact(o): # <<<<<<<<<<<<<< - * d = o - * L = len(d) - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":234 - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - * elif not strict_types and PyDict_Check(o): # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - __pyx_t_3 = ((!(__pyx_v_strict_types != 0)) != 0); - if (__pyx_t_3) { - } else { - __pyx_t_12 = __pyx_t_3; - goto __pyx_L46_bool_binop_done; - } - __pyx_t_3 = (PyDict_Check(__pyx_v_o) != 0); - __pyx_t_12 = __pyx_t_3; - __pyx_L46_bool_binop_done:; - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":235 - * if ret != 0: break - * elif not strict_types and PyDict_Check(o): - * L = len(o) # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") - */ - __pyx_t_30 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_30 == ((Py_ssize_t)-1))) __PYX_ERR(0, 235, __pyx_L1_error) - __pyx_v_L = __pyx_t_30; - - /* "msgpack/_packer.pyx":236 - * elif not strict_types and PyDict_Check(o): - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - */ - __pyx_t_12 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":237 - * L = len(o) - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: - */ - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 237, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_Raise(__pyx_t_11, 0, 0, 0); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __PYX_ERR(0, 237, __pyx_L1_error) - - /* "msgpack/_packer.pyx":236 - * elif not strict_types and PyDict_Check(o): - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - */ - } - - /* "msgpack/_packer.pyx":238 - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) # <<<<<<<<<<<<<< - * if ret == 0: - * for k, v in o.items(): - */ - __pyx_v_ret = msgpack_pack_map((&__pyx_v_self->pk), __pyx_v_L); - - /* "msgpack/_packer.pyx":239 - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * for k, v in o.items(): - * ret = self._pack(k, nest_limit-1) - */ - __pyx_t_12 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":240 - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: - * for k, v in o.items(): # <<<<<<<<<<<<<< - * ret = self._pack(k, nest_limit-1) - * if ret != 0: break - */ - __pyx_t_30 = 0; - if (unlikely(__pyx_v_o == Py_None)) { - PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "items"); - __PYX_ERR(0, 240, __pyx_L1_error) - } - __pyx_t_2 = __Pyx_dict_iterator(__pyx_v_o, 0, __pyx_n_s_items, (&__pyx_t_27), (&__pyx_t_16)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 240, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_11); - __pyx_t_11 = __pyx_t_2; - __pyx_t_2 = 0; - while (1) { - __pyx_t_9 = __Pyx_dict_iter_next(__pyx_t_11, __pyx_t_27, &__pyx_t_30, &__pyx_t_2, &__pyx_t_10, NULL, __pyx_t_16); - if (unlikely(__pyx_t_9 == 0)) break; - if (unlikely(__pyx_t_9 == -1)) __PYX_ERR(0, 240, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_t_10); - __Pyx_XDECREF_SET(__pyx_v_k, __pyx_t_2); - __pyx_t_2 = 0; - __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_10); - __pyx_t_10 = 0; - - /* "msgpack/_packer.pyx":241 - * if ret == 0: - * for k, v in o.items(): - * ret = self._pack(k, nest_limit-1) # <<<<<<<<<<<<<< - * if ret != 0: break - * ret = self._pack(v, nest_limit-1) - */ - __pyx_t_31.__pyx_n = 1; - __pyx_t_31.nest_limit = (__pyx_v_nest_limit - 1); - __pyx_t_9 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_k, &__pyx_t_31); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 241, __pyx_L1_error) - __pyx_v_ret = __pyx_t_9; - - /* "msgpack/_packer.pyx":242 - * for k, v in o.items(): - * ret = self._pack(k, nest_limit-1) - * if ret != 0: break # <<<<<<<<<<<<<< - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - */ - __pyx_t_12 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_12) { - goto __pyx_L51_break; - } - - /* "msgpack/_packer.pyx":243 - * ret = self._pack(k, nest_limit-1) - * if ret != 0: break - * ret = self._pack(v, nest_limit-1) # <<<<<<<<<<<<<< - * if ret != 0: break - * elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - */ - __pyx_t_31.__pyx_n = 1; - __pyx_t_31.nest_limit = (__pyx_v_nest_limit - 1); - __pyx_t_9 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_v, &__pyx_t_31); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 243, __pyx_L1_error) - __pyx_v_ret = __pyx_t_9; - - /* "msgpack/_packer.pyx":244 - * if ret != 0: break - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break # <<<<<<<<<<<<<< - * elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - * # This should be before Tuple because ExtType is namedtuple. - */ - __pyx_t_12 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_12) { - goto __pyx_L51_break; - } - } - __pyx_L51_break:; - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - - /* "msgpack/_packer.pyx":239 - * raise ValueError("dict is too large") - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * for k, v in o.items(): - * ret = self._pack(k, nest_limit-1) - */ - } - - /* "msgpack/_packer.pyx":234 - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - * elif not strict_types and PyDict_Check(o): # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":245 - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - * elif type(o) is ExtType if strict_types else isinstance(o, ExtType): # <<<<<<<<<<<<<< - * # This should be before Tuple because ExtType is namedtuple. - * longval = o.code - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_11 = __Pyx_PyObject_CallOneArg(((PyObject *)__pyx_ptype_7cpython_4type_type), __pyx_v_o); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 245, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_n_s_ExtType); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 245, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __pyx_t_3 = (__pyx_t_11 == __pyx_t_10); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_t_12 = __pyx_t_3; - } else { - __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_n_s_ExtType); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 245, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_o, __pyx_t_10); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(0, 245, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_t_12 = __pyx_t_3; - } - __pyx_t_3 = (__pyx_t_12 != 0); - if (__pyx_t_3) { - - /* "msgpack/_packer.pyx":247 - * elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - * # This should be before Tuple because ExtType is namedtuple. - * longval = o.code # <<<<<<<<<<<<<< - * rawval = o.data - * L = len(o.data) - */ - __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_o, __pyx_n_s_code); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 247, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __pyx_t_24 = __Pyx_PyInt_As_long(__pyx_t_10); if (unlikely((__pyx_t_24 == (long)-1) && PyErr_Occurred())) __PYX_ERR(0, 247, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_v_longval = __pyx_t_24; - - /* "msgpack/_packer.pyx":248 - * # This should be before Tuple because ExtType is namedtuple. - * longval = o.code - * rawval = o.data # <<<<<<<<<<<<<< - * L = len(o.data) - * if L > ITEM_LIMIT: - */ - __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_o, __pyx_n_s_data); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 248, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __pyx_t_29 = __Pyx_PyObject_AsWritableString(__pyx_t_10); if (unlikely((!__pyx_t_29) && PyErr_Occurred())) __PYX_ERR(0, 248, __pyx_L1_error) - __pyx_v_rawval = __pyx_t_29; - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - - /* "msgpack/_packer.pyx":249 - * longval = o.code - * rawval = o.data - * L = len(o.data) # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * raise ValueError("EXT data is too large") - */ - __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_o, __pyx_n_s_data); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 249, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __pyx_t_27 = PyObject_Length(__pyx_t_10); if (unlikely(__pyx_t_27 == ((Py_ssize_t)-1))) __PYX_ERR(0, 249, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_v_L = __pyx_t_27; - - /* "msgpack/_packer.pyx":250 - * rawval = o.data - * L = len(o.data) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("EXT data is too large") - * ret = msgpack_pack_ext(&self.pk, longval, L) - */ - __pyx_t_3 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_3)) { - - /* "msgpack/_packer.pyx":251 - * L = len(o.data) - * if L > ITEM_LIMIT: - * raise ValueError("EXT data is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_ext(&self.pk, longval, L) - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - */ - __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 251, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_Raise(__pyx_t_10, 0, 0, 0); - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(0, 251, __pyx_L1_error) - - /* "msgpack/_packer.pyx":250 - * rawval = o.data - * L = len(o.data) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("EXT data is too large") - * ret = msgpack_pack_ext(&self.pk, longval, L) - */ - } - - /* "msgpack/_packer.pyx":252 - * if L > ITEM_LIMIT: - * raise ValueError("EXT data is too large") - * ret = msgpack_pack_ext(&self.pk, longval, L) # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - */ - __pyx_v_ret = msgpack_pack_ext((&__pyx_v_self->pk), __pyx_v_longval, __pyx_v_L); - - /* "msgpack/_packer.pyx":253 - * raise ValueError("EXT data is too large") - * ret = msgpack_pack_ext(&self.pk, longval, L) - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) # <<<<<<<<<<<<<< - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - * L = len(o) - */ - __pyx_v_ret = msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_v_L); - - /* "msgpack/_packer.pyx":245 - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - * elif type(o) is ExtType if strict_types else isinstance(o, ExtType): # <<<<<<<<<<<<<< - * # This should be before Tuple because ExtType is namedtuple. - * longval = o.code - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":254 - * ret = msgpack_pack_ext(&self.pk, longval, L) - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - if ((__pyx_v_strict_types != 0)) { - __pyx_t_3 = PyList_CheckExact(__pyx_v_o); - } else { - __pyx_t_1 = PyTuple_Check(__pyx_v_o); - if (!__pyx_t_1) { - } else { - __pyx_t_12 = __pyx_t_1; - goto __pyx_L55_bool_binop_done; - } - __pyx_t_1 = PyList_Check(__pyx_v_o); - __pyx_t_12 = __pyx_t_1; - __pyx_L55_bool_binop_done:; - __pyx_t_3 = __pyx_t_12; - } - __pyx_t_12 = (__pyx_t_3 != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":255 - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - * L = len(o) # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * raise ValueError("list is too large") - */ - __pyx_t_27 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_27 == ((Py_ssize_t)-1))) __PYX_ERR(0, 255, __pyx_L1_error) - __pyx_v_L = __pyx_t_27; - - /* "msgpack/_packer.pyx":256 - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("list is too large") - * ret = msgpack_pack_array(&self.pk, L) - */ - __pyx_t_12 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":257 - * L = len(o) - * if L > ITEM_LIMIT: - * raise ValueError("list is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_array(&self.pk, L) - * if ret == 0: - */ - __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 257, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_Raise(__pyx_t_10, 0, 0, 0); - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(0, 257, __pyx_L1_error) - - /* "msgpack/_packer.pyx":256 - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - * L = len(o) - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError("list is too large") - * ret = msgpack_pack_array(&self.pk, L) - */ - } - - /* "msgpack/_packer.pyx":258 - * if L > ITEM_LIMIT: - * raise ValueError("list is too large") - * ret = msgpack_pack_array(&self.pk, L) # <<<<<<<<<<<<<< - * if ret == 0: - * for v in o: - */ - __pyx_v_ret = msgpack_pack_array((&__pyx_v_self->pk), __pyx_v_L); - - /* "msgpack/_packer.pyx":259 - * raise ValueError("list is too large") - * ret = msgpack_pack_array(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * for v in o: - * ret = self._pack(v, nest_limit-1) - */ - __pyx_t_12 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":260 - * ret = msgpack_pack_array(&self.pk, L) - * if ret == 0: - * for v in o: # <<<<<<<<<<<<<< - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - */ - if (likely(PyList_CheckExact(__pyx_v_o)) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_10 = __pyx_v_o; __Pyx_INCREF(__pyx_t_10); __pyx_t_27 = 0; - __pyx_t_32 = NULL; - } else { - __pyx_t_27 = -1; __pyx_t_10 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 260, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __pyx_t_32 = Py_TYPE(__pyx_t_10)->tp_iternext; if (unlikely(!__pyx_t_32)) __PYX_ERR(0, 260, __pyx_L1_error) - } - for (;;) { - if (likely(!__pyx_t_32)) { - if (likely(PyList_CheckExact(__pyx_t_10))) { - if (__pyx_t_27 >= PyList_GET_SIZE(__pyx_t_10)) break; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_11 = PyList_GET_ITEM(__pyx_t_10, __pyx_t_27); __Pyx_INCREF(__pyx_t_11); __pyx_t_27++; if (unlikely(0 < 0)) __PYX_ERR(0, 260, __pyx_L1_error) - #else - __pyx_t_11 = PySequence_ITEM(__pyx_t_10, __pyx_t_27); __pyx_t_27++; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 260, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - #endif - } else { - if (__pyx_t_27 >= PyTuple_GET_SIZE(__pyx_t_10)) break; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_11 = PyTuple_GET_ITEM(__pyx_t_10, __pyx_t_27); __Pyx_INCREF(__pyx_t_11); __pyx_t_27++; if (unlikely(0 < 0)) __PYX_ERR(0, 260, __pyx_L1_error) - #else - __pyx_t_11 = PySequence_ITEM(__pyx_t_10, __pyx_t_27); __pyx_t_27++; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 260, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_11); - #endif - } - } else { - __pyx_t_11 = __pyx_t_32(__pyx_t_10); - if (unlikely(!__pyx_t_11)) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 260, __pyx_L1_error) - } - break; - } - __Pyx_GOTREF(__pyx_t_11); - } - __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_11); - __pyx_t_11 = 0; - - /* "msgpack/_packer.pyx":261 - * if ret == 0: - * for v in o: - * ret = self._pack(v, nest_limit-1) # <<<<<<<<<<<<<< - * if ret != 0: break - * elif PyMemoryView_Check(o): - */ - __pyx_t_31.__pyx_n = 1; - __pyx_t_31.nest_limit = (__pyx_v_nest_limit - 1); - __pyx_t_16 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_v, &__pyx_t_31); if (unlikely(__pyx_t_16 == ((int)-1))) __PYX_ERR(0, 261, __pyx_L1_error) - __pyx_v_ret = __pyx_t_16; - - /* "msgpack/_packer.pyx":262 - * for v in o: - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break # <<<<<<<<<<<<<< - * elif PyMemoryView_Check(o): - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - */ - __pyx_t_12 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_12) { - goto __pyx_L60_break; - } - - /* "msgpack/_packer.pyx":260 - * ret = msgpack_pack_array(&self.pk, L) - * if ret == 0: - * for v in o: # <<<<<<<<<<<<<< - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - */ - } - __pyx_L60_break:; - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - - /* "msgpack/_packer.pyx":259 - * raise ValueError("list is too large") - * ret = msgpack_pack_array(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * for v in o: - * ret = self._pack(v, nest_limit-1) - */ - } - - /* "msgpack/_packer.pyx":254 - * ret = msgpack_pack_ext(&self.pk, longval, L) - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - * elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): # <<<<<<<<<<<<<< - * L = len(o) - * if L > ITEM_LIMIT: - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":263 - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - * elif PyMemoryView_Check(o): # <<<<<<<<<<<<<< - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - * raise ValueError("could not get buffer for memoryview") - */ - __pyx_t_12 = (PyMemoryView_Check(__pyx_v_o) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":264 - * if ret != 0: break - * elif PyMemoryView_Check(o): - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: # <<<<<<<<<<<<<< - * raise ValueError("could not get buffer for memoryview") - * L = view.len - */ - __pyx_t_16 = PyObject_GetBuffer(__pyx_v_o, (&__pyx_v_view), PyBUF_SIMPLE); if (unlikely(__pyx_t_16 == ((int)-1))) __PYX_ERR(0, 264, __pyx_L1_error) - __pyx_t_12 = ((__pyx_t_16 != 0) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":265 - * elif PyMemoryView_Check(o): - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - * raise ValueError("could not get buffer for memoryview") # <<<<<<<<<<<<<< - * L = view.len - * if L > ITEM_LIMIT: - */ - __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 265, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_Raise(__pyx_t_10, 0, 0, 0); - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(0, 265, __pyx_L1_error) - - /* "msgpack/_packer.pyx":264 - * if ret != 0: break - * elif PyMemoryView_Check(o): - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: # <<<<<<<<<<<<<< - * raise ValueError("could not get buffer for memoryview") - * L = view.len - */ - } - - /* "msgpack/_packer.pyx":266 - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - * raise ValueError("could not get buffer for memoryview") - * L = view.len # <<<<<<<<<<<<<< - * if L > ITEM_LIMIT: - * PyBuffer_Release(&view); - */ - __pyx_t_27 = __pyx_v_view.len; - __pyx_v_L = __pyx_t_27; - - /* "msgpack/_packer.pyx":267 - * raise ValueError("could not get buffer for memoryview") - * L = view.len - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * PyBuffer_Release(&view); - * raise ValueError("memoryview is too large") - */ - __pyx_t_12 = ((__pyx_v_L > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_12)) { - - /* "msgpack/_packer.pyx":268 - * L = view.len - * if L > ITEM_LIMIT: - * PyBuffer_Release(&view); # <<<<<<<<<<<<<< - * raise ValueError("memoryview is too large") - * ret = msgpack_pack_bin(&self.pk, L) - */ - PyBuffer_Release((&__pyx_v_view)); - - /* "msgpack/_packer.pyx":269 - * if L > ITEM_LIMIT: - * PyBuffer_Release(&view); - * raise ValueError("memoryview is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: - */ - __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__11, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 269, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_Raise(__pyx_t_10, 0, 0, 0); - __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(0, 269, __pyx_L1_error) - - /* "msgpack/_packer.pyx":267 - * raise ValueError("could not get buffer for memoryview") - * L = view.len - * if L > ITEM_LIMIT: # <<<<<<<<<<<<<< - * PyBuffer_Release(&view); - * raise ValueError("memoryview is too large") - */ - } - - /* "msgpack/_packer.pyx":270 - * PyBuffer_Release(&view); - * raise ValueError("memoryview is too large") - * ret = msgpack_pack_bin(&self.pk, L) # <<<<<<<<<<<<<< - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - */ - __pyx_v_ret = msgpack_pack_bin((&__pyx_v_self->pk), __pyx_v_L); - - /* "msgpack/_packer.pyx":271 - * raise ValueError("memoryview is too large") - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - * PyBuffer_Release(&view); - */ - __pyx_t_12 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":272 - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) # <<<<<<<<<<<<<< - * PyBuffer_Release(&view); - * elif not default_used and self._default: - */ - __pyx_v_ret = msgpack_pack_raw_body((&__pyx_v_self->pk), ((char *)__pyx_v_view.buf), __pyx_v_L); - - /* "msgpack/_packer.pyx":271 - * raise ValueError("memoryview is too large") - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: # <<<<<<<<<<<<<< - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - * PyBuffer_Release(&view); - */ - } - - /* "msgpack/_packer.pyx":273 - * if ret == 0: - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - * PyBuffer_Release(&view); # <<<<<<<<<<<<<< - * elif not default_used and self._default: - * o = self._default(o) - */ - PyBuffer_Release((&__pyx_v_view)); - - /* "msgpack/_packer.pyx":263 - * ret = self._pack(v, nest_limit-1) - * if ret != 0: break - * elif PyMemoryView_Check(o): # <<<<<<<<<<<<<< - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - * raise ValueError("could not get buffer for memoryview") - */ - goto __pyx_L6; - } - - /* "msgpack/_packer.pyx":274 - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - * PyBuffer_Release(&view); - * elif not default_used and self._default: # <<<<<<<<<<<<<< - * o = self._default(o) - * default_used = 1 - */ - __pyx_t_3 = ((!(__pyx_v_default_used != 0)) != 0); - if (__pyx_t_3) { - } else { - __pyx_t_12 = __pyx_t_3; - goto __pyx_L65_bool_binop_done; - } - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_self->_default); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 274, __pyx_L1_error) - __pyx_t_12 = __pyx_t_3; - __pyx_L65_bool_binop_done:; - if (__pyx_t_12) { - - /* "msgpack/_packer.pyx":275 - * PyBuffer_Release(&view); - * elif not default_used and self._default: - * o = self._default(o) # <<<<<<<<<<<<<< - * default_used = 1 - * continue - */ - __Pyx_INCREF(__pyx_v_self->_default); - __pyx_t_11 = __pyx_v_self->_default; __pyx_t_2 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_11); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_11, function); - } - } - __pyx_t_10 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_11, __pyx_t_2, __pyx_v_o) : __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_v_o); - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 275, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_10); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __Pyx_DECREF_SET(__pyx_v_o, __pyx_t_10); - __pyx_t_10 = 0; - - /* "msgpack/_packer.pyx":276 - * elif not default_used and self._default: - * o = self._default(o) - * default_used = 1 # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_v_default_used = 1; - - /* "msgpack/_packer.pyx":277 - * o = self._default(o) - * default_used = 1 - * continue # <<<<<<<<<<<<<< - * else: - * PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) - */ - goto __pyx_L4_continue; - - /* "msgpack/_packer.pyx":274 - * ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - * PyBuffer_Release(&view); - * elif not default_used and self._default: # <<<<<<<<<<<<<< - * o = self._default(o) - * default_used = 1 - */ - } - - /* "msgpack/_packer.pyx":279 - * continue - * else: - * PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) # <<<<<<<<<<<<<< - * return ret - * - */ - /*else*/ { - __pyx_t_28 = PyErr_Format(__pyx_builtin_TypeError, ((char *)"can not serialize '%.200s' object"), Py_TYPE(__pyx_v_o)->tp_name); if (unlikely(__pyx_t_28 == ((PyObject *)NULL))) __PYX_ERR(0, 279, __pyx_L1_error) - } - __pyx_L6:; - - /* "msgpack/_packer.pyx":280 - * else: - * PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) - * return ret # <<<<<<<<<<<<<< - * - * cpdef pack(self, object obj): - */ - __pyx_r = __pyx_v_ret; - goto __pyx_L0; - __pyx_L4_continue:; - } - - /* "msgpack/_packer.pyx":148 - * self.pk.buf = NULL - * - * cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: # <<<<<<<<<<<<<< - * cdef long long llval - * cdef unsigned long long ullval - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_10); - __Pyx_XDECREF(__pyx_t_11); - __Pyx_XDECREF(__pyx_t_13); - __Pyx_XDECREF(__pyx_t_14); - __Pyx_XDECREF(__pyx_t_15); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer._pack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_d); - __Pyx_XDECREF(__pyx_v_oe); - __Pyx_XDECREF(__pyx_v_k); - __Pyx_XDECREF(__pyx_v_v); - __Pyx_XDECREF(__pyx_v_o); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":282 - * return ret - * - * cpdef pack(self, object obj): # <<<<<<<<<<<<<< - * cdef int ret - * try: - */ - -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_7pack(PyObject *__pyx_v_self, PyObject *__pyx_v_obj); /*proto*/ -static PyObject *__pyx_f_7msgpack_9_cmsgpack_6Packer_pack(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_skip_dispatch) { - int __pyx_v_ret; - PyObject *__pyx_v_buf = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - int __pyx_t_8; - struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack __pyx_t_9; - int __pyx_t_10; - __Pyx_RefNannySetupContext("pack", 0); - /* Check if called by wrapper */ - if (unlikely(__pyx_skip_dispatch)) ; - /* Check if overridden in Python */ - else if (unlikely((Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0) || (Py_TYPE(((PyObject *)__pyx_v_self))->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) { - #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS - static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT; - if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) { - PY_UINT64_T __pyx_type_dict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self)); - #endif - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 282, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (!PyCFunction_Check(__pyx_t_1) || (PyCFunction_GET_FUNCTION(__pyx_t_1) != (PyCFunction)(void*)__pyx_pw_7msgpack_9_cmsgpack_6Packer_7pack)) { - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_t_1); - __pyx_t_3 = __pyx_t_1; __pyx_t_4 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - __pyx_t_2 = (__pyx_t_4) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_4, __pyx_v_obj) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_v_obj); - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 282, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - goto __pyx_L0; - } - #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS - __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self)); - __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self)); - if (unlikely(__pyx_type_dict_guard != __pyx_tp_dict_version)) { - __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT; - } - #endif - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS - } - #endif - } - - /* "msgpack/_packer.pyx":284 - * cpdef pack(self, object obj): - * cdef int ret - * try: # <<<<<<<<<<<<<< - * ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - * except: - */ - { - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ExceptionSave(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7); - __Pyx_XGOTREF(__pyx_t_5); - __Pyx_XGOTREF(__pyx_t_6); - __Pyx_XGOTREF(__pyx_t_7); - /*try:*/ { - - /* "msgpack/_packer.pyx":285 - * cdef int ret - * try: - * ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) # <<<<<<<<<<<<<< - * except: - * self.pk.length = 0 - */ - __pyx_t_9.__pyx_n = 1; - __pyx_t_9.nest_limit = __pyx_v_7msgpack_9_cmsgpack_DEFAULT_RECURSE_LIMIT; - __pyx_t_8 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_obj, &__pyx_t_9); if (unlikely(__pyx_t_8 == ((int)-1))) __PYX_ERR(0, 285, __pyx_L3_error) - __pyx_v_ret = __pyx_t_8; - - /* "msgpack/_packer.pyx":284 - * cpdef pack(self, object obj): - * cdef int ret - * try: # <<<<<<<<<<<<<< - * ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - * except: - */ - } - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L8_try_end; - __pyx_L3_error:; - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "msgpack/_packer.pyx":286 - * try: - * ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - * except: # <<<<<<<<<<<<<< - * self.pk.length = 0 - * raise - */ - /*except:*/ { - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3) < 0) __PYX_ERR(0, 286, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_t_3); - - /* "msgpack/_packer.pyx":287 - * ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - * except: - * self.pk.length = 0 # <<<<<<<<<<<<<< - * raise - * if ret: # should not happen. - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":288 - * except: - * self.pk.length = 0 - * raise # <<<<<<<<<<<<<< - * if ret: # should not happen. - * raise RuntimeError("internal error") - */ - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_2); - __Pyx_XGIVEREF(__pyx_t_3); - __Pyx_ErrRestoreWithState(__pyx_t_1, __pyx_t_2, __pyx_t_3); - __pyx_t_1 = 0; __pyx_t_2 = 0; __pyx_t_3 = 0; - __PYX_ERR(0, 288, __pyx_L5_except_error) - } - __pyx_L5_except_error:; - - /* "msgpack/_packer.pyx":284 - * cpdef pack(self, object obj): - * cdef int ret - * try: # <<<<<<<<<<<<<< - * ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - * except: - */ - __Pyx_XGIVEREF(__pyx_t_5); - __Pyx_XGIVEREF(__pyx_t_6); - __Pyx_XGIVEREF(__pyx_t_7); - __Pyx_ExceptionReset(__pyx_t_5, __pyx_t_6, __pyx_t_7); - goto __pyx_L1_error; - __pyx_L8_try_end:; - } - - /* "msgpack/_packer.pyx":289 - * self.pk.length = 0 - * raise - * if ret: # should not happen. # <<<<<<<<<<<<<< - * raise RuntimeError("internal error") - * if self.autoreset: - */ - __pyx_t_10 = (__pyx_v_ret != 0); - if (unlikely(__pyx_t_10)) { - - /* "msgpack/_packer.pyx":290 - * raise - * if ret: # should not happen. - * raise RuntimeError("internal error") # <<<<<<<<<<<<<< - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__12, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 290, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(0, 290, __pyx_L1_error) - - /* "msgpack/_packer.pyx":289 - * self.pk.length = 0 - * raise - * if ret: # should not happen. # <<<<<<<<<<<<<< - * raise RuntimeError("internal error") - * if self.autoreset: - */ - } - - /* "msgpack/_packer.pyx":291 - * if ret: # should not happen. - * raise RuntimeError("internal error") - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - __pyx_t_10 = (__pyx_v_self->autoreset != 0); - if (__pyx_t_10) { - - /* "msgpack/_packer.pyx":292 - * raise RuntimeError("internal error") - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) # <<<<<<<<<<<<<< - * self.pk.length = 0 - * return buf - */ - __pyx_t_3 = PyBytes_FromStringAndSize(__pyx_v_self->pk.buf, __pyx_v_self->pk.length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 292, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_v_buf = ((PyObject*)__pyx_t_3); - __pyx_t_3 = 0; - - /* "msgpack/_packer.pyx":293 - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 # <<<<<<<<<<<<<< - * return buf - * - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":294 - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - * return buf # <<<<<<<<<<<<<< - * - * def pack_ext_type(self, typecode, data): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_buf); - __pyx_r = __pyx_v_buf; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":291 - * if ret: # should not happen. - * raise RuntimeError("internal error") - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - } - - /* "msgpack/_packer.pyx":282 - * return ret - * - * cpdef pack(self, object obj): # <<<<<<<<<<<<<< - * cdef int ret - * try: - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_buf); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_7pack(PyObject *__pyx_v_self, PyObject *__pyx_v_obj); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_6pack[] = "Packer.pack(self, obj)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_7pack(PyObject *__pyx_v_self, PyObject *__pyx_v_obj) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("pack (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_6pack(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), ((PyObject *)__pyx_v_obj)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_6pack(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_obj) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("pack", 0); - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_7msgpack_9_cmsgpack_6Packer_pack(__pyx_v_self, __pyx_v_obj, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 282, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":296 - * return buf - * - * def pack_ext_type(self, typecode, data): # <<<<<<<<<<<<<< - * msgpack_pack_ext(&self.pk, typecode, len(data)) - * msgpack_pack_raw_body(&self.pk, data, len(data)) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_9pack_ext_type(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_8pack_ext_type[] = "Packer.pack_ext_type(self, typecode, data)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_9pack_ext_type(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_typecode = 0; - PyObject *__pyx_v_data = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("pack_ext_type (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_typecode,&__pyx_n_s_data,0}; - PyObject* values[2] = {0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_typecode)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - CYTHON_FALLTHROUGH; - case 1: - if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_data)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("pack_ext_type", 1, 2, 2, 1); __PYX_ERR(0, 296, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "pack_ext_type") < 0)) __PYX_ERR(0, 296, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - } - __pyx_v_typecode = values[0]; - __pyx_v_data = values[1]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack_ext_type", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 296, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_ext_type", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_8pack_ext_type(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), __pyx_v_typecode, __pyx_v_data); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_8pack_ext_type(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_typecode, PyObject *__pyx_v_data) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - char __pyx_t_1; - Py_ssize_t __pyx_t_2; - char *__pyx_t_3; - __Pyx_RefNannySetupContext("pack_ext_type", 0); - - /* "msgpack/_packer.pyx":297 - * - * def pack_ext_type(self, typecode, data): - * msgpack_pack_ext(&self.pk, typecode, len(data)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, data, len(data)) - * - */ - __pyx_t_1 = __Pyx_PyInt_As_char(__pyx_v_typecode); if (unlikely((__pyx_t_1 == (char)-1) && PyErr_Occurred())) __PYX_ERR(0, 297, __pyx_L1_error) - __pyx_t_2 = PyObject_Length(__pyx_v_data); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 297, __pyx_L1_error) - (void)(msgpack_pack_ext((&__pyx_v_self->pk), __pyx_t_1, __pyx_t_2)); - - /* "msgpack/_packer.pyx":298 - * def pack_ext_type(self, typecode, data): - * msgpack_pack_ext(&self.pk, typecode, len(data)) - * msgpack_pack_raw_body(&self.pk, data, len(data)) # <<<<<<<<<<<<<< - * - * def pack_array_header(self, long long size): - */ - __pyx_t_3 = __Pyx_PyObject_AsWritableString(__pyx_v_data); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) __PYX_ERR(0, 298, __pyx_L1_error) - __pyx_t_2 = PyObject_Length(__pyx_v_data); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 298, __pyx_L1_error) - (void)(msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_t_3, __pyx_t_2)); - - /* "msgpack/_packer.pyx":296 - * return buf - * - * def pack_ext_type(self, typecode, data): # <<<<<<<<<<<<<< - * msgpack_pack_ext(&self.pk, typecode, len(data)) - * msgpack_pack_raw_body(&self.pk, data, len(data)) - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_ext_type", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":300 - * msgpack_pack_raw_body(&self.pk, data, len(data)) - * - * def pack_array_header(self, long long size): # <<<<<<<<<<<<<< - * if size > ITEM_LIMIT: - * raise ValueError - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_11pack_array_header(PyObject *__pyx_v_self, PyObject *__pyx_arg_size); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_10pack_array_header[] = "Packer.pack_array_header(self, long long size)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_11pack_array_header(PyObject *__pyx_v_self, PyObject *__pyx_arg_size) { - PY_LONG_LONG __pyx_v_size; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("pack_array_header (wrapper)", 0); - assert(__pyx_arg_size); { - __pyx_v_size = __Pyx_PyInt_As_PY_LONG_LONG(__pyx_arg_size); if (unlikely((__pyx_v_size == (PY_LONG_LONG)-1) && PyErr_Occurred())) __PYX_ERR(0, 300, __pyx_L3_error) - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_array_header", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_10pack_array_header(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), ((PY_LONG_LONG)__pyx_v_size)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_10pack_array_header(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PY_LONG_LONG __pyx_v_size) { - int __pyx_v_ret; - PyObject *__pyx_v_buf = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("pack_array_header", 0); - - /* "msgpack/_packer.pyx":301 - * - * def pack_array_header(self, long long size): - * if size > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError - * cdef int ret = msgpack_pack_array(&self.pk, size) - */ - __pyx_t_1 = ((__pyx_v_size > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":302 - * def pack_array_header(self, long long size): - * if size > ITEM_LIMIT: - * raise ValueError # <<<<<<<<<<<<<< - * cdef int ret = msgpack_pack_array(&self.pk, size) - * if ret == -1: - */ - __Pyx_Raise(__pyx_builtin_ValueError, 0, 0, 0); - __PYX_ERR(0, 302, __pyx_L1_error) - - /* "msgpack/_packer.pyx":301 - * - * def pack_array_header(self, long long size): - * if size > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError - * cdef int ret = msgpack_pack_array(&self.pk, size) - */ - } - - /* "msgpack/_packer.pyx":303 - * if size > ITEM_LIMIT: - * raise ValueError - * cdef int ret = msgpack_pack_array(&self.pk, size) # <<<<<<<<<<<<<< - * if ret == -1: - * raise MemoryError - */ - __pyx_v_ret = msgpack_pack_array((&__pyx_v_self->pk), __pyx_v_size); - - /* "msgpack/_packer.pyx":304 - * raise ValueError - * cdef int ret = msgpack_pack_array(&self.pk, size) - * if ret == -1: # <<<<<<<<<<<<<< - * raise MemoryError - * elif ret: # should not happen - */ - __pyx_t_1 = ((__pyx_v_ret == -1L) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":305 - * cdef int ret = msgpack_pack_array(&self.pk, size) - * if ret == -1: - * raise MemoryError # <<<<<<<<<<<<<< - * elif ret: # should not happen - * raise TypeError - */ - PyErr_NoMemory(); __PYX_ERR(0, 305, __pyx_L1_error) - - /* "msgpack/_packer.pyx":304 - * raise ValueError - * cdef int ret = msgpack_pack_array(&self.pk, size) - * if ret == -1: # <<<<<<<<<<<<<< - * raise MemoryError - * elif ret: # should not happen - */ - } - - /* "msgpack/_packer.pyx":306 - * if ret == -1: - * raise MemoryError - * elif ret: # should not happen # <<<<<<<<<<<<<< - * raise TypeError - * if self.autoreset: - */ - __pyx_t_1 = (__pyx_v_ret != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":307 - * raise MemoryError - * elif ret: # should not happen - * raise TypeError # <<<<<<<<<<<<<< - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - __Pyx_Raise(__pyx_builtin_TypeError, 0, 0, 0); - __PYX_ERR(0, 307, __pyx_L1_error) - - /* "msgpack/_packer.pyx":306 - * if ret == -1: - * raise MemoryError - * elif ret: # should not happen # <<<<<<<<<<<<<< - * raise TypeError - * if self.autoreset: - */ - } - - /* "msgpack/_packer.pyx":308 - * elif ret: # should not happen - * raise TypeError - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - __pyx_t_1 = (__pyx_v_self->autoreset != 0); - if (__pyx_t_1) { - - /* "msgpack/_packer.pyx":309 - * raise TypeError - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) # <<<<<<<<<<<<<< - * self.pk.length = 0 - * return buf - */ - __pyx_t_2 = PyBytes_FromStringAndSize(__pyx_v_self->pk.buf, __pyx_v_self->pk.length); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 309, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_v_buf = ((PyObject*)__pyx_t_2); - __pyx_t_2 = 0; - - /* "msgpack/_packer.pyx":310 - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 # <<<<<<<<<<<<<< - * return buf - * - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":311 - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - * return buf # <<<<<<<<<<<<<< - * - * def pack_map_header(self, long long size): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_buf); - __pyx_r = __pyx_v_buf; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":308 - * elif ret: # should not happen - * raise TypeError - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - } - - /* "msgpack/_packer.pyx":300 - * msgpack_pack_raw_body(&self.pk, data, len(data)) - * - * def pack_array_header(self, long long size): # <<<<<<<<<<<<<< - * if size > ITEM_LIMIT: - * raise ValueError - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_array_header", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_buf); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":313 - * return buf - * - * def pack_map_header(self, long long size): # <<<<<<<<<<<<<< - * if size > ITEM_LIMIT: - * raise ValueError - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_13pack_map_header(PyObject *__pyx_v_self, PyObject *__pyx_arg_size); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_12pack_map_header[] = "Packer.pack_map_header(self, long long size)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_13pack_map_header(PyObject *__pyx_v_self, PyObject *__pyx_arg_size) { - PY_LONG_LONG __pyx_v_size; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("pack_map_header (wrapper)", 0); - assert(__pyx_arg_size); { - __pyx_v_size = __Pyx_PyInt_As_PY_LONG_LONG(__pyx_arg_size); if (unlikely((__pyx_v_size == (PY_LONG_LONG)-1) && PyErr_Occurred())) __PYX_ERR(0, 313, __pyx_L3_error) - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_map_header", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_12pack_map_header(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), ((PY_LONG_LONG)__pyx_v_size)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_12pack_map_header(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PY_LONG_LONG __pyx_v_size) { - int __pyx_v_ret; - PyObject *__pyx_v_buf = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("pack_map_header", 0); - - /* "msgpack/_packer.pyx":314 - * - * def pack_map_header(self, long long size): - * if size > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError - * cdef int ret = msgpack_pack_map(&self.pk, size) - */ - __pyx_t_1 = ((__pyx_v_size > __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":315 - * def pack_map_header(self, long long size): - * if size > ITEM_LIMIT: - * raise ValueError # <<<<<<<<<<<<<< - * cdef int ret = msgpack_pack_map(&self.pk, size) - * if ret == -1: - */ - __Pyx_Raise(__pyx_builtin_ValueError, 0, 0, 0); - __PYX_ERR(0, 315, __pyx_L1_error) - - /* "msgpack/_packer.pyx":314 - * - * def pack_map_header(self, long long size): - * if size > ITEM_LIMIT: # <<<<<<<<<<<<<< - * raise ValueError - * cdef int ret = msgpack_pack_map(&self.pk, size) - */ - } - - /* "msgpack/_packer.pyx":316 - * if size > ITEM_LIMIT: - * raise ValueError - * cdef int ret = msgpack_pack_map(&self.pk, size) # <<<<<<<<<<<<<< - * if ret == -1: - * raise MemoryError - */ - __pyx_v_ret = msgpack_pack_map((&__pyx_v_self->pk), __pyx_v_size); - - /* "msgpack/_packer.pyx":317 - * raise ValueError - * cdef int ret = msgpack_pack_map(&self.pk, size) - * if ret == -1: # <<<<<<<<<<<<<< - * raise MemoryError - * elif ret: # should not happen - */ - __pyx_t_1 = ((__pyx_v_ret == -1L) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":318 - * cdef int ret = msgpack_pack_map(&self.pk, size) - * if ret == -1: - * raise MemoryError # <<<<<<<<<<<<<< - * elif ret: # should not happen - * raise TypeError - */ - PyErr_NoMemory(); __PYX_ERR(0, 318, __pyx_L1_error) - - /* "msgpack/_packer.pyx":317 - * raise ValueError - * cdef int ret = msgpack_pack_map(&self.pk, size) - * if ret == -1: # <<<<<<<<<<<<<< - * raise MemoryError - * elif ret: # should not happen - */ - } - - /* "msgpack/_packer.pyx":319 - * if ret == -1: - * raise MemoryError - * elif ret: # should not happen # <<<<<<<<<<<<<< - * raise TypeError - * if self.autoreset: - */ - __pyx_t_1 = (__pyx_v_ret != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_packer.pyx":320 - * raise MemoryError - * elif ret: # should not happen - * raise TypeError # <<<<<<<<<<<<<< - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - __Pyx_Raise(__pyx_builtin_TypeError, 0, 0, 0); - __PYX_ERR(0, 320, __pyx_L1_error) - - /* "msgpack/_packer.pyx":319 - * if ret == -1: - * raise MemoryError - * elif ret: # should not happen # <<<<<<<<<<<<<< - * raise TypeError - * if self.autoreset: - */ - } - - /* "msgpack/_packer.pyx":321 - * elif ret: # should not happen - * raise TypeError - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - __pyx_t_1 = (__pyx_v_self->autoreset != 0); - if (__pyx_t_1) { - - /* "msgpack/_packer.pyx":322 - * raise TypeError - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) # <<<<<<<<<<<<<< - * self.pk.length = 0 - * return buf - */ - __pyx_t_2 = PyBytes_FromStringAndSize(__pyx_v_self->pk.buf, __pyx_v_self->pk.length); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 322, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_v_buf = ((PyObject*)__pyx_t_2); - __pyx_t_2 = 0; - - /* "msgpack/_packer.pyx":323 - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 # <<<<<<<<<<<<<< - * return buf - * - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":324 - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - * return buf # <<<<<<<<<<<<<< - * - * def pack_map_pairs(self, object pairs): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_buf); - __pyx_r = __pyx_v_buf; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":321 - * elif ret: # should not happen - * raise TypeError - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - } - - /* "msgpack/_packer.pyx":313 - * return buf - * - * def pack_map_header(self, long long size): # <<<<<<<<<<<<<< - * if size > ITEM_LIMIT: - * raise ValueError - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_map_header", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_buf); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":326 - * return buf - * - * def pack_map_pairs(self, object pairs): # <<<<<<<<<<<<<< - * """ - * Pack *pairs* as msgpack map type. - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_15pack_map_pairs(PyObject *__pyx_v_self, PyObject *__pyx_v_pairs); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_14pack_map_pairs[] = "Packer.pack_map_pairs(self, pairs)\n\n Pack *pairs* as msgpack map type.\n\n *pairs* should be a sequence of pairs.\n (`len(pairs)` and `for k, v in pairs:` should be supported.)\n "; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_15pack_map_pairs(PyObject *__pyx_v_self, PyObject *__pyx_v_pairs) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("pack_map_pairs (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_14pack_map_pairs(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), ((PyObject *)__pyx_v_pairs)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_14pack_map_pairs(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, PyObject *__pyx_v_pairs) { - int __pyx_v_ret; - PyObject *__pyx_v_k = NULL; - PyObject *__pyx_v_v = NULL; - PyObject *__pyx_v_buf = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - Py_ssize_t __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *(*__pyx_t_4)(PyObject *); - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - PyObject *(*__pyx_t_9)(PyObject *); - int __pyx_t_10; - __Pyx_RefNannySetupContext("pack_map_pairs", 0); - - /* "msgpack/_packer.pyx":333 - * (`len(pairs)` and `for k, v in pairs:` should be supported.) - * """ - * cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) # <<<<<<<<<<<<<< - * if ret == 0: - * for k, v in pairs: - */ - __pyx_t_1 = PyObject_Length(__pyx_v_pairs); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 333, __pyx_L1_error) - __pyx_v_ret = msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_1); - - /* "msgpack/_packer.pyx":334 - * """ - * cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - * if ret == 0: # <<<<<<<<<<<<<< - * for k, v in pairs: - * ret = self._pack(k) - */ - __pyx_t_2 = ((__pyx_v_ret == 0) != 0); - if (__pyx_t_2) { - - /* "msgpack/_packer.pyx":335 - * cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - * if ret == 0: - * for k, v in pairs: # <<<<<<<<<<<<<< - * ret = self._pack(k) - * if ret != 0: break - */ - if (likely(PyList_CheckExact(__pyx_v_pairs)) || PyTuple_CheckExact(__pyx_v_pairs)) { - __pyx_t_3 = __pyx_v_pairs; __Pyx_INCREF(__pyx_t_3); __pyx_t_1 = 0; - __pyx_t_4 = NULL; - } else { - __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_pairs); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 335, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) - } - for (;;) { - if (likely(!__pyx_t_4)) { - if (likely(PyList_CheckExact(__pyx_t_3))) { - if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_3)) break; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_5 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_5); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(0, 335, __pyx_L1_error) - #else - __pyx_t_5 = PySequence_ITEM(__pyx_t_3, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 335, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - #endif - } else { - if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_3)) break; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_5); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(0, 335, __pyx_L1_error) - #else - __pyx_t_5 = PySequence_ITEM(__pyx_t_3, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 335, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - #endif - } - } else { - __pyx_t_5 = __pyx_t_4(__pyx_t_3); - if (unlikely(!__pyx_t_5)) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(0, 335, __pyx_L1_error) - } - break; - } - __Pyx_GOTREF(__pyx_t_5); - } - if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { - PyObject* sequence = __pyx_t_5; - Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - __PYX_ERR(0, 335, __pyx_L1_error) - } - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - if (likely(PyTuple_CheckExact(sequence))) { - __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); - __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); - } else { - __pyx_t_6 = PyList_GET_ITEM(sequence, 0); - __pyx_t_7 = PyList_GET_ITEM(sequence, 1); - } - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(__pyx_t_7); - #else - __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 335, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 335, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - #endif - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } else { - Py_ssize_t index = -1; - __pyx_t_8 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 335, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext; - index = 0; __pyx_t_6 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_6)) goto __pyx_L6_unpacking_failed; - __Pyx_GOTREF(__pyx_t_6); - index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L6_unpacking_failed; - __Pyx_GOTREF(__pyx_t_7); - if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) __PYX_ERR(0, 335, __pyx_L1_error) - __pyx_t_9 = NULL; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - goto __pyx_L7_unpacking_done; - __pyx_L6_unpacking_failed:; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_9 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); - __PYX_ERR(0, 335, __pyx_L1_error) - __pyx_L7_unpacking_done:; - } - __Pyx_XDECREF_SET(__pyx_v_k, __pyx_t_6); - __pyx_t_6 = 0; - __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_7); - __pyx_t_7 = 0; - - /* "msgpack/_packer.pyx":336 - * if ret == 0: - * for k, v in pairs: - * ret = self._pack(k) # <<<<<<<<<<<<<< - * if ret != 0: break - * ret = self._pack(v) - */ - __pyx_t_10 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_k, NULL); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(0, 336, __pyx_L1_error) - __pyx_v_ret = __pyx_t_10; - - /* "msgpack/_packer.pyx":337 - * for k, v in pairs: - * ret = self._pack(k) - * if ret != 0: break # <<<<<<<<<<<<<< - * ret = self._pack(v) - * if ret != 0: break - */ - __pyx_t_2 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_2) { - goto __pyx_L5_break; - } - - /* "msgpack/_packer.pyx":338 - * ret = self._pack(k) - * if ret != 0: break - * ret = self._pack(v) # <<<<<<<<<<<<<< - * if ret != 0: break - * if ret == -1: - */ - __pyx_t_10 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer *)__pyx_v_self->__pyx_vtab)->_pack(__pyx_v_self, __pyx_v_v, NULL); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(0, 338, __pyx_L1_error) - __pyx_v_ret = __pyx_t_10; - - /* "msgpack/_packer.pyx":339 - * if ret != 0: break - * ret = self._pack(v) - * if ret != 0: break # <<<<<<<<<<<<<< - * if ret == -1: - * raise MemoryError - */ - __pyx_t_2 = ((__pyx_v_ret != 0) != 0); - if (__pyx_t_2) { - goto __pyx_L5_break; - } - - /* "msgpack/_packer.pyx":335 - * cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - * if ret == 0: - * for k, v in pairs: # <<<<<<<<<<<<<< - * ret = self._pack(k) - * if ret != 0: break - */ - } - __pyx_L5_break:; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "msgpack/_packer.pyx":334 - * """ - * cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - * if ret == 0: # <<<<<<<<<<<<<< - * for k, v in pairs: - * ret = self._pack(k) - */ - } - - /* "msgpack/_packer.pyx":340 - * ret = self._pack(v) - * if ret != 0: break - * if ret == -1: # <<<<<<<<<<<<<< - * raise MemoryError - * elif ret: # should not happen - */ - __pyx_t_2 = ((__pyx_v_ret == -1L) != 0); - if (unlikely(__pyx_t_2)) { - - /* "msgpack/_packer.pyx":341 - * if ret != 0: break - * if ret == -1: - * raise MemoryError # <<<<<<<<<<<<<< - * elif ret: # should not happen - * raise TypeError - */ - PyErr_NoMemory(); __PYX_ERR(0, 341, __pyx_L1_error) - - /* "msgpack/_packer.pyx":340 - * ret = self._pack(v) - * if ret != 0: break - * if ret == -1: # <<<<<<<<<<<<<< - * raise MemoryError - * elif ret: # should not happen - */ - } - - /* "msgpack/_packer.pyx":342 - * if ret == -1: - * raise MemoryError - * elif ret: # should not happen # <<<<<<<<<<<<<< - * raise TypeError - * if self.autoreset: - */ - __pyx_t_2 = (__pyx_v_ret != 0); - if (unlikely(__pyx_t_2)) { - - /* "msgpack/_packer.pyx":343 - * raise MemoryError - * elif ret: # should not happen - * raise TypeError # <<<<<<<<<<<<<< - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - __Pyx_Raise(__pyx_builtin_TypeError, 0, 0, 0); - __PYX_ERR(0, 343, __pyx_L1_error) - - /* "msgpack/_packer.pyx":342 - * if ret == -1: - * raise MemoryError - * elif ret: # should not happen # <<<<<<<<<<<<<< - * raise TypeError - * if self.autoreset: - */ - } - - /* "msgpack/_packer.pyx":344 - * elif ret: # should not happen - * raise TypeError - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - __pyx_t_2 = (__pyx_v_self->autoreset != 0); - if (__pyx_t_2) { - - /* "msgpack/_packer.pyx":345 - * raise TypeError - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) # <<<<<<<<<<<<<< - * self.pk.length = 0 - * return buf - */ - __pyx_t_3 = PyBytes_FromStringAndSize(__pyx_v_self->pk.buf, __pyx_v_self->pk.length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 345, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_v_buf = ((PyObject*)__pyx_t_3); - __pyx_t_3 = 0; - - /* "msgpack/_packer.pyx":346 - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 # <<<<<<<<<<<<<< - * return buf - * - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":347 - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - * return buf # <<<<<<<<<<<<<< - * - * def reset(self): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_buf); - __pyx_r = __pyx_v_buf; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":344 - * elif ret: # should not happen - * raise TypeError - * if self.autoreset: # <<<<<<<<<<<<<< - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * self.pk.length = 0 - */ - } - - /* "msgpack/_packer.pyx":326 - * return buf - * - * def pack_map_pairs(self, object pairs): # <<<<<<<<<<<<<< - * """ - * Pack *pairs* as msgpack map type. - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.pack_map_pairs", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_k); - __Pyx_XDECREF(__pyx_v_v); - __Pyx_XDECREF(__pyx_v_buf); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":349 - * return buf - * - * def reset(self): # <<<<<<<<<<<<<< - * """Reset internal buffer. - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_17reset(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_16reset[] = "Packer.reset(self)\nReset internal buffer.\n\n This method is usaful only when autoreset=False.\n "; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_17reset(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("reset (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_16reset(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_16reset(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("reset", 0); - - /* "msgpack/_packer.pyx":354 - * This method is usaful only when autoreset=False. - * """ - * self.pk.length = 0 # <<<<<<<<<<<<<< - * - * def bytes(self): - */ - __pyx_v_self->pk.length = 0; - - /* "msgpack/_packer.pyx":349 - * return buf - * - * def reset(self): # <<<<<<<<<<<<<< - * """Reset internal buffer. - * - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":356 - * self.pk.length = 0 - * - * def bytes(self): # <<<<<<<<<<<<<< - * """Return internal buffer contents as bytes object""" - * return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_19bytes(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_18bytes[] = "Packer.bytes(self)\nReturn internal buffer contents as bytes object"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_19bytes(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("bytes (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_18bytes(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_18bytes(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("bytes", 0); - - /* "msgpack/_packer.pyx":358 - * def bytes(self): - * """Return internal buffer contents as bytes object""" - * return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) # <<<<<<<<<<<<<< - * - * def getbuffer(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyBytes_FromStringAndSize(__pyx_v_self->pk.buf, __pyx_v_self->pk.length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 358, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":356 - * self.pk.length = 0 - * - * def bytes(self): # <<<<<<<<<<<<<< - * """Return internal buffer contents as bytes object""" - * return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_packer.pyx":360 - * return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * - * def getbuffer(self): # <<<<<<<<<<<<<< - * """Return view of internal buffer.""" - * return buff_to_buff(self.pk.buf, self.pk.length) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_21getbuffer(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_20getbuffer[] = "Packer.getbuffer(self)\nReturn view of internal buffer."; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_21getbuffer(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("getbuffer (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_20getbuffer(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_20getbuffer(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("getbuffer", 0); - - /* "msgpack/_packer.pyx":362 - * def getbuffer(self): - * """Return view of internal buffer.""" - * return buff_to_buff(self.pk.buf, self.pk.length) # <<<<<<<<<<<<<< - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = buff_to_buff(__pyx_v_self->pk.buf, __pyx_v_self->pk.length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 362, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_packer.pyx":360 - * return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - * - * def getbuffer(self): # <<<<<<<<<<<<<< - * """Return view of internal buffer.""" - * return buff_to_buff(self.pk.buf, self.pk.length) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.getbuffer", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "(tree fragment)":1 - * def __reduce_cython__(self): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_23__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_22__reduce_cython__[] = "Packer.__reduce_cython__(self)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_23__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_22__reduce_cython__(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_22__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__reduce_cython__", 0); - - /* "(tree fragment)":2 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__13, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_Raise(__pyx_t_1, 0, 0, 0); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 2, __pyx_L1_error) - - /* "(tree fragment)":1 - * def __reduce_cython__(self): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "(tree fragment)":3 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_25__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_6Packer_24__setstate_cython__[] = "Packer.__setstate_cython__(self, __pyx_state)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_6Packer_25__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_6Packer_24__setstate_cython__(((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_6Packer_24__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Packer *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__setstate_cython__", 0); - - /* "(tree fragment)":4 - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - */ - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_Raise(__pyx_t_1, 0, 0, 0); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 4, __pyx_L1_error) - - /* "(tree fragment)":3 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Packer.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":57 - * void unpack_clear(unpack_context* ctx) - * - * cdef inline init_ctx(unpack_context *ctx, # <<<<<<<<<<<<<< - * object object_hook, object object_pairs_hook, - * object list_hook, object ext_hook, - */ - -static CYTHON_INLINE PyObject *__pyx_f_7msgpack_9_cmsgpack_init_ctx(unpack_context *__pyx_v_ctx, PyObject *__pyx_v_object_hook, PyObject *__pyx_v_object_pairs_hook, PyObject *__pyx_v_list_hook, PyObject *__pyx_v_ext_hook, int __pyx_v_use_list, int __pyx_v_raw, int __pyx_v_strict_map_key, char const *__pyx_v_encoding, char const *__pyx_v_unicode_errors, Py_ssize_t __pyx_v_max_str_len, Py_ssize_t __pyx_v_max_bin_len, Py_ssize_t __pyx_v_max_array_len, Py_ssize_t __pyx_v_max_map_len, Py_ssize_t __pyx_v_max_ext_len) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("init_ctx", 0); - - /* "msgpack/_unpacker.pyx":65 - * Py_ssize_t max_array_len, Py_ssize_t max_map_len, - * Py_ssize_t max_ext_len): - * unpack_init(ctx) # <<<<<<<<<<<<<< - * ctx.user.use_list = use_list - * ctx.user.raw = raw - */ - unpack_init(__pyx_v_ctx); - - /* "msgpack/_unpacker.pyx":66 - * Py_ssize_t max_ext_len): - * unpack_init(ctx) - * ctx.user.use_list = use_list # <<<<<<<<<<<<<< - * ctx.user.raw = raw - * ctx.user.strict_map_key = strict_map_key - */ - __pyx_v_ctx->user.use_list = __pyx_v_use_list; - - /* "msgpack/_unpacker.pyx":67 - * unpack_init(ctx) - * ctx.user.use_list = use_list - * ctx.user.raw = raw # <<<<<<<<<<<<<< - * ctx.user.strict_map_key = strict_map_key - * ctx.user.object_hook = ctx.user.list_hook = NULL - */ - __pyx_v_ctx->user.raw = __pyx_v_raw; - - /* "msgpack/_unpacker.pyx":68 - * ctx.user.use_list = use_list - * ctx.user.raw = raw - * ctx.user.strict_map_key = strict_map_key # <<<<<<<<<<<<<< - * ctx.user.object_hook = ctx.user.list_hook = NULL - * ctx.user.max_str_len = max_str_len - */ - __pyx_v_ctx->user.strict_map_key = __pyx_v_strict_map_key; - - /* "msgpack/_unpacker.pyx":69 - * ctx.user.raw = raw - * ctx.user.strict_map_key = strict_map_key - * ctx.user.object_hook = ctx.user.list_hook = NULL # <<<<<<<<<<<<<< - * ctx.user.max_str_len = max_str_len - * ctx.user.max_bin_len = max_bin_len - */ - __pyx_v_ctx->user.object_hook = ((PyObject *)NULL); - __pyx_v_ctx->user.list_hook = ((PyObject *)NULL); - - /* "msgpack/_unpacker.pyx":70 - * ctx.user.strict_map_key = strict_map_key - * ctx.user.object_hook = ctx.user.list_hook = NULL - * ctx.user.max_str_len = max_str_len # <<<<<<<<<<<<<< - * ctx.user.max_bin_len = max_bin_len - * ctx.user.max_array_len = max_array_len - */ - __pyx_v_ctx->user.max_str_len = __pyx_v_max_str_len; - - /* "msgpack/_unpacker.pyx":71 - * ctx.user.object_hook = ctx.user.list_hook = NULL - * ctx.user.max_str_len = max_str_len - * ctx.user.max_bin_len = max_bin_len # <<<<<<<<<<<<<< - * ctx.user.max_array_len = max_array_len - * ctx.user.max_map_len = max_map_len - */ - __pyx_v_ctx->user.max_bin_len = __pyx_v_max_bin_len; - - /* "msgpack/_unpacker.pyx":72 - * ctx.user.max_str_len = max_str_len - * ctx.user.max_bin_len = max_bin_len - * ctx.user.max_array_len = max_array_len # <<<<<<<<<<<<<< - * ctx.user.max_map_len = max_map_len - * ctx.user.max_ext_len = max_ext_len - */ - __pyx_v_ctx->user.max_array_len = __pyx_v_max_array_len; - - /* "msgpack/_unpacker.pyx":73 - * ctx.user.max_bin_len = max_bin_len - * ctx.user.max_array_len = max_array_len - * ctx.user.max_map_len = max_map_len # <<<<<<<<<<<<<< - * ctx.user.max_ext_len = max_ext_len - * - */ - __pyx_v_ctx->user.max_map_len = __pyx_v_max_map_len; - - /* "msgpack/_unpacker.pyx":74 - * ctx.user.max_array_len = max_array_len - * ctx.user.max_map_len = max_map_len - * ctx.user.max_ext_len = max_ext_len # <<<<<<<<<<<<<< - * - * if object_hook is not None and object_pairs_hook is not None: - */ - __pyx_v_ctx->user.max_ext_len = __pyx_v_max_ext_len; - - /* "msgpack/_unpacker.pyx":76 - * ctx.user.max_ext_len = max_ext_len - * - * if object_hook is not None and object_pairs_hook is not None: # <<<<<<<<<<<<<< - * raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") - * - */ - __pyx_t_2 = (__pyx_v_object_hook != Py_None); - __pyx_t_3 = (__pyx_t_2 != 0); - if (__pyx_t_3) { - } else { - __pyx_t_1 = __pyx_t_3; - goto __pyx_L4_bool_binop_done; - } - __pyx_t_3 = (__pyx_v_object_pairs_hook != Py_None); - __pyx_t_2 = (__pyx_t_3 != 0); - __pyx_t_1 = __pyx_t_2; - __pyx_L4_bool_binop_done:; - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":77 - * - * if object_hook is not None and object_pairs_hook is not None: - * raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") # <<<<<<<<<<<<<< - * - * if object_hook is not None: - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__15, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 77, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 77, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":76 - * ctx.user.max_ext_len = max_ext_len - * - * if object_hook is not None and object_pairs_hook is not None: # <<<<<<<<<<<<<< - * raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") - * - */ - } - - /* "msgpack/_unpacker.pyx":79 - * raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") - * - * if object_hook is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(object_hook): - * raise TypeError("object_hook must be a callable.") - */ - __pyx_t_1 = (__pyx_v_object_hook != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "msgpack/_unpacker.pyx":80 - * - * if object_hook is not None: - * if not PyCallable_Check(object_hook): # <<<<<<<<<<<<<< - * raise TypeError("object_hook must be a callable.") - * ctx.user.object_hook = object_hook - */ - __pyx_t_2 = ((!(PyCallable_Check(__pyx_v_object_hook) != 0)) != 0); - if (unlikely(__pyx_t_2)) { - - /* "msgpack/_unpacker.pyx":81 - * if object_hook is not None: - * if not PyCallable_Check(object_hook): - * raise TypeError("object_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.object_hook = object_hook - * - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 81, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":80 - * - * if object_hook is not None: - * if not PyCallable_Check(object_hook): # <<<<<<<<<<<<<< - * raise TypeError("object_hook must be a callable.") - * ctx.user.object_hook = object_hook - */ - } - - /* "msgpack/_unpacker.pyx":82 - * if not PyCallable_Check(object_hook): - * raise TypeError("object_hook must be a callable.") - * ctx.user.object_hook = object_hook # <<<<<<<<<<<<<< - * - * if object_pairs_hook is None: - */ - __pyx_v_ctx->user.object_hook = ((PyObject *)__pyx_v_object_hook); - - /* "msgpack/_unpacker.pyx":79 - * raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") - * - * if object_hook is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(object_hook): - * raise TypeError("object_hook must be a callable.") - */ - } - - /* "msgpack/_unpacker.pyx":84 - * ctx.user.object_hook = object_hook - * - * if object_pairs_hook is None: # <<<<<<<<<<<<<< - * ctx.user.has_pairs_hook = False - * else: - */ - __pyx_t_2 = (__pyx_v_object_pairs_hook == Py_None); - __pyx_t_1 = (__pyx_t_2 != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":85 - * - * if object_pairs_hook is None: - * ctx.user.has_pairs_hook = False # <<<<<<<<<<<<<< - * else: - * if not PyCallable_Check(object_pairs_hook): - */ - __pyx_v_ctx->user.has_pairs_hook = 0; - - /* "msgpack/_unpacker.pyx":84 - * ctx.user.object_hook = object_hook - * - * if object_pairs_hook is None: # <<<<<<<<<<<<<< - * ctx.user.has_pairs_hook = False - * else: - */ - goto __pyx_L8; - } - - /* "msgpack/_unpacker.pyx":87 - * ctx.user.has_pairs_hook = False - * else: - * if not PyCallable_Check(object_pairs_hook): # <<<<<<<<<<<<<< - * raise TypeError("object_pairs_hook must be a callable.") - * ctx.user.object_hook = object_pairs_hook - */ - /*else*/ { - __pyx_t_1 = ((!(PyCallable_Check(__pyx_v_object_pairs_hook) != 0)) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":88 - * else: - * if not PyCallable_Check(object_pairs_hook): - * raise TypeError("object_pairs_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.object_hook = object_pairs_hook - * ctx.user.has_pairs_hook = True - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 88, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 88, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":87 - * ctx.user.has_pairs_hook = False - * else: - * if not PyCallable_Check(object_pairs_hook): # <<<<<<<<<<<<<< - * raise TypeError("object_pairs_hook must be a callable.") - * ctx.user.object_hook = object_pairs_hook - */ - } - - /* "msgpack/_unpacker.pyx":89 - * if not PyCallable_Check(object_pairs_hook): - * raise TypeError("object_pairs_hook must be a callable.") - * ctx.user.object_hook = object_pairs_hook # <<<<<<<<<<<<<< - * ctx.user.has_pairs_hook = True - * - */ - __pyx_v_ctx->user.object_hook = ((PyObject *)__pyx_v_object_pairs_hook); - - /* "msgpack/_unpacker.pyx":90 - * raise TypeError("object_pairs_hook must be a callable.") - * ctx.user.object_hook = object_pairs_hook - * ctx.user.has_pairs_hook = True # <<<<<<<<<<<<<< - * - * if list_hook is not None: - */ - __pyx_v_ctx->user.has_pairs_hook = 1; - } - __pyx_L8:; - - /* "msgpack/_unpacker.pyx":92 - * ctx.user.has_pairs_hook = True - * - * if list_hook is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(list_hook): - * raise TypeError("list_hook must be a callable.") - */ - __pyx_t_1 = (__pyx_v_list_hook != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "msgpack/_unpacker.pyx":93 - * - * if list_hook is not None: - * if not PyCallable_Check(list_hook): # <<<<<<<<<<<<<< - * raise TypeError("list_hook must be a callable.") - * ctx.user.list_hook = list_hook - */ - __pyx_t_2 = ((!(PyCallable_Check(__pyx_v_list_hook) != 0)) != 0); - if (unlikely(__pyx_t_2)) { - - /* "msgpack/_unpacker.pyx":94 - * if list_hook is not None: - * if not PyCallable_Check(list_hook): - * raise TypeError("list_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.list_hook = list_hook - * - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 94, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 94, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":93 - * - * if list_hook is not None: - * if not PyCallable_Check(list_hook): # <<<<<<<<<<<<<< - * raise TypeError("list_hook must be a callable.") - * ctx.user.list_hook = list_hook - */ - } - - /* "msgpack/_unpacker.pyx":95 - * if not PyCallable_Check(list_hook): - * raise TypeError("list_hook must be a callable.") - * ctx.user.list_hook = list_hook # <<<<<<<<<<<<<< - * - * if ext_hook is not None: - */ - __pyx_v_ctx->user.list_hook = ((PyObject *)__pyx_v_list_hook); - - /* "msgpack/_unpacker.pyx":92 - * ctx.user.has_pairs_hook = True - * - * if list_hook is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(list_hook): - * raise TypeError("list_hook must be a callable.") - */ - } - - /* "msgpack/_unpacker.pyx":97 - * ctx.user.list_hook = list_hook - * - * if ext_hook is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(ext_hook): - * raise TypeError("ext_hook must be a callable.") - */ - __pyx_t_2 = (__pyx_v_ext_hook != Py_None); - __pyx_t_1 = (__pyx_t_2 != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":98 - * - * if ext_hook is not None: - * if not PyCallable_Check(ext_hook): # <<<<<<<<<<<<<< - * raise TypeError("ext_hook must be a callable.") - * ctx.user.ext_hook = ext_hook - */ - __pyx_t_1 = ((!(PyCallable_Check(__pyx_v_ext_hook) != 0)) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":99 - * if ext_hook is not None: - * if not PyCallable_Check(ext_hook): - * raise TypeError("ext_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.ext_hook = ext_hook - * - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__19, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 99, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":98 - * - * if ext_hook is not None: - * if not PyCallable_Check(ext_hook): # <<<<<<<<<<<<<< - * raise TypeError("ext_hook must be a callable.") - * ctx.user.ext_hook = ext_hook - */ - } - - /* "msgpack/_unpacker.pyx":100 - * if not PyCallable_Check(ext_hook): - * raise TypeError("ext_hook must be a callable.") - * ctx.user.ext_hook = ext_hook # <<<<<<<<<<<<<< - * - * ctx.user.encoding = encoding - */ - __pyx_v_ctx->user.ext_hook = ((PyObject *)__pyx_v_ext_hook); - - /* "msgpack/_unpacker.pyx":97 - * ctx.user.list_hook = list_hook - * - * if ext_hook is not None: # <<<<<<<<<<<<<< - * if not PyCallable_Check(ext_hook): - * raise TypeError("ext_hook must be a callable.") - */ - } - - /* "msgpack/_unpacker.pyx":102 - * ctx.user.ext_hook = ext_hook - * - * ctx.user.encoding = encoding # <<<<<<<<<<<<<< - * ctx.user.unicode_errors = unicode_errors - * - */ - __pyx_v_ctx->user.encoding = __pyx_v_encoding; - - /* "msgpack/_unpacker.pyx":103 - * - * ctx.user.encoding = encoding - * ctx.user.unicode_errors = unicode_errors # <<<<<<<<<<<<<< - * - * def default_read_extended_type(typecode, data): - */ - __pyx_v_ctx->user.unicode_errors = __pyx_v_unicode_errors; - - /* "msgpack/_unpacker.pyx":57 - * void unpack_clear(unpack_context* ctx) - * - * cdef inline init_ctx(unpack_context *ctx, # <<<<<<<<<<<<<< - * object object_hook, object object_pairs_hook, - * object list_hook, object ext_hook, - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack._cmsgpack.init_ctx", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":105 - * ctx.user.unicode_errors = unicode_errors - * - * def default_read_extended_type(typecode, data): # <<<<<<<<<<<<<< - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_1default_read_extended_type(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_default_read_extended_type[] = "default_read_extended_type(typecode, data)"; -static PyMethodDef __pyx_mdef_7msgpack_9_cmsgpack_1default_read_extended_type = {"default_read_extended_type", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_7msgpack_9_cmsgpack_1default_read_extended_type, METH_VARARGS|METH_KEYWORDS, __pyx_doc_7msgpack_9_cmsgpack_default_read_extended_type}; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_1default_read_extended_type(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_typecode = 0; - CYTHON_UNUSED PyObject *__pyx_v_data = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("default_read_extended_type (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_typecode,&__pyx_n_s_data,0}; - PyObject* values[2] = {0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_typecode)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - CYTHON_FALLTHROUGH; - case 1: - if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_data)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("default_read_extended_type", 1, 2, 2, 1); __PYX_ERR(1, 105, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "default_read_extended_type") < 0)) __PYX_ERR(1, 105, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - } - __pyx_v_typecode = values[0]; - __pyx_v_data = values[1]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("default_read_extended_type", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 105, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.default_read_extended_type", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_default_read_extended_type(__pyx_self, __pyx_v_typecode, __pyx_v_data); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_default_read_extended_type(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_typecode, CYTHON_UNUSED PyObject *__pyx_v_data) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("default_read_extended_type", 0); - - /* "msgpack/_unpacker.pyx":106 - * - * def default_read_extended_type(typecode, data): - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) # <<<<<<<<<<<<<< - * - * cdef inline int get_data_from_buffer(object obj, - */ - __pyx_t_1 = __Pyx_PyUnicode_FormatSafe(__pyx_kp_u_Cannot_decode_extended_type_with, __pyx_v_typecode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 106, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_builtin_NotImplementedError, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 106, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":105 - * ctx.user.unicode_errors = unicode_errors - * - * def default_read_extended_type(typecode, data): # <<<<<<<<<<<<<< - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack._cmsgpack.default_read_extended_type", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":108 - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - * - * cdef inline int get_data_from_buffer(object obj, # <<<<<<<<<<<<<< - * Py_buffer *view, - * char **buf, - */ - -static CYTHON_INLINE int __pyx_f_7msgpack_9_cmsgpack_get_data_from_buffer(PyObject *__pyx_v_obj, Py_buffer *__pyx_v_view, char **__pyx_v_buf, Py_ssize_t *__pyx_v_buffer_len, int *__pyx_v_new_protocol) { - PyObject *__pyx_v_contiguous = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - Py_ssize_t __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - char *__pyx_t_6; - __Pyx_RefNannySetupContext("get_data_from_buffer", 0); - - /* "msgpack/_unpacker.pyx":115 - * cdef object contiguous - * cdef Py_buffer tmp - * if PyObject_CheckBuffer(obj): # <<<<<<<<<<<<<< - * new_protocol[0] = 1 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - */ - __pyx_t_1 = (PyObject_CheckBuffer(__pyx_v_obj) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":116 - * cdef Py_buffer tmp - * if PyObject_CheckBuffer(obj): - * new_protocol[0] = 1 # <<<<<<<<<<<<<< - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - * raise - */ - (__pyx_v_new_protocol[0]) = 1; - - /* "msgpack/_unpacker.pyx":117 - * if PyObject_CheckBuffer(obj): - * new_protocol[0] = 1 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: # <<<<<<<<<<<<<< - * raise - * if view.itemsize != 1: - */ - __pyx_t_2 = PyObject_GetBuffer(__pyx_v_obj, __pyx_v_view, PyBUF_FULL_RO); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 117, __pyx_L1_error) - __pyx_t_1 = ((__pyx_t_2 == -1L) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":118 - * new_protocol[0] = 1 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - * raise # <<<<<<<<<<<<<< - * if view.itemsize != 1: - * PyBuffer_Release(view) - */ - __Pyx_ReraiseException(); __PYX_ERR(1, 118, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":117 - * if PyObject_CheckBuffer(obj): - * new_protocol[0] = 1 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: # <<<<<<<<<<<<<< - * raise - * if view.itemsize != 1: - */ - } - - /* "msgpack/_unpacker.pyx":119 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - * raise - * if view.itemsize != 1: # <<<<<<<<<<<<<< - * PyBuffer_Release(view) - * raise BufferError("cannot unpack from multi-byte object") - */ - __pyx_t_1 = ((__pyx_v_view->itemsize != 1) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":120 - * raise - * if view.itemsize != 1: - * PyBuffer_Release(view) # <<<<<<<<<<<<<< - * raise BufferError("cannot unpack from multi-byte object") - * if PyBuffer_IsContiguous(view, b'A') == 0: - */ - PyBuffer_Release(__pyx_v_view); - - /* "msgpack/_unpacker.pyx":121 - * if view.itemsize != 1: - * PyBuffer_Release(view) - * raise BufferError("cannot unpack from multi-byte object") # <<<<<<<<<<<<<< - * if PyBuffer_IsContiguous(view, b'A') == 0: - * PyBuffer_Release(view) - */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_BufferError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 121, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 121, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":119 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - * raise - * if view.itemsize != 1: # <<<<<<<<<<<<<< - * PyBuffer_Release(view) - * raise BufferError("cannot unpack from multi-byte object") - */ - } - - /* "msgpack/_unpacker.pyx":122 - * PyBuffer_Release(view) - * raise BufferError("cannot unpack from multi-byte object") - * if PyBuffer_IsContiguous(view, b'A') == 0: # <<<<<<<<<<<<<< - * PyBuffer_Release(view) - * # create a contiguous copy and get buffer - */ - __pyx_t_1 = ((PyBuffer_IsContiguous(__pyx_v_view, 'A') == 0) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":123 - * raise BufferError("cannot unpack from multi-byte object") - * if PyBuffer_IsContiguous(view, b'A') == 0: - * PyBuffer_Release(view) # <<<<<<<<<<<<<< - * # create a contiguous copy and get buffer - * contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') - */ - PyBuffer_Release(__pyx_v_view); - - /* "msgpack/_unpacker.pyx":125 - * PyBuffer_Release(view) - * # create a contiguous copy and get buffer - * contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') # <<<<<<<<<<<<<< - * PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) - * # view must hold the only reference to contiguous, - */ - __pyx_t_3 = PyMemoryView_GetContiguous(__pyx_v_obj, PyBUF_READ, 'C'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 125, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_v_contiguous = __pyx_t_3; - __pyx_t_3 = 0; - - /* "msgpack/_unpacker.pyx":126 - * # create a contiguous copy and get buffer - * contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') - * PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) # <<<<<<<<<<<<<< - * # view must hold the only reference to contiguous, - * # so memory is freed when view is released - */ - __pyx_t_2 = PyObject_GetBuffer(__pyx_v_contiguous, __pyx_v_view, PyBUF_SIMPLE); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 126, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":129 - * # view must hold the only reference to contiguous, - * # so memory is freed when view is released - * Py_DECREF(contiguous) # <<<<<<<<<<<<<< - * buffer_len[0] = view.len - * buf[0] = view.buf - */ - Py_DECREF(__pyx_v_contiguous); - - /* "msgpack/_unpacker.pyx":122 - * PyBuffer_Release(view) - * raise BufferError("cannot unpack from multi-byte object") - * if PyBuffer_IsContiguous(view, b'A') == 0: # <<<<<<<<<<<<<< - * PyBuffer_Release(view) - * # create a contiguous copy and get buffer - */ - } - - /* "msgpack/_unpacker.pyx":130 - * # so memory is freed when view is released - * Py_DECREF(contiguous) - * buffer_len[0] = view.len # <<<<<<<<<<<<<< - * buf[0] = view.buf - * return 1 - */ - __pyx_t_4 = __pyx_v_view->len; - (__pyx_v_buffer_len[0]) = __pyx_t_4; - - /* "msgpack/_unpacker.pyx":131 - * Py_DECREF(contiguous) - * buffer_len[0] = view.len - * buf[0] = view.buf # <<<<<<<<<<<<<< - * return 1 - * else: - */ - (__pyx_v_buf[0]) = ((char *)__pyx_v_view->buf); - - /* "msgpack/_unpacker.pyx":132 - * buffer_len[0] = view.len - * buf[0] = view.buf - * return 1 # <<<<<<<<<<<<<< - * else: - * new_protocol[0] = 0 - */ - __pyx_r = 1; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":115 - * cdef object contiguous - * cdef Py_buffer tmp - * if PyObject_CheckBuffer(obj): # <<<<<<<<<<<<<< - * new_protocol[0] = 1 - * if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - */ - } - - /* "msgpack/_unpacker.pyx":134 - * return 1 - * else: - * new_protocol[0] = 0 # <<<<<<<<<<<<<< - * if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: - * raise BufferError("could not get memoryview") - */ - /*else*/ { - (__pyx_v_new_protocol[0]) = 0; - - /* "msgpack/_unpacker.pyx":135 - * else: - * new_protocol[0] = 0 - * if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: # <<<<<<<<<<<<<< - * raise BufferError("could not get memoryview") - * PyErr_WarnEx(RuntimeWarning, - */ - __pyx_t_2 = PyObject_AsReadBuffer(__pyx_v_obj, ((void const **)__pyx_v_buf), __pyx_v_buffer_len); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 135, __pyx_L1_error) - __pyx_t_1 = ((__pyx_t_2 == -1L) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":136 - * new_protocol[0] = 0 - * if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: - * raise BufferError("could not get memoryview") # <<<<<<<<<<<<<< - * PyErr_WarnEx(RuntimeWarning, - * "using old buffer interface to unpack %s; " - */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_BufferError, __pyx_tuple__21, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 136, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 136, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":135 - * else: - * new_protocol[0] = 0 - * if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: # <<<<<<<<<<<<<< - * raise BufferError("could not get memoryview") - * PyErr_WarnEx(RuntimeWarning, - */ - } - - /* "msgpack/_unpacker.pyx":140 - * "using old buffer interface to unpack %s; " - * "this leads to unpacking errors if slicing is used and " - * "will be removed in a future version" % type(obj), # <<<<<<<<<<<<<< - * 1) - * return 1 - */ - __pyx_t_3 = __Pyx_PyObject_CallOneArg(((PyObject *)__pyx_ptype_7cpython_4type_type), __pyx_v_obj); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyUnicode_Format(__pyx_kp_u_using_old_buffer_interface_to_un, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 140, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_AsWritableString(__pyx_t_5); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) __PYX_ERR(1, 140, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":137 - * if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: - * raise BufferError("could not get memoryview") - * PyErr_WarnEx(RuntimeWarning, # <<<<<<<<<<<<<< - * "using old buffer interface to unpack %s; " - * "this leads to unpacking errors if slicing is used and " - */ - __pyx_t_2 = PyErr_WarnEx(__pyx_builtin_RuntimeWarning, __pyx_t_6, 1); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 137, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - - /* "msgpack/_unpacker.pyx":142 - * "will be removed in a future version" % type(obj), - * 1) - * return 1 # <<<<<<<<<<<<<< - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, - */ - __pyx_r = 1; - goto __pyx_L0; - } - - /* "msgpack/_unpacker.pyx":108 - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - * - * cdef inline int get_data_from_buffer(object obj, # <<<<<<<<<<<<<< - * Py_buffer *view, - * char **buf, - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("msgpack._cmsgpack.get_data_from_buffer", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_contiguous); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":144 - * return 1 - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_3unpackb(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_2unpackb[] = "unpackb(packed, object_hook=None, list_hook=None, bool use_list=True, bool raw=True, bool strict_map_key=False, encoding=None, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, Py_ssize_t max_array_len=-1, Py_ssize_t max_map_len=-1, Py_ssize_t max_ext_len=-1)\n\n Unpack packed_bytes to object. Returns an unpacked object.\n\n Raises ``ExtraData`` when *packed* contains extra bytes.\n Raises ``ValueError`` when *packed* is incomplete.\n Raises ``FormatError`` when *packed* is not valid msgpack.\n Raises ``StackError`` when *packed* contains too nested.\n Other exceptions can be raised during unpacking.\n\n See :class:`Unpacker` for options.\n\n *max_xxx_len* options are configured automatically from ``len(packed)``.\n "; -static PyMethodDef __pyx_mdef_7msgpack_9_cmsgpack_3unpackb = {"unpackb", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_7msgpack_9_cmsgpack_3unpackb, METH_VARARGS|METH_KEYWORDS, __pyx_doc_7msgpack_9_cmsgpack_2unpackb}; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_3unpackb(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_packed = 0; - PyObject *__pyx_v_object_hook = 0; - PyObject *__pyx_v_list_hook = 0; - int __pyx_v_use_list; - int __pyx_v_raw; - int __pyx_v_strict_map_key; - PyObject *__pyx_v_encoding = 0; - PyObject *__pyx_v_unicode_errors = 0; - PyObject *__pyx_v_object_pairs_hook = 0; - PyObject *__pyx_v_ext_hook = 0; - Py_ssize_t __pyx_v_max_str_len; - Py_ssize_t __pyx_v_max_bin_len; - Py_ssize_t __pyx_v_max_array_len; - Py_ssize_t __pyx_v_max_map_len; - Py_ssize_t __pyx_v_max_ext_len; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("unpackb (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_packed,&__pyx_n_s_object_hook,&__pyx_n_s_list_hook,&__pyx_n_s_use_list,&__pyx_n_s_raw,&__pyx_n_s_strict_map_key,&__pyx_n_s_encoding,&__pyx_n_s_unicode_errors,&__pyx_n_s_object_pairs_hook,&__pyx_n_s_ext_hook,&__pyx_n_s_max_str_len,&__pyx_n_s_max_bin_len,&__pyx_n_s_max_array_len,&__pyx_n_s_max_map_len,&__pyx_n_s_max_ext_len,0}; - PyObject* values[15] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; - values[1] = ((PyObject *)Py_None); - values[2] = ((PyObject *)Py_None); - - /* "msgpack/_unpacker.pyx":146 - * def unpackb(object packed, object object_hook=None, object list_hook=None, - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, # <<<<<<<<<<<<<< - * object_pairs_hook=None, ext_hook=ExtType, - * Py_ssize_t max_str_len=-1, - */ - values[6] = ((PyObject *)Py_None); - values[7] = ((PyObject *)Py_None); - - /* "msgpack/_unpacker.pyx":147 - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - * object_pairs_hook=None, ext_hook=ExtType, # <<<<<<<<<<<<<< - * Py_ssize_t max_str_len=-1, - * Py_ssize_t max_bin_len=-1, - */ - values[8] = ((PyObject *)Py_None); - values[9] = __pyx_k__22; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 15: values[14] = PyTuple_GET_ITEM(__pyx_args, 14); - CYTHON_FALLTHROUGH; - case 14: values[13] = PyTuple_GET_ITEM(__pyx_args, 13); - CYTHON_FALLTHROUGH; - case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); - CYTHON_FALLTHROUGH; - case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); - CYTHON_FALLTHROUGH; - case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); - CYTHON_FALLTHROUGH; - case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); - CYTHON_FALLTHROUGH; - case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); - CYTHON_FALLTHROUGH; - case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); - CYTHON_FALLTHROUGH; - case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - CYTHON_FALLTHROUGH; - case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - CYTHON_FALLTHROUGH; - case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - CYTHON_FALLTHROUGH; - case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - CYTHON_FALLTHROUGH; - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_packed)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - CYTHON_FALLTHROUGH; - case 1: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_object_hook); - if (value) { values[1] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 2: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_list_hook); - if (value) { values[2] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 3: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_use_list); - if (value) { values[3] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 4: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_raw); - if (value) { values[4] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 5: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_strict_map_key); - if (value) { values[5] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 6: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_encoding); - if (value) { values[6] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 7: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_unicode_errors); - if (value) { values[7] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 8: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_object_pairs_hook); - if (value) { values[8] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 9: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_ext_hook); - if (value) { values[9] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 10: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_str_len); - if (value) { values[10] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 11: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_bin_len); - if (value) { values[11] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 12: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_array_len); - if (value) { values[12] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 13: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_map_len); - if (value) { values[13] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 14: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_ext_len); - if (value) { values[14] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "unpackb") < 0)) __PYX_ERR(1, 144, __pyx_L3_error) - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 15: values[14] = PyTuple_GET_ITEM(__pyx_args, 14); - CYTHON_FALLTHROUGH; - case 14: values[13] = PyTuple_GET_ITEM(__pyx_args, 13); - CYTHON_FALLTHROUGH; - case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); - CYTHON_FALLTHROUGH; - case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); - CYTHON_FALLTHROUGH; - case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); - CYTHON_FALLTHROUGH; - case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); - CYTHON_FALLTHROUGH; - case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); - CYTHON_FALLTHROUGH; - case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); - CYTHON_FALLTHROUGH; - case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - CYTHON_FALLTHROUGH; - case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - CYTHON_FALLTHROUGH; - case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - CYTHON_FALLTHROUGH; - case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - CYTHON_FALLTHROUGH; - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_packed = values[0]; - __pyx_v_object_hook = values[1]; - __pyx_v_list_hook = values[2]; - if (values[3]) { - __pyx_v_use_list = __Pyx_PyObject_IsTrue(values[3]); if (unlikely((__pyx_v_use_list == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L3_error) - } else { - - /* "msgpack/_unpacker.pyx":145 - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, - * bint use_list=True, bint raw=True, bint strict_map_key=False, # <<<<<<<<<<<<<< - * encoding=None, unicode_errors=None, - * object_pairs_hook=None, ext_hook=ExtType, - */ - __pyx_v_use_list = ((int)1); - } - if (values[4]) { - __pyx_v_raw = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_raw == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L3_error) - } else { - __pyx_v_raw = ((int)1); - } - if (values[5]) { - __pyx_v_strict_map_key = __Pyx_PyObject_IsTrue(values[5]); if (unlikely((__pyx_v_strict_map_key == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L3_error) - } else { - __pyx_v_strict_map_key = ((int)0); - } - __pyx_v_encoding = values[6]; - __pyx_v_unicode_errors = values[7]; - __pyx_v_object_pairs_hook = values[8]; - __pyx_v_ext_hook = values[9]; - if (values[10]) { - __pyx_v_max_str_len = __Pyx_PyIndex_AsSsize_t(values[10]); if (unlikely((__pyx_v_max_str_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L3_error) - } else { - __pyx_v_max_str_len = ((Py_ssize_t)-1L); - } - if (values[11]) { - __pyx_v_max_bin_len = __Pyx_PyIndex_AsSsize_t(values[11]); if (unlikely((__pyx_v_max_bin_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 149, __pyx_L3_error) - } else { - __pyx_v_max_bin_len = ((Py_ssize_t)-1L); - } - if (values[12]) { - __pyx_v_max_array_len = __Pyx_PyIndex_AsSsize_t(values[12]); if (unlikely((__pyx_v_max_array_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 150, __pyx_L3_error) - } else { - __pyx_v_max_array_len = ((Py_ssize_t)-1L); - } - if (values[13]) { - __pyx_v_max_map_len = __Pyx_PyIndex_AsSsize_t(values[13]); if (unlikely((__pyx_v_max_map_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 151, __pyx_L3_error) - } else { - __pyx_v_max_map_len = ((Py_ssize_t)-1L); - } - if (values[14]) { - __pyx_v_max_ext_len = __Pyx_PyIndex_AsSsize_t(values[14]); if (unlikely((__pyx_v_max_ext_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 152, __pyx_L3_error) - } else { - __pyx_v_max_ext_len = ((Py_ssize_t)-1L); - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("unpackb", 0, 1, 15, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 144, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.unpackb", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_2unpackb(__pyx_self, __pyx_v_packed, __pyx_v_object_hook, __pyx_v_list_hook, __pyx_v_use_list, __pyx_v_raw, __pyx_v_strict_map_key, __pyx_v_encoding, __pyx_v_unicode_errors, __pyx_v_object_pairs_hook, __pyx_v_ext_hook, __pyx_v_max_str_len, __pyx_v_max_bin_len, __pyx_v_max_array_len, __pyx_v_max_map_len, __pyx_v_max_ext_len); - - /* "msgpack/_unpacker.pyx":144 - * return 1 - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_2unpackb(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_packed, PyObject *__pyx_v_object_hook, PyObject *__pyx_v_list_hook, int __pyx_v_use_list, int __pyx_v_raw, int __pyx_v_strict_map_key, PyObject *__pyx_v_encoding, PyObject *__pyx_v_unicode_errors, PyObject *__pyx_v_object_pairs_hook, PyObject *__pyx_v_ext_hook, Py_ssize_t __pyx_v_max_str_len, Py_ssize_t __pyx_v_max_bin_len, Py_ssize_t __pyx_v_max_array_len, Py_ssize_t __pyx_v_max_map_len, Py_ssize_t __pyx_v_max_ext_len) { - unpack_context __pyx_v_ctx; - Py_ssize_t __pyx_v_off; - int __pyx_v_ret; - Py_buffer __pyx_v_view; - char *__pyx_v_buf; - Py_ssize_t __pyx_v_buf_len; - char const *__pyx_v_cenc; - char const *__pyx_v_cerr; - int __pyx_v_new_protocol; - PyObject *__pyx_v_obj = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - char const *__pyx_t_4; - char const *__pyx_t_5; - PyObject *__pyx_t_6 = NULL; - int __pyx_t_7; - char const *__pyx_t_8; - PyObject *__pyx_t_9 = NULL; - PyObject *__pyx_t_10 = NULL; - PyObject *__pyx_t_11 = NULL; - PyObject *__pyx_t_12 = NULL; - PyObject *__pyx_t_13 = NULL; - PyObject *__pyx_t_14 = NULL; - PyObject *__pyx_t_15 = NULL; - PyObject *__pyx_t_16 = NULL; - PyObject *__pyx_t_17 = NULL; - PyObject *__pyx_t_18 = NULL; - __Pyx_RefNannySetupContext("unpackb", 0); - - /* "msgpack/_unpacker.pyx":167 - * """ - * cdef unpack_context ctx - * cdef Py_ssize_t off = 0 # <<<<<<<<<<<<<< - * cdef int ret - * - */ - __pyx_v_off = 0; - - /* "msgpack/_unpacker.pyx":171 - * - * cdef Py_buffer view - * cdef char* buf = NULL # <<<<<<<<<<<<<< - * cdef Py_ssize_t buf_len - * cdef const char* cenc = NULL - */ - __pyx_v_buf = NULL; - - /* "msgpack/_unpacker.pyx":173 - * cdef char* buf = NULL - * cdef Py_ssize_t buf_len - * cdef const char* cenc = NULL # <<<<<<<<<<<<<< - * cdef const char* cerr = NULL - * cdef int new_protocol = 0 - */ - __pyx_v_cenc = NULL; - - /* "msgpack/_unpacker.pyx":174 - * cdef Py_ssize_t buf_len - * cdef const char* cenc = NULL - * cdef const char* cerr = NULL # <<<<<<<<<<<<<< - * cdef int new_protocol = 0 - * - */ - __pyx_v_cerr = NULL; - - /* "msgpack/_unpacker.pyx":175 - * cdef const char* cenc = NULL - * cdef const char* cerr = NULL - * cdef int new_protocol = 0 # <<<<<<<<<<<<<< - * - * if encoding is not None: - */ - __pyx_v_new_protocol = 0; - - /* "msgpack/_unpacker.pyx":177 - * cdef int new_protocol = 0 - * - * if encoding is not None: # <<<<<<<<<<<<<< - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * cenc = encoding - */ - __pyx_t_1 = (__pyx_v_encoding != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "msgpack/_unpacker.pyx":178 - * - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) # <<<<<<<<<<<<<< - * cenc = encoding - * - */ - __pyx_t_3 = PyErr_WarnEx(__pyx_builtin_DeprecationWarning, ((char *)"encoding is deprecated, Use raw=False instead."), 1); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 178, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":179 - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * cenc = encoding # <<<<<<<<<<<<<< - * - * if unicode_errors is not None: - */ - __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_v_encoding); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) __PYX_ERR(1, 179, __pyx_L1_error) - __pyx_v_cenc = __pyx_t_4; - - /* "msgpack/_unpacker.pyx":177 - * cdef int new_protocol = 0 - * - * if encoding is not None: # <<<<<<<<<<<<<< - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * cenc = encoding - */ - } - - /* "msgpack/_unpacker.pyx":181 - * cenc = encoding - * - * if unicode_errors is not None: # <<<<<<<<<<<<<< - * cerr = unicode_errors - * - */ - __pyx_t_2 = (__pyx_v_unicode_errors != Py_None); - __pyx_t_1 = (__pyx_t_2 != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":182 - * - * if unicode_errors is not None: - * cerr = unicode_errors # <<<<<<<<<<<<<< - * - * get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) - */ - __pyx_t_5 = __Pyx_PyObject_AsString(__pyx_v_unicode_errors); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) __PYX_ERR(1, 182, __pyx_L1_error) - __pyx_v_cerr = __pyx_t_5; - - /* "msgpack/_unpacker.pyx":181 - * cenc = encoding - * - * if unicode_errors is not None: # <<<<<<<<<<<<<< - * cerr = unicode_errors - * - */ - } - - /* "msgpack/_unpacker.pyx":184 - * cerr = unicode_errors - * - * get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) # <<<<<<<<<<<<<< - * - * if max_str_len == -1: - */ - __pyx_t_3 = __pyx_f_7msgpack_9_cmsgpack_get_data_from_buffer(__pyx_v_packed, (&__pyx_v_view), (&__pyx_v_buf), (&__pyx_v_buf_len), (&__pyx_v_new_protocol)); if (unlikely(__pyx_t_3 == ((int)0))) __PYX_ERR(1, 184, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":186 - * get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) - * - * if max_str_len == -1: # <<<<<<<<<<<<<< - * max_str_len = buf_len - * if max_bin_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_str_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":187 - * - * if max_str_len == -1: - * max_str_len = buf_len # <<<<<<<<<<<<<< - * if max_bin_len == -1: - * max_bin_len = buf_len - */ - __pyx_v_max_str_len = __pyx_v_buf_len; - - /* "msgpack/_unpacker.pyx":186 - * get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) - * - * if max_str_len == -1: # <<<<<<<<<<<<<< - * max_str_len = buf_len - * if max_bin_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":188 - * if max_str_len == -1: - * max_str_len = buf_len - * if max_bin_len == -1: # <<<<<<<<<<<<<< - * max_bin_len = buf_len - * if max_array_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_bin_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":189 - * max_str_len = buf_len - * if max_bin_len == -1: - * max_bin_len = buf_len # <<<<<<<<<<<<<< - * if max_array_len == -1: - * max_array_len = buf_len - */ - __pyx_v_max_bin_len = __pyx_v_buf_len; - - /* "msgpack/_unpacker.pyx":188 - * if max_str_len == -1: - * max_str_len = buf_len - * if max_bin_len == -1: # <<<<<<<<<<<<<< - * max_bin_len = buf_len - * if max_array_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":190 - * if max_bin_len == -1: - * max_bin_len = buf_len - * if max_array_len == -1: # <<<<<<<<<<<<<< - * max_array_len = buf_len - * if max_map_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_array_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":191 - * max_bin_len = buf_len - * if max_array_len == -1: - * max_array_len = buf_len # <<<<<<<<<<<<<< - * if max_map_len == -1: - * max_map_len = buf_len//2 - */ - __pyx_v_max_array_len = __pyx_v_buf_len; - - /* "msgpack/_unpacker.pyx":190 - * if max_bin_len == -1: - * max_bin_len = buf_len - * if max_array_len == -1: # <<<<<<<<<<<<<< - * max_array_len = buf_len - * if max_map_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":192 - * if max_array_len == -1: - * max_array_len = buf_len - * if max_map_len == -1: # <<<<<<<<<<<<<< - * max_map_len = buf_len//2 - * if max_ext_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_map_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":193 - * max_array_len = buf_len - * if max_map_len == -1: - * max_map_len = buf_len//2 # <<<<<<<<<<<<<< - * if max_ext_len == -1: - * max_ext_len = buf_len - */ - __pyx_v_max_map_len = __Pyx_div_Py_ssize_t(__pyx_v_buf_len, 2); - - /* "msgpack/_unpacker.pyx":192 - * if max_array_len == -1: - * max_array_len = buf_len - * if max_map_len == -1: # <<<<<<<<<<<<<< - * max_map_len = buf_len//2 - * if max_ext_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":194 - * if max_map_len == -1: - * max_map_len = buf_len//2 - * if max_ext_len == -1: # <<<<<<<<<<<<<< - * max_ext_len = buf_len - * - */ - __pyx_t_1 = ((__pyx_v_max_ext_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":195 - * max_map_len = buf_len//2 - * if max_ext_len == -1: - * max_ext_len = buf_len # <<<<<<<<<<<<<< - * - * try: - */ - __pyx_v_max_ext_len = __pyx_v_buf_len; - - /* "msgpack/_unpacker.pyx":194 - * if max_map_len == -1: - * max_map_len = buf_len//2 - * if max_ext_len == -1: # <<<<<<<<<<<<<< - * max_ext_len = buf_len - * - */ - } - - /* "msgpack/_unpacker.pyx":197 - * max_ext_len = buf_len - * - * try: # <<<<<<<<<<<<<< - * init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - * use_list, raw, strict_map_key, cenc, cerr, - */ - /*try:*/ { - - /* "msgpack/_unpacker.pyx":198 - * - * try: - * init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, # <<<<<<<<<<<<<< - * use_list, raw, strict_map_key, cenc, cerr, - * max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) - */ - __pyx_t_6 = __pyx_f_7msgpack_9_cmsgpack_init_ctx((&__pyx_v_ctx), __pyx_v_object_hook, __pyx_v_object_pairs_hook, __pyx_v_list_hook, __pyx_v_ext_hook, __pyx_v_use_list, __pyx_v_raw, __pyx_v_strict_map_key, __pyx_v_cenc, __pyx_v_cerr, __pyx_v_max_str_len, __pyx_v_max_bin_len, __pyx_v_max_array_len, __pyx_v_max_map_len, __pyx_v_max_ext_len); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 198, __pyx_L11_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - - /* "msgpack/_unpacker.pyx":201 - * use_list, raw, strict_map_key, cenc, cerr, - * max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) - * ret = unpack_construct(&ctx, buf, buf_len, &off) # <<<<<<<<<<<<<< - * finally: - * if new_protocol: - */ - __pyx_t_3 = unpack_construct((&__pyx_v_ctx), __pyx_v_buf, __pyx_v_buf_len, (&__pyx_v_off)); if (unlikely(__pyx_t_3 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 201, __pyx_L11_error) - __pyx_v_ret = __pyx_t_3; - } - - /* "msgpack/_unpacker.pyx":203 - * ret = unpack_construct(&ctx, buf, buf_len, &off) - * finally: - * if new_protocol: # <<<<<<<<<<<<<< - * PyBuffer_Release(&view); - * - */ - /*finally:*/ { - /*normal exit:*/{ - __pyx_t_1 = (__pyx_v_new_protocol != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":204 - * finally: - * if new_protocol: - * PyBuffer_Release(&view); # <<<<<<<<<<<<<< - * - * if ret == 1: - */ - PyBuffer_Release((&__pyx_v_view)); - - /* "msgpack/_unpacker.pyx":203 - * ret = unpack_construct(&ctx, buf, buf_len, &off) - * finally: - * if new_protocol: # <<<<<<<<<<<<<< - * PyBuffer_Release(&view); - * - */ - } - goto __pyx_L12; - } - __pyx_L11_error:; - /*exception exit:*/{ - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_14 = 0; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - if (PY_MAJOR_VERSION >= 3) __Pyx_ExceptionSwap(&__pyx_t_12, &__pyx_t_13, &__pyx_t_14); - if ((PY_MAJOR_VERSION < 3) || unlikely(__Pyx_GetException(&__pyx_t_9, &__pyx_t_10, &__pyx_t_11) < 0)) __Pyx_ErrFetch(&__pyx_t_9, &__pyx_t_10, &__pyx_t_11); - __Pyx_XGOTREF(__pyx_t_9); - __Pyx_XGOTREF(__pyx_t_10); - __Pyx_XGOTREF(__pyx_t_11); - __Pyx_XGOTREF(__pyx_t_12); - __Pyx_XGOTREF(__pyx_t_13); - __Pyx_XGOTREF(__pyx_t_14); - __pyx_t_3 = __pyx_lineno; __pyx_t_7 = __pyx_clineno; __pyx_t_8 = __pyx_filename; - { - __pyx_t_1 = (__pyx_v_new_protocol != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":204 - * finally: - * if new_protocol: - * PyBuffer_Release(&view); # <<<<<<<<<<<<<< - * - * if ret == 1: - */ - PyBuffer_Release((&__pyx_v_view)); - - /* "msgpack/_unpacker.pyx":203 - * ret = unpack_construct(&ctx, buf, buf_len, &off) - * finally: - * if new_protocol: # <<<<<<<<<<<<<< - * PyBuffer_Release(&view); - * - */ - } - } - if (PY_MAJOR_VERSION >= 3) { - __Pyx_XGIVEREF(__pyx_t_12); - __Pyx_XGIVEREF(__pyx_t_13); - __Pyx_XGIVEREF(__pyx_t_14); - __Pyx_ExceptionReset(__pyx_t_12, __pyx_t_13, __pyx_t_14); - } - __Pyx_XGIVEREF(__pyx_t_9); - __Pyx_XGIVEREF(__pyx_t_10); - __Pyx_XGIVEREF(__pyx_t_11); - __Pyx_ErrRestore(__pyx_t_9, __pyx_t_10, __pyx_t_11); - __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_14 = 0; - __pyx_lineno = __pyx_t_3; __pyx_clineno = __pyx_t_7; __pyx_filename = __pyx_t_8; - goto __pyx_L1_error; - } - __pyx_L12:; - } - - /* "msgpack/_unpacker.pyx":206 - * PyBuffer_Release(&view); - * - * if ret == 1: # <<<<<<<<<<<<<< - * obj = unpack_data(&ctx) - * if off < buf_len: - */ - __pyx_t_1 = ((__pyx_v_ret == 1) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":207 - * - * if ret == 1: - * obj = unpack_data(&ctx) # <<<<<<<<<<<<<< - * if off < buf_len: - * raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - */ - __pyx_t_6 = unpack_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 207, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_v_obj = __pyx_t_6; - __pyx_t_6 = 0; - - /* "msgpack/_unpacker.pyx":208 - * if ret == 1: - * obj = unpack_data(&ctx) - * if off < buf_len: # <<<<<<<<<<<<<< - * raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - * return obj - */ - __pyx_t_1 = ((__pyx_v_off < __pyx_v_buf_len) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":209 - * obj = unpack_data(&ctx) - * if off < buf_len: - * raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) # <<<<<<<<<<<<<< - * return obj - * unpack_clear(&ctx) - */ - __Pyx_GetModuleGlobalName(__pyx_t_15, __pyx_n_s_ExtraData); if (unlikely(!__pyx_t_15)) __PYX_ERR(1, 209, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_15); - __pyx_t_16 = PyBytes_FromStringAndSize((__pyx_v_buf + __pyx_v_off), (__pyx_v_buf_len - __pyx_v_off)); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 209, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_16); - __pyx_t_17 = NULL; - __pyx_t_7 = 0; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_15))) { - __pyx_t_17 = PyMethod_GET_SELF(__pyx_t_15); - if (likely(__pyx_t_17)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_15); - __Pyx_INCREF(__pyx_t_17); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_15, function); - __pyx_t_7 = 1; - } - } - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_15)) { - PyObject *__pyx_temp[3] = {__pyx_t_17, __pyx_v_obj, __pyx_t_16}; - __pyx_t_6 = __Pyx_PyFunction_FastCall(__pyx_t_15, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 209, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0; - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_15)) { - PyObject *__pyx_temp[3] = {__pyx_t_17, __pyx_v_obj, __pyx_t_16}; - __pyx_t_6 = __Pyx_PyCFunction_FastCall(__pyx_t_15, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 209, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0; - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - } else - #endif - { - __pyx_t_18 = PyTuple_New(2+__pyx_t_7); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 209, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_18); - if (__pyx_t_17) { - __Pyx_GIVEREF(__pyx_t_17); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_17); __pyx_t_17 = NULL; - } - __Pyx_INCREF(__pyx_v_obj); - __Pyx_GIVEREF(__pyx_v_obj); - PyTuple_SET_ITEM(__pyx_t_18, 0+__pyx_t_7, __pyx_v_obj); - __Pyx_GIVEREF(__pyx_t_16); - PyTuple_SET_ITEM(__pyx_t_18, 1+__pyx_t_7, __pyx_t_16); - __pyx_t_16 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_15, __pyx_t_18, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 209, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; - } - __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; - __Pyx_Raise(__pyx_t_6, 0, 0, 0); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __PYX_ERR(1, 209, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":208 - * if ret == 1: - * obj = unpack_data(&ctx) - * if off < buf_len: # <<<<<<<<<<<<<< - * raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - * return obj - */ - } - - /* "msgpack/_unpacker.pyx":210 - * if off < buf_len: - * raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - * return obj # <<<<<<<<<<<<<< - * unpack_clear(&ctx) - * if ret == 0: - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_obj); - __pyx_r = __pyx_v_obj; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":206 - * PyBuffer_Release(&view); - * - * if ret == 1: # <<<<<<<<<<<<<< - * obj = unpack_data(&ctx) - * if off < buf_len: - */ - } - - /* "msgpack/_unpacker.pyx":211 - * raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - * return obj - * unpack_clear(&ctx) # <<<<<<<<<<<<<< - * if ret == 0: - * raise ValueError("Unpack failed: incomplete input") - */ - unpack_clear((&__pyx_v_ctx)); - - /* "msgpack/_unpacker.pyx":212 - * return obj - * unpack_clear(&ctx) - * if ret == 0: # <<<<<<<<<<<<<< - * raise ValueError("Unpack failed: incomplete input") - * elif ret == -2: - */ - switch (__pyx_v_ret) { - case 0: - - /* "msgpack/_unpacker.pyx":213 - * unpack_clear(&ctx) - * if ret == 0: - * raise ValueError("Unpack failed: incomplete input") # <<<<<<<<<<<<<< - * elif ret == -2: - * raise FormatError - */ - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__23, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 213, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_Raise(__pyx_t_6, 0, 0, 0); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __PYX_ERR(1, 213, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":212 - * return obj - * unpack_clear(&ctx) - * if ret == 0: # <<<<<<<<<<<<<< - * raise ValueError("Unpack failed: incomplete input") - * elif ret == -2: - */ - break; - case -2L: - - /* "msgpack/_unpacker.pyx":215 - * raise ValueError("Unpack failed: incomplete input") - * elif ret == -2: - * raise FormatError # <<<<<<<<<<<<<< - * elif ret == -3: - * raise StackError - */ - __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_FormatError); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 215, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_Raise(__pyx_t_6, 0, 0, 0); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __PYX_ERR(1, 215, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":214 - * if ret == 0: - * raise ValueError("Unpack failed: incomplete input") - * elif ret == -2: # <<<<<<<<<<<<<< - * raise FormatError - * elif ret == -3: - */ - break; - case -3L: - - /* "msgpack/_unpacker.pyx":217 - * raise FormatError - * elif ret == -3: - * raise StackError # <<<<<<<<<<<<<< - * raise ValueError("Unpack failed: error = %d" % (ret,)) - * - */ - __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_StackError); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 217, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_Raise(__pyx_t_6, 0, 0, 0); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __PYX_ERR(1, 217, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":216 - * elif ret == -2: - * raise FormatError - * elif ret == -3: # <<<<<<<<<<<<<< - * raise StackError - * raise ValueError("Unpack failed: error = %d" % (ret,)) - */ - break; - default: break; - } - - /* "msgpack/_unpacker.pyx":218 - * elif ret == -3: - * raise StackError - * raise ValueError("Unpack failed: error = %d" % (ret,)) # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_6 = __Pyx_PyUnicode_From_int(__pyx_v_ret, 0, ' ', 'd'); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 218, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_15 = __Pyx_PyUnicode_Concat(__pyx_kp_u_Unpack_failed_error, __pyx_t_6); if (unlikely(!__pyx_t_15)) __PYX_ERR(1, 218, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_15); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_15); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 218, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; - __Pyx_Raise(__pyx_t_6, 0, 0, 0); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __PYX_ERR(1, 218, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":144 - * return 1 - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_15); - __Pyx_XDECREF(__pyx_t_16); - __Pyx_XDECREF(__pyx_t_17); - __Pyx_XDECREF(__pyx_t_18); - __Pyx_AddTraceback("msgpack._cmsgpack.unpackb", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_obj); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":221 - * - * - * def unpack(object stream, **kwargs): # <<<<<<<<<<<<<< - * PyErr_WarnEx( - * DeprecationWarning, - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_5unpack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_4unpack[] = "unpack(stream, **kwargs)"; -static PyMethodDef __pyx_mdef_7msgpack_9_cmsgpack_5unpack = {"unpack", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_7msgpack_9_cmsgpack_5unpack, METH_VARARGS|METH_KEYWORDS, __pyx_doc_7msgpack_9_cmsgpack_4unpack}; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_5unpack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_stream = 0; - PyObject *__pyx_v_kwargs = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("unpack (wrapper)", 0); - __pyx_v_kwargs = PyDict_New(); if (unlikely(!__pyx_v_kwargs)) return NULL; - __Pyx_GOTREF(__pyx_v_kwargs); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_stream,0}; - PyObject* values[1] = {0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_stream)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, pos_args, "unpack") < 0)) __PYX_ERR(1, 221, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - } - __pyx_v_stream = values[0]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("unpack", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 221, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; - __Pyx_AddTraceback("msgpack._cmsgpack.unpack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_4unpack(__pyx_self, __pyx_v_stream, __pyx_v_kwargs); - - /* function exit code */ - __Pyx_XDECREF(__pyx_v_kwargs); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_4unpack(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_stream, PyObject *__pyx_v_kwargs) { - PyObject *__pyx_v_data = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("unpack", 0); - - /* "msgpack/_unpacker.pyx":222 - * - * def unpack(object stream, **kwargs): - * PyErr_WarnEx( # <<<<<<<<<<<<<< - * DeprecationWarning, - * "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) - */ - __pyx_t_1 = PyErr_WarnEx(__pyx_builtin_DeprecationWarning, ((char *)"Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead."), 1); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(1, 222, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":225 - * DeprecationWarning, - * "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) - * data = stream.read() # <<<<<<<<<<<<<< - * return unpackb(data, **kwargs) - * - */ - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_stream, __pyx_n_s_read); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 225, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - __pyx_t_2 = (__pyx_t_4) ? __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4) : __Pyx_PyObject_CallNoArg(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 225, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_v_data = __pyx_t_2; - __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":226 - * "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) - * data = stream.read() - * return unpackb(data, **kwargs) # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_n_s_unpackb); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 226, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 226, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_INCREF(__pyx_v_data); - __Pyx_GIVEREF(__pyx_v_data); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_data); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, __pyx_v_kwargs); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 226, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":221 - * - * - * def unpack(object stream, **kwargs): # <<<<<<<<<<<<<< - * PyErr_WarnEx( - * DeprecationWarning, - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack._cmsgpack.unpack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_data); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":337 - * cdef uint64_t stream_offset - * - * def __cinit__(self): # <<<<<<<<<<<<<< - * self.buf = NULL - * - */ - -/* Python wrapper */ -static int __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} - if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__cinit__", 0))) return -1; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker___cinit__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_7msgpack_9_cmsgpack_8Unpacker___cinit__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "msgpack/_unpacker.pyx":338 - * - * def __cinit__(self): - * self.buf = NULL # <<<<<<<<<<<<<< - * - * def __dealloc__(self): - */ - __pyx_v_self->buf = NULL; - - /* "msgpack/_unpacker.pyx":337 - * cdef uint64_t stream_offset - * - * def __cinit__(self): # <<<<<<<<<<<<<< - * self.buf = NULL - * - */ - - /* function exit code */ - __pyx_r = 0; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":340 - * self.buf = NULL - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * PyMem_Free(self.buf) - * self.buf = NULL - */ - -/* Python wrapper */ -static void __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ -static void __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_3__dealloc__(PyObject *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); - __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_2__dealloc__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -static void __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_2__dealloc__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__", 0); - - /* "msgpack/_unpacker.pyx":341 - * - * def __dealloc__(self): - * PyMem_Free(self.buf) # <<<<<<<<<<<<<< - * self.buf = NULL - * - */ - PyMem_Free(__pyx_v_self->buf); - - /* "msgpack/_unpacker.pyx":342 - * def __dealloc__(self): - * PyMem_Free(self.buf) - * self.buf = NULL # <<<<<<<<<<<<<< - * - * def __init__(self, file_like=None, Py_ssize_t read_size=0, - */ - __pyx_v_self->buf = NULL; - - /* "msgpack/_unpacker.pyx":340 - * self.buf = NULL - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * PyMem_Free(self.buf) - * self.buf = NULL - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -/* "msgpack/_unpacker.pyx":344 - * self.buf = NULL - * - * def __init__(self, file_like=None, Py_ssize_t read_size=0, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, - */ - -/* Python wrapper */ -static int __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_5__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_5__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_file_like = 0; - Py_ssize_t __pyx_v_read_size; - int __pyx_v_use_list; - int __pyx_v_raw; - int __pyx_v_strict_map_key; - PyObject *__pyx_v_object_hook = 0; - PyObject *__pyx_v_object_pairs_hook = 0; - PyObject *__pyx_v_list_hook = 0; - PyObject *__pyx_v_encoding = 0; - PyObject *__pyx_v_unicode_errors = 0; - Py_ssize_t __pyx_v_max_buffer_size; - PyObject *__pyx_v_ext_hook = 0; - Py_ssize_t __pyx_v_max_str_len; - Py_ssize_t __pyx_v_max_bin_len; - Py_ssize_t __pyx_v_max_array_len; - Py_ssize_t __pyx_v_max_map_len; - Py_ssize_t __pyx_v_max_ext_len; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_file_like,&__pyx_n_s_read_size,&__pyx_n_s_use_list,&__pyx_n_s_raw,&__pyx_n_s_strict_map_key,&__pyx_n_s_object_hook,&__pyx_n_s_object_pairs_hook,&__pyx_n_s_list_hook,&__pyx_n_s_encoding,&__pyx_n_s_unicode_errors,&__pyx_n_s_max_buffer_size,&__pyx_n_s_ext_hook,&__pyx_n_s_max_str_len,&__pyx_n_s_max_bin_len,&__pyx_n_s_max_array_len,&__pyx_n_s_max_map_len,&__pyx_n_s_max_ext_len,0}; - PyObject* values[17] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; - values[0] = ((PyObject *)Py_None); - - /* "msgpack/_unpacker.pyx":346 - * def __init__(self, file_like=None, Py_ssize_t read_size=0, - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, # <<<<<<<<<<<<<< - * encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, - * object ext_hook=ExtType, - */ - values[5] = ((PyObject *)Py_None); - values[6] = ((PyObject *)Py_None); - values[7] = ((PyObject *)Py_None); - - /* "msgpack/_unpacker.pyx":347 - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, - * encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, # <<<<<<<<<<<<<< - * object ext_hook=ExtType, - * Py_ssize_t max_str_len=-1, - */ - values[8] = ((PyObject *)Py_None); - values[9] = ((PyObject *)Py_None); - values[11] = __pyx_k__24; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 17: values[16] = PyTuple_GET_ITEM(__pyx_args, 16); - CYTHON_FALLTHROUGH; - case 16: values[15] = PyTuple_GET_ITEM(__pyx_args, 15); - CYTHON_FALLTHROUGH; - case 15: values[14] = PyTuple_GET_ITEM(__pyx_args, 14); - CYTHON_FALLTHROUGH; - case 14: values[13] = PyTuple_GET_ITEM(__pyx_args, 13); - CYTHON_FALLTHROUGH; - case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); - CYTHON_FALLTHROUGH; - case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); - CYTHON_FALLTHROUGH; - case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); - CYTHON_FALLTHROUGH; - case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); - CYTHON_FALLTHROUGH; - case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); - CYTHON_FALLTHROUGH; - case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); - CYTHON_FALLTHROUGH; - case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - CYTHON_FALLTHROUGH; - case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - CYTHON_FALLTHROUGH; - case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - CYTHON_FALLTHROUGH; - case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - CYTHON_FALLTHROUGH; - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_file_like); - if (value) { values[0] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 1: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_read_size); - if (value) { values[1] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 2: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_use_list); - if (value) { values[2] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 3: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_raw); - if (value) { values[3] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 4: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_strict_map_key); - if (value) { values[4] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 5: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_object_hook); - if (value) { values[5] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 6: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_object_pairs_hook); - if (value) { values[6] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 7: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_list_hook); - if (value) { values[7] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 8: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_encoding); - if (value) { values[8] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 9: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_unicode_errors); - if (value) { values[9] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 10: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_buffer_size); - if (value) { values[10] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 11: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_ext_hook); - if (value) { values[11] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 12: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_str_len); - if (value) { values[12] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 13: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_bin_len); - if (value) { values[13] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 14: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_array_len); - if (value) { values[14] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 15: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_map_len); - if (value) { values[15] = value; kw_args--; } - } - CYTHON_FALLTHROUGH; - case 16: - if (kw_args > 0) { - PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_ext_len); - if (value) { values[16] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 344, __pyx_L3_error) - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 17: values[16] = PyTuple_GET_ITEM(__pyx_args, 16); - CYTHON_FALLTHROUGH; - case 16: values[15] = PyTuple_GET_ITEM(__pyx_args, 15); - CYTHON_FALLTHROUGH; - case 15: values[14] = PyTuple_GET_ITEM(__pyx_args, 14); - CYTHON_FALLTHROUGH; - case 14: values[13] = PyTuple_GET_ITEM(__pyx_args, 13); - CYTHON_FALLTHROUGH; - case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); - CYTHON_FALLTHROUGH; - case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); - CYTHON_FALLTHROUGH; - case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); - CYTHON_FALLTHROUGH; - case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); - CYTHON_FALLTHROUGH; - case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); - CYTHON_FALLTHROUGH; - case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); - CYTHON_FALLTHROUGH; - case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - CYTHON_FALLTHROUGH; - case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - CYTHON_FALLTHROUGH; - case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - CYTHON_FALLTHROUGH; - case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - CYTHON_FALLTHROUGH; - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - CYTHON_FALLTHROUGH; - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - CYTHON_FALLTHROUGH; - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - CYTHON_FALLTHROUGH; - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_file_like = values[0]; - if (values[1]) { - __pyx_v_read_size = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_read_size == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 344, __pyx_L3_error) - } else { - __pyx_v_read_size = ((Py_ssize_t)0); - } - if (values[2]) { - __pyx_v_use_list = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_use_list == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 345, __pyx_L3_error) - } else { - - /* "msgpack/_unpacker.pyx":345 - * - * def __init__(self, file_like=None, Py_ssize_t read_size=0, - * bint use_list=True, bint raw=True, bint strict_map_key=False, # <<<<<<<<<<<<<< - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, - * encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, - */ - __pyx_v_use_list = ((int)1); - } - if (values[3]) { - __pyx_v_raw = __Pyx_PyObject_IsTrue(values[3]); if (unlikely((__pyx_v_raw == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 345, __pyx_L3_error) - } else { - __pyx_v_raw = ((int)1); - } - if (values[4]) { - __pyx_v_strict_map_key = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_strict_map_key == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 345, __pyx_L3_error) - } else { - __pyx_v_strict_map_key = ((int)0); - } - __pyx_v_object_hook = values[5]; - __pyx_v_object_pairs_hook = values[6]; - __pyx_v_list_hook = values[7]; - __pyx_v_encoding = values[8]; - __pyx_v_unicode_errors = values[9]; - if (values[10]) { - __pyx_v_max_buffer_size = __Pyx_PyIndex_AsSsize_t(values[10]); if (unlikely((__pyx_v_max_buffer_size == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 347, __pyx_L3_error) - } else { - __pyx_v_max_buffer_size = ((Py_ssize_t)0); - } - __pyx_v_ext_hook = values[11]; - if (values[12]) { - __pyx_v_max_str_len = __Pyx_PyIndex_AsSsize_t(values[12]); if (unlikely((__pyx_v_max_str_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 349, __pyx_L3_error) - } else { - __pyx_v_max_str_len = ((Py_ssize_t)-1L); - } - if (values[13]) { - __pyx_v_max_bin_len = __Pyx_PyIndex_AsSsize_t(values[13]); if (unlikely((__pyx_v_max_bin_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 350, __pyx_L3_error) - } else { - __pyx_v_max_bin_len = ((Py_ssize_t)-1L); - } - if (values[14]) { - __pyx_v_max_array_len = __Pyx_PyIndex_AsSsize_t(values[14]); if (unlikely((__pyx_v_max_array_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 351, __pyx_L3_error) - } else { - __pyx_v_max_array_len = ((Py_ssize_t)-1L); - } - if (values[15]) { - __pyx_v_max_map_len = __Pyx_PyIndex_AsSsize_t(values[15]); if (unlikely((__pyx_v_max_map_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 352, __pyx_L3_error) - } else { - __pyx_v_max_map_len = ((Py_ssize_t)-1L); - } - if (values[16]) { - __pyx_v_max_ext_len = __Pyx_PyIndex_AsSsize_t(values[16]); if (unlikely((__pyx_v_max_ext_len == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 353, __pyx_L3_error) - } else { - __pyx_v_max_ext_len = ((Py_ssize_t)-1L); - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 0, 17, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 344, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_4__init__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self), __pyx_v_file_like, __pyx_v_read_size, __pyx_v_use_list, __pyx_v_raw, __pyx_v_strict_map_key, __pyx_v_object_hook, __pyx_v_object_pairs_hook, __pyx_v_list_hook, __pyx_v_encoding, __pyx_v_unicode_errors, __pyx_v_max_buffer_size, __pyx_v_ext_hook, __pyx_v_max_str_len, __pyx_v_max_bin_len, __pyx_v_max_array_len, __pyx_v_max_map_len, __pyx_v_max_ext_len); - - /* "msgpack/_unpacker.pyx":344 - * self.buf = NULL - * - * def __init__(self, file_like=None, Py_ssize_t read_size=0, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_4__init__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, PyObject *__pyx_v_file_like, Py_ssize_t __pyx_v_read_size, int __pyx_v_use_list, int __pyx_v_raw, int __pyx_v_strict_map_key, PyObject *__pyx_v_object_hook, PyObject *__pyx_v_object_pairs_hook, PyObject *__pyx_v_list_hook, PyObject *__pyx_v_encoding, PyObject *__pyx_v_unicode_errors, Py_ssize_t __pyx_v_max_buffer_size, PyObject *__pyx_v_ext_hook, Py_ssize_t __pyx_v_max_str_len, Py_ssize_t __pyx_v_max_bin_len, Py_ssize_t __pyx_v_max_array_len, Py_ssize_t __pyx_v_max_map_len, Py_ssize_t __pyx_v_max_ext_len) { - char const *__pyx_v_cenc; - char const *__pyx_v_cerr; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - Py_ssize_t __pyx_t_3; - Py_ssize_t __pyx_t_4; - long __pyx_t_5; - int __pyx_t_6; - int __pyx_t_7; - char const *__pyx_t_8; - char const *__pyx_t_9; - __Pyx_RefNannySetupContext("__init__", 0); - - /* "msgpack/_unpacker.pyx":354 - * Py_ssize_t max_map_len=-1, - * Py_ssize_t max_ext_len=-1): - * cdef const char *cenc=NULL, # <<<<<<<<<<<<<< - * cdef const char *cerr=NULL - * - */ - __pyx_v_cenc = NULL; - - /* "msgpack/_unpacker.pyx":355 - * Py_ssize_t max_ext_len=-1): - * cdef const char *cenc=NULL, - * cdef const char *cerr=NULL # <<<<<<<<<<<<<< - * - * self.object_hook = object_hook - */ - __pyx_v_cerr = NULL; - - /* "msgpack/_unpacker.pyx":357 - * cdef const char *cerr=NULL - * - * self.object_hook = object_hook # <<<<<<<<<<<<<< - * self.object_pairs_hook = object_pairs_hook - * self.list_hook = list_hook - */ - __Pyx_INCREF(__pyx_v_object_hook); - __Pyx_GIVEREF(__pyx_v_object_hook); - __Pyx_GOTREF(__pyx_v_self->object_hook); - __Pyx_DECREF(__pyx_v_self->object_hook); - __pyx_v_self->object_hook = __pyx_v_object_hook; - - /* "msgpack/_unpacker.pyx":358 - * - * self.object_hook = object_hook - * self.object_pairs_hook = object_pairs_hook # <<<<<<<<<<<<<< - * self.list_hook = list_hook - * self.ext_hook = ext_hook - */ - __Pyx_INCREF(__pyx_v_object_pairs_hook); - __Pyx_GIVEREF(__pyx_v_object_pairs_hook); - __Pyx_GOTREF(__pyx_v_self->object_pairs_hook); - __Pyx_DECREF(__pyx_v_self->object_pairs_hook); - __pyx_v_self->object_pairs_hook = __pyx_v_object_pairs_hook; - - /* "msgpack/_unpacker.pyx":359 - * self.object_hook = object_hook - * self.object_pairs_hook = object_pairs_hook - * self.list_hook = list_hook # <<<<<<<<<<<<<< - * self.ext_hook = ext_hook - * - */ - __Pyx_INCREF(__pyx_v_list_hook); - __Pyx_GIVEREF(__pyx_v_list_hook); - __Pyx_GOTREF(__pyx_v_self->list_hook); - __Pyx_DECREF(__pyx_v_self->list_hook); - __pyx_v_self->list_hook = __pyx_v_list_hook; - - /* "msgpack/_unpacker.pyx":360 - * self.object_pairs_hook = object_pairs_hook - * self.list_hook = list_hook - * self.ext_hook = ext_hook # <<<<<<<<<<<<<< - * - * self.file_like = file_like - */ - __Pyx_INCREF(__pyx_v_ext_hook); - __Pyx_GIVEREF(__pyx_v_ext_hook); - __Pyx_GOTREF(__pyx_v_self->ext_hook); - __Pyx_DECREF(__pyx_v_self->ext_hook); - __pyx_v_self->ext_hook = __pyx_v_ext_hook; - - /* "msgpack/_unpacker.pyx":362 - * self.ext_hook = ext_hook - * - * self.file_like = file_like # <<<<<<<<<<<<<< - * if file_like: - * self.file_like_read = file_like.read - */ - __Pyx_INCREF(__pyx_v_file_like); - __Pyx_GIVEREF(__pyx_v_file_like); - __Pyx_GOTREF(__pyx_v_self->file_like); - __Pyx_DECREF(__pyx_v_self->file_like); - __pyx_v_self->file_like = __pyx_v_file_like; - - /* "msgpack/_unpacker.pyx":363 - * - * self.file_like = file_like - * if file_like: # <<<<<<<<<<<<<< - * self.file_like_read = file_like.read - * if not PyCallable_Check(self.file_like_read): - */ - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_file_like); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 363, __pyx_L1_error) - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":364 - * self.file_like = file_like - * if file_like: - * self.file_like_read = file_like.read # <<<<<<<<<<<<<< - * if not PyCallable_Check(self.file_like_read): - * raise TypeError("`file_like.read` must be a callable.") - */ - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_file_like, __pyx_n_s_read); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 364, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GIVEREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_v_self->file_like_read); - __Pyx_DECREF(__pyx_v_self->file_like_read); - __pyx_v_self->file_like_read = __pyx_t_2; - __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":365 - * if file_like: - * self.file_like_read = file_like.read - * if not PyCallable_Check(self.file_like_read): # <<<<<<<<<<<<<< - * raise TypeError("`file_like.read` must be a callable.") - * - */ - __pyx_t_2 = __pyx_v_self->file_like_read; - __Pyx_INCREF(__pyx_t_2); - __pyx_t_1 = ((!(PyCallable_Check(__pyx_t_2) != 0)) != 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":366 - * self.file_like_read = file_like.read - * if not PyCallable_Check(self.file_like_read): - * raise TypeError("`file_like.read` must be a callable.") # <<<<<<<<<<<<<< - * - * if max_str_len == -1: - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__25, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 366, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 366, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":365 - * if file_like: - * self.file_like_read = file_like.read - * if not PyCallable_Check(self.file_like_read): # <<<<<<<<<<<<<< - * raise TypeError("`file_like.read` must be a callable.") - * - */ - } - - /* "msgpack/_unpacker.pyx":363 - * - * self.file_like = file_like - * if file_like: # <<<<<<<<<<<<<< - * self.file_like_read = file_like.read - * if not PyCallable_Check(self.file_like_read): - */ - } - - /* "msgpack/_unpacker.pyx":368 - * raise TypeError("`file_like.read` must be a callable.") - * - * if max_str_len == -1: # <<<<<<<<<<<<<< - * max_str_len = max_buffer_size or 1024*1024 - * if max_bin_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_str_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":369 - * - * if max_str_len == -1: - * max_str_len = max_buffer_size or 1024*1024 # <<<<<<<<<<<<<< - * if max_bin_len == -1: - * max_bin_len = max_buffer_size or 1024*1024 - */ - if (!__pyx_v_max_buffer_size) { - } else { - __pyx_t_3 = __pyx_v_max_buffer_size; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_3 = 0x100000; - __pyx_L6_bool_binop_done:; - __pyx_v_max_str_len = __pyx_t_3; - - /* "msgpack/_unpacker.pyx":368 - * raise TypeError("`file_like.read` must be a callable.") - * - * if max_str_len == -1: # <<<<<<<<<<<<<< - * max_str_len = max_buffer_size or 1024*1024 - * if max_bin_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":370 - * if max_str_len == -1: - * max_str_len = max_buffer_size or 1024*1024 - * if max_bin_len == -1: # <<<<<<<<<<<<<< - * max_bin_len = max_buffer_size or 1024*1024 - * if max_array_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_bin_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":371 - * max_str_len = max_buffer_size or 1024*1024 - * if max_bin_len == -1: - * max_bin_len = max_buffer_size or 1024*1024 # <<<<<<<<<<<<<< - * if max_array_len == -1: - * max_array_len = max_buffer_size or 128*1024 - */ - if (!__pyx_v_max_buffer_size) { - } else { - __pyx_t_3 = __pyx_v_max_buffer_size; - goto __pyx_L9_bool_binop_done; - } - __pyx_t_3 = 0x100000; - __pyx_L9_bool_binop_done:; - __pyx_v_max_bin_len = __pyx_t_3; - - /* "msgpack/_unpacker.pyx":370 - * if max_str_len == -1: - * max_str_len = max_buffer_size or 1024*1024 - * if max_bin_len == -1: # <<<<<<<<<<<<<< - * max_bin_len = max_buffer_size or 1024*1024 - * if max_array_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":372 - * if max_bin_len == -1: - * max_bin_len = max_buffer_size or 1024*1024 - * if max_array_len == -1: # <<<<<<<<<<<<<< - * max_array_len = max_buffer_size or 128*1024 - * if max_map_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_array_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":373 - * max_bin_len = max_buffer_size or 1024*1024 - * if max_array_len == -1: - * max_array_len = max_buffer_size or 128*1024 # <<<<<<<<<<<<<< - * if max_map_len == -1: - * max_map_len = max_buffer_size//2 or 32*1024 - */ - if (!__pyx_v_max_buffer_size) { - } else { - __pyx_t_3 = __pyx_v_max_buffer_size; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_3 = 0x20000; - __pyx_L12_bool_binop_done:; - __pyx_v_max_array_len = __pyx_t_3; - - /* "msgpack/_unpacker.pyx":372 - * if max_bin_len == -1: - * max_bin_len = max_buffer_size or 1024*1024 - * if max_array_len == -1: # <<<<<<<<<<<<<< - * max_array_len = max_buffer_size or 128*1024 - * if max_map_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":374 - * if max_array_len == -1: - * max_array_len = max_buffer_size or 128*1024 - * if max_map_len == -1: # <<<<<<<<<<<<<< - * max_map_len = max_buffer_size//2 or 32*1024 - * if max_ext_len == -1: - */ - __pyx_t_1 = ((__pyx_v_max_map_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":375 - * max_array_len = max_buffer_size or 128*1024 - * if max_map_len == -1: - * max_map_len = max_buffer_size//2 or 32*1024 # <<<<<<<<<<<<<< - * if max_ext_len == -1: - * max_ext_len = max_buffer_size or 1024*1024 - */ - __pyx_t_4 = __Pyx_div_Py_ssize_t(__pyx_v_max_buffer_size, 2); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L15_bool_binop_done; - } - __pyx_t_3 = 0x8000; - __pyx_L15_bool_binop_done:; - __pyx_v_max_map_len = __pyx_t_3; - - /* "msgpack/_unpacker.pyx":374 - * if max_array_len == -1: - * max_array_len = max_buffer_size or 128*1024 - * if max_map_len == -1: # <<<<<<<<<<<<<< - * max_map_len = max_buffer_size//2 or 32*1024 - * if max_ext_len == -1: - */ - } - - /* "msgpack/_unpacker.pyx":376 - * if max_map_len == -1: - * max_map_len = max_buffer_size//2 or 32*1024 - * if max_ext_len == -1: # <<<<<<<<<<<<<< - * max_ext_len = max_buffer_size or 1024*1024 - * - */ - __pyx_t_1 = ((__pyx_v_max_ext_len == -1L) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":377 - * max_map_len = max_buffer_size//2 or 32*1024 - * if max_ext_len == -1: - * max_ext_len = max_buffer_size or 1024*1024 # <<<<<<<<<<<<<< - * - * if not max_buffer_size: - */ - if (!__pyx_v_max_buffer_size) { - } else { - __pyx_t_3 = __pyx_v_max_buffer_size; - goto __pyx_L18_bool_binop_done; - } - __pyx_t_3 = 0x100000; - __pyx_L18_bool_binop_done:; - __pyx_v_max_ext_len = __pyx_t_3; - - /* "msgpack/_unpacker.pyx":376 - * if max_map_len == -1: - * max_map_len = max_buffer_size//2 or 32*1024 - * if max_ext_len == -1: # <<<<<<<<<<<<<< - * max_ext_len = max_buffer_size or 1024*1024 - * - */ - } - - /* "msgpack/_unpacker.pyx":379 - * max_ext_len = max_buffer_size or 1024*1024 - * - * if not max_buffer_size: # <<<<<<<<<<<<<< - * max_buffer_size = INT_MAX - * if read_size > max_buffer_size: - */ - __pyx_t_1 = ((!(__pyx_v_max_buffer_size != 0)) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":380 - * - * if not max_buffer_size: - * max_buffer_size = INT_MAX # <<<<<<<<<<<<<< - * if read_size > max_buffer_size: - * raise ValueError("read_size should be less or equal to max_buffer_size") - */ - __pyx_v_max_buffer_size = INT_MAX; - - /* "msgpack/_unpacker.pyx":379 - * max_ext_len = max_buffer_size or 1024*1024 - * - * if not max_buffer_size: # <<<<<<<<<<<<<< - * max_buffer_size = INT_MAX - * if read_size > max_buffer_size: - */ - } - - /* "msgpack/_unpacker.pyx":381 - * if not max_buffer_size: - * max_buffer_size = INT_MAX - * if read_size > max_buffer_size: # <<<<<<<<<<<<<< - * raise ValueError("read_size should be less or equal to max_buffer_size") - * if not read_size: - */ - __pyx_t_1 = ((__pyx_v_read_size > __pyx_v_max_buffer_size) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":382 - * max_buffer_size = INT_MAX - * if read_size > max_buffer_size: - * raise ValueError("read_size should be less or equal to max_buffer_size") # <<<<<<<<<<<<<< - * if not read_size: - * read_size = min(max_buffer_size, 1024**2) - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__26, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 382, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 382, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":381 - * if not max_buffer_size: - * max_buffer_size = INT_MAX - * if read_size > max_buffer_size: # <<<<<<<<<<<<<< - * raise ValueError("read_size should be less or equal to max_buffer_size") - * if not read_size: - */ - } - - /* "msgpack/_unpacker.pyx":383 - * if read_size > max_buffer_size: - * raise ValueError("read_size should be less or equal to max_buffer_size") - * if not read_size: # <<<<<<<<<<<<<< - * read_size = min(max_buffer_size, 1024**2) - * self.max_buffer_size = max_buffer_size - */ - __pyx_t_1 = ((!(__pyx_v_read_size != 0)) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":384 - * raise ValueError("read_size should be less or equal to max_buffer_size") - * if not read_size: - * read_size = min(max_buffer_size, 1024**2) # <<<<<<<<<<<<<< - * self.max_buffer_size = max_buffer_size - * self.read_size = read_size - */ - __pyx_t_5 = 0x100000; - __pyx_t_3 = __pyx_v_max_buffer_size; - if (((__pyx_t_5 < __pyx_t_3) != 0)) { - __pyx_t_4 = __pyx_t_5; - } else { - __pyx_t_4 = __pyx_t_3; - } - __pyx_v_read_size = __pyx_t_4; - - /* "msgpack/_unpacker.pyx":383 - * if read_size > max_buffer_size: - * raise ValueError("read_size should be less or equal to max_buffer_size") - * if not read_size: # <<<<<<<<<<<<<< - * read_size = min(max_buffer_size, 1024**2) - * self.max_buffer_size = max_buffer_size - */ - } - - /* "msgpack/_unpacker.pyx":385 - * if not read_size: - * read_size = min(max_buffer_size, 1024**2) - * self.max_buffer_size = max_buffer_size # <<<<<<<<<<<<<< - * self.read_size = read_size - * self.buf = PyMem_Malloc(read_size) - */ - __pyx_v_self->max_buffer_size = __pyx_v_max_buffer_size; - - /* "msgpack/_unpacker.pyx":386 - * read_size = min(max_buffer_size, 1024**2) - * self.max_buffer_size = max_buffer_size - * self.read_size = read_size # <<<<<<<<<<<<<< - * self.buf = PyMem_Malloc(read_size) - * if self.buf == NULL: - */ - __pyx_v_self->read_size = __pyx_v_read_size; - - /* "msgpack/_unpacker.pyx":387 - * self.max_buffer_size = max_buffer_size - * self.read_size = read_size - * self.buf = PyMem_Malloc(read_size) # <<<<<<<<<<<<<< - * if self.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") - */ - __pyx_v_self->buf = ((char *)PyMem_Malloc(__pyx_v_read_size)); - - /* "msgpack/_unpacker.pyx":388 - * self.read_size = read_size - * self.buf = PyMem_Malloc(read_size) - * if self.buf == NULL: # <<<<<<<<<<<<<< - * raise MemoryError("Unable to allocate internal buffer.") - * self.buf_size = read_size - */ - __pyx_t_1 = ((__pyx_v_self->buf == NULL) != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":389 - * self.buf = PyMem_Malloc(read_size) - * if self.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") # <<<<<<<<<<<<<< - * self.buf_size = read_size - * self.buf_head = 0 - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 389, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 389, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":388 - * self.read_size = read_size - * self.buf = PyMem_Malloc(read_size) - * if self.buf == NULL: # <<<<<<<<<<<<<< - * raise MemoryError("Unable to allocate internal buffer.") - * self.buf_size = read_size - */ - } - - /* "msgpack/_unpacker.pyx":390 - * if self.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") - * self.buf_size = read_size # <<<<<<<<<<<<<< - * self.buf_head = 0 - * self.buf_tail = 0 - */ - __pyx_v_self->buf_size = __pyx_v_read_size; - - /* "msgpack/_unpacker.pyx":391 - * raise MemoryError("Unable to allocate internal buffer.") - * self.buf_size = read_size - * self.buf_head = 0 # <<<<<<<<<<<<<< - * self.buf_tail = 0 - * self.stream_offset = 0 - */ - __pyx_v_self->buf_head = 0; - - /* "msgpack/_unpacker.pyx":392 - * self.buf_size = read_size - * self.buf_head = 0 - * self.buf_tail = 0 # <<<<<<<<<<<<<< - * self.stream_offset = 0 - * - */ - __pyx_v_self->buf_tail = 0; - - /* "msgpack/_unpacker.pyx":393 - * self.buf_head = 0 - * self.buf_tail = 0 - * self.stream_offset = 0 # <<<<<<<<<<<<<< - * - * if encoding is not None: - */ - __pyx_v_self->stream_offset = 0; - - /* "msgpack/_unpacker.pyx":395 - * self.stream_offset = 0 - * - * if encoding is not None: # <<<<<<<<<<<<<< - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * self.encoding = encoding - */ - __pyx_t_1 = (__pyx_v_encoding != Py_None); - __pyx_t_6 = (__pyx_t_1 != 0); - if (__pyx_t_6) { - - /* "msgpack/_unpacker.pyx":396 - * - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) # <<<<<<<<<<<<<< - * self.encoding = encoding - * cenc = encoding - */ - __pyx_t_7 = PyErr_WarnEx(__pyx_builtin_DeprecationWarning, ((char *)"encoding is deprecated, Use raw=False instead."), 1); if (unlikely(__pyx_t_7 == ((int)-1))) __PYX_ERR(1, 396, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":397 - * if encoding is not None: - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * self.encoding = encoding # <<<<<<<<<<<<<< - * cenc = encoding - * - */ - __Pyx_INCREF(__pyx_v_encoding); - __Pyx_GIVEREF(__pyx_v_encoding); - __Pyx_GOTREF(__pyx_v_self->encoding); - __Pyx_DECREF(__pyx_v_self->encoding); - __pyx_v_self->encoding = __pyx_v_encoding; - - /* "msgpack/_unpacker.pyx":398 - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * self.encoding = encoding - * cenc = encoding # <<<<<<<<<<<<<< - * - * if unicode_errors is not None: - */ - __pyx_t_8 = __Pyx_PyObject_AsString(__pyx_v_encoding); if (unlikely((!__pyx_t_8) && PyErr_Occurred())) __PYX_ERR(1, 398, __pyx_L1_error) - __pyx_v_cenc = __pyx_t_8; - - /* "msgpack/_unpacker.pyx":395 - * self.stream_offset = 0 - * - * if encoding is not None: # <<<<<<<<<<<<<< - * PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - * self.encoding = encoding - */ - } - - /* "msgpack/_unpacker.pyx":400 - * cenc = encoding - * - * if unicode_errors is not None: # <<<<<<<<<<<<<< - * self.unicode_errors = unicode_errors - * cerr = unicode_errors - */ - __pyx_t_6 = (__pyx_v_unicode_errors != Py_None); - __pyx_t_1 = (__pyx_t_6 != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":401 - * - * if unicode_errors is not None: - * self.unicode_errors = unicode_errors # <<<<<<<<<<<<<< - * cerr = unicode_errors - * - */ - __Pyx_INCREF(__pyx_v_unicode_errors); - __Pyx_GIVEREF(__pyx_v_unicode_errors); - __Pyx_GOTREF(__pyx_v_self->unicode_errors); - __Pyx_DECREF(__pyx_v_self->unicode_errors); - __pyx_v_self->unicode_errors = __pyx_v_unicode_errors; - - /* "msgpack/_unpacker.pyx":402 - * if unicode_errors is not None: - * self.unicode_errors = unicode_errors - * cerr = unicode_errors # <<<<<<<<<<<<<< - * - * init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - */ - __pyx_t_9 = __Pyx_PyObject_AsString(__pyx_v_unicode_errors); if (unlikely((!__pyx_t_9) && PyErr_Occurred())) __PYX_ERR(1, 402, __pyx_L1_error) - __pyx_v_cerr = __pyx_t_9; - - /* "msgpack/_unpacker.pyx":400 - * cenc = encoding - * - * if unicode_errors is not None: # <<<<<<<<<<<<<< - * self.unicode_errors = unicode_errors - * cerr = unicode_errors - */ - } - - /* "msgpack/_unpacker.pyx":404 - * cerr = unicode_errors - * - * init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, # <<<<<<<<<<<<<< - * ext_hook, use_list, raw, strict_map_key, cenc, cerr, - * max_str_len, max_bin_len, max_array_len, - */ - __pyx_t_2 = __pyx_f_7msgpack_9_cmsgpack_init_ctx((&__pyx_v_self->ctx), __pyx_v_object_hook, __pyx_v_object_pairs_hook, __pyx_v_list_hook, __pyx_v_ext_hook, __pyx_v_use_list, __pyx_v_raw, __pyx_v_strict_map_key, __pyx_v_cenc, __pyx_v_cerr, __pyx_v_max_str_len, __pyx_v_max_bin_len, __pyx_v_max_array_len, __pyx_v_max_map_len, __pyx_v_max_ext_len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 404, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":344 - * self.buf = NULL - * - * def __init__(self, file_like=None, Py_ssize_t read_size=0, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":409 - * max_map_len, max_ext_len) - * - * def feed(self, object next_bytes): # <<<<<<<<<<<<<< - * """Append `next_bytes` to internal buffer.""" - * cdef Py_buffer pybuff - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_7feed(PyObject *__pyx_v_self, PyObject *__pyx_v_next_bytes); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_6feed[] = "Unpacker.feed(self, next_bytes)\nAppend `next_bytes` to internal buffer."; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_7feed(PyObject *__pyx_v_self, PyObject *__pyx_v_next_bytes) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("feed (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_6feed(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self), ((PyObject *)__pyx_v_next_bytes)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_6feed(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, PyObject *__pyx_v_next_bytes) { - Py_buffer __pyx_v_pybuff; - int __pyx_v_new_protocol; - char *__pyx_v_buf; - Py_ssize_t __pyx_v_buf_len; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - int __pyx_t_5; - char const *__pyx_t_6; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - PyObject *__pyx_t_9 = NULL; - PyObject *__pyx_t_10 = NULL; - PyObject *__pyx_t_11 = NULL; - PyObject *__pyx_t_12 = NULL; - __Pyx_RefNannySetupContext("feed", 0); - - /* "msgpack/_unpacker.pyx":412 - * """Append `next_bytes` to internal buffer.""" - * cdef Py_buffer pybuff - * cdef int new_protocol = 0 # <<<<<<<<<<<<<< - * cdef char* buf - * cdef Py_ssize_t buf_len - */ - __pyx_v_new_protocol = 0; - - /* "msgpack/_unpacker.pyx":416 - * cdef Py_ssize_t buf_len - * - * if self.file_like is not None: # <<<<<<<<<<<<<< - * raise AssertionError( - * "unpacker.feed() is not be able to use with `file_like`.") - */ - __pyx_t_1 = (__pyx_v_self->file_like != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (unlikely(__pyx_t_2)) { - - /* "msgpack/_unpacker.pyx":417 - * - * if self.file_like is not None: - * raise AssertionError( # <<<<<<<<<<<<<< - * "unpacker.feed() is not be able to use with `file_like`.") - * - */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_AssertionError, __pyx_tuple__27, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 417, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 417, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":416 - * cdef Py_ssize_t buf_len - * - * if self.file_like is not None: # <<<<<<<<<<<<<< - * raise AssertionError( - * "unpacker.feed() is not be able to use with `file_like`.") - */ - } - - /* "msgpack/_unpacker.pyx":420 - * "unpacker.feed() is not be able to use with `file_like`.") - * - * get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol) # <<<<<<<<<<<<<< - * try: - * self.append_buffer(buf, buf_len) - */ - __pyx_t_4 = __pyx_f_7msgpack_9_cmsgpack_get_data_from_buffer(__pyx_v_next_bytes, (&__pyx_v_pybuff), (&__pyx_v_buf), (&__pyx_v_buf_len), (&__pyx_v_new_protocol)); if (unlikely(__pyx_t_4 == ((int)0))) __PYX_ERR(1, 420, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":421 - * - * get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol) - * try: # <<<<<<<<<<<<<< - * self.append_buffer(buf, buf_len) - * finally: - */ - /*try:*/ { - - /* "msgpack/_unpacker.pyx":422 - * get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol) - * try: - * self.append_buffer(buf, buf_len) # <<<<<<<<<<<<<< - * finally: - * if new_protocol: - */ - __pyx_t_3 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->append_buffer(__pyx_v_self, __pyx_v_buf, __pyx_v_buf_len); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 422, __pyx_L5_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } - - /* "msgpack/_unpacker.pyx":424 - * self.append_buffer(buf, buf_len) - * finally: - * if new_protocol: # <<<<<<<<<<<<<< - * PyBuffer_Release(&pybuff) - * - */ - /*finally:*/ { - /*normal exit:*/{ - __pyx_t_2 = (__pyx_v_new_protocol != 0); - if (__pyx_t_2) { - - /* "msgpack/_unpacker.pyx":425 - * finally: - * if new_protocol: - * PyBuffer_Release(&pybuff) # <<<<<<<<<<<<<< - * - * cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): - */ - PyBuffer_Release((&__pyx_v_pybuff)); - - /* "msgpack/_unpacker.pyx":424 - * self.append_buffer(buf, buf_len) - * finally: - * if new_protocol: # <<<<<<<<<<<<<< - * PyBuffer_Release(&pybuff) - * - */ - } - goto __pyx_L6; - } - __pyx_L5_error:; - /*exception exit:*/{ - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PY_MAJOR_VERSION >= 3) __Pyx_ExceptionSwap(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12); - if ((PY_MAJOR_VERSION < 3) || unlikely(__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9) < 0)) __Pyx_ErrFetch(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); - __Pyx_XGOTREF(__pyx_t_7); - __Pyx_XGOTREF(__pyx_t_8); - __Pyx_XGOTREF(__pyx_t_9); - __Pyx_XGOTREF(__pyx_t_10); - __Pyx_XGOTREF(__pyx_t_11); - __Pyx_XGOTREF(__pyx_t_12); - __pyx_t_4 = __pyx_lineno; __pyx_t_5 = __pyx_clineno; __pyx_t_6 = __pyx_filename; - { - __pyx_t_2 = (__pyx_v_new_protocol != 0); - if (__pyx_t_2) { - - /* "msgpack/_unpacker.pyx":425 - * finally: - * if new_protocol: - * PyBuffer_Release(&pybuff) # <<<<<<<<<<<<<< - * - * cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): - */ - PyBuffer_Release((&__pyx_v_pybuff)); - - /* "msgpack/_unpacker.pyx":424 - * self.append_buffer(buf, buf_len) - * finally: - * if new_protocol: # <<<<<<<<<<<<<< - * PyBuffer_Release(&pybuff) - * - */ - } - } - if (PY_MAJOR_VERSION >= 3) { - __Pyx_XGIVEREF(__pyx_t_10); - __Pyx_XGIVEREF(__pyx_t_11); - __Pyx_XGIVEREF(__pyx_t_12); - __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); - } - __Pyx_XGIVEREF(__pyx_t_7); - __Pyx_XGIVEREF(__pyx_t_8); - __Pyx_XGIVEREF(__pyx_t_9); - __Pyx_ErrRestore(__pyx_t_7, __pyx_t_8, __pyx_t_9); - __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; - __pyx_lineno = __pyx_t_4; __pyx_clineno = __pyx_t_5; __pyx_filename = __pyx_t_6; - goto __pyx_L1_error; - } - __pyx_L6:; - } - - /* "msgpack/_unpacker.pyx":409 - * max_map_len, max_ext_len) - * - * def feed(self, object next_bytes): # <<<<<<<<<<<<<< - * """Append `next_bytes` to internal buffer.""" - * cdef Py_buffer pybuff - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.feed", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":427 - * PyBuffer_Release(&pybuff) - * - * cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): # <<<<<<<<<<<<<< - * cdef: - * char* buf = self.buf - */ - -static PyObject *__pyx_f_7msgpack_9_cmsgpack_8Unpacker_append_buffer(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, void *__pyx_v__buf, Py_ssize_t __pyx_v__buf_len) { - char *__pyx_v_buf; - char *__pyx_v_new_buf; - Py_ssize_t __pyx_v_head; - Py_ssize_t __pyx_v_tail; - Py_ssize_t __pyx_v_buf_size; - Py_ssize_t __pyx_v_new_size; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - char *__pyx_t_1; - Py_ssize_t __pyx_t_2; - int __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - Py_ssize_t __pyx_t_5; - Py_ssize_t __pyx_t_6; - __Pyx_RefNannySetupContext("append_buffer", 0); - - /* "msgpack/_unpacker.pyx":429 - * cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): - * cdef: - * char* buf = self.buf # <<<<<<<<<<<<<< - * char* new_buf - * Py_ssize_t head = self.buf_head - */ - __pyx_t_1 = __pyx_v_self->buf; - __pyx_v_buf = __pyx_t_1; - - /* "msgpack/_unpacker.pyx":431 - * char* buf = self.buf - * char* new_buf - * Py_ssize_t head = self.buf_head # <<<<<<<<<<<<<< - * Py_ssize_t tail = self.buf_tail - * Py_ssize_t buf_size = self.buf_size - */ - __pyx_t_2 = __pyx_v_self->buf_head; - __pyx_v_head = __pyx_t_2; - - /* "msgpack/_unpacker.pyx":432 - * char* new_buf - * Py_ssize_t head = self.buf_head - * Py_ssize_t tail = self.buf_tail # <<<<<<<<<<<<<< - * Py_ssize_t buf_size = self.buf_size - * Py_ssize_t new_size - */ - __pyx_t_2 = __pyx_v_self->buf_tail; - __pyx_v_tail = __pyx_t_2; - - /* "msgpack/_unpacker.pyx":433 - * Py_ssize_t head = self.buf_head - * Py_ssize_t tail = self.buf_tail - * Py_ssize_t buf_size = self.buf_size # <<<<<<<<<<<<<< - * Py_ssize_t new_size - * - */ - __pyx_t_2 = __pyx_v_self->buf_size; - __pyx_v_buf_size = __pyx_t_2; - - /* "msgpack/_unpacker.pyx":436 - * Py_ssize_t new_size - * - * if tail + _buf_len > buf_size: # <<<<<<<<<<<<<< - * if ((tail - head) + _buf_len) <= buf_size: - * # move to front. - */ - __pyx_t_3 = (((__pyx_v_tail + __pyx_v__buf_len) > __pyx_v_buf_size) != 0); - if (__pyx_t_3) { - - /* "msgpack/_unpacker.pyx":437 - * - * if tail + _buf_len > buf_size: - * if ((tail - head) + _buf_len) <= buf_size: # <<<<<<<<<<<<<< - * # move to front. - * memmove(buf, buf + head, tail - head) - */ - __pyx_t_3 = ((((__pyx_v_tail - __pyx_v_head) + __pyx_v__buf_len) <= __pyx_v_buf_size) != 0); - if (__pyx_t_3) { - - /* "msgpack/_unpacker.pyx":439 - * if ((tail - head) + _buf_len) <= buf_size: - * # move to front. - * memmove(buf, buf + head, tail - head) # <<<<<<<<<<<<<< - * tail -= head - * head = 0 - */ - (void)(memmove(__pyx_v_buf, (__pyx_v_buf + __pyx_v_head), (__pyx_v_tail - __pyx_v_head))); - - /* "msgpack/_unpacker.pyx":440 - * # move to front. - * memmove(buf, buf + head, tail - head) - * tail -= head # <<<<<<<<<<<<<< - * head = 0 - * else: - */ - __pyx_v_tail = (__pyx_v_tail - __pyx_v_head); - - /* "msgpack/_unpacker.pyx":441 - * memmove(buf, buf + head, tail - head) - * tail -= head - * head = 0 # <<<<<<<<<<<<<< - * else: - * # expand buffer. - */ - __pyx_v_head = 0; - - /* "msgpack/_unpacker.pyx":437 - * - * if tail + _buf_len > buf_size: - * if ((tail - head) + _buf_len) <= buf_size: # <<<<<<<<<<<<<< - * # move to front. - * memmove(buf, buf + head, tail - head) - */ - goto __pyx_L4; - } - - /* "msgpack/_unpacker.pyx":444 - * else: - * # expand buffer. - * new_size = (tail-head) + _buf_len # <<<<<<<<<<<<<< - * if new_size > self.max_buffer_size: - * raise BufferFull - */ - /*else*/ { - __pyx_v_new_size = ((__pyx_v_tail - __pyx_v_head) + __pyx_v__buf_len); - - /* "msgpack/_unpacker.pyx":445 - * # expand buffer. - * new_size = (tail-head) + _buf_len - * if new_size > self.max_buffer_size: # <<<<<<<<<<<<<< - * raise BufferFull - * new_size = min(new_size*2, self.max_buffer_size) - */ - __pyx_t_3 = ((__pyx_v_new_size > __pyx_v_self->max_buffer_size) != 0); - if (unlikely(__pyx_t_3)) { - - /* "msgpack/_unpacker.pyx":446 - * new_size = (tail-head) + _buf_len - * if new_size > self.max_buffer_size: - * raise BufferFull # <<<<<<<<<<<<<< - * new_size = min(new_size*2, self.max_buffer_size) - * new_buf = PyMem_Malloc(new_size) - */ - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_BufferFull); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 446, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 446, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":445 - * # expand buffer. - * new_size = (tail-head) + _buf_len - * if new_size > self.max_buffer_size: # <<<<<<<<<<<<<< - * raise BufferFull - * new_size = min(new_size*2, self.max_buffer_size) - */ - } - - /* "msgpack/_unpacker.pyx":447 - * if new_size > self.max_buffer_size: - * raise BufferFull - * new_size = min(new_size*2, self.max_buffer_size) # <<<<<<<<<<<<<< - * new_buf = PyMem_Malloc(new_size) - * if new_buf == NULL: - */ - __pyx_t_2 = __pyx_v_self->max_buffer_size; - __pyx_t_5 = (__pyx_v_new_size * 2); - if (((__pyx_t_2 < __pyx_t_5) != 0)) { - __pyx_t_6 = __pyx_t_2; - } else { - __pyx_t_6 = __pyx_t_5; - } - __pyx_v_new_size = __pyx_t_6; - - /* "msgpack/_unpacker.pyx":448 - * raise BufferFull - * new_size = min(new_size*2, self.max_buffer_size) - * new_buf = PyMem_Malloc(new_size) # <<<<<<<<<<<<<< - * if new_buf == NULL: - * # self.buf still holds old buffer and will be freed during - */ - __pyx_v_new_buf = ((char *)PyMem_Malloc(__pyx_v_new_size)); - - /* "msgpack/_unpacker.pyx":449 - * new_size = min(new_size*2, self.max_buffer_size) - * new_buf = PyMem_Malloc(new_size) - * if new_buf == NULL: # <<<<<<<<<<<<<< - * # self.buf still holds old buffer and will be freed during - * # obj destruction - */ - __pyx_t_3 = ((__pyx_v_new_buf == NULL) != 0); - if (unlikely(__pyx_t_3)) { - - /* "msgpack/_unpacker.pyx":452 - * # self.buf still holds old buffer and will be freed during - * # obj destruction - * raise MemoryError("Unable to enlarge internal buffer.") # <<<<<<<<<<<<<< - * memcpy(new_buf, buf + head, tail - head) - * PyMem_Free(buf) - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__28, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 452, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 452, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":449 - * new_size = min(new_size*2, self.max_buffer_size) - * new_buf = PyMem_Malloc(new_size) - * if new_buf == NULL: # <<<<<<<<<<<<<< - * # self.buf still holds old buffer and will be freed during - * # obj destruction - */ - } - - /* "msgpack/_unpacker.pyx":453 - * # obj destruction - * raise MemoryError("Unable to enlarge internal buffer.") - * memcpy(new_buf, buf + head, tail - head) # <<<<<<<<<<<<<< - * PyMem_Free(buf) - * - */ - (void)(memcpy(__pyx_v_new_buf, (__pyx_v_buf + __pyx_v_head), (__pyx_v_tail - __pyx_v_head))); - - /* "msgpack/_unpacker.pyx":454 - * raise MemoryError("Unable to enlarge internal buffer.") - * memcpy(new_buf, buf + head, tail - head) - * PyMem_Free(buf) # <<<<<<<<<<<<<< - * - * buf = new_buf - */ - PyMem_Free(__pyx_v_buf); - - /* "msgpack/_unpacker.pyx":456 - * PyMem_Free(buf) - * - * buf = new_buf # <<<<<<<<<<<<<< - * buf_size = new_size - * tail -= head - */ - __pyx_v_buf = __pyx_v_new_buf; - - /* "msgpack/_unpacker.pyx":457 - * - * buf = new_buf - * buf_size = new_size # <<<<<<<<<<<<<< - * tail -= head - * head = 0 - */ - __pyx_v_buf_size = __pyx_v_new_size; - - /* "msgpack/_unpacker.pyx":458 - * buf = new_buf - * buf_size = new_size - * tail -= head # <<<<<<<<<<<<<< - * head = 0 - * - */ - __pyx_v_tail = (__pyx_v_tail - __pyx_v_head); - - /* "msgpack/_unpacker.pyx":459 - * buf_size = new_size - * tail -= head - * head = 0 # <<<<<<<<<<<<<< - * - * memcpy(buf + tail, (_buf), _buf_len) - */ - __pyx_v_head = 0; - } - __pyx_L4:; - - /* "msgpack/_unpacker.pyx":436 - * Py_ssize_t new_size - * - * if tail + _buf_len > buf_size: # <<<<<<<<<<<<<< - * if ((tail - head) + _buf_len) <= buf_size: - * # move to front. - */ - } - - /* "msgpack/_unpacker.pyx":461 - * head = 0 - * - * memcpy(buf + tail, (_buf), _buf_len) # <<<<<<<<<<<<<< - * self.buf = buf - * self.buf_head = head - */ - (void)(memcpy((__pyx_v_buf + __pyx_v_tail), ((char *)__pyx_v__buf), __pyx_v__buf_len)); - - /* "msgpack/_unpacker.pyx":462 - * - * memcpy(buf + tail, (_buf), _buf_len) - * self.buf = buf # <<<<<<<<<<<<<< - * self.buf_head = head - * self.buf_size = buf_size - */ - __pyx_v_self->buf = __pyx_v_buf; - - /* "msgpack/_unpacker.pyx":463 - * memcpy(buf + tail, (_buf), _buf_len) - * self.buf = buf - * self.buf_head = head # <<<<<<<<<<<<<< - * self.buf_size = buf_size - * self.buf_tail = tail + _buf_len - */ - __pyx_v_self->buf_head = __pyx_v_head; - - /* "msgpack/_unpacker.pyx":464 - * self.buf = buf - * self.buf_head = head - * self.buf_size = buf_size # <<<<<<<<<<<<<< - * self.buf_tail = tail + _buf_len - * - */ - __pyx_v_self->buf_size = __pyx_v_buf_size; - - /* "msgpack/_unpacker.pyx":465 - * self.buf_head = head - * self.buf_size = buf_size - * self.buf_tail = tail + _buf_len # <<<<<<<<<<<<<< - * - * cdef read_from_file(self): - */ - __pyx_v_self->buf_tail = (__pyx_v_tail + __pyx_v__buf_len); - - /* "msgpack/_unpacker.pyx":427 - * PyBuffer_Release(&pybuff) - * - * cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): # <<<<<<<<<<<<<< - * cdef: - * char* buf = self.buf - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.append_buffer", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":467 - * self.buf_tail = tail + _buf_len - * - * cdef read_from_file(self): # <<<<<<<<<<<<<< - * next_bytes = self.file_like_read( - * min(self.read_size, - */ - -static PyObject *__pyx_f_7msgpack_9_cmsgpack_8Unpacker_read_from_file(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_v_next_bytes = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - Py_ssize_t __pyx_t_2; - Py_ssize_t __pyx_t_3; - Py_ssize_t __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - int __pyx_t_8; - char *__pyx_t_9; - __Pyx_RefNannySetupContext("read_from_file", 0); - - /* "msgpack/_unpacker.pyx":470 - * next_bytes = self.file_like_read( - * min(self.read_size, - * self.max_buffer_size - (self.buf_tail - self.buf_head) # <<<<<<<<<<<<<< - * )) - * if next_bytes: - */ - __pyx_t_2 = (__pyx_v_self->max_buffer_size - (__pyx_v_self->buf_tail - __pyx_v_self->buf_head)); - - /* "msgpack/_unpacker.pyx":469 - * cdef read_from_file(self): - * next_bytes = self.file_like_read( - * min(self.read_size, # <<<<<<<<<<<<<< - * self.max_buffer_size - (self.buf_tail - self.buf_head) - * )) - */ - __pyx_t_3 = __pyx_v_self->read_size; - - /* "msgpack/_unpacker.pyx":470 - * next_bytes = self.file_like_read( - * min(self.read_size, - * self.max_buffer_size - (self.buf_tail - self.buf_head) # <<<<<<<<<<<<<< - * )) - * if next_bytes: - */ - if (((__pyx_t_2 < __pyx_t_3) != 0)) { - __pyx_t_4 = __pyx_t_2; - } else { - __pyx_t_4 = __pyx_t_3; - } - __pyx_t_5 = PyInt_FromSsize_t(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 470, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_INCREF(__pyx_v_self->file_like_read); - __pyx_t_6 = __pyx_v_self->file_like_read; __pyx_t_7 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_7)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_7); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_6, function); - } - } - __pyx_t_1 = (__pyx_t_7) ? __Pyx_PyObject_Call2Args(__pyx_t_6, __pyx_t_7, __pyx_t_5) : __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_5); - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 468, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_v_next_bytes = __pyx_t_1; - __pyx_t_1 = 0; - - /* "msgpack/_unpacker.pyx":472 - * self.max_buffer_size - (self.buf_tail - self.buf_head) - * )) - * if next_bytes: # <<<<<<<<<<<<<< - * self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) - * else: - */ - __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_v_next_bytes); if (unlikely(__pyx_t_8 < 0)) __PYX_ERR(1, 472, __pyx_L1_error) - if (__pyx_t_8) { - - /* "msgpack/_unpacker.pyx":473 - * )) - * if next_bytes: - * self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) # <<<<<<<<<<<<<< - * else: - * self.file_like = None - */ - __pyx_t_9 = PyBytes_AsString(__pyx_v_next_bytes); if (unlikely(__pyx_t_9 == ((char *)NULL))) __PYX_ERR(1, 473, __pyx_L1_error) - __pyx_t_4 = PyBytes_Size(__pyx_v_next_bytes); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1L))) __PYX_ERR(1, 473, __pyx_L1_error) - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->append_buffer(__pyx_v_self, __pyx_t_9, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 473, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "msgpack/_unpacker.pyx":472 - * self.max_buffer_size - (self.buf_tail - self.buf_head) - * )) - * if next_bytes: # <<<<<<<<<<<<<< - * self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) - * else: - */ - goto __pyx_L3; - } - - /* "msgpack/_unpacker.pyx":475 - * self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) - * else: - * self.file_like = None # <<<<<<<<<<<<<< - * - * cdef object _unpack(self, execute_fn execute, bint iter=0): - */ - /*else*/ { - __Pyx_INCREF(Py_None); - __Pyx_GIVEREF(Py_None); - __Pyx_GOTREF(__pyx_v_self->file_like); - __Pyx_DECREF(__pyx_v_self->file_like); - __pyx_v_self->file_like = Py_None; - } - __pyx_L3:; - - /* "msgpack/_unpacker.pyx":467 - * self.buf_tail = tail + _buf_len - * - * cdef read_from_file(self): # <<<<<<<<<<<<<< - * next_bytes = self.file_like_read( - * min(self.read_size, - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.read_from_file", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_next_bytes); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":477 - * self.file_like = None - * - * cdef object _unpack(self, execute_fn execute, bint iter=0): # <<<<<<<<<<<<<< - * cdef int ret - * cdef object obj - */ - -static PyObject *__pyx_f_7msgpack_9_cmsgpack_8Unpacker__unpack(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, execute_fn __pyx_v_execute, struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack *__pyx_optional_args) { - int __pyx_v_iter = ((int)0); - int __pyx_v_ret; - PyObject *__pyx_v_obj = 0; - Py_ssize_t __pyx_v_prev_head; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - int __pyx_t_8; - __Pyx_RefNannySetupContext("_unpack", 0); - if (__pyx_optional_args) { - if (__pyx_optional_args->__pyx_n > 0) { - __pyx_v_iter = __pyx_optional_args->iter; - } - } - - /* "msgpack/_unpacker.pyx":482 - * cdef Py_ssize_t prev_head - * - * if self.buf_head >= self.buf_tail and self.file_like is not None: # <<<<<<<<<<<<<< - * self.read_from_file() - * - */ - __pyx_t_2 = ((__pyx_v_self->buf_head >= __pyx_v_self->buf_tail) != 0); - if (__pyx_t_2) { - } else { - __pyx_t_1 = __pyx_t_2; - goto __pyx_L4_bool_binop_done; - } - __pyx_t_2 = (__pyx_v_self->file_like != Py_None); - __pyx_t_3 = (__pyx_t_2 != 0); - __pyx_t_1 = __pyx_t_3; - __pyx_L4_bool_binop_done:; - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":483 - * - * if self.buf_head >= self.buf_tail and self.file_like is not None: - * self.read_from_file() # <<<<<<<<<<<<<< - * - * while 1: - */ - __pyx_t_4 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->read_from_file(__pyx_v_self); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 483, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "msgpack/_unpacker.pyx":482 - * cdef Py_ssize_t prev_head - * - * if self.buf_head >= self.buf_tail and self.file_like is not None: # <<<<<<<<<<<<<< - * self.read_from_file() - * - */ - } - - /* "msgpack/_unpacker.pyx":485 - * self.read_from_file() - * - * while 1: # <<<<<<<<<<<<<< - * prev_head = self.buf_head - * if prev_head >= self.buf_tail: - */ - while (1) { - - /* "msgpack/_unpacker.pyx":486 - * - * while 1: - * prev_head = self.buf_head # <<<<<<<<<<<<<< - * if prev_head >= self.buf_tail: - * if iter: - */ - __pyx_t_5 = __pyx_v_self->buf_head; - __pyx_v_prev_head = __pyx_t_5; - - /* "msgpack/_unpacker.pyx":487 - * while 1: - * prev_head = self.buf_head - * if prev_head >= self.buf_tail: # <<<<<<<<<<<<<< - * if iter: - * raise StopIteration("No more data to unpack.") - */ - __pyx_t_1 = ((__pyx_v_prev_head >= __pyx_v_self->buf_tail) != 0); - if (__pyx_t_1) { - - /* "msgpack/_unpacker.pyx":488 - * prev_head = self.buf_head - * if prev_head >= self.buf_tail: - * if iter: # <<<<<<<<<<<<<< - * raise StopIteration("No more data to unpack.") - * else: - */ - __pyx_t_1 = (__pyx_v_iter != 0); - if (unlikely(__pyx_t_1)) { - - /* "msgpack/_unpacker.pyx":489 - * if prev_head >= self.buf_tail: - * if iter: - * raise StopIteration("No more data to unpack.") # <<<<<<<<<<<<<< - * else: - * raise OutOfData("No more data to unpack.") - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_StopIteration, __pyx_tuple__29, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 489, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 489, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":488 - * prev_head = self.buf_head - * if prev_head >= self.buf_tail: - * if iter: # <<<<<<<<<<<<<< - * raise StopIteration("No more data to unpack.") - * else: - */ - } - - /* "msgpack/_unpacker.pyx":491 - * raise StopIteration("No more data to unpack.") - * else: - * raise OutOfData("No more data to unpack.") # <<<<<<<<<<<<<< - * - * ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - */ - /*else*/ { - __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_OutOfData); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 491, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_7)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_7); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_6, function); - } - } - __pyx_t_4 = (__pyx_t_7) ? __Pyx_PyObject_Call2Args(__pyx_t_6, __pyx_t_7, __pyx_kp_u_No_more_data_to_unpack) : __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_kp_u_No_more_data_to_unpack); - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 491, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 491, __pyx_L1_error) - } - - /* "msgpack/_unpacker.pyx":487 - * while 1: - * prev_head = self.buf_head - * if prev_head >= self.buf_tail: # <<<<<<<<<<<<<< - * if iter: - * raise StopIteration("No more data to unpack.") - */ - } - - /* "msgpack/_unpacker.pyx":493 - * raise OutOfData("No more data to unpack.") - * - * ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) # <<<<<<<<<<<<<< - * self.stream_offset += self.buf_head - prev_head - * - */ - __pyx_t_8 = __pyx_v_execute((&__pyx_v_self->ctx), __pyx_v_self->buf, __pyx_v_self->buf_tail, (&__pyx_v_self->buf_head)); if (unlikely(__pyx_t_8 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 493, __pyx_L1_error) - __pyx_v_ret = __pyx_t_8; - - /* "msgpack/_unpacker.pyx":494 - * - * ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - * self.stream_offset += self.buf_head - prev_head # <<<<<<<<<<<<<< - * - * if ret == 1: - */ - __pyx_v_self->stream_offset = (__pyx_v_self->stream_offset + (__pyx_v_self->buf_head - __pyx_v_prev_head)); - - /* "msgpack/_unpacker.pyx":496 - * self.stream_offset += self.buf_head - prev_head - * - * if ret == 1: # <<<<<<<<<<<<<< - * obj = unpack_data(&self.ctx) - * unpack_init(&self.ctx) - */ - switch (__pyx_v_ret) { - case 1: - - /* "msgpack/_unpacker.pyx":497 - * - * if ret == 1: - * obj = unpack_data(&self.ctx) # <<<<<<<<<<<<<< - * unpack_init(&self.ctx) - * return obj - */ - __pyx_t_4 = unpack_data((&__pyx_v_self->ctx)); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 497, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_v_obj = __pyx_t_4; - __pyx_t_4 = 0; - - /* "msgpack/_unpacker.pyx":498 - * if ret == 1: - * obj = unpack_data(&self.ctx) - * unpack_init(&self.ctx) # <<<<<<<<<<<<<< - * return obj - * elif ret == 0: - */ - unpack_init((&__pyx_v_self->ctx)); - - /* "msgpack/_unpacker.pyx":499 - * obj = unpack_data(&self.ctx) - * unpack_init(&self.ctx) - * return obj # <<<<<<<<<<<<<< - * elif ret == 0: - * if self.file_like is not None: - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_obj); - __pyx_r = __pyx_v_obj; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":496 - * self.stream_offset += self.buf_head - prev_head - * - * if ret == 1: # <<<<<<<<<<<<<< - * obj = unpack_data(&self.ctx) - * unpack_init(&self.ctx) - */ - break; - case 0: - - /* "msgpack/_unpacker.pyx":501 - * return obj - * elif ret == 0: - * if self.file_like is not None: # <<<<<<<<<<<<<< - * self.read_from_file() - * continue - */ - __pyx_t_1 = (__pyx_v_self->file_like != Py_None); - __pyx_t_3 = (__pyx_t_1 != 0); - if (__pyx_t_3) { - - /* "msgpack/_unpacker.pyx":502 - * elif ret == 0: - * if self.file_like is not None: - * self.read_from_file() # <<<<<<<<<<<<<< - * continue - * if iter: - */ - __pyx_t_4 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->read_from_file(__pyx_v_self); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 502, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "msgpack/_unpacker.pyx":503 - * if self.file_like is not None: - * self.read_from_file() - * continue # <<<<<<<<<<<<<< - * if iter: - * raise StopIteration("No more data to unpack.") - */ - goto __pyx_L6_continue; - - /* "msgpack/_unpacker.pyx":501 - * return obj - * elif ret == 0: - * if self.file_like is not None: # <<<<<<<<<<<<<< - * self.read_from_file() - * continue - */ - } - - /* "msgpack/_unpacker.pyx":504 - * self.read_from_file() - * continue - * if iter: # <<<<<<<<<<<<<< - * raise StopIteration("No more data to unpack.") - * else: - */ - __pyx_t_3 = (__pyx_v_iter != 0); - if (unlikely(__pyx_t_3)) { - - /* "msgpack/_unpacker.pyx":505 - * continue - * if iter: - * raise StopIteration("No more data to unpack.") # <<<<<<<<<<<<<< - * else: - * raise OutOfData("No more data to unpack.") - */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_StopIteration, __pyx_tuple__29, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 505, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 505, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":504 - * self.read_from_file() - * continue - * if iter: # <<<<<<<<<<<<<< - * raise StopIteration("No more data to unpack.") - * else: - */ - } - - /* "msgpack/_unpacker.pyx":507 - * raise StopIteration("No more data to unpack.") - * else: - * raise OutOfData("No more data to unpack.") # <<<<<<<<<<<<<< - * elif ret == -2: - * raise FormatError - */ - /*else*/ { - __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_OutOfData); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 507, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_7)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_7); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_6, function); - } - } - __pyx_t_4 = (__pyx_t_7) ? __Pyx_PyObject_Call2Args(__pyx_t_6, __pyx_t_7, __pyx_kp_u_No_more_data_to_unpack) : __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_kp_u_No_more_data_to_unpack); - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 507, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 507, __pyx_L1_error) - } - - /* "msgpack/_unpacker.pyx":500 - * unpack_init(&self.ctx) - * return obj - * elif ret == 0: # <<<<<<<<<<<<<< - * if self.file_like is not None: - * self.read_from_file() - */ - break; - case -2L: - - /* "msgpack/_unpacker.pyx":509 - * raise OutOfData("No more data to unpack.") - * elif ret == -2: - * raise FormatError # <<<<<<<<<<<<<< - * elif ret == -3: - * raise StackError - */ - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_FormatError); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 509, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 509, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":508 - * else: - * raise OutOfData("No more data to unpack.") - * elif ret == -2: # <<<<<<<<<<<<<< - * raise FormatError - * elif ret == -3: - */ - break; - case -3L: - - /* "msgpack/_unpacker.pyx":511 - * raise FormatError - * elif ret == -3: - * raise StackError # <<<<<<<<<<<<<< - * else: - * raise ValueError("Unpack failed: error = %d" % (ret,)) - */ - __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_n_s_StackError); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 511, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 511, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":510 - * elif ret == -2: - * raise FormatError - * elif ret == -3: # <<<<<<<<<<<<<< - * raise StackError - * else: - */ - break; - default: - - /* "msgpack/_unpacker.pyx":513 - * raise StackError - * else: - * raise ValueError("Unpack failed: error = %d" % (ret,)) # <<<<<<<<<<<<<< - * - * def read_bytes(self, Py_ssize_t nbytes): - */ - __pyx_t_4 = __Pyx_PyUnicode_From_int(__pyx_v_ret, 0, ' ', 'd'); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 513, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyUnicode_Concat(__pyx_kp_u_Unpack_failed_error, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 513, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 513, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 513, __pyx_L1_error) - break; - } - __pyx_L6_continue:; - } - - /* "msgpack/_unpacker.pyx":477 - * self.file_like = None - * - * cdef object _unpack(self, execute_fn execute, bint iter=0): # <<<<<<<<<<<<<< - * cdef int ret - * cdef object obj - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker._unpack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_obj); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":515 - * raise ValueError("Unpack failed: error = %d" % (ret,)) - * - * def read_bytes(self, Py_ssize_t nbytes): # <<<<<<<<<<<<<< - * """Read a specified number of raw bytes from the stream""" - * cdef Py_ssize_t nread - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_9read_bytes(PyObject *__pyx_v_self, PyObject *__pyx_arg_nbytes); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_8read_bytes[] = "Unpacker.read_bytes(self, Py_ssize_t nbytes)\nRead a specified number of raw bytes from the stream"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_9read_bytes(PyObject *__pyx_v_self, PyObject *__pyx_arg_nbytes) { - Py_ssize_t __pyx_v_nbytes; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("read_bytes (wrapper)", 0); - assert(__pyx_arg_nbytes); { - __pyx_v_nbytes = __Pyx_PyIndex_AsSsize_t(__pyx_arg_nbytes); if (unlikely((__pyx_v_nbytes == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 515, __pyx_L3_error) - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.read_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_8read_bytes(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self), ((Py_ssize_t)__pyx_v_nbytes)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_8read_bytes(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, Py_ssize_t __pyx_v_nbytes) { - Py_ssize_t __pyx_v_nread; - PyObject *__pyx_v_ret = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - Py_ssize_t __pyx_t_1; - Py_ssize_t __pyx_t_2; - Py_ssize_t __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - int __pyx_t_5; - int __pyx_t_6; - int __pyx_t_7; - PyObject *__pyx_t_8 = NULL; - PyObject *__pyx_t_9 = NULL; - PyObject *__pyx_t_10 = NULL; - __Pyx_RefNannySetupContext("read_bytes", 0); - - /* "msgpack/_unpacker.pyx":518 - * """Read a specified number of raw bytes from the stream""" - * cdef Py_ssize_t nread - * nread = min(self.buf_tail - self.buf_head, nbytes) # <<<<<<<<<<<<<< - * ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - * self.buf_head += nread - */ - __pyx_t_1 = __pyx_v_nbytes; - __pyx_t_2 = (__pyx_v_self->buf_tail - __pyx_v_self->buf_head); - if (((__pyx_t_1 < __pyx_t_2) != 0)) { - __pyx_t_3 = __pyx_t_1; - } else { - __pyx_t_3 = __pyx_t_2; - } - __pyx_v_nread = __pyx_t_3; - - /* "msgpack/_unpacker.pyx":519 - * cdef Py_ssize_t nread - * nread = min(self.buf_tail - self.buf_head, nbytes) - * ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) # <<<<<<<<<<<<<< - * self.buf_head += nread - * if len(ret) < nbytes and self.file_like is not None: - */ - __pyx_t_4 = PyBytes_FromStringAndSize((__pyx_v_self->buf + __pyx_v_self->buf_head), __pyx_v_nread); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 519, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_v_ret = __pyx_t_4; - __pyx_t_4 = 0; - - /* "msgpack/_unpacker.pyx":520 - * nread = min(self.buf_tail - self.buf_head, nbytes) - * ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - * self.buf_head += nread # <<<<<<<<<<<<<< - * if len(ret) < nbytes and self.file_like is not None: - * ret += self.file_like.read(nbytes - len(ret)) - */ - __pyx_v_self->buf_head = (__pyx_v_self->buf_head + __pyx_v_nread); - - /* "msgpack/_unpacker.pyx":521 - * ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - * self.buf_head += nread - * if len(ret) < nbytes and self.file_like is not None: # <<<<<<<<<<<<<< - * ret += self.file_like.read(nbytes - len(ret)) - * return ret - */ - __pyx_t_3 = PyObject_Length(__pyx_v_ret); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1))) __PYX_ERR(1, 521, __pyx_L1_error) - __pyx_t_6 = ((__pyx_t_3 < __pyx_v_nbytes) != 0); - if (__pyx_t_6) { - } else { - __pyx_t_5 = __pyx_t_6; - goto __pyx_L4_bool_binop_done; - } - __pyx_t_6 = (__pyx_v_self->file_like != Py_None); - __pyx_t_7 = (__pyx_t_6 != 0); - __pyx_t_5 = __pyx_t_7; - __pyx_L4_bool_binop_done:; - if (__pyx_t_5) { - - /* "msgpack/_unpacker.pyx":522 - * self.buf_head += nread - * if len(ret) < nbytes and self.file_like is not None: - * ret += self.file_like.read(nbytes - len(ret)) # <<<<<<<<<<<<<< - * return ret - * - */ - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_self->file_like, __pyx_n_s_read); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 522, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_3 = PyObject_Length(__pyx_v_ret); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1))) __PYX_ERR(1, 522, __pyx_L1_error) - __pyx_t_9 = PyInt_FromSsize_t((__pyx_v_nbytes - __pyx_t_3)); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 522, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_9); - __pyx_t_10 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_8); - if (likely(__pyx_t_10)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); - __Pyx_INCREF(__pyx_t_10); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_8, function); - } - } - __pyx_t_4 = (__pyx_t_10) ? __Pyx_PyObject_Call2Args(__pyx_t_8, __pyx_t_10, __pyx_t_9) : __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_t_9); - __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 522, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyNumber_InPlaceAdd(__pyx_v_ret, __pyx_t_4); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 522, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF_SET(__pyx_v_ret, __pyx_t_8); - __pyx_t_8 = 0; - - /* "msgpack/_unpacker.pyx":521 - * ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - * self.buf_head += nread - * if len(ret) < nbytes and self.file_like is not None: # <<<<<<<<<<<<<< - * ret += self.file_like.read(nbytes - len(ret)) - * return ret - */ - } - - /* "msgpack/_unpacker.pyx":523 - * if len(ret) < nbytes and self.file_like is not None: - * ret += self.file_like.read(nbytes - len(ret)) - * return ret # <<<<<<<<<<<<<< - * - * def unpack(self): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_ret); - __pyx_r = __pyx_v_ret; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":515 - * raise ValueError("Unpack failed: error = %d" % (ret,)) - * - * def read_bytes(self, Py_ssize_t nbytes): # <<<<<<<<<<<<<< - * """Read a specified number of raw bytes from the stream""" - * cdef Py_ssize_t nread - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_XDECREF(__pyx_t_9); - __Pyx_XDECREF(__pyx_t_10); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.read_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_ret); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":525 - * return ret - * - * def unpack(self): # <<<<<<<<<<<<<< - * """Unpack one object - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_11unpack(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_10unpack[] = "Unpacker.unpack(self)\nUnpack one object\n\n Raises `OutOfData` when there are no more bytes to unpack.\n "; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_11unpack(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("unpack (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_10unpack(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_10unpack(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("unpack", 0); - - /* "msgpack/_unpacker.pyx":530 - * Raises `OutOfData` when there are no more bytes to unpack. - * """ - * return self._unpack(unpack_construct) # <<<<<<<<<<<<<< - * - * def skip(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->_unpack(__pyx_v_self, unpack_construct, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 530, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":525 - * return ret - * - * def unpack(self): # <<<<<<<<<<<<<< - * """Unpack one object - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.unpack", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":532 - * return self._unpack(unpack_construct) - * - * def skip(self): # <<<<<<<<<<<<<< - * """Read and ignore one object, returning None - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_13skip(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_12skip[] = "Unpacker.skip(self)\nRead and ignore one object, returning None\n\n Raises `OutOfData` when there are no more bytes to unpack.\n "; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_13skip(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("skip (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_12skip(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_12skip(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("skip", 0); - - /* "msgpack/_unpacker.pyx":537 - * Raises `OutOfData` when there are no more bytes to unpack. - * """ - * return self._unpack(unpack_skip) # <<<<<<<<<<<<<< - * - * def read_array_header(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->_unpack(__pyx_v_self, unpack_skip, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 537, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":532 - * return self._unpack(unpack_construct) - * - * def skip(self): # <<<<<<<<<<<<<< - * """Read and ignore one object, returning None - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.skip", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":539 - * return self._unpack(unpack_skip) - * - * def read_array_header(self): # <<<<<<<<<<<<<< - * """assuming the next object is an array, return its size n, such that - * the next n unpack() calls will iterate over its contents. - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_15read_array_header(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_14read_array_header[] = "Unpacker.read_array_header(self)\nassuming the next object is an array, return its size n, such that\n the next n unpack() calls will iterate over its contents.\n\n Raises `OutOfData` when there are no more bytes to unpack.\n "; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_15read_array_header(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("read_array_header (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_14read_array_header(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_14read_array_header(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("read_array_header", 0); - - /* "msgpack/_unpacker.pyx":545 - * Raises `OutOfData` when there are no more bytes to unpack. - * """ - * return self._unpack(read_array_header) # <<<<<<<<<<<<<< - * - * def read_map_header(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->_unpack(__pyx_v_self, read_array_header, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 545, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":539 - * return self._unpack(unpack_skip) - * - * def read_array_header(self): # <<<<<<<<<<<<<< - * """assuming the next object is an array, return its size n, such that - * the next n unpack() calls will iterate over its contents. - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.read_array_header", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":547 - * return self._unpack(read_array_header) - * - * def read_map_header(self): # <<<<<<<<<<<<<< - * """assuming the next object is a map, return its size n, such that the - * next n * 2 unpack() calls will iterate over its key-value pairs. - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_17read_map_header(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_16read_map_header[] = "Unpacker.read_map_header(self)\nassuming the next object is a map, return its size n, such that the\n next n * 2 unpack() calls will iterate over its key-value pairs.\n\n Raises `OutOfData` when there are no more bytes to unpack.\n "; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_17read_map_header(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("read_map_header (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_16read_map_header(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_16read_map_header(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("read_map_header", 0); - - /* "msgpack/_unpacker.pyx":553 - * Raises `OutOfData` when there are no more bytes to unpack. - * """ - * return self._unpack(read_map_header) # <<<<<<<<<<<<<< - * - * def tell(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->_unpack(__pyx_v_self, read_map_header, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 553, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":547 - * return self._unpack(read_array_header) - * - * def read_map_header(self): # <<<<<<<<<<<<<< - * """assuming the next object is a map, return its size n, such that the - * next n * 2 unpack() calls will iterate over its key-value pairs. - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.read_map_header", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":555 - * return self._unpack(read_map_header) - * - * def tell(self): # <<<<<<<<<<<<<< - * return self.stream_offset - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_19tell(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_18tell[] = "Unpacker.tell(self)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_19tell(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tell (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_18tell(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_18tell(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("tell", 0); - - /* "msgpack/_unpacker.pyx":556 - * - * def tell(self): - * return self.stream_offset # <<<<<<<<<<<<<< - * - * def __iter__(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_unsigned_PY_LONG_LONG(__pyx_v_self->stream_offset); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 556, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":555 - * return self._unpack(read_map_header) - * - * def tell(self): # <<<<<<<<<<<<<< - * return self.stream_offset - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.tell", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":558 - * return self.stream_offset - * - * def __iter__(self): # <<<<<<<<<<<<<< - * return self - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_21__iter__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_21__iter__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_20__iter__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_20__iter__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__iter__", 0); - - /* "msgpack/_unpacker.pyx":559 - * - * def __iter__(self): - * return self # <<<<<<<<<<<<<< - * - * def __next__(self): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(((PyObject *)__pyx_v_self)); - __pyx_r = ((PyObject *)__pyx_v_self); - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":558 - * return self.stream_offset - * - * def __iter__(self): # <<<<<<<<<<<<<< - * return self - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "msgpack/_unpacker.pyx":561 - * return self - * - * def __next__(self): # <<<<<<<<<<<<<< - * return self._unpack(unpack_construct, 1) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_23__next__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_23__next__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__next__ (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_22__next__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_22__next__(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack __pyx_t_2; - __Pyx_RefNannySetupContext("__next__", 0); - - /* "msgpack/_unpacker.pyx":562 - * - * def __next__(self): - * return self._unpack(unpack_construct, 1) # <<<<<<<<<<<<<< - * - * # for debug. - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_2.__pyx_n = 1; - __pyx_t_2.iter = 1; - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self->__pyx_vtab)->_unpack(__pyx_v_self, unpack_construct, &__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 562, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "msgpack/_unpacker.pyx":561 - * return self - * - * def __next__(self): # <<<<<<<<<<<<<< - * return self._unpack(unpack_construct, 1) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.__next__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "(tree fragment)":1 - * def __reduce_cython__(self): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_25__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_24__reduce_cython__[] = "Unpacker.__reduce_cython__(self)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_25__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_24__reduce_cython__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_24__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__reduce_cython__", 0); - - /* "(tree fragment)":2 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__30, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_Raise(__pyx_t_1, 0, 0, 0); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 2, __pyx_L1_error) - - /* "(tree fragment)":1 - * def __reduce_cython__(self): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "(tree fragment)":3 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_27__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ -static char __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_26__setstate_cython__[] = "Unpacker.__setstate_cython__(self, __pyx_state)"; -static PyObject *__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_27__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); - __pyx_r = __pyx_pf_7msgpack_9_cmsgpack_8Unpacker_26__setstate_cython__(((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_7msgpack_9_cmsgpack_8Unpacker_26__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__setstate_cython__", 0); - - /* "(tree fragment)":4 - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - */ - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__31, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_Raise(__pyx_t_1, 0, 0, 0); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 4, __pyx_L1_error) - - /* "(tree fragment)":3 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("msgpack._cmsgpack.Unpacker.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Packer __pyx_vtable_7msgpack_9_cmsgpack_Packer; - -static PyObject *__pyx_tp_new_7msgpack_9_cmsgpack_Packer(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_7msgpack_9_cmsgpack_Packer *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)o); - p->__pyx_vtab = __pyx_vtabptr_7msgpack_9_cmsgpack_Packer; - p->_default = Py_None; Py_INCREF(Py_None); - p->_bencoding = Py_None; Py_INCREF(Py_None); - p->_berrors = Py_None; Py_INCREF(Py_None); - p->use_float = ((PyBoolObject *)Py_None); Py_INCREF(Py_None); - if (unlikely(__pyx_pw_7msgpack_9_cmsgpack_6Packer_1__cinit__(o, __pyx_empty_tuple, NULL) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_7msgpack_9_cmsgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_9_cmsgpack_Packer *p = (struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)o; - #if CYTHON_USE_TP_FINALIZE - if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && !_PyGC_FINALIZED(o)) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - PyObject_GC_UnTrack(o); - { - PyObject *etype, *eval, *etb; - PyErr_Fetch(&etype, &eval, &etb); - ++Py_REFCNT(o); - __pyx_pw_7msgpack_9_cmsgpack_6Packer_5__dealloc__(o); - --Py_REFCNT(o); - PyErr_Restore(etype, eval, etb); - } - Py_CLEAR(p->_default); - Py_CLEAR(p->_bencoding); - Py_CLEAR(p->_berrors); - Py_CLEAR(p->use_float); - (*Py_TYPE(o)->tp_free)(o); -} - -static int __pyx_tp_traverse_7msgpack_9_cmsgpack_Packer(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_7msgpack_9_cmsgpack_Packer *p = (struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)o; - if (p->_default) { - e = (*v)(p->_default, a); if (e) return e; - } - if (p->_bencoding) { - e = (*v)(p->_bencoding, a); if (e) return e; - } - if (p->_berrors) { - e = (*v)(p->_berrors, a); if (e) return e; - } - if (p->use_float) { - e = (*v)(((PyObject *)p->use_float), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_7msgpack_9_cmsgpack_Packer(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_7msgpack_9_cmsgpack_Packer *p = (struct __pyx_obj_7msgpack_9_cmsgpack_Packer *)o; - tmp = ((PyObject*)p->_default); - p->_default = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->_bencoding); - p->_bencoding = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->_berrors); - p->_berrors = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->use_float); - p->use_float = ((PyBoolObject *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyMethodDef __pyx_methods_7msgpack_9_cmsgpack_Packer[] = { - {"pack", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_7pack, METH_O, __pyx_doc_7msgpack_9_cmsgpack_6Packer_6pack}, - {"pack_ext_type", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_7msgpack_9_cmsgpack_6Packer_9pack_ext_type, METH_VARARGS|METH_KEYWORDS, __pyx_doc_7msgpack_9_cmsgpack_6Packer_8pack_ext_type}, - {"pack_array_header", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_11pack_array_header, METH_O, __pyx_doc_7msgpack_9_cmsgpack_6Packer_10pack_array_header}, - {"pack_map_header", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_13pack_map_header, METH_O, __pyx_doc_7msgpack_9_cmsgpack_6Packer_12pack_map_header}, - {"pack_map_pairs", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_15pack_map_pairs, METH_O, __pyx_doc_7msgpack_9_cmsgpack_6Packer_14pack_map_pairs}, - {"reset", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_17reset, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_6Packer_16reset}, - {"bytes", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_19bytes, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_6Packer_18bytes}, - {"getbuffer", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_21getbuffer, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_6Packer_20getbuffer}, - {"__reduce_cython__", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_23__reduce_cython__, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_6Packer_22__reduce_cython__}, - {"__setstate_cython__", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_6Packer_25__setstate_cython__, METH_O, __pyx_doc_7msgpack_9_cmsgpack_6Packer_24__setstate_cython__}, - {0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_7msgpack_9_cmsgpack_Packer = { - PyVarObject_HEAD_INIT(0, 0) - "msgpack._cmsgpack.Packer", /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_9_cmsgpack_Packer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_9_cmsgpack_Packer, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - "Packer(default=None, encoding=None, unicode_errors=None, bool use_single_float=False, bool autoreset=True, bool use_bin_type=False, bool strict_types=False)\n\n MessagePack Packer\n\n usage::\n\n packer = Packer()\n astream.write(packer.pack(a))\n astream.write(packer.pack(b))\n\n Packer's constructor has some keyword arguments:\n\n :param callable default:\n Convert user type to builtin type that Packer supports.\n See also simplejson's document.\n\n :param bool use_single_float:\n Use single precision float type for float. (default: False)\n\n :param bool autoreset:\n Reset buffer after each pack and return its content as `bytes`. (default: True).\n If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.\n\n :param bool use_bin_type:\n Use bin type introduced in msgpack spec 2.0 for bytes.\n It also enables str8 type for unicode.\n Current default value is false, but it will be changed to true\n in future version. You should specify it explicitly.\n\n :param bool strict_types:\n If set to true, types will be checked to be exact. Derived classes\n from serializeable types will not be serialized and will be\n treated as unsupported type and forwarded to default.\n Additionally tuples will not be serialized as lists.\n This is useful when trying to implement accurate serialization\n for python types.\n\n :param str unicode_errors:\n Error handler for encoding unicode. (default: 'strict')\n\n :param str encoding:\n (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')\n ", /*tp_doc*/ - __pyx_tp_traverse_7msgpack_9_cmsgpack_Packer, /*tp_traverse*/ - __pyx_tp_clear_7msgpack_9_cmsgpack_Packer, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_9_cmsgpack_Packer, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pw_7msgpack_9_cmsgpack_6Packer_3__init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_9_cmsgpack_Packer, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; -static struct __pyx_vtabstruct_7msgpack_9_cmsgpack_Unpacker __pyx_vtable_7msgpack_9_cmsgpack_Unpacker; - -static PyObject *__pyx_tp_new_7msgpack_9_cmsgpack_Unpacker(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)o); - p->__pyx_vtab = __pyx_vtabptr_7msgpack_9_cmsgpack_Unpacker; - p->file_like = Py_None; Py_INCREF(Py_None); - p->file_like_read = Py_None; Py_INCREF(Py_None); - p->object_hook = Py_None; Py_INCREF(Py_None); - p->object_pairs_hook = Py_None; Py_INCREF(Py_None); - p->list_hook = Py_None; Py_INCREF(Py_None); - p->ext_hook = Py_None; Py_INCREF(Py_None); - p->encoding = Py_None; Py_INCREF(Py_None); - p->unicode_errors = Py_None; Py_INCREF(Py_None); - if (unlikely(__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_1__cinit__(o, __pyx_empty_tuple, NULL) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_7msgpack_9_cmsgpack_Unpacker(PyObject *o) { - struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *p = (struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)o; - #if CYTHON_USE_TP_FINALIZE - if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && !_PyGC_FINALIZED(o)) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - PyObject_GC_UnTrack(o); - { - PyObject *etype, *eval, *etb; - PyErr_Fetch(&etype, &eval, &etb); - ++Py_REFCNT(o); - __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_3__dealloc__(o); - --Py_REFCNT(o); - PyErr_Restore(etype, eval, etb); - } - Py_CLEAR(p->file_like); - Py_CLEAR(p->file_like_read); - Py_CLEAR(p->object_hook); - Py_CLEAR(p->object_pairs_hook); - Py_CLEAR(p->list_hook); - Py_CLEAR(p->ext_hook); - Py_CLEAR(p->encoding); - Py_CLEAR(p->unicode_errors); - (*Py_TYPE(o)->tp_free)(o); -} - -static int __pyx_tp_traverse_7msgpack_9_cmsgpack_Unpacker(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *p = (struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)o; - if (p->file_like) { - e = (*v)(p->file_like, a); if (e) return e; - } - if (p->file_like_read) { - e = (*v)(p->file_like_read, a); if (e) return e; - } - if (p->object_hook) { - e = (*v)(p->object_hook, a); if (e) return e; - } - if (p->object_pairs_hook) { - e = (*v)(p->object_pairs_hook, a); if (e) return e; - } - if (p->list_hook) { - e = (*v)(p->list_hook, a); if (e) return e; - } - if (p->ext_hook) { - e = (*v)(p->ext_hook, a); if (e) return e; - } - if (p->encoding) { - e = (*v)(p->encoding, a); if (e) return e; - } - if (p->unicode_errors) { - e = (*v)(p->unicode_errors, a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_7msgpack_9_cmsgpack_Unpacker(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *p = (struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *)o; - tmp = ((PyObject*)p->file_like); - p->file_like = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->file_like_read); - p->file_like_read = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->object_hook); - p->object_hook = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->object_pairs_hook); - p->object_pairs_hook = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->list_hook); - p->list_hook = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->ext_hook); - p->ext_hook = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->encoding); - p->encoding = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->unicode_errors); - p->unicode_errors = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyObject *__pyx_specialmethod___pyx_pw_7msgpack_9_cmsgpack_8Unpacker_23__next__(PyObject *self, CYTHON_UNUSED PyObject *arg) {return __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_23__next__(self);} - -static PyMethodDef __pyx_methods_7msgpack_9_cmsgpack_Unpacker[] = { - {"feed", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_7feed, METH_O, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_6feed}, - {"read_bytes", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_9read_bytes, METH_O, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_8read_bytes}, - {"unpack", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_11unpack, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_10unpack}, - {"skip", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_13skip, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_12skip}, - {"read_array_header", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_15read_array_header, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_14read_array_header}, - {"read_map_header", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_17read_map_header, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_16read_map_header}, - {"tell", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_19tell, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_18tell}, - {"__next__", (PyCFunction)__pyx_specialmethod___pyx_pw_7msgpack_9_cmsgpack_8Unpacker_23__next__, METH_NOARGS|METH_COEXIST, 0}, - {"__reduce_cython__", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_25__reduce_cython__, METH_NOARGS, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_24__reduce_cython__}, - {"__setstate_cython__", (PyCFunction)__pyx_pw_7msgpack_9_cmsgpack_8Unpacker_27__setstate_cython__, METH_O, __pyx_doc_7msgpack_9_cmsgpack_8Unpacker_26__setstate_cython__}, - {0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_7msgpack_9_cmsgpack_Unpacker = { - PyVarObject_HEAD_INIT(0, 0) - "msgpack._cmsgpack.Unpacker", /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_9_cmsgpack_Unpacker, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - "Unpacker(file_like=None, Py_ssize_t read_size=0, bool use_list=True, bool raw=True, bool strict_map_key=False, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, Py_ssize_t max_array_len=-1, Py_ssize_t max_map_len=-1, Py_ssize_t max_ext_len=-1)\nStreaming unpacker.\n\n Arguments:\n\n :param file_like:\n File-like object having `.read(n)` method.\n If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.\n\n :param int read_size:\n Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)\n\n :param bool use_list:\n If true, unpack msgpack array to Python list.\n Otherwise, unpack to Python tuple. (default: True)\n\n :param bool raw:\n If true, unpack msgpack raw to Python bytes (default).\n Otherwise, unpack to Python str (or unicode on Python 2) by decoding\n with UTF-8 encoding (recommended).\n Currently, the default is true, but it will be changed to false in\n near future. So you must specify it explicitly for keeping backward\n compatibility.\n\n *encoding* option which is deprecated overrides this option.\n\n :param bool strict_map_key:\n If true, only str or bytes are accepted for map (dict) keys.\n It's False by default for backward-compatibility.\n But it will be True from msgpack 1.0.\n\n :param callable object_hook:\n When specified, it should be callable.\n Unpacker calls it with a dict argument after unpacking msgpack map.\n (See also simplejson)\n\n :param callable object_pairs_hook:\n When specified, it should be callable.\n Unpacker calls it with a list of key-value pairs after unpacking msgpack map.\n (See also simplejson)\n\n :param int max_buffer_size:\n Limits size of data w""aiting unpacked. 0 means system's INT_MAX (default).\n Raises `BufferFull` exception when it is insufficient.\n You should set this parameter when unpacking data from untrusted source.\n\n :param int max_str_len:\n Deprecated, use *max_buffer_size* instead.\n Limits max length of str. (default: max_buffer_size or 1024*1024)\n\n :param int max_bin_len:\n Deprecated, use *max_buffer_size* instead.\n Limits max length of bin. (default: max_buffer_size or 1024*1024)\n\n :param int max_array_len:\n Limits max length of array. (default: max_buffer_size or 128*1024)\n\n :param int max_map_len:\n Limits max length of map. (default: max_buffer_size//2 or 32*1024)\n\n :param int max_ext_len:\n Deprecated, use *max_buffer_size* instead.\n Limits max size of ext type. (default: max_buffer_size or 1024*1024)\n\n :param str encoding:\n Deprecated, use ``raw=False`` instead.\n Encoding used for decoding msgpack raw.\n If it is None (default), msgpack raw is deserialized to Python bytes.\n\n :param str unicode_errors:\n Error handler used for decoding str type. (default: `'strict'`)\n\n\n Example of streaming deserialize from file-like object::\n\n unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024)\n for o in unpacker:\n process(o)\n\n Example of streaming deserialize from socket::\n\n unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024)\n while True:\n buf = sock.recv(1024**2)\n if not buf:\n break\n unpacker.feed(buf)\n for o in unpacker:\n process(o)\n\n Raises ``ExtraData`` when *packed* contains extra bytes.\n Raises ``OutOfData`` when *packed* is incomplete.\n Raises ``FormatError`` when *packed* is not valid msgpack.\n Raises ``StackError`` when *packed* contains too nested.\n Other exceptions ca""n be raised during unpacking.\n ", /*tp_doc*/ - __pyx_tp_traverse_7msgpack_9_cmsgpack_Unpacker, /*tp_traverse*/ - __pyx_tp_clear_7msgpack_9_cmsgpack_Unpacker, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_21__iter__, /*tp_iter*/ - __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_23__next__, /*tp_iternext*/ - __pyx_methods_7msgpack_9_cmsgpack_Unpacker, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pw_7msgpack_9_cmsgpack_8Unpacker_5__init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_9_cmsgpack_Unpacker, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -#if CYTHON_PEP489_MULTI_PHASE_INIT -static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ -static int __pyx_pymod_exec__cmsgpack(PyObject* module); /*proto*/ -static PyModuleDef_Slot __pyx_moduledef_slots[] = { - {Py_mod_create, (void*)__pyx_pymod_create}, - {Py_mod_exec, (void*)__pyx_pymod_exec__cmsgpack}, - {0, NULL} -}; -#endif - -static struct PyModuleDef __pyx_moduledef = { - PyModuleDef_HEAD_INIT, - "_cmsgpack", - 0, /* m_doc */ - #if CYTHON_PEP489_MULTI_PHASE_INIT - 0, /* m_size */ - #else - -1, /* m_size */ - #endif - __pyx_methods /* m_methods */, - #if CYTHON_PEP489_MULTI_PHASE_INIT - __pyx_moduledef_slots, /* m_slots */ - #else - NULL, /* m_reload */ - #endif - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif -#ifndef CYTHON_SMALL_CODE -#if defined(__clang__) - #define CYTHON_SMALL_CODE -#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) - #define CYTHON_SMALL_CODE __attribute__((cold)) -#else - #define CYTHON_SMALL_CODE -#endif -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_n_s_AssertionError, __pyx_k_AssertionError, sizeof(__pyx_k_AssertionError), 0, 0, 1, 1}, - {&__pyx_n_s_BufferError, __pyx_k_BufferError, sizeof(__pyx_k_BufferError), 0, 0, 1, 1}, - {&__pyx_n_s_BufferFull, __pyx_k_BufferFull, sizeof(__pyx_k_BufferFull), 0, 0, 1, 1}, - {&__pyx_kp_u_Cannot_decode_extended_type_with, __pyx_k_Cannot_decode_extended_type_with, sizeof(__pyx_k_Cannot_decode_extended_type_with), 0, 1, 0, 0}, - {&__pyx_n_s_DeprecationWarning, __pyx_k_DeprecationWarning, sizeof(__pyx_k_DeprecationWarning), 0, 0, 1, 1}, - {&__pyx_kp_u_EXT_data_is_too_large, __pyx_k_EXT_data_is_too_large, sizeof(__pyx_k_EXT_data_is_too_large), 0, 1, 0, 0}, - {&__pyx_n_s_ExtType, __pyx_k_ExtType, sizeof(__pyx_k_ExtType), 0, 0, 1, 1}, - {&__pyx_n_s_ExtraData, __pyx_k_ExtraData, sizeof(__pyx_k_ExtraData), 0, 0, 1, 1}, - {&__pyx_n_s_FormatError, __pyx_k_FormatError, sizeof(__pyx_k_FormatError), 0, 0, 1, 1}, - {&__pyx_kp_u_Integer_value_out_of_range, __pyx_k_Integer_value_out_of_range, sizeof(__pyx_k_Integer_value_out_of_range), 0, 1, 0, 0}, - {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, - {&__pyx_kp_u_No_more_data_to_unpack, __pyx_k_No_more_data_to_unpack, sizeof(__pyx_k_No_more_data_to_unpack), 0, 1, 0, 0}, - {&__pyx_n_s_NotImplementedError, __pyx_k_NotImplementedError, sizeof(__pyx_k_NotImplementedError), 0, 0, 1, 1}, - {&__pyx_n_s_OutOfData, __pyx_k_OutOfData, sizeof(__pyx_k_OutOfData), 0, 0, 1, 1}, - {&__pyx_n_s_OverflowError, __pyx_k_OverflowError, sizeof(__pyx_k_OverflowError), 0, 0, 1, 1}, - {&__pyx_n_s_Packer, __pyx_k_Packer, sizeof(__pyx_k_Packer), 0, 0, 1, 1}, - {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, - {&__pyx_n_s_RuntimeWarning, __pyx_k_RuntimeWarning, sizeof(__pyx_k_RuntimeWarning), 0, 0, 1, 1}, - {&__pyx_n_s_StackError, __pyx_k_StackError, sizeof(__pyx_k_StackError), 0, 0, 1, 1}, - {&__pyx_n_s_StopIteration, __pyx_k_StopIteration, sizeof(__pyx_k_StopIteration), 0, 0, 1, 1}, - {&__pyx_n_s_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 0, 0, 1, 1}, - {&__pyx_kp_u_Unable_to_allocate_internal_buff, __pyx_k_Unable_to_allocate_internal_buff, sizeof(__pyx_k_Unable_to_allocate_internal_buff), 0, 1, 0, 0}, - {&__pyx_kp_u_Unable_to_enlarge_internal_buffe, __pyx_k_Unable_to_enlarge_internal_buffe, sizeof(__pyx_k_Unable_to_enlarge_internal_buffe), 0, 1, 0, 0}, - {&__pyx_kp_u_Unpack_failed_error, __pyx_k_Unpack_failed_error, sizeof(__pyx_k_Unpack_failed_error), 0, 1, 0, 0}, - {&__pyx_kp_u_Unpack_failed_incomplete_input, __pyx_k_Unpack_failed_incomplete_input, sizeof(__pyx_k_Unpack_failed_incomplete_input), 0, 1, 0, 0}, - {&__pyx_n_s_Unpacker, __pyx_k_Unpacker, sizeof(__pyx_k_Unpacker), 0, 0, 1, 1}, - {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s_autoreset, __pyx_k_autoreset, sizeof(__pyx_k_autoreset), 0, 0, 1, 1}, - {&__pyx_n_s_buf, __pyx_k_buf, sizeof(__pyx_k_buf), 0, 0, 1, 1}, - {&__pyx_n_s_buf_len, __pyx_k_buf_len, sizeof(__pyx_k_buf_len), 0, 0, 1, 1}, - {&__pyx_kp_u_cannot_unpack_from_multi_byte_ob, __pyx_k_cannot_unpack_from_multi_byte_ob, sizeof(__pyx_k_cannot_unpack_from_multi_byte_ob), 0, 1, 0, 0}, - {&__pyx_n_s_cenc, __pyx_k_cenc, sizeof(__pyx_k_cenc), 0, 0, 1, 1}, - {&__pyx_n_s_cerr, __pyx_k_cerr, sizeof(__pyx_k_cerr), 0, 0, 1, 1}, - {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, - {&__pyx_n_s_code, __pyx_k_code, sizeof(__pyx_k_code), 0, 0, 1, 1}, - {&__pyx_kp_u_could_not_get_buffer_for_memoryv, __pyx_k_could_not_get_buffer_for_memoryv, sizeof(__pyx_k_could_not_get_buffer_for_memoryv), 0, 1, 0, 0}, - {&__pyx_kp_u_could_not_get_memoryview, __pyx_k_could_not_get_memoryview, sizeof(__pyx_k_could_not_get_memoryview), 0, 1, 0, 0}, - {&__pyx_n_s_ctx, __pyx_k_ctx, sizeof(__pyx_k_ctx), 0, 0, 1, 1}, - {&__pyx_n_u_d, __pyx_k_d, sizeof(__pyx_k_d), 0, 1, 0, 1}, - {&__pyx_n_s_data, __pyx_k_data, sizeof(__pyx_k_data), 0, 0, 1, 1}, - {&__pyx_n_s_ddtrace_vendor_msgpack, __pyx_k_ddtrace_vendor_msgpack, sizeof(__pyx_k_ddtrace_vendor_msgpack), 0, 0, 1, 1}, - {&__pyx_n_s_ddtrace_vendor_msgpack_exception, __pyx_k_ddtrace_vendor_msgpack_exception, sizeof(__pyx_k_ddtrace_vendor_msgpack_exception), 0, 0, 1, 1}, - {&__pyx_n_s_default, __pyx_k_default, sizeof(__pyx_k_default), 0, 0, 1, 1}, - {&__pyx_kp_u_default_must_be_a_callable, __pyx_k_default_must_be_a_callable, sizeof(__pyx_k_default_must_be_a_callable), 0, 1, 0, 0}, - {&__pyx_n_s_default_read_extended_type, __pyx_k_default_read_extended_type, sizeof(__pyx_k_default_read_extended_type), 0, 0, 1, 1}, - {&__pyx_kp_u_dict_is_too_large, __pyx_k_dict_is_too_large, sizeof(__pyx_k_dict_is_too_large), 0, 1, 0, 0}, - {&__pyx_n_s_encoding, __pyx_k_encoding, sizeof(__pyx_k_encoding), 0, 0, 1, 1}, - {&__pyx_n_s_ext_hook, __pyx_k_ext_hook, sizeof(__pyx_k_ext_hook), 0, 0, 1, 1}, - {&__pyx_kp_u_ext_hook_must_be_a_callable, __pyx_k_ext_hook_must_be_a_callable, sizeof(__pyx_k_ext_hook_must_be_a_callable), 0, 1, 0, 0}, - {&__pyx_n_s_file_like, __pyx_k_file_like, sizeof(__pyx_k_file_like), 0, 0, 1, 1}, - {&__pyx_kp_u_file_like_read_must_be_a_callab, __pyx_k_file_like_read_must_be_a_callab, sizeof(__pyx_k_file_like_read_must_be_a_callab), 0, 1, 0, 0}, - {&__pyx_n_s_getstate, __pyx_k_getstate, sizeof(__pyx_k_getstate), 0, 0, 1, 1}, - {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_kp_u_internal_error, __pyx_k_internal_error, sizeof(__pyx_k_internal_error), 0, 1, 0, 0}, - {&__pyx_n_s_items, __pyx_k_items, sizeof(__pyx_k_items), 0, 0, 1, 1}, - {&__pyx_n_s_kwargs, __pyx_k_kwargs, sizeof(__pyx_k_kwargs), 0, 0, 1, 1}, - {&__pyx_n_s_list_hook, __pyx_k_list_hook, sizeof(__pyx_k_list_hook), 0, 0, 1, 1}, - {&__pyx_kp_u_list_hook_must_be_a_callable, __pyx_k_list_hook_must_be_a_callable, sizeof(__pyx_k_list_hook_must_be_a_callable), 0, 1, 0, 0}, - {&__pyx_kp_u_list_is_too_large, __pyx_k_list_is_too_large, sizeof(__pyx_k_list_is_too_large), 0, 1, 0, 0}, - {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, - {&__pyx_n_s_max_array_len, __pyx_k_max_array_len, sizeof(__pyx_k_max_array_len), 0, 0, 1, 1}, - {&__pyx_n_s_max_bin_len, __pyx_k_max_bin_len, sizeof(__pyx_k_max_bin_len), 0, 0, 1, 1}, - {&__pyx_n_s_max_buffer_size, __pyx_k_max_buffer_size, sizeof(__pyx_k_max_buffer_size), 0, 0, 1, 1}, - {&__pyx_n_s_max_ext_len, __pyx_k_max_ext_len, sizeof(__pyx_k_max_ext_len), 0, 0, 1, 1}, - {&__pyx_n_s_max_map_len, __pyx_k_max_map_len, sizeof(__pyx_k_max_map_len), 0, 0, 1, 1}, - {&__pyx_n_s_max_str_len, __pyx_k_max_str_len, sizeof(__pyx_k_max_str_len), 0, 0, 1, 1}, - {&__pyx_kp_u_memoryview_is_too_large, __pyx_k_memoryview_is_too_large, sizeof(__pyx_k_memoryview_is_too_large), 0, 1, 0, 0}, - {&__pyx_n_s_msgpack__cmsgpack, __pyx_k_msgpack__cmsgpack, sizeof(__pyx_k_msgpack__cmsgpack), 0, 0, 1, 1}, - {&__pyx_kp_s_msgpack__unpacker_pyx, __pyx_k_msgpack__unpacker_pyx, sizeof(__pyx_k_msgpack__unpacker_pyx), 0, 0, 1, 0}, - {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, - {&__pyx_n_s_new_protocol, __pyx_k_new_protocol, sizeof(__pyx_k_new_protocol), 0, 0, 1, 1}, - {&__pyx_kp_s_no_default___reduce___due_to_non, __pyx_k_no_default___reduce___due_to_non, sizeof(__pyx_k_no_default___reduce___due_to_non), 0, 0, 1, 0}, - {&__pyx_n_s_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 0, 0, 1, 1}, - {&__pyx_n_s_object_hook, __pyx_k_object_hook, sizeof(__pyx_k_object_hook), 0, 0, 1, 1}, - {&__pyx_kp_u_object_hook_must_be_a_callable, __pyx_k_object_hook_must_be_a_callable, sizeof(__pyx_k_object_hook_must_be_a_callable), 0, 1, 0, 0}, - {&__pyx_n_s_object_pairs_hook, __pyx_k_object_pairs_hook, sizeof(__pyx_k_object_pairs_hook), 0, 0, 1, 1}, - {&__pyx_kp_u_object_pairs_hook_and_object_hoo, __pyx_k_object_pairs_hook_and_object_hoo, sizeof(__pyx_k_object_pairs_hook_and_object_hoo), 0, 1, 0, 0}, - {&__pyx_kp_u_object_pairs_hook_must_be_a_call, __pyx_k_object_pairs_hook_must_be_a_call, sizeof(__pyx_k_object_pairs_hook_must_be_a_call), 0, 1, 0, 0}, - {&__pyx_n_s_off, __pyx_k_off, sizeof(__pyx_k_off), 0, 0, 1, 1}, - {&__pyx_n_s_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 0, 0, 1, 1}, - {&__pyx_n_s_packed, __pyx_k_packed, sizeof(__pyx_k_packed), 0, 0, 1, 1}, - {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, - {&__pyx_n_s_raw, __pyx_k_raw, sizeof(__pyx_k_raw), 0, 0, 1, 1}, - {&__pyx_n_s_read, __pyx_k_read, sizeof(__pyx_k_read), 0, 0, 1, 1}, - {&__pyx_n_s_read_size, __pyx_k_read_size, sizeof(__pyx_k_read_size), 0, 0, 1, 1}, - {&__pyx_kp_u_read_size_should_be_less_or_equa, __pyx_k_read_size_should_be_less_or_equa, sizeof(__pyx_k_read_size_should_be_less_or_equa), 0, 1, 0, 0}, - {&__pyx_kp_u_recursion_limit_exceeded, __pyx_k_recursion_limit_exceeded, sizeof(__pyx_k_recursion_limit_exceeded), 0, 1, 0, 0}, - {&__pyx_n_s_reduce, __pyx_k_reduce, sizeof(__pyx_k_reduce), 0, 0, 1, 1}, - {&__pyx_n_s_reduce_cython, __pyx_k_reduce_cython, sizeof(__pyx_k_reduce_cython), 0, 0, 1, 1}, - {&__pyx_n_s_reduce_ex, __pyx_k_reduce_ex, sizeof(__pyx_k_reduce_ex), 0, 0, 1, 1}, - {&__pyx_n_s_ret, __pyx_k_ret, sizeof(__pyx_k_ret), 0, 0, 1, 1}, - {&__pyx_n_s_setstate, __pyx_k_setstate, sizeof(__pyx_k_setstate), 0, 0, 1, 1}, - {&__pyx_n_s_setstate_cython, __pyx_k_setstate_cython, sizeof(__pyx_k_setstate_cython), 0, 0, 1, 1}, - {&__pyx_n_s_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 0, 0, 1, 1}, - {&__pyx_n_s_strict_map_key, __pyx_k_strict_map_key, sizeof(__pyx_k_strict_map_key), 0, 0, 1, 1}, - {&__pyx_n_s_strict_types, __pyx_k_strict_types, sizeof(__pyx_k_strict_types), 0, 0, 1, 1}, - {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_typecode, __pyx_k_typecode, sizeof(__pyx_k_typecode), 0, 0, 1, 1}, - {&__pyx_n_s_unicode_errors, __pyx_k_unicode_errors, sizeof(__pyx_k_unicode_errors), 0, 0, 1, 1}, - {&__pyx_kp_u_unicode_string_is_too_large, __pyx_k_unicode_string_is_too_large, sizeof(__pyx_k_unicode_string_is_too_large), 0, 1, 0, 0}, - {&__pyx_n_s_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 0, 0, 1, 1}, - {&__pyx_n_s_unpackb, __pyx_k_unpackb, sizeof(__pyx_k_unpackb), 0, 0, 1, 1}, - {&__pyx_kp_u_unpacker_feed_is_not_be_able_to, __pyx_k_unpacker_feed_is_not_be_able_to, sizeof(__pyx_k_unpacker_feed_is_not_be_able_to), 0, 1, 0, 0}, - {&__pyx_n_s_use_bin_type, __pyx_k_use_bin_type, sizeof(__pyx_k_use_bin_type), 0, 0, 1, 1}, - {&__pyx_n_s_use_list, __pyx_k_use_list, sizeof(__pyx_k_use_list), 0, 0, 1, 1}, - {&__pyx_n_s_use_single_float, __pyx_k_use_single_float, sizeof(__pyx_k_use_single_float), 0, 0, 1, 1}, - {&__pyx_kp_u_using_old_buffer_interface_to_un, __pyx_k_using_old_buffer_interface_to_un, sizeof(__pyx_k_using_old_buffer_interface_to_un), 0, 1, 0, 0}, - {&__pyx_n_s_view, __pyx_k_view, sizeof(__pyx_k_view), 0, 0, 1, 1}, - {0, 0, 0, 0, 0, 0, 0} -}; -static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 111, __pyx_L1_error) - __pyx_builtin_DeprecationWarning = __Pyx_GetBuiltinName(__pyx_n_s_DeprecationWarning); if (!__pyx_builtin_DeprecationWarning) __PYX_ERR(0, 119, __pyx_L1_error) - __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(0, 126, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 163, __pyx_L1_error) - __pyx_builtin_OverflowError = __Pyx_GetBuiltinName(__pyx_n_s_OverflowError); if (!__pyx_builtin_OverflowError) __PYX_ERR(0, 183, __pyx_L1_error) - __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 290, __pyx_L1_error) - __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 106, __pyx_L1_error) - __pyx_builtin_BufferError = __Pyx_GetBuiltinName(__pyx_n_s_BufferError); if (!__pyx_builtin_BufferError) __PYX_ERR(1, 121, __pyx_L1_error) - __pyx_builtin_RuntimeWarning = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeWarning); if (!__pyx_builtin_RuntimeWarning) __PYX_ERR(1, 137, __pyx_L1_error) - __pyx_builtin_AssertionError = __Pyx_GetBuiltinName(__pyx_n_s_AssertionError); if (!__pyx_builtin_AssertionError) __PYX_ERR(1, 417, __pyx_L1_error) - __pyx_builtin_StopIteration = __Pyx_GetBuiltinName(__pyx_n_s_StopIteration); if (!__pyx_builtin_StopIteration) __PYX_ERR(1, 489, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - - /* "msgpack/_packer.pyx":111 - * self.pk.buf = PyMem_Malloc(buf_size) - * if self.pk.buf == NULL: - * raise MemoryError("Unable to allocate internal buffer.") # <<<<<<<<<<<<<< - * self.pk.buf_size = buf_size - * self.pk.length = 0 - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_u_Unable_to_allocate_internal_buff); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - - /* "msgpack/_packer.pyx":126 - * if default is not None: - * if not PyCallable_Check(default): - * raise TypeError("default must be a callable.") # <<<<<<<<<<<<<< - * self._default = default - * - */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_u_default_must_be_a_callable); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 126, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__2); - __Pyx_GIVEREF(__pyx_tuple__2); - - /* "msgpack/_packer.pyx":163 - * - * if nest_limit < 0: - * raise ValueError("recursion limit exceeded.") # <<<<<<<<<<<<<< - * - * while True: - */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_u_recursion_limit_exceeded); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(0, 163, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__4); - __Pyx_GIVEREF(__pyx_tuple__4); - - /* "msgpack/_packer.pyx":189 - * continue - * else: - * raise OverflowError("Integer value out of range") # <<<<<<<<<<<<<< - * elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - * longval = o - */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_u_Integer_value_out_of_range); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(0, 189, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); - - /* "msgpack/_packer.pyx":212 - * ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - * if ret == -2: - * raise ValueError("unicode string is too large") # <<<<<<<<<<<<<< - * else: - * o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - */ - __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_u_unicode_string_is_too_large); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(0, 212, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); - - /* "msgpack/_packer.pyx":226 - * L = len(d) - * if L > ITEM_LIMIT: - * raise ValueError("dict is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_map(&self.pk, L) - * if ret == 0: - */ - __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_u_dict_is_too_large); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(0, 226, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); - - /* "msgpack/_packer.pyx":251 - * L = len(o.data) - * if L > ITEM_LIMIT: - * raise ValueError("EXT data is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_ext(&self.pk, longval, L) - * ret = msgpack_pack_raw_body(&self.pk, rawval, L) - */ - __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_u_EXT_data_is_too_large); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(0, 251, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); - - /* "msgpack/_packer.pyx":257 - * L = len(o) - * if L > ITEM_LIMIT: - * raise ValueError("list is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_array(&self.pk, L) - * if ret == 0: - */ - __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_u_list_is_too_large); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(0, 257, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__9); - __Pyx_GIVEREF(__pyx_tuple__9); - - /* "msgpack/_packer.pyx":265 - * elif PyMemoryView_Check(o): - * if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - * raise ValueError("could not get buffer for memoryview") # <<<<<<<<<<<<<< - * L = view.len - * if L > ITEM_LIMIT: - */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_u_could_not_get_buffer_for_memoryv); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(0, 265, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - - /* "msgpack/_packer.pyx":269 - * if L > ITEM_LIMIT: - * PyBuffer_Release(&view); - * raise ValueError("memoryview is too large") # <<<<<<<<<<<<<< - * ret = msgpack_pack_bin(&self.pk, L) - * if ret == 0: - */ - __pyx_tuple__11 = PyTuple_Pack(1, __pyx_kp_u_memoryview_is_too_large); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 269, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__11); - __Pyx_GIVEREF(__pyx_tuple__11); - - /* "msgpack/_packer.pyx":290 - * raise - * if ret: # should not happen. - * raise RuntimeError("internal error") # <<<<<<<<<<<<<< - * if self.autoreset: - * buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - */ - __pyx_tuple__12 = PyTuple_Pack(1, __pyx_kp_u_internal_error); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(0, 290, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__12); - __Pyx_GIVEREF(__pyx_tuple__12); - - /* "(tree fragment)":2 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - __pyx_tuple__13 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__13); - __Pyx_GIVEREF(__pyx_tuple__13); - - /* "(tree fragment)":4 - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - */ - __pyx_tuple__14 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(2, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__14); - __Pyx_GIVEREF(__pyx_tuple__14); - - /* "msgpack/_unpacker.pyx":77 - * - * if object_hook is not None and object_pairs_hook is not None: - * raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") # <<<<<<<<<<<<<< - * - * if object_hook is not None: - */ - __pyx_tuple__15 = PyTuple_Pack(1, __pyx_kp_u_object_pairs_hook_and_object_hoo); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 77, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__15); - __Pyx_GIVEREF(__pyx_tuple__15); - - /* "msgpack/_unpacker.pyx":81 - * if object_hook is not None: - * if not PyCallable_Check(object_hook): - * raise TypeError("object_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.object_hook = object_hook - * - */ - __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_u_object_hook_must_be_a_callable); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 81, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__16); - __Pyx_GIVEREF(__pyx_tuple__16); - - /* "msgpack/_unpacker.pyx":88 - * else: - * if not PyCallable_Check(object_pairs_hook): - * raise TypeError("object_pairs_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.object_hook = object_pairs_hook - * ctx.user.has_pairs_hook = True - */ - __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_u_object_pairs_hook_must_be_a_call); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 88, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__17); - __Pyx_GIVEREF(__pyx_tuple__17); - - /* "msgpack/_unpacker.pyx":94 - * if list_hook is not None: - * if not PyCallable_Check(list_hook): - * raise TypeError("list_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.list_hook = list_hook - * - */ - __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_u_list_hook_must_be_a_callable); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 94, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__18); - __Pyx_GIVEREF(__pyx_tuple__18); - - /* "msgpack/_unpacker.pyx":99 - * if ext_hook is not None: - * if not PyCallable_Check(ext_hook): - * raise TypeError("ext_hook must be a callable.") # <<<<<<<<<<<<<< - * ctx.user.ext_hook = ext_hook - * - */ - __pyx_tuple__19 = PyTuple_Pack(1, __pyx_kp_u_ext_hook_must_be_a_callable); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(1, 99, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__19); - __Pyx_GIVEREF(__pyx_tuple__19); - - /* "msgpack/_unpacker.pyx":121 - * if view.itemsize != 1: - * PyBuffer_Release(view) - * raise BufferError("cannot unpack from multi-byte object") # <<<<<<<<<<<<<< - * if PyBuffer_IsContiguous(view, b'A') == 0: - * PyBuffer_Release(view) - */ - __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_u_cannot_unpack_from_multi_byte_ob); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(1, 121, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__20); - __Pyx_GIVEREF(__pyx_tuple__20); - - /* "msgpack/_unpacker.pyx":136 - * new_protocol[0] = 0 - * if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: - * raise BufferError("could not get memoryview") # <<<<<<<<<<<<<< - * PyErr_WarnEx(RuntimeWarning, - * "using old buffer interface to unpack %s; " - */ - __pyx_tuple__21 = PyTuple_Pack(1, __pyx_kp_u_could_not_get_memoryview); if (unlikely(!__pyx_tuple__21)) __PYX_ERR(1, 136, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__21); - __Pyx_GIVEREF(__pyx_tuple__21); - - /* "msgpack/_unpacker.pyx":213 - * unpack_clear(&ctx) - * if ret == 0: - * raise ValueError("Unpack failed: incomplete input") # <<<<<<<<<<<<<< - * elif ret == -2: - * raise FormatError - */ - __pyx_tuple__23 = PyTuple_Pack(1, __pyx_kp_u_Unpack_failed_incomplete_input); if (unlikely(!__pyx_tuple__23)) __PYX_ERR(1, 213, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__23); - __Pyx_GIVEREF(__pyx_tuple__23); - - /* "msgpack/_unpacker.pyx":366 - * self.file_like_read = file_like.read - * if not PyCallable_Check(self.file_like_read): - * raise TypeError("`file_like.read` must be a callable.") # <<<<<<<<<<<<<< - * - * if max_str_len == -1: - */ - __pyx_tuple__25 = PyTuple_Pack(1, __pyx_kp_u_file_like_read_must_be_a_callab); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(1, 366, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__25); - __Pyx_GIVEREF(__pyx_tuple__25); - - /* "msgpack/_unpacker.pyx":382 - * max_buffer_size = INT_MAX - * if read_size > max_buffer_size: - * raise ValueError("read_size should be less or equal to max_buffer_size") # <<<<<<<<<<<<<< - * if not read_size: - * read_size = min(max_buffer_size, 1024**2) - */ - __pyx_tuple__26 = PyTuple_Pack(1, __pyx_kp_u_read_size_should_be_less_or_equa); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(1, 382, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__26); - __Pyx_GIVEREF(__pyx_tuple__26); - - /* "msgpack/_unpacker.pyx":417 - * - * if self.file_like is not None: - * raise AssertionError( # <<<<<<<<<<<<<< - * "unpacker.feed() is not be able to use with `file_like`.") - * - */ - __pyx_tuple__27 = PyTuple_Pack(1, __pyx_kp_u_unpacker_feed_is_not_be_able_to); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(1, 417, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__27); - __Pyx_GIVEREF(__pyx_tuple__27); - - /* "msgpack/_unpacker.pyx":452 - * # self.buf still holds old buffer and will be freed during - * # obj destruction - * raise MemoryError("Unable to enlarge internal buffer.") # <<<<<<<<<<<<<< - * memcpy(new_buf, buf + head, tail - head) - * PyMem_Free(buf) - */ - __pyx_tuple__28 = PyTuple_Pack(1, __pyx_kp_u_Unable_to_enlarge_internal_buffe); if (unlikely(!__pyx_tuple__28)) __PYX_ERR(1, 452, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__28); - __Pyx_GIVEREF(__pyx_tuple__28); - - /* "msgpack/_unpacker.pyx":489 - * if prev_head >= self.buf_tail: - * if iter: - * raise StopIteration("No more data to unpack.") # <<<<<<<<<<<<<< - * else: - * raise OutOfData("No more data to unpack.") - */ - __pyx_tuple__29 = PyTuple_Pack(1, __pyx_kp_u_No_more_data_to_unpack); if (unlikely(!__pyx_tuple__29)) __PYX_ERR(1, 489, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__29); - __Pyx_GIVEREF(__pyx_tuple__29); - - /* "(tree fragment)":2 - * def __reduce_cython__(self): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - */ - __pyx_tuple__30 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__30)) __PYX_ERR(2, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__30); - __Pyx_GIVEREF(__pyx_tuple__30); - - /* "(tree fragment)":4 - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") - * def __setstate_cython__(self, __pyx_state): - * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< - */ - __pyx_tuple__31 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__31)) __PYX_ERR(2, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__31); - __Pyx_GIVEREF(__pyx_tuple__31); - - /* "msgpack/_unpacker.pyx":105 - * ctx.user.unicode_errors = unicode_errors - * - * def default_read_extended_type(typecode, data): # <<<<<<<<<<<<<< - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - * - */ - __pyx_tuple__32 = PyTuple_Pack(2, __pyx_n_s_typecode, __pyx_n_s_data); if (unlikely(!__pyx_tuple__32)) __PYX_ERR(1, 105, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__32); - __Pyx_GIVEREF(__pyx_tuple__32); - __pyx_codeobj__33 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__32, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_msgpack__unpacker_pyx, __pyx_n_s_default_read_extended_type, 105, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__33)) __PYX_ERR(1, 105, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":144 - * return 1 - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - */ - __pyx_tuple__34 = PyTuple_Pack(25, __pyx_n_s_packed, __pyx_n_s_object_hook, __pyx_n_s_list_hook, __pyx_n_s_use_list, __pyx_n_s_raw, __pyx_n_s_strict_map_key, __pyx_n_s_encoding, __pyx_n_s_unicode_errors, __pyx_n_s_object_pairs_hook, __pyx_n_s_ext_hook, __pyx_n_s_max_str_len, __pyx_n_s_max_bin_len, __pyx_n_s_max_array_len, __pyx_n_s_max_map_len, __pyx_n_s_max_ext_len, __pyx_n_s_ctx, __pyx_n_s_off, __pyx_n_s_ret, __pyx_n_s_view, __pyx_n_s_buf, __pyx_n_s_buf_len, __pyx_n_s_cenc, __pyx_n_s_cerr, __pyx_n_s_new_protocol, __pyx_n_s_obj); if (unlikely(!__pyx_tuple__34)) __PYX_ERR(1, 144, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__34); - __Pyx_GIVEREF(__pyx_tuple__34); - __pyx_codeobj__35 = (PyObject*)__Pyx_PyCode_New(15, 0, 25, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__34, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_msgpack__unpacker_pyx, __pyx_n_s_unpackb, 144, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__35)) __PYX_ERR(1, 144, __pyx_L1_error) - - /* "msgpack/_unpacker.pyx":221 - * - * - * def unpack(object stream, **kwargs): # <<<<<<<<<<<<<< - * PyErr_WarnEx( - * DeprecationWarning, - */ - __pyx_tuple__36 = PyTuple_Pack(3, __pyx_n_s_stream, __pyx_n_s_kwargs, __pyx_n_s_data); if (unlikely(!__pyx_tuple__36)) __PYX_ERR(1, 221, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__36); - __Pyx_GIVEREF(__pyx_tuple__36); - __pyx_codeobj__37 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__36, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_msgpack__unpacker_pyx, __pyx_n_s_unpack, 221, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__37)) __PYX_ERR(1, 221, __pyx_L1_error) - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(3, 1, __pyx_L1_error); - __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(3, 1, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(void); /*proto*/ -static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(void); /*proto*/ -static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(void); /*proto*/ -static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(void); /*proto*/ -static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(void); /*proto*/ -static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(void); /*proto*/ -static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(void); /*proto*/ - -static int __Pyx_modinit_global_init_code(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); - /*--- Global init code ---*/ - __Pyx_RefNannyFinishContext(); - return 0; -} - -static int __Pyx_modinit_variable_export_code(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); - /*--- Variable export code ---*/ - __Pyx_RefNannyFinishContext(); - return 0; -} - -static int __Pyx_modinit_function_export_code(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); - /*--- Function export code ---*/ - __Pyx_RefNannyFinishContext(); - return 0; -} - -static int __Pyx_modinit_type_init_code(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); - /*--- Type init code ---*/ - __pyx_vtabptr_7msgpack_9_cmsgpack_Packer = &__pyx_vtable_7msgpack_9_cmsgpack_Packer; - __pyx_vtable_7msgpack_9_cmsgpack_Packer._pack = (int (*)(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *, PyObject *, struct __pyx_opt_args_7msgpack_9_cmsgpack_6Packer__pack *__pyx_optional_args))__pyx_f_7msgpack_9_cmsgpack_6Packer__pack; - __pyx_vtable_7msgpack_9_cmsgpack_Packer.pack = (PyObject *(*)(struct __pyx_obj_7msgpack_9_cmsgpack_Packer *, PyObject *, int __pyx_skip_dispatch))__pyx_f_7msgpack_9_cmsgpack_6Packer_pack; - if (PyType_Ready(&__pyx_type_7msgpack_9_cmsgpack_Packer) < 0) __PYX_ERR(0, 54, __pyx_L1_error) - __pyx_type_7msgpack_9_cmsgpack_Packer.tp_print = 0; - if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_7msgpack_9_cmsgpack_Packer.tp_dictoffset && __pyx_type_7msgpack_9_cmsgpack_Packer.tp_getattro == PyObject_GenericGetAttr)) { - __pyx_type_7msgpack_9_cmsgpack_Packer.tp_getattro = __Pyx_PyObject_GenericGetAttr; - } - if (__Pyx_SetVtable(__pyx_type_7msgpack_9_cmsgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_9_cmsgpack_Packer) < 0) __PYX_ERR(0, 54, __pyx_L1_error) - if (PyObject_SetAttr(__pyx_m, __pyx_n_s_Packer, (PyObject *)&__pyx_type_7msgpack_9_cmsgpack_Packer) < 0) __PYX_ERR(0, 54, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_7msgpack_9_cmsgpack_Packer) < 0) __PYX_ERR(0, 54, __pyx_L1_error) - __pyx_ptype_7msgpack_9_cmsgpack_Packer = &__pyx_type_7msgpack_9_cmsgpack_Packer; - __pyx_vtabptr_7msgpack_9_cmsgpack_Unpacker = &__pyx_vtable_7msgpack_9_cmsgpack_Unpacker; - __pyx_vtable_7msgpack_9_cmsgpack_Unpacker.append_buffer = (PyObject *(*)(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *, void *, Py_ssize_t))__pyx_f_7msgpack_9_cmsgpack_8Unpacker_append_buffer; - __pyx_vtable_7msgpack_9_cmsgpack_Unpacker.read_from_file = (PyObject *(*)(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *))__pyx_f_7msgpack_9_cmsgpack_8Unpacker_read_from_file; - __pyx_vtable_7msgpack_9_cmsgpack_Unpacker._unpack = (PyObject *(*)(struct __pyx_obj_7msgpack_9_cmsgpack_Unpacker *, execute_fn, struct __pyx_opt_args_7msgpack_9_cmsgpack_8Unpacker__unpack *__pyx_optional_args))__pyx_f_7msgpack_9_cmsgpack_8Unpacker__unpack; - if (PyType_Ready(&__pyx_type_7msgpack_9_cmsgpack_Unpacker) < 0) __PYX_ERR(1, 229, __pyx_L1_error) - __pyx_type_7msgpack_9_cmsgpack_Unpacker.tp_print = 0; - if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_7msgpack_9_cmsgpack_Unpacker.tp_dictoffset && __pyx_type_7msgpack_9_cmsgpack_Unpacker.tp_getattro == PyObject_GenericGetAttr)) { - __pyx_type_7msgpack_9_cmsgpack_Unpacker.tp_getattro = __Pyx_PyObject_GenericGetAttr; - } - if (__Pyx_SetVtable(__pyx_type_7msgpack_9_cmsgpack_Unpacker.tp_dict, __pyx_vtabptr_7msgpack_9_cmsgpack_Unpacker) < 0) __PYX_ERR(1, 229, __pyx_L1_error) - if (PyObject_SetAttr(__pyx_m, __pyx_n_s_Unpacker, (PyObject *)&__pyx_type_7msgpack_9_cmsgpack_Unpacker) < 0) __PYX_ERR(1, 229, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_7msgpack_9_cmsgpack_Unpacker) < 0) __PYX_ERR(1, 229, __pyx_L1_error) - __pyx_ptype_7msgpack_9_cmsgpack_Unpacker = &__pyx_type_7msgpack_9_cmsgpack_Unpacker; - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_modinit_type_import_code(void) { - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); - /*--- Type import code ---*/ - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 9, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", - #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 - sizeof(PyTypeObject), - #else - sizeof(PyHeapTypeObject), - #endif - __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_4type_type) __PYX_ERR(4, 9, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(5, 8, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_ptype_7cpython_4bool_bool = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "bool", sizeof(PyBoolObject), __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_4bool_bool) __PYX_ERR(5, 8, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(6, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_ptype_7cpython_7complex_complex = __Pyx_ImportType(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "complex", sizeof(PyComplexObject), __Pyx_ImportType_CheckSize_Warn); - if (!__pyx_ptype_7cpython_7complex_complex) __PYX_ERR(6, 15, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_modinit_variable_import_code(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); - /*--- Variable import code ---*/ - __Pyx_RefNannyFinishContext(); - return 0; -} - -static int __Pyx_modinit_function_import_code(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); - /*--- Function import code ---*/ - __Pyx_RefNannyFinishContext(); - return 0; -} - - -#if PY_MAJOR_VERSION < 3 -#ifdef CYTHON_NO_PYINIT_EXPORT -#define __Pyx_PyMODINIT_FUNC void -#else -#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC -#endif -#else -#ifdef CYTHON_NO_PYINIT_EXPORT -#define __Pyx_PyMODINIT_FUNC PyObject * -#else -#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC -#endif -#endif - - -#if PY_MAJOR_VERSION < 3 -__Pyx_PyMODINIT_FUNC init_cmsgpack(void) CYTHON_SMALL_CODE; /*proto*/ -__Pyx_PyMODINIT_FUNC init_cmsgpack(void) -#else -__Pyx_PyMODINIT_FUNC PyInit__cmsgpack(void) CYTHON_SMALL_CODE; /*proto*/ -__Pyx_PyMODINIT_FUNC PyInit__cmsgpack(void) -#if CYTHON_PEP489_MULTI_PHASE_INIT -{ - return PyModuleDef_Init(&__pyx_moduledef); -} -static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { - #if PY_VERSION_HEX >= 0x030700A1 - static PY_INT64_T main_interpreter_id = -1; - PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); - if (main_interpreter_id == -1) { - main_interpreter_id = current_id; - return (unlikely(current_id == -1)) ? -1 : 0; - } else if (unlikely(main_interpreter_id != current_id)) - #else - static PyInterpreterState *main_interpreter = NULL; - PyInterpreterState *current_interpreter = PyThreadState_Get()->interp; - if (!main_interpreter) { - main_interpreter = current_interpreter; - } else if (unlikely(main_interpreter != current_interpreter)) - #endif - { - PyErr_SetString( - PyExc_ImportError, - "Interpreter change detected - this module can only be loaded into one interpreter per process."); - return -1; - } - return 0; -} -static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) { - PyObject *value = PyObject_GetAttrString(spec, from_name); - int result = 0; - if (likely(value)) { - if (allow_none || value != Py_None) { - result = PyDict_SetItemString(moddict, to_name, value); - } - Py_DECREF(value); - } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - } else { - result = -1; - } - return result; -} -static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { - PyObject *module = NULL, *moddict, *modname; - if (__Pyx_check_single_interpreter()) - return NULL; - if (__pyx_m) - return __Pyx_NewRef(__pyx_m); - modname = PyObject_GetAttrString(spec, "name"); - if (unlikely(!modname)) goto bad; - module = PyModule_NewObject(modname); - Py_DECREF(modname); - if (unlikely(!module)) goto bad; - moddict = PyModule_GetDict(module); - if (unlikely(!moddict)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; - return module; -bad: - Py_XDECREF(module); - return NULL; -} - - -static CYTHON_SMALL_CODE int __pyx_pymod_exec__cmsgpack(PyObject *__pyx_pyinit_module) -#endif -#endif -{ - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannyDeclarations - #if CYTHON_PEP489_MULTI_PHASE_INIT - if (__pyx_m) { - if (__pyx_m == __pyx_pyinit_module) return 0; - PyErr_SetString(PyExc_RuntimeError, "Module '_cmsgpack' has already been imported. Re-initialisation is not supported."); - return -1; - } - #elif PY_MAJOR_VERSION >= 3 - if (__pyx_m) return __Pyx_NewRef(__pyx_m); - #endif - #if CYTHON_REFNANNY -__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); -if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); -} -#endif - __Pyx_RefNannySetupContext("__Pyx_PyMODINIT_FUNC PyInit__cmsgpack(void)", 0); - if (__Pyx_check_binary_version() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #ifdef __Pxy_PyFrame_Initialize_Offsets - __Pxy_PyFrame_Initialize_Offsets(); - #endif - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(3, 1, __pyx_L1_error) - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(3, 1, __pyx_L1_error) - __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(3, 1, __pyx_L1_error) - #ifdef __Pyx_CyFunction_USED - if (__pyx_CyFunction_init() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Coroutine_USED - if (__pyx_Coroutine_init() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_AsyncGen_USED - if (__pyx_AsyncGen_init() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_StopAsyncIteration_USED - if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - /*--- Library function declarations ---*/ - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Module creation code ---*/ - #if CYTHON_PEP489_MULTI_PHASE_INIT - __pyx_m = __pyx_pyinit_module; - Py_INCREF(__pyx_m); - #else - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4("_cmsgpack", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (unlikely(!__pyx_m)) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(3, 1, __pyx_L1_error) - Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(3, 1, __pyx_L1_error) - __pyx_cython_runtime = PyImport_AddModule((char *) "cython_runtime"); if (unlikely(!__pyx_cython_runtime)) __PYX_ERR(3, 1, __pyx_L1_error) - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(3, 1, __pyx_L1_error); - /*--- Initialize various global constants etc. ---*/ - if (__Pyx_InitGlobals() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - if (__pyx_module_is_main_msgpack___cmsgpack) { - if (PyObject_SetAttr(__pyx_m, __pyx_n_s_name, __pyx_n_s_main) < 0) __PYX_ERR(3, 1, __pyx_L1_error) - } - #if PY_MAJOR_VERSION >= 3 - { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(3, 1, __pyx_L1_error) - if (!PyDict_GetItemString(modules, "msgpack._cmsgpack")) { - if (unlikely(PyDict_SetItemString(modules, "msgpack._cmsgpack", __pyx_m) < 0)) __PYX_ERR(3, 1, __pyx_L1_error) - } - } - #endif - /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - /*--- Global type/function init code ---*/ - (void)__Pyx_modinit_global_init_code(); - (void)__Pyx_modinit_variable_export_code(); - (void)__Pyx_modinit_function_export_code(); - if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; - if (unlikely(__Pyx_modinit_type_import_code() != 0)) goto __pyx_L1_error; - (void)__Pyx_modinit_variable_import_code(); - (void)__Pyx_modinit_function_import_code(); - /*--- Execution code ---*/ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - if (__Pyx_patch_abc() < 0) __PYX_ERR(3, 1, __pyx_L1_error) - #endif - - /* "msgpack/_packer.pyx":6 - * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact - * - * from ddtrace.vendor.msgpack import ExtType # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 6, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_n_s_ExtType); - __Pyx_GIVEREF(__pyx_n_s_ExtType); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_ExtType); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_ddtrace_vendor_msgpack, __pyx_t_1, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 6, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_ExtType); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 6, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ExtType, __pyx_t_1) < 0) __PYX_ERR(0, 6, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_packer.pyx":42 - * object buff_to_buff(char *, Py_ssize_t) - * - * cdef int DEFAULT_RECURSE_LIMIT=511 # <<<<<<<<<<<<<< - * cdef long long ITEM_LIMIT = (2**32)-1 - * - */ - __pyx_v_7msgpack_9_cmsgpack_DEFAULT_RECURSE_LIMIT = 0x1FF; - - /* "msgpack/_packer.pyx":43 - * - * cdef int DEFAULT_RECURSE_LIMIT=511 - * cdef long long ITEM_LIMIT = (2**32)-1 # <<<<<<<<<<<<<< - * - * - */ - __pyx_v_7msgpack_9_cmsgpack_ITEM_LIMIT = 0xFFFFFFFF; - - /* "msgpack/_packer.pyx":148 - * self.pk.buf = NULL - * - * cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: # <<<<<<<<<<<<<< - * cdef long long llval - * cdef unsigned long long ullval - */ - __pyx_k__3 = __pyx_v_7msgpack_9_cmsgpack_DEFAULT_RECURSE_LIMIT; - - /* "msgpack/_unpacker.pyx":16 - * - * from ddtrace.vendor.msgpack.exceptions import ( - * BufferFull, # <<<<<<<<<<<<<< - * OutOfData, - * ExtraData, - */ - __pyx_t_2 = PyList_New(5); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 16, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(__pyx_n_s_BufferFull); - __Pyx_GIVEREF(__pyx_n_s_BufferFull); - PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_BufferFull); - __Pyx_INCREF(__pyx_n_s_OutOfData); - __Pyx_GIVEREF(__pyx_n_s_OutOfData); - PyList_SET_ITEM(__pyx_t_2, 1, __pyx_n_s_OutOfData); - __Pyx_INCREF(__pyx_n_s_ExtraData); - __Pyx_GIVEREF(__pyx_n_s_ExtraData); - PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_ExtraData); - __Pyx_INCREF(__pyx_n_s_FormatError); - __Pyx_GIVEREF(__pyx_n_s_FormatError); - PyList_SET_ITEM(__pyx_t_2, 3, __pyx_n_s_FormatError); - __Pyx_INCREF(__pyx_n_s_StackError); - __Pyx_GIVEREF(__pyx_n_s_StackError); - PyList_SET_ITEM(__pyx_t_2, 4, __pyx_n_s_StackError); - - /* "msgpack/_unpacker.pyx":15 - * ctypedef unsigned long long uint64_t - * - * from ddtrace.vendor.msgpack.exceptions import ( # <<<<<<<<<<<<<< - * BufferFull, - * OutOfData, - */ - __pyx_t_1 = __Pyx_Import(__pyx_n_s_ddtrace_vendor_msgpack_exception, __pyx_t_2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_BufferFull); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_BufferFull, __pyx_t_2) < 0) __PYX_ERR(1, 16, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_OutOfData); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_OutOfData, __pyx_t_2) < 0) __PYX_ERR(1, 17, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_ExtraData); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ExtraData, __pyx_t_2) < 0) __PYX_ERR(1, 18, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_FormatError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FormatError, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_StackError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_StackError, __pyx_t_2) < 0) __PYX_ERR(1, 20, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "msgpack/_unpacker.pyx":22 - * StackError, - * ) - * from ddtrace.vendor.msgpack import ExtType # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 22, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_n_s_ExtType); - __Pyx_GIVEREF(__pyx_n_s_ExtType); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_ExtType); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_ddtrace_vendor_msgpack, __pyx_t_1, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 22, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_ExtType); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 22, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ExtType, __pyx_t_1) < 0) __PYX_ERR(1, 22, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":105 - * ctx.user.unicode_errors = unicode_errors - * - * def default_read_extended_type(typecode, data): # <<<<<<<<<<<<<< - * raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - * - */ - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_7msgpack_9_cmsgpack_1default_read_extended_type, NULL, __pyx_n_s_msgpack__cmsgpack); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 105, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_default_read_extended_type, __pyx_t_2) < 0) __PYX_ERR(1, 105, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":147 - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - * object_pairs_hook=None, ext_hook=ExtType, # <<<<<<<<<<<<<< - * Py_ssize_t max_str_len=-1, - * Py_ssize_t max_bin_len=-1, - */ - __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_n_s_ExtType); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 147, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_k__22 = __pyx_t_2; - __Pyx_GIVEREF(__pyx_t_2); - __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":144 - * return 1 - * - * def unpackb(object packed, object object_hook=None, object list_hook=None, # <<<<<<<<<<<<<< - * bint use_list=True, bint raw=True, bint strict_map_key=False, - * encoding=None, unicode_errors=None, - */ - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_7msgpack_9_cmsgpack_3unpackb, NULL, __pyx_n_s_msgpack__cmsgpack); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 144, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_unpackb, __pyx_t_2) < 0) __PYX_ERR(1, 144, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":221 - * - * - * def unpack(object stream, **kwargs): # <<<<<<<<<<<<<< - * PyErr_WarnEx( - * DeprecationWarning, - */ - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_7msgpack_9_cmsgpack_5unpack, NULL, __pyx_n_s_msgpack__cmsgpack); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 221, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_unpack, __pyx_t_2) < 0) __PYX_ERR(1, 221, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "msgpack/_unpacker.pyx":348 - * object object_hook=None, object object_pairs_hook=None, object list_hook=None, - * encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, - * object ext_hook=ExtType, # <<<<<<<<<<<<<< - * Py_ssize_t max_str_len=-1, - * Py_ssize_t max_bin_len=-1, - */ - __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_n_s_ExtType); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 348, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_k__24 = __pyx_t_2; - __Pyx_GIVEREF(__pyx_t_2); - __pyx_t_2 = 0; - - /* "msgpack/_cmsgpack.pyx":1 - * # coding: utf-8 # <<<<<<<<<<<<<< - * #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 - * include "_packer.pyx" - */ - __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(3, 1, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_2) < 0) __PYX_ERR(3, 1, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /*--- Wrapped vars code ---*/ - - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init msgpack._cmsgpack", __pyx_clineno, __pyx_lineno, __pyx_filename); - } - Py_CLEAR(__pyx_m); - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init msgpack._cmsgpack"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if CYTHON_PEP489_MULTI_PHASE_INIT - return (__pyx_m != NULL) ? 0 : -1; - #elif PY_MAJOR_VERSION >= 3 - return __pyx_m; - #else - return; - #endif -} - -/* --- Runtime support code --- */ -/* Refnanny */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule(modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, "RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; -} -#endif - -/* PyObjectGetAttrStr */ -#if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} -#endif - -/* GetBuiltinName */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); - if (unlikely(!result)) { - PyErr_Format(PyExc_NameError, -#if PY_MAJOR_VERSION >= 3 - "name '%U' is not defined", name); -#else - "name '%.200s' is not defined", PyString_AS_STRING(name)); -#endif - } - return result; -} - -/* RaiseArgTupleInvalid */ -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); -} - -/* KeywordStringCheck */ -static int __Pyx_CheckKeywordStrings( - PyObject *kwdict, - const char* function_name, - int kw_allowed) -{ - PyObject* key = 0; - Py_ssize_t pos = 0; -#if CYTHON_COMPILING_IN_PYPY - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) - goto invalid_keyword; - return 1; -#else - while (PyDict_Next(kwdict, &pos, &key, 0)) { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_Check(key))) - #endif - if (unlikely(!PyUnicode_Check(key))) - goto invalid_keyword_type; - } - if ((!kw_allowed) && unlikely(key)) - goto invalid_keyword; - return 1; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - return 0; -#endif -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif - return 0; -} - -/* PyObjectCall */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyErrFetchRestore */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} -#endif - -/* RaiseException */ -#if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - __Pyx_PyThreadState_declare - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } -#endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - } - __Pyx_PyThreadState_assign - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} -#else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; - } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; - } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - int is_subclass = PyObject_IsSubclass(instance_class, type); - if (!is_subclass) { - instance_class = NULL; - } else if (unlikely(is_subclass == -1)) { - goto bad; - } else { - type = instance_class; - } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } - if (cause) { - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; -} -#endif - -/* RaiseDoubleKeywords */ -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); - #endif -} - -/* ParseKeywords */ -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; - #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else - #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -/* ExtTypeTest */ -static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (likely(__Pyx_TypeCheck(obj, type))) - return 1; - PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", - Py_TYPE(obj)->tp_name, type->tp_name); - return 0; -} - -/* GetTopmostException */ -#if CYTHON_USE_EXC_INFO_STACK -static _PyErr_StackItem * -__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) -{ - _PyErr_StackItem *exc_info = tstate->exc_info; - while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) && - exc_info->previous_item != NULL) - { - exc_info = exc_info->previous_item; - } - return exc_info; -} -#endif - -/* SaveResetException */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - #if CYTHON_USE_EXC_INFO_STACK - _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); - *type = exc_info->exc_type; - *value = exc_info->exc_value; - *tb = exc_info->exc_traceback; - #else - *type = tstate->exc_type; - *value = tstate->exc_value; - *tb = tstate->exc_traceback; - #endif - Py_XINCREF(*type); - Py_XINCREF(*value); - Py_XINCREF(*tb); -} -static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - #if CYTHON_USE_EXC_INFO_STACK - _PyErr_StackItem *exc_info = tstate->exc_info; - tmp_type = exc_info->exc_type; - tmp_value = exc_info->exc_value; - tmp_tb = exc_info->exc_traceback; - exc_info->exc_type = type; - exc_info->exc_value = value; - exc_info->exc_traceback = tb; - #else - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - #endif - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -#endif - -/* PyErrExceptionMatches */ -#if CYTHON_FAST_THREAD_STATE -static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { - Py_ssize_t i, n; - n = PyTuple_GET_SIZE(tuple); -#if PY_MAJOR_VERSION >= 3 - for (i=0; icurexc_type; - if (exc_type == err) return 1; - if (unlikely(!exc_type)) return 0; - if (unlikely(PyTuple_Check(err))) - return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); - return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); -} -#endif - -/* GetException */ -#if CYTHON_FAST_THREAD_STATE -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) -#else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) -#endif -{ - PyObject *local_type, *local_value, *local_tb; -#if CYTHON_FAST_THREAD_STATE - PyObject *tmp_type, *tmp_value, *tmp_tb; - local_type = tstate->curexc_type; - local_value = tstate->curexc_value; - local_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -#else - PyErr_Fetch(&local_type, &local_value, &local_tb); -#endif - PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_FAST_THREAD_STATE - if (unlikely(tstate->curexc_type)) -#else - if (unlikely(PyErr_Occurred())) -#endif - goto bad; - #if PY_MAJOR_VERSION >= 3 - if (local_tb) { - if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) - goto bad; - } - #endif - Py_XINCREF(local_tb); - Py_XINCREF(local_type); - Py_XINCREF(local_value); - *type = local_type; - *value = local_value; - *tb = local_tb; -#if CYTHON_FAST_THREAD_STATE - #if CYTHON_USE_EXC_INFO_STACK - { - _PyErr_StackItem *exc_info = tstate->exc_info; - tmp_type = exc_info->exc_type; - tmp_value = exc_info->exc_value; - tmp_tb = exc_info->exc_traceback; - exc_info->exc_type = local_type; - exc_info->exc_value = local_value; - exc_info->exc_traceback = local_tb; - } - #else - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = local_type; - tstate->exc_value = local_value; - tstate->exc_traceback = local_tb; - #endif - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif - return 0; -bad: - *type = 0; - *value = 0; - *tb = 0; - Py_XDECREF(local_type); - Py_XDECREF(local_value); - Py_XDECREF(local_tb); - return -1; -} - -/* PyCFunctionFastCall */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif - -/* PyFunctionFastCall */ -#if CYTHON_FAST_PYCALL -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = __Pyx_PyFrame_GetLocalsplus(f); - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif -#endif - -/* PyObjectCall2Args */ -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; -} - -/* PyObjectCallMethO */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* SwapException */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - #if CYTHON_USE_EXC_INFO_STACK - _PyErr_StackItem *exc_info = tstate->exc_info; - tmp_type = exc_info->exc_type; - tmp_value = exc_info->exc_value; - tmp_tb = exc_info->exc_traceback; - exc_info->exc_type = *type; - exc_info->exc_value = *value; - exc_info->exc_traceback = *tb; - #else - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = *type; - tstate->exc_value = *value; - tstate->exc_traceback = *tb; - #endif - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); - PyErr_SetExcInfo(*type, *value, *tb); - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#endif - -/* IterFinish */ -static CYTHON_INLINE int __Pyx_IterFinish(void) { -#if CYTHON_FAST_THREAD_STATE - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject* exc_type = tstate->curexc_type; - if (unlikely(exc_type)) { - if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) { - PyObject *exc_value, *exc_tb; - exc_value = tstate->curexc_value; - exc_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; - Py_DECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - return 0; - } else { - return -1; - } - } - return 0; -#else - if (unlikely(PyErr_Occurred())) { - if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { - PyErr_Clear(); - return 0; - } else { - return -1; - } - } - return 0; -#endif -} - -/* PyObjectCallNoArg */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || __Pyx_CyFunction_Check(func))) -#else - if (likely(PyCFunction_Check(func))) -#endif - { - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - return __Pyx_PyObject_CallMethO(func, NULL); - } - } - return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); -} -#endif - -/* PyObjectGetMethod */ -static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { - PyObject *attr; -#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP - PyTypeObject *tp = Py_TYPE(obj); - PyObject *descr; - descrgetfunc f = NULL; - PyObject **dictptr, *dict; - int meth_found = 0; - assert (*method == NULL); - if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { - attr = __Pyx_PyObject_GetAttrStr(obj, name); - goto try_unpack; - } - if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { - return 0; - } - descr = _PyType_Lookup(tp, name); - if (likely(descr != NULL)) { - Py_INCREF(descr); -#if PY_MAJOR_VERSION >= 3 - #ifdef __Pyx_CyFunction_USED - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) - #else - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type))) - #endif -#else - #ifdef __Pyx_CyFunction_USED - if (likely(PyFunction_Check(descr) || __Pyx_CyFunction_Check(descr))) - #else - if (likely(PyFunction_Check(descr))) - #endif -#endif - { - meth_found = 1; - } else { - f = Py_TYPE(descr)->tp_descr_get; - if (f != NULL && PyDescr_IsData(descr)) { - attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); - Py_DECREF(descr); - goto try_unpack; - } - } - } - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr != NULL && (dict = *dictptr) != NULL) { - Py_INCREF(dict); - attr = __Pyx_PyDict_GetItemStr(dict, name); - if (attr != NULL) { - Py_INCREF(attr); - Py_DECREF(dict); - Py_XDECREF(descr); - goto try_unpack; - } - Py_DECREF(dict); - } - if (meth_found) { - *method = descr; - return 1; - } - if (f != NULL) { - attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); - Py_DECREF(descr); - goto try_unpack; - } - if (descr != NULL) { - *method = descr; - return 0; - } - PyErr_Format(PyExc_AttributeError, -#if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, name); -#else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(name)); -#endif - return 0; -#else - attr = __Pyx_PyObject_GetAttrStr(obj, name); - goto try_unpack; -#endif -try_unpack: -#if CYTHON_UNPACK_METHODS - if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { - PyObject *function = PyMethod_GET_FUNCTION(attr); - Py_INCREF(function); - Py_DECREF(attr); - *method = function; - return 1; - } -#endif - *method = attr; - return 0; -} - -/* PyObjectCallMethod0 */ -static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { - PyObject *method = NULL, *result = NULL; - int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); - if (likely(is_method)) { - result = __Pyx_PyObject_CallOneArg(method, obj); - Py_DECREF(method); - return result; - } - if (unlikely(!method)) goto bad; - result = __Pyx_PyObject_CallNoArg(method); - Py_DECREF(method); -bad: - return result; -} - -/* RaiseNeedMoreValuesToUnpack */ -static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", - index, (index == 1) ? "" : "s"); -} - -/* RaiseTooManyValuesToUnpack */ -static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { - PyErr_Format(PyExc_ValueError, - "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); -} - -/* UnpackItemEndCheck */ -static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { - if (unlikely(retval)) { - Py_DECREF(retval); - __Pyx_RaiseTooManyValuesError(expected); - return -1; - } else { - return __Pyx_IterFinish(); - } - return 0; -} - -/* RaiseNoneIterError */ -static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); -} - -/* UnpackTupleError */ -static void __Pyx_UnpackTupleError(PyObject *t, Py_ssize_t index) { - if (t == Py_None) { - __Pyx_RaiseNoneNotIterableError(); - } else if (PyTuple_GET_SIZE(t) < index) { - __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(t)); - } else { - __Pyx_RaiseTooManyValuesError(index); - } -} - -/* UnpackTuple2 */ -static CYTHON_INLINE int __Pyx_unpack_tuple2_exact( - PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, int decref_tuple) { - PyObject *value1 = NULL, *value2 = NULL; -#if CYTHON_COMPILING_IN_PYPY - value1 = PySequence_ITEM(tuple, 0); if (unlikely(!value1)) goto bad; - value2 = PySequence_ITEM(tuple, 1); if (unlikely(!value2)) goto bad; -#else - value1 = PyTuple_GET_ITEM(tuple, 0); Py_INCREF(value1); - value2 = PyTuple_GET_ITEM(tuple, 1); Py_INCREF(value2); -#endif - if (decref_tuple) { - Py_DECREF(tuple); - } - *pvalue1 = value1; - *pvalue2 = value2; - return 0; -#if CYTHON_COMPILING_IN_PYPY -bad: - Py_XDECREF(value1); - Py_XDECREF(value2); - if (decref_tuple) { Py_XDECREF(tuple); } - return -1; -#endif -} -static int __Pyx_unpack_tuple2_generic(PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, - int has_known_size, int decref_tuple) { - Py_ssize_t index; - PyObject *value1 = NULL, *value2 = NULL, *iter = NULL; - iternextfunc iternext; - iter = PyObject_GetIter(tuple); - if (unlikely(!iter)) goto bad; - if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } - iternext = Py_TYPE(iter)->tp_iternext; - value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } - value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } - if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; - Py_DECREF(iter); - *pvalue1 = value1; - *pvalue2 = value2; - return 0; -unpacking_failed: - if (!has_known_size && __Pyx_IterFinish() == 0) - __Pyx_RaiseNeedMoreValuesError(index); -bad: - Py_XDECREF(iter); - Py_XDECREF(value1); - Py_XDECREF(value2); - if (decref_tuple) { Py_XDECREF(tuple); } - return -1; -} - -/* dict_iter */ -static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* iterable, int is_dict, PyObject* method_name, - Py_ssize_t* p_orig_length, int* p_source_is_dict) { - is_dict = is_dict || likely(PyDict_CheckExact(iterable)); - *p_source_is_dict = is_dict; - if (is_dict) { -#if !CYTHON_COMPILING_IN_PYPY - *p_orig_length = PyDict_Size(iterable); - Py_INCREF(iterable); - return iterable; -#elif PY_MAJOR_VERSION >= 3 - static PyObject *py_items = NULL, *py_keys = NULL, *py_values = NULL; - PyObject **pp = NULL; - if (method_name) { - const char *name = PyUnicode_AsUTF8(method_name); - if (strcmp(name, "iteritems") == 0) pp = &py_items; - else if (strcmp(name, "iterkeys") == 0) pp = &py_keys; - else if (strcmp(name, "itervalues") == 0) pp = &py_values; - if (pp) { - if (!*pp) { - *pp = PyUnicode_FromString(name + 4); - if (!*pp) - return NULL; - } - method_name = *pp; - } - } -#endif - } - *p_orig_length = 0; - if (method_name) { - PyObject* iter; - iterable = __Pyx_PyObject_CallMethod0(iterable, method_name); - if (!iterable) - return NULL; -#if !CYTHON_COMPILING_IN_PYPY - if (PyTuple_CheckExact(iterable) || PyList_CheckExact(iterable)) - return iterable; -#endif - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - return iter; - } - return PyObject_GetIter(iterable); -} -static CYTHON_INLINE int __Pyx_dict_iter_next( - PyObject* iter_obj, CYTHON_NCP_UNUSED Py_ssize_t orig_length, CYTHON_NCP_UNUSED Py_ssize_t* ppos, - PyObject** pkey, PyObject** pvalue, PyObject** pitem, int source_is_dict) { - PyObject* next_item; -#if !CYTHON_COMPILING_IN_PYPY - if (source_is_dict) { - PyObject *key, *value; - if (unlikely(orig_length != PyDict_Size(iter_obj))) { - PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); - return -1; - } - if (unlikely(!PyDict_Next(iter_obj, ppos, &key, &value))) { - return 0; - } - if (pitem) { - PyObject* tuple = PyTuple_New(2); - if (unlikely(!tuple)) { - return -1; - } - Py_INCREF(key); - Py_INCREF(value); - PyTuple_SET_ITEM(tuple, 0, key); - PyTuple_SET_ITEM(tuple, 1, value); - *pitem = tuple; - } else { - if (pkey) { - Py_INCREF(key); - *pkey = key; - } - if (pvalue) { - Py_INCREF(value); - *pvalue = value; - } - } - return 1; - } else if (PyTuple_CheckExact(iter_obj)) { - Py_ssize_t pos = *ppos; - if (unlikely(pos >= PyTuple_GET_SIZE(iter_obj))) return 0; - *ppos = pos + 1; - next_item = PyTuple_GET_ITEM(iter_obj, pos); - Py_INCREF(next_item); - } else if (PyList_CheckExact(iter_obj)) { - Py_ssize_t pos = *ppos; - if (unlikely(pos >= PyList_GET_SIZE(iter_obj))) return 0; - *ppos = pos + 1; - next_item = PyList_GET_ITEM(iter_obj, pos); - Py_INCREF(next_item); - } else -#endif - { - next_item = PyIter_Next(iter_obj); - if (unlikely(!next_item)) { - return __Pyx_IterFinish(); - } - } - if (pitem) { - *pitem = next_item; - } else if (pkey && pvalue) { - if (__Pyx_unpack_tuple2(next_item, pkey, pvalue, source_is_dict, source_is_dict, 1)) - return -1; - } else if (pkey) { - *pkey = next_item; - } else { - *pvalue = next_item; - } - return 1; -} - -/* PyDictVersioning */ -#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { - PyObject *dict = Py_TYPE(obj)->tp_dict; - return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; -} -static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { - PyObject **dictptr = NULL; - Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; - if (offset) { -#if CYTHON_COMPILING_IN_CPYTHON - dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); -#else - dictptr = _PyObject_GetDictPtr(obj); -#endif - } - return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; -} -static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { - PyObject *dict = Py_TYPE(obj)->tp_dict; - if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) - return 0; - return obj_dict_version == __Pyx_get_object_dict_version(obj); -} -#endif - -/* GetModuleGlobalName */ -#if CYTHON_USE_DICT_VERSIONS -static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) -#else -static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) -#endif -{ - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 - result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } else if (unlikely(PyErr_Occurred())) { - return NULL; - } -#else - result = PyDict_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } -#endif -#else - result = PyObject_GetItem(__pyx_d, name); - __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) - if (likely(result)) { - return __Pyx_NewRef(result); - } - PyErr_Clear(); -#endif - return __Pyx_GetBuiltinName(name); -} - -/* ReRaiseException */ -static CYTHON_INLINE void __Pyx_ReraiseException(void) { - PyObject *type = NULL, *value = NULL, *tb = NULL; -#if CYTHON_FAST_THREAD_STATE - PyThreadState *tstate = PyThreadState_GET(); - #if CYTHON_USE_EXC_INFO_STACK - _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); - type = exc_info->exc_type; - value = exc_info->exc_value; - tb = exc_info->exc_traceback; - #else - type = tstate->exc_type; - value = tstate->exc_value; - tb = tstate->exc_traceback; - #endif -#else - PyErr_GetExcInfo(&type, &value, &tb); -#endif - if (!type || type == Py_None) { -#if !CYTHON_FAST_THREAD_STATE - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tb); -#endif - PyErr_SetString(PyExc_RuntimeError, - "No active exception to reraise"); - } else { -#if CYTHON_FAST_THREAD_STATE - Py_INCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); -#endif - PyErr_Restore(type, value, tb); - } -} - -/* None */ -static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t a, Py_ssize_t b) { - Py_ssize_t q = a / b; - Py_ssize_t r = a - q*b; - q -= ((r != 0) & ((r ^ b) < 0)); - return q; -} - -/* CIntToDigits */ -static const char DIGIT_PAIRS_10[2*10*10+1] = { - "00010203040506070809" - "10111213141516171819" - "20212223242526272829" - "30313233343536373839" - "40414243444546474849" - "50515253545556575859" - "60616263646566676869" - "70717273747576777879" - "80818283848586878889" - "90919293949596979899" -}; -static const char DIGIT_PAIRS_8[2*8*8+1] = { - "0001020304050607" - "1011121314151617" - "2021222324252627" - "3031323334353637" - "4041424344454647" - "5051525354555657" - "6061626364656667" - "7071727374757677" -}; -static const char DIGITS_HEX[2*16+1] = { - "0123456789abcdef" - "0123456789ABCDEF" -}; - -/* BuildPyUnicode */ -static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, char* chars, int clength, - int prepend_sign, char padding_char) { - PyObject *uval; - Py_ssize_t uoffset = ulength - clength; -#if CYTHON_USE_UNICODE_INTERNALS - Py_ssize_t i; -#if CYTHON_PEP393_ENABLED - void *udata; - uval = PyUnicode_New(ulength, 127); - if (unlikely(!uval)) return NULL; - udata = PyUnicode_DATA(uval); -#else - Py_UNICODE *udata; - uval = PyUnicode_FromUnicode(NULL, ulength); - if (unlikely(!uval)) return NULL; - udata = PyUnicode_AS_UNICODE(uval); -#endif - if (uoffset > 0) { - i = 0; - if (prepend_sign) { - __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, 0, '-'); - i++; - } - for (; i < uoffset; i++) { - __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, i, padding_char); - } - } - for (i=0; i < clength; i++) { - __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, uoffset+i, chars[i]); - } -#else - { - PyObject *sign = NULL, *padding = NULL; - uval = NULL; - if (uoffset > 0) { - prepend_sign = !!prepend_sign; - if (uoffset > prepend_sign) { - padding = PyUnicode_FromOrdinal(padding_char); - if (likely(padding) && uoffset > prepend_sign + 1) { - PyObject *tmp; - PyObject *repeat = PyInt_FromSize_t(uoffset - prepend_sign); - if (unlikely(!repeat)) goto done_or_error; - tmp = PyNumber_Multiply(padding, repeat); - Py_DECREF(repeat); - Py_DECREF(padding); - padding = tmp; - } - if (unlikely(!padding)) goto done_or_error; - } - if (prepend_sign) { - sign = PyUnicode_FromOrdinal('-'); - if (unlikely(!sign)) goto done_or_error; - } - } - uval = PyUnicode_DecodeASCII(chars, clength, NULL); - if (likely(uval) && padding) { - PyObject *tmp = PyNumber_Add(padding, uval); - Py_DECREF(uval); - uval = tmp; - } - if (likely(uval) && sign) { - PyObject *tmp = PyNumber_Add(sign, uval); - Py_DECREF(uval); - uval = tmp; - } -done_or_error: - Py_XDECREF(padding); - Py_XDECREF(sign); - } -#endif - return uval; -} - -/* CIntToPyUnicode */ -#ifdef _MSC_VER - #ifndef _MSC_STDINT_H_ - #if _MSC_VER < 1300 - typedef unsigned short uint16_t; - #else - typedef unsigned __int16 uint16_t; - #endif - #endif -#else - #include -#endif -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) -#define GCC_DIAGNOSTIC -#endif -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char) { - char digits[sizeof(int)*3+2]; - char *dpos, *end = digits + sizeof(int)*3+2; - const char *hex_digits = DIGITS_HEX; - Py_ssize_t length, ulength; - int prepend_sign, last_one_off; - int remaining; -#ifdef GCC_DIAGNOSTIC -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wconversion" -#endif - const int neg_one = (int) -1, const_zero = (int) 0; -#ifdef GCC_DIAGNOSTIC -#pragma GCC diagnostic pop -#endif - const int is_unsigned = neg_one > const_zero; - if (format_char == 'X') { - hex_digits += 16; - format_char = 'x'; - } - remaining = value; - last_one_off = 0; - dpos = end; - do { - int digit_pos; - switch (format_char) { - case 'o': - digit_pos = abs((int)(remaining % (8*8))); - remaining = (int) (remaining / (8*8)); - dpos -= 2; - *(uint16_t*)dpos = ((const uint16_t*)DIGIT_PAIRS_8)[digit_pos]; - last_one_off = (digit_pos < 8); - break; - case 'd': - digit_pos = abs((int)(remaining % (10*10))); - remaining = (int) (remaining / (10*10)); - dpos -= 2; - *(uint16_t*)dpos = ((const uint16_t*)DIGIT_PAIRS_10)[digit_pos]; - last_one_off = (digit_pos < 10); - break; - case 'x': - *(--dpos) = hex_digits[abs((int)(remaining % 16))]; - remaining = (int) (remaining / 16); - break; - default: - assert(0); - break; - } - } while (unlikely(remaining != 0)); - if (last_one_off) { - assert(*dpos == '0'); - dpos++; - } - length = end - dpos; - ulength = length; - prepend_sign = 0; - if (!is_unsigned && value <= neg_one) { - if (padding_char == ' ' || width <= length + 1) { - *(--dpos) = '-'; - ++length; - } else { - prepend_sign = 1; - } - ++ulength; - } - if (width > ulength) { - ulength = width; - } - if (ulength == 1) { - return PyUnicode_FromOrdinal(*dpos); - } - return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); -} - -/* PyObject_GenericGetAttrNoDict */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { - PyErr_Format(PyExc_AttributeError, -#if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); -#else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); -#endif - return NULL; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - PyObject *descr; - PyTypeObject *tp = Py_TYPE(obj); - if (unlikely(!PyString_Check(attr_name))) { - return PyObject_GenericGetAttr(obj, attr_name); - } - assert(!tp->tp_dictoffset); - descr = _PyType_Lookup(tp, attr_name); - if (unlikely(!descr)) { - return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); - } - Py_INCREF(descr); - #if PY_MAJOR_VERSION < 3 - if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) - #endif - { - descrgetfunc f = Py_TYPE(descr)->tp_descr_get; - if (unlikely(f)) { - PyObject *res = f(descr, obj, (PyObject *)tp); - Py_DECREF(descr); - return res; - } - } - return descr; -} -#endif - -/* PyObject_GenericGetAttr */ -#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 -static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name) { - if (unlikely(Py_TYPE(obj)->tp_dictoffset)) { - return PyObject_GenericGetAttr(obj, attr_name); - } - return __Pyx_PyObject_GenericGetAttrNoDict(obj, attr_name); -} -#endif - -/* SetVTable */ -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject *ob = PyCapsule_New(vtable, 0, 0); -#else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); -#endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) - goto bad; - Py_DECREF(ob); - return 0; -bad: - Py_XDECREF(ob); - return -1; -} - -/* SetupReduce */ -static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { - int ret; - PyObject *name_attr; - name_attr = __Pyx_PyObject_GetAttrStr(meth, __pyx_n_s_name); - if (likely(name_attr)) { - ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); - } else { - ret = -1; - } - if (unlikely(ret < 0)) { - PyErr_Clear(); - ret = 0; - } - Py_XDECREF(name_attr); - return ret; -} -static int __Pyx_setup_reduce(PyObject* type_obj) { - int ret = 0; - PyObject *object_reduce = NULL; - PyObject *object_reduce_ex = NULL; - PyObject *reduce = NULL; - PyObject *reduce_ex = NULL; - PyObject *reduce_cython = NULL; - PyObject *setstate = NULL; - PyObject *setstate_cython = NULL; -#if CYTHON_USE_PYTYPE_LOOKUP - if (_PyType_Lookup((PyTypeObject*)type_obj, __pyx_n_s_getstate)) goto GOOD; -#else - if (PyObject_HasAttr(type_obj, __pyx_n_s_getstate)) goto GOOD; -#endif -#if CYTHON_USE_PYTYPE_LOOKUP - object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; -#else - object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; -#endif - reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_ex); if (unlikely(!reduce_ex)) goto BAD; - if (reduce_ex == object_reduce_ex) { -#if CYTHON_USE_PYTYPE_LOOKUP - object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; -#else - object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; -#endif - reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce); if (unlikely(!reduce)) goto BAD; - if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_n_s_reduce_cython)) { - reduce_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_cython); if (unlikely(!reduce_cython)) goto BAD; - ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce, reduce_cython); if (unlikely(ret < 0)) goto BAD; - ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce_cython); if (unlikely(ret < 0)) goto BAD; - setstate = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate); - if (!setstate) PyErr_Clear(); - if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_n_s_setstate_cython)) { - setstate_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate_cython); if (unlikely(!setstate_cython)) goto BAD; - ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate, setstate_cython); if (unlikely(ret < 0)) goto BAD; - ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate_cython); if (unlikely(ret < 0)) goto BAD; - } - PyType_Modified((PyTypeObject*)type_obj); - } - } - goto GOOD; -BAD: - if (!PyErr_Occurred()) - PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); - ret = -1; -GOOD: -#if !CYTHON_USE_PYTYPE_LOOKUP - Py_XDECREF(object_reduce); - Py_XDECREF(object_reduce_ex); -#endif - Py_XDECREF(reduce); - Py_XDECREF(reduce_ex); - Py_XDECREF(reduce_cython); - Py_XDECREF(setstate); - Py_XDECREF(setstate_cython); - return ret; -} - -/* TypeImport */ -#ifndef __PYX_HAVE_RT_ImportType -#define __PYX_HAVE_RT_ImportType -static PyTypeObject *__Pyx_ImportType(PyObject *module, const char *module_name, const char *class_name, - size_t size, enum __Pyx_ImportType_CheckSize check_size) -{ - PyObject *result = 0; - char warning[200]; - Py_ssize_t basicsize; -#ifdef Py_LIMITED_API - PyObject *py_basicsize; -#endif - result = PyObject_GetAttrString(module, class_name); - if (!result) - goto bad; - if (!PyType_Check(result)) { - PyErr_Format(PyExc_TypeError, - "%.200s.%.200s is not a type object", - module_name, class_name); - goto bad; - } -#ifndef Py_LIMITED_API - basicsize = ((PyTypeObject *)result)->tp_basicsize; -#else - py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); - if (!py_basicsize) - goto bad; - basicsize = PyLong_AsSsize_t(py_basicsize); - Py_DECREF(py_basicsize); - py_basicsize = 0; - if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) - goto bad; -#endif - if ((size_t)basicsize < size) { - PyErr_Format(PyExc_ValueError, - "%.200s.%.200s size changed, may indicate binary incompatibility. " - "Expected %zd from C header, got %zd from PyObject", - module_name, class_name, size, basicsize); - goto bad; - } - if (check_size == __Pyx_ImportType_CheckSize_Error && (size_t)basicsize != size) { - PyErr_Format(PyExc_ValueError, - "%.200s.%.200s size changed, may indicate binary incompatibility. " - "Expected %zd from C header, got %zd from PyObject", - module_name, class_name, size, basicsize); - goto bad; - } - else if (check_size == __Pyx_ImportType_CheckSize_Warn && (size_t)basicsize > size) { - PyOS_snprintf(warning, sizeof(warning), - "%s.%s size changed, may indicate binary incompatibility. " - "Expected %zd from C header, got %zd from PyObject", - module_name, class_name, size, basicsize); - if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; - } - return (PyTypeObject *)result; -bad: - Py_XDECREF(result); - return NULL; -} -#endif - -/* Import */ -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - #if PY_MAJOR_VERSION < 3 - PyObject *py_import; - py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); - if (!py_import) - goto bad; - #endif - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - { - #if PY_MAJOR_VERSION >= 3 - if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - goto bad; - PyErr_Clear(); - } - } - level = 0; - } - #endif - if (!module) { - #if PY_MAJOR_VERSION < 3 - PyObject *py_level = PyInt_FromLong(level); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, (PyObject *)NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); - #endif - } - } -bad: - #if PY_MAJOR_VERSION < 3 - Py_XDECREF(py_import); - #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); - return module; -} - -/* ImportFrom */ -static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { - PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); - if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_ImportError, - #if PY_MAJOR_VERSION < 3 - "cannot import name %.230s", PyString_AS_STRING(name)); - #else - "cannot import name %S", name); - #endif - } - return value; -} - -/* CLineInTraceback */ -#ifndef CYTHON_CLINE_IN_TRACEBACK -static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { - PyObject *use_cline; - PyObject *ptype, *pvalue, *ptraceback; -#if CYTHON_COMPILING_IN_CPYTHON - PyObject **cython_runtime_dict; -#endif - if (unlikely(!__pyx_cython_runtime)) { - return c_line; - } - __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); -#if CYTHON_COMPILING_IN_CPYTHON - cython_runtime_dict = _PyObject_GetDictPtr(__pyx_cython_runtime); - if (likely(cython_runtime_dict)) { - __PYX_PY_DICT_LOOKUP_IF_MODIFIED( - use_cline, *cython_runtime_dict, - __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_n_s_cline_in_traceback)) - } else -#endif - { - PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback); - if (use_cline_obj) { - use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; - Py_DECREF(use_cline_obj); - } else { - PyErr_Clear(); - use_cline = NULL; - } - } - if (!use_cline) { - c_line = 0; - PyObject_SetAttr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback, Py_False); - } - else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { - c_line = 0; - } - __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); - return c_line; -} -#endif - -/* CodeObjectCache */ -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = start + (end - start) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; - } else { - return mid; - } - } - if (code_line <= entries[mid].code_line) { - return mid; - } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; - Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -/* AddTraceback */ -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - PyThreadState *tstate = __Pyx_PyThreadState_Current; - if (c_line) { - c_line = __Pyx_CLineForTraceback(tstate, c_line); - } - py_code = __pyx_find_code_object(c_line ? -c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); - } - py_frame = PyFrame_New( - tstate, /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - __Pyx_PyFrame_SetLineNumber(py_frame, py_line); - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -/* CIntFromPyVerify */ -#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) -#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) -#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ - {\ - func_type value = func_value;\ - if (sizeof(target_type) < sizeof(func_type)) {\ - if (unlikely(value != (func_type) (target_type) value)) {\ - func_type zero = 0;\ - if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ - return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_PY_LONG_LONG(unsigned PY_LONG_LONG value) { - const unsigned PY_LONG_LONG neg_one = (unsigned PY_LONG_LONG) ((unsigned PY_LONG_LONG) 0 - (unsigned PY_LONG_LONG) 1), const_zero = (unsigned PY_LONG_LONG) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(unsigned PY_LONG_LONG) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned PY_LONG_LONG) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned PY_LONG_LONG) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(unsigned PY_LONG_LONG) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned PY_LONG_LONG) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned PY_LONG_LONG), - little, !is_unsigned); - } -} - -/* CIntFromPy */ -static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_As_PY_LONG_LONG(PyObject *x) { - const PY_LONG_LONG neg_one = (PY_LONG_LONG) ((PY_LONG_LONG) 0 - (PY_LONG_LONG) 1), const_zero = (PY_LONG_LONG) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(PY_LONG_LONG) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (PY_LONG_LONG) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (PY_LONG_LONG) 0; - case 1: __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, digit, digits[0]) - case 2: - if (8 * sizeof(PY_LONG_LONG) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) >= 2 * PyLong_SHIFT) { - return (PY_LONG_LONG) (((((PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(PY_LONG_LONG) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) >= 3 * PyLong_SHIFT) { - return (PY_LONG_LONG) (((((((PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(PY_LONG_LONG) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) >= 4 * PyLong_SHIFT) { - return (PY_LONG_LONG) (((((((((PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (PY_LONG_LONG) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(PY_LONG_LONG) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(PY_LONG_LONG, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(PY_LONG_LONG) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(PY_LONG_LONG, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (PY_LONG_LONG) 0; - case -1: __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, digit, +digits[0]) - case -2: - if (8 * sizeof(PY_LONG_LONG) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - return (PY_LONG_LONG) (((PY_LONG_LONG)-1)*(((((PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(PY_LONG_LONG) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - return (PY_LONG_LONG) ((((((PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - return (PY_LONG_LONG) (((PY_LONG_LONG)-1)*(((((((PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(PY_LONG_LONG) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - return (PY_LONG_LONG) ((((((((PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { - return (PY_LONG_LONG) (((PY_LONG_LONG)-1)*(((((((((PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(PY_LONG_LONG) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(PY_LONG_LONG, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { - return (PY_LONG_LONG) ((((((((((PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (PY_LONG_LONG)digits[0]))); - } - } - break; - } -#endif - if (sizeof(PY_LONG_LONG) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(PY_LONG_LONG, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(PY_LONG_LONG) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(PY_LONG_LONG, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - PY_LONG_LONG val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (PY_LONG_LONG) -1; - } - } else { - PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (PY_LONG_LONG) -1; - val = __Pyx_PyInt_As_PY_LONG_LONG(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to PY_LONG_LONG"); - return (PY_LONG_LONG) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to PY_LONG_LONG"); - return (PY_LONG_LONG) -1; -} - -/* CIntFromPy */ -static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_As_unsigned_PY_LONG_LONG(PyObject *x) { - const unsigned PY_LONG_LONG neg_one = (unsigned PY_LONG_LONG) ((unsigned PY_LONG_LONG) 0 - (unsigned PY_LONG_LONG) 1), const_zero = (unsigned PY_LONG_LONG) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(unsigned PY_LONG_LONG) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (unsigned PY_LONG_LONG) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (unsigned PY_LONG_LONG) 0; - case 1: __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, digit, digits[0]) - case 2: - if (8 * sizeof(unsigned PY_LONG_LONG) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) >= 2 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(unsigned PY_LONG_LONG) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) >= 3 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(unsigned PY_LONG_LONG) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) >= 4 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (unsigned PY_LONG_LONG) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(unsigned PY_LONG_LONG) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned PY_LONG_LONG, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned PY_LONG_LONG) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned PY_LONG_LONG, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (unsigned PY_LONG_LONG) 0; - case -1: __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, digit, +digits[0]) - case -2: - if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) (((unsigned PY_LONG_LONG)-1)*(((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(unsigned PY_LONG_LONG) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) ((((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) (((unsigned PY_LONG_LONG)-1)*(((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(unsigned PY_LONG_LONG) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) ((((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) (((unsigned PY_LONG_LONG)-1)*(((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(unsigned PY_LONG_LONG) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(unsigned PY_LONG_LONG, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(unsigned PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { - return (unsigned PY_LONG_LONG) ((((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]))); - } - } - break; - } -#endif - if (sizeof(unsigned PY_LONG_LONG) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned PY_LONG_LONG, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned PY_LONG_LONG) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(unsigned PY_LONG_LONG, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - unsigned PY_LONG_LONG val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (unsigned PY_LONG_LONG) -1; - } - } else { - unsigned PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (unsigned PY_LONG_LONG) -1; - val = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG) -1; -} - -/* CIntFromPy */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (long) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { - return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { - return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { - return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (long) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) - case -2: - if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - } -#endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - long val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (long) -1; - } - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; -} - -/* CIntFromPy */ -static CYTHON_INLINE char __Pyx_PyInt_As_char(PyObject *x) { - const char neg_one = (char) ((char) 0 - (char) 1), const_zero = (char) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(char) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(char, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (char) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (char) 0; - case 1: __PYX_VERIFY_RETURN_INT(char, digit, digits[0]) - case 2: - if (8 * sizeof(char) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) >= 2 * PyLong_SHIFT) { - return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(char) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) >= 3 * PyLong_SHIFT) { - return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(char) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) >= 4 * PyLong_SHIFT) { - return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (char) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(char) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(char) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (char) 0; - case -1: __PYX_VERIFY_RETURN_INT(char, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(char, digit, +digits[0]) - case -2: - if (8 * sizeof(char) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { - return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(char) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { - return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(char) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { - return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(char) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { - return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(char) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { - return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(char) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(char) - 1 > 4 * PyLong_SHIFT) { - return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); - } - } - break; - } -#endif - if (sizeof(char) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(char) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - char val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (char) -1; - } - } else { - char val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (char) -1; - val = __Pyx_PyInt_As_char(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to char"); - return (char) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to char"); - return (char) -1; -} - -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - -/* CIntFromPy */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (int) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (int) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) - case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - } -#endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - int val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (int) -1; - } - } else { - int val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; -} - -/* FastTypeChecks */ -#if CYTHON_COMPILING_IN_CPYTHON -static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { - while (a) { - a = a->tp_base; - if (a == b) - return 1; - } - return b == &PyBaseObject_Type; -} -static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { - PyObject *mro; - if (a == b) return 1; - mro = a->tp_mro; - if (likely(mro)) { - Py_ssize_t i, n; - n = PyTuple_GET_SIZE(mro); - for (i = 0; i < n; i++) { - if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) - return 1; - } - return 0; - } - return __Pyx_InBases(a, b); -} -#if PY_MAJOR_VERSION == 2 -static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { - PyObject *exception, *value, *tb; - int res; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&exception, &value, &tb); - res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; - if (unlikely(res == -1)) { - PyErr_WriteUnraisable(err); - res = 0; - } - if (!res) { - res = PyObject_IsSubclass(err, exc_type2); - if (unlikely(res == -1)) { - PyErr_WriteUnraisable(err); - res = 0; - } - } - __Pyx_ErrRestore(exception, value, tb); - return res; -} -#else -static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { - int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; - if (!res) { - res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); - } - return res; -} -#endif -static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { - Py_ssize_t i, n; - assert(PyExceptionClass_Check(exc_type)); - n = PyTuple_GET_SIZE(tuple); -#if PY_MAJOR_VERSION >= 3 - for (i=0; ip) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - if (PyObject_Hash(*t->p) == -1) - return -1; - ++t; - } - return 0; -} - -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { - return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); -} -static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { - Py_ssize_t ignore; - return __Pyx_PyObject_AsStringAndSize(o, &ignore); -} -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -#if !CYTHON_PEP393_ENABLED -static const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } - } - } -#endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -} -#else -static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { - if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (likely(PyUnicode_IS_ASCII(o))) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } -#else - return PyUnicode_AsUTF8AndSize(o, length); -#endif -} -#endif -#endif -static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { - return __Pyx_PyUnicode_AsStringAndSize(o, length); - } else -#endif -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) - if (PyByteArray_Check(o)) { - *length = PyByteArray_GET_SIZE(o); - return PyByteArray_AS_STRING(o); - } else -#endif - { - char* result; - int r = PyBytes_AsStringAndSize(o, &result, length); - if (unlikely(r < 0)) { - return NULL; - } else { - return result; - } - } -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - int is_true = x == Py_True; - if (is_true | (x == Py_False) | (x == Py_None)) return is_true; - else return PyObject_IsTrue(x); -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { - int retval; - if (unlikely(!x)) return -1; - retval = __Pyx_PyObject_IsTrue(x); - Py_DECREF(x); - return retval; -} -static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { -#if PY_MAJOR_VERSION >= 3 - if (PyLong_Check(result)) { - if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__int__ returned non-int (type %.200s). " - "The ability to return an instance of a strict subclass of int " - "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { - Py_DECREF(result); - return NULL; - } - return result; - } -#endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - type_name, type_name, Py_TYPE(result)->tp_name); - Py_DECREF(result); - return NULL; -} -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { -#if CYTHON_USE_TYPE_SLOTS - PyNumberMethods *m; -#endif - const char *name = NULL; - PyObject *res = NULL; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x) || PyLong_Check(x))) -#else - if (likely(PyLong_Check(x))) -#endif - return __Pyx_NewRef(x); -#if CYTHON_USE_TYPE_SLOTS - m = Py_TYPE(x)->tp_as_number; - #if PY_MAJOR_VERSION < 3 - if (m && m->nb_int) { - name = "int"; - res = m->nb_int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = m->nb_long(x); - } - #else - if (likely(m && m->nb_int)) { - name = "int"; - res = m->nb_int(x); - } - #endif -#else - if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { - res = PyNumber_Int(x); - } -#endif - if (likely(res)) { -#if PY_MAJOR_VERSION < 3 - if (unlikely(!PyInt_Check(res) && !PyLong_Check(res))) { -#else - if (unlikely(!PyLong_CheckExact(res))) { -#endif - return __Pyx_PyNumber_IntOrLongWrongResultType(res, name); - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject *x; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(b))) { - if (sizeof(Py_ssize_t) >= sizeof(long)) - return PyInt_AS_LONG(b); - else - return PyInt_AsSsize_t(b); - } -#endif - if (likely(PyLong_CheckExact(b))) { - #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)b)->ob_digit; - const Py_ssize_t size = Py_SIZE(b); - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; - } else { - switch (size) { - case 2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - } - } - #endif - return PyLong_AsSsize_t(b); - } - x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} -static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { - return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); -} -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { - return PyInt_FromSize_t(ival); -} - - -#endif /* Py_PYTHON_H */ diff --git a/ddtrace/vendor/msgpack/_cmsgpack.pyx b/ddtrace/vendor/msgpack/_cmsgpack.pyx deleted file mode 100644 index 8ebdbf58b25..00000000000 --- a/ddtrace/vendor/msgpack/_cmsgpack.pyx +++ /dev/null @@ -1,4 +0,0 @@ -# coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii, language_level=3 -include "_packer.pyx" -include "_unpacker.pyx" diff --git a/ddtrace/vendor/msgpack/_packer.pyx b/ddtrace/vendor/msgpack/_packer.pyx deleted file mode 100644 index c0e5a5c4b55..00000000000 --- a/ddtrace/vendor/msgpack/_packer.pyx +++ /dev/null @@ -1,362 +0,0 @@ -# coding: utf-8 - -from cpython cimport * -from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact - -from ddtrace.vendor.msgpack import ExtType - - -cdef extern from "Python.h": - - int PyMemoryView_Check(object obj) - char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL - - -cdef extern from "pack.h": - struct msgpack_packer: - char* buf - size_t length - size_t buf_size - bint use_bin_type - - int msgpack_pack_int(msgpack_packer* pk, int d) - int msgpack_pack_nil(msgpack_packer* pk) - int msgpack_pack_true(msgpack_packer* pk) - int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long(msgpack_packer* pk, long d) - int msgpack_pack_long_long(msgpack_packer* pk, long long d) - int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) - int msgpack_pack_float(msgpack_packer* pk, float d) - int msgpack_pack_double(msgpack_packer* pk, double d) - int msgpack_pack_array(msgpack_packer* pk, size_t l) - int msgpack_pack_map(msgpack_packer* pk, size_t l) - int msgpack_pack_raw(msgpack_packer* pk, size_t l) - int msgpack_pack_bin(msgpack_packer* pk, size_t l) - int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) - int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) - int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) - -cdef extern from "buff_converter.h": - object buff_to_buff(char *, Py_ssize_t) - -cdef int DEFAULT_RECURSE_LIMIT=511 -cdef long long ITEM_LIMIT = (2**32)-1 - - -cdef inline int PyBytesLike_Check(object o): - return PyBytes_Check(o) or PyByteArray_Check(o) - - -cdef inline int PyBytesLike_CheckExact(object o): - return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) - - -cdef class Packer(object): - """ - MessagePack Packer - - usage:: - - packer = Packer() - astream.write(packer.pack(a)) - astream.write(packer.pack(b)) - - Packer's constructor has some keyword arguments: - - :param callable default: - Convert user type to builtin type that Packer supports. - See also simplejson's document. - - :param bool use_single_float: - Use single precision float type for float. (default: False) - - :param bool autoreset: - Reset buffer after each pack and return its content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. - - :param bool use_bin_type: - Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. - Current default value is false, but it will be changed to true - in future version. You should specify it explicitly. - - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializeable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. - - :param str unicode_errors: - Error handler for encoding unicode. (default: 'strict') - - :param str encoding: - (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') - """ - cdef msgpack_packer pk - cdef object _default - cdef object _bencoding - cdef object _berrors - cdef const char *encoding - cdef const char *unicode_errors - cdef bint strict_types - cdef bool use_float - cdef bint autoreset - - def __cinit__(self): - cdef int buf_size = 1024*1024 - self.pk.buf = PyMem_Malloc(buf_size) - if self.pk.buf == NULL: - raise MemoryError("Unable to allocate internal buffer.") - self.pk.buf_size = buf_size - self.pk.length = 0 - - def __init__(self, default=None, encoding=None, unicode_errors=None, - bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, - bint strict_types=False): - if encoding is not None: - PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) - self.use_float = use_single_float - self.strict_types = strict_types - self.autoreset = autoreset - self.pk.use_bin_type = use_bin_type - if default is not None: - if not PyCallable_Check(default): - raise TypeError("default must be a callable.") - self._default = default - - self._bencoding = encoding - if encoding is None: - if PY_MAJOR_VERSION < 3: - self.encoding = 'utf-8' - else: - self.encoding = NULL - else: - self.encoding = self._bencoding - - self._berrors = unicode_errors - if unicode_errors is None: - self.unicode_errors = NULL - else: - self.unicode_errors = self._berrors - - def __dealloc__(self): - PyMem_Free(self.pk.buf) - self.pk.buf = NULL - - cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: - cdef long long llval - cdef unsigned long long ullval - cdef long longval - cdef float fval - cdef double dval - cdef char* rawval - cdef int ret - cdef dict d - cdef Py_ssize_t L - cdef int default_used = 0 - cdef bint strict_types = self.strict_types - cdef Py_buffer view - - if nest_limit < 0: - raise ValueError("recursion limit exceeded.") - - while True: - if o is None: - ret = msgpack_pack_nil(&self.pk) - elif PyBool_Check(o) if strict_types else isinstance(o, bool): - if o: - ret = msgpack_pack_true(&self.pk) - else: - ret = msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): - # PyInt_Check(long) is True for Python 3. - # So we should test long before int. - try: - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) - except OverflowError as oe: - if not default_used and self._default is not None: - o = self._default(o) - default_used = True - continue - else: - raise OverflowError("Integer value out of range") - elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - longval = o - ret = msgpack_pack_long(&self.pk, longval) - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - if self.use_float: - fval = o - ret = msgpack_pack_float(&self.pk, fval) - else: - dval = o - ret = msgpack_pack_double(&self.pk, dval) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - L = len(o) - if L > ITEM_LIMIT: - PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - rawval = o - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if self.encoding == NULL and self.unicode_errors == NULL: - ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - if ret == -2: - raise ValueError("unicode string is too large") - else: - o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - L = len(o) - if L > ITEM_LIMIT: - raise ValueError("unicode string is too large") - ret = msgpack_pack_raw(&self.pk, L) - if ret == 0: - rawval = o - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o): - d = o - L = len(d) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in d.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif not strict_types and PyDict_Check(o): - L = len(o) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in o.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - # This should be before Tuple because ExtType is namedtuple. - longval = o.code - rawval = o.data - L = len(o.data) - if L > ITEM_LIMIT: - raise ValueError("EXT data is too large") - ret = msgpack_pack_ext(&self.pk, longval, L) - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - L = len(o) - if L > ITEM_LIMIT: - raise ValueError("list is too large") - ret = msgpack_pack_array(&self.pk, L) - if ret == 0: - for v in o: - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif PyMemoryView_Check(o): - if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - raise ValueError("could not get buffer for memoryview") - L = view.len - if L > ITEM_LIMIT: - PyBuffer_Release(&view); - raise ValueError("memoryview is too large") - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - PyBuffer_Release(&view); - elif not default_used and self._default: - o = self._default(o) - default_used = 1 - continue - else: - PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) - return ret - - cpdef pack(self, object obj): - cdef int ret - try: - ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - except: - self.pk.length = 0 - raise - if ret: # should not happen. - raise RuntimeError("internal error") - if self.autoreset: - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf - - def pack_ext_type(self, typecode, data): - msgpack_pack_ext(&self.pk, typecode, len(data)) - msgpack_pack_raw_body(&self.pk, data, len(data)) - - def pack_array_header(self, long long size): - if size > ITEM_LIMIT: - raise ValueError - cdef int ret = msgpack_pack_array(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError - if self.autoreset: - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf - - def pack_map_header(self, long long size): - if size > ITEM_LIMIT: - raise ValueError - cdef int ret = msgpack_pack_map(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError - if self.autoreset: - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf - - def pack_map_pairs(self, object pairs): - """ - Pack *pairs* as msgpack map type. - - *pairs* should be a sequence of pairs. - (`len(pairs)` and `for k, v in pairs:` should be supported.) - """ - cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - if ret == 0: - for k, v in pairs: - ret = self._pack(k) - if ret != 0: break - ret = self._pack(v) - if ret != 0: break - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError - if self.autoreset: - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf - - def reset(self): - """Reset internal buffer. - - This method is usaful only when autoreset=False. - """ - self.pk.length = 0 - - def bytes(self): - """Return internal buffer contents as bytes object""" - return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - - def getbuffer(self): - """Return view of internal buffer.""" - return buff_to_buff(self.pk.buf, self.pk.length) diff --git a/ddtrace/vendor/msgpack/_unpacker.pyx b/ddtrace/vendor/msgpack/_unpacker.pyx deleted file mode 100644 index 5239ba7bc08..00000000000 --- a/ddtrace/vendor/msgpack/_unpacker.pyx +++ /dev/null @@ -1,569 +0,0 @@ -# coding: utf-8 - -from cpython cimport * - -cdef extern from "Python.h": - ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 - object PyMemoryView_GetContiguous(object obj, int buffertype, char order) - -from libc.stdlib cimport * -from libc.string cimport * -from libc.limits cimport * -ctypedef unsigned long long uint64_t - -from ddtrace.vendor.msgpack.exceptions import ( - BufferFull, - OutOfData, - ExtraData, - FormatError, - StackError, -) -from ddtrace.vendor.msgpack import ExtType - - -cdef extern from "unpack.h": - ctypedef struct msgpack_user: - bint use_list - bint raw - bint has_pairs_hook # call object_hook with k-v pairs - bint strict_map_key - PyObject* object_hook - PyObject* list_hook - PyObject* ext_hook - char *encoding - char *unicode_errors - Py_ssize_t max_str_len - Py_ssize_t max_bin_len - Py_ssize_t max_array_len - Py_ssize_t max_map_len - Py_ssize_t max_ext_len - - ctypedef struct unpack_context: - msgpack_user user - PyObject* obj - Py_ssize_t count - - ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, - Py_ssize_t len, Py_ssize_t* off) except? -1 - execute_fn unpack_construct - execute_fn unpack_skip - execute_fn read_array_header - execute_fn read_map_header - void unpack_init(unpack_context* ctx) - object unpack_data(unpack_context* ctx) - void unpack_clear(unpack_context* ctx) - -cdef inline init_ctx(unpack_context *ctx, - object object_hook, object object_pairs_hook, - object list_hook, object ext_hook, - bint use_list, bint raw, bint strict_map_key, - const char* encoding, const char* unicode_errors, - Py_ssize_t max_str_len, Py_ssize_t max_bin_len, - Py_ssize_t max_array_len, Py_ssize_t max_map_len, - Py_ssize_t max_ext_len): - unpack_init(ctx) - ctx.user.use_list = use_list - ctx.user.raw = raw - ctx.user.strict_map_key = strict_map_key - ctx.user.object_hook = ctx.user.list_hook = NULL - ctx.user.max_str_len = max_str_len - ctx.user.max_bin_len = max_bin_len - ctx.user.max_array_len = max_array_len - ctx.user.max_map_len = max_map_len - ctx.user.max_ext_len = max_ext_len - - if object_hook is not None and object_pairs_hook is not None: - raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") - - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - ctx.user.object_hook = object_hook - - if object_pairs_hook is None: - ctx.user.has_pairs_hook = False - else: - if not PyCallable_Check(object_pairs_hook): - raise TypeError("object_pairs_hook must be a callable.") - ctx.user.object_hook = object_pairs_hook - ctx.user.has_pairs_hook = True - - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - ctx.user.list_hook = list_hook - - if ext_hook is not None: - if not PyCallable_Check(ext_hook): - raise TypeError("ext_hook must be a callable.") - ctx.user.ext_hook = ext_hook - - ctx.user.encoding = encoding - ctx.user.unicode_errors = unicode_errors - -def default_read_extended_type(typecode, data): - raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) - -cdef inline int get_data_from_buffer(object obj, - Py_buffer *view, - char **buf, - Py_ssize_t *buffer_len, - int *new_protocol) except 0: - cdef object contiguous - cdef Py_buffer tmp - if PyObject_CheckBuffer(obj): - new_protocol[0] = 1 - if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - raise - if view.itemsize != 1: - PyBuffer_Release(view) - raise BufferError("cannot unpack from multi-byte object") - if PyBuffer_IsContiguous(view, b'A') == 0: - PyBuffer_Release(view) - # create a contiguous copy and get buffer - contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') - PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) - # view must hold the only reference to contiguous, - # so memory is freed when view is released - Py_DECREF(contiguous) - buffer_len[0] = view.len - buf[0] = view.buf - return 1 - else: - new_protocol[0] = 0 - if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: - raise BufferError("could not get memoryview") - PyErr_WarnEx(RuntimeWarning, - "using old buffer interface to unpack %s; " - "this leads to unpacking errors if slicing is used and " - "will be removed in a future version" % type(obj), - 1) - return 1 - -def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw=True, bint strict_map_key=False, - encoding=None, unicode_errors=None, - object_pairs_hook=None, ext_hook=ExtType, - Py_ssize_t max_str_len=-1, - Py_ssize_t max_bin_len=-1, - Py_ssize_t max_array_len=-1, - Py_ssize_t max_map_len=-1, - Py_ssize_t max_ext_len=-1): - """ - Unpack packed_bytes to object. Returns an unpacked object. - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``ValueError`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. - - See :class:`Unpacker` for options. - - *max_xxx_len* options are configured automatically from ``len(packed)``. - """ - cdef unpack_context ctx - cdef Py_ssize_t off = 0 - cdef int ret - - cdef Py_buffer view - cdef char* buf = NULL - cdef Py_ssize_t buf_len - cdef const char* cenc = NULL - cdef const char* cerr = NULL - cdef int new_protocol = 0 - - if encoding is not None: - PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - cenc = encoding - - if unicode_errors is not None: - cerr = unicode_errors - - get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) - - if max_str_len == -1: - max_str_len = buf_len - if max_bin_len == -1: - max_bin_len = buf_len - if max_array_len == -1: - max_array_len = buf_len - if max_map_len == -1: - max_map_len = buf_len//2 - if max_ext_len == -1: - max_ext_len = buf_len - - try: - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, raw, strict_map_key, cenc, cerr, - max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) - ret = unpack_construct(&ctx, buf, buf_len, &off) - finally: - if new_protocol: - PyBuffer_Release(&view); - - if ret == 1: - obj = unpack_data(&ctx) - if off < buf_len: - raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - return obj - unpack_clear(&ctx) - if ret == 0: - raise ValueError("Unpack failed: incomplete input") - elif ret == -2: - raise FormatError - elif ret == -3: - raise StackError - raise ValueError("Unpack failed: error = %d" % (ret,)) - - -def unpack(object stream, **kwargs): - PyErr_WarnEx( - DeprecationWarning, - "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) - data = stream.read() - return unpackb(data, **kwargs) - - -cdef class Unpacker(object): - """Streaming unpacker. - - Arguments: - - :param file_like: - File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - - :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) - - :param bool use_list: - If true, unpack msgpack array to Python list. - Otherwise, unpack to Python tuple. (default: True) - - :param bool raw: - If true, unpack msgpack raw to Python bytes (default). - Otherwise, unpack to Python str (or unicode on Python 2) by decoding - with UTF-8 encoding (recommended). - Currently, the default is true, but it will be changed to false in - near future. So you must specify it explicitly for keeping backward - compatibility. - - *encoding* option which is deprecated overrides this option. - - :param bool strict_map_key: - If true, only str or bytes are accepted for map (dict) keys. - It's False by default for backward-compatibility. - But it will be True from msgpack 1.0. - - :param callable object_hook: - When specified, it should be callable. - Unpacker calls it with a dict argument after unpacking msgpack map. - (See also simplejson) - - :param callable object_pairs_hook: - When specified, it should be callable. - Unpacker calls it with a list of key-value pairs after unpacking msgpack map. - (See also simplejson) - - :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). - Raises `BufferFull` exception when it is insufficient. - You should set this parameter when unpacking data from untrusted source. - - :param int max_str_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) - - :param int max_bin_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) - - :param int max_array_len: - Limits max length of array. (default: max_buffer_size or 128*1024) - - :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2 or 32*1024) - - :param int max_ext_len: - Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) - - :param str encoding: - Deprecated, use ``raw=False`` instead. - Encoding used for decoding msgpack raw. - If it is None (default), msgpack raw is deserialized to Python bytes. - - :param str unicode_errors: - Error handler used for decoding str type. (default: `'strict'`) - - - Example of streaming deserialize from file-like object:: - - unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) - for o in unpacker: - process(o) - - Example of streaming deserialize from socket:: - - unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) - while True: - buf = sock.recv(1024**2) - if not buf: - break - unpacker.feed(buf) - for o in unpacker: - process(o) - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``OutOfData`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. - """ - cdef unpack_context ctx - cdef char* buf - cdef Py_ssize_t buf_size, buf_head, buf_tail - cdef object file_like - cdef object file_like_read - cdef Py_ssize_t read_size - # To maintain refcnt. - cdef object object_hook, object_pairs_hook, list_hook, ext_hook - cdef object encoding, unicode_errors - cdef Py_ssize_t max_buffer_size - cdef uint64_t stream_offset - - def __cinit__(self): - self.buf = NULL - - def __dealloc__(self): - PyMem_Free(self.buf) - self.buf = NULL - - def __init__(self, file_like=None, Py_ssize_t read_size=0, - bint use_list=True, bint raw=True, bint strict_map_key=False, - object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, - object ext_hook=ExtType, - Py_ssize_t max_str_len=-1, - Py_ssize_t max_bin_len=-1, - Py_ssize_t max_array_len=-1, - Py_ssize_t max_map_len=-1, - Py_ssize_t max_ext_len=-1): - cdef const char *cenc=NULL, - cdef const char *cerr=NULL - - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.list_hook = list_hook - self.ext_hook = ext_hook - - self.file_like = file_like - if file_like: - self.file_like_read = file_like.read - if not PyCallable_Check(self.file_like_read): - raise TypeError("`file_like.read` must be a callable.") - - if max_str_len == -1: - max_str_len = max_buffer_size or 1024*1024 - if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024*1024 - if max_array_len == -1: - max_array_len = max_buffer_size or 128*1024 - if max_map_len == -1: - max_map_len = max_buffer_size//2 or 32*1024 - if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024*1024 - - if not max_buffer_size: - max_buffer_size = INT_MAX - if read_size > max_buffer_size: - raise ValueError("read_size should be less or equal to max_buffer_size") - if not read_size: - read_size = min(max_buffer_size, 1024**2) - self.max_buffer_size = max_buffer_size - self.read_size = read_size - self.buf = PyMem_Malloc(read_size) - if self.buf == NULL: - raise MemoryError("Unable to allocate internal buffer.") - self.buf_size = read_size - self.buf_head = 0 - self.buf_tail = 0 - self.stream_offset = 0 - - if encoding is not None: - PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - self.encoding = encoding - cenc = encoding - - if unicode_errors is not None: - self.unicode_errors = unicode_errors - cerr = unicode_errors - - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, raw, strict_map_key, cenc, cerr, - max_str_len, max_bin_len, max_array_len, - max_map_len, max_ext_len) - - def feed(self, object next_bytes): - """Append `next_bytes` to internal buffer.""" - cdef Py_buffer pybuff - cdef int new_protocol = 0 - cdef char* buf - cdef Py_ssize_t buf_len - - if self.file_like is not None: - raise AssertionError( - "unpacker.feed() is not be able to use with `file_like`.") - - get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol) - try: - self.append_buffer(buf, buf_len) - finally: - if new_protocol: - PyBuffer_Release(&pybuff) - - cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): - cdef: - char* buf = self.buf - char* new_buf - Py_ssize_t head = self.buf_head - Py_ssize_t tail = self.buf_tail - Py_ssize_t buf_size = self.buf_size - Py_ssize_t new_size - - if tail + _buf_len > buf_size: - if ((tail - head) + _buf_len) <= buf_size: - # move to front. - memmove(buf, buf + head, tail - head) - tail -= head - head = 0 - else: - # expand buffer. - new_size = (tail-head) + _buf_len - if new_size > self.max_buffer_size: - raise BufferFull - new_size = min(new_size*2, self.max_buffer_size) - new_buf = PyMem_Malloc(new_size) - if new_buf == NULL: - # self.buf still holds old buffer and will be freed during - # obj destruction - raise MemoryError("Unable to enlarge internal buffer.") - memcpy(new_buf, buf + head, tail - head) - PyMem_Free(buf) - - buf = new_buf - buf_size = new_size - tail -= head - head = 0 - - memcpy(buf + tail, (_buf), _buf_len) - self.buf = buf - self.buf_head = head - self.buf_size = buf_size - self.buf_tail = tail + _buf_len - - cdef read_from_file(self): - next_bytes = self.file_like_read( - min(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) - if next_bytes: - self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) - else: - self.file_like = None - - cdef object _unpack(self, execute_fn execute, bint iter=0): - cdef int ret - cdef object obj - cdef Py_ssize_t prev_head - - if self.buf_head >= self.buf_tail and self.file_like is not None: - self.read_from_file() - - while 1: - prev_head = self.buf_head - if prev_head >= self.buf_tail: - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head - - if ret == 1: - obj = unpack_data(&self.ctx) - unpack_init(&self.ctx) - return obj - elif ret == 0: - if self.file_like is not None: - self.read_from_file() - continue - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - elif ret == -2: - raise FormatError - elif ret == -3: - raise StackError - else: - raise ValueError("Unpack failed: error = %d" % (ret,)) - - def read_bytes(self, Py_ssize_t nbytes): - """Read a specified number of raw bytes from the stream""" - cdef Py_ssize_t nread - nread = min(self.buf_tail - self.buf_head, nbytes) - ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - self.buf_head += nread - if len(ret) < nbytes and self.file_like is not None: - ret += self.file_like.read(nbytes - len(ret)) - return ret - - def unpack(self): - """Unpack one object - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(unpack_construct) - - def skip(self): - """Read and ignore one object, returning None - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(unpack_skip) - - def read_array_header(self): - """assuming the next object is an array, return its size n, such that - the next n unpack() calls will iterate over its contents. - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(read_array_header) - - def read_map_header(self): - """assuming the next object is a map, return its size n, such that the - next n * 2 unpack() calls will iterate over its key-value pairs. - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(read_map_header) - - def tell(self): - return self.stream_offset - - def __iter__(self): - return self - - def __next__(self): - return self._unpack(unpack_construct, 1) - - # for debug. - #def _buf(self): - # return PyString_FromStringAndSize(self.buf, self.buf_tail) - - #def _off(self): - # return self.buf_head diff --git a/ddtrace/vendor/msgpack/_version.py b/ddtrace/vendor/msgpack/_version.py deleted file mode 100644 index 926c5e7b02b..00000000000 --- a/ddtrace/vendor/msgpack/_version.py +++ /dev/null @@ -1 +0,0 @@ -version = (0, 6, 1) diff --git a/ddtrace/vendor/msgpack/buff_converter.h b/ddtrace/vendor/msgpack/buff_converter.h deleted file mode 100644 index bc7227ae9dd..00000000000 --- a/ddtrace/vendor/msgpack/buff_converter.h +++ /dev/null @@ -1,28 +0,0 @@ -#include "Python.h" - -/* cython does not support this preprocessor check => write it in raw C */ -#if PY_MAJOR_VERSION == 2 -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - return PyBuffer_FromMemory(buff, size); -} - -#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3) -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - return PyMemoryView_FromMemory(buff, size, PyBUF_READ); -} -#else -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - Py_buffer pybuf; - if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) { - return NULL; - } - - return PyMemoryView_FromBuffer(&pybuf); -} -#endif diff --git a/ddtrace/vendor/msgpack/exceptions.py b/ddtrace/vendor/msgpack/exceptions.py deleted file mode 100644 index d6d2615cfdd..00000000000 --- a/ddtrace/vendor/msgpack/exceptions.py +++ /dev/null @@ -1,48 +0,0 @@ -class UnpackException(Exception): - """Base class for some exceptions raised while unpacking. - - NOTE: unpack may raise exception other than subclass of - UnpackException. If you want to catch all error, catch - Exception instead. - """ - - -class BufferFull(UnpackException): - pass - - -class OutOfData(UnpackException): - pass - - -class FormatError(ValueError, UnpackException): - """Invalid msgpack format""" - - -class StackError(ValueError, UnpackException): - """Too nested""" - - -# Deprecated. Use ValueError instead -UnpackValueError = ValueError - - -class ExtraData(UnpackValueError): - """ExtraData is raised when there is trailing data. - - This exception is raised while only one-shot (not streaming) - unpack. - """ - - def __init__(self, unpacked, extra): - self.unpacked = unpacked - self.extra = extra - - def __str__(self): - return "unpack(b) received extra data." - - -# Deprecated. Use Exception instead to catch all exception during packing. -PackException = Exception -PackValueError = ValueError -PackOverflowError = OverflowError diff --git a/ddtrace/vendor/msgpack/fallback.py b/ddtrace/vendor/msgpack/fallback.py deleted file mode 100644 index 3836e830b8f..00000000000 --- a/ddtrace/vendor/msgpack/fallback.py +++ /dev/null @@ -1,1027 +0,0 @@ -"""Fallback pure Python implementation of msgpack""" - -import sys -import struct -import warnings - - -if sys.version_info[0] == 2: - PY2 = True - int_types = (int, long) - def dict_iteritems(d): - return d.iteritems() -else: - PY2 = False - int_types = int - unicode = str - xrange = range - def dict_iteritems(d): - return d.items() - -if sys.version_info < (3, 5): - # Ugly hack... - RecursionError = RuntimeError - - def _is_recursionerror(e): - return len(e.args) == 1 and isinstance(e.args[0], str) and \ - e.args[0].startswith('maximum recursion depth exceeded') -else: - def _is_recursionerror(e): - return True - -if hasattr(sys, 'pypy_version_info'): - # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own - # StringBuilder is fastest. - from __pypy__ import newlist_hint - try: - from __pypy__.builders import BytesBuilder as StringBuilder - except ImportError: - from __pypy__.builders import StringBuilder - USING_STRINGBUILDER = True - class StringIO(object): - def __init__(self, s=b''): - if s: - self.builder = StringBuilder(len(s)) - self.builder.append(s) - else: - self.builder = StringBuilder() - def write(self, s): - if isinstance(s, memoryview): - s = s.tobytes() - elif isinstance(s, bytearray): - s = bytes(s) - self.builder.append(s) - def getvalue(self): - return self.builder.build() -else: - USING_STRINGBUILDER = False - from io import BytesIO as StringIO - newlist_hint = lambda size: [] - - -from .exceptions import ( - BufferFull, - OutOfData, - ExtraData, - FormatError, - StackError, -) - -from . import ExtType - - -EX_SKIP = 0 -EX_CONSTRUCT = 1 -EX_READ_ARRAY_HEADER = 2 -EX_READ_MAP_HEADER = 3 - -TYPE_IMMEDIATE = 0 -TYPE_ARRAY = 1 -TYPE_MAP = 2 -TYPE_RAW = 3 -TYPE_BIN = 4 -TYPE_EXT = 5 - -DEFAULT_RECURSE_LIMIT = 511 - - -def _check_type_strict(obj, t, type=type, tuple=tuple): - if type(t) is tuple: - return type(obj) in t - else: - return type(obj) is t - - -def _get_data_from_buffer(obj): - try: - view = memoryview(obj) - except TypeError: - # try to use legacy buffer protocol if 2.7, otherwise re-raise - if PY2: - view = memoryview(buffer(obj)) - warnings.warn("using old buffer interface to unpack %s; " - "this leads to unpacking errors if slicing is used and " - "will be removed in a future version" % type(obj), - RuntimeWarning, stacklevel=3) - else: - raise - if view.itemsize != 1: - raise ValueError("cannot unpack from multi-byte object") - return view - - -def unpack(stream, **kwargs): - warnings.warn( - "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", - DeprecationWarning, stacklevel=2) - data = stream.read() - return unpackb(data, **kwargs) - - -def unpackb(packed, **kwargs): - """ - Unpack an object from `packed`. - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``ValueError`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. - - See :class:`Unpacker` for options. - """ - unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs) - unpacker.feed(packed) - try: - ret = unpacker._unpack() - except OutOfData: - raise ValueError("Unpack failed: incomplete input") - except RecursionError as e: - if _is_recursionerror(e): - raise StackError - raise - if unpacker._got_extradata(): - raise ExtraData(ret, unpacker._get_extradata()) - return ret - - -if sys.version_info < (2, 7, 6): - def _unpack_from(f, b, o=0): - """Explicit typcast for legacy struct.unpack_from""" - return struct.unpack_from(f, bytes(b), o) -else: - _unpack_from = struct.unpack_from - - -class Unpacker(object): - """Streaming unpacker. - - arguments: - - :param file_like: - File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - - :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) - - :param bool use_list: - If true, unpack msgpack array to Python list. - Otherwise, unpack to Python tuple. (default: True) - - :param bool raw: - If true, unpack msgpack raw to Python bytes (default). - Otherwise, unpack to Python str (or unicode on Python 2) by decoding - with UTF-8 encoding (recommended). - Currently, the default is true, but it will be changed to false in - near future. So you must specify it explicitly for keeping backward - compatibility. - - *encoding* option which is deprecated overrides this option. - - :param bool strict_map_key: - If true, only str or bytes are accepted for map (dict) keys. - It's False by default for backward-compatibility. - But it will be True from msgpack 1.0. - - :param callable object_hook: - When specified, it should be callable. - Unpacker calls it with a dict argument after unpacking msgpack map. - (See also simplejson) - - :param callable object_pairs_hook: - When specified, it should be callable. - Unpacker calls it with a list of key-value pairs after unpacking msgpack map. - (See also simplejson) - - :param str encoding: - Encoding used for decoding msgpack raw. - If it is None (default), msgpack raw is deserialized to Python bytes. - - :param str unicode_errors: - (deprecated) Used for decoding msgpack raw with *encoding*. - (default: `'strict'`) - - :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). - Raises `BufferFull` exception when it is insufficient. - You should set this parameter when unpacking data from untrusted source. - - :param int max_str_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) - - :param int max_bin_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) - - :param int max_array_len: - Limits max length of array. - (default: max_buffer_size or 128*1024) - - :param int max_map_len: - Limits max length of map. - (default: max_buffer_size//2 or 32*1024) - - :param int max_ext_len: - Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) - - Example of streaming deserialize from file-like object:: - - unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) - for o in unpacker: - process(o) - - Example of streaming deserialize from socket:: - - unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) - while True: - buf = sock.recv(1024**2) - if not buf: - break - unpacker.feed(buf) - for o in unpacker: - process(o) - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``OutOfData`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. - """ - - def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False, - object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors=None, max_buffer_size=0, - ext_hook=ExtType, - max_str_len=-1, - max_bin_len=-1, - max_array_len=-1, - max_map_len=-1, - max_ext_len=-1): - if encoding is not None: - warnings.warn( - "encoding is deprecated, Use raw=False instead.", - DeprecationWarning, stacklevel=2) - - if unicode_errors is None: - unicode_errors = 'strict' - - if file_like is None: - self._feeding = True - else: - if not callable(file_like.read): - raise TypeError("`file_like.read` must be callable") - self.file_like = file_like - self._feeding = False - - #: array of bytes fed. - self._buffer = bytearray() - #: Which position we currently reads - self._buff_i = 0 - - # When Unpacker is used as an iterable, between the calls to next(), - # the buffer is not "consumed" completely, for efficiency sake. - # Instead, it is done sloppily. To make sure we raise BufferFull at - # the correct moments, we have to keep track of how sloppy we were. - # Furthermore, when the buffer is incomplete (that is: in the case - # we raise an OutOfData) we need to rollback the buffer to the correct - # state, which _buf_checkpoint records. - self._buf_checkpoint = 0 - - if max_str_len == -1: - max_str_len = max_buffer_size or 1024*1024 - if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024*1024 - if max_array_len == -1: - max_array_len = max_buffer_size or 128*1024 - if max_map_len == -1: - max_map_len = max_buffer_size//2 or 32*1024 - if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024*1024 - - self._max_buffer_size = max_buffer_size or 2**31-1 - if read_size > self._max_buffer_size: - raise ValueError("read_size must be smaller than max_buffer_size") - self._read_size = read_size or min(self._max_buffer_size, 16*1024) - self._raw = bool(raw) - self._strict_map_key = bool(strict_map_key) - self._encoding = encoding - self._unicode_errors = unicode_errors - self._use_list = use_list - self._list_hook = list_hook - self._object_hook = object_hook - self._object_pairs_hook = object_pairs_hook - self._ext_hook = ext_hook - self._max_str_len = max_str_len - self._max_bin_len = max_bin_len - self._max_array_len = max_array_len - self._max_map_len = max_map_len - self._max_ext_len = max_ext_len - self._stream_offset = 0 - - if list_hook is not None and not callable(list_hook): - raise TypeError('`list_hook` is not callable') - if object_hook is not None and not callable(object_hook): - raise TypeError('`object_hook` is not callable') - if object_pairs_hook is not None and not callable(object_pairs_hook): - raise TypeError('`object_pairs_hook` is not callable') - if object_hook is not None and object_pairs_hook is not None: - raise TypeError("object_pairs_hook and object_hook are mutually " - "exclusive") - if not callable(ext_hook): - raise TypeError("`ext_hook` is not callable") - - def feed(self, next_bytes): - assert self._feeding - view = _get_data_from_buffer(next_bytes) - if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size): - raise BufferFull - - # Strip buffer before checkpoint before reading file. - if self._buf_checkpoint > 0: - del self._buffer[:self._buf_checkpoint] - self._buff_i -= self._buf_checkpoint - self._buf_checkpoint = 0 - - # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython - self._buffer.extend(view) - - def _consume(self): - """ Gets rid of the used parts of the buffer. """ - self._stream_offset += self._buff_i - self._buf_checkpoint - self._buf_checkpoint = self._buff_i - - def _got_extradata(self): - return self._buff_i < len(self._buffer) - - def _get_extradata(self): - return self._buffer[self._buff_i:] - - def read_bytes(self, n): - return self._read(n) - - def _read(self, n): - # (int) -> bytearray - self._reserve(n) - i = self._buff_i - self._buff_i = i+n - return self._buffer[i:i+n] - - def _reserve(self, n): - remain_bytes = len(self._buffer) - self._buff_i - n - - # Fast path: buffer has n bytes already - if remain_bytes >= 0: - return - - if self._feeding: - self._buff_i = self._buf_checkpoint - raise OutOfData - - # Strip buffer before checkpoint before reading file. - if self._buf_checkpoint > 0: - del self._buffer[:self._buf_checkpoint] - self._buff_i -= self._buf_checkpoint - self._buf_checkpoint = 0 - - # Read from file - remain_bytes = -remain_bytes - while remain_bytes > 0: - to_read_bytes = max(self._read_size, remain_bytes) - read_data = self.file_like.read(to_read_bytes) - if not read_data: - break - assert isinstance(read_data, bytes) - self._buffer += read_data - remain_bytes -= len(read_data) - - if len(self._buffer) < n + self._buff_i: - self._buff_i = 0 # rollback - raise OutOfData - - def _read_header(self, execute=EX_CONSTRUCT): - typ = TYPE_IMMEDIATE - n = 0 - obj = None - self._reserve(1) - b = self._buffer[self._buff_i] - self._buff_i += 1 - if b & 0b10000000 == 0: - obj = b - elif b & 0b11100000 == 0b11100000: - obj = -1 - (b ^ 0xff) - elif b & 0b11100000 == 0b10100000: - n = b & 0b00011111 - typ = TYPE_RAW - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b & 0b11110000 == 0b10010000: - n = b & 0b00001111 - typ = TYPE_ARRAY - if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b & 0b11110000 == 0b10000000: - n = b & 0b00001111 - typ = TYPE_MAP - if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - elif b == 0xc0: - obj = None - elif b == 0xc2: - obj = False - elif b == 0xc3: - obj = True - elif b == 0xc4: - typ = TYPE_BIN - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xc5: - typ = TYPE_BIN - self._reserve(2) - n = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xc6: - typ = TYPE_BIN - self._reserve(4) - n = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xc7: # ext 8 - typ = TYPE_EXT - self._reserve(2) - L, n = _unpack_from('Bb', self._buffer, self._buff_i) - self._buff_i += 2 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xc8: # ext 16 - typ = TYPE_EXT - self._reserve(3) - L, n = _unpack_from('>Hb', self._buffer, self._buff_i) - self._buff_i += 3 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xc9: # ext 32 - typ = TYPE_EXT - self._reserve(5) - L, n = _unpack_from('>Ib', self._buffer, self._buff_i) - self._buff_i += 5 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xca: - self._reserve(4) - obj = _unpack_from(">f", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xcb: - self._reserve(8) - obj = _unpack_from(">d", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xcc: - self._reserve(1) - obj = self._buffer[self._buff_i] - self._buff_i += 1 - elif b == 0xcd: - self._reserve(2) - obj = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xce: - self._reserve(4) - obj = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xcf: - self._reserve(8) - obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xd0: - self._reserve(1) - obj = _unpack_from("b", self._buffer, self._buff_i)[0] - self._buff_i += 1 - elif b == 0xd1: - self._reserve(2) - obj = _unpack_from(">h", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xd2: - self._reserve(4) - obj = _unpack_from(">i", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xd3: - self._reserve(8) - obj = _unpack_from(">q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xd4: # fixext 1 - typ = TYPE_EXT - if self._max_ext_len < 1: - raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - self._reserve(2) - n, obj = _unpack_from("b1s", self._buffer, self._buff_i) - self._buff_i += 2 - elif b == 0xd5: # fixext 2 - typ = TYPE_EXT - if self._max_ext_len < 2: - raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - self._reserve(3) - n, obj = _unpack_from("b2s", self._buffer, self._buff_i) - self._buff_i += 3 - elif b == 0xd6: # fixext 4 - typ = TYPE_EXT - if self._max_ext_len < 4: - raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - self._reserve(5) - n, obj = _unpack_from("b4s", self._buffer, self._buff_i) - self._buff_i += 5 - elif b == 0xd7: # fixext 8 - typ = TYPE_EXT - if self._max_ext_len < 8: - raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - self._reserve(9) - n, obj = _unpack_from("b8s", self._buffer, self._buff_i) - self._buff_i += 9 - elif b == 0xd8: # fixext 16 - typ = TYPE_EXT - if self._max_ext_len < 16: - raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - self._reserve(17) - n, obj = _unpack_from("b16s", self._buffer, self._buff_i) - self._buff_i += 17 - elif b == 0xd9: - typ = TYPE_RAW - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xda: - typ = TYPE_RAW - self._reserve(2) - n, = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xdb: - typ = TYPE_RAW - self._reserve(4) - n, = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xdc: - typ = TYPE_ARRAY - self._reserve(2) - n, = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xdd: - typ = TYPE_ARRAY - self._reserve(4) - n, = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xde: - self._reserve(2) - n, = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP - elif b == 0xdf: - self._reserve(4) - n, = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP - else: - raise FormatError("Unknown header: 0x%x" % b) - return typ, n, obj - - def _unpack(self, execute=EX_CONSTRUCT): - typ, n, obj = self._read_header(execute) - - if execute == EX_READ_ARRAY_HEADER: - if typ != TYPE_ARRAY: - raise ValueError("Expected array") - return n - if execute == EX_READ_MAP_HEADER: - if typ != TYPE_MAP: - raise ValueError("Expected map") - return n - # TODO should we eliminate the recursion? - if typ == TYPE_ARRAY: - if execute == EX_SKIP: - for i in xrange(n): - # TODO check whether we need to call `list_hook` - self._unpack(EX_SKIP) - return - ret = newlist_hint(n) - for i in xrange(n): - ret.append(self._unpack(EX_CONSTRUCT)) - if self._list_hook is not None: - ret = self._list_hook(ret) - # TODO is the interaction between `list_hook` and `use_list` ok? - return ret if self._use_list else tuple(ret) - if typ == TYPE_MAP: - if execute == EX_SKIP: - for i in xrange(n): - # TODO check whether we need to call hooks - self._unpack(EX_SKIP) - self._unpack(EX_SKIP) - return - if self._object_pairs_hook is not None: - ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), - self._unpack(EX_CONSTRUCT)) - for _ in xrange(n)) - else: - ret = {} - for _ in xrange(n): - key = self._unpack(EX_CONSTRUCT) - if self._strict_map_key and type(key) not in (unicode, bytes): - raise ValueError("%s is not allowed for map key" % str(type(key))) - ret[key] = self._unpack(EX_CONSTRUCT) - if self._object_hook is not None: - ret = self._object_hook(ret) - return ret - if execute == EX_SKIP: - return - if typ == TYPE_RAW: - if self._encoding is not None: - obj = obj.decode(self._encoding, self._unicode_errors) - elif self._raw: - obj = bytes(obj) - else: - obj = obj.decode('utf_8') - return obj - if typ == TYPE_EXT: - return self._ext_hook(n, bytes(obj)) - if typ == TYPE_BIN: - return bytes(obj) - assert typ == TYPE_IMMEDIATE - return obj - - def __iter__(self): - return self - - def __next__(self): - try: - ret = self._unpack(EX_CONSTRUCT) - self._consume() - return ret - except OutOfData: - self._consume() - raise StopIteration - except RecursionError: - raise StackError - - next = __next__ - - def skip(self): - self._unpack(EX_SKIP) - self._consume() - - def unpack(self): - try: - ret = self._unpack(EX_CONSTRUCT) - except RecursionError: - raise StackError - self._consume() - return ret - - def read_array_header(self): - ret = self._unpack(EX_READ_ARRAY_HEADER) - self._consume() - return ret - - def read_map_header(self): - ret = self._unpack(EX_READ_MAP_HEADER) - self._consume() - return ret - - def tell(self): - return self._stream_offset - - -class Packer(object): - """ - MessagePack Packer - - usage: - - packer = Packer() - astream.write(packer.pack(a)) - astream.write(packer.pack(b)) - - Packer's constructor has some keyword arguments: - - :param callable default: - Convert user type to builtin type that Packer supports. - See also simplejson's document. - - :param bool use_single_float: - Use single precision float type for float. (default: False) - - :param bool autoreset: - Reset buffer after each pack and return its content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. - - :param bool use_bin_type: - Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. - - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializeable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. - - :param str encoding: - (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') - - :param str unicode_errors: - Error handler for encoding unicode. (default: 'strict') - """ - def __init__(self, default=None, encoding=None, unicode_errors=None, - use_single_float=False, autoreset=True, use_bin_type=False, - strict_types=False): - if encoding is None: - encoding = 'utf_8' - else: - warnings.warn( - "encoding is deprecated, Use raw=False instead.", - DeprecationWarning, stacklevel=2) - - if unicode_errors is None: - unicode_errors = 'strict' - - self._strict_types = strict_types - self._use_float = use_single_float - self._autoreset = autoreset - self._use_bin_type = use_bin_type - self._encoding = encoding - self._unicode_errors = unicode_errors - self._buffer = StringIO() - if default is not None: - if not callable(default): - raise TypeError("default must be callable") - self._default = default - - def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, - check=isinstance, check_type_strict=_check_type_strict): - default_used = False - if self._strict_types: - check = check_type_strict - list_types = list - else: - list_types = (list, tuple) - while True: - if nest_limit < 0: - raise ValueError("recursion limit exceeded") - if obj is None: - return self._buffer.write(b"\xc0") - if check(obj, bool): - if obj: - return self._buffer.write(b"\xc3") - return self._buffer.write(b"\xc2") - if check(obj, int_types): - if 0 <= obj < 0x80: - return self._buffer.write(struct.pack("B", obj)) - if -0x20 <= obj < 0: - return self._buffer.write(struct.pack("b", obj)) - if 0x80 <= obj <= 0xff: - return self._buffer.write(struct.pack("BB", 0xcc, obj)) - if -0x80 <= obj < 0: - return self._buffer.write(struct.pack(">Bb", 0xd0, obj)) - if 0xff < obj <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xcd, obj)) - if -0x8000 <= obj < -0x80: - return self._buffer.write(struct.pack(">Bh", 0xd1, obj)) - if 0xffff < obj <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xce, obj)) - if -0x80000000 <= obj < -0x8000: - return self._buffer.write(struct.pack(">Bi", 0xd2, obj)) - if 0xffffffff < obj <= 0xffffffffffffffff: - return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) - if -0x8000000000000000 <= obj < -0x80000000: - return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) - if not default_used and self._default is not None: - obj = self._default(obj) - default_used = True - continue - raise OverflowError("Integer value out of range") - if check(obj, (bytes, bytearray)): - n = len(obj) - if n >= 2**32: - raise ValueError("%s is too large" % type(obj).__name__) - self._pack_bin_header(n) - return self._buffer.write(obj) - if check(obj, unicode): - if self._encoding is None: - raise TypeError( - "Can't encode unicode string: " - "no encoding is specified") - obj = obj.encode(self._encoding, self._unicode_errors) - n = len(obj) - if n >= 2**32: - raise ValueError("String is too large") - self._pack_raw_header(n) - return self._buffer.write(obj) - if check(obj, memoryview): - n = len(obj) * obj.itemsize - if n >= 2**32: - raise ValueError("Memoryview is too large") - self._pack_bin_header(n) - return self._buffer.write(obj) - if check(obj, float): - if self._use_float: - return self._buffer.write(struct.pack(">Bf", 0xca, obj)) - return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) - if check(obj, ExtType): - code = obj.code - data = obj.data - assert isinstance(code, int) - assert isinstance(data, bytes) - L = len(data) - if L == 1: - self._buffer.write(b'\xd4') - elif L == 2: - self._buffer.write(b'\xd5') - elif L == 4: - self._buffer.write(b'\xd6') - elif L == 8: - self._buffer.write(b'\xd7') - elif L == 16: - self._buffer.write(b'\xd8') - elif L <= 0xff: - self._buffer.write(struct.pack(">BB", 0xc7, L)) - elif L <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xc8, L)) - else: - self._buffer.write(struct.pack(">BI", 0xc9, L)) - self._buffer.write(struct.pack("b", code)) - self._buffer.write(data) - return - if check(obj, list_types): - n = len(obj) - self._pack_array_header(n) - for i in xrange(n): - self._pack(obj[i], nest_limit - 1) - return - if check(obj, dict): - return self._pack_map_pairs(len(obj), dict_iteritems(obj), - nest_limit - 1) - if not default_used and self._default is not None: - obj = self._default(obj) - default_used = 1 - continue - raise TypeError("Cannot serialize %r" % (obj, )) - - def pack(self, obj): - try: - self._pack(obj) - except: - self._buffer = StringIO() # force reset - raise - if self._autoreset: - ret = self._buffer.getvalue() - self._buffer = StringIO() - return ret - - def pack_map_pairs(self, pairs): - self._pack_map_pairs(len(pairs), pairs) - if self._autoreset: - ret = self._buffer.getvalue() - self._buffer = StringIO() - return ret - - def pack_array_header(self, n): - if n >= 2**32: - raise ValueError - self._pack_array_header(n) - if self._autoreset: - ret = self._buffer.getvalue() - self._buffer = StringIO() - return ret - - def pack_map_header(self, n): - if n >= 2**32: - raise ValueError - self._pack_map_header(n) - if self._autoreset: - ret = self._buffer.getvalue() - self._buffer = StringIO() - return ret - - def pack_ext_type(self, typecode, data): - if not isinstance(typecode, int): - raise TypeError("typecode must have int type.") - if not 0 <= typecode <= 127: - raise ValueError("typecode should be 0-127") - if not isinstance(data, bytes): - raise TypeError("data must have bytes type") - L = len(data) - if L > 0xffffffff: - raise ValueError("Too large data") - if L == 1: - self._buffer.write(b'\xd4') - elif L == 2: - self._buffer.write(b'\xd5') - elif L == 4: - self._buffer.write(b'\xd6') - elif L == 8: - self._buffer.write(b'\xd7') - elif L == 16: - self._buffer.write(b'\xd8') - elif L <= 0xff: - self._buffer.write(b'\xc7' + struct.pack('B', L)) - elif L <= 0xffff: - self._buffer.write(b'\xc8' + struct.pack('>H', L)) - else: - self._buffer.write(b'\xc9' + struct.pack('>I', L)) - self._buffer.write(struct.pack('B', typecode)) - self._buffer.write(data) - - def _pack_array_header(self, n): - if n <= 0x0f: - return self._buffer.write(struct.pack('B', 0x90 + n)) - if n <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xdc, n)) - if n <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xdd, n)) - raise ValueError("Array is too large") - - def _pack_map_header(self, n): - if n <= 0x0f: - return self._buffer.write(struct.pack('B', 0x80 + n)) - if n <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xde, n)) - if n <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xdf, n)) - raise ValueError("Dict is too large") - - def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): - self._pack_map_header(n) - for (k, v) in pairs: - self._pack(k, nest_limit - 1) - self._pack(v, nest_limit - 1) - - def _pack_raw_header(self, n): - if n <= 0x1f: - self._buffer.write(struct.pack('B', 0xa0 + n)) - elif self._use_bin_type and n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xd9, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xda, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xdb, n)) - else: - raise ValueError('Raw is too large') - - def _pack_bin_header(self, n): - if not self._use_bin_type: - return self._pack_raw_header(n) - elif n <= 0xff: - return self._buffer.write(struct.pack('>BB', 0xc4, n)) - elif n <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xc5, n)) - elif n <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xc6, n)) - else: - raise ValueError('Bin is too large') - - def bytes(self): - """Return internal buffer contents as bytes object""" - return self._buffer.getvalue() - - def reset(self): - """Reset internal buffer. - - This method is usaful only when autoreset=False. - """ - self._buffer = StringIO() - - def getbuffer(self): - """Return view of internal buffer.""" - if USING_STRINGBUILDER or PY2: - return memoryview(self.bytes()) - else: - return self._buffer.getbuffer() diff --git a/ddtrace/vendor/msgpack/pack.h b/ddtrace/vendor/msgpack/pack.h deleted file mode 100644 index 4f3ce1d99ec..00000000000 --- a/ddtrace/vendor/msgpack/pack.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - * MessagePack for Python packing routine - * - * Copyright (C) 2009 Naoki INADA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "sysdep.h" -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef _MSC_VER -#define inline __inline -#endif - -typedef struct msgpack_packer { - char *buf; - size_t length; - size_t buf_size; - bool use_bin_type; -} msgpack_packer; - -typedef struct Packer Packer; - -static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) -{ - char* buf = pk->buf; - size_t bs = pk->buf_size; - size_t len = pk->length; - - if (len + l > bs) { - bs = (len + l) * 2; - buf = (char*)PyMem_Realloc(buf, bs); - if (!buf) { - PyErr_NoMemory(); - return -1; - } - } - memcpy(buf + len, data, l); - len += l; - - pk->buf = buf; - pk->buf_size = bs; - pk->length = len; - return 0; -} - -#define msgpack_pack_append_buffer(user, buf, len) \ - return msgpack_pack_write(user, (const char*)buf, len) - -#include "pack_template.h" - -// return -2 when o is too long -static inline int -msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) -{ -#if PY_MAJOR_VERSION >= 3 - assert(PyUnicode_Check(o)); - - Py_ssize_t len; - const char* buf = PyUnicode_AsUTF8AndSize(o, &len); - if (buf == NULL) - return -1; - - if (len > limit) { - return -2; - } - - int ret = msgpack_pack_raw(pk, len); - if (ret) return ret; - - return msgpack_pack_raw_body(pk, buf, len); -#else - PyObject *bytes; - Py_ssize_t len; - int ret; - - // py2 - bytes = PyUnicode_AsUTF8String(o); - if (bytes == NULL) - return -1; - - len = PyString_GET_SIZE(bytes); - if (len > limit) { - Py_DECREF(bytes); - return -2; - } - - ret = msgpack_pack_raw(pk, len); - if (ret) { - Py_DECREF(bytes); - return -1; - } - ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len); - Py_DECREF(bytes); - return ret; -#endif -} - -#ifdef __cplusplus -} -#endif diff --git a/ddtrace/vendor/msgpack/pack_template.h b/ddtrace/vendor/msgpack/pack_template.h deleted file mode 100644 index 69982f4d291..00000000000 --- a/ddtrace/vendor/msgpack/pack_template.h +++ /dev/null @@ -1,778 +0,0 @@ -/* - * MessagePack packing routine template - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#if defined(__LITTLE_ENDIAN__) -#define TAKE8_8(d) ((uint8_t*)&d)[0] -#define TAKE8_16(d) ((uint8_t*)&d)[0] -#define TAKE8_32(d) ((uint8_t*)&d)[0] -#define TAKE8_64(d) ((uint8_t*)&d)[0] -#elif defined(__BIG_ENDIAN__) -#define TAKE8_8(d) ((uint8_t*)&d)[0] -#define TAKE8_16(d) ((uint8_t*)&d)[1] -#define TAKE8_32(d) ((uint8_t*)&d)[3] -#define TAKE8_64(d) ((uint8_t*)&d)[7] -#endif - -#ifndef msgpack_pack_append_buffer -#error msgpack_pack_append_buffer callback is not defined -#endif - - -/* - * Integer - */ - -#define msgpack_pack_real_uint8(x, d) \ -do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ -} while(0) - -#define msgpack_pack_real_uint16(x, d) \ -do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ - } else if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } \ -} while(0) - -#define msgpack_pack_real_uint32(x, d) \ -do { \ - if(d < (1<<8)) { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1<<16)) { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } \ -} while(0) - -#define msgpack_pack_real_uint64(x, d) \ -do { \ - if(d < (1ULL<<8)) { \ - if(d < (1ULL<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1ULL<<16)) { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else if(d < (1ULL<<32)) { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else { \ - /* unsigned 64 */ \ - unsigned char buf[9]; \ - buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ - msgpack_pack_append_buffer(x, buf, 9); \ - } \ - } \ -} while(0) - -#define msgpack_pack_real_int8(x, d) \ -do { \ - if(d < -(1<<5)) { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } \ -} while(0) - -#define msgpack_pack_real_int16(x, d) \ -do { \ - if(d < -(1<<5)) { \ - if(d < -(1<<7)) { \ - /* signed 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ - } else { \ - if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } \ - } \ -} while(0) - -#define msgpack_pack_real_int32(x, d) \ -do { \ - if(d < -(1<<5)) { \ - if(d < -(1<<15)) { \ - /* signed 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else if(d < -(1<<7)) { \ - /* signed 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ - } else { \ - if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else if(d < (1<<16)) { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } \ -} while(0) - -#define msgpack_pack_real_int64(x, d) \ -do { \ - if(d < -(1LL<<5)) { \ - if(d < -(1LL<<15)) { \ - if(d < -(1LL<<31)) { \ - /* signed 64 */ \ - unsigned char buf[9]; \ - buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \ - msgpack_pack_append_buffer(x, buf, 9); \ - } else { \ - /* signed 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } else { \ - if(d < -(1<<7)) { \ - /* signed 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } \ - } else if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ - } else { \ - if(d < (1LL<<16)) { \ - if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } \ - } else { \ - if(d < (1LL<<32)) { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else { \ - /* unsigned 64 */ \ - unsigned char buf[9]; \ - buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ - msgpack_pack_append_buffer(x, buf, 9); \ - } \ - } \ - } \ -} while(0) - - -static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d) -{ - msgpack_pack_real_uint8(x, d); -} - -static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d) -{ - msgpack_pack_real_uint16(x, d); -} - -static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d) -{ - msgpack_pack_real_uint32(x, d); -} - -static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d) -{ - msgpack_pack_real_uint64(x, d); -} - -static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d) -{ - msgpack_pack_real_int8(x, d); -} - -static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d) -{ - msgpack_pack_real_int16(x, d); -} - -static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d) -{ - msgpack_pack_real_int32(x, d); -} - -static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) -{ - msgpack_pack_real_int64(x, d); -} - - -//#ifdef msgpack_pack_inline_func_cint - -static inline int msgpack_pack_short(msgpack_packer* x, short d) -{ -#if defined(SIZEOF_SHORT) -#if SIZEOF_SHORT == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_SHORT == 4 - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#elif defined(SHRT_MAX) -#if SHRT_MAX == 0x7fff - msgpack_pack_real_int16(x, d); -#elif SHRT_MAX == 0x7fffffff - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(short) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(short) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} -#endif -} - -static inline int msgpack_pack_int(msgpack_packer* x, int d) -{ -#if defined(SIZEOF_INT) -#if SIZEOF_INT == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_INT == 4 - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#elif defined(INT_MAX) -#if INT_MAX == 0x7fff - msgpack_pack_real_int16(x, d); -#elif INT_MAX == 0x7fffffff - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(int) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(int) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} -#endif -} - -static inline int msgpack_pack_long(msgpack_packer* x, long d) -{ -#if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#elif defined(LONG_MAX) -#if LONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} -#endif -} - -static inline int msgpack_pack_long_long(msgpack_packer* x, long long d) -{ -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#elif defined(LLONG_MAX) -#if LLONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LLONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(long long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long long) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d) -{ -#if defined(SIZEOF_SHORT) -#if SIZEOF_SHORT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(USHRT_MAX) -#if USHRT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif USHRT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned short) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned short) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d) -{ -#if defined(SIZEOF_INT) -#if SIZEOF_INT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(UINT_MAX) -#if UINT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif UINT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned int) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d) -{ -#if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULONG_MAX) -#if ULONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d) -{ -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULLONG_MAX) -#if ULLONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULLONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -//#undef msgpack_pack_inline_func_cint -//#endif - - - -/* - * Float - */ - -static inline int msgpack_pack_float(msgpack_packer* x, float d) -{ - unsigned char buf[5]; - buf[0] = 0xca; - _PyFloat_Pack4(d, &buf[1], 0); - msgpack_pack_append_buffer(x, buf, 5); -} - -static inline int msgpack_pack_double(msgpack_packer* x, double d) -{ - unsigned char buf[9]; - buf[0] = 0xcb; - _PyFloat_Pack8(d, &buf[1], 0); - msgpack_pack_append_buffer(x, buf, 9); -} - - -/* - * Nil - */ - -static inline int msgpack_pack_nil(msgpack_packer* x) -{ - static const unsigned char d = 0xc0; - msgpack_pack_append_buffer(x, &d, 1); -} - - -/* - * Boolean - */ - -static inline int msgpack_pack_true(msgpack_packer* x) -{ - static const unsigned char d = 0xc3; - msgpack_pack_append_buffer(x, &d, 1); -} - -static inline int msgpack_pack_false(msgpack_packer* x) -{ - static const unsigned char d = 0xc2; - msgpack_pack_append_buffer(x, &d, 1); -} - - -/* - * Array - */ - -static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n) -{ - if(n < 16) { - unsigned char d = 0x90 | n; - msgpack_pack_append_buffer(x, &d, 1); - } else if(n < 65536) { - unsigned char buf[3]; - buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5]; - buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n); - msgpack_pack_append_buffer(x, buf, 5); - } -} - - -/* - * Map - */ - -static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n) -{ - if(n < 16) { - unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); - } else if(n < 65536) { - unsigned char buf[3]; - buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5]; - buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n); - msgpack_pack_append_buffer(x, buf, 5); - } -} - - -/* - * Raw - */ - -static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) -{ - if (l < 32) { - unsigned char d = 0xa0 | (uint8_t)l; - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); - } else if (x->use_bin_type && l < 256) { // str8 is new format introduced with bin. - unsigned char buf[2] = {0xd9, (uint8_t)l}; - msgpack_pack_append_buffer(x, buf, 2); - } else if (l < 65536) { - unsigned char buf[3]; - buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5]; - buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l); - msgpack_pack_append_buffer(x, buf, 5); - } -} - -/* - * bin - */ -static inline int msgpack_pack_bin(msgpack_packer *x, size_t l) -{ - if (!x->use_bin_type) { - return msgpack_pack_raw(x, l); - } - if (l < 256) { - unsigned char buf[2] = {0xc4, (unsigned char)l}; - msgpack_pack_append_buffer(x, buf, 2); - } else if (l < 65536) { - unsigned char buf[3] = {0xc5}; - _msgpack_store16(&buf[1], (uint16_t)l); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5] = {0xc6}; - _msgpack_store32(&buf[1], (uint32_t)l); - msgpack_pack_append_buffer(x, buf, 5); - } -} - -static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l) -{ - if (l > 0) msgpack_pack_append_buffer(x, (const unsigned char*)b, l); - return 0; -} - -/* - * Ext - */ -static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l) -{ - if (l == 1) { - unsigned char buf[2]; - buf[0] = 0xd4; - buf[1] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 2); - } - else if(l == 2) { - unsigned char buf[2]; - buf[0] = 0xd5; - buf[1] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 2); - } - else if(l == 4) { - unsigned char buf[2]; - buf[0] = 0xd6; - buf[1] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 2); - } - else if(l == 8) { - unsigned char buf[2]; - buf[0] = 0xd7; - buf[1] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 2); - } - else if(l == 16) { - unsigned char buf[2]; - buf[0] = 0xd8; - buf[1] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 2); - } - else if(l < 256) { - unsigned char buf[3]; - buf[0] = 0xc7; - buf[1] = l; - buf[2] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 3); - } else if(l < 65536) { - unsigned char buf[4]; - buf[0] = 0xc8; - _msgpack_store16(&buf[1], (uint16_t)l); - buf[3] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 4); - } else { - unsigned char buf[6]; - buf[0] = 0xc9; - _msgpack_store32(&buf[1], (uint32_t)l); - buf[5] = (unsigned char)typecode; - msgpack_pack_append_buffer(x, buf, 6); - } - -} - - - -#undef msgpack_pack_append_buffer - -#undef TAKE8_8 -#undef TAKE8_16 -#undef TAKE8_32 -#undef TAKE8_64 - -#undef msgpack_pack_real_uint8 -#undef msgpack_pack_real_uint16 -#undef msgpack_pack_real_uint32 -#undef msgpack_pack_real_uint64 -#undef msgpack_pack_real_int8 -#undef msgpack_pack_real_int16 -#undef msgpack_pack_real_int32 -#undef msgpack_pack_real_int64 diff --git a/ddtrace/vendor/msgpack/setup.py b/ddtrace/vendor/msgpack/setup.py deleted file mode 100644 index addc81cbd99..00000000000 --- a/ddtrace/vendor/msgpack/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -__all__ = ["get_extensions"] - -from setuptools import Extension -import sys - - -def get_extensions(): - libraries = [] - if sys.platform == "win32": - libraries.append("ws2_32") - - macros = [] - if sys.byteorder == "big": - macros = [("__BIG_ENDIAN__", "1")] - else: - macros = [("__LITTLE_ENDIAN__", "1")] - - ext = Extension( - "ddtrace.vendor.msgpack._cmsgpack", - sources=["ddtrace/vendor/msgpack/_cmsgpack.cpp"], - libraries=libraries, - include_dirs=["ddtrace/vendor/"], - define_macros=macros, - ) - - return [ext] diff --git a/ddtrace/vendor/msgpack/sysdep.h b/ddtrace/vendor/msgpack/sysdep.h deleted file mode 100644 index ed9c1bc0b80..00000000000 --- a/ddtrace/vendor/msgpack/sysdep.h +++ /dev/null @@ -1,194 +0,0 @@ -/* - * MessagePack system dependencies - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_SYSDEP_H__ -#define MSGPACK_SYSDEP_H__ - -#include -#include -#if defined(_MSC_VER) && _MSC_VER < 1600 -typedef __int8 int8_t; -typedef unsigned __int8 uint8_t; -typedef __int16 int16_t; -typedef unsigned __int16 uint16_t; -typedef __int32 int32_t; -typedef unsigned __int32 uint32_t; -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -#elif defined(_MSC_VER) // && _MSC_VER >= 1600 -#include -#else -#include -#include -#endif - -#ifdef _WIN32 -#define _msgpack_atomic_counter_header -typedef long _msgpack_atomic_counter_t; -#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) -#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) -#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41) -#define _msgpack_atomic_counter_header "gcc_atomic.h" -#else -typedef unsigned int _msgpack_atomic_counter_t; -#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) -#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) -#endif - -#ifdef _WIN32 - -#ifdef __cplusplus -/* numeric_limits::min,max */ -#ifdef max -#undef max -#endif -#ifdef min -#undef min -#endif -#endif - -#else -#include /* __BYTE_ORDER */ -#endif - -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#elif _WIN32 -#define __LITTLE_ENDIAN__ -#endif -#endif - - -#ifdef __LITTLE_ENDIAN__ - -#ifdef _WIN32 -# if defined(ntohs) -# define _msgpack_be16(x) ntohs(x) -# elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400) -# define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x)) -# else -# define _msgpack_be16(x) ( \ - ((((uint16_t)x) << 8) ) | \ - ((((uint16_t)x) >> 8) ) ) -# endif -#else -# define _msgpack_be16(x) ntohs(x) -#endif - -#ifdef _WIN32 -# if defined(ntohl) -# define _msgpack_be32(x) ntohl(x) -# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400) -# define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x)) -# else -# define _msgpack_be32(x) \ - ( ((((uint32_t)x) << 24) ) | \ - ((((uint32_t)x) << 8) & 0x00ff0000U ) | \ - ((((uint32_t)x) >> 8) & 0x0000ff00U ) | \ - ((((uint32_t)x) >> 24) ) ) -# endif -#else -# define _msgpack_be32(x) ntohl(x) -#endif - -#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400) -# define _msgpack_be64(x) (_byteswap_uint64(x)) -#elif defined(bswap_64) -# define _msgpack_be64(x) bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) -#else -#define _msgpack_be64(x) \ - ( ((((uint64_t)x) << 56) ) | \ - ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ - ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ - ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ - ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ - ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ - ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ - ((((uint64_t)x) >> 56) ) ) -#endif - -#define _msgpack_load16(cast, from) ((cast)( \ - (((uint16_t)((uint8_t*)(from))[0]) << 8) | \ - (((uint16_t)((uint8_t*)(from))[1]) ) )) - -#define _msgpack_load32(cast, from) ((cast)( \ - (((uint32_t)((uint8_t*)(from))[0]) << 24) | \ - (((uint32_t)((uint8_t*)(from))[1]) << 16) | \ - (((uint32_t)((uint8_t*)(from))[2]) << 8) | \ - (((uint32_t)((uint8_t*)(from))[3]) ) )) - -#define _msgpack_load64(cast, from) ((cast)( \ - (((uint64_t)((uint8_t*)(from))[0]) << 56) | \ - (((uint64_t)((uint8_t*)(from))[1]) << 48) | \ - (((uint64_t)((uint8_t*)(from))[2]) << 40) | \ - (((uint64_t)((uint8_t*)(from))[3]) << 32) | \ - (((uint64_t)((uint8_t*)(from))[4]) << 24) | \ - (((uint64_t)((uint8_t*)(from))[5]) << 16) | \ - (((uint64_t)((uint8_t*)(from))[6]) << 8) | \ - (((uint64_t)((uint8_t*)(from))[7]) ) )) - -#else - -#define _msgpack_be16(x) (x) -#define _msgpack_be32(x) (x) -#define _msgpack_be64(x) (x) - -#define _msgpack_load16(cast, from) ((cast)( \ - (((uint16_t)((uint8_t*)from)[0]) << 8) | \ - (((uint16_t)((uint8_t*)from)[1]) ) )) - -#define _msgpack_load32(cast, from) ((cast)( \ - (((uint32_t)((uint8_t*)from)[0]) << 24) | \ - (((uint32_t)((uint8_t*)from)[1]) << 16) | \ - (((uint32_t)((uint8_t*)from)[2]) << 8) | \ - (((uint32_t)((uint8_t*)from)[3]) ) )) - -#define _msgpack_load64(cast, from) ((cast)( \ - (((uint64_t)((uint8_t*)from)[0]) << 56) | \ - (((uint64_t)((uint8_t*)from)[1]) << 48) | \ - (((uint64_t)((uint8_t*)from)[2]) << 40) | \ - (((uint64_t)((uint8_t*)from)[3]) << 32) | \ - (((uint64_t)((uint8_t*)from)[4]) << 24) | \ - (((uint64_t)((uint8_t*)from)[5]) << 16) | \ - (((uint64_t)((uint8_t*)from)[6]) << 8) | \ - (((uint64_t)((uint8_t*)from)[7]) ) )) -#endif - - -#define _msgpack_store16(to, num) \ - do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0) -#define _msgpack_store32(to, num) \ - do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0) -#define _msgpack_store64(to, num) \ - do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0) - -/* -#define _msgpack_load16(cast, from) \ - ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); }) -#define _msgpack_load32(cast, from) \ - ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); }) -#define _msgpack_load64(cast, from) \ - ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); }) -*/ - - -#endif /* msgpack/sysdep.h */ diff --git a/ddtrace/vendor/msgpack/unpack.h b/ddtrace/vendor/msgpack/unpack.h deleted file mode 100644 index 85dbbed5b67..00000000000 --- a/ddtrace/vendor/msgpack/unpack.h +++ /dev/null @@ -1,287 +0,0 @@ -/* - * MessagePack for Python unpacking routine - * - * Copyright (C) 2009 Naoki INADA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define MSGPACK_EMBED_STACK_SIZE (1024) -#include "unpack_define.h" - -typedef struct unpack_user { - bool use_list; - bool raw; - bool has_pairs_hook; - bool strict_map_key; - PyObject *object_hook; - PyObject *list_hook; - PyObject *ext_hook; - const char *encoding; - const char *unicode_errors; - Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len; -} unpack_user; - -typedef PyObject* msgpack_unpack_object; -struct unpack_context; -typedef struct unpack_context unpack_context; -typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off); - -static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) -{ - return NULL; -} - -static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ - PyObject *p = PyInt_FromLong((long)d); - if (!p) - return -1; - *o = p; - return 0; -} -static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ - return unpack_callback_uint16(u, d, o); -} - - -static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) -{ - PyObject *p = PyInt_FromSize_t((size_t)d); - if (!p) - return -1; - *o = p; - return 0; -} - -static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ - PyObject *p; - if (d > LONG_MAX) { - p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); - } else { - p = PyInt_FromLong((long)d); - } - if (!p) - return -1; - *o = p; - return 0; -} - -static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ - PyObject *p = PyInt_FromLong(d); - if (!p) - return -1; - *o = p; - return 0; -} - -static inline int unpack_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ - return unpack_callback_int32(u, d, o); -} - -static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ - return unpack_callback_int32(u, d, o); -} - -static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ - PyObject *p; - if (d > LONG_MAX || d < LONG_MIN) { - p = PyLong_FromLongLong((PY_LONG_LONG)d); - } else { - p = PyInt_FromLong((long)d); - } - *o = p; - return 0; -} - -static inline int unpack_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ - PyObject *p = PyFloat_FromDouble(d); - if (!p) - return -1; - *o = p; - return 0; -} - -static inline int unpack_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ - return unpack_callback_double(u, d, o); -} - -static inline int unpack_callback_nil(unpack_user* u, msgpack_unpack_object* o) -{ Py_INCREF(Py_None); *o = Py_None; return 0; } - -static inline int unpack_callback_true(unpack_user* u, msgpack_unpack_object* o) -{ Py_INCREF(Py_True); *o = Py_True; return 0; } - -static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o) -{ Py_INCREF(Py_False); *o = Py_False; return 0; } - -static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > u->max_array_len) { - PyErr_Format(PyExc_ValueError, "%u exceeds max_array_len(%zd)", n, u->max_array_len); - return -1; - } - PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); - - if (!p) - return -1; - *o = p; - return 0; -} - -static inline int unpack_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - if (u->use_list) - PyList_SET_ITEM(*c, current, o); - else - PyTuple_SET_ITEM(*c, current, o); - return 0; -} - -static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c) -{ - if (u->list_hook) { - PyObject *new_c = PyObject_CallFunctionObjArgs(u->list_hook, *c, NULL); - if (!new_c) - return -1; - Py_DECREF(*c); - *c = new_c; - } - return 0; -} - -static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > u->max_map_len) { - PyErr_Format(PyExc_ValueError, "%u exceeds max_map_len(%zd)", n, u->max_map_len); - return -1; - } - PyObject *p; - if (u->has_pairs_hook) { - p = PyList_New(n); // Or use tuple? - } - else { - p = PyDict_New(); - } - if (!p) - return -1; - *o = p; - return 0; -} - -static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) -{ - if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) { - PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name); - return -1; - } - if (u->has_pairs_hook) { - msgpack_unpack_object item = PyTuple_Pack(2, k, v); - if (!item) - return -1; - Py_DECREF(k); - Py_DECREF(v); - PyList_SET_ITEM(*c, current, item); - return 0; - } - else if (PyDict_SetItem(*c, k, v) == 0) { - Py_DECREF(k); - Py_DECREF(v); - return 0; - } - return -1; -} - -static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c) -{ - if (u->object_hook) { - PyObject *new_c = PyObject_CallFunctionObjArgs(u->object_hook, *c, NULL); - if (!new_c) - return -1; - - Py_DECREF(*c); - *c = new_c; - } - return 0; -} - -static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) -{ - if (l > u->max_str_len) { - PyErr_Format(PyExc_ValueError, "%u exceeds max_str_len(%zd)", l, u->max_str_len); - return -1; - } - - PyObject *py; - - if (u->encoding) { - py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); - } else if (u->raw) { - py = PyBytes_FromStringAndSize(p, l); - } else { - py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors); - } - if (!py) - return -1; - *o = py; - return 0; -} - -static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) -{ - if (l > u->max_bin_len) { - PyErr_Format(PyExc_ValueError, "%u exceeds max_bin_len(%zd)", l, u->max_bin_len); - return -1; - } - - PyObject *py = PyBytes_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; -} - -static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, - unsigned int length, msgpack_unpack_object* o) -{ - PyObject *py; - int8_t typecode = (int8_t)*pos++; - if (!u->ext_hook) { - PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL"); - return -1; - } - if (length-1 > u->max_ext_len) { - PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len); - return -1; - } - // length also includes the typecode, so the actual data is length-1 -#if PY_MAJOR_VERSION == 2 - py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1); -#else - py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); -#endif - if (!py) - return -1; - *o = py; - return 0; -} - -#include "unpack_template.h" diff --git a/ddtrace/vendor/msgpack/unpack_define.h b/ddtrace/vendor/msgpack/unpack_define.h deleted file mode 100644 index 0dd708d17c3..00000000000 --- a/ddtrace/vendor/msgpack/unpack_define.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * MessagePack unpacking routine template - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_UNPACK_DEFINE_H__ -#define MSGPACK_UNPACK_DEFINE_H__ - -#include "msgpack/sysdep.h" -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - - -#ifndef MSGPACK_EMBED_STACK_SIZE -#define MSGPACK_EMBED_STACK_SIZE 32 -#endif - - -// CS is first byte & 0x1f -typedef enum { - CS_HEADER = 0x00, // nil - - //CS_ = 0x01, - //CS_ = 0x02, // false - //CS_ = 0x03, // true - - CS_BIN_8 = 0x04, - CS_BIN_16 = 0x05, - CS_BIN_32 = 0x06, - - CS_EXT_8 = 0x07, - CS_EXT_16 = 0x08, - CS_EXT_32 = 0x09, - - CS_FLOAT = 0x0a, - CS_DOUBLE = 0x0b, - CS_UINT_8 = 0x0c, - CS_UINT_16 = 0x0d, - CS_UINT_32 = 0x0e, - CS_UINT_64 = 0x0f, - CS_INT_8 = 0x10, - CS_INT_16 = 0x11, - CS_INT_32 = 0x12, - CS_INT_64 = 0x13, - - //CS_FIXEXT1 = 0x14, - //CS_FIXEXT2 = 0x15, - //CS_FIXEXT4 = 0x16, - //CS_FIXEXT8 = 0x17, - //CS_FIXEXT16 = 0x18, - - CS_RAW_8 = 0x19, - CS_RAW_16 = 0x1a, - CS_RAW_32 = 0x1b, - CS_ARRAY_16 = 0x1c, - CS_ARRAY_32 = 0x1d, - CS_MAP_16 = 0x1e, - CS_MAP_32 = 0x1f, - - ACS_RAW_VALUE, - ACS_BIN_VALUE, - ACS_EXT_VALUE, -} msgpack_unpack_state; - - -typedef enum { - CT_ARRAY_ITEM, - CT_MAP_KEY, - CT_MAP_VALUE, -} msgpack_container_type; - - -#ifdef __cplusplus -} -#endif - -#endif /* msgpack/unpack_define.h */ diff --git a/ddtrace/vendor/msgpack/unpack_template.h b/ddtrace/vendor/msgpack/unpack_template.h deleted file mode 100644 index 9924b9c6f20..00000000000 --- a/ddtrace/vendor/msgpack/unpack_template.h +++ /dev/null @@ -1,454 +0,0 @@ -/* - * MessagePack unpacking routine template - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef USE_CASE_RANGE -#if !defined(_MSC_VER) -#define USE_CASE_RANGE -#endif -#endif - -typedef struct unpack_stack { - PyObject* obj; - Py_ssize_t size; - Py_ssize_t count; - unsigned int ct; - PyObject* map_key; -} unpack_stack; - -struct unpack_context { - unpack_user user; - unsigned int cs; - unsigned int trail; - unsigned int top; - /* - unpack_stack* stack; - unsigned int stack_size; - unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ - unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; -}; - - -static inline void unpack_init(unpack_context* ctx) -{ - ctx->cs = CS_HEADER; - ctx->trail = 0; - ctx->top = 0; - /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ - ctx->stack[0].obj = unpack_callback_root(&ctx->user); -} - -/* -static inline void unpack_destroy(unpack_context* ctx) -{ - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } -} -*/ - -static inline PyObject* unpack_data(unpack_context* ctx) -{ - return (ctx)->stack[0].obj; -} - -static inline void unpack_clear(unpack_context *ctx) -{ - Py_CLEAR(ctx->stack[0].obj); -} - -template -static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) -{ - assert(len >= *off); - - const unsigned char* p = (unsigned char*)data + *off; - const unsigned char* const pe = (unsigned char*)data + len; - const void* n = p; - - unsigned int trail = ctx->trail; - unsigned int cs = ctx->cs; - unsigned int top = ctx->top; - unpack_stack* stack = ctx->stack; - /* - unsigned int stack_size = ctx->stack_size; - */ - unpack_user* user = &ctx->user; - - PyObject* obj = NULL; - unpack_stack* c = NULL; - - int ret; - -#define construct_cb(name) \ - construct && unpack_callback ## name - -#define push_simple_value(func) \ - if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ - goto _push -#define push_fixed_value(func, arg) \ - if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \ - goto _push -#define push_variable_value(func, base, pos, len) \ - if(construct_cb(func)(user, \ - (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ - goto _push - -#define again_fixed_trail(_cs, trail_len) \ - trail = trail_len; \ - cs = _cs; \ - goto _fixed_trail_again -#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ - trail = trail_len; \ - if(trail == 0) { goto ifzero; } \ - cs = _cs; \ - goto _fixed_trail_again - -#define start_container(func, count_, ct_) \ - if(top >= MSGPACK_EMBED_STACK_SIZE) { ret = -3; goto _end; } \ - if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ - if((count_) == 0) { obj = stack[top].obj; \ - if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \ - goto _push; } \ - stack[top].ct = ct_; \ - stack[top].size = count_; \ - stack[top].count = 0; \ - ++top; \ - goto _header_again - -#define NEXT_CS(p) ((unsigned int)*p & 0x1f) - -#ifdef USE_CASE_RANGE -#define SWITCH_RANGE_BEGIN switch(*p) { -#define SWITCH_RANGE(FROM, TO) case FROM ... TO: -#define SWITCH_RANGE_DEFAULT default: -#define SWITCH_RANGE_END } -#else -#define SWITCH_RANGE_BEGIN { if(0) { -#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { -#define SWITCH_RANGE_DEFAULT } else { -#define SWITCH_RANGE_END } } -#endif - - if(p == pe) { goto _out; } - do { - switch(cs) { - case CS_HEADER: - SWITCH_RANGE_BEGIN - SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum - push_fixed_value(_uint8, *(uint8_t*)p); - SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum - push_fixed_value(_int8, *(int8_t*)p); - SWITCH_RANGE(0xc0, 0xdf) // Variable - switch(*p) { - case 0xc0: // nil - push_simple_value(_nil); - //case 0xc1: // never used - case 0xc2: // false - push_simple_value(_false); - case 0xc3: // true - push_simple_value(_true); - case 0xc4: // bin 8 - again_fixed_trail(NEXT_CS(p), 1); - case 0xc5: // bin 16 - again_fixed_trail(NEXT_CS(p), 2); - case 0xc6: // bin 32 - again_fixed_trail(NEXT_CS(p), 4); - case 0xc7: // ext 8 - again_fixed_trail(NEXT_CS(p), 1); - case 0xc8: // ext 16 - again_fixed_trail(NEXT_CS(p), 2); - case 0xc9: // ext 32 - again_fixed_trail(NEXT_CS(p), 4); - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - case 0xd4: // fixext 1 - case 0xd5: // fixext 2 - case 0xd6: // fixext 4 - case 0xd7: // fixext 8 - again_fixed_trail_if_zero(ACS_EXT_VALUE, - (1 << (((unsigned int)*p) & 0x03))+1, - _ext_zero); - case 0xd8: // fixext 16 - again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero); - case 0xd9: // str 8 - again_fixed_trail(NEXT_CS(p), 1); - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); - default: - ret = -2; - goto _end; - } - SWITCH_RANGE(0xa0, 0xbf) // FixRaw - again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - SWITCH_RANGE(0x90, 0x9f) // FixArray - start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - SWITCH_RANGE(0x80, 0x8f) // FixMap - start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - - SWITCH_RANGE_DEFAULT - ret = -2; - goto _end; - SWITCH_RANGE_END - // end CS_HEADER - - - _fixed_trail_again: - ++p; - - default: - if((size_t)(pe - p) < trail) { goto _out; } - n = p; p += trail - 1; - switch(cs) { - case CS_EXT_8: - again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero); - case CS_EXT_16: - again_fixed_trail_if_zero(ACS_EXT_VALUE, - _msgpack_load16(uint16_t,n)+1, - _ext_zero); - case CS_EXT_32: - again_fixed_trail_if_zero(ACS_EXT_VALUE, - _msgpack_load32(uint32_t,n)+1, - _ext_zero); - case CS_FLOAT: { - double f = _PyFloat_Unpack4((unsigned char*)n, 0); - push_fixed_value(_float, f); } - case CS_DOUBLE: { - double f = _PyFloat_Unpack8((unsigned char*)n, 0); - push_fixed_value(_double, f); } - case CS_UINT_8: - push_fixed_value(_uint8, *(uint8_t*)n); - case CS_UINT_16: - push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); - case CS_UINT_32: - push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); - case CS_UINT_64: - push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); - - case CS_INT_8: - push_fixed_value(_int8, *(int8_t*)n); - case CS_INT_16: - push_fixed_value(_int16, _msgpack_load16(int16_t,n)); - case CS_INT_32: - push_fixed_value(_int32, _msgpack_load32(int32_t,n)); - case CS_INT_64: - push_fixed_value(_int64, _msgpack_load64(int64_t,n)); - - case CS_BIN_8: - again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero); - case CS_BIN_16: - again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero); - case CS_BIN_32: - again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero); - case ACS_BIN_VALUE: - _bin_zero: - push_variable_value(_bin, data, n, trail); - - case CS_RAW_8: - again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero); - case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); - case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); - case ACS_RAW_VALUE: - _raw_zero: - push_variable_value(_raw, data, n, trail); - - case ACS_EXT_VALUE: - _ext_zero: - push_variable_value(_ext, data, n, trail); - - case CS_ARRAY_16: - start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); - case CS_ARRAY_32: - /* FIXME security guard */ - start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); - - case CS_MAP_16: - start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); - case CS_MAP_32: - /* FIXME security guard */ - start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); - - default: - goto _failed; - } - } - -_push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; - - default: - goto _failed; - } - -_header_again: - cs = CS_HEADER; - ++p; - } while(p != pe); - goto _out; - - -_finish: - if (!construct) - unpack_callback_nil(user, &obj); - stack[0].obj = obj; - ++p; - ret = 1; - /*printf("-- finish --\n"); */ - goto _end; - -_failed: - /*printf("** FAILED **\n"); */ - ret = -1; - goto _end; - -_out: - ret = 0; - goto _end; - -_end: - ctx->cs = cs; - ctx->trail = trail; - ctx->top = top; - *off = p - (const unsigned char*)data; - - return ret; -#undef construct_cb -} - -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END -#undef push_simple_value -#undef push_fixed_value -#undef push_variable_value -#undef again_fixed_trail -#undef again_fixed_trail_if_zero -#undef start_container - -template -static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) -{ - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; - -#define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; - - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; -#ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: -#else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: -#endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); - return 1; -} - -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END - -static const execute_fn unpack_construct = &unpack_execute; -static const execute_fn unpack_skip = &unpack_execute; -static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>; -static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; - -#undef NEXT_CS - -/* vim: set ts=4 sw=4 sts=4 expandtab */ diff --git a/setup.py b/setup.py index 8d6693102f6..c1357225a62 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,8 @@ def run_tests(self): # enum34 is an enum backport for earlier versions of python # funcsigs backport required for vendored debtcollector -install_requires = ["enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'"] +# encoding using msgpack +install_requires = ["enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'", "msgpack"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( @@ -165,7 +166,7 @@ def get_exts_for(name): # Try to build with C extensions first, fallback to only pure-Python if building fails try: all_exts = [] - for extname in ("msgpack", "wrapt", "psutil"): + for extname in ("wrapt", "psutil"): exts = get_exts_for(extname) if exts: all_exts.extend(exts) diff --git a/tests/test_encoders.py b/tests/test_encoders.py index 682a5507311..24a6ce824bd 100644 --- a/tests/test_encoders.py +++ b/tests/test_encoders.py @@ -1,11 +1,11 @@ import json +import msgpack from unittest import TestCase from ddtrace.span import Span from ddtrace.compat import msgpack_type, string_type from ddtrace.encoding import JSONEncoder, MsgpackEncoder -from ddtrace.vendor import msgpack class TestEncoders(TestCase): diff --git a/tests/test_integration.py b/tests/test_integration.py index 8752413342a..ccfcf18ee18 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -3,6 +3,7 @@ import logging import mock import ddtrace +import msgpack from unittest import TestCase, skip, skipUnless @@ -15,7 +16,6 @@ from ddtrace.compat import httplib, PYTHON_INTERPRETER, PYTHON_VERSION from ddtrace.internal.runtime.container import CGroupInfo from ddtrace.vendor import monotonic -from ddtrace.vendor import msgpack from tests.test_tracer import get_dummy_tracer diff --git a/tox.ini b/tox.ini index 2aa05e56397..0b5a103d6fe 100644 --- a/tox.ini +++ b/tox.ini @@ -20,9 +20,9 @@ envlist = flake8 black wait - {py27,py34,py35,py36,py37}-tracer + {py27,py34,py35,py36,py37}-tracer{,-msgpackmin,-msgpack056,-msgpack060} {py27,py34,py35,py36,py37}-internal - {py27,py34,py35,py36,py37}-integration + {py27,py34,py35,py36,py37}-integration{,-msgpackmin,-msgpack056,-msgpack060} {py27,py34,py35,py36,py37}-ddtracerun {py27,py34,py35,py36,py37}-test_utils {py27,py34,py35,py36,py37}-test_logging @@ -313,6 +313,9 @@ deps = mako100: mako>=1.0.0,<1.1.0 mako010: mako>=0.1.0,<1.0.0 memcached: python-memcached + msgpackmin: msgpack==0.5.0 + msgpack056: msgpack>=0.5.6,<0.6.0 + msgpack060: msgpack>=0.6.0,<0.6.1 molten070: molten>=0.7.0,<0.7.2 molten072: molten>=0.7.2,<0.8.0 mongoengine015: mongoengine>=0.15<0.16 From 30ffa830fa5ac8bb2b70e148436e96333f579018 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 10:27:44 +0100 Subject: [PATCH 57/81] chord(encoding): remove unnused global variable --- ddtrace/encoding.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ddtrace/encoding.py b/ddtrace/encoding.py index c6159645aff..8b7bdadddda 100644 --- a/ddtrace/encoding.py +++ b/ddtrace/encoding.py @@ -7,12 +7,9 @@ # DEV: We are ok with the pure Python fallback for msgpack if the C-extension failed to install try: import msgpack - # DEV: `use_bin_type` only exists since `0.4.0`, but we vendor a more recent version - MSGPACK_PARAMS = {'use_bin_type': True} MSGPACK_ENCODING = True except ImportError: # fallback to JSON - MSGPACK_PARAMS = {} MSGPACK_ENCODING = False log = get_logger(__name__) From 20be025dc372a72794386cf6b2fdd33902757ee1 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 10:33:58 +0100 Subject: [PATCH 58/81] refactor(encoding): simplify msgpack/json encoders This refactor the code to make it simpler: - we only define the useful class at load time based on msgpack availability - we declare methods are static when they are - we use class attributes rather than useless per-instance attribute --- ddtrace/api.py | 4 +- ddtrace/encoding.py | 116 +++++++++++++++++--------------------- ddtrace/payload.py | 4 +- tests/test_integration.py | 20 +------ tests/test_payload.py | 4 +- 5 files changed, 59 insertions(+), 89 deletions(-) diff --git a/ddtrace/api.py b/ddtrace/api.py index 5654d098b7e..6809d41aa9b 100644 --- a/ddtrace/api.py +++ b/ddtrace/api.py @@ -4,7 +4,7 @@ import socket # project -from .encoding import get_encoder, JSONEncoder +from .encoding import Encoder, JSONEncoder from .compat import httplib, PYTHON_VERSION, PYTHON_INTERPRETER, get_connection_response from .internal.logger import get_logger from .internal.runtime import container @@ -185,7 +185,7 @@ def _set_version(self, version, encoder=None): if self._compatibility_mode: self._encoder = JSONEncoder() else: - self._encoder = encoder or get_encoder() + self._encoder = encoder or Encoder() # overwrite the Content-type with the one chosen in the Encoder self._headers.update({'Content-Type': self._encoder.content_type}) diff --git a/ddtrace/encoding.py b/ddtrace/encoding.py index 8b7bdadddda..621286a0e7c 100644 --- a/ddtrace/encoding.py +++ b/ddtrace/encoding.py @@ -4,30 +4,13 @@ from .internal.logger import get_logger -# DEV: We are ok with the pure Python fallback for msgpack if the C-extension failed to install -try: - import msgpack - MSGPACK_ENCODING = True -except ImportError: - # fallback to JSON - MSGPACK_ENCODING = False - log = get_logger(__name__) -class Encoder(object): +class _EncoderBase(object): """ Encoder interface that provides the logic to encode traces and service. """ - def __init__(self): - """ - When extending the ``Encoder`` class, ``headers`` must be set because - they're returned by the encoding methods, so that the API transport doesn't - need to know what is the right header to suggest the decoding format to the - agent - """ - self.content_type = '' - def encode_traces(self, traces): """ Encodes a list of traces, expecting a list of items where each items @@ -50,75 +33,78 @@ def encode_trace(self, trace): """ return self.encode([span.to_dict() for span in trace]) - def encode(self, obj): + @staticmethod + def encode(obj): """ Defines the underlying format used during traces or services encoding. This method must be implemented and should only be used by the internal functions. """ raise NotImplementedError - def decode(self, data): + @staticmethod + def decode(data): """ Defines the underlying format used during traces or services encoding. This method must be implemented and should only be used by the internal functions. """ raise NotImplementedError - def join_encoded(self, objs): + @staticmethod + def join_encoded(objs): """Helper used to join a list of encoded objects into an encoded list of objects""" raise NotImplementedError -class JSONEncoder(Encoder): - def __init__(self): - # TODO[manu]: add instructions about how users can switch to Msgpack - log.debug('using JSON encoder; application performance may be degraded') - self.content_type = 'application/json' +class JSONEncoder(_EncoderBase): + content_type = 'application/json' - def encode(self, obj): + @staticmethod + def encode(obj): return json.dumps(obj) - def decode(self, data): + @staticmethod + def decode(data): return json.loads(data) - def join_encoded(self, objs): + @staticmethod + def join_encoded(objs): """Join a list of encoded objects together as a json array""" return '[' + ','.join(objs) + ']' -class MsgpackEncoder(Encoder): - def __init__(self): - log.debug('using Msgpack encoder') - self.content_type = 'application/msgpack' - - def encode(self, obj): - return msgpack.packb(obj) - - def decode(self, data): - return msgpack.unpackb(data) - - def join_encoded(self, objs): - """Join a list of encoded objects together as a msgpack array""" - buf = b''.join(objs) - - # Prepend array header to buffer - # https://github.com/msgpack/msgpack-python/blob/f46523b1af7ff2d408da8500ea36a4f9f2abe915/msgpack/fallback.py#L948-L955 - count = len(objs) - if count <= 0xf: - return struct.pack('B', 0x90 + count) + buf - elif count <= 0xffff: - return struct.pack('>BH', 0xdc, count) + buf - else: - return struct.pack('>BI', 0xdd, count) + buf - - -def get_encoder(): - """ - Switching logic that choose the best encoder for the API transport. - The default behavior is to use Msgpack if we have a CPP implementation - installed, falling back to the Python built-in JSON encoder. - """ - if MSGPACK_ENCODING: - return MsgpackEncoder() - else: - return JSONEncoder() +# DEV: We are ok with the pure Python fallback for msgpack if the C-extension failed to install +try: + import msgpack +except ImportError: + log.warning('using JSON encoder; application performance may be degraded; please install msgpack') + Encoder = JSONEncoder +else: + log.debug('using MessagePack encoder') + + class MsgpackEncoder(_EncoderBase): + content_type = 'application/msgpack' + + @staticmethod + def encode(obj): + return msgpack.packb(obj) + + @staticmethod + def decode(data): + return msgpack.unpackb(data) + + @staticmethod + def join_encoded(objs): + """Join a list of encoded objects together as a msgpack array""" + buf = b''.join(objs) + + # Prepend array header to buffer + # https://github.com/msgpack/msgpack-python/blob/f46523b1af7ff2d408da8500ea36a4f9f2abe915/msgpack/fallback.py#L948-L955 + count = len(objs) + if count <= 0xf: + return struct.pack('B', 0x90 + count) + buf + elif count <= 0xffff: + return struct.pack('>BH', 0xdc, count) + buf + else: + return struct.pack('>BI', 0xdd, count) + buf + + Encoder = MsgpackEncoder diff --git a/ddtrace/payload.py b/ddtrace/payload.py index acbede4fbcd..2af9d3ae21c 100644 --- a/ddtrace/payload.py +++ b/ddtrace/payload.py @@ -1,4 +1,4 @@ -from .encoding import get_encoder +from .encoding import Encoder class PayloadFull(Exception): @@ -32,7 +32,7 @@ def __init__(self, encoder=None, max_payload_size=DEFAULT_MAX_PAYLOAD_SIZE): being considered full (default: 5mb) """ self.max_payload_size = max_payload_size - self.encoder = encoder or get_encoder() + self.encoder = encoder or Encoder() self.traces = [] self.size = 0 diff --git a/tests/test_integration.py b/tests/test_integration.py index ccfcf18ee18..bc24bd28438 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -12,7 +12,7 @@ from ddtrace.filters import FilterRequestsOnUrl from ddtrace.constants import FILTERS_KEY from ddtrace.tracer import Tracer -from ddtrace.encoding import JSONEncoder, MsgpackEncoder, get_encoder +from ddtrace.encoding import JSONEncoder, MsgpackEncoder from ddtrace.compat import httplib, PYTHON_INTERPRETER, PYTHON_VERSION from ddtrace.internal.runtime.container import CGroupInfo from ddtrace.vendor import monotonic @@ -273,9 +273,7 @@ def test_send_presampler_headers(self, mocked_http): } params, _ = request_call.call_args_list[0] headers = params[3] - assert len(expected_headers) == len(headers) - for k, v in expected_headers.items(): - assert v == headers[k] + assert expected_headers == headers def _send_traces_and_check(self, traces, nresponses=1): # test JSON encoder @@ -362,20 +360,6 @@ class TestAPIDowngrade(TestCase): Ensures that if the tracing client found an earlier trace agent, it will downgrade the current connection to a stable API version """ - @skip('msgpack package split breaks this test; it works for newer version of msgpack') - def test_get_encoder_default(self): - # get_encoder should return MsgpackEncoder instance if - # msgpack and the CPP implementaiton are available - encoder = get_encoder() - assert isinstance(encoder, MsgpackEncoder) - - @mock.patch('ddtrace.encoding.MSGPACK_ENCODING', False) - def test_get_encoder_fallback(self): - # get_encoder should return JSONEncoder instance if - # msgpack or the CPP implementaiton, are not available - encoder = get_encoder() - assert isinstance(encoder, JSONEncoder) - @skip('msgpack package split breaks this test; it works for newer version of msgpack') def test_downgrade_api(self): # make a call to a not existing endpoint, downgrades diff --git a/tests/test_payload.py b/tests/test_payload.py index fc2cd25e6bc..97675504b6e 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -1,6 +1,6 @@ import math -from ddtrace.encoding import get_encoder, JSONEncoder +from ddtrace.encoding import Encoder, JSONEncoder from ddtrace.payload import Payload, PayloadFull from ddtrace.span import Span @@ -18,7 +18,7 @@ def test_init(self): With no encoder We use the default encoder """ - default_encoder_type = type(get_encoder()) + default_encoder_type = type(Encoder()) payload = Payload() self.assertIsInstance(payload.encoder, default_encoder_type) From abb0902c09ebffa861e25718b80bf5a8125aafd1 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 11:14:19 +0100 Subject: [PATCH 59/81] refactor(encoding): msgpack is a dependency, don't "try" to import it We just import msgpack for sure since ddtrace depends on it, it has to be there. --- ddtrace/encoding.py | 66 +++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/ddtrace/encoding.py b/ddtrace/encoding.py index 621286a0e7c..229b4f1da05 100644 --- a/ddtrace/encoding.py +++ b/ddtrace/encoding.py @@ -1,6 +1,8 @@ import json import struct +import msgpack + from .internal.logger import get_logger @@ -72,39 +74,31 @@ def join_encoded(objs): return '[' + ','.join(objs) + ']' -# DEV: We are ok with the pure Python fallback for msgpack if the C-extension failed to install -try: - import msgpack -except ImportError: - log.warning('using JSON encoder; application performance may be degraded; please install msgpack') - Encoder = JSONEncoder -else: - log.debug('using MessagePack encoder') - - class MsgpackEncoder(_EncoderBase): - content_type = 'application/msgpack' - - @staticmethod - def encode(obj): - return msgpack.packb(obj) - - @staticmethod - def decode(data): - return msgpack.unpackb(data) - - @staticmethod - def join_encoded(objs): - """Join a list of encoded objects together as a msgpack array""" - buf = b''.join(objs) - - # Prepend array header to buffer - # https://github.com/msgpack/msgpack-python/blob/f46523b1af7ff2d408da8500ea36a4f9f2abe915/msgpack/fallback.py#L948-L955 - count = len(objs) - if count <= 0xf: - return struct.pack('B', 0x90 + count) + buf - elif count <= 0xffff: - return struct.pack('>BH', 0xdc, count) + buf - else: - return struct.pack('>BI', 0xdd, count) + buf - - Encoder = MsgpackEncoder +class MsgpackEncoder(_EncoderBase): + content_type = 'application/msgpack' + + @staticmethod + def encode(obj): + return msgpack.packb(obj) + + @staticmethod + def decode(data): + return msgpack.unpackb(data) + + @staticmethod + def join_encoded(objs): + """Join a list of encoded objects together as a msgpack array""" + buf = b''.join(objs) + + # Prepend array header to buffer + # https://github.com/msgpack/msgpack-python/blob/f46523b1af7ff2d408da8500ea36a4f9f2abe915/msgpack/fallback.py#L948-L955 + count = len(objs) + if count <= 0xf: + return struct.pack('B', 0x90 + count) + buf + elif count <= 0xffff: + return struct.pack('>BH', 0xdc, count) + buf + else: + return struct.pack('>BI', 0xdd, count) + buf + + +Encoder = MsgpackEncoder From 839b582d82d0f50b100fbb95bf10a3b15e92f398 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 11:19:02 +0100 Subject: [PATCH 60/81] fix(setup): set minimum msgpack version we support We test with 0.5.0 and above in the CI. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c1357225a62..e6dc04721d2 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def run_tests(self): # enum34 is an enum backport for earlier versions of python # funcsigs backport required for vendored debtcollector # encoding using msgpack -install_requires = ["enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'", "msgpack"] +install_requires = ["enum34; python_version<'3.4'", "funcsigs>=1.0.0;python_version=='2.7'", "msgpack>=0.5.0"] # Base `setup()` kwargs without any C-extension registering setup_kwargs = dict( From 12de872770907427cbc44da31e98f0e95b130686 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 15:58:36 +0100 Subject: [PATCH 61/81] ci(tox): do not install ddtrace for running black (#1204) This is a follow-up of #1200, doing the same thing for black. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 0b5a103d6fe..894f6eef676 100644 --- a/tox.ini +++ b/tox.ini @@ -491,6 +491,8 @@ deps= ignore_outcome=true [testenv:black] +commands_pre= +skip_install=true deps=black commands=black --check . basepython=python3.7 From 406bd0b0a7c3091bb3c06e79b96f01bfc0ed5dc1 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 16:20:49 +0100 Subject: [PATCH 62/81] ci(wait): do not install ddtrace in the wait target We don't use ddtrace at all, we just run a Python script. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 894f6eef676..310e36a69c8 100644 --- a/tox.ini +++ b/tox.ini @@ -476,6 +476,8 @@ commands = benchmarks: pytest --benchmark-only {posargs} tests/benchmark.py [testenv:wait] +commands_pre= +skip_install=true commands=python tests/wait-for-services.py {posargs} basepython=python deps= From 37df32949ece1bc593b3af32b1e43ded22c5d86f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 16:08:43 +0100 Subject: [PATCH 63/81] ci(docs): modernize documentation building This streamlines the doc building process by: - Using the latest shiny Python 3.8 in the CI - Leveraging tox to create a venv to build the doc - Upgrade to the latest polished Sphinx version to build the doc - Remove the bulky Makefile - Remove the esoteric Ruby build code, brrrr --- .circleci/config.yml | 8 +- Rakefile | 8 -- docs/Makefile | 225 ------------------------------------------- tox.ini | 7 ++ 4 files changed, 10 insertions(+), 238 deletions(-) delete mode 100644 docs/Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml index d54ecf172c9..fe970816b2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -794,14 +794,12 @@ jobs: build_docs: # deploy official documentation docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 resource_class: *resource_class steps: - checkout - - run: sudo apt-get -y install rake - # Sphinx 1.7.5 is required otherwise docs are not properly built - - run: sudo pip install mkwheelhouse sphinx==1.7.5 wrapt - - run: rake docs + - run: sudo pip install tox + - run: tox -e docs - run: command: | mkdir -p /tmp/docs diff --git a/Rakefile b/Rakefile index ea90004a40f..04c1a995ca7 100644 --- a/Rakefile +++ b/Rakefile @@ -1,11 +1,3 @@ -desc "build the docs" -task :docs do - sh "pip install sphinx" - Dir.chdir 'docs' do - sh "make html" - end -end - # Deploy tasks S3_DIR = ENV['S3_DIR'] S3_BUCKET = "pypi.datadoghq.com" diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 7b1ce33c234..00000000000 --- a/docs/Makefile +++ /dev/null @@ -1,225 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " epub3 to make an epub3" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - @echo " dummy to check syntax errors of document sources" - -.PHONY: clean -clean: - rm -rf $(BUILDDIR)/* - -.PHONY: html -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: dirhtml -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -.PHONY: singlehtml -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -.PHONY: pickle -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -.PHONY: json -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -.PHONY: htmlhelp -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -.PHONY: qthelp -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ddtrace.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ddtrace.qhc" - -.PHONY: applehelp -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" \ - "~/Library/Documentation/Help or install it in your application" \ - "bundle." - -.PHONY: devhelp -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/ddtrace" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ddtrace" - @echo "# devhelp" - -.PHONY: epub -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -.PHONY: epub3 -epub3: - $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 - @echo - @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." - -.PHONY: latex -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -.PHONY: latexpdf -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: latexpdfja -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: text -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -.PHONY: man -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -.PHONY: texinfo -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -.PHONY: info -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -.PHONY: gettext -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -.PHONY: changes -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -.PHONY: linkcheck -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -.PHONY: doctest -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -.PHONY: coverage -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." - -.PHONY: xml -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -.PHONY: pseudoxml -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." - -.PHONY: dummy -dummy: - $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy - @echo - @echo "Build finished. Dummy builder generates no files." diff --git a/tox.ini b/tox.ini index 894f6eef676..f01aa4b76ca 100644 --- a/tox.ini +++ b/tox.ini @@ -512,6 +512,13 @@ deps= commands=flake8 . basepython=python3.7 +[testenv:docs] +commands_pre= +skip_install=true +deps = sphinx +commands = sphinx-build -b html docs docs/_build/html +basepython = python + # do not use develop mode with celery as running multiple python versions within # same job will cause problem for tests that use ddtrace-run [celery_contrib] From 32dbbf4cac3a6e7cb5ee9caf6218536432bc2eb2 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 13 Feb 2020 16:13:38 +0100 Subject: [PATCH 64/81] ci(update): use standard Python images for black and flake8 jobs There's no need to pull our custom images there. --- .circleci/config.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe970816b2f..77d2ce823d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,25 +37,23 @@ persist_to_workspace_step: &persist_to_workspace_step jobs: black: docker: - - *test_runner + - image: circleci/python:3.8 resource_class: *resource_class steps: - checkout - - *restore_cache_step + - run: pip install tox - run: tox -e 'black' --result-json /tmp/black.results - *persist_to_workspace_step - - *save_cache_step flake8: docker: - - *test_runner + - image: circleci/python:3.8 resource_class: *resource_class steps: - checkout - - *restore_cache_step + - run: pip install tox - run: tox -e 'flake8' --result-json /tmp/flake8.results - *persist_to_workspace_step - - *save_cache_step # Test that we can build the package properly and package long description will render test_build: From ae9ea2fcea188dd2b087441a3d3b67e6fe24bc99 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 14 Feb 2020 14:11:03 +0100 Subject: [PATCH 65/81] ci(docs): fix deploy_dev target for releasing docs (#1212) This currently fails since the docs are not built by rake anymore (I presume). - Remove unneeded `sudo` usage in docs target - Upgrade deploy_dev to Python 3.8 --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe970816b2f..574650594dd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -749,12 +749,13 @@ jobs: deploy_dev: # build the master branch releasing development docs and wheels docker: - - image: circleci/python:3.6 + - image: circleci/python:3.8 resource_class: *resource_class steps: - checkout - run: sudo apt-get -y install rake - - run: sudo pip install mkwheelhouse sphinx awscli + - run: pip install tox mkwheelhouse awscli + - run: tox -e docs - run: S3_DIR=trace-dev rake release:docs - run: S3_DIR=trace-dev rake release:wheel @@ -798,7 +799,7 @@ jobs: resource_class: *resource_class steps: - checkout - - run: sudo pip install tox + - run: pip install tox - run: tox -e docs - run: command: | From 4653e97f332ae12a8a5bd8e465c127c5b74bb7c2 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 14 Feb 2020 08:36:06 -0500 Subject: [PATCH 66/81] utils: Update get_env to support any number of parts (#1208) * utils: Update get_env to support any number of parts Changes get_env(integration, key) to get_env(*parts) to make it easier to get an env built from 1 part or more than 2 parts the major change here to pay attention to is the third positional argument used to be the default, not the default *must* be provided as a keyword argument * keyword argument after *args is not supported in Python2 * remove * --- ddtrace/context.py | 4 ++-- ddtrace/contrib/celery/patch.py | 4 ++-- ddtrace/contrib/dbapi/__init__.py | 2 +- ddtrace/contrib/falcon/patch.py | 2 +- ddtrace/contrib/jinja2/patch.py | 2 +- ddtrace/contrib/kombu/patch.py | 2 +- ddtrace/contrib/molten/patch.py | 4 ++-- ddtrace/contrib/pylons/patch.py | 2 +- ddtrace/contrib/pyramid/patch.py | 4 ++-- ddtrace/contrib/requests/patch.py | 6 +++--- ddtrace/settings/integration.py | 2 +- ddtrace/tracer.py | 5 +++-- ddtrace/utils/formats.py | 13 +++++++++++-- tests/test_utils.py | 7 ++++++- 14 files changed, 37 insertions(+), 22 deletions(-) diff --git a/ddtrace/context.py b/ddtrace/context.py index 7feff0c79f9..e2937b80eb6 100644 --- a/ddtrace/context.py +++ b/ddtrace/context.py @@ -25,8 +25,8 @@ class Context(object): This data structure is thread-safe. """ - _partial_flush_enabled = asbool(get_env('tracer', 'partial_flush_enabled', 'false')) - _partial_flush_min_spans = int(get_env('tracer', 'partial_flush_min_spans', 500)) + _partial_flush_enabled = asbool(get_env('tracer', 'partial_flush_enabled', default=False)) + _partial_flush_min_spans = int(get_env('tracer', 'partial_flush_min_spans', default=500)) def __init__(self, trace_id=None, span_id=None, sampling_priority=None, _dd_origin=None): """ diff --git a/ddtrace/contrib/celery/patch.py b/ddtrace/contrib/celery/patch.py index b6e9793840e..bb3d4320e1d 100644 --- a/ddtrace/contrib/celery/patch.py +++ b/ddtrace/contrib/celery/patch.py @@ -9,8 +9,8 @@ # Celery default settings config._add('celery', { - 'producer_service_name': get_env('celery', 'producer_service_name', PRODUCER_SERVICE), - 'worker_service_name': get_env('celery', 'worker_service_name', WORKER_SERVICE), + 'producer_service_name': get_env('celery', 'producer_service_name', default=PRODUCER_SERVICE), + 'worker_service_name': get_env('celery', 'worker_service_name', default=WORKER_SERVICE), }) diff --git a/ddtrace/contrib/dbapi/__init__.py b/ddtrace/contrib/dbapi/__init__.py index e8c577c7741..4c48ef66382 100644 --- a/ddtrace/contrib/dbapi/__init__.py +++ b/ddtrace/contrib/dbapi/__init__.py @@ -14,7 +14,7 @@ log = get_logger(__name__) config._add('dbapi2', dict( - trace_fetch_methods=asbool(get_env('dbapi2', 'trace_fetch_methods', 'false')), + trace_fetch_methods=asbool(get_env('dbapi2', 'trace_fetch_methods', default=False)), )) diff --git a/ddtrace/contrib/falcon/patch.py b/ddtrace/contrib/falcon/patch.py index 5eef31f6d6b..471c711d51d 100644 --- a/ddtrace/contrib/falcon/patch.py +++ b/ddtrace/contrib/falcon/patch.py @@ -23,7 +23,7 @@ def patch(): def traced_init(wrapped, instance, args, kwargs): mw = kwargs.pop('middleware', []) service = os.environ.get('DATADOG_SERVICE_NAME') or 'falcon' - distributed_tracing = asbool(get_env('falcon', 'distributed_tracing', True)) + distributed_tracing = asbool(get_env('falcon', 'distributed_tracing', default=True)) mw.insert(0, TraceMiddleware(tracer, service, distributed_tracing)) kwargs['middleware'] = mw diff --git a/ddtrace/contrib/jinja2/patch.py b/ddtrace/contrib/jinja2/patch.py index 988c33db9ea..65ac216a4f6 100644 --- a/ddtrace/contrib/jinja2/patch.py +++ b/ddtrace/contrib/jinja2/patch.py @@ -12,7 +12,7 @@ # default settings config._add('jinja2', { - 'service_name': get_env('jinja2', 'service_name', None), + 'service_name': get_env('jinja2', 'service_name'), }) diff --git a/ddtrace/contrib/kombu/patch.py b/ddtrace/contrib/kombu/patch.py index a82079f3e0c..828cc044b08 100644 --- a/ddtrace/contrib/kombu/patch.py +++ b/ddtrace/contrib/kombu/patch.py @@ -22,7 +22,7 @@ # kombu default settings config._add('kombu', { - 'service_name': get_env('kombu', 'service_name', DEFAULT_SERVICE) + 'service_name': get_env('kombu', 'service_name', default=DEFAULT_SERVICE) }) propagator = HTTPPropagator() diff --git a/ddtrace/contrib/molten/patch.py b/ddtrace/contrib/molten/patch.py index 967d06899f5..61e4de627f4 100644 --- a/ddtrace/contrib/molten/patch.py +++ b/ddtrace/contrib/molten/patch.py @@ -17,9 +17,9 @@ # Configure default configuration config._add('molten', dict( - service_name=get_env('molten', 'service_name', 'molten'), + service_name=get_env('molten', 'service_name', default='molten'), app='molten', - distributed_tracing=asbool(get_env('molten', 'distributed_tracing', True)), + distributed_tracing=asbool(get_env('molten', 'distributed_tracing', default=True)), )) diff --git a/ddtrace/contrib/pylons/patch.py b/ddtrace/contrib/pylons/patch.py index ad437d8c207..3ca18afedf2 100644 --- a/ddtrace/contrib/pylons/patch.py +++ b/ddtrace/contrib/pylons/patch.py @@ -32,7 +32,7 @@ def traced_init(wrapped, instance, args, kwargs): # set tracing options and create the TraceMiddleware service = os.environ.get('DATADOG_SERVICE_NAME', 'pylons') - distributed_tracing = asbool(get_env('pylons', 'distributed_tracing', True)) + distributed_tracing = asbool(get_env('pylons', 'distributed_tracing', default=True)) Pin(service=service, tracer=tracer).onto(instance) traced_app = PylonsTraceMiddleware(instance, tracer, service=service, distributed_tracing=distributed_tracing) diff --git a/ddtrace/contrib/pyramid/patch.py b/ddtrace/contrib/pyramid/patch.py index 8224d183def..c5d429eaa24 100644 --- a/ddtrace/contrib/pyramid/patch.py +++ b/ddtrace/contrib/pyramid/patch.py @@ -30,13 +30,13 @@ def patch(): def traced_init(wrapped, instance, args, kwargs): settings = kwargs.pop('settings', {}) service = os.environ.get('DATADOG_SERVICE_NAME') or 'pyramid' - distributed_tracing = asbool(get_env('pyramid', 'distributed_tracing', True)) + distributed_tracing = asbool(get_env('pyramid', 'distributed_tracing', default=True)) # DEV: integration-specific analytics flag can be not set but still enabled # globally for web frameworks analytics_enabled = get_env('pyramid', 'analytics_enabled') if analytics_enabled is not None: analytics_enabled = asbool(analytics_enabled) - analytics_sample_rate = get_env('pyramid', 'analytics_sample_rate', True) + analytics_sample_rate = get_env('pyramid', 'analytics_sample_rate', default=True) trace_settings = { SETTINGS_SERVICE: service, SETTINGS_DISTRIBUTED_TRACING: distributed_tracing, diff --git a/ddtrace/contrib/requests/patch.py b/ddtrace/contrib/requests/patch.py index 0072481514c..45939a9bac8 100644 --- a/ddtrace/contrib/requests/patch.py +++ b/ddtrace/contrib/requests/patch.py @@ -13,9 +13,9 @@ # requests default settings config._add('requests', { - 'service_name': get_env('requests', 'service_name', DEFAULT_SERVICE), - 'distributed_tracing': asbool(get_env('requests', 'distributed_tracing', True)), - 'split_by_domain': asbool(get_env('requests', 'split_by_domain', False)), + 'service_name': get_env('requests', 'service_name', default=DEFAULT_SERVICE), + 'distributed_tracing': asbool(get_env('requests', 'distributed_tracing', default=True)), + 'split_by_domain': asbool(get_env('requests', 'split_by_domain', default=False)), }) diff --git a/ddtrace/settings/integration.py b/ddtrace/settings/integration.py index 3d8b2288e8f..82cdd37d3e4 100644 --- a/ddtrace/settings/integration.py +++ b/ddtrace/settings/integration.py @@ -44,7 +44,7 @@ def __init__(self, global_config, name, *args, **kwargs): if analytics_enabled_env is not None: analytics_enabled_env = asbool(analytics_enabled_env) self.setdefault('analytics_enabled', analytics_enabled_env) - self.setdefault('analytics_sample_rate', float(get_env(name, 'analytics_sample_rate', 1.0))) + self.setdefault('analytics_sample_rate', float(get_env(name, 'analytics_sample_rate', default=1.0))) def __deepcopy__(self, memodict=None): new = IntegrationConfig(self.global_config, deepcopy(dict(self))) diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 112eafe92d2..78606d89ceb 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -67,8 +67,9 @@ class Tracer(object): DEFAULT_HOSTNAME = environ.get('DD_AGENT_HOST', environ.get('DATADOG_TRACE_AGENT_HOSTNAME', 'localhost')) DEFAULT_PORT = int(environ.get('DD_TRACE_AGENT_PORT', 8126)) - DEFAULT_DOGSTATSD_PORT = int(get_env('dogstatsd', 'port', 8125)) - DEFAULT_DOGSTATSD_URL = get_env('dogstatsd', 'url', 'udp://{}:{}'.format(DEFAULT_HOSTNAME, DEFAULT_DOGSTATSD_PORT)) + DEFAULT_DOGSTATSD_PORT = int(get_env('dogstatsd', 'port', default=8125)) + DEFAULT_DOGSTATSD_URL = get_env('dogstatsd', 'url', + default='udp://{}:{}'.format(DEFAULT_HOSTNAME, DEFAULT_DOGSTATSD_PORT)) DEFAULT_AGENT_URL = environ.get('DD_TRACE_AGENT_URL', 'http://%s:%d' % (DEFAULT_HOSTNAME, DEFAULT_PORT)) def __init__(self, url=DEFAULT_AGENT_URL, dogstatsd_url=DEFAULT_DOGSTATSD_URL): diff --git a/ddtrace/utils/formats.py b/ddtrace/utils/formats.py index 7fe13dab553..66a089b8687 100644 --- a/ddtrace/utils/formats.py +++ b/ddtrace/utils/formats.py @@ -3,7 +3,7 @@ from .deprecation import deprecation -def get_env(integration, variable, default=None): +def get_env(*parts, **kwargs): """Retrieves environment variables value for the given integration. It must be used for consistency between integrations. The implementation is backward compatible with legacy nomenclature: @@ -14,8 +14,17 @@ def get_env(integration, variable, default=None): arguments * return `default` otherwise + :param parts: evironment variable parts that will be joined with ``_`` to generate the name + :type parts: :obj:`str` + :param kwargs: ``default`` is the only supported keyword argument which sets the default value + if no environment variable is found + :rtype: :obj:`str` | ``kwargs["default"]`` + :returns: The string environment variable value or the value of ``kwargs["default"]`` if not found """ - key = "{}_{}".format(integration, variable).upper() + default = kwargs.get("default") + + key = "_".join(parts) + key = key.upper() legacy_env = "DATADOG_{}".format(key) env = "DD_{}".format(key) diff --git a/tests/test_utils.py b/tests/test_utils.py index 959c8acc7da..ccf44390b52 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -24,9 +24,14 @@ def test_get_env(self): # are not set value = get_env('django', 'distributed_tracing') self.assertIsNone(value) - value = get_env('django', 'distributed_tracing', False) + value = get_env('django', 'distributed_tracing', default=False) self.assertFalse(value) + def test_get_env_long(self): + os.environ['DD_SOME_VERY_LONG_TEST_KEY'] = '1' + value = get_env('some', 'very', 'long', 'test', 'key', default='2') + assert value == '1' + def test_get_env_found(self): # ensure `get_env` returns a value if the environment variable is set os.environ['DD_REQUESTS_DISTRIBUTED_TRACING'] = '1' From 9d896c3c414280a09b4698b5217bad5377bd9e5f Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 14 Feb 2020 08:57:48 -0500 Subject: [PATCH 67/81] tests: fix ordering of PYTHONPATH (#1210) * tests: fix ordering of PYTHONPATH If you run the sitecustomize test cases on their own they fail because they import the ddtrace/bootstrap/sitecustomize.py module and not the tests/commands/bootstrap/sitecustomize.py module. This is because this helper code would put the test bootstrap module on the end of the PYTHONPATH, which means it gets looked at last. I have no idea how this seems to work ok when you run the entire test suite together, must be an odd dependency on a previous test case? * fix flake8 --- tests/commands/test_runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_runner.py b/tests/commands/test_runner.py index 8596b5be6d3..0310f754e7a 100644 --- a/tests/commands/test_runner.py +++ b/tests/commands/test_runner.py @@ -22,10 +22,10 @@ def inject_sitecustomize(path): env = os.environ.copy() sitecustomize = os.path.join(root_folder, '..', path) - # Add `boostrap` module so that `sitecustomize.py` is at the bottom - # of the PYTHONPATH - python_path = list(sys.path) + [sitecustomize] - env['PYTHONPATH'] = ':'.join(python_path)[1:] + # Add `bootstrap` directory to the beginning of PYTHONTPATH so we know + # if `import sitecustomize` is run that it'll be the one we specify + python_path = [sitecustomize] + list(sys.path) + env['PYTHONPATH'] = ':'.join(python_path) return env From 00ac506094541687d4190c4eaa0519aeb0bb4d05 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 14 Feb 2020 10:39:05 -0500 Subject: [PATCH 68/81] ci(update) use standard Python image for test_build job (#1209) * ci(update) use standard Python image for test_build job * no longer use virtualenv * Update .circleci/config.yml --- .circleci/config.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c91af556783..0ffd4692dd9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,31 +58,28 @@ jobs: # Test that we can build the package properly and package long description will render test_build: docker: - - *test_runner + - image: circleci/python:3.8 resource_class: *resource_class steps: - checkout - # Create and activate a Python3.7 virtualenv - - run: virtualenv --python python3.7 .venv/build - # Install required dependencies # DEV: `pyopenssl` needed until the following PR is released # https://github.com/pypa/twine/pull/447 # DEV: `wheel` is needed to run `bdist_wheel` - - run: .venv/build/bin/pip install twine readme_renderer[md] pyopenssl wheel + - run: pip install twine readme_renderer[md] pyopenssl wheel # Ensure we didn't cache from previous runs - run: rm -rf build/ dist/ # Manually build any extensions to ensure they succeed # DEV: `DDTRACE_BUILD_RAISE=TRUE` will ensure we don't swallow any build errors - - run: DDTRACE_BUILD_RAISE=TRUE .venv/build/bin/python setup.py build_ext --force + - run: DDTRACE_BUILD_RAISE=TRUE python setup.py build_ext --force # Ensure source package will build - - run: .venv/build/bin/python setup.py sdist + - run: python setup.py sdist # Ensure wheel will build # DEV: `DDTRACE_BUILD_RAISE=TRUE` will ensure we don't swallow any build errors - - run: DDTRACE_BUILD_RAISE=TRUE .venv/build/bin/python setup.py bdist_wheel + - run: DDTRACE_BUILD_RAISE=TRUE python setup.py bdist_wheel # Ensure package long description is valid and will render # https://github.com/pypa/twine/tree/6c4d5ecf2596c72b89b969ccc37b82c160645df8#twine-check - - run: .venv/build/bin/twine check dist/* + - run: twine check dist/* tracer: docker: From eac74425ff819ab98fcf565f952c4a58e37e287b Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 17 Feb 2020 14:56:36 +0100 Subject: [PATCH 69/81] ci(tox): remove useless basepython specifiers (#1214) --- tox.ini | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tox.ini b/tox.ini index cd06dbddad5..3345475ec27 100644 --- a/tox.ini +++ b/tox.ini @@ -143,12 +143,6 @@ envlist = # https://stackoverflow.com/questions/57459123/why-do-i-need-to-run-tox-twice-to-test-a-python-package-with-c-extension commands_pre={envpython} {toxinidir}/setup.py develop usedevelop = True -basepython = - py27: python2.7 - py34: python3.4 - py35: python3.5 - py36: python3.6 - py37: python3.7 deps = pdbpp From 9ced4ec8d0431bf8f09681d75810e43eee053c61 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Mon, 17 Feb 2020 11:45:35 -0500 Subject: [PATCH 70/81] ci: rework .circleci/config.yml (#1213) * ci: rework .circleci/config.yml * add labels to docker images * fix test_build job * don't forget to checkout code * run test build on all versions * restructure test builds * fix docker definition * run flask python tests in parallel * organize more * fix pip install for python34 * use sudo with py34 to install pip deps * use python:3.4 image instead * reorganize the build flow * break out more test suites * fix job keys * rename wait_all_tests * we don't need a placeholder for pre_test * remove build_docs from pre_test requirement * remove python specific integration jobs Co-authored-by: Tahir H. Butt --- .circleci/config.yml | 1289 +++++++++++++++++------------------------- 1 file changed, 511 insertions(+), 778 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ffd4692dd9..b5f3e0a9168 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,67 +1,95 @@ -version: 2 +version: 2.1 - -# Common configuration blocks as YAML anchors -# See: https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ -httpbin_local: &httpbin_local - image: kennethreitz/httpbin@sha256:2c7abc4803080c22928265744410173b6fea3b898872c01c5fd0f0f9df4a59fb - name: httpbin.org -test_runner: &test_runner - image: datadog/docker-library:ddtrace_py -restore_cache_step: &restore_cache_step - restore_cache: - keys: - # In the cache key: - # - .Environment.CIRCLE_JOB: We do separate tox environments by job name, so caching and restoring is - # much faster. - - tox-cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "tox.ini" }} resource_class: &resource_class small -save_cache_step: &save_cache_step - save_cache: - key: tox-cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "tox.ini" }} - paths: - - .tox -deploy_docs_filters: &deploy_docs_filters - filters: - tags: - only: /(^docs$)|(^v[0-9]+(\.[0-9]+)*$)/ - branches: - ignore: /.*/ -persist_to_workspace_step: &persist_to_workspace_step - persist_to_workspace: - root: /tmp - paths: - - "*.results" - - -jobs: - black: - docker: - - image: circleci/python:3.8 - resource_class: *resource_class +busybox_image: &busybox_image busybox:latest +python38_image: &python38_image circleci/python:3.8 +python37_image: &python37_image circleci/python:3.7 +python36_image: &python36_image circleci/python:3.6 +python35_image: &python35_image circleci/python:3.5 +# The circleci/python:3.4 images are broken, we cannot do `pip install ` +# https://github.com/circleci/circleci-images/issues/466 +python34_image: &python34_image python:3.4 +python27_image: &python27_image circleci/python:2.7 +ddtrace_dev_image: &ddtrace_dev_image datadog/docker-library:ddtrace_py +datadog_agent_image: &datadog_agent_image datadog/docker-dd-agent:latest +redis_image: &redis_image redis:4.0-alpine +rediscluster_image: &rediscluster_image grokzen/redis-cluster:4.0.9 +memcached_image: &memcached_image memcached:1.5-alpine +cassandra_image: &cassandra_image spotify/cassandra:latest +consul_image: &consul_image consul:1.6.0 +moto_image: &moto_image palazzem/moto:1.0.1 +elasticsearch_image: &elasticsearch_image elasticsearch:2.3 +mysql_image: &mysql_image mysql:5.7 +postgres_image: &postgres_image postgres:10.5-alpine +mongo_image: &mongo_image mongo:3.6 +httpbin_image: &httpbin_image kennethreitz/httpbin@sha256:2c7abc4803080c22928265744410173b6fea3b898872c01c5fd0f0f9df4a59fb +vertica_image: &vertica_image sumitchawla/vertica:latest +rabbitmq_image: &rabbitmq_image rabbitmq:3.7-alpine + +commands: + setup_tox: + description: "Install tox" steps: - - checkout - run: pip install tox - - run: tox -e 'black' --result-json /tmp/black.results - - *persist_to_workspace_step - flake8: - docker: - - image: circleci/python:3.8 - resource_class: *resource_class - steps: - - checkout - - run: pip install tox - - run: tox -e 'flake8' --result-json /tmp/flake8.results - - *persist_to_workspace_step + restore_tox_cache: + description: "Restore .tox directory from previous runs for faster installs" + steps: + - restore_cache: + # In the cache key: + # - .Environment.CIRCLE_JOB: We do separate tox environments by job name, so caching and restoring is + # much faster. + key: tox-cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "tox.ini" }} + + save_tox_cache: + description: "Save .tox directory into cache for faster installs next time" + steps: + - save_cache: + # In the cache key: + # - .Environment.CIRCLE_JOB: We do separate tox environments by job name, so caching and restoring is + # much faster. + key: tox-cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "tox.ini" }} + paths: + - ".tox" + + save_results: + description: "Persist tox *.results files to /tmp directory" + steps: + - persist_to_workspace: + root: /tmp + paths: + - "*.results" + + run_tox_scenario: + description: "Run scripts/run-tox-scenario with setup, caching and persistence" + parameters: + pattern: + type: string + wait: + type: string + default: "" + steps: + - checkout + - setup_tox + - restore_tox_cache + - when: + condition: + << parameters.wait >> + steps: + - run: + name: "Waiting for << parameters.wait >>" + command: tox -e 'wait' << parameters.wait >> + - run: + name: "Run scripts/run-tox-scenario" + command: scripts/run-tox-scenario '<< parameters.pattern >>' + - save_results + - save_tox_cache - # Test that we can build the package properly and package long description will render test_build: - docker: - - image: circleci/python:3.8 - resource_class: *resource_class + description: "Build the package extensions and wheel to validate builds work" steps: - checkout + # Install required dependencies # DEV: `pyopenssl` needed until the following PR is released # https://github.com/pypa/twine/pull/447 @@ -81,655 +109,522 @@ jobs: # https://github.com/pypa/twine/tree/6c4d5ecf2596c72b89b969ccc37b82c160645df8#twine-check - run: twine check dist/* - tracer: + +executors: + python38: + docker: + - image: *python38_image + resource_class: *resource_class + python37: + docker: + - image: *python37_image + resource_class: *resource_class + python36: docker: - - *test_runner + - image: *python36_image resource_class: *resource_class + python35: + docker: + - image: *python35_image + resource_class: *resource_class + python34: + docker: + - image: *python34_image + resource_class: *resource_class + python27: + docker: + - image: *python27_image + resource_class: *resource_class + ddtrace_dev: + docker: + - image: *ddtrace_dev_image + resource_class: *resource_class + +# Common configuration blocks as YAML anchors +# See: https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ +httpbin_local: &httpbin_local + image: *httpbin_image + name: httpbin.org + +deploy_docs_filters: &deploy_docs_filters + filters: + tags: + only: /(^docs$)|(^v[0-9]+(\.[0-9]+)*$)/ + branches: + ignore: /.*/ + +datadog_agent: &datadog_agent + image: *datadog_agent_image + environment: + DD_APM_ENABLED: true + DD_BIND_HOST: 0.0.0.0 + DD_API_KEY: invalid_key_but_this_is_fine + +mysql_server: &mysql_server + image: *mysql_image + environment: + - MYSQL_ROOT_PASSWORD=admin + - MYSQL_PASSWORD=test + - MYSQL_USER=test + - MYSQL_DATABASE=test + +postgres_server: &postgres_server + image: *postgres_image + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=postgres + +jobs: + black: + executor: python38 steps: - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-tracer' - - *persist_to_workspace_step - - *save_cache_step + - setup_tox + - run: tox -e 'black' --result-json /tmp/black.results + - save_results - internal: - docker: - - *test_runner - resource_class: *resource_class + flake8: + executor: python38 steps: - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-internal' - - *persist_to_workspace_step - - *save_cache_step + - setup_tox + - run: tox -e 'flake8' --result-json /tmp/flake8.results + - save_results + + test_build_py38: + executor: python38 + steps: + - test_build + test_build_py37: + executor: python37 + steps: + - test_build + test_build_py36: + executor: python36 + steps: + - test_build + test_build_py35: + executor: python35 + steps: + - test_build + test_build_py34: + executor: python34 + steps: + - test_build + test_build_py27: + executor: python27 + steps: + - test_build + + tracer: + executor: ddtrace_dev + steps: + - run_tox_scenario: + pattern: '^py..-tracer' + + internal: + executor: ddtrace_dev + steps: + - run_tox_scenario: + pattern: '^py..-internal' opentracer: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-opentracer' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^py..-opentracer' integration: + executor: ddtrace_dev docker: - - <<: *test_runner - env: + - image: *ddtrace_dev_image + environment: TEST_DATADOG_INTEGRATION: 1 - - image: datadog/docker-dd-agent - env: - - DD_APM_ENABLED=true - - DD_BIND_HOST=0.0.0.0 - - DD_API_KEY=invalid_key_but_this_is_fine - resource_class: *resource_class + - *datadog_agent steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-integration' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^py..-integration' futures: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^futures_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^futures_contrib-' boto: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^boto\(core\)\?_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^boto\(core\)\?_contrib-' ddtracerun: + executor: ddtrace_dev docker: - - *test_runner - - image: redis:4.0-alpine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *redis_image steps: - - checkout - - run: scripts/run-tox-scenario '^py..-ddtracerun$' - - *persist_to_workspace_step + - run_tox_scenario: + pattern: '^py..-ddtracerun$' test_utils: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-test_utils$' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^py..-test_utils$' test_logging: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^py..-test_logging$' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^py..-test_logging$' asyncio: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^asyncio_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^asyncio_contrib-' pylons: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^pylons_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^pylons_contrib-' aiohttp: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^aiohttp_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^aiohttp_contrib-' tornado: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^tornado_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^tornado_contrib-' bottle: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^bottle_contrib\(_autopatch\)\?-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^bottle_contrib\(_autopatch\)\?-' cassandra: + executor: ddtrace_dev docker: - - <<: *test_runner - env: + - image: *ddtrace_dev_image + environment: CASS_DRIVER_NO_EXTENSIONS: 1 - - image: spotify/cassandra:latest - env: + - image: *cassandra_image + environment: - MAX_HEAP_SIZE=512M - HEAP_NEWSIZE=256M - resource_class: *resource_class steps: - - checkout - - *restore_cache_step - - run: tox -e wait cassandra - - run: scripts/run-tox-scenario '^cassandra_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: cassandra + pattern: '^cassandra_contrib-' celery: + executor: ddtrace_dev docker: - - *test_runner + - image: *ddtrace_dev_image - image: redis:4.0-alpine - resource_class: *resource_class steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^celery_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^celery_contrib-' consul: + executor: ddtrace_dev docker: - - *test_runner - - image: consul:1.6.0 - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *consul_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^consul_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^consul_contrib-' dogpile_cache: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^dogpile_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^dogpile_contrib-' elasticsearch: + executor: ddtrace_dev docker: - - *test_runner - - image: elasticsearch:2.3 - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *elasticsearch_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^elasticsearch_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^elasticsearch_contrib-' falcon: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: tox -e 'falcon_contrib{,_autopatch}-{py27,py34,py35,py36}-falcon{10,11,12,13,14}' --result-json /tmp/falcon.results - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^falcon_contrib' django: + executor: ddtrace_dev docker: - - *test_runner - - image: redis:4.0-alpine - - image: memcached:1.5-alpine - - image: datadog/docker-dd-agent - env: - - DD_APM_ENABLED=true - - DD_BIND_HOST=0.0.0.0 - - DD_API_KEY=invalid_key_but_this_is_fine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *redis_image + - image: *memcached_image + - *datadog_agent steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^django_contrib' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^django_contrib' djangorestframework: + executor: ddtrace_dev docker: - - *test_runner - - image: redis:4.0-alpine - - image: memcached:1.5-alpine - - image: datadog/docker-dd-agent - env: - - DD_APM_ENABLED=true - - DD_BIND_HOST=0.0.0.0 - - DD_API_KEY=invalid_key_but_this_is_fine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *redis_image + - image: *memcached_image + - *datadog_agent steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^django_drf_contrib' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^django_drf_contrib' flask: + executor: ddtrace_dev docker: - - *test_runner - - image: redis:4.0-alpine - - image: memcached:1.5-alpine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *redis_image + - image: *memcached_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^flask_\(cache_\)\?contrib\(_autopatch\)\?-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^flask_\(cache_\)\?contrib\(_autopatch\)\?-' gevent: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^gevent_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^gevent_contrib-' httplib: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^httplib_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^httplib_contrib' grpc: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^grpc_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^grpc_contrib-' molten: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^molten_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^molten_contrib-' mysqlconnector: + executor: ddtrace_dev docker: - - *test_runner - - image: mysql:5.7 - env: - - MYSQL_ROOT_PASSWORD=admin - - MYSQL_PASSWORD=test - - MYSQL_USER=test - - MYSQL_DATABASE=test - resource_class: *resource_class + - image: *ddtrace_dev_image + - *mysql_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' mysql - - run: scripts/run-tox-scenario '^mysql_contrib-.*-mysqlconnector' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: mysql + pattern: '^mysql_contrib-.*-mysqlconnector' mysqlpython: + executor: ddtrace_dev docker: - - *test_runner - - image: mysql:5.7 - env: - - MYSQL_ROOT_PASSWORD=admin - - MYSQL_PASSWORD=test - - MYSQL_USER=test - - MYSQL_DATABASE=test - resource_class: *resource_class + - image: *ddtrace_dev_image + - *mysql_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' mysql - - run: scripts/run-tox-scenario '^mysqldb_contrib-.*-mysqlclient' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: mysql + pattern: '^mysqldb_contrib-.*-mysqlclient' mysqldb: + executor: ddtrace_dev docker: - - *test_runner - - image: mysql:5.7 - env: - - MYSQL_ROOT_PASSWORD=admin - - MYSQL_PASSWORD=test - - MYSQL_USER=test - - MYSQL_DATABASE=test - resource_class: *resource_class + - image: *ddtrace_dev_image + - *mysql_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' mysql - - run: scripts/run-tox-scenario '^mysqldb_contrib-.*-mysqldb' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: mysql + pattern: '^mysqldb_contrib-.*-mysqldb' pymysql: + executor: ddtrace_dev docker: - - *test_runner - - image: mysql:5.7 - env: - - MYSQL_ROOT_PASSWORD=admin - - MYSQL_PASSWORD=test - - MYSQL_USER=test - - MYSQL_DATABASE=test - resource_class: *resource_class + - image: *ddtrace_dev_image + - *mysql_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' mysql - - run: scripts/run-tox-scenario '^pymysql_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: mysql + pattern: '^pymysql_contrib-' pylibmc: + executor: ddtrace_dev docker: - - *test_runner - - image: memcached:1.5-alpine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *memcached_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^pylibmc_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^pylibmc_contrib-' pymemcache: + executor: ddtrace_dev docker: - - *test_runner - - image: memcached:1.5-alpine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *memcached_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^pymemcache_contrib\(_autopatch\)\?-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^pymemcache_contrib\(_autopatch\)\?-' mongoengine: + executor: ddtrace_dev docker: - - *test_runner - - image: mongo:3.6 - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *mongo_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^mongoengine_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^mongoengine_contrib-' pymongo: + executor: ddtrace_dev docker: - - *test_runner - - image: mongo:3.6 - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *mongo_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^pymongo_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^pymongo_contrib-' pyramid: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^pyramid_contrib\(_autopatch\)\?-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^pyramid_contrib\(_autopatch\)\?-' requests: + executor: ddtrace_dev docker: - - *test_runner + - image: *ddtrace_dev_image - *httpbin_local - resource_class: *resource_class steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^requests_contrib\(_autopatch\)\?-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^requests_contrib\(_autopatch\)\?-' requestsgevent: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^requests_gevent_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^requests_gevent_contrib-' sqlalchemy: + executor: ddtrace_dev docker: - - *test_runner - - image: postgres:10.5-alpine - env: - - POSTGRES_PASSWORD=postgres - - POSTGRES_USER=postgres - - POSTGRES_DB=postgres - - image: mysql:5.7 - env: - - MYSQL_ROOT_PASSWORD=admin - - MYSQL_PASSWORD=test - - MYSQL_USER=test - - MYSQL_DATABASE=test - resource_class: *resource_class + - image: *ddtrace_dev_image + - *postgres_server + - *mysql_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' postgres mysql - - run: scripts/run-tox-scenario '^sqlalchemy_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: postgres mysql + pattern: '^sqlalchemy_contrib-' dbapi: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^dbapi_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^dbapi_contrib-' psycopg: + executor: ddtrace_dev docker: - - *test_runner - - image: postgres:10.5-alpine - env: - - POSTGRES_PASSWORD=postgres - - POSTGRES_USER=postgres - - POSTGRES_DB=postgres - resource_class: *resource_class + - image: *ddtrace_dev_image + - *postgres_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' postgres - - run: scripts/run-tox-scenario '^psycopg_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: postgres + pattern: '^psycopg_contrib-' aiobotocore: + executor: ddtrace_dev docker: - - *test_runner - - image: palazzem/moto:1.0.1 - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *moto_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^aiobotocore_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^aiobotocore_contrib' aiopg: + executor: ddtrace_dev docker: - - *test_runner - - image: postgres:10.5-alpine - env: - - POSTGRES_PASSWORD=postgres - - POSTGRES_USER=postgres - - POSTGRES_DB=postgres - resource_class: *resource_class + - image: *ddtrace_dev_image + - *postgres_server steps: - - checkout - - *restore_cache_step - - run: tox -e 'wait' postgres - - run: scripts/run-tox-scenario '^aiopg_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: postgres + pattern: '^aiopg_contrib-' redis: + executor: ddtrace_dev docker: - - *test_runner - - image: redis:4.0-alpine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *redis_image steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^redis_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^redis_contrib-' rediscluster: + executor: ddtrace_dev docker: - - *test_runner - - image: grokzen/redis-cluster:4.0.9 - env: + - image: *ddtrace_dev_image + - image: *rediscluster_image + environment: - IP=0.0.0.0 - resource_class: *resource_class steps: - - checkout - - *restore_cache_step - - run: tox -e wait rediscluster - - run: scripts/run-tox-scenario '^rediscluster_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: rediscluster + pattern: '^rediscluster_contrib-' vertica: + executor: ddtrace_dev docker: - - *test_runner - - image: sumitchawla/vertica - env: + - image: *ddtrace_dev_image + - image: *vertica_image + environment: - VP_TEST_USER=dbadmin - VP_TEST_PASSWORD=abc123 - VP_TEST_DATABASE=docker - resource_class: *resource_class steps: - - checkout - - *restore_cache_step - - run: tox -e wait vertica - - run: scripts/run-tox-scenario '^vertica_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: vertica + pattern: '^vertica_contrib-' kombu: + executor: ddtrace_dev docker: - - *test_runner - - image: rabbitmq:3.7-alpine - resource_class: *resource_class + - image: *ddtrace_dev_image + - image: *rabbitmq_image steps: - - checkout - - *restore_cache_step - - run: tox -e wait rabbitmq - - run: scripts/run-tox-scenario '^kombu_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + wait: rabbitmq + pattern: '^kombu_contrib-' sqlite3: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^sqlite3_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^sqlite3_contrib-' unit_tests: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^unit_tests-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^unit_tests-' benchmarks: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - checkout - - *restore_cache_step + - setup_tox + - restore_tox_cache - run: command: | mkdir -p /tmp/test-reports @@ -738,16 +633,15 @@ jobs: path: /tmp/test-reports - store_artifacts: path: /tmp/test-reports - - *persist_to_workspace_step - - *save_cache_step + - save_results + - save_tox_cache deploy_dev: # build the master branch releasing development docs and wheels - docker: - - image: circleci/python:3.8 - resource_class: *resource_class + executor: python38 steps: - checkout + - setup_tox - run: sudo apt-get -y install rake - run: pip install tox mkwheelhouse awscli - run: tox -e docs @@ -755,46 +649,29 @@ jobs: - run: S3_DIR=trace-dev rake release:wheel jinja2: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^jinja2_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^jinja2_contrib-' mako: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^mako_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^mako_contrib-' algoliasearch: - docker: - - *test_runner - resource_class: *resource_class + executor: ddtrace_dev steps: - - checkout - - *restore_cache_step - - run: scripts/run-tox-scenario '^algoliasearch_contrib-' - - *persist_to_workspace_step - - *save_cache_step + - run_tox_scenario: + pattern: '^algoliasearch_contrib-' build_docs: # deploy official documentation - docker: - - image: circleci/python:3.8 - resource_class: *resource_class + executor: python38 steps: - checkout - - run: pip install tox + - setup_tox - run: tox -e docs - run: command: | @@ -805,9 +682,7 @@ jobs: deploy_to_s3: # deploy official documentation - docker: - - image: circleci/python:3.6 - resource_class: *resource_class + executor: python38 steps: - checkout - run: sudo apt-get -y install rake @@ -815,15 +690,13 @@ jobs: - run: sudo pip install mkwheelhouse sphinx==1.7.5 awscli wrapt - run: S3_DIR=trace rake release:docs - wait_all_tests: - # this step ensures all `tox` environments are properly executed - docker: - - *test_runner - resource_class: *resource_class + verify_all_tests_ran: + executor: python38 steps: - attach_workspace: at: /tmp/workspace - checkout + - setup_tox - run: ls /tmp/workspace/* # debug: shows how many time each test was executed - run: jq -s ".[]|.testenvs|keys|.[]" /tmp/workspace/* | grep -v GLOB | sed 's/"//g' | sort | uniq -c | sort -rn @@ -835,6 +708,11 @@ jobs: - run: diff all_tests all_executed_tests +requires_pre_test: &requires_pre_test + requires: + - black + - flake8 + workflows: version: 2 @@ -853,229 +731,78 @@ workflows: - approve_docs_deployment test: jobs: + # Jobs that should run before individual integration test suites - build_docs - black - flake8 - - test_build - - aiobotocore: - requires: - - flake8 - - black - - aiohttp: - requires: - - flake8 - - black - - aiopg: - requires: - - flake8 - - black - - asyncio: - requires: - - flake8 - - black - - algoliasearch: - requires: - - flake8 - - black - - benchmarks: - requires: - - flake8 - - black - - boto: - requires: - - flake8 - - black - - bottle: - requires: - - flake8 - - black - - cassandra: - requires: - - flake8 - - black - - celery: - requires: - - flake8 - - black - - consul: - requires: - - flake8 - - black - - dbapi: - requires: - - flake8 - - black - - ddtracerun: - requires: - - flake8 - - black - - django: - requires: - - flake8 - - black - - djangorestframework: - requires: - - flake8 - - black - - dogpile_cache: - requires: - - flake8 - - black - - elasticsearch: - requires: - - flake8 - - black - - falcon: - requires: - - flake8 - - black - - flask: - requires: - - flake8 - - black - - futures: - requires: - - flake8 - - black - - gevent: - requires: - - flake8 - - black - - grpc: - requires: - - flake8 - - black - - httplib: - requires: - - flake8 - - black - - integration: - requires: - - flake8 - - black - - internal: - requires: - - flake8 - - black - - jinja2: - requires: - - flake8 - - black - - kombu: - requires: - - flake8 - - black - - mako: - requires: - - flake8 - - black - - molten: - requires: - - flake8 - - black - - mongoengine: - requires: - - flake8 - - black - - mysqlconnector: - requires: - - flake8 - - black - - mysqldb: - requires: - - flake8 - - black - - mysqlpython: - requires: - - flake8 - - black - - opentracer: - requires: - - flake8 - - black - - psycopg: - requires: - - flake8 - - black - - pylibmc: - requires: - - flake8 - - black - - pylons: - requires: - - flake8 - - black - - pymemcache: - requires: - - flake8 - - black - - pymongo: - requires: - - flake8 - - black - - pymysql: - requires: - - flake8 - - black - - pyramid: - requires: - - flake8 - - black - - redis: - requires: - - flake8 - - black - - rediscluster: - requires: - - flake8 - - black - - requests: - requires: - - flake8 - - black - - requestsgevent: - requires: - - flake8 - - black - - sqlalchemy: - requires: - - flake8 - - black - - sqlite3: - requires: - - flake8 - - black - - test_utils: - requires: - - flake8 - - black - - test_logging: - requires: - - flake8 - - black - - tornado: - requires: - - flake8 - - black - - tracer: - requires: - - flake8 - - black - - unit_tests: - requires: - - flake8 - - black - - vertica: - requires: - - flake8 - - black - - wait_all_tests: - requires: - # Initial jobs + + # Test building the package + - test_build_py38: *requires_pre_test + - test_build_py37: *requires_pre_test + - test_build_py36: *requires_pre_test + - test_build_py35: *requires_pre_test + - test_build_py34: *requires_pre_test + - test_build_py27: *requires_pre_test + + # Integration test suites + - aiobotocore: *requires_pre_test + - aiohttp: *requires_pre_test + - aiopg: *requires_pre_test + - asyncio: *requires_pre_test + - algoliasearch: *requires_pre_test + - benchmarks: *requires_pre_test + - boto: *requires_pre_test + - bottle: *requires_pre_test + - cassandra: *requires_pre_test + - celery: *requires_pre_test + - consul: *requires_pre_test + - dbapi: *requires_pre_test + - ddtracerun: *requires_pre_test + - django: *requires_pre_test + - djangorestframework: *requires_pre_test + - dogpile_cache: *requires_pre_test + - elasticsearch: *requires_pre_test + - falcon: *requires_pre_test + - flask: *requires_pre_test + - futures: *requires_pre_test + - gevent: *requires_pre_test + - grpc: *requires_pre_test + - httplib: *requires_pre_test + - integration: *requires_pre_test + - internal: *requires_pre_test + - jinja2: *requires_pre_test + - kombu: *requires_pre_test + - mako: *requires_pre_test + - molten: *requires_pre_test + - mongoengine: *requires_pre_test + - mysqlconnector: *requires_pre_test + - mysqldb: *requires_pre_test + - mysqlpython: *requires_pre_test + - opentracer: *requires_pre_test + - psycopg: *requires_pre_test + - pylibmc: *requires_pre_test + - pylons: *requires_pre_test + - pymemcache: *requires_pre_test + - pymongo: *requires_pre_test + - pymysql: *requires_pre_test + - pyramid: *requires_pre_test + - redis: *requires_pre_test + - rediscluster: *requires_pre_test + - requests: *requires_pre_test + - requestsgevent: *requires_pre_test + - sqlalchemy: *requires_pre_test + - sqlite3: *requires_pre_test + - test_utils: *requires_pre_test + - test_logging: *requires_pre_test + - tornado: *requires_pre_test + # tracer + - tracer: *requires_pre_test + - unit_tests: *requires_pre_test + - vertica: *requires_pre_test + - verify_all_tests_ran: + requires: + # Individual tests do not need this to start running - build_docs - - black - - flake8 - - test_build # flake8 dependent jobs - aiobotocore @@ -1125,15 +852,21 @@ workflows: - requestsgevent - sqlalchemy - sqlite3 + - test_build_py38 + - test_build_py37 + - test_build_py36 + - test_build_py35 + - test_build_py34 + - test_build_py27 - test_utils - test_logging - - tornado - tracer + - tornado - unit_tests - vertica - deploy_dev: requires: - - wait_all_tests + - verify_all_tests_ran filters: branches: only: master From 69cd010f37e392bd73e7a9040d0b2c1fee393bc1 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Tue, 18 Feb 2020 11:01:50 -0500 Subject: [PATCH 71/81] core: Set _dd.measured tag on integration spans (#1196) * internal: Add ability to easily mark a span to be measured * contrib: start to update integrations to indicate measured spans * fix flake8 * tests: start work on _dd.measured test updates * finish up tests for existing modified integrations * fix flake8 issues * fix s/assert_span_is_measured/assert_is_measured/g * fix issue * ensure we measure old patching method * measure more integrations * fix black formatting * more integrations and tests * add missing import * finish up integration changes * fix flake8 issues * fix failed tests * fix double dot * fix failing test * use span.set_tag(SPAN_MEASURED_KEY) instead * move _dd.measured to metrics * more consistently set _dd.measured value Co-authored-by: Tahir H. Butt --- ddtrace/constants.py | 1 + ddtrace/contrib/aiobotocore/patch.py | 3 +- ddtrace/contrib/aiohttp/middlewares.py | 3 +- ddtrace/contrib/aiopg/connection.py | 3 +- ddtrace/contrib/algoliasearch/patch.py | 3 ++ ddtrace/contrib/boto/patch.py | 5 +- ddtrace/contrib/botocore/patch.py | 4 +- ddtrace/contrib/bottle/trace.py | 5 +- ddtrace/contrib/cassandra/session.py | 3 +- ddtrace/contrib/celery/signals.py | 3 ++ ddtrace/contrib/consul/patch.py | 3 +- ddtrace/contrib/dbapi/__init__.py | 5 +- ddtrace/contrib/dogpile_cache/region.py | 3 ++ ddtrace/contrib/elasticsearch/patch.py | 4 +- ddtrace/contrib/elasticsearch/transport.py | 5 +- ddtrace/contrib/falcon/middleware.py | 3 +- ddtrace/contrib/flask/patch.py | 3 +- ddtrace/contrib/flask_cache/tracers.py | 9 ++-- ddtrace/contrib/grpc/client_interceptor.py | 3 +- ddtrace/contrib/grpc/server_interceptor.py | 3 +- ddtrace/contrib/httplib/patch.py | 3 +- ddtrace/contrib/jinja2/patch.py | 2 + ddtrace/contrib/kombu/patch.py | 4 +- ddtrace/contrib/mako/patch.py | 2 + ddtrace/contrib/molten/patch.py | 3 +- ddtrace/contrib/psycopg/connection.py | 2 + ddtrace/contrib/pylibmc/client.py | 6 ++- ddtrace/contrib/pylons/middleware.py | 3 +- ddtrace/contrib/pymemcache/client.py | 3 +- ddtrace/contrib/pymongo/client.py | 4 +- ddtrace/contrib/pyramid/trace.py | 3 +- ddtrace/contrib/redis/patch.py | 4 +- ddtrace/contrib/rediscluster/patch.py | 3 +- ddtrace/contrib/requests/connection.py | 3 +- ddtrace/contrib/sqlalchemy/engine.py | 3 +- ddtrace/contrib/tornado/handlers.py | 5 +- ddtrace/contrib/vertica/patch.py | 9 +++- ddtrace/span.py | 17 +++++- tests/contrib/aiobotocore/py35/test.py | 5 +- tests/contrib/aiobotocore/test.py | 21 +++++++- tests/contrib/aiohttp/test_request.py | 3 ++ tests/contrib/aiohttp/test_request_safety.py | 2 + tests/contrib/aiopg/test.py | 2 + tests/contrib/algoliasearch/test.py | 2 + tests/contrib/boto/test.py | 11 +++- tests/contrib/botocore/test.py | 9 +++- tests/contrib/bottle/test.py | 13 ++++- tests/contrib/cassandra/test.py | 3 ++ tests/contrib/celery/test_integration.py | 24 +++++++++ tests/contrib/consul/test.py | 9 ++++ tests/contrib/dbapi/test_unit.py | 9 ++++ tests/contrib/dogpile_cache/test_tracing.py | 9 ++++ tests/contrib/elasticsearch/test.py | 14 ++++- tests/contrib/falcon/test_suite.py | 17 +++++- tests/contrib/flask/test_request.py | 11 +++- .../flask_autopatch/test_flask_autopatch.py | 12 ++--- tests/contrib/flask_cache/test.py | 10 ++++ tests/contrib/grpc/test_grpc.py | 3 ++ tests/contrib/httplib/test_httplib.py | 15 +++++- tests/contrib/jinja2/__init__.py | 0 tests/contrib/jinja2/test_jinja2.py | 10 ++++ tests/contrib/kombu/test.py | 3 ++ tests/contrib/mako/__init__.py | 0 tests/contrib/mako/test_mako.py | 5 ++ tests/contrib/molten/test_molten.py | 6 ++- tests/contrib/mongoengine/test.py | 9 ++++ tests/contrib/mysql/test_mysql.py | 6 +++ tests/contrib/mysqldb/test_mysql.py | 8 +++ tests/contrib/psycopg/test_psycopg.py | 6 +++ tests/contrib/pylibmc/test.py | 2 + tests/contrib/pylons/test_pylons.py | 8 ++- tests/contrib/pymemcache/test_client_mixin.py | 2 + tests/contrib/pymongo/test.py | 5 ++ tests/contrib/pymysql/test_pymysql.py | 6 +++ tests/contrib/pyramid/utils.py | 12 ++++- tests/contrib/redis/test.py | 7 +++ tests/contrib/rediscluster/test.py | 3 ++ tests/contrib/requests/test_requests.py | 17 +++++- tests/contrib/sqlalchemy/test_mysql.py | 2 + tests/contrib/sqlalchemy/test_patch.py | 4 ++ tests/contrib/sqlalchemy/test_postgres.py | 2 + tests/contrib/sqlalchemy/test_sqlite.py | 2 + tests/contrib/sqlite3/test_sqlite3.py | 11 ++++ tests/contrib/tornado/test_tornado_web.py | 13 ++++- tests/contrib/vertica/test_vertica.py | 4 ++ tests/test_span.py | 52 ++++++++++++++++++- tests/utils/__init__.py | 17 ++++++ 87 files changed, 517 insertions(+), 62 deletions(-) create mode 100644 tests/contrib/jinja2/__init__.py create mode 100644 tests/contrib/mako/__init__.py diff --git a/ddtrace/constants.py b/ddtrace/constants.py index 803e98a53fa..4d2075c57d3 100644 --- a/ddtrace/constants.py +++ b/ddtrace/constants.py @@ -8,6 +8,7 @@ ORIGIN_KEY = '_dd.origin' HOSTNAME_KEY = '_dd.hostname' ENV_KEY = 'env' +SPAN_MEASURED_KEY = '_dd.measured' NUMERIC_TAGS = (ANALYTICS_SAMPLE_RATE_KEY, ) diff --git a/ddtrace/contrib/aiobotocore/patch.py b/ddtrace/contrib/aiobotocore/patch.py index ca903f170d1..fd35b26a048 100644 --- a/ddtrace/contrib/aiobotocore/patch.py +++ b/ddtrace/contrib/aiobotocore/patch.py @@ -5,7 +5,7 @@ from aiobotocore.endpoint import ClientResponseContentProxy -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...pin import Pin from ...ext import SpanTypes, http, aws from ...compat import PYTHON_VERSION_INFO @@ -81,6 +81,7 @@ def _wrapped_api_call(original_func, instance, args, kwargs): with pin.tracer.trace('{}.command'.format(endpoint_name), service='{}.{}'.format(pin.service, endpoint_name), span_type=SpanTypes.HTTP) as span: + span.set_tag(SPAN_MEASURED_KEY) if len(args) > 0: operation = args[0] diff --git a/ddtrace/contrib/aiohttp/middlewares.py b/ddtrace/contrib/aiohttp/middlewares.py index 52269b91939..7be8e08189c 100644 --- a/ddtrace/contrib/aiohttp/middlewares.py +++ b/ddtrace/contrib/aiohttp/middlewares.py @@ -2,7 +2,7 @@ from ..asyncio import context_provider from ...compat import stringify -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...settings import config @@ -45,6 +45,7 @@ def attach_context(request): service=service, span_type=SpanTypes.WEB, ) + request_span.set_tag(SPAN_MEASURED_KEY) # Configure trace search sample rate # DEV: aiohttp is special case maintains separate configuration from config api diff --git a/ddtrace/contrib/aiopg/connection.py b/ddtrace/contrib/aiopg/connection.py index f5dc3afb47f..5dabf4b5b4c 100644 --- a/ddtrace/contrib/aiopg/connection.py +++ b/ddtrace/contrib/aiopg/connection.py @@ -4,7 +4,7 @@ from aiopg.utils import _ContextManager from .. import dbapi -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, sql from ...pin import Pin from ...settings import config @@ -29,6 +29,7 @@ def _trace_method(self, method, resource, extra_tags, *args, **kwargs): with pin.tracer.trace(self._datadog_name, service=service, resource=resource, span_type=SpanTypes.SQL) as s: + s.set_tag(SPAN_MEASURED_KEY) s.set_tag(sql.QUERY, resource) s.set_tags(pin.tags) s.set_tags(extra_tags) diff --git a/ddtrace/contrib/algoliasearch/patch.py b/ddtrace/contrib/algoliasearch/patch.py index 859b6eb0f7d..b00db7376f3 100644 --- a/ddtrace/contrib/algoliasearch/patch.py +++ b/ddtrace/contrib/algoliasearch/patch.py @@ -1,3 +1,4 @@ +from ...constants import SPAN_MEASURED_KEY from ddtrace.pin import Pin from ddtrace.settings import config from ddtrace.utils.wrappers import unwrap as _u @@ -101,6 +102,8 @@ def _patched_search(func, instance, wrapt_args, wrapt_kwargs): return func(*wrapt_args, **wrapt_kwargs) with pin.tracer.trace('algoliasearch.search', service=pin.service) as span: + span.set_tag(SPAN_MEASURED_KEY) + if not span.sampled: return func(*wrapt_args, **wrapt_kwargs) diff --git a/ddtrace/contrib/boto/patch.py b/ddtrace/contrib/boto/patch.py index c1de9e9cb5b..fbeab218511 100644 --- a/ddtrace/contrib/boto/patch.py +++ b/ddtrace/contrib/boto/patch.py @@ -3,7 +3,7 @@ import inspect from ddtrace import config -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...pin import Pin from ...ext import SpanTypes, http, aws from ...utils.wrappers import unwrap @@ -69,6 +69,7 @@ def patched_query_request(original_func, instance, args, kwargs): service='{}.{}'.format(pin.service, endpoint_name), span_type=SpanTypes.HTTP, ) as span: + span.set_tag(SPAN_MEASURED_KEY) operation_name = None if args: @@ -137,7 +138,7 @@ def patched_auth_request(original_func, instance, args, kwargs): service='{}.{}'.format(pin.service, endpoint_name), span_type=SpanTypes.HTTP, ) as span: - + span.set_tag(SPAN_MEASURED_KEY) if args: http_method = args[0] span.resource = '%s.%s' % (endpoint_name, http_method.lower()) diff --git a/ddtrace/contrib/botocore/patch.py b/ddtrace/contrib/botocore/patch.py index 87ed79ee96b..a700c912972 100644 --- a/ddtrace/contrib/botocore/patch.py +++ b/ddtrace/contrib/botocore/patch.py @@ -7,7 +7,7 @@ import botocore.client # project -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...pin import Pin from ...ext import SpanTypes, http, aws from ...utils.formats import deep_getattr @@ -47,7 +47,7 @@ def patched_api_call(original_func, instance, args, kwargs): with pin.tracer.trace('{}.command'.format(endpoint_name), service='{}.{}'.format(pin.service, endpoint_name), span_type=SpanTypes.HTTP) as span: - + span.set_tag(SPAN_MEASURED_KEY) operation = None if args: operation = args[0] diff --git a/ddtrace/contrib/bottle/trace.py b/ddtrace/contrib/bottle/trace.py index 12c196b32ef..efd87661945 100644 --- a/ddtrace/contrib/bottle/trace.py +++ b/ddtrace/contrib/bottle/trace.py @@ -5,7 +5,7 @@ import ddtrace # project -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...settings import config @@ -36,8 +36,9 @@ def wrapped(*args, **kwargs): self.tracer.context_provider.activate(context) with self.tracer.trace( - 'bottle.request', service=self.service, resource=resource, span_type=SpanTypes.WEB + 'bottle.request', service=self.service, resource=resource, span_type=SpanTypes.WEB, ) as s: + s.set_tag(SPAN_MEASURED_KEY) # set analytics sample rate with global config enabled s.set_tag( ANALYTICS_SAMPLE_RATE_KEY, diff --git a/ddtrace/contrib/cassandra/session.py b/ddtrace/contrib/cassandra/session.py index 512aba7758d..171ec3e2f74 100644 --- a/ddtrace/contrib/cassandra/session.py +++ b/ddtrace/contrib/cassandra/session.py @@ -8,7 +8,7 @@ # project from ...compat import stringify -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, net, cassandra as cassx, errors from ...internal.logger import get_logger from ...pin import Pin @@ -182,6 +182,7 @@ def _start_span_and_set_tags(pin, query, session, cluster): service = pin.service tracer = pin.tracer span = tracer.trace('cassandra.query', service=service, span_type=SpanTypes.CASSANDRA) + span.set_tag(SPAN_MEASURED_KEY) _sanitize_query(span, query) span.set_tags(_extract_session_metas(session)) # FIXME[matt] do once? span.set_tags(_extract_cluster_metas(cluster)) diff --git a/ddtrace/contrib/celery/signals.py b/ddtrace/contrib/celery/signals.py index 2afcce556ac..b5c2923a2ca 100644 --- a/ddtrace/contrib/celery/signals.py +++ b/ddtrace/contrib/celery/signals.py @@ -2,6 +2,7 @@ from celery import registry +from ...constants import SPAN_MEASURED_KEY from ...ext import SpanTypes from ...internal.logger import get_logger from . import constants as c @@ -29,6 +30,7 @@ def trace_prerun(*args, **kwargs): # propagate the `Span` in the current task Context service = config.celery['worker_service_name'] span = pin.tracer.trace(c.WORKER_ROOT_SPAN, service=service, resource=task.name, span_type=SpanTypes.WORKER) + span.set_tag(SPAN_MEASURED_KEY) attach_span(task, task_id, span) @@ -79,6 +81,7 @@ def trace_before_publish(*args, **kwargs): # in the task_after_publish signal service = config.celery['producer_service_name'] span = pin.tracer.trace(c.PRODUCER_ROOT_SPAN, service=service, resource=task_name) + span.set_tag(SPAN_MEASURED_KEY) span.set_tag(c.TASK_TAG_KEY, c.TASK_APPLY_ASYNC) span.set_tag('celery.id', task_id) span.set_tags(tags_from_context(kwargs)) diff --git a/ddtrace/contrib/consul/patch.py b/ddtrace/contrib/consul/patch.py index aec3390aeb7..3827d06515a 100644 --- a/ddtrace/contrib/consul/patch.py +++ b/ddtrace/contrib/consul/patch.py @@ -3,7 +3,7 @@ from ddtrace.vendor.wrapt import wrap_function_wrapper as _w from ddtrace import config -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import consul as consulx from ...pin import Pin from ...utils.wrappers import unwrap as _u @@ -47,6 +47,7 @@ def trace_func(wrapped, instance, args, kwargs): resource = name.upper() with pin.tracer.trace(consulx.CMD, service=pin.service, resource=resource) as span: + span.set_tag(SPAN_MEASURED_KEY) rate = config.consul.get_analytics_sample_rate() if rate is not None: span.set_tag(ANALYTICS_SAMPLE_RATE_KEY, rate) diff --git a/ddtrace/contrib/dbapi/__init__.py b/ddtrace/contrib/dbapi/__init__.py index 4c48ef66382..ed12986c9b9 100644 --- a/ddtrace/contrib/dbapi/__init__.py +++ b/ddtrace/contrib/dbapi/__init__.py @@ -2,7 +2,7 @@ Generic dbapi tracing code. """ -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, sql from ...internal.logger import get_logger from ...pin import Pin @@ -43,7 +43,10 @@ def _trace_method(self, method, name, resource, extra_tags, *args, **kwargs): if not pin or not pin.enabled(): return method(*args, **kwargs) service = pin.service + measured = name == self._self_datadog_name with pin.tracer.trace(name, service=service, resource=resource, span_type=SpanTypes.SQL) as s: + if measured: + s.set_tag(SPAN_MEASURED_KEY) # No reason to tag the query since it is set as the resource by the agent. See: # https://github.com/DataDog/datadog-trace-agent/blob/bda1ebbf170dd8c5879be993bdd4dbae70d10fda/obfuscate/sql.go#L232 s.set_tags(pin.tags) diff --git a/ddtrace/contrib/dogpile_cache/region.py b/ddtrace/contrib/dogpile_cache/region.py index 61d1cdb6183..e7cc4fecd59 100644 --- a/ddtrace/contrib/dogpile_cache/region.py +++ b/ddtrace/contrib/dogpile_cache/region.py @@ -1,5 +1,6 @@ import dogpile +from ...constants import SPAN_MEASURED_KEY from ...pin import Pin @@ -10,6 +11,7 @@ def _wrap_get_create(func, instance, args, kwargs): key = args[0] with pin.tracer.trace('dogpile.cache', resource='get_or_create', span_type='cache') as span: + span.set_tag(SPAN_MEASURED_KEY) span.set_tag('key', key) span.set_tag('region', instance.name) span.set_tag('backend', instance.actual_backend.__class__.__name__) @@ -23,6 +25,7 @@ def _wrap_get_create_multi(func, instance, args, kwargs): keys = args[0] with pin.tracer.trace('dogpile.cache', resource='get_or_create_multi', span_type='cache') as span: + span.set_tag(SPAN_MEASURED_KEY) span.set_tag('keys', keys) span.set_tag('region', instance.name) span.set_tag('backend', instance.actual_backend.__class__.__name__) diff --git a/ddtrace/contrib/elasticsearch/patch.py b/ddtrace/contrib/elasticsearch/patch.py index 16b9d6e0e0c..d291b6b7ef8 100644 --- a/ddtrace/contrib/elasticsearch/patch.py +++ b/ddtrace/contrib/elasticsearch/patch.py @@ -5,7 +5,7 @@ from .quantize import quantize from ...compat import urlencode -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, elasticsearch as metadata, http from ...pin import Pin from ...utils.wrappers import unwrap as _u @@ -53,6 +53,8 @@ def _perform_request(func, instance, args, kwargs): return func(*args, **kwargs) with pin.tracer.trace('elasticsearch.query', span_type=SpanTypes.ELASTICSEARCH) as span: + span.set_tag(SPAN_MEASURED_KEY) + # Don't instrument if the trace is not sampled if not span.sampled: return func(*args, **kwargs) diff --git a/ddtrace/contrib/elasticsearch/transport.py b/ddtrace/contrib/elasticsearch/transport.py index 347f710bd6e..7bf8d42f418 100644 --- a/ddtrace/contrib/elasticsearch/transport.py +++ b/ddtrace/contrib/elasticsearch/transport.py @@ -4,6 +4,7 @@ from .quantize import quantize +from ...constants import SPAN_MEASURED_KEY from ...utils.deprecation import deprecated from ...compat import urlencode from ...ext import SpanTypes, http, elasticsearch as metadata @@ -24,12 +25,14 @@ class TracedTransport(elasticsearch.Transport): _datadog_service = datadog_service def perform_request(self, method, url, params=None, body=None): - with self._datadog_tracer.trace('elasticsearch.query', span_type=SpanTypes.ELASTICSEARCH) as s: + with self._datadog_tracer.trace('elasticsearch.query', + span_type=SpanTypes.ELASTICSEARCH) as s: # Don't instrument if the trace is not sampled if not s.sampled: return super(TracedTransport, self).perform_request( method, url, params=params, body=body) + s.set_tag(SPAN_MEASURED_KEY) s.service = self._datadog_service s.set_tag(metadata.METHOD, method) s.set_tag(metadata.URL, url) diff --git a/ddtrace/contrib/falcon/middleware.py b/ddtrace/contrib/falcon/middleware.py index e30b0704944..2ff86d69963 100644 --- a/ddtrace/contrib/falcon/middleware.py +++ b/ddtrace/contrib/falcon/middleware.py @@ -5,7 +5,7 @@ from ddtrace.propagation.http import HTTPPropagator from ...compat import iteritems -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...settings import config @@ -32,6 +32,7 @@ def process_request(self, req, resp): service=self.service, span_type=SpanTypes.WEB, ) + span.set_tag(SPAN_MEASURED_KEY) # set analytics sample rate with global config enabled span.set_tag( diff --git a/ddtrace/contrib/flask/patch.py b/ddtrace/contrib/flask/patch.py index 243a8cc56bb..e01805b2966 100644 --- a/ddtrace/contrib/flask/patch.py +++ b/ddtrace/contrib/flask/patch.py @@ -7,7 +7,7 @@ from ddtrace import compat from ddtrace import config, Pin -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator @@ -283,6 +283,7 @@ def traced_wsgi_app(pin, wrapped, instance, args, kwargs): # We will override this below in `traced_dispatch_request` when we have a `RequestContext` and possibly a url rule resource = u'{} {}'.format(request.method, request.path) with pin.tracer.trace('flask.request', service=pin.service, resource=resource, span_type=SpanTypes.WEB) as s: + s.set_tag(SPAN_MEASURED_KEY) # set analytics sample rate with global config enabled sample_rate = config.flask.get_analytics_sample_rate(use_global_config=True) if sample_rate is not None: diff --git a/ddtrace/contrib/flask_cache/tracers.py b/ddtrace/contrib/flask_cache/tracers.py index 31c7ea9b22c..072d5be3f50 100644 --- a/ddtrace/contrib/flask_cache/tracers.py +++ b/ddtrace/contrib/flask_cache/tracers.py @@ -7,7 +7,7 @@ # project from .utils import _extract_conn_tags, _resource_from_cache_prefix -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes from ...settings import config @@ -45,11 +45,8 @@ def __trace(self, cmd): Start a tracing with default attributes and tags """ # create a new span - s = self._datadog_tracer.trace( - cmd, - span_type=SpanTypes.CACHE, - service=self._datadog_service - ) + s = self._datadog_tracer.trace(cmd, span_type=SpanTypes.CACHE, service=self._datadog_service) + s.set_tag(SPAN_MEASURED_KEY) # set span tags s.set_tag(CACHE_BACKEND, self.config.get('CACHE_TYPE')) s.set_tags(self._datadog_meta) diff --git a/ddtrace/contrib/grpc/client_interceptor.py b/ddtrace/contrib/grpc/client_interceptor.py index be1e66efa91..1791b981b72 100644 --- a/ddtrace/contrib/grpc/client_interceptor.py +++ b/ddtrace/contrib/grpc/client_interceptor.py @@ -7,7 +7,7 @@ from ddtrace.ext import SpanTypes, errors from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from . import constants from .utils import parse_method_path @@ -154,6 +154,7 @@ def _intercept_client_call(self, method_kind, client_call_details): span = tracer.trace( "grpc", span_type=SpanTypes.GRPC, service=self._pin.service, resource=client_call_details.method, ) + span.set_tag(SPAN_MEASURED_KEY) # tags for method details method_path = client_call_details.method diff --git a/ddtrace/contrib/grpc/server_interceptor.py b/ddtrace/contrib/grpc/server_interceptor.py index dbce643543a..ab5bef869ae 100644 --- a/ddtrace/contrib/grpc/server_interceptor.py +++ b/ddtrace/contrib/grpc/server_interceptor.py @@ -5,7 +5,7 @@ from ddtrace.ext import errors from ddtrace.compat import to_unicode -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes from ...propagation.http import HTTPPropagator from . import constants @@ -69,6 +69,7 @@ def _fn(self, method_kind, behavior, args, kwargs): service=self._pin.service, resource=self._handler_call_details.method, ) + span.set_tag(SPAN_MEASURED_KEY) method_path = self._handler_call_details.method method_package, method_service, method_name = parse_method_path(method_path) diff --git a/ddtrace/contrib/httplib/patch.py b/ddtrace/contrib/httplib/patch.py index 61297cbd741..71685500f7f 100644 --- a/ddtrace/contrib/httplib/patch.py +++ b/ddtrace/contrib/httplib/patch.py @@ -3,7 +3,7 @@ # Project from ...compat import PY2, httplib, parse -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http as ext_http from ...http import store_request_headers, store_response_headers from ...internal.logger import get_logger @@ -56,6 +56,7 @@ def _wrap_putrequest(func, instance, args, kwargs): try: # Create a new span and attach to this instance (so we can retrieve/update/close later on the response) span = pin.tracer.trace(span_name, span_type=SpanTypes.HTTP) + span.set_tag(SPAN_MEASURED_KEY) setattr(instance, '_datadog_span', span) method, path = args[:2] diff --git a/ddtrace/contrib/jinja2/patch.py b/ddtrace/contrib/jinja2/patch.py index 65ac216a4f6..f7be9147bbf 100644 --- a/ddtrace/contrib/jinja2/patch.py +++ b/ddtrace/contrib/jinja2/patch.py @@ -3,6 +3,7 @@ from ddtrace import config +from ...constants import SPAN_MEASURED_KEY from ...ext import SpanTypes from ...utils.formats import get_env from ...pin import Pin @@ -50,6 +51,7 @@ def _wrap_render(wrapped, instance, args, kwargs): template_name = instance.name or DEFAULT_TEMPLATE_NAME with pin.tracer.trace('jinja2.render', pin.service, span_type=SpanTypes.TEMPLATE) as span: + span.set_tag(SPAN_MEASURED_KEY) try: return wrapped(*args, **kwargs) finally: diff --git a/ddtrace/contrib/kombu/patch.py b/ddtrace/contrib/kombu/patch.py index 828cc044b08..0101b42edd8 100644 --- a/ddtrace/contrib/kombu/patch.py +++ b/ddtrace/contrib/kombu/patch.py @@ -3,7 +3,7 @@ from ddtrace.vendor import wrapt # project -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, kombu as kombux from ...pin import Pin from ...propagation.http import HTTPPropagator @@ -79,6 +79,7 @@ def traced_receive(func, instance, args, kwargs): if context.trace_id: pin.tracer.context_provider.activate(context) with pin.tracer.trace(kombux.RECEIVE_NAME, service=pin.service, span_type=SpanTypes.WORKER) as s: + s.set_tag(SPAN_MEASURED_KEY) # run the command exchange = message.delivery_info['exchange'] s.resource = exchange @@ -100,6 +101,7 @@ def traced_publish(func, instance, args, kwargs): return func(*args, **kwargs) with pin.tracer.trace(kombux.PUBLISH_NAME, service=pin.service, span_type=SpanTypes.WORKER) as s: + s.set_tag(SPAN_MEASURED_KEY) exchange_name = get_exchange_from_args(args) s.resource = exchange_name s.set_tag(kombux.EXCHANGE, exchange_name) diff --git a/ddtrace/contrib/mako/patch.py b/ddtrace/contrib/mako/patch.py index 5f6da9c2c45..1a7dcc1fa70 100644 --- a/ddtrace/contrib/mako/patch.py +++ b/ddtrace/contrib/mako/patch.py @@ -1,6 +1,7 @@ import mako from mako.template import Template +from ...constants import SPAN_MEASURED_KEY from ...ext import SpanTypes from ...pin import Pin from ...utils.importlib import func_name @@ -39,6 +40,7 @@ def _wrap_render(wrapped, instance, args, kwargs): template_name = instance.filename or DEFAULT_TEMPLATE_NAME with pin.tracer.trace(func_name(wrapped), pin.service, span_type=SpanTypes.TEMPLATE) as span: + span.set_tag(SPAN_MEASURED_KEY) try: template = wrapped(*args, **kwargs) return template diff --git a/ddtrace/contrib/molten/patch.py b/ddtrace/contrib/molten/patch.py index 61e4de627f4..be145c7bf82 100644 --- a/ddtrace/contrib/molten/patch.py +++ b/ddtrace/contrib/molten/patch.py @@ -5,7 +5,7 @@ from ... import Pin, config from ...compat import urlencode -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...utils.formats import asbool, get_env @@ -83,6 +83,7 @@ def patch_app_call(wrapped, instance, args, kwargs): pin.tracer.context_provider.activate(context) with pin.tracer.trace('molten.request', service=pin.service, resource=resource, span_type=SpanTypes.WEB) as span: + span.set_tag(SPAN_MEASURED_KEY) # set analytics sample rate with global config enabled span.set_tag( ANALYTICS_SAMPLE_RATE_KEY, diff --git a/ddtrace/contrib/psycopg/connection.py b/ddtrace/contrib/psycopg/connection.py index d3e4eb6e955..7accc2c0bc5 100644 --- a/ddtrace/contrib/psycopg/connection.py +++ b/ddtrace/contrib/psycopg/connection.py @@ -5,6 +5,7 @@ # stdlib import functools +from ...constants import SPAN_MEASURED_KEY from ...ext import SpanTypes, db, net, sql from ...utils.deprecation import deprecated @@ -43,6 +44,7 @@ def execute(self, query, vars=None): # noqa: A002 return cursor.execute(self, query, vars) with self._datadog_tracer.trace('postgres.query', service=self._datadog_service, span_type=SpanTypes.SQL) as s: + s.set_tag(SPAN_MEASURED_KEY) if not s.sampled: return super(TracedCursor, self).execute(query, vars) diff --git a/ddtrace/contrib/pylibmc/client.py b/ddtrace/contrib/pylibmc/client.py index 415a0ef9f49..5713f4ba9f4 100644 --- a/ddtrace/contrib/pylibmc/client.py +++ b/ddtrace/contrib/pylibmc/client.py @@ -7,7 +7,7 @@ # project import ddtrace -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, memcached, net from ...internal.logger import get_logger from ...settings import config @@ -135,7 +135,9 @@ def _span(self, cmd_name): 'memcached.cmd', service=pin.service, resource=cmd_name, - span_type=SpanTypes.CACHE) + span_type=SpanTypes.CACHE, + ) + span.set_tag(SPAN_MEASURED_KEY) try: self._tag_span(span) diff --git a/ddtrace/contrib/pylons/middleware.py b/ddtrace/contrib/pylons/middleware.py index 2d2d5df359b..d40391288c3 100644 --- a/ddtrace/contrib/pylons/middleware.py +++ b/ddtrace/contrib/pylons/middleware.py @@ -7,7 +7,7 @@ from .constants import CONFIG_MIDDLEWARE from ...compat import reraise -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator @@ -42,6 +42,7 @@ def __call__(self, environ, start_response): self._tracer.context_provider.activate(context) with self._tracer.trace('pylons.request', service=self._service, span_type=SpanTypes.WEB) as span: + span.set_tag(SPAN_MEASURED_KEY) # Set the service in tracer.trace() as priority sampling requires it to be # set as early as possible when different services share one single agent. diff --git a/ddtrace/contrib/pymemcache/client.py b/ddtrace/contrib/pymemcache/client.py index 891425814b0..74e59db0b0e 100644 --- a/ddtrace/contrib/pymemcache/client.py +++ b/ddtrace/contrib/pymemcache/client.py @@ -13,7 +13,7 @@ ) # project -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...compat import reraise from ...ext import SpanTypes, net, memcached as memcachedx from ...internal.logger import get_logger @@ -143,6 +143,7 @@ def _traced_cmd(self, method_name, *args, **kwargs): resource=method_name, span_type=SpanTypes.CACHE, ) as span: + span.set_tag(SPAN_MEASURED_KEY) # set analytics sample rate span.set_tag( ANALYTICS_SAMPLE_RATE_KEY, diff --git a/ddtrace/contrib/pymongo/client.py b/ddtrace/contrib/pymongo/client.py index 3aa5d22320a..425553c4217 100644 --- a/ddtrace/contrib/pymongo/client.py +++ b/ddtrace/contrib/pymongo/client.py @@ -9,7 +9,7 @@ # project import ddtrace from ...compat import iteritems -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, mongo as mongox, net as netx from ...internal.logger import get_logger from ...settings import config @@ -103,6 +103,7 @@ def _datadog_trace_operation(self, operation): return None span = pin.tracer.trace('pymongo.cmd', span_type=SpanTypes.MONGODB, service=pin.service) + span.set_tag(SPAN_MEASURED_KEY) span.set_tag(mongox.DB, cmd.db) span.set_tag(mongox.COLLECTION, cmd.coll) span.set_tags(cmd.tags) @@ -224,6 +225,7 @@ def __trace(self, cmd): span_type=SpanTypes.MONGODB, service=pin.service) + s.set_tag(SPAN_MEASURED_KEY) if cmd.db: s.set_tag(mongox.DB, cmd.db) if cmd: diff --git a/ddtrace/contrib/pyramid/trace.py b/ddtrace/contrib/pyramid/trace.py index 029352759b2..3c6bc61ed15 100644 --- a/ddtrace/contrib/pyramid/trace.py +++ b/ddtrace/contrib/pyramid/trace.py @@ -5,7 +5,7 @@ # project import ddtrace -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator @@ -71,6 +71,7 @@ def trace_tween(request): if context.trace_id: tracer.context_provider.activate(context) with tracer.trace('pyramid.request', service=service, resource='404', span_type=SpanTypes.WEB) as span: + span.set_tag(SPAN_MEASURED_KEY) # Configure trace search sample rate # DEV: pyramid is special case maintains separate configuration from config api analytics_enabled = settings.get(SETTINGS_ANALYTICS_ENABLED) diff --git a/ddtrace/contrib/redis/patch.py b/ddtrace/contrib/redis/patch.py index e1dddd2c9ad..2b43c4272e3 100644 --- a/ddtrace/contrib/redis/patch.py +++ b/ddtrace/contrib/redis/patch.py @@ -4,7 +4,7 @@ # project from ddtrace import config -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...pin import Pin from ...ext import SpanTypes, redis as redisx from ...utils.wrappers import unwrap @@ -63,6 +63,7 @@ def traced_execute_command(func, instance, args, kwargs): return func(*args, **kwargs) with pin.tracer.trace(redisx.CMD, service=pin.service, span_type=SpanTypes.REDIS) as s: + s.set_tag(SPAN_MEASURED_KEY) query = format_command_args(args) s.resource = query s.set_tag(redisx.RAWCMD, query) @@ -97,6 +98,7 @@ def traced_execute_pipeline(func, instance, args, kwargs): resource = '\n'.join(cmds) tracer = pin.tracer with tracer.trace(redisx.CMD, resource=resource, service=pin.service, span_type=SpanTypes.REDIS) as s: + s.set_tag(SPAN_MEASURED_KEY) s.set_tag(redisx.RAWCMD, resource) s.set_tags(_get_tags(instance)) s.set_metric(redisx.PIPELINE_LEN, len(instance.command_stack)) diff --git a/ddtrace/contrib/rediscluster/patch.py b/ddtrace/contrib/rediscluster/patch.py index b224174dcd1..52437de96a0 100644 --- a/ddtrace/contrib/rediscluster/patch.py +++ b/ddtrace/contrib/rediscluster/patch.py @@ -4,7 +4,7 @@ # project from ddtrace import config -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...pin import Pin from ...ext import SpanTypes, redis as redisx from ...utils.wrappers import unwrap @@ -47,6 +47,7 @@ def traced_execute_pipeline(func, instance, args, kwargs): resource = '\n'.join(cmds) tracer = pin.tracer with tracer.trace(redisx.CMD, resource=resource, service=pin.service, span_type=SpanTypes.REDIS) as s: + s.set_tag(SPAN_MEASURED_KEY) s.set_tag(redisx.RAWCMD, resource) s.set_metric(redisx.PIPELINE_LEN, len(instance.command_stack)) diff --git a/ddtrace/contrib/requests/connection.py b/ddtrace/contrib/requests/connection.py index 503d4a56c27..6c0413f58ac 100644 --- a/ddtrace/contrib/requests/connection.py +++ b/ddtrace/contrib/requests/connection.py @@ -3,7 +3,7 @@ from ddtrace.http import store_request_headers, store_response_headers from ...compat import parse -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...internal.logger import get_logger from ...propagation.http import HTTPPropagator @@ -68,6 +68,7 @@ def _wrap_send(func, instance, args, kwargs): ) with tracer.trace("requests.request", span_type=SpanTypes.HTTP) as span: + span.set_tag(SPAN_MEASURED_KEY) # update the span service name before doing any action span.service = _extract_service_name(instance, span, hostname=hostname) diff --git a/ddtrace/contrib/sqlalchemy/engine.py b/ddtrace/contrib/sqlalchemy/engine.py index 06308e8110d..2b46a32be9e 100644 --- a/ddtrace/contrib/sqlalchemy/engine.py +++ b/ddtrace/contrib/sqlalchemy/engine.py @@ -17,7 +17,7 @@ # project import ddtrace -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, sql as sqlx, net as netx from ...pin import Pin from ...settings import config @@ -81,6 +81,7 @@ def _before_cur_exec(self, conn, cursor, statement, *args): span_type=SpanTypes.SQL, resource=statement, ) + span.set_tag(SPAN_MEASURED_KEY) if not _set_tags_from_url(span, conn.engine.url): _set_tags_from_cursor(span, self.vendor, cursor) diff --git a/ddtrace/contrib/tornado/handlers.py b/ddtrace/contrib/tornado/handlers.py index 2699c2e15c2..63be74d5c64 100644 --- a/ddtrace/contrib/tornado/handlers.py +++ b/ddtrace/contrib/tornado/handlers.py @@ -2,7 +2,7 @@ from .constants import CONFIG_KEY, REQUEST_CONTEXT_KEY, REQUEST_SPAN_KEY from .stack_context import TracerStackContext -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, http from ...propagation.http import HTTPPropagator from ...settings import config @@ -35,8 +35,9 @@ def execute(func, handler, args, kwargs): request_span = tracer.trace( 'tornado.request', service=service, - span_type=SpanTypes.WEB + span_type=SpanTypes.WEB, ) + request_span.set_tag(SPAN_MEASURED_KEY) # set analytics sample rate # DEV: tornado is special case maintains separate configuration from config api analytics_enabled = settings['analytics_enabled'] diff --git a/ddtrace/contrib/vertica/patch.py b/ddtrace/contrib/vertica/patch.py index f893b94b6fd..5c08b03dec8 100644 --- a/ddtrace/contrib/vertica/patch.py +++ b/ddtrace/contrib/vertica/patch.py @@ -3,7 +3,7 @@ from ddtrace.vendor import wrapt import ddtrace -from ...constants import ANALYTICS_SAMPLE_RATE_KEY +from ...constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ...ext import SpanTypes, db as dbx from ...ext import net from ...internal.logger import get_logger @@ -74,26 +74,31 @@ def cursor_span_end(instance, cursor, _, conf, *args, **kwargs): 'span_type': SpanTypes.SQL, 'span_start': execute_span_start, 'span_end': execute_span_end, + 'measured': True, }, 'copy': { 'operation_name': 'vertica.copy', 'span_type': SpanTypes.SQL, 'span_start': copy_span_start, + 'measured': False, }, 'fetchone': { 'operation_name': 'vertica.fetchone', 'span_type': SpanTypes.SQL, 'span_end': fetch_span_end, + 'measured': False, }, 'fetchall': { 'operation_name': 'vertica.fetchall', 'span_type': SpanTypes.SQL, 'span_end': fetch_span_end, + 'measured': False, }, 'nextset': { 'operation_name': 'vertica.nextset', 'span_type': SpanTypes.SQL, 'span_end': fetch_span_end, + 'measured': False, }, }, }, @@ -199,6 +204,8 @@ def wrapper(wrapped, instance, args, kwargs): operation_name = conf['operation_name'] tracer = pin.tracer with tracer.trace(operation_name, service=pin.service, span_type=conf.get('span_type')) as span: + if conf.get('measured', False): + span.set_tag(SPAN_MEASURED_KEY) span.set_tags(pin.tags) if 'span_start' in conf: diff --git a/ddtrace/span.py b/ddtrace/span.py index 9c9c2e1fb1e..2c99004d261 100644 --- a/ddtrace/span.py +++ b/ddtrace/span.py @@ -4,7 +4,7 @@ import traceback from .compat import StringIO, stringify, iteritems, numeric_types, time_ns, is_integer -from .constants import NUMERIC_TAGS, MANUAL_DROP_KEY, MANUAL_KEEP_KEY +from .constants import NUMERIC_TAGS, MANUAL_DROP_KEY, MANUAL_KEEP_KEY, SPAN_MEASURED_KEY from .ext import SpanTypes, errors, priority, net, http from .internal.logger import get_logger @@ -204,6 +204,13 @@ def set_tag(self, key, value=None): elif key == MANUAL_DROP_KEY: self.context.sampling_priority = priority.USER_REJECT return + elif key == SPAN_MEASURED_KEY: + # Set `_dd.measured` tag as a metric + # DEV: `set_metric` will ensure it is an integer 0 or 1 + if value is None: + value = 1 + self.set_metric(key, value) + return try: self.meta[key] = stringify(value) @@ -239,6 +246,14 @@ def set_metric(self, key, value): # This method sets a numeric tag value for the given key. It acts # like `set_meta()` and it simply add a tag without further processing. + # Enforce a specific connstant for `_dd.measured` + if key == SPAN_MEASURED_KEY: + try: + value = int(bool(value)) + except (ValueError, TypeError): + log.warning('failed to convert %r tag to an integer from %r', key, value) + return + # FIXME[matt] we could push this check to serialization time as well. # only permit types that are commonly serializable (don't use # isinstance so that we convert unserializable types like numpy diff --git a/tests/contrib/aiobotocore/py35/test.py b/tests/contrib/aiobotocore/py35/test.py index fb12611b083..8bcc80ab4c4 100644 --- a/tests/contrib/aiobotocore/py35/test.py +++ b/tests/contrib/aiobotocore/py35/test.py @@ -5,7 +5,7 @@ from ..utils import aiobotocore_client from ...asyncio.utils import AsyncioTestCase, mark_asyncio from ....test_tracer import get_dummy_tracer -from ....utils import assert_span_http_status_code +from ....utils import assert_span_http_status_code, assert_is_measured class AIOBotocoreTest(AsyncioTestCase): @@ -45,12 +45,14 @@ async def test_response_context_manager(self): assert len(traces[1]) == 1 span = traces[0][0] + assert_is_measured(span) assert span.get_tag('aws.operation') == 'GetObject' assert_span_http_status_code(span, 200) assert span.service == 'aws.s3' assert span.resource == 's3.getobject' read_span = traces[1][0] + assert_is_measured(read_span) assert read_span.get_tag('aws.operation') == 'GetObject' assert_span_http_status_code(read_span, 200) assert read_span.service == 'aws.s3' @@ -64,6 +66,7 @@ async def test_response_context_manager(self): assert len(traces[0]) == 1 span = traces[0][0] + assert_is_measured(span) assert span.get_tag('aws.operation') == 'GetObject' assert_span_http_status_code(span, 200) assert span.service == 'aws.s3' diff --git a/tests/contrib/aiobotocore/test.py b/tests/contrib/aiobotocore/test.py index 700d593c8ba..f5e5d367f2a 100644 --- a/tests/contrib/aiobotocore/test.py +++ b/tests/contrib/aiobotocore/test.py @@ -8,7 +8,7 @@ from .utils import aiobotocore_client from ..asyncio.utils import AsyncioTestCase, mark_asyncio from ...test_tracer import get_dummy_tracer -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class AIOBotocoreTest(AsyncioTestCase): @@ -33,6 +33,7 @@ def test_traced_client(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + assert_is_measured(span) self.assertEqual(span.get_tag('aws.agent'), 'aiobotocore') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') @@ -69,6 +70,7 @@ def test_s3_client(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + assert_is_measured(span) self.assertEqual(span.get_tag('aws.operation'), 'ListBuckets') assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.s3') @@ -87,9 +89,13 @@ def test_s3_put(self): assert spans self.assertEqual(len(spans), 2) self.assertEqual(spans[0].get_tag('aws.operation'), 'CreateBucket') + + assert_is_measured(spans[0]) assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].service, 'aws.s3') self.assertEqual(spans[0].resource, 's3.createbucket') + + assert_is_measured(spans[1]) self.assertEqual(spans[1].get_tag('aws.operation'), 'PutObject') self.assertEqual(spans[1].resource, 's3.putobject') self.assertEqual(spans[1].get_tag('params.Key'), stringify(params['Key'])) @@ -108,6 +114,7 @@ def test_s3_client_error(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + assert_is_measured(span) self.assertEqual(span.resource, 's3.listobjects') self.assertEqual(span.error, 1) self.assertTrue('NoSuchBucket' in span.get_tag('error.msg')) @@ -135,6 +142,8 @@ def test_s3_client_read(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + + assert_is_measured(span) self.assertEqual(span.get_tag('aws.operation'), 'GetObject') assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'aws.s3') @@ -161,6 +170,8 @@ def test_sqs_client(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + + assert_is_measured(span) self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListQueues') assert_span_http_status_code(span, 200) @@ -177,6 +188,8 @@ def test_kinesis_client(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + + assert_is_measured(span) self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListStreams') assert_span_http_status_code(span, 200) @@ -194,6 +207,8 @@ def test_lambda_client(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + + assert_is_measured(span) self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListFunctions') assert_span_http_status_code(span, 200) @@ -210,6 +225,8 @@ def test_kms_client(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] + + assert_is_measured(span) self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'ListKeys') assert_span_http_status_code(span, 200) @@ -261,6 +278,7 @@ def test_opentraced_client(self): self.assertEqual(ot_span.parent_id, None) self.assertEqual(dd_span.parent_id, ot_span.span_id) + assert_is_measured(dd_span) self.assertEqual(dd_span.get_tag('aws.agent'), 'aiobotocore') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') @@ -304,6 +322,7 @@ def test_opentraced_s3_client(self): self.assertEqual(dd_span2.parent_id, ot_inner_span.span_id) self.assertEqual(ot_inner_span2.parent_id, ot_outer_span.span_id) + assert_is_measured(dd_span) self.assertEqual(dd_span.get_tag('aws.operation'), 'ListBuckets') assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.service, 'aws.s3') diff --git a/tests/contrib/aiohttp/test_request.py b/tests/contrib/aiohttp/test_request.py index b13e848f408..93afcdb90aa 100644 --- a/tests/contrib/aiohttp/test_request.py +++ b/tests/contrib/aiohttp/test_request.py @@ -10,6 +10,7 @@ from ddtrace.contrib.aiohttp.middlewares import trace_app from .utils import TraceTestCase +from ...utils import assert_is_measured class TestRequestTracing(TraceTestCase): @@ -40,6 +41,8 @@ def test_full_request(self): assert 1 == len(traces) assert 2 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) + template_span = traces[0][1] # request assert 'aiohttp-web' == request_span.service diff --git a/tests/contrib/aiohttp/test_request_safety.py b/tests/contrib/aiohttp/test_request_safety.py index 6f4c93947f3..a58e7421a39 100644 --- a/tests/contrib/aiohttp/test_request_safety.py +++ b/tests/contrib/aiohttp/test_request_safety.py @@ -11,6 +11,7 @@ from ddtrace.contrib.aiohttp.middlewares import trace_app from .utils import TraceTestCase +from ...utils import assert_is_measured class TestAiohttpSafety(TraceTestCase): @@ -44,6 +45,7 @@ def test_full_request(self): request_span = traces[0][0] template_span = traces[0][1] # request + assert_is_measured(request_span) assert 'aiohttp-web' == request_span.service assert 'aiohttp.request' == request_span.name assert 'GET /template/' == request_span.resource diff --git a/tests/contrib/aiopg/test.py b/tests/contrib/aiopg/test.py index faf7f17a12f..abc93a1888e 100644 --- a/tests/contrib/aiopg/test.py +++ b/tests/contrib/aiopg/test.py @@ -16,6 +16,7 @@ from tests.contrib.config import POSTGRES_CONFIG from tests.test_tracer import get_dummy_tracer from tests.contrib.asyncio.utils import AsyncioTestCase, mark_asyncio +from ...utils import assert_is_measured TEST_PORT = POSTGRES_CONFIG['port'] @@ -68,6 +69,7 @@ def assert_conn_is_traced(self, tracer, db, service): assert spans assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.name == 'postgres.query' assert span.resource == q assert span.service == service diff --git a/tests/contrib/algoliasearch/test.py b/tests/contrib/algoliasearch/test.py index 45ecb0db3bc..173616d5509 100644 --- a/tests/contrib/algoliasearch/test.py +++ b/tests/contrib/algoliasearch/test.py @@ -2,6 +2,7 @@ from ddtrace.contrib.algoliasearch.patch import (patch, unpatch, algoliasearch_version) from ddtrace.pin import Pin from tests.base import BaseTracerTestCase +from ...utils import assert_is_measured class AlgoliasearchTest(BaseTracerTestCase): @@ -71,6 +72,7 @@ def test_algoliasearch(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'algoliasearch' assert span.name == 'algoliasearch.search' assert span.span_type is None diff --git a/tests/contrib/boto/test.py b/tests/contrib/boto/test.py index d785a5021a9..d32b5223445 100644 --- a/tests/contrib/boto/test.py +++ b/tests/contrib/boto/test.py @@ -18,7 +18,7 @@ from unittest import skipUnless from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class BotoTest(BaseTracerTestCase): @@ -53,6 +53,7 @@ def test_ec2_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.get_tag('aws.operation'), 'RunInstances') assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'POST') @@ -108,6 +109,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.operation'), 'get_all_buckets') @@ -118,6 +120,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'PUT') self.assertEqual(span.get_tag('path'), '/') @@ -129,6 +132,7 @@ def test_s3_client(self): assert spans self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'HEAD') self.assertEqual(span.get_tag('aws.operation'), 'head_bucket') @@ -162,13 +166,16 @@ def test_s3_put(self): # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket') + assert_is_measured(spans[0]) assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].service, 'test-boto-tracing.s3') self.assertEqual(spans[0].resource, 's3.put') # get bucket + assert_is_measured(spans[1]) self.assertEqual(spans[1].get_tag('aws.operation'), 'head_bucket') self.assertEqual(spans[1].resource, 's3.head') # put object + assert_is_measured(spans[2]) self.assertEqual(spans[2].get_tag('aws.operation'), '_send_file_internal') self.assertEqual(spans[2].resource, 's3.put') @@ -216,6 +223,7 @@ def test_lambda_client(self): assert spans self.assertEqual(len(spans), 2) span = spans[0] + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.region'), 'us-east-2') @@ -235,6 +243,7 @@ def test_sts_client(self): spans = writer.pop() assert spans span = spans[0] + assert_is_measured(span) self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'GetFederationToken') self.assertEqual(span.service, 'test-boto-tracing.sts') diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index e99aa8ce8b1..2afd0690aa4 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -11,7 +11,7 @@ # testing from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class BotocoreTest(BaseTracerTestCase): @@ -43,6 +43,7 @@ def test_traced_client(self): assert spans span = spans[0] self.assertEqual(len(spans), 1) + assert_is_measured(span) self.assertEqual(span.get_tag('aws.agent'), 'botocore') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') @@ -81,6 +82,7 @@ def test_s3_client(self): assert spans span = spans[0] self.assertEqual(len(spans), 2) + assert_is_measured(span) self.assertEqual(span.get_tag('aws.operation'), 'ListBuckets') assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.s3') @@ -110,6 +112,7 @@ def test_s3_put(self): span = spans[0] self.assertEqual(len(spans), 2) self.assertEqual(span.get_tag('aws.operation'), 'CreateBucket') + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.s3') self.assertEqual(span.resource, 's3.createbucket') @@ -133,6 +136,7 @@ def test_sqs_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListQueues') + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.sqs') self.assertEqual(span.resource, 'sqs.listqueues') @@ -150,6 +154,7 @@ def test_kinesis_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListStreams') + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.kinesis') self.assertEqual(span.resource, 'kinesis.liststreams') @@ -192,6 +197,7 @@ def test_lambda_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListFunctions') + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.lambda') self.assertEqual(span.resource, 'lambda.listfunctions') @@ -209,6 +215,7 @@ def test_kms_client(self): self.assertEqual(len(spans), 1) self.assertEqual(span.get_tag('aws.region'), 'us-east-1') self.assertEqual(span.get_tag('aws.operation'), 'ListKeys') + assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.service, 'test-botocore-tracing.kms') self.assertEqual(span.resource, 'kms.listkeys') diff --git a/tests/contrib/bottle/test.py b/tests/contrib/bottle/test.py index 28af1a808f5..8abd7b5ecb8 100644 --- a/tests/contrib/bottle/test.py +++ b/tests/contrib/bottle/test.py @@ -4,7 +4,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured from ddtrace import compat from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY @@ -55,6 +55,8 @@ def hi(name): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.span_type == 'web' @@ -118,6 +120,8 @@ def handled400(): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /400_return' @@ -141,6 +145,8 @@ def handled400(): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /400_raise' @@ -164,6 +170,8 @@ def hi(): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' @@ -231,6 +239,8 @@ def hi(): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.name == 'bottle.request' assert s.service == 'bottle-app' assert s.resource == 'GET /hi' @@ -402,6 +412,7 @@ def hi(name): assert ot_span.resource == 'ot_span' + assert_is_measured(dd_span) assert dd_span.name == 'bottle.request' assert dd_span.service == 'bottle-app' assert dd_span.resource == 'GET /hi/' diff --git a/tests/contrib/cassandra/test.py b/tests/contrib/cassandra/test.py index e0288366583..520e842910b 100644 --- a/tests/contrib/cassandra/test.py +++ b/tests/contrib/cassandra/test.py @@ -19,6 +19,7 @@ from tests.contrib.config import CASSANDRA_CONFIG from tests.opentracer.utils import init_tracer from tests.test_tracer import get_dummy_tracer +from ...utils import assert_is_measured # Oftentimes our tests fails because Cassandra connection timeouts during keyspace drop. Slowness in keyspace drop # is known and is due to 'auto_snapshot' configuration. In our test env we should disable it, but the official cassandra @@ -119,6 +120,8 @@ def _test_query_base(self, execute_fn): assert len(spans) == 1 query = spans[0] + + assert_is_measured(query) assert query.service == self.TEST_SERVICE assert query.resource == self.TEST_QUERY assert query.span_type == 'cassandra' diff --git a/tests/contrib/celery/test_integration.py b/tests/contrib/celery/test_integration.py index a93ec0f95f2..5f89ba6c80c 100644 --- a/tests/contrib/celery/test_integration.py +++ b/tests/contrib/celery/test_integration.py @@ -6,6 +6,7 @@ from .base import CeleryBaseTestCase from tests.opentracer.utils import init_tracer +from ...utils import assert_is_measured class MyException(Exception): @@ -101,6 +102,8 @@ def fn_task(): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.error == 0 assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.fn_task' @@ -124,6 +127,8 @@ def fn_task(self): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.error == 0 assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.fn_task' @@ -145,6 +150,8 @@ def fn_task_parameters(user, force_logout=False): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.error == 0 assert span.name == 'celery.apply' assert span.resource == 'tests.contrib.celery.test_integration.fn_task_parameters' @@ -166,6 +173,8 @@ def fn_task_parameters(user, force_logout=False): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.error == 0 assert span.name == 'celery.apply' assert span.resource == 'tests.contrib.celery.test_integration.fn_task_parameters' @@ -188,6 +197,8 @@ def fn_exception(): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.fn_exception' assert span.service == 'celery-worker' @@ -213,6 +224,8 @@ def fn_exception(): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.fn_exception' assert span.service == 'celery-worker' @@ -235,6 +248,8 @@ def fn_exception(): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.fn_exception' assert span.service == 'celery-worker' @@ -268,6 +283,8 @@ def run(self): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.error == 0 assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.BaseTask' @@ -296,6 +313,8 @@ def run(self): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.BaseTask' assert span.service == 'celery-worker' @@ -329,6 +348,8 @@ def run(self): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.name == 'celery.run' assert span.resource == 'tests.contrib.celery.test_integration.BaseTask' assert span.service == 'celery-worker' @@ -350,6 +371,8 @@ def add(x, y): assert 1 == len(traces) assert 1 == len(traces[0]) span = traces[0][0] + + assert_is_measured(span) assert span.error == 0 assert span.name == 'celery.run' assert span.service == 'celery-worker' @@ -419,6 +442,7 @@ def fn_task_parameters(user, force_logout=False): assert ot_span.name == 'celery_op' assert ot_span.service == 'celery_svc' + assert_is_measured(dd_span) assert dd_span.error == 0 assert dd_span.name == 'celery.apply' assert dd_span.resource == 'tests.contrib.celery.test_integration.fn_task_parameters' diff --git a/tests/contrib/consul/test.py b/tests/contrib/consul/test.py index b68f6b85ab4..574bc2757a9 100644 --- a/tests/contrib/consul/test.py +++ b/tests/contrib/consul/test.py @@ -7,6 +7,7 @@ from ..config import CONSUL_CONFIG from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class TestConsulPatch(BaseTracerTestCase): @@ -37,6 +38,8 @@ def test_put(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == consulx.CMD assert span.resource == 'PUT' @@ -56,6 +59,8 @@ def test_get(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == consulx.CMD assert span.resource == 'GET' @@ -75,6 +80,8 @@ def test_delete(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == consulx.CMD assert span.resource == 'DELETE' @@ -95,6 +102,8 @@ def test_kwargs(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == consulx.CMD assert span.resource == 'PUT' diff --git a/tests/contrib/dbapi/test_unit.py b/tests/contrib/dbapi/test_unit.py index 18b78f6051c..2f87861a802 100644 --- a/tests/contrib/dbapi/test_unit.py +++ b/tests/contrib/dbapi/test_unit.py @@ -5,6 +5,7 @@ from ddtrace.contrib.dbapi import FetchTracedCursor, TracedCursor, TracedConnection from ddtrace.span import Span from ...base import BaseTracerTestCase +from ...utils import assert_is_measured, assert_is_not_measured class TestTracedCursor(BaseTracerTestCase): @@ -71,14 +72,17 @@ def test_correct_span_names(self): traced_cursor.execute('arg_1', kwarg1='kwarg1') self.assert_structure(dict(name='sql.query')) + assert_is_measured(self.get_root_span()) self.reset() traced_cursor.executemany('arg_1', kwarg1='kwarg1') self.assert_structure(dict(name='sql.query')) + assert_is_measured(self.get_root_span()) self.reset() traced_cursor.callproc('arg_1', 'arg2') self.assert_structure(dict(name='sql.query')) + assert_is_measured(self.get_root_span()) self.reset() traced_cursor.fetchone('arg_1', kwarg1='kwarg1') @@ -99,14 +103,17 @@ def test_correct_span_names_can_be_overridden_by_pin(self): traced_cursor.execute('arg_1', kwarg1='kwarg1') self.assert_structure(dict(name='changed.query')) + assert_is_measured(self.get_root_span()) self.reset() traced_cursor.executemany('arg_1', kwarg1='kwarg1') self.assert_structure(dict(name='changed.query')) + assert_is_measured(self.get_root_span()) self.reset() traced_cursor.callproc('arg_1', 'arg2') self.assert_structure(dict(name='changed.query')) + assert_is_measured(self.get_root_span()) self.reset() traced_cursor.fetchone('arg_1', kwarg1='kwarg1') @@ -163,6 +170,8 @@ def method(): traced_cursor._trace_method(method, 'my_name', 'my_resource', {'extra1': 'value_extra1'}) span = tracer.writer.pop()[0] # type: Span + # Only measure if the name passed matches the default name (e.g. `sql.query` and not `sql.query.fetchall`) + assert_is_not_measured(span) assert span.meta['pin1'] == 'value_pin1', 'Pin tags are preserved' assert span.meta['extra1'] == 'value_extra1', 'Extra tags are merged into pin tags' assert span.name == 'my_name', 'Span name is respected' diff --git a/tests/contrib/dogpile_cache/test_tracing.py b/tests/contrib/dogpile_cache/test_tracing.py index 1548bf24e11..dc20afa8914 100644 --- a/tests/contrib/dogpile_cache/test_tracing.py +++ b/tests/contrib/dogpile_cache/test_tracing.py @@ -5,6 +5,7 @@ from ddtrace.contrib.dogpile_cache.patch import patch, unpatch from tests.test_tracer import get_dummy_tracer +from ...utils import assert_is_measured @pytest.fixture @@ -75,6 +76,8 @@ def test_traces_get_or_create(tracer, single_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.name == "dogpile.cache" assert span.resource == "get_or_create" assert span.meta["key"] == "tests.contrib.dogpile_cache.test_tracing:fn|1" @@ -90,6 +93,8 @@ def test_traces_get_or_create(tracer, single_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.name == "dogpile.cache" assert span.resource == "get_or_create" assert span.meta["key"] == "tests.contrib.dogpile_cache.test_tracing:fn|1" @@ -106,6 +111,8 @@ def test_traces_get_or_create_multi(tracer, multi_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.meta["keys"] == ( "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + "'tests.contrib.dogpile_cache.test_tracing:fn|3']" ) @@ -121,6 +128,7 @@ def test_traces_get_or_create_multi(tracer, multi_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.meta["keys"] == ( "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + "'tests.contrib.dogpile_cache.test_tracing:fn|4']" ) @@ -136,6 +144,7 @@ def test_traces_get_or_create_multi(tracer, multi_cache): spans = traces[0] assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.meta["keys"] == ( "['tests.contrib.dogpile_cache.test_tracing:fn|2', " + "'tests.contrib.dogpile_cache.test_tracing:fn|4']" ) diff --git a/tests/contrib/elasticsearch/test.py b/tests/contrib/elasticsearch/test.py index 3d705b3e468..4c609736d86 100644 --- a/tests/contrib/elasticsearch/test.py +++ b/tests/contrib/elasticsearch/test.py @@ -14,7 +14,7 @@ from ..config import ELASTICSEARCH_CONFIG from ...test_tracer import get_dummy_tracer from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class ElasticsearchTest(unittest.TestCase): @@ -60,6 +60,8 @@ def test_elasticsearch(self): assert spans assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'elasticsearch.query' assert span.span_type == 'elasticsearch' @@ -78,6 +80,7 @@ def test_elasticsearch(self): assert spans assert len(spans) == 3 span = spans[0] + assert_is_measured(span) assert span.error == 0 assert span.get_tag('elasticsearch.method') == 'PUT' assert span.get_tag('elasticsearch.url') == '/%s/%s/%s' % (self.ES_INDEX, self.ES_TYPE, 10) @@ -90,6 +93,7 @@ def test_elasticsearch(self): assert spans, spans assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == 'POST /%s/_refresh' % self.ES_INDEX assert span.get_tag('elasticsearch.method') == 'POST' assert span.get_tag('elasticsearch.url') == '/%s/_refresh' % self.ES_INDEX @@ -107,6 +111,7 @@ def test_elasticsearch(self): assert spans assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == 'GET /%s/%s/_search' % (self.ES_INDEX, self.ES_TYPE) assert span.get_tag('elasticsearch.method') == 'GET' assert span.get_tag('elasticsearch.url') == '/%s/%s/_search' % (self.ES_INDEX, self.ES_TYPE) @@ -132,6 +137,7 @@ def test_elasticsearch(self): spans = writer.pop() assert spans span = spans[0] + assert_is_measured(span) assert_span_http_status_code(span, 404) # Raise error 400, the index 10 is created twice @@ -143,6 +149,7 @@ def test_elasticsearch(self): spans = writer.pop() assert spans span = spans[-1] + assert_is_measured(span) assert_span_http_status_code(span, 400) # Drop the index, checking it won't raise exception on success or failure @@ -180,6 +187,7 @@ def test_elasticsearch_ot(self): assert ot_span.service == 'my_svc' assert ot_span.resource == 'ot_span' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'elasticsearch.query' assert dd_span.span_type == 'elasticsearch' @@ -232,6 +240,7 @@ def test_elasticsearch(self): assert spans, spans assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'elasticsearch.query' assert span.span_type == 'elasticsearch' @@ -250,6 +259,7 @@ def test_elasticsearch(self): assert spans, spans assert len(spans) == 3 span = spans[0] + assert_is_measured(span) assert span.error == 0 assert span.get_tag('elasticsearch.method') == 'PUT' assert span.get_tag('elasticsearch.url') == '/%s/%s/%s' % (self.ES_INDEX, self.ES_TYPE, 10) @@ -263,6 +273,7 @@ def test_elasticsearch(self): assert spans, spans assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == 'POST /%s/_refresh' % self.ES_INDEX assert span.get_tag('elasticsearch.method') == 'POST' assert span.get_tag('elasticsearch.url') == '/%s/_refresh' % self.ES_INDEX @@ -286,6 +297,7 @@ def test_elasticsearch(self): assert spans, spans assert len(spans) == 4 span = spans[-1] + assert_is_measured(span) assert span.resource == 'GET /%s/%s/_search' % (self.ES_INDEX, self.ES_TYPE) assert span.get_tag('elasticsearch.method') == 'GET' assert span.get_tag('elasticsearch.url') == '/%s/%s/_search' % (self.ES_INDEX, self.ES_TYPE) diff --git a/tests/contrib/falcon/test_suite.py b/tests/contrib/falcon/test_suite.py index 6797ce62d32..9837aa398dc 100644 --- a/tests/contrib/falcon/test_suite.py +++ b/tests/contrib/falcon/test_suite.py @@ -3,7 +3,7 @@ from ddtrace.ext import errors as errx, http as httpx from tests.opentracer.utils import init_tracer -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class FalconTestCase(object): @@ -19,6 +19,8 @@ def test_404(self): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET 404' @@ -39,6 +41,8 @@ def test_exception(self): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.ResourceException' @@ -55,6 +59,8 @@ def test_200(self, query_string=''): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.Resource200' @@ -152,6 +158,8 @@ def test_201(self): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'POST tests.contrib.falcon.app.resources.Resource201' @@ -168,6 +176,8 @@ def test_500(self): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.Resource500' @@ -183,6 +193,8 @@ def test_404_exception(self): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert span.resource == 'GET tests.contrib.falcon.app.resources.ResourceNotFound' @@ -199,6 +211,8 @@ def test_404_exception_no_stacktracer(self): assert len(traces) == 1 assert len(traces[0]) == 1 span = traces[0][0] + + assert_is_measured(span) assert span.name == 'falcon.request' assert span.service == self._service assert_span_http_status_code(span, 404) @@ -227,6 +241,7 @@ def test_200_ot(self): assert ot_span.service == 'my_svc' assert ot_span.resource == 'ot_span' + assert_is_measured(dd_span) assert dd_span.name == 'falcon.request' assert dd_span.service == self._service assert dd_span.resource == 'GET tests.contrib.falcon.app.resources.Resource200' diff --git a/tests/contrib/flask/test_request.py b/tests/contrib/flask/test_request.py index 86c0f5e44bf..46432e3a007 100644 --- a/tests/contrib/flask/test_request.py +++ b/tests/contrib/flask/test_request.py @@ -7,7 +7,7 @@ from flask import abort from . import BaseFlaskTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured base_exception_name = 'builtins.Exception' @@ -53,6 +53,7 @@ def index(): # Root request span req_span = spans[0] + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /') @@ -285,6 +286,8 @@ def index(): # Root request span req_span = spans[0] + + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') # Note: contains no query string @@ -403,6 +406,7 @@ def test_request_404(self): # Root request span req_span = spans[0] + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET 404') @@ -464,6 +468,8 @@ def not_found(): # Root request span req_span = spans[0] + + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /not-found') @@ -536,6 +542,7 @@ def fivehundred(): # Root request span req_span = spans[0] + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /500') @@ -619,6 +626,7 @@ def fivehundredone(): # Root request span req_span = spans[0] + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /501') @@ -726,6 +734,7 @@ def fivehundred(): # Root request span req_span = spans[0] + assert_is_measured(req_span) self.assertEqual(req_span.service, 'flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /500') diff --git a/tests/contrib/flask_autopatch/test_flask_autopatch.py b/tests/contrib/flask_autopatch/test_flask_autopatch.py index 1ce50499b93..679c44c5ea8 100644 --- a/tests/contrib/flask_autopatch/test_flask_autopatch.py +++ b/tests/contrib/flask_autopatch/test_flask_autopatch.py @@ -8,7 +8,7 @@ from ddtrace import Pin from ...test_tracer import get_dummy_tracer -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class FlaskAutopatchTestCase(unittest.TestCase): @@ -74,6 +74,7 @@ def index(): # Root request span req_span = spans[0] + assert_is_measured(req_span) self.assertEqual(req_span.service, 'test-flask') self.assertEqual(req_span.name, 'flask.request') self.assertEqual(req_span.resource, 'GET /') @@ -82,11 +83,10 @@ def index(): self.assertIsNone(req_span.parent_id) # Request tags - self.assertEqual( - set(['flask.version', 'http.url', 'http.method', 'http.status_code', - 'flask.endpoint', 'flask.url_rule']), - set(req_span.meta.keys()), - ) + for tag in ['flask.version', 'http.url', 'http.method', 'http.status_code', + 'flask.endpoint', 'flask.url_rule']: + assert tag in req_span.meta + self.assertEqual(req_span.get_tag('flask.endpoint'), 'index') self.assertEqual(req_span.get_tag('flask.url_rule'), '/') self.assertEqual(req_span.get_tag('http.method'), 'GET') diff --git a/tests/contrib/flask_cache/test.py b/tests/contrib/flask_cache/test.py index 1515cd93f8d..bd305a679e7 100644 --- a/tests/contrib/flask_cache/test.py +++ b/tests/contrib/flask_cache/test.py @@ -14,6 +14,7 @@ from ..config import REDIS_CONFIG, MEMCACHED_CONFIG from ...base import BaseTracerTestCase from ...util import assert_dict_issuperset +from ...utils import assert_is_measured class FlaskCacheTest(BaseTracerTestCase): @@ -34,6 +35,7 @@ def test_simple_cache_get(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'get') self.assertEqual(span.name, 'flask_cache.cmd') @@ -52,6 +54,7 @@ def test_simple_cache_set(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'set') self.assertEqual(span.name, 'flask_cache.cmd') @@ -70,6 +73,7 @@ def test_simple_cache_add(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'add') self.assertEqual(span.name, 'flask_cache.cmd') @@ -88,6 +92,7 @@ def test_simple_cache_delete(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'delete') self.assertEqual(span.name, 'flask_cache.cmd') @@ -106,6 +111,7 @@ def test_simple_cache_delete_many(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'delete_many') self.assertEqual(span.name, 'flask_cache.cmd') @@ -124,6 +130,7 @@ def test_simple_cache_clear(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'clear') self.assertEqual(span.name, 'flask_cache.cmd') @@ -141,6 +148,7 @@ def test_simple_cache_get_many(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'get_many') self.assertEqual(span.name, 'flask_cache.cmd') @@ -162,6 +170,7 @@ def test_simple_cache_set_many(self): spans = self.get_spans() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'set_many') self.assertEqual(span.name, 'flask_cache.cmd') @@ -238,6 +247,7 @@ def test_simple_cache_get_ot(self): self.assertEqual(ot_span.resource, 'ot_span') self.assertEqual(ot_span.service, 'my_svc') + assert_is_measured(dd_span) self.assertEqual(dd_span.service, self.SERVICE) self.assertEqual(dd_span.resource, 'get') self.assertEqual(dd_span.name, 'flask_cache.cmd') diff --git a/tests/contrib/grpc/test_grpc.py b/tests/contrib/grpc/test_grpc.py index 56dc9eb514f..9c013dcc94f 100644 --- a/tests/contrib/grpc/test_grpc.py +++ b/tests/contrib/grpc/test_grpc.py @@ -10,6 +10,7 @@ from ddtrace import Pin from ...base import BaseTracerTestCase +from ...utils import assert_is_measured from .hello_pb2 import HelloRequest, HelloReply from .hello_pb2_grpc import add_HelloServicer_to_server, HelloStub, HelloServicer @@ -63,6 +64,7 @@ def _stop_server(self): self._server.stop(0) def _check_client_span(self, span, service, method_name, method_kind): + assert_is_measured(span) assert span.name == 'grpc' assert span.resource == '/helloworld.Hello/{}'.format(method_name) assert span.service == service @@ -78,6 +80,7 @@ def _check_client_span(self, span, service, method_name, method_kind): assert span.get_tag('grpc.port') == '50531' def _check_server_span(self, span, service, method_name, method_kind): + assert_is_measured(span) assert span.name == 'grpc' assert span.resource == '/helloworld.Hello/{}'.format(method_name) assert span.service == service diff --git a/tests/contrib/httplib/test_httplib.py b/tests/contrib/httplib/test_httplib.py index 9f0037c27ee..103fa5d4bf8 100644 --- a/tests/contrib/httplib/test_httplib.py +++ b/tests/contrib/httplib/test_httplib.py @@ -18,7 +18,7 @@ from ...base import BaseTracerTestCase from ...util import override_global_tracer -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured if PY2: from urllib2 import urlopen, build_opener, Request @@ -149,6 +149,7 @@ def test_httplib_request_get_request(self, query_string=''): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -186,6 +187,7 @@ def test_httplib_request_get_request_https(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -210,6 +212,7 @@ def test_httplib_request_post_request(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -233,6 +236,7 @@ def test_httplib_request_get_request_query_string(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -262,6 +266,7 @@ def test_httplib_request_500_request(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -291,6 +296,7 @@ def test_httplib_request_non_200_request(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -375,6 +381,7 @@ def test_urllib_request(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -399,6 +406,7 @@ def test_urllib_request_https(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -424,6 +432,7 @@ def test_urllib_request_object(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -448,6 +457,7 @@ def test_urllib_request_opener(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, self.SPAN_NAME) @@ -479,6 +489,7 @@ def test_httplib_request_get_request_ot(self): self.assertEqual(ot_span.service, 'my_svc') self.assertEqual(ot_span.name, 'ot_span') + assert_is_measured(dd_span) self.assertEqual(dd_span.span_type, 'http') self.assertEqual(dd_span.name, self.SPAN_NAME) self.assertEqual(dd_span.error, 0) @@ -551,6 +562,7 @@ def test_urllib_request(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, 'httplib.request') @@ -575,6 +587,7 @@ def test_urllib_request_https(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) span = spans[0] + assert_is_measured(span) self.assertEqual(span.span_type, 'http') self.assertIsNone(span.service) self.assertEqual(span.name, 'httplib.request') diff --git a/tests/contrib/jinja2/__init__.py b/tests/contrib/jinja2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/contrib/jinja2/test_jinja2.py b/tests/contrib/jinja2/test_jinja2.py index f7d1d0eea6a..29b7f206fe3 100644 --- a/tests/contrib/jinja2/test_jinja2.py +++ b/tests/contrib/jinja2/test_jinja2.py @@ -7,6 +7,7 @@ from ddtrace import Pin, config from ddtrace.contrib.jinja2 import patch, unpatch from tests.test_tracer import get_dummy_tracer +from ...utils import assert_is_measured, assert_is_not_measured TEST_DIR = os.path.dirname(os.path.realpath(__file__)) TMPL_DIR = os.path.join(TEST_DIR, 'templates') @@ -39,7 +40,9 @@ def test_render_inline_template(self): assert span.get_tag('jinja2.template_name') == '' assert spans[0].name == 'jinja2.compile' + assert_is_not_measured(spans[0]) assert spans[1].name == 'jinja2.render' + assert_is_measured(spans[1]) def test_generate_inline_template(self): t = jinja2.environment.Template('Hello {{name}}!') @@ -55,7 +58,9 @@ def test_generate_inline_template(self): assert span.get_tag('jinja2.template_name') == '' assert spans[0].name == 'jinja2.compile' + assert_is_not_measured(spans[0]) assert spans[1].name == 'jinja2.render' + assert_is_measured(spans[1]) def test_file_template(self): loader = jinja2.loaders.FileSystemLoader(TMPL_DIR) @@ -76,10 +81,15 @@ def get_def(s): return s.name, s.get_tag('jinja2.template_name') assert get_def(spans[0]) == ('jinja2.load', 'template.html') + assert_is_not_measured(spans[0]) assert get_def(spans[1]) == ('jinja2.compile', 'template.html') + assert_is_not_measured(spans[1]) assert get_def(spans[2]) == ('jinja2.render', 'template.html') + assert_is_measured(spans[2]) assert get_def(spans[3]) == ('jinja2.load', 'base.html') + assert_is_not_measured(spans[3]) assert get_def(spans[4]) == ('jinja2.compile', 'base.html') + assert_is_not_measured(spans[4]) # additionnal checks for jinja2.load assert spans[0].get_tag('jinja2.template_path') == os.path.join(TMPL_DIR, 'template.html') diff --git a/tests/contrib/kombu/test.py b/tests/contrib/kombu/test.py index 82fd8a93ab5..6249cb42299 100644 --- a/tests/contrib/kombu/test.py +++ b/tests/contrib/kombu/test.py @@ -8,6 +8,7 @@ from ddtrace.ext import kombu as kombux from ..config import RABBITMQ_CONFIG from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class TestKombuPatch(BaseTracerTestCase): @@ -67,6 +68,7 @@ def _assert_spans(self): spans = self.get_spans() self.assertEqual(len(spans), 2) consumer_span = spans[0] + assert_is_measured(consumer_span) self.assertEqual(consumer_span.service, self.TEST_SERVICE) self.assertEqual(consumer_span.name, kombux.PUBLISH_NAME) self.assertEqual(consumer_span.span_type, 'worker') @@ -79,6 +81,7 @@ def _assert_spans(self): self.assertEqual(consumer_span.resource, 'tasks') producer_span = spans[1] + assert_is_measured(producer_span) self.assertEqual(producer_span.service, self.TEST_SERVICE) self.assertEqual(producer_span.name, kombux.RECEIVE_NAME) self.assertEqual(producer_span.span_type, 'worker') diff --git a/tests/contrib/mako/__init__.py b/tests/contrib/mako/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/contrib/mako/test_mako.py b/tests/contrib/mako/test_mako.py index b663fbcb1a4..81c347950f2 100644 --- a/tests/contrib/mako/test_mako.py +++ b/tests/contrib/mako/test_mako.py @@ -10,6 +10,7 @@ from ddtrace.contrib.mako import patch, unpatch from ddtrace.compat import StringIO, to_unicode from tests.test_tracer import get_dummy_tracer +from ...utils import assert_is_measured TEST_DIR = os.path.dirname(os.path.realpath(__file__)) TMPL_DIR = os.path.join(TEST_DIR, 'templates') @@ -32,6 +33,7 @@ def test_render(self): spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) + assert_is_measured(spans[0]) self.assertEqual(spans[0].service, 'mako') self.assertEqual(spans[0].span_type, 'template') self.assertEqual(spans[0].get_tag('mako.template_name'), '') @@ -43,6 +45,7 @@ def test_render(self): self.assertEqual(t.render_unicode(name='mako'), to_unicode('Hello mako!')) spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) + assert_is_measured(spans[0]) self.assertEqual(spans[0].service, 'mako') self.assertEqual(spans[0].span_type, 'template') self.assertEqual(spans[0].get_tag('mako.template_name'), '') @@ -57,6 +60,7 @@ def test_render(self): self.assertEqual(buf.getvalue(), 'Hello mako!') spans = self.tracer.writer.pop() self.assertEqual(len(spans), 1) + assert_is_measured(spans[0]) self.assertEqual(spans[0].service, 'mako') self.assertEqual(spans[0].span_type, 'template') self.assertEqual(spans[0].get_tag('mako.template_name'), '') @@ -73,6 +77,7 @@ def test_file_template(self): template_name = os.path.join(TMPL_DIR, 'template.html') + assert_is_measured(spans[0]) self.assertEqual(spans[0].span_type, 'template') self.assertEqual(spans[0].service, 'mako') self.assertEqual(spans[0].get_tag('mako.template_name'), template_name) diff --git a/tests/contrib/molten/test_molten.py b/tests/contrib/molten/test_molten.py index 8564234bd5e..8b91c108530 100644 --- a/tests/contrib/molten/test_molten.py +++ b/tests/contrib/molten/test_molten.py @@ -9,7 +9,7 @@ from ddtrace.contrib.molten.patch import MOLTEN_VERSION from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured # NOTE: Type annotations required by molten otherwise parameters cannot be coerced @@ -47,6 +47,7 @@ def test_route_success(self): # access data property self.assertEqual(response.data, '"Hello 24 year old named Jim!"') span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, 'molten') self.assertEqual(span.name, 'molten.request') self.assertEqual(span.span_type, 'web') @@ -77,6 +78,7 @@ def test_route_success_query_string(self): # access data property self.assertEqual(response.data, '"Hello 24 year old named Jim!"') span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, 'molten') self.assertEqual(span.name, 'molten.request') self.assertEqual(span.resource, 'GET /hello/{name}/{age}') @@ -164,6 +166,7 @@ def test_route_failure(self): spans = self.tracer.writer.pop() self.assertEqual(response.status_code, 404) span = spans[0] + assert_is_measured(span) self.assertEqual(span.service, 'molten') self.assertEqual(span.name, 'molten.request') self.assertEqual(span.resource, 'GET 404') @@ -180,6 +183,7 @@ def route_error() -> str: spans = self.tracer.writer.pop() self.assertEqual(response.status_code, 500) span = spans[0] + assert_is_measured(span) route_error_span = spans[-1] self.assertEqual(span.service, 'molten') self.assertEqual(span.name, 'molten.request') diff --git a/tests/contrib/mongoengine/test.py b/tests/contrib/mongoengine/test.py index fad28a549f2..3c3c30c7e82 100644 --- a/tests/contrib/mongoengine/test.py +++ b/tests/contrib/mongoengine/test.py @@ -17,6 +17,7 @@ from ..config import MONGO_CONFIG from ...base import override_config from ...test_tracer import get_dummy_tracer +from ...utils import assert_is_measured class Artist(mongoengine.Document): @@ -45,6 +46,8 @@ def test_insert_update_delete_query(self): spans = tracer.writer.pop() assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.resource == 'drop artist' assert span.span_type == 'mongodb' assert span.service == self.TEST_SERVICE @@ -61,6 +64,7 @@ def test_insert_update_delete_query(self): spans = tracer.writer.pop() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == 'insert artist' assert span.span_type == 'mongodb' assert span.service == self.TEST_SERVICE @@ -80,6 +84,7 @@ def test_insert_update_delete_query(self): spans = tracer.writer.pop() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == '{} artist'.format(name) assert span.span_type == 'mongodb' assert span.service == self.TEST_SERVICE @@ -97,6 +102,7 @@ def test_insert_update_delete_query(self): spans = tracer.writer.pop() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == '{} artist {{"first_name": "?"}}'.format(name) assert span.span_type == 'mongodb' assert span.service == self.TEST_SERVICE @@ -111,6 +117,7 @@ def test_insert_update_delete_query(self): spans = tracer.writer.pop() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == 'update artist {"_id": "?"}' assert span.span_type == 'mongodb' assert span.service == self.TEST_SERVICE @@ -124,6 +131,7 @@ def test_insert_update_delete_query(self): spans = tracer.writer.pop() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.resource == 'delete artist {"_id": "?"}' assert span.span_type == 'mongodb' assert span.service == self.TEST_SERVICE @@ -151,6 +159,7 @@ def test_opentracing(self): assert ot_span.name == 'ot_span' assert ot_span.service == 'my_svc' + assert_is_measured(dd_span) assert dd_span.resource == 'drop artist' assert dd_span.span_type == 'mongodb' assert dd_span.service == self.TEST_SERVICE diff --git a/tests/contrib/mysql/test_mysql.py b/tests/contrib/mysql/test_mysql.py index d9872c3f922..87f5aa50bc6 100644 --- a/tests/contrib/mysql/test_mysql.py +++ b/tests/contrib/mysql/test_mysql.py @@ -11,6 +11,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase from ...util import assert_dict_issuperset +from ...utils import assert_is_measured class MySQLCore(object): @@ -46,6 +47,7 @@ def test_simple_query(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -69,6 +71,7 @@ def test_simple_query_fetchll(self): assert len(spans) == 2 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -209,6 +212,7 @@ def test_query_proc(self): # typically, internal calls to execute, but at least we # can expect the last closed span to be our proc. span = spans[len(spans) - 1] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -246,6 +250,7 @@ def test_simple_query_ot(self): assert ot_span.service == 'mysql_svc' assert ot_span.name == 'mysql_op' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' @@ -283,6 +288,7 @@ def test_simple_query_ot_fetchall(self): assert ot_span.service == 'mysql_svc' assert ot_span.name == 'mysql_op' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' diff --git a/tests/contrib/mysqldb/test_mysql.py b/tests/contrib/mysqldb/test_mysql.py index 15581e1f1df..0ca075dd924 100644 --- a/tests/contrib/mysqldb/test_mysql.py +++ b/tests/contrib/mysqldb/test_mysql.py @@ -8,6 +8,7 @@ from ..config import MYSQL_CONFIG from ...base import BaseTracerTestCase from ...util import assert_dict_issuperset +from ...utils import assert_is_measured class MySQLCore(object): @@ -49,6 +50,7 @@ def test_simple_query(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -72,6 +74,7 @@ def test_simple_query_fetchall(self): assert len(spans) == 2 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -96,6 +99,7 @@ def test_simple_query_with_positional_args(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -119,6 +123,7 @@ def test_simple_query_with_positional_args_fetchall(self): assert len(spans) == 2 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -263,6 +268,7 @@ def test_query_proc(self): # typically, internal calls to execute, but at least we # can expect the next to the last closed span to be our proc. span = spans[-2] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'mysql.query' assert span.span_type == 'sql' @@ -297,6 +303,7 @@ def test_simple_query_ot(self): assert ot_span.service == 'mysql_svc' assert ot_span.name == 'mysql_op' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' @@ -331,6 +338,7 @@ def test_simple_query_ot_fetchall(self): assert ot_span.service == 'mysql_svc' assert ot_span.name == 'mysql_op' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'mysql.query' assert dd_span.span_type == 'sql' diff --git a/tests/contrib/psycopg/test_psycopg.py b/tests/contrib/psycopg/test_psycopg.py index 2955b0ec362..038397e4d04 100644 --- a/tests/contrib/psycopg/test_psycopg.py +++ b/tests/contrib/psycopg/test_psycopg.py @@ -18,6 +18,7 @@ from tests.opentracer.utils import init_tracer from tests.contrib.config import POSTGRES_CONFIG from ...base import BaseTracerTestCase +from ...utils import assert_is_measured from ...utils.tracer import DummyTracer @@ -133,6 +134,7 @@ def assert_conn_is_traced(self, db, service): ), ) root = self.get_root_span() + assert_is_measured(root) self.assertIsNone(root.get_tag('sql.query')) self.reset() @@ -156,6 +158,7 @@ def test_opentracing_propagation(self): dict(name='postgres.query', resource=query, service='postgres', error=0, span_type='sql'), ), ) + assert_is_measured(self.get_spans()[1]) self.reset() with self.override_config('dbapi2', dict(trace_fetch_methods=True)): @@ -176,6 +179,7 @@ def test_opentracing_propagation(self): dict(name='postgres.query.fetchall', resource=query, service='postgres', error=0, span_type='sql'), ), ) + assert_is_measured(self.get_spans()[1]) @skipIf(PSYCOPG2_VERSION < (2, 5), 'context manager not available in psycopg2==2.4') def test_cursor_ctx_manager(self): @@ -190,6 +194,7 @@ def test_cursor_ctx_manager(self): assert len(rows) == 1, rows assert rows[0][0] == 'blah' + assert_is_measured(self.get_root_span()) self.assert_structure( dict(name='postgres.query'), ) @@ -284,6 +289,7 @@ def test_composed_query(self): assert rows[0][0] == 'one' assert rows[1][0] == 'two' + assert_is_measured(self.get_root_span()) self.assert_structure( dict(name='postgres.query', resource=query.as_string(db)), ) diff --git a/tests/contrib/pylibmc/test.py b/tests/contrib/pylibmc/test.py index b9444f0e429..d6320c69a93 100644 --- a/tests/contrib/pylibmc/test.py +++ b/tests/contrib/pylibmc/test.py @@ -16,6 +16,7 @@ from ...opentracer.utils import init_tracer from ...contrib.config import MEMCACHED_CONFIG as cfg from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class PylibmcCore(object): @@ -178,6 +179,7 @@ def test_get_set_delete(self): assert expected_resources == resources def _verify_cache_span(self, s, start, end): + assert_is_measured(s) assert s.start > start assert s.start + s.duration < end assert s.service == self.TEST_SERVICE diff --git a/tests/contrib/pylons/test_pylons.py b/tests/contrib/pylons/test_pylons.py index 3bf903e01a0..1e9831f38ad 100644 --- a/tests/contrib/pylons/test_pylons.py +++ b/tests/contrib/pylons/test_pylons.py @@ -12,7 +12,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class PylonsTestCase(BaseTracerTestCase): @@ -48,6 +48,7 @@ def test_controller_exception(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'web' assert span.resource == 'root.raise_exception' assert span.error == 0 @@ -78,6 +79,7 @@ def test_mw_exc_success(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'web' assert span.resource == 'None.None' assert span.error == 0 @@ -106,6 +108,7 @@ def test_middleware_exception(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'web' assert span.resource == 'None.None' assert span.error == 1 @@ -128,6 +131,7 @@ def test_exc_success(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'web' assert span.resource == 'root.raise_exception' assert span.error == 0 @@ -150,6 +154,7 @@ def test_exc_client_failure(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'web' assert span.resource == 'root.raise_exception' assert span.error == 0 @@ -172,6 +177,7 @@ def test_success_200(self, query_string=''): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == 'web' assert span.resource == 'root.index' assert_span_http_status_code(span, 200) diff --git a/tests/contrib/pymemcache/test_client_mixin.py b/tests/contrib/pymemcache/test_client_mixin.py index edd9fe292ac..60f19a383a8 100644 --- a/tests/contrib/pymemcache/test_client_mixin.py +++ b/tests/contrib/pymemcache/test_client_mixin.py @@ -11,6 +11,7 @@ from tests.test_tracer import get_dummy_tracer from ...base import override_config +from ...utils import assert_is_measured _Client = pymemcache.client.base.Client @@ -34,6 +35,7 @@ def check_spans(self, num_expected, resources_expected, queries_expected): self.assertEqual(num_expected, len(spans)) for span, resource, query in zip(spans, resources_expected, queries_expected): + assert_is_measured(span) self.assertEqual(span.get_tag(net.TARGET_HOST), TEST_HOST) self.assertEqual(span.get_metric(net.TARGET_PORT), TEST_PORT) self.assertEqual(span.name, memcachedx.CMD) diff --git a/tests/contrib/pymongo/test.py b/tests/contrib/pymongo/test.py index 1e69ad1228e..935983b30dc 100644 --- a/tests/contrib/pymongo/test.py +++ b/tests/contrib/pymongo/test.py @@ -17,6 +17,7 @@ from ..config import MONGO_CONFIG from ...base import override_config from ...test_tracer import get_dummy_tracer +from ...utils import assert_is_measured def test_normalize_filter(): @@ -104,6 +105,7 @@ def test_update(self): assert spans, spans for span in spans: # ensure all the of the common metadata is set + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.span_type == 'mongodb' assert span.meta.get('mongodb.collection') == 'songs' @@ -153,6 +155,7 @@ def test_delete(self): assert spans, spans for span in spans: # ensure all the of the common metadata is set + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.span_type == 'mongodb' assert span.meta.get('mongodb.collection') == collection_name @@ -218,6 +221,7 @@ def test_insert_find(self): spans = writer.pop() for span in spans: # ensure all the of the common metadata is set + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.span_type == 'mongodb' assert span.meta.get('mongodb.collection') == 'teams' @@ -287,6 +291,7 @@ def test_update_ot(self): # ensure the parenting assert span.parent_id == ot_span.span_id # ensure all the of the common metadata is set + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.span_type == 'mongodb' assert span.meta.get('mongodb.collection') == 'songs' diff --git a/tests/contrib/pymysql/test_pymysql.py b/tests/contrib/pymysql/test_pymysql.py index 5951c33ce37..7006194986b 100644 --- a/tests/contrib/pymysql/test_pymysql.py +++ b/tests/contrib/pymysql/test_pymysql.py @@ -12,6 +12,7 @@ from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase from ...util import assert_dict_issuperset +from ...utils import assert_is_measured from ...contrib.config import MYSQL_CONFIG @@ -63,6 +64,7 @@ def test_simple_query(self): assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'pymysql.query' assert span.span_type == 'sql' @@ -84,6 +86,7 @@ def test_simple_query_fetchall(self): assert len(spans) == 2 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'pymysql.query' assert span.span_type == 'sql' @@ -227,6 +230,7 @@ def test_query_proc(self): # typically, internal calls to execute, but at least we # can expect the last closed span to be our proc. span = spans[len(spans) - 2] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'pymysql.query' assert span.span_type == 'sql' @@ -258,6 +262,7 @@ def test_simple_query_ot(self): assert ot_span.service == 'mysql_svc' assert ot_span.name == 'mysql_op' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'pymysql.query' assert dd_span.span_type == 'sql' @@ -290,6 +295,7 @@ def test_simple_query_ot_fetchall(self): assert ot_span.service == 'mysql_svc' assert ot_span.name == 'mysql_op' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'pymysql.query' assert dd_span.span_type == 'sql' diff --git a/tests/contrib/pyramid/utils.py b/tests/contrib/pyramid/utils.py index 3e9ee9dc81a..e82ff9fe49e 100644 --- a/tests/contrib/pyramid/utils.py +++ b/tests/contrib/pyramid/utils.py @@ -14,7 +14,7 @@ from ...opentracer.utils import init_tracer from ...base import BaseTracerTestCase -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class PyramidBase(BaseTracerTestCase): @@ -63,6 +63,7 @@ def test_200(self, query_string=""): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET index" assert s.error == 0 @@ -146,6 +147,7 @@ def test_404(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "404" assert s.error == 0 @@ -161,6 +163,7 @@ def test_302(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET raise_redirect" assert s.error == 0 @@ -176,6 +179,7 @@ def test_204(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET raise_no_content" assert s.error == 0 @@ -194,6 +198,7 @@ def test_exception(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET exception" assert s.error == 1 @@ -210,6 +215,7 @@ def test_500(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET error" assert s.error == 1 @@ -230,6 +236,7 @@ def test_json(self): assert len(spans) == 2 spans_by_name = {s.name: s for s in spans} s = spans_by_name["pyramid.request"] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET json" assert s.error == 0 @@ -254,6 +261,7 @@ def test_renderer(self): assert len(spans) == 2 spans_by_name = {s.name: s for s in spans} s = spans_by_name["pyramid.request"] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "GET renderer" assert s.error == 0 @@ -276,6 +284,7 @@ def test_http_exception_response(self): spans = writer.pop() assert len(spans) == 1 s = spans[0] + assert_is_measured(s) assert s.service == "foobar" assert s.resource == "404" assert s.error == 1 @@ -347,6 +356,7 @@ def test_200_ot(self): assert ot_span.name == "pyramid_get" assert ot_span.service == "pyramid_svc" + assert_is_measured(dd_span) assert dd_span.service == "foobar" assert dd_span.resource == "GET index" assert dd_span.error == 0 diff --git a/tests/contrib/redis/test.py b/tests/contrib/redis/test.py index bf746cd1182..92ce63d0e21 100644 --- a/tests/contrib/redis/test.py +++ b/tests/contrib/redis/test.py @@ -10,6 +10,7 @@ from ..config import REDIS_CONFIG from ...test_tracer import get_dummy_tracer from ...base import BaseTracerTestCase +from ...utils import assert_is_measured def test_redis_legacy(): @@ -46,6 +47,8 @@ def test_long_command(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'redis.command' assert span.span_type == 'redis' @@ -71,6 +74,7 @@ def test_basics(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'redis.command' assert span.span_type == 'redis' @@ -116,6 +120,7 @@ def test_pipeline_traced(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'redis.command' assert span.resource == u'SET blah 32\nRPUSH foo éé\nHGETALL xxx' @@ -137,6 +142,7 @@ def test_pipeline_immediate(self): spans = self.get_spans() assert len(spans) == 2 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'redis.command' assert span.resource == u'SET a 1' @@ -213,6 +219,7 @@ def test_opentracing(self): assert ot_span.name == 'redis_get' assert ot_span.service == 'redis_svc' + assert_is_measured(dd_span) assert dd_span.service == self.TEST_SERVICE assert dd_span.name == 'redis.command' assert dd_span.span_type == 'redis' diff --git a/tests/contrib/rediscluster/test.py b/tests/contrib/rediscluster/test.py index f2224cbc044..da6d2a864bd 100644 --- a/tests/contrib/rediscluster/test.py +++ b/tests/contrib/rediscluster/test.py @@ -6,6 +6,7 @@ from ..config import REDISCLUSTER_CONFIG from ...test_tracer import get_dummy_tracer from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class TestRedisPatch(BaseTracerTestCase): @@ -39,6 +40,7 @@ def test_basics(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'redis.command' assert span.span_type == 'redis' @@ -57,6 +59,7 @@ def test_pipeline(self): spans = self.get_spans() assert len(spans) == 1 span = spans[0] + assert_is_measured(span) assert span.service == self.TEST_SERVICE assert span.name == 'redis.command' assert span.resource == u'SET blah 32\nRPUSH foo éé\nHGETALL xxx' diff --git a/tests/contrib/requests/test_requests.py b/tests/contrib/requests/test_requests.py index 97faa06d2f2..cb5efd596f3 100644 --- a/tests/contrib/requests/test_requests.py +++ b/tests/contrib/requests/test_requests.py @@ -13,7 +13,7 @@ from ...base import BaseTracerTestCase from ...util import override_global_tracer -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured # socket name comes from https://english.stackexchange.com/a/44048 SOCKET = 'httpbin.org' @@ -105,6 +105,8 @@ def test_200(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'GET' assert_span_http_status_code(s, 200) assert s.error == 0 @@ -122,6 +124,8 @@ def test_200_send(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'GET' assert_span_http_status_code(s, 200) assert s.error == 0 @@ -137,6 +141,8 @@ def test_200_query_string(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'GET' assert_span_http_status_code(s, 200) assert s.get_tag(http.URL) == URL_200 @@ -154,6 +160,8 @@ def test_requests_module_200(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'GET' assert_span_http_status_code(s, 200) assert s.error == 0 @@ -166,6 +174,8 @@ def test_post_500(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'POST' assert_span_http_status_code(s, 500) assert s.error == 1 @@ -181,6 +191,8 @@ def test_non_existant_url(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'GET' assert s.error == 1 assert 'Failed to establish a new connection' in s.get_tag(errors.MSG) @@ -195,6 +207,8 @@ def test_500(self): spans = self.tracer.writer.pop() assert len(spans) == 1 s = spans[0] + + assert_is_measured(s) assert s.get_tag(http.METHOD) == 'GET' assert_span_http_status_code(s, 500) assert s.error == 1 @@ -368,6 +382,7 @@ def test_200_ot(self): assert ot_span.name == 'requests_get' assert ot_span.service == 'requests_svc' + assert_is_measured(dd_span) assert dd_span.get_tag(http.METHOD) == 'GET' assert_span_http_status_code(dd_span, 200) assert dd_span.error == 0 diff --git a/tests/contrib/sqlalchemy/test_mysql.py b/tests/contrib/sqlalchemy/test_mysql.py index 2e2037c100d..6420dbea238 100644 --- a/tests/contrib/sqlalchemy/test_mysql.py +++ b/tests/contrib/sqlalchemy/test_mysql.py @@ -4,6 +4,7 @@ from .mixins import SQLAlchemyTestMixin from ..config import MYSQL_CONFIG from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class MysqlConnectorTestCase(SQLAlchemyTestMixin, BaseTracerTestCase): @@ -36,6 +37,7 @@ def test_engine_execute_errors(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] # span fields + assert_is_measured(span) self.assertEqual(span.name, '{}.query'.format(self.VENDOR)) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'SELECT * FROM a_wrong_table') diff --git a/tests/contrib/sqlalchemy/test_patch.py b/tests/contrib/sqlalchemy/test_patch.py index 05a9a0e27c8..3cbecfe6199 100644 --- a/tests/contrib/sqlalchemy/test_patch.py +++ b/tests/contrib/sqlalchemy/test_patch.py @@ -6,6 +6,7 @@ from ..config import POSTGRES_CONFIG from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class SQLAlchemyPatchTestCase(BaseTracerTestCase): @@ -44,6 +45,7 @@ def test_engine_traced(self): assert len(traces[0]) == 1 span = traces[0][0] # check subset of span fields + assert_is_measured(span) assert span.name == 'postgres.query' assert span.service == 'postgres' assert span.error == 0 @@ -61,6 +63,7 @@ def test_engine_pin_service(self): assert len(traces[0]) == 1 span = traces[0][0] # check subset of span fields + assert_is_measured(span) assert span.name == 'postgres.query' assert span.service == 'replica-db' assert span.error == 0 @@ -93,6 +96,7 @@ def test_analytics_sample_rate(self): self.conn.execute('SELECT 1').fetchall() root = self.get_root_span() + assert_is_measured(root) root.assert_matches(name='postgres.query') # If the value is None assert it was not set, otherwise assert the expected value diff --git a/tests/contrib/sqlalchemy/test_postgres.py b/tests/contrib/sqlalchemy/test_postgres.py index 7832fa4d19a..b2b82440360 100644 --- a/tests/contrib/sqlalchemy/test_postgres.py +++ b/tests/contrib/sqlalchemy/test_postgres.py @@ -7,6 +7,7 @@ from .mixins import SQLAlchemyTestMixin from ..config import POSTGRES_CONFIG from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class PostgresTestCase(SQLAlchemyTestMixin, BaseTracerTestCase): @@ -39,6 +40,7 @@ def test_engine_execute_errors(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] # span fields + assert_is_measured(span) self.assertEqual(span.name, '{}.query'.format(self.VENDOR)) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'SELECT * FROM a_wrong_table') diff --git a/tests/contrib/sqlalchemy/test_sqlite.py b/tests/contrib/sqlalchemy/test_sqlite.py index b3a78b4e909..313d5268894 100644 --- a/tests/contrib/sqlalchemy/test_sqlite.py +++ b/tests/contrib/sqlalchemy/test_sqlite.py @@ -4,6 +4,7 @@ from .mixins import SQLAlchemyTestMixin from ...base import BaseTracerTestCase +from ...utils import assert_is_measured class SQLiteTestCase(SQLAlchemyTestMixin, BaseTracerTestCase): @@ -31,6 +32,7 @@ def test_engine_execute_errors(self): self.assertEqual(len(traces[0]), 1) span = traces[0][0] # span fields + assert_is_measured(span) self.assertEqual(span.name, '{}.query'.format(self.VENDOR)) self.assertEqual(span.service, self.SERVICE) self.assertEqual(span.resource, 'SELECT * FROM a_wrong_table') diff --git a/tests/contrib/sqlite3/test_sqlite3.py b/tests/contrib/sqlite3/test_sqlite3.py index e1294bd053a..00dcee92b11 100644 --- a/tests/contrib/sqlite3/test_sqlite3.py +++ b/tests/contrib/sqlite3/test_sqlite3.py @@ -13,6 +13,7 @@ # testing from tests.opentracer.utils import init_tracer from ...base import BaseTracerTestCase +from ...utils import assert_is_measured, assert_is_not_measured class TestSQLite(BaseTracerTestCase): @@ -69,6 +70,7 @@ def test_sqlite(self): dict(name='sqlite.query', span_type='sql', resource=q, service=service, error=0), ) root = self.get_root_span() + assert_is_measured(root) self.assertIsNone(root.get_tag('sql.query')) assert start <= root.start <= end assert root.duration <= end - start @@ -87,6 +89,7 @@ def test_sqlite(self): dict(name='sqlite.query', span_type='sql', resource=q, service=service, error=1), ) root = self.get_root_span() + assert_is_measured(root) self.assertIsNone(root.get_tag('sql.query')) self.assertIsNotNone(root.get_tag(errors.ERROR_STACK)) self.assertIn('OperationalError', root.get_tag(errors.ERROR_TYPE)) @@ -113,9 +116,11 @@ def test_sqlite_fetchall_is_traced(self): # Assert query query_span.assert_structure(dict(name='sqlite.query', resource=q)) + assert_is_measured(query_span) # Assert fetchall fetchall_span.assert_structure(dict(name='sqlite.query.fetchall', resource=q, span_type='sql', error=0)) + assert_is_not_measured(fetchall_span) self.assertIsNone(fetchall_span.get_tag('sql.query')) def test_sqlite_fetchone_is_traced(self): @@ -137,9 +142,11 @@ def test_sqlite_fetchone_is_traced(self): query_span, fetchone_span = self.get_root_spans() # Assert query + assert_is_measured(query_span) query_span.assert_structure(dict(name='sqlite.query', resource=q)) # Assert fetchone + assert_is_not_measured(fetchone_span) fetchone_span.assert_structure( dict( name='sqlite.query.fetchone', @@ -169,9 +176,11 @@ def test_sqlite_fetchmany_is_traced(self): query_span, fetchmany_span = self.get_root_spans() # Assert query + assert_is_measured(query_span) query_span.assert_structure(dict(name='sqlite.query', resource=q)) # Assert fetchmany + assert_is_not_measured(fetchmany_span) fetchmany_span.assert_structure( dict( name='sqlite.query.fetchmany', @@ -204,6 +213,7 @@ def test_sqlite_ot(self): dict(name='sqlite.query', span_type='sql', resource=q, error=0), ) ) + assert_is_measured(self.get_spans()[1]) self.reset() with self.override_config('dbapi2', dict(trace_fetch_methods=True)): @@ -223,6 +233,7 @@ def test_sqlite_ot(self): dict(name='sqlite.query.fetchall', span_type='sql', resource=q, error=0), ), ) + assert_is_measured(self.get_spans()[1]) def test_commit(self): connection = self._given_a_traced_connection(self.tracer) diff --git a/tests/contrib/tornado/test_tornado_web.py b/tests/contrib/tornado/test_tornado_web.py index e710da71039..d61172d0a94 100644 --- a/tests/contrib/tornado/test_tornado_web.py +++ b/tests/contrib/tornado/test_tornado_web.py @@ -8,7 +8,7 @@ import tornado from tests.opentracer.utils import init_tracer -from ...utils import assert_span_http_status_code +from ...utils import assert_span_http_status_code, assert_is_measured class TestTornadoWeb(TornadoTestCase): @@ -29,6 +29,7 @@ def test_success_handler(self, query_string=''): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -59,6 +60,7 @@ def test_nested_handler(self): assert 2 == len(traces[0]) # check request span request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -86,6 +88,7 @@ def test_exception_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -107,6 +110,7 @@ def test_http_exception_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -128,6 +132,7 @@ def test_http_exception_500_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -149,6 +154,7 @@ def test_sync_success_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -168,6 +174,7 @@ def test_sync_exception_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -189,6 +196,7 @@ def test_404_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -210,6 +218,7 @@ def test_redirect_handler(self): assert 1 == len(traces[1]) redirect_span = traces[0][0] + assert_is_measured(redirect_span) assert 'tornado-web' == redirect_span.service assert 'tornado.request' == redirect_span.name assert 'web' == redirect_span.span_type @@ -240,6 +249,7 @@ def test_static_handler(self): assert 1 == len(traces[0]) request_span = traces[0][0] + assert_is_measured(request_span) assert 'tornado-web' == request_span.service assert 'tornado.request' == request_span.name assert 'web' == request_span.span_type @@ -302,6 +312,7 @@ def test_success_handler_ot(self): assert ot_span.name == 'tornado_op' assert ot_span.service == 'tornado_svc' + assert_is_measured(dd_span) assert 'tornado-web' == dd_span.service assert 'tornado.request' == dd_span.name assert 'web' == dd_span.span_type diff --git a/tests/contrib/vertica/test_vertica.py b/tests/contrib/vertica/test_vertica.py index 753915cb737..431b41b7f17 100644 --- a/tests/contrib/vertica/test_vertica.py +++ b/tests/contrib/vertica/test_vertica.py @@ -15,6 +15,7 @@ from tests.contrib.config import VERTICA_CONFIG from tests.opentracer.utils import init_tracer from tests.test_tracer import get_dummy_tracer +from ...utils import assert_is_measured TEST_TABLE = 'test_table' @@ -204,6 +205,7 @@ def test_execute_metadata(self): assert len(spans) == 2 # check all the metadata + assert_is_measured(spans[0]) assert spans[0].service == 'vertica' assert spans[0].span_type == 'sql' assert spans[0].name == 'vertica.query' @@ -231,6 +233,7 @@ def test_cursor_override(self): assert len(spans) == 2 # check all the metadata + assert_is_measured(spans[0]) assert spans[0].service == 'vertica' assert spans[0].span_type == 'sql' assert spans[0].name == 'vertica.query' @@ -373,6 +376,7 @@ def test_opentracing(self): assert ot_span.parent_id is None assert dd_span.parent_id == ot_span.span_id + assert_is_measured(dd_span) assert dd_span.service == 'vertica' assert dd_span.span_type == 'sql' assert dd_span.name == 'vertica.query' diff --git a/tests/test_span.py b/tests/test_span.py index 426ab8341fd..4d794a01410 100644 --- a/tests/test_span.py +++ b/tests/test_span.py @@ -1,13 +1,15 @@ import mock import time +import pytest from unittest.case import SkipTest from ddtrace.context import Context -from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY +from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY, SPAN_MEASURED_KEY from ddtrace.span import Span from ddtrace.ext import SpanTypes, errors, priority from .base import BaseTracerTestCase +from .utils import assert_is_measured, assert_is_not_measured class SpanTestCase(BaseTracerTestCase): @@ -419,3 +421,51 @@ def test_duration_int(self): s.finish(finish_time=123) assert s.duration_ns == 1000000000 assert s.duration == 1 + + +@pytest.mark.parametrize( + "value,assertion", + [ + (None, assert_is_measured), + (1, assert_is_measured), + (1.0, assert_is_measured), + (-1, assert_is_measured), + (True, assert_is_measured), + ("true", assert_is_measured), + + # DEV: Ends up being measured because we do `bool("false")` which is `True` + ("false", assert_is_measured), + + (0, assert_is_not_measured), + (0.0, assert_is_not_measured), + (False, assert_is_not_measured), + ], +) +def test_set_tag_measured(value, assertion): + s = Span(tracer=None, name="test.span") + s.set_tag(SPAN_MEASURED_KEY, value) + assertion(s) + + +def test_set_tag_measured_not_set(): + # Span is not measured by default + s = Span(tracer=None, name="test.span") + assert_is_not_measured(s) + + +def test_set_tag_measured_no_value(): + s = Span(tracer=None, name="test.span") + s.set_tag(SPAN_MEASURED_KEY) + assert_is_measured(s) + + +def test_set_tag_measured_change_value(): + s = Span(tracer=None, name="test.span") + s.set_tag(SPAN_MEASURED_KEY, True) + assert_is_measured(s) + + s.set_tag(SPAN_MEASURED_KEY, False) + assert_is_not_measured(s) + + s.set_tag(SPAN_MEASURED_KEY) + assert_is_measured(s) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 526a52a3c0a..8a4c7f19780 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,9 +1,26 @@ import contextlib import os +from ddtrace.constants import SPAN_MEASURED_KEY from ddtrace.ext import http +def assert_is_measured(span): + """Assert that the span has the proper _dd.measured tag set""" + assert SPAN_MEASURED_KEY in span.metrics + assert SPAN_MEASURED_KEY not in span.meta + assert span.get_metric(SPAN_MEASURED_KEY) == 1 + + +def assert_is_not_measured(span): + """Assert that the span does not set _dd.measured""" + assert SPAN_MEASURED_KEY not in span.meta + if SPAN_MEASURED_KEY in span.meta: + assert span.get_metric(SPAN_MEASURED_KEY) == 0 + else: + assert SPAN_MEASURED_KEY not in span.meta + + def assert_span_http_status_code(span, code): """Assert on the span's 'http.status_code' tag""" tag = span.get_tag(http.STATUS_CODE) From 40af97022f200baadd29388c1078ab089257891f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 19 Feb 2020 17:32:29 +0100 Subject: [PATCH 72/81] Support msgpack 1.0.0 (#1216) * fix(tests): remove encoding argument to msgpack.encode This has been removed in msgpack 1.0.0. Just use the decoder decode method. * style(black): black test_integrations --- pyproject.toml | 1 - tests/test_integration.py | 204 ++++++++++++++++++-------------------- 2 files changed, 98 insertions(+), 107 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ba35d7871c5..b2492929ae9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,7 +178,6 @@ exclude = ''' | test_helpers.py | test_hook.py | test_instance_config.py - | test_integration.py | test_payload.py | test_pin.py | test_sampler.py diff --git a/tests/test_integration.py b/tests/test_integration.py index bc24bd28438..b6aec4c65a5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,9 +1,7 @@ import os -import json import logging import mock import ddtrace -import msgpack from unittest import TestCase, skip, skipUnless @@ -23,7 +21,7 @@ class MockedLogHandler(logging.Handler): """Record log messages to verify error logging logic""" def __init__(self, *args, **kwargs): - self.messages = {'debug': [], 'info': [], 'warning': [], 'error': [], 'critical': []} + self.messages = {"debug": [], "info": [], "warning": [], "error": [], "critical": []} super(MockedLogHandler, self).__init__(*args, **kwargs) def emit(self, record): @@ -38,29 +36,22 @@ class FlawedAPI(API): """ Deliberately report data with an incorrect method to trigger a 4xx response """ + def _put(self, endpoint, data, count=0): conn = httplib.HTTPConnection(self.hostname, self.port) - conn.request('HEAD', endpoint, data, self._headers) + conn.request("HEAD", endpoint, data, self._headers) return Response.from_http_response(conn.getresponse()) @skipUnless( - os.environ.get('TEST_DATADOG_INTEGRATION', False), - 'You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable' + os.environ.get("TEST_DATADOG_INTEGRATION", False), + "You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable", ) class TestWorkers(TestCase): """ Ensures that a workers interacts correctly with the main thread. These are part of integration tests so real calls are triggered. """ - def _decode(self, payload): - """ - Helper function that decodes data based on the given Encoder. - """ - if isinstance(self.api._encoder, JSONEncoder): - return json.loads(payload) - elif isinstance(self.api._encoder, MsgpackEncoder): - return msgpack.unpackb(payload, encoding='utf-8') def setUp(self): """ @@ -93,23 +84,21 @@ def _get_endpoint_payload(self, calls, endpoint): """ for call, _ in calls: if endpoint in call[0]: - return call[0], self._decode(call[1]) + return call[0], self.api._encoder.decode(call[1]) return None, None @skipUnless( - os.environ.get('TEST_DATADOG_INTEGRATION_UDS', False), - 'You should have a running trace agent on a socket and set TEST_DATADOG_INTEGRATION_UDS=1 env variable' + os.environ.get("TEST_DATADOG_INTEGRATION_UDS", False), + "You should have a running trace agent on a socket and set TEST_DATADOG_INTEGRATION_UDS=1 env variable", ) def test_worker_single_trace_uds(self): - self.tracer.configure(uds_path='/tmp/ddagent/trace.sock') + self.tracer.configure(uds_path="/tmp/ddagent/trace.sock") # Write a first trace so we get a _worker - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() worker = self.tracer.writer - worker._log_error_status = mock.Mock( - worker._log_error_status, wraps=worker._log_error_status, - ) - self.tracer.trace('client.testing').finish() + worker._log_error_status = mock.Mock(worker._log_error_status, wraps=worker._log_error_status,) + self.tracer.trace("client.testing").finish() # one send is expected self._wait_thread_flush() @@ -117,14 +106,12 @@ def test_worker_single_trace_uds(self): assert worker._log_error_status.call_count == 0 def test_worker_single_trace_uds_wrong_socket_path(self): - self.tracer.configure(uds_path='/tmp/ddagent/nosockethere') + self.tracer.configure(uds_path="/tmp/ddagent/nosockethere") # Write a first trace so we get a _worker - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() worker = self.tracer.writer - worker._log_error_status = mock.Mock( - worker._log_error_status, wraps=worker._log_error_status, - ) - self.tracer.trace('client.testing').finish() + worker._log_error_status = mock.Mock(worker._log_error_status, wraps=worker._log_error_status,) + self.tracer.trace("client.testing").finish() # one send is expected self._wait_thread_flush() @@ -134,101 +121,102 @@ def test_worker_single_trace_uds_wrong_socket_path(self): def test_worker_single_trace(self): # create a trace block and send it using the transport system tracer = self.tracer - tracer.trace('client.testing').finish() + tracer.trace("client.testing").finish() # one send is expected self._wait_thread_flush() assert self.api._put.call_count == 1 # check and retrieve the right call - endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, '/v0.4/traces') - assert endpoint == '/v0.4/traces' + endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, "/v0.4/traces") + assert endpoint == "/v0.4/traces" assert len(payload) == 1 assert len(payload[0]) == 1 - assert payload[0][0]['name'] == 'client.testing' + assert payload[0][0][b"name"] == b"client.testing" # DEV: If we can make the writer flushing deterministic for the case of tests, then we can re-enable this - @skip('Writer flush intervals are impossible to time correctly to make this test not flaky') + @skip("Writer flush intervals are impossible to time correctly to make this test not flaky") def test_worker_multiple_traces(self): # make a single send() if multiple traces are created before the flush interval tracer = self.tracer - tracer.trace('client.testing').finish() - tracer.trace('client.testing').finish() + tracer.trace("client.testing").finish() + tracer.trace("client.testing").finish() # one send is expected self._wait_thread_flush() assert self.api._put.call_count == 1 # check and retrieve the right call - endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, '/v0.4/traces') - assert endpoint == '/v0.4/traces' + endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, "/v0.4/traces") + assert endpoint == "/v0.4/traces" assert len(payload) == 2 assert len(payload[0]) == 1 assert len(payload[1]) == 1 - assert payload[0][0]['name'] == 'client.testing' - assert payload[1][0]['name'] == 'client.testing' + assert payload[0][0][b"name"] == b"client.testing" + assert payload[1][0][b"name"] == b"client.testing" def test_worker_single_trace_multiple_spans(self): # make a single send() if a single trace with multiple spans is created before the flush tracer = self.tracer - parent = tracer.trace('client.testing') - tracer.trace('client.testing').finish() + parent = tracer.trace("client.testing") + tracer.trace("client.testing").finish() parent.finish() # one send is expected self._wait_thread_flush() assert self.api._put.call_count == 1 # check and retrieve the right call - endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, '/v0.4/traces') - assert endpoint == '/v0.4/traces' + endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, "/v0.4/traces") + assert endpoint == "/v0.4/traces" assert len(payload) == 1 assert len(payload[0]) == 2 - assert payload[0][0]['name'] == 'client.testing' - assert payload[0][1]['name'] == 'client.testing' + assert payload[0][0][b"name"] == b"client.testing" + assert payload[0][1][b"name"] == b"client.testing" def test_worker_http_error_logging(self): # Tests the logging http error logic tracer = self.tracer self.tracer.writer.api = FlawedAPI(Tracer.DEFAULT_HOSTNAME, Tracer.DEFAULT_PORT) - tracer.trace('client.testing').finish() + tracer.trace("client.testing").finish() - log = logging.getLogger('ddtrace.internal.writer') - log_handler = MockedLogHandler(level='DEBUG') + log = logging.getLogger("ddtrace.internal.writer") + log_handler = MockedLogHandler(level="DEBUG") log.addHandler(log_handler) self._wait_thread_flush() assert tracer.writer._last_error_ts < monotonic.monotonic() - logged_errors = log_handler.messages['error'] + logged_errors = log_handler.messages["error"] assert len(logged_errors) == 1 - assert 'Failed to send traces to Datadog Agent at http://localhost:8126: ' \ - 'HTTP error status 400, reason Bad Request, message Content-Type:' \ - in logged_errors[0] + assert ( + "Failed to send traces to Datadog Agent at http://localhost:8126: " + "HTTP error status 400, reason Bad Request, message Content-Type:" in logged_errors[0] + ) def test_worker_filter_request(self): - self.tracer.configure(settings={FILTERS_KEY: [FilterRequestsOnUrl(r'http://example\.com/health')]}) + self.tracer.configure(settings={FILTERS_KEY: [FilterRequestsOnUrl(r"http://example\.com/health")]}) # spy the send() method self.api = self.tracer.writer.api self.api._put = mock.Mock(self.api._put, wraps=self.api._put) - span = self.tracer.trace('testing.filteredurl') - span.set_tag(http.URL, 'http://example.com/health') + span = self.tracer.trace("testing.filteredurl") + span.set_tag(http.URL, "http://example.com/health") span.finish() - span = self.tracer.trace('testing.nonfilteredurl') - span.set_tag(http.URL, 'http://example.com/api/resource') + span = self.tracer.trace("testing.nonfilteredurl") + span.set_tag(http.URL, "http://example.com/api/resource") span.finish() self._wait_thread_flush() # Only the second trace should have been sent assert self.api._put.call_count == 1 # check and retrieve the right call - endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, '/v0.4/traces') - assert endpoint == '/v0.4/traces' + endpoint, payload = self._get_endpoint_payload(self.api._put.call_args_list, "/v0.4/traces") + assert endpoint == "/v0.4/traces" assert len(payload) == 1 - assert payload[0][0]['name'] == 'testing.nonfilteredurl' + assert payload[0][0][b"name"] == b"testing.nonfilteredurl" @skipUnless( - os.environ.get('TEST_DATADOG_INTEGRATION', False), - 'You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable' + os.environ.get("TEST_DATADOG_INTEGRATION", False), + "You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable", ) class TestAPITransport(TestCase): """ @@ -236,23 +224,24 @@ class TestAPITransport(TestCase): of integration tests so real calls are triggered and you have to execute a real trace-agent to let them pass. """ - @mock.patch('ddtrace.internal.runtime.container.get_container_info') + + @mock.patch("ddtrace.internal.runtime.container.get_container_info") def setUp(self, get_container_info): """ Create a tracer without workers, while spying the ``send()`` method """ # Mock the container id we use for making requests - get_container_info.return_value = CGroupInfo(container_id='test-container-id') + get_container_info.return_value = CGroupInfo(container_id="test-container-id") # create a new API object to test the transport using synchronous calls self.tracer = get_dummy_tracer() - self.api_json = API('localhost', 8126, encoder=JSONEncoder()) - self.api_msgpack = API('localhost', 8126, encoder=MsgpackEncoder()) + self.api_json = API("localhost", 8126, encoder=JSONEncoder()) + self.api_msgpack = API("localhost", 8126, encoder=MsgpackEncoder()) - @mock.patch('ddtrace.api.httplib.HTTPConnection') + @mock.patch("ddtrace.api.httplib.HTTPConnection") def test_send_presampler_headers(self, mocked_http): # register a single trace with a span and send them to the trace agent - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() trace = self.tracer.writer.pop() traces = [trace] @@ -263,13 +252,13 @@ def test_send_presampler_headers(self, mocked_http): # retrieve the headers from the mocked request call expected_headers = { - 'Datadog-Container-Id': 'test-container-id', # mocked in setUp() - 'Datadog-Meta-Lang': 'python', - 'Datadog-Meta-Lang-Interpreter': PYTHON_INTERPRETER, - 'Datadog-Meta-Lang-Version': PYTHON_VERSION, - 'Datadog-Meta-Tracer-Version': ddtrace.__version__, - 'X-Datadog-Trace-Count': '1', - 'Content-Type': 'application/msgpack', + "Datadog-Container-Id": "test-container-id", # mocked in setUp() + "Datadog-Meta-Lang": "python", + "Datadog-Meta-Lang-Interpreter": PYTHON_INTERPRETER, + "Datadog-Meta-Lang-Version": PYTHON_VERSION, + "Datadog-Meta-Tracer-Version": ddtrace.__version__, + "X-Datadog-Trace-Count": "1", + "Content-Type": "application/msgpack", } params, _ = request_call.call_args_list[0] headers = params[3] @@ -290,7 +279,7 @@ def _send_traces_and_check(self, traces, nresponses=1): def test_send_single_trace(self): # register a single trace with a span and send them to the trace agent - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() trace = self.tracer.writer.pop() traces = [trace] @@ -298,7 +287,7 @@ def test_send_single_trace(self): def test_send_many_traces(self): # register a single trace with a span and send them to the trace agent - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() trace = self.tracer.writer.pop() # 30k is a right number to have both json and msgpack send 2 payload :) traces = [trace] * 30000 @@ -309,7 +298,7 @@ def test_send_single_with_wrong_errors(self): # if the error field is set to True, it must be cast as int so # that the agent decoder handles that properly without providing # a decoding error - span = self.tracer.trace('client.testing') + span = self.tracer.trace("client.testing") span.error = True span.finish() trace = self.tracer.writer.pop() @@ -319,9 +308,9 @@ def test_send_single_with_wrong_errors(self): def test_send_multiple_traces(self): # register some traces and send them to the trace agent - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() trace_1 = self.tracer.writer.pop() - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() trace_2 = self.tracer.writer.pop() traces = [trace_1, trace_2] @@ -329,8 +318,8 @@ def test_send_multiple_traces(self): def test_send_single_trace_multiple_spans(self): # register some traces and send them to the trace agent - with self.tracer.trace('client.testing'): - self.tracer.trace('client.testing').finish() + with self.tracer.trace("client.testing"): + self.tracer.trace("client.testing").finish() trace = self.tracer.writer.pop() traces = [trace] @@ -338,12 +327,12 @@ def test_send_single_trace_multiple_spans(self): def test_send_multiple_traces_multiple_spans(self): # register some traces and send them to the trace agent - with self.tracer.trace('client.testing'): - self.tracer.trace('client.testing').finish() + with self.tracer.trace("client.testing"): + self.tracer.trace("client.testing").finish() trace_1 = self.tracer.writer.pop() - with self.tracer.trace('client.testing'): - self.tracer.trace('client.testing').finish() + with self.tracer.trace("client.testing"): + self.tracer.trace("client.testing").finish() trace_2 = self.tracer.writer.pop() traces = [trace_1, trace_2] @@ -352,26 +341,27 @@ def test_send_multiple_traces_multiple_spans(self): @skipUnless( - os.environ.get('TEST_DATADOG_INTEGRATION', False), - 'You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable' + os.environ.get("TEST_DATADOG_INTEGRATION", False), + "You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable", ) class TestAPIDowngrade(TestCase): """ Ensures that if the tracing client found an earlier trace agent, it will downgrade the current connection to a stable API version """ - @skip('msgpack package split breaks this test; it works for newer version of msgpack') + + @skip("msgpack package split breaks this test; it works for newer version of msgpack") def test_downgrade_api(self): # make a call to a not existing endpoint, downgrades # the current API to a stable one tracer = get_dummy_tracer() - tracer.trace('client.testing').finish() + tracer.trace("client.testing").finish() trace = tracer.writer.pop() # the encoder is right but we're targeting an API # endpoint that is not available - api = API('localhost', 8126) - api._traces = '/v0.0/traces' + api = API("localhost", 8126) + api._traces = "/v0.0/traces" assert isinstance(api._encoder, MsgpackEncoder) # after the call, we downgrade to a working endpoint @@ -382,25 +372,26 @@ def test_downgrade_api(self): @skipUnless( - os.environ.get('TEST_DATADOG_INTEGRATION', False), - 'You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable' + os.environ.get("TEST_DATADOG_INTEGRATION", False), + "You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable", ) class TestRateByService(TestCase): """ Check we get feedback from the agent and we're able to process it. """ + def setUp(self): """ Create a tracer without workers, while spying the ``send()`` method """ # create a new API object to test the transport using synchronous calls self.tracer = get_dummy_tracer() - self.api_json = API('localhost', 8126, encoder=JSONEncoder(), priority_sampling=True) - self.api_msgpack = API('localhost', 8126, encoder=MsgpackEncoder(), priority_sampling=True) + self.api_json = API("localhost", 8126, encoder=JSONEncoder(), priority_sampling=True) + self.api_msgpack = API("localhost", 8126, encoder=MsgpackEncoder(), priority_sampling=True) def test_send_single_trace(self): # register a single trace with a span and send them to the trace agent - self.tracer.trace('client.testing').finish() + self.tracer.trace("client.testing").finish() trace = self.tracer.writer.pop() traces = [trace] @@ -413,31 +404,32 @@ def test_send_single_trace(self): responses = self.api_json.send_traces(traces) assert len(responses) == 1 assert responses[0].status == 200 - assert responses[0].get_json() == dict(rate_by_service={'service:,env:': 1}) + assert responses[0].get_json() == dict(rate_by_service={"service:,env:": 1}) # test Msgpack encoder responses = self.api_msgpack.send_traces(traces) assert len(responses) == 1 assert responses[0].status == 200 - assert responses[0].get_json() == dict(rate_by_service={'service:,env:': 1}) + assert responses[0].get_json() == dict(rate_by_service={"service:,env:": 1}) @skipUnless( - os.environ.get('TEST_DATADOG_INTEGRATION', False), - 'You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable' + os.environ.get("TEST_DATADOG_INTEGRATION", False), + "You should have a running trace agent and set TEST_DATADOG_INTEGRATION=1 env variable", ) class TestConfigure(TestCase): """ Ensures that when calling configure without specifying hostname and port, previous overrides have been kept. """ + def test_configure_keeps_api_hostname_and_port(self): tracer = Tracer() # use real tracer with real api - assert 'localhost' == tracer.writer.api.hostname + assert "localhost" == tracer.writer.api.hostname assert 8126 == tracer.writer.api.port - tracer.configure(hostname='127.0.0.1', port=8127) - assert '127.0.0.1' == tracer.writer.api.hostname + tracer.configure(hostname="127.0.0.1", port=8127) + assert "127.0.0.1" == tracer.writer.api.hostname assert 8127 == tracer.writer.api.port tracer.configure(priority_sampling=True) - assert '127.0.0.1' == tracer.writer.api.hostname + assert "127.0.0.1" == tracer.writer.api.hostname assert 8127 == tracer.writer.api.port From 47d96b2d914f7b4b130e47be673a680483650f44 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 20 Feb 2020 20:14:54 +0100 Subject: [PATCH 73/81] fix(encoding): always return bytes when decoding (#1220) msgpack 1.0.0 has a different default behaviour which returns string rather than bytes. Keep it to bytes by default for all versions. Note that this is only used in testing anyway. --- ddtrace/encoding.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddtrace/encoding.py b/ddtrace/encoding.py index 229b4f1da05..817ff4d6b26 100644 --- a/ddtrace/encoding.py +++ b/ddtrace/encoding.py @@ -83,7 +83,9 @@ def encode(obj): @staticmethod def decode(data): - return msgpack.unpackb(data) + if msgpack.version[:2] < (0, 6): + return msgpack.unpackb(data) + return msgpack.unpackb(data, raw=True) @staticmethod def join_encoded(objs): From 50b3170156f2f5f7aad98d75c778ab0e11226084 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 20 Feb 2020 21:40:59 -0500 Subject: [PATCH 74/81] fix(tests): use encoder decode in tests (#1221) --- tests/test_encoders.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_encoders.py b/tests/test_encoders.py index 24a6ce824bd..18ffb74d150 100644 --- a/tests/test_encoders.py +++ b/tests/test_encoders.py @@ -1,5 +1,4 @@ import json -import msgpack from unittest import TestCase @@ -88,7 +87,7 @@ def test_encode_traces_msgpack(self): encoder = MsgpackEncoder() spans = encoder.encode_traces(traces) - items = msgpack.unpackb(spans) + items = encoder.decode(spans) # test the encoded output that should be a string # and the output must be flatten @@ -123,7 +122,7 @@ def test_join_encoded_msgpack(self): data = encoder.join_encoded(encoded_traces) # Parse the encoded data - items = msgpack.unpackb(data) + items = encoder.decode(data) # test the encoded output that should be a string # and the output must be flatten From 8852b97116397eb052ee3db483c5beb48e678405 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 20 Feb 2020 09:42:24 -0500 Subject: [PATCH 75/81] fix(tox): do not skip install for docs --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3345475ec27..a09293bce3d 100644 --- a/tox.ini +++ b/tox.ini @@ -509,8 +509,6 @@ commands=flake8 . basepython=python3.7 [testenv:docs] -commands_pre= -skip_install=true deps = sphinx commands = sphinx-build -b html docs docs/_build/html basepython = python From 40da729eb76e0b4647f7599f1179ff354d1f2dd0 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 20 Feb 2020 13:00:24 -0500 Subject: [PATCH 76/81] fix: treat spinx warnings as errors --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a09293bce3d..3db966341ad 100644 --- a/tox.ini +++ b/tox.ini @@ -510,7 +510,7 @@ basepython=python3.7 [testenv:docs] deps = sphinx -commands = sphinx-build -b html docs docs/_build/html +commands = sphinx-build -W -b html docs docs/_build/html basepython = python # do not use develop mode with celery as running multiple python versions within From 4c9d6c904bed1c7cce2c5c63e64007f02052c2f4 Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 20 Feb 2020 13:34:51 -0500 Subject: [PATCH 77/81] fix: fix sphinx errors --- docs/installation_quickstart.rst | 6 +++--- tox.ini | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/installation_quickstart.rst b/docs/installation_quickstart.rst index 0f158f30ffe..adf0b1c68d6 100644 --- a/docs/installation_quickstart.rst +++ b/docs/installation_quickstart.rst @@ -42,7 +42,7 @@ variable before starting your application and importing the library: .. list-table:: :header-rows: 1 - :widths: 1 1 2 + :widths: 1 1 1 2 * - Configuration Variable - Configuration Type @@ -64,7 +64,7 @@ that you can use the Datadog tracer in your OpenTracing-compatible applications. Installation -^^^^^^^^^^^^ +~~~~~~~~~~~~ Include OpenTracing with ``ddtrace``:: @@ -78,7 +78,7 @@ you have the following in ``setup.py``:: ], Configuration -^^^^^^^^^^^^^ +~~~~~~~~~~~~~ The OpenTracing convention for initializing a tracer is to define an initialization method that will configure and instantiate a new tracer and diff --git a/tox.ini b/tox.ini index 3db966341ad..0c1ce803d4e 100644 --- a/tox.ini +++ b/tox.ini @@ -509,9 +509,13 @@ commands=flake8 . basepython=python3.7 [testenv:docs] -deps = sphinx -commands = sphinx-build -W -b html docs docs/_build/html -basepython = python +commands_pre= +extras= + opentracing +deps= + sphinx +commands=sphinx-build -W -b html docs docs/_build/html +basepython=python # do not use develop mode with celery as running multiple python versions within # same job will cause problem for tests that use ddtrace-run From e5af3b8c68d8eaad5a46b0dcfa85eed8062d509a Mon Sep 17 00:00:00 2001 From: "Tahir H. Butt" Date: Thu, 20 Feb 2020 21:46:04 -0500 Subject: [PATCH 78/81] fix: typo --- ddtrace/contrib/django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/contrib/django/__init__.py b/ddtrace/contrib/django/__init__.py index 9a38d888c09..b58e72add40 100644 --- a/ddtrace/contrib/django/__init__.py +++ b/ddtrace/contrib/django/__init__.py @@ -93,7 +93,7 @@ The Django integration provides automatic migration from enabling tracing using a middleware to the method consistent with our integrations. Application developers are encouraged to convert their configuration of the tracer to the -later. +latter. 1. Remove ``'ddtrace.config.django'`` from ``INSTALLED_APPS`` in ``settings.py``. From 6a5d1841d5e69324e9e7de1b3b884f2f823cb32f Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 2 Mar 2020 16:04:48 -0800 Subject: [PATCH 79/81] lint/black fixes Signed-off-by: Alex Boten --- ddtrace/contrib/logging/__init__.py | 6 ++-- ddtrace/propagation/datadog.py | 43 ++++++++++------------------- tests/propagation/test_http_b3.py | 18 ++++++------ 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/ddtrace/contrib/logging/__init__.py b/ddtrace/contrib/logging/__init__.py index 3605645b034..330deb10cd7 100644 --- a/ddtrace/contrib/logging/__init__.py +++ b/ddtrace/contrib/logging/__init__.py @@ -10,7 +10,7 @@ ``dd.trace_id=%(dd.trace_id)s`` and ``dd.span_id=%(dd.span_id)s``. ls-trace-run ------------ +------------ When using ``ls-trace-run``, enable patching by setting the environment variable ``DD_LOGS_INJECTION=true``. The logger by default will have a format that @@ -57,10 +57,10 @@ def hello(): from ...utils.importlib import require_modules -required_modules = ['logging'] +required_modules = ["logging"] with require_modules(required_modules) as missing_modules: if not missing_modules: from .patch import patch, unpatch - __all__ = ['patch', 'unpatch'] + __all__ = ["patch", "unpatch"] diff --git a/ddtrace/propagation/datadog.py b/ddtrace/propagation/datadog.py index 5dc651a8af6..ac1567fa463 100644 --- a/ddtrace/propagation/datadog.py +++ b/ddtrace/propagation/datadog.py @@ -7,26 +7,20 @@ # HTTP headers one should set for distributed tracing. # These are cross-language (eg: Python, Go and other implementations should honor these) -HTTP_HEADER_TRACE_ID = 'x-datadog-trace-id' -HTTP_HEADER_PARENT_ID = 'x-datadog-parent-id' -HTTP_HEADER_SAMPLING_PRIORITY = 'x-datadog-sampling-priority' -HTTP_HEADER_ORIGIN = 'x-datadog-origin' +HTTP_HEADER_TRACE_ID = "x-datadog-trace-id" +HTTP_HEADER_PARENT_ID = "x-datadog-parent-id" +HTTP_HEADER_SAMPLING_PRIORITY = "x-datadog-sampling-priority" +HTTP_HEADER_ORIGIN = "x-datadog-origin" # Note that due to WSGI spec we have to also check for uppercased and prefixed # versions of these headers -POSSIBLE_HTTP_HEADER_TRACE_IDS = frozenset( - [HTTP_HEADER_TRACE_ID, get_wsgi_header(HTTP_HEADER_TRACE_ID)] -) -POSSIBLE_HTTP_HEADER_PARENT_IDS = frozenset( - [HTTP_HEADER_PARENT_ID, get_wsgi_header(HTTP_HEADER_PARENT_ID)] -) +POSSIBLE_HTTP_HEADER_TRACE_IDS = frozenset([HTTP_HEADER_TRACE_ID, get_wsgi_header(HTTP_HEADER_TRACE_ID)]) +POSSIBLE_HTTP_HEADER_PARENT_IDS = frozenset([HTTP_HEADER_PARENT_ID, get_wsgi_header(HTTP_HEADER_PARENT_ID)]) POSSIBLE_HTTP_HEADER_SAMPLING_PRIORITIES = frozenset( [HTTP_HEADER_SAMPLING_PRIORITY, get_wsgi_header(HTTP_HEADER_SAMPLING_PRIORITY)] ) -POSSIBLE_HTTP_HEADER_ORIGIN = frozenset( - [HTTP_HEADER_ORIGIN, get_wsgi_header(HTTP_HEADER_ORIGIN)] -) +POSSIBLE_HTTP_HEADER_ORIGIN = frozenset([HTTP_HEADER_ORIGIN, get_wsgi_header(HTTP_HEADER_ORIGIN)]) def _extract_header_value(possible_header_names, headers, default=None): @@ -69,15 +63,11 @@ def parent_call(): @staticmethod def extract_trace_id(headers): - return int( - _extract_header_value(POSSIBLE_HTTP_HEADER_TRACE_IDS, headers, default=0) - ) + return int(_extract_header_value(POSSIBLE_HTTP_HEADER_TRACE_IDS, headers, default=0)) @staticmethod def extract_parent_span_id(headers): - return int( - _extract_header_value(POSSIBLE_HTTP_HEADER_PARENT_IDS, headers, default=0) - ) + return int(_extract_header_value(POSSIBLE_HTTP_HEADER_PARENT_IDS, headers, default=0)) @staticmethod def extract_sampling_priority(headers): @@ -118,23 +108,20 @@ def my_controller(url, headers): sampling_priority = int(sampling_priority) return Context( - trace_id=trace_id, - span_id=parent_span_id, - sampling_priority=sampling_priority, - _dd_origin=origin, + trace_id=trace_id, span_id=parent_span_id, sampling_priority=sampling_priority, _dd_origin=origin, ) # If headers are invalid and cannot be parsed, return a new context and log the issue. - except Exception as error: + except Exception: try: log.debug( - 'invalid x-datadog-* headers, trace-id: %s, parent-id: %s, priority: %s, origin: %s, error: %s', + "invalid x-datadog-* headers, trace-id: %s, parent-id: %s, priority: %s, origin: %s, error: %s", headers.get(HTTP_HEADER_TRACE_ID, 0), headers.get(HTTP_HEADER_PARENT_ID, 0), headers.get(HTTP_HEADER_SAMPLING_PRIORITY), - headers.get(HTTP_HEADER_ORIGIN, ''), - error, + headers.get(HTTP_HEADER_ORIGIN, ""), + exc_info=True, ) # We might fail on string formatting errors ; in that case only format the first error except Exception: - log.debug(error) + log.debug(exc_info=True) return Context() diff --git a/tests/propagation/test_http_b3.py b/tests/propagation/test_http_b3.py index 25cec76ba64..d65dba58b1d 100644 --- a/tests/propagation/test_http_b3.py +++ b/tests/propagation/test_http_b3.py @@ -11,7 +11,7 @@ def test_b3_inject(self): tracer = get_dummy_tracer() tracer.configure(http_propagator=B3HTTPPropagator) - with tracer.trace('global_root_span') as span: + with tracer.trace("global_root_span") as span: headers = {} set_http_propagator_factory(B3HTTPPropagator) propagator = HTTPPropagator() @@ -26,29 +26,29 @@ def test_b3_extract(self): tracer.configure(http_propagator=B3HTTPPropagator) headers = { - 'x-b3-traceid': '4d2', - 'x-b3-spanid': '162e', - 'x-b3-sampled': '1', + "x-b3-traceid": "4d2", + "x-b3-spanid": "162e", + "x-b3-sampled": "1", } propagator = HTTPPropagator() context = propagator.extract(headers) tracer.context_provider.activate(context) - with tracer.trace('local_root_span') as span: + with tracer.trace("local_root_span") as span: assert span.trace_id == 1234 assert span.parent_id == 5678 assert span.context.sampling_priority == AUTO_KEEP headers = { - 'x-b3-traceid': '4d2', - 'x-b3-spanid': '162e', - 'x-b3-sampled': '0', + "x-b3-traceid": "4d2", + "x-b3-spanid": "162e", + "x-b3-sampled": "0", } propagator = HTTPPropagator() context = propagator.extract(headers) tracer.context_provider.activate(context) - with tracer.trace('local_root_span') as span: + with tracer.trace("local_root_span") as span: assert span.context.sampling_priority == AUTO_REJECT From 02ad6a0ad35c776a8b53a352708aba350c4b946c Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Tue, 3 Mar 2020 17:44:51 -0800 Subject: [PATCH 80/81] fix docs and django builds Signed-off-by: Alex Boten --- docs/advanced_usage.rst | 4 ++-- docs/basic_usage.rst | 2 +- tests/contrib/django/test_django.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index c894c28b9d7..9b3d902cd82 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -72,7 +72,7 @@ on the other side, the metadata is retrieved and the trace can continue. To propagate the tracing information, HTTP headers are used to transmit the required metadata to piece together the trace. -.. autoclass:: ddtrace.propagation.http.HTTPPropagator +.. autoclass:: ddtrace.propagation.datadog.DatadogHTTPPropagator :members: Custom @@ -538,7 +538,7 @@ application. .. _ddtracerun: ``ls-trace-run`` ---------------- +---------------- ``ls-trace-run`` will trace :ref:`supported` web frameworks and database modules without the need for changing your code:: diff --git a/docs/basic_usage.rst b/docs/basic_usage.rst index 291ddf1c99d..84b72eeece6 100644 --- a/docs/basic_usage.rst +++ b/docs/basic_usage.rst @@ -10,7 +10,7 @@ Auto Instrumentation -------------------- ``ls-trace-run`` -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^ Python applications can easily be instrumented with ``ddtrace`` by using the included ``ls-trace-run`` command. Simply prefix your Python execution command diff --git a/tests/contrib/django/test_django.py b/tests/contrib/django/test_django.py index 6862e9efb0f..90eba8d1d2c 100644 --- a/tests/contrib/django/test_django.py +++ b/tests/contrib/django/test_django.py @@ -6,7 +6,7 @@ from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY, SAMPLING_PRIORITY_KEY from ddtrace.ext import http, errors from ddtrace.ext.priority import USER_KEEP -from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID, HTTP_HEADER_PARENT_ID, HTTP_HEADER_SAMPLING_PRIORITY +from ddtrace.propagation.datadog import HTTP_HEADER_TRACE_ID, HTTP_HEADER_PARENT_ID, HTTP_HEADER_SAMPLING_PRIORITY from ddtrace.propagation.utils import get_wsgi_header from tests.base import BaseTestCase From 74c637bdeaec317a06961ffe3a7e3e7440c2e58d Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 26 Feb 2020 13:26:17 +0100 Subject: [PATCH 81/81] ci: requires virtualenv < 20 There's a regression in virtualenv >= 20 that makes one of the ddtrace-run + Celery test to fails. virtualenv now adds a site.py which modifies sys.path and breaks the PYTHONPATH that is set by ddtrace-run. --- .circleci/config.yml | 4 ++-- tox.ini | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ad0b5e47e7..47de367194f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -703,9 +703,9 @@ jobs: # list all executed test - run: jq -s ".[]|.testenvs|keys|.[]" /tmp/workspace/* | grep -v GLOB | sed 's/"//g' | sort | uniq | tee all_executed_tests # list all tests in tox.ini - - run: tox -l | grep -v "^wait$" | sort > all_tests + - run: tox -l | grep -v -E "^(wait$|\.)" | sort > all_tests # checks that all tests were executed - - run: diff all_tests all_executed_tests + - run: diff -u all_tests all_executed_tests requires_pre_test: &requires_pre_test diff --git a/tox.ini b/tox.ini index 0c1ce803d4e..2648eee2191 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ # versions. [tox] +requires = virtualenv<20 # Our various test environments. The py*-all tasks will run the core # library tests and all contrib tests with the latest library versions. # The others will test specific versions of libraries.