From b28b80173c623de2b108bcd687e2e0091fc2d1c6 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 31 May 2017 15:08:55 +0200 Subject: [PATCH 01/29] Switch to psutil and update announce --- instana/fsm.py | 10 +++++----- requirements.txt | 1 + setup.py | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/instana/fsm.py b/instana/fsm.py index fb1717fc..13b81bbc 100644 --- a/instana/fsm.py +++ b/instana/fsm.py @@ -1,6 +1,6 @@ import subprocess import os -import sys +import psutil import threading as t import fysom as f import instana.log as l @@ -91,10 +91,10 @@ def check_host(self, host): def announce_sensor(self, e): l.debug("announcing sensor to the agent") - - d = Discovery(pid=os.getpid(), - name=sys.executable, - args=sys.argv[0:]) + p = psutil.Process(os.getpid()) + d = Discovery(pid=p.pid, + name=p.cmdline()[0], + args=p.cmdline()[1:]) (b, _) = self.agent.request_response( self.agent.make_url(a.AGENT_DISCOVERY_URL), "PUT", d) diff --git a/requirements.txt b/requirements.txt index 3b0983cc..80a82923 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ nose>=1.0 fysom>=2.1.2 opentracing>=1.2.1 basictracer>=2.2.0 +psutil>=5.1.3 diff --git a/setup.py b/setup.py index bdac1e4f..64650659 100644 --- a/setup.py +++ b/setup.py @@ -13,5 +13,6 @@ setup_requires=['nose>=1.0', 'fysom>=2.1.2', 'opentracing>=1.2.1,<1.3', - 'basictracer>=2.2.0'], + 'basictracer>=2.2.0', + 'psutil>=5.1.3'], test_suite='nose.collector') From 9047f7ab0fa264de1852b9b7355a8d57ab1e477b Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 31 May 2017 15:28:39 +0200 Subject: [PATCH 02/29] Add travis test suite setup. --- .travis.yml | 6 ++++++ tests/__init__.py | 0 2 files changed, 6 insertions(+) create mode 100644 .travis.yml create mode 100644 tests/__init__.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..65e4f3dc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: python +python: + - "2.7" + - "3.6" +install: "pip install -r requirements.txt" +script: nosetests diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 9ed7b340ebff7d5d436e4f10cdc3c389611e2b27 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 1 Jun 2017 12:32:20 +0200 Subject: [PATCH 03/29] Proper ID generation and conversion --- instana/util.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 instana/util.py diff --git a/instana/util.py b/instana/util.py new file mode 100644 index 00000000..8eb6ff5e --- /dev/null +++ b/instana/util.py @@ -0,0 +1,41 @@ +import random +import os +import time +import struct +import binascii + +_rnd = random.Random() +_current_pid = 0 + + +def generate_id(): + """ Generate a 64bit signed integer for use as a Span or Trace ID """ + global _current_pid + + pid = os.getpid() + if (_current_pid != pid): + _current_pid = pid + _rnd.seed(int(1000000 * time.time()) ^ pid) + return _rnd.randint(-9223372036854775808, 9223372036854775807) + + +def id_to_header(id): + """ Convert a 64bit signed integer to an unsigned base 16 hex string """ + + if not isinstance(id, int): + return "" + + byteString = struct.pack('>q', id) + return binascii.hexlify(byteString).lstrip('0') + + +def header_to_id(header): + """ Convert an unsigned base 16 hex string into a 64bit signed integer """ + + if not isinstance(header, basestring): + return 0 + + # Pad the header to 16 chars + header = header.zfill(16) + r = binascii.unhexlify(header) + return struct.unpack('>q', r)[0] From 758f28776393c57d6730ec19a637e6914576aa65 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 1 Jun 2017 12:32:36 +0200 Subject: [PATCH 04/29] Add tests to validate ID management --- tests/__init__.py | 2 + tests/test_id_management.py | 130 ++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 tests/test_id_management.py diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..abe0cabf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +import sys +sys.path.append("/Users/pglombardo/Projects/instana/python-sensor") diff --git a/tests/test_id_management.py b/tests/test_id_management.py new file mode 100644 index 00000000..0c2877be --- /dev/null +++ b/tests/test_id_management.py @@ -0,0 +1,130 @@ +import instana.util +import string +from nose.tools import assert_equals + + +def test_id_generation(): + count = 0 + while count <= 1000: + id = instana.util.generate_id() + assert id >= -9223372036854775808 + assert id <= 9223372036854775807 + count += 1 + + +def test_id_max_value_and_conversion(): + max_id = 9223372036854775807 + min_id = -9223372036854775808 + max_hex = "7fffffffffffffff" + min_hex = "8000000000000000" + + assert_equals(max_hex, instana.util.id_to_header(max_id)) + assert_equals(min_hex, instana.util.id_to_header(min_id)) + + assert_equals(max_id, instana.util.header_to_id(max_hex)) + assert_equals(min_id, instana.util.header_to_id(min_hex)) + + +def test_id_conversion_back_and_forth(): + # id --> header --> id + original_id = instana.util.generate_id() + header_id = instana.util.id_to_header(original_id) + converted_back_id = instana.util.header_to_id(header_id) + assert original_id == converted_back_id + + # header --> id --> header + original_header_id = "c025ee93b1aeda7b" + id = instana.util.header_to_id(original_header_id) + converted_back_header_id = instana.util.id_to_header(id) + assert_equals(original_header_id, converted_back_header_id) + + # Test a random value + id = -7815363404733516491 + header = "938a406416457535" + + result = instana.util.header_to_id(header) + assert_equals(id, result) + + result = instana.util.id_to_header(id) + assert_equals(header, result) + + +def test_that_leading_zeros_handled_correctly(): + header = instana.util.id_to_header(16) + assert_equals("10", header) + + id = instana.util.header_to_id("10") + assert_equals(16, id) + + id = instana.util.header_to_id("0000000000000010") + assert_equals(16, id) + + id = instana.util.header_to_id("88b6c735206ca42") + assert_equals(615705016619420226, id) + + id = instana.util.header_to_id("088b6c735206ca42") + assert_equals(615705016619420226, id) + + +def test_id_to_header_conversion(): + # Test passing a standard Integer ID + original_id = instana.util.generate_id() + converted_id = instana.util.id_to_header(original_id) + + # Assert that it is a string and there are no non-hex characters + assert isinstance(converted_id, basestring) + assert all(c in string.hexdigits for c in converted_id) + + # Test passing a standard Integer ID as a String + original_id = instana.util.generate_id() + converted_id = instana.util.id_to_header(original_id) + + # Assert that it is a string and there are no non-hex characters + assert isinstance(converted_id, basestring) + assert all(c in string.hexdigits for c in converted_id) + + +def test_id_to_header_conversion_with_bogus_id(): + # Test passing an empty String + converted_id = instana.util.id_to_header('') + + # Assert that it is a string and there are no non-hex characters + assert isinstance(converted_id, basestring) + assert converted_id == '' + + # Test passing a nil + converted_id = instana.util.id_to_header(None) + + # Assert that it is a string and there are no non-hex characters + assert isinstance(converted_id, basestring) + assert converted_id == '' + + # Test passing an Array + converted_id = instana.util.id_to_header([]) + + # Assert that it is a string and there are no non-hex characters + assert isinstance(converted_id, basestring) + assert converted_id == '' + + +def test_header_to_id_conversion(): + # Get a hex string to test against & convert + header_id = instana.util.id_to_header(instana.util.generate_id) + converted_id = instana.util.header_to_id(header_id) + + # Assert that it is an Integer + assert isinstance(converted_id, int) + + +def test_header_to_id_conversion_with_bogus_header(): + # Bogus nil arg + bogus_result = instana.util.header_to_id(None) + assert_equals(0, bogus_result) + + # Bogus Integer arg + bogus_result = instana.util.header_to_id(1234) + assert_equals(0, bogus_result) + + # Bogus Array arg + bogus_result = instana.util.header_to_id([1234]) + assert_equals(0, bogus_result) From 842cad3809bf8620651c856d3c86f119d1f2578a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 7 Jun 2017 15:33:43 +0200 Subject: [PATCH 05/29] Store timer + set daemon flag so procs exit timely --- instana/fsm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/instana/fsm.py b/instana/fsm.py index 13b81bbc..cbf1642d 100644 --- a/instana/fsm.py +++ b/instana/fsm.py @@ -26,6 +26,7 @@ class Fsm(object): agent = None fsm = None + timer = None def __init__(self, agent): l.debug("initializing fsm") @@ -106,7 +107,9 @@ def announce_sensor(self, e): self.fsm.announce() def schedule_retry(self, fun, e): - t.Timer(self.RETRY_PERIOD, fun, [e]).start() + self.timer = t.Timer(self.RETRY_PERIOD, fun, [e]) + self.timer.setDaemon(True) + self.timer.start() def test_agent(self, e): l.debug("testing communication with the agent") From 7616d3b53526d485447305abdace5424f8beb868 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 8 Jun 2017 13:47:19 +0200 Subject: [PATCH 06/29] Basic tracer & span tests --- tests/test_ot_span.py | 17 +++++++++++++++++ tests/test_ot_tracer.py | 11 +++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/test_ot_span.py create mode 100644 tests/test_ot_tracer.py diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py new file mode 100644 index 00000000..59fe79f5 --- /dev/null +++ b/tests/test_ot_span.py @@ -0,0 +1,17 @@ +import opentracing +import instana.tracer +import string +from nose.tools import assert_equals + + +def test_span_basics(): + opentracing.global_tracer = instana.tracer.InstanaTracer() + span = opentracing.global_tracer.start_span("blah") + assert hasattr(span, "finish") + assert hasattr(span, "set_tag") + assert hasattr(span, "tags") + assert hasattr(span, "operation_name") + assert hasattr(span, "set_baggage_item") + assert hasattr(span, "get_baggage_item") + assert hasattr(span, "context") + assert hasattr(span, "log") diff --git a/tests/test_ot_tracer.py b/tests/test_ot_tracer.py new file mode 100644 index 00000000..3302d581 --- /dev/null +++ b/tests/test_ot_tracer.py @@ -0,0 +1,11 @@ +import opentracing +import instana.tracer +from nose.tools import assert_equals + + +def test_tracer_basics(): + assert hasattr(instana.tracer, 'InstanaTracer') + opentracing.global_tracer = instana.tracer.InstanaTracer() + assert hasattr(opentracing.global_tracer, "start_span") + assert hasattr(opentracing.global_tracer, "inject") + assert hasattr(opentracing.global_tracer, "extract") From 32c3ec848321c356141dac02dbd5ae1d7c871204 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Jun 2017 23:25:33 +0200 Subject: [PATCH 07/29] Fix hang, simplify finite state machine events --- instana/fsm.py | 51 +++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/instana/fsm.py b/instana/fsm.py index cbf1642d..50f14b29 100644 --- a/instana/fsm.py +++ b/instana/fsm.py @@ -17,11 +17,6 @@ def __init__(self, **kwds): class Fsm(object): - E_START = "start" - E_LOOKUP = "lookup" - E_ANNOUNCE = "announce" - E_TEST = "test" - RETRY_PERIOD = 30 agent = None @@ -33,20 +28,23 @@ def __init__(self, agent): self.agent = agent self.fsm = f.Fysom({ - "initial": "none", + "initial": "lostandalone", "events": [ - {"name": self.E_START, "src": [ - "none", "unannounced", "announced", "ready"], "dst": "init"}, - {"name": self.E_LOOKUP, "src": "init", "dst": "unannounced"}, - {"name": self.E_ANNOUNCE, "src": "unannounced", "dst": "announced"}, - {"name": self.E_TEST, "src": "announced", "dst": "ready"}], + ("startup", "*", "lostandalone"), + ("lookup", "lostandalone", "found"), + ("announce", "found", "announced"), + ("ready", "announced", "good2go")], "callbacks": { - "onstart": self.lookup_agent_host, - "onenterunannounced": self.announce_sensor, - "onenterannounced": self.test_agent}}) + "onlookup": self.lookup_agent_host, + "onannounce": self.announce_sensor, + "onchangestate": self.printstatechange}}) + + def printstatechange(self, e): + l.debug('========= (%i#%s) FSM event: %s, src: %s, dst: %s ==========' % \ + (os.getpid(), t.current_thread().name, e.event, e.src, e.dst)) def reset(self): - self.fsm.start() + self.fsm.lookup() def lookup_agent_host(self, e): if self.agent.sensor.options.agent_host != "": @@ -57,17 +55,19 @@ def lookup_agent_host(self, e): h = self.check_host(host) if h == a.AGENT_HEADER: self.agent.set_host(host) - self.fsm.lookup() + self.fsm.announce() else: host = self.get_default_gateway() if host: self.check_host(host) if h == a.AGENT_HEADER: self.agent.set_host(host) - self.fsm.lookup() + self.fsm.announce() else: l.error("Cannot lookup agent host. Scheduling retry.") - self.schedule_retry(self.lookup_agent_host, e) + self.schedule_retry(self.lookup_agent_host, e, "agent_lookup") + return False + return True def get_default_gateway(self): l.debug("checking default gateway") @@ -101,15 +101,20 @@ def announce_sensor(self, e): self.agent.make_url(a.AGENT_DISCOVERY_URL), "PUT", d) if not b: l.error("Cannot announce sensor. Scheduling retry.") - self.schedule_retry(self.announce_sensor, e) + self.schedule_retry(self.announce_sensor, e, "announce") + return False else: self.agent.set_from(b) - self.fsm.announce() + self.fsm.ready() + return True - def schedule_retry(self, fun, e): + def schedule_retry(self, fun, e, name): + l.error("Scheduling: " + name) self.timer = t.Timer(self.RETRY_PERIOD, fun, [e]) - self.timer.setDaemon(True) + self.timer.daemon = True + self.timer.name = name self.timer.start() + l.debug('Threadlist: %s', str(t.enumerate())) def test_agent(self, e): l.debug("testing communication with the agent") @@ -117,6 +122,6 @@ def test_agent(self, e): (b, _) = self.agent.head(self.agent.make_url(a.AGENT_DATA_URL)) if not b: - self.schedule_retry(self.test_agent, e) + self.schedule_retry(self.test_agent, e, "agent test") else: self.fsm.test() From bfa8827dbaa7b1b6c643f7e9d389dbbcd4e45709 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Jun 2017 23:26:36 +0200 Subject: [PATCH 08/29] Linter fixes; Update fsm state check --- instana/agent.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/instana/agent.py b/instana/agent.py index 79121836..4dd4ee95 100644 --- a/instana/agent.py +++ b/instana/agent.py @@ -8,6 +8,7 @@ except ImportError: import urllib2 + class From(object): pid = "" hostId = "" @@ -42,10 +43,11 @@ def __init__(self, sensor): self.reset() def to_json(self, o): - return json.dumps(o, default=lambda o: o.__dict__, sort_keys=False,separators=(',', ':')).encode() + return json.dumps(o, default=lambda o: o.__dict__, + sort_keys=False, separators=(',', ':')).encode() def can_send(self): - return self.fsm.fsm.current == "ready" + return self.fsm.fsm.current == "good2go" def head(self, url): return self.request(url, "HEAD", None) @@ -94,7 +96,7 @@ def full_request_response(self, url, method, o, body, header): if method == "HEAD": b = True except Exception as e: - l.error(str(e)) + l.error("full_request_response: " + str(e)) return (b, h) From 1530b1cf5545bd4d8f4aefad6ca3ec5b6bf90bf5 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Jun 2017 23:28:52 +0200 Subject: [PATCH 09/29] Include pid in log messages --- instana/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instana/log.py b/instana/log.py index baad4911..d16029f7 100644 --- a/instana/log.py +++ b/instana/log.py @@ -1,6 +1,7 @@ import logging as l +import os -logger = l.getLogger('instana') +logger = l.getLogger('instana(' + str(os.getpid()) + ')') def init(level): From b19e6d59259b68eb5675f40a6253bd867a137dd6 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Jun 2017 23:29:36 +0200 Subject: [PATCH 10/29] Set timer name/daemon for debug; linter improvements --- instana/meter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/instana/meter.py b/instana/meter.py index d6b48870..b72520a6 100644 --- a/instana/meter.py +++ b/instana/meter.py @@ -79,13 +79,17 @@ class Meter(object): sensor = None last_usage = None last_collect = None + timer = None def __init__(self, sensor): self.sensor = sensor self.tick() def tick(self): - t.Timer(1, self.process).start() + timer = t.Timer(1, self.process) + timer.daemon = True + timer.name = "Instana Metric Collection" + timer.start() def process(self): if self.sensor.agent.can_send(): @@ -99,7 +103,9 @@ def process(self): d = EntityData(pid=os.getpid(), snapshot=s, metrics=m) t.Thread(target=self.sensor.agent.request, - args=(self.sensor.agent.make_url(a.AGENT_DATA_URL), "POST", d)).start() + args=(self.sensor.agent.make_url(a.AGENT_DATA_URL), + "POST", d), + name="Metrics POST").start() self.tick() From d8595b14ee53623d74b99562fc4c7ee6150c8b0a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 12 Jun 2017 23:30:16 +0200 Subject: [PATCH 11/29] Override start_span so Instana IDs are used via generate_id --- instana/tracer.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/instana/tracer.py b/instana/tracer.py index 1e018110..16a63eac 100644 --- a/instana/tracer.py +++ b/instana/tracer.py @@ -1,9 +1,14 @@ +import time from basictracer import BasicTracer import instana.recorder as r import opentracing import instana.options as o import instana.sensor as s +from basictracer.context import SpanContext +from basictracer.span import BasicSpan +from instana.util import generate_id + class InstanaTracer(BasicTracer): sensor = None @@ -13,6 +18,47 @@ def __init__(self, options=o.Options()): super(InstanaTracer, self).__init__( r.InstanaRecorder(self.sensor), r.InstanaSampler()) + def start_span( + self, + operation_name=None, + child_of=None, + references=None, + tags=None, + start_time=None): + "Taken from BasicTracer so we can override generate_id calls to ours" + + start_time = time.time() if start_time is None else start_time + + # See if we have a parent_ctx in `references` + parent_ctx = None + if child_of is not None: + parent_ctx = ( + child_of if isinstance(child_of, opentracing.SpanContext) + else child_of.context) + elif references is not None and len(references) > 0: + # TODO only the first reference is currently used + parent_ctx = references[0].referenced_context + + # Assemble the child ctx + ctx = SpanContext(span_id=generate_id()) + if parent_ctx is not None: + if parent_ctx._baggage is not None: + ctx._baggage = parent_ctx._baggage.copy() + ctx.trace_id = parent_ctx.trace_id + ctx.sampled = parent_ctx.sampled + else: + ctx.trace_id = generate_id() + ctx.sampled = self.sampler.sampled(ctx.trace_id) + + # Tie it all together + return BasicSpan( + self, + operation_name=operation_name, + context=ctx, + parent_id=(None if parent_ctx is None else parent_ctx.span_id), + tags=tags, + start_time=start_time) + def init(options): opentracing.tracer = InstanaTracer(options) From 479de65226f3a02522556d7d66e44f432322b563 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 13 Jun 2017 14:26:49 +0200 Subject: [PATCH 12/29] Set test env var --- tests/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index abe0cabf..26bdda87 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,4 @@ +import os import sys sys.path.append("/Users/pglombardo/Projects/instana/python-sensor") +os.environ["INSTANA_TEST"] = "true" From 937e02f152d31c835133d17cc64fcb979f20c05f Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 13 Jun 2017 14:40:01 +0200 Subject: [PATCH 13/29] Quick fix: delay sending snapshot This allows process recognition to happen before missing the snapshot were originally sending on first metrics post (and then again 5 minutes later). --- instana/meter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/meter.py b/instana/meter.py index b72520a6..87de37aa 100644 --- a/instana/meter.py +++ b/instana/meter.py @@ -75,7 +75,7 @@ def __init__(self, **kwds): class Meter(object): SNAPSHOT_PERIOD = 600 - snapshot_countdown = 1 + snapshot_countdown = 30 sensor = None last_usage = None last_collect = None From 2a8d0cfa91026612b04642a2a5ba3aa3f17116d5 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 13 Jun 2017 17:38:14 +0200 Subject: [PATCH 14/29] Proper delta reporting of metrics --- instana/meter.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/instana/meter.py b/instana/meter.py index 87de37aa..8031767b 100644 --- a/instana/meter.py +++ b/instana/meter.py @@ -5,6 +5,7 @@ import gc as gc_ import sys import instana.agent_const as a +import copy class Snapshot(object): @@ -63,6 +64,17 @@ class Metrics(object): def __init__(self, **kwds): self.__dict__.update(kwds) + def delta_data(self, delta): + data = self.__dict__ + if delta is None: + return data + + unchanged_items = set(data.items()) & set(delta.items()) + for x in unchanged_items: + data.pop(x[0]) + + return data + class EntityData(object): pid = 0 @@ -80,6 +92,7 @@ class Meter(object): last_usage = None last_collect = None timer = None + last_metrics = None def __init__(self, sensor): self.sensor = sensor @@ -95,17 +108,18 @@ def process(self): if self.sensor.agent.can_send(): self.snapshot_countdown = self.snapshot_countdown - 1 s = None + cm = self.collect_metrics() if self.snapshot_countdown == 0: self.snapshot_countdown = self.SNAPSHOT_PERIOD s = self.collect_snapshot() - - m = self.collect_metrics() - d = EntityData(pid=os.getpid(), snapshot=s, metrics=m) - - t.Thread(target=self.sensor.agent.request, - args=(self.sensor.agent.make_url(a.AGENT_DATA_URL), - "POST", d), - name="Metrics POST").start() + md = cm.delta_data(None) + else: + md = copy.deepcopy(cm).delta_data(self.last_metrics) + + d = EntityData(pid=os.getpid(), snapshot=s, metrics=md) + self.sensor.agent.request( + self.sensor.agent.make_url(a.AGENT_DATA_URL), "POST", d) + self.last_metrics = cm.__dict__ self.tick() From c1fc36f5b08c45ce5112ebd7779f2b91a547d867 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 23 Jun 2017 10:29:21 +0200 Subject: [PATCH 15/29] Unite all Data objects under a single namespace --- instana/custom.py | 6 ------ instana/data.py | 8 -------- instana/http.py | 11 ----------- instana/span_data.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 28 insertions(+), 25 deletions(-) delete mode 100644 instana/custom.py delete mode 100644 instana/data.py delete mode 100644 instana/http.py create mode 100644 instana/span_data.py diff --git a/instana/custom.py b/instana/custom.py deleted file mode 100644 index 5bf5dcba..00000000 --- a/instana/custom.py +++ /dev/null @@ -1,6 +0,0 @@ -class CustomData(object): - tags = None - logs = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) diff --git a/instana/data.py b/instana/data.py deleted file mode 100644 index 2b126853..00000000 --- a/instana/data.py +++ /dev/null @@ -1,8 +0,0 @@ -class Data(object): - service = None - http = None - baggage = None - custom = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) diff --git a/instana/http.py b/instana/http.py deleted file mode 100644 index d7871a78..00000000 --- a/instana/http.py +++ /dev/null @@ -1,11 +0,0 @@ -HTTP_CLIENT = "g.hc" -HTTP_SERVER = "g.http" - -class HttpData(object): - host = None - url = None - status = 0 - method = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) diff --git a/instana/span_data.py b/instana/span_data.py new file mode 100644 index 00000000..2d30bd92 --- /dev/null +++ b/instana/span_data.py @@ -0,0 +1,28 @@ +HTTP_CLIENT = "g.hc" +HTTP_SERVER = "g.http" + +class Data(object): + service = None + http = None + baggage = None + custom = None + sdk = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) + +class HttpData(object): + host = None + url = None + status = 0 + method = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) + +class CustomData(object): + tags = None + logs = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) From fb45cb5ec300d63e788ce0445c0ff5eb8d238738 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 16:12:55 +0200 Subject: [PATCH 16/29] More tests; use a single tracer --- tests/__init__.py | 6 ++++-- tests/test_ot_span.py | 25 +++++++++++++++++++++++-- tests/test_ot_tracer.py | 1 - 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 26bdda87..e95723bf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,6 @@ import os -import sys -sys.path.append("/Users/pglombardo/Projects/instana/python-sensor") +import opentracing +import instana.tracer os.environ["INSTANA_TEST"] = "true" + +opentracing.global_tracer = instana.tracer.InstanaTracer() diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index 59fe79f5..d3ee6ce3 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -4,8 +4,7 @@ from nose.tools import assert_equals -def test_span_basics(): - opentracing.global_tracer = instana.tracer.InstanaTracer() +def test_span_interface(): span = opentracing.global_tracer.start_span("blah") assert hasattr(span, "finish") assert hasattr(span, "set_tag") @@ -15,3 +14,25 @@ def test_span_basics(): assert hasattr(span, "get_baggage_item") assert hasattr(span, "context") assert hasattr(span, "log") + + +def test_span_ids(): + count = 0 + while count <= 1000: + count += 1 + span = opentracing.global_tracer.start_span("test_span_ids") + context = span.context + assert -9223372036854775808 <= context.span_id <= 9223372036854775807 + assert -9223372036854775808 <= context.trace_id <= 9223372036854775807 + + +def test_span_fields(): + span = opentracing.global_tracer.start_span("mycustom") + assert_equals("mycustom", span.operation_name) + assert span.context + + span.set_tag("tagone", "string") + span.set_tag("tagtwo", 150) + + assert_equals("string", span.tags['tagone']) + assert_equals(150, span.tags['tagtwo']) diff --git a/tests/test_ot_tracer.py b/tests/test_ot_tracer.py index 3302d581..1e3706d4 100644 --- a/tests/test_ot_tracer.py +++ b/tests/test_ot_tracer.py @@ -5,7 +5,6 @@ def test_tracer_basics(): assert hasattr(instana.tracer, 'InstanaTracer') - opentracing.global_tracer = instana.tracer.InstanaTracer() assert hasattr(opentracing.global_tracer, "start_span") assert hasattr(opentracing.global_tracer, "inject") assert hasattr(opentracing.global_tracer, "extract") From 1d36e66178fee419be50ea90900dd61a2c9c37fc Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 16:15:06 +0200 Subject: [PATCH 17/29] Remove unused imports. --- tests/test_ot_span.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index d3ee6ce3..c7e9dbe9 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -1,6 +1,4 @@ import opentracing -import instana.tracer -import string from nose.tools import assert_equals From a61125e9c7e5f7a5d8612f20364c384486e22bf9 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 17:30:44 +0200 Subject: [PATCH 18/29] If no incoming context, use same id for s & t --- instana/tracer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/instana/tracer.py b/instana/tracer.py index 16a63eac..7833692e 100644 --- a/instana/tracer.py +++ b/instana/tracer.py @@ -40,14 +40,15 @@ def start_span( parent_ctx = references[0].referenced_context # Assemble the child ctx - ctx = SpanContext(span_id=generate_id()) + instana_id = generate_id() + ctx = SpanContext(span_id=instana_id) if parent_ctx is not None: if parent_ctx._baggage is not None: ctx._baggage = parent_ctx._baggage.copy() ctx.trace_id = parent_ctx.trace_id ctx.sampled = parent_ctx.sampled else: - ctx.trace_id = generate_id() + ctx.trace_id = instana_id ctx.sampled = self.sampler.sampled(ctx.trace_id) # Tie it all together From 261410dd034d281e3600a8b1479acb3ab99f6312 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 18:36:14 +0200 Subject: [PATCH 19/29] Span buffering; SDK Span type support: * SDK span type definition and management * Use a Queue to periodically batch send spans * Add support for 'span.kind' tag and translation * Add tests to validate SDK span types * Add tests to validate proper span buffering * Add functions to manage span queue * Add Setup/Teardown functions for tests --- instana/recorder.py | 135 +++++++++++++++++++++++++++++++----------- instana/span_data.py | 25 ++++++++ tests/test_ot_span.py | 106 +++++++++++++++++++++++++-------- 3 files changed, 206 insertions(+), 60 deletions(-) diff --git a/instana/recorder.py b/instana/recorder.py index c83bd1cb..52f530d8 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -1,53 +1,120 @@ from basictracer import Sampler, SpanRecorder import instana.agent_const as a -import instana.http as http -import instana.custom as c import threading as t import opentracing.ext.tags as ext import socket -import instana.data as d +import instana.span_data as sd +import Queue +import time -class InstanaSpan(object): - t = 0 - p = None - s = 0 - ts = 0 - d = 0 - n = None - f = None - data = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) class InstanaRecorder(SpanRecorder): sensor = None + registered_spans = ("memcache", "rpc-client", "rpc-server") + queue = Queue.Queue() def __init__(self, sensor): super(InstanaRecorder, self).__init__() self.sensor = sensor + self.run() + + def run(self): + """ Span a background thread to periodically report queued spans """ + self.timer = t.Thread(target=self.report_spans) + self.timer.daemon = True + self.timer.name = "Instana Span Reporting" + self.timer.start() + + def report_spans(self): + """ Periodically report the queued spans """ + while 1: + if self.sensor.agent.can_send() and self.queue.qsize: + url = self.sensor.agent.make_url(a.AGENT_TRACES_URL) + self.sensor.agent.request(url, "POST", self.queued_spans()) + time.sleep(2) + + def queue_size(self): + """ Return the size of the queue; how may spans are queued, """ + return self.queue.qsize() + + def queued_spans(self): + """ Get all of the spans in the queue """ + spans = [] + while True: + try: + s = self.queue.get(False) + except Queue.Empty: + break + else: + spans.append(s) + return spans + + def clear_spans(self): + """ Clear the queue of spans """ + self.queued_spans() def record_span(self, span): + """ + Convert the passed BasicSpan into an InstanaSpan and + add it to the span queue + """ if self.sensor.agent.can_send(): - data = d.Data(service=self.get_service_name(span), - http=http.HttpData(host=self.get_host_name(span), - url=self.get_string_tag(span, ext.HTTP_URL), - method=self.get_string_tag(span, ext.HTTP_METHOD), - status=self.get_tag(span, ext.HTTP_STATUS_CODE)), - baggage=span.context.baggage, - custom=c.CustomData(tags=span.tags, - logs=self.collect_logs(span))) - - t.Thread(target=self.sensor.agent.request, - args=(self.sensor.agent.make_url(a.AGENT_TRACES_URL), "POST", - [InstanaSpan(t=span.context.trace_id, - p=span.parent_id, - s=span.context.span_id, - ts=int(round(span.start_time * 1000)), - d=int(round(span.duration * 1000)), - n=self.get_http_type(span), - f=self.sensor.agent.from_, - data=data)])).start() + instana_span = None + + if span.operation_name in self.registered_spans: + instana_span = self.build_registered_span(span) + else: + instana_span = self.build_sdk_span(span) + + self.queue.put(instana_span) + + def build_registered_span(self, span): + """ Takes a BasicSpan and converts it into a registered InstanaSpan """ + data = sd.Data(service=self.get_service_name(span), + http=sd.HttpData(host=self.get_host_name(span), + url=self.get_string_tag(span, ext.HTTP_URL), + method=self.get_string_tag(span, ext.HTTP_METHOD), + status=self.get_tag(span, ext.HTTP_STATUS_CODE)), + baggage=span.context.baggage, + custom=sd.CustomData(tags=span.tags, + logs=self.collect_logs(span))) + return sd.InstanaSpan( + t=span.context.trace_id, + p=span.parent_id, + s=span.context.span_id, + ts=int(round(span.start_time * 1000)), + d=int(round(span.duration * 1000)), + n=self.get_http_type(span), + f=self.sensor.agent.from_, + data=data) + + def build_sdk_span(self, span): + """ Takes a BasicSpan and converts into an SDK type InstanaSpan """ + + custom_data = sd.CustomData( + tags=span.tags, + logs=self.collect_logs(span)) + + sdk_data = sd.SDKData( + Name=span.operation_name, + Custom=custom_data + ) + + if "span.kind" in span.tags: + sdk_data.Type = span.tags["span.kind"] + + data = sd.Data(service=self.get_service_name(span), + sdk=sdk_data) + + return sd.InstanaSpan( + t=span.context.trace_id, + p=span.parent_id, + s=span.context.span_id, + ts=int(round(span.start_time * 1000)), + d=int(round(span.duration * 1000)), + n="sdk", + f=self.sensor.agent.from_, + data=data) def get_tag(self, span, tag): if tag in span.tags: diff --git a/instana/span_data.py b/instana/span_data.py index 2d30bd92..b28a8666 100644 --- a/instana/span_data.py +++ b/instana/span_data.py @@ -1,6 +1,21 @@ HTTP_CLIENT = "g.hc" HTTP_SERVER = "g.http" +class InstanaSpan(object): + t = 0 + p = None + s = 0 + ts = 0 + ta = "py" + d = 0 + n = None + f = None + ec = 0 + data = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) + class Data(object): service = None http = None @@ -26,3 +41,13 @@ class CustomData(object): def __init__(self, **kwds): self.__dict__.update(kwds) + +class SDKData(object): + Name = None + Type = None + Arguments = None + Return = None + Custom = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index c7e9dbe9..f61da6fc 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -1,36 +1,90 @@ import opentracing from nose.tools import assert_equals +import time +class OTSpanTest: + def setUp(): + """ Clear all spans before a test run """ + recorder = opentracing.global_tracer.recorder + recorder.clear_spans() -def test_span_interface(): - span = opentracing.global_tracer.start_span("blah") - assert hasattr(span, "finish") - assert hasattr(span, "set_tag") - assert hasattr(span, "tags") - assert hasattr(span, "operation_name") - assert hasattr(span, "set_baggage_item") - assert hasattr(span, "get_baggage_item") - assert hasattr(span, "context") - assert hasattr(span, "log") + def tearDown(): + """ Do nothing for now """ + return None -def test_span_ids(): - count = 0 - while count <= 1000: - count += 1 - span = opentracing.global_tracer.start_span("test_span_ids") - context = span.context - assert -9223372036854775808 <= context.span_id <= 9223372036854775807 - assert -9223372036854775808 <= context.trace_id <= 9223372036854775807 + def test_span_interface(): + span = opentracing.global_tracer.start_span("blah") + assert hasattr(span, "finish") + assert hasattr(span, "set_tag") + assert hasattr(span, "tags") + assert hasattr(span, "operation_name") + assert hasattr(span, "set_baggage_item") + assert hasattr(span, "get_baggage_item") + assert hasattr(span, "context") + assert hasattr(span, "log") -def test_span_fields(): - span = opentracing.global_tracer.start_span("mycustom") - assert_equals("mycustom", span.operation_name) - assert span.context + def test_span_ids(): + count = 0 + while count <= 1000: + count += 1 + span = opentracing.global_tracer.start_span("test_span_ids") + context = span.context + assert -9223372036854775808 <= context.span_id <= 9223372036854775807 + assert -9223372036854775808 <= context.trace_id <= 9223372036854775807 - span.set_tag("tagone", "string") - span.set_tag("tagtwo", 150) - assert_equals("string", span.tags['tagone']) - assert_equals(150, span.tags['tagtwo']) + def test_span_fields(): + span = opentracing.global_tracer.start_span("mycustom") + assert_equals("mycustom", span.operation_name) + assert span.context + + span.set_tag("tagone", "string") + span.set_tag("tagtwo", 150) + + assert_equals("string", span.tags['tagone']) + assert_equals(150, span.tags['tagtwo']) + + def test_span_queueing(): + recorder = opentracing.global_tracer.recorder + + count = 1 + while count <= 20: + count += 1 + span = opentracing.global_tracer.start_span("queuethisplz") + span.set_tag("tagone", "string") + span.set_tag("tagtwo", 150) + span.finish() + + assert_equals(20, recorder.queue_size()) + + + def test_sdk_spans(): + recorder = opentracing.global_tracer.recorder + + span = opentracing.global_tracer.start_span("custom_sdk_span") + span.set_tag("tagone", "string") + span.set_tag("tagtwo", 150) + span.set_tag('span.kind', "entry") + time.sleep(0.5) + span.finish() + + spans = recorder.queued_spans() + assert 1, len(spans) + + sdk_span = spans[0] + assert_equals('sdk', sdk_span.n) + assert_equals(None, sdk_span.p) + assert_equals(sdk_span.s, sdk_span.t) + assert sdk_span.ts + assert sdk_span.ts > 0 + assert sdk_span.d + assert sdk_span.d > 0 + assert_equals("py", sdk_span.ta) + + assert sdk_span.data + assert sdk_span.data.sdk + assert_equals('entry', sdk_span.data.sdk.Type) + assert sdk_span.data.sdk.Custom + assert sdk_span.data.sdk.Custom.tags From 936a6009fe8fc55486c93af2df59d172d3204a71 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 18:41:53 +0200 Subject: [PATCH 20/29] Better name for unified InstanaSpan module --- instana/{span_data.py => span.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename instana/{span_data.py => span.py} (100%) diff --git a/instana/span_data.py b/instana/span.py similarity index 100% rename from instana/span_data.py rename to instana/span.py From 14f516fc361cbf001e51d70c0ce18eb7e0a6f84c Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 18:42:31 +0200 Subject: [PATCH 21/29] Pylint improvements --- tests/test_ot_span.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index f61da6fc..b543951d 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -2,6 +2,7 @@ from nose.tools import assert_equals import time + class OTSpanTest: def setUp(): """ Clear all spans before a test run """ @@ -12,7 +13,6 @@ def tearDown(): """ Do nothing for now """ return None - def test_span_interface(): span = opentracing.global_tracer.start_span("blah") assert hasattr(span, "finish") @@ -24,7 +24,6 @@ def test_span_interface(): assert hasattr(span, "context") assert hasattr(span, "log") - def test_span_ids(): count = 0 while count <= 1000: @@ -34,7 +33,6 @@ def test_span_ids(): assert -9223372036854775808 <= context.span_id <= 9223372036854775807 assert -9223372036854775808 <= context.trace_id <= 9223372036854775807 - def test_span_fields(): span = opentracing.global_tracer.start_span("mycustom") assert_equals("mycustom", span.operation_name) @@ -59,7 +57,6 @@ def test_span_queueing(): assert_equals(20, recorder.queue_size()) - def test_sdk_spans(): recorder = opentracing.global_tracer.recorder From 1fa68630cbfdb6191e7b9fc79eb20c7d80c324c4 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 18:46:40 +0200 Subject: [PATCH 22/29] Migrate to class style test structure --- tests/test_ot_span.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index b543951d..ff7df7d5 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -3,17 +3,17 @@ import time -class OTSpanTest: - def setUp(): +class TestOTSpan: + def setUp(self): """ Clear all spans before a test run """ recorder = opentracing.global_tracer.recorder recorder.clear_spans() - def tearDown(): + def tearDown(self): """ Do nothing for now """ return None - def test_span_interface(): + def test_span_interface(self): span = opentracing.global_tracer.start_span("blah") assert hasattr(span, "finish") assert hasattr(span, "set_tag") @@ -24,7 +24,7 @@ def test_span_interface(): assert hasattr(span, "context") assert hasattr(span, "log") - def test_span_ids(): + def test_span_ids(self): count = 0 while count <= 1000: count += 1 @@ -33,7 +33,7 @@ def test_span_ids(): assert -9223372036854775808 <= context.span_id <= 9223372036854775807 assert -9223372036854775808 <= context.trace_id <= 9223372036854775807 - def test_span_fields(): + def test_span_fields(self): span = opentracing.global_tracer.start_span("mycustom") assert_equals("mycustom", span.operation_name) assert span.context @@ -44,7 +44,7 @@ def test_span_fields(): assert_equals("string", span.tags['tagone']) assert_equals(150, span.tags['tagtwo']) - def test_span_queueing(): + def test_span_queueing(self): recorder = opentracing.global_tracer.recorder count = 1 @@ -57,7 +57,7 @@ def test_span_queueing(): assert_equals(20, recorder.queue_size()) - def test_sdk_spans(): + def test_sdk_spans(self): recorder = opentracing.global_tracer.recorder span = opentracing.global_tracer.start_span("custom_sdk_span") From 1986bd1beecfca625afb7ddffa9e6594f8981804 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 22:59:56 +0200 Subject: [PATCH 23/29] First draft for a repo README --- README.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0df274df..1c3e9ac4 100644 --- a/README.md +++ b/README.md @@ -1 +1,37 @@ -# python-sensor +
+ +
+ +# Instana + +The Instana package provides Python metrics and traces (request, queue & cross-host) for [Instana](https://www.instana.com/). + +## Note + +This package supports Python 2.7 or greater. + +Any and all feedback is welcome. Happy Python visibility. + +## Installation + + $ pip install instana + +## Usage + +The instana package is a zero configuration tool that will automatically collect key metrics from your Python processes. Just install and go. + +## Tracing + +This Python package supports [OpenTracing](http://opentracing.io/). + +## Documentation + +You can find more documentation covering supported components and minimum versions in the Instana [documentation portal](https://instana.atlassian.net/wiki/display/DOCS/Python). + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/instana/python-sensor. + +## More + +Want to instrument other languages? See our [Nodejs instrumentation](https://github.com/instana/nodejs-sensor), [Go instrumentation](https://github.com/instana/golang-sensor), [Ruby instrumentation](https://github.com/instana/ruby-sensor) or [many other supported technologies](https://www.instana.com/supported-technologies/). From 5a6e165df64a1b0de7f383124491059d5442e9f0 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 29 Jun 2017 23:59:31 +0200 Subject: [PATCH 24/29] Rename package to stan; update README --- README.md | 8 ++++---- setup.cfg | 3 +++ setup.py | 7 ++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c3e9ac4..9825bc01 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ -# Instana +# Stan -The Instana package provides Python metrics and traces (request, queue & cross-host) for [Instana](https://www.instana.com/). +The stan package provides Python metrics and traces (request, queue & cross-host) for [Instana](https://www.instana.com/). ## Note @@ -14,11 +14,11 @@ Any and all feedback is welcome. Happy Python visibility. ## Installation - $ pip install instana + $ pip install stan ## Usage -The instana package is a zero configuration tool that will automatically collect key metrics from your Python processes. Just install and go. +The stan package is a zero configuration tool that will automatically collect key metrics from your Python processes. Just install and go. ## Tracing diff --git a/setup.cfg b/setup.cfg index 5061eeff..1e9fde20 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ [nosetests] verbose=1 nocapture=1 + +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py index 64650659..3f17da8f 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ from setuptools import setup, find_packages -setup(name='instana', +setup(name='stan', version='0.0.1', url='https://github.com/instana/python-sensor', license='MIT', author='Instana Inc.', - author_email='pavlo.baron@instana.com', + author_email='peter.lombardo@instana.com', description='Metrics sensor and trace collector for Instana', packages=find_packages(exclude=['tests', 'examples']), long_description=open('README.md').read(), @@ -15,4 +15,5 @@ 'opentracing>=1.2.1,<1.3', 'basictracer>=2.2.0', 'psutil>=5.1.3'], - test_suite='nose.collector') + test_suite='nose.collector', + keywords=['performance', 'opentracing', 'metrics', 'monitoring']) From d5bc33f7a314e54fac88a8fbff7c6b7a1e8c8b54 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 30 Jun 2017 12:07:09 +0200 Subject: [PATCH 25/29] Expanded span.kind support & tests. --- instana/recorder.py | 12 ++++++++++-- tests/test_ot_span.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/instana/recorder.py b/instana/recorder.py index 52f530d8..20a2d4e0 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -3,7 +3,7 @@ import threading as t import opentracing.ext.tags as ext import socket -import instana.span_data as sd +import instana.span as sd import Queue import time @@ -11,6 +11,8 @@ class InstanaRecorder(SpanRecorder): sensor = None registered_spans = ("memcache", "rpc-client", "rpc-server") + entry_kind = ["entry", "server", "consumer"] + exit_kind = ["exit", "client", "producer"] queue = Queue.Queue() def __init__(self, sensor): @@ -101,7 +103,13 @@ def build_sdk_span(self, span): ) if "span.kind" in span.tags: - sdk_data.Type = span.tags["span.kind"] + if span.tags["span.kind"] in self.entry_kind: + sdk_data.Type = "entry" + elif span.tags["span.kind"] in self.exit_kind: + sdk_data.Type = "exit" + else: + sdk_data.Type = "local" + data = sd.Data(service=self.get_service_name(span), sdk=sdk_data) diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index ff7df7d5..4afd2167 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -85,3 +85,44 @@ def test_sdk_spans(self): assert_equals('entry', sdk_span.data.sdk.Type) assert sdk_span.data.sdk.Custom assert sdk_span.data.sdk.Custom.tags + + def test_span_kind(self): + recorder = opentracing.global_tracer.recorder + + span = opentracing.global_tracer.start_span("custom_sdk_span") + span.set_tag('span.kind', "consumer") + span.finish() + + span = opentracing.global_tracer.start_span("custom_sdk_span") + span.set_tag('span.kind', "server") + span.finish() + + span = opentracing.global_tracer.start_span("custom_sdk_span") + span.set_tag('span.kind', "producer") + span.finish() + + span = opentracing.global_tracer.start_span("custom_sdk_span") + span.set_tag('span.kind', "client") + span.finish() + + span = opentracing.global_tracer.start_span("custom_sdk_span") + span.set_tag('span.kind', "blah") + span.finish() + + spans = recorder.queued_spans() + assert 5, len(spans) + + span = spans[0] + assert_equals('entry', span.data.sdk.Type) + + span = spans[1] + assert_equals('entry', span.data.sdk.Type) + + span = spans[2] + assert_equals('exit', span.data.sdk.Type) + + span = spans[3] + assert_equals('exit', span.data.sdk.Type) + + span = spans[4] + assert_equals('local', span.data.sdk.Type) From 15a5f2497957a5649b9fa48d765f6bc3b01dbec1 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 3 Jul 2017 11:51:20 +0200 Subject: [PATCH 26/29] Allow only one sensor per process - Avoids duplicated, triplicated metric reporting --- instana/tracer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/instana/tracer.py b/instana/tracer.py index 7833692e..6d0ab7ac 100644 --- a/instana/tracer.py +++ b/instana/tracer.py @@ -4,17 +4,28 @@ import opentracing import instana.options as o import instana.sensor as s +import instana.log as ilog from basictracer.context import SpanContext from basictracer.span import BasicSpan from instana.util import generate_id +# In case a user or app creates multiple tracers, we limit to just +# one sensor per process otherwise metrics collection is duplicated, +# triplicated etc. +gSensor = None + class InstanaTracer(BasicTracer): sensor = None def __init__(self, options=o.Options()): - self.sensor = s.Sensor(options) + global gSensor + if gSensor is None: + self.sensor = s.Sensor(options) + gSensor = self.sensor + else: + self.sensor = gSensor super(InstanaTracer, self).__init__( r.InstanaRecorder(self.sensor), r.InstanaSampler()) From 440ec9ed48ceb8d643e234cc4d7b4c869e40d407 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 3 Jul 2017 13:41:39 +0200 Subject: [PATCH 27/29] Use a dedicated metric collection thread. --- instana/meter.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/instana/meter.py b/instana/meter.py index 8031767b..96d33f8a 100644 --- a/instana/meter.py +++ b/instana/meter.py @@ -6,6 +6,7 @@ import sys import instana.agent_const as a import copy +import time class Snapshot(object): @@ -96,13 +97,18 @@ class Meter(object): def __init__(self, sensor): self.sensor = sensor - self.tick() + self.run() - def tick(self): - timer = t.Timer(1, self.process) - timer.daemon = True - timer.name = "Instana Metric Collection" - timer.start() + def run(self): + self.timer = t.Thread(target=self.collect_and_report) + self.timer.daemon = True + self.timer.name = "Instana Metric Collection" + self.timer.start() + + def collect_and_report(self): + while 1: + self.process() + time.sleep(1) def process(self): if self.sensor.agent.can_send(): @@ -121,8 +127,6 @@ def process(self): self.sensor.agent.make_url(a.AGENT_DATA_URL), "POST", d) self.last_metrics = cm.__dict__ - self.tick() - def collect_snapshot(self): try: s = Snapshot(name=self.sensor.service_name, From 396a279b1974d3a4c48cbc4665a04cd1611d557c Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 3 Jul 2017 13:44:50 +0200 Subject: [PATCH 28/29] Lowercase what fields we can for now. --- instana/recorder.py | 5 ++--- instana/span.py | 6 +++--- tests/test_ot_span.py | 5 +++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/instana/recorder.py b/instana/recorder.py index 20a2d4e0..5e038d2f 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -98,8 +98,8 @@ def build_sdk_span(self, span): logs=self.collect_logs(span)) sdk_data = sd.SDKData( - Name=span.operation_name, - Custom=custom_data + name=span.operation_name, + custom=custom_data ) if "span.kind" in span.tags: @@ -110,7 +110,6 @@ def build_sdk_span(self, span): else: sdk_data.Type = "local" - data = sd.Data(service=self.get_service_name(span), sdk=sdk_data) diff --git a/instana/span.py b/instana/span.py index b28a8666..442a1809 100644 --- a/instana/span.py +++ b/instana/span.py @@ -43,11 +43,11 @@ def __init__(self, **kwds): self.__dict__.update(kwds) class SDKData(object): - Name = None + name = None Type = None - Arguments = None + arguments = None Return = None - Custom = None + custom = None def __init__(self, **kwds): self.__dict__.update(kwds) diff --git a/tests/test_ot_span.py b/tests/test_ot_span.py index 4afd2167..170d3b88 100644 --- a/tests/test_ot_span.py +++ b/tests/test_ot_span.py @@ -83,8 +83,9 @@ def test_sdk_spans(self): assert sdk_span.data assert sdk_span.data.sdk assert_equals('entry', sdk_span.data.sdk.Type) - assert sdk_span.data.sdk.Custom - assert sdk_span.data.sdk.Custom.tags + assert_equals('custom_sdk_span', sdk_span.data.sdk.name) + assert sdk_span.data.sdk.custom + assert sdk_span.data.sdk.custom.tags def test_span_kind(self): recorder = opentracing.global_tracer.recorder From d26591988d339ade28241958ae88679b85e2278a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 3 Jul 2017 13:50:31 +0200 Subject: [PATCH 29/29] Rename package to instana --- README.md | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9825bc01..99f244ca 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ -# Stan +# Instana -The stan package provides Python metrics and traces (request, queue & cross-host) for [Instana](https://www.instana.com/). +The instana package provides Python metrics and traces (request, queue & cross-host) for [Instana](https://www.instana.com/). ## Note @@ -14,11 +14,11 @@ Any and all feedback is welcome. Happy Python visibility. ## Installation - $ pip install stan + $ pip install instana ## Usage -The stan package is a zero configuration tool that will automatically collect key metrics from your Python processes. Just install and go. +The instana package is a zero configuration tool that will automatically collect key metrics from your Python processes. Just install and go. ## Tracing diff --git a/setup.py b/setup.py index 3f17da8f..3ac334da 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -setup(name='stan', +setup(name='instana', version='0.0.1', url='https://github.com/instana/python-sensor', license='MIT',