From 42dc067fc6101dbe6de9dcb2d0386d893714e402 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 7 Jul 2020 12:56:30 +0200 Subject: [PATCH 01/91] AWS Fargate Agent, Option & Recorder --- instana/agent/aws_fargate.py | 104 +++++++++++++++++++++++++++++++++++ instana/options.py | 21 +++++++ instana/recorder.py | 21 +++++++ instana/singletons.py | 9 ++- 4 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 instana/agent/aws_fargate.py diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py new file mode 100644 index 00000000..bc553110 --- /dev/null +++ b/instana/agent/aws_fargate.py @@ -0,0 +1,104 @@ +""" +The Instana agent (for AWS Fargate functions) that manages +monitoring state and reporting that data. +""" +import os +import time +from ..log import logger +from ..util import to_json +from .base import BaseAgent +from instana.collector import Collector +from instana.options import AWSFargateOptions + + +class AWSFargateFrom(object): + """ The source identifier for AWSFargateAgent """ + hl = True + cp = "aws" + e = "taskDefinition" + + def __init__(self, **kwds): + self.__dict__.update(kwds) + + +class AWSFargateAgent(BaseAgent): + """ In-process agent for AWS Fargate """ + def __init__(self): + super(AWSFargateAgent, self).__init__() + + self.from_ = AWSFargateFrom() + self.collector = None + self.options = AWSFargateOptions() + self.report_headers = None + self._can_send = False + self.extra_headers = self.options.extra_http_headers + + if self._validate_options(): + self._can_send = True + self.collector = Collector(self) + self.collector.start() + else: + logger.warning("Required INSTANA_AGENT_KEY and/or INSTANA_ENDPOINT_URL environment variables not set. " + "We will not be able monitor this function.") + + def can_send(self): + """ + Are we in a state where we can send data? + @return: Boolean + """ + return self._can_send + + def get_from_structure(self): + """ + Retrieves the From data that is reported alongside monitoring data. + @return: dict() + """ + return {'hl': True, 'cp': 'aws', 'e': self.collector.get_fq_arn()} + + def report_data_payload(self, payload): + """ + Used to report metrics and span data to the endpoint URL in self.options.endpoint_url + """ + response = None + try: + if self.report_headers is None: + # Prepare request headers + self.report_headers = dict() + self.report_headers["Content-Type"] = "application/json" + self.report_headers["X-Instana-Host"] = self.collector.get_fq_arn() + self.report_headers["X-Instana-Key"] = self.options.agent_key + self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) + + # logger.debug("using these headers: %s", self.report_headers) + + if 'INSTANA_DISABLE_CA_CHECK' in os.environ: + ssl_verify = False + else: + ssl_verify = True + + response = self.client.post(self.__data_bundle_url(), + data=to_json(payload), + headers=self.report_headers, + timeout=self.options.timeout, + verify=ssl_verify) + + if 200 <= response.status_code < 300: + logger.debug("report_data_payload: Instana responded with status code %s", response.status_code) + else: + logger.info("report_data_payload: Instana responded with status code %s", response.status_code) + except Exception as e: + logger.debug("report_data_payload: connection error (%s)", type(e)) + finally: + return response + + def _validate_options(self): + """ + Validate that the options used by this Agent are valid. e.g. can we report data? + """ + return self.options.endpoint_url is not None and self.options.agent_key is not None + + def __data_bundle_url(self): + """ + URL for posting metrics to the host agent. Only valid when announced. + """ + return "%s/bundle" % self.options.endpoint_url diff --git a/instana/options.py b/instana/options.py index 467c413c..b3594d30 100644 --- a/instana/options.py +++ b/instana/options.py @@ -63,3 +63,24 @@ def __init__(self, **kwds): self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) + +class AWSFargateOptions(BaseOptions): + endpoint_url = None + agent_key = None + extra_http_headers = None + timeout = None + + def __init__(self, **kwds): + super(AWSFargateOptions, self).__init__() + + self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) + + # Remove any trailing slash (if any) + if self.endpoint_url is not None and self.endpoint_url[-1] == "/": + self.endpoint_url = self.endpoint_url[:-1] + + self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) + self.service_name = os.environ.get("INSTANA_SERVICE_NAME", None) + self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) + self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) + diff --git a/instana/recorder.py b/instana/recorder.py index abc147e6..946d3051 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -134,6 +134,27 @@ def record_span(self, span): self.agent.collector.span_queue.put(json_span) +class AWSFargateRecorder(StandardRecorder): + def __init__(self, agent): + self.agent = agent + super(AWSFargateRecorder, self).__init__() + + def record_span(self, span): + """ + Convert the passed BasicSpan and add it to the span queue + """ + source = self.agent.get_from_structure() + service_name = self.agent.options.service_name + + if span.operation_name in self.REGISTERED_SPANS: + json_span = RegisteredSpan(span, source, service_name) + else: + json_span = SDKSpan(span, source, service_name) + + # logger.debug("Recorded span: %s", json_span) + self.agent.collector.span_queue.put(json_span) + + class InstanaSampler(Sampler): def sampled(self, _): return False diff --git a/instana/singletons.py b/instana/singletons.py index 6b4c4bfc..53091a72 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -16,12 +16,19 @@ agent = TestAgent() span_recorder = StandardRecorder() -elif os.environ.get("INSTANA_ENDPOINT_URL", False): +elif os.environ.get("LAMBDA_HANDLER", False): from .agent.aws_lambda import AWSLambdaAgent from .recorder import AWSLambdaRecorder agent = AWSLambdaAgent() span_recorder = AWSLambdaRecorder(agent) + +elif os.environ.get("FARGATE", False): + from .agent.aws_fargate import AWSFargateAgent + from .recorder import AWSFargateRecorder + + agent = AWSFargateAgent() + span_recorder = AWSFargateRecorder(agent) else: from .agent.host import HostAgent from .recorder import StandardRecorder From b9baf513f1e9acb2c1c03546b899d48576ea0571 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 7 Jul 2020 14:20:16 +0200 Subject: [PATCH 02/91] Break Collector into platform parts --- instana/agent/aws_lambda.py | 4 +-- instana/collector/aws_fargate.py | 25 ++++++++++++++ instana/collector/aws_lambda.py | 38 +++++++++++++++++++++ instana/{collector.py => collector/base.py} | 34 +++--------------- 4 files changed, 69 insertions(+), 32 deletions(-) create mode 100644 instana/collector/aws_fargate.py create mode 100644 instana/collector/aws_lambda.py rename instana/{collector.py => collector/base.py} (72%) diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index cf5c6882..2af1666a 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -7,7 +7,7 @@ from ..log import logger from ..util import to_json from .base import BaseAgent -from instana.collector import Collector +from instana.collector.aws_lambda import AWSLambdaCollector from instana.options import AWSLambdaOptions @@ -35,7 +35,7 @@ def __init__(self): if self._validate_options(): self._can_send = True - self.collector = Collector(self) + self.collector = AWSLambdaCollector(self) self.collector.start() else: logger.warning("Required INSTANA_AGENT_KEY and/or INSTANA_ENDPOINT_URL environment variables not set. " diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py new file mode 100644 index 00000000..c6b6cb45 --- /dev/null +++ b/instana/collector/aws_fargate.py @@ -0,0 +1,25 @@ + +from ..log import logger +from ..util import DictionaryOfStan + + +class AWSFargateCollector(object): + def __init__(self, agent): + super(AWSFargateCollector, self).__init__(agent) + logger.debug("Loading AWS Fargate Collector") + + def collect_snapshot(self, event, context): + self.snapshot_data = DictionaryOfStan() + + self.context = context + self.event = event + + try: + plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.aws.lambda" + self.snapshot_data["plugins"] = [plugin_data] + except: + logger.debug("collect_snapshot error", exc_info=True) + finally: + return self.snapshot_data + diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py new file mode 100644 index 00000000..4c945bde --- /dev/null +++ b/instana/collector/aws_lambda.py @@ -0,0 +1,38 @@ +from ..log import logger +from .base import BaseCollector +from ..util import DictionaryOfStan, normalize_aws_lambda_arn + + +class AWSLambdaCollector(BaseCollector): + def __init__(self, agent): + super(AWSLambdaCollector, self).__init__(agent) + logger.debug("Loading AWS Lambda Collector") + self._fq_arn = None + + def collect_snapshot(self, event, context): + self.snapshot_data = DictionaryOfStan() + + self.context = context + self.event = event + + try: + plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.aws.lambda" + plugin_data["entityId"] = self.get_fq_arn() + self.snapshot_data["plugins"] = [plugin_data] + except: + logger.debug("collect_snapshot error", exc_info=True) + finally: + return self.snapshot_data + + def get_fq_arn(self): + if self._fq_arn is not None: + return self._fq_arn + + if self.context is None: + logger.debug("Attempt to get qualified ARN before the context object is available") + return '' + + self._fq_arn = normalize_aws_lambda_arn(self.context) + return self._fq_arn + diff --git a/instana/collector.py b/instana/collector/base.py similarity index 72% rename from instana/collector.py rename to instana/collector/base.py index df2230d5..e0e83b5c 100644 --- a/instana/collector.py +++ b/instana/collector/base.py @@ -2,8 +2,8 @@ import sys import threading -from .log import logger -from .util import every, DictionaryOfStan, normalize_aws_lambda_arn +from ..log import logger +from ..util import every, DictionaryOfStan if sys.version_info.major == 2: @@ -12,9 +12,8 @@ import queue -class Collector(object): +class BaseCollector(object): def __init__(self, agent): - logger.debug("Loading collector") self.agent = agent self.span_queue = queue.Queue() self.thread_shutdown = threading.Event() @@ -24,7 +23,6 @@ def __init__(self, agent): self.snapshot_data = None self.snapshot_data_sent = False self.lock = threading.Lock() - self._fq_arn = None def start(self): if self.agent.can_send(): @@ -80,31 +78,7 @@ def prepare_and_report_data(self): return True def collect_snapshot(self, event, context): - self.snapshot_data = DictionaryOfStan() - - self.context = context - self.event = event - - try: - plugin_data = dict() - plugin_data["name"] = "com.instana.plugin.aws.lambda" - plugin_data["entityId"] = self.get_fq_arn() - self.snapshot_data["plugins"] = [plugin_data] - except: - logger.debug("collect_snapshot error", exc_info=True) - finally: - return self.snapshot_data - - def get_fq_arn(self): - if self._fq_arn is not None: - return self._fq_arn - - if self.context is None: - logger.debug("Attempt to get qualified ARN before the context object is available") - return '' - - self._fq_arn = normalize_aws_lambda_arn(self.context) - return self._fq_arn + logger.debug("BaseCollector: collect_snapshot needs to be overridden") def __queued_spans(self): """ Get all of the spans in the queue """ From c7748255bfb0a89c2eb107ee49a9ef16dd793be6 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 7 Jul 2020 14:20:41 +0200 Subject: [PATCH 03/91] Initial Fargate tests; AWSFargateAgent/Collector/Options --- tests/platforms/test_fargate.py | 146 ++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/platforms/test_fargate.py diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py new file mode 100644 index 00000000..5593acdd --- /dev/null +++ b/tests/platforms/test_fargate.py @@ -0,0 +1,146 @@ +from __future__ import absolute_import + +import os +import sys +import json +import pytest +import unittest + +from instana.tracer import InstanaTracer +from instana.options import AWSFargateOptions +from instana.recorder import AWSFargateRecorder +from instana.agent.aws_fargate import AWSFargateAgent +from instana.singletons import get_agent, set_agent, get_tracer, set_tracer + + +class TestFargate(unittest.TestCase): + def __init__(self, methodName='runTest'): + super(TestFargate, self).__init__(methodName) + self.agent = None + self.span_recorder = None + self.tracer = None + self.pwd = os.path.dirname(os.path.realpath(__file__)) + + self.original_agent = get_agent() + self.original_tracer = get_tracer() + + def setUp(self): + os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" + os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" + + def tearDown(self): + """ Reset all environment variables of consequence """ + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: + os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") + if "INSTANA_ENDPOINT_URL" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_AGENT_KEY" in os.environ: + os.environ.pop("INSTANA_AGENT_KEY") + + set_agent(self.original_agent) + set_tracer(self.original_tracer) + + def create_agent_and_setup_tracer(self): + self.agent = AWSFargateAgent() + self.span_recorder = AWSFargateRecorder(self.agent) + self.tracer = InstanaTracer(recorder=self.span_recorder) + set_agent(self.agent) + set_tracer(self.tracer) + + def test_invalid_options(self): + # None of the required env vars are available... + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: + os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") + if "INSTANA_ENDPOINT_URL" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_AGENT_KEY" in os.environ: + os.environ.pop("INSTANA_AGENT_KEY") + + agent = AWSFargateAgent() + self.assertFalse(agent._can_send) + self.assertIsNone(agent.collector) + + def test_secrets(self): + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent, 'secrets_matcher')) + self.assertEqual(self.agent.secrets_matcher, 'contains-ignore-case') + self.assertTrue(hasattr(self.agent, 'secrets_list')) + self.assertEqual(self.agent.secrets_list, ['key', 'pass', 'secret']) + + def test_has_options(self): + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent, 'options')) + self.assertTrue(type(self.agent.options) is AWSFargateOptions) + + def test_has_extra_headers(self): + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent, 'extra_headers')) + + def test_agent_extra_headers(self): + os.environ['INSTANA_EXTRA_HTTP_HEADERS'] = "X-Test-Header;X-Another-Header;X-And-Another-Header" + self.create_agent_and_setup_tracer() + self.assertIsNotNone(self.agent.extra_headers) + should_headers = ['x-test-header', 'x-another-header', 'x-and-another-header'] + self.assertEqual(should_headers, self.agent.extra_headers) + + @pytest.mark.skip("todo") + def test_custom_service_name(self): + os.environ['INSTANA_SERVICE_NAME'] = "Legion" + with open(self.pwd + '/../data/lambda/api_gateway_event.json', 'r') as json_file: + event = json.load(json_file) + + self.create_agent_and_setup_tracer() + + # Call the Instana Fargate Handler as we do in the real world. It will initiate tracing and then + # figure out the original (the users') Fargate Handler and execute it. + # The original Fargate handler is set in os.environ["LAMBDA_HANDLER"] + # result = lambda_handler(event, self.context) + os.environ.pop('INSTANA_SERVICE_NAME') + + # self.assertEqual('All Ok', result) + payload = self.agent.collector.prepare_payload() + + self.assertTrue("metrics" in payload) + self.assertTrue("spans" in payload) + self.assertEqual(2, len(payload.keys())) + + self.assertTrue(type(payload['metrics']['plugins']) is list) + self.assertTrue(len(payload['metrics']['plugins']) == 1) + plugin_data = payload['metrics']['plugins'][0] + + self.assertEqual('com.instana.plugin.aws.lambda', plugin_data['name']) + self.assertEqual('arn:aws:lambda:us-east-2:12345:function:TestPython:1', plugin_data['entityId']) + + self.assertEqual(1, len(payload['spans'])) + + span = payload['spans'][0] + self.assertEqual('aws.lambda.entry', span.n) + self.assertIsNotNone(span.t) + self.assertIsNotNone(span.s) + self.assertIsNone(span.p) + self.assertIsNotNone(span.ts) + self.assertIsNotNone(span.d) + + self.assertEqual({'hl': True, 'cp': 'aws', 'e': 'arn:aws:lambda:us-east-2:12345:function:TestPython:1'}, + span.f) + + self.assertIsNone(span.ec) + self.assertIsNone(span.data['lambda']['error']) + + self.assertEqual('arn:aws:lambda:us-east-2:12345:function:TestPython:1', span.data['lambda']['arn']) + self.assertEqual(None, span.data['lambda']['alias']) + self.assertEqual('python', span.data['lambda']['runtime']) + self.assertEqual('TestPython', span.data['lambda']['functionName']) + self.assertEqual('1', span.data['lambda']['functionVersion']) + + self.assertEqual('Legion', span.data['service']) + + self.assertEqual('aws:api.gateway', span.data['lambda']['trigger']) + self.assertEqual('POST', span.data['http']['method']) + self.assertEqual('/path/to/resource', span.data['http']['url']) + self.assertEqual('/{proxy+}', span.data['http']['path_tpl']) + if sys.version[:3] == '2.7': + self.assertEqual(u"foo=[u'bar']", span.data['http']['params']) + else: + self.assertEqual("foo=['bar']", span.data['http']['params']) + From e3d25bc5a0080edc1bb9d6bf58e3be5705285ade Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 7 Jul 2020 14:32:03 +0200 Subject: [PATCH 04/91] Fix Fargate Collector instantiation --- instana/agent/aws_fargate.py | 6 +++--- instana/collector/aws_fargate.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index bc553110..531b08a6 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -1,5 +1,5 @@ """ -The Instana agent (for AWS Fargate functions) that manages +The Instana agent (for AWS Fargate) that manages monitoring state and reporting that data. """ import os @@ -7,7 +7,7 @@ from ..log import logger from ..util import to_json from .base import BaseAgent -from instana.collector import Collector +from instana.collector.aws_fargate import AWSFargateCollector from instana.options import AWSFargateOptions @@ -35,7 +35,7 @@ def __init__(self): if self._validate_options(): self._can_send = True - self.collector = Collector(self) + self.collector = AWSFargateCollector(self) self.collector.start() else: logger.warning("Required INSTANA_AGENT_KEY and/or INSTANA_ENDPOINT_URL environment variables not set. " diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index c6b6cb45..9a5875dd 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,9 +1,10 @@ from ..log import logger +from .base import BaseCollector from ..util import DictionaryOfStan -class AWSFargateCollector(object): +class AWSFargateCollector(BaseCollector): def __init__(self, agent): super(AWSFargateCollector, self).__init__(agent) logger.debug("Loading AWS Fargate Collector") From 2dc85361a635788ae5d669a72b30cfa98602b553 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 9 Jul 2020 15:30:59 +0200 Subject: [PATCH 05/91] Include package file --- instana/collector/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 instana/collector/__init__.py diff --git a/instana/collector/__init__.py b/instana/collector/__init__.py new file mode 100644 index 00000000..e69de29b From d603b63fec136c790b9cd82a9c5652f8496ab492 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 10 Jul 2020 14:47:40 +0200 Subject: [PATCH 06/91] Fargate snapshot collection for Task, Containers, Docker & Process --- instana/agent/aws_fargate.py | 5 + instana/collector/aws_fargate.py | 195 +++++++++++++++++++++++++++++-- instana/collector/aws_lambda.py | 14 +++ instana/collector/base.py | 15 +-- 4 files changed, 207 insertions(+), 22 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 531b08a6..b0f4f6a1 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -33,6 +33,11 @@ def __init__(self): self._can_send = False self.extra_headers = self.options.extra_http_headers + if "ECS_CONTAINER_METADATA_URI" not in os.environ: + logger.debug("AWSFargateAgent: ECS_CONTAINER_METADATA_URI not in environment. This won't work.") + + self.ecmu = os.environ.get("ECS_CONTAINER_METADATA_URI", None) + if self._validate_options(): self._can_send = True self.collector = AWSFargateCollector(self) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 9a5875dd..a8724cda 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,7 +1,8 @@ - +import os +import getpass from ..log import logger from .base import BaseCollector -from ..util import DictionaryOfStan +from ..util import DictionaryOfStan, get_proc_cmdline class AWSFargateCollector(BaseCollector): @@ -9,18 +10,194 @@ def __init__(self, agent): super(AWSFargateCollector, self).__init__(agent) logger.debug("Loading AWS Fargate Collector") - def collect_snapshot(self, event, context): - self.snapshot_data = DictionaryOfStan() + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/ + self.root_metadata = None + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/task + self.task_metadata = None + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/stats + self.stats_metadata = None - self.context = context - self.event = event + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/task/stats + self.task_stats_metadata = None + + def prepare_payload(self): + pass + + def collect_snapshot(self, *argv, **kwargs): + plugins = [] + self.snapshot_data = DictionaryOfStan() try: - plugin_data = dict() - plugin_data["name"] = "com.instana.plugin.aws.lambda" - self.snapshot_data["plugins"] = [plugin_data] + plugins.extend(self._collect_task_snapshot()) + plugins.extend(self._collect_container_snapshots()) + plugins.extend(self._collect_docker_snapshot()) + plugins.extend(self._collect_process_snapshot()) + plugins.extend(self._collect_runtime_snapshot()) + self.snapshot_data["plugins"] = plugins except: logger.debug("collect_snapshot error", exc_info=True) finally: return self.snapshot_data + def _collect_task_snapshot(self): + """ + Collect and return snapshot data for the task + @return: list - with one plugin entity + """ + plugin_data = dict() + try: + plugin_data["name"] = "com.instana.plugin.aws.ecs.task" + plugin_data["entityId"] = "metadata.TaskARN" # FIXME + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["taskArn"] = self.task_metadata.get("TaskARN", None) + plugin_data["data"]["clusterArn"] = self.task_metadata.get("Cluster", None) + plugin_data["data"]["taskDefinition"] = self.task_metadata.get("Family", None) + plugin_data["data"]["taskDefinitionVersion"] = self.task_metadata.get("Revision", None) + plugin_data["data"]["availabilityZone"] = self.task_metadata.get("AvailabilityZone", None) + plugin_data["data"]["desiredStatus"] = self.task_metadata.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = self.task_metadata.get("KnownStatus", None) + plugin_data["data"]["pullStartedAt"] = self.task_metadata.get("PullStartedAt", None) + plugin_data["data"]["pullStoppedAt"] = self.task_metadata.get("PullStoppeddAt", None) + limits = self.task_metadata.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + except: + logger.debug("_collect_task_snapshot: ", exc_info=True) + return [plugin_data] + + def _collect_container_snapshots(self): + """ + Collect and return snapshot data for every container in this task + @return: list - with one or more plugin entities + """ + plugins = [] + try: + containers = self.task_metadata.get("Containers", []) + for container in containers: + plugin_data = dict() + try: + labels = container.get("Labels", {}) + name = container.get("Name", "") + taskArn = labels.get("com.amazonaws.ecs.container-name", "") + + plugin_data["name"] = "com.instana.plugin.aws.ecs.container" + # "entityId": $taskARN + "::" + $containerName + plugin_data["entityId"] = "%s::%s" % (taskArn, name) + + plugin_data["data"] = DictionaryOfStan() + if self.root_metadata["Name"] == name: + plugin_data["data"]["instrumented"] = True + plugin_data["data"]["runtime"] = "python" + plugin_data["data"]["dockerId"] = container.get("DockerId", None) + plugin_data["data"]["dockerName"] = container.get("DockerName", None) + plugin_data["data"]["containerName"] = container.get("Name", None) + plugin_data["data"]["image"] = container.get("Image", None) + plugin_data["data"]["imageId"] = container.get("ImageID", None) + plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) + plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) + plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) + plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) + plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) + plugin_data["data"]["ports"] = container.get("Ports", None) + plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) + plugin_data["data"]["startedAt"] = container.get("StartedAt", None) + plugin_data["data"]["type"] = container.get("Type", None) + limits = container.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + finally: + plugins.append(plugin_data) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + return plugins + + def _collect_docker_snapshot(self): + plugins = [] + try: + containers = self.task_metadata.get("Containers", []) + for container in containers: + plugin_data = dict() + try: + labels = container.get("Labels", {}) + name = container.get("Name", "") + taskArn = labels.get("com.amazonaws.ecs.container-name", "") + + plugin_data["name"] = "com.instana.plugin.docker" + # "entityId": $taskARN + "::" + $containerName + plugin_data["entityId"] = "%s::%s" % (taskArn, name) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["Id"] = container.get("DockerId", None) + plugin_data["data"]["Created"] = container.get("CreatedAt", None) + plugin_data["data"]["Started"] = container.get("StartedAt", None) + plugin_data["data"]["Image"] = container.get("Image", None) + plugin_data["data"]["Labels"] = container.get("Labels", None) + plugin_data["data"]["Ports"] = container.get("Ports", None) + + networks = container.get("Networks", []) + if len(networks) >= 1: + plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + finally: + plugins.append(plugin_data) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + return plugins + + def _collect_process_snapshot(self): + plugin_data = dict() + try: + plugin_data["name"] = "com.instana.plugin.process" + plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["pid"] = int(os.getpid()) + env = dict() + for key in os.environ: + env[key] = os.environ[key] + plugin_data["data"]["env"] = env + plugin_data["data"]["exec"] = os.readlink("/proc/self/exe") + + cmdline = get_proc_cmdline() + if len(cmdline) > 1: + # drop the exe + cmdline.pop(0) + plugin_data["data"]["args"] = cmdline + plugin_data["data"]["user"] = getpass.getuser() + try: + plugin_data["data"]["group"] = getpass.getuser(os.getegid()).gr_name + except: + logger.debug("getpass.getuser: ", exc_info=True) + + plugin_data["data"]["start"] = 1 # ¯\_(ツ)_/¯ FIXME + plugin_data["data"]["containerType"] = "docker" + plugin_data["data"]["container"] = self.root_metadata.get("DockerId") + plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # ¯\_(ツ)_/¯ FIXME + plugin_data["data"]["com.instana.plugin.host.name"] = self.task_metadata.get("TaskArn") + + + except: + logger.debug("_collect_process_snapshot: ", exc_info=True) + return plugin_data + + def _collect_runtime_snapshot(self): + plugin_data = dict() + lock_acquired = self.process_metadata_mutex.acquire(False) + if lock_acquired: + try: + plugin_data["name"] = "com.instana.plugin.python" + plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"] = DictionaryOfStan() + except: + logger.debug("_collect_runtime_snapshot: ", exc_info=True) + finally: + self.process_metadata_mutex.release() + return plugin_data diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index 4c945bde..c4274319 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -25,6 +25,20 @@ def collect_snapshot(self, event, context): finally: return self.snapshot_data + def prepare_payload(self): + payload = DictionaryOfStan() + payload["spans"] = None + payload["metrics"] = None + + if not self.span_queue.empty(): + payload["spans"] = self.__queued_spans() + + if self.snapshot_data and self.snapshot_data_sent is False: + payload["metrics"] = self.snapshot_data + self.snapshot_data_sent = True + + return payload + def get_fq_arn(self): if self._fq_arn is not None: return self._fq_arn diff --git a/instana/collector/base.py b/instana/collector/base.py index e0e83b5c..ac59bdad 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -47,18 +47,7 @@ def background_report(self): return self.prepare_and_report_data() def prepare_payload(self): - payload = DictionaryOfStan() - payload["spans"] = None - payload["metrics"] = None - - if not self.span_queue.empty(): - payload["spans"] = self.__queued_spans() - - if self.snapshot_data and self.snapshot_data_sent is False: - payload["metrics"] = self.snapshot_data - self.snapshot_data_sent = True - - return payload + logger.debug("BaseCollector: prepare_payload needs to be overridden") def prepare_and_report_data(self): if "INSTANA_TEST" in os.environ: @@ -77,7 +66,7 @@ def prepare_and_report_data(self): logger.debug("prepare_and_report_data: Couldn't acquire lock") return True - def collect_snapshot(self, event, context): + def collect_snapshot(self, *argv, **kwargs): logger.debug("BaseCollector: collect_snapshot needs to be overridden") def __queued_spans(self): From ebfe5af8619844d11208cc0eb9da76c53b5bcf80 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 20 Jul 2020 19:16:57 +0200 Subject: [PATCH 07/91] Centralized environment detection --- instana/agent/aws_fargate.py | 2 +- instana/instrumentation/aws/lambda_inst.py | 9 ++++----- instana/log.py | 7 ++++--- instana/singletons.py | 13 ++++++++++--- tests/platforms/test_fargate.py | 3 +++ tests/platforms/test_lambda.py | 3 +++ 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index b0f4f6a1..5980b19a 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -44,7 +44,7 @@ def __init__(self): self.collector.start() else: logger.warning("Required INSTANA_AGENT_KEY and/or INSTANA_ENDPOINT_URL environment variables not set. " - "We will not be able monitor this function.") + "We will not be able monitor this AWS Fargate cluster.") def can_send(self): """ diff --git a/instana/instrumentation/aws/lambda_inst.py b/instana/instrumentation/aws/lambda_inst.py index 0342d393..54f68499 100644 --- a/instana/instrumentation/aws/lambda_inst.py +++ b/instana/instrumentation/aws/lambda_inst.py @@ -1,15 +1,14 @@ """ Instrumentation for AWS Lambda functions """ -import os import sys import wrapt -from .triggers import enrich_lambda_span, get_context - from ...log import logger -from ...singletons import get_agent, get_tracer +from ...singletons import env_is_aws_lambda from ... import get_lambda_handler_or_default +from ...singletons import get_agent, get_tracer +from .triggers import enrich_lambda_span, get_context def lambda_handler_with_instana(wrapped, instance, args, kwargs): @@ -34,7 +33,7 @@ def lambda_handler_with_instana(wrapped, instance, args, kwargs): return result -if os.environ.get("INSTANA_ENDPOINT_URL", False): +if env_is_aws_lambda: handler_module, handler_function = get_lambda_handler_or_default() if handler_module is not None and handler_function is not None: diff --git a/instana/log.py b/instana/log.py index 5c262d4d..31482805 100644 --- a/instana/log.py +++ b/instana/log.py @@ -1,7 +1,8 @@ -from __future__ import print_function -import logging import os import sys +import logging + +from .singletons import env_is_aws_lambda logger = None @@ -86,7 +87,7 @@ def running_in_gunicorn(): if running_in_gunicorn(): logger = logging.getLogger("gunicorn.error") -elif os.environ.get("INSTANA_ENDPOINT_URL", False): +elif env_is_aws_lambda is True: logger = get_aws_lambda_logger() else: logger = get_standard_logger() diff --git a/instana/singletons.py b/instana/singletons.py index 53091a72..a6e5768a 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -9,26 +9,33 @@ tracer = None span_recorder = None -if os.environ.get("INSTANA_TEST", False): +# Detect the environment where we are running ahead of time +aws_env = os.environ.get("AWS_EXECUTION_ENV", "") +env_is_test = os.environ.get("INSTANA_TEST", False) +env_is_aws_fargate = aws_env == "AWS_ECS_FARGATE" +env_is_aws_lambda = "AWS_Lambda_" in aws_env + +if env_is_test: from .agent.test import TestAgent from .recorder import StandardRecorder agent = TestAgent() span_recorder = StandardRecorder() -elif os.environ.get("LAMBDA_HANDLER", False): +elif env_is_aws_lambda: from .agent.aws_lambda import AWSLambdaAgent from .recorder import AWSLambdaRecorder agent = AWSLambdaAgent() span_recorder = AWSLambdaRecorder(agent) -elif os.environ.get("FARGATE", False): +elif env_is_aws_fargate: from .agent.aws_fargate import AWSFargateAgent from .recorder import AWSFargateRecorder agent = AWSFargateAgent() span_recorder = AWSFargateRecorder(agent) + else: from .agent.host import HostAgent from .recorder import StandardRecorder diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index 5593acdd..a2ec211d 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -25,11 +25,14 @@ def __init__(self, methodName='runTest'): self.original_tracer = get_tracer() def setUp(self): + os.environ["AWS_EXECUTION_ENV"] = "AWS_ECS_FARGATE" os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" def tearDown(self): """ Reset all environment variables of consequence """ + if "AWS_EXECUTION_ENV" in os.environ: + os.environ.pop("AWS_EXECUTION_ENV") if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") if "INSTANA_ENDPOINT_URL" in os.environ: diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index 166b9b68..086a366f 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -50,6 +50,7 @@ def __init__(self, methodName='runTest'): self.original_tracer = get_tracer() def setUp(self): + os.environ["AWS_EXECUTION_ENV"] = "AWS_Lambda_python_3.8" os.environ["LAMBDA_HANDLER"] = "tests.platforms.test_lambda.my_lambda_handler" os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" @@ -57,6 +58,8 @@ def setUp(self): def tearDown(self): """ Reset all environment variables of consequence """ + if "AWS_EXECUTION_ENV" in os.environ: + os.environ.pop("AWS_EXECUTION_ENV") if "LAMBDA_HANDLER" in os.environ: os.environ.pop("LAMBDA_HANDLER") if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: From 34a70a42b275957a014ed6fdf56bd9654dd58014 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 20 Jul 2020 19:19:34 +0200 Subject: [PATCH 08/91] Manual check to avoid circular dependency --- instana/log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/instana/log.py b/instana/log.py index 31482805..46cbea34 100644 --- a/instana/log.py +++ b/instana/log.py @@ -2,8 +2,6 @@ import sys import logging -from .singletons import env_is_aws_lambda - logger = None @@ -85,6 +83,9 @@ def running_in_gunicorn(): return False +aws_env = os.environ.get("AWS_EXECUTION_ENV", "") +env_is_aws_lambda = "AWS_Lambda_" in aws_env + if running_in_gunicorn(): logger = logging.getLogger("gunicorn.error") elif env_is_aws_lambda is True: From 31e4abf5acc1d9c875f9f38012b09c5464dc26dc Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 20 Jul 2020 19:33:49 +0200 Subject: [PATCH 09/91] Restore print_function --- instana/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/instana/log.py b/instana/log.py index 46cbea34..71aeba48 100644 --- a/instana/log.py +++ b/instana/log.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import logging From 2629d54c3042bcf5928d065491106c1739a642c8 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 23 Jul 2020 11:06:11 +0200 Subject: [PATCH 10/91] Better background thread; docs & tests --- instana/agent/aws_fargate.py | 5 -- instana/collector/aws_fargate.py | 109 ++++++++++++++++++++++++++++++- instana/collector/aws_lambda.py | 7 +- instana/collector/base.py | 73 ++++++++++++++++++--- instana/util.py | 18 +++++ tests/test_utils.py | 24 +++++++ 6 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 tests/test_utils.py diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 5980b19a..38b28dac 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -33,11 +33,6 @@ def __init__(self): self._can_send = False self.extra_headers = self.options.extra_http_headers - if "ECS_CONTAINER_METADATA_URI" not in os.environ: - logger.debug("AWSFargateAgent: ECS_CONTAINER_METADATA_URI not in environment. This won't work.") - - self.ecmu = os.environ.get("ECS_CONTAINER_METADATA_URI", None) - if self._validate_options(): self._can_send = True self.collector = AWSFargateCollector(self) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index a8724cda..ec64fb47 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,8 +1,11 @@ import os import getpass +import requests +import threading +from time import time from ..log import logger from .base import BaseCollector -from ..util import DictionaryOfStan, get_proc_cmdline +from ..util import DictionaryOfStan, get_proc_cmdline, every, validate_url class AWSFargateCollector(BaseCollector): @@ -10,6 +13,36 @@ def __init__(self, agent): super(AWSFargateCollector, self).__init__(agent) logger.debug("Loading AWS Fargate Collector") + # Indicates if this Collector has all requirements to run successfully + self.ready_to_start = True + + # Prepare the URLS that we will collect data from + self.ecmu = os.environ.get("ECS_CONTAINER_METADATA_URI", "") + + if self.ecmu == "" or validate_url(self.ecmu) is False: + logger.warn("AWSFargateCollector: ECS_CONTAINER_METADATA_URI not in environment or invalid URL. " + "Instana will not be able to monitor this environment") + self.ready_to_start = False + + self.ecmu_url_root = self.ecmu + '/' + self.ecmu_url_task = self.ecmu + '/task' + self.ecmu_url_stats = self.ecmu + '/stats' + self.ecmu_url_task_stats = self.ecmu + '/task/stats' + + # Lock used synchronize data collection + self.ecmu_lock = threading.Lock() + + # How often to report data + self.report_interval = 1 + self.http_client = requests.Session() + + # Saved snapshot data + self.snapshot_data = None + # Timestamp in seconds of the last time we sent snapshot data + self.snapshot_data_last_sent = 0 + # How often to report snapshot data (in seconds) + self.snapshot_data_interval = 600 + # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/ self.root_metadata = None @@ -26,8 +59,80 @@ def __init__(self, agent): # ${ECS_CONTAINER_METADATA_URI}/task/stats self.task_stats_metadata = None + def start(self): + if self.ready_to_start is False: + logger.warn("AWS Fargate Collector is missing requirements and cannot monitor this environment.") + return + + # Launch a thread here to periodically collect data from the ECS Metadata Container API + if self.agent.can_send(): + logger.debug("AWSFargateCollector.start: launching ecs metadata collection thread") + self.ecs_metadata_thread = threading.Thread(target=self.metadata_thread_loop, args=()) + self.ecs_metadata_thread.setDaemon(True) + self.ecs_metadata_thread.start() + else: + logger.warning("Collector started but the agent tells us we can't send anything out.") + + super(AWSFargateCollector, self).start() + + def metadata_thread_loop(self): + """ + Just a loop that is run in the background thread. + @return: None + """ + every(self.report_interval, self.get_ecs_metadata, "AWSFargateCollector: metadata_thread_loop") + + def get_ecs_metadata(self): + """ + Get the latest data from the ECS metadata container API and store on the class + @return: Boolean + """ + lock_acquired = self.ecmu_lock.acquire(False) + if lock_acquired: + try: + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/ + self.root_metadata = self.http_client.get(self.ecmu_url_root, timeout=3).content + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/task + self.task_metadata = self.http_client.get(self.ecmu_url_task, timeout=3).content + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/stats + self.stats_metadata = self.http_client.get(self.ecmu_url_stats, timeout=3).content + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/task/stats + self.task_stats_metadata = self.http_client.get(self.ecmu_url_task_stats, timeout=3).content + except Exception: + logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) + finally: + self.ecmu_lock.release() + else: + logger.debug("AWSFargateCollector.get_ecs_metadata: skipping because data collection already in progress") + + def should_send_snapshot_data(self): + delta = int(time()) - self.snapshot_data_last_sent + if delta > self.snapshot_data_interval: + return True + return False + def prepare_payload(self): - pass + payload = DictionaryOfStan() + payload["spans"] = None + payload["metrics"] = None + + if not self.span_queue.empty(): + payload["spans"] = self.__queued_spans() + + if self.should_send_snapshot_data(): + if self.snapshot_data is None: + self.snapshot_data = self.collect_snapshot() + payload["metrics"] = self.snapshot_data + self.snapshot_data_last_sent = int(time()) + + return payload def collect_snapshot(self, *argv, **kwargs): plugins = [] diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index c4274319..3451827d 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -7,6 +7,8 @@ class AWSLambdaCollector(BaseCollector): def __init__(self, agent): super(AWSLambdaCollector, self).__init__(agent) logger.debug("Loading AWS Lambda Collector") + self.context = None + self.event = None self._fq_arn = None def collect_snapshot(self, event, context): @@ -25,6 +27,9 @@ def collect_snapshot(self, event, context): finally: return self.snapshot_data + def should_send_snapshot_data(self): + return self.snapshot_data and self.snapshot_data_sent is False + def prepare_payload(self): payload = DictionaryOfStan() payload["spans"] = None @@ -33,7 +38,7 @@ def prepare_payload(self): if not self.span_queue.empty(): payload["spans"] = self.__queued_spans() - if self.snapshot_data and self.snapshot_data_sent is False: + if self.should_send_snapshot_data(): payload["metrics"] = self.snapshot_data self.snapshot_data_sent = True diff --git a/instana/collector/base.py b/instana/collector/base.py index ac59bdad..bb619ed9 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -1,3 +1,7 @@ +""" +A Collector launches a background thread and continually collects & reports data. The data +can be any combination of metrics, snapshot data and spans. +""" import os import sys import threading @@ -13,43 +17,94 @@ class BaseCollector(object): + """ + Base class to handle the collection & reporting of snapshot and metric data + This class launches a background thread to do this work. + """ def __init__(self, agent): + # The agent for this process. Can be Standard, AWSLambda or Fargate self.agent = agent + + # The Queue where we store finished spans before they are sent self.span_queue = queue.Queue() + + # The background thread that reports data in a loop every self.REPORT_INTERVAL seconds + self.reporting_thread = None + + # Signal for background thread(s) to shutdown self.thread_shutdown = threading.Event() self.thread_shutdown.clear() - self.context = None - self.event = None + + # The collected Snapshot Data and info about when it was sent self.snapshot_data = None self.snapshot_data_sent = False + + # Lock used syncronize reporting - no updates when sending self.lock = threading.Lock() + # Reporting interval for the background thread(s) + self.report_interval = 5 + def start(self): + """ + Starts the collector and starts reporting as long as the agent is in a ready state. + @return: None + """ if self.agent.can_send(): - t = threading.Thread(target=self.thread_loop, args=()) - t.setDaemon(True) - t.start() + logger.debug("BaseCollector.start: launching collection thread") + self.reporting_thread = threading.Thread(target=self.thread_loop, args=()) + self.reporting_thread.setDaemon(True) + self.reporting_thread.start() else: logger.warning("Collector started but the agent tells us we can't send anything out.") def shutdown(self): + """ + Shuts down the collector and reports any final data. + @return: None + """ logger.debug("Collector.shutdown: Reporting final data.") self.thread_shutdown.set() self.prepare_and_report_data() def thread_loop(self): - every(5, self.background_report, "Instana Collector: prepare_and_report_data") + """ + Just a loop that is run in the background thread. + @return: None + """ + every(self.report_interval, self.background_report, "Instana Collector: prepare_and_report_data") def background_report(self): + """ + The main work-horse method to report data in the background thread. + @return: Boolean + """ if self.thread_shutdown.is_set(): logger.debug("Thread shutdown signal is active: Shutting down reporting thread") return False return self.prepare_and_report_data() + def should_send_snapshot_data(self): + """ + Determines if snapshot data should be sent + @return: Boolean + """ + logger.debug("BaseCollector: should_send_snapshot_data needs to be overridden") + return False + def prepare_payload(self): + """ + Method to prepare the data to be reported. + @return: DictionaryOfStan() + """ logger.debug("BaseCollector: prepare_payload needs to be overridden") + return DictionaryOfStan() def prepare_and_report_data(self): + """ + Prepare and report the data payload. + @return: Boolean + """ if "INSTANA_TEST" in os.environ: return True @@ -70,8 +125,10 @@ def collect_snapshot(self, *argv, **kwargs): logger.debug("BaseCollector: collect_snapshot needs to be overridden") def __queued_spans(self): - """ Get all of the spans in the queue """ - span = None + """ + Get all of the queued spans + @return: list + """ spans = [] while True: try: diff --git a/instana/util.py b/instana/util.py index 628c5734..1d735a5a 100644 --- a/instana/util.py +++ b/instana/util.py @@ -401,3 +401,21 @@ def normalize_aws_lambda_arn(context): except: logger.debug("normalize_arn: ", exc_info=True) + +def validate_url(url): + """ + Validate if is a valid url + + Examples: + - "http://localhost:5000" - valid + - "http://localhost:5000/path" - valid + - "sandwich" - invalid + + @param url: string + @return: Boolean + """ + try: + result = parse.urlparse(url) + return all([result.scheme, result.netloc]) + except: + return False \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..6c5539cd --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,24 @@ +from __future__ import absolute_import + +from instana.util import validate_url + + +def setup_method(): + pass + + +def test_validate_url(): + assert(validate_url("http://localhost:3000")) + assert(validate_url("http://localhost:3000/")) + assert(validate_url("https://localhost:3000/path/item")) + assert(validate_url("http://localhost")) + assert(validate_url("https://localhost/")) + assert(validate_url("https://localhost/path/item")) + assert(validate_url("http://127.0.0.1")) + assert(validate_url("https://10.0.12.221/")) + assert(validate_url("http://[2001:db8:85a3:8d3:1319:8a2e:370:7348]/")) + assert(validate_url("https://[2001:db8:85a3:8d3:1319:8a2e:370:7348]:443/")) + assert(validate_url("boligrafo") is False) + assert(validate_url("http:boligrafo") is False) + assert(validate_url(None) is False) + From 1b1b24695fe78c89e11f99a32f8633b3a97939bc Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 08:56:24 +0000 Subject: [PATCH 11/91] Check existence before referencing --- instana/collector/aws_fargate.py | 161 ++++++++++++++++--------------- 1 file changed, 82 insertions(+), 79 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index ec64fb47..5522583b 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -157,21 +157,22 @@ def _collect_task_snapshot(self): """ plugin_data = dict() try: - plugin_data["name"] = "com.instana.plugin.aws.ecs.task" - plugin_data["entityId"] = "metadata.TaskARN" # FIXME - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["taskArn"] = self.task_metadata.get("TaskARN", None) - plugin_data["data"]["clusterArn"] = self.task_metadata.get("Cluster", None) - plugin_data["data"]["taskDefinition"] = self.task_metadata.get("Family", None) - plugin_data["data"]["taskDefinitionVersion"] = self.task_metadata.get("Revision", None) - plugin_data["data"]["availabilityZone"] = self.task_metadata.get("AvailabilityZone", None) - plugin_data["data"]["desiredStatus"] = self.task_metadata.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = self.task_metadata.get("KnownStatus", None) - plugin_data["data"]["pullStartedAt"] = self.task_metadata.get("PullStartedAt", None) - plugin_data["data"]["pullStoppedAt"] = self.task_metadata.get("PullStoppeddAt", None) - limits = self.task_metadata.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + if self.task_metadata is not None: + plugin_data["name"] = "com.instana.plugin.aws.ecs.task" + plugin_data["entityId"] = "metadata.TaskARN" # FIXME + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["taskArn"] = self.task_metadata.get("TaskARN", None) + plugin_data["data"]["clusterArn"] = self.task_metadata.get("Cluster", None) + plugin_data["data"]["taskDefinition"] = self.task_metadata.get("Family", None) + plugin_data["data"]["taskDefinitionVersion"] = self.task_metadata.get("Revision", None) + plugin_data["data"]["availabilityZone"] = self.task_metadata.get("AvailabilityZone", None) + plugin_data["data"]["desiredStatus"] = self.task_metadata.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = self.task_metadata.get("KnownStatus", None) + plugin_data["data"]["pullStartedAt"] = self.task_metadata.get("PullStartedAt", None) + plugin_data["data"]["pullStoppedAt"] = self.task_metadata.get("PullStoppeddAt", None) + limits = self.task_metadata.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) except: logger.debug("_collect_task_snapshot: ", exc_info=True) return [plugin_data] @@ -183,44 +184,45 @@ def _collect_container_snapshots(self): """ plugins = [] try: - containers = self.task_metadata.get("Containers", []) - for container in containers: - plugin_data = dict() - try: - labels = container.get("Labels", {}) - name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.container-name", "") - - plugin_data["name"] = "com.instana.plugin.aws.ecs.container" - # "entityId": $taskARN + "::" + $containerName - plugin_data["entityId"] = "%s::%s" % (taskArn, name) - - plugin_data["data"] = DictionaryOfStan() - if self.root_metadata["Name"] == name: - plugin_data["data"]["instrumented"] = True - plugin_data["data"]["runtime"] = "python" - plugin_data["data"]["dockerId"] = container.get("DockerId", None) - plugin_data["data"]["dockerName"] = container.get("DockerName", None) - plugin_data["data"]["containerName"] = container.get("Name", None) - plugin_data["data"]["image"] = container.get("Image", None) - plugin_data["data"]["imageId"] = container.get("ImageID", None) - plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) - plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) - plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) - plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) - plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) - plugin_data["data"]["ports"] = container.get("Ports", None) - plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) - plugin_data["data"]["startedAt"] = container.get("StartedAt", None) - plugin_data["data"]["type"] = container.get("Type", None) - limits = container.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - finally: - plugins.append(plugin_data) + if self.task_metadata is not None: + containers = self.task_metadata.get("Containers", []) + for container in containers: + plugin_data = dict() + try: + labels = container.get("Labels", {}) + name = container.get("Name", "") + taskArn = labels.get("com.amazonaws.ecs.container-name", "") + + plugin_data["name"] = "com.instana.plugin.aws.ecs.container" + # "entityId": $taskARN + "::" + $containerName + plugin_data["entityId"] = "%s::%s" % (taskArn, name) + + plugin_data["data"] = DictionaryOfStan() + if self.root_metadata["Name"] == name: + plugin_data["data"]["instrumented"] = True + plugin_data["data"]["runtime"] = "python" + plugin_data["data"]["dockerId"] = container.get("DockerId", None) + plugin_data["data"]["dockerName"] = container.get("DockerName", None) + plugin_data["data"]["containerName"] = container.get("Name", None) + plugin_data["data"]["image"] = container.get("Image", None) + plugin_data["data"]["imageId"] = container.get("ImageID", None) + plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) + plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) + plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) + plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) + plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) + plugin_data["data"]["ports"] = container.get("Ports", None) + plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) + plugin_data["data"]["startedAt"] = container.get("StartedAt", None) + plugin_data["data"]["type"] = container.get("Type", None) + limits = container.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + finally: + plugins.append(plugin_data) except: logger.debug("_collect_container_snapshots: ", exc_info=True) return plugins @@ -228,32 +230,33 @@ def _collect_container_snapshots(self): def _collect_docker_snapshot(self): plugins = [] try: - containers = self.task_metadata.get("Containers", []) - for container in containers: - plugin_data = dict() - try: - labels = container.get("Labels", {}) - name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.container-name", "") - - plugin_data["name"] = "com.instana.plugin.docker" - # "entityId": $taskARN + "::" + $containerName - plugin_data["entityId"] = "%s::%s" % (taskArn, name) - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["Id"] = container.get("DockerId", None) - plugin_data["data"]["Created"] = container.get("CreatedAt", None) - plugin_data["data"]["Started"] = container.get("StartedAt", None) - plugin_data["data"]["Image"] = container.get("Image", None) - plugin_data["data"]["Labels"] = container.get("Labels", None) - plugin_data["data"]["Ports"] = container.get("Ports", None) - - networks = container.get("Networks", []) - if len(networks) >= 1: - plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - finally: - plugins.append(plugin_data) + if self.task_metadata is not None: + containers = self.task_metadata.get("Containers", []) + for container in containers: + plugin_data = dict() + try: + labels = container.get("Labels", {}) + name = container.get("Name", "") + taskArn = labels.get("com.amazonaws.ecs.container-name", "") + + plugin_data["name"] = "com.instana.plugin.docker" + # "entityId": $taskARN + "::" + $containerName + plugin_data["entityId"] = "%s::%s" % (taskArn, name) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["Id"] = container.get("DockerId", None) + plugin_data["data"]["Created"] = container.get("CreatedAt", None) + plugin_data["data"]["Started"] = container.get("StartedAt", None) + plugin_data["data"]["Image"] = container.get("Image", None) + plugin_data["data"]["Labels"] = container.get("Labels", None) + plugin_data["data"]["Ports"] = container.get("Ports", None) + + networks = container.get("Networks", []) + if len(networks) >= 1: + plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + finally: + plugins.append(plugin_data) except: logger.debug("_collect_container_snapshots: ", exc_info=True) return plugins From 5525ca558adcd173811d2b210063e11f9bc96d16 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 22:17:32 +0200 Subject: [PATCH 12/91] No unicode characters for py2 --- instana/collector/aws_fargate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 5522583b..1dcc8bd9 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -285,7 +285,7 @@ def _collect_process_snapshot(self): except: logger.debug("getpass.getuser: ", exc_info=True) - plugin_data["data"]["start"] = 1 # ¯\_(ツ)_/¯ FIXME + plugin_data["data"]["start"] = 1 # FIXME plugin_data["data"]["containerType"] = "docker" plugin_data["data"]["container"] = self.root_metadata.get("DockerId") plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # ¯\_(ツ)_/¯ FIXME From 5810b6e5d3d4a60084f82ff205913c219c098210 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 22:19:44 +0200 Subject: [PATCH 13/91] Rename method --- instana/collector/aws_fargate.py | 2 +- instana/collector/aws_lambda.py | 2 +- instana/collector/base.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 1dcc8bd9..c17023d8 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -124,7 +124,7 @@ def prepare_payload(self): payload["metrics"] = None if not self.span_queue.empty(): - payload["spans"] = self.__queued_spans() + payload["spans"] = self.queued_spans() if self.should_send_snapshot_data(): if self.snapshot_data is None: diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index 3451827d..61838f15 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -36,7 +36,7 @@ def prepare_payload(self): payload["metrics"] = None if not self.span_queue.empty(): - payload["spans"] = self.__queued_spans() + payload["spans"] = self.queued_spans() if self.should_send_snapshot_data(): payload["metrics"] = self.snapshot_data diff --git a/instana/collector/base.py b/instana/collector/base.py index bb619ed9..9ca6d0b2 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -124,7 +124,7 @@ def prepare_and_report_data(self): def collect_snapshot(self, *argv, **kwargs): logger.debug("BaseCollector: collect_snapshot needs to be overridden") - def __queued_spans(self): + def queued_spans(self): """ Get all of the queued spans @return: list From 976e8f3e1250794f4b0d4dc0b3b0c6f87f2290e4 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 22:42:27 +0200 Subject: [PATCH 14/91] Remove more unicode chars --- instana/collector/aws_fargate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index c17023d8..9b3fd584 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -288,7 +288,7 @@ def _collect_process_snapshot(self): plugin_data["data"]["start"] = 1 # FIXME plugin_data["data"]["containerType"] = "docker" plugin_data["data"]["container"] = self.root_metadata.get("DockerId") - plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # ¯\_(ツ)_/¯ FIXME + plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME plugin_data["data"]["com.instana.plugin.host.name"] = self.task_metadata.get("TaskArn") From 349f62504004f9384fdf7fd42c226f5adb8a15e3 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 23:23:38 +0200 Subject: [PATCH 15/91] Test: Show extra spans if any --- tests/frameworks/test_django.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index 7ae7b4dd..c270819a 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import pytest import urllib3 from django.apps import apps from django.contrib.staticfiles.testing import StaticLiveServerTestCase @@ -103,6 +104,18 @@ def test_request_with_error(self): self.assertEqual(500, response.status) spans = self.recorder.queued_spans() + + span_count = len(spans) + if span_count != 4: + msg = "Expected 4 spans but got %d: " % span_count + span_list = "" + if span_count > 0: + for span in spans: + if span.n == 'sdk': + span_list += "%s, " % span.data['sdk']['name'] + else: + span_list += "%s, " % span.n + pytest.fail(msg + span_list) self.assertEqual(4, len(spans)) test_span = spans[3] From 5ee1f2c4cfb5bf495fa69a96c2ce30e9082cfc73 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 23:34:41 +0200 Subject: [PATCH 16/91] Dump the full span --- tests/frameworks/test_django.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index c270819a..c2235e76 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -105,16 +105,17 @@ def test_request_with_error(self): spans = self.recorder.queued_spans() + import ipdb + ipdb.set_trace() + span_count = len(spans) if span_count != 4: - msg = "Expected 4 spans but got %d: " % span_count + msg = "Expected 4 spans but got %d\n: " % span_count span_list = "" if span_count > 0: for span in spans: - if span.n == 'sdk': - span_list += "%s, " % span.data['sdk']['name'] - else: - span_list += "%s, " % span.n + span.stack = '' + span_list += repr(span) + '\n' pytest.fail(msg + span_list) self.assertEqual(4, len(spans)) From 832c8241915e472f11cb97c2adf38cb7957734a6 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 24 Jul 2020 23:39:10 +0200 Subject: [PATCH 17/91] Remove debug --- tests/frameworks/test_django.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index c2235e76..dd2dcff8 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -105,9 +105,6 @@ def test_request_with_error(self): spans = self.recorder.queued_spans() - import ipdb - ipdb.set_trace() - span_count = len(spans) if span_count != 4: msg = "Expected 4 spans but got %d\n: " % span_count From 1c22e586c3f6bad00dedc1c922aebeaa450c5003 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sat, 25 Jul 2020 00:00:05 +0200 Subject: [PATCH 18/91] Move span dump into its own helper method --- tests/frameworks/test_django.py | 11 +++-------- tests/helpers.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index dd2dcff8..d0d63f16 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -7,6 +7,7 @@ from instana.singletons import agent, tracer +from ..helpers import fail_with_message_and_span_dump from ..apps.app_django import INSTALLED_APPS apps.populate(INSTALLED_APPS) @@ -107,14 +108,8 @@ def test_request_with_error(self): span_count = len(spans) if span_count != 4: - msg = "Expected 4 spans but got %d\n: " % span_count - span_list = "" - if span_count > 0: - for span in spans: - span.stack = '' - span_list += repr(span) + '\n' - pytest.fail(msg + span_list) - self.assertEqual(4, len(spans)) + msg = "Expected 4 spans but got %d" % span_count + fail_with_message_and_span_dump(msg, spans) test_span = spans[3] urllib3_span = spans[2] diff --git a/tests/helpers.py b/tests/helpers.py index ef1f5e12..6e6aa65f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,4 +1,5 @@ import os +import pytest testenv = {} @@ -58,6 +59,24 @@ testenv['mongodb_pw'] = os.environ.get('MONGO_PW', None) +def fail_with_message_and_span_dump(msg, spans): + """ + Helper method to fail a test when the number of spans isn't what was expected. This helper + will print and dump the list of spans in . + + @param msg: Descriptive message to print with the failure + @param spans: the list of spans to dump + @return: None + """ + span_count = len(spans) + span_dump = "\nDumping all collected spans (%d) -->\n" % span_count + if span_count > 0: + for span in spans: + span.stack = '' + span_dump += repr(span) + '\n' + pytest.fail(msg + span_dump, True) + + def get_first_span_by_name(spans, name): for span in spans: if span.n == name: From aa332a210d452118bd98640f3cfd25431d9c87f7 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sat, 25 Jul 2020 00:09:09 +0200 Subject: [PATCH 19/91] Add debug checks --- instana/instrumentation/aws/lambda_inst.py | 2 +- instana/singletons.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/instana/instrumentation/aws/lambda_inst.py b/instana/instrumentation/aws/lambda_inst.py index 54f68499..502413ab 100644 --- a/instana/instrumentation/aws/lambda_inst.py +++ b/instana/instrumentation/aws/lambda_inst.py @@ -33,7 +33,7 @@ def lambda_handler_with_instana(wrapped, instance, args, kwargs): return result -if env_is_aws_lambda: +if env_is_aws_lambda is True: handler_module, handler_function = get_lambda_handler_or_default() if handler_module is not None and handler_function is not None: diff --git a/instana/singletons.py b/instana/singletons.py index a6e5768a..632cec9c 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -15,6 +15,10 @@ env_is_aws_fargate = aws_env == "AWS_ECS_FARGATE" env_is_aws_lambda = "AWS_Lambda_" in aws_env +print("test: %s" % env_is_test) +print("fargate: %s" % env_is_aws_fargate) +print("lambda: %s" % env_is_aws_lambda) + if env_is_test: from .agent.test import TestAgent from .recorder import StandardRecorder From 9e18b74bc38a2a15a0fefbf6543a5a52edc97b0a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sat, 25 Jul 2020 00:49:29 +0200 Subject: [PATCH 20/91] Dont test log spans here --- tests/frameworks/test_django.py | 29 ++++++++++++++++------------- tests/helpers.py | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index d0d63f16..4433ca85 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -1,14 +1,13 @@ from __future__ import absolute_import -import pytest import urllib3 from django.apps import apps +from ..apps.app_django import INSTALLED_APPS from django.contrib.staticfiles.testing import StaticLiveServerTestCase from instana.singletons import agent, tracer -from ..helpers import fail_with_message_and_span_dump -from ..apps.app_django import INSTALLED_APPS +from ..helpers import fail_with_message_and_span_dump, get_first_span_by_filter, drop_log_spans_from_list apps.populate(INSTALLED_APPS) @@ -104,17 +103,24 @@ def test_request_with_error(self): assert response self.assertEqual(500, response.status) - spans = self.recorder.queued_spans() + spans = drop_log_spans_from_list(self.recorder.queued_spans()) span_count = len(spans) - if span_count != 4: - msg = "Expected 4 spans but got %d" % span_count + if span_count != 3: + msg = "Expected 3 spans but got %d" % span_count fail_with_message_and_span_dump(msg, spans) - test_span = spans[3] - urllib3_span = spans[2] - django_span = spans[1] - log_span = spans[0] + filter = lambda span: span.n == 'sdk' and span.data['sdk']['name'] == 'test' + test_span = get_first_span_by_filter(spans, filter) + assert(test_span) + + filter = lambda span: span.n == 'urllib3' + urllib3_span = get_first_span_by_filter(spans, filter) + assert(urllib3_span) + + filter = lambda span: span.n == 'django' + django_span = get_first_span_by_filter(spans, filter) + assert(django_span) assert ('X-Instana-T' in response.headers) assert (int(response.headers['X-Instana-T'], 16)) @@ -134,15 +140,12 @@ def test_request_with_error(self): self.assertEqual("test", test_span.data["sdk"]["name"]) self.assertEqual("urllib3", urllib3_span.n) self.assertEqual("django", django_span.n) - self.assertEqual("log", log_span.n) self.assertEqual(test_span.t, urllib3_span.t) self.assertEqual(urllib3_span.t, django_span.t) - self.assertEqual(django_span.t, log_span.t) self.assertEqual(urllib3_span.p, test_span.s) self.assertEqual(django_span.p, urllib3_span.s) - self.assertEqual(log_span.p, django_span.s) self.assertEqual(1, django_span.ec) diff --git a/tests/helpers.py b/tests/helpers.py index 6e6aa65f..ee36c7f1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -59,6 +59,20 @@ testenv['mongodb_pw'] = os.environ.get('MONGO_PW', None) +def drop_log_spans_from_list(spans): + """ + Log spans may occur randomly in test runs because of various intentional errors (for testing). This + helper method will remove all of the log spans from and return the remaining list. Helpful + for those tests where we are not testing log spans - where log spans are just noise. + @param spans: the list of spans to filter + @return: a filtered list of spans + """ + for span in spans: + if span.n == 'log': + spans.remove(span) + return spans + + def fail_with_message_and_span_dump(msg, spans): """ Helper method to fail a test when the number of spans isn't what was expected. This helper @@ -78,6 +92,12 @@ def fail_with_message_and_span_dump(msg, spans): def get_first_span_by_name(spans, name): + """ + Get the first span in that has a span.n value of + @param spans: the list of spans to search + @param name: the name to search for + @return: Span or None if nothing found + """ for span in spans: if span.n == name: return span From 5f8cec087b2b02c200b927d99c50324ab05491ea Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sat, 25 Jul 2020 10:30:37 +0200 Subject: [PATCH 21/91] better way to filter log spans --- tests/frameworks/test_django.py | 3 ++- tests/helpers.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index 4433ca85..59d3e0f3 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -103,7 +103,8 @@ def test_request_with_error(self): assert response self.assertEqual(500, response.status) - spans = drop_log_spans_from_list(self.recorder.queued_spans()) + spans = self.recorder.queued_spans() + spans = drop_log_spans_from_list(spans) span_count = len(spans) if span_count != 3: diff --git a/tests/helpers.py b/tests/helpers.py index ee36c7f1..dc9d17e4 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -67,10 +67,11 @@ def drop_log_spans_from_list(spans): @param spans: the list of spans to filter @return: a filtered list of spans """ + new_list = [] for span in spans: - if span.n == 'log': - spans.remove(span) - return spans + if span.n != 'log': + new_list.append(span) + return new_list def fail_with_message_and_span_dump(msg, spans): From 626115d0f6853a63af9538eda5db2bd9bcc0192b Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 13:19:25 +0200 Subject: [PATCH 22/91] Fargate checks --- instana/collector/aws_fargate.py | 19 +++++++++++++++++++ instana/recorder.py | 18 ++++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 9b3fd584..5ac35177 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -36,6 +36,9 @@ def __init__(self, agent): self.report_interval = 1 self.http_client = requests.Session() + # The fully qualified ARN for this process + self._fq_arn = None + # Saved snapshot data self.snapshot_data = None # Timestamp in seconds of the last time we sent snapshot data @@ -309,3 +312,19 @@ def _collect_runtime_snapshot(self): finally: self.process_metadata_mutex.release() return plugin_data + + def get_fq_arn(self): + if self._fq_arn is not None: + return self._fq_arn + + if self.root_metadata is not None: + labels = self.root_metadata.get("Labels", None) + if labels is not None: + taskArn = labels.get("com.amazonaws.ecs.task-arn", "") + + container_name = self.root_metadata.get("Name", "") + + self._fq_arn = taskArn + "::" + container_name + return self._fq_arn + else: + return "Missing ECMU metadata" diff --git a/instana/recorder.py b/instana/recorder.py index 946d3051..3dca3d1a 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -143,16 +143,18 @@ def record_span(self, span): """ Convert the passed BasicSpan and add it to the span queue """ - source = self.agent.get_from_structure() - service_name = self.agent.options.service_name + if self.agent.can_send(): + logger.debug("AWSFargateAgent not ready. Not tracing.") + source = self.agent.get_from_structure() + service_name = self.agent.options.service_name - if span.operation_name in self.REGISTERED_SPANS: - json_span = RegisteredSpan(span, source, service_name) - else: - json_span = SDKSpan(span, source, service_name) + if span.operation_name in self.REGISTERED_SPANS: + json_span = RegisteredSpan(span, source, service_name) + else: + json_span = SDKSpan(span, source, service_name) - # logger.debug("Recorded span: %s", json_span) - self.agent.collector.span_queue.put(json_span) + # logger.debug("Recorded span: %s", json_span) + self.agent.collector.span_queue.put(json_span) class InstanaSampler(Sampler): From deab44f77463f377e246cdd7cbfeb6fa01254c23 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 13:19:46 +0200 Subject: [PATCH 23/91] Temporarily disable lambda instrumentation --- instana/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/__init__.py b/instana/__init__.py index e030e0de..9a9138ee 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -115,7 +115,7 @@ def boot_agent(): # Instrumentation if "INSTANA_DISABLE_AUTO_INSTR" not in os.environ: # Import & initialize instrumentation - from .instrumentation.aws import lambda_inst + # from .instrumentation.aws import lambda_inst if sys.version_info >= (3, 5, 3): from .instrumentation import asyncio From e13e0feaa2e9a651d718d483c82f1a8c4842ef70 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 14:23:22 +0200 Subject: [PATCH 24/91] Logger vs print --- instana/singletons.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instana/singletons.py b/instana/singletons.py index 632cec9c..3420d761 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -15,9 +15,9 @@ env_is_aws_fargate = aws_env == "AWS_ECS_FARGATE" env_is_aws_lambda = "AWS_Lambda_" in aws_env -print("test: %s" % env_is_test) -print("fargate: %s" % env_is_aws_fargate) -print("lambda: %s" % env_is_aws_lambda) +logger.warn("test: %s" % env_is_test) +logger.warn("fargate: %s" % env_is_aws_fargate) +logger.warn("lambda: %s" % env_is_aws_lambda) if env_is_test: from .agent.test import TestAgent From 4b3dfa4bf4614e28fe7b6a54f1b26fc32dbb40c9 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 14:59:15 +0200 Subject: [PATCH 25/91] Fix egid + euid calcs --- instana/collector/aws_fargate.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 5ac35177..f80701b7 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,5 +1,6 @@ import os -import getpass +import pwd +import grp import requests import threading from time import time @@ -282,11 +283,13 @@ def _collect_process_snapshot(self): # drop the exe cmdline.pop(0) plugin_data["data"]["args"] = cmdline - plugin_data["data"]["user"] = getpass.getuser() try: - plugin_data["data"]["group"] = getpass.getuser(os.getegid()).gr_name + euid = os.geteuid() + egid = os.getegid() + plugin_data["data"]["user"] = pwd.getpwuid(euid) + plugin_data["data"]["group"] = grp.getgrgid(egid).gr_name except: - logger.debug("getpass.getuser: ", exc_info=True) + logger.debug("euid/egid detection: ", exc_info=True) plugin_data["data"]["start"] = 1 # FIXME plugin_data["data"]["containerType"] = "docker" @@ -301,16 +304,12 @@ def _collect_process_snapshot(self): def _collect_runtime_snapshot(self): plugin_data = dict() - lock_acquired = self.process_metadata_mutex.acquire(False) - if lock_acquired: - try: - plugin_data["name"] = "com.instana.plugin.python" - plugin_data["entityId"] = str(os.getpid()) - plugin_data["data"] = DictionaryOfStan() - except: - logger.debug("_collect_runtime_snapshot: ", exc_info=True) - finally: - self.process_metadata_mutex.release() + try: + plugin_data["name"] = "com.instana.plugin.python" + plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"] = DictionaryOfStan() + except: + logger.debug("_collect_runtime_snapshot: ", exc_info=True) return plugin_data def get_fq_arn(self): From a05992a066a0364ed81fe726d8423009a5ec73a4 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 15:06:59 +0200 Subject: [PATCH 26/91] Less debug and more safeties --- instana/agent/aws_fargate.py | 4 +--- instana/collector/aws_fargate.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 38b28dac..f96b3be1 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -82,9 +82,7 @@ def report_data_payload(self, payload): timeout=self.options.timeout, verify=ssl_verify) - if 200 <= response.status_code < 300: - logger.debug("report_data_payload: Instana responded with status code %s", response.status_code) - else: + if not 200 <= response.status_code < 300: logger.info("report_data_payload: Instana responded with status code %s", response.status_code) except Exception as e: logger.debug("report_data_payload: connection error (%s)", type(e)) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index f80701b7..9762987d 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -293,11 +293,11 @@ def _collect_process_snapshot(self): plugin_data["data"]["start"] = 1 # FIXME plugin_data["data"]["containerType"] = "docker" - plugin_data["data"]["container"] = self.root_metadata.get("DockerId") + if self.root_metadata is not None: + plugin_data["data"]["container"] = self.root_metadata.get("DockerId") plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME - plugin_data["data"]["com.instana.plugin.host.name"] = self.task_metadata.get("TaskArn") - - + if self.task_metadata is not None: + plugin_data["data"]["com.instana.plugin.host.name"] = self.task_metadata.get("TaskArn") except: logger.debug("_collect_process_snapshot: ", exc_info=True) return plugin_data From be3a95ebf840121f565b66a0527bd2fc09ae3afb Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 15:12:08 +0200 Subject: [PATCH 27/91] No root path in root url --- instana/collector/aws_fargate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 9762987d..5a66c4e3 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -25,7 +25,7 @@ def __init__(self, agent): "Instana will not be able to monitor this environment") self.ready_to_start = False - self.ecmu_url_root = self.ecmu + '/' + self.ecmu_url_root = self.ecmu self.ecmu_url_task = self.ecmu + '/task' self.ecmu_url_stats = self.ecmu + '/stats' self.ecmu_url_task_stats = self.ecmu + '/task/stats' From e5a67f6f825d7f319cafd8bffa4635298f1363bd Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 17:21:34 +0200 Subject: [PATCH 28/91] Fix payload & add tests --- instana/collector/aws_fargate.py | 12 +- tests/data/fargate/root_metadata.json | 31 ++ tests/data/fargate/stats_metadata.json | 184 ++++++++++ tests/data/fargate/task_metadata.json | 78 +++++ tests/data/fargate/task_stats_metadata.json | 370 ++++++++++++++++++++ tests/platforms/test_fargate_collector.py | 76 ++++ 6 files changed, 748 insertions(+), 3 deletions(-) create mode 100644 tests/data/fargate/root_metadata.json create mode 100644 tests/data/fargate/stats_metadata.json create mode 100644 tests/data/fargate/task_metadata.json create mode 100644 tests/data/fargate/task_stats_metadata.json create mode 100644 tests/platforms/test_fargate_collector.py diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 5a66c4e3..2df8e196 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -37,6 +37,9 @@ def __init__(self, agent): self.report_interval = 1 self.http_client = requests.Session() + # This is the collecter thread querying the metadata url + self.ecs_metadata_thread = None + # The fully qualified ARN for this process self._fq_arn = None @@ -46,6 +49,7 @@ def __init__(self, agent): self.snapshot_data_last_sent = 0 # How often to report snapshot data (in seconds) self.snapshot_data_interval = 600 + self.last_payload = None # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/ @@ -124,7 +128,7 @@ def should_send_snapshot_data(self): def prepare_payload(self): payload = DictionaryOfStan() - payload["spans"] = None + payload["spans"] = [] payload["metrics"] = None if not self.span_queue.empty(): @@ -136,6 +140,8 @@ def prepare_payload(self): payload["metrics"] = self.snapshot_data self.snapshot_data_last_sent = int(time()) + self.last_payload = payload + return payload def collect_snapshot(self, *argv, **kwargs): @@ -300,7 +306,7 @@ def _collect_process_snapshot(self): plugin_data["data"]["com.instana.plugin.host.name"] = self.task_metadata.get("TaskArn") except: logger.debug("_collect_process_snapshot: ", exc_info=True) - return plugin_data + return [plugin_data] def _collect_runtime_snapshot(self): plugin_data = dict() @@ -310,7 +316,7 @@ def _collect_runtime_snapshot(self): plugin_data["data"] = DictionaryOfStan() except: logger.debug("_collect_runtime_snapshot: ", exc_info=True) - return plugin_data + return [plugin_data] def get_fq_arn(self): if self._fq_arn is not None: diff --git a/tests/data/fargate/root_metadata.json b/tests/data/fargate/root_metadata.json new file mode 100644 index 00000000..cae53388 --- /dev/null +++ b/tests/data/fargate/root_metadata.json @@ -0,0 +1,31 @@ +{ + "DockerId": "63dc7ac9f3130bba35c785ed90ff12aad82087b5c5a0a45a922c45a64128eb45", + "Name": "docker-ssh-aws-fargate", + "DockerName": "ecs-docker-ssh-aws-fargate-1-docker-ssh-aws-fargate-9ef9a8edfefcaac95100", + "Image": "410797082306.dkr.ecr.us-east-2.amazonaws.com/fargate-docker-ssh:latest", + "ImageID": "sha256:c67110b16eb3ea771ff00d536023b9f07ffb4bcd07f6b535b525318d5033a368", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-2:410797082306:cluster/lombardo-ssh-cluster", + "com.amazonaws.ecs.container-name": "docker-ssh-aws-fargate", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-2:410797082306:task/2d60afb1-e7fd-4761-9430-a375293a9b82", + "com.amazonaws.ecs.task-definition-family": "docker-ssh-aws-fargate", + "com.amazonaws.ecs.task-definition-version": "1" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 256, + "Memory": 512 + }, + "CreatedAt": "2020-07-27T12:14:12.583114444Z", + "StartedAt": "2020-07-27T12:14:13.545410186Z", + "Type": "NORMAL", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.10.96" + ] + } + ] +} \ No newline at end of file diff --git a/tests/data/fargate/stats_metadata.json b/tests/data/fargate/stats_metadata.json new file mode 100644 index 00000000..0478a2d1 --- /dev/null +++ b/tests/data/fargate/stats_metadata.json @@ -0,0 +1,184 @@ +{ + "read": "2020-07-27T13:52:00.740080345Z", + "preread": "2020-07-27T13:51:59.738544869Z", + "pids_stats": { + "current": 10 + }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 202, + "minor": 26368, + "op": "Read", + "value": 0 + }, + { + "major": 202, + "minor": 26368, + "op": "Write", + "value": 128319488 + }, + { + "major": 202, + "minor": 26368, + "op": "Sync", + "value": 8933376 + }, + { + "major": 202, + "minor": 26368, + "op": "Async", + "value": 119386112 + }, + { + "major": 202, + "minor": 26368, + "op": "Total", + "value": 128319488 + } + ], + "io_serviced_recursive": [ + { + "major": 202, + "minor": 26368, + "op": "Read", + "value": 0 + }, + { + "major": 202, + "minor": 26368, + "op": "Write", + "value": 2538 + }, + { + "major": 202, + "minor": 26368, + "op": "Sync", + "value": 567 + }, + { + "major": 202, + "minor": 26368, + "op": "Async", + "value": 1971 + }, + { + "major": 202, + "minor": 26368, + "op": "Total", + "value": 2538 + } + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": [] + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 65637595575, + "percpu_usage": [ + 33807663526, + 31829932049, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "usage_in_kernelmode": 5310000000, + "usage_in_usermode": 58930000000 + }, + "system_cpu_usage": 11897300000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 65608183513, + "percpu_usage": [ + 33793294462, + 31814889051, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "usage_in_kernelmode": 5310000000, + "usage_in_usermode": 58900000000 + }, + "system_cpu_usage": 11895320000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 193757184, + "max_usage": 195305472, + "stats": { + "active_anon": 78704640, + "active_file": 18501632, + "cache": 90185728, + "dirty": 0, + "hierarchical_memory_limit": 536870912, + "hierarchical_memsw_limit": 1073741824, + "inactive_anon": 0, + "inactive_file": 71684096, + "mapped_file": 32768, + "pgfault": 1088220, + "pgmajfault": 0, + "pgpgin": 690027, + "pgpgout": 648793, + "rss": 78708736, + "rss_huge": 0, + "total_active_anon": 78704640, + "total_active_file": 18501632, + "total_cache": 90185728, + "total_dirty": 0, + "total_inactive_anon": 0, + "total_inactive_file": 71684096, + "total_mapped_file": 32768, + "total_pgfault": 1088220, + "total_pgmajfault": 0, + "total_pgpgin": 690027, + "total_pgpgout": 648793, + "total_rss": 78708736, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "limit": 536870912 + }, + "name": "/ecs-docker-ssh-aws-fargate-1-docker-ssh-aws-fargate-9ef9a8edfefcaac95100", + "id": "63dc7ac9f3130bba35c785ed90ff12aad82087b5c5a0a45a922c45a64128eb45" +} \ No newline at end of file diff --git a/tests/data/fargate/task_metadata.json b/tests/data/fargate/task_metadata.json new file mode 100644 index 00000000..52cda703 --- /dev/null +++ b/tests/data/fargate/task_metadata.json @@ -0,0 +1,78 @@ +{ + "Cluster": "arn:aws:ecs:us-east-2:410797082306:cluster/lombardo-ssh-cluster", + "TaskARN": "arn:aws:ecs:us-east-2:410797082306:task/2d60afb1-e7fd-4761-9430-a375293a9b82", + "Family": "docker-ssh-aws-fargate", + "Revision": "1", + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Containers": [ + { + "DockerId": "bfb22a5acd6c9695fba80ae542d12f047baa6a63521cad975001ed25c3ce19c2", + "Name": "~internal~ecs~pause", + "DockerName": "ecs-docker-ssh-aws-fargate-1-internalecspause-82bdec9beeffb9907c00", + "Image": "fg-proxy:tinyproxy", + "ImageID": "", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-2:410797082306:cluster/lombardo-ssh-cluster", + "com.amazonaws.ecs.container-name": "~internal~ecs~pause", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-2:410797082306:task/2d60afb1-e7fd-4761-9430-a375293a9b82", + "com.amazonaws.ecs.task-definition-family": "docker-ssh-aws-fargate", + "com.amazonaws.ecs.task-definition-version": "1" + }, + "DesiredStatus": "RESOURCES_PROVISIONED", + "KnownStatus": "RESOURCES_PROVISIONED", + "Limits": { + "CPU": 0, + "Memory": 0 + }, + "CreatedAt": "2020-07-27T12:13:51.454846803Z", + "StartedAt": "2020-07-27T12:13:52.449238716Z", + "Type": "CNI_PAUSE", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.10.96" + ] + } + ] + }, + { + "DockerId": "63dc7ac9f3130bba35c785ed90ff12aad82087b5c5a0a45a922c45a64128eb45", + "Name": "docker-ssh-aws-fargate", + "DockerName": "ecs-docker-ssh-aws-fargate-1-docker-ssh-aws-fargate-9ef9a8edfefcaac95100", + "Image": "410797082306.dkr.ecr.us-east-2.amazonaws.com/fargate-docker-ssh:latest", + "ImageID": "sha256:c67110b16eb3ea771ff00d536023b9f07ffb4bcd07f6b535b525318d5033a368", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-2:410797082306:cluster/lombardo-ssh-cluster", + "com.amazonaws.ecs.container-name": "docker-ssh-aws-fargate", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-2:410797082306:task/2d60afb1-e7fd-4761-9430-a375293a9b82", + "com.amazonaws.ecs.task-definition-family": "docker-ssh-aws-fargate", + "com.amazonaws.ecs.task-definition-version": "1" + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": { + "CPU": 256, + "Memory": 512 + }, + "CreatedAt": "2020-07-27T12:14:12.583114444Z", + "StartedAt": "2020-07-27T12:14:13.545410186Z", + "Type": "NORMAL", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": [ + "10.0.10.96" + ] + } + ] + } + ], + "Limits": { + "CPU": 0.25, + "Memory": 512 + }, + "PullStartedAt": "2020-07-27T12:13:52.586240564Z", + "PullStoppedAt": "2020-07-27T12:14:12.577606317Z" +} \ No newline at end of file diff --git a/tests/data/fargate/task_stats_metadata.json b/tests/data/fargate/task_stats_metadata.json new file mode 100644 index 00000000..55dadd4b --- /dev/null +++ b/tests/data/fargate/task_stats_metadata.json @@ -0,0 +1,370 @@ +{ + "63dc7ac9f3130bba35c785ed90ff12aad82087b5c5a0a45a922c45a64128eb45": { + "read": "2020-07-27T13:52:40.859305224Z", + "preread": "2020-07-27T13:52:39.855550726Z", + "pids_stats": { + "current": 10 + }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 202, + "minor": 26368, + "op": "Read", + "value": 0 + }, + { + "major": 202, + "minor": 26368, + "op": "Write", + "value": 128352256 + }, + { + "major": 202, + "minor": 26368, + "op": "Sync", + "value": 8966144 + }, + { + "major": 202, + "minor": 26368, + "op": "Async", + "value": 119386112 + }, + { + "major": 202, + "minor": 26368, + "op": "Total", + "value": 128352256 + } + ], + "io_serviced_recursive": [ + { + "major": 202, + "minor": 26368, + "op": "Read", + "value": 0 + }, + { + "major": 202, + "minor": 26368, + "op": "Write", + "value": 2542 + }, + { + "major": 202, + "minor": 26368, + "op": "Sync", + "value": 571 + }, + { + "major": 202, + "minor": 26368, + "op": "Async", + "value": 1971 + }, + { + "major": 202, + "minor": 26368, + "op": "Total", + "value": 2542 + } + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": [] + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 66070557631, + "percpu_usage": [ + 34054097656, + 32016459975, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "usage_in_kernelmode": 5330000000, + "usage_in_usermode": 59390000000 + }, + "system_cpu_usage": 11976670000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 66050861012, + "percpu_usage": [ + 34040562270, + 32010298742, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "usage_in_kernelmode": 5330000000, + "usage_in_usermode": 59370000000 + }, + "system_cpu_usage": 11974670000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 193769472, + "max_usage": 195305472, + "stats": { + "active_anon": 78721024, + "active_file": 18501632, + "cache": 90185728, + "dirty": 0, + "hierarchical_memory_limit": 536870912, + "hierarchical_memsw_limit": 1073741824, + "inactive_anon": 0, + "inactive_file": 71684096, + "mapped_file": 32768, + "pgfault": 1088223, + "pgmajfault": 0, + "pgpgin": 690034, + "pgpgout": 648797, + "rss": 78721024, + "rss_huge": 0, + "total_active_anon": 78721024, + "total_active_file": 18501632, + "total_cache": 90185728, + "total_dirty": 0, + "total_inactive_anon": 0, + "total_inactive_file": 71684096, + "total_mapped_file": 32768, + "total_pgfault": 1088223, + "total_pgmajfault": 0, + "total_pgpgin": 690034, + "total_pgpgout": 648797, + "total_rss": 78721024, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "limit": 536870912 + }, + "name": "/ecs-docker-ssh-aws-fargate-1-docker-ssh-aws-fargate-9ef9a8edfefcaac95100", + "id": "63dc7ac9f3130bba35c785ed90ff12aad82087b5c5a0a45a922c45a64128eb45" + }, + "bfb22a5acd6c9695fba80ae542d12f047baa6a63521cad975001ed25c3ce19c2": { + "read": "2020-07-27T13:52:40.858238762Z", + "preread": "2020-07-27T13:52:39.856756864Z", + "pids_stats": { + "current": 7 + }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { + "major": 202, + "minor": 26368, + "op": "Read", + "value": 5926912 + }, + { + "major": 202, + "minor": 26368, + "op": "Write", + "value": 8192 + }, + { + "major": 202, + "minor": 26368, + "op": "Sync", + "value": 5935104 + }, + { + "major": 202, + "minor": 26368, + "op": "Async", + "value": 0 + }, + { + "major": 202, + "minor": 26368, + "op": "Total", + "value": 5935104 + } + ], + "io_serviced_recursive": [ + { + "major": 202, + "minor": 26368, + "op": "Read", + "value": 344 + }, + { + "major": 202, + "minor": 26368, + "op": "Write", + "value": 2 + }, + { + "major": 202, + "minor": 26368, + "op": "Sync", + "value": 346 + }, + { + "major": 202, + "minor": 26368, + "op": "Async", + "value": 0 + }, + { + "major": 202, + "minor": 26368, + "op": "Total", + "value": 346 + } + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": [] + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 1764671369, + "percpu_usage": [ + 788582076, + 976089293, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "usage_in_kernelmode": 1120000000, + "usage_in_usermode": 380000000 + }, + "system_cpu_usage": 11976660000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 1764637941, + "percpu_usage": [ + 788548648, + 976089293, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "usage_in_kernelmode": 1120000000, + "usage_in_usermode": 380000000 + }, + "system_cpu_usage": 11974670000000, + "online_cpus": 2, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 11923456, + "max_usage": 14852096, + "stats": { + "active_anon": 3878912, + "active_file": 4464640, + "cache": 6004736, + "dirty": 0, + "hierarchical_memory_limit": 536870912, + "hierarchical_memsw_limit": 9223372036854772000, + "inactive_anon": 0, + "inactive_file": 1540096, + "mapped_file": 2039808, + "pgfault": 6185, + "pgmajfault": 52, + "pgpgin": 7526, + "pgpgout": 5113, + "rss": 3878912, + "rss_huge": 0, + "total_active_anon": 3878912, + "total_active_file": 4464640, + "total_cache": 6004736, + "total_dirty": 0, + "total_inactive_anon": 0, + "total_inactive_file": 1540096, + "total_mapped_file": 2039808, + "total_pgfault": 6185, + "total_pgmajfault": 52, + "total_pgpgin": 7526, + "total_pgpgout": 5113, + "total_rss": 3878912, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "limit": 4134510592 + }, + "name": "/ecs-docker-ssh-aws-fargate-1-internalecspause-82bdec9beeffb9907c00", + "id": "bfb22a5acd6c9695fba80ae542d12f047baa6a63521cad975001ed25c3ce19c2" + } +} \ No newline at end of file diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py new file mode 100644 index 00000000..cfcb71a9 --- /dev/null +++ b/tests/platforms/test_fargate_collector.py @@ -0,0 +1,76 @@ +from __future__ import absolute_import + +import os +import sys +import json +import pytest +import unittest + +from instana.tracer import InstanaTracer +from instana.options import AWSFargateOptions +from instana.recorder import AWSFargateRecorder +from instana.agent.aws_fargate import AWSFargateAgent +from instana.singletons import get_agent, set_agent, get_tracer, set_tracer + + +class TestFargate(unittest.TestCase): + def __init__(self, methodName='runTest'): + super(TestFargate, self).__init__(methodName) + self.agent = None + self.span_recorder = None + self.tracer = None + self.pwd = os.path.dirname(os.path.realpath(__file__)) + + self.original_agent = get_agent() + self.original_tracer = get_tracer() + + def setUp(self): + os.environ["AWS_EXECUTION_ENV"] = "AWS_ECS_FARGATE" + os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" + os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" + + def tearDown(self): + """ Reset all environment variables of consequence """ + if "AWS_EXECUTION_ENV" in os.environ: + os.environ.pop("AWS_EXECUTION_ENV") + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: + os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") + if "INSTANA_ENDPOINT_URL" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_AGENT_KEY" in os.environ: + os.environ.pop("INSTANA_AGENT_KEY") + + set_agent(self.original_agent) + set_tracer(self.original_tracer) + + def create_agent_and_setup_tracer(self): + self.agent = AWSFargateAgent() + self.span_recorder = AWSFargateRecorder(self.agent) + self.tracer = InstanaTracer(recorder=self.span_recorder) + set_agent(self.agent) + set_tracer(self.tracer) + + # Manually set the ECS Metadata API results on the collector + with open(self.pwd + '/../data/fargate/root_metadata.json', 'r') as json_file: + self.agent.collector.root_metadata = json.load(json_file) + with open(self.pwd + '/../data/fargate/task_metadata.json', 'r') as json_file: + self.agent.collector.task_metadata = json.load(json_file) + with open(self.pwd + '/../data/fargate/stats_metadata.json', 'r') as json_file: + self.agent.collector.stats_metadata = json.load(json_file) + with open(self.pwd + '/../data/fargate/task_stats_metadata.json', 'r') as json_file: + self.agent.collector.task_stats_metadata = json.load(json_file) + + def test_prepare_payload(self): + self.create_agent_and_setup_tracer() + + payload = self.agent.collector.prepare_payload() + assert(payload) + assert('spans' in payload) + assert(type(payload['spans']) is list) + assert(len(payload['spans']) == 0) + assert('metrics' in payload) + assert(len(payload['metrics'].keys()) == 1) + assert('plugins' in payload['metrics']) + assert(type(payload['metrics']['plugins']) is list) + assert(len(payload['metrics']['plugins']) == 7) + From b6220d4175130b6f460b29ac63ff6b3d19e9cc93 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 17:36:29 +0200 Subject: [PATCH 29/91] Convert JSON responses --- instana/collector/aws_fargate.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 2df8e196..b15e1af2 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,6 +1,7 @@ import os import pwd import grp +import json import requests import threading from time import time @@ -100,19 +101,23 @@ def get_ecs_metadata(self): try: # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/ - self.root_metadata = self.http_client.get(self.ecmu_url_root, timeout=3).content + json_body = self.http_client.get(self.ecmu_url_root, timeout=3).content + self.root_metadata = json.loads(json_body) # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/task - self.task_metadata = self.http_client.get(self.ecmu_url_task, timeout=3).content + json_body = self.http_client.get(self.ecmu_url_task, timeout=3).content + self.task_metadata = json.loads(json_body) # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/stats - self.stats_metadata = self.http_client.get(self.ecmu_url_stats, timeout=3).content + json_body = self.http_client.get(self.ecmu_url_stats, timeout=3).content + self.stats_metadata = json.loads(json_body) # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/task/stats - self.task_stats_metadata = self.http_client.get(self.ecmu_url_task_stats, timeout=3).content + json_body = self.http_client.get(self.ecmu_url_task_stats, timeout=3).content + self.task_stats_metadata = json.loads(json_body) except Exception: logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) finally: From 6475923005cc4e48fc806c8ee8bec5a6e4c94187 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 27 Jul 2020 17:41:11 +0200 Subject: [PATCH 30/91] Remake headers each time for now --- instana/agent/aws_fargate.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index f96b3be1..05947da4 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -61,13 +61,12 @@ def report_data_payload(self, payload): """ response = None try: - if self.report_headers is None: - # Prepare request headers - self.report_headers = dict() - self.report_headers["Content-Type"] = "application/json" - self.report_headers["X-Instana-Host"] = self.collector.get_fq_arn() - self.report_headers["X-Instana-Key"] = self.options.agent_key - self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) + # Prepare request headers + self.report_headers = dict() + self.report_headers["Content-Type"] = "application/json" + self.report_headers["X-Instana-Host"] = self.collector.get_fq_arn() + self.report_headers["X-Instana-Key"] = self.options.agent_key + self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) # logger.debug("using these headers: %s", self.report_headers) From 49935e76c9d31deaf2e8d6642b1b34ef505ce7e3 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 11:48:59 +0200 Subject: [PATCH 31/91] Log detected environment --- instana/singletons.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/instana/singletons.py b/instana/singletons.py index 3420d761..e9d001d1 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -15,14 +15,11 @@ env_is_aws_fargate = aws_env == "AWS_ECS_FARGATE" env_is_aws_lambda = "AWS_Lambda_" in aws_env -logger.warn("test: %s" % env_is_test) -logger.warn("fargate: %s" % env_is_aws_fargate) -logger.warn("lambda: %s" % env_is_aws_lambda) - if env_is_test: from .agent.test import TestAgent from .recorder import StandardRecorder + logger.debug("TEST environment detected.") agent = TestAgent() span_recorder = StandardRecorder() @@ -30,6 +27,7 @@ from .agent.aws_lambda import AWSLambdaAgent from .recorder import AWSLambdaRecorder + logger.debug("AWS Lambda environment detected.") agent = AWSLambdaAgent() span_recorder = AWSLambdaRecorder(agent) @@ -37,6 +35,7 @@ from .agent.aws_fargate import AWSFargateAgent from .recorder import AWSFargateRecorder + logger.debug("AWS Fargate environment detected.") agent = AWSFargateAgent() span_recorder = AWSFargateRecorder(agent) @@ -44,6 +43,7 @@ from .agent.host import HostAgent from .recorder import StandardRecorder + logger.debug("Standard Host environment detected.") agent = HostAgent() span_recorder = StandardRecorder() From 27123c87a759f33fe2230b358eaf0ca45ae23a48 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 11:49:53 +0200 Subject: [PATCH 32/91] Breakout data collection into helpers for simplicity --- instana/agent/host.py | 1 + instana/collector/aws_fargate.py | 211 +++--------------- instana/collector/helpers/__init__.py | 0 instana/collector/helpers/base.py | 9 + instana/collector/helpers/fargate/__init__.py | 0 .../collector/helpers/fargate/container.py | 55 +++++ instana/collector/helpers/fargate/docker.py | 43 ++++ instana/collector/helpers/fargate/task.py | 32 +++ instana/collector/helpers/process/__init__.py | 0 instana/collector/helpers/process/helper.py | 45 ++++ instana/collector/helpers/runtime/__init__.py | 0 instana/collector/helpers/runtime/helper.py | 19 ++ tests/platforms/test_fargate_collector.py | 8 + 13 files changed, 239 insertions(+), 184 deletions(-) create mode 100644 instana/collector/helpers/__init__.py create mode 100644 instana/collector/helpers/base.py create mode 100644 instana/collector/helpers/fargate/__init__.py create mode 100644 instana/collector/helpers/fargate/container.py create mode 100644 instana/collector/helpers/fargate/docker.py create mode 100644 instana/collector/helpers/fargate/task.py create mode 100644 instana/collector/helpers/process/__init__.py create mode 100644 instana/collector/helpers/process/helper.py create mode 100644 instana/collector/helpers/runtime/__init__.py create mode 100644 instana/collector/helpers/runtime/helper.py diff --git a/instana/agent/host.py b/instana/agent/host.py index 4fc3738c..2df70c4e 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -214,6 +214,7 @@ def report_data_payload(self, entity_data): """ response = None try: + logger.debug(to_json(entity_data)) response = self.client.post(self.__data_url(), data=to_json(entity_data), headers={"Content-Type": "application/json"}, diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index b15e1af2..ab373bb1 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,14 +1,17 @@ import os -import pwd -import grp import json import requests import threading from time import time from ..log import logger from .base import BaseCollector -from ..util import DictionaryOfStan, get_proc_cmdline, every, validate_url +from ..util import DictionaryOfStan, every, validate_url +from .helpers.process.helper import ProcessHelper +from .helpers.runtime.helper import RuntimeHelper +from .helpers.fargate.task import TaskHelper +from .helpers.fargate.docker import DockerHelper +from .helpers.fargate.container import ContainerHelper class AWSFargateCollector(BaseCollector): def __init__(self, agent): @@ -22,7 +25,7 @@ def __init__(self, agent): self.ecmu = os.environ.get("ECS_CONTAINER_METADATA_URI", "") if self.ecmu == "" or validate_url(self.ecmu) is False: - logger.warn("AWSFargateCollector: ECS_CONTAINER_METADATA_URI not in environment or invalid URL. " + logger.warning("AWSFargateCollector: ECS_CONTAINER_METADATA_URI not in environment or invalid URL. " "Instana will not be able to monitor this environment") self.ready_to_start = False @@ -68,9 +71,19 @@ def __init__(self, agent): # ${ECS_CONTAINER_METADATA_URI}/task/stats self.task_stats_metadata = None + # List of helpers that help out in data collection + self.helpers = [] + + # Populate the collection helpers + self.helpers.append(TaskHelper(self)) + self.helpers.append(DockerHelper(self)) + self.helpers.append(ProcessHelper(self)) + self.helpers.append(RuntimeHelper(self)) + self.helpers.append(ContainerHelper(self)) + def start(self): if self.ready_to_start is False: - logger.warn("AWS Fargate Collector is missing requirements and cannot monitor this environment.") + logger.warning("AWS Fargate Collector is missing requirements and cannot monitor this environment.") return # Launch a thread here to periodically collect data from the ECS Metadata Container API @@ -134,194 +147,24 @@ def should_send_snapshot_data(self): def prepare_payload(self): payload = DictionaryOfStan() payload["spans"] = [] - payload["metrics"] = None + payload["metrics"]["plugins"] = [] if not self.span_queue.empty(): payload["spans"] = self.queued_spans() - if self.should_send_snapshot_data(): - if self.snapshot_data is None: - self.snapshot_data = self.collect_snapshot() - payload["metrics"] = self.snapshot_data - self.snapshot_data_last_sent = int(time()) - - self.last_payload = payload - - return payload - - def collect_snapshot(self, *argv, **kwargs): - plugins = [] - self.snapshot_data = DictionaryOfStan() - - try: - plugins.extend(self._collect_task_snapshot()) - plugins.extend(self._collect_container_snapshots()) - plugins.extend(self._collect_docker_snapshot()) - plugins.extend(self._collect_process_snapshot()) - plugins.extend(self._collect_runtime_snapshot()) - self.snapshot_data["plugins"] = plugins - except: - logger.debug("collect_snapshot error", exc_info=True) - finally: - return self.snapshot_data - - def _collect_task_snapshot(self): - """ - Collect and return snapshot data for the task - @return: list - with one plugin entity - """ - plugin_data = dict() - try: - if self.task_metadata is not None: - plugin_data["name"] = "com.instana.plugin.aws.ecs.task" - plugin_data["entityId"] = "metadata.TaskARN" # FIXME - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["taskArn"] = self.task_metadata.get("TaskARN", None) - plugin_data["data"]["clusterArn"] = self.task_metadata.get("Cluster", None) - plugin_data["data"]["taskDefinition"] = self.task_metadata.get("Family", None) - plugin_data["data"]["taskDefinitionVersion"] = self.task_metadata.get("Revision", None) - plugin_data["data"]["availabilityZone"] = self.task_metadata.get("AvailabilityZone", None) - plugin_data["data"]["desiredStatus"] = self.task_metadata.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = self.task_metadata.get("KnownStatus", None) - plugin_data["data"]["pullStartedAt"] = self.task_metadata.get("PullStartedAt", None) - plugin_data["data"]["pullStoppedAt"] = self.task_metadata.get("PullStoppeddAt", None) - limits = self.task_metadata.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - except: - logger.debug("_collect_task_snapshot: ", exc_info=True) - return [plugin_data] - - def _collect_container_snapshots(self): - """ - Collect and return snapshot data for every container in this task - @return: list - with one or more plugin entities - """ plugins = [] + with_snapshot = self.should_send_snapshot_data() try: - if self.task_metadata is not None: - containers = self.task_metadata.get("Containers", []) - for container in containers: - plugin_data = dict() - try: - labels = container.get("Labels", {}) - name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.container-name", "") - - plugin_data["name"] = "com.instana.plugin.aws.ecs.container" - # "entityId": $taskARN + "::" + $containerName - plugin_data["entityId"] = "%s::%s" % (taskArn, name) - - plugin_data["data"] = DictionaryOfStan() - if self.root_metadata["Name"] == name: - plugin_data["data"]["instrumented"] = True - plugin_data["data"]["runtime"] = "python" - plugin_data["data"]["dockerId"] = container.get("DockerId", None) - plugin_data["data"]["dockerName"] = container.get("DockerName", None) - plugin_data["data"]["containerName"] = container.get("Name", None) - plugin_data["data"]["image"] = container.get("Image", None) - plugin_data["data"]["imageId"] = container.get("ImageID", None) - plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) - plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) - plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) - plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) - plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) - plugin_data["data"]["ports"] = container.get("Ports", None) - plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) - plugin_data["data"]["startedAt"] = container.get("StartedAt", None) - plugin_data["data"]["type"] = container.get("Type", None) - limits = container.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - finally: - plugins.append(plugin_data) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - return plugins + for helper in self.helpers: + plugins.extend(helper.collect_metrics(with_snapshot)) - def _collect_docker_snapshot(self): - plugins = [] - try: - if self.task_metadata is not None: - containers = self.task_metadata.get("Containers", []) - for container in containers: - plugin_data = dict() - try: - labels = container.get("Labels", {}) - name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.container-name", "") - - plugin_data["name"] = "com.instana.plugin.docker" - # "entityId": $taskARN + "::" + $containerName - plugin_data["entityId"] = "%s::%s" % (taskArn, name) - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["Id"] = container.get("DockerId", None) - plugin_data["data"]["Created"] = container.get("CreatedAt", None) - plugin_data["data"]["Started"] = container.get("StartedAt", None) - plugin_data["data"]["Image"] = container.get("Image", None) - plugin_data["data"]["Labels"] = container.get("Labels", None) - plugin_data["data"]["Ports"] = container.get("Ports", None) - - networks = container.get("Networks", []) - if len(networks) >= 1: - plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - finally: - plugins.append(plugin_data) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - return plugins + payload["metrics"]["plugins"] = plugins - def _collect_process_snapshot(self): - plugin_data = dict() - try: - plugin_data["name"] = "com.instana.plugin.process" - plugin_data["entityId"] = str(os.getpid()) - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["pid"] = int(os.getpid()) - env = dict() - for key in os.environ: - env[key] = os.environ[key] - plugin_data["data"]["env"] = env - plugin_data["data"]["exec"] = os.readlink("/proc/self/exe") - - cmdline = get_proc_cmdline() - if len(cmdline) > 1: - # drop the exe - cmdline.pop(0) - plugin_data["data"]["args"] = cmdline - try: - euid = os.geteuid() - egid = os.getegid() - plugin_data["data"]["user"] = pwd.getpwuid(euid) - plugin_data["data"]["group"] = grp.getgrgid(egid).gr_name - except: - logger.debug("euid/egid detection: ", exc_info=True) - - plugin_data["data"]["start"] = 1 # FIXME - plugin_data["data"]["containerType"] = "docker" - if self.root_metadata is not None: - plugin_data["data"]["container"] = self.root_metadata.get("DockerId") - plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME - if self.task_metadata is not None: - plugin_data["data"]["com.instana.plugin.host.name"] = self.task_metadata.get("TaskArn") + if with_snapshot is True: + self.snapshot_data_last_sent = int(time()) except: - logger.debug("_collect_process_snapshot: ", exc_info=True) - return [plugin_data] - - def _collect_runtime_snapshot(self): - plugin_data = dict() - try: - plugin_data["name"] = "com.instana.plugin.python" - plugin_data["entityId"] = str(os.getpid()) - plugin_data["data"] = DictionaryOfStan() - except: - logger.debug("_collect_runtime_snapshot: ", exc_info=True) - return [plugin_data] + logger.debug("collect_snapshot error", exc_info=True) + return payload def get_fq_arn(self): if self._fq_arn is not None: diff --git a/instana/collector/helpers/__init__.py b/instana/collector/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instana/collector/helpers/base.py b/instana/collector/helpers/base.py new file mode 100644 index 00000000..b48e978b --- /dev/null +++ b/instana/collector/helpers/base.py @@ -0,0 +1,9 @@ +from ...log import logger + + +class BaseHelper: + def __init__(self, collector): + self.collector = collector + + def collect_metrics(self, with_sendsnapshot = False): + logger.debug("BaseHelper.collect_metrics must be overridden") \ No newline at end of file diff --git a/instana/collector/helpers/fargate/__init__.py b/instana/collector/helpers/fargate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py new file mode 100644 index 00000000..a07b90f3 --- /dev/null +++ b/instana/collector/helpers/fargate/container.py @@ -0,0 +1,55 @@ +from ....log import logger +from ....util import DictionaryOfStan +from ..base import BaseHelper + + +class ContainerHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + """ + Collect and return metrics (and optionally snapshot data) for every container in this task + @return: list - with one or more plugin entities + """ + plugins = [] + try: + if self.collector.task_metadata is not None: + containers = self.collector.task_metadata.get("Containers", []) + for container in containers: + plugin_data = dict() + try: + labels = container.get("Labels", {}) + name = container.get("Name", "") + taskArn = labels.get("com.amazonaws.ecs.container-name", "") + + plugin_data["name"] = "com.instana.plugin.aws.ecs.container" + # "entityId": $taskARN + "::" + $containerName + plugin_data["entityId"] = "%s::%s" % (taskArn, name) + + plugin_data["data"] = DictionaryOfStan() + if self.collector.root_metadata["Name"] == name: + plugin_data["data"]["instrumented"] = True + plugin_data["data"]["runtime"] = "python" + plugin_data["data"]["dockerId"] = container.get("DockerId", None) + plugin_data["data"]["dockerName"] = container.get("DockerName", None) + plugin_data["data"]["containerName"] = container.get("Name", None) + plugin_data["data"]["image"] = container.get("Image", None) + plugin_data["data"]["imageId"] = container.get("ImageID", None) + plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) + plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) + plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) + plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) + plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) + plugin_data["data"]["ports"] = container.get("Ports", None) + plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) + plugin_data["data"]["startedAt"] = container.get("StartedAt", None) + plugin_data["data"]["type"] = container.get("Type", None) + limits = container.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + finally: + plugins.append(plugin_data) + except: + logger.debug("collect_container_metrics: ", exc_info=True) + return plugins diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py new file mode 100644 index 00000000..091d5ece --- /dev/null +++ b/instana/collector/helpers/fargate/docker.py @@ -0,0 +1,43 @@ +from ....log import logger +from ..base import BaseHelper +from ....util import DictionaryOfStan + + +class DockerHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + """ + Collect and return docker metrics (and optionally snapshot data) for this task + @return: list - with one or more plugin entities + """ + plugins = [] + try: + if self.collector.task_metadata is not None: + containers = self.collector.task_metadata.get("Containers", []) + for container in containers: + plugin_data = dict() + try: + labels = container.get("Labels", {}) + name = container.get("Name", "") + taskArn = labels.get("com.amazonaws.ecs.container-name", "") + + plugin_data["name"] = "com.instana.plugin.docker" + # "entityId": $taskARN + "::" + $containerName + plugin_data["entityId"] = "%s::%s" % (taskArn, name) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["Id"] = container.get("DockerId", None) + plugin_data["data"]["Created"] = container.get("CreatedAt", None) + plugin_data["data"]["Started"] = container.get("StartedAt", None) + plugin_data["data"]["Image"] = container.get("Image", None) + plugin_data["data"]["Labels"] = container.get("Labels", None) + plugin_data["data"]["Ports"] = container.get("Ports", None) + + networks = container.get("Networks", []) + if len(networks) >= 1: + plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) + except: + logger.debug("_collect_container_snapshots: ", exc_info=True) + finally: + plugins.append(plugin_data) + except: + logger.debug("collect_docker_metrics: ", exc_info=True) + return plugins diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py new file mode 100644 index 00000000..f79d39eb --- /dev/null +++ b/instana/collector/helpers/fargate/task.py @@ -0,0 +1,32 @@ +from ....log import logger +from ..base import BaseHelper +from ....util import DictionaryOfStan + + +class TaskHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + """ + Collect and return metrics data (and optionally snapshot data) for this task + @return: list - with one plugin entity + """ + plugin_data = dict() + try: + if self.collector.task_metadata is not None: + plugin_data["name"] = "com.instana.plugin.aws.ecs.task" + plugin_data["entityId"] = "metadata.TaskARN" # FIXME + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["taskArn"] = self.collector.task_metadata.get("TaskARN", None) + plugin_data["data"]["clusterArn"] = self.collector.task_metadata.get("Cluster", None) + plugin_data["data"]["taskDefinition"] = self.collector.task_metadata.get("Family", None) + plugin_data["data"]["taskDefinitionVersion"] = self.collector.task_metadata.get("Revision", None) + plugin_data["data"]["availabilityZone"] = self.collector.task_metadata.get("AvailabilityZone", None) + plugin_data["data"]["desiredStatus"] = self.collector.task_metadata.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = self.collector.task_metadata.get("KnownStatus", None) + plugin_data["data"]["pullStartedAt"] = self.collector.task_metadata.get("PullStartedAt", None) + plugin_data["data"]["pullStoppedAt"] = self.collector.task_metadata.get("PullStoppeddAt", None) + limits = self.collector.task_metadata.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + except: + logger.debug("collect_task_metrics: ", exc_info=True) + return [plugin_data] diff --git a/instana/collector/helpers/process/__init__.py b/instana/collector/helpers/process/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process/helper.py new file mode 100644 index 00000000..22408d5c --- /dev/null +++ b/instana/collector/helpers/process/helper.py @@ -0,0 +1,45 @@ +import os +import pwd +import grp +from ..base import BaseHelper +from instana.log import logger +from instana.util import DictionaryOfStan, get_proc_cmdline, every, validate_url + + +class ProcessHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + plugin_data = dict() + try: + plugin_data["name"] = "com.instana.plugin.process" + plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["pid"] = int(os.getpid()) + env = dict() + for key in os.environ: + env[key] = os.environ[key] + plugin_data["data"]["env"] = env + plugin_data["data"]["exec"] = os.readlink("/proc/self/exe") + + cmdline = get_proc_cmdline() + if len(cmdline) > 1: + # drop the exe + cmdline.pop(0) + plugin_data["data"]["args"] = cmdline + try: + euid = os.geteuid() + egid = os.getegid() + plugin_data["data"]["user"] = pwd.getpwuid(euid) + plugin_data["data"]["group"] = grp.getgrgid(egid).gr_name + except: + logger.debug("euid/egid detection: ", exc_info=True) + + plugin_data["data"]["start"] = 1 # FIXME + plugin_data["data"]["containerType"] = "docker" + if self.collector.root_metadata is not None: + plugin_data["data"]["container"] = self.collector.root_metadata.get("DockerId") + plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME + if self.collector.task_metadata is not None: + plugin_data["data"]["com.instana.plugin.host.name"] = self.collector.task_metadata.get("TaskArn") + except: + logger.debug("_collect_process_snapshot: ", exc_info=True) + return [plugin_data] diff --git a/instana/collector/helpers/runtime/__init__.py b/instana/collector/helpers/runtime/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instana/collector/helpers/runtime/helper.py b/instana/collector/helpers/runtime/helper.py new file mode 100644 index 00000000..cb0348b9 --- /dev/null +++ b/instana/collector/helpers/runtime/helper.py @@ -0,0 +1,19 @@ +import os +from ..base import BaseHelper +from instana.log import logger +from instana.util import DictionaryOfStan + + +class RuntimeHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + plugin_data = dict() + try: + plugin_data["name"] = "com.instana.plugin.python" + plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["metrics"] = self.collect_runtime_metrics() + plugin_data["data"]["snapshot"] = self._collect_runtime_snapshot() + except: + logger.debug("_collect_runtime_snapshot: ", exc_info=True) + return [plugin_data] + diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index cfcb71a9..d5dbe163 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -65,6 +65,8 @@ def test_prepare_payload(self): payload = self.agent.collector.prepare_payload() assert(payload) + + assert(len(payload.keys()) == 2) assert('spans' in payload) assert(type(payload['spans']) is list) assert(len(payload['spans']) == 0) @@ -74,3 +76,9 @@ def test_prepare_payload(self): assert(type(payload['metrics']['plugins']) is list) assert(len(payload['metrics']['plugins']) == 7) + plugins = payload['metrics']['plugins'] + for plugin in plugins: + assert('name' in plugin) + assert('entityId' in plugin) + assert('data' in plugin) + From 8833c0cb0a19629dcb90f05c4ceab8f7aa56491a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 11:50:39 +0000 Subject: [PATCH 33/91] Set INSTANA_TEST for test runs --- .circleci/config.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df641d03..1bfbbee8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,8 @@ jobs: pip install -e '.[test]' - run: name: run tests + environment: + INSTANA_TEST: true command: | . venv/bin/activate pytest -v @@ -67,6 +69,8 @@ jobs: pip install -e '.[test]' - run: name: run tests + environment: + INSTANA_TEST: true command: | . venv/bin/activate pytest -v @@ -94,9 +98,12 @@ jobs: pip install -e '.[test-cassandra]' - run: name: run tests + environment: + INSTANA_TEST: true + CASSANDRA_TEST: true command: | . venv/bin/activate - CASSANDRA_TEST=1 pytest -v tests/clients/test_cassandra-driver.py + pytest -v tests/clients/test_cassandra-driver.py py36cassandra: docker: @@ -118,9 +125,12 @@ jobs: pip install -e '.[test-cassandra]' - run: name: run tests + environment: + INSTANA_TEST: true + CASSANDRA_TEST: true command: | . venv/bin/activate - CASSANDRA_TEST=1 pytest -v tests/clients/test_cassandra-driver.py + pytest -v tests/clients/test_cassandra-driver.py gevent38: docker: @@ -138,9 +148,12 @@ jobs: pip install -e '.[test-gevent]' - run: name: run tests + environment: + INSTANA_TEST: true + GEVENT_TEST: true command: | . venv/bin/activate - GEVENT_TEST=1 pytest -v tests/frameworks/test_gevent.py + pytest -v tests/frameworks/test_gevent.py workflows: version: 2 build: From 03e4222f6fa44bb5ccec212105369d09bef3e189 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 11:51:28 +0000 Subject: [PATCH 34/91] Migrate Python metric collection to dedicated helper --- instana/collector/aws_fargate.py | 7 + instana/collector/base.py | 3 +- instana/collector/helpers/runtime/helper.py | 213 +++++++++++++++++- instana/collector/helpers/runtime/snapshot.py | 0 instana/meter.py | 5 +- instana/singletons.py | 2 +- 6 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 instana/collector/helpers/runtime/snapshot.py diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index ab373bb1..35480247 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -6,6 +6,7 @@ from ..log import logger from .base import BaseCollector from ..util import DictionaryOfStan, every, validate_url +from ..singletons import env_is_test from .helpers.process.helper import ProcessHelper from .helpers.runtime.helper import RuntimeHelper @@ -13,6 +14,7 @@ from .helpers.fargate.docker import DockerHelper from .helpers.fargate.container import ContainerHelper + class AWSFargateCollector(BaseCollector): def __init__(self, agent): super(AWSFargateCollector, self).__init__(agent) @@ -109,6 +111,11 @@ def get_ecs_metadata(self): Get the latest data from the ECS metadata container API and store on the class @return: Boolean """ + + if env_is_test is True: + # For test, we are using mock ECS metadata + return + lock_acquired = self.ecmu_lock.acquire(False) if lock_acquired: try: diff --git a/instana/collector/base.py b/instana/collector/base.py index 9ca6d0b2..76092535 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -7,6 +7,7 @@ import threading from ..log import logger +from ..singletons import env_is_test from ..util import every, DictionaryOfStan @@ -105,7 +106,7 @@ def prepare_and_report_data(self): Prepare and report the data payload. @return: Boolean """ - if "INSTANA_TEST" in os.environ: + if env_is_test is True: return True lock_acquired = self.lock.acquire(False) diff --git a/instana/collector/helpers/runtime/helper.py b/instana/collector/helpers/runtime/helper.py index cb0348b9..054e6a5e 100644 --- a/instana/collector/helpers/runtime/helper.py +++ b/instana/collector/helpers/runtime/helper.py @@ -1,19 +1,226 @@ import os +import copy +import gc as gc_ +import sys +import platform +import resource +import threading +from types import ModuleType +from pkg_resources import DistributionNotFound, get_distribution + from ..base import BaseHelper from instana.log import logger -from instana.util import DictionaryOfStan +from instana.util import DictionaryOfStan, determine_service_name class RuntimeHelper(BaseHelper): + def __init__(self, collector): + super(RuntimeHelper, self).__init__(collector) + self.last_collect = None + self.last_usage = None + self.last_metrics = None + def collect_metrics(self, with_snapshot = False): plugin_data = dict() try: plugin_data["name"] = "com.instana.plugin.python" plugin_data["entityId"] = str(os.getpid()) plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["metrics"] = self.collect_runtime_metrics() - plugin_data["data"]["snapshot"] = self._collect_runtime_snapshot() + + snapshot_payload = None + metrics_payload = self.gather_metrics() + + if with_snapshot is True: + snapshot_payload = self.gather_snapshot() + metrics_payload = copy.deepcopy(metrics_payload).delta_data(None) + else: + metrics_payload = copy.deepcopy(metrics_payload).delta_data(self.last_metrics) + + plugin_data["data"]["pid"] = str(os.getpid()) + plugin_data["data"]["metrics"] = metrics_payload + plugin_data["data"]["snapshot"] = snapshot_payload + + self.last_metrics = metrics_payload except: logger.debug("_collect_runtime_snapshot: ", exc_info=True) return [plugin_data] + def gather_metrics(self): + """ Collect up and return various metrics """ + try: + g = None + u = resource.getrusage(resource.RUSAGE_SELF) + if gc_.isenabled(): + c = list(gc_.get_count()) + th = list(gc_.get_threshold()) + g = GC(collect0=c[0] if not self.last_collect else c[0] - self.last_collect[0], + collect1=c[1] if not self.last_collect else c[ + 1] - self.last_collect[1], + collect2=c[2] if not self.last_collect else c[ + 2] - self.last_collect[2], + threshold0=th[0], + threshold1=th[1], + threshold2=th[2]) + + thr = threading.enumerate() + daemon_threads = [tr.daemon is True for tr in thr].count(True) + alive_threads = [tr.daemon is False for tr in thr].count(True) + dummy_threads = [type(tr) is threading._DummyThread for tr in thr].count(True) + + m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0], + ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1], + ru_maxrss=u[2], + ru_ixrss=u[3], + ru_idrss=u[4], + ru_isrss=u[5], + ru_minflt=u[6] if not self.last_usage else u[6] - self.last_usage[6], + ru_majflt=u[7] if not self.last_usage else u[7] - self.last_usage[7], + ru_nswap=u[8] if not self.last_usage else u[8] - self.last_usage[8], + ru_inblock=u[9] if not self.last_usage else u[9] - self.last_usage[9], + ru_oublock=u[10] if not self.last_usage else u[10] - self.last_usage[10], + ru_msgsnd=u[11] if not self.last_usage else u[11] - self.last_usage[11], + ru_msgrcv=u[12] if not self.last_usage else u[12] - self.last_usage[12], + ru_nsignals=u[13] if not self.last_usage else u[13] - self.last_usage[13], + ru_nvcs=u[14] if not self.last_usage else u[14] - self.last_usage[14], + ru_nivcsw=u[15] if not self.last_usage else u[15] - self.last_usage[15], + alive_threads=alive_threads, + dummy_threads=dummy_threads, + daemon_threads=daemon_threads, + gc=g) + + self.last_usage = u + if gc_.isenabled(): + self.last_collect = c + + return m + except Exception: + logger.debug("collect_metrics", exc_info=True) + + def gather_snapshot(self): + """ Gathers Python specific Snapshot information for this process """ + snapshot_payload = {} + try: + snapshot_payload['name'] = determine_service_name() + snapshot_payload['version'] = sys.version + snapshot_payload['f'] = platform.python_implementation() # flavor + snapshot_payload['a'] = platform.architecture()[0] # architecture + snapshot_payload['versions'] = self.gather_python_packages() + snapshot_payload['djmw'] = None # FIXME + except Exception as e: + logger.debug("collect_snapshot: ", exc_info=True) + finally: + return snapshot_payload + + def gather_python_packages(self): + """ Collect up the list of modules in use """ + try: + res = {} + m = sys.modules.copy() + + for k in m: + # Don't report submodules (e.g. django.x, django.y, django.z) + # Skip modules that begin with underscore + if ('.' in k) or k[0] == '_': + continue + if m[k]: + try: + d = m[k].__dict__ + if "version" in d and d["version"]: + res[k] = self.jsonable(d["version"]) + elif "__version__" in d and d["__version__"]: + res[k] = self.jsonable(d["__version__"]) + else: + res[k] = get_distribution(k).version + except DistributionNotFound: + pass + except Exception: + logger.debug("gather_python_packages: could not process module: %s", k) + + except Exception: + logger.debug("gather_python_packages", exc_info=True) + else: + return res + + def jsonable(self, value): + try: + if callable(value): + try: + result = value() + except: + result = 'Unknown' + elif type(value) is ModuleType: + result = value + else: + result = value + return str(result) + except Exception: + logger.debug("jsonable: ", exc_info=True) + +class GC(object): + collect0 = 0 + collect1 = 0 + collect2 = 0 + threshold0 = 0 + threshold1 = 0 + threshold2 = 0 + + def __init__(self, **kwds): + self.__dict__.update(kwds) + + def to_dict(self): + return self.__dict__ + + +class Metrics(object): + ru_utime = .0 + ru_stime = .0 + ru_maxrss = 0 + ru_ixrss = 0 + ru_idrss = 0 + ru_isrss = 0 + ru_minflt = 0 + ru_majflt = 0 + ru_nswap = 0 + ru_inblock = 0 + ru_oublock = 0 + ru_msgsnd = 0 + ru_msgrcv = 0 + ru_nsignals = 0 + ru_nvcs = 0 + ru_nivcsw = 0 + dummy_threads = 0 + alive_threads = 0 + daemon_threads = 0 + gc = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) + + def delta_data(self, delta): + data = self.__dict__ + if delta is None: + return data + + unchanged_items = set(data.items()) & set(delta.items()) + for x in unchanged_items: + data.pop(x[0]) + + return data + + def to_dict(self): + return self.__dict__ + + +class EntityData(object): + pid = 0 + snapshot = None + metrics = None + + def __init__(self, **kwds): + self.__dict__.update(kwds) + + def to_dict(self): + return self.__dict__ + + + diff --git a/instana/collector/helpers/runtime/snapshot.py b/instana/collector/helpers/runtime/snapshot.py new file mode 100644 index 00000000..e69de29b diff --git a/instana/meter.py b/instana/meter.py index 8302ae83..bd9eb340 100644 --- a/instana/meter.py +++ b/instana/meter.py @@ -1,13 +1,12 @@ import copy import gc as gc_ import json +import sys import platform import resource -import sys import threading -from types import ModuleType from fysom import FysomError - +from types import ModuleType from pkg_resources import DistributionNotFound, get_distribution from .log import logger diff --git a/instana/singletons.py b/instana/singletons.py index e9d001d1..a633b2c1 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -11,7 +11,7 @@ # Detect the environment where we are running ahead of time aws_env = os.environ.get("AWS_EXECUTION_ENV", "") -env_is_test = os.environ.get("INSTANA_TEST", False) +env_is_test = "INSTANA_TEST" in os.environ env_is_aws_fargate = aws_env == "AWS_ECS_FARGATE" env_is_aws_lambda = "AWS_Lambda_" in aws_env From 8a8a06856ba3995dc716844fceeb536c0b77d900 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 11:59:34 +0000 Subject: [PATCH 35/91] Fix entityId --- instana/collector/helpers/fargate/task.py | 2 +- instana/collector/helpers/process/helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index f79d39eb..8ecb2d40 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -13,7 +13,7 @@ def collect_metrics(self, with_snapshot = False): try: if self.collector.task_metadata is not None: plugin_data["name"] = "com.instana.plugin.aws.ecs.task" - plugin_data["entityId"] = "metadata.TaskARN" # FIXME + plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) plugin_data["data"] = DictionaryOfStan() plugin_data["data"]["taskArn"] = self.collector.task_metadata.get("TaskARN", None) plugin_data["data"]["clusterArn"] = self.collector.task_metadata.get("Cluster", None) diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process/helper.py index 22408d5c..13a551c6 100644 --- a/instana/collector/helpers/process/helper.py +++ b/instana/collector/helpers/process/helper.py @@ -37,7 +37,7 @@ def collect_metrics(self, with_snapshot = False): plugin_data["data"]["containerType"] = "docker" if self.collector.root_metadata is not None: plugin_data["data"]["container"] = self.collector.root_metadata.get("DockerId") - plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME + # plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME: the pid in the root namespace (very optional) if self.collector.task_metadata is not None: plugin_data["data"]["com.instana.plugin.host.name"] = self.collector.task_metadata.get("TaskArn") except: From 62f72fa553270cb5e4329c390b7f137ea521e10d Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 13:42:36 +0000 Subject: [PATCH 36/91] Cleanup Options; Add all supported env vars --- instana/options.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/instana/options.py b/instana/options.py index b3594d30..30a50c37 100644 --- a/instana/options.py +++ b/instana/options.py @@ -6,13 +6,12 @@ class BaseOptions(object): - service_name = None - extra_http_headers = None - log_level = logging.WARN - debug = None - def __init__(self, **kwds): try: + self.debug = False + self.log_level = logging.WARN + self.service_name = determine_service_name() + if "INSTANA_DEBUG" in os.environ: self.log_level = logging.DEBUG self.debug = True @@ -29,13 +28,9 @@ class StandardOptions(BaseOptions): AGENT_DEFAULT_HOST = "localhost" AGENT_DEFAULT_PORT = 42699 - agent_host = None - agent_port = None - def __init__(self, **kwds): super(StandardOptions, self).__init__() - self.service_name = determine_service_name() self.agent_host = os.environ.get("INSTANA_AGENT_HOST", self.AGENT_DEFAULT_HOST) self.agent_port = os.environ.get("INSTANA_AGENT_PORT", self.AGENT_DEFAULT_PORT) @@ -44,9 +39,9 @@ def __init__(self, **kwds): class AWSLambdaOptions(BaseOptions): + """ Configurable option bits for AWS Lambda """ endpoint_url = None agent_key = None - extra_http_headers = None timeout = None def __init__(self, **kwds): @@ -59,28 +54,25 @@ def __init__(self, **kwds): self.endpoint_url = self.endpoint_url[:-1] self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) - self.service_name = os.environ.get("INSTANA_SERVICE_NAME", None) self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) class AWSFargateOptions(BaseOptions): - endpoint_url = None - agent_key = None - extra_http_headers = None - timeout = None - + """ Configurable option bits for AWS Fargate """ def __init__(self, **kwds): super(AWSFargateOptions, self).__init__() - self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) + self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) + self.endpoint_proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) + self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) # Remove any trailing slash (if any) if self.endpoint_url is not None and self.endpoint_url[-1] == "/": self.endpoint_url = self.endpoint_url[:-1] - self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) - self.service_name = os.environ.get("INSTANA_SERVICE_NAME", None) - self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) + self.tags = os.environ.get("INSTANA_TAGS", None) + self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) + self.zone = os.environ.get("INSTANA_ZONE", None) From cf206d806ce8e99479c16f3927296db753c89fd7 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 14:18:07 +0000 Subject: [PATCH 37/91] Normalize and standardize on extra_http_header handling --- instana/agent/aws_fargate.py | 1 - instana/agent/aws_lambda.py | 1 - instana/agent/base.py | 1 - instana/agent/host.py | 8 +++-- instana/instrumentation/aiohttp/client.py | 4 +-- instana/instrumentation/aiohttp/server.py | 4 +-- instana/instrumentation/aws/triggers.py | 8 ++--- instana/instrumentation/django/middleware.py | 5 ++-- instana/instrumentation/flask/vanilla.py | 4 +-- instana/instrumentation/flask/with_blinker.py | 4 +-- instana/instrumentation/pyramid/tweens.py | 6 ++-- instana/instrumentation/tornado/server.py | 4 +-- instana/instrumentation/urllib3.py | 4 +-- instana/instrumentation/webapp2_inst.py | 4 +-- instana/options.py | 29 ++++++++----------- instana/wsgi.py | 4 +-- tests/clients/test_urllib3.py | 6 ++-- tests/frameworks/test_aiohttp.py | 8 ++--- tests/frameworks/test_django.py | 2 +- tests/frameworks/test_tornado_server.py | 2 +- tests/frameworks/test_wsgi.py | 2 +- tests/platforms/test_fargate.py | 11 +++---- tests/platforms/test_lambda.py | 11 +++---- tests/test_agent.py | 5 ++-- 24 files changed, 70 insertions(+), 68 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 05947da4..a2abc981 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -31,7 +31,6 @@ def __init__(self): self.options = AWSFargateOptions() self.report_headers = None self._can_send = False - self.extra_headers = self.options.extra_http_headers if self._validate_options(): self._can_send = True diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index 2af1666a..cb00cc1e 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -31,7 +31,6 @@ def __init__(self): self.options = AWSLambdaOptions() self.report_headers = None self._can_send = False - self.extra_headers = self.options.extra_http_headers if self._validate_options(): self._can_send = True diff --git a/instana/agent/base.py b/instana/agent/base.py index 172d6bab..b50e0d7c 100644 --- a/instana/agent/base.py +++ b/instana/agent/base.py @@ -7,7 +7,6 @@ class BaseAgent(object): sensor = None secrets_matcher = 'contains-ignore-case' secrets_list = ['key', 'pass', 'secret'] - extra_headers = None options = None def __init__(self): diff --git a/instana/agent/host.py b/instana/agent/host.py index 2df70c4e..86e4cdc4 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -139,8 +139,12 @@ def set_from(self, json_string): self.secrets_list = res_data['secrets']['list'] if "extraHeaders" in res_data: - self.extra_headers = res_data['extraHeaders'] - logger.info("Will also capture these custom headers: %s", self.extra_headers) + # FIXME: add tests + if self.options.extra_http_headers is None: + self.options.extra_http_headers = res_data['extraHeaders'] + else: + self.options.extra_http_headers.extend(res_data['extraHeaders']) + logger.info("Will also capture these custom headers: %s", self.options.extra_http_headers) self.announce_data = AnnounceData(pid=res_data['pid'], agentUuid=res_data['agentUuid']) diff --git a/instana/instrumentation/aiohttp/client.py b/instana/instrumentation/aiohttp/client.py index 683e0b28..d63b2a49 100644 --- a/instana/instrumentation/aiohttp/client.py +++ b/instana/instrumentation/aiohttp/client.py @@ -41,8 +41,8 @@ async def stan_request_end(session, trace_config_ctx, params): if scope is not None: scope.span.set_tag('http.status_code', params.response.status) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: if custom_header in params.response.headers: scope.span.set_tag("http.%s" % custom_header, params.response.headers[custom_header]) diff --git a/instana/instrumentation/aiohttp/server.py b/instana/instrumentation/aiohttp/server.py index 6508ab90..a341d7aa 100644 --- a/instana/instrumentation/aiohttp/server.py +++ b/instana/instrumentation/aiohttp/server.py @@ -32,8 +32,8 @@ async def stan_middleware(request, handler): scope.span.set_tag("http.method", request.method) # Custom header tracking support - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: if custom_header in request.headers: scope.span.set_tag("http.%s" % custom_header, request.headers[custom_header]) diff --git a/instana/instrumentation/aws/triggers.py b/instana/instrumentation/aws/triggers.py index 19acc38b..4398a020 100644 --- a/instana/instrumentation/aws/triggers.py +++ b/instana/instrumentation/aws/triggers.py @@ -137,8 +137,8 @@ def enrich_lambda_span(agent, span, event, context): span.set_tag('http.path_tpl', event["resource"]) span.set_tag('http.params', read_http_query_params(event)) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - capture_extra_headers(event, span, agent.extra_headers) + if agent.options.extra_http_headers is not None: + capture_extra_headers(event, span, agent.options.extra_http_headers) elif is_application_load_balancer_trigger(event): span.set_tag('lambda.trigger', 'aws:application.load.balancer') @@ -146,8 +146,8 @@ def enrich_lambda_span(agent, span, event, context): span.set_tag('http.url', event["path"]) span.set_tag('http.params', read_http_query_params(event)) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - capture_extra_headers(event, span, agent.extra_headers) + if agent.options.extra_http_headers is not None: + capture_extra_headers(event, span, agent.options.extra_http_headers) elif is_cloudwatch_trigger(event): span.set_tag('lambda.trigger', 'aws:cloudwatch.events') diff --git a/instana/instrumentation/django/middleware.py b/instana/instrumentation/django/middleware.py index 78e746c7..98555d53 100644 --- a/instana/instrumentation/django/middleware.py +++ b/instana/instrumentation/django/middleware.py @@ -22,6 +22,7 @@ class InstanaMiddleware(MiddlewareMixin): """ Django Middleware to provide request tracing for Instana """ def __init__(self, get_response=None): + super(InstanaMiddleware, self).__init__(get_response) self.get_response = get_response def process_request(self, request): @@ -31,8 +32,8 @@ def process_request(self, request): ctx = tracer.extract(ot.Format.HTTP_HEADERS, env) request.iscope = tracer.start_active_span('django', child_of=ctx) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: # Headers are available in this format: HTTP_X_CAPTURE_THIS django_header = ('HTTP_' + custom_header.upper()).replace('-', '_') if django_header in env: diff --git a/instana/instrumentation/flask/vanilla.py b/instana/instrumentation/flask/vanilla.py index 174bfe49..e62ee020 100644 --- a/instana/instrumentation/flask/vanilla.py +++ b/instana/instrumentation/flask/vanilla.py @@ -25,8 +25,8 @@ def before_request_with_instana(*argv, **kwargs): flask.g.scope = tracer.start_active_span('wsgi', child_of=ctx) span = flask.g.scope.span - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: # Headers are available in this format: HTTP_X_CAPTURE_THIS header = ('HTTP_' + custom_header.upper()).replace('-', '_') if header in env: diff --git a/instana/instrumentation/flask/with_blinker.py b/instana/instrumentation/flask/with_blinker.py index 5a95c3a1..183b2c8a 100644 --- a/instana/instrumentation/flask/with_blinker.py +++ b/instana/instrumentation/flask/with_blinker.py @@ -26,8 +26,8 @@ def request_started_with_instana(sender, **extra): flask.g.scope = tracer.start_active_span('wsgi', child_of=ctx) span = flask.g.scope.span - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: # Headers are available in this format: HTTP_X_CAPTURE_THIS header = ('HTTP_' + custom_header.upper()).replace('-', '_') if header in env: diff --git a/instana/instrumentation/pyramid/tweens.py b/instana/instrumentation/pyramid/tweens.py index 923f8de2..78d1da90 100644 --- a/instana/instrumentation/pyramid/tweens.py +++ b/instana/instrumentation/pyramid/tweens.py @@ -9,6 +9,7 @@ from ...singletons import tracer, agent from ...util import strip_secrets + class InstanaTweenFactory(object): """A factory that provides Instana instrumentation tween for Pyramid apps""" @@ -27,8 +28,8 @@ def __call__(self, request): if request.matched_route is not None: scope.span.set_tag("http.path_tpl", request.matched_route.pattern) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: # Headers are available in this format: HTTP_X_CAPTURE_THIS h = ('HTTP_' + custom_header.upper()).replace('-', '_') if h in request.headers: @@ -74,6 +75,7 @@ def __call__(self, request): return response + def includeme(config): logger.debug("Instrumenting pyramid") config.add_tween(__name__ + '.InstanaTweenFactory') diff --git a/instana/instrumentation/tornado/server.py b/instana/instrumentation/tornado/server.py index 0c65968a..cb71628a 100644 --- a/instana/instrumentation/tornado/server.py +++ b/instana/instrumentation/tornado/server.py @@ -39,8 +39,8 @@ def execute_with_instana(wrapped, instance, argv, kwargs): scope.span.set_tag("handler", instance.__class__.__name__) # Custom header tracking support - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: if custom_header in instance.request.headers: scope.span.set_tag("http.%s" % custom_header, instance.request.headers[custom_header]) diff --git a/instana/instrumentation/urllib3.py b/instana/instrumentation/urllib3.py index ab5f2889..f36713eb 100644 --- a/instana/instrumentation/urllib3.py +++ b/instana/instrumentation/urllib3.py @@ -48,8 +48,8 @@ def collect_response(scope, response): try: scope.span.set_tag(ext.HTTP_STATUS_CODE, response.status) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: if custom_header in response.headers: scope.span.set_tag("http.%s" % custom_header, response.headers[custom_header]) diff --git a/instana/instrumentation/webapp2_inst.py b/instana/instrumentation/webapp2_inst.py index 08f2662d..435a57bb 100644 --- a/instana/instrumentation/webapp2_inst.py +++ b/instana/instrumentation/webapp2_inst.py @@ -41,8 +41,8 @@ def new_start_response(status, headers, exc_info=None): ctx = tracer.extract(ot.Format.HTTP_HEADERS, env) scope = env['stan_scope'] = tracer.start_active_span("wsgi", child_of=ctx) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: # Headers are available in this format: HTTP_X_CAPTURE_THIS wsgi_header = ('HTTP_' + custom_header.upper()).replace('-', '_') if wsgi_header in env: diff --git a/instana/options.py b/instana/options.py index 30a50c37..7c727772 100644 --- a/instana/options.py +++ b/instana/options.py @@ -7,19 +7,18 @@ class BaseOptions(object): def __init__(self, **kwds): - try: - self.debug = False - self.log_level = logging.WARN - self.service_name = determine_service_name() - - if "INSTANA_DEBUG" in os.environ: - self.log_level = logging.DEBUG - self.debug = True - if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: - self.extra_http_headers = str(os.environ["INSTANA_EXTRA_HTTP_HEADERS"]).lower().split(';') - except: - pass - + self.debug = False + self.log_level = logging.WARN + self.service_name = determine_service_name() + self.extra_http_headers = None + + if "INSTANA_DEBUG" in os.environ: + self.log_level = logging.DEBUG + self.debug = True + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: + self.extra_http_headers = str(os.environ["INSTANA_EXTRA_HTTP_HEADERS"]).lower().split(';') + + self.secrets = os.environ.get("INSTANA_SECRETS", None) self.__dict__.update(kwds) @@ -40,10 +39,6 @@ def __init__(self, **kwds): class AWSLambdaOptions(BaseOptions): """ Configurable option bits for AWS Lambda """ - endpoint_url = None - agent_key = None - timeout = None - def __init__(self, **kwds): super(AWSLambdaOptions, self).__init__() diff --git a/instana/wsgi.py b/instana/wsgi.py index 9e2990b0..5c72d130 100644 --- a/instana/wsgi.py +++ b/instana/wsgi.py @@ -35,8 +35,8 @@ def new_start_response(status, headers, exc_info=None): ctx = tracer.extract(ot.Format.HTTP_HEADERS, env) self.scope = tracer.start_active_span("wsgi", child_of=ctx) - if hasattr(agent, 'extra_headers') and agent.extra_headers is not None: - for custom_header in agent.extra_headers: + if agent.options.extra_http_headers is not None: + for custom_header in agent.options.extra_http_headers: # Headers are available in this format: HTTP_X_CAPTURE_THIS wsgi_header = ('HTTP_' + custom_header.upper()).replace('-', '_') if wsgi_header in env: diff --git a/tests/clients/test_urllib3.py b/tests/clients/test_urllib3.py index c1f57c59..b4922a4c 100644 --- a/tests/clients/test_urllib3.py +++ b/tests/clients/test_urllib3.py @@ -657,8 +657,8 @@ def test_requestspkg_put(self): self.assertTrue(len(urllib3_span.stack) > 1) def test_response_header_capture(self): - original_extra_headers = agent.extra_headers - agent.extra_headers = ['X-Capture-This'] + original_extra_http_headers = agent.options.extra_http_headers + agent.options.extra_http_headers = ['X-Capture-This'] with tracer.start_active_span('test'): r = self.http.request('GET', testenv["wsgi_server"] + '/response_headers') @@ -708,5 +708,5 @@ def test_response_header_capture(self): self.assertTrue(len(urllib3_span.stack) > 1) self.assertTrue('http.X-Capture-This' in urllib3_span.data["custom"]["tags"]) - agent.extra_headers = original_extra_headers + agent.options.extra_http_headers = original_extra_http_headers diff --git a/tests/frameworks/test_aiohttp.py b/tests/frameworks/test_aiohttp.py index 64aa4c19..97e499b0 100644 --- a/tests/frameworks/test_aiohttp.py +++ b/tests/frameworks/test_aiohttp.py @@ -326,8 +326,8 @@ async def test(): self.assertEqual(response.headers["Server-Timing"], "intid;desc=%s" % traceId) def test_client_response_header_capture(self): - original_extra_headers = agent.extra_headers - agent.extra_headers = ['X-Capture-This'] + original_extra_http_headers = agent.options.extra_http_headers + agent.options.extra_http_headers = ['X-Capture-This'] async def test(): with async_tracer.start_active_span('test'): @@ -377,7 +377,7 @@ async def test(): assert("Server-Timing" in response.headers) self.assertEqual(response.headers["Server-Timing"], "intid;desc=%s" % traceId) - agent.extra_headers = original_extra_headers + agent.options.extra_http_headers = original_extra_http_headers def test_client_error(self): async def test(): @@ -569,7 +569,7 @@ async def test(): with async_tracer.start_active_span('test'): async with aiohttp.ClientSession() as session: # Hack together a manual custom headers list - agent.extra_headers = [u'X-Capture-This', u'X-Capture-That'] + agent.options.extra_http_headers = [u'X-Capture-This', u'X-Capture-That'] headers = dict() headers['X-Capture-This'] = 'this' diff --git a/tests/frameworks/test_django.py b/tests/frameworks/test_django.py index 59d3e0f3..f1c4c331 100644 --- a/tests/frameworks/test_django.py +++ b/tests/frameworks/test_django.py @@ -213,7 +213,7 @@ def test_complex_request(self): def test_custom_header_capture(self): # Hack together a manual custom headers list - agent.extra_headers = [u'X-Capture-This', u'X-Capture-That'] + agent.options.extra_http_headers = [u'X-Capture-This', u'X-Capture-That'] request_headers = dict() request_headers['X-Capture-This'] = 'this' diff --git a/tests/frameworks/test_tornado_server.py b/tests/frameworks/test_tornado_server.py index ffd160d2..b93ee72e 100644 --- a/tests/frameworks/test_tornado_server.py +++ b/tests/frameworks/test_tornado_server.py @@ -540,7 +540,7 @@ async def test(): with async_tracer.start_active_span('test'): async with aiohttp.ClientSession() as session: # Hack together a manual custom headers list - agent.extra_headers = [u'X-Capture-This', u'X-Capture-That'] + agent.options.extra_http_headers = [u'X-Capture-This', u'X-Capture-That'] headers = dict() headers['X-Capture-This'] = 'this' diff --git a/tests/frameworks/test_wsgi.py b/tests/frameworks/test_wsgi.py index 5f644bde..dc12f1d8 100644 --- a/tests/frameworks/test_wsgi.py +++ b/tests/frameworks/test_wsgi.py @@ -171,7 +171,7 @@ def test_complex_request(self): def test_custom_header_capture(self): # Hack together a manual custom headers list - agent.extra_headers = [u'X-Capture-This', u'X-Capture-That'] + agent.options.extra_http_headers = [u'X-Capture-This', u'X-Capture-That'] request_headers = {} request_headers['X-Capture-This'] = 'this' diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index a2ec211d..9f3fb374 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -75,16 +75,17 @@ def test_has_options(self): self.assertTrue(hasattr(self.agent, 'options')) self.assertTrue(type(self.agent.options) is AWSFargateOptions) - def test_has_extra_headers(self): + def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() - self.assertTrue(hasattr(self.agent, 'extra_headers')) + self.assertTrue(hasattr(self.agent, 'options')) + self.assertTrue(hasattr(self.agent.options, 'extra_http_headers')) - def test_agent_extra_headers(self): + def test_agent_extra_http_headers(self): os.environ['INSTANA_EXTRA_HTTP_HEADERS'] = "X-Test-Header;X-Another-Header;X-And-Another-Header" self.create_agent_and_setup_tracer() - self.assertIsNotNone(self.agent.extra_headers) + self.assertIsNotNone(self.agent.options.extra_http_headers) should_headers = ['x-test-header', 'x-another-header', 'x-and-another-header'] - self.assertEqual(should_headers, self.agent.extra_headers) + self.assertEqual(should_headers, self.agent.options.extra_http_headers) @pytest.mark.skip("todo") def test_custom_service_name(self): diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index 086a366f..395f1369 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -101,9 +101,10 @@ def test_secrets(self): self.assertTrue(hasattr(self.agent, 'secrets_list')) self.assertEqual(self.agent.secrets_list, ['key', 'pass', 'secret']) - def test_has_extra_headers(self): + def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() - self.assertTrue(hasattr(self.agent, 'extra_headers')) + self.assertTrue(hasattr(self.agent, 'options')) + self.assertTrue(hasattr(self.agent.options, 'extra_http_headers')) def test_has_options(self): self.create_agent_and_setup_tracer() @@ -117,12 +118,12 @@ def test_get_handler(self): self.assertEqual("tests", handler_module) self.assertEqual("lambda_handler", handler_function) - def test_agent_extra_headers(self): + def test_agent_extra_http_headers(self): os.environ['INSTANA_EXTRA_HTTP_HEADERS'] = "X-Test-Header;X-Another-Header;X-And-Another-Header" self.create_agent_and_setup_tracer() - self.assertIsNotNone(self.agent.extra_headers) + self.assertIsNotNone(self.agent.options.extra_http_headers) should_headers = ['x-test-header', 'x-another-header', 'x-and-another-header'] - self.assertEqual(should_headers, self.agent.extra_headers) + self.assertEqual(should_headers, self.agent.options.extra_http_headers) def test_custom_service_name(self): os.environ['INSTANA_SERVICE_NAME'] = "Legion" diff --git a/tests/test_agent.py b/tests/test_agent.py index e16a1f4e..6dde387f 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -19,8 +19,9 @@ def test_secrets(self): self.assertTrue(hasattr(agent, 'secrets_list')) self.assertEqual(agent.secrets_list, ['key', 'pass', 'secret']) - def test_has_extra_headers(self): - self.assertTrue(hasattr(agent, 'extra_headers')) + def test_options_have_extra_http_headers(self): + self.assertTrue(hasattr(agent, 'options')) + self.assertTrue(hasattr(agent.options, 'extra_http_headers')) def test_has_options(self): self.assertTrue(hasattr(agent, 'options')) From 9894c86bc3e7c692f8631651db42eeae1b3dff2a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 14:25:16 +0000 Subject: [PATCH 38/91] Uncommong lambda inst --- instana/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/__init__.py b/instana/__init__.py index 9a9138ee..e030e0de 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -115,7 +115,7 @@ def boot_agent(): # Instrumentation if "INSTANA_DISABLE_AUTO_INSTR" not in os.environ: # Import & initialize instrumentation - # from .instrumentation.aws import lambda_inst + from .instrumentation.aws import lambda_inst if sys.version_info >= (3, 5, 3): from .instrumentation import asyncio From f29b1b362916d8833fd9c46a08c3a628bad35fdd Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 15:47:28 +0000 Subject: [PATCH 39/91] Same service_name handling regardless of environment --- instana/recorder.py | 25 ++++++++++++------------- instana/singletons.py | 4 ++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/instana/recorder.py b/instana/recorder.py index 3dca3d1a..be0e485a 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -28,7 +28,8 @@ class StandardRecorder(object): # Recorder thread for collection/reporting of spans thread = None - def __init__(self): + def __init__(self, agent): + self.agent = agent self.queue = queue.Queue() def start(self): @@ -102,10 +103,13 @@ def record_span(self, span): Convert the passed BasicSpan into and add it to the span queue """ if instana.singletons.agent.can_send() or "INSTANA_TEST" in os.environ: + service_name = None source = instana.singletons.agent.get_from_structure() + if "INSTANA_SERVICE_NAME" in os.environ: + service_name = self.agent.options.service_name if span.operation_name in self.REGISTERED_SPANS: - json_span = RegisteredSpan(span, source, None) + json_span = RegisteredSpan(span, source, service_name) else: service_name = instana.singletons.agent.options.service_name json_span = SDKSpan(span, source, service_name) @@ -114,16 +118,14 @@ def record_span(self, span): class AWSLambdaRecorder(StandardRecorder): - def __init__(self, agent): - self.agent = agent - super(AWSLambdaRecorder, self).__init__() - def record_span(self, span): """ Convert the passed BasicSpan and add it to the span queue """ + service_name = None source = self.agent.get_from_structure() - service_name = self.agent.options.service_name + if "INSTANA_SERVICE_NAME" in os.environ: + service_name = self.agent.options.service_name if span.operation_name in self.REGISTERED_SPANS: json_span = RegisteredSpan(span, source, service_name) @@ -135,18 +137,15 @@ def record_span(self, span): class AWSFargateRecorder(StandardRecorder): - def __init__(self, agent): - self.agent = agent - super(AWSFargateRecorder, self).__init__() - def record_span(self, span): """ Convert the passed BasicSpan and add it to the span queue """ if self.agent.can_send(): - logger.debug("AWSFargateAgent not ready. Not tracing.") + service_name = None source = self.agent.get_from_structure() - service_name = self.agent.options.service_name + if "INSTANA_SERVICE_NAME" in os.environ: + service_name = self.agent.options.service_name if span.operation_name in self.REGISTERED_SPANS: json_span = RegisteredSpan(span, source, service_name) diff --git a/instana/singletons.py b/instana/singletons.py index a633b2c1..0f5dda40 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -21,7 +21,7 @@ logger.debug("TEST environment detected.") agent = TestAgent() - span_recorder = StandardRecorder() + span_recorder = StandardRecorder(agent) elif env_is_aws_lambda: from .agent.aws_lambda import AWSLambdaAgent @@ -45,7 +45,7 @@ logger.debug("Standard Host environment detected.") agent = HostAgent() - span_recorder = StandardRecorder() + span_recorder = StandardRecorder(agent) def get_agent(): From 8a8b27daed2f75b09c4778374fe8b7a7e5a5582f Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 28 Jul 2020 16:28:33 +0000 Subject: [PATCH 40/91] Skip asynqp tests which are breaking in Python 3.8 --- tests/clients/test_asynqp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/clients/test_asynqp.py b/tests/clients/test_asynqp.py index ae2f7996..603a59fb 100644 --- a/tests/clients/test_asynqp.py +++ b/tests/clients/test_asynqp.py @@ -20,7 +20,8 @@ else: rabbitmq_host = "localhost" -@pytest.mark.skipif(LooseVersion(sys.version) < LooseVersion('3.5.3'), reason="") +#@pytest.mark.skipif(LooseVersion(sys.version) < LooseVersion('3.5.3'), reason="") +@pytest.mark.skip("FIXME: Abandoned asynqp is now causing issues in later Python versions.") class TestAsynqp(unittest.TestCase): @asyncio.coroutine def connect(self): From 8c4f8708145ef6f3b03541fdd056fe963c2bd71c Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 29 Jul 2020 08:33:02 +0000 Subject: [PATCH 41/91] Use a default agent if not specified --- instana/recorder.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/instana/recorder.py b/instana/recorder.py index be0e485a..78f50a3d 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -6,7 +6,7 @@ from .log import logger from .util import every -import instana.singletons +from .singletons import get_agent from basictracer import Sampler from .span import (RegisteredSpan, SDKSpan) @@ -28,8 +28,12 @@ class StandardRecorder(object): # Recorder thread for collection/reporting of spans thread = None - def __init__(self, agent): - self.agent = agent + def __init__(self, agent = None): + if agent is None: + self.agent = get_agent() + else: + self.agent = agent + self.queue = queue.Queue() def start(self): @@ -63,13 +67,13 @@ def report_spans(self): logger.debug(" -> Span reporting thread is now alive") def span_work(): - if instana.singletons.agent.should_threads_shutdown.is_set(): + if self.agent.should_threads_shutdown.is_set(): logger.debug("Thread shutdown signal from agent is active: Shutting down span reporting thread") return False queue_size = self.queue.qsize() - if queue_size > 0 and instana.singletons.agent.can_send(): - response = instana.singletons.agent.report_traces(self.queued_spans()) + if queue_size > 0 and self.agent.can_send(): + response = self.agent.report_traces(self.queued_spans()) if response: logger.debug("reported %d spans", queue_size) return True @@ -102,16 +106,16 @@ def record_span(self, span): """ Convert the passed BasicSpan into and add it to the span queue """ - if instana.singletons.agent.can_send() or "INSTANA_TEST" in os.environ: + if self.agent.can_send() or "INSTANA_TEST" in os.environ: service_name = None - source = instana.singletons.agent.get_from_structure() + source = self.agent.get_from_structure() if "INSTANA_SERVICE_NAME" in os.environ: service_name = self.agent.options.service_name if span.operation_name in self.REGISTERED_SPANS: json_span = RegisteredSpan(span, source, service_name) else: - service_name = instana.singletons.agent.options.service_name + service_name = self.agent.options.service_name json_span = SDKSpan(span, source, service_name) self.queue.put(json_span) From 31214f9b9b8d4ee78af9640d831be2b726bca55f Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 29 Jul 2020 08:43:34 +0000 Subject: [PATCH 42/91] Late import to avoid circular import --- instana/recorder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instana/recorder.py b/instana/recorder.py index 78f50a3d..6cba416a 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -6,7 +6,6 @@ from .log import logger from .util import every -from .singletons import get_agent from basictracer import Sampler from .span import (RegisteredSpan, SDKSpan) @@ -30,6 +29,8 @@ class StandardRecorder(object): def __init__(self, agent = None): if agent is None: + # Late import to avoid circular import + from .singletons import get_agent self.agent = get_agent() else: self.agent = agent From aaf9b08cb780864441e2541b8543145b15ce2fa7 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 29 Jul 2020 08:43:53 +0000 Subject: [PATCH 43/91] Remove unecessary logging --- tests/clients/test_mysql-python.py | 1 - tests/clients/test_mysqlclient.py | 1 - tests/clients/test_psycopg2.py | 1 - tests/clients/test_pymongo.py | 3 --- tests/clients/test_pymysql.py | 1 - 5 files changed, 7 deletions(-) diff --git a/tests/clients/test_mysql-python.py b/tests/clients/test_mysql-python.py index f028318d..c5193b2a 100644 --- a/tests/clients/test_mysql-python.py +++ b/tests/clients/test_mysql-python.py @@ -51,7 +51,6 @@ class TestMySQLPython(unittest.TestCase): def setUp(self): - logger.warning("MySQL connecting: %s:@%s:3306/%s", testenv['mysql_user'], testenv['mysql_host'], testenv['mysql_db']) self.db = MySQLdb.connect(host=testenv['mysql_host'], port=testenv['mysql_port'], user=testenv['mysql_user'], passwd=testenv['mysql_pw'], db=testenv['mysql_db']) diff --git a/tests/clients/test_mysqlclient.py b/tests/clients/test_mysqlclient.py index 1a1e18be..e67ee91d 100644 --- a/tests/clients/test_mysqlclient.py +++ b/tests/clients/test_mysqlclient.py @@ -51,7 +51,6 @@ class TestMySQLPython(unittest.TestCase): def setUp(self): - logger.info("MySQL connecting: %s:@%s:3306/%s", testenv['mysql_user'], testenv['mysql_host'], testenv['mysql_db']) self.db = MySQLdb.connect(host=testenv['mysql_host'], port=testenv['mysql_port'], user=testenv['mysql_user'], passwd=testenv['mysql_pw'], db=testenv['mysql_db']) diff --git a/tests/clients/test_psycopg2.py b/tests/clients/test_psycopg2.py index 5a889550..d72e8d73 100644 --- a/tests/clients/test_psycopg2.py +++ b/tests/clients/test_psycopg2.py @@ -48,7 +48,6 @@ class TestPsycoPG2(unittest.TestCase): def setUp(self): - logger.warning("Postgresql connecting: %s:@%s:5432/%s", testenv['postgresql_user'], testenv['postgresql_host'], testenv['postgresql_db']) self.db = psycopg2.connect(host=testenv['postgresql_host'], port=testenv['postgresql_port'], user=testenv['postgresql_user'], password=testenv['postgresql_pw'], database=testenv['postgresql_db']) diff --git a/tests/clients/test_pymongo.py b/tests/clients/test_pymongo.py index 5edd400c..55f2f324 100644 --- a/tests/clients/test_pymongo.py +++ b/tests/clients/test_pymongo.py @@ -18,9 +18,6 @@ class TestPyMongo(unittest.TestCase): def setUp(self): - logger.warning("Connecting to MongoDB mongo://%s:@%s:%s", - testenv['mongodb_user'], testenv['mongodb_host'], testenv['mongodb_port']) - self.conn = pymongo.MongoClient(host=testenv['mongodb_host'], port=int(testenv['mongodb_port']), username=testenv['mongodb_user'], password=testenv['mongodb_pw']) self.conn.test.records.delete_many(filter={}) diff --git a/tests/clients/test_pymysql.py b/tests/clients/test_pymysql.py index 3b398d61..aa3c9490 100644 --- a/tests/clients/test_pymysql.py +++ b/tests/clients/test_pymysql.py @@ -45,7 +45,6 @@ class TestPyMySQL(unittest.TestCase): def setUp(self): - logger.warning("MySQL connecting: %s:@%s:3306/%s", testenv['mysql_user'], testenv['mysql_host'], testenv['mysql_db']) self.db = pymysql.connect(host=testenv['mysql_host'], port=testenv['mysql_port'], user=testenv['mysql_user'], passwd=testenv['mysql_pw'], db=testenv['mysql_db']) From 0e565d64e7e9c793b536e608470a47ec5cc9c6c4 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 29 Jul 2020 08:59:00 +0000 Subject: [PATCH 44/91] Py27 compatibility --- instana/collector/helpers/base.py | 2 +- instana/recorder.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/instana/collector/helpers/base.py b/instana/collector/helpers/base.py index b48e978b..e9f05f5c 100644 --- a/instana/collector/helpers/base.py +++ b/instana/collector/helpers/base.py @@ -1,7 +1,7 @@ from ...log import logger -class BaseHelper: +class BaseHelper(object): def __init__(self, collector): self.collector = collector diff --git a/instana/recorder.py b/instana/recorder.py index 6cba416a..ece7e3ae 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -163,4 +163,5 @@ def record_span(self, span): class InstanaSampler(Sampler): def sampled(self, _): + # We never sample return False From b4df611846e164e823db2f68883999188aaf3d73 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 29 Jul 2020 14:57:46 +0000 Subject: [PATCH 45/91] Add support for INSTANA_ZONE --- instana/collector/aws_fargate.py | 6 ++-- instana/collector/helpers/fargate/docker.py | 2 ++ instana/collector/helpers/fargate/hardware.py | 26 ++++++++++++++++ tests/platforms/test_fargate_collector.py | 30 +++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 instana/collector/helpers/fargate/hardware.py diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 35480247..bf9acac3 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -8,10 +8,11 @@ from ..util import DictionaryOfStan, every, validate_url from ..singletons import env_is_test -from .helpers.process.helper import ProcessHelper -from .helpers.runtime.helper import RuntimeHelper from .helpers.fargate.task import TaskHelper from .helpers.fargate.docker import DockerHelper +from .helpers.process.helper import ProcessHelper +from .helpers.runtime.helper import RuntimeHelper +from .helpers.fargate.hardware import HardwareHelper from .helpers.fargate.container import ContainerHelper @@ -81,6 +82,7 @@ def __init__(self, agent): self.helpers.append(DockerHelper(self)) self.helpers.append(ProcessHelper(self)) self.helpers.append(RuntimeHelper(self)) + self.helpers.append(HardwareHelper(self)) self.helpers.append(ContainerHelper(self)) def start(self): diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index 091d5ece..4d2b2542 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -24,6 +24,8 @@ def collect_metrics(self, with_snapshot = False): # "entityId": $taskARN + "::" + $containerName plugin_data["entityId"] = "%s::%s" % (taskArn, name) plugin_data["data"] = DictionaryOfStan() + + # Snapshot Data plugin_data["data"]["Id"] = container.get("DockerId", None) plugin_data["data"]["Created"] = container.get("CreatedAt", None) plugin_data["data"]["Started"] = container.get("StartedAt", None) diff --git a/instana/collector/helpers/fargate/hardware.py b/instana/collector/helpers/fargate/hardware.py new file mode 100644 index 00000000..3f504627 --- /dev/null +++ b/instana/collector/helpers/fargate/hardware.py @@ -0,0 +1,26 @@ +from ....log import logger +from ....util import DictionaryOfStan +from ..base import BaseHelper + + +class HardwareHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + """ + # This helper only sends snapshot data related to the INSTANA_ZONE environment variable + @return: list + """ + plugins = [] + if with_snapshot is False or self.collector.agent.options.zone is None: + return plugins + + try: + if self.collector.task_metadata is not None: + plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.generic.hardware" + plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["availability-zone"] = self.collector.agent.options.zone + plugins.append(plugin_data) + except: + logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) + return plugins diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index d5dbe163..0bb98a91 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -29,6 +29,9 @@ def setUp(self): os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" + if "INSTANA_ZONE" in os.environ: + os.environ.pop("INSTANA_ZONE") + def tearDown(self): """ Reset all environment variables of consequence """ if "AWS_EXECUTION_ENV" in os.environ: @@ -39,6 +42,8 @@ def tearDown(self): os.environ.pop("INSTANA_ENDPOINT_URL") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_ZONE" in os.environ: + os.environ.pop("INSTANA_ZONE") set_agent(self.original_agent) set_tracer(self.original_tracer) @@ -82,3 +87,28 @@ def test_prepare_payload(self): assert('entityId' in plugin) assert('data' in plugin) + def test_no_instana_zone(self): + self.create_agent_and_setup_tracer() + assert(self.agent.options.zone is None) + + def test_instana_zone(self): + os.environ["INSTANA_ZONE"] = "YellowDog" + self.create_agent_and_setup_tracer() + + assert(self.agent.options.zone == "YellowDog") + + payload = self.agent.collector.prepare_payload() + assert(payload) + + plugins = payload['metrics']['plugins'] + assert(type(plugins) is list) + + hardware_plugin = None + for plugin in plugins: + if plugin["name"] == "com.instana.plugin.generic.hardware": + hardware_plugin = plugin + + assert(hardware_plugin) + assert("data" in hardware_plugin) + assert("availability-zone" in hardware_plugin["data"]) + assert(hardware_plugin["data"]["availability-zone"] == "YellowDog") From 85c9b17716e3e2c860d7aabef38ea0751a6a5005 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 30 Jul 2020 09:42:37 +0000 Subject: [PATCH 46/91] Global INSTANA_SECRETS support --- instana/agent/base.py | 2 -- instana/agent/host.py | 4 +-- instana/collector/helpers/process/helper.py | 2 +- instana/instrumentation/aiohttp/client.py | 2 +- instana/instrumentation/aiohttp/server.py | 2 +- instana/instrumentation/django/middleware.py | 2 +- instana/instrumentation/flask/vanilla.py | 2 +- instana/instrumentation/flask/with_blinker.py | 2 +- instana/instrumentation/pyramid/tweens.py | 2 +- instana/instrumentation/tornado/client.py | 2 +- instana/instrumentation/tornado/server.py | 2 +- instana/instrumentation/urllib3.py | 2 +- instana/instrumentation/webapp2_inst.py | 2 +- instana/options.py | 17 ++++++++++- instana/wsgi.py | 2 +- tests/platforms/test_fargate.py | 30 +++++++++++++------ tests/platforms/test_lambda.py | 4 +-- tests/test_agent.py | 4 +-- 18 files changed, 55 insertions(+), 30 deletions(-) diff --git a/instana/agent/base.py b/instana/agent/base.py index b50e0d7c..c22f4ade 100644 --- a/instana/agent/base.py +++ b/instana/agent/base.py @@ -5,8 +5,6 @@ class BaseAgent(object): """ Base class for all agent flavors """ client = None sensor = None - secrets_matcher = 'contains-ignore-case' - secrets_list = ['key', 'pass', 'secret'] options = None def __init__(self): diff --git a/instana/agent/host.py b/instana/agent/host.py index 86e4cdc4..c6f768c2 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -135,8 +135,8 @@ def set_from(self, json_string): res_data = json.loads(raw_json) if "secrets" in res_data: - self.secrets_matcher = res_data['secrets']['matcher'] - self.secrets_list = res_data['secrets']['list'] + self.options.secrets_matcher = res_data['secrets']['matcher'] + self.options.secrets_list = res_data['secrets']['list'] if "extraHeaders" in res_data: # FIXME: add tests diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process/helper.py index 13a551c6..f6cf9095 100644 --- a/instana/collector/helpers/process/helper.py +++ b/instana/collector/helpers/process/helper.py @@ -3,7 +3,7 @@ import grp from ..base import BaseHelper from instana.log import logger -from instana.util import DictionaryOfStan, get_proc_cmdline, every, validate_url +from instana.util import DictionaryOfStan, get_proc_cmdline, strip_secrets class ProcessHelper(BaseHelper): diff --git a/instana/instrumentation/aiohttp/client.py b/instana/instrumentation/aiohttp/client.py index d63b2a49..142665a9 100644 --- a/instana/instrumentation/aiohttp/client.py +++ b/instana/instrumentation/aiohttp/client.py @@ -28,7 +28,7 @@ async def stan_request_start(session, trace_config_ctx, params): parts = str(params.url).split('?') if len(parts) > 1: - cleaned_qp = strip_secrets(parts[1], agent.secrets_matcher, agent.secrets_list) + cleaned_qp = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) scope.span.set_tag("http.url", parts[0]) scope.span.set_tag('http.method', params.method) diff --git a/instana/instrumentation/aiohttp/server.py b/instana/instrumentation/aiohttp/server.py index a341d7aa..0c8fa9d6 100644 --- a/instana/instrumentation/aiohttp/server.py +++ b/instana/instrumentation/aiohttp/server.py @@ -25,7 +25,7 @@ async def stan_middleware(request, handler): url = str(request.url) parts = url.split('?') if len(parts) > 1: - cleaned_qp = strip_secrets(parts[1], agent.secrets_matcher, agent.secrets_list) + cleaned_qp = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) scope.span.set_tag("http.url", parts[0]) diff --git a/instana/instrumentation/django/middleware.py b/instana/instrumentation/django/middleware.py index 98555d53..f272946e 100644 --- a/instana/instrumentation/django/middleware.py +++ b/instana/instrumentation/django/middleware.py @@ -43,7 +43,7 @@ def process_request(self, request): if 'PATH_INFO' in env: request.iscope.span.set_tag(ext.HTTP_URL, env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.secrets_matcher, agent.secrets_list) + scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) request.iscope.span.set_tag("http.params", scrubbed_params) if 'HTTP_HOST' in env: request.iscope.span.set_tag("http.host", env['HTTP_HOST']) diff --git a/instana/instrumentation/flask/vanilla.py b/instana/instrumentation/flask/vanilla.py index e62ee020..ef9b1dbf 100644 --- a/instana/instrumentation/flask/vanilla.py +++ b/instana/instrumentation/flask/vanilla.py @@ -36,7 +36,7 @@ def before_request_with_instana(*argv, **kwargs): if 'PATH_INFO' in env: span.set_tag(ext.HTTP_URL, env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.secrets_matcher, agent.secrets_list) + scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) span.set_tag("http.params", scrubbed_params) if 'HTTP_HOST' in env: span.set_tag("http.host", env['HTTP_HOST']) diff --git a/instana/instrumentation/flask/with_blinker.py b/instana/instrumentation/flask/with_blinker.py index 183b2c8a..b422572d 100644 --- a/instana/instrumentation/flask/with_blinker.py +++ b/instana/instrumentation/flask/with_blinker.py @@ -37,7 +37,7 @@ def request_started_with_instana(sender, **extra): if 'PATH_INFO' in env: span.set_tag(ext.HTTP_URL, env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.secrets_matcher, agent.secrets_list) + scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) span.set_tag("http.params", scrubbed_params) if 'HTTP_HOST' in env: span.set_tag("http.host", env['HTTP_HOST']) diff --git a/instana/instrumentation/pyramid/tweens.py b/instana/instrumentation/pyramid/tweens.py index 78d1da90..642fdc7c 100644 --- a/instana/instrumentation/pyramid/tweens.py +++ b/instana/instrumentation/pyramid/tweens.py @@ -36,7 +36,7 @@ def __call__(self, request): scope.span.set_tag("http.%s" % custom_header, request.headers[h]) if len(request.query_string): - scrubbed_params = strip_secrets(request.query_string, agent.secrets_matcher, agent.secrets_list) + scrubbed_params = strip_secrets(request.query_string, agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", scrubbed_params) response = None diff --git a/instana/instrumentation/tornado/client.py b/instana/instrumentation/tornado/client.py index f3f2890d..01c3e06b 100644 --- a/instana/instrumentation/tornado/client.py +++ b/instana/instrumentation/tornado/client.py @@ -49,7 +49,7 @@ def fetch_with_instana(wrapped, instance, argv, kwargs): # Query param scrubbing parts = request.url.split('?') if len(parts) > 1: - cleaned_qp = strip_secrets(parts[1], agent.secrets_matcher, agent.secrets_list) + cleaned_qp = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) scope.span.set_tag("http.url", parts[0]) diff --git a/instana/instrumentation/tornado/server.py b/instana/instrumentation/tornado/server.py index cb71628a..638db1b7 100644 --- a/instana/instrumentation/tornado/server.py +++ b/instana/instrumentation/tornado/server.py @@ -29,7 +29,7 @@ def execute_with_instana(wrapped, instance, argv, kwargs): # Query param scrubbing if instance.request.query is not None and len(instance.request.query) > 0: - cleaned_qp = strip_secrets(instance.request.query, agent.secrets_matcher, agent.secrets_list) + cleaned_qp = strip_secrets(instance.request.query, agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) url = "%s://%s%s" % (instance.request.protocol, instance.request.host, instance.request.path) diff --git a/instana/instrumentation/urllib3.py b/instana/instrumentation/urllib3.py index f36713eb..716402e6 100644 --- a/instana/instrumentation/urllib3.py +++ b/instana/instrumentation/urllib3.py @@ -32,7 +32,7 @@ def collect(instance, args, kwargs): parts = kvs['path'].split('?') kvs['path'] = parts[0] if len(parts) == 2: - kvs['query'] = strip_secrets(parts[1], agent.secrets_matcher, agent.secrets_list) + kvs['query'] = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) if type(instance) is urllib3.connectionpool.HTTPSConnectionPool: kvs['url'] = 'https://%s:%d%s' % (kvs['host'], kvs['port'], kvs['path']) diff --git a/instana/instrumentation/webapp2_inst.py b/instana/instrumentation/webapp2_inst.py index 435a57bb..5462bb2f 100644 --- a/instana/instrumentation/webapp2_inst.py +++ b/instana/instrumentation/webapp2_inst.py @@ -51,7 +51,7 @@ def new_start_response(status, headers, exc_info=None): if 'PATH_INFO' in env: scope.span.set_tag('http.path', env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.secrets_matcher, agent.secrets_list) + scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", scrubbed_params) if 'REQUEST_METHOD' in env: scope.span.set_tag(tags.HTTP_METHOD, env['REQUEST_METHOD']) diff --git a/instana/options.py b/instana/options.py index 7c727772..1242078a 100644 --- a/instana/options.py +++ b/instana/options.py @@ -1,7 +1,8 @@ """ Options for the in-process Instana agent """ -import logging import os +import logging +from .log import logger from .util import determine_service_name @@ -18,7 +19,21 @@ def __init__(self, **kwds): if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: self.extra_http_headers = str(os.environ["INSTANA_EXTRA_HTTP_HEADERS"]).lower().split(';') + # Defaults + self.secrets_matcher = 'contains-ignore-case' + self.secrets_list = ['key', 'pass', 'secret'] + + # Env var format: :[,] self.secrets = os.environ.get("INSTANA_SECRETS", None) + + if self.secrets is not None: + parts = self.secrets.split(':') + if len(parts) == 2: + self.secrets_matcher = parts[0] + self.secrets_list = parts[1].split(',') + else: + logger.warning("Couldn't parse INSTANA_SECRETS env var: %s", self.secrets) + self.__dict__.update(kwds) diff --git a/instana/wsgi.py b/instana/wsgi.py index 5c72d130..47824150 100644 --- a/instana/wsgi.py +++ b/instana/wsgi.py @@ -45,7 +45,7 @@ def new_start_response(status, headers, exc_info=None): if 'PATH_INFO' in env: self.scope.span.set_tag('http.path', env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.secrets_matcher, agent.secrets_list) + scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) self.scope.span.set_tag("http.params", scrubbed_params) if 'REQUEST_METHOD' in env: self.scope.span.set_tag(tags.HTTP_METHOD, env['REQUEST_METHOD']) diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index 9f3fb374..4d2531ed 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -39,6 +39,8 @@ def tearDown(self): os.environ.pop("INSTANA_ENDPOINT_URL") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_SECRETS" in os.environ: + os.environ.pop("INSTANA_SECRETS") set_agent(self.original_agent) set_tracer(self.original_tracer) @@ -50,6 +52,11 @@ def create_agent_and_setup_tracer(self): set_agent(self.agent) set_tracer(self.tracer) + def test_has_options(self): + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent, 'options')) + self.assertTrue(type(self.agent.options) is AWSFargateOptions) + def test_invalid_options(self): # None of the required env vars are available... if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: @@ -63,17 +70,22 @@ def test_invalid_options(self): self.assertFalse(agent._can_send) self.assertIsNone(agent.collector) - def test_secrets(self): + def test_default_secrets(self): self.create_agent_and_setup_tracer() - self.assertTrue(hasattr(self.agent, 'secrets_matcher')) - self.assertEqual(self.agent.secrets_matcher, 'contains-ignore-case') - self.assertTrue(hasattr(self.agent, 'secrets_list')) - self.assertEqual(self.agent.secrets_list, ['key', 'pass', 'secret']) - - def test_has_options(self): + self.assertIsNone(self.agent.options.secrets) + self.assertTrue(hasattr(self.agent.options, 'secrets_matcher')) + self.assertEqual(self.agent.options.secrets_matcher, 'contains-ignore-case') + self.assertTrue(hasattr(self.agent.options, 'secrets_list')) + self.assertEqual(self.agent.options.secrets_list, ['key', 'pass', 'secret']) + + def test_custom_secrets(self): + os.environ["INSTANA_SECRETS"] = "equals:love,war,games" self.create_agent_and_setup_tracer() - self.assertTrue(hasattr(self.agent, 'options')) - self.assertTrue(type(self.agent.options) is AWSFargateOptions) + + self.assertTrue(hasattr(self.agent.options, 'secrets_matcher')) + self.assertEqual(self.agent.options.secrets_matcher, 'equals') + self.assertTrue(hasattr(self.agent.options, 'secrets_list')) + self.assertEqual(self.agent.options.secrets_list, ['love', 'war', 'games']) def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index 395f1369..bea2a054 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -97,9 +97,9 @@ def test_invalid_options(self): def test_secrets(self): self.create_agent_and_setup_tracer() self.assertTrue(hasattr(self.agent, 'secrets_matcher')) - self.assertEqual(self.agent.secrets_matcher, 'contains-ignore-case') + self.assertEqual(self.agent.options.secrets_matcher, 'contains-ignore-case') self.assertTrue(hasattr(self.agent, 'secrets_list')) - self.assertEqual(self.agent.secrets_list, ['key', 'pass', 'secret']) + self.assertEqual(self.agent.options.secrets_list, ['key', 'pass', 'secret']) def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() diff --git a/tests/test_agent.py b/tests/test_agent.py index 6dde387f..93075612 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -15,9 +15,9 @@ def tearDown(self): def test_secrets(self): self.assertTrue(hasattr(agent, 'secrets_matcher')) - self.assertEqual(agent.secrets_matcher, 'contains-ignore-case') + self.assertEqual(agent.options.secrets_matcher, 'contains-ignore-case') self.assertTrue(hasattr(agent, 'secrets_list')) - self.assertEqual(agent.secrets_list, ['key', 'pass', 'secret']) + self.assertEqual(agent.options.secrets_list, ['key', 'pass', 'secret']) def test_options_have_extra_http_headers(self): self.assertTrue(hasattr(agent, 'options')) From 8fc3464975cbcb2e87767710939135bf0944f0f5 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 30 Jul 2020 10:02:31 +0000 Subject: [PATCH 47/91] Apply secrets check to procces env vars; cleanup --- instana/collector/helpers/process/helper.py | 9 +++- instana/instrumentation/aiohttp/client.py | 4 +- instana/instrumentation/aiohttp/server.py | 4 +- instana/instrumentation/django/middleware.py | 4 +- instana/instrumentation/flask/vanilla.py | 4 +- instana/instrumentation/flask/with_blinker.py | 4 +- instana/instrumentation/pyramid/tweens.py | 4 +- instana/instrumentation/tornado/client.py | 4 +- instana/instrumentation/tornado/server.py | 4 +- instana/instrumentation/urllib3.py | 4 +- instana/instrumentation/webapp2_inst.py | 4 +- instana/util.py | 54 +++++++++++++++++-- instana/wsgi.py | 4 +- tests/test_secrets.py | 30 +++++------ 14 files changed, 94 insertions(+), 43 deletions(-) diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process/helper.py index f6cf9095..69c733d4 100644 --- a/instana/collector/helpers/process/helper.py +++ b/instana/collector/helpers/process/helper.py @@ -3,7 +3,7 @@ import grp from ..base import BaseHelper from instana.log import logger -from instana.util import DictionaryOfStan, get_proc_cmdline, strip_secrets +from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret class ProcessHelper(BaseHelper): @@ -16,7 +16,12 @@ def collect_metrics(self, with_snapshot = False): plugin_data["data"]["pid"] = int(os.getpid()) env = dict() for key in os.environ: - env[key] = os.environ[key] + if contains_secret(key, + self.collector.agent.options.secrets_matcher, + self.collector.agent.options.secrets_list): + env[key] = "" + else: + env[key] = os.environ[key] plugin_data["data"]["env"] = env plugin_data["data"]["exec"] = os.readlink("/proc/self/exe") diff --git a/instana/instrumentation/aiohttp/client.py b/instana/instrumentation/aiohttp/client.py index 142665a9..31d5b26a 100644 --- a/instana/instrumentation/aiohttp/client.py +++ b/instana/instrumentation/aiohttp/client.py @@ -5,7 +5,7 @@ from ...log import logger from ...singletons import agent, async_tracer -from ...util import strip_secrets +from ...util import strip_secrets_from_query try: @@ -28,7 +28,7 @@ async def stan_request_start(session, trace_config_ctx, params): parts = str(params.url).split('?') if len(parts) > 1: - cleaned_qp = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) + cleaned_qp = strip_secrets_from_query(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) scope.span.set_tag("http.url", parts[0]) scope.span.set_tag('http.method', params.method) diff --git a/instana/instrumentation/aiohttp/server.py b/instana/instrumentation/aiohttp/server.py index 0c8fa9d6..ac62e1bb 100644 --- a/instana/instrumentation/aiohttp/server.py +++ b/instana/instrumentation/aiohttp/server.py @@ -5,7 +5,7 @@ from ...log import logger from ...singletons import agent, async_tracer -from ...util import strip_secrets +from ...util import strip_secrets_from_query try: @@ -25,7 +25,7 @@ async def stan_middleware(request, handler): url = str(request.url) parts = url.split('?') if len(parts) > 1: - cleaned_qp = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) + cleaned_qp = strip_secrets_from_query(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) scope.span.set_tag("http.url", parts[0]) diff --git a/instana/instrumentation/django/middleware.py b/instana/instrumentation/django/middleware.py index f272946e..5d2c9b43 100644 --- a/instana/instrumentation/django/middleware.py +++ b/instana/instrumentation/django/middleware.py @@ -9,7 +9,7 @@ from ...log import logger from ...singletons import agent, tracer -from ...util import strip_secrets +from ...util import strip_secrets_from_query DJ_INSTANA_MIDDLEWARE = 'instana.instrumentation.django.middleware.InstanaMiddleware' @@ -43,7 +43,7 @@ def process_request(self, request): if 'PATH_INFO' in env: request.iscope.span.set_tag(ext.HTTP_URL, env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) + scrubbed_params = strip_secrets_from_query(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) request.iscope.span.set_tag("http.params", scrubbed_params) if 'HTTP_HOST' in env: request.iscope.span.set_tag("http.host", env['HTTP_HOST']) diff --git a/instana/instrumentation/flask/vanilla.py b/instana/instrumentation/flask/vanilla.py index ef9b1dbf..e025876e 100644 --- a/instana/instrumentation/flask/vanilla.py +++ b/instana/instrumentation/flask/vanilla.py @@ -9,7 +9,7 @@ from ...log import logger from ...singletons import agent, tracer -from ...util import strip_secrets +from ...util import strip_secrets_from_query path_tpl_re = re.compile('<.*>') @@ -36,7 +36,7 @@ def before_request_with_instana(*argv, **kwargs): if 'PATH_INFO' in env: span.set_tag(ext.HTTP_URL, env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) + scrubbed_params = strip_secrets_from_query(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) span.set_tag("http.params", scrubbed_params) if 'HTTP_HOST' in env: span.set_tag("http.host", env['HTTP_HOST']) diff --git a/instana/instrumentation/flask/with_blinker.py b/instana/instrumentation/flask/with_blinker.py index b422572d..6ffb58d9 100644 --- a/instana/instrumentation/flask/with_blinker.py +++ b/instana/instrumentation/flask/with_blinker.py @@ -6,7 +6,7 @@ import opentracing.ext.tags as ext from ...log import logger -from ...util import strip_secrets +from ...util import strip_secrets_from_query from ...singletons import agent, tracer import flask @@ -37,7 +37,7 @@ def request_started_with_instana(sender, **extra): if 'PATH_INFO' in env: span.set_tag(ext.HTTP_URL, env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) + scrubbed_params = strip_secrets_from_query(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) span.set_tag("http.params", scrubbed_params) if 'HTTP_HOST' in env: span.set_tag("http.host", env['HTTP_HOST']) diff --git a/instana/instrumentation/pyramid/tweens.py b/instana/instrumentation/pyramid/tweens.py index 642fdc7c..5ac1d3ec 100644 --- a/instana/instrumentation/pyramid/tweens.py +++ b/instana/instrumentation/pyramid/tweens.py @@ -7,7 +7,7 @@ from ...log import logger from ...singletons import tracer, agent -from ...util import strip_secrets +from ...util import strip_secrets_from_query class InstanaTweenFactory(object): @@ -36,7 +36,7 @@ def __call__(self, request): scope.span.set_tag("http.%s" % custom_header, request.headers[h]) if len(request.query_string): - scrubbed_params = strip_secrets(request.query_string, agent.options.secrets_matcher, agent.options.secrets_list) + scrubbed_params = strip_secrets_from_query(request.query_string, agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", scrubbed_params) response = None diff --git a/instana/instrumentation/tornado/client.py b/instana/instrumentation/tornado/client.py index 01c3e06b..df26eb7d 100644 --- a/instana/instrumentation/tornado/client.py +++ b/instana/instrumentation/tornado/client.py @@ -6,7 +6,7 @@ from ...log import logger from ...singletons import agent, setup_tornado_tracer, tornado_tracer -from ...util import strip_secrets +from ...util import strip_secrets_from_query from distutils.version import LooseVersion @@ -49,7 +49,7 @@ def fetch_with_instana(wrapped, instance, argv, kwargs): # Query param scrubbing parts = request.url.split('?') if len(parts) > 1: - cleaned_qp = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) + cleaned_qp = strip_secrets_from_query(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) scope.span.set_tag("http.url", parts[0]) diff --git a/instana/instrumentation/tornado/server.py b/instana/instrumentation/tornado/server.py index 638db1b7..d563ef00 100644 --- a/instana/instrumentation/tornado/server.py +++ b/instana/instrumentation/tornado/server.py @@ -5,7 +5,7 @@ from ...log import logger from ...singletons import agent, setup_tornado_tracer, tornado_tracer -from ...util import strip_secrets +from ...util import strip_secrets_from_query from distutils.version import LooseVersion @@ -29,7 +29,7 @@ def execute_with_instana(wrapped, instance, argv, kwargs): # Query param scrubbing if instance.request.query is not None and len(instance.request.query) > 0: - cleaned_qp = strip_secrets(instance.request.query, agent.options.secrets_matcher, agent.options.secrets_list) + cleaned_qp = strip_secrets_from_query(instance.request.query, agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", cleaned_qp) url = "%s://%s%s" % (instance.request.protocol, instance.request.host, instance.request.path) diff --git a/instana/instrumentation/urllib3.py b/instana/instrumentation/urllib3.py index 716402e6..21a11e81 100644 --- a/instana/instrumentation/urllib3.py +++ b/instana/instrumentation/urllib3.py @@ -6,7 +6,7 @@ from ..log import logger from ..singletons import agent, tracer -from ..util import strip_secrets +from ..util import strip_secrets_from_query try: import urllib3 @@ -32,7 +32,7 @@ def collect(instance, args, kwargs): parts = kvs['path'].split('?') kvs['path'] = parts[0] if len(parts) == 2: - kvs['query'] = strip_secrets(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) + kvs['query'] = strip_secrets_from_query(parts[1], agent.options.secrets_matcher, agent.options.secrets_list) if type(instance) is urllib3.connectionpool.HTTPSConnectionPool: kvs['url'] = 'https://%s:%d%s' % (kvs['host'], kvs['port'], kvs['path']) diff --git a/instana/instrumentation/webapp2_inst.py b/instana/instrumentation/webapp2_inst.py index 5462bb2f..2452862a 100644 --- a/instana/instrumentation/webapp2_inst.py +++ b/instana/instrumentation/webapp2_inst.py @@ -6,7 +6,7 @@ from ..log import logger from ..singletons import agent, tracer -from ..util import strip_secrets +from ..util import strip_secrets_from_query try: @@ -51,7 +51,7 @@ def new_start_response(status, headers, exc_info=None): if 'PATH_INFO' in env: scope.span.set_tag('http.path', env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) + scrubbed_params = strip_secrets_from_query(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) scope.span.set_tag("http.params", scrubbed_params) if 'REQUEST_METHOD' in env: scope.span.set_tag(tags.HTTP_METHOD, env['REQUEST_METHOD']) diff --git a/instana/util.py b/instana/util.py index 1d735a5a..fd6edc76 100644 --- a/instana/util.py +++ b/instana/util.py @@ -139,7 +139,53 @@ def package_version(): return version -def strip_secrets(qp, matcher, kwlist): +def contains_secret(candidate, matcher, kwlist): + """ + This function will indicate whether contains a secret as described here: + https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#secrets + + :param candidate: string to check + :param matcher: the matcher to use + :param kwlist: the list of keywords to match + :return: boolean + """ + try: + if candidate is None or candidate == "INSTANA_AGENT_KEY": + return False + + if type(kwlist) is not list: + logger.debug("contains_secret: bad keyword list") + return False + + if matcher == 'equals-ignore-case': + for keyword in kwlist: + if candidate.lower() == keyword.lower(): + return True + elif matcher == 'equals': + for keyword in kwlist: + if candidate == keyword: + return True + elif matcher == 'contains-ignore-case': + for keyword in kwlist: + if keyword.lower() in candidate: + return True + elif matcher == 'contains': + for keyword in kwlist: + if keyword in candidate: + return True + elif matcher == 'regex': + for regexp in kwlist: + if re.match(regexp, candidate): + return True + else: + logger.debug("contains_secret: unknown matcher") + return False + + except Exception: + logger.debug("contains_secret", exc_info=True) + + +def strip_secrets_from_query(qp, matcher, kwlist): """ This function will scrub the secrets from a query param string based on the passed in matcher and kwlist. @@ -161,7 +207,7 @@ def strip_secrets(qp, matcher, kwlist): return '' if type(kwlist) is not list: - logger.debug("strip_secrets: bad keyword list") + logger.debug("strip_secrets_from_query: bad keyword list") return qp # If there are no key=values, then just return @@ -202,7 +248,7 @@ def strip_secrets(qp, matcher, kwlist): if re.match(regexp, kv[0]): params[index] = (kv[0], redacted) else: - logger.debug("strip_secrets: unknown matcher") + logger.debug("strip_secrets_from_query: unknown matcher") return qp if sys.version_info < (3, 0): @@ -216,7 +262,7 @@ def strip_secrets(qp, matcher, kwlist): return query except Exception: - logger.debug("strip_secrets", exc_info=True) + logger.debug("strip_secrets_from_query", exc_info=True) def sql_sanitizer(sql): diff --git a/instana/wsgi.py b/instana/wsgi.py index 47824150..77dfa8b9 100644 --- a/instana/wsgi.py +++ b/instana/wsgi.py @@ -4,7 +4,7 @@ import opentracing.ext.tags as tags from .singletons import agent, tracer -from .util import strip_secrets +from .util import strip_secrets_from_query class iWSGIMiddleware(object): @@ -45,7 +45,7 @@ def new_start_response(status, headers, exc_info=None): if 'PATH_INFO' in env: self.scope.span.set_tag('http.path', env['PATH_INFO']) if 'QUERY_STRING' in env and len(env['QUERY_STRING']): - scrubbed_params = strip_secrets(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) + scrubbed_params = strip_secrets_from_query(env['QUERY_STRING'], agent.options.secrets_matcher, agent.options.secrets_list) self.scope.span.set_tag("http.params", scrubbed_params) if 'REQUEST_METHOD' in env: self.scope.span.set_tag(tags.HTTP_METHOD, env['REQUEST_METHOD']) diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 51b1bc9d..5a6214d6 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -2,7 +2,7 @@ import unittest -from instana.util import strip_secrets +from instana.util import strip_secrets_from_query class TestSecrets(unittest.TestCase): @@ -18,7 +18,7 @@ def test_equals_ignore_case(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=&THREE=&4='+'&five='okyeah'") @@ -28,7 +28,7 @@ def test_equals(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=&THREE=&4='+'&five='okyeah'") @@ -38,7 +38,7 @@ def test_equals_no_match(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five='okyeah'") @@ -48,7 +48,7 @@ def test_contains_ignore_case(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five=") @@ -58,7 +58,7 @@ def test_contains_ignore_case_no_match(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five='okyeah'") @@ -68,7 +68,7 @@ def test_contains(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five=") @@ -78,7 +78,7 @@ def test_contains_no_match(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five='okyeah'") @@ -88,7 +88,7 @@ def test_regex(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4=&five='okyeah'") @@ -98,7 +98,7 @@ def test_regex_no_match(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five='okyeah'") @@ -108,7 +108,7 @@ def test_equals_with_path_component(self): query_params = "/signup?one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "/signup?one=1&Two=&THREE=&4='+'&five='okyeah'") @@ -118,7 +118,7 @@ def test_equals_with_full_url(self): query_params = "http://www.x.org/signup?one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "http://www.x.org/signup?one=1&Two=&THREE=&4='+'&five='okyeah'") @@ -128,7 +128,7 @@ def test_equals_with_none(self): query_params = None - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual('', stripped) @@ -138,7 +138,7 @@ def test_bad_matcher(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five='okyeah'") @@ -148,6 +148,6 @@ def test_bad_kwlist(self): query_params = "one=1&Two=two&THREE=&4='+'&five='okyeah'" - stripped = strip_secrets(query_params, matcher, kwlist) + stripped = strip_secrets_from_query(query_params, matcher, kwlist) self.assertEqual(stripped, "one=1&Two=two&THREE=&4='+'&five='okyeah'") From d260ef730b2031142998f9634298109a0a403356 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 30 Jul 2020 11:14:01 +0000 Subject: [PATCH 48/91] INSTANA_TAGS support; Helper stability improvements --- instana/collector/aws_fargate.py | 2 + .../collector/helpers/fargate/container.py | 2 +- instana/collector/helpers/fargate/docker.py | 2 +- instana/collector/helpers/fargate/hardware.py | 12 ++++-- instana/collector/helpers/fargate/host.py | 27 +++++++++++++ instana/collector/helpers/fargate/task.py | 40 +++++++++++-------- instana/options.py | 6 ++- tests/platforms/test_fargate.py | 25 ++++++++++++ 8 files changed, 92 insertions(+), 24 deletions(-) create mode 100644 instana/collector/helpers/fargate/host.py diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index bf9acac3..b7548fcc 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -8,6 +8,7 @@ from ..util import DictionaryOfStan, every, validate_url from ..singletons import env_is_test +from .helpers.fargate.host import HostHelper from .helpers.fargate.task import TaskHelper from .helpers.fargate.docker import DockerHelper from .helpers.process.helper import ProcessHelper @@ -78,6 +79,7 @@ def __init__(self, agent): self.helpers = [] # Populate the collection helpers + self.helpers.append(HostHelper(self)) self.helpers.append(TaskHelper(self)) self.helpers.append(DockerHelper(self)) self.helpers.append(ProcessHelper(self)) diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index a07b90f3..e3ea29bb 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -15,12 +15,12 @@ def collect_metrics(self, with_snapshot = False): containers = self.collector.task_metadata.get("Containers", []) for container in containers: plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.aws.ecs.container" try: labels = container.get("Labels", {}) name = container.get("Name", "") taskArn = labels.get("com.amazonaws.ecs.container-name", "") - plugin_data["name"] = "com.instana.plugin.aws.ecs.container" # "entityId": $taskARN + "::" + $containerName plugin_data["entityId"] = "%s::%s" % (taskArn, name) diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index 4d2b2542..31718928 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -15,12 +15,12 @@ def collect_metrics(self, with_snapshot = False): containers = self.collector.task_metadata.get("Containers", []) for container in containers: plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.docker" try: labels = container.get("Labels", {}) name = container.get("Name", "") taskArn = labels.get("com.amazonaws.ecs.container-name", "") - plugin_data["name"] = "com.instana.plugin.docker" # "entityId": $taskARN + "::" + $containerName plugin_data["entityId"] = "%s::%s" % (taskArn, name) plugin_data["data"] = DictionaryOfStan() diff --git a/instana/collector/helpers/fargate/hardware.py b/instana/collector/helpers/fargate/hardware.py index 3f504627..c8b868e3 100644 --- a/instana/collector/helpers/fargate/hardware.py +++ b/instana/collector/helpers/fargate/hardware.py @@ -17,10 +17,14 @@ def collect_metrics(self, with_snapshot = False): if self.collector.task_metadata is not None: plugin_data = dict() plugin_data["name"] = "com.instana.plugin.generic.hardware" - plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["availability-zone"] = self.collector.agent.options.zone - plugins.append(plugin_data) + try: + plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["availability-zone"] = self.collector.agent.options.zone + except: + logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) + finally: + plugins.append(plugin_data) except: logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) return plugins diff --git a/instana/collector/helpers/fargate/host.py b/instana/collector/helpers/fargate/host.py new file mode 100644 index 00000000..2efa6dcd --- /dev/null +++ b/instana/collector/helpers/fargate/host.py @@ -0,0 +1,27 @@ +from ....log import logger +from ..base import BaseHelper +from ....util import DictionaryOfStan + + +class HostHelper(BaseHelper): + def collect_metrics(self, with_snapshot = False): + """ + # This helper only sends snapshot data related to the INSTANA_TAGS environment variable + @return: list + """ + plugins = [] + if with_snapshot is False or self.collector.agent.options.tags is None: + return plugins + + plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.host" + try: + plugin_data["entityId"] = "h" + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["tags"] = self.collector.agent.options.tags + plugins.append(plugin_data) + except: + logger.debug("HostHelper.collect_metrics: ", exc_info=True) + finally: + plugins.append(plugin_data) + return plugins diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index 8ecb2d40..7e8ddd99 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -9,24 +9,30 @@ def collect_metrics(self, with_snapshot = False): Collect and return metrics data (and optionally snapshot data) for this task @return: list - with one plugin entity """ - plugin_data = dict() + plugins = [] try: if self.collector.task_metadata is not None: - plugin_data["name"] = "com.instana.plugin.aws.ecs.task" - plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["taskArn"] = self.collector.task_metadata.get("TaskARN", None) - plugin_data["data"]["clusterArn"] = self.collector.task_metadata.get("Cluster", None) - plugin_data["data"]["taskDefinition"] = self.collector.task_metadata.get("Family", None) - plugin_data["data"]["taskDefinitionVersion"] = self.collector.task_metadata.get("Revision", None) - plugin_data["data"]["availabilityZone"] = self.collector.task_metadata.get("AvailabilityZone", None) - plugin_data["data"]["desiredStatus"] = self.collector.task_metadata.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = self.collector.task_metadata.get("KnownStatus", None) - plugin_data["data"]["pullStartedAt"] = self.collector.task_metadata.get("PullStartedAt", None) - plugin_data["data"]["pullStoppedAt"] = self.collector.task_metadata.get("PullStoppeddAt", None) - limits = self.collector.task_metadata.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + try: + plugin_data = dict() + plugin_data["name"] = "com.instana.plugin.aws.ecs.task" + plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) + plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["taskArn"] = self.collector.task_metadata.get("TaskARN", None) + plugin_data["data"]["clusterArn"] = self.collector.task_metadata.get("Cluster", None) + plugin_data["data"]["taskDefinition"] = self.collector.task_metadata.get("Family", None) + plugin_data["data"]["taskDefinitionVersion"] = self.collector.task_metadata.get("Revision", None) + plugin_data["data"]["availabilityZone"] = self.collector.task_metadata.get("AvailabilityZone", None) + plugin_data["data"]["desiredStatus"] = self.collector.task_metadata.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = self.collector.task_metadata.get("KnownStatus", None) + plugin_data["data"]["pullStartedAt"] = self.collector.task_metadata.get("PullStartedAt", None) + plugin_data["data"]["pullStoppedAt"] = self.collector.task_metadata.get("PullStoppeddAt", None) + limits = self.collector.task_metadata.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + except: + logger.debug("collect_task_metrics: ", exc_info=True) + finally: + plugins.append(plugin_data) except: logger.debug("collect_task_metrics: ", exc_info=True) - return [plugin_data] + return plugins diff --git a/instana/options.py b/instana/options.py index 1242078a..7911b920 100644 --- a/instana/options.py +++ b/instana/options.py @@ -81,8 +81,12 @@ def __init__(self, **kwds): if self.endpoint_url is not None and self.endpoint_url[-1] == "/": self.endpoint_url = self.endpoint_url[:-1] + self.tags = None + tag_list = os.environ.get("INSTANA_TAGS", None) + if tag_list is not None: + self.tags = tag_list.split(',') + self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) - self.tags = os.environ.get("INSTANA_TAGS", None) self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.zone = os.environ.get("INSTANA_ZONE", None) diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index 4d2531ed..e25b2fec 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -41,6 +41,8 @@ def tearDown(self): os.environ.pop("INSTANA_AGENT_KEY") if "INSTANA_SECRETS" in os.environ: os.environ.pop("INSTANA_SECRETS") + if "INSTANA_TAGS" in os.environ: + os.environ.pop("INSTANA_TAGS") set_agent(self.original_agent) set_tracer(self.original_tracer) @@ -87,6 +89,29 @@ def test_custom_secrets(self): self.assertTrue(hasattr(self.agent.options, 'secrets_list')) self.assertEqual(self.agent.options.secrets_list, ['love', 'war', 'games']) + def test_default_tags(self): + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent.options, 'tags')) + self.assertIsNone(self.agent.options.tags) + + def test_custom_tags(self): + os.environ["INSTANA_TAGS"] = "love,war,games" + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent.options, 'tags')) + self.assertEqual(self.agent.options.tags, ["love", "war", "games"]) + + payload = self.agent.collector.prepare_payload() + + assert(payload) + host_plugin = None + plugins = payload['metrics']['plugins'] + for plugin in plugins: + if plugin["name"] == "com.instana.plugin.host": + host_plugin = plugin + assert(host_plugin) + assert(host_plugin["entityId"] == "h") + assert(host_plugin["data"]["tags"] == ['love', 'war', 'games']) + def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() self.assertTrue(hasattr(self.agent, 'options')) From b63a257b8daf5dbba2ddc2331257144bbd52af9f Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 30 Jul 2020 12:11:16 +0000 Subject: [PATCH 49/91] Fix test stragglers --- tests/platforms/test_lambda.py | 4 ++-- tests/test_agent.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index bea2a054..3be351fc 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -96,9 +96,9 @@ def test_invalid_options(self): def test_secrets(self): self.create_agent_and_setup_tracer() - self.assertTrue(hasattr(self.agent, 'secrets_matcher')) + self.assertTrue(hasattr(self.agent.options, 'secrets_matcher')) self.assertEqual(self.agent.options.secrets_matcher, 'contains-ignore-case') - self.assertTrue(hasattr(self.agent, 'secrets_list')) + self.assertTrue(hasattr(self.agent.options, 'secrets_list')) self.assertEqual(self.agent.options.secrets_list, ['key', 'pass', 'secret']) def test_has_extra_http_headers(self): diff --git a/tests/test_agent.py b/tests/test_agent.py index 93075612..ba349b22 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -14,9 +14,9 @@ def tearDown(self): pass def test_secrets(self): - self.assertTrue(hasattr(agent, 'secrets_matcher')) + self.assertTrue(hasattr(agent.options, 'secrets_matcher')) self.assertEqual(agent.options.secrets_matcher, 'contains-ignore-case') - self.assertTrue(hasattr(agent, 'secrets_list')) + self.assertTrue(hasattr(agent.options, 'secrets_list')) self.assertEqual(agent.options.secrets_list, ['key', 'pass', 'secret']) def test_options_have_extra_http_headers(self): From fa4869c015e373e09dac1998ce1a8ed4b1464a8a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 30 Jul 2020 12:59:46 +0000 Subject: [PATCH 50/91] Add support for INSTANA_ENDPOINT_PROXY --- instana/agent/aws_fargate.py | 5 +++-- instana/agent/aws_lambda.py | 3 ++- instana/options.py | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index a2abc981..9fc8177f 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -26,9 +26,9 @@ class AWSFargateAgent(BaseAgent): def __init__(self): super(AWSFargateAgent, self).__init__() + self.options = AWSFargateOptions() self.from_ = AWSFargateFrom() self.collector = None - self.options = AWSFargateOptions() self.report_headers = None self._can_send = False @@ -78,7 +78,8 @@ def report_data_payload(self, payload): data=to_json(payload), headers=self.report_headers, timeout=self.options.timeout, - verify=ssl_verify) + verify=ssl_verify, + proxies=self.options.endpoint_proxy) if not 200 <= response.status_code < 300: logger.info("report_data_payload: Instana responded with status code %s", response.status_code) diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index cb00cc1e..122a7495 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -79,7 +79,8 @@ def report_data_payload(self, payload): data=to_json(payload), headers=self.report_headers, timeout=self.options.timeout, - verify=ssl_verify) + verify=ssl_verify, + proxies = self.options.endpoint_proxy) if 200 <= response.status_code < 300: logger.debug("report_data_payload: Instana responded with status code %s", response.status_code) diff --git a/instana/options.py b/instana/options.py index 7911b920..e4df83db 100644 --- a/instana/options.py +++ b/instana/options.py @@ -57,13 +57,18 @@ class AWSLambdaOptions(BaseOptions): def __init__(self, **kwds): super(AWSLambdaOptions, self).__init__() + self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) - # Remove any trailing slash (if any) if self.endpoint_url is not None and self.endpoint_url[-1] == "/": self.endpoint_url = self.endpoint_url[:-1] - self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) + proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) + if proxy is None: + self.endpoint_proxy = { } + else: + self.endpoint_proxy = {'https': proxy } + self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) @@ -74,19 +79,23 @@ def __init__(self, **kwds): super(AWSFargateOptions, self).__init__() self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) - self.endpoint_proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) - self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) # Remove any trailing slash (if any) if self.endpoint_url is not None and self.endpoint_url[-1] == "/": self.endpoint_url = self.endpoint_url[:-1] + proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) + if proxy is None: + self.endpoint_proxy = { } + else: + self.endpoint_proxy = {'https': proxy } + self.tags = None tag_list = os.environ.get("INSTANA_TAGS", None) if tag_list is not None: self.tags = tag_list.split(',') - self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) + self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) self.zone = os.environ.get("INSTANA_ZONE", None) From 94563847cfdf5c11fedbf8d54e75c43accdb711a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 5 Aug 2020 10:10:39 +0200 Subject: [PATCH 51/91] Endpoint proxy tests --- tests/platforms/test_fargate.py | 7 +++++++ tests/platforms/test_lambda.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index e25b2fec..a9371877 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -37,6 +37,8 @@ def tearDown(self): os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") if "INSTANA_ENDPOINT_URL" in os.environ: os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_ENDPOINT_PROXY" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_PROXY") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") if "INSTANA_SECRETS" in os.environ: @@ -124,6 +126,11 @@ def test_agent_extra_http_headers(self): should_headers = ['x-test-header', 'x-another-header', 'x-and-another-header'] self.assertEqual(should_headers, self.agent.options.extra_http_headers) + def test_custom_proxy(self): + os.environ["INSTANA_ENDPOINT_PROXY"] = "http://myproxy.123" + self.create_agent_and_setup_tracer() + assert(self.agent.options.endpoint_proxy == { 'https': "http://myproxy.123" }) + @pytest.mark.skip("todo") def test_custom_service_name(self): os.environ['INSTANA_SERVICE_NAME'] = "Legion" diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index 3be351fc..d25f3f9d 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -110,6 +110,7 @@ def test_has_options(self): self.create_agent_and_setup_tracer() self.assertTrue(hasattr(self.agent, 'options')) self.assertTrue(type(self.agent.options) is AWSLambdaOptions) + assert(self.agent.options.endpoint_proxy == { }) def test_get_handler(self): os.environ["LAMBDA_HANDLER"] = "tests.lambda_handler" @@ -125,6 +126,11 @@ def test_agent_extra_http_headers(self): should_headers = ['x-test-header', 'x-another-header', 'x-and-another-header'] self.assertEqual(should_headers, self.agent.options.extra_http_headers) + def test_custom_proxy(self): + os.environ["INSTANA_ENDPOINT_PROXY"] = "http://myproxy.123" + self.create_agent_and_setup_tracer() + assert(self.agent.options.endpoint_proxy == { 'https': "http://myproxy.123" }) + def test_custom_service_name(self): os.environ['INSTANA_SERVICE_NAME'] = "Legion" with open(self.pwd + '/../data/lambda/api_gateway_event.json', 'r') as json_file: From 7e284717196e200e52109ae7ea605077e5317b73 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 5 Aug 2020 10:24:46 +0200 Subject: [PATCH 52/91] Reset proxy var in tests --- tests/platforms/test_lambda.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index d25f3f9d..43891ffa 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -66,6 +66,8 @@ def tearDown(self): os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") if "INSTANA_ENDPOINT_URL" in os.environ: os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_ENDPOINT_PROXY" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_PROXY") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") From fc78660a4caa8a1ed7be9ea87dd92b40f31820a5 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 5 Aug 2020 11:57:15 +0200 Subject: [PATCH 53/91] Add Lambda & Fargate boot messages --- instana/agent/aws_fargate.py | 8 ++++++++ instana/agent/aws_lambda.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 9fc8177f..0e198c31 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -4,6 +4,7 @@ """ import os import time +import pkg_resources from ..log import logger from ..util import to_json from .base import BaseAgent @@ -32,6 +33,13 @@ def __init__(self): self.report_headers = None self._can_send = False + package_version = 'unknown' + try: + package_version = pkg_resources.get_distribution('instana').version + except pkg_resources.DistributionNotFound: + pass + logger.info("Stan is on the AWS Fargate scene. Starting Instana instrumentation version: %s", package_version) + if self._validate_options(): self._can_send = True self.collector = AWSFargateCollector(self) diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index 122a7495..dd232ebe 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -4,6 +4,7 @@ """ import os import time +import pkg_resources from ..log import logger from ..util import to_json from .base import BaseAgent @@ -32,6 +33,13 @@ def __init__(self): self.report_headers = None self._can_send = False + package_version = 'unknown' + try: + package_version = pkg_resources.get_distribution('instana').version + except pkg_resources.DistributionNotFound: + pass + logger.info("Stan is on the AWS Lambda scene. Starting Instana instrumentation version: %s", package_version) + if self._validate_options(): self._can_send = True self.collector = AWSLambdaCollector(self) From 9f60d880757ae0b6db0edf21fe5b043c00874f7a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 5 Aug 2020 14:50:23 +0200 Subject: [PATCH 54/91] Version path for fargate --- tests/data/fargate/1.3.0/README.md | 2 + .../fargate/{ => 1.3.0}/root_metadata.json | 0 .../fargate/{ => 1.3.0}/stats_metadata.json | 0 .../fargate/{ => 1.3.0}/task_metadata.json | 0 .../{ => 1.3.0}/task_stats_metadata.json | 0 tests/platforms/test_fargate.py | 64 +------------------ tests/platforms/test_fargate_collector.py | 8 +-- 7 files changed, 7 insertions(+), 67 deletions(-) create mode 100644 tests/data/fargate/1.3.0/README.md rename tests/data/fargate/{ => 1.3.0}/root_metadata.json (100%) rename tests/data/fargate/{ => 1.3.0}/stats_metadata.json (100%) rename tests/data/fargate/{ => 1.3.0}/task_metadata.json (100%) rename tests/data/fargate/{ => 1.3.0}/task_stats_metadata.json (100%) diff --git a/tests/data/fargate/1.3.0/README.md b/tests/data/fargate/1.3.0/README.md new file mode 100644 index 00000000..ad4dc277 --- /dev/null +++ b/tests/data/fargate/1.3.0/README.md @@ -0,0 +1,2 @@ +... 1.3.0 being the AWS Fargate Platform version: +https://docs.aws.amazon.com/AmazonECS/latest/developerguide/platform_versions.html \ No newline at end of file diff --git a/tests/data/fargate/root_metadata.json b/tests/data/fargate/1.3.0/root_metadata.json similarity index 100% rename from tests/data/fargate/root_metadata.json rename to tests/data/fargate/1.3.0/root_metadata.json diff --git a/tests/data/fargate/stats_metadata.json b/tests/data/fargate/1.3.0/stats_metadata.json similarity index 100% rename from tests/data/fargate/stats_metadata.json rename to tests/data/fargate/1.3.0/stats_metadata.json diff --git a/tests/data/fargate/task_metadata.json b/tests/data/fargate/1.3.0/task_metadata.json similarity index 100% rename from tests/data/fargate/task_metadata.json rename to tests/data/fargate/1.3.0/task_metadata.json diff --git a/tests/data/fargate/task_stats_metadata.json b/tests/data/fargate/1.3.0/task_stats_metadata.json similarity index 100% rename from tests/data/fargate/task_stats_metadata.json rename to tests/data/fargate/1.3.0/task_stats_metadata.json diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index a9371877..a38446a5 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -129,66 +129,4 @@ def test_agent_extra_http_headers(self): def test_custom_proxy(self): os.environ["INSTANA_ENDPOINT_PROXY"] = "http://myproxy.123" self.create_agent_and_setup_tracer() - assert(self.agent.options.endpoint_proxy == { 'https': "http://myproxy.123" }) - - @pytest.mark.skip("todo") - def test_custom_service_name(self): - os.environ['INSTANA_SERVICE_NAME'] = "Legion" - with open(self.pwd + '/../data/lambda/api_gateway_event.json', 'r') as json_file: - event = json.load(json_file) - - self.create_agent_and_setup_tracer() - - # Call the Instana Fargate Handler as we do in the real world. It will initiate tracing and then - # figure out the original (the users') Fargate Handler and execute it. - # The original Fargate handler is set in os.environ["LAMBDA_HANDLER"] - # result = lambda_handler(event, self.context) - os.environ.pop('INSTANA_SERVICE_NAME') - - # self.assertEqual('All Ok', result) - payload = self.agent.collector.prepare_payload() - - self.assertTrue("metrics" in payload) - self.assertTrue("spans" in payload) - self.assertEqual(2, len(payload.keys())) - - self.assertTrue(type(payload['metrics']['plugins']) is list) - self.assertTrue(len(payload['metrics']['plugins']) == 1) - plugin_data = payload['metrics']['plugins'][0] - - self.assertEqual('com.instana.plugin.aws.lambda', plugin_data['name']) - self.assertEqual('arn:aws:lambda:us-east-2:12345:function:TestPython:1', plugin_data['entityId']) - - self.assertEqual(1, len(payload['spans'])) - - span = payload['spans'][0] - self.assertEqual('aws.lambda.entry', span.n) - self.assertIsNotNone(span.t) - self.assertIsNotNone(span.s) - self.assertIsNone(span.p) - self.assertIsNotNone(span.ts) - self.assertIsNotNone(span.d) - - self.assertEqual({'hl': True, 'cp': 'aws', 'e': 'arn:aws:lambda:us-east-2:12345:function:TestPython:1'}, - span.f) - - self.assertIsNone(span.ec) - self.assertIsNone(span.data['lambda']['error']) - - self.assertEqual('arn:aws:lambda:us-east-2:12345:function:TestPython:1', span.data['lambda']['arn']) - self.assertEqual(None, span.data['lambda']['alias']) - self.assertEqual('python', span.data['lambda']['runtime']) - self.assertEqual('TestPython', span.data['lambda']['functionName']) - self.assertEqual('1', span.data['lambda']['functionVersion']) - - self.assertEqual('Legion', span.data['service']) - - self.assertEqual('aws:api.gateway', span.data['lambda']['trigger']) - self.assertEqual('POST', span.data['http']['method']) - self.assertEqual('/path/to/resource', span.data['http']['url']) - self.assertEqual('/{proxy+}', span.data['http']['path_tpl']) - if sys.version[:3] == '2.7': - self.assertEqual(u"foo=[u'bar']", span.data['http']['params']) - else: - self.assertEqual("foo=['bar']", span.data['http']['params']) - + assert(self.agent.options.endpoint_proxy == { 'https': "http://myproxy.123" }) \ No newline at end of file diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 0bb98a91..0d93ba37 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -56,13 +56,13 @@ def create_agent_and_setup_tracer(self): set_tracer(self.tracer) # Manually set the ECS Metadata API results on the collector - with open(self.pwd + '/../data/fargate/root_metadata.json', 'r') as json_file: + with open(self.pwd + '/../data/fargate/1.3.0/root_metadata.json', 'r') as json_file: self.agent.collector.root_metadata = json.load(json_file) - with open(self.pwd + '/../data/fargate/task_metadata.json', 'r') as json_file: + with open(self.pwd + '/../data/fargate/1.3.0/task_metadata.json', 'r') as json_file: self.agent.collector.task_metadata = json.load(json_file) - with open(self.pwd + '/../data/fargate/stats_metadata.json', 'r') as json_file: + with open(self.pwd + '/../data/fargate/1.3.0/stats_metadata.json', 'r') as json_file: self.agent.collector.stats_metadata = json.load(json_file) - with open(self.pwd + '/../data/fargate/task_stats_metadata.json', 'r') as json_file: + with open(self.pwd + '/../data/fargate/1.3.0/task_stats_metadata.json', 'r') as json_file: self.agent.collector.task_stats_metadata = json.load(json_file) def test_prepare_payload(self): From 08e6955b9f9f39c0c73173e19a338cf6f88f946c Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 5 Aug 2020 15:11:21 +0200 Subject: [PATCH 55/91] Pylint fixes --- .pylintrc | 580 ++++++++++++++++++++ instana/agent/aws_fargate.py | 11 +- instana/agent/base.py | 1 - instana/agent/host.py | 40 +- instana/agent/test.py | 2 - instana/collector/aws_fargate.py | 11 +- instana/collector/aws_lambda.py | 5 +- instana/collector/base.py | 1 - instana/collector/helpers/runtime/helper.py | 10 +- tests/platforms/test_fargate.py | 17 +- tests/platforms/test_fargate_collector.py | 9 +- 11 files changed, 624 insertions(+), 63 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..64485097 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,580 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions= diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 0e198c31..c0a6ee1d 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -5,11 +5,11 @@ import os import time import pkg_resources +from instana.options import AWSFargateOptions +from instana.collector.aws_fargate import AWSFargateCollector from ..log import logger from ..util import to_json from .base import BaseAgent -from instana.collector.aws_fargate import AWSFargateCollector -from instana.options import AWSFargateOptions class AWSFargateFrom(object): @@ -91,10 +91,9 @@ def report_data_payload(self, payload): if not 200 <= response.status_code < 300: logger.info("report_data_payload: Instana responded with status code %s", response.status_code) - except Exception as e: - logger.debug("report_data_payload: connection error (%s)", type(e)) - finally: - return response + except Exception as exc: + logger.debug("report_data_payload: connection error (%s)", type(exc)) + return response def _validate_options(self): """ diff --git a/instana/agent/base.py b/instana/agent/base.py index c22f4ade..c1beff9c 100644 --- a/instana/agent/base.py +++ b/instana/agent/base.py @@ -9,4 +9,3 @@ class BaseAgent(object): def __init__(self): self.client = requests.Session() - diff --git a/instana/agent/host.py b/instana/agent/host.py index c6f768c2..5de7ca7a 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -172,10 +172,9 @@ def is_agent_listening(self, host, port): else: logger.debug("...something is listening on %s:%d but it's not the Instana Host Agent: %s", host, port, server_header) - except: + except Exception: logger.debug("Instana Host Agent not found on %s:%d", host, port) - finally: - return result + return result def announce(self, discovery): """ @@ -192,10 +191,9 @@ def announce(self, discovery): if response.status_code == 200: self.last_seen = datetime.now() - except Exception as e: - logger.debug("announce: connection error (%s)", type(e)) - finally: - return response + except Exception as exc: + logger.debug("announce: connection error (%s)", type(exc)) + return response def is_agent_ready(self): """ @@ -207,10 +205,9 @@ def is_agent_ready(self): if response.status_code == 200: ready = True - except Exception as e: - logger.debug("is_agent_ready: connection error (%s)", type(e)) - finally: - return ready + except Exception as exc: + logger.debug("is_agent_ready: connection error (%s)", type(exc)) + return ready def report_data_payload(self, entity_data): """ @@ -228,10 +225,9 @@ def report_data_payload(self, entity_data): if response.status_code == 200: self.last_seen = datetime.now() - except Exception as e: - logger.debug("report_data_payload: Instana host agent connection error (%s)", type(e)) - finally: - return response + except Exception as exc: + logger.debug("report_data_payload: Instana host agent connection error (%s)", type(exc)) + return response def report_traces(self, spans): """ @@ -253,10 +249,9 @@ def report_traces(self, spans): if response.status_code == 200: self.last_seen = datetime.now() - except Exception as e: - logger.debug("report_traces: Instana host agent connection error (%s)", type(e)) - finally: - return response + except Exception as exc: + logger.debug("report_traces: Instana host agent connection error (%s)", type(exc)) + return response def handle_agent_tasks(self, task): """ @@ -291,10 +286,9 @@ def __task_response(self, message_id, data): data=payload, headers={"Content-Type": "application/json"}, timeout=0.8) - except Exception as e: - logger.debug("__task_response: Instana host agent connection error (%s)", type(e)) - finally: - return response + except Exception as exc: + logger.debug("__task_response: Instana host agent connection error (%s)", type(exc)) + return response def __discovery_url(self): """ diff --git a/instana/agent/test.py b/instana/agent/test.py index 46d0b7e1..d8da94d7 100644 --- a/instana/agent/test.py +++ b/instana/agent/test.py @@ -28,5 +28,3 @@ def can_send(self): def report_traces(self, spans): logger.warning("Tried to report_traces with a TestAgent!") - - diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index b7548fcc..40523c6c 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,8 +1,9 @@ import os import json -import requests import threading from time import time +import requests + from ..log import logger from .base import BaseCollector from ..util import DictionaryOfStan, every, validate_url @@ -30,7 +31,7 @@ def __init__(self, agent): if self.ecmu == "" or validate_url(self.ecmu) is False: logger.warning("AWSFargateCollector: ECS_CONTAINER_METADATA_URI not in environment or invalid URL. " - "Instana will not be able to monitor this environment") + "Instana will not be able to monitor this environment") self.ready_to_start = False self.ecmu_url_root = self.ecmu @@ -173,7 +174,7 @@ def prepare_payload(self): if with_snapshot is True: self.snapshot_data_last_sent = int(time()) - except: + except Exception: logger.debug("collect_snapshot error", exc_info=True) return payload @@ -184,11 +185,11 @@ def get_fq_arn(self): if self.root_metadata is not None: labels = self.root_metadata.get("Labels", None) if labels is not None: - taskArn = labels.get("com.amazonaws.ecs.task-arn", "") + task_arn = labels.get("com.amazonaws.ecs.task-arn", "") container_name = self.root_metadata.get("Name", "") - self._fq_arn = taskArn + "::" + container_name + self._fq_arn = task_arn + "::" + container_name return self._fq_arn else: return "Missing ECMU metadata" diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index 61838f15..97f638e2 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -22,10 +22,9 @@ def collect_snapshot(self, event, context): plugin_data["name"] = "com.instana.plugin.aws.lambda" plugin_data["entityId"] = self.get_fq_arn() self.snapshot_data["plugins"] = [plugin_data] - except: + except Exception: logger.debug("collect_snapshot error", exc_info=True) - finally: - return self.snapshot_data + return self.snapshot_data def should_send_snapshot_data(self): return self.snapshot_data and self.snapshot_data_sent is False diff --git a/instana/collector/base.py b/instana/collector/base.py index 76092535..81940e50 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -2,7 +2,6 @@ A Collector launches a background thread and continually collects & reports data. The data can be any combination of metrics, snapshot data and spans. """ -import os import sys import threading diff --git a/instana/collector/helpers/runtime/helper.py b/instana/collector/helpers/runtime/helper.py index 054e6a5e..f41a6d2b 100644 --- a/instana/collector/helpers/runtime/helper.py +++ b/instana/collector/helpers/runtime/helper.py @@ -106,15 +106,14 @@ def gather_snapshot(self): snapshot_payload['a'] = platform.architecture()[0] # architecture snapshot_payload['versions'] = self.gather_python_packages() snapshot_payload['djmw'] = None # FIXME - except Exception as e: + except Exception: logger.debug("collect_snapshot: ", exc_info=True) - finally: - return snapshot_payload + return snapshot_payload def gather_python_packages(self): """ Collect up the list of modules in use """ + res = {} try: - res = {} m = sys.modules.copy() for k in m: @@ -138,8 +137,7 @@ def gather_python_packages(self): except Exception: logger.debug("gather_python_packages", exc_info=True) - else: - return res + return res def jsonable(self, value): try: diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index a38446a5..cfff4ad0 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -1,9 +1,6 @@ from __future__ import absolute_import import os -import sys -import json -import pytest import unittest from instana.tracer import InstanaTracer @@ -59,7 +56,7 @@ def create_agent_and_setup_tracer(self): def test_has_options(self): self.create_agent_and_setup_tracer() self.assertTrue(hasattr(self.agent, 'options')) - self.assertTrue(type(self.agent.options) is AWSFargateOptions) + self.assertTrue(isinstance(self.agent.options, AWSFargateOptions)) def test_invalid_options(self): # None of the required env vars are available... @@ -71,7 +68,7 @@ def test_invalid_options(self): os.environ.pop("INSTANA_AGENT_KEY") agent = AWSFargateAgent() - self.assertFalse(agent._can_send) + self.assertFalse(agent.can_send()) self.assertIsNone(agent.collector) def test_default_secrets(self): @@ -104,15 +101,15 @@ def test_custom_tags(self): payload = self.agent.collector.prepare_payload() - assert(payload) + assert payload host_plugin = None plugins = payload['metrics']['plugins'] for plugin in plugins: if plugin["name"] == "com.instana.plugin.host": host_plugin = plugin - assert(host_plugin) - assert(host_plugin["entityId"] == "h") - assert(host_plugin["data"]["tags"] == ['love', 'war', 'games']) + assert host_plugin + assert host_plugin["entityId"] == "h" + assert host_plugin["data"]["tags"] == ['love', 'war', 'games'] def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() @@ -129,4 +126,4 @@ def test_agent_extra_http_headers(self): def test_custom_proxy(self): os.environ["INSTANA_ENDPOINT_PROXY"] = "http://myproxy.123" self.create_agent_and_setup_tracer() - assert(self.agent.options.endpoint_proxy == { 'https': "http://myproxy.123" }) \ No newline at end of file + assert self.agent.options.endpoint_proxy == {'https': "http://myproxy.123"} diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 0d93ba37..5abdf705 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -1,13 +1,10 @@ from __future__ import absolute_import import os -import sys import json -import pytest import unittest from instana.tracer import InstanaTracer -from instana.options import AWSFargateOptions from instana.recorder import AWSFargateRecorder from instana.agent.aws_fargate import AWSFargateAgent from instana.singletons import get_agent, set_agent, get_tracer, set_tracer @@ -73,12 +70,12 @@ def test_prepare_payload(self): assert(len(payload.keys()) == 2) assert('spans' in payload) - assert(type(payload['spans']) is list) + assert(isinstance(payload['spans'], list)) assert(len(payload['spans']) == 0) assert('metrics' in payload) assert(len(payload['metrics'].keys()) == 1) assert('plugins' in payload['metrics']) - assert(type(payload['metrics']['plugins']) is list) + assert(isinstance(payload['metrics']['plugins'], list)) assert(len(payload['metrics']['plugins']) == 7) plugins = payload['metrics']['plugins'] @@ -101,7 +98,7 @@ def test_instana_zone(self): assert(payload) plugins = payload['metrics']['plugins'] - assert(type(plugins) is list) + assert(isinstance(plugins, list)) hardware_plugin = None for plugin in plugins: From 5ee5057398ac087910216464abe4f4389bf90f02 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 10:47:30 +0200 Subject: [PATCH 56/91] Docker metrics support --- instana/collector/aws_fargate.py | 6 + instana/collector/helpers/base.py | 59 ++++++- .../collector/helpers/fargate/container.py | 4 +- instana/collector/helpers/fargate/docker.py | 153 ++++++++++++++---- tests/platforms/test_fargate_collector.py | 110 ++++++++++++- 5 files changed, 300 insertions(+), 32 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 40523c6c..53f45ba0 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -1,3 +1,6 @@ +""" +Snapshot & metrics collection for AWS Fargate +""" import os import json import threading @@ -19,6 +22,7 @@ class AWSFargateCollector(BaseCollector): + """ Collector for AWS Fargate """ def __init__(self, agent): super(AWSFargateCollector, self).__init__(agent) logger.debug("Loading AWS Fargate Collector") @@ -124,11 +128,13 @@ def get_ecs_metadata(self): lock_acquired = self.ecmu_lock.acquire(False) if lock_acquired: try: + # FIXME: Only get once? or every 5 mins? # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/ json_body = self.http_client.get(self.ecmu_url_root, timeout=3).content self.root_metadata = json.loads(json_body) + # FIXME: Only get once? or every 5 mins? # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/task json_body = self.http_client.get(self.ecmu_url_task, timeout=3).content diff --git a/instana/collector/helpers/base.py b/instana/collector/helpers/base.py index e9f05f5c..613e05a2 100644 --- a/instana/collector/helpers/base.py +++ b/instana/collector/helpers/base.py @@ -5,5 +5,60 @@ class BaseHelper(object): def __init__(self, collector): self.collector = collector - def collect_metrics(self, with_sendsnapshot = False): - logger.debug("BaseHelper.collect_metrics must be overridden") \ No newline at end of file + def get_delta(self, source, previous, metric): + """ + Given a metric, see if the value varies from the previous reported metrics + + @param source [dict or value]: the dict to retrieve the new value of (as source[metric]) or + if not a dict, then the new value of the metric + @param previous [dict]: the previous value of that was reported (as previous[metric]) + @param metric [String or Tuple]: the name of the metric in question. If the keys for source[metric], + and previous[metric] vary, you can pass a tuple in the form of (src, dst) + @return: None (meaning no difference) or the new value (source[metric]) + """ + if isinstance(metric, tuple): + src_metric = metric[0] + dst_metric = metric[1] + else: + src_metric = metric + dst_metric = metric + + if isinstance(source, dict): + new_value = source.get(src_metric, None) + else: + new_value = source + + if previous[dst_metric] != new_value: + return new_value + else: + return None + + def apply_delta(self, source, previous, new, metric): + """ + Helper method to assist in delta reporting of metrics. + + @param source [dict or value]: the dict to retrieve the new value of (as source[metric]) or + if not a dict, then the new value of the metric + @param previous [dict]: the previous value of that was reported (as previous[metric]) + @param new [dict]: the new value of the metric that will be sent new (as new[metric]) + @param metric [String or Tuple]: the name of the metric in question. If the keys for source[metric], + previous[metric] and new[metric] vary, you can pass a tuple in the form of (src, dst) + @return: None + """ + if isinstance(metric, tuple): + src_metric = metric[0] + dst_metric = metric[1] + else: + src_metric = metric + dst_metric = metric + + if isinstance(source, dict): + new_value = source.get(src_metric, None) + else: + new_value = source + + if previous[dst_metric] != new_value: + previous[dst_metric] = new[dst_metric] = new_value + + def collect_metrics(self, with_snapshot=False): + logger.debug("BaseHelper.collect_metrics must be overridden") diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index e3ea29bb..f4d7e49c 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -19,7 +19,7 @@ def collect_metrics(self, with_snapshot = False): try: labels = container.get("Labels", {}) name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.container-name", "") + taskArn = labels.get("com.amazonaws.ecs.task-arn", "") # "entityId": $taskARN + "::" + $containerName plugin_data["entityId"] = "%s::%s" % (taskArn, name) @@ -50,6 +50,6 @@ def collect_metrics(self, with_snapshot = False): logger.debug("_collect_container_snapshots: ", exc_info=True) finally: plugins.append(plugin_data) - except: + except Exception: logger.debug("collect_container_metrics: ", exc_info=True) return plugins diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index 31718928..0849c7ad 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -4,7 +4,13 @@ class DockerHelper(BaseHelper): - def collect_metrics(self, with_snapshot = False): + def __init__(self, collector): + super(DockerHelper, self).__init__(collector) + + # The metrics from the previous report cycle + self.previous = DictionaryOfStan() + + def collect_metrics(self, with_snapshot=False): """ Collect and return docker metrics (and optionally snapshot data) for this task @return: list - with one or more plugin entities @@ -16,30 +22,123 @@ def collect_metrics(self, with_snapshot = False): for container in containers: plugin_data = dict() plugin_data["name"] = "com.instana.plugin.docker" - try: - labels = container.get("Labels", {}) - name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.container-name", "") - - # "entityId": $taskARN + "::" + $containerName - plugin_data["entityId"] = "%s::%s" % (taskArn, name) - plugin_data["data"] = DictionaryOfStan() - - # Snapshot Data - plugin_data["data"]["Id"] = container.get("DockerId", None) - plugin_data["data"]["Created"] = container.get("CreatedAt", None) - plugin_data["data"]["Started"] = container.get("StartedAt", None) - plugin_data["data"]["Image"] = container.get("Image", None) - plugin_data["data"]["Labels"] = container.get("Labels", None) - plugin_data["data"]["Ports"] = container.get("Ports", None) - - networks = container.get("Networks", []) - if len(networks) >= 1: - plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) - except: - logger.debug("_collect_container_snapshots: ", exc_info=True) - finally: - plugins.append(plugin_data) - except: - logger.debug("collect_docker_metrics: ", exc_info=True) + docker_id = container.get("DockerId") + + name = container.get("Name", "") + labels = container.get("Labels", {}) + task_arn = labels.get("com.amazonaws.ecs.task-arn", "") + + plugin_data["entityId"] = "%s::%s" % (task_arn, name) + plugin_data["data"] = DictionaryOfStan() + + # Metrics + self._collect_container_metrics(plugin_data, docker_id) + + # Snapshot + if with_snapshot: + self._collect_container_snapshot(plugin_data, container) + + plugins.append(plugin_data) + except Exception: + logger.debug("DockerHelper.collect_metrics: ", exc_info=True) return plugins + + def _collect_container_snapshot(self, plugin_data, container): + try: + # Snapshot Data + plugin_data["data"]["Id"] = container.get("DockerId", None) + plugin_data["data"]["Created"] = container.get("CreatedAt", None) + plugin_data["data"]["Started"] = container.get("StartedAt", None) + plugin_data["data"]["Image"] = container.get("Image", None) + plugin_data["data"]["Labels"] = container.get("Labels", None) + plugin_data["data"]["Ports"] = container.get("Ports", None) + + networks = container.get("Networks", []) + if len(networks) >= 1: + plugin_data["data"]["NetworkMode"] = networks[0].get("NetworkMode", None) + except Exception: + logger.debug("_collect_container_snapshot: ", exc_info=True) + + def _collect_container_metrics(self, plugin_data, docker_id): + try: + container = self.collector.task_stats_metadata.get(docker_id, None) + if container is not None: + # TODO + # networks = container.get("networks", None) + # if networks is not None: + # # sum of all delta(metadata.networks.$interface_ids.rx_bytes) for all interface IDs + # plugin_data["data"]["network"]["rx"]["bytes"] = 0 + # plugin_data["data"]["network"]["rx"]["dropped"] = 0 + # plugin_data["data"]["network"]["rx"]["errors"] = 0 + # plugin_data["data"]["network"]["rx"]["packets"] = 0 + # plugin_data["data"]["network"]["tx"]["bytes"] = 0 + # plugin_data["data"]["network"]["tx"]["dropped"] = 0 + # plugin_data["data"]["network"]["tx"]["errors"] = 0 + # plugin_data["data"]["network"]["tx"]["packets"] = 0 + + cpu_stats = container.get("cpu_stats", {}) + cpu_usage = cpu_stats.get("cpu_usage", None) + throttling_data = cpu_stats.get("throttling_data", None) + + if cpu_usage is not None: + online_cpus = cpu_stats.get("online_cpus", 1) + system_cpu_usage = cpu_stats.get("system_cpu_usage", 0) + + metric_value = (cpu_usage["total_usage"] / system_cpu_usage) * online_cpus + self.apply_delta(metric_value, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], "total_usage") + + metric_value = (cpu_usage["usage_in_usermode"] / system_cpu_usage) * online_cpus + self.apply_delta(metric_value, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], "user_usage") + + metric_value = (cpu_usage["usage_in_kernelmode"] / system_cpu_usage) * online_cpus + self.apply_delta(metric_value, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], "system_usage") + + if throttling_data is not None: + self.apply_delta(throttling_data, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], ("periods", "throttling_count")) + self.apply_delta(throttling_data, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], ("throttled_time", "throttling_time")) + + memory = container.get("memory_stats", {}) + memory_stats = memory.get("stats", None) + + self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "usage") + self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "max_usage") + self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "limit") + + if memory_stats is not None: + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "active_anon") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "active_file") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "inactive_anon") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "inactive_file") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "total_cache") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "total_rss") + + blkio_stats = container.get("blkio_stats", None) + if blkio_stats is not None: + service_bytes = blkio_stats.get("io_service_bytes_recursive", None) + if service_bytes is not None: + for entry in service_bytes: + if entry["op"] == "Read": + self.apply_delta(entry, self.previous[docker_id]["blkio"], + plugin_data["data"]["blkio"], ("value", "blk_read")) + elif entry["op"] == "Write": + self.apply_delta(entry, self.previous[docker_id]["blkio"], + plugin_data["data"]["blkio"], ("value", "blk_write")) + + except Exception: + logger.debug("_collect_container_metrics: ", exc_info=True) diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 5abdf705..d190dca3 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -10,6 +10,17 @@ from instana.singletons import get_agent, set_agent, get_tracer, set_tracer +def get_docker_plugin(plugins): + """ + Given a list of plugins, find and return the docker plugin that we're interested in from the mock data + """ + docker_plugin = None + for plugin in plugins: + if plugin["name"] == "com.instana.plugin.docker" and plugin["entityId"] == "arn:aws:ecs:us-east-2:410797082306:task/2d60afb1-e7fd-4761-9430-a375293a9b82::docker-ssh-aws-fargate": + docker_plugin = plugin + return docker_plugin + + class TestFargate(unittest.TestCase): def __init__(self, methodName='runTest'): super(TestFargate, self).__init__(methodName) @@ -62,7 +73,7 @@ def create_agent_and_setup_tracer(self): with open(self.pwd + '/../data/fargate/1.3.0/task_stats_metadata.json', 'r') as json_file: self.agent.collector.task_stats_metadata = json.load(json_file) - def test_prepare_payload(self): + def test_prepare_payload_basics(self): self.create_agent_and_setup_tracer() payload = self.agent.collector.prepare_payload() @@ -80,10 +91,107 @@ def test_prepare_payload(self): plugins = payload['metrics']['plugins'] for plugin in plugins: + # print("%s - %s" % (plugin["name"], plugin["entityId"])) assert('name' in plugin) assert('entityId' in plugin) assert('data' in plugin) + def test_docker_plugin_snapshot_data(self): + self.create_agent_and_setup_tracer() + + first_payload = self.agent.collector.prepare_payload() + second_payload = self.agent.collector.prepare_payload() + + assert(first_payload) + assert(second_payload) + + plugin_first_report = get_docker_plugin(first_payload['metrics']['plugins']) + plugin_second_report = get_docker_plugin(second_payload['metrics']['plugins']) + + assert(plugin_first_report) + assert("data" in plugin_first_report) + + # First report should have snapshot data + data = plugin_first_report["data"] + assert(data["Id"] == "63dc7ac9f3130bba35c785ed90ff12aad82087b5c5a0a45a922c45a64128eb45") + assert(data["Created"] == "2020-07-27T12:14:12.583114444Z") + assert(data["Started"] == "2020-07-27T12:14:13.545410186Z") + assert(data["Image"] == "410797082306.dkr.ecr.us-east-2.amazonaws.com/fargate-docker-ssh:latest") + assert(data["Labels"] == {'com.amazonaws.ecs.cluster': 'arn:aws:ecs:us-east-2:410797082306:cluster/lombardo-ssh-cluster', 'com.amazonaws.ecs.container-name': 'docker-ssh-aws-fargate', 'com.amazonaws.ecs.task-arn': 'arn:aws:ecs:us-east-2:410797082306:task/2d60afb1-e7fd-4761-9430-a375293a9b82', 'com.amazonaws.ecs.task-definition-family': 'docker-ssh-aws-fargate', 'com.amazonaws.ecs.task-definition-version': '1'}) + assert(data["Ports"] is None) + + # Second report should have no snapshot data + assert(plugin_second_report) + assert("data" in plugin_second_report) + data = plugin_second_report["data"] + assert("Id" not in data) + assert("Created" not in data) + assert("Started" not in data) + assert("Image" not in data) + assert("Labels" not in data) + assert("Ports" not in data) + + def test_docker_plugin_metrics(self): + self.create_agent_and_setup_tracer() + + first_payload = self.agent.collector.prepare_payload() + second_payload = self.agent.collector.prepare_payload() + + assert(first_payload) + assert(second_payload) + + plugin_first_report = get_docker_plugin(first_payload['metrics']['plugins']) + assert(plugin_first_report) + assert("data" in plugin_first_report) + + plugin_second_report = get_docker_plugin(second_payload['metrics']['plugins']) + assert(plugin_second_report) + assert("data" in plugin_second_report) + + # First report should report all metrics + data = plugin_first_report.get("data", None) + assert(data) + + # FIXME + # network = data.get("network", None) + # assert(network) + # assert("rx" in network) + # assert("tx" in network) + + cpu = data.get("cpu", None) + assert(cpu) + assert(cpu["total_usage"] == 0.011033210004283327) + assert(cpu["user_usage"] == 0.009917614829497682) + assert(cpu["system_usage"] == 0.0008900637656376939) + assert(cpu["throttling_count"] == 0) + assert(cpu["throttling_time"] == 0) + + memory = data.get("memory", None) + assert(memory) + assert(memory["active_anon"] == 78721024) + assert(memory["active_file"] == 18501632) + assert(memory["inactive_anon"] == 0) + assert(memory["inactive_file"] == 71684096) + assert(memory["total_cache"] == 90185728) + assert(memory["total_rss"] == 78721024) + assert(memory["usage"] == 193769472) + assert(memory["max_usage"] == 195305472) + assert(memory["limit"] == 536870912) + + blkio = data.get("blkio", None) + assert(blkio) + assert(blkio["blk_read"] == 0) + assert(blkio["blk_write"] == 128352256) + + # Second report should report the delta (in the test case, nothing) + data = plugin_second_report["data"] + assert("cpu" in data) + assert(len(data["cpu"]) == 0) + assert("memory" in data) + assert(len(data["memory"]) == 0) + assert("blkio" in data) + assert(len(data["blkio"]) == 0) + def test_no_instana_zone(self): self.create_agent_and_setup_tracer() assert(self.agent.options.zone is None) From 3367e9688a45a04008921c3b4e42f634a5106bfe Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:02:35 +0200 Subject: [PATCH 57/91] Refactor, normalization and all the other cool words --- instana/collector/helpers/fargate/docker.py | 199 ++++++++++++-------- 1 file changed, 120 insertions(+), 79 deletions(-) diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index 0849c7ad..a663b0b9 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -1,9 +1,11 @@ +""" Module to handle the collection of Docker metrics in AWS Fargate """ from ....log import logger from ..base import BaseHelper from ....util import DictionaryOfStan class DockerHelper(BaseHelper): + """ This class acts as a helper to collect Docker snapshot and metric information """ def __init__(self, collector): super(DockerHelper, self).__init__(collector) @@ -60,85 +62,124 @@ def _collect_container_snapshot(self, plugin_data, container): logger.debug("_collect_container_snapshot: ", exc_info=True) def _collect_container_metrics(self, plugin_data, docker_id): + container = self.collector.task_stats_metadata.get(docker_id, None) + if container is not None: + self._collect_network_metrics(container, plugin_data, docker_id) + self._collect_cpu_metrics(container, plugin_data, docker_id) + self._collect_memory_metrics(container, plugin_data, docker_id) + self._collect_blkio_metrics(container, plugin_data, docker_id) + + def _collect_network_metrics(self, container, plugin_data, docker_id): try: - container = self.collector.task_stats_metadata.get(docker_id, None) - if container is not None: - # TODO - # networks = container.get("networks", None) - # if networks is not None: - # # sum of all delta(metadata.networks.$interface_ids.rx_bytes) for all interface IDs - # plugin_data["data"]["network"]["rx"]["bytes"] = 0 - # plugin_data["data"]["network"]["rx"]["dropped"] = 0 - # plugin_data["data"]["network"]["rx"]["errors"] = 0 - # plugin_data["data"]["network"]["rx"]["packets"] = 0 - # plugin_data["data"]["network"]["tx"]["bytes"] = 0 - # plugin_data["data"]["network"]["tx"]["dropped"] = 0 - # plugin_data["data"]["network"]["tx"]["errors"] = 0 - # plugin_data["data"]["network"]["tx"]["packets"] = 0 - - cpu_stats = container.get("cpu_stats", {}) - cpu_usage = cpu_stats.get("cpu_usage", None) - throttling_data = cpu_stats.get("throttling_data", None) - - if cpu_usage is not None: - online_cpus = cpu_stats.get("online_cpus", 1) - system_cpu_usage = cpu_stats.get("system_cpu_usage", 0) - - metric_value = (cpu_usage["total_usage"] / system_cpu_usage) * online_cpus - self.apply_delta(metric_value, - self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], "total_usage") - - metric_value = (cpu_usage["usage_in_usermode"] / system_cpu_usage) * online_cpus - self.apply_delta(metric_value, - self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], "user_usage") - - metric_value = (cpu_usage["usage_in_kernelmode"] / system_cpu_usage) * online_cpus - self.apply_delta(metric_value, - self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], "system_usage") - - if throttling_data is not None: - self.apply_delta(throttling_data, - self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], ("periods", "throttling_count")) - self.apply_delta(throttling_data, - self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], ("throttled_time", "throttling_time")) - - memory = container.get("memory_stats", {}) - memory_stats = memory.get("stats", None) - - self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "usage") - self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "max_usage") - self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "limit") - - if memory_stats is not None: - self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "active_anon") - self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "active_file") - self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "inactive_anon") - self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "inactive_file") - self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "total_cache") - self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "total_rss") - - blkio_stats = container.get("blkio_stats", None) - if blkio_stats is not None: - service_bytes = blkio_stats.get("io_service_bytes_recursive", None) - if service_bytes is not None: - for entry in service_bytes: - if entry["op"] == "Read": - self.apply_delta(entry, self.previous[docker_id]["blkio"], - plugin_data["data"]["blkio"], ("value", "blk_read")) - elif entry["op"] == "Write": - self.apply_delta(entry, self.previous[docker_id]["blkio"], - plugin_data["data"]["blkio"], ("value", "blk_write")) + networks = container.get("networks", None) + tx_bytes_total = tx_dropped_total = tx_errors_total = tx_packets_total = 0 + rx_bytes_total = rx_dropped_total = rx_errors_total = rx_packets_total = 0 + + if networks is not None: + for key in networks.keys(): + if "eth" in key: + tx_bytes_total += networks[key].get("tx_bytes", 0) + tx_dropped_total += networks[key].get("tx_dropped", 0) + tx_errors_total += networks[key].get("tx_errors", 0) + tx_packets_total += networks[key].get("tx_packets", 0) + + rx_bytes_total += networks[key].get("rx_bytes", 0) + rx_dropped_total += networks[key].get("rx_dropped", 0) + rx_errors_total += networks[key].get("rx_errors", 0) + rx_packets_total += networks[key].get("rx_packets", 0) + + self.apply_delta(tx_bytes_total, self.previous[docker_id]["network"]["tx"], + plugin_data["data"]["tx"], "bytes") + self.apply_delta(tx_dropped_total, self.previous[docker_id]["network"]["tx"], + plugin_data["data"]["tx"], "dropped") + self.apply_delta(tx_errors_total, self.previous[docker_id]["network"]["tx"], + plugin_data["data"]["tx"], "errors") + self.apply_delta(tx_packets_total, self.previous[docker_id]["network"]["tx"], + plugin_data["data"]["tx"], "packets") + + self.apply_delta(rx_bytes_total, self.previous[docker_id]["network"]["rx"], + plugin_data["data"]["rx"], "bytes") + self.apply_delta(rx_dropped_total, self.previous[docker_id]["network"]["rx"], + plugin_data["data"]["rx"], "dropped") + self.apply_delta(rx_errors_total, self.previous[docker_id]["network"]["rx"], + plugin_data["data"]["rx"], "errors") + self.apply_delta(rx_packets_total, self.previous[docker_id]["network"]["rx"], + plugin_data["data"]["rx"], "packets") + except Exception: + logger.debug("_collect_network_metrics: ", exc_info=True) + + def _collect_cpu_metrics(self, container, plugin_data, docker_id): + try: + cpu_stats = container.get("cpu_stats", {}) + cpu_usage = cpu_stats.get("cpu_usage", None) + throttling_data = cpu_stats.get("throttling_data", None) + + if cpu_usage is not None: + online_cpus = cpu_stats.get("online_cpus", 1) + system_cpu_usage = cpu_stats.get("system_cpu_usage", 0) + + metric_value = (cpu_usage["total_usage"] / system_cpu_usage) * online_cpus + self.apply_delta(metric_value, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], "total_usage") + + metric_value = (cpu_usage["usage_in_usermode"] / system_cpu_usage) * online_cpus + self.apply_delta(metric_value, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], "user_usage") + + metric_value = (cpu_usage["usage_in_kernelmode"] / system_cpu_usage) * online_cpus + self.apply_delta(metric_value, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], "system_usage") + + if throttling_data is not None: + self.apply_delta(throttling_data, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], ("periods", "throttling_count")) + self.apply_delta(throttling_data, + self.previous[docker_id]["cpu"], + plugin_data["data"]["cpu"], ("throttled_time", "throttling_time")) + except Exception: + logger.debug("_collect_cpu_metrics: ", exc_info=True) + def _collect_memory_metrics(self, container, plugin_data, docker_id): + try: + memory = container.get("memory_stats", {}) + memory_stats = memory.get("stats", None) + + self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "usage") + self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "max_usage") + self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "limit") + + if memory_stats is not None: + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "active_anon") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "active_file") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "inactive_anon") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "inactive_file") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "total_cache") + self.apply_delta(memory_stats, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "total_rss") + except Exception: + logger.debug("_collect_memory_metrics: ", exc_info=True) + + def _collect_blkio_metrics(self, container, plugin_data, docker_id): + try: + blkio_stats = container.get("blkio_stats", None) + if blkio_stats is not None: + service_bytes = blkio_stats.get("io_service_bytes_recursive", None) + if service_bytes is not None: + for entry in service_bytes: + if entry["op"] == "Read": + self.apply_delta(entry, self.previous[docker_id]["blkio"], + plugin_data["data"]["blkio"], ("value", "blk_read")) + elif entry["op"] == "Write": + self.apply_delta(entry, self.previous[docker_id]["blkio"], + plugin_data["data"]["blkio"], ("value", "blk_write")) except Exception: - logger.debug("_collect_container_metrics: ", exc_info=True) + logger.debug("_collect_blkio_metrics: ", exc_info=True) From 6fcdb59e7fdf40463602d80706879f0d085baca1 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:02:54 +0200 Subject: [PATCH 58/91] PyLint told me to do it --- instana/collector/helpers/base.py | 8 +++++ .../collector/helpers/fargate/container.py | 12 +++---- instana/collector/helpers/fargate/hardware.py | 8 +++-- instana/collector/helpers/fargate/host.py | 6 ++-- instana/collector/helpers/fargate/task.py | 8 +++-- instana/collector/helpers/process/helper.py | 8 ++--- instana/collector/helpers/runtime/helper.py | 32 +++++-------------- 7 files changed, 40 insertions(+), 42 deletions(-) diff --git a/instana/collector/helpers/base.py b/instana/collector/helpers/base.py index 613e05a2..297e881b 100644 --- a/instana/collector/helpers/base.py +++ b/instana/collector/helpers/base.py @@ -1,7 +1,15 @@ +""" +Base class for the various helpers that can be used by Collectors. Helpers assist +in the data collection for various entities such as host, hardware, AWS Task, ec2, +memory, cpu, docker etc etc.. +""" from ...log import logger class BaseHelper(object): + """ + Base class for all helpers. Descendants must override and implement `self.collect_metrics`. + """ def __init__(self, collector): self.collector = collector diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index f4d7e49c..a383de9f 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -1,10 +1,12 @@ +""" Module to handle the collection of container metrics in AWS Fargate """ from ....log import logger from ....util import DictionaryOfStan from ..base import BaseHelper class ContainerHelper(BaseHelper): - def collect_metrics(self, with_snapshot = False): + """ This class acts as a helper to collect container snapshot and metric information """ + def collect_metrics(self, with_snapshot=False): """ Collect and return metrics (and optionally snapshot data) for every container in this task @return: list - with one or more plugin entities @@ -19,10 +21,8 @@ def collect_metrics(self, with_snapshot = False): try: labels = container.get("Labels", {}) name = container.get("Name", "") - taskArn = labels.get("com.amazonaws.ecs.task-arn", "") - - # "entityId": $taskARN + "::" + $containerName - plugin_data["entityId"] = "%s::%s" % (taskArn, name) + task_arn = labels.get("com.amazonaws.ecs.task-arn", "") + plugin_data["entityId"] = "%s::%s" % (task_arn, name) plugin_data["data"] = DictionaryOfStan() if self.collector.root_metadata["Name"] == name: @@ -46,7 +46,7 @@ def collect_metrics(self, with_snapshot = False): limits = container.get("Limits", {}) plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - except: + except Exception: logger.debug("_collect_container_snapshots: ", exc_info=True) finally: plugins.append(plugin_data) diff --git a/instana/collector/helpers/fargate/hardware.py b/instana/collector/helpers/fargate/hardware.py index c8b868e3..aa19aaf8 100644 --- a/instana/collector/helpers/fargate/hardware.py +++ b/instana/collector/helpers/fargate/hardware.py @@ -1,10 +1,12 @@ +""" Module to handle the reporting of the hardware plugin in AWS Fargate """ from ....log import logger from ....util import DictionaryOfStan from ..base import BaseHelper class HardwareHelper(BaseHelper): - def collect_metrics(self, with_snapshot = False): + """ This class acts as a helper to collect data for the hardware plugin """ + def collect_metrics(self, with_snapshot=False): """ # This helper only sends snapshot data related to the INSTANA_ZONE environment variable @return: list @@ -21,10 +23,10 @@ def collect_metrics(self, with_snapshot = False): plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) plugin_data["data"] = DictionaryOfStan() plugin_data["data"]["availability-zone"] = self.collector.agent.options.zone - except: + except Exception: logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) finally: plugins.append(plugin_data) - except: + except Exception: logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) return plugins diff --git a/instana/collector/helpers/fargate/host.py b/instana/collector/helpers/fargate/host.py index 2efa6dcd..f0212c19 100644 --- a/instana/collector/helpers/fargate/host.py +++ b/instana/collector/helpers/fargate/host.py @@ -1,10 +1,12 @@ +""" Module to help in collecting data for the host plugin in AWS Fargate """ from ....log import logger from ..base import BaseHelper from ....util import DictionaryOfStan class HostHelper(BaseHelper): - def collect_metrics(self, with_snapshot = False): + """ This class acts as a helper to collect data for the host plugin """ + def collect_metrics(self, with_snapshot=False): """ # This helper only sends snapshot data related to the INSTANA_TAGS environment variable @return: list @@ -20,7 +22,7 @@ def collect_metrics(self, with_snapshot = False): plugin_data["data"] = DictionaryOfStan() plugin_data["data"]["tags"] = self.collector.agent.options.tags plugins.append(plugin_data) - except: + except Exception: logger.debug("HostHelper.collect_metrics: ", exc_info=True) finally: plugins.append(plugin_data) diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index 7e8ddd99..d5c45e0f 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -1,10 +1,12 @@ +""" Module to assist in the data collection about the AWS Fargate task that is running this process """ from ....log import logger from ..base import BaseHelper from ....util import DictionaryOfStan class TaskHelper(BaseHelper): - def collect_metrics(self, with_snapshot = False): + """ This class helps in collecting data about the AWS Fargate task that is running """ + def collect_metrics(self, with_snapshot=False): """ Collect and return metrics data (and optionally snapshot data) for this task @return: list - with one plugin entity @@ -29,10 +31,10 @@ def collect_metrics(self, with_snapshot = False): limits = self.collector.task_metadata.get("Limits", {}) plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - except: + except Exception: logger.debug("collect_task_metrics: ", exc_info=True) finally: plugins.append(plugin_data) - except: + except Exception: logger.debug("collect_task_metrics: ", exc_info=True) return plugins diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process/helper.py index 69c733d4..03f4c84f 100644 --- a/instana/collector/helpers/process/helper.py +++ b/instana/collector/helpers/process/helper.py @@ -1,13 +1,13 @@ import os import pwd import grp -from ..base import BaseHelper from instana.log import logger from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret +from ..base import BaseHelper class ProcessHelper(BaseHelper): - def collect_metrics(self, with_snapshot = False): + def collect_metrics(self, with_snapshot=False): plugin_data = dict() try: plugin_data["name"] = "com.instana.plugin.process" @@ -35,7 +35,7 @@ def collect_metrics(self, with_snapshot = False): egid = os.getegid() plugin_data["data"]["user"] = pwd.getpwuid(euid) plugin_data["data"]["group"] = grp.getgrgid(egid).gr_name - except: + except Exception: logger.debug("euid/egid detection: ", exc_info=True) plugin_data["data"]["start"] = 1 # FIXME @@ -45,6 +45,6 @@ def collect_metrics(self, with_snapshot = False): # plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME: the pid in the root namespace (very optional) if self.collector.task_metadata is not None: plugin_data["data"]["com.instana.plugin.host.name"] = self.collector.task_metadata.get("TaskArn") - except: + except Exception: logger.debug("_collect_process_snapshot: ", exc_info=True) return [plugin_data] diff --git a/instana/collector/helpers/runtime/helper.py b/instana/collector/helpers/runtime/helper.py index f41a6d2b..cd2c2cc4 100644 --- a/instana/collector/helpers/runtime/helper.py +++ b/instana/collector/helpers/runtime/helper.py @@ -8,10 +8,11 @@ from types import ModuleType from pkg_resources import DistributionNotFound, get_distribution -from ..base import BaseHelper from instana.log import logger from instana.util import DictionaryOfStan, determine_service_name +from ..base import BaseHelper + class RuntimeHelper(BaseHelper): def __init__(self, collector): @@ -20,7 +21,7 @@ def __init__(self, collector): self.last_usage = None self.last_metrics = None - def collect_metrics(self, with_snapshot = False): + def collect_metrics(self, with_snapshot=False): plugin_data = dict() try: plugin_data["name"] = "com.instana.plugin.python" @@ -41,7 +42,7 @@ def collect_metrics(self, with_snapshot = False): plugin_data["data"]["snapshot"] = snapshot_payload self.last_metrics = metrics_payload - except: + except Exception: logger.debug("_collect_runtime_snapshot: ", exc_info=True) return [plugin_data] @@ -54,10 +55,8 @@ def gather_metrics(self): c = list(gc_.get_count()) th = list(gc_.get_threshold()) g = GC(collect0=c[0] if not self.last_collect else c[0] - self.last_collect[0], - collect1=c[1] if not self.last_collect else c[ - 1] - self.last_collect[1], - collect2=c[2] if not self.last_collect else c[ - 2] - self.last_collect[2], + collect1=c[1] if not self.last_collect else c[1] - self.last_collect[1], + collect2=c[2] if not self.last_collect else c[2] - self.last_collect[2], threshold0=th[0], threshold1=th[1], threshold2=th[2]) @@ -65,7 +64,7 @@ def gather_metrics(self): thr = threading.enumerate() daemon_threads = [tr.daemon is True for tr in thr].count(True) alive_threads = [tr.daemon is False for tr in thr].count(True) - dummy_threads = [type(tr) is threading._DummyThread for tr in thr].count(True) + dummy_threads = [isinstance(tr, threading._DummyThread) for tr in thr].count(True) m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0], ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1], @@ -146,7 +145,7 @@ def jsonable(self, value): result = value() except: result = 'Unknown' - elif type(value) is ModuleType: + elif isinstance(value, ModuleType): result = value else: result = value @@ -207,18 +206,3 @@ def delta_data(self, delta): def to_dict(self): return self.__dict__ - - -class EntityData(object): - pid = 0 - snapshot = None - metrics = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def to_dict(self): - return self.__dict__ - - - From 42494dd953615fe9a98fcd6e69b845916673b085 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:26:41 +0200 Subject: [PATCH 59/91] Refactor and respect snapshot reporting flag --- instana/collector/helpers/process/helper.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process/helper.py index 03f4c84f..52781684 100644 --- a/instana/collector/helpers/process/helper.py +++ b/instana/collector/helpers/process/helper.py @@ -7,11 +7,23 @@ class ProcessHelper(BaseHelper): + """ Helper class to collect metrics for this process """ def collect_metrics(self, with_snapshot=False): plugin_data = dict() try: plugin_data["name"] = "com.instana.plugin.process" plugin_data["entityId"] = str(os.getpid()) + + # Snapshot + if with_snapshot: + self._collect_process_snapshot(plugin_data) + + except Exception: + logger.debug("ProcessHelper.collect_metrics: ", exc_info=True) + return [plugin_data] + + def _collect_process_snapshot(self, plugin_data): + try: plugin_data["data"] = DictionaryOfStan() plugin_data["data"]["pid"] = int(os.getpid()) env = dict() @@ -46,5 +58,4 @@ def collect_metrics(self, with_snapshot=False): if self.collector.task_metadata is not None: plugin_data["data"]["com.instana.plugin.host.name"] = self.collector.task_metadata.get("TaskArn") except Exception: - logger.debug("_collect_process_snapshot: ", exc_info=True) - return [plugin_data] + logger.debug("ProcessHelper._collect_process_snapshot: ", exc_info=True) From b9f19cf6929a1d3c18774525c692b3a594090077 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:29:14 +0200 Subject: [PATCH 60/91] Subpackages not needed --- instana/collector/helpers/{process/helper.py => process.py} | 0 instana/collector/helpers/process/__init__.py | 0 instana/collector/helpers/{runtime/helper.py => runtime.py} | 0 instana/collector/helpers/runtime/__init__.py | 0 instana/collector/helpers/runtime/snapshot.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename instana/collector/helpers/{process/helper.py => process.py} (100%) delete mode 100644 instana/collector/helpers/process/__init__.py rename instana/collector/helpers/{runtime/helper.py => runtime.py} (100%) delete mode 100644 instana/collector/helpers/runtime/__init__.py delete mode 100644 instana/collector/helpers/runtime/snapshot.py diff --git a/instana/collector/helpers/process/helper.py b/instana/collector/helpers/process.py similarity index 100% rename from instana/collector/helpers/process/helper.py rename to instana/collector/helpers/process.py diff --git a/instana/collector/helpers/process/__init__.py b/instana/collector/helpers/process/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/instana/collector/helpers/runtime/helper.py b/instana/collector/helpers/runtime.py similarity index 100% rename from instana/collector/helpers/runtime/helper.py rename to instana/collector/helpers/runtime.py diff --git a/instana/collector/helpers/runtime/__init__.py b/instana/collector/helpers/runtime/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/instana/collector/helpers/runtime/snapshot.py b/instana/collector/helpers/runtime/snapshot.py deleted file mode 100644 index e69de29b..00000000 From b36918c3aa6e8cf847b3e647697d6aa2edf19188 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:31:25 +0200 Subject: [PATCH 61/91] Linter fixes --- instana/collector/helpers/process.py | 2 +- instana/collector/helpers/runtime.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/instana/collector/helpers/process.py b/instana/collector/helpers/process.py index 52781684..107cfcc8 100644 --- a/instana/collector/helpers/process.py +++ b/instana/collector/helpers/process.py @@ -3,7 +3,7 @@ import grp from instana.log import logger from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret -from ..base import BaseHelper +from .base import BaseHelper class ProcessHelper(BaseHelper): diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index cd2c2cc4..008ac336 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -11,10 +11,11 @@ from instana.log import logger from instana.util import DictionaryOfStan, determine_service_name -from ..base import BaseHelper +from .base import BaseHelper class RuntimeHelper(BaseHelper): + """ Helper class to collect snapshot and metrics for this Python runtime """ def __init__(self, collector): super(RuntimeHelper, self).__init__(collector) self.last_collect = None From 7dd43f78779b044c2907f39f06a321a4ab709439 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:34:15 +0200 Subject: [PATCH 62/91] Update imports with new path --- instana/collector/aws_fargate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 53f45ba0..791d8b92 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -12,11 +12,11 @@ from ..util import DictionaryOfStan, every, validate_url from ..singletons import env_is_test +from .helpers.process import ProcessHelper +from .helpers.runtime import RuntimeHelper from .helpers.fargate.host import HostHelper from .helpers.fargate.task import TaskHelper from .helpers.fargate.docker import DockerHelper -from .helpers.process.helper import ProcessHelper -from .helpers.runtime.helper import RuntimeHelper from .helpers.fargate.hardware import HardwareHelper from .helpers.fargate.container import ContainerHelper From f627da9ea3ffb707a4788dbe1762957694d0ccba Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Sun, 9 Aug 2020 12:40:08 +0200 Subject: [PATCH 63/91] Without snapshot flag, do nothing --- instana/collector/helpers/fargate/container.py | 4 ++++ instana/collector/helpers/fargate/task.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index a383de9f..f517521c 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -12,6 +12,10 @@ def collect_metrics(self, with_snapshot=False): @return: list - with one or more plugin entities """ plugins = [] + + if with_snapshot is False: + return plugins + try: if self.collector.task_metadata is not None: containers = self.collector.task_metadata.get("Containers", []) diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index d5c45e0f..a77f1ff2 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -12,6 +12,10 @@ def collect_metrics(self, with_snapshot=False): @return: list - with one plugin entity """ plugins = [] + + if with_snapshot is False: + return plugins + try: if self.collector.task_metadata is not None: try: From a5bf1895a616cd251a27103edc81e7a93a0b8906 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 10 Aug 2020 12:11:05 +0200 Subject: [PATCH 64/91] Make sure file exists b4 trying to read it --- instana/collector/helpers/process.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/instana/collector/helpers/process.py b/instana/collector/helpers/process.py index 107cfcc8..96a4619e 100644 --- a/instana/collector/helpers/process.py +++ b/instana/collector/helpers/process.py @@ -35,7 +35,10 @@ def _collect_process_snapshot(self, plugin_data): else: env[key] = os.environ[key] plugin_data["data"]["env"] = env - plugin_data["data"]["exec"] = os.readlink("/proc/self/exe") + if os.path.isfile("/proc/self/exe"): + plugin_data["data"]["exec"] = os.readlink("/proc/self/exe") + else: + logger.debug("Can't access /proc/self/exe...") cmdline = get_proc_cmdline() if len(cmdline) > 1: From 434fc0b0aa7da88130b90d5a4457290605ef5ccb Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 10 Aug 2020 12:11:30 +0200 Subject: [PATCH 65/91] Fix test class names --- tests/platforms/test_fargate_collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index d190dca3..4f59f9c3 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -21,9 +21,9 @@ def get_docker_plugin(plugins): return docker_plugin -class TestFargate(unittest.TestCase): +class TestFargateCollector(unittest.TestCase): def __init__(self, methodName='runTest'): - super(TestFargate, self).__init__(methodName) + super(TestFargateCollector, self).__init__(methodName) self.agent = None self.span_recorder = None self.tracer = None From acc964649eef364e05bec796ae6404f03947a665 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 10 Aug 2020 12:27:44 +0200 Subject: [PATCH 66/91] Minor cleanup --- instana/collector/helpers/runtime.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index 008ac336..3246e7f2 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -28,6 +28,7 @@ def collect_metrics(self, with_snapshot=False): plugin_data["name"] = "com.instana.plugin.python" plugin_data["entityId"] = str(os.getpid()) plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["pid"] = str(os.getpid()) snapshot_payload = None metrics_payload = self.gather_metrics() @@ -35,12 +36,11 @@ def collect_metrics(self, with_snapshot=False): if with_snapshot is True: snapshot_payload = self.gather_snapshot() metrics_payload = copy.deepcopy(metrics_payload).delta_data(None) + plugin_data["data"]["snapshot"] = snapshot_payload else: metrics_payload = copy.deepcopy(metrics_payload).delta_data(self.last_metrics) - plugin_data["data"]["pid"] = str(os.getpid()) plugin_data["data"]["metrics"] = metrics_payload - plugin_data["data"]["snapshot"] = snapshot_payload self.last_metrics = metrics_payload except Exception: @@ -105,7 +105,7 @@ def gather_snapshot(self): snapshot_payload['f'] = platform.python_implementation() # flavor snapshot_payload['a'] = platform.architecture()[0] # architecture snapshot_payload['versions'] = self.gather_python_packages() - snapshot_payload['djmw'] = None # FIXME + #snapshot_payload['djmw'] = None # FIXME except Exception: logger.debug("collect_snapshot: ", exc_info=True) return snapshot_payload From 4aab66d979be4e3d95797d5f26357e7796041817 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 10 Aug 2020 12:40:06 +0200 Subject: [PATCH 67/91] Update python containers to latest --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bfbbee8..173577e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ version: 2 jobs: python27: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:2.7.18 - image: circleci/postgres:9.6.5-alpine-ram - image: circleci/mariadb:10.1-ram - image: circleci/redis:5.0.4 @@ -43,7 +43,7 @@ jobs: python38: docker: - - image: circleci/python:3.7.7-stretch + - image: circleci/python:3.7.8-stretch - image: circleci/postgres:9.6.5-alpine-ram - image: circleci/mariadb:10-ram - image: circleci/redis:5.0.4 @@ -77,7 +77,7 @@ jobs: py27cassandra: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:2.7.18 - image: circleci/cassandra:3.10 environment: MAX_HEAP_SIZE: 2048m @@ -107,7 +107,7 @@ jobs: py36cassandra: docker: - - image: circleci/python:3.6.8 + - image: circleci/python:3.6.11 - image: circleci/cassandra:3.10 environment: MAX_HEAP_SIZE: 2048m @@ -134,7 +134,7 @@ jobs: gevent38: docker: - - image: circleci/python:3.8.2 + - image: circleci/python:3.8.5 working_directory: ~/repo steps: - checkout From 0ccb9357071326d699fdf70a304c73ba9cf1fc8c Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 11 Aug 2020 16:40:29 +0200 Subject: [PATCH 68/91] Update zone and tags handling --- instana/collector/aws_fargate.py | 5 +-- instana/collector/helpers/fargate/hardware.py | 32 --------------- instana/collector/helpers/fargate/host.py | 29 -------------- instana/collector/helpers/fargate/task.py | 6 +++ instana/options.py | 13 ++++++- tests/platforms/test_fargate.py | 18 --------- tests/platforms/test_fargate_collector.py | 39 +++++++++++++++---- 7 files changed, 51 insertions(+), 91 deletions(-) delete mode 100644 instana/collector/helpers/fargate/hardware.py delete mode 100644 instana/collector/helpers/fargate/host.py diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 791d8b92..23087609 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -14,10 +14,8 @@ from .helpers.process import ProcessHelper from .helpers.runtime import RuntimeHelper -from .helpers.fargate.host import HostHelper from .helpers.fargate.task import TaskHelper from .helpers.fargate.docker import DockerHelper -from .helpers.fargate.hardware import HardwareHelper from .helpers.fargate.container import ContainerHelper @@ -84,12 +82,10 @@ def __init__(self, agent): self.helpers = [] # Populate the collection helpers - self.helpers.append(HostHelper(self)) self.helpers.append(TaskHelper(self)) self.helpers.append(DockerHelper(self)) self.helpers.append(ProcessHelper(self)) self.helpers.append(RuntimeHelper(self)) - self.helpers.append(HardwareHelper(self)) self.helpers.append(ContainerHelper(self)) def start(self): @@ -174,6 +170,7 @@ def prepare_payload(self): with_snapshot = self.should_send_snapshot_data() try: for helper in self.helpers: + # logger.debug("FargateCollector calling %s" % helper) plugins.extend(helper.collect_metrics(with_snapshot)) payload["metrics"]["plugins"] = plugins diff --git a/instana/collector/helpers/fargate/hardware.py b/instana/collector/helpers/fargate/hardware.py deleted file mode 100644 index aa19aaf8..00000000 --- a/instana/collector/helpers/fargate/hardware.py +++ /dev/null @@ -1,32 +0,0 @@ -""" Module to handle the reporting of the hardware plugin in AWS Fargate """ -from ....log import logger -from ....util import DictionaryOfStan -from ..base import BaseHelper - - -class HardwareHelper(BaseHelper): - """ This class acts as a helper to collect data for the hardware plugin """ - def collect_metrics(self, with_snapshot=False): - """ - # This helper only sends snapshot data related to the INSTANA_ZONE environment variable - @return: list - """ - plugins = [] - if with_snapshot is False or self.collector.agent.options.zone is None: - return plugins - - try: - if self.collector.task_metadata is not None: - plugin_data = dict() - plugin_data["name"] = "com.instana.plugin.generic.hardware" - try: - plugin_data["entityId"] = self.collector.task_metadata.get("TaskARN", None) - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["availability-zone"] = self.collector.agent.options.zone - except Exception: - logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) - finally: - plugins.append(plugin_data) - except Exception: - logger.debug("HardwareHelper.collect_metrics: ", exc_info=True) - return plugins diff --git a/instana/collector/helpers/fargate/host.py b/instana/collector/helpers/fargate/host.py deleted file mode 100644 index f0212c19..00000000 --- a/instana/collector/helpers/fargate/host.py +++ /dev/null @@ -1,29 +0,0 @@ -""" Module to help in collecting data for the host plugin in AWS Fargate """ -from ....log import logger -from ..base import BaseHelper -from ....util import DictionaryOfStan - - -class HostHelper(BaseHelper): - """ This class acts as a helper to collect data for the host plugin """ - def collect_metrics(self, with_snapshot=False): - """ - # This helper only sends snapshot data related to the INSTANA_TAGS environment variable - @return: list - """ - plugins = [] - if with_snapshot is False or self.collector.agent.options.tags is None: - return plugins - - plugin_data = dict() - plugin_data["name"] = "com.instana.plugin.host" - try: - plugin_data["entityId"] = "h" - plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["tags"] = self.collector.agent.options.tags - plugins.append(plugin_data) - except Exception: - logger.debug("HostHelper.collect_metrics: ", exc_info=True) - finally: - plugins.append(plugin_data) - return plugins diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index a77f1ff2..94367df0 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -35,6 +35,12 @@ def collect_metrics(self, with_snapshot=False): limits = self.collector.task_metadata.get("Limits", {}) plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + + if self.collector.agent.options.zone is not None: + plugin_data["data"]["instanaZone"] = self.collector.agent.options.zone + + if self.collector.agent.options.tags is not None: + plugin_data["data"]["tags"] = self.collector.agent.options.tags except Exception: logger.debug("collect_task_metrics: ", exc_info=True) finally: diff --git a/instana/options.py b/instana/options.py index e4df83db..e050738a 100644 --- a/instana/options.py +++ b/instana/options.py @@ -93,7 +93,18 @@ def __init__(self, **kwds): self.tags = None tag_list = os.environ.get("INSTANA_TAGS", None) if tag_list is not None: - self.tags = tag_list.split(',') + try: + self.tags = dict() + tags = tag_list.split(',') + for tag_and_value in tags: + parts = tag_and_value.split('=') + length = len(parts) + if length == 1: + self.tags[parts[0]] = None + elif length == 2: + self.tags[parts[0]] = parts[1] + except Exception: + logger.debug("Error parsing INSTANA_TAGS env var: %s", tag_list) self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index cfff4ad0..5f032837 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -93,24 +93,6 @@ def test_default_tags(self): self.assertTrue(hasattr(self.agent.options, 'tags')) self.assertIsNone(self.agent.options.tags) - def test_custom_tags(self): - os.environ["INSTANA_TAGS"] = "love,war,games" - self.create_agent_and_setup_tracer() - self.assertTrue(hasattr(self.agent.options, 'tags')) - self.assertEqual(self.agent.options.tags, ["love", "war", "games"]) - - payload = self.agent.collector.prepare_payload() - - assert payload - host_plugin = None - plugins = payload['metrics']['plugins'] - for plugin in plugins: - if plugin["name"] == "com.instana.plugin.host": - host_plugin = plugin - assert host_plugin - assert host_plugin["entityId"] == "h" - assert host_plugin["data"]["tags"] == ['love', 'war', 'games'] - def test_has_extra_http_headers(self): self.create_agent_and_setup_tracer() self.assertTrue(hasattr(self.agent, 'options')) diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 4f59f9c3..216ff5bb 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -39,6 +39,8 @@ def setUp(self): if "INSTANA_ZONE" in os.environ: os.environ.pop("INSTANA_ZONE") + if "INSTANA_TAGS" in os.environ: + os.environ.pop("INSTANA_TAGS") def tearDown(self): """ Reset all environment variables of consequence """ @@ -52,6 +54,8 @@ def tearDown(self): os.environ.pop("INSTANA_AGENT_KEY") if "INSTANA_ZONE" in os.environ: os.environ.pop("INSTANA_ZONE") + if "INSTANA_TAGS" in os.environ: + os.environ.pop("INSTANA_TAGS") set_agent(self.original_agent) set_tracer(self.original_tracer) @@ -208,12 +212,33 @@ def test_instana_zone(self): plugins = payload['metrics']['plugins'] assert(isinstance(plugins, list)) - hardware_plugin = None + task_plugin = None for plugin in plugins: - if plugin["name"] == "com.instana.plugin.generic.hardware": - hardware_plugin = plugin + if plugin["name"] == "com.instana.plugin.aws.ecs.task": + task_plugin = plugin - assert(hardware_plugin) - assert("data" in hardware_plugin) - assert("availability-zone" in hardware_plugin["data"]) - assert(hardware_plugin["data"]["availability-zone"] == "YellowDog") + assert(task_plugin) + assert("data" in task_plugin) + assert("instanaZone" in task_plugin["data"]) + assert(task_plugin["data"]["instanaZone"] == "YellowDog") + + def test_custom_tags(self): + os.environ["INSTANA_TAGS"] = "love,war=1,games" + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent.options, 'tags')) + self.assertEqual(self.agent.options.tags, {"love": None, "war": "1", "games": None}) + + payload = self.agent.collector.prepare_payload() + + assert payload + task_plugin = None + plugins = payload['metrics']['plugins'] + for plugin in plugins: + if plugin["name"] == "com.instana.plugin.aws.ecs.task": + task_plugin = plugin + assert task_plugin + assert "tags" in task_plugin["data"] + tags = task_plugin["data"]["tags"] + assert tags["war"] == "1" + assert tags["love"] is None + assert tags["games"] is None From 8599490e66242b147529d24d502127e901b24198 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 11 Aug 2020 17:13:53 +0200 Subject: [PATCH 69/91] Package collection cleanup --- instana/collector/aws_fargate.py | 1 - instana/collector/helpers/runtime.py | 32 ++++++++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 23087609..1c829365 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -170,7 +170,6 @@ def prepare_payload(self): with_snapshot = self.should_send_snapshot_data() try: for helper in self.helpers: - # logger.debug("FargateCollector calling %s" % helper) plugins.extend(helper.collect_metrics(with_snapshot)) payload["metrics"]["plugins"] = plugins diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index 3246e7f2..4cc207cd 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -112,39 +112,43 @@ def gather_snapshot(self): def gather_python_packages(self): """ Collect up the list of modules in use """ - res = {} + versions = dict() try: - m = sys.modules.copy() + sys_packages = sys.modules.copy() - for k in m: + for pkg_name in sys_packages: # Don't report submodules (e.g. django.x, django.y, django.z) # Skip modules that begin with underscore - if ('.' in k) or k[0] == '_': + if ('.' in pkg_name) or pkg_name[0] == '_': continue - if m[k]: + if sys_packages[pkg_name]: try: - d = m[k].__dict__ - if "version" in d and d["version"]: - res[k] = self.jsonable(d["version"]) - elif "__version__" in d and d["__version__"]: - res[k] = self.jsonable(d["__version__"]) + pkg_info = sys_packages[pkg_name].__dict__ + if "version" in pkg_info: + versions[pkg_name] = self.jsonable(pkg_info["version"]) + elif "__version__" in pkg_info: + if isinstance(pkg_info["__version__"], str): + versions[pkg_name] = pkg_info["__version__"] + else: + versions[pkg_name] = self.jsonable(pkg_info["__version__"]) else: - res[k] = get_distribution(k).version + versions[pkg_name] = get_distribution(pkg_name).version except DistributionNotFound: pass except Exception: - logger.debug("gather_python_packages: could not process module: %s", k) + logger.debug("gather_python_packages: could not process module: %s", pkg_name) except Exception: logger.debug("gather_python_packages", exc_info=True) - return res + finally: + return versions def jsonable(self, value): try: if callable(value): try: result = value() - except: + except Exception: result = 'Unknown' elif isinstance(value, ModuleType): result = value From 0b93c6b34e30722b45132f39fe9ace43b9bca411 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 11 Aug 2020 18:26:11 +0200 Subject: [PATCH 70/91] Add to_pretty_json helper --- instana/util.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/instana/util.py b/instana/util.py index fd6edc76..7ca2861a 100644 --- a/instana/util.py +++ b/instana/util.py @@ -95,6 +95,25 @@ def extractor(o): except Exception: logger.debug("to_json non-fatal encoding issue: ", exc_info=True) +def to_pretty_json(obj): + """ + Convert obj to pretty json. Used mostly in logging/debugging. + + :param obj: the object to serialize to json + :return: json string + """ + try: + def extractor(o): + if not hasattr(o, '__dict__'): + logger.debug("Couldn't serialize non dict type: %s", type(o)) + return {} + else: + return {k.lower(): v for k, v in o.__dict__.items() if v is not None} + + return json.dumps(obj, default=extractor, sort_keys=True, indent=4, separators=(',', ':')).encode() + except Exception: + logger.debug("to_pretty_json non-fatal encoding issue: ", exc_info=True) + def get_proc_cmdline(as_string=False): """ From ee380fa4a87e13066ab8a1cc0823e08f17e38e73 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 11 Aug 2020 18:45:15 +0200 Subject: [PATCH 71/91] Debug helpers --- instana/collector/helpers/fargate/container.py | 3 ++- instana/collector/helpers/fargate/docker.py | 3 ++- instana/collector/helpers/fargate/task.py | 4 +++- instana/collector/helpers/process.py | 3 ++- instana/collector/helpers/runtime.py | 3 ++- instana/util.py | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index f517521c..70373608 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -1,6 +1,6 @@ """ Module to handle the collection of container metrics in AWS Fargate """ from ....log import logger -from ....util import DictionaryOfStan +from ....util import DictionaryOfStan, to_pretty_json from ..base import BaseHelper @@ -50,6 +50,7 @@ def collect_metrics(self, with_snapshot=False): limits = container.get("Limits", {}) plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + #logger.debug(to_pretty_json(plugin_data)) except Exception: logger.debug("_collect_container_snapshots: ", exc_info=True) finally: diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index a663b0b9..4a4d77fb 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -1,7 +1,7 @@ """ Module to handle the collection of Docker metrics in AWS Fargate """ from ....log import logger from ..base import BaseHelper -from ....util import DictionaryOfStan +from ....util import DictionaryOfStan, to_pretty_json class DockerHelper(BaseHelper): @@ -41,6 +41,7 @@ def collect_metrics(self, with_snapshot=False): self._collect_container_snapshot(plugin_data, container) plugins.append(plugin_data) + #logger.debug(to_pretty_json(plugin_data)) except Exception: logger.debug("DockerHelper.collect_metrics: ", exc_info=True) return plugins diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index 94367df0..ff089683 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -1,7 +1,7 @@ """ Module to assist in the data collection about the AWS Fargate task that is running this process """ from ....log import logger from ..base import BaseHelper -from ....util import DictionaryOfStan +from ....util import DictionaryOfStan, to_pretty_json class TaskHelper(BaseHelper): @@ -41,6 +41,8 @@ def collect_metrics(self, with_snapshot=False): if self.collector.agent.options.tags is not None: plugin_data["data"]["tags"] = self.collector.agent.options.tags + + #logger.debug(to_pretty_json(plugin_data)) except Exception: logger.debug("collect_task_metrics: ", exc_info=True) finally: diff --git a/instana/collector/helpers/process.py b/instana/collector/helpers/process.py index 96a4619e..cc1a498b 100644 --- a/instana/collector/helpers/process.py +++ b/instana/collector/helpers/process.py @@ -2,7 +2,7 @@ import pwd import grp from instana.log import logger -from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret +from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret, to_pretty_json from .base import BaseHelper @@ -18,6 +18,7 @@ def collect_metrics(self, with_snapshot=False): if with_snapshot: self._collect_process_snapshot(plugin_data) + #logger.debug(to_pretty_json(plugin_data)) except Exception: logger.debug("ProcessHelper.collect_metrics: ", exc_info=True) return [plugin_data] diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index 4cc207cd..69fc6ee4 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -9,7 +9,7 @@ from pkg_resources import DistributionNotFound, get_distribution from instana.log import logger -from instana.util import DictionaryOfStan, determine_service_name +from instana.util import DictionaryOfStan, determine_service_name, to_pretty_json from .base import BaseHelper @@ -43,6 +43,7 @@ def collect_metrics(self, with_snapshot=False): plugin_data["data"]["metrics"] = metrics_payload self.last_metrics = metrics_payload + #logger.debug(to_pretty_json(plugin_data)) except Exception: logger.debug("_collect_runtime_snapshot: ", exc_info=True) return [plugin_data] diff --git a/instana/util.py b/instana/util.py index 7ca2861a..686702ba 100644 --- a/instana/util.py +++ b/instana/util.py @@ -110,7 +110,7 @@ def extractor(o): else: return {k.lower(): v for k, v in o.__dict__.items() if v is not None} - return json.dumps(obj, default=extractor, sort_keys=True, indent=4, separators=(',', ':')).encode() + return json.dumps(obj, default=extractor, sort_keys=True, indent=4, separators=(',', ':')) except Exception: logger.debug("to_pretty_json non-fatal encoding issue: ", exc_info=True) From 85a8535cc9b77c56d20ec842cef3d9854b3d3a06 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 11 Aug 2020 19:19:20 +0200 Subject: [PATCH 72/91] Round CPU floats and rootbeer floats too --- instana/collector/helpers/fargate/docker.py | 6 +++--- tests/platforms/test_fargate_collector.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index 4a4d77fb..bda9ed56 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -120,17 +120,17 @@ def _collect_cpu_metrics(self, container, plugin_data, docker_id): system_cpu_usage = cpu_stats.get("system_cpu_usage", 0) metric_value = (cpu_usage["total_usage"] / system_cpu_usage) * online_cpus - self.apply_delta(metric_value, + self.apply_delta(round(metric_value, 6), self.previous[docker_id]["cpu"], plugin_data["data"]["cpu"], "total_usage") metric_value = (cpu_usage["usage_in_usermode"] / system_cpu_usage) * online_cpus - self.apply_delta(metric_value, + self.apply_delta(round(metric_value, 6), self.previous[docker_id]["cpu"], plugin_data["data"]["cpu"], "user_usage") metric_value = (cpu_usage["usage_in_kernelmode"] / system_cpu_usage) * online_cpus - self.apply_delta(metric_value, + self.apply_delta(round(metric_value, 6), self.previous[docker_id]["cpu"], plugin_data["data"]["cpu"], "system_usage") diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 216ff5bb..bb05681c 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -164,9 +164,9 @@ def test_docker_plugin_metrics(self): cpu = data.get("cpu", None) assert(cpu) - assert(cpu["total_usage"] == 0.011033210004283327) - assert(cpu["user_usage"] == 0.009917614829497682) - assert(cpu["system_usage"] == 0.0008900637656376939) + assert(cpu["total_usage"] == 0.011033) + assert(cpu["user_usage"] == 0.009918) + assert(cpu["system_usage"] == 0.00089) assert(cpu["throttling_count"] == 0) assert(cpu["throttling_time"] == 0) From 44712fa7e8b90fa96e76d2b5da34658db736756f Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Tue, 11 Aug 2020 21:36:57 +0200 Subject: [PATCH 73/91] Fetch ECMU metadata only on interval --- instana/collector/aws_fargate.py | 35 +++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 1c829365..4a41f428 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -43,6 +43,10 @@ def __init__(self, agent): # Lock used synchronize data collection self.ecmu_lock = threading.Lock() + # Timestamp in seconds of the last time we fetched all ECMU data + self.last_ecmu_full_fetch = 0 + # How often to do a full fetch of ECMU data + self.ecmu_full_fetch_interval = 605 # How often to report data self.report_interval = 1 @@ -121,29 +125,32 @@ def get_ecs_metadata(self): # For test, we are using mock ECS metadata return - lock_acquired = self.ecmu_lock.acquire(False) - if lock_acquired: + self.ecmu_lock.acquire(False) + if self.ecmu_lock.locked(): try: - # FIXME: Only get once? or every 5 mins? - # Response from the last call to - # ${ECS_CONTAINER_METADATA_URI}/ - json_body = self.http_client.get(self.ecmu_url_root, timeout=3).content - self.root_metadata = json.loads(json_body) + delta = int(time()) - self.last_ecmu_full_fetch + if delta > self.ecmu_full_fetch_interval: + # Refetch the ECMU snapshot data + self.last_ecmu_full_fetch = int(time()) - # FIXME: Only get once? or every 5 mins? - # Response from the last call to - # ${ECS_CONTAINER_METADATA_URI}/task - json_body = self.http_client.get(self.ecmu_url_task, timeout=3).content - self.task_metadata = json.loads(json_body) + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/ + json_body = self.http_client.get(self.ecmu_url_root, timeout=1).content + self.root_metadata = json.loads(json_body) + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/task + json_body = self.http_client.get(self.ecmu_url_task, timeout=1).content + self.task_metadata = json.loads(json_body) # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/stats - json_body = self.http_client.get(self.ecmu_url_stats, timeout=3).content + json_body = self.http_client.get(self.ecmu_url_stats, timeout=2).content self.stats_metadata = json.loads(json_body) # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/task/stats - json_body = self.http_client.get(self.ecmu_url_task_stats, timeout=3).content + json_body = self.http_client.get(self.ecmu_url_task_stats, timeout=1).content self.task_stats_metadata = json.loads(json_body) except Exception: logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) From 8f8eadab33d726144870be40c24e5e7b03ac492a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 12 Aug 2020 16:37:10 +0200 Subject: [PATCH 74/91] Lock cleanup --- instana/collector/base.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/instana/collector/base.py b/instana/collector/base.py index 81940e50..0c5ec02c 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -40,7 +40,9 @@ def __init__(self, agent): self.snapshot_data_sent = False # Lock used syncronize reporting - no updates when sending - self.lock = threading.Lock() + # Used by the background reporting thread. Used to syncronize report attempts and so + # that we never have two in progress at once. + self.background_report_lock = threading.Lock() # Reporting interval for the background thread(s) self.report_interval = 5 @@ -108,15 +110,15 @@ def prepare_and_report_data(self): if env_is_test is True: return True - lock_acquired = self.lock.acquire(False) + lock_acquired = self.background_report_lock.acquire(False) if lock_acquired: - payload = self.prepare_payload() + try: + payload = self.prepare_payload() - if len(payload) > 0: - self.agent.report_data_payload(payload) - else: - logger.debug("prepare_and_report_data: No data to report") - self.lock.release() + if len(payload) > 0: + self.agent.report_data_payload(payload) + finally: + self.background_report_lock.release() else: logger.debug("prepare_and_report_data: Couldn't acquire lock") return True From 5bd18420dfd340233d2a5e081bf708d9f78792bb Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 12 Aug 2020 16:44:18 +0200 Subject: [PATCH 75/91] Better lock syncronization --- instana/collector/aws_fargate.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 4a41f428..d3f7e7ad 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -41,8 +41,12 @@ def __init__(self, agent): self.ecmu_url_stats = self.ecmu + '/stats' self.ecmu_url_task_stats = self.ecmu + '/task/stats' - # Lock used synchronize data collection + # In AWS Fargate, there are two threads: + # 1. ECS Container Metadata data collection + # 2. Payload preparation and data reporting + # This lock makes sure that only one is executing at a time. self.ecmu_lock = threading.Lock() + # Timestamp in seconds of the last time we fetched all ECMU data self.last_ecmu_full_fetch = 0 # How often to do a full fetch of ECMU data @@ -173,18 +177,24 @@ def prepare_payload(self): if not self.span_queue.empty(): payload["spans"] = self.queued_spans() - plugins = [] with_snapshot = self.should_send_snapshot_data() - try: - for helper in self.helpers: - plugins.extend(helper.collect_metrics(with_snapshot)) - payload["metrics"]["plugins"] = plugins + self.ecmu_lock.acquire(False) + if self.ecmu_lock.locked(): + try: + plugins = [] + for helper in self.helpers: + plugins.extend(helper.collect_metrics(with_snapshot)) + + payload["metrics"]["plugins"] = plugins + + if with_snapshot is True: + self.snapshot_data_last_sent = int(time()) + except Exception: + logger.debug("collect_snapshot error", exc_info=True) + finally: + self.ecmu_lock.release() - if with_snapshot is True: - self.snapshot_data_last_sent = int(time()) - except Exception: - logger.debug("collect_snapshot error", exc_info=True) return payload def get_fq_arn(self): From 8e8b84fe34358a4a2e32761e842d3d47be507c54 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 12 Aug 2020 17:42:14 +0200 Subject: [PATCH 76/91] Fix lock check --- instana/collector/aws_fargate.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index d3f7e7ad..724b526b 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -129,8 +129,8 @@ def get_ecs_metadata(self): # For test, we are using mock ECS metadata return - self.ecmu_lock.acquire(False) - if self.ecmu_lock.locked(): + lock_acquired = self.ecmu_lock.acquire(False) + if lock_acquired is True: try: delta = int(time()) - self.last_ecmu_full_fetch if delta > self.ecmu_full_fetch_interval: @@ -161,7 +161,7 @@ def get_ecs_metadata(self): finally: self.ecmu_lock.release() else: - logger.debug("AWSFargateCollector.get_ecs_metadata: skipping because data collection already in progress") + logger.debug("AWSFargateCollector.get_ecs_metadata: skipping because data collection in progress") def should_send_snapshot_data(self): delta = int(time()) - self.snapshot_data_last_sent @@ -179,8 +179,8 @@ def prepare_payload(self): with_snapshot = self.should_send_snapshot_data() - self.ecmu_lock.acquire(False) - if self.ecmu_lock.locked(): + lock_acquired = self.ecmu_lock.acquire(False) + if lock_acquired is True: try: plugins = [] for helper in self.helpers: @@ -194,6 +194,8 @@ def prepare_payload(self): logger.debug("collect_snapshot error", exc_info=True) finally: self.ecmu_lock.release() + else: + logger.debug("prepare_payload: Could not acquire lock - skipping") return payload From bfd7087e56950337d5ff3f3bb4c5172b43256904 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 12 Aug 2020 18:33:07 +0200 Subject: [PATCH 77/91] Set lock acquire to blocking --- instana/collector/aws_fargate.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 724b526b..df7f3021 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -129,7 +129,7 @@ def get_ecs_metadata(self): # For test, we are using mock ECS metadata return - lock_acquired = self.ecmu_lock.acquire(False) + lock_acquired = self.ecmu_lock.acquire(True, timeout=1) if lock_acquired is True: try: delta = int(time()) - self.last_ecmu_full_fetch @@ -160,8 +160,6 @@ def get_ecs_metadata(self): logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) finally: self.ecmu_lock.release() - else: - logger.debug("AWSFargateCollector.get_ecs_metadata: skipping because data collection in progress") def should_send_snapshot_data(self): delta = int(time()) - self.snapshot_data_last_sent @@ -179,7 +177,7 @@ def prepare_payload(self): with_snapshot = self.should_send_snapshot_data() - lock_acquired = self.ecmu_lock.acquire(False) + lock_acquired = self.ecmu_lock.acquire(True, timeout=1) if lock_acquired is True: try: plugins = [] @@ -194,8 +192,6 @@ def prepare_payload(self): logger.debug("collect_snapshot error", exc_info=True) finally: self.ecmu_lock.release() - else: - logger.debug("prepare_payload: Could not acquire lock - skipping") return payload From 95be15565e7edd39c2fc37e7ef45c7ed7649f88d Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Wed, 12 Aug 2020 20:06:40 +0200 Subject: [PATCH 78/91] Consider with_snapshot in delta reporting --- instana/collector/aws_fargate.py | 2 +- instana/collector/helpers/base.py | 7 +- instana/collector/helpers/fargate/docker.py | 73 +++++++++++---------- instana/collector/helpers/fargate/task.py | 2 +- 4 files changed, 44 insertions(+), 40 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index df7f3021..31570b4e 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -67,7 +67,7 @@ def __init__(self, agent): # Timestamp in seconds of the last time we sent snapshot data self.snapshot_data_last_sent = 0 # How often to report snapshot data (in seconds) - self.snapshot_data_interval = 600 + self.snapshot_data_interval = 300 self.last_payload = None # Response from the last call to diff --git a/instana/collector/helpers/base.py b/instana/collector/helpers/base.py index 297e881b..d0860c55 100644 --- a/instana/collector/helpers/base.py +++ b/instana/collector/helpers/base.py @@ -1,4 +1,4 @@ -""" +""" Base class for the various helpers that can be used by Collectors. Helpers assist in the data collection for various entities such as host, hardware, AWS Task, ec2, memory, cpu, docker etc etc.. @@ -41,7 +41,7 @@ def get_delta(self, source, previous, metric): else: return None - def apply_delta(self, source, previous, new, metric): + def apply_delta(self, source, previous, new, metric, with_snapshot): """ Helper method to assist in delta reporting of metrics. @@ -51,6 +51,7 @@ def apply_delta(self, source, previous, new, metric): @param new [dict]: the new value of the metric that will be sent new (as new[metric]) @param metric [String or Tuple]: the name of the metric in question. If the keys for source[metric], previous[metric] and new[metric] vary, you can pass a tuple in the form of (src, dst) + @param with_snapshot [Bool]: if this metric is being sent with snapshot data @return: None """ if isinstance(metric, tuple): @@ -65,7 +66,7 @@ def apply_delta(self, source, previous, new, metric): else: new_value = source - if previous[dst_metric] != new_value: + if previous[dst_metric] != new_value or with_snapshot is True: previous[dst_metric] = new[dst_metric] = new_value def collect_metrics(self, with_snapshot=False): diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index bda9ed56..662cc1aa 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -1,7 +1,7 @@ """ Module to handle the collection of Docker metrics in AWS Fargate """ from ....log import logger from ..base import BaseHelper -from ....util import DictionaryOfStan, to_pretty_json +from ....util import DictionaryOfStan class DockerHelper(BaseHelper): @@ -34,7 +34,7 @@ def collect_metrics(self, with_snapshot=False): plugin_data["data"] = DictionaryOfStan() # Metrics - self._collect_container_metrics(plugin_data, docker_id) + self._collect_container_metrics(plugin_data, docker_id, with_snapshot) # Snapshot if with_snapshot: @@ -62,15 +62,15 @@ def _collect_container_snapshot(self, plugin_data, container): except Exception: logger.debug("_collect_container_snapshot: ", exc_info=True) - def _collect_container_metrics(self, plugin_data, docker_id): + def _collect_container_metrics(self, plugin_data, docker_id, with_snapshot): container = self.collector.task_stats_metadata.get(docker_id, None) if container is not None: - self._collect_network_metrics(container, plugin_data, docker_id) - self._collect_cpu_metrics(container, plugin_data, docker_id) - self._collect_memory_metrics(container, plugin_data, docker_id) - self._collect_blkio_metrics(container, plugin_data, docker_id) + self._collect_network_metrics(container, plugin_data, docker_id, with_snapshot) + self._collect_cpu_metrics(container, plugin_data, docker_id, with_snapshot) + self._collect_memory_metrics(container, plugin_data, docker_id, with_snapshot) + self._collect_blkio_metrics(container, plugin_data, docker_id, with_snapshot) - def _collect_network_metrics(self, container, plugin_data, docker_id): + def _collect_network_metrics(self, container, plugin_data, docker_id, with_snapshot): try: networks = container.get("networks", None) tx_bytes_total = tx_dropped_total = tx_errors_total = tx_packets_total = 0 @@ -90,26 +90,26 @@ def _collect_network_metrics(self, container, plugin_data, docker_id): rx_packets_total += networks[key].get("rx_packets", 0) self.apply_delta(tx_bytes_total, self.previous[docker_id]["network"]["tx"], - plugin_data["data"]["tx"], "bytes") + plugin_data["data"]["tx"], "bytes", with_snapshot) self.apply_delta(tx_dropped_total, self.previous[docker_id]["network"]["tx"], - plugin_data["data"]["tx"], "dropped") + plugin_data["data"]["tx"], "dropped", with_snapshot) self.apply_delta(tx_errors_total, self.previous[docker_id]["network"]["tx"], - plugin_data["data"]["tx"], "errors") + plugin_data["data"]["tx"], "errors", with_snapshot) self.apply_delta(tx_packets_total, self.previous[docker_id]["network"]["tx"], - plugin_data["data"]["tx"], "packets") + plugin_data["data"]["tx"], "packets", with_snapshot) self.apply_delta(rx_bytes_total, self.previous[docker_id]["network"]["rx"], - plugin_data["data"]["rx"], "bytes") + plugin_data["data"]["rx"], "bytes", with_snapshot) self.apply_delta(rx_dropped_total, self.previous[docker_id]["network"]["rx"], - plugin_data["data"]["rx"], "dropped") + plugin_data["data"]["rx"], "dropped", with_snapshot) self.apply_delta(rx_errors_total, self.previous[docker_id]["network"]["rx"], - plugin_data["data"]["rx"], "errors") + plugin_data["data"]["rx"], "errors", with_snapshot) self.apply_delta(rx_packets_total, self.previous[docker_id]["network"]["rx"], - plugin_data["data"]["rx"], "packets") + plugin_data["data"]["rx"], "packets", with_snapshot) except Exception: logger.debug("_collect_network_metrics: ", exc_info=True) - def _collect_cpu_metrics(self, container, plugin_data, docker_id): + def _collect_cpu_metrics(self, container, plugin_data, docker_id, with_snapshot): try: cpu_stats = container.get("cpu_stats", {}) cpu_usage = cpu_stats.get("cpu_usage", None) @@ -122,54 +122,57 @@ def _collect_cpu_metrics(self, container, plugin_data, docker_id): metric_value = (cpu_usage["total_usage"] / system_cpu_usage) * online_cpus self.apply_delta(round(metric_value, 6), self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], "total_usage") + plugin_data["data"]["cpu"], "total_usage", with_snapshot) metric_value = (cpu_usage["usage_in_usermode"] / system_cpu_usage) * online_cpus self.apply_delta(round(metric_value, 6), self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], "user_usage") + plugin_data["data"]["cpu"], "user_usage", with_snapshot) metric_value = (cpu_usage["usage_in_kernelmode"] / system_cpu_usage) * online_cpus self.apply_delta(round(metric_value, 6), self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], "system_usage") + plugin_data["data"]["cpu"], "system_usage", with_snapshot) if throttling_data is not None: self.apply_delta(throttling_data, self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], ("periods", "throttling_count")) + plugin_data["data"]["cpu"], ("periods", "throttling_count"), with_snapshot) self.apply_delta(throttling_data, self.previous[docker_id]["cpu"], - plugin_data["data"]["cpu"], ("throttled_time", "throttling_time")) + plugin_data["data"]["cpu"], ("throttled_time", "throttling_time"), with_snapshot) except Exception: logger.debug("_collect_cpu_metrics: ", exc_info=True) - def _collect_memory_metrics(self, container, plugin_data, docker_id): + def _collect_memory_metrics(self, container, plugin_data, docker_id, with_snapshot): try: memory = container.get("memory_stats", {}) memory_stats = memory.get("stats", None) - self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "usage") - self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "max_usage") - self.apply_delta(memory, self.previous[docker_id]["memory"], plugin_data["data"]["memory"], "limit") + self.apply_delta(memory, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "usage", with_snapshot) + self.apply_delta(memory, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "max_usage", with_snapshot) + self.apply_delta(memory, self.previous[docker_id]["memory"], + plugin_data["data"]["memory"], "limit", with_snapshot) if memory_stats is not None: self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "active_anon") + plugin_data["data"]["memory"], "active_anon", with_snapshot) self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "active_file") + plugin_data["data"]["memory"], "active_file", with_snapshot) self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "inactive_anon") + plugin_data["data"]["memory"], "inactive_anon", with_snapshot) self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "inactive_file") + plugin_data["data"]["memory"], "inactive_file", with_snapshot) self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "total_cache") + plugin_data["data"]["memory"], "total_cache", with_snapshot) self.apply_delta(memory_stats, self.previous[docker_id]["memory"], - plugin_data["data"]["memory"], "total_rss") + plugin_data["data"]["memory"], "total_rss", with_snapshot) except Exception: logger.debug("_collect_memory_metrics: ", exc_info=True) - def _collect_blkio_metrics(self, container, plugin_data, docker_id): + def _collect_blkio_metrics(self, container, plugin_data, docker_id, with_snapshot): try: blkio_stats = container.get("blkio_stats", None) if blkio_stats is not None: @@ -178,9 +181,9 @@ def _collect_blkio_metrics(self, container, plugin_data, docker_id): for entry in service_bytes: if entry["op"] == "Read": self.apply_delta(entry, self.previous[docker_id]["blkio"], - plugin_data["data"]["blkio"], ("value", "blk_read")) + plugin_data["data"]["blkio"], ("value", "blk_read"), with_snapshot) elif entry["op"] == "Write": self.apply_delta(entry, self.previous[docker_id]["blkio"], - plugin_data["data"]["blkio"], ("value", "blk_write")) + plugin_data["data"]["blkio"], ("value", "blk_write"), with_snapshot) except Exception: logger.debug("_collect_blkio_metrics: ", exc_info=True) diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index ff089683..87731de3 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -1,7 +1,7 @@ """ Module to assist in the data collection about the AWS Fargate task that is running this process """ from ....log import logger from ..base import BaseHelper -from ....util import DictionaryOfStan, to_pretty_json +from ....util import DictionaryOfStan class TaskHelper(BaseHelper): From b5de03cba243bf00bce08f91badd2aff05c741d9 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 11:06:05 +0200 Subject: [PATCH 79/91] Fix reporting of mandatory fields --- .../collector/helpers/fargate/container.py | 40 +++++++++---------- instana/collector/helpers/fargate/docker.py | 2 +- instana/collector/helpers/fargate/task.py | 27 ++++++------- instana/collector/helpers/process.py | 11 ++--- 4 files changed, 36 insertions(+), 44 deletions(-) diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index 70373608..7bbac55c 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -13,9 +13,6 @@ def collect_metrics(self, with_snapshot=False): """ plugins = [] - if with_snapshot is False: - return plugins - try: if self.collector.task_metadata is not None: containers = self.collector.task_metadata.get("Containers", []) @@ -31,26 +28,27 @@ def collect_metrics(self, with_snapshot=False): plugin_data["data"] = DictionaryOfStan() if self.collector.root_metadata["Name"] == name: plugin_data["data"]["instrumented"] = True - plugin_data["data"]["runtime"] = "python" plugin_data["data"]["dockerId"] = container.get("DockerId", None) - plugin_data["data"]["dockerName"] = container.get("DockerName", None) - plugin_data["data"]["containerName"] = container.get("Name", None) - plugin_data["data"]["image"] = container.get("Image", None) - plugin_data["data"]["imageId"] = container.get("ImageID", None) plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) - plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) - plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) - plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) - plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) - plugin_data["data"]["ports"] = container.get("Ports", None) - plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) - plugin_data["data"]["startedAt"] = container.get("StartedAt", None) - plugin_data["data"]["type"] = container.get("Type", None) - limits = container.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - #logger.debug(to_pretty_json(plugin_data)) + + if with_snapshot is True: + plugin_data["data"]["runtime"] = "python" + plugin_data["data"]["dockerName"] = container.get("DockerName", None) + plugin_data["data"]["containerName"] = container.get("Name", None) + plugin_data["data"]["image"] = container.get("Image", None) + plugin_data["data"]["imageId"] = container.get("ImageID", None) + plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) + plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) + plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) + plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) + plugin_data["data"]["ports"] = container.get("Ports", None) + plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) + plugin_data["data"]["startedAt"] = container.get("StartedAt", None) + plugin_data["data"]["type"] = container.get("Type", None) + limits = container.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) except Exception: logger.debug("_collect_container_snapshots: ", exc_info=True) finally: diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index 662cc1aa..ed4caeb3 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -32,6 +32,7 @@ def collect_metrics(self, with_snapshot=False): plugin_data["entityId"] = "%s::%s" % (task_arn, name) plugin_data["data"] = DictionaryOfStan() + plugin_data["data"]["Id"] = container.get("DockerId", None) # Metrics self._collect_container_metrics(plugin_data, docker_id, with_snapshot) @@ -49,7 +50,6 @@ def collect_metrics(self, with_snapshot=False): def _collect_container_snapshot(self, plugin_data, container): try: # Snapshot Data - plugin_data["data"]["Id"] = container.get("DockerId", None) plugin_data["data"]["Created"] = container.get("CreatedAt", None) plugin_data["data"]["Started"] = container.get("StartedAt", None) plugin_data["data"]["Image"] = container.get("Image", None) diff --git a/instana/collector/helpers/fargate/task.py b/instana/collector/helpers/fargate/task.py index 87731de3..c21c4e3b 100644 --- a/instana/collector/helpers/fargate/task.py +++ b/instana/collector/helpers/fargate/task.py @@ -13,9 +13,6 @@ def collect_metrics(self, with_snapshot=False): """ plugins = [] - if with_snapshot is False: - return plugins - try: if self.collector.task_metadata is not None: try: @@ -28,21 +25,21 @@ def collect_metrics(self, with_snapshot=False): plugin_data["data"]["taskDefinition"] = self.collector.task_metadata.get("Family", None) plugin_data["data"]["taskDefinitionVersion"] = self.collector.task_metadata.get("Revision", None) plugin_data["data"]["availabilityZone"] = self.collector.task_metadata.get("AvailabilityZone", None) - plugin_data["data"]["desiredStatus"] = self.collector.task_metadata.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = self.collector.task_metadata.get("KnownStatus", None) - plugin_data["data"]["pullStartedAt"] = self.collector.task_metadata.get("PullStartedAt", None) - plugin_data["data"]["pullStoppedAt"] = self.collector.task_metadata.get("PullStoppeddAt", None) - limits = self.collector.task_metadata.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - if self.collector.agent.options.zone is not None: - plugin_data["data"]["instanaZone"] = self.collector.agent.options.zone + if with_snapshot is True: + plugin_data["data"]["desiredStatus"] = self.collector.task_metadata.get("DesiredStatus", None) + plugin_data["data"]["knownStatus"] = self.collector.task_metadata.get("KnownStatus", None) + plugin_data["data"]["pullStartedAt"] = self.collector.task_metadata.get("PullStartedAt", None) + plugin_data["data"]["pullStoppedAt"] = self.collector.task_metadata.get("PullStoppeddAt", None) + limits = self.collector.task_metadata.get("Limits", {}) + plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) + plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) - if self.collector.agent.options.tags is not None: - plugin_data["data"]["tags"] = self.collector.agent.options.tags + if self.collector.agent.options.zone is not None: + plugin_data["data"]["instanaZone"] = self.collector.agent.options.zone - #logger.debug(to_pretty_json(plugin_data)) + if self.collector.agent.options.tags is not None: + plugin_data["data"]["tags"] = self.collector.agent.options.tags except Exception: logger.debug("collect_task_metrics: ", exc_info=True) finally: diff --git a/instana/collector/helpers/process.py b/instana/collector/helpers/process.py index cc1a498b..c663350a 100644 --- a/instana/collector/helpers/process.py +++ b/instana/collector/helpers/process.py @@ -13,12 +13,13 @@ def collect_metrics(self, with_snapshot=False): try: plugin_data["name"] = "com.instana.plugin.process" plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"]["pid"] = int(os.getpid()) + plugin_data["data"]["containerType"] = "docker" + if self.collector.root_metadata is not None: + plugin_data["data"]["container"] = self.collector.root_metadata.get("DockerId") - # Snapshot if with_snapshot: self._collect_process_snapshot(plugin_data) - - #logger.debug(to_pretty_json(plugin_data)) except Exception: logger.debug("ProcessHelper.collect_metrics: ", exc_info=True) return [plugin_data] @@ -26,7 +27,6 @@ def collect_metrics(self, with_snapshot=False): def _collect_process_snapshot(self, plugin_data): try: plugin_data["data"] = DictionaryOfStan() - plugin_data["data"]["pid"] = int(os.getpid()) env = dict() for key in os.environ: if contains_secret(key, @@ -55,9 +55,6 @@ def _collect_process_snapshot(self, plugin_data): logger.debug("euid/egid detection: ", exc_info=True) plugin_data["data"]["start"] = 1 # FIXME - plugin_data["data"]["containerType"] = "docker" - if self.collector.root_metadata is not None: - plugin_data["data"]["container"] = self.collector.root_metadata.get("DockerId") # plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME: the pid in the root namespace (very optional) if self.collector.task_metadata is not None: plugin_data["data"]["com.instana.plugin.host.name"] = self.collector.task_metadata.get("TaskArn") From 7745a46760c79c9f657d907cfa3770abb244d6ac Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 11:29:33 +0200 Subject: [PATCH 80/91] Fix data payload init --- instana/collector/helpers/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/collector/helpers/process.py b/instana/collector/helpers/process.py index c663350a..1549bf18 100644 --- a/instana/collector/helpers/process.py +++ b/instana/collector/helpers/process.py @@ -13,6 +13,7 @@ def collect_metrics(self, with_snapshot=False): try: plugin_data["name"] = "com.instana.plugin.process" plugin_data["entityId"] = str(os.getpid()) + plugin_data["data"] = DictionaryOfStan() plugin_data["data"]["pid"] = int(os.getpid()) plugin_data["data"]["containerType"] = "docker" if self.collector.root_metadata is not None: @@ -26,7 +27,6 @@ def collect_metrics(self, with_snapshot=False): def _collect_process_snapshot(self, plugin_data): try: - plugin_data["data"] = DictionaryOfStan() env = dict() for key in os.environ: if contains_secret(key, From c1246c094f9462bf92c7ee586a264d97652dab2a Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 12:26:11 +0200 Subject: [PATCH 81/91] Simplify things and remove a thread --- instana/collector/aws_fargate.py | 107 +++++++++------------- instana/collector/base.py | 2 +- tests/platforms/test_fargate_collector.py | 9 +- 3 files changed, 44 insertions(+), 74 deletions(-) diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 31570b4e..47597b7e 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -41,19 +41,16 @@ def __init__(self, agent): self.ecmu_url_stats = self.ecmu + '/stats' self.ecmu_url_task_stats = self.ecmu + '/task/stats' - # In AWS Fargate, there are two threads: - # 1. ECS Container Metadata data collection - # 2. Payload preparation and data reporting - # This lock makes sure that only one is executing at a time. - self.ecmu_lock = threading.Lock() - # Timestamp in seconds of the last time we fetched all ECMU data self.last_ecmu_full_fetch = 0 + # How often to do a full fetch of ECMU data - self.ecmu_full_fetch_interval = 605 + self.ecmu_full_fetch_interval = 304 # How often to report data self.report_interval = 1 + + # HTTP client with keep-alive self.http_client = requests.Session() # This is the collecter thread querying the metadata url @@ -101,65 +98,44 @@ def start(self): logger.warning("AWS Fargate Collector is missing requirements and cannot monitor this environment.") return - # Launch a thread here to periodically collect data from the ECS Metadata Container API - if self.agent.can_send(): - logger.debug("AWSFargateCollector.start: launching ecs metadata collection thread") - self.ecs_metadata_thread = threading.Thread(target=self.metadata_thread_loop, args=()) - self.ecs_metadata_thread.setDaemon(True) - self.ecs_metadata_thread.start() - else: - logger.warning("Collector started but the agent tells us we can't send anything out.") - super(AWSFargateCollector, self).start() - def metadata_thread_loop(self): - """ - Just a loop that is run in the background thread. - @return: None - """ - every(self.report_interval, self.get_ecs_metadata, "AWSFargateCollector: metadata_thread_loop") - def get_ecs_metadata(self): """ Get the latest data from the ECS metadata container API and store on the class @return: Boolean """ - if env_is_test is True: # For test, we are using mock ECS metadata return - lock_acquired = self.ecmu_lock.acquire(True, timeout=1) - if lock_acquired is True: - try: - delta = int(time()) - self.last_ecmu_full_fetch - if delta > self.ecmu_full_fetch_interval: - # Refetch the ECMU snapshot data - self.last_ecmu_full_fetch = int(time()) - - # Response from the last call to - # ${ECS_CONTAINER_METADATA_URI}/ - json_body = self.http_client.get(self.ecmu_url_root, timeout=1).content - self.root_metadata = json.loads(json_body) - - # Response from the last call to - # ${ECS_CONTAINER_METADATA_URI}/task - json_body = self.http_client.get(self.ecmu_url_task, timeout=1).content - self.task_metadata = json.loads(json_body) + try: + delta = int(time()) - self.last_ecmu_full_fetch + if delta > self.ecmu_full_fetch_interval: + # Refetch the ECMU snapshot data + self.last_ecmu_full_fetch = int(time()) # Response from the last call to - # ${ECS_CONTAINER_METADATA_URI}/stats - json_body = self.http_client.get(self.ecmu_url_stats, timeout=2).content - self.stats_metadata = json.loads(json_body) + # ${ECS_CONTAINER_METADATA_URI}/ + json_body = self.http_client.get(self.ecmu_url_root, timeout=1).content + self.root_metadata = json.loads(json_body) # Response from the last call to - # ${ECS_CONTAINER_METADATA_URI}/task/stats - json_body = self.http_client.get(self.ecmu_url_task_stats, timeout=1).content - self.task_stats_metadata = json.loads(json_body) - except Exception: - logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) - finally: - self.ecmu_lock.release() + # ${ECS_CONTAINER_METADATA_URI}/task + json_body = self.http_client.get(self.ecmu_url_task, timeout=1).content + self.task_metadata = json.loads(json_body) + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/stats + json_body = self.http_client.get(self.ecmu_url_stats, timeout=2).content + self.stats_metadata = json.loads(json_body) + + # Response from the last call to + # ${ECS_CONTAINER_METADATA_URI}/task/stats + json_body = self.http_client.get(self.ecmu_url_task_stats, timeout=1).content + self.task_stats_metadata = json.loads(json_body) + except Exception: + logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) def should_send_snapshot_data(self): delta = int(time()) - self.snapshot_data_last_sent @@ -177,21 +153,20 @@ def prepare_payload(self): with_snapshot = self.should_send_snapshot_data() - lock_acquired = self.ecmu_lock.acquire(True, timeout=1) - if lock_acquired is True: - try: - plugins = [] - for helper in self.helpers: - plugins.extend(helper.collect_metrics(with_snapshot)) - - payload["metrics"]["plugins"] = plugins - - if with_snapshot is True: - self.snapshot_data_last_sent = int(time()) - except Exception: - logger.debug("collect_snapshot error", exc_info=True) - finally: - self.ecmu_lock.release() + # Fetch the latest metrics + self.get_ecs_metadata() + + try: + plugins = [] + for helper in self.helpers: + plugins.extend(helper.collect_metrics(with_snapshot)) + + payload["metrics"]["plugins"] = plugins + + if with_snapshot is True: + self.snapshot_data_last_sent = int(time()) + except Exception: + logger.debug("collect_snapshot error", exc_info=True) return payload diff --git a/instana/collector/base.py b/instana/collector/base.py index 0c5ec02c..8d012e50 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -28,7 +28,7 @@ def __init__(self, agent): # The Queue where we store finished spans before they are sent self.span_queue = queue.Queue() - # The background thread that reports data in a loop every self.REPORT_INTERVAL seconds + # The background thread that reports data in a loop every self.report_interval seconds self.reporting_thread = None # Signal for background thread(s) to shutdown diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index bb05681c..06d5a616 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -128,7 +128,7 @@ def test_docker_plugin_snapshot_data(self): assert(plugin_second_report) assert("data" in plugin_second_report) data = plugin_second_report["data"] - assert("Id" not in data) + assert("Id" in data) assert("Created" not in data) assert("Started" not in data) assert("Image" not in data) @@ -155,12 +155,7 @@ def test_docker_plugin_metrics(self): # First report should report all metrics data = plugin_first_report.get("data", None) assert(data) - - # FIXME - # network = data.get("network", None) - # assert(network) - # assert("rx" in network) - # assert("tx" in network) + assert "network" not in data cpu = data.get("cpu", None) assert(cpu) From 6f968836cb0762500a265d65284ef719a8007325 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 15:10:08 +0200 Subject: [PATCH 82/91] Change test container image to make cassandra happy --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 173577e5..6b6eb64d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,7 +77,7 @@ jobs: py27cassandra: docker: - - image: circleci/python:2.7.18 + - image: circleci/python:2.7.16-stretch - image: circleci/cassandra:3.10 environment: MAX_HEAP_SIZE: 2048m From fb3d10968f652d14b73aedd8ff89f3502ed0c334 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 15:29:22 +0200 Subject: [PATCH 83/91] Py27 tests use stretch --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b6eb64d..a4946168 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ version: 2 jobs: python27: docker: - - image: circleci/python:2.7.18 + - image: circleci/python:2.7.16-stretch - image: circleci/postgres:9.6.5-alpine-ram - image: circleci/mariadb:10.1-ram - image: circleci/redis:5.0.4 From e0fdf8168977e85fefc5275f254daa6c7e90e657 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 16:09:10 +0200 Subject: [PATCH 84/91] Add Python 2.7 compatibility division --- instana/collector/helpers/fargate/docker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index ed4caeb3..c69a075b 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -1,4 +1,5 @@ """ Module to handle the collection of Docker metrics in AWS Fargate """ +from __future__ import division from ....log import logger from ..base import BaseHelper from ....util import DictionaryOfStan From 88cc2fcdad4b5dca3b1290c1abd8ad05c5d3a20d Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Thu, 13 Aug 2020 16:56:02 +0200 Subject: [PATCH 85/91] pylint all the things --- instana/agent/aws_lambda.py | 10 +++--- instana/agent/base.py | 3 ++ instana/agent/host.py | 1 - instana/collector/aws_fargate.py | 15 ++++---- instana/collector/aws_lambda.py | 5 ++- instana/collector/base.py | 6 ++-- .../collector/helpers/fargate/container.py | 2 +- instana/collector/helpers/process.py | 6 ++-- instana/collector/helpers/runtime.py | 11 +++--- instana/options.py | 12 +++---- instana/util.py | 34 +++++++++---------- 11 files changed, 54 insertions(+), 51 deletions(-) diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index dd232ebe..b790647b 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -8,8 +8,8 @@ from ..log import logger from ..util import to_json from .base import BaseAgent -from instana.collector.aws_lambda import AWSLambdaCollector -from instana.options import AWSLambdaOptions +from ..collector.aws_lambda import AWSLambdaCollector +from ..options import AWSLambdaOptions class AWSLambdaFrom(object): @@ -88,7 +88,7 @@ def report_data_payload(self, payload): headers=self.report_headers, timeout=self.options.timeout, verify=ssl_verify, - proxies = self.options.endpoint_proxy) + proxies=self.options.endpoint_proxy) if 200 <= response.status_code < 300: logger.debug("report_data_payload: Instana responded with status code %s", response.status_code) @@ -96,8 +96,8 @@ def report_data_payload(self, payload): logger.info("report_data_payload: Instana responded with status code %s", response.status_code) except Exception as e: logger.debug("report_data_payload: connection error (%s)", type(e)) - finally: - return response + + return response def _validate_options(self): """ diff --git a/instana/agent/base.py b/instana/agent/base.py index c1beff9c..98559369 100644 --- a/instana/agent/base.py +++ b/instana/agent/base.py @@ -1,3 +1,6 @@ +""" +Base class for all the agent flavors +""" import requests diff --git a/instana/agent/host.py b/instana/agent/host.py index 5de7ca7a..47820189 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -139,7 +139,6 @@ def set_from(self, json_string): self.options.secrets_list = res_data['secrets']['list'] if "extraHeaders" in res_data: - # FIXME: add tests if self.options.extra_http_headers is None: self.options.extra_http_headers = res_data['extraHeaders'] else: diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 47597b7e..6db92b72 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -3,13 +3,12 @@ """ import os import json -import threading from time import time import requests from ..log import logger from .base import BaseCollector -from ..util import DictionaryOfStan, every, validate_url +from ..util import DictionaryOfStan, validate_url from ..singletons import env_is_test from .helpers.process import ProcessHelper @@ -148,15 +147,15 @@ def prepare_payload(self): payload["spans"] = [] payload["metrics"]["plugins"] = [] - if not self.span_queue.empty(): - payload["spans"] = self.queued_spans() + try: + if not self.span_queue.empty(): + payload["spans"] = self.queued_spans() - with_snapshot = self.should_send_snapshot_data() + with_snapshot = self.should_send_snapshot_data() - # Fetch the latest metrics - self.get_ecs_metadata() + # Fetch the latest metrics + self.get_ecs_metadata() - try: plugins = [] for helper in self.helpers: plugins.extend(helper.collect_metrics(with_snapshot)) diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index 97f638e2..d83181e9 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -1,9 +1,13 @@ +""" +Snapshot & metrics collection for AWS Lambda +""" from ..log import logger from .base import BaseCollector from ..util import DictionaryOfStan, normalize_aws_lambda_arn class AWSLambdaCollector(BaseCollector): + """ Collector for AWS Lambda """ def __init__(self, agent): super(AWSLambdaCollector, self).__init__(agent) logger.debug("Loading AWS Lambda Collector") @@ -53,4 +57,3 @@ def get_fq_arn(self): self._fq_arn = normalize_aws_lambda_arn(self.context) return self._fq_arn - diff --git a/instana/collector/base.py b/instana/collector/base.py index 8d012e50..43cfed56 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -13,7 +13,7 @@ if sys.version_info.major == 2: import Queue as queue else: - import queue + import queue # pylint: disable=import-error class BaseCollector(object): @@ -114,9 +114,7 @@ def prepare_and_report_data(self): if lock_acquired: try: payload = self.prepare_payload() - - if len(payload) > 0: - self.agent.report_data_payload(payload) + self.agent.report_data_payload(payload) finally: self.background_report_lock.release() else: diff --git a/instana/collector/helpers/fargate/container.py b/instana/collector/helpers/fargate/container.py index 7bbac55c..4193e64e 100644 --- a/instana/collector/helpers/fargate/container.py +++ b/instana/collector/helpers/fargate/container.py @@ -1,6 +1,6 @@ """ Module to handle the collection of container metrics in AWS Fargate """ from ....log import logger -from ....util import DictionaryOfStan, to_pretty_json +from ....util import DictionaryOfStan from ..base import BaseHelper diff --git a/instana/collector/helpers/process.py b/instana/collector/helpers/process.py index 1549bf18..1f7dd853 100644 --- a/instana/collector/helpers/process.py +++ b/instana/collector/helpers/process.py @@ -1,8 +1,9 @@ +""" Collection helper for the process """ import os import pwd import grp from instana.log import logger -from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret, to_pretty_json +from instana.util import DictionaryOfStan, get_proc_cmdline, contains_secret from .base import BaseHelper @@ -54,8 +55,7 @@ def _collect_process_snapshot(self, plugin_data): except Exception: logger.debug("euid/egid detection: ", exc_info=True) - plugin_data["data"]["start"] = 1 # FIXME - # plugin_data["data"]["com.instana.plugin.host.pid"] = 1 # FIXME: the pid in the root namespace (very optional) + plugin_data["data"]["start"] = 1 # FIXME: process start time reporting if self.collector.task_metadata is not None: plugin_data["data"]["com.instana.plugin.host.name"] = self.collector.task_metadata.get("TaskArn") except Exception: diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index 69fc6ee4..11284479 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -1,3 +1,4 @@ +""" Collection helper for the Python runtime """ import os import copy import gc as gc_ @@ -9,7 +10,7 @@ from pkg_resources import DistributionNotFound, get_distribution from instana.log import logger -from instana.util import DictionaryOfStan, determine_service_name, to_pretty_json +from instana.util import DictionaryOfStan, determine_service_name from .base import BaseHelper @@ -66,7 +67,7 @@ def gather_metrics(self): thr = threading.enumerate() daemon_threads = [tr.daemon is True for tr in thr].count(True) alive_threads = [tr.daemon is False for tr in thr].count(True) - dummy_threads = [isinstance(tr, threading._DummyThread) for tr in thr].count(True) + dummy_threads = [isinstance(tr, threading._DummyThread) for tr in thr].count(True) # pylint: disable=protected-access m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0], ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1], @@ -106,7 +107,7 @@ def gather_snapshot(self): snapshot_payload['f'] = platform.python_implementation() # flavor snapshot_payload['a'] = platform.architecture()[0] # architecture snapshot_payload['versions'] = self.gather_python_packages() - #snapshot_payload['djmw'] = None # FIXME + #snapshot_payload['djmw'] = None # FIXME: django middleware reporting except Exception: logger.debug("collect_snapshot: ", exc_info=True) return snapshot_payload @@ -141,8 +142,8 @@ def gather_python_packages(self): except Exception: logger.debug("gather_python_packages", exc_info=True) - finally: - return versions + + return versions def jsonable(self, value): try: diff --git a/instana/options.py b/instana/options.py index e050738a..3b2c6cbc 100644 --- a/instana/options.py +++ b/instana/options.py @@ -7,6 +7,7 @@ class BaseOptions(object): + """ Base class for all option classes. Holds items common to all """ def __init__(self, **kwds): self.debug = False self.log_level = logging.WARN @@ -48,7 +49,7 @@ def __init__(self, **kwds): self.agent_host = os.environ.get("INSTANA_AGENT_HOST", self.AGENT_DEFAULT_HOST) self.agent_port = os.environ.get("INSTANA_AGENT_PORT", self.AGENT_DEFAULT_PORT) - if type(self.agent_port) is str: + if not isinstance(self.agent_port, int): self.agent_port = int(self.agent_port) @@ -65,9 +66,9 @@ def __init__(self, **kwds): proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) if proxy is None: - self.endpoint_proxy = { } + self.endpoint_proxy = {} else: - self.endpoint_proxy = {'https': proxy } + self.endpoint_proxy = {'https': proxy} self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) @@ -86,9 +87,9 @@ def __init__(self, **kwds): proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) if proxy is None: - self.endpoint_proxy = { } + self.endpoint_proxy = {} else: - self.endpoint_proxy = {'https': proxy } + self.endpoint_proxy = {'https': proxy} self.tags = None tag_list = os.environ.get("INSTANA_TAGS", None) @@ -109,4 +110,3 @@ def __init__(self, **kwds): self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) self.zone = os.environ.get("INSTANA_ZONE", None) - diff --git a/instana/util.py b/instana/util.py index 686702ba..ad5bac89 100644 --- a/instana/util.py +++ b/instana/util.py @@ -5,8 +5,8 @@ import sys import time -import pkg_resources from collections import defaultdict +import pkg_resources try: from urllib import parse @@ -154,8 +154,8 @@ def package_version(): version = pkg_resources.get_distribution('instana').version except pkg_resources.DistributionNotFound: version = 'unknown' - finally: - return version + + return version def contains_secret(candidate, matcher, kwlist): @@ -172,7 +172,7 @@ def contains_secret(candidate, matcher, kwlist): if candidate is None or candidate == "INSTANA_AGENT_KEY": return False - if type(kwlist) is not list: + if not isinstance(kwlist, list): logger.debug("contains_secret: bad keyword list") return False @@ -225,7 +225,7 @@ def strip_secrets_from_query(qp, matcher, kwlist): if qp is None: return '' - if type(kwlist) is not list: + if not isinstance(kwlist, list): logger.debug("strip_secrets_from_query: bad keyword list") return qp @@ -312,7 +312,7 @@ def get_default_gateway(): with open("/proc/self/net/route") as routes: for line in routes: parts = line.split('\t') - if '00000000' == parts[1]: + if parts[1] == '00000000': hip = parts[2] if hip is not None and len(hip) == 8: @@ -323,28 +323,28 @@ def get_default_gateway(): logger.warning("get_default_gateway: ", exc_info=True) -def get_py_source(file): +def get_py_source(filename): """ Retrieves and returns the source code for any Python files requested by the UI via the host agent - @param file [String] The fully qualified path to a file + @param filename [String] The fully qualified path to a file """ response = None try: - if regexp_py.search(file) is None: + if regexp_py.search(filename) is None: response = {"error": "Only Python source files are allowed. (*.py)"} else: pysource = "" - with open(file, 'r') as pyfile: + with open(filename, 'r') as pyfile: pysource = pyfile.read() response = {"data": pysource} - except Exception as e: - response = {"error": str(e)} - finally: - return response + except Exception as exc: + response = {"error": str(exc)} + + return response # Used by get_py_source @@ -354,7 +354,7 @@ def get_py_source(file): def every(delay, task, name): """ Executes a task every `delay` seconds - + :param delay: the delay in seconds :param task: the method to run. The method should return False if you want the loop to stop. :return: None @@ -436,7 +436,7 @@ def determine_service_name(): except ImportError: pass return app_name - except Exception as e: + except Exception: logger.debug("get_application_name: ", exc_info=True) return app_name @@ -483,4 +483,4 @@ def validate_url(url): result = parse.urlparse(url) return all([result.scheme, result.netloc]) except: - return False \ No newline at end of file + return False From 0e3100d8254c8a660ba04f6a56e87d7c8c100641 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 14 Aug 2020 10:12:13 +0200 Subject: [PATCH 86/91] Updated hierarchy of Option classes; INSTANA_TIMEOUT in ms --- instana/agent/aws_fargate.py | 19 +++++------ instana/agent/aws_lambda.py | 14 +++----- instana/options.py | 62 ++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index c0a6ee1d..0af072ff 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -68,25 +68,22 @@ def report_data_payload(self, payload): """ response = None try: - # Prepare request headers - self.report_headers = dict() - self.report_headers["Content-Type"] = "application/json" - self.report_headers["X-Instana-Host"] = self.collector.get_fq_arn() - self.report_headers["X-Instana-Key"] = self.options.agent_key + if self.report_headers is None: + # Prepare request headers + self.report_headers = dict() + self.report_headers["Content-Type"] = "application/json" + self.report_headers["X-Instana-Host"] = self.collector.get_fq_arn() + self.report_headers["X-Instana-Key"] = self.options.agent_key + self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) # logger.debug("using these headers: %s", self.report_headers) - if 'INSTANA_DISABLE_CA_CHECK' in os.environ: - ssl_verify = False - else: - ssl_verify = True - response = self.client.post(self.__data_bundle_url(), data=to_json(payload), headers=self.report_headers, timeout=self.options.timeout, - verify=ssl_verify, + verify=self.options.ssl_verify, proxies=self.options.endpoint_proxy) if not 200 <= response.status_code < 300: diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index b790647b..bf3b6409 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -74,28 +74,24 @@ def report_data_payload(self, payload): self.report_headers["Content-Type"] = "application/json" self.report_headers["X-Instana-Host"] = self.collector.get_fq_arn() self.report_headers["X-Instana-Key"] = self.options.agent_key - self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) - # logger.debug("using these headers: %s", self.report_headers) + self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) - if 'INSTANA_DISABLE_CA_CHECK' in os.environ: - ssl_verify = False - else: - ssl_verify = True + # logger.debug("using these headers: %s", self.report_headers) response = self.client.post(self.__data_bundle_url(), data=to_json(payload), headers=self.report_headers, timeout=self.options.timeout, - verify=ssl_verify, + verify=self.options.ssl_verify, proxies=self.options.endpoint_proxy) if 200 <= response.status_code < 300: logger.debug("report_data_payload: Instana responded with status code %s", response.status_code) else: logger.info("report_data_payload: Instana responded with status code %s", response.status_code) - except Exception as e: - logger.debug("report_data_payload: connection error (%s)", type(e)) + except Exception as exc: + logger.debug("report_data_payload: connection error (%s)", type(exc)) return response diff --git a/instana/options.py b/instana/options.py index 3b2c6cbc..68ed563e 100644 --- a/instana/options.py +++ b/instana/options.py @@ -1,4 +1,14 @@ -""" Options for the in-process Instana agent """ +""" +Option classes for the in-process Instana agent + +The description and hierarchy of the classes in this file are as follows: + +BaseOptions - base class for all environments. Holds settings common to all. + - StandardOptions - The options class used when running directly on a host/node with an Instana agent + - ServerlessOptions - Base class for serverless environments. Holds settings common to all serverless environments. + - AWSLambdaOptions - Options class for AWS Lambda. Holds settings specific to AWS Lambda. + - AWSFargateOptions - Options class for AWS Fargate. Holds settings specific to AWS Fargate. +""" import os import logging @@ -39,7 +49,7 @@ def __init__(self, **kwds): class StandardOptions(BaseOptions): - """ Configurable option bits for this package """ + """ The options class used when running directly on a host/node with an Instana agent """ AGENT_DEFAULT_HOST = "localhost" AGENT_DEFAULT_PORT = 42699 @@ -53,16 +63,22 @@ def __init__(self, **kwds): self.agent_port = int(self.agent_port) -class AWSLambdaOptions(BaseOptions): - """ Configurable option bits for AWS Lambda """ +class ServerlessOptions(BaseOptions): + """ Base class for serverless environments. Holds settings common to all serverless environments. """ def __init__(self, **kwds): - super(AWSLambdaOptions, self).__init__() + super(ServerlessOptions, self).__init__() self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) + # Remove any trailing slash (if any) if self.endpoint_url is not None and self.endpoint_url[-1] == "/": self.endpoint_url = self.endpoint_url[:-1] + + if 'INSTANA_DISABLE_CA_CHECK' in os.environ: + self.ssl_verify = False + else: + self.ssl_verify = True proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) if proxy is None: @@ -70,26 +86,32 @@ def __init__(self, **kwds): else: self.endpoint_proxy = {'https': proxy} - self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) + timeout_in_ms = os.environ.get("INSTANA_TIMEOUT", None) + if timeout_in_ms is None: + self.timeout = 0.8 + else: + # Convert the value from milliseconds to seconds for the requests package + try: + self.timeout = int(timeout_in_ms) / 1000 + except ValueError: + logger.warning("Likely invalid INSTANA_TIMEOUT=%s value. Using default.", timeout_in_ms) + logger.warning("INSTANA_TIMEOUT should specify timeout in milliseconds. See " + "https://www.instana.com/docs/reference/environment_variables/#serverless-monitoring") + self.timeout = 0.8 + self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) -class AWSFargateOptions(BaseOptions): - """ Configurable option bits for AWS Fargate """ +class AWSLambdaOptions(ServerlessOptions): + """ Options class for AWS Lambda. Holds settings specific to AWS Lambda. """ def __init__(self, **kwds): - super(AWSFargateOptions, self).__init__() + super(AWSLambdaOptions, self).__init__() - self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) - self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) - # Remove any trailing slash (if any) - if self.endpoint_url is not None and self.endpoint_url[-1] == "/": - self.endpoint_url = self.endpoint_url[:-1] - proxy = os.environ.get("INSTANA_ENDPOINT_PROXY", None) - if proxy is None: - self.endpoint_proxy = {} - else: - self.endpoint_proxy = {'https': proxy} +class AWSFargateOptions(ServerlessOptions): + """ Options class for AWS Fargate. Holds settings specific to AWS Fargate. """ + def __init__(self, **kwds): + super(AWSFargateOptions, self).__init__() self.tags = None tag_list = os.environ.get("INSTANA_TAGS", None) @@ -107,6 +129,4 @@ def __init__(self, **kwds): except Exception: logger.debug("Error parsing INSTANA_TAGS env var: %s", tag_list) - self.timeout = os.environ.get("INSTANA_TIMEOUT", 0.5) - self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) self.zone = os.environ.get("INSTANA_ZONE", None) From 0ab5bec6bdc256713e7f762b88353d0750a5073b Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Fri, 14 Aug 2020 16:22:41 +0200 Subject: [PATCH 87/91] Improved log level handling & tests --- instana/__init__.py | 10 ++-- instana/agent/aws_fargate.py | 4 +- instana/agent/aws_lambda.py | 4 +- instana/agent/base.py | 14 ++++++ instana/agent/host.py | 21 +++++---- instana/log.py | 13 +----- instana/options.py | 23 ++++++++-- tests/platforms/test_fargate.py | 13 +++++- tests/platforms/test_lambda.py | 12 +++++ tests/test_agent.py | 81 ++++++++++++++++++++++++++++----- 10 files changed, 152 insertions(+), 43 deletions(-) diff --git a/instana/__init__.py b/instana/__init__.py index e030e0de..70d20491 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -22,8 +22,8 @@ import os import sys import importlib -import pkg_resources from threading import Timer +import pkg_resources __author__ = 'Instana Inc.' __copyright__ = 'Copyright 2020 Instana Inc.' @@ -66,7 +66,7 @@ def get_lambda_handler_or_default(): parts = handler.split(".") handler_function = parts.pop() handler_module = ".".join(parts) - except: + except Exception: pass return handler_module, handler_function @@ -101,8 +101,7 @@ def boot_agent_later(): import gevent gevent.spawn_later(2.0, boot_agent) else: - t = Timer(2.0, boot_agent) - t.start() + Timer(2.0, boot_agent).start() def boot_agent(): @@ -182,7 +181,8 @@ def boot_agent(): # and some Pipenv installs. If this is the case, it's best effort. if hasattr(sys, 'argv') and len(sys.argv) > 0 and (os.path.basename(sys.argv[0]) in do_not_load_list): if "INSTANA_DEBUG" in os.environ: - print("Instana: No use in monitoring this process type (%s). Will go sit in a corner quietly." % os.path.basename(sys.argv[0])) + print("Instana: No use in monitoring this process type (%s). " + "Will go sit in a corner quietly." % os.path.basename(sys.argv[0])) else: if "INSTANA_MAGIC" in os.environ: # If we're being loaded into an already running process, then delay agent initialization diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 0af072ff..102f1eaf 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -2,7 +2,6 @@ The Instana agent (for AWS Fargate) that manages monitoring state and reporting that data. """ -import os import time import pkg_resources from instana.options import AWSFargateOptions @@ -33,6 +32,9 @@ def __init__(self): self.report_headers = None self._can_send = False + # Update log level (if INSTANA_LOG_LEVEL was set) + self.update_log_level() + package_version = 'unknown' try: package_version = pkg_resources.get_distribution('instana').version diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index bf3b6409..23ce5ff2 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -2,7 +2,6 @@ The Instana agent (for AWS Lambda functions) that manages monitoring state and reporting that data. """ -import os import time import pkg_resources from ..log import logger @@ -33,6 +32,9 @@ def __init__(self): self.report_headers = None self._can_send = False + # Update log level from what Options detected + self.update_log_level() + package_version = 'unknown' try: package_version = pkg_resources.get_distribution('instana').version diff --git a/instana/agent/base.py b/instana/agent/base.py index 98559369..79804e3f 100644 --- a/instana/agent/base.py +++ b/instana/agent/base.py @@ -1,7 +1,9 @@ """ Base class for all the agent flavors """ +import logging import requests +from ..log import logger class BaseAgent(object): @@ -12,3 +14,15 @@ class BaseAgent(object): def __init__(self): self.client = requests.Session() + + def update_log_level(self): + """ Uses the value in to update the global logger """ + if self.options is None or self.options.log_level not in [logging.DEBUG, + logging.INFO, + logging.WARN, + logging.ERROR]: + logger.warning("BaseAgent.update_log_level: Unknown log level set") + return + + logger.setLevel(self.options.log_level) + diff --git a/instana/agent/host.py b/instana/agent/host.py index 47820189..d108c350 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -44,17 +44,20 @@ class HostAgent(BaseAgent): AGENT_DATA_PATH = "com.instana.plugin.python.%d" AGENT_HEADER = "Instana Agent" - announce_data = None - options = StandardOptions() - - machine = None - last_seen = None - last_fork_check = None - _boot_pid = os.getpid() - should_threads_shutdown = threading.Event() - def __init__(self): super(HostAgent, self).__init__() + + self.announce_data = None + self.machine = None + self.last_seen = None + self.last_fork_check = None + self._boot_pid = os.getpid() + self.should_threads_shutdown = threading.Event() + self.options = StandardOptions() + + # Update log level from what Options detected + self.update_log_level() + logger.debug("initializing agent") self.sensor = Sensor(self) self.machine = TheMachine(self) diff --git a/instana/log.py b/instana/log.py index 71aeba48..9b117121 100644 --- a/instana/log.py +++ b/instana/log.py @@ -18,11 +18,7 @@ def get_standard_logger(): f = logging.Formatter('%(asctime)s: %(process)d %(levelname)s %(name)s: %(message)s') ch.setFormatter(f) standard_logger.addHandler(ch) - if "INSTANA_DEBUG" in os.environ: - standard_logger.setLevel(logging.DEBUG) - else: - standard_logger.setLevel(logging.WARN) - + standard_logger.setLevel(logging.DEBUG) return standard_logger @@ -33,12 +29,7 @@ def get_aws_lambda_logger(): @return: Logger """ aws_lambda_logger = logging.getLogger() - - if "INSTANA_DEBUG" in os.environ: - aws_lambda_logger.setLevel(logging.DEBUG) - else: - aws_lambda_logger.setLevel(logging.WARN) - + aws_lambda_logger.setLevel(logging.INFO) return aws_lambda_logger diff --git a/instana/options.py b/instana/options.py index 68ed563e..6ca5321f 100644 --- a/instana/options.py +++ b/instana/options.py @@ -27,6 +27,7 @@ def __init__(self, **kwds): if "INSTANA_DEBUG" in os.environ: self.log_level = logging.DEBUG self.debug = True + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: self.extra_http_headers = str(os.environ["INSTANA_EXTRA_HTTP_HEADERS"]).lower().split(';') @@ -70,11 +71,11 @@ def __init__(self, **kwds): self.agent_key = os.environ.get("INSTANA_AGENT_KEY", None) self.endpoint_url = os.environ.get("INSTANA_ENDPOINT_URL", None) - + # Remove any trailing slash (if any) if self.endpoint_url is not None and self.endpoint_url[-1] == "/": self.endpoint_url = self.endpoint_url[:-1] - + if 'INSTANA_DISABLE_CA_CHECK' in os.environ: self.ssl_verify = False else: @@ -99,8 +100,22 @@ def __init__(self, **kwds): "https://www.instana.com/docs/reference/environment_variables/#serverless-monitoring") self.timeout = 0.8 - self.log_level = os.environ.get("INSTANA_LOG_LEVEL", None) - + value = os.environ.get("INSTANA_LOG_LEVEL", None) + if value is not None: + try: + value = value.lower() + if value == "debug": + self.log_level = logging.DEBUG + elif value == "info": + self.log_level = logging.INFO + elif value == "warn" or value == "warning": + self.log_level = logging.WARNING + elif value == "error": + self.log_level = logging.ERROR + else: + logger.warning("Unknown INSTANA_LOG_LEVEL specified: %s", value) + except Exception: + logger.debug("BaseAgent.update_log_level: ", exc_info=True) class AWSLambdaOptions(ServerlessOptions): """ Options class for AWS Lambda. Holds settings specific to AWS Lambda. """ diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index 5f032837..c8d3feb7 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import os +import logging import unittest from instana.tracer import InstanaTracer @@ -16,7 +17,6 @@ def __init__(self, methodName='runTest'): self.agent = None self.span_recorder = None self.tracer = None - self.pwd = os.path.dirname(os.path.realpath(__file__)) self.original_agent = get_agent() self.original_tracer = get_tracer() @@ -38,6 +38,8 @@ def tearDown(self): os.environ.pop("INSTANA_ENDPOINT_PROXY") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_LOG_LEVEL" in os.environ: + os.environ.pop("INSTANA_LOG_LEVEL") if "INSTANA_SECRETS" in os.environ: os.environ.pop("INSTANA_SECRETS") if "INSTANA_TAGS" in os.environ: @@ -105,6 +107,15 @@ def test_agent_extra_http_headers(self): should_headers = ['x-test-header', 'x-another-header', 'x-and-another-header'] self.assertEqual(should_headers, self.agent.options.extra_http_headers) + def test_agent_default_log_level(self): + self.create_agent_and_setup_tracer() + assert self.agent.options.log_level == logging.WARNING + + def test_agent_custom_log_level(self): + os.environ['INSTANA_LOG_LEVEL'] = "eRror" + self.create_agent_and_setup_tracer() + assert self.agent.options.log_level == logging.ERROR + def test_custom_proxy(self): os.environ["INSTANA_ENDPOINT_PROXY"] = "http://myproxy.123" self.create_agent_and_setup_tracer() diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index 43891ffa..817eb6e2 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -4,6 +4,7 @@ import sys import json import wrapt +import logging import unittest from instana.tracer import InstanaTracer @@ -70,6 +71,8 @@ def tearDown(self): os.environ.pop("INSTANA_ENDPOINT_PROXY") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_LOG_LEVEL" in os.environ: + os.environ.pop("INSTANA_LOG_LEVEL") set_agent(self.original_agent) set_tracer(self.original_tracer) @@ -575,3 +578,12 @@ def test_arn_parsing(self): # Fully qualified already with the '$LATEST' special tag ctx.invoked_function_arn = "arn:aws:lambda:us-east-2:12345:function:TestPython:$LATEST" assert(normalize_aws_lambda_arn(ctx) == "arn:aws:lambda:us-east-2:12345:function:TestPython:$LATEST") + + def test_agent_default_log_level(self): + self.create_agent_and_setup_tracer() + assert self.agent.options.log_level == logging.WARNING + + def test_agent_custom_log_level(self): + os.environ['INSTANA_LOG_LEVEL'] = "eRror" + self.create_agent_and_setup_tracer() + assert self.agent.options.log_level == logging.ERROR \ No newline at end of file diff --git a/tests/test_agent.py b/tests/test_agent.py index ba349b22..25b23dce 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -1,29 +1,88 @@ from __future__ import absolute_import +import os +import logging import unittest -from instana.singletons import agent +from instana.agent.host import HostAgent +from instana.tracer import InstanaTracer from instana.options import StandardOptions +from instana.recorder import StandardRecorder +from instana.singletons import get_agent, set_agent, get_tracer, set_tracer class TestAgent(unittest.TestCase): + def __init__(self, methodName='runTest'): + super(TestAgent, self).__init__(methodName) + self.agent = None + self.span_recorder = None + self.tracer = None + self.pwd = os.path.dirname(os.path.realpath(__file__)) + + self.original_agent = get_agent() + self.original_tracer = get_tracer() + def setUp(self): - pass + os.environ["AWS_EXECUTION_ENV"] = "AWS_ECS_FARGATE" + os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" + os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" def tearDown(self): - pass + """ Reset all environment variables of consequence """ + if "AWS_EXECUTION_ENV" in os.environ: + os.environ.pop("AWS_EXECUTION_ENV") + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: + os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") + if "INSTANA_ENDPOINT_URL" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_ENDPOINT_PROXY" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_PROXY") + if "INSTANA_AGENT_KEY" in os.environ: + os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_LOG_LEVEL" in os.environ: + os.environ.pop("INSTANA_LOG_LEVEL") + if "INSTANA_SECRETS" in os.environ: + os.environ.pop("INSTANA_SECRETS") + if "INSTANA_TAGS" in os.environ: + os.environ.pop("INSTANA_TAGS") + + set_agent(self.original_agent) + set_tracer(self.original_tracer) + + def create_agent_and_setup_tracer(self): + self.agent = HostAgent() + self.span_recorder = StandardRecorder(self.agent) + self.tracer = InstanaTracer(recorder=self.span_recorder) + set_agent(self.agent) + set_tracer(self.tracer) def test_secrets(self): - self.assertTrue(hasattr(agent.options, 'secrets_matcher')) - self.assertEqual(agent.options.secrets_matcher, 'contains-ignore-case') - self.assertTrue(hasattr(agent.options, 'secrets_list')) - self.assertEqual(agent.options.secrets_list, ['key', 'pass', 'secret']) + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent.options, 'secrets_matcher')) + self.assertEqual(self.agent.options.secrets_matcher, 'contains-ignore-case') + self.assertTrue(hasattr(self.agent.options, 'secrets_list')) + self.assertEqual(self.agent.options.secrets_list, ['key', 'pass', 'secret']) def test_options_have_extra_http_headers(self): - self.assertTrue(hasattr(agent, 'options')) - self.assertTrue(hasattr(agent.options, 'extra_http_headers')) + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent, 'options')) + self.assertTrue(hasattr(self.agent.options, 'extra_http_headers')) def test_has_options(self): - self.assertTrue(hasattr(agent, 'options')) - self.assertTrue(type(agent.options) is StandardOptions) + self.create_agent_and_setup_tracer() + self.assertTrue(hasattr(self.agent, 'options')) + self.assertTrue(isinstance(self.agent.options, StandardOptions)) + + def test_agent_default_log_level(self): + self.create_agent_and_setup_tracer() + assert self.agent.options.log_level == logging.WARNING + def test_agent_instana_debug(self): + os.environ['INSTANA_DEBUG'] = "asdf" + self.create_agent_and_setup_tracer() + assert self.agent.options.log_level == logging.DEBUG + + def test_agent_instana_service_name(self): + os.environ['INSTANA_SERVICE_NAME'] = "greycake" + self.create_agent_and_setup_tracer() + assert self.agent.options.service_name == "greycake" \ No newline at end of file From c3425581fd94310ee8929faf16140f1e23ea3ee1 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 17 Aug 2020 12:55:20 +0200 Subject: [PATCH 88/91] Refactor Host agent metric collection --- instana/__init__.py | 30 +- instana/__main__.py | 5 +- instana/agent/aws_fargate.py | 10 +- instana/agent/aws_lambda.py | 10 +- instana/agent/host.py | 97 ++--- instana/collector/aws_fargate.py | 14 - instana/collector/aws_lambda.py | 6 +- instana/collector/base.py | 22 +- instana/collector/helpers/base.py | 6 +- instana/collector/helpers/runtime.py | 251 ++++++------- instana/collector/host.py | 80 +++++ instana/fsm.py | 24 +- instana/instrumentation/django/middleware.py | 14 +- instana/meter.py | 339 ------------------ instana/recorder.py | 99 +---- instana/sensor.py | 20 -- instana/singletons.py | 16 +- instana/tracer.py | 8 +- pytest.ini | 2 +- tests/platforms/test_fargate.py | 6 +- tests/platforms/test_fargate_collector.py | 4 +- .../{test_agent.py => platforms/test_host.py} | 17 +- tests/platforms/test_host_collector.py | 127 +++++++ tests/platforms/test_lambda.py | 8 +- 24 files changed, 465 insertions(+), 750 deletions(-) create mode 100644 instana/collector/host.py delete mode 100644 instana/meter.py delete mode 100644 instana/sensor.py rename tests/{test_agent.py => platforms/test_host.py} (85%) create mode 100644 tests/platforms/test_host_collector.py diff --git a/instana/__init__.py b/instana/__init__.py index 70d20491..860f1aea 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -1,20 +1,17 @@ +# coding=utf-8 """ -The Instana package has two core components: the agent and the tracer. - -The agent is individual to each python process and handles process metric -collection and reporting. - -The tracer upholds the OpenTracing API and is responsible for reporting -span data to Instana. - -The following outlines the hierarchy of classes for these two components. - -Agent - Sensor - Meter - -Tracer - Recorder +▀████▀███▄ ▀███▀▄█▀▀▀█▄███▀▀██▀▀███ ██ ▀███▄ ▀███▀ ██ + ██ ███▄ █ ▄██ ▀█▀ ██ ▀█ ▄██▄ ███▄ █ ▄██▄ + ██ █ ███ █ ▀███▄ ██ ▄█▀██▄ █ ███ █ ▄█▀██▄ + ██ █ ▀██▄ █ ▀█████▄ ██ ▄█ ▀██ █ ▀██▄ █ ▄█ ▀██ + ██ █ ▀██▄█ ▄ ▀██ ██ ████████ █ ▀██▄█ ████████ + ██ █ ███ ██ ██ ██ █▀ ██ █ ███ █▀ ██ +▄████▄███▄ ██ █▀█████▀ ▄████▄ ▄███▄ ▄████▄███▄ ██ ▄███▄ ▄████▄ + +https://www.instana.com/ + +Documentation: https://www.instana.com/docs/ +Source Code: https://github.com/instana/python-sensor """ from __future__ import absolute_import @@ -108,6 +105,7 @@ def boot_agent(): """Initialize the Instana agent and conditionally load auto-instrumentation.""" # Disable all the unused-import violations in this function # pylint: disable=unused-import + # pylint: disable=import-outside-toplevel import instana.singletons diff --git a/instana/__main__.py b/instana/__main__.py index c2f89ea6..67b226d3 100644 --- a/instana/__main__.py +++ b/instana/__main__.py @@ -39,7 +39,8 @@ ============================================================================ Monitoring Python Documentation: -https://docs.instana.io/ecosystem/python +https://www.instana.com/docs/ecosystem/python/ + Help & Support: https://support.instana.com/ @@ -74,7 +75,7 @@ ============================================================================ Monitoring Python Documentation: -https://docs.instana.io/ecosystem/python +https://www.instana.com/docs/ecosystem/python/ Help & Support: https://support.instana.com/ diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index 102f1eaf..bc852bf8 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -3,11 +3,10 @@ monitoring state and reporting that data. """ import time -import pkg_resources from instana.options import AWSFargateOptions from instana.collector.aws_fargate import AWSFargateCollector from ..log import logger -from ..util import to_json +from ..util import to_json, package_version from .base import BaseAgent @@ -35,11 +34,6 @@ def __init__(self): # Update log level (if INSTANA_LOG_LEVEL was set) self.update_log_level() - package_version = 'unknown' - try: - package_version = pkg_resources.get_distribution('instana').version - except pkg_resources.DistributionNotFound: - pass logger.info("Stan is on the AWS Fargate scene. Starting Instana instrumentation version: %s", package_version) if self._validate_options(): @@ -79,8 +73,6 @@ def report_data_payload(self, payload): self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) - # logger.debug("using these headers: %s", self.report_headers) - response = self.client.post(self.__data_bundle_url(), data=to_json(payload), headers=self.report_headers, diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index 23ce5ff2..2222c60e 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -3,9 +3,8 @@ monitoring state and reporting that data. """ import time -import pkg_resources from ..log import logger -from ..util import to_json +from ..util import to_json, package_version from .base import BaseAgent from ..collector.aws_lambda import AWSLambdaCollector from ..options import AWSLambdaOptions @@ -35,11 +34,6 @@ def __init__(self): # Update log level from what Options detected self.update_log_level() - package_version = 'unknown' - try: - package_version = pkg_resources.get_distribution('instana').version - except pkg_resources.DistributionNotFound: - pass logger.info("Stan is on the AWS Lambda scene. Starting Instana instrumentation version: %s", package_version) if self._validate_options(): @@ -79,8 +73,6 @@ def report_data_payload(self, payload): self.report_headers["X-Instana-Time"] = str(round(time.time() * 1000)) - # logger.debug("using these headers: %s", self.report_headers) - response = self.client.post(self.__data_bundle_url(), data=to_json(payload), headers=self.report_headers, diff --git a/instana/agent/host.py b/instana/agent/host.py index d108c350..a1a7ea20 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -7,17 +7,13 @@ import json import os from datetime import datetime -import threading -import instana.singletons - -from ..fsm import TheMachine from ..log import logger -from ..sensor import Sensor -from ..util import to_json, get_py_source, package_version -from ..options import StandardOptions - from .base import BaseAgent +from ..fsm import TheMachine +from ..options import StandardOptions +from ..collector.host import HostCollector +from ..util import to_json, get_py_source, package_version class AnnounceData(object): @@ -34,11 +30,6 @@ class HostAgent(BaseAgent): The Agent class is the central controlling entity for the Instana Python language sensor. The key parts it handles are the announce state and the collection and reporting of metrics and spans to the Instana Host agent. - - To do this, there are 3 major components to this class: - 1. TheMachine - finite state machine related to announce state - 2. Sensor -> Meter - metric collection and reporting - 3. Tracer -> Recorder - span queueing and reporting """ AGENT_DISCOVERY_PATH = "com.instana.plugin.python.discovery" AGENT_DATA_PATH = "com.instana.plugin.python.%d" @@ -52,30 +43,28 @@ def __init__(self): self.last_seen = None self.last_fork_check = None self._boot_pid = os.getpid() - self.should_threads_shutdown = threading.Event() self.options = StandardOptions() # Update log level from what Options detected self.update_log_level() + + logger.info("Stan is on the scene. Starting Instana instrumentation version: %s", package_version) - logger.debug("initializing agent") - self.sensor = Sensor(self) + self.collector = HostCollector(self) self.machine = TheMachine(self) - def start(self, _): + def start(self): """ Starts the agent and required threads This method is called after a successful announce. See fsm.py """ - logger.debug("Spawning metric & span reporting threads") - self.should_threads_shutdown.clear() - self.sensor.start() - instana.singletons.tracer.recorder.start() + logger.debug("Starting Host Collector") + self.collector.start() def handle_fork(self): """ - Forks happen. Here we handle them. Affected components are the singletons: Agent, Sensor & Tracers + Forks happen. Here we handle them. """ # Reset the Agent self.reset() @@ -85,11 +74,9 @@ def reset(self): This will reset the agent to a fresh unannounced state. :return: None """ - # Will signal to any running background threads to shutdown. - self.should_threads_shutdown.set() - self.last_seen = None self.announce_data = None + self.collector.shutdown(report_final=False) # Will schedule a restart of the announce cycle in the future self.machine.reset() @@ -119,7 +106,7 @@ def can_send(self): self.handle_fork() return False - if self.machine.fsm.current == "good2go": + if self.machine.fsm.current in ["wait4init", "good2go"]: return True return False @@ -185,13 +172,12 @@ def announce(self, discovery): response = None try: url = self.__discovery_url() - # logger.debug("making announce request to %s", url) response = self.client.put(url, data=to_json(discovery), headers={"Content-Type": "application/json"}, timeout=0.8) - if response.status_code == 200: + if 200 <= response.status_code <= 204: self.last_seen = datetime.now() except Exception as exc: logger.debug("announce: connection error (%s)", type(exc)) @@ -211,48 +197,41 @@ def is_agent_ready(self): logger.debug("is_agent_ready: connection error (%s)", type(exc)) return ready - def report_data_payload(self, entity_data): + def report_data_payload(self, payload): """ - Used to report entity data (metrics & snapshot) to the host agent. + Used to report collection payload to the host agent. This can be metrics, spans and snapshot data. """ response = None try: - logger.debug(to_json(entity_data)) - response = self.client.post(self.__data_url(), - data=to_json(entity_data), - headers={"Content-Type": "application/json"}, - timeout=0.8) - - # logger.warning("report_data: response.status_code is %s" % response.status_code) - - if response.status_code == 200: + # Report spans (if any) + span_count = len(payload['spans']) + if span_count > 0: + logger.debug("Reporting %d spans", span_count) + response = self.client.post(self.__traces_url(), + data=to_json(payload['spans']), + headers={"Content-Type": "application/json"}, + timeout=0.8) + + if response is not None and 200 <= response.status_code <= 204: self.last_seen = datetime.now() - except Exception as exc: - logger.debug("report_data_payload: Instana host agent connection error (%s)", type(exc)) - return response - - def report_traces(self, spans): - """ - Used to report entity data (metrics & snapshot) to the host agent. - """ - response = None - try: - # Concurrency double check: Don't report if we don't have - # any spans - if len(spans) == 0: - return 0 - response = self.client.post(self.__traces_url(), - data=to_json(spans), + # Report metrics + metric_bundle = payload["metrics"]["plugins"][0]["data"] + # logger.debug(to_json(metric_bundle)) + response = self.client.post(self.__data_url(), + data=to_json(metric_bundle), headers={"Content-Type": "application/json"}, timeout=0.8) - # logger.debug("report_traces: response.status_code is %s" % response.status_code) - - if response.status_code == 200: + if response is not None and 200 <= response.status_code <= 204: self.last_seen = datetime.now() + + if response.status_code == 200 and len(response.content) > 2: + # The host agent returned something indicating that is has a request for us that we + # need to process. + self.handle_agent_tasks(json.loads(response.content)[0]) except Exception as exc: - logger.debug("report_traces: Instana host agent connection error (%s)", type(exc)) + logger.debug("report_data_payload: Instana host agent connection error (%s)", type(exc), exc_info=True) return response def handle_agent_tasks(self, task): diff --git a/instana/collector/aws_fargate.py b/instana/collector/aws_fargate.py index 6db92b72..8c13eec5 100644 --- a/instana/collector/aws_fargate.py +++ b/instana/collector/aws_fargate.py @@ -46,9 +46,6 @@ def __init__(self, agent): # How often to do a full fetch of ECMU data self.ecmu_full_fetch_interval = 304 - # How often to report data - self.report_interval = 1 - # HTTP client with keep-alive self.http_client = requests.Session() @@ -58,14 +55,6 @@ def __init__(self, agent): # The fully qualified ARN for this process self._fq_arn = None - # Saved snapshot data - self.snapshot_data = None - # Timestamp in seconds of the last time we sent snapshot data - self.snapshot_data_last_sent = 0 - # How often to report snapshot data (in seconds) - self.snapshot_data_interval = 300 - self.last_payload = None - # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/ self.root_metadata = None @@ -82,9 +71,6 @@ def __init__(self, agent): # ${ECS_CONTAINER_METADATA_URI}/task/stats self.task_stats_metadata = None - # List of helpers that help out in data collection - self.helpers = [] - # Populate the collection helpers self.helpers.append(TaskHelper(self)) self.helpers.append(DockerHelper(self)) diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index d83181e9..fd10d5d5 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -15,9 +15,13 @@ def __init__(self, agent): self.event = None self._fq_arn = None - def collect_snapshot(self, event, context): + # How often to report data + self.report_interval = 5 + self.snapshot_data = DictionaryOfStan() + self.snapshot_data_sent = True + def collect_snapshot(self, event, context): self.context = context self.event = event diff --git a/instana/collector/base.py b/instana/collector/base.py index 43cfed56..d8c553ac 100644 --- a/instana/collector/base.py +++ b/instana/collector/base.py @@ -33,11 +33,14 @@ def __init__(self, agent): # Signal for background thread(s) to shutdown self.thread_shutdown = threading.Event() - self.thread_shutdown.clear() - # The collected Snapshot Data and info about when it was sent - self.snapshot_data = None - self.snapshot_data_sent = False + # Timestamp in seconds of the last time we sent snapshot data + self.snapshot_data_last_sent = 0 + # How often to report snapshot data (in seconds) + self.snapshot_data_interval = 300 + + # List of helpers that help out in data collection + self.helpers = [] # Lock used syncronize reporting - no updates when sending # Used by the background reporting thread. Used to syncronize report attempts and so @@ -45,7 +48,7 @@ def __init__(self, agent): self.background_report_lock = threading.Lock() # Reporting interval for the background thread(s) - self.report_interval = 5 + self.report_interval = 1 def start(self): """ @@ -54,20 +57,23 @@ def start(self): """ if self.agent.can_send(): logger.debug("BaseCollector.start: launching collection thread") + self.thread_shutdown.clear() self.reporting_thread = threading.Thread(target=self.thread_loop, args=()) self.reporting_thread.setDaemon(True) self.reporting_thread.start() else: - logger.warning("Collector started but the agent tells us we can't send anything out.") + logger.warning("BaseCollector.start: the agent tells us we can't send anything out.") - def shutdown(self): + def shutdown(self, report_final=True): """ Shuts down the collector and reports any final data. @return: None """ logger.debug("Collector.shutdown: Reporting final data.") self.thread_shutdown.set() - self.prepare_and_report_data() + + if report_final is True: + self.prepare_and_report_data() def thread_loop(self): """ diff --git a/instana/collector/helpers/base.py b/instana/collector/helpers/base.py index d0860c55..9a325061 100644 --- a/instana/collector/helpers/base.py +++ b/instana/collector/helpers/base.py @@ -66,8 +66,10 @@ def apply_delta(self, source, previous, new, metric, with_snapshot): else: new_value = source - if previous[dst_metric] != new_value or with_snapshot is True: - previous[dst_metric] = new[dst_metric] = new_value + previous_value = previous.get(dst_metric, 0) + if previous_value != new_value or with_snapshot is True: + previous[dst_metric] = new[dst_metric] = new_value + def collect_metrics(self, with_snapshot=False): logger.debug("BaseHelper.collect_metrics must be overridden") diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index 11284479..7499bf44 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -1,7 +1,6 @@ """ Collection helper for the Python runtime """ import os -import copy -import gc as gc_ +import gc import sys import platform import resource @@ -19,10 +18,14 @@ class RuntimeHelper(BaseHelper): """ Helper class to collect snapshot and metrics for this Python runtime """ def __init__(self, collector): super(RuntimeHelper, self).__init__(collector) - self.last_collect = None - self.last_usage = None - self.last_metrics = None - + self.previous = DictionaryOfStan() + self.previous_rusage = resource.getrusage(resource.RUSAGE_SELF) + + if gc.isenabled(): + self.previous_gc_count = gc.get_count() + else: + self.previous_gc_count = None + def collect_metrics(self, with_snapshot=False): plugin_data = dict() try: @@ -31,74 +34,123 @@ def collect_metrics(self, with_snapshot=False): plugin_data["data"] = DictionaryOfStan() plugin_data["data"]["pid"] = str(os.getpid()) - snapshot_payload = None - metrics_payload = self.gather_metrics() + self._collect_runtime_metrics(plugin_data, with_snapshot) if with_snapshot is True: - snapshot_payload = self.gather_snapshot() - metrics_payload = copy.deepcopy(metrics_payload).delta_data(None) - plugin_data["data"]["snapshot"] = snapshot_payload - else: - metrics_payload = copy.deepcopy(metrics_payload).delta_data(self.last_metrics) - - plugin_data["data"]["metrics"] = metrics_payload - - self.last_metrics = metrics_payload - #logger.debug(to_pretty_json(plugin_data)) + self._collect_runtime_snapshot(plugin_data) except Exception: - logger.debug("_collect_runtime_snapshot: ", exc_info=True) + logger.debug("_collect_metrics: ", exc_info=True) return [plugin_data] - def gather_metrics(self): - """ Collect up and return various metrics """ + def _collect_runtime_metrics(self, plugin_data, with_snapshot): + """ Collect up and return the runtime metrics """ + try: + rusage = resource.getrusage(resource.RUSAGE_SELF) + if gc.isenabled(): + self._collect_gc_metrics(plugin_data, with_snapshot) + + self._collect_thread_metrics(plugin_data, with_snapshot) + + value_diff = rusage.ru_utime - self.previous_rusage.ru_utime + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_utime", with_snapshot) + + value_diff = rusage.ru_stime - self.previous_rusage.ru_stime + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_stime", with_snapshot) + + self.apply_delta(rusage.ru_maxrss, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_maxrss", with_snapshot) + self.apply_delta(rusage.ru_ixrss, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_ixrss", with_snapshot) + self.apply_delta(rusage.ru_idrss, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_idrss", with_snapshot) + self.apply_delta(rusage.ru_isrss, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_isrss", with_snapshot) + + value_diff = rusage.ru_minflt - self.previous_rusage.ru_minflt + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_minflt", with_snapshot) + + value_diff = rusage.ru_majflt - self.previous_rusage.ru_majflt + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_majflt", with_snapshot) + + value_diff = rusage.ru_nswap - self.previous_rusage.ru_nswap + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_nswap", with_snapshot) + + value_diff = rusage.ru_inblock - self.previous_rusage.ru_inblock + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_inblock", with_snapshot) + + value_diff = rusage.ru_oublock - self.previous_rusage.ru_oublock + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_oublock", with_snapshot) + + value_diff = rusage.ru_msgsnd - self.previous_rusage.ru_msgsnd + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_msgsnd", with_snapshot) + + value_diff = rusage.ru_msgrcv - self.previous_rusage.ru_msgrcv + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_msgrcv", with_snapshot) + + value_diff = rusage.ru_nsignals - self.previous_rusage.ru_nsignals + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_nsignals", with_snapshot) + + value_diff = rusage.ru_nvcsw - self.previous_rusage.ru_nvcsw + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_nvcsw", with_snapshot) + + value_diff = rusage.ru_nivcsw - self.previous_rusage.ru_nivcsw + self.apply_delta(value_diff, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "ru_nivcsw", with_snapshot) + except Exception: + logger.debug("_collect_runtime_metrics", exc_info=True) + finally: + self.previous_rusage = rusage + + def _collect_gc_metrics(self, plugin_data, with_snapshot): try: - g = None - u = resource.getrusage(resource.RUSAGE_SELF) - if gc_.isenabled(): - c = list(gc_.get_count()) - th = list(gc_.get_threshold()) - g = GC(collect0=c[0] if not self.last_collect else c[0] - self.last_collect[0], - collect1=c[1] if not self.last_collect else c[1] - self.last_collect[1], - collect2=c[2] if not self.last_collect else c[2] - self.last_collect[2], - threshold0=th[0], - threshold1=th[1], - threshold2=th[2]) - - thr = threading.enumerate() - daemon_threads = [tr.daemon is True for tr in thr].count(True) - alive_threads = [tr.daemon is False for tr in thr].count(True) - dummy_threads = [isinstance(tr, threading._DummyThread) for tr in thr].count(True) # pylint: disable=protected-access - - m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0], - ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1], - ru_maxrss=u[2], - ru_ixrss=u[3], - ru_idrss=u[4], - ru_isrss=u[5], - ru_minflt=u[6] if not self.last_usage else u[6] - self.last_usage[6], - ru_majflt=u[7] if not self.last_usage else u[7] - self.last_usage[7], - ru_nswap=u[8] if not self.last_usage else u[8] - self.last_usage[8], - ru_inblock=u[9] if not self.last_usage else u[9] - self.last_usage[9], - ru_oublock=u[10] if not self.last_usage else u[10] - self.last_usage[10], - ru_msgsnd=u[11] if not self.last_usage else u[11] - self.last_usage[11], - ru_msgrcv=u[12] if not self.last_usage else u[12] - self.last_usage[12], - ru_nsignals=u[13] if not self.last_usage else u[13] - self.last_usage[13], - ru_nvcs=u[14] if not self.last_usage else u[14] - self.last_usage[14], - ru_nivcsw=u[15] if not self.last_usage else u[15] - self.last_usage[15], - alive_threads=alive_threads, - dummy_threads=dummy_threads, - daemon_threads=daemon_threads, - gc=g) - - self.last_usage = u - if gc_.isenabled(): - self.last_collect = c - - return m + gc_count = gc.get_count() + gc_threshold = gc.get_threshold() + + self.apply_delta(gc_count[0], self.previous['data']['metrics']['gc'], + plugin_data['data']['metrics']['gc'], "collect0", with_snapshot) + self.apply_delta(gc_count[1], self.previous['data']['metrics']['gc'], + plugin_data['data']['metrics']['gc'], "collect1", with_snapshot) + self.apply_delta(gc_count[2], self.previous['data']['metrics']['gc'], + plugin_data['data']['metrics']['gc'], "collect2", with_snapshot) + + self.apply_delta(gc_threshold[0], self.previous['data']['metrics']['gc'], + plugin_data['data']['metrics']['gc'], "threshold0", with_snapshot) + self.apply_delta(gc_threshold[1], self.previous['data']['metrics']['gc'], + plugin_data['data']['metrics']['gc'], "threshold1", with_snapshot) + self.apply_delta(gc_threshold[2], self.previous['data']['metrics']['gc'], + plugin_data['data']['metrics']['gc'], "threshold2", with_snapshot) except Exception: - logger.debug("collect_metrics", exc_info=True) + logger.debug("_collect_gc_metrics", exc_info=True) - def gather_snapshot(self): + def _collect_thread_metrics(self, plugin_data, with_snapshot): + try: + threads = threading.enumerate() + daemon_threads = [thread.daemon is True for thread in threads].count(True) + self.apply_delta(daemon_threads, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "daemon_threads", with_snapshot) + + alive_threads = [thread.daemon is False for thread in threads].count(True) + self.apply_delta(alive_threads, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "alive_threads", with_snapshot) + + dummy_threads = [isinstance(thread, threading._DummyThread) for thread in threads].count(True) # pylint: disable=protected-access + self.apply_delta(dummy_threads, self.previous['data']['metrics'], + plugin_data['data']['metrics'], "dummy_threads", with_snapshot) + except Exception: + logger.debug("_collect_thread_metrics", exc_info=True) + + def _collect_runtime_snapshot(self,plugin_data): """ Gathers Python specific Snapshot information for this process """ snapshot_payload = {} try: @@ -107,10 +159,19 @@ def gather_snapshot(self): snapshot_payload['f'] = platform.python_implementation() # flavor snapshot_payload['a'] = platform.architecture()[0] # architecture snapshot_payload['versions'] = self.gather_python_packages() - #snapshot_payload['djmw'] = None # FIXME: django middleware reporting + + try: + from django.conf import settings + if hasattr(settings, 'MIDDLEWARE') and settings.MIDDLEWARE is not None: + snapshot_payload['djmw'] = settings.MIDDLEWARE + elif hasattr(settings, 'MIDDLEWARE_CLASSES') and settings.MIDDLEWARE_CLASSES is not None: + snapshot_payload['djmw'] = settings.MIDDLEWARE_CLASSES + except ImportError: + pass except Exception: logger.debug("collect_snapshot: ", exc_info=True) - return snapshot_payload + + plugin_data['data']['snapshot'] = snapshot_payload def gather_python_packages(self): """ Collect up the list of modules in use """ @@ -159,57 +220,3 @@ def jsonable(self, value): return str(result) except Exception: logger.debug("jsonable: ", exc_info=True) - -class GC(object): - collect0 = 0 - collect1 = 0 - collect2 = 0 - threshold0 = 0 - threshold1 = 0 - threshold2 = 0 - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def to_dict(self): - return self.__dict__ - - -class Metrics(object): - ru_utime = .0 - ru_stime = .0 - ru_maxrss = 0 - ru_ixrss = 0 - ru_idrss = 0 - ru_isrss = 0 - ru_minflt = 0 - ru_majflt = 0 - ru_nswap = 0 - ru_inblock = 0 - ru_oublock = 0 - ru_msgsnd = 0 - ru_msgrcv = 0 - ru_nsignals = 0 - ru_nvcs = 0 - ru_nivcsw = 0 - dummy_threads = 0 - alive_threads = 0 - daemon_threads = 0 - gc = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def delta_data(self, delta): - data = self.__dict__ - if delta is None: - return data - - unchanged_items = set(data.items()) & set(delta.items()) - for x in unchanged_items: - data.pop(x[0]) - - return data - - def to_dict(self): - return self.__dict__ diff --git a/instana/collector/host.py b/instana/collector/host.py new file mode 100644 index 00000000..4bc46780 --- /dev/null +++ b/instana/collector/host.py @@ -0,0 +1,80 @@ +""" +Snapshot & metrics collection for AWS Fargate +""" +from time import time +from ..log import logger +from .base import BaseCollector +from ..util import DictionaryOfStan +from ..singletons import env_is_test +from .helpers.runtime import RuntimeHelper + + +class HostCollector(BaseCollector): + """ Collector for AWS Fargate """ + def __init__(self, agent): + super(HostCollector, self).__init__(agent) + logger.debug("Loading Host Collector") + + # Indicates if this Collector has all requirements to run successfully + self.ready_to_start = True + + # Populate the collection helpers + self.helpers.append(RuntimeHelper(self)) + + def start(self): + if self.ready_to_start is False: + logger.warning("Host Collector is missing requirements and cannot monitor this environment.") + return + + super(HostCollector, self).start() + + def prepare_and_report_data(self): + """ + We override this method from the base class so that we can handle the wait4init + state machine case. + """ + try: + if self.agent.machine.fsm.current == "wait4init": + # Test the host agent if we're ready to send data + if self.agent.is_agent_ready(): + if self.agent.machine.fsm.current != "good2go": + logger.debug("Agent is ready. Getting to work.") + self.agent.machine.fsm.ready() + else: + return + except Exception: + logger.debug('Harmless state machine thread disagreement. Will self-correct on next timer cycle.') + + super(HostCollector, self).prepare_and_report_data() + + def should_send_snapshot_data(self): + delta = int(time()) - self.snapshot_data_last_sent + if delta > self.snapshot_data_interval: + return True + return False + + def prepare_payload(self): + payload = DictionaryOfStan() + payload["spans"] = [] + payload["metrics"]["plugins"] = [] + + try: + if not self.span_queue.empty(): + payload["spans"] = self.queued_spans() + + with_snapshot = self.should_send_snapshot_data() + + plugins = [] + for helper in self.helpers: + plugins.extend(helper.collect_metrics(with_snapshot)) + + payload["metrics"]["plugins"] = plugins + + if with_snapshot is True: + logger.debug("Sending snapshot") + logger.debug(payload) + self.snapshot_data_last_sent = int(time()) + except Exception: + logger.debug("collect_snapshot error", exc_info=True) + + return payload diff --git a/instana/fsm.py b/instana/fsm.py index aaec7452..3c5e1d69 100644 --- a/instana/fsm.py +++ b/instana/fsm.py @@ -5,10 +5,9 @@ import socket import subprocess import sys -import threading as t +import threading from fysom import Fysom -import pkg_resources from .log import logger from .util import get_default_gateway @@ -45,14 +44,7 @@ class TheMachine(object): warnedPeriodic = False def __init__(self, agent): - package_version = 'unknown' - try: - package_version = pkg_resources.get_distribution('instana').version - except pkg_resources.DistributionNotFound: - pass - - logger.info("Stan is on the scene. Starting Instana instrumentation version: %s", package_version) - logger.debug("initializing fsm") + logger.debug("Initializing host agent state machine") self.agent = agent self.fsm = Fysom({ @@ -66,10 +58,9 @@ def __init__(self, agent): # "onchangestate": self.print_state_change, "onlookup": self.lookup_agent_host, "onannounce": self.announce_sensor, - "onpending": self.agent.start, - "onready": self.on_ready}}) + "onpending": self.on_ready}}) - self.timer = t.Timer(1, self.fsm.lookup) + self.timer = threading.Timer(1, self.fsm.lookup) self.timer.daemon = True self.timer.name = self.THREAD_NAME @@ -80,7 +71,7 @@ def __init__(self, agent): @staticmethod def print_state_change(e): logger.debug('========= (%i#%s) FSM event: %s, src: %s, dst: %s ==========', - os.getpid(), t.current_thread().name, e.event, e.src, e.dst) + os.getpid(), threading.current_thread().name, e.event, e.src, e.dst) def reset(self): """ @@ -96,8 +87,6 @@ def reset(self): self.fsm.lookup() def lookup_agent_host(self, e): - self.agent.should_threads_shutdown.clear() - host = self.agent.options.agent_host port = self.agent.options.agent_port @@ -177,12 +166,13 @@ def announce_sensor(self, e): return False def schedule_retry(self, fun, e, name): - self.timer = t.Timer(self.RETRY_PERIOD, fun, [e]) + self.timer = threading.Timer(self.RETRY_PERIOD, fun, [e]) self.timer.daemon = True self.timer.name = name self.timer.start() def on_ready(self, _): + self.agent.start() logger.info("Instana host agent available. We're in business. Announced pid: %s (true pid: %s)", str(os.getpid()), str(self.agent.announce_data.pid)) diff --git a/instana/instrumentation/django/middleware.py b/instana/instrumentation/django/middleware.py index 5d2c9b43..44ee7a68 100644 --- a/instana/instrumentation/django/middleware.py +++ b/instana/instrumentation/django/middleware.py @@ -83,12 +83,9 @@ def load_middleware_wrapper(wrapped, instance, args, kwargs): if DJ_INSTANA_MIDDLEWARE in settings.MIDDLEWARE: return wrapped(*args, **kwargs) - # Save the list of middleware for Snapshot reporting - agent.sensor.meter.djmw = settings.MIDDLEWARE - - if type(settings.MIDDLEWARE) is tuple: + if isinstance(settings.MIDDLEWARE, tuple): settings.MIDDLEWARE = (DJ_INSTANA_MIDDLEWARE,) + settings.MIDDLEWARE - elif type(settings.MIDDLEWARE) is list: + elif isinstance(settings.MIDDLEWARE, list): settings.MIDDLEWARE = [DJ_INSTANA_MIDDLEWARE] + settings.MIDDLEWARE else: logger.warning("Instana: Couldn't add InstanaMiddleware to Django") @@ -97,12 +94,9 @@ def load_middleware_wrapper(wrapped, instance, args, kwargs): if DJ_INSTANA_MIDDLEWARE in settings.MIDDLEWARE_CLASSES: return wrapped(*args, **kwargs) - # Save the list of middleware for Snapshot reporting - agent.sensor.meter.djmw = settings.MIDDLEWARE_CLASSES - - if type(settings.MIDDLEWARE_CLASSES) is tuple: + if isinstance(settings.MIDDLEWARE_CLASSES, tuple): settings.MIDDLEWARE_CLASSES = (DJ_INSTANA_MIDDLEWARE,) + settings.MIDDLEWARE_CLASSES - elif type(settings.MIDDLEWARE_CLASSES) is list: + elif isinstance(settings.MIDDLEWARE_CLASSES, list): settings.MIDDLEWARE_CLASSES = [DJ_INSTANA_MIDDLEWARE] + settings.MIDDLEWARE_CLASSES else: logger.warning("Instana: Couldn't add InstanaMiddleware to Django") diff --git a/instana/meter.py b/instana/meter.py deleted file mode 100644 index bd9eb340..00000000 --- a/instana/meter.py +++ /dev/null @@ -1,339 +0,0 @@ -import copy -import gc as gc_ -import json -import sys -import platform -import resource -import threading -from fysom import FysomError -from types import ModuleType -from pkg_resources import DistributionNotFound, get_distribution - -from .log import logger -from .util import every, determine_service_name - - -class Snapshot(object): - name = None - version = None - f = None # flavor: CPython, Jython, IronPython, PyPy - a = None # architecture: i386, x86, x86_64, AMD64 - versions = None - djmw = [] - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def to_dict(self): - kvs = dict() - kvs['name'] = self.name - kvs['version'] = self.version - kvs['f'] = self.f # flavor - kvs['a'] = self.a # architecture - kvs['versions'] = self.versions - kvs['djmw'] = list(self.djmw) - return kvs - - -class GC(object): - collect0 = 0 - collect1 = 0 - collect2 = 0 - threshold0 = 0 - threshold1 = 0 - threshold2 = 0 - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def to_dict(self): - return self.__dict__ - - -class Metrics(object): - ru_utime = .0 - ru_stime = .0 - ru_maxrss = 0 - ru_ixrss = 0 - ru_idrss = 0 - ru_isrss = 0 - ru_minflt = 0 - ru_majflt = 0 - ru_nswap = 0 - ru_inblock = 0 - ru_oublock = 0 - ru_msgsnd = 0 - ru_msgrcv = 0 - ru_nsignals = 0 - ru_nvcs = 0 - ru_nivcsw = 0 - dummy_threads = 0 - alive_threads = 0 - daemon_threads = 0 - gc = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def delta_data(self, delta): - data = self.__dict__ - if delta is None: - return data - - unchanged_items = set(data.items()) & set(delta.items()) - for x in unchanged_items: - data.pop(x[0]) - - return data - - def to_dict(self): - return self.__dict__ - - -class EntityData(object): - pid = 0 - snapshot = None - metrics = None - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def to_dict(self): - return self.__dict__ - - -class Meter(object): - SNAPSHOT_PERIOD = 600 - THREAD_NAME = "Instana Metric Collection" - - # The agent that this instance belongs to - agent = None - - # We send Snapshot data every 10 minutes. This is the countdown variable. - snapshot_countdown = 0 - - # Collect the Snapshot only once and store the resulting Snapshot object here. - # We use this for every repeated snapshot send (every 10 minutes) - cached_snapshot = None - - last_usage = None - last_collect = None - last_metrics = None - djmw = None - thread = None - - # A True value signals the metric reporting thread to shutdown - _shutdown = False - - def __init__(self, agent): - self.agent = agent - - def start(self): - """ - This function can be called at first boot or after a fork. In either case, it will - assure that the Meter is in a proper state (via reset()) and spawn a new background - thread to periodically report the metrics payload. - - Note that this will abandon any previous thread object that (in the case of an `os.fork()`) - should no longer exist in the forked process. - - (Forked processes carry forward only the thread that called `os.fork()` - into the new process space. All other background threads need to be recreated.) - - Calling this directly more than once without an actual fork will cause errors. - """ - self.reset() - self.thread.start() - - def reset(self): - """" Reset the state as new """ - self.last_usage = None - self.last_collect = None - self.last_metrics = None - self.snapshot_countdown = 0 - self.cached_snapshot = None - self.thread = None - - self.thread = threading.Thread(target=self.collect_and_report) - self.thread.daemon = True - self.thread.name = self.THREAD_NAME - - def handle_fork(self): - self.start() - - def collect_and_report(self): - """ - Target function for the metric reporting thread. This is a simple loop to - collect and report entity data every 1 second. - """ - logger.debug(" -> Metric reporting thread is now alive") - - def metric_work(): - if self.agent.should_threads_shutdown.is_set(): - logger.debug("Thread shutdown signal from agent is active: Shutting down metric reporting thread") - return False - - self.process() - - if self.agent.is_timed_out(): - logger.warning("Instana host agent unreachable for >1 min. Going to sit in a corner...") - self.agent.reset() - return False - return True - - every(1, metric_work, "Metrics Collection") - - def process(self): - """ Collects, processes & reports metrics """ - try: - if self.agent.machine.fsm.current == "wait4init": - # Test the host agent if we're ready to send data - if self.agent.is_agent_ready(): - if self.agent.machine.fsm.current != "good2go": - self.agent.machine.fsm.ready() - else: - return - except FysomError: - logger.debug('Harmless state machine thread disagreement. Will self-correct on next timer cycle.') - return - - if self.agent.can_send(): - self.snapshot_countdown = self.snapshot_countdown - 1 - ss = None - cm = self.collect_metrics() - - if self.snapshot_countdown < 1: - logger.debug("Sending process snapshot data") - self.snapshot_countdown = self.SNAPSHOT_PERIOD - ss = self.collect_snapshot() - md = copy.deepcopy(cm).delta_data(None) - else: - md = copy.deepcopy(cm).delta_data(self.last_metrics) - - ed = EntityData(pid=self.agent.announce_data.pid, snapshot=ss, metrics=md) - response = self.agent.report_data_payload(ed) - - if response: - if response.status_code == 200 and len(response.content) > 2: - # The host agent returned something indicating that is has a request for us that we - # need to process. - self.agent.handle_agent_tasks(json.loads(response.content)[0]) - - self.last_metrics = cm.__dict__ - - def collect_snapshot(self): - """ Collects snapshot related information to this process and environment """ - try: - if self.cached_snapshot is not None: - return self.cached_snapshot - - service_name = determine_service_name() - - s = Snapshot(name=service_name, version=platform.version(), - f=platform.python_implementation(), - a=platform.architecture()[0], - djmw=self.djmw) - s.version = sys.version - s.versions = self.collect_modules() - - # Cache the snapshot - self.cached_snapshot = s - except Exception as e: - logger.debug("collect_snapshot: ", exc_info=True) - else: - return s - - def jsonable(self, value): - try: - if callable(value): - try: - result = value() - except: - result = 'Unknown' - elif type(value) is ModuleType: - result = value - else: - result = value - return str(result) - except Exception: - logger.debug("jsonable: ", exc_info=True) - - def collect_modules(self): - """ Collect up the list of modules in use """ - try: - res = {} - m = sys.modules.copy() - for k in m: - # Don't report submodules (e.g. django.x, django.y, django.z) - # Skip modules that begin with underscore - if ('.' in k) or k[0] == '_': - continue - if m[k]: - try: - d = m[k].__dict__ - if "version" in d and d["version"]: - res[k] = self.jsonable(d["version"]) - elif "__version__" in d and d["__version__"]: - res[k] = self.jsonable(d["__version__"]) - else: - res[k] = get_distribution(k).version - except DistributionNotFound: - pass - except Exception: - logger.debug("collect_modules: could not process module: %s", k) - - except Exception: - logger.debug("collect_modules", exc_info=True) - else: - return res - - def collect_metrics(self): - """ Collect up and return various metrics """ - try: - g = None - u = resource.getrusage(resource.RUSAGE_SELF) - if gc_.isenabled(): - c = list(gc_.get_count()) - th = list(gc_.get_threshold()) - g = GC(collect0=c[0] if not self.last_collect else c[0] - self.last_collect[0], - collect1=c[1] if not self.last_collect else c[ - 1] - self.last_collect[1], - collect2=c[2] if not self.last_collect else c[ - 2] - self.last_collect[2], - threshold0=th[0], - threshold1=th[1], - threshold2=th[2]) - - thr = threading.enumerate() - daemon_threads = [tr.daemon is True for tr in thr].count(True) - alive_threads = [tr.daemon is False for tr in thr].count(True) - dummy_threads = [type(tr) is threading._DummyThread for tr in thr].count(True) - - m = Metrics(ru_utime=u[0] if not self.last_usage else u[0] - self.last_usage[0], - ru_stime=u[1] if not self.last_usage else u[1] - self.last_usage[1], - ru_maxrss=u[2], - ru_ixrss=u[3], - ru_idrss=u[4], - ru_isrss=u[5], - ru_minflt=u[6] if not self.last_usage else u[6] - self.last_usage[6], - ru_majflt=u[7] if not self.last_usage else u[7] - self.last_usage[7], - ru_nswap=u[8] if not self.last_usage else u[8] - self.last_usage[8], - ru_inblock=u[9] if not self.last_usage else u[9] - self.last_usage[9], - ru_oublock=u[10] if not self.last_usage else u[10] - self.last_usage[10], - ru_msgsnd=u[11] if not self.last_usage else u[11] - self.last_usage[11], - ru_msgrcv=u[12] if not self.last_usage else u[12] - self.last_usage[12], - ru_nsignals=u[13] if not self.last_usage else u[13] - self.last_usage[13], - ru_nvcs=u[14] if not self.last_usage else u[14] - self.last_usage[14], - ru_nivcsw=u[15] if not self.last_usage else u[15] - self.last_usage[15], - alive_threads=alive_threads, - dummy_threads=dummy_threads, - daemon_threads=daemon_threads, - gc=g) - - self.last_usage = u - if gc_.isenabled(): - self.last_collect = c - - return m - except Exception: - logger.debug("collect_metrics", exc_info=True) diff --git a/instana/recorder.py b/instana/recorder.py index ece7e3ae..437c1862 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -2,11 +2,10 @@ import os import sys -import threading -from .log import logger -from .util import every from basictracer import Sampler + +from .log import logger from .span import (RegisteredSpan, SDKSpan) if sys.version_info.major == 2: @@ -15,7 +14,7 @@ import queue -class StandardRecorder(object): +class StanRecorder(object): THREAD_NAME = "Instana Span Reporting" REGISTERED_SPANS = ("aiohttp-client", "aiohttp-server", "aws.lambda.entry", "cassandra", @@ -30,61 +29,15 @@ class StandardRecorder(object): def __init__(self, agent = None): if agent is None: # Late import to avoid circular import + # pylint: disable=import-outside-toplevel from .singletons import get_agent self.agent = get_agent() else: self.agent = agent - self.queue = queue.Queue() - - def start(self): - """ - This function can be called at first boot or after a fork. In either case, it will - assure that the Recorder is in a proper state (via reset()) and spawn a new background - thread to periodically report queued spans - - Note that this will abandon any previous thread object that (in the case of an `os.fork()`) - should no longer exist in the forked process. - - (Forked processes carry forward only the thread that called `os.fork()` - into the new process space. All other background threads need to be recreated.) - - Calling this directly more than once without an actual fork will cause errors. - """ - self.reset() - self.thread.start() - - def reset(self): - # Prepare the thread for span collection/reporting - self.thread = threading.Thread(target=self.report_spans) - self.thread.daemon = True - self.thread.name = self.THREAD_NAME - - def handle_fork(self): - self.start() - - def report_spans(self): - """ Periodically report the queued spans """ - logger.debug(" -> Span reporting thread is now alive") - - def span_work(): - if self.agent.should_threads_shutdown.is_set(): - logger.debug("Thread shutdown signal from agent is active: Shutting down span reporting thread") - return False - - queue_size = self.queue.qsize() - if queue_size > 0 and self.agent.can_send(): - response = self.agent.report_traces(self.queued_spans()) - if response: - logger.debug("reported %d spans", queue_size) - return True - - if "INSTANA_TEST" not in os.environ: - every(2, span_work, "Span Reporting") - def queue_size(self): """ Return the size of the queue; how may spans are queued, """ - return self.queue.qsize() + return self.agent.collector.span_queue.qsize() def queued_spans(self): """ Get all of the spans in the queue """ @@ -92,7 +45,7 @@ def queued_spans(self): spans = [] while True: try: - span = self.queue.get(False) + span = self.agent.collector.span_queue.get(False) except queue.Empty: break else: @@ -107,45 +60,6 @@ def record_span(self, span): """ Convert the passed BasicSpan into and add it to the span queue """ - if self.agent.can_send() or "INSTANA_TEST" in os.environ: - service_name = None - source = self.agent.get_from_structure() - if "INSTANA_SERVICE_NAME" in os.environ: - service_name = self.agent.options.service_name - - if span.operation_name in self.REGISTERED_SPANS: - json_span = RegisteredSpan(span, source, service_name) - else: - service_name = self.agent.options.service_name - json_span = SDKSpan(span, source, service_name) - - self.queue.put(json_span) - - -class AWSLambdaRecorder(StandardRecorder): - def record_span(self, span): - """ - Convert the passed BasicSpan and add it to the span queue - """ - service_name = None - source = self.agent.get_from_structure() - if "INSTANA_SERVICE_NAME" in os.environ: - service_name = self.agent.options.service_name - - if span.operation_name in self.REGISTERED_SPANS: - json_span = RegisteredSpan(span, source, service_name) - else: - json_span = SDKSpan(span, source, service_name) - - # logger.debug("Recorded span: %s", json_span) - self.agent.collector.span_queue.put(json_span) - - -class AWSFargateRecorder(StandardRecorder): - def record_span(self, span): - """ - Convert the passed BasicSpan and add it to the span queue - """ if self.agent.can_send(): service_name = None source = self.agent.get_from_structure() @@ -155,6 +69,7 @@ def record_span(self, span): if span.operation_name in self.REGISTERED_SPANS: json_span = RegisteredSpan(span, source, service_name) else: + service_name = self.agent.options.service_name json_span = SDKSpan(span, source, service_name) # logger.debug("Recorded span: %s", json_span) diff --git a/instana/sensor.py b/instana/sensor.py deleted file mode 100644 index 7453e0df..00000000 --- a/instana/sensor.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import absolute_import - -from .meter import Meter - - -class Sensor(object): - agent = None - meter = None - - def __init__(self, agent): - self.agent = agent - self.meter = Meter(agent) - - def start(self): - # Nothing to do for the Sensor; Pass onto Meter - self.meter.start() - - def handle_fork(self): - # Nothing to do for the Sensor; Pass onto Meter - self.meter.handle_fork() diff --git a/instana/singletons.py b/instana/singletons.py index 0f5dda40..fe55c8eb 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -17,35 +17,35 @@ if env_is_test: from .agent.test import TestAgent - from .recorder import StandardRecorder + from .recorder import StanRecorder logger.debug("TEST environment detected.") agent = TestAgent() - span_recorder = StandardRecorder(agent) + span_recorder = StanRecorder(agent) elif env_is_aws_lambda: from .agent.aws_lambda import AWSLambdaAgent - from .recorder import AWSLambdaRecorder + from .recorder import StanRecorder logger.debug("AWS Lambda environment detected.") agent = AWSLambdaAgent() - span_recorder = AWSLambdaRecorder(agent) + span_recorder = StanRecorder(agent) elif env_is_aws_fargate: from .agent.aws_fargate import AWSFargateAgent - from .recorder import AWSFargateRecorder + from .recorder import StanRecorder logger.debug("AWS Fargate environment detected.") agent = AWSFargateAgent() - span_recorder = AWSFargateRecorder(agent) + span_recorder = StanRecorder(agent) else: from .agent.host import HostAgent - from .recorder import StandardRecorder + from .recorder import StanRecorder logger.debug("Standard Host environment detected.") agent = HostAgent() - span_recorder = StandardRecorder(agent) + span_recorder = StanRecorder(agent) def get_agent(): diff --git a/instana/tracer.py b/instana/tracer.py index c47dfe25..fa12fd46 100644 --- a/instana/tracer.py +++ b/instana/tracer.py @@ -11,7 +11,7 @@ from .binary_propagator import BinaryPropagator from .http_propagator import HTTPPropagator from .text_propagator import TextPropagator -from .recorder import StandardRecorder, InstanaSampler +from .recorder import StanRecorder, InstanaSampler from .span import InstanaSpan, RegisteredSpan, SpanContext from .util import generate_id @@ -20,7 +20,7 @@ class InstanaTracer(BasicTracer): def __init__(self, scope_manager=None, recorder=None): if recorder is None: - recorder = StandardRecorder() + recorder = StanRecorder() super(InstanaTracer, self).__init__( recorder, InstanaSampler(), scope_manager) @@ -29,10 +29,6 @@ def __init__(self, scope_manager=None, recorder=None): self._propagators[ot.Format.TEXT_MAP] = TextPropagator() self._propagators[ot.Format.BINARY] = BinaryPropagator() - def handle_fork(self): - # Nothing to do for the Tracer; Pass onto Recorder - self.recorder.handle_fork() - def start_active_span(self, operation_name, child_of=None, diff --git a/pytest.ini b/pytest.ini index 3474cd54..c3dd3042 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] log_cli = 1 -log_cli_level = INFO +log_cli_level = DEBUG log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %H:%M:%S diff --git a/tests/platforms/test_fargate.py b/tests/platforms/test_fargate.py index c8d3feb7..301114f9 100644 --- a/tests/platforms/test_fargate.py +++ b/tests/platforms/test_fargate.py @@ -6,7 +6,7 @@ from instana.tracer import InstanaTracer from instana.options import AWSFargateOptions -from instana.recorder import AWSFargateRecorder +from instana.recorder import StanRecorder from instana.agent.aws_fargate import AWSFargateAgent from instana.singletons import get_agent, set_agent, get_tracer, set_tracer @@ -42,6 +42,8 @@ def tearDown(self): os.environ.pop("INSTANA_LOG_LEVEL") if "INSTANA_SECRETS" in os.environ: os.environ.pop("INSTANA_SECRETS") + if "INSTANA_DEBUG" in os.environ: + os.environ.pop("INSTANA_DEBUG") if "INSTANA_TAGS" in os.environ: os.environ.pop("INSTANA_TAGS") @@ -50,7 +52,7 @@ def tearDown(self): def create_agent_and_setup_tracer(self): self.agent = AWSFargateAgent() - self.span_recorder = AWSFargateRecorder(self.agent) + self.span_recorder = StanRecorder(self.agent) self.tracer = InstanaTracer(recorder=self.span_recorder) set_agent(self.agent) set_tracer(self.tracer) diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 06d5a616..1cb68c84 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -5,7 +5,7 @@ import unittest from instana.tracer import InstanaTracer -from instana.recorder import AWSFargateRecorder +from instana.recorder import StanRecorder from instana.agent.aws_fargate import AWSFargateAgent from instana.singletons import get_agent, set_agent, get_tracer, set_tracer @@ -62,7 +62,7 @@ def tearDown(self): def create_agent_and_setup_tracer(self): self.agent = AWSFargateAgent() - self.span_recorder = AWSFargateRecorder(self.agent) + self.span_recorder = StanRecorder(self.agent) self.tracer = InstanaTracer(recorder=self.span_recorder) set_agent(self.agent) set_tracer(self.tracer) diff --git a/tests/test_agent.py b/tests/platforms/test_host.py similarity index 85% rename from tests/test_agent.py rename to tests/platforms/test_host.py index 25b23dce..28ef5660 100644 --- a/tests/test_agent.py +++ b/tests/platforms/test_host.py @@ -7,25 +7,22 @@ from instana.agent.host import HostAgent from instana.tracer import InstanaTracer from instana.options import StandardOptions -from instana.recorder import StandardRecorder +from instana.recorder import StanRecorder from instana.singletons import get_agent, set_agent, get_tracer, set_tracer -class TestAgent(unittest.TestCase): +class TestHost(unittest.TestCase): def __init__(self, methodName='runTest'): - super(TestAgent, self).__init__(methodName) + super(TestHost, self).__init__(methodName) self.agent = None self.span_recorder = None self.tracer = None - self.pwd = os.path.dirname(os.path.realpath(__file__)) self.original_agent = get_agent() self.original_tracer = get_tracer() def setUp(self): - os.environ["AWS_EXECUTION_ENV"] = "AWS_ECS_FARGATE" - os.environ["INSTANA_ENDPOINT_URL"] = "https://localhost/notreal" - os.environ["INSTANA_AGENT_KEY"] = "Fake_Key" + pass def tearDown(self): """ Reset all environment variables of consequence """ @@ -41,6 +38,8 @@ def tearDown(self): os.environ.pop("INSTANA_AGENT_KEY") if "INSTANA_LOG_LEVEL" in os.environ: os.environ.pop("INSTANA_LOG_LEVEL") + if "INSTANA_SERVICE_NAME" in os.environ: + os.environ.pop("INSTANA_SERVICE_NAME") if "INSTANA_SECRETS" in os.environ: os.environ.pop("INSTANA_SECRETS") if "INSTANA_TAGS" in os.environ: @@ -51,7 +50,7 @@ def tearDown(self): def create_agent_and_setup_tracer(self): self.agent = HostAgent() - self.span_recorder = StandardRecorder(self.agent) + self.span_recorder = StanRecorder(self.agent) self.tracer = InstanaTracer(recorder=self.span_recorder) set_agent(self.agent) set_tracer(self.tracer) @@ -85,4 +84,4 @@ def test_agent_instana_debug(self): def test_agent_instana_service_name(self): os.environ['INSTANA_SERVICE_NAME'] = "greycake" self.create_agent_and_setup_tracer() - assert self.agent.options.service_name == "greycake" \ No newline at end of file + assert self.agent.options.service_name == "greycake" diff --git a/tests/platforms/test_host_collector.py b/tests/platforms/test_host_collector.py new file mode 100644 index 00000000..5ca4209d --- /dev/null +++ b/tests/platforms/test_host_collector.py @@ -0,0 +1,127 @@ +from __future__ import absolute_import + +import os +import json +import unittest + +from instana.tracer import InstanaTracer +from instana.recorder import StanRecorder +from instana.agent.host import HostAgent +from instana.singletons import get_agent, set_agent, get_tracer, set_tracer + + +class TestHostCollector(unittest.TestCase): + def __init__(self, methodName='runTest'): + super(TestHostCollector, self).__init__(methodName) + self.agent = None + self.span_recorder = None + self.tracer = None + + self.original_agent = get_agent() + self.original_tracer = get_tracer() + + def setUp(self): + pass + + def tearDown(self): + """ Reset all environment variables of consequence """ + if "AWS_EXECUTION_ENV" in os.environ: + os.environ.pop("AWS_EXECUTION_ENV") + if "INSTANA_EXTRA_HTTP_HEADERS" in os.environ: + os.environ.pop("INSTANA_EXTRA_HTTP_HEADERS") + if "INSTANA_ENDPOINT_URL" in os.environ: + os.environ.pop("INSTANA_ENDPOINT_URL") + if "INSTANA_AGENT_KEY" in os.environ: + os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_ZONE" in os.environ: + os.environ.pop("INSTANA_ZONE") + if "INSTANA_TAGS" in os.environ: + os.environ.pop("INSTANA_TAGS") + + set_agent(self.original_agent) + set_tracer(self.original_tracer) + + def create_agent_and_setup_tracer(self): + self.agent = HostAgent() + self.span_recorder = StanRecorder(self.agent) + self.tracer = InstanaTracer(recorder=self.span_recorder) + set_agent(self.agent) + set_tracer(self.tracer) + + def test_prepare_payload_basics(self): + self.create_agent_and_setup_tracer() + + payload = self.agent.collector.prepare_payload() + assert(payload) + + assert(len(payload.keys()) == 2) + assert('spans' in payload) + assert(isinstance(payload['spans'], list)) + assert(len(payload['spans']) == 0) + assert('metrics' in payload) + assert(len(payload['metrics'].keys()) == 1) + assert('plugins' in payload['metrics']) + assert(isinstance(payload['metrics']['plugins'], list)) + assert(len(payload['metrics']['plugins']) == 1) + + python_plugin = payload['metrics']['plugins'][0] + assert python_plugin['name'] == 'com.instana.plugin.python' + assert python_plugin['entityId'] == str(os.getpid()) + assert 'data' in python_plugin + assert 'snapshot' in python_plugin['data'] + assert 'metrics' in python_plugin['data'] + + # Validate that all metrics are reported on the first run + assert 'ru_utime' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_utime']) in [float, int] + assert 'ru_stime' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_stime']) in [float, int] + assert 'ru_maxrss' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_maxrss']) in [float, int] + assert 'ru_ixrss' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_ixrss']) in [float, int] + assert 'ru_idrss' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_idrss']) in [float, int] + assert 'ru_isrss' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_isrss']) in [float, int] + assert 'ru_minflt' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_minflt']) in [float, int] + assert 'ru_majflt' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_majflt']) in [float, int] + assert 'ru_nswap' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_nswap']) in [float, int] + assert 'ru_inblock' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_inblock']) in [float, int] + assert 'ru_oublock' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_oublock']) in [float, int] + assert 'ru_msgsnd' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_msgsnd']) in [float, int] + assert 'ru_msgrcv' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_msgrcv']) in [float, int] + assert 'ru_nsignals' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_nsignals']) in [float, int] + assert 'ru_nvcsw' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_nvcsw']) in [float, int] + assert 'ru_nivcsw' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['ru_nivcsw']) in [float, int] + assert 'alive_threads' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['alive_threads']) in [float, int] + assert 'dummy_threads' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['dummy_threads']) in [float, int] + assert 'daemon_threads' in python_plugin['data']['metrics'] + assert type(python_plugin['data']['metrics']['daemon_threads']) in [float, int] + + assert 'gc' in python_plugin['data']['metrics'] + assert isinstance(python_plugin['data']['metrics']['gc'], dict) + assert 'collect0' in python_plugin['data']['metrics']['gc'] + assert type(python_plugin['data']['metrics']['gc']['collect0']) in [float, int] + assert 'collect1' in python_plugin['data']['metrics']['gc'] + assert type(python_plugin['data']['metrics']['gc']['collect1']) in [float, int] + assert 'collect2' in python_plugin['data']['metrics']['gc'] + assert type(python_plugin['data']['metrics']['gc']['collect2']) in [float, int] + assert 'threshold0' in python_plugin['data']['metrics']['gc'] + assert type(python_plugin['data']['metrics']['gc']['threshold0']) in [float, int] + assert 'threshold1' in python_plugin['data']['metrics']['gc'] + assert type(python_plugin['data']['metrics']['gc']['threshold1']) in [float, int] + assert 'threshold2' in python_plugin['data']['metrics']['gc'] + assert type(python_plugin['data']['metrics']['gc']['threshold2']) in [float, int] diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index 817eb6e2..9fb1d25e 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -10,7 +10,7 @@ from instana.tracer import InstanaTracer from instana.agent.aws_lambda import AWSLambdaAgent from instana.options import AWSLambdaOptions -from instana.recorder import AWSLambdaRecorder +from instana.recorder import StanRecorder from instana import lambda_handler from instana import get_lambda_handler_or_default from instana.instrumentation.aws.lambda_inst import lambda_handler_with_instana @@ -71,6 +71,10 @@ def tearDown(self): os.environ.pop("INSTANA_ENDPOINT_PROXY") if "INSTANA_AGENT_KEY" in os.environ: os.environ.pop("INSTANA_AGENT_KEY") + if "INSTANA_SERVICE_NAME" in os.environ: + os.environ.pop("INSTANA_SERVICE_NAME") + if "INSTANA_DEBUG" in os.environ: + os.environ.pop("INSTANA_DEBUG") if "INSTANA_LOG_LEVEL" in os.environ: os.environ.pop("INSTANA_LOG_LEVEL") @@ -79,7 +83,7 @@ def tearDown(self): def create_agent_and_setup_tracer(self): self.agent = AWSLambdaAgent() - self.span_recorder = AWSLambdaRecorder(self.agent) + self.span_recorder = StanRecorder(self.agent) self.tracer = InstanaTracer(recorder=self.span_recorder) set_agent(self.agent) set_tracer(self.tracer) From 124dc134790456e6416dff27795d1f1714500913 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 17 Aug 2020 13:58:50 +0200 Subject: [PATCH 89/91] Dot the Is --- instana/agent/aws_fargate.py | 2 +- instana/agent/aws_lambda.py | 2 +- instana/agent/host.py | 2 +- instana/collector/aws_lambda.py | 2 +- instana/collector/helpers/runtime.py | 4 ++-- instana/collector/host.py | 2 -- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/instana/agent/aws_fargate.py b/instana/agent/aws_fargate.py index bc852bf8..194cb26b 100644 --- a/instana/agent/aws_fargate.py +++ b/instana/agent/aws_fargate.py @@ -34,7 +34,7 @@ def __init__(self): # Update log level (if INSTANA_LOG_LEVEL was set) self.update_log_level() - logger.info("Stan is on the AWS Fargate scene. Starting Instana instrumentation version: %s", package_version) + logger.info("Stan is on the AWS Fargate scene. Starting Instana instrumentation version: %s", package_version()) if self._validate_options(): self._can_send = True diff --git a/instana/agent/aws_lambda.py b/instana/agent/aws_lambda.py index 2222c60e..4ce3160a 100644 --- a/instana/agent/aws_lambda.py +++ b/instana/agent/aws_lambda.py @@ -34,7 +34,7 @@ def __init__(self): # Update log level from what Options detected self.update_log_level() - logger.info("Stan is on the AWS Lambda scene. Starting Instana instrumentation version: %s", package_version) + logger.info("Stan is on the AWS Lambda scene. Starting Instana instrumentation version: %s", package_version()) if self._validate_options(): self._can_send = True diff --git a/instana/agent/host.py b/instana/agent/host.py index a1a7ea20..ac510b67 100644 --- a/instana/agent/host.py +++ b/instana/agent/host.py @@ -48,7 +48,7 @@ def __init__(self): # Update log level from what Options detected self.update_log_level() - logger.info("Stan is on the scene. Starting Instana instrumentation version: %s", package_version) + logger.info("Stan is on the scene. Starting Instana instrumentation version: %s", package_version()) self.collector = HostCollector(self) self.machine = TheMachine(self) diff --git a/instana/collector/aws_lambda.py b/instana/collector/aws_lambda.py index fd10d5d5..ad018363 100644 --- a/instana/collector/aws_lambda.py +++ b/instana/collector/aws_lambda.py @@ -19,7 +19,7 @@ def __init__(self, agent): self.report_interval = 5 self.snapshot_data = DictionaryOfStan() - self.snapshot_data_sent = True + self.snapshot_data_sent = False def collect_snapshot(self, event, context): self.context = context diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index 7499bf44..aeee6879 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -161,12 +161,12 @@ def _collect_runtime_snapshot(self,plugin_data): snapshot_payload['versions'] = self.gather_python_packages() try: - from django.conf import settings + from django.conf import settings # pylint: disable=import-outside-toplevel if hasattr(settings, 'MIDDLEWARE') and settings.MIDDLEWARE is not None: snapshot_payload['djmw'] = settings.MIDDLEWARE elif hasattr(settings, 'MIDDLEWARE_CLASSES') and settings.MIDDLEWARE_CLASSES is not None: snapshot_payload['djmw'] = settings.MIDDLEWARE_CLASSES - except ImportError: + except Exception: pass except Exception: logger.debug("collect_snapshot: ", exc_info=True) diff --git a/instana/collector/host.py b/instana/collector/host.py index 4bc46780..57184b13 100644 --- a/instana/collector/host.py +++ b/instana/collector/host.py @@ -71,8 +71,6 @@ def prepare_payload(self): payload["metrics"]["plugins"] = plugins if with_snapshot is True: - logger.debug("Sending snapshot") - logger.debug(payload) self.snapshot_data_last_sent = int(time()) except Exception: logger.debug("collect_snapshot error", exc_info=True) From 96e02f0ff1537c72832a8bff7271953ce933aab9 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 17 Aug 2020 15:30:35 +0200 Subject: [PATCH 90/91] Remove debug --- instana/singletons.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/instana/singletons.py b/instana/singletons.py index fe55c8eb..8a7bdc5d 100644 --- a/instana/singletons.py +++ b/instana/singletons.py @@ -19,7 +19,6 @@ from .agent.test import TestAgent from .recorder import StanRecorder - logger.debug("TEST environment detected.") agent = TestAgent() span_recorder = StanRecorder(agent) @@ -27,7 +26,6 @@ from .agent.aws_lambda import AWSLambdaAgent from .recorder import StanRecorder - logger.debug("AWS Lambda environment detected.") agent = AWSLambdaAgent() span_recorder = StanRecorder(agent) @@ -35,7 +33,6 @@ from .agent.aws_fargate import AWSFargateAgent from .recorder import StanRecorder - logger.debug("AWS Fargate environment detected.") agent = AWSFargateAgent() span_recorder = StanRecorder(agent) @@ -43,7 +40,6 @@ from .agent.host import HostAgent from .recorder import StanRecorder - logger.debug("Standard Host environment detected.") agent = HostAgent() span_recorder = StanRecorder(agent) From 0d585f900472ffa14a47a5bb3b213c90b84ae378 Mon Sep 17 00:00:00 2001 From: Peter Giacomo Lombardo Date: Mon, 17 Aug 2020 15:52:52 +0200 Subject: [PATCH 91/91] Docker blkio metrics are accumalative --- instana/collector/helpers/fargate/docker.py | 18 +++++++--- instana/collector/helpers/runtime.py | 38 ++++++++++----------- tests/platforms/test_fargate_collector.py | 4 ++- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/instana/collector/helpers/fargate/docker.py b/instana/collector/helpers/fargate/docker.py index c69a075b..9cf6bfb8 100644 --- a/instana/collector/helpers/fargate/docker.py +++ b/instana/collector/helpers/fargate/docker.py @@ -13,6 +13,10 @@ def __init__(self, collector): # The metrics from the previous report cycle self.previous = DictionaryOfStan() + # For metrics that are accumalative, store their previous values here + # Indexed by docker_id: self.previous_blkio[docker_id][metric] + self.previous_blkio = DictionaryOfStan() + def collect_metrics(self, with_snapshot=False): """ Collect and return docker metrics (and optionally snapshot data) for this task @@ -181,10 +185,16 @@ def _collect_blkio_metrics(self, container, plugin_data, docker_id, with_snapsho if service_bytes is not None: for entry in service_bytes: if entry["op"] == "Read": - self.apply_delta(entry, self.previous[docker_id]["blkio"], - plugin_data["data"]["blkio"], ("value", "blk_read"), with_snapshot) + previous_value = self.previous_blkio[docker_id].get("blk_read", 0) + value_diff = entry["value"] - previous_value + self.apply_delta(value_diff, self.previous[docker_id]["blkio"], + plugin_data["data"]["blkio"], "blk_read", with_snapshot) + self.previous_blkio[docker_id]["blk_read"] = entry["value"] elif entry["op"] == "Write": - self.apply_delta(entry, self.previous[docker_id]["blkio"], - plugin_data["data"]["blkio"], ("value", "blk_write"), with_snapshot) + previous_value = self.previous_blkio[docker_id].get("blk_write", 0) + value_diff = entry["value"] - previous_value + self.apply_delta(value_diff, self.previous[docker_id]["blkio"], + plugin_data["data"]["blkio"], "blk_write", with_snapshot) + self.previous_blkio[docker_id]["blk_write"] = entry["value"] except Exception: logger.debug("_collect_blkio_metrics: ", exc_info=True) diff --git a/instana/collector/helpers/runtime.py b/instana/collector/helpers/runtime.py index aeee6879..e7789c99 100644 --- a/instana/collector/helpers/runtime.py +++ b/instana/collector/helpers/runtime.py @@ -18,14 +18,14 @@ class RuntimeHelper(BaseHelper): """ Helper class to collect snapshot and metrics for this Python runtime """ def __init__(self, collector): super(RuntimeHelper, self).__init__(collector) - self.previous = DictionaryOfStan() + self.previous = DictionaryOfStan() self.previous_rusage = resource.getrusage(resource.RUSAGE_SELF) if gc.isenabled(): self.previous_gc_count = gc.get_count() else: self.previous_gc_count = None - + def collect_metrics(self, with_snapshot=False): plugin_data = dict() try: @@ -58,7 +58,7 @@ def _collect_runtime_metrics(self, plugin_data, with_snapshot): value_diff = rusage.ru_stime - self.previous_rusage.ru_stime self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_stime", with_snapshot) - + self.apply_delta(rusage.ru_maxrss, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_maxrss", with_snapshot) self.apply_delta(rusage.ru_ixrss, self.previous['data']['metrics'], @@ -67,7 +67,7 @@ def _collect_runtime_metrics(self, plugin_data, with_snapshot): plugin_data['data']['metrics'], "ru_idrss", with_snapshot) self.apply_delta(rusage.ru_isrss, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_isrss", with_snapshot) - + value_diff = rusage.ru_minflt - self.previous_rusage.ru_minflt self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_minflt", with_snapshot) @@ -75,31 +75,31 @@ def _collect_runtime_metrics(self, plugin_data, with_snapshot): value_diff = rusage.ru_majflt - self.previous_rusage.ru_majflt self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_majflt", with_snapshot) - + value_diff = rusage.ru_nswap - self.previous_rusage.ru_nswap self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_nswap", with_snapshot) - + value_diff = rusage.ru_inblock - self.previous_rusage.ru_inblock self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_inblock", with_snapshot) - + value_diff = rusage.ru_oublock - self.previous_rusage.ru_oublock self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_oublock", with_snapshot) - + value_diff = rusage.ru_msgsnd - self.previous_rusage.ru_msgsnd self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_msgsnd", with_snapshot) - + value_diff = rusage.ru_msgrcv - self.previous_rusage.ru_msgrcv self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_msgrcv", with_snapshot) - + value_diff = rusage.ru_nsignals - self.previous_rusage.ru_nsignals self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_nsignals", with_snapshot) - + value_diff = rusage.ru_nvcsw - self.previous_rusage.ru_nvcsw self.apply_delta(value_diff, self.previous['data']['metrics'], plugin_data['data']['metrics'], "ru_nvcsw", with_snapshot) @@ -111,25 +111,25 @@ def _collect_runtime_metrics(self, plugin_data, with_snapshot): logger.debug("_collect_runtime_metrics", exc_info=True) finally: self.previous_rusage = rusage - + def _collect_gc_metrics(self, plugin_data, with_snapshot): try: gc_count = gc.get_count() gc_threshold = gc.get_threshold() self.apply_delta(gc_count[0], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "collect0", with_snapshot) + plugin_data['data']['metrics']['gc'], "collect0", with_snapshot) self.apply_delta(gc_count[1], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "collect1", with_snapshot) + plugin_data['data']['metrics']['gc'], "collect1", with_snapshot) self.apply_delta(gc_count[2], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "collect2", with_snapshot) + plugin_data['data']['metrics']['gc'], "collect2", with_snapshot) self.apply_delta(gc_threshold[0], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "threshold0", with_snapshot) + plugin_data['data']['metrics']['gc'], "threshold0", with_snapshot) self.apply_delta(gc_threshold[1], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "threshold1", with_snapshot) + plugin_data['data']['metrics']['gc'], "threshold1", with_snapshot) self.apply_delta(gc_threshold[2], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "threshold2", with_snapshot) + plugin_data['data']['metrics']['gc'], "threshold2", with_snapshot) except Exception: logger.debug("_collect_gc_metrics", exc_info=True) @@ -159,7 +159,7 @@ def _collect_runtime_snapshot(self,plugin_data): snapshot_payload['f'] = platform.python_implementation() # flavor snapshot_payload['a'] = platform.architecture()[0] # architecture snapshot_payload['versions'] = self.gather_python_packages() - + try: from django.conf import settings # pylint: disable=import-outside-toplevel if hasattr(settings, 'MIDDLEWARE') and settings.MIDDLEWARE is not None: diff --git a/tests/platforms/test_fargate_collector.py b/tests/platforms/test_fargate_collector.py index 1cb68c84..a2b11fee 100644 --- a/tests/platforms/test_fargate_collector.py +++ b/tests/platforms/test_fargate_collector.py @@ -189,7 +189,9 @@ def test_docker_plugin_metrics(self): assert("memory" in data) assert(len(data["memory"]) == 0) assert("blkio" in data) - assert(len(data["blkio"]) == 0) + assert(len(data["blkio"]) == 1) + assert(data["blkio"]['blk_write'] == 0) + assert('blk_read' not in data["blkio"]) def test_no_instana_zone(self): self.create_agent_and_setup_tracer()