From 489aefa92fe81e9f62d6dc5b7fff47b455a37909 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 28 Jun 2023 18:48:35 +0200 Subject: [PATCH 1/5] feat(iast): iast metrics core and executed.source metric --- ddtrace/appsec/_constants.py | 2 + ddtrace/appsec/iast/_metrics.py | 89 +++++++++++++++++++ .../appsec/iast/_taint_tracking/__init__.py | 2 + ddtrace/appsec/iast/constants.py | 2 + ddtrace/appsec/iast/taint_sinks/_base.py | 4 + ddtrace/internal/telemetry/constants.py | 1 + tests/appsec/iast/test_telemety.py | 30 +++++++ 7 files changed, 130 insertions(+) create mode 100644 ddtrace/appsec/iast/_metrics.py create mode 100644 tests/appsec/iast/test_telemety.py diff --git a/ddtrace/appsec/_constants.py b/ddtrace/appsec/_constants.py index a3c470acedf..85a509a11b2 100644 --- a/ddtrace/appsec/_constants.py +++ b/ddtrace/appsec/_constants.py @@ -63,6 +63,8 @@ class IAST(object): """Specific constants for IAST""" ENV = "DD_IAST_ENABLED" + ENV_DEBUG = "_DD_IAST_DEBUG" + TELEMETRY_REPORT_LVL = "DD_IAST_TELEMETRY_VERBOSITY" JSON = "_dd.iast.json" ENABLED = "_dd.iast.enabled" CONTEXT_KEY = "_iast_data" diff --git a/ddtrace/appsec/iast/_metrics.py b/ddtrace/appsec/iast/_metrics.py new file mode 100644 index 00000000000..6fcdb8ffb8e --- /dev/null +++ b/ddtrace/appsec/iast/_metrics.py @@ -0,0 +1,89 @@ +import os + +from ddtrace.internal.logger import get_logger + +from ddtrace.internal.telemetry import telemetry_metrics_writer +from ddtrace.internal.utils.cache import cached +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_IAST +from ddtrace.appsec._constants import IAST + +log = get_logger(__name__) + +TELEMETRY_OFF_NAME = "OFF" +TELEMETRY_DEBUG_NAME = "DEBUG" +TELEMETRY_MANDATORY_NAME = "MANDATORY" +TELEMETRY_INFORMATION_NAME = "INFORMATION" + +TELEMETRY_DEBUG_VERBOSITY = 10 +TELEMETRY_INFORMATION_VERBOSITY = 20 +TELEMETRY_MANDATORY_VERBOSITY = 30 +TELEMETRY_OFF_VERBOSITY = 40 + +METRICS_REPORT_LVLS = ( + (TELEMETRY_DEBUG_VERBOSITY, TELEMETRY_DEBUG_NAME), + (TELEMETRY_INFORMATION_VERBOSITY, TELEMETRY_INFORMATION_NAME), + (TELEMETRY_MANDATORY_VERBOSITY, TELEMETRY_MANDATORY_NAME), + (TELEMETRY_OFF_VERBOSITY, TELEMETRY_OFF_NAME), +) + + +def get_iast_metrics_report_lvl(*args, **kwargs): + report_lvl_name = os.environ.get(IAST.TELEMETRY_REPORT_LVL, TELEMETRY_INFORMATION_NAME).upper() + report_lvl = 3 + for lvl, lvl_name in METRICS_REPORT_LVLS: + if report_lvl_name == lvl_name: + return lvl + return report_lvl + + +def metric_verbosity(lvl): + def wrapper(f): + if lvl >= get_iast_metrics_report_lvl(): + try: + return f + except Exception: + log.warning("Error reporting IAST metrics", exc_info=True) + return lambda: None + + return wrapper + + +@metric_verbosity(TELEMETRY_MANDATORY_VERBOSITY) +def _set_metric_iast_instrumented_source(source_type): + telemetry_metrics_writer.add_count_metric( + TELEMETRY_NAMESPACE_TAG_IAST, "instrumented.source", 1, (("source_type", source_type),) + ) + + +@metric_verbosity(TELEMETRY_MANDATORY_VERBOSITY) +def _set_metric_iast_instrumented_propagation(): + telemetry_metrics_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_IAST, "instrumented.propagation", 1) + + +@metric_verbosity(TELEMETRY_MANDATORY_VERBOSITY) +def _set_metric_iast_instrumented_sink(vulnerability_type): + telemetry_metrics_writer.add_count_metric( + TELEMETRY_NAMESPACE_TAG_IAST, "instrumented.sink", 1, (("vulnerability_type", vulnerability_type),) + ) + + +@metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY) +def _set_metric_iast_executed_source(source_type): + telemetry_metrics_writer.add_count_metric( + TELEMETRY_NAMESPACE_TAG_IAST, "executed.source", 1, (("source_type", source_type),) + ) + + +@metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY) +def _set_metric_iast_executed_sink(vulnerability_type): + telemetry_metrics_writer.add_count_metric( + TELEMETRY_NAMESPACE_TAG_IAST, "executed.sink", 1, (("vulnerability_type", vulnerability_type),) + ) + + +@metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY) +def _set_metric_iast_request_tainted(): + # TODO: function num_objects_tainted is in other PR + # from ddtrace.appsec.iast._taint_tracking import num_objects_tainted + num_objects_tainted = lambda: 1 + telemetry_metrics_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_IAST, "request.tainted", num_objects_tainted()) diff --git a/ddtrace/appsec/iast/_taint_tracking/__init__.py b/ddtrace/appsec/iast/_taint_tracking/__init__.py index f52e2fcc1bc..66dcd4e0e10 100644 --- a/ddtrace/appsec/iast/_taint_tracking/__init__.py +++ b/ddtrace/appsec/iast/_taint_tracking/__init__.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING from ddtrace.appsec.iast import oce +from ddtrace.appsec.iast._metrics import _set_metric_iast_executed_source from ddtrace.appsec.iast._taint_dict import get_taint_dict from ddtrace.appsec.iast._taint_tracking._native import new_pyobject_id from ddtrace.appsec.iast._taint_tracking._native import setup # noqa: F401 @@ -52,6 +53,7 @@ def taint_pyobject(pyobject, input_info): # type: (Any, Input_info) -> Any pyobject = new_pyobject_id(pyobject, len_pyobject) taint_dict = get_taint_dict() taint_dict[id(pyobject)] = ((input_info, 0, len_pyobject),) + _set_metric_iast_executed_source(source_origin) return pyobject diff --git a/ddtrace/appsec/iast/constants.py b/ddtrace/appsec/iast/constants.py index 6c9a2d69ec6..bee8fc9ca07 100644 --- a/ddtrace/appsec/iast/constants.py +++ b/ddtrace/appsec/iast/constants.py @@ -16,6 +16,8 @@ RC4_DEF = "rc4" IDEA_DEF = "idea" +DD_IAST_TELEMETRY_VERBOSITY = "DD_IAST_TELEMETRY_VERBOSITY" + DEFAULT_WEAK_HASH_ALGORITHMS = {MD5_DEF, SHA1_DEF} DEFAULT_WEAK_CIPHER_ALGORITHMS = {DES_DEF, BLOWFISH_DEF, RC2_DEF, RC4_DEF, IDEA_DEF} diff --git a/ddtrace/appsec/iast/taint_sinks/_base.py b/ddtrace/appsec/iast/taint_sinks/_base.py index 135470da8b7..11b15693f1a 100644 --- a/ddtrace/appsec/iast/taint_sinks/_base.py +++ b/ddtrace/appsec/iast/taint_sinks/_base.py @@ -4,6 +4,7 @@ from ddtrace import tracer from ddtrace.appsec._constants import IAST from ddtrace.appsec.iast import oce +from ddtrace.appsec.iast._metrics import _set_metric_iast_executed_sink from ddtrace.appsec.iast._overhead_control_engine import Operation from ddtrace.appsec.iast.reporter import Evidence from ddtrace.appsec.iast.reporter import IastSpanReporter @@ -84,6 +85,9 @@ def report(cls, evidence_value="", sources=None): log.debug("Unexpected evidence_value type: %s", type(evidence_value)) if cls.is_not_reported(file_name, line_number): + + _set_metric_iast_executed_sink(cls.vulnerability_type) + report = _context.get_item(IAST.CONTEXT_KEY, span=span) if report: report.vulnerabilities.add( diff --git a/ddtrace/internal/telemetry/constants.py b/ddtrace/internal/telemetry/constants.py index 9f800a42190..e1c03f6e7be 100644 --- a/ddtrace/internal/telemetry/constants.py +++ b/ddtrace/internal/telemetry/constants.py @@ -5,6 +5,7 @@ TELEMETRY_NAMESPACE_TAG_TRACER = "tracers" TELEMETRY_NAMESPACE_TAG_APPSEC = "appsec" +TELEMETRY_NAMESPACE_TAG_IAST = "iast" TELEMETRY_TYPE_GENERATE_METRICS = "generate-metrics" TELEMETRY_TYPE_DISTRIBUTION = "distributions" diff --git a/tests/appsec/iast/test_telemety.py b/tests/appsec/iast/test_telemety.py new file mode 100644 index 00000000000..61cea317752 --- /dev/null +++ b/tests/appsec/iast/test_telemety.py @@ -0,0 +1,30 @@ +import pytest + +from ddtrace.appsec.iast._metrics import metric_verbosity +from ddtrace.appsec.iast._metrics import TELEMETRY_DEBUG_VERBOSITY +from ddtrace.appsec.iast._metrics import TELEMETRY_MANDATORY_VERBOSITY +from ddtrace.appsec.iast._metrics import TELEMETRY_INFORMATION_VERBOSITY + +from tests.utils import override_env + + +@pytest.mark.parametrize( + "lvl, env_lvl, expected_result", + [ + (TELEMETRY_DEBUG_VERBOSITY, "OFF", None), + (TELEMETRY_MANDATORY_VERBOSITY, "OFF", None), + (TELEMETRY_INFORMATION_VERBOSITY, "OFF", None), + (TELEMETRY_DEBUG_VERBOSITY, "DEBUG", 1), + (TELEMETRY_MANDATORY_VERBOSITY, "DEBUG", 1), + (TELEMETRY_INFORMATION_VERBOSITY, "DEBUG", 1), + (TELEMETRY_DEBUG_VERBOSITY, "INFORMATION", None), + (TELEMETRY_INFORMATION_VERBOSITY, "INFORMATION", 1), + (TELEMETRY_MANDATORY_VERBOSITY, "INFORMATION", 1), + (TELEMETRY_DEBUG_VERBOSITY, "MANDATORY", None), + (TELEMETRY_INFORMATION_VERBOSITY, "MANDATORY", None), + (TELEMETRY_MANDATORY_VERBOSITY, "MANDATORY", 1), + ], +) +def test_metric_verbosity(lvl, env_lvl, expected_result): + with override_env(dict(DD_IAST_TELEMETRY_VERBOSITY=env_lvl)): + assert metric_verbosity(lvl)(lambda: 1)() == expected_result From 95c54c23e4b90147b7953a0fdeadbc9c181437a4 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 28 Jun 2023 18:55:05 +0200 Subject: [PATCH 2/5] feat(iast): iast metrics core and executed.source metric --- ddtrace/appsec/iast/_metrics.py | 5 ++--- tests/appsec/iast/test_telemety.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ddtrace/appsec/iast/_metrics.py b/ddtrace/appsec/iast/_metrics.py index 6fcdb8ffb8e..55d17d3ace9 100644 --- a/ddtrace/appsec/iast/_metrics.py +++ b/ddtrace/appsec/iast/_metrics.py @@ -1,11 +1,10 @@ import os +from ddtrace.appsec._constants import IAST from ddtrace.internal.logger import get_logger - from ddtrace.internal.telemetry import telemetry_metrics_writer -from ddtrace.internal.utils.cache import cached from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_IAST -from ddtrace.appsec._constants import IAST + log = get_logger(__name__) diff --git a/tests/appsec/iast/test_telemety.py b/tests/appsec/iast/test_telemety.py index 61cea317752..e9690364969 100644 --- a/tests/appsec/iast/test_telemety.py +++ b/tests/appsec/iast/test_telemety.py @@ -1,10 +1,9 @@ import pytest -from ddtrace.appsec.iast._metrics import metric_verbosity from ddtrace.appsec.iast._metrics import TELEMETRY_DEBUG_VERBOSITY -from ddtrace.appsec.iast._metrics import TELEMETRY_MANDATORY_VERBOSITY from ddtrace.appsec.iast._metrics import TELEMETRY_INFORMATION_VERBOSITY - +from ddtrace.appsec.iast._metrics import TELEMETRY_MANDATORY_VERBOSITY +from ddtrace.appsec.iast._metrics import metric_verbosity from tests.utils import override_env From 21552ecd1aaebce3e42aa652076b4d33ce5d113f Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 28 Jun 2023 18:59:37 +0200 Subject: [PATCH 3/5] feat(iast): iast metrics core and executed.source metric --- ddtrace/appsec/iast/_taint_tracking/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/appsec/iast/_taint_tracking/__init__.py b/ddtrace/appsec/iast/_taint_tracking/__init__.py index 66dcd4e0e10..b88d76f37bb 100644 --- a/ddtrace/appsec/iast/_taint_tracking/__init__.py +++ b/ddtrace/appsec/iast/_taint_tracking/__init__.py @@ -53,7 +53,7 @@ def taint_pyobject(pyobject, input_info): # type: (Any, Input_info) -> Any pyobject = new_pyobject_id(pyobject, len_pyobject) taint_dict = get_taint_dict() taint_dict[id(pyobject)] = ((input_info, 0, len_pyobject),) - _set_metric_iast_executed_source(source_origin) + _set_metric_iast_executed_source(input_info.origin) return pyobject From 6ca12432d7a28e45c111646955b555949677061e Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 28 Jun 2023 19:11:28 +0200 Subject: [PATCH 4/5] feat(iast): iast metrics core and executed.source metric --- ddtrace/appsec/iast/_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/appsec/iast/_metrics.py b/ddtrace/appsec/iast/_metrics.py index 55d17d3ace9..2dc4d01af41 100644 --- a/ddtrace/appsec/iast/_metrics.py +++ b/ddtrace/appsec/iast/_metrics.py @@ -42,7 +42,7 @@ def wrapper(f): return f except Exception: log.warning("Error reporting IAST metrics", exc_info=True) - return lambda: None + return lambda: None # noqa: E731 return wrapper From 2e54b113a132d699b5b0cd170ca7b35ce349ddf0 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 28 Jun 2023 19:15:45 +0200 Subject: [PATCH 5/5] feat(iast): iast metrics core and executed.source metric --- ddtrace/appsec/iast/_metrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ddtrace/appsec/iast/_metrics.py b/ddtrace/appsec/iast/_metrics.py index 2dc4d01af41..ca172b0a423 100644 --- a/ddtrace/appsec/iast/_metrics.py +++ b/ddtrace/appsec/iast/_metrics.py @@ -82,7 +82,4 @@ def _set_metric_iast_executed_sink(vulnerability_type): @metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY) def _set_metric_iast_request_tainted(): - # TODO: function num_objects_tainted is in other PR - # from ddtrace.appsec.iast._taint_tracking import num_objects_tainted - num_objects_tainted = lambda: 1 - telemetry_metrics_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_IAST, "request.tainted", num_objects_tainted()) + telemetry_metrics_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_IAST, "request.tainted", 1)