feat(settings): fuzzy env variable matching

ddtrace/_monkey.py

@@ -1,5 +1,4 @@
 import importlib
-import os
 import sys
 import threading
 from typing import TYPE_CHECKING
@@ -12,6 +11,7 @@
 from .internal.utils import formats
 from .internal.utils.importlib import require_modules
 from .settings import _config as config
+from .settings.matching import getenv
 
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -177,11 +177,10 @@ def patch_all(**patch_modules):
     # The enabled setting can be overridden by environment variables
     for module, enabled in modules.items():
         env_var = "DD_TRACE_%s_ENABLED" % module.upper()
-        if env_var not in os.environ:
-            continue
-
-        override_enabled = formats.asbool(os.environ[env_var])
-        modules[module] = override_enabled
+        value = getenv(env_var)
+        if value is not None:
+            override_enabled = formats.asbool(value)
+            modules[module] = override_enabled
 
     # Arguments take precedence over the environment and the defaults.
     modules.update(patch_modules)
@@ -196,7 +195,7 @@ def patch_iast(**patch_modules):
 
     IAST_PATCH: list of implemented vulnerabilities
     """
-    iast_enabled = formats.asbool(os.environ.get(IAST_ENV, "false"))
+    iast_enabled = formats.asbool(getenv(IAST_ENV, "false"))
     if iast_enabled:
         patch(raise_errors=False, patch_modules_prefix="ddtrace.appsec.iast.taint_sinks", **patch_modules)
 

ddtrace/appsec/_remoteconfiguration.py

@@ -1,4 +1,3 @@
-import os
 from typing import TYPE_CHECKING
 
 from ddtrace.appsec.utils import _appsec_rc_features_is_enabled
@@ -7,6 +6,7 @@
 from ddtrace.internal.remoteconfig import RemoteConfig
 from ddtrace.internal.remoteconfig.constants import ASM_FEATURES_PRODUCT
 from ddtrace.internal.utils.formats import asbool
+from ddtrace.settings.matching import getenv
 
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -51,8 +51,8 @@ def _reload_features(metadata, features):
 
             _appsec_enabled = True
 
-            if not (APPSEC_ENV not in os.environ and rc_appsec_enabled is True) and (
-                asbool(os.environ.get(APPSEC_ENV)) is False or rc_appsec_enabled is False
+            if not (getenv(APPSEC_ENV) is None and rc_appsec_enabled is True) and (
+                asbool(getenv(APPSEC_ENV)) is False or rc_appsec_enabled is False
             ):
                 _appsec_enabled = False
 

ddtrace/appsec/iast/_overhead_control_engine.py

@@ -3,18 +3,19 @@
 limit. It will measure operations being executed in a request and it will deactivate detection
 (and therefore reduce the overhead to nearly 0) if a certain threshold is reached.
 """
-import os
 import threading
 from typing import TYPE_CHECKING
 
+from ddtrace.settings.matching import getenv
+
 
 if TYPE_CHECKING:  # pragma: no cover
     from typing import Set
     from typing import Tuple
     from typing import Type
 
-MAX_REQUESTS = int(os.environ.get("DD_IAST_MAX_CONCURRENT_REQUESTS", 2))
-MAX_VULNERABILITIES_PER_REQUEST = int(os.environ.get("DD_IAST_VULNERABILITIES_PER_REQUEST", 2))
+MAX_REQUESTS = int(getenv("DD_IAST_MAX_CONCURRENT_REQUESTS", 2))
+MAX_VULNERABILITIES_PER_REQUEST = int(getenv("DD_IAST_VULNERABILITIES_PER_REQUEST", 2))
 
 
 class Operation(object):

ddtrace/appsec/processor.py

@@ -32,6 +32,7 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.processor import SpanProcessor
 from ddtrace.internal.rate_limiter import RateLimiter
+from ddtrace.settings.matching import getenv
 
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -80,20 +81,20 @@ def _transform_headers(data):
 
 def get_rules():
     # type: () -> str
-    return os.getenv("DD_APPSEC_RULES", default=DEFAULT_RULES)
+    return getenv("DD_APPSEC_RULES", default=DEFAULT_RULES)
 
 
 def get_appsec_obfuscation_parameter_key_regexp():
     # type: () -> bytes
     return ensure_binary(
-        os.getenv("DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP", DEFAULT_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP)
+        getenv("DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP", DEFAULT_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP)
     )
 
 
 def get_appsec_obfuscation_parameter_value_regexp():
     # type: () -> bytes
     return ensure_binary(
-        os.getenv("DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP", DEFAULT_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP)
+        getenv("DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP", DEFAULT_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP)
     )
 
 
@@ -143,12 +144,12 @@ def _set_headers(span, headers, kind):
 
 def _get_rate_limiter():
     # type: () -> RateLimiter
-    return RateLimiter(int(os.getenv("DD_APPSEC_TRACE_RATE_LIMIT", DEFAULT_TRACE_RATE_LIMIT)))
+    return RateLimiter(int(getenv("DD_APPSEC_TRACE_RATE_LIMIT", DEFAULT_TRACE_RATE_LIMIT)))
 
 
 def _get_waf_timeout():
     # type: () -> int
-    return int(os.getenv("DD_APPSEC_WAF_TIMEOUT", DEFAULT_WAF_TIMEOUT))
+    return int(getenv("DD_APPSEC_WAF_TIMEOUT", DEFAULT_WAF_TIMEOUT))
 
 
 @attr.s(eq=False)

ddtrace/appsec/utils.py

@@ -1,13 +1,12 @@
-import os
-
 from ddtrace.constants import APPSEC_ENV
 from ddtrace.internal.utils.formats import asbool
+from ddtrace.settings.matching import getenv
 
 
 def _appsec_rc_features_is_enabled():
     # type: () -> bool
-    if asbool(os.environ.get("DD_REMOTE_CONFIGURATION_ENABLED", "false")):
-        return APPSEC_ENV not in os.environ
+    if asbool(getenv("DD_REMOTE_CONFIGURATION_ENABLED", "false")):
+        return getenv(APPSEC_ENV) is None
     return False
 
 

ddtrace/bootstrap/sitecustomize.py

@@ -24,6 +24,7 @@
 from ddtrace.internal.runtime.runtime_metrics import RuntimeWorker
 from ddtrace.internal.utils.formats import asbool  # noqa
 from ddtrace.internal.utils.formats import parse_tags_str
+from ddtrace.settings.matching import getenv
 from ddtrace.tracer import DD_LOG_FORMAT  # noqa
 from ddtrace.tracer import debug_mode
 from ddtrace.vendor.debtcollector import deprecate
@@ -66,7 +67,7 @@
 
 
 def update_patched_modules():
-    modules_to_patch = os.getenv("DD_PATCH_MODULES")
+    modules_to_patch = getenv("DD_PATCH_MODULES")
     if not modules_to_patch:
         return
 
@@ -78,8 +79,8 @@ def update_patched_modules():
 try:
     from ddtrace import tracer
 
-    priority_sampling = os.getenv("DD_PRIORITY_SAMPLING")
-    profiling = asbool(os.getenv("DD_PROFILING_ENABLED", False))
+    priority_sampling = getenv("DD_PRIORITY_SAMPLING")
+    profiling = asbool(getenv("DD_PROFILING_ENABLED", False))
 
     if profiling:
         log.debug("profiler enabled via environment variable")
@@ -90,12 +91,12 @@ def update_patched_modules():
 
         DynamicInstrumentation.enable()
 
-    if asbool(os.getenv("DD_RUNTIME_METRICS_ENABLED")):
+    if asbool(getenv("DD_RUNTIME_METRICS_ENABLED")):
         RuntimeWorker.enable()
 
     opts = {}  # type: Dict[str, Any]
 
-    dd_trace_enabled = os.getenv("DD_TRACE_ENABLED", default=True)
+    dd_trace_enabled = getenv("DD_TRACE_ENABLED", default=True)
     if asbool(dd_trace_enabled):
         trace_enabled = True
     else:
@@ -114,12 +115,12 @@ def update_patched_modules():
 
         patch_all(**EXTRA_PATCHED_MODULES)
 
-    dd_env = os.getenv("DD_ENV")
+    dd_env = getenv("DD_ENV")
     if dd_env:
         tracer.set_tags({constants.ENV_KEY: dd_env})
 
-    if "DD_TRACE_GLOBAL_TAGS" in os.environ:
-        env_tags = os.getenv("DD_TRACE_GLOBAL_TAGS")
+    env_tags = getenv("DD_TRACE_GLOBAL_TAGS")
+    if env_tags is not None:
         tracer.set_tags(parse_tags_str(env_tags))
 
     # Check for and import any sitecustomize that would have normally been used

ddtrace/internal/agent.py

@@ -3,6 +3,7 @@
 from typing import Union
 
 from ddtrace.internal.compat import parse
+from ddtrace.settings.matching import getenv
 
 from .http import HTTPConnection
 from .http import HTTPSConnection
@@ -24,33 +25,33 @@
 
 def get_trace_hostname(default=DEFAULT_HOSTNAME):
     # type: (Union[T, str]) -> Union[T, str]
-    return os.environ.get("DD_AGENT_HOST", os.environ.get("DD_TRACE_AGENT_HOSTNAME", default))
+    return getenv("DD_AGENT_HOST", getenv("DD_TRACE_AGENT_HOSTNAME", default))
 
 
 def get_stats_hostname(default=DEFAULT_HOSTNAME):
     # type: (Union[T, str]) -> Union[T, str]
-    return os.environ.get("DD_AGENT_HOST", os.environ.get("DD_DOGSTATSD_HOST", default))
+    return getenv("DD_AGENT_HOST", getenv("DD_DOGSTATSD_HOST", default))
 
 
 def get_trace_port(default=DEFAULT_TRACE_PORT):
     # type: (Union[T, int]) -> Union[T,int]
-    v = os.environ.get("DD_AGENT_PORT", os.environ.get("DD_TRACE_AGENT_PORT"))
+    v = getenv("DD_AGENT_PORT", getenv("DD_TRACE_AGENT_PORT"))
     if v is not None:
         return int(v)
     return default
 
 
 def get_stats_port(default=DEFAULT_STATS_PORT):
     # type: (Union[T, int]) -> Union[T,int]
-    v = os.getenv("DD_DOGSTATSD_PORT", default=None)
+    v = getenv("DD_DOGSTATSD_PORT", default=None)
     if v is not None:
         return int(v)
     return default
 
 
 def get_trace_agent_timeout():
     # type: () -> float
-    return float(os.getenv("DD_TRACE_AGENT_TIMEOUT_SECONDS", default=DEFAULT_TIMEOUT))
+    return float(getenv("DD_TRACE_AGENT_TIMEOUT_SECONDS", default=DEFAULT_TIMEOUT))
 
 
 def get_trace_url():
@@ -62,7 +63,7 @@ def get_trace_url():
     user_supplied_host = get_trace_hostname(None) is not None
     user_supplied_port = get_trace_port(None) is not None
 
-    url = os.environ.get("DD_TRACE_AGENT_URL")
+    url = getenv("DD_TRACE_AGENT_URL")
 
     if not url:
         if user_supplied_host or user_supplied_port:
@@ -80,7 +81,7 @@ def get_stats_url():
     user_supplied_host = get_stats_hostname(None) is not None
     user_supplied_port = get_stats_port(None) is not None
 
-    url = os.getenv("DD_DOGSTATSD_URL", default=None)
+    url = getenv("DD_DOGSTATSD_URL", default=None)
 
     if not url:
         if user_supplied_host or user_supplied_port:

ddtrace/internal/debug.py

@@ -13,6 +13,7 @@
 from ddtrace.internal.writer import AgentWriter
 from ddtrace.internal.writer import LogWriter
 from ddtrace.sampler import DatadogSampler
+from ddtrace.settings.matching import getenv
 
 from .logger import get_logger
 
@@ -143,7 +144,7 @@ def collect(tracer):
         runtime_metrics_enabled=RuntimeWorker.enabled,
         dd_version=ddtrace.config.version or "",
         priority_sampling_enabled=tracer._priority_sampler is not None,
-        global_tags=os.getenv("DD_TAGS", ""),
+        global_tags=getenv("DD_TAGS", ""),
         tracer_tags=tags_to_str(tracer._tags),
         integrations=integration_configs,
         partial_flush_enabled=tracer._partial_flush_enabled,

ddtrace/internal/remoteconfig/worker.py

@@ -1,10 +1,10 @@
 import logging
-import os
 
 from ddtrace.internal import periodic
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.remoteconfig.client import RemoteConfigClient
 from ddtrace.internal.utils.time import StopWatch
+from ddtrace.settings.matching import getenv
 
 
 log = get_logger(__name__)
@@ -15,7 +15,7 @@
 
 def get_poll_interval_seconds():
     # type:() -> float
-    return float(os.getenv("DD_REMOTECONFIG_POLL_SECONDS", default=DEFAULT_REMOTECONFIG_POLL_SECONDS))
+    return float(getenv("DD_REMOTECONFIG_POLL_SECONDS", default=DEFAULT_REMOTECONFIG_POLL_SECONDS))
 
 
 class RemoteConfigWorker(periodic.PeriodicService):

ddtrace/internal/runtime/runtime_metrics.py

@@ -1,5 +1,4 @@
 import itertools
-import os
 from typing import ClassVar
 from typing import Optional
 from typing import Set
@@ -13,6 +12,7 @@
 
 import ddtrace
 from ddtrace.internal import forksafe
+from ddtrace.settings.matching import getenv
 
 from .. import periodic
 from ..dogstatsd import get_dogstatsd_client
@@ -62,7 +62,7 @@ class RuntimeMetrics(RuntimeCollectorsIterable):
 
 
 def _get_interval_or_default():
-    return float(os.getenv("DD_RUNTIME_METRICS_INTERVAL", default=10))
+    return float(getenv("DD_RUNTIME_METRICS_INTERVAL", default=10))
 
 
 @attr.s(eq=False)

ddtrace/internal/sampling.py

@@ -1,5 +1,4 @@
 import json
-import os
 import re
 from typing import Optional
 from typing import TYPE_CHECKING
@@ -19,6 +18,7 @@
 from ddtrace.constants import _SINGLE_SPAN_SAMPLING_RATE
 from ddtrace.internal.glob_matching import GlobMatcher
 from ddtrace.internal.logger import get_logger
+from ddtrace.settings.matching import getenv
 
 from .rate_limiter import RateLimiter
 
@@ -267,7 +267,7 @@ def _get_span_sampling_json():
 
 def _get_file_json():
     # type: () -> Optional[List[Dict[str, Any]]]
-    file_json_raw = os.getenv("DD_SPAN_SAMPLING_RULES_FILE")
+    file_json_raw = getenv("DD_SPAN_SAMPLING_RULES_FILE")
     if file_json_raw:
         with open(file_json_raw) as f:
             return _load_span_sampling_json(f.read())
@@ -276,7 +276,7 @@ def _get_file_json():
 
 def _get_env_json():
     # type: () -> Optional[List[Dict[str, Any]]]
-    env_json_raw = os.getenv("DD_SPAN_SAMPLING_RULES")
+    env_json_raw = getenv("DD_SPAN_SAMPLING_RULES")
     if env_json_raw:
         return _load_span_sampling_json(env_json_raw)
     return None

ddtrace/internal/telemetry/writer.py

@@ -7,6 +7,7 @@
 from ...internal import atexit
 from ...internal import forksafe
 from ...settings import _config as config
+from ...settings.matching import getenv
 from ..agent import get_connection
 from ..agent import get_trace_url
 from ..compat import get_connection_response
@@ -28,7 +29,7 @@
 
 def _get_interval_or_default():
     # type: () -> float
-    return float(os.getenv("DD_TELEMETRY_HEARTBEAT_INTERVAL", default=60))
+    return float(getenv("DD_TELEMETRY_HEARTBEAT_INTERVAL", default=60))
 
 
 class TelemetryWriter(PeriodicService):