Merge branch 'main' into evan.li/migrate-token-metric-key-names

.github/CODEOWNERS

@@ -81,6 +81,7 @@ scripts/iast/*                      @DataDog/asm-python
 
 # Profiling
 ddtrace/profiling                   @DataDog/profiling-python
+ddtrace/settings/profiling.py       @DataDog/profiling-python
 ddtrace/internal/datadog/profiling  @DataDog/profiling-python
 tests/profiling                     @DataDog/profiling-python
 

ddtrace/appsec/_capabilities.py

@@ -6,11 +6,6 @@
 from ddtrace.settings.asm import config as asm_config
 
 
-def _asm_feature_is_required():
-    flags = _rc_capabilities()
-    return Flags.ASM_ACTIVATION in flags or Flags.ASM_API_SECURITY_SAMPLE_RATE in flags
-
-
 class Flags(enum.IntFlag):
     ASM_ACTIVATION = 1 << 1
     ASM_IP_BLOCKING = 1 << 2
@@ -22,7 +17,6 @@ class Flags(enum.IntFlag):
     ASM_CUSTOM_RULES = 1 << 8
     ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
     ASM_TRUSTED_IPS = 1 << 10
-    ASM_API_SECURITY_SAMPLE_RATE = 1 << 11
     ASM_RASP_SQLI = 1 << 21
     ASM_RASP_LFI = 1 << 22
     ASM_RASP_SSRF = 1 << 23
@@ -31,6 +25,7 @@ class Flags(enum.IntFlag):
     ASM_RASP_RCE = 1 << 26
     ASM_RASP_NOSQLI = 1 << 27
     ASM_RASP_XSS = 1 << 28
+    ASM_AUTO_USER = 1 << 31
 
 
 _ALL_ASM_BLOCKING = (
@@ -45,6 +40,12 @@ class Flags(enum.IntFlag):
 )
 
 _ALL_RASP = Flags.ASM_RASP_SQLI | Flags.ASM_RASP_LFI | Flags.ASM_RASP_SSRF
+_FEATURE_REQUIRED = Flags.ASM_ACTIVATION | Flags.ASM_AUTO_USER
+
+
+def _asm_feature_is_required() -> bool:
+    flags = _rc_capabilities()
+    return (_FEATURE_REQUIRED & flags) != 0
 
 
 def _rc_capabilities(test_tracer: Optional[ddtrace.Tracer] = None) -> Flags:
@@ -57,8 +58,8 @@ def _rc_capabilities(test_tracer: Optional[ddtrace.Tracer] = None) -> Flags:
             value |= _ALL_ASM_BLOCKING
             if asm_config._ep_enabled:
                 value |= _ALL_RASP
-        if asm_config._api_security_enabled:
-            value |= Flags.ASM_API_SECURITY_SAMPLE_RATE
+        if asm_config._auto_user_instrumentation_enabled:
+            value |= Flags.ASM_AUTO_USER
     return value
 
 

ddtrace/appsec/_constants.py

@@ -69,7 +69,9 @@ class APPSEC(metaclass=Constant_Class):
     AUTO_LOGIN_EVENTS_FAILURE_MODE = "_dd.appsec.events.users.login.failure.auto.mode"
     BLOCKED = "appsec.blocked"
     EVENT = "appsec.event"
-    AUTOMATIC_USER_EVENTS_TRACKING = "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING"
+    AUTOMATIC_USER_EVENTS_TRACKING = "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING"  # DEPRECATED
+    AUTO_USER_INSTRUMENTATION_MODE = "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE"
+    AUTO_USER_INSTRUMENTATION_MODE_ENABLED = "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING_ENABLED"
     USER_MODEL_LOGIN_FIELD = "DD_USER_MODEL_LOGIN_FIELD"
     USER_MODEL_EMAIL_FIELD = "DD_USER_MODEL_EMAIL_FIELD"
     USER_MODEL_NAME_FIELD = "DD_USER_MODEL_NAME_FIELD"
@@ -218,17 +220,18 @@ class PRODUCTS(metaclass=Constant_Class):
 class LOGIN_EVENTS_MODE(metaclass=Constant_Class):
     """
     string identifier for the mode of the user login events. Can be:
-    DISABLED: automatic login events are disabled.
-    SAFE: automatic login events are enabled but will only store non-PII fields (id, pk uid...)
+    DISABLED: automatic login events are disabled. Can still be enabled by Remote Config.
+    ANONYMIZATION: automatic login events are enabled but will only store non-PII fields (id, pk uid...)
     EXTENDED: automatic login events are enabled and will store potentially PII fields (username,
     email, ...).
     SDK: manually issued login events using the SDK.
     """
 
     DISABLED = "disabled"
-    SAFE = "safe"
-    EXTENDED = "extended"
+    IDENT = "identification"
+    ANON = "anonymization"
     SDK = "sdk"
+    AUTO = "auto"
 
 
 class DEFAULT(metaclass=Constant_Class):

ddtrace/appsec/_remoteconfiguration.py

@@ -7,7 +7,6 @@
 from typing import Optional
 
 from ddtrace import Tracer
-from ddtrace import config
 from ddtrace.appsec._capabilities import _asm_feature_is_required
 from ddtrace.appsec._constants import PRODUCTS
 from ddtrace.internal import forksafe
@@ -97,6 +96,7 @@ def _add_rules_to_list(features: Mapping[str, Any], feature: str, message: str,
 def _appsec_callback(features: Mapping[str, Any], test_tracer: Optional[Tracer] = None) -> None:
     config = features.get("config", {})
     _appsec_1click_activation(config, test_tracer)
+    _appsec_auto_user_mode(config, test_tracer)
     _appsec_rules_data(config, test_tracer)
 
 
@@ -143,9 +143,7 @@ def _preprocess_results_appsec_1click_activation(
             if features == {}:
                 rc_asm_enabled = False
             else:
-                asm_features = features.get("asm", {})
-                if asm_features is not None:
-                    rc_asm_enabled = asm_features.get("enabled")
+                rc_asm_enabled = features.get("asm", {}).get("enabled")
             log.debug(
                 "[%s][P: %s] ASM Remote Configuration ASM_FEATURES. Appsec enabled: %s",
                 os.getpid(),
@@ -224,17 +222,8 @@ def _appsec_1click_activation(features: Mapping[str, Any], test_tracer: Optional
                     asm_config._asm_enabled = False
 
 
-def _appsec_api_security_settings(features: Mapping[str, Any], test_tracer: Optional[Tracer] = None) -> None:
+def _appsec_auto_user_mode(features: Mapping[str, Any], test_tracer: Optional[Tracer] = None) -> None:
     """
-    Deprecated
-    Update API Security settings from remote config
-    Actually: Update sample rate
+    Update Auto User settings from remote config
     """
-    if config._remote_config_enabled and asm_config._api_security_enabled:
-        rc_api_security_sample_rate = features.get("api_security", {}).get("request_sample_rate", None)
-        if rc_api_security_sample_rate is not None:
-            try:
-                sample_rate = max(0.0, min(1.0, float(rc_api_security_sample_rate)))
-                asm_config._api_security_sample_rate = sample_rate
-            except Exception:  # nosec
-                pass
+    asm_config._auto_user_instrumentation_rc_mode = features.get("auto_user_instrum", {}).get("mode", None)

ddtrace/appsec/_trace_utils.py

@@ -7,7 +7,7 @@
 from ddtrace.appsec._constants import APPSEC
 from ddtrace.appsec._constants import LOGIN_EVENTS_MODE
 from ddtrace.appsec._constants import WAF_CONTEXT_NAMES
-from ddtrace.appsec._utils import _safe_userid
+from ddtrace.appsec._utils import _hash_user_id
 from ddtrace.contrib.trace_utils import set_user
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import user
@@ -54,14 +54,14 @@ def _track_user_login_common(
             span.set_tag_str(APPSEC.USER_LOGIN_EVENT_FAILURE_TRACK, "true")
 
         # This is used to mark if the call was done from the SDK of the automatic login events
-        if login_events_mode == LOGIN_EVENTS_MODE.SDK:
+        if login_events_mode in (LOGIN_EVENTS_MODE.SDK, LOGIN_EVENTS_MODE.AUTO):
             span.set_tag_str("%s.sdk" % tag_prefix, "true")
+            reported_mode = asm_config._user_event_mode
+        else:
+            reported_mode = login_events_mode
 
         mode_tag = APPSEC.AUTO_LOGIN_EVENTS_SUCCESS_MODE if success else APPSEC.AUTO_LOGIN_EVENTS_FAILURE_MODE
-        auto_tag_mode = (
-            login_events_mode if login_events_mode != LOGIN_EVENTS_MODE.SDK else asm_config._automatic_login_events_mode
-        )
-        span.set_tag_str(mode_tag, auto_tag_mode)
+        span.set_tag_str(mode_tag, reported_mode)
 
         tag_metadata_prefix = "%s.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, success_str)
         if metadata is not None:
@@ -114,16 +114,15 @@ def track_user_login_success_event(
     :param metadata: a dictionary with additional metadata information to be stored with the event
     """
 
+    real_mode = login_events_mode if login_events_mode != LOGIN_EVENTS_MODE.AUTO else asm_config._user_event_mode
+    if real_mode == LOGIN_EVENTS_MODE.DISABLED:
+        return
     span = _track_user_login_common(tracer, True, metadata, login_events_mode, login, name, email, span)
     if not span:
         return
 
-    if (
-        user_id
-        and (login_events_mode not in (LOGIN_EVENTS_MODE.SDK, LOGIN_EVENTS_MODE.EXTENDED))
-        and not asm_config._user_model_login_field
-    ):
-        user_id = _safe_userid(user_id)
+    if real_mode == LOGIN_EVENTS_MODE.ANON and isinstance(user_id, str):
+        user_id = _hash_user_id(user_id)
 
     set_user(tracer, user_id, name, email, scope, role, session_id, propagate, span)
 
@@ -146,27 +145,27 @@ def track_user_login_failure_event(
     :param metadata: a dictionary with additional metadata information to be stored with the event
     """
 
-    if (
-        user_id
-        and (login_events_mode not in (LOGIN_EVENTS_MODE.SDK, LOGIN_EVENTS_MODE.EXTENDED))
-        and not asm_config._user_model_login_field
-    ):
-        user_id = _safe_userid(user_id)
-
+    real_mode = login_events_mode if login_events_mode != LOGIN_EVENTS_MODE.AUTO else asm_config._user_event_mode
+    if real_mode == LOGIN_EVENTS_MODE.DISABLED:
+        return
     span = _track_user_login_common(tracer, False, metadata, login_events_mode)
     if not span:
         return
-    if user_id:
-        span.set_tag_str("%s.failure.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, user.ID), str(user_id))
     if exists is not None:
         exists_str = "true" if exists else "false"
         span.set_tag_str("%s.failure.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, user.EXISTS), exists_str)
-    if login:
-        span.set_tag_str("%s.failure.login" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, login)
-    if email:
-        span.set_tag_str("%s.failure.email" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, email)
-    if name:
-        span.set_tag_str("%s.failure.username" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, name)
+    if user_id:
+        if real_mode == LOGIN_EVENTS_MODE.ANON and isinstance(user_id, str):
+            user_id = _hash_user_id(user_id)
+        span.set_tag_str("%s.failure.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, user.ID), str(user_id))
+    # if called from the SDK, set the login, email and name
+    if login_events_mode in (LOGIN_EVENTS_MODE.SDK, LOGIN_EVENTS_MODE.AUTO):
+        if login:
+            span.set_tag_str("%s.failure.login" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, login)
+        if email:
+            span.set_tag_str("%s.failure.email" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, email)
+        if name:
+            span.set_tag_str("%s.failure.username" % APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, name)
 
 
 def track_user_signup_event(
@@ -305,14 +304,11 @@ def _on_django_login(
     mode,
     info_retriever,
 ):
-    if not asm_config._asm_enabled:
-        return
-
     if user:
         from ddtrace.contrib.django.compat import user_is_authenticated
 
         if user_is_authenticated(user):
-            user_id, user_extra = info_retriever.get_user_info()
+            user_id = info_retriever.get_userid()
 
             with pin.tracer.trace("django.contrib.auth.login", span_type=SpanTypes.AUTH):
                 session_key = getattr(request, "session_key", None)
@@ -322,7 +318,6 @@ def _on_django_login(
                     session_id=session_key,
                     propagate=True,
                     login_events_mode=mode,
-                    **user_extra,
                 )
         else:
             # Login failed and the user is unknown (may exist or not)
@@ -334,8 +329,7 @@ def _on_django_auth(result_user, mode, kwargs, pin, info_retriever):
     if not asm_config._asm_enabled:
         return True, result_user
 
-    extended_userid_fields = info_retriever.possible_user_id_fields + info_retriever.possible_login_fields
-    userid_list = info_retriever.possible_user_id_fields if mode == "safe" else extended_userid_fields
+    userid_list = info_retriever.possible_user_id_fields + info_retriever.possible_login_fields
 
     for possible_key in userid_list:
         if possible_key in kwargs:
@@ -348,16 +342,15 @@ def _on_django_auth(result_user, mode, kwargs, pin, info_retriever):
         with pin.tracer.trace("django.contrib.auth.login", span_type=SpanTypes.AUTH):
             exists = info_retriever.user_exists()
             if exists:
-                user_id, user_extra = info_retriever.get_user_info()
+                user_id = info_retriever.get_userid()
                 track_user_login_failure_event(
                     pin.tracer,
                     user_id=user_id,
                     login_events_mode=mode,
                     exists=True,
-                    **user_extra,
                 )
             else:
-                track_user_login_failure_event(pin.tracer, user_id=user_id, login_events_mode=mode, exists=exists)
+                track_user_login_failure_event(pin.tracer, user_id=user_id, login_events_mode=mode, exists=False)
 
     return False, None
 

ddtrace/appsec/_utils.py

@@ -63,6 +63,12 @@ def access_body(bd):
         return req_body
 
 
+def _hash_user_id(user_id: str) -> str:
+    import hashlib
+
+    return f"anon_{hashlib.sha256(user_id.encode()).hexdigest()[:32]}"
+
+
 def _safe_userid(user_id):
     try:
         _ = int(user_id)
@@ -108,10 +114,7 @@ def get_userid(self):
             return user_login
 
         user_login = self.find_in_user_model(self.possible_user_id_fields)
-        if asm_config._automatic_login_events_mode == "extended":
-            return user_login
-
-        return _safe_userid(user_login)
+        return user_login
 
     def get_username(self):
         username = getattr(self.user, asm_config._user_model_name_field, None)
@@ -149,19 +152,15 @@ def get_user_info(self):
         user_extra_info = {}
 
         user_id = self.get_userid()
-        if asm_config._automatic_login_events_mode == "extended":
-            if not user_id:
-                user_id = self.find_in_user_model(self.possible_user_id_fields)
-
-            user_extra_info = {
-                "login": self.get_username(),
-                "email": self.get_user_email(),
-                "name": self.get_name(),
-            }
-
         if not user_id:
             return None, {}
 
+        user_extra_info = {
+            "login": self.get_username(),
+            "email": self.get_user_email(),
+            "name": self.get_name(),
+        }
+
         return user_id, user_extra_info
 
 

ddtrace/contrib/django/patch.py

@@ -763,50 +763,29 @@ def get_user_email(self):
 @trace_utils.with_traced_module
 def traced_login(django, pin, func, instance, args, kwargs):
     func(*args, **kwargs)
-
+    mode = asm_config._user_event_mode
+    if mode == "disabled":
+        return
     try:
-        mode = asm_config._automatic_login_events_mode
         request = get_argument_value(args, kwargs, 0, "request")
         user = get_argument_value(args, kwargs, 1, "user")
-
-        if mode == "disabled":
-            return
-
-        core.dispatch(
-            "django.login",
-            (
-                pin,
-                request,
-                user,
-                mode,
-                _DjangoUserInfoRetriever(user),
-            ),
-        )
+        core.dispatch("django.login", (pin, request, user, mode, _DjangoUserInfoRetriever(user)))
     except Exception:
         log.debug("Error while trying to trace Django login", exc_info=True)
 
 
 @trace_utils.with_traced_module
 def traced_authenticate(django, pin, func, instance, args, kwargs):
     result_user = func(*args, **kwargs)
+    mode = asm_config._user_event_mode
+    if mode == "disabled":
+        return result_user
     try:
-        mode = asm_config._automatic_login_events_mode
-        if mode == "disabled":
-            return result_user
-
         result = core.dispatch_with_results(
-            "django.auth",
-            (
-                result_user,
-                mode,
-                kwargs,
-                pin,
-                _DjangoUserInfoRetriever(result_user, credentials=kwargs),
-            ),
+            "django.auth", (result_user, mode, kwargs, pin, _DjangoUserInfoRetriever(result_user, credentials=kwargs))
         ).user
         if result and result.value[0]:
             return result.value[1]
-
     except Exception:
         log.debug("Error while trying to trace Django authenticate", exc_info=True)