Merge branch 'master' into levi/overhead-bench

appsec/CMakeLists.txt

@@ -9,7 +9,7 @@ HunterGate(
     URL "https://github.com/cpp-pm/hunter/archive/v0.23.314.tar.gz"
     SHA1 "95c47c92f68edb091b5d6d18924baabe02a6962a")
 
-project(ddappsec VERSION 0.92.0)
+project(ddappsec VERSION 0.93.0)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_C_STANDARD 11)

appsec/src/extension/commands/client_init.c

@@ -122,13 +122,18 @@ static dd_result _pack_command(
     // We send this empty for now. The helper will check for empty and if so it
     // will generate it
     dd_mpack_write_lstr(w, "runtime_id");
-    dd_mpack_write_nullable_cstr(w, "");
-
+    zend_string *runtime_id = dd_trace_get_formatted_runtime_id(false);
+    if (runtime_id == NULL) {
+        dd_mpack_write_nullable_cstr(w, "");
+    } else {
+        dd_mpack_write_nullable_zstr(w, runtime_id);
+        zend_string_free(runtime_id);
+    }
     mpack_finish_map(w);
 
     // Engine settings
     // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers)
-    mpack_start_map(w, 5);
+    mpack_start_map(w, 6);
     {
         dd_mpack_write_lstr(w, "rules_file");
         const char *rules_file = ZSTR_VAL(get_global_DD_APPSEC_RULES());
@@ -155,6 +160,17 @@ static dd_result _pack_command(
     dd_mpack_write_nullable_cstr(
         w, ZSTR_VAL(get_global_DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP()));
 
+    dd_mpack_write_lstr(w, "schema_extraction");
+    mpack_start_map(w, 2);
+
+    dd_mpack_write_lstr(w, "enabled");
+    mpack_write_bool(w, get_global_DD_EXPERIMENTAL_API_SECURITY_ENABLED());
+
+    dd_mpack_write_lstr(w, "sample_rate");
+    mpack_write(w, get_global_DD_API_SECURITY_REQUEST_SAMPLE_RATE());
+
+    mpack_finish_map(w);
+
     mpack_finish_map(w);
 
     // Remote config settings

appsec/src/extension/commands/request_init.c

@@ -171,15 +171,11 @@ static void _pack_headers(mpack_writer_t *nonnull w)
             continue;
         }
 
-        if (Z_TYPE_P(val) != IS_STRING) {
-            continue;
-        }
-
         if (_is_relevant_header(key)) {
             zend_string *transf_header_name = _transform_header_name(key);
             dd_mpack_write_zstr(w, transf_header_name);
             zend_string_efree(transf_header_name);
-            dd_mpack_write_zstr(w, Z_STR_P(val));
+            dd_mpack_write_zval(w, val);
         }
     }
     ZEND_HASH_FOREACH_END();

appsec/src/extension/configuration.h

@@ -31,7 +31,7 @@ extern bool runtime_config_first_init;
 
 // clang-format off
 #define DD_CONFIGURATION \
-    SYSCFG(BOOL, DD_APPSEC_ENABLED, "false")                                                                                          \
+    CONFIG(BOOL, DD_APPSEC_ENABLED, "false")                                                                                          \
     SYSCFG(STRING, DD_APPSEC_RULES, "")                                                                                               \
     SYSCFG(CUSTOM(uint64_t), DD_APPSEC_WAF_TIMEOUT, "10000", .parser = _parse_uint64)                                                 \
     SYSCFG(CUSTOM(uint32_t), DD_APPSEC_TRACE_RATE_LIMIT, "100", .parser = _parse_uint32)                                              \
@@ -47,7 +47,7 @@ extern bool runtime_config_first_init;
     CONFIG(STRING, DD_APPSEC_HELPER_PATH, DD_BASE("bin/ddappsec-helper"))                                                             \
     CONFIG(STRING, DD_APPSEC_HELPER_RUNTIME_PATH, "/tmp", .ini_change = dd_on_runtime_path_update)                                    \
     SYSCFG(STRING, DD_APPSEC_HELPER_LOG_FILE, "/dev/null")                                                                            \
-    CONFIG(CUSTOM(SET), DD_EXTRA_SERVICES, "", .parser = _parse_list)                                                                  \
+    CONFIG(CUSTOM(SET), DD_EXTRA_SERVICES, "", .parser = _parse_list)                                                                 \
     CONFIG(STRING, DD_APPSEC_HELPER_EXTRA_ARGS, "")                                                                                   \
     CALIAS(STRING, DD_SERVICE, "", CALIASES("DD_SERVICE_NAME"))                                                                       \
     CONFIG(STRING, DD_ENV, "")                                                                                                        \
@@ -61,7 +61,9 @@ extern bool runtime_config_first_init;
     CONFIG(BOOL, DD_TRACE_ENABLED, "true")                                                                                            \
     CONFIG(CUSTOM(STRING), DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING, "safe", .parser = dd_parse_automated_user_events_tracking)       \
     CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, "")                                                                          \
-    CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, "")
+    CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, "")                                                                          \
+    CONFIG(BOOL, DD_EXPERIMENTAL_API_SECURITY_ENABLED, "false")                                                                       \
+    CONFIG(DOUBLE, DD_API_SECURITY_REQUEST_SAMPLE_RATE, "0.1")
 // clang-format on
 
 #define CALIAS CONFIG

appsec/src/extension/ddappsec.c

@@ -247,13 +247,7 @@ static PHP_RINIT_FUNCTION(ddappsec)
     pthread_once(&_rinit_once_control, _rinit_once);
     zai_config_rinit();
 
-    //_check_enabled should be run only once. However, pthread_once approach
-    // does not work with ZTS.
-    if (DDAPPSEC_G(enabled) == NOT_CONFIGURED) {
-        mlog_g(
-            dd_log_trace, "Enabled not configured, computing enabled status");
-        _check_enabled();
-    }
+    _check_enabled();
 
     if (DDAPPSEC_G(enabled_by_configuration) == DISABLED) {
         return SUCCESS;
@@ -446,11 +440,11 @@ static void _check_enabled()
 {
     if (!get_global_DD_APPSEC_TESTING() &&
         (!dd_trace_enabled() || strcmp(sapi_module.name, "cli") == 0 ||
-            sapi_module.phpinfo_as_text)) {
+            (sapi_module.phpinfo_as_text != 0))) {
         DDAPPSEC_G(enabled_by_configuration) = DISABLED;
     } else if (!dd_is_config_using_default(DDAPPSEC_CONFIG_DD_APPSEC_ENABLED)) {
         DDAPPSEC_G(enabled_by_configuration) =
-            get_global_DD_APPSEC_ENABLED() ? ENABLED : DISABLED;
+            get_DD_APPSEC_ENABLED() ? ENABLED : DISABLED;
     } else {
         DDAPPSEC_G(enabled_by_configuration) = NOT_CONFIGURED;
     };

appsec/src/extension/ddtrace.c

@@ -6,6 +6,7 @@
 #include "ddtrace.h"
 #include <fcntl.h>
 #include <unistd.h>
+#include <zend_API.h>
 
 #include "configuration.h"
 #include "ddappsec.h"
@@ -19,13 +20,14 @@ static int (*_orig_ddtrace_shutdown)(SHUTDOWN_FUNC_ARGS);
 static int _mod_type;
 static int _mod_number;
 static const char *_mod_version;
-static bool _ddtrace_enabled;
+static bool _ddtrace_loaded;
 static zend_string *_ddtrace_root_span_fname;
 static zend_string *_meta_propname;
 static zend_string *_metrics_propname;
 static THREAD_LOCAL_ON_ZTS bool _suppress_ddtrace_rshutdown;
 static THREAD_LOCAL_ON_ZTS zval *_span_meta;
 static THREAD_LOCAL_ON_ZTS zval *_span_metrics;
+static uint8_t *_ddtrace_runtime_id = NULL;
 
 static zend_module_entry *_find_ddtrace_module(void);
 static int _ddtrace_rshutdown_testing(SHUTDOWN_FUNC_ARGS);
@@ -34,6 +36,8 @@ static void _register_testing_objects(void);
 static zval *(*nullable _ddtrace_root_span_get_meta)();
 static zval *(*nullable _ddtrace_root_span_get_metrics)();
 static void (*nullable _ddtrace_close_all_spans_and_flush)();
+static void (*nullable _ddtrace_set_priority_sampling_on_root)(
+    zend_long priority, enum dd_sampling_mechanism mechanism);
 
 static void dd_trace_load_symbols(void)
 {
@@ -70,16 +74,25 @@ static void dd_trace_load_symbols(void)
             dlerror());
     }
 
+    _ddtrace_runtime_id = dlsym(handle, "ddtrace_runtime_id");
+    if (_ddtrace_runtime_id == NULL) {
+        // NOLINTNEXTLINE(concurrency-mt-unsafe)
+        mlog(dd_log_debug, "Failed to load ddtrace_runtime_id: %s", dlerror());
+    }
+
+    _ddtrace_set_priority_sampling_on_root =
+        dlsym(handle, "ddtrace_set_priority_sampling_on_root");
+    if (_ddtrace_set_priority_sampling_on_root == NULL) {
+        // NOLINTNEXTLINE(concurrency-mt-unsafe)
+        mlog(dd_log_error,
+            "Failed to load ddtrace_set_priority_sampling_on_root: %s",
+            dlerror());
+    }
     dlclose(handle);
 }
 
 void dd_trace_startup()
 {
-    _ddtrace_enabled = false;
-    if (!get_global_DD_TRACE_ENABLED()) {
-        return;
-    }
-
     _ddtrace_root_span_fname = zend_string_init_interned(
         LSTRARG("ddtrace\\root_span"), 1 /* permanent */);
     _meta_propname = zend_string_init_interned(LSTRARG("meta"), 1);
@@ -90,14 +103,16 @@ void dd_trace_startup()
     }
 
     zend_module_entry *mod = _find_ddtrace_module();
-    if (!mod) {
+    if (mod == NULL) {
         mlog(dd_log_debug, "Cannot find ddtrace extension");
+        _ddtrace_loaded = false;
         return;
     }
+
+    _ddtrace_loaded = true;
     _mod_type = mod->type;
     _mod_number = mod->module_number;
     _mod_version = mod->version;
-    _ddtrace_enabled = true;
 
     dd_trace_load_symbols();
 
@@ -112,7 +127,7 @@ void dd_trace_rinit()
     _span_meta = NULL;
     _span_metrics = NULL;
     // DDTrace might not be loaded during tests
-    if (!_ddtrace_enabled) {
+    if (!dd_trace_enabled()) {
         return;
     }
     _span_meta = dd_trace_root_span_get_meta();
@@ -151,7 +166,8 @@ static int _ddtrace_rshutdown_testing(SHUTDOWN_FUNC_ARGS)
 
 const char *nullable dd_trace_version() { return _mod_version; }
 
-bool dd_trace_enabled() { return _ddtrace_enabled; }
+bool dd_trace_loaded() { return _ddtrace_loaded; }
+bool dd_trace_enabled() { return _ddtrace_loaded && get_DD_TRACE_ENABLED(); }
 
 bool dd_trace_root_span_add_tag(zend_string *nonnull tag, zval *nonnull value)
 {
@@ -245,6 +261,44 @@ zval *nullable dd_trace_root_span_get_metrics()
     return _ddtrace_root_span_get_metrics();
 }
 
+// NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers)
+zend_string *nullable dd_trace_get_formatted_runtime_id(bool persistent)
+{
+    if (_ddtrace_runtime_id == NULL) {
+        return NULL;
+    }
+
+    zend_string *encoded_id = zend_string_alloc(36, persistent);
+
+    size_t length = sprintf(ZSTR_VAL(encoded_id),
+        "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+        _ddtrace_runtime_id[0], _ddtrace_runtime_id[1], _ddtrace_runtime_id[2],
+        _ddtrace_runtime_id[3], _ddtrace_runtime_id[4], _ddtrace_runtime_id[5],
+        _ddtrace_runtime_id[6], _ddtrace_runtime_id[7], _ddtrace_runtime_id[8],
+        _ddtrace_runtime_id[9], _ddtrace_runtime_id[10],
+        _ddtrace_runtime_id[11], _ddtrace_runtime_id[12],
+        _ddtrace_runtime_id[13], _ddtrace_runtime_id[14],
+        _ddtrace_runtime_id[15]);
+
+    if (length != 36) {
+        zend_string_free(encoded_id);
+        encoded_id = NULL;
+    }
+
+    return encoded_id;
+}
+// NOLINTEND(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers)
+
+void dd_trace_set_priority_sampling_on_root(
+    zend_long priority, enum dd_sampling_mechanism mechanism)
+{
+    if (_ddtrace_set_priority_sampling_on_root == NULL) {
+        return;
+    }
+
+    _ddtrace_set_priority_sampling_on_root(priority, mechanism);
+}
+
 static PHP_FUNCTION(datadog_appsec_testing_ddtrace_rshutdown)
 {
     if (zend_parse_parameters_none() == FAILURE) {
@@ -309,13 +363,30 @@ static PHP_FUNCTION(datadog_appsec_testing_root_span_get_metrics) // NOLINT
     }
 }
 
+static PHP_FUNCTION(datadog_appsec_testing_get_formatted_runtime_id) // NOLINT
+{
+    if (zend_parse_parameters_none() == FAILURE) {
+        RETURN_FALSE;
+    }
+
+    zend_string *id = dd_trace_get_formatted_runtime_id(false);
+    if (id != NULL) {
+        RETURN_STR(id);
+    }
+    RETURN_EMPTY_STRING();
+}
+
 // clang-format off
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(void_ret_bool_arginfo, 0, 0, _IS_BOOL, 0)
 ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(void_ret_nullable_array, 0, 0, IS_ARRAY, 1)
 ZEND_END_ARG_INFO()
 
+    ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(void_ret_nullable_string, 0, 0, IS_STRING, 1)
+ZEND_END_ARG_INFO()
+
+
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_root_span_add_tag, 0, 0, _IS_BOOL, 2)
 ZEND_ARG_TYPE_INFO(0, tag, IS_STRING, 0)
 ZEND_ARG_TYPE_INFO(0, value, IS_STRING, 0)
@@ -326,6 +397,7 @@ static const zend_function_entry functions[] = {
     ZEND_RAW_FENTRY(DD_TESTING_NS "root_span_add_tag", PHP_FN(datadog_appsec_testing_root_span_add_tag), arginfo_root_span_add_tag, 0)
     ZEND_RAW_FENTRY(DD_TESTING_NS "root_span_get_meta", PHP_FN(datadog_appsec_testing_root_span_get_meta), void_ret_nullable_array, 0)
     ZEND_RAW_FENTRY(DD_TESTING_NS "root_span_get_metrics", PHP_FN(datadog_appsec_testing_root_span_get_metrics), void_ret_nullable_array, 0)
+    ZEND_RAW_FENTRY(DD_TESTING_NS "get_formatted_runtime_id", PHP_FN(datadog_appsec_testing_get_formatted_runtime_id), void_ret_nullable_string, 0)
     PHP_FE_END
 };
 // clang-format on

appsec/src/extension/ddtrace.h

@@ -9,6 +9,19 @@
 #include <stdbool.h>
 #include <zend.h>
 
+static const int PRIORITY_SAMPLING_AUTO_KEEP = 1;
+static const int PRIORITY_SAMPLING_AUTO_REJECT = 0;
+static const int PRIORITY_SAMPLING_USER_KEEP = 2;
+static const int PRIORITY_SAMPLING_USER_REJECT = -1;
+
+enum dd_sampling_mechanism {
+    DD_MECHANISM_DEFAULT = 0,
+    DD_MECHANISM_AGENT_RATE = 1,
+    DD_MECHANISM_REMOTE_RATE = 2,
+    DD_MECHANISM_RULE = 3,
+    DD_MECHANISM_MANUAL = 4,
+};
+
 typedef zend_object root_span_t;
 
 void dd_trace_startup(void);
@@ -17,7 +30,10 @@ void dd_trace_shutdown(void);
 void dd_trace_rinit(void);
 // Returns the tracer version
 const char *nullable dd_trace_version(void);
+// This function should only be used in RINIT
 bool dd_trace_enabled(void);
+// This function should be used before loading tracer symbols
+bool dd_trace_loaded(void);
 
 // increases the refcount of tag, but not value (like zval_hash_add)
 // however, it destroy value if the operation fails (unlike zval_hash_add)
@@ -33,3 +49,8 @@ void dd_trace_close_all_spans_and_flush(void);
 // It is ready for modification, with refcount == 1
 zval *nullable dd_trace_root_span_get_meta(void);
 zval *nullable dd_trace_root_span_get_metrics(void);
+zend_string *nullable dd_trace_get_formatted_runtime_id(bool persistent);
+
+// Set sampling priority on root span
+void dd_trace_set_priority_sampling_on_root(zend_long priority,
+    enum dd_sampling_mechanism mechanism);