ISSUES-5436 support custom http

programs/server/CMakeLists.txt

@@ -4,14 +4,13 @@ set(CLICKHOUSE_SERVER_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/PingRequestHandler.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusMetricsWriter.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusRequestHandler.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/StaticRequestHandler.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp
-   )
+)
 
 set(CLICKHOUSE_SERVER_SOURCES
     ${CLICKHOUSE_SERVER_SOURCES}

programs/server/HTTPHandler.cpp

@@ -1,12 +1,16 @@
 #include "HTTPHandler.h"
 
+#include "HTTPHandlerFactory.h"
+#include "HTTPHandlerRequestFilter.h"
+
 #include <chrono>
 #include <iomanip>
 #include <Poco/File.h>
 #include <Poco/Net/HTTPBasicCredentials.h>
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerRequestImpl.h>
 #include <Poco/Net/HTTPServerResponse.h>
+#include <Poco/Net/HTTPRequestHandlerFactory.h>
 #include <Poco/Net/NetException.h>
 #include <ext/scope_guard.h>
 #include <Core/ExternalTable.h>
@@ -32,6 +36,7 @@
 #include <IO/WriteBufferFromTemporaryFile.h>
 #include <DataStreams/IBlockInputStream.h>
 #include <Interpreters/executeQuery.h>
+#include <Interpreters/QueryParameterVisitor.h>
 #include <Common/typeid_cast.h>
 #include <Poco/Net/HTTPStream.h>
 
@@ -54,6 +59,7 @@ namespace ErrorCodes
     extern const int CANNOT_PARSE_DATETIME;
     extern const int CANNOT_PARSE_NUMBER;
     extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_COMPILE_REGEXP;
 
     extern const int UNKNOWN_ELEMENT_IN_AST;
     extern const int UNKNOWN_TYPE_OF_AST_NODE;
@@ -78,6 +84,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_FORMAT;
     extern const int UNKNOWN_DATABASE_ENGINE;
     extern const int UNKNOWN_TYPE_OF_QUERY;
+    extern const int NO_ELEMENTS_IN_CONFIG;
 
     extern const int QUERY_IS_TOO_LARGE;
 
@@ -204,9 +211,9 @@ void HTTPHandler::pushDelayedResults(Output & used_output)
 }
 
 
-HTTPHandler::HTTPHandler(IServer & server_)
+HTTPHandler::HTTPHandler(IServer & server_, const std::string & name)
     : server(server_)
-    , log(&Logger::get("HTTPHandler"))
+    , log(&Logger::get(name))
 {
     server_display_name = server.config().getString("display_name", getFQDNOrHostName());
 }
@@ -226,13 +233,6 @@ void HTTPHandler::processQuery(
 
     std::istream & istr = request.stream();
 
-    /// Part of the query can be passed in the 'query' parameter and the rest in the request body
-    /// (http method need not necessarily be POST). In this case the entire query consists of the
-    /// contents of the 'query' parameter, a line break and the request body.
-    std::string query_param = params.get("query", "");
-    if (!query_param.empty())
-        query_param += '\n';
-
     /// The user and password can be passed by headers (similar to X-Auth-*),
     /// which is used by load balancers to pass authentication information.
     std::string user = request.get("X-ClickHouse-User", "");
@@ -390,8 +390,6 @@ void HTTPHandler::processQuery(
         used_output.out_maybe_delayed_and_compressed = used_output.out_maybe_compressed;
     }
 
-    std::unique_ptr<ReadBuffer> in_param = std::make_unique<ReadBufferFromString>(query_param);
-
     std::unique_ptr<ReadBuffer> in_post_raw = std::make_unique<ReadBufferFromIStream>(istr);
 
     /// Request body can be compressed using algorithm specified in the Content-Encoding header.
@@ -413,7 +411,7 @@ void HTTPHandler::processQuery(
 
     std::unique_ptr<ReadBuffer> in;
 
-    static const NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
+    static const NameSet reserved_param_names{"compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
         "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"};
 
     Names reserved_param_suffixes;
@@ -478,42 +476,21 @@ void HTTPHandler::processQuery(
         else if (param_could_be_skipped(key))
         {
         }
-        else if (startsWith(key, "param_"))
-        {
-            /// Save name and values of substitution in dictionary.
-            const String parameter_name = key.substr(strlen("param_"));
-            context.setQueryParameter(parameter_name, value);
-        }
         else
         {
-            /// All other query parameters are treated as settings.
-            settings_changes.push_back({key, value});
+            /// Other than query parameters are treated as settings.
+            if (!customizeQueryParam(context, key, value))
+                settings_changes.push_back({key, value});
         }
     }
 
     /// For external data we also want settings
     context.checkSettingsConstraints(settings_changes);
     context.applySettingsChanges(settings_changes);
 
-    /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
-    std::string full_query;
-
-    /// Support for "external data for query processing".
-    if (has_external_data)
-    {
-        ExternalTablesHandler handler(context, params);
-        params.load(request, istr, handler);
-
-        /// Params are of both form params POST and uri (GET params)
-        for (const auto & it : params)
-            if (it.first == "query")
-                full_query += it.second;
-
-        in = std::make_unique<ReadBufferFromString>(full_query);
-    }
-    else
-        in = std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
-
+    const auto & query = getQuery(request, params, context);
+    std::unique_ptr<ReadBuffer> in_param = std::make_unique<ReadBufferFromString>(query);
+    in = has_external_data ? std::move(in_param) : std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
 
     /// HTTP response compression is turned on only if the client signalled that they support it
     /// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on.
@@ -593,7 +570,7 @@ void HTTPHandler::processQuery(
         });
     }
 
-    customizeContext(context);
+    customizeContext(request, context);
 
     executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context,
         [&response] (const String & current_query_id, const String & content_type, const String & format, const String & timezone)
@@ -731,5 +708,190 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
     }
 }
 
+DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string & param_name_)
+    : HTTPHandler(server_, "DynamicQueryHandler"), param_name(param_name_)
+{
+}
+
+bool DynamicQueryHandler::customizeQueryParam(Context & context, const std::string & key, const std::string & value)
+{
+    if (key == param_name)
+        return true;    /// do nothing
+
+    if (startsWith(key, "param_"))
+    {
+        /// Save name and values of substitution in dictionary.
+        const String parameter_name = key.substr(strlen("param_"));
+        context.setQueryParameter(parameter_name, value);
+        return true;
+    }
+
+    return false;
+}
+
+std::string DynamicQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context)
+{
+
+    if (likely(!startsWith(request.getContentType(), "multipart/form-data")))
+    {
+        /// Part of the query can be passed in the 'query' parameter and the rest in the request body
+        /// (http method need not necessarily be POST). In this case the entire query consists of the
+        /// contents of the 'query' parameter, a line break and the request body.
+        std::string query_param = params.get(param_name, "");
+        return query_param.empty() ? query_param : query_param + "\n";
+    }
+
+    /// Support for "external data for query processing".
+    /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
+    ExternalTablesHandler handler(context, params);
+    params.load(request, request.stream(), handler);
+
+    std::string full_query;
+    /// Params are of both form params POST and uri (GET params)
+    for (const auto & it : params)
+        if (it.first == param_name)
+            full_query += it.second;
+
+    return full_query;
+}
+
+PredefineQueryHandler::PredefineQueryHandler(
+    IServer & server, const NameSet & receive_params_, const std::string & predefine_query_
+    , const std::optional<String> & url_regex_, const std::unordered_map<String, String> & header_name_with_regex_)
+    : HTTPHandler(server, "PredefineQueryHandler"), receive_params(receive_params_), predefine_query(predefine_query_)
+    , url_regex(url_regex_), header_name_with_capture_regex(header_name_with_regex_)
+{
+}
+
+bool PredefineQueryHandler::customizeQueryParam(Context & context, const std::string & key, const std::string & value)
+{
+    if (receive_params.count(key))
+    {
+        context.setQueryParameter(key, value);
+        return true;
+    }
+
+    return false;
+}
+
+void PredefineQueryHandler::customizeContext(Poco::Net::HTTPServerRequest & request, DB::Context & context)
+{
+    /// If in the configuration file, the handler's header is regex and contains named capture group
+    /// We will extract regex named capture groups as query parameters
+
+    const auto & set_query_params = [&](const char * begin, const char * end, const std::string & regex)
+    {
+        auto compiled_regex = std::make_shared<re2_st::RE2>(regex);
+
+        if (!compiled_regex->ok())
+            throw Exception("cannot compile re2: " + regex + " for routing_rule, error: " + compiled_regex->error() +
+                ". Look at https://github.com/google/re2/wiki/Syntax for reference.", ErrorCodes::CANNOT_COMPILE_REGEXP);
+
+        int num_captures = compiled_regex->NumberOfCapturingGroups() + 1;
+
+        re2_st::StringPiece matches[num_captures];
+        re2_st::StringPiece input(begin, end - begin);
+        if (compiled_regex->Match(input, 0, end - begin, re2_st::RE2::Anchor::ANCHOR_BOTH, matches, num_captures))
+        {
+            for (const auto & [capturing_name, capturing_index] : compiled_regex->NamedCapturingGroups())
+            {
+                const auto & capturing_value = matches[capturing_index];
+
+                if (capturing_value.data())
+                    context.setQueryParameter(capturing_name, String(capturing_value.data(), capturing_value.size()));
+            }
+        }
+    };
+
+    if (url_regex)
+    {
+        const auto & uri = request.getURI();
+        set_query_params(uri.data(), find_first_symbols<'?'>(uri.data(), uri.data() + uri.size()), *url_regex);
+    }
+
+    for (const auto & [header_name, regex] : header_name_with_capture_regex)
+    {
+        const auto & header_value = request.get(header_name);
+        set_query_params(header_value.data(), header_value.data() + header_value.size(), regex);
+    }
+}
+
+std::string PredefineQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context)
+{
+    if (unlikely(startsWith(request.getContentType(), "multipart/form-data")))
+    {
+        /// Support for "external data for query processing".
+        ExternalTablesHandler handler(context, params);
+        params.load(request, request.stream(), handler);
+    }
+
+    return predefine_query;
+}
+
+Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & server, const std::string & config_prefix)
+{
+    std::string query_param_name = server.config().getString(config_prefix + ".handler.query_param_name", "query");
+    return addFiltersFromConfig(new RoutingRuleHTTPHandlerFactory<DynamicQueryHandler>(server, std::move(query_param_name)), server.config(), config_prefix);
+}
+
+static inline bool capturingNamedQueryParam(NameSet receive_params, const std::string & expression)
+{
+    auto compiled_regex = std::make_shared<re2_st::RE2>(expression);
+
+    if (!compiled_regex->ok())
+        throw Exception("Cannot compile re2: " + expression + " for routing_rule, error: " +
+            compiled_regex->error() + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", ErrorCodes::CANNOT_COMPILE_REGEXP);
+
+    const auto & capturing_names = compiled_regex->NamedCapturingGroups();
+    return std::count_if(capturing_names.begin(), capturing_names.end(), [&](const auto & iterator)
+    {
+        return std::count_if(receive_params.begin(), receive_params.end(),
+            [&](const auto & param_name) { return param_name == iterator.first; });
+    });
+}
+
+Poco::Net::HTTPRequestHandlerFactory * createPredefineHandlerFactory(IServer & server, const std::string & config_prefix)
+{
+    Poco::Util::AbstractConfiguration & configuration = server.config();
+
+    if (!configuration.has(config_prefix + ".handler.query"))
+        throw Exception("There is no path '" + config_prefix + ".handler.query" + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
+
+    std::string predefine_query = configuration.getString(config_prefix + ".handler.query");
+    NameSet analyze_receive_params = analyzeReceiveQueryParams(predefine_query);
+
+    std::unordered_map<String, String> headers_name_with_regex;
+    Poco::Util::AbstractConfiguration::Keys headers_name;
+    configuration.keys(config_prefix + ".headers", headers_name);
+
+    for (const auto & header_name : headers_name)
+    {
+        auto expression = configuration.getString(config_prefix + ".headers." + header_name);
+
+        if (!startsWith(expression, "regex:"))
+            continue;
+
+        expression = expression.substr(6);
+        if (capturingNamedQueryParam(analyze_receive_params, expression))
+            headers_name_with_regex.emplace(std::make_pair(header_name, expression));
+    }
+
+    if (configuration.has(config_prefix + ".url"))
+    {
+        auto url_expression = configuration.getString(config_prefix + ".url");
+
+        if (startsWith(url_expression, "regex:"))
+            url_expression = url_expression.substr(6);
+
+        if (capturingNamedQueryParam(analyze_receive_params, url_expression))
+            return addFiltersFromConfig(new RoutingRuleHTTPHandlerFactory<PredefineQueryHandler>(
+                server, std::move(analyze_receive_params), std::move(predefine_query), std::optional<String>(url_expression),
+                std::move(headers_name_with_regex)), configuration, config_prefix);
+    }
+
+    return addFiltersFromConfig(new RoutingRuleHTTPHandlerFactory<PredefineQueryHandler>(
+        server, std::move(analyze_receive_params), std::move(predefine_query), std::optional<String>{} ,std::move(headers_name_with_regex)),
+        configuration, config_prefix);
+}
 
 }