From b25ea3ef3e62590f6cc0b0c23b5dc28016143266 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Wed, 31 Mar 2021 10:20:15 +0900 Subject: [PATCH] Pre-warming TLS Tunnel --- .gitignore | 1 + doc/admin-guide/files/records.config.en.rst | 24 + doc/admin-guide/files/sni.yaml.en.rst | 21 + doc/admin-guide/layer-4-routing.en.rst | 48 + .../monitoring/statistics/core/ssl.en.rst | 42 + doc/uml/l4-pre-warming-overview.uml | 32 + iocore/eventsystem/I_EThread.h | 3 + iocore/eventsystem/I_Thread.h | 1 + iocore/net/P_SNIActionPerformer.h | 12 +- iocore/net/P_SSLNetVConnection.h | 23 +- iocore/net/SSLSNIConfig.cc | 8 +- iocore/net/YamlSNIConfig.cc | 64 +- iocore/net/YamlSNIConfig.h | 16 + iocore/net/libinknet_stub.cc | 10 + mgmt/RecordsConfig.cc | 8 + proxy/http/HttpConfig.h | 2 + proxy/http/HttpProxyServerMain.cc | 3 + proxy/http/HttpSM.cc | 54 +- proxy/http/HttpSM.h | 2 + proxy/http/Makefile.am | 16 +- proxy/http/PreWarmAlgorithm.h | 122 ++ proxy/http/PreWarmConfig.cc | 75 ++ proxy/http/PreWarmConfig.h | 57 + proxy/http/PreWarmManager.cc | 1170 +++++++++++++++++ proxy/http/PreWarmManager.h | 341 +++++ proxy/http/unit_tests/test_PreWarm.cc | 223 ++++ src/traffic_quic/traffic_quic.cc | 8 + 27 files changed, 2367 insertions(+), 19 deletions(-) create mode 100644 doc/uml/l4-pre-warming-overview.uml create mode 100644 proxy/http/PreWarmAlgorithm.h create mode 100644 proxy/http/PreWarmConfig.cc create mode 100644 proxy/http/PreWarmConfig.h create mode 100644 proxy/http/PreWarmManager.cc create mode 100644 proxy/http/PreWarmManager.h create mode 100644 proxy/http/unit_tests/test_PreWarm.cc diff --git a/.gitignore b/.gitignore index 801e96e33f6..0de925609c6 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,7 @@ proxy/http/remap/test_PluginDso proxy/http/remap/test_PluginFactory proxy/http/remap/test_RemapPluginInfo proxy/http/test_proxy_http +proxy/http/test_PreWarm proxy/http/remap/test_* proxy/http2/test_libhttp2 proxy/http2/test_Http2DependencyTree diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index b73c834b4c2..03e9e294b0d 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3949,6 +3949,30 @@ SNI Routing Frequency of checking the activity of SNI Routing Tunnel. Set to ``0`` to disable monitoring of the activity of the SNI tunnels. The feature is disabled by default. +.. ts:cv:: CONFIG proxy.config.tunnel.prewarm INT 0 + + Enable :ref:`pre-warming-tls-tunnel`. The feature is disabled by default. + +.. ts:cv:: CONFIG proxy.config.tunnel.prewarm.max_stats_size INT 100 + + Max size of :ref:`dynamic stats for Pre-warming TLS Tunnel `. + +.. ts:cv:: CONFIG proxy.config.tunnel.prewarm.algorithm INT 2 + + Version of pre-warming algorithm. + + ===== ====================================================================== + Value Description + ===== ====================================================================== + ``1`` Periodical pre-warming only + ``2`` Event based pre-warming + Periodical pre-warming + ===== ====================================================================== + +.. ts:cv:: CONFIG proxy.config.tunnel.prewarm.event_period INT 1000 + :units: milliseconds + + Frequency of periodical pre-warming in milli-seconds. + OCSP Stapling Configuration =========================== diff --git a/doc/admin-guide/files/sni.yaml.en.rst b/doc/admin-guide/files/sni.yaml.en.rst index c1ff0a6709a..7a04e4b152a 100644 --- a/doc/admin-guide/files/sni.yaml.en.rst +++ b/doc/admin-guide/files/sni.yaml.en.rst @@ -159,6 +159,27 @@ tunnel_alpn Inbound List of ALPN Protocol Ids for Partial Blind This only works with ``partial_blind_route``. ========================= ========= ======================================================================================== +Pre-warming TLS Tunnel +---------------------- + +=============================== ======================================================================================== +Key Meaning +=============================== ======================================================================================== +tunnel_prewarm Override :ts:cv:`proxy.config.tunnel.prewarm` in records.config. + +tunnel_prewarm_srv Enable SRV record lookup on pre-warming. Default is ``false``. + +tunnel_prewarm_rate Rate of how many connections to pre-warm. Default is ``1.0``. + +tunnel_prewarm_min Minimum number of pre-warming queue size (per thread). Default is ``0``. + +tunnel_prewarm_max Maximum number of pre-warming queue size (per thread). Default is ``-1`` (unlimited). + +tunnel_prewarm_connect_timeout Timeout for TCP/TLS handshake (in seconds). + +tunnel_prewarm_inactive_timeout Inactive timeout for connections in the pool (in seconds). +=============================== ======================================================================================== + Client verification, via ``verify_client``, corresponds to setting :ts:cv:`proxy.config.ssl.client.certification_level` for this connection as noted below. diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst index b6528973a2d..2485a43ab0a 100644 --- a/doc/admin-guide/layer-4-routing.en.rst +++ b/doc/admin-guide/layer-4-routing.en.rst @@ -129,3 +129,51 @@ tunneled connection like this, the only transaction hooks that will be triggered :c:macro:`TS_HTTP_TXN_START_HOOK` and :c:macro:`TS_HTTP_TXN_CLOSE_HOOK`. In addition, because |TS| does not terminate (and therefore does not decrypt) the connection, it cannot be cached or served from cache. + +.. _pre-warming-tls-tunnel: + +Pre-warming TLS Tunnel +====================== + +Pre-warming TLS Tunnel reduces the latency of TLS connections (``forward_route`` and ``partial_blind_route`` type SNI +Routing). When this feature is enabled, each ET_NET thread makes TLS connections pool per routing type, SNI, and ALPN. + +.. figure:: ../uml/images/l4-pre-warming-overview.svg + :align: center + +Stats for connection pools are registered dynamically on start up. Details in :ref:`pre-warming-tls-tunnel-stats`. + +Examples +-------- + +.. code:: yaml + + sni: + - fqdn: foo.com + http2: off + partial_blind_route: bar.com + client_sni_policy: server_name + tunnel_prewarm: true + tunnel_prewarm_connect_timeout: 10 + tunnel_prewarm_inactive_timeout: 150 + tunnel_prewarm_max: 100 + tunnel_prewarm_min: 10 + tunnel_alpn: + - h2 + +.. code:: + + proxy.process.tunnel.prewarm.bar.com:443.tls.current_init 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.current_open 10 + proxy.process.tunnel.prewarm.bar.com:443.tls.total_hit 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.total_miss 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.total_handshake_time 1106250000 + proxy.process.tunnel.prewarm.bar.com:443.tls.total_handshake_count 10 + proxy.process.tunnel.prewarm.bar.com:443.tls.total_retry 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.current_init 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.current_open 10 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_hit 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_miss 0 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_handshake_time 1142368000 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_handshake_count 10 + proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_retry 0 diff --git a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst index 4f51c6c79e3..e18bae11c6f 100644 --- a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst @@ -237,3 +237,45 @@ SSL/TLS :type: gauge A gauge of current active SNI Routing Tunnels. + +.. _pre-warming-tls-tunnel-stats: + +Pre-warming TLS Tunnel +---------------------- + +Stats for Pre-warming TLS Tunnel is registered dynamically. The ``POOL`` in below represents combination of ``..``. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.current_init integer + :type: gauge + + Represents the current number of initializing connections in the pool. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.current_open integer + :type: gauge + + Represents the current number of established connections in the pool. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_hit integer + :type: counter + + Represents the total number of pre-warmed connection is used. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_miss integer + :type: counter + + Represents the total number of pre-warmed connection is not used. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_handshake_time integer + :type: counter + + Represents the total number of handshake duration of pre-warming. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_handshake_count integer + :type: counter + + Represents the total number of handshake time of pre-warming. + +.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_retry integer + :type: counter + + Represents the total number of pre-warming retry. diff --git a/doc/uml/l4-pre-warming-overview.uml b/doc/uml/l4-pre-warming-overview.uml new file mode 100644 index 00000000000..5367de03031 --- /dev/null +++ b/doc/uml/l4-pre-warming-overview.uml @@ -0,0 +1,32 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. + +@startuml +'title Pre-warm TLS Tunnel +skinparam sequenceMessageAlign direction +skinparam ParticipantPadding 75 +'skinparam monochrome reverse +skinparam backgroundColor #white +hide footbox + +participant client +participant proxy +participant origin_server + +group pre-warm connection pool + proxy -> origin_server : open connection + proxy -> origin_server : open connection + proxy -> origin_server : open connection + ... +end +... +group pool size > 0 + client -> proxy : open connection + hnote over proxy #white: use pre-warmed connection from pool + rnote over client, origin_server #lightgreen: TLS Partial Blind Tunnel +end +@enduml diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 49d74d896a3..8da21e65c5e 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -43,6 +43,8 @@ struct DiskHandler; struct EventIO; class ServerSessionPool; +class PreWarmQueue; + class Event; class Continuation; @@ -349,6 +351,7 @@ class EThread : public Thread Event *start_event = nullptr; ServerSessionPool *server_session_pool = nullptr; + PreWarmQueue *prewarm_queue = nullptr; /** Default handler used until it is overridden. diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 8037d2a10c4..432d14aa7be 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -135,6 +135,7 @@ class Thread ProxyAllocator ioDataAllocator; ProxyAllocator ioAllocator; ProxyAllocator ioBlockAllocator; + ProxyAllocator preWarmSMAllocator; // From InkAPI (plugins wrappers) ProxyAllocator apiHookAllocator; ProxyAllocator INKContAllocator; diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 11e269196aa..7fc7924b606 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -98,8 +98,9 @@ class ControlH2 : public ActionItem class TunnelDestination : public ActionItem { public: - TunnelDestination(const std::string_view &dest, SNIRoutingType type, const std::vector &alpn) - : destination(dest), type(type), alpn_ids(alpn) + TunnelDestination(const std::string_view &dest, SNIRoutingType type, YamlSNIConfig::TunnelPreWarm prewarm, + const std::vector &alpn) + : destination(dest), type(type), tunnel_prewarm(prewarm), alpn_ids(alpn) { need_fix = (destination.find_first_of('$') != std::string::npos); } @@ -115,10 +116,10 @@ class TunnelDestination : public ActionItem // If needed, we will try to amend the tunnel destination. if (ctx._fqdn_wildcard_captured_groups && need_fix) { const auto &fixed_dst = replace_match_groups(destination, *ctx._fqdn_wildcard_captured_groups); - ssl_netvc->set_tunnel_destination(fixed_dst, type); + ssl_netvc->set_tunnel_destination(fixed_dst, type, tunnel_prewarm); Debug("ssl_sni", "Destination now is [%s], configured [%s], fqdn [%s]", fixed_dst.c_str(), destination.c_str(), servername); } else { - ssl_netvc->set_tunnel_destination(destination, type); + ssl_netvc->set_tunnel_destination(destination, type, tunnel_prewarm); Debug("ssl_sni", "Destination now is [%s], fqdn [%s]", destination.c_str(), servername); } @@ -202,7 +203,8 @@ class TunnelDestination : public ActionItem } std::string destination; - SNIRoutingType type = SNIRoutingType::NONE; + SNIRoutingType type = SNIRoutingType::NONE; + YamlSNIConfig::TunnelPreWarm tunnel_prewarm = YamlSNIConfig::TunnelPreWarm::UNSET; const std::vector &alpn_ids; bool need_fix; }; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 7c8b94ae1b6..491fa55460c 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -298,11 +298,13 @@ class SSLNetVConnection : public UnixNetVConnection, bool decrypt_tunnel() const; bool upstream_tls() const; SNIRoutingType tunnel_type() const; + YamlSNIConfig::TunnelPreWarm tunnel_prewarm() const; void - set_tunnel_destination(const std::string_view &destination, SNIRoutingType type) + set_tunnel_destination(const std::string_view &destination, SNIRoutingType type, YamlSNIConfig::TunnelPreWarm prewarm) { - _tunnel_type = type; + _tunnel_type = type; + _tunnel_prewarm = prewarm; auto pos = destination.find(":"); if (nullptr != tunnel_host) { @@ -484,10 +486,13 @@ class SSLNetVConnection : public UnixNetVConnection, HANDSHAKE_HOOKS_DONE } sslHandshakeHookState = HANDSHAKE_HOOKS_PRE; - int64_t redoWriteSize = 0; - char *tunnel_host = nullptr; - in_port_t tunnel_port = 0; - SNIRoutingType _tunnel_type = SNIRoutingType::NONE; + int64_t redoWriteSize = 0; + + char *tunnel_host = nullptr; + in_port_t tunnel_port = 0; + SNIRoutingType _tunnel_type = SNIRoutingType::NONE; + YamlSNIConfig::TunnelPreWarm _tunnel_prewarm = YamlSNIConfig::TunnelPreWarm::UNSET; + X509_STORE_CTX *verify_cert = nullptr; // Null-terminated string, or nullptr if there is no SNI server name. @@ -521,6 +526,12 @@ SSLNetVConnection::tunnel_type() const return _tunnel_type; } +inline YamlSNIConfig::TunnelPreWarm +SSLNetVConnection::tunnel_prewarm() const +{ + return _tunnel_prewarm; +} + /** Returns true if this vc was configured for forward_route or partial_blind_route */ diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index ae8227545b7..c134c82421d 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -30,6 +30,9 @@ ****************************************************************************/ #include "P_SSLSNI.h" + +#include "PreWarmManager.h" + #include "tscore/Diags.h" #include "tscore/SimpleTokenizer.h" #include "tscore/ink_memory.h" @@ -77,7 +80,8 @@ SNIConfigParams::loadSNIConfig() ai->actions.push_back(std::make_unique(item.protocol_mask)); } if (item.tunnel_destination.length() > 0) { - ai->actions.push_back(std::make_unique(item.tunnel_destination, item.tunnel_type, item.tunnel_alpn)); + ai->actions.push_back( + std::make_unique(item.tunnel_destination, item.tunnel_type, item.tunnel_prewarm, item.tunnel_alpn)); } if (!item.client_sni_policy.empty()) { ai->actions.push_back(std::make_unique(item.client_sni_policy)); @@ -200,6 +204,8 @@ SNIConfig::reconfigure() params->Initialize(); configid = configProcessor.set(configid, params); + + prewarmManager.reconfigure(); } SNIConfigParams * diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 2142e84cedc..cadeb77148d 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -130,6 +130,13 @@ std::set valid_sni_config_keys = {TS_fqdn, TS_forward_route, TS_partial_blind_route, TS_tunnel_alpn, + TS_tunnel_prewarm, + TS_tunnel_prewarm_min, + TS_tunnel_prewarm_max, + TS_tunnel_prewarm_rate, + TS_tunnel_prewarm_connect_timeout, + TS_tunnel_prewarm_inactive_timeout, + TS_tunnel_prewarm_srv, TS_verify_server_policy, TS_verify_server_properties, TS_client_cert, @@ -232,15 +239,64 @@ template <> struct convert { item.host_sni_policy = static_cast(policy); } + YamlSNIConfig::TunnelPreWarm t_prewarm = YamlSNIConfig::TunnelPreWarm::UNSET; + uint32_t t_min = item.tunnel_prewarm_min; + int32_t t_max = item.tunnel_prewarm_max; + double t_rate = item.tunnel_prewarm_rate; + uint32_t t_connect_timeout = item.tunnel_prewarm_connect_timeout; + uint32_t t_inactive_timeout = item.tunnel_prewarm_inactive_timeout; + bool t_srv = item.tunnel_prewarm_srv; + + if (node[TS_tunnel_prewarm]) { + auto is_prewarm_enabled = node[TS_tunnel_prewarm].as(); + if (is_prewarm_enabled) { + t_prewarm = YamlSNIConfig::TunnelPreWarm::ENABLED; + } else { + t_prewarm = YamlSNIConfig::TunnelPreWarm::DISABLED; + } + } + if (node[TS_tunnel_prewarm_min]) { + t_min = node[TS_tunnel_prewarm_min].as(); + } + if (node[TS_tunnel_prewarm_max]) { + t_max = node[TS_tunnel_prewarm_max].as(); + } + if (node[TS_tunnel_prewarm_rate]) { + t_rate = node[TS_tunnel_prewarm_rate].as(); + } + if (node[TS_tunnel_prewarm_connect_timeout]) { + t_connect_timeout = node[TS_tunnel_prewarm_connect_timeout].as(); + } + if (node[TS_tunnel_prewarm_inactive_timeout]) { + t_inactive_timeout = node[TS_tunnel_prewarm_inactive_timeout].as(); + } + if (node[TS_tunnel_prewarm_srv]) { + t_srv = node[TS_tunnel_prewarm_srv].as(); + } + if (node[TS_tunnel_route]) { item.tunnel_destination = node[TS_tunnel_route].as(); item.tunnel_type = SNIRoutingType::BLIND; } else if (node[TS_forward_route]) { - item.tunnel_destination = node[TS_forward_route].as(); - item.tunnel_type = SNIRoutingType::FORWARD; + item.tunnel_destination = node[TS_forward_route].as(); + item.tunnel_type = SNIRoutingType::FORWARD; + item.tunnel_prewarm = t_prewarm; + item.tunnel_prewarm_min = t_min; + item.tunnel_prewarm_max = t_max; + item.tunnel_prewarm_rate = t_rate; + item.tunnel_prewarm_connect_timeout = t_connect_timeout; + item.tunnel_prewarm_inactive_timeout = t_inactive_timeout; + item.tunnel_prewarm_srv = t_srv; } else if (node[TS_partial_blind_route]) { - item.tunnel_destination = node[TS_partial_blind_route].as(); - item.tunnel_type = SNIRoutingType::PARTIAL_BLIND; + item.tunnel_destination = node[TS_partial_blind_route].as(); + item.tunnel_type = SNIRoutingType::PARTIAL_BLIND; + item.tunnel_prewarm = t_prewarm; + item.tunnel_prewarm_min = t_min; + item.tunnel_prewarm_max = t_max; + item.tunnel_prewarm_rate = t_rate; + item.tunnel_prewarm_connect_timeout = t_connect_timeout; + item.tunnel_prewarm_inactive_timeout = t_inactive_timeout; + item.tunnel_prewarm_srv = t_srv; if (node[TS_tunnel_alpn]) { load_tunnel_alpn(item.tunnel_alpn, node[TS_tunnel_alpn]); diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index 5b10ff3f3ea..700e740cfc2 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -39,6 +39,13 @@ TSDECL(tunnel_route); TSDECL(forward_route); TSDECL(partial_blind_route); TSDECL(tunnel_alpn); +TSDECL(tunnel_prewarm); +TSDECL(tunnel_prewarm_min); +TSDECL(tunnel_prewarm_max); +TSDECL(tunnel_prewarm_rate); +TSDECL(tunnel_prewarm_connect_timeout); +TSDECL(tunnel_prewarm_inactive_timeout); +TSDECL(tunnel_prewarm_srv); TSDECL(verify_server_policy); TSDECL(verify_server_properties); TSDECL(verify_origin_server); @@ -55,6 +62,7 @@ struct YamlSNIConfig { enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED, UNSET }; enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3, UNSET }; enum class TLSProtocol : uint8_t { TLSv1 = 0, TLSv1_1, TLSv1_2, TLSv1_3, TLS_MAX = TLSv1_3 }; + enum class TunnelPreWarm : uint8_t { DISABLED = 0, ENABLED, UNSET }; YamlSNIConfig() {} @@ -77,6 +85,14 @@ struct YamlSNIConfig { unsigned long protocol_mask; std::vector tunnel_alpn{}; + bool tunnel_prewarm_srv = false; + uint32_t tunnel_prewarm_min = 0; + int32_t tunnel_prewarm_max = -1; + double tunnel_prewarm_rate = 1.0; + uint32_t tunnel_prewarm_connect_timeout = 0; + uint32_t tunnel_prewarm_inactive_timeout = 0; + TunnelPreWarm tunnel_prewarm = TunnelPreWarm::UNSET; + void EnableProtocol(YamlSNIConfig::TLSProtocol proto); }; diff --git a/iocore/net/libinknet_stub.cc b/iocore/net/libinknet_stub.cc index 74091db53cc..1c65bd33251 100644 --- a/iocore/net/libinknet_stub.cc +++ b/iocore/net/libinknet_stub.cc @@ -175,3 +175,13 @@ ProcessManager::signalManager(int, char const *) ink_assert(false); return; } + +#include "PreWarmManager.h" +void +PreWarmManager::reconfigure() +{ + ink_assert(false); + return; +} + +PreWarmManager prewarmManager; diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index fd00ca027fd..997d5fcabbb 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -987,6 +987,14 @@ static const RecordElement RecordsConfig[] = //########################################################################## {RECT_CONFIG, "proxy.config.tunnel.activity_check_period", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-100]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.tunnel.prewarm", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.tunnel.prewarm.max_stats_size", RECD_INT, "100", RECU_RESTART_TS, RR_NULL, RECC_INT, "[5-65536]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.tunnel.prewarm.event_period", RECD_INT, "1000", RECU_DYNAMIC, RR_NULL, RECC_INT, "[10-3600000]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.tunnel.prewarm.algorithm", RECD_INT, "2", RECU_DYNAMIC, RR_NULL, RECC_INT, "[1-2]", RECA_NULL} + , //########################################################################## //# diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 5f77ece86c0..31af2fa902c 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -864,6 +864,8 @@ struct HttpConfigParams : public ConfigInfo { class HttpConfig { public: + using scoped_config = ConfigProcessor::scoped_config; + static void startup(); static void reconfigure(); diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 1a38a8bd259..f5f1c3b7658 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -44,6 +44,7 @@ #include "P_QUICNextProtocolAccept.h" #include "http3/Http3SessionAccept.h" #endif +#include "PreWarmManager.h" #include @@ -383,6 +384,8 @@ start_HttpProxyServer() hook->invoke(TS_EVENT_LIFECYCLE_PORTS_READY, nullptr); hook = hook->next(); } + + prewarmManager.start(); } void diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 3b2bfd38ca5..337cebccd40 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -32,6 +32,8 @@ #include "HttpSessionManager.h" #include "P_Cache.h" #include "P_Net.h" +#include "PreWarmConfig.h" +#include "PreWarmManager.h" #include "StatPages.h" #include "Log.h" #include "LogAccess.h" @@ -378,6 +380,12 @@ HttpSM::cleanup() transform_cache_sm.mutex.clear(); magic = HTTP_SM_MAGIC_DEAD; debug_on = false; + + if (_prewarm_sm) { + _prewarm_sm->destroy(); + THREAD_FREE(_prewarm_sm, preWarmSMAllocator, this_ethread()); + _prewarm_sm = nullptr; + } } void @@ -5312,13 +5320,51 @@ HttpSM::do_http_server_open(bool raw) if (ssl_vc && raw) { tls_upstream = ssl_vc->upstream_tls(); _tunnel_type = ssl_vc->tunnel_type(); + // ALPN on TLS Partial Blind Tunnel - set negotiated ALPN id + int pid = SessionProtocolNameRegistry::INVALID; if (ssl_vc->tunnel_type() == SNIRoutingType::PARTIAL_BLIND) { - int pid = ssl_vc->get_negotiated_protocol_id(); + pid = ssl_vc->get_negotiated_protocol_id(); if (pid != SessionProtocolNameRegistry::INVALID) { opt.alpn_protos = SessionProtocolNameRegistry::convert_openssl_alpn_wire_format(pid); } } + + // + // Grab pre-warmed NetVConnection if possible + // + PreWarmConfig::scoped_config prewarm_conf; + bool use_prewarm = prewarm_conf->enabled; + + // override "proxy.config.tunnel.prewarm" by "tunnel_prewarm" in sni.yaml + if (YamlSNIConfig::TunnelPreWarm sni_use_prewarm = ssl_vc->tunnel_prewarm(); + sni_use_prewarm != YamlSNIConfig::TunnelPreWarm::UNSET) { + use_prewarm = static_cast(sni_use_prewarm); + } + + if (use_prewarm) { + // TODO: avoid copy of string -> make map key std::variant + PreWarm::SPtrConstDst dst = + std::make_shared(ssl_vc->get_tunnel_host(), ssl_vc->get_tunnel_port(), + tls_upstream ? SNIRoutingType::PARTIAL_BLIND : SNIRoutingType::FORWARD, pid); + + EThread *ethread = this_ethread(); + _prewarm_sm = ethread->prewarm_queue->dequeue(dst); + + if (_prewarm_sm != nullptr) { + NetVConnection *netvc = _prewarm_sm->move_netvc(); + ink_release_assert(_prewarm_sm->handler == &PreWarmSM::state_closed); + + SMDebug("http_ss", "using pre-warmed tunnel netvc=%p", netvc); + + t_state.current.attempts = 0; + + ink_release_assert(default_handler == HttpSM::default_handler); + handleEvent(NET_EVENT_OPEN, netvc); + return; + } + SMDebug("http_ss", "no pre-warmed tunnel"); + } } opt.local_port = ua_txn->get_outbound_port(); @@ -7011,6 +7057,12 @@ HttpSM::setup_blind_tunnel(bool send_response_hdr, IOBufferReader *initial) ua_raw_buffer_reader = nullptr; } + // if pre-warmed connection is used and it has data from origin server, foward it to ua + if (_prewarm_sm && _prewarm_sm->has_data_from_origin_server()) { + ink_release_assert(_prewarm_sm->handler == &PreWarmSM::state_closed); + client_response_hdr_bytes += to_ua_buf->write(_prewarm_sm->server_buf_reader()); + } + // Next order of business if copy the remaining data from the // header buffer into new buffer client_request_body_bytes += from_ua_buf->write(ua_txn->get_remote_reader()); diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 8154d4d7e1d..9dd737541c7 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -61,6 +61,7 @@ static size_t const HTTP_SERVER_RESP_HDR_BUFFER_INDEX = BUFFER_SIZE_INDEX_8K; class Http1ServerSession; class AuthHttpAdapter; +class PreWarmSM; class HttpSM; typedef int (HttpSM::*HttpSMHandler)(int event, void *data); @@ -665,6 +666,7 @@ class HttpSM : public Continuation, public PluginUserArgs int _client_transaction_priority_weight = -1, _client_transaction_priority_dependence = -1; bool _from_early_data = false; SNIRoutingType _tunnel_type = SNIRoutingType::NONE; + PreWarmSM *_prewarm_sm = nullptr; }; //// diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index a29d5155b52..53f983251c9 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -77,13 +77,15 @@ libhttp_a_SOURCES = \ HttpTransactHeaders.h \ HttpTunnel.cc \ HttpTunnel.h \ - ForwardedConfig.cc + ForwardedConfig.cc \ + PreWarmConfig.cc \ + PreWarmManager.cc if BUILD_TESTS libhttp_a_SOURCES += RegressionHttpTransact.cc endif -check_PROGRAMS = test_proxy_http +check_PROGRAMS = test_proxy_http test_PreWarm TESTS = $(check_PROGRAMS) @@ -111,6 +113,16 @@ test_proxy_http_LDADD = \ @HWLOC_LIBS@ \ @LIBCAP@ +test_PreWarm_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -I$(abs_top_srcdir)/tests/include + +test_PreWarm_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la + +test_PreWarm_SOURCES = \ + unit_tests/test_PreWarm.cc + clang-tidy-local: $(libhttp_a_SOURCES) $(noinst_HEADERS) $(CXX_Clang_Tidy) diff --git a/proxy/http/PreWarmAlgorithm.h b/proxy/http/PreWarmAlgorithm.h new file mode 100644 index 00000000000..8b1d240fcec --- /dev/null +++ b/proxy/http/PreWarmAlgorithm.h @@ -0,0 +1,122 @@ +/** @file + + Pre-Warming Pool Size Algorithm + + v1: periodical pre-warming only + v2: periodical pre-warming + event based pre-warming + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "tscore/ink_assert.h" +#include "tscore/ink_error.h" + +#include +#include + +namespace PreWarm +{ +enum class Algorithm { + V1 = 1, + V2, +}; + +inline PreWarm::Algorithm +algorithm_version(int i) +{ + switch (i) { + case 2: + return PreWarm::Algorithm::V2; + case 1: + return PreWarm::Algorithm::V1; + default: + ink_abort("unsupported version v=%d", i); + } +} + +/** + Periodical pre-warming for algorithm v1 + + Expand the pool size to @requested_size + + @params min : min connections (configured) + @params max : max connections (configured), -1 : unlimited + + @return how many connections needs to be pre-warmed for next period + */ +inline uint32_t +prewarm_size_v1_on_event_interval(uint32_t requested_size, uint32_t current_size, uint32_t min, int32_t max) +{ + uint32_t n = requested_size; + + // keep tunnel_min connections pre-warmed at least + n = std::max(n, min); + + if (max >= 0) { + n = std::min(n, static_cast(max)); + } + + if (current_size >= n) { + // we already have enough connections, don't need to open new connection + return 0; + } else { + n -= current_size; + } + + return n; +} + +/** + Periodical pre-warming for algorithm v2 + + Expand the pool size to @current_size + @miss * @rate. The event based pre-warming handles the hit cases. + + @params min : min connections (configured) + @params max : max connections (configured), -1 : unlimited + + @return how many connections needs to be pre-warmed for next period + */ +inline uint32_t +prewarm_size_v2_on_event_interval(uint32_t hit, uint32_t miss, uint32_t current_size, uint32_t min, int32_t max, double rate) +{ + if (hit + miss + current_size < min) { + // fallback to v1 to keep min size + return prewarm_size_v1_on_event_interval(hit + miss, current_size, min, max); + } + + // Reached limit - do nothing + if (max >= 0 && current_size >= static_cast(max)) { + return 0; + } + + // Add #miss connections to the pool + uint32_t n = miss * rate; + + // Check limit + if (max >= 0 && n + current_size > static_cast(max)) { + ink_release_assert(static_cast(max) > current_size); + n = max - current_size; + } + + return n; +} + +} // namespace PreWarm diff --git a/proxy/http/PreWarmConfig.cc b/proxy/http/PreWarmConfig.cc new file mode 100644 index 00000000000..f2149219810 --- /dev/null +++ b/proxy/http/PreWarmConfig.cc @@ -0,0 +1,75 @@ +/** @file + + Configs for PreWarming Tunnel + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "PreWarmConfig.h" +#include "PreWarmManager.h" + +//// +// PreWarmConfigParams +// +PreWarmConfigParams::PreWarmConfigParams() +{ + // RECU_RESTART_TS + REC_EstablishStaticConfigByte(enabled, "proxy.config.tunnel.prewarm"); + REC_EstablishStaticConfigInteger(max_stats_size, "proxy.config.tunnel.prewarm.max_stats_size"); + + // RECU_DYNAMIC + REC_ReadConfigInteger(event_period, "proxy.config.tunnel.prewarm.event_period"); + REC_ReadConfigInteger(algorithm, "proxy.config.tunnel.prewarm.algorithm"); +} + +//// +// PreWarmConfig +// +void +PreWarmConfig::startup() +{ + _config_update_handler = std::make_unique>(); + + // dynamic configs + _config_update_handler->attach("proxy.config.tunnel.prewarm.event_period"); + _config_update_handler->attach("proxy.config.tunnel.prewarm.algorithm"); + + reconfigure(); +} + +void +PreWarmConfig::reconfigure() +{ + PreWarmConfigParams *params = new PreWarmConfigParams(); + _config_id = configProcessor.set(_config_id, params); + + prewarmManager.reconfigure(); +} + +PreWarmConfigParams * +PreWarmConfig::acquire() +{ + return static_cast(configProcessor.get(_config_id)); +} + +void +PreWarmConfig::release(PreWarmConfigParams *params) +{ + configProcessor.release(_config_id, params); +} diff --git a/proxy/http/PreWarmConfig.h b/proxy/http/PreWarmConfig.h new file mode 100644 index 00000000000..12b22100f58 --- /dev/null +++ b/proxy/http/PreWarmConfig.h @@ -0,0 +1,57 @@ +/** @file + + Configs for PreWarming Tunnel + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "HttpConfig.h" + +struct PreWarmConfigParams : public ConfigInfo { + PreWarmConfigParams(); + + // noncopyable + PreWarmConfigParams(const HttpConfigParams &) = delete; + PreWarmConfigParams &operator=(const HttpConfigParams &) = delete; + + // Config Params + int8_t enabled = 0; + int8_t algorithm = 0; + int64_t event_period = 0; + int64_t max_stats_size = 0; +}; + +class PreWarmConfig +{ +public: + using scoped_config = ConfigProcessor::scoped_config; + + static void startup(); + + // ConfigUpdateContinuation interface + static void reconfigure(); + + // ConfigProcessor::scoped_config interface + static PreWarmConfigParams *acquire(); + static void release(PreWarmConfigParams *params); + +private: + inline static int _config_id = 0; + inline static std::unique_ptr> _config_update_handler; +}; diff --git a/proxy/http/PreWarmManager.cc b/proxy/http/PreWarmManager.cc new file mode 100644 index 00000000000..3a878d7c44e --- /dev/null +++ b/proxy/http/PreWarmManager.cc @@ -0,0 +1,1170 @@ +/** @file + + Pre-Warming NetVConnection + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "PreWarmManager.h" +#include "PreWarmConfig.h" + +#include "HttpConfig.h" +#include "P_SSLSNI.h" + +#include "tscpp/util/PostScript.h" + +#include + +#define PreWarmSMDebug(fmt, ...) Debug("prewarm_sm", "[%p] " fmt, this, ##__VA_ARGS__); +#define PreWarmSMVDebug(fmt, ...) Debug("v_prewarm_sm", "[%p] " fmt, this, ##__VA_ARGS__); + +ClassAllocator preWarmSMAllocator("preWarmSMAllocator"); +PreWarmManager prewarmManager; + +namespace +{ +using namespace std::literals; + +constexpr int DOWN_SERVER_TIMEOUT = 300; +constexpr size_t STAT_NAME_BUF_LEN = 1024; + +constexpr std::string_view SRV_TUNNEL_TCP = "_tunnel._tcp."sv; +constexpr std::string_view CLIENT_SNI_POLICY_SERVER_NAME = "server_name"sv; + +std::string_view +alpn_name_for_stat(int alpn_id) +{ + if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_1_0) { + return "http1_0"sv; + } else if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_1_1) { + return "http1_1"sv; + } else if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_2_0) { + return "http2"sv; + } else if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_3) { + return "http3"sv; + } else { + return "unknown"sv; + } +} + +void +parse_authority(std::string &fqdn, int32_t &port, std::string_view authority) +{ + if (auto pos = authority.find(":"); pos != std::string::npos) { + fqdn = authority.substr(0, pos); + port = static_cast(std::stoi(authority.substr(pos + 1).data())); + } else { + fqdn = authority; + port = -1; + } +} + +//// +// Stats +// +constexpr std::string_view STAT_NAME_PREFIX = "proxy.process.tunnel.prewarm"sv; + +struct StatEntry { + std::string_view name; + RecRawStatSyncCb cb; +}; + +// the order is the same as PreWarm::Stat +// clang-format off +constexpr StatEntry STAT_ENTRIES[] = { + {"current_init"sv, RecRawStatSyncSum}, + {"current_open"sv, RecRawStatSyncSum}, + {"total_hit"sv, RecRawStatSyncSum}, + {"total_miss"sv, RecRawStatSyncSum}, + {"total_handshake_time"sv, RecRawStatSyncSum}, + {"total_handshake_count"sv, RecRawStatSyncSum}, + {"total_retry"sv, RecRawStatSyncSum}, +}; +// clang-format on + +} // namespace + +//// +// PreWarmSM +// +PreWarmSM::PreWarmSM(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf, + const PreWarm::SPtrConstStatsIds &stats_ids) + : Continuation(new_ProxyMutex()), _dst(dst), _conf(conf), _stats_ids(stats_ids) +{ + SET_HANDLER(&PreWarmSM::state_init); + + Debug("v_prewarm_conf", "host=%p _dst=%ld _conf=%ld _stats_ids=%ld", dst->host.data(), _dst.use_count(), _conf.use_count(), + _stats_ids.use_count()); +} + +PreWarmSM::~PreWarmSM() {} + +/** + Start opening netvc directly + */ +void +PreWarmSM::start() +{ + _reset(); + _retry_counter = 0; + + handleEvent(EVENT_IMMEDIATE); +} + +/** + Retry with Exponential Backoff (through EventProcessor) + */ +void +PreWarmSM::retry() +{ + _reset(); + + ink_hrtime delay = HRTIME_SECONDS(1 << _retry_counter); + ++_retry_counter; + prewarmManager.stats.increment(_stats_ids->at(static_cast(PreWarm::Stat::RETRY)), 1); + + EThread *ethread = this_ethread(); + _retry_event = ethread->schedule_in_local(this, delay, EVENT_IMMEDIATE); + + if (_retry_counter % 10 == 0) { + Warning("retry pre-warming dst=%.*s:%d type=%d alpn=%d retry=%" PRIu32, (int)_dst->host.size(), _dst->host.data(), _dst->port, + (int)_dst->type, _dst->alpn_index, _retry_counter); + } +} + +/** + Stop pre-warming. Move to state_closed from any state. + */ +void +PreWarmSM::stop() +{ + if (handler == &PreWarmSM::state_closed) { + // do nothing + return; + } + + _reset(); + SET_HANDLER(&PreWarmSM::state_closed); + _milestones.mark(Milestone::CLOSED); +} + +void +PreWarmSM::destroy() +{ + _reset(); + + _dst.reset(); + _conf.reset(); + _stats_ids.reset(); + + this->mutex = nullptr; +} + +/** + @brief Give ownership of netvc to the caller + + PreWarmSM will not reveice any event as Continuation anymore. + Caller can read _read_buf through server_buf_reader(). + */ +NetVConnection * +PreWarmSM::move_netvc() +{ + if (handler != &PreWarmSM::state_open) { + return nullptr; + } + + NetVConnection *netvc = _netvc; + _netvc = nullptr; + + // clear the reference from netvc + netvc->do_io_read(nullptr, 0, nullptr); + netvc->do_io_write(nullptr, 0, nullptr); + + _timeout.cancel_active_timeout(); + _timeout.cancel_inactive_timeout(); + + if (_pending_action != nullptr) { + _pending_action->cancel(); + _pending_action = nullptr; + } + + SET_HANDLER(&PreWarmSM::state_closed); + + return netvc; +} + +int +PreWarmSM::state_init(int event, void *data) +{ + switch (event) { + case EVENT_IMMEDIATE: { + if (_retry_event != nullptr && data == _retry_event) { + _retry_event = nullptr; + } + + SET_HANDLER(&PreWarmSM::state_dns_lookup); + _timeout.set_active_timeout(_conf->connect_timeout); + _milestones.mark(Milestone::INIT); + + PreWarmSMDebug("pre-warming a netvc dst=%.*s:%d type=%d alpn=%d retry=%" PRIu32, (int)_dst->host.size(), _dst->host.data(), + _dst->port, (int)_dst->type, _dst->alpn_index, _retry_counter); + + if (_conf->srv_enabled) { + char target[MAXDNAME]; + size_t target_len = 0; + + memcpy(target, SRV_TUNNEL_TCP.data(), SRV_TUNNEL_TCP.size()); + target_len += SRV_TUNNEL_TCP.size(); + + memcpy(target + target_len, _dst->host.data(), _dst->host.size()); + target_len += _dst->host.size(); + + PreWarmSMVDebug("lookup SRV by %.*s", (int)target_len, target); + + Action *srv_lookup_action_handle = hostDBProcessor.getSRVbyname_imm( + this, static_cast(&PreWarmSM::process_srv_info), target, target_len); + if (srv_lookup_action_handle != ACTION_RESULT_DONE) { + _pending_action = srv_lookup_action_handle; + } + } else { + PreWarmSMVDebug("lookup A/AAAA by %.*s", (int)_dst->host.size(), _dst->host.data()); + + Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm( + this, static_cast(&PreWarmSM::process_hostdb_info), _dst->host.data(), _dst->host.size()); + if (dns_lookup_action_handle != ACTION_RESULT_DONE) { + _pending_action = dns_lookup_action_handle; + } + } + + break; + } + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +int +PreWarmSM::state_dns_lookup(int event, void *data) +{ + HostDBInfo *info = static_cast(data); + + switch (event) { + case EVENT_HOST_DB_LOOKUP: { + _pending_action = nullptr; + + if (info == nullptr || info->is_failed()) { + PreWarmSMVDebug("hostdb lookup is failed"); + + retry(); + return EVENT_DONE; + } + + IpEndpoint addr; + + ats_ip_copy(addr, info->ip()); + addr.port() = htons(_dst->port); + + if (is_debug_tag_set("v_prewarm_sm")) { + char addrbuf[INET6_ADDRPORTSTRLEN]; + PreWarmSMVDebug("hostdb lookup is done %s", ats_ip_nptop(addr, addrbuf, sizeof(addrbuf))); + } + + SET_HANDLER(&PreWarmSM::state_net_open); + _milestones.mark(Milestone::DNS_LOOKUP_DONE); + + Action *connect_action_handle = _connect(addr); + if (connect_action_handle != ACTION_RESULT_DONE) { + _pending_action = connect_action_handle; + } + + break; + } + case EVENT_SRV_LOOKUP: { + _pending_action = nullptr; + std::string_view hostname; + + if (info == nullptr || !info->is_srv || !info->round_robin) { + // no SRV record, fallback to default lookup + hostname = _dst->host; + } else { + HostDBRoundRobin *rr = info->rr(); + HostDBInfo *srv = nullptr; + if (rr) { + char srv_hostname[MAXDNAME] = {0}; + + ink_hrtime now = Thread::get_hrtime(); + srv = rr->select_best_srv(srv_hostname, &mutex->thread_holding->generator, ink_hrtime_to_sec(now), DOWN_SERVER_TIMEOUT); + hostname = std::string_view(srv_hostname); + + if (srv == nullptr) { + // lookup SRV record failed, fallback to default lookup + hostname = _dst->host; + } + } + } + + Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm( + this, static_cast(&PreWarmSM::process_hostdb_info), hostname.data(), hostname.size()); + if (dns_lookup_action_handle != ACTION_RESULT_DONE) { + _pending_action = dns_lookup_action_handle; + } + + break; + } + case VC_EVENT_ACTIVE_TIMEOUT: + retry(); + break; + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +int +PreWarmSM::state_net_open(int event, void *data) +{ + switch (event) { + case NET_EVENT_OPEN: { + _pending_action = nullptr; + _netvc = static_cast(data); + + // set buffers and (re)enable read/write for check status + // when TCP/TLS connection is established, VC_EVENT_WRITE_READY will be signaled + _read_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + _read_buf_reader = _read_buf->alloc_reader(); + _netvc->do_io_read(this, INT64_MAX, _read_buf); + + _write_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + _write_buf_reader = _write_buf->alloc_reader(); + _netvc->do_io_write(this, INT64_MAX, _write_buf_reader); + + break; + } + case VC_EVENT_READ_READY: + [[fallthrough]]; + case VC_EVENT_WRITE_READY: { + VIO *vio = static_cast(data); + NetVConnection *netvc = static_cast(vio->vc_server); + + ink_release_assert(netvc == _netvc); + PreWarmSMVDebug("%s Handshake is done netvc=%p", (_dst->type == SNIRoutingType::FORWARD) ? "TCP" : "TLS", _netvc); + + SET_HANDLER(&PreWarmSM::state_open); + _timeout.cancel_active_timeout(); + _timeout.set_inactive_timeout(_conf->inactive_timeout); + _milestones.mark(Milestone::ESTABLISHED); + _record_handshake_time(); + + // disable write op of pre-warmed connection + // keep read op enabled to get EOS event from origin server + netvc->do_io_write(nullptr, 0, nullptr); + + EThread *ethread = this_ethread(); + ethread->prewarm_queue->push(_dst, this); + + break; + } + case NET_EVENT_OPEN_FAILED: { + if (data != nullptr) { + const int errnum = -(reinterpret_cast(data)); + if (errnum == EADDRNOTAVAIL) { + // exhaust all ephemeral ports, do not retry + stop(); + break; + } + Warning("NET_EVENT_OPEN_FAILED: error message=%s (%d)", strerror(errnum), errnum); + } + [[fallthrough]]; + } + case VC_EVENT_ACTIVE_TIMEOUT: + [[fallthrough]]; + case VC_EVENT_ERROR: + [[fallthrough]]; + case VC_EVENT_EOS: + retry(); + break; + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +int +PreWarmSM::state_open(int event, void *data) +{ + switch (event) { + case VC_EVENT_READ_READY: { + // When the origin server sends something, keep it in the buffer. Forward it to the UA when a tunnel is setup + // (HttpSM::setup_blind_tunnel) + // - e.g. some HTTP/2 implementations send SETTINGS frame & WINDOW_UPDATE frame immediately when TLS handshake is done. + VIO *vio = static_cast(data); + NetVConnection *netvc = static_cast(vio->vc_server); + + ink_release_assert(netvc == _netvc); + + if (is_debug_tag_set("prewarm_sm")) { + if (_read_buf_reader->is_read_avail_more_than(0)) { + uint64_t read_len = _read_buf_reader->read_avail(); + PreWarmSMDebug("buffering data from origin server len=%" PRIu64, read_len); + + if (is_debug_tag_set("v_prewarm_sm")) { + uint8_t buf[1024]; + read_len = std::min(static_cast(sizeof(buf)), read_len); + _read_buf_reader->memcpy(buf, read_len); + + ts::LocalBufferWriter<2048> bw; + bw.print("{}", ts::bwf::Hex_Dump(buf)); + + PreWarmSMVDebug("\n%.*s\n", (int)read_len * 2, bw.data()); + } + } + } + + break; + } + case VC_EVENT_EOS: + // possibly inactive timeout at origin server + [[fallthrough]]; + case VC_EVENT_INACTIVITY_TIMEOUT: { + PreWarmSMDebug("%s (%d)", get_vc_event_name(event), event); + stop(); + break; + } + case VC_EVENT_ACTIVE_TIMEOUT: + [[fallthrough]]; + case VC_EVENT_ERROR: + [[fallthrough]]; + case VC_EVENT_WRITE_READY: + [[fallthrough]]; + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +int +PreWarmSM::state_closed(int event, void *data) +{ + switch (event) { + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +IOBufferReader * +PreWarmSM::server_buf_reader() +{ + return _read_buf_reader; +} + +bool +PreWarmSM::has_data_from_origin_server() const +{ + if (_read_buf_reader == nullptr) { + return false; + } + + return _read_buf_reader->is_read_avail_more_than(0); +} + +bool +PreWarmSM::is_active_timeout_expired(ink_hrtime now) +{ + return _timeout.is_active_timeout_expired(now); +} + +bool +PreWarmSM::is_inactive_timeout_expired(ink_hrtime now) +{ + return _timeout.is_inactive_timeout_expired(now); +} + +void +PreWarmSM::process_hostdb_info(HostDBInfo *r) +{ + ink_release_assert(this->handler == &PreWarmSM::state_dns_lookup); + + this->handleEvent(EVENT_HOST_DB_LOOKUP, r); +} + +void +PreWarmSM::process_srv_info(HostDBInfo *r) +{ + ink_release_assert(this->handler == &PreWarmSM::state_dns_lookup); + + this->handleEvent(EVENT_SRV_LOOKUP, r); +} + +Action * +PreWarmSM::_connect(const IpEndpoint &addr) +{ + Action *connect_action_handle = nullptr; + + HttpConfig::scoped_config http_conf_params; + + NetVCOptions opt; + opt.reset(); + opt.f_blocking_connect = false; + opt.set_sock_param(http_conf_params->oride.sock_recv_buffer_size_out, http_conf_params->oride.sock_send_buffer_size_out, + http_conf_params->oride.sock_option_flag_out, http_conf_params->oride.sock_packet_mark_out, + http_conf_params->oride.sock_packet_tos_out); + opt.f_tcp_fastopen = (http_conf_params->oride.sock_option_flag_out & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN); + + switch (_dst->type) { + case SNIRoutingType::FORWARD: { + SCOPED_MUTEX_LOCK(lock, mutex, this_ethread()); + // TODO: constify UnixNetProcessor::connect_re_internal() + connect_action_handle = netProcessor.connect_re(this, &addr.sa, &opt); + break; + } + case SNIRoutingType::PARTIAL_BLIND: { + // SNI + opt.set_sni_servername(_conf->sni.data(), _conf->sni.size()); + + // ALPN + opt.alpn_protos = SessionProtocolNameRegistry::convert_openssl_alpn_wire_format(_dst->alpn_index); + + // Verify Server Configs + opt.verifyServerPolicy = _conf->verify_server_policy; + opt.verifyServerProperties = _conf->verify_server_properties; + + // Client Cert + opt.ssl_client_cert_name = http_conf_params->oride.ssl_client_cert_filename; + opt.ssl_client_private_key_name = http_conf_params->oride.ssl_client_private_key_filename; + opt.ssl_client_ca_cert_name = http_conf_params->oride.ssl_client_ca_cert_filename; + + SCOPED_MUTEX_LOCK(lock, mutex, this_ethread()); + connect_action_handle = sslNetProcessor.connect_re(this, &addr.sa, &opt); + break; + } + default: + // do nothing + break; + } + + return connect_action_handle; +} + +/** + Reset state & *some* members + */ +void +PreWarmSM::_reset() +{ + SET_HANDLER(&PreWarmSM::state_init); + + _timeout.cancel_active_timeout(); + _timeout.cancel_inactive_timeout(); + + if (_netvc != nullptr) { + _netvc->do_io_close(); + _netvc = nullptr; + } + + if (_pending_action != nullptr) { + _pending_action->cancel(); + _pending_action = nullptr; + } + + if (_read_buf != nullptr) { + // free_MIOBuffer dealloc all readers + free_MIOBuffer(_read_buf); + _read_buf = nullptr; + _read_buf_reader = nullptr; + } + + if (_write_buf != nullptr) { + // free_MIOBuffer dealloc all readers + free_MIOBuffer(_write_buf); + _write_buf = nullptr; + _write_buf_reader = nullptr; + } + + if (_retry_event != nullptr) { + _retry_event->cancel(); + _retry_event = nullptr; + } +} + +void +PreWarmSM::_record_handshake_time() +{ + ink_hrtime duration = _milestones.elapsed(Milestone::INIT, Milestone::ESTABLISHED); + + ink_assert(duration > 0); + if (duration <= 0) { + return; + } + + prewarmManager.stats.increment(_stats_ids->at(static_cast(PreWarm::Stat::HANDSHAKE_TIME)), duration); + prewarmManager.stats.increment(_stats_ids->at(static_cast(PreWarm::Stat::HANDSHAKE_COUNT)), 1); +} + +//// +// PreWarmQueue +// +PreWarmQueue::PreWarmQueue() : Continuation(new_ProxyMutex()) +{ + SET_HANDLER(&PreWarmQueue::state_init); +} + +PreWarmQueue::~PreWarmQueue() +{ + _tick_event->cancel(); + _tick_event = nullptr; + + for (auto &e : _map) { + Info info = e.second; + + _make_queue_empty(info.init_list); + delete info.init_list; + + _make_queue_empty(info.open_list); + delete info.open_list; + } + + this->mutex = nullptr; +} + +int +PreWarmQueue::state_init(int event, void *data) +{ + switch (event) { + case EVENT_IMMEDIATE: { + _reconfigure(); + + _cop = ActivityCop(this->mutex, &_cop_list, 1); + _cop.start(); + + // schedule tick event + EThread *ethread = this_ethread(); + _tick_event = ethread->schedule_every_local(this, _event_period); + + SET_HANDLER(&PreWarmQueue::state_running); + + break; + } + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +int +PreWarmQueue::state_running(int event, void *data) +{ + switch (event) { + case EVENT_INTERVAL: { + for (auto &[dst, info] : _map) { + // mentain queues + _delete_closed_sm(info.init_list); + _delete_closed_sm(info.open_list); + + // pre-warm new connections + _prewarm_on_event_interval(dst, info); + + // set prewarmManager.stats + Debug("v_prewarm_q", "dst=%.*s:%d type=%d alpn=%d miss=%d hit=%d init=%d open=%d", (int)dst->host.size(), dst->host.data(), + dst->port, (int)dst->type, dst->alpn_index, info.stat.miss, info.stat.hit, (int)info.init_list->size(), + (int)info.open_list->size()); + + prewarmManager.stats.set_sum(info.stats_ids->at(static_cast(PreWarm::Stat::INIT_LIST_SIZE)), info.init_list->size()); + prewarmManager.stats.set_sum(info.stats_ids->at(static_cast(PreWarm::Stat::OPEN_LIST_SIZE)), info.open_list->size()); + prewarmManager.stats.increment(info.stats_ids->at(static_cast(PreWarm::Stat::HIT)), info.stat.hit); + prewarmManager.stats.increment(info.stats_ids->at(static_cast(PreWarm::Stat::MISS)), info.stat.miss); + + // clear PreWarmQueue::Stat + info.stat.miss = 0; + info.stat.hit = 0; + } + break; + } + case EVENT_IMMEDIATE: { + _reconfigure(); + + // reschedule tick event + EThread *ethread = this_ethread(); + _tick_event->cancel(); + _tick_event = ethread->schedule_every_local(this, _event_period); + + break; + } + default: + ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event); + break; + } + + return EVENT_DONE; +} + +void +PreWarmQueue::push(const PreWarm::SPtrConstDst &dst, PreWarmSM *sm) +{ + ink_release_assert(sm->handler == &PreWarmSM::state_open); + + if (auto res = _map.find(dst); res != _map.end()) { + Queue *init_list = res->second.init_list; + + // expecting init_list.front() is sm in many cases, if not we need to change container + for (auto it = init_list->begin(); it != init_list->end(); ++it) { + if (*it == sm) { + it = init_list->erase(it); + break; + } + } + + res->second.open_list->push_front(sm); + } +} + +/** + Use open_list as FILO to adjust size of list. ( A new sm is pushed in front of the list by PreWarmQeueu::push() ) + When the list has redundant sm(s), they will be closed by inactivity timeout. + */ +PreWarmSM * +PreWarmQueue::dequeue(const PreWarm::SPtrConstDst &target) +{ + PreWarmSM *sm = nullptr; + + auto res = _map.find(target); + if (res == _map.end()) { + // no such pool + return nullptr; + } + + const PreWarm::SPtrConstDst &dst = res->first; + Info &info = res->second; + + Queue *q = info.open_list; + while (!q->empty()) { + sm = q->front(); + q->pop_front(); + + if (sm->handler == &PreWarmSM::state_open) { + _cop_list.remove(sm); + break; + } + + _delete_prewarm_sm(sm); + sm = nullptr; + } + + // stat + if (sm == nullptr) { + ++info.stat.miss; + } else { + ++info.stat.hit; + } + + _prewarm_on_dequeue(dst, info); + + return sm; +} + +void +PreWarmQueue::_new_prewarm_sm(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf, + const PreWarm::SPtrConstStatsIds &stats_ids) +{ + EThread *ethread = this_ethread(); + + PreWarmSM *sm = THREAD_ALLOC(preWarmSMAllocator, ethread); + new (sm) PreWarmSM(dst, conf, stats_ids); + _cop_list.push(sm); + + _map[dst].init_list->push_back(sm); + + SCOPED_MUTEX_LOCK(lock, sm->mutex, ethread); + sm->start(); +} + +void +PreWarmQueue::_delete_prewarm_sm(PreWarmSM *sm) +{ + ink_release_assert(sm->handler == &PreWarmSM::state_closed); + + _cop_list.remove(sm); + + sm->destroy(); + THREAD_FREE(sm, preWarmSMAllocator, this_ethread()); +} + +/** + Periodical pre-warming + + Try to keep (min <= pool size && pool size <= max) + + V1: Expand the pool size to requested size + V2: Expand the pool size to current size + miss * rate + */ +void +PreWarmQueue::_prewarm_on_event_interval(const PreWarm::SPtrConstDst &dst, const Info &info) +{ + const uint32_t current_size = info.init_list->size() + info.open_list->size(); + uint32_t n = 0; + + switch (_algorithm) { + case PreWarm::Algorithm::V2: { + n = PreWarm::prewarm_size_v2_on_event_interval(info.stat.hit, info.stat.miss, current_size, info.conf->min, info.conf->max, + info.conf->rate); + break; + } + case PreWarm::Algorithm::V1: + [[fallthrough]]; + default: + n = PreWarm::prewarm_size_v1_on_event_interval(info.stat.miss + info.stat.hit, current_size, info.conf->min, info.conf->max); + break; + } + + Debug("v_prewarm_q", "prewarm_size=%" PRId32, n); + + for (uint32_t i = 0; i < n; ++i) { + _new_prewarm_sm(dst, info.conf, info.stats_ids); + } +} + +/** + Event based pre-warming + + V1: Do nothing + V2: Start pre-warming a new netvc + */ +void +PreWarmQueue::_prewarm_on_dequeue(const PreWarm::SPtrConstDst &dst, const Info &info) +{ + switch (_algorithm) { + case PreWarm::Algorithm::V2: { + const int32_t current_size = info.init_list->size() + info.open_list->size(); + if (current_size < info.conf->max) { + _new_prewarm_sm(dst, info.conf, info.stats_ids); + } + break; + } + case PreWarm::Algorithm::V1: + [[fallthrough]]; + default: + // do nothing + break; + } +} + +/** + Reconfigure _map based on new SNIConfig + */ +void +PreWarmQueue::_reconfigure() +{ + { + PreWarmConfig::scoped_config prewarm_conf; + + _event_period = HRTIME_MSECONDS(prewarm_conf->event_period); + _algorithm = PreWarm::algorithm_version(prewarm_conf->algorithm); + } + + // build new map based on new SNIConfig + const PreWarm::ParsedSNIConf &new_conf_list = prewarmManager.get_parsed_conf(); + const PreWarm::StatsIdMap &new_stats_id_map = prewarmManager.get_stats_id_map(); + + Map new_map; + + for (auto &entry : new_conf_list) { + const PreWarm::SPtrConstDst &dst = entry.first; + PreWarm::SPtrConstConf conf = entry.second; + + if (const auto &res = _map.find(dst); res != _map.end()) { + // copy from old info + const Info &old_info = res->second; + + new_map[dst] = Info{old_info.init_list, old_info.open_list, conf, old_info.stats_ids, old_info.stat}; + } else { + // make new info + PreWarm::SPtrConstStatsIds stats_ids; + if (const auto &res = new_stats_id_map.find(dst); res != new_stats_id_map.end()) { + stats_ids = res->second; + } else { + Error("no stats ids found for %s", dst->host.c_str()); + continue; + } + + Queue *init_list = new Queue(); + Queue *open_list = new Queue(); + new_map[dst] = Info{init_list, open_list, conf, stats_ids, {}}; + } + } + + // free unexisting entries + for (auto &[dst, info] : _map) { + if (auto entry = new_conf_list.find(dst); entry == new_conf_list.end()) { + prewarmManager.stats.set_sum(info.stats_ids->at(static_cast(PreWarm::Stat::INIT_LIST_SIZE)), 0); + prewarmManager.stats.set_sum(info.stats_ids->at(static_cast(PreWarm::Stat::OPEN_LIST_SIZE)), 0); + + _make_queue_empty(info.init_list); + delete info.init_list; + + _make_queue_empty(info.open_list); + delete info.open_list; + + info.conf.reset(); + info.stats_ids.reset(); + } + } + + std::swap(_map, new_map); +} + +/** + Delete all PreWarmSM in the queue + */ +void +PreWarmQueue::_make_queue_empty(Queue *q) +{ + while (!q->empty()) { + PreWarmSM *sm = q->front(); + q->pop_front(); + sm->stop(); + _delete_prewarm_sm(sm); + } +} + +/** + Delete closed state PreWarmSM in the queue + */ +void +PreWarmQueue::_delete_closed_sm(Queue *q) +{ + for (auto it = q->begin(); it != q->end();) { + if ((*it)->handler == &PreWarmSM::state_closed) { + _delete_prewarm_sm(*it); + it = q->erase(it); + } else { + ++it; + } + } +} + +//// +// PreWarmManager +// +void +PreWarmManager::reconfigure_prewarming_on_threads() +{ + EventProcessor::ThreadGroupDescriptor *tg = &(eventProcessor.thread_group[0]); + ink_release_assert(memcmp(tg->_name.data(), "ET_NET", 6) == 0); + + Debug("prewarm", "reconfigure prewarming"); + + for (int i = 0; i < tg->_count; ++i) { + EThread *ethread = tg->_thread[i]; + if (ethread->prewarm_queue == nullptr) { + ethread->prewarm_queue = new PreWarmQueue(); + } + ethread->schedule_imm_local(ethread->prewarm_queue); + } +} + +void +PreWarmManager::start() +{ + PreWarmConfig::startup(); + + _mutex = new_ProxyMutex(); + + this->reconfigure(); +} + +void +PreWarmManager::reconfigure() +{ + if (_mutex == nullptr) { + // don't reconfigure before start + return; + } + + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + + PreWarmConfig::scoped_config prewarm_conf; + bool is_prewarm_enabled = prewarm_conf->enabled; + + SNIConfig::scoped_config sni_conf; + for (const auto &item : sni_conf->Y_sni.items) { + if (item.tunnel_prewarm == YamlSNIConfig::TunnelPreWarm::ENABLED) { + is_prewarm_enabled = true; + break; + } + } + + if (is_prewarm_enabled) { + if (!stats.is_allocated()) { + stats.init(prewarm_conf->max_stats_size); + } + + _parsed_conf.clear(); + _parse_sni_conf(_parsed_conf, sni_conf); + _register_stats(_parsed_conf); + + reconfigure_prewarming_on_threads(); + } +} + +/** + TODO: stop pre-warming + */ +void +PreWarmManager::stop() +{ + _mutex->free(); +} + +const PreWarm::ParsedSNIConf & +PreWarmManager::get_parsed_conf() const +{ + return _parsed_conf; +} + +const PreWarm::StatsIdMap & +PreWarmManager::get_stats_id_map() const +{ + return _stats_id_map; +} + +/** + Convert SNIConfigParams to PreWarm::ParsedSNIConf + */ +void +PreWarmManager::_parse_sni_conf(PreWarm::ParsedSNIConf &parsed_conf, const SNIConfigParams *sni_conf) const +{ + PreWarmConfig::scoped_config prewarm_conf; + + for (const auto &item : sni_conf->Y_sni.items) { + if (item.tunnel_type != SNIRoutingType::FORWARD && item.tunnel_type != SNIRoutingType::PARTIAL_BLIND) { + continue; + } + + if (item.tunnel_prewarm == YamlSNIConfig::TunnelPreWarm::DISABLED || + (item.tunnel_prewarm == YamlSNIConfig::TunnelPreWarm::UNSET && !prewarm_conf->enabled)) { + continue; + } + + std::vector alpn_ids = {SessionProtocolNameRegistry::INVALID}; + if (!item.tunnel_alpn.empty() && item.tunnel_type == SNIRoutingType::PARTIAL_BLIND) { + for (auto &id : item.tunnel_alpn) { + alpn_ids.push_back(id); + } + } + + for (int id : alpn_ids) { + Debug("prewarm_m", "sni=%s dst=%s type=%d alpn=%d min=%d max=%d c_timeout=%d i_timeout=%d srv=%d", item.fqdn.c_str(), + item.tunnel_destination.c_str(), (int)item.tunnel_type, id, (int)item.tunnel_prewarm_min, (int)item.tunnel_prewarm_max, + (int)item.tunnel_prewarm_connect_timeout, (int)item.tunnel_prewarm_inactive_timeout, item.tunnel_prewarm_srv); + + std::string dst_fqdn; + int32_t port; + parse_authority(dst_fqdn, port, item.tunnel_destination); + + if (port < 0) { + if (item.tunnel_type == SNIRoutingType::PARTIAL_BLIND) { + port = 443; + } else { + port = 80; + } + } + + PreWarm::SPtrConstDst dst = std::make_shared(dst_fqdn, port, item.tunnel_type, id); + + // clang-format off + PreWarm::SPtrConstConf conf = std::make_shared( + item.tunnel_prewarm_min, + item.tunnel_prewarm_max, + item.tunnel_prewarm_rate, + HRTIME_SECONDS(item.tunnel_prewarm_connect_timeout), + HRTIME_SECONDS(item.tunnel_prewarm_inactive_timeout), + item.tunnel_prewarm_srv, + item.verify_server_policy, + item.verify_server_properties, + (strcmp(item.client_sni_policy, CLIENT_SNI_POLICY_SERVER_NAME) == 0)? item.fqdn : dst_fqdn + ); + // clang-format on + + parsed_conf[dst] = conf; + } + } +} + +/** + Create stats per pool. + Registered stats id is stored in _stat_id_map. + */ +void +PreWarmManager::_register_stats(const PreWarm::ParsedSNIConf &parsed_conf) +{ + int stats_counter = 0; + + for (auto &entry : parsed_conf) { + const PreWarm::SPtrConstDst &dst = entry.first; + + PreWarm::StatsIds ids; + for (int j = 0; j < static_cast(PreWarm::Stat::LAST_ENTRY); ++j) { + char name[STAT_NAME_BUF_LEN]; + + if (dst->alpn_index != SessionProtocolNameRegistry::INVALID) { + std::string_view alpn_name = alpn_name_for_stat(dst->alpn_index); + + snprintf(name, sizeof(name), "%s.%.*s:%d.tls.%s.%s", STAT_NAME_PREFIX.data(), static_cast(dst->host.size()), + dst->host.data(), dst->port, alpn_name.data(), STAT_ENTRIES[j].name.data()); + } else { + snprintf(name, sizeof(name), "%s.%.*s:%d.%s.%s", STAT_NAME_PREFIX.data(), static_cast(dst->host.size()), + dst->host.data(), dst->port, (dst->type == SNIRoutingType::PARTIAL_BLIND) ? "tls" : "tcp", + STAT_ENTRIES[j].name.data()); + } + + int stats_id = stats.find(name); + if (stats_id < 0) { + stats_id = stats.create(RECT_PROCESS, name, RECD_INT, STAT_ENTRIES[j].cb); + + if (stats_id < 0) { + // proxy.config.tunnel.prewarm.max_stats_size is enough? + Error("couldn't register stat name=%s", name); + } else { + ++stats_counter; + } + } + + ids[j] = stats_id; + + Debug("v_prewarm_init", "stat id=%d name=%s", stats_id, name); + } + + _stats_id_map[dst] = std::make_shared(ids); + } + + Note("%d dynamic stats are registered for pre-warming tunnel", stats_counter); +} diff --git a/proxy/http/PreWarmManager.h b/proxy/http/PreWarmManager.h new file mode 100644 index 00000000000..e7da819c267 --- /dev/null +++ b/proxy/http/PreWarmManager.h @@ -0,0 +1,341 @@ +/** @file + + Pre-Warming NetVConnection + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "PreWarmAlgorithm.h" + +#include "P_Net.h" +#include "I_NetVConnection.h" +#include "P_SSLSNI.h" +#include "P_HostDB.h" +#include "YamlSNIConfig.h" +#include "NetTimeout.h" +#include "Milestones.h" + +#include "records/DynamicStats.h" + +#include +#include +#include +#include + +class PreWarmSM; +class PreWarmManager; + +extern ClassAllocator preWarmSMAllocator; +extern PreWarmManager prewarmManager; + +namespace PreWarm +{ +//// +// Dst +// +struct Dst { + Dst(const std::string &h, in_port_t p, SNIRoutingType t, int a) : host(h), port(p), type(t), alpn_index(a) {} + + std::string host; + in_port_t port = 0; + SNIRoutingType type = SNIRoutingType::NONE; + int alpn_index = SessionProtocolNameRegistry::INVALID; +}; + +using SPtrConstDst = std::shared_ptr; + +struct DstHash { + size_t + operator()(const PreWarm::SPtrConstDst &dst) const + { + CryptoHash hash; + CryptoContext context{}; + + context.update(dst->host.data(), dst->host.size()); + context.update(&dst->port, sizeof(in_port_t)); + context.update(&dst->type, sizeof(SNIRoutingType)); + context.update(&dst->alpn_index, sizeof(int)); + + context.finalize(hash); + + return static_cast(hash.fold()); + } +}; + +struct DstKeyEqual { + bool + operator()(const PreWarm::SPtrConstDst &x, const PreWarm::SPtrConstDst &y) const + { + return x->host == y->host && x->port == y->port && x->type == y->type && x->alpn_index == y->alpn_index; + } +}; + +//// +// Conf +// +struct Conf { + Conf(uint32_t min, int32_t max, double rate, ink_hrtime connect_timeout, ink_hrtime inactive_timeout, bool srv_enabled, + YamlSNIConfig::Policy verify_server_policy, YamlSNIConfig::Property verify_server_properties, const std::string &sni) + : min(min), + max(max), + rate(rate), + connect_timeout(connect_timeout), + inactive_timeout(inactive_timeout), + srv_enabled(srv_enabled), + verify_server_policy(verify_server_policy), + verify_server_properties(verify_server_properties), + sni(sni) + { + } + + uint32_t min = 0; + int32_t max = 0; + double rate = 1.0; + ink_hrtime connect_timeout = 0; + ink_hrtime inactive_timeout = 0; + bool srv_enabled = false; + YamlSNIConfig::Policy verify_server_policy = YamlSNIConfig::Policy::UNSET; + YamlSNIConfig::Property verify_server_properties = YamlSNIConfig::Property::UNSET; + std::string sni; +}; + +using SPtrConstConf = std::shared_ptr; +using ParsedSNIConf = std::unordered_map; + +//// +// Stats +// +enum class Stat { + INIT_LIST_SIZE = 0, + OPEN_LIST_SIZE, + HIT, + MISS, + HANDSHAKE_TIME, + HANDSHAKE_COUNT, + RETRY, + LAST_ENTRY, +}; + +using StatsIds = std::array(PreWarm::Stat::LAST_ENTRY)>; +using SPtrConstStatsIds = std::shared_ptr; +using StatsIdMap = std::unordered_map; + +} // namespace PreWarm + +/** + @class PreWarmSM + @brief A state machine to pre-warm connection + + @startuml + hide empty description + [*] --> state_init : new + state_init --> state_dns_lookup : start() + state_init --> state_closed : stop() + state_dns_lookup --> state_net_open : HostDB lookup is done + state_dns_lookup --> state_init : retry() + state_dns_lookup --> state_closed : stop() + state_net_open --> state_open : TCP/TLS Handshake is done + state_net_open --> state_init : retry() + state_net_open --> state_closed : stop() + state_open --> state_closed : move_netvc()\nstop() + state_closed --> [*] : delete + @enduml + */ +class PreWarmSM : public Continuation +{ +public: + PreWarmSM(){}; + PreWarmSM(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf, const PreWarm::SPtrConstStatsIds &stats_ids); + ~PreWarmSM() override; + + // States + int state_init(int event, void *data); + int state_dns_lookup(int event, void *data); + int state_net_open(int event, void *data); + int state_open(int event, void *data); + int state_closed(int event, void *data); + + // Controllers + void start(); + void retry(); + void stop(); + void destroy(); + + // Modifiers + NetVConnection *move_netvc(); + IOBufferReader *server_buf_reader(); + + // References + bool has_data_from_origin_server() const; + + // NetTimeout + // TODO: constify + bool is_active_timeout_expired(ink_hrtime now); + bool is_inactive_timeout_expired(ink_hrtime now); + + // HostDB inline completion functions + void process_hostdb_info(HostDBInfo *r); + void process_srv_info(HostDBInfo *r); + +private: + enum class Milestone { + INIT = 0, + DNS_LOOKUP_DONE, + ESTABLISHED, + CLOSED, + LAST_ENTRY, + }; + + Action *_connect(const IpEndpoint &addr); + void _reset(); + void _record_handshake_time(); + + //// + // Variables + // + NetTimeout _timeout{}; + Milestones(Milestone::LAST_ENTRY)> _milestones; + + uint32_t _retry_counter = 0; + + PreWarm::SPtrConstDst _dst; + PreWarm::SPtrConstConf _conf; + PreWarm::SPtrConstStatsIds _stats_ids; + + NetVConnection *_netvc = nullptr; + Action *_pending_action = nullptr; + MIOBuffer *_read_buf = nullptr; + IOBufferReader *_read_buf_reader = nullptr; + MIOBuffer *_write_buf = nullptr; + IOBufferReader *_write_buf_reader = nullptr; + Event *_retry_event = nullptr; +}; + +/** + @class PreWarmQueue + @detail + - Each ET_NET thread has this queue + - Responsible for the life cycle of PreWarmSM until giving it to HttpSM + + @startuml + hide empty description + [*] --> state_init : new + state_init --> state_running : start pre-warming + @enduml + */ +class PreWarmQueue : public Continuation +{ +public: + PreWarmQueue(); + ~PreWarmQueue(); + + // States + int state_init(int event, void *data); + int state_running(int event, void *data); + + // Modifiers for queue + void push(const PreWarm::SPtrConstDst &dst, PreWarmSM *sm); + PreWarmSM *dequeue(const PreWarm::SPtrConstDst &dst); + +private: + using Queue = std::deque; + + struct Stat { + uint32_t miss = 0; + uint32_t hit = 0; + }; + + struct Info { + Queue *init_list; + Queue *open_list; + PreWarm::SPtrConstConf conf; + PreWarm::SPtrConstStatsIds stats_ids; + Stat stat; + }; + + using Map = std::unordered_map; + + // construct/destruct PreWarmSM + void _new_prewarm_sm(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf, + const PreWarm::SPtrConstStatsIds &stats_ids); + void _delete_prewarm_sm(PreWarmSM *sm); + + void _reconfigure(); + void _make_queue_empty(Queue *q); + void _delete_closed_sm(Queue *q); + + // hooks for pre-warming pool size algorithm + void _prewarm_on_event_interval(const PreWarm::SPtrConstDst &dst, const Info &info); + void _prewarm_on_dequeue(const PreWarm::SPtrConstDst &dst, const Info &info); + + //// + // Variables + // + PreWarm::Algorithm _algorithm = PreWarm::Algorithm::V1; + + Event *_tick_event = nullptr; + ink_hrtime _event_period = HRTIME_SECONDS(1); + + // Force PreWarmSM to open new netvc to keep the connection warm periodically + ActivityCop _cop; + DLL _cop_list; + + Map _map; +}; + +/** + @class PreWarmManager + @details + - Global singleton object + - Responsible for stats & configs management + */ +class PreWarmManager +{ +public: + static void reconfigure_prewarming_on_threads(); + + // Controllers + void start(); + void reconfigure(); + void stop(); + + // References + const PreWarm::ParsedSNIConf &get_parsed_conf() const; + const PreWarm::StatsIdMap &get_stats_id_map() const; + + //// + // Variables + // + DynamicStats stats; + +private: + void _parse_sni_conf(PreWarm::ParsedSNIConf &parsed_conf, const SNIConfigParams *sni_conf) const; + void _register_stats(const PreWarm::ParsedSNIConf &parsed_conf); + + //// + // Variables + // + // For the race of Main Thread (start up) vs Task Thread (config reload) + Ptr _mutex; + + PreWarm::ParsedSNIConf _parsed_conf; + PreWarm::StatsIdMap _stats_id_map; +}; diff --git a/proxy/http/unit_tests/test_PreWarm.cc b/proxy/http/unit_tests/test_PreWarm.cc new file mode 100644 index 00000000000..3f22b8909d4 --- /dev/null +++ b/proxy/http/unit_tests/test_PreWarm.cc @@ -0,0 +1,223 @@ +/** @file + + Unit Tests for Pre-Warming Pool Size Algorithm + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "PreWarmAlgorithm.h" + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +TEST_CASE("PreWarm Algorithm", "[prewarm]") +{ + SECTION("prewarm_size_v1_on_event_interval") + { + SECTION("{min, max} = {10, 100}") + { + const uint32_t min = 10; + const uint32_t max = 100; + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 0, min, max) == min); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 0, min, max) == 20); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 0, min, max) == max); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 5, min, max) == 5); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 5, min, max) == 15); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 5, min, max) == 95); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 10, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 10, min, max) == 10); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 10, min, max) == 90); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 50, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 50, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 50, min, max) == 50); + } + + SECTION("{min, max} = {0, 0}") + { + const uint32_t min = 0; + const uint32_t max = 0; + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 0, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 0, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 0, min, max) == 0); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 5, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 5, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 5, min, max) == 0); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 10, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 10, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 10, min, max) == 0); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 50, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 50, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 50, min, max) == 0); + } + + SECTION("{min, max} = {10, -1}") + { + const uint32_t min = 10; + const uint32_t max = -1; + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 0, min, max) == min); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 0, min, max) == 20); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 0, min, max) == 101); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 5, min, max) == 5); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 5, min, max) == 15); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 5, min, max) == 96); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 10, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 10, min, max) == 10); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 10, min, max) == 91); + + CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 50, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 50, min, max) == 0); + CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 50, min, max) == 51); + } + } + + SECTION("prewarm_size_v4_on_event_interval") + { + const uint32_t min = 10; + const uint32_t max = 100; + + // same as v3 + SECTION("rate = 1.0") + { + const double rate = 1.0; + + SECTION("hit + miss + current_size < min ") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + } + + SECTION("min <= miss + hit + current_size") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 90, min, max, rate) == 10); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 91, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 91, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 91, min, max, rate) == 9); + } + } + + SECTION("rate = 0.0") + { + const double rate = 0.0; + + SECTION("hit + miss + current_size < min ") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + } + + SECTION("min <= miss + hit + current_size") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 0); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 0); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 0); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 90, min, max, rate) == 0); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 91, min, max, rate) == 0); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 91, min, max, rate) == 0); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 91, min, max, rate) == 0); + } + } + + SECTION("rate = 0.5") + { + const double rate = 0.5; + + SECTION("hit + miss + current_size < min ") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + } + + SECTION("min <= miss + hit + current_size") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 5); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 4); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 5); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 90, min, max, rate) == 5); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 18, 90, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 19, 90, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 20, 90, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 21, 90, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 22, 90, min, max, rate) == 10); + } + } + + SECTION("rate = 1.5") + { + const double rate = 1.5; + + SECTION("hit + miss + current_size < min ") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9); + } + + SECTION("min <= miss + hit + current_size") + { + CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 15); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0); + + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 5, 90, min, max, rate) == 7); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 6, 90, min, max, rate) == 9); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 7, 90, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 8, 90, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 10); + CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 10); + } + } + } +} diff --git a/src/traffic_quic/traffic_quic.cc b/src/traffic_quic/traffic_quic.cc index 4ee2f5fb0ac..8f42a9bb2bb 100644 --- a/src/traffic_quic/traffic_quic.cc +++ b/src/traffic_quic/traffic_quic.cc @@ -343,3 +343,11 @@ void HttpCacheAction::cancel(Continuation *c) { } + +#include "PreWarmManager.h" +void +PreWarmManager::reconfigure() +{ +} + +PreWarmManager prewarmManager;