Merge branch 'master' into quic-latest

* master:
  Implement TLSBasicSupport for QUICNetVC (#7959)
  Reload server session inactivity timeout before placing a session into the pool (#7618)
  Use OpeSSL EVP API if SHA1 API is unavailable  (cache_promote) (#7447)
  Cleanup: Get rid of HTTP2_SESSION_EVENT_RECV (#7879)
  Timing and permissions update for regex_revalidate test (#7998)
  limit m_current_range to max value in RangeTransform (#4843)
  Allow to TLS handshake to error out on TSVConnReenable (#7994)
  Cleanup: Get rid of HTTP2_SESSION_EVENT_INIT (#7878)
  Add hook for loading certificate and key data from plugin  (#6609)
  Doc: Now's Minute invocation error (#7990)
  Fix typo in configure.ac (#7993)

configure.ac

@@ -837,7 +837,7 @@ case $host_os_def in
       #  #279 is "controlling expression is constant" (which is e.g. TSReleaseAssert(!"Unexpected Event");
       common_opt="-pipe -Wall -wd873 -wd279"
       debug_opt="-g $common_opt"
-      release_opt="-g $common_opt $optimization_flags -axsse4.2 -fno-strict-aliasing"
+      release_opt="-g $common_opt $optimizing_flags -axsse4.2 -fno-strict-aliasing"
       cxx_opt="-Wno-invalid-offsetof"
     ])
 
@@ -1302,6 +1302,7 @@ AC_CHECK_FUNCS([ \
   HMAC_CTX_new \
   X509_get0_signature \
   ERR_get_error_all \
+  SHA1 \
 ])
 
 AC_CHECK_FUNC([ASN1_STRING_get0_data], [],

doc/admin-guide/plugins/header_rewrite.en.rst

@@ -418,7 +418,7 @@ values, such as year, month etc.
     %{NOW:MONTH}     Current month (0-11, 0 == January)
     %{NOW:DAY}       Current day of the month (1-31)
     %{NOW:HOUR}      Current hour (0-23, in the 24h system)
-    %{NOW:MIN}       Current minute (0-59}
+    %{NOW:MINUTE}    Current minute (0-59}
     %{NOW:WEEKDAY}   Current weekday (0-6, 0 == Sunday)
     %{NOW:YEARDAY}   Current day of the year (0-365, 0 == Jan 1st)
 

doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst

@@ -106,6 +106,14 @@ Types
       Invoked with the event :c:data:`TS_EVENT_LIFECYCLE_TASK_THREADS_READY` and ``NULL``
       data.
 
+   .. cpp:enumerator:: TS_LIFECYCLE_SSL_SECRET_HOOK
+
+      Called before the data for the certificate or key is loaded.  The data argument to the callback is a pointer to a :type:`TSSecretID` which
+      contains a pointer to the name of the certificate or key and the relevant version if applicable.
+
+      This hook gives the plugin a chance to load the certificate or key from an alternative source and set via the :c:func:`TSSslSecretSet` API.
+      If there is no plugin override, the certificate or key will be loaded from disk and the secret name will be interpreted as a file path.
+
    .. cpp:enumerator:: TS_LIFECYCLE_SHUTDOWN_HOOK
 
       Called after |TS| receiving a shutdown signal, such as SIGTERM.

doc/developer-guide/api/functions/TSSslSecret.en.rst

@@ -0,0 +1,77 @@
+.. Licensed to the Apache Software Foundation (ASF) under one or more
+      contributor license agreements.  See the NOTICE file distributed
+   with this work for additional information regarding copyright
+   ownership.  The ASF licenses this file to you under the Apache
+   License, Version 2.0 (the "License"); you may not use this file
+   except in compliance with the License.  You may obtain a copy of
+   the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied.  See the License for the specific language governing
+   permissions and limitations under the License.
+.. include:: /common.defs
+
+.. default-domain:: c
+
+TSSslSecretSet
+**************
+
+Set the data associated with a secret name specified in the config.
+
+Synopsis
+========
+
+.. code-block:: cpp
+
+    #include <ts/ts.h>
+
+.. function:: TSReturnCode TSSslSecretSet(const char * secret_name, int secret_name_length, const char * secret_data, int secret_data_len)
+
+Description
+===========
+
+:func:`TSSslSecretSet` updates the current secret map. Generally the secret name corresponds to the name of a certificate or a key.
+Future creation of SSL_CTX objects that use the secret will use the newly specified data. It can be useful to call this function
+from the :data:`TS_LIFECYCLE_SSL_SECRET_HOOK`.
+
+TSSslSecretGet
+**************
+
+Get the data associated with a secret name specified in the config.
+
+Synopsis
+========
+
+.. code-block:: cpp
+
+    #include <ts/ts.h>
+
+.. function:: TSReturnCode TSSslSecretGet(const char * secret_name, int secret_name_length, const char ** secret_data_return, int * secret_data_len)
+
+Description
+===========
+
+:func:`TSSslSecretGet` fetches the named secret from the current secret map. TS_ERROR is returned if there is no entry for the secret.
+
+TSSslSecretUpdate
+*****************
+
+Tell |TS| to update the SSL objects dependent on the secret.
+
+Synopsis
+========
+
+.. code-block:: cpp
+
+    #include <ts/ts.h>
+
+.. function:: TSReturnCode TSSslSecretGet(const char * secret_name, int secret_name_length)
+
+Description
+===========
+
+:func:`TSSslSecretUpdate` causes |TS| to update the SSL objects that depend on the specified secret.  This enables a plugin to look for
+multiple secret updates and make calls to :func:`TSSslSecretSet` to update the secret table.  Then once everything is updated call
+:func:`TSSslSecretUpdate` to update the SSL objects with a consistent updated set of secrets.

include/ts/apidefs.h.in

@@ -437,6 +437,7 @@ typedef enum {
   TS_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED_HOOK,
   TS_LIFECYCLE_MSG_HOOK,
   TS_LIFECYCLE_TASK_THREADS_READY_HOOK,
+  TS_LIFECYCLE_SSL_SECRET_HOOK,
   TS_LIFECYCLE_SHUTDOWN_HOOK,
   TS_LIFECYCLE_LAST_HOOK
 } TSLifecycleHookID;
@@ -551,6 +552,7 @@ typedef enum {
   TS_EVENT_SSL_VERIFY_SERVER = 60205,
   TS_EVENT_SSL_VERIFY_CLIENT = 60206,
   TS_EVENT_SSL_CLIENT_HELLO  = 60207,
+  TS_EVENT_SSL_SECRET        = 60208,
 
   TS_EVENT_MGMT_UPDATE = 60300
 } TSEvent;
@@ -1031,6 +1033,13 @@ typedef struct TSSslSessionID_s {
   char bytes[TS_SSL_MAX_SSL_SESSION_ID_LENGTH];
 } TSSslSessionID;
 
+typedef struct TSSecretID_s {
+  const char *cert_name;
+  size_t cert_name_len;
+  const char *key_name;
+  size_t key_name_len;
+} TSSecretID;
+
 /* --------------------------------------------------------------------------
    Init */
 

include/ts/ts.h

@@ -1300,6 +1300,13 @@ tsapi TSSslContext TSSslClientContextFindByName(const char *ca_paths, const char
 tsapi TSReturnCode TSSslClientCertUpdate(const char *cert_path, const char *key_path);
 tsapi TSReturnCode TSSslServerCertUpdate(const char *cert_path, const char *key_path);
 
+/* Update the transient secret table for SSL_CTX loading */
+tsapi TSReturnCode TSSslSecretSet(const char *secret_name, int secret_name_length, const char *secret_data, int secret_data_len);
+tsapi TSReturnCode TSSslSecretGet(const char *secret_name, int secret_name_length, const char **secret_data_return,
+                                  int *secret_data_len);
+
+tsapi TSReturnCode TSSslSecretUpdate(const char *secret_name, int secret_name_length);
+
 /* Create a new SSL context based on the settings in records.config */
 tsapi TSSslContext TSSslServerContextCreate(TSSslX509 cert, const char *certname, const char *rsp_file);
 tsapi void TSSslContextDestroy(TSSslContext ctx);

iocore/net/Makefile.am

@@ -143,6 +143,7 @@ libinknet_a_SOURCES = \
 	P_Socks.h \
 	P_SSLCertLookup.h \
 	P_SSLConfig.h \
+        P_SSLSecret.h \
 	P_SSLNetAccept.h \
 	P_SSLNetProcessor.h \
 	P_SSLNetVConnection.h \
@@ -171,6 +172,7 @@ libinknet_a_SOURCES = \
 	SSLClientCoordinator.cc \
 	SSLClientUtils.cc \
 	SSLConfig.cc \
+        SSLSecret.cc \
 	SSLDiags.cc \
 	SSLInternal.cc \
 	SSLNetAccept.cc \

iocore/net/P_QUICNetVConnection.h

@@ -38,6 +38,7 @@
 #include "P_UnixNet.h"
 #include "P_UDPNet.h"
 #include "P_ALPNSupport.h"
+#include "TLSBasicSupport.h"
 #include "TLSSessionResumptionSupport.h"
 #include "tscore/ink_apidefs.h"
 #include "tscore/List.h"
@@ -140,6 +141,7 @@ class QUICNetVConnection : public UnixNetVConnection,
                            public QUICConnection,
                            public RefCountObj,
                            public ALPNSupport,
+                           public TLSBasicSupport,
                            public TLSSessionResumptionSupport
 {
   using super = UnixNetVConnection; ///< Parent type.
@@ -223,6 +225,11 @@ class QUICNetVConnection : public UnixNetVConnection,
   SLINK(QUICNetVConnection, closed_alink);
 
 protected:
+  // TLSBasicSupport
+  SSL *_get_ssl_object() const override;
+  ssl_curve_id _get_tls_curve() const override;
+
+  // TLSSessionResumptionSupport
   const IpEndpoint &_getLocalEndpoint() override;
 
 private:

iocore/net/P_SSLCertLookup.h

@@ -23,6 +23,7 @@
 
 #pragma once
 
+#include <set>
 #include <openssl/ssl.h>
 
 #include "ProxyConfig.h"
@@ -140,7 +141,7 @@ struct SSLCertLookup : public ConfigInfo {
       Exact matches have priority, then wildcards. Only destination based matches are checked.
       @return @c A pointer to the matched context, @c nullptr if no match is found.
   */
-  SSLCertContext *find(const char *name) const;
+  SSLCertContext *find(const std::string &name) const;
 
   // Return the last-resort default TLS context if there is no name or address match.
   SSL_CTX *
@@ -152,8 +153,15 @@ struct SSLCertLookup : public ConfigInfo {
   unsigned count() const;
   SSLCertContext *get(unsigned i) const;
 
+  void register_cert_secrets(std::vector<std::string> const &cert_secrets, std::set<std::string> &lookup_names);
+  void getPolicies(const std::string &secret_name, std::set<shared_SSLMultiCertConfigParams> &policies) const;
+
   SSLCertLookup();
   ~SSLCertLookup() override;
+
+private:
+  // Map cert_secret name to lookup keys
+  std::unordered_map<std::string, std::vector<std::string>> cert_secret_registry;
 };
 
 void ticket_block_free(void *ptr);

iocore/net/P_SSLConfig.h

@@ -41,6 +41,7 @@
 #include "YamlSNIConfig.h"
 
 #include "P_SSLUtils.h"
+#include "P_SSLSecret.h"
 
 struct SSLCertLookup;
 struct ssl_ticket_key_block;
@@ -144,9 +145,16 @@ struct SSLConfigParams : public ConfigInfo {
   mutable std::unordered_map<std::string, CTX_MAP> top_level_ctx_map;
   mutable ink_mutex ctxMapLock;
 
+  mutable SSLSecret secrets;
+
   shared_SSL_CTX getClientSSL_CTX() const;
+  shared_SSL_CTX getCTX(const std::string &client_cert, const std::string &key_file, const char *ca_bundle_file,
+                        const char *ca_bundle_path) const;
   shared_SSL_CTX getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file,
                         const char *ca_bundle_path) const;
+  void updateCTX(const std::string &secret_string_name) const;
+
+  void clearCTX(const std::string &client_cert) const;
 
   void cleanupCTXTable();
 
@@ -166,11 +174,22 @@ struct SSLConfig {
   static void startup();
   static void reconfigure();
   static SSLConfigParams *acquire();
+  static SSLConfigParams *load_acquire();
   static void release(SSLConfigParams *params);
+  static void load_release(SSLConfigParams *params);
+
+  // These methods manipulate the double buffering of the configs
+  // The "loading" version is only active during loading.  Once
+  // it is fliped to the active by comit_config_id, it/ becomes the
+  // version accessble to the rest of the system.
+  static int get_config_index();
+  static int get_loading_config_index();
+  static void commit_config_id();
   typedef ConfigProcessor::scoped_config<SSLConfig, SSLConfigParams> scoped_config;
 
 private:
-  static int configid;
+  static int config_index;
+  static int configids[2];
 };
 
 struct SSLCertificateConfig {

iocore/net/P_SSLSecret.h

@@ -0,0 +1,36 @@
+/** @file
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+class SSLSecret
+{
+public:
+  SSLSecret() {}
+  bool getSecret(const std::string &name, std::string_view &data) const;
+  bool setSecret(const std::string &name, const char *data, int data_len);
+  bool getOrLoadSecret(const std::string &name, const std::string &name2, std::string_view &data, std::string_view &data2);
+
+private:
+  const std::string *getSecretItem(const std::string &name) const;
+  bool loadSecret(const std::string &name, const std::string &name2, std::string &data_item, std::string &data_item2);
+  bool loadFile(const std::string &name, std::string &data_item);
+
+  std::unordered_map<std::string, std::string> secret_map;
+};

iocore/net/P_SSLUtils.h

@@ -78,10 +78,11 @@ class SSLMultiCertConfigLoader
   static bool set_session_id_context(SSL_CTX *ctx, const SSLConfigParams *params,
                                      const SSLMultiCertConfigParams *sslMultCertSettings);
 
-  static bool index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, const char *sni_name);
   static int check_server_cert_now(X509 *cert, const char *certname);
   static void clear_pw_references(SSL_CTX *ssl_ctx);
 
+  bool update_ssl_ctx(const std::string &secret_name);
+
 protected:
   const SSLConfigParams *_params;
 
@@ -90,7 +91,9 @@ class SSLMultiCertConfigLoader
 
 private:
   virtual const char *_debug_tag() const;
-  bool _store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams &ssl_multi_cert_params);
+  virtual bool _store_ssl_ctx(SSLCertLookup *lookup, shared_SSLMultiCertConfigParams ssl_multi_cert_params);
+  bool _prep_ssl_ctx(const shared_SSLMultiCertConfigParams sslMultCertSettings, SSLMultiCertConfigLoader::CertLoadData &data,
+                     std::set<std::string> &common_names, std::unordered_map<int, std::set<std::string>> &unique_names);
   virtual void _set_handshake_callbacks(SSL_CTX *ctx);
   virtual bool _setup_session_cache(SSL_CTX *ctx);
   virtual bool _setup_dialog(SSL_CTX *ctx, const SSLMultiCertConfigParams *sslMultCertSettings);