Merge pull request apache#662 from bneradt/asf_92x_merge_2022_03_29

Asf 92x merge 2022 03 29

CHANGELOG-9.2.0

@@ -108,6 +108,7 @@ Changes with Apache Traffic Server 9.2.0
   #7836 - build_h3_tools: use OpenSSL_1_1_1k+quic
   #7837 - Update NextHop strategies so that unavailable server retry codes are configurable.
   #7841 - Unifdef test code for MMH and moved it into its own test file
+  #7843 - DNS: Fix lack of nameserver failover in low use circumstances.
   #7844 - Clean up lua plugin doc for overridable configurations
   #7847 - Fixed double declaration types for log buffer tracking
   #7848 - Remove bucket search from IntrusiveHashMap::erase
@@ -334,6 +335,7 @@ Changes with Apache Traffic Server 9.2.0
   #8439 - Fix Multiplexer POST Body Handling
   #8440 - Update the Multiplexer Docs for Multplexed HTTPS Connections
   #8441 - bigobj: use automake to build test utilities
+  #8445 - Include CPPFLAGS when building CompileParseRules
   #8449 - Add AuTest for HTTP CONNECT
   #8450 - Document proxy.config.memory.max_usage
   #8453 - Ignore ECONNABORTED on blocking accept
@@ -365,6 +367,9 @@ Changes with Apache Traffic Server 9.2.0
   #8543 - doc: Convert miscased Traffic Server references to |TS| macro
   #8550 - TSUserArg: add value type checking
   #8568 - Update descriptions of sni.yaml.default
+  #8570 - Port #6816 from core strategy to plugin
+  #8571 - Port #7749 from core strategy to plugin
+  #8573 - Port #7837 from core strategy to plugin
   #8578 - Add SSLSessionDup for older OpenSSL and BoringSSL
   #8580 - Ports #7897 from core strategies to parent_select plugin.
   #8585 - Clean up of next hop HostRecord class.
@@ -389,10 +394,32 @@ Changes with Apache Traffic Server 9.2.0
   #8650 - Fix strategies to initialize scheme
   #8651 - Add parent_select plugin strategy caching
   #8656 - Cleanup strategy debug logs
+  #8657 - Support transforming range requests when origin returns full resource.
   #8661 - Making 9.2.x backwards compatible with 9.1.x
   #8663 - Revert "DNS: Fix lack of nameserver failover in low use circumstances…
   #8667 - TLS Session Resumption: fix timed out session
   #8673 - Fix to allow running autopep8 from outside top_srcdir
+  #8677 - Update iocore/cache/test to fix the build
   #8680 - Fix missing unique_lock definition.
+  #8688 - update FREELIST macros for AArch64
   #8690 - 9.2.x: Fix Clang 13.0.1 and GCC 12.0.1 Compiler Warnings
   #8691 - 9.2.x: Fix test_QUIC unit test builds. (#8678)
+  #8692 - Add docs for parent.config defaults
+  #8700 - Change api TSSslTicketKeyUpdate to return TSReturnCode
+  #8705 - Add back "DNS: Fix lack of nameserver failover in low use circumstances."
+  #8706 - Provide libquic.a definition to VIO::VIO()
+  #8707 - cache-request-method autest: Extending IO delay
+  #8711 - Move hwloc-based functions into it's own header/compilation unit
+  #8715 - Fixes build when hwloc is not installed
+  #8718 - Enable all iocore/cache tests
+  #8722 - Adds user-agent to OCSP requests
+  #8725 - Update THREAD_FREE macro to only evaluate the _t parameter once.
+  #8729 - Address issues with python 3.10
+  #8736 - [ink_base64] Fix buffer size computation
+  #8738 - ts_lua: change type in stats struct from 'int' to 'TSMgmtInt' to avoid overflow
+  #8742 - Fixes Issue #7824 - The strategies.yaml parser can incorrectly interpret
+  #8752 - docs: add some obvious units to some http2 overridables
+  #8753 - Update to proxy-verifier-v2.3.1
+  #8763 - Fix error handling in SSL cert/key load failures
+  #8765 - Fixing the httpbin AuTests by pinning Werkzeug
+  #8770 - Remove null check before ats_free calls (#8744)

doc/admin-guide/files/parent.config.en.rst

@@ -222,6 +222,7 @@ The following list shows the possible actions and their allowed values.
       parent is marked down and a new parent is selected to retry the request.  The number of
       retries is controlled by ``max_unavailable_server_retries`` which is set to 1 by default.
     - ``both`` - This enables both ``simple_retry`` and ``unavailable_server_retry`` as described above.
+    - If not set, by default all response codes will be considered a success, and parents will not be retried based on any HTTP response code.
 
 .. _parent-config-format-simple-server-retry-responses:
 
@@ -264,7 +265,7 @@ The following list shows the possible actions and their allowed values.
     -  ``strict`` - Traffic Server machines serve requests strictly in
        turn. For example: machine ``proxy1`` serves the first request,
        ``proxy2`` serves the second request, and so on.
-    -  ``false`` - Round robin selection does not occur.
+    -  ``false`` - The default. Round robin selection does not occur.
     -  ``consistent_hash`` - consistent hash of the url so that one parent
        is chosen for a given url. If a parent is down, the traffic that
        would go to the down parent is rehashed amongst the remaining parents.
@@ -284,7 +285,7 @@ The following list shows the possible actions and their allowed values.
 ``go_direct``
     One of the following values:
 
-    -  ``true`` - requests bypass parent hierarchies and go directly to
+    -  ``true`` - The default. Requests bypass parent hierarchies and go directly to
        the origin server.
 
     -  ``false`` - requests do not bypass parent hierarchies.
@@ -294,7 +295,7 @@ The following list shows the possible actions and their allowed values.
 ``qstring``
     One of the following values:
 
-    -  ``consider`` - Use the query string when finding a parent.
+    -  ``consider`` - The default. Use the query string when finding a parent.
 
     -  ``ignore`` - Do not consider the query string when finding a parent. This
        is especially useful when using the ``consistent_hash`` selection strategy,

doc/admin-guide/files/records.config.en.rst

@@ -2278,10 +2278,11 @@ Cache Control
    :overridable:
 
    When enabled (``1``), |TS| will attempt to write (lock) the URL
-   to cache. This is rarely useful (at the moment), since it'll only be able
-   to write to cache if the origin has ignored the ``Range:`` header. For a use
-   case where you know the origin will respond with a full (``200``) response,
-   you can turn this on to allow it to be cached.
+   to cache for a request specifying a range. This is useful when the origin server
+   might ignore a range request and respond with a full (``200``) response.
+   Additionally, this setting will attempt to transform a 200 response from the origin
+   server to a partial (``206``) response, honoring the requested range, while
+   caching the full response.
 
 .. ts:cv:: CONFIG proxy.config.http.cache.ignore_accept_mismatch INT 2
    :reloadable:
@@ -4049,15 +4050,13 @@ OCSP Stapling Configuration
 
    Number of seconds before an OCSP response expires in the stapling cache.
 
-   See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts.
-
 .. ts:cv:: CONFIG proxy.config.ssl.ocsp.request_timeout INT 10
+   :units: seconds
 
    Timeout (in seconds) for queries to OCSP responders.
 
-   See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts.
-
 .. ts:cv:: CONFIG proxy.config.ssl.ocsp.update_period INT 60
+   :units: seconds
 
    Update period (in seconds) for stapling caches.
 
@@ -4101,6 +4100,7 @@ HTTP/2 Configuration
 
 .. ts:cv:: CONFIG proxy.config.http2.initial_window_size_in INT 65535
    :reloadable:
+   :units: bytes
 
    The initial window size for inbound connection streams.
 
@@ -4116,6 +4116,7 @@ HTTP/2 Configuration
 
 .. ts:cv:: CONFIG proxy.config.http2.max_frame_size INT 16384
    :reloadable:
+   :units: bytes
 
    Indicates the size of the largest frame payload that the sender is willing
    to receive.
@@ -4148,6 +4149,7 @@ HTTP/2 Configuration
 
 .. ts:cv:: CONFIG proxy.config.http2.active_timeout_in INT 0
    :reloadable:
+   :units: seconds
 
    This is the active timeout of the http2 connection. It is set when the connection is opened
    and keeps ticking regardless of activity level.
@@ -4156,6 +4158,7 @@ HTTP/2 Configuration
 
 .. ts:cv:: CONFIG proxy.config.http2.accept_no_activity_timeout INT 120
    :reloadable:
+   :units: seconds
 
    Specifies how long |TS| keeps connections to clients open if no
    activity is received on the connection. Lowering this timeout can ease
@@ -4164,6 +4167,7 @@ HTTP/2 Configuration
 
 .. ts:cv:: CONFIG proxy.config.http2.no_activity_timeout_in INT 120
    :reloadable:
+   :units: seconds
 
    Specifies how long |TS| keeps connections to clients open if a
    transaction stalls. Lowering this timeout can ease pressure on the proxy if
@@ -4244,6 +4248,7 @@ HTTP/2 Configuration
 
 .. ts:cv:: CONFIG proxy.config.http2.write_buffer_block_size INT 262144
    :reloadable:
+   :units: bytes
 
    Specifies the size of a buffer block that is used for buffering outgoing
    HTTP/2 frames. The size will be rounded up based on power of 2.

include/ts/ts.h

@@ -1312,7 +1312,7 @@ tsapi TSReturnCode TSSslSecretUpdate(const char *secret_name, int secret_name_le
 /* Create a new SSL context based on the settings in records.config */
 tsapi TSSslContext TSSslServerContextCreate(TSSslX509 cert, const char *certname, const char *rsp_file);
 tsapi void TSSslContextDestroy(TSSslContext ctx);
-tsapi void TSSslTicketKeyUpdate(char *ticketData, int ticketDataLen);
+tsapi TSReturnCode TSSslTicketKeyUpdate(char *ticketData, int ticketDataLen);
 TSAcceptor TSAcceptorGet(TSVConn sslp);
 TSAcceptor TSAcceptorGetbyID(int ID);
 int TSAcceptorCount();

include/tscore/ink_base64.h

@@ -42,5 +42,16 @@ bool ats_base64_encode(const unsigned char *inBuffer, size_t inBufferSize, char
 bool ats_base64_decode(const char *inBuffer, size_t inBufferSize, unsigned char *outBuffer, size_t outBufSize, size_t *length);
 
 // Little helper functions to calculate minimum required output buffer for encoding/decoding.
-#define ATS_BASE64_ENCODE_DSTLEN(_length) ((_length * 8) / 6 + 4)
-#define ATS_BASE64_DECODE_DSTLEN(_length) (((_length + 3) / 4) * 3)
+// These sizes include one byte for null termination, because ats_base64_encode and ats_base64_decode will always write a null
+// terminator.
+inline constexpr size_t
+ats_base64_encode_dstlen(size_t length)
+{
+  return ((length + 2) / 3) * 4 + 1;
+}
+
+inline constexpr size_t
+ats_base64_decode_dstlen(size_t length)
+{
+  return ((length + 3) / 4) * 3 + 1;
+}

include/tscore/ink_defs.h

@@ -100,10 +100,6 @@ countof(const T (&)[N])
 #define unlikely(x) __builtin_expect(!!(x), 0)
 #endif
 
-#if TS_USE_HWLOC
-#include <hwloc.h>
-#endif
-
 #if defined(MAP_NORESERVE)
 #define MAP_SHARED_MAP_NORESERVE (MAP_SHARED | MAP_NORESERVE)
 #else
@@ -121,7 +117,6 @@ extern int on;
 /* Functions
  */
 int ink_sys_name_release(char *name, int namelen, char *release, int releaselen);
-int ink_number_of_processors();
 int ink_login_name_max();
 
 #ifdef __cplusplus
@@ -138,8 +133,3 @@ ROUNDUP(ArithmeticV value, ArithmeticM m)
   return value;
 }
 #endif
-
-#if TS_USE_HWLOC
-// Get the hardware topology
-hwloc_topology_t ink_get_topology();
-#endif

include/tscore/ink_hw.h

@@ -0,0 +1,33 @@
+/** @file
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#pragma once
+
+#include "tscore/ink_config.h"
+
+#if TS_USE_HWLOC
+#include <hwloc.h>
+
+// Get the hardware topology
+hwloc_topology_t ink_get_topology();
+#endif
+
+int ink_number_of_processors();

include/tscore/ink_queue.h

@@ -139,10 +139,10 @@ union head_p {
 #define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \
   (_x).s.pointer = _p;                           \
   (_x).s.version = _v
-#elif defined(__x86_64__) || defined(__ia64__) || defined(__powerpc64__) || defined(__aarch64__) || defined(__mips64)
+#elif defined(__x86_64__) || defined(__ia64__) || defined(__powerpc64__) || defined(__mips64)
 /* Layout of FREELIST_POINTER
  *
- *  0 ~ 47 bits : 48 bits, Virtual Address (47 bits for AMD64 and 48 bits for AArch64)
+ *  0 ~ 47 bits : 48 bits, Virtual Address
  * 48 ~ 62 bits : 15 bits, Freelist Version
  *      63 bits :  1 bits, The type of Virtual Address (0 = user space, 1 = kernel space)
  */
@@ -158,11 +158,30 @@ union head_p {
 #else
 /* the shift is `logical' */
 #define FREELIST_POINTER(_x) \
-  ((void *)((((intptr_t)(_x).data) & 0x0000FFFFFFFFFFFFLL) | (((~((((intptr_t)(_x).data) >> 63) - 1)) >> 48) << 48)))
+  ((void *)((((intptr_t)(_x).data) & 0x0000FFFFFFFFFFFFLL) | ((~((((intptr_t)(_x).data) >> 63) - 1)) << 48)))
 #endif
 
 #define FREELIST_VERSION(_x) ((((intptr_t)(_x).data) & 0x7FFF000000000000LL) >> 48)
 #define SET_FREELIST_POINTER_VERSION(_x, _p, _v) (_x).data = ((((intptr_t)(_p)) & 0x8000FFFFFFFFFFFFLL) | (((_v)&0x7FFFLL) << 48))
+#elif defined(__aarch64__)
+/* Layout of FREELIST_POINTER
+ *
+ *  0 ~ 51 bits : 52 bits, Virtual Address
+ * 52 ~ 62 bits : 11 bits, Freelist Version
+ *      63 bits :  1 bits, The type of Virtual Address (0 = user space, 1 = kernel space)
+ */
+#if ((~0 >> 1) < 0)
+/* the shift is `arithmetic' */
+#define FREELIST_POINTER(_x) \
+  ((void *)((((intptr_t)(_x).data) & 0x000FFFFFFFFFFFFFLL) | ((((intptr_t)(_x).data) >> 63) << 52))) // sign extend
+#else
+/* the shift is `logical' */
+#define FREELIST_POINTER(_x) \
+  ((void *)((((intptr_t)(_x).data) & 0x000FFFFFFFFFFFFFLL) | ((~((((intptr_t)(_x).data) >> 63) - 1)) << 52)))
+#endif
+
+#define FREELIST_VERSION(_x) ((((intptr_t)(_x).data) & 0x7FF0000000000000LL) >> 52)
+#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) (_x).data = ((((intptr_t)(_p)) & 0x800FFFFFFFFFFFFFLL) | (((_v)&0x7FFLL) << 52))
 #else
 #error "unsupported processor"
 #endif

iocore/aio/AIO.cc

@@ -25,7 +25,8 @@
  * Async Disk IO operations.
  */
 
-#include <tscore/TSSystemState.h>
+#include "tscore/TSSystemState.h"
+#include "tscore/ink_hw.h"
 
 #include "P_AIO.h"
 

iocore/aio/test_AIO.cc

@@ -23,6 +23,7 @@
 
 #include "P_AIO.h"
 #include "InkAPIInternal.h"
+#include "tscore/ink_hw.h"
 #include "tscore/I_Layout.h"
 #include "tscore/TSSystemState.h"
 #include <iostream>

iocore/cache/CacheHosting.cc

@@ -490,18 +490,14 @@ CacheHostRecord::Init(matcher_line *line_info, CacheType typ)
             const char *errptr = "A volume number expected";
             RecSignalWarning(REC_SIGNAL_CONFIG_ERROR, "%s discarding %s entry at line %d :%s", "[CacheHosting]", config_file,
                              line_info->line_num, errptr);
-            if (val != nullptr) {
-              ats_free(val);
-            }
+            ats_free(val);
             return -1;
           }
         }
         if ((*s < '0') || (*s > '9')) {
           RecSignalWarning(REC_SIGNAL_CONFIG_ERROR, "%s discarding %s entry at line %d : bad token [%c]", "[CacheHosting]",
                            config_file, line_info->line_num, *s);
-          if (val != nullptr) {
-            ats_free(val);
-          }
+          ats_free(val);
           return -1;
         }
         s++;
@@ -537,9 +533,7 @@ CacheHostRecord::Init(matcher_line *line_info, CacheType typ)
           if (!is_vol_present) {
             RecSignalWarning(REC_SIGNAL_CONFIG_ERROR, "%s discarding %s entry at line %d : bad volume number [%d]",
                              "[CacheHosting]", config_file, line_info->line_num, volume_number);
-            if (val != nullptr) {
-              ats_free(val);
-            }
+            ats_free(val);
             return -1;
           }
           if (c == '\0') {

iocore/cache/Makefile.am

@@ -105,6 +105,7 @@ test_LDADD = \
 	$(top_builddir)/iocore/aio/libinkaio.a \
 	$(top_builddir)/src/tscore/libtscore.la \
 	$(top_builddir)/lib/records/librecords_p.a \
+	$(top_builddir)/lib/fastlz/libfastlz.a \
 	$(top_builddir)/iocore/eventsystem/libinkevent.a \
 	@HWLOC_LIBS@ \
 	@LIBPCRE@ \
@@ -116,7 +117,6 @@ test_LDADD = \
 	@YAMLCPP_LIBS@ \
 	-lm
 
-if EXPENSIVE_TESTS
 check_PROGRAMS = \
   test_Cache \
   test_RWW \
@@ -129,7 +129,6 @@ check_PROGRAMS = \
   test_Update_L_to_S \
   test_Update_S_to_L \
   test_Update_header
-endif
 
 test_main_SOURCES = \
   ./test/main.cc \

iocore/cache/P_CacheInternal.h

@@ -614,9 +614,9 @@ free_CacheVC(CacheVC *cont)
   cont->blocks.clear();
   cont->writer_buf.clear();
   cont->alternate_index = CACHE_ALT_INDEX_DEFAULT;
-  if (cont->scan_vol_map) {
-    ats_free(cont->scan_vol_map);
-  }
+
+  ats_free(cont->scan_vol_map);
+
   memset((char *)&cont->vio, 0, cont->size_to_init);
 #ifdef CACHE_STAT_PAGES
   ink_assert(!cont->stat_link.next && !cont->stat_link.prev);