Merge branch 'master' into quic-latest

* master:
  Adds a shell script to help build the H3 toolchains (#7299)
  Remove unfinished h2c support (#7286)
  Allow disabling SO_MARK and IP_TOS usage (#7292)
  Enable all h2spec test (#7289)
  Fix bad HTTP/2 post client causing stuck HttpSM (#7237)
  Sticky server does not work with H2 client (#7261)
  7096: Synchronize Server Session Management and Network I/O (#7278)
  HostDB: remove unused field in HostDBApplicationInfo, and update remaining types in http_data to fix broken padding. (#7264)
  Add support for a new (TSMgmtDataTypeGet) mgmt API function to retrieve the record data type (#7221)
  Fix example in default sni.yaml configuration. (#7277)
  Fix proxy.process.http.current_client_transactions (#7258)
  Add AuTest for HTTP/2 Graceful Shutdown (#7271)
  Fix truncated reponse on HTTP/2 graceful shutdown (#7267)
  url_sig add 'ignore_expiry = true' option for log replay testing (#7231)
  Respecting default rolling_enabled in plugins. (#7275)
  gracefully handle TSReleaseAsserts in statichit and generator plugins (#7269)
  Removes commented out code from esi plugin (#7273)
  Allow initial // in request targets. (#7266)
  Document external log rotation support via SIGUSR2 (#7265)
  Let Dedicated EThreads use `EThread::schedule` (#7228)
  HostDB: Fix cache data version checking to use full version, not major version. (#7263)
  Bugfix: set a default inactivity timeout only if a read or write I/O operation was set (#7226)
  Treat objects with negative max-age CC directives as stale. (#7260)
  Remove some usless defines, which just obsfucates code (#7252)
  Remove useless if for port set assertion. (#7250)
  Fix test_error_page_selection memory leaks and logic errors (#7248)
  [multiplexer] option to skip post/put requests (#7233)
  Incorporates the latest CI build changes (#7251)
  Add support for server protocol stack API (#7239)
  Fix for plugins ASAN suppression file (#7249)
  RolledLogDeleter: do not sort on each candidate consideration. (#7243)
  Make double Au test more reliable. (#7216)
  Ensure that ca override does not get lost (#7219)
  Stop crash on disk failure (#7218)
  Do not cache Transfer-Encoding header (#7234)
  clean up body factory tests (#7236)
  Revert "Create an explicit runroot.yaml for AuTests (#7177)" (#7235)
  New option to dead server to not retry during dead period (#7142)
  Increment ssl_error_syscall only if not EOF (#7225)
  Fix renamed setting in default config (#7224)
  Log config reload: use new config for initialization (#7215)
  Introduce proxy-verifier to AuTests (#7211)
  Follow redirection responses when refreshing stale cache objects. (#7213)
  Create an explicit runroot.yaml for AuTests (#7177)
  Support external log rotation tools via SIGUSR2 (#6806)
  Add support for TS API for Note, Status, Warning, Alert (#7208)
  If the weight is 0, the SRV record should be selected from the highest priority group (#7206)
  Cleanup: remove unnecessary memset() within dns_process() (#7209)
  Docs cleanup (#7210)
  Strip whitespaces after field-name and before the colon in headers from the origin (#7202)
  Adds new plugin: statichit (#7173)
  Add duplicate header field processing when creating outgoing response (#7207)

.gitignore

@@ -193,6 +193,7 @@ RELEASE
 
 # autest
 tests/env-test/
+tests/proxy-verifier
 tests/Pipfile.lock
 tests/gold_tests/chunked_encoding/smuggle-client
 tests/gold_tests/tls/ssl-post

ci/jenkins/bin/autest.sh

@@ -19,8 +19,10 @@
 set +x
 
 cd src
+sleep 30
 
-if [ ! -z "$ghprbActualCommit" ]; then
+git branch --contains ${ghprbActualCommit} > /dev/null
+if [ $? = 0 -a ! -z "$ghprbActualCommit" ]; then
     git diff ${ghprbActualCommit}^...${ghprbActualCommit} --name-only | egrep -E '^(build|iocore|proxy|tests|include|mgmt|plugins|proxy|src)/' > /dev/null
     if [ $? = 1 ]; then
         echo "No relevant files changed, skipping run"

ci/jenkins/bin/build.sh

@@ -28,9 +28,14 @@ CCACHE="--enable-ccache"
 # When to enable -Werror
 WERROR="--enable-werror"
 
+# Optional settings
+ASAN=""
+[ "1" == "$enable_asan" ] && ASAN="--enable-asan"
+
 echo "DEBUG: $DEBUG"
 echo "CCACHE: $CCACHE"
 echo "WERROR: $WERROR"
+echo "ASAN: $ASAN"
 
 # Change to the build area (this is previously setup in extract.sh)
 cd "${ATS_BUILD_BASEDIR}/build"
@@ -46,7 +51,8 @@ set -x
     --with-user=jenkins \
     ${CCACHE} \
     ${WERROR} \
-    ${DEBUG}
+    ${DEBUG} \
+    ${ASAN}
 
 echo
 echo -n "Main build started at " && date

ci/jenkins/bin/cache-tests.sh

@@ -37,9 +37,10 @@ ${ATS_MAKE} -i ${ATS_MAKE_FLAGS} V=1 Q=
 ${ATS_MAKE} -i install
 
 [ -x ${PREFIX}/bin/traffic_server ] || exit 1
+ldd  ${PREFIX}/bin/traffic_server
 
-# Get NPM v10
-source /opt/rh/rh-nodejs10/enable
+# Get NPM v12
+source /opt/rh/rh-nodejs12/enable
 
 # Setup and start ATS with the required remap rule
 echo "map http://127.0.0.1:8080 http://192.168.3.1:8000" >> $REMAP
@@ -49,7 +50,7 @@ ${PREFIX}/bin/trafficserver start
 set -x
 
 cd /home/jenkins/cache-tests
-npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}.json
+npm run --silent cli --base=http://127.0.0.1:8080 > /CA/cache-tests/${ATS_BRANCH}.json
 cat /CA/cache-tests/${ATS_BRANCH}.json
 
 ${PREFIX}/bin/trafficserver stop
@@ -61,7 +62,7 @@ if [ "" != "$TWEAK" ]; then
     echo "CONFIG proxy.config.http.negative_caching_enabled INT 1" >> $RECORDS
     ${PREFIX}/bin/trafficserver start
     cd /home/jenkins/cache-tests
-    npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json
+    npm run --silent cli --base=http://127.0.0.1:8080 > /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json
     echo "TWEAKED RESULTS"
     cat /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json
 

ci/jenkins/bin/clang-analyzer.sh

@@ -38,9 +38,6 @@ checkers="-enable-checker alpha.unix.cstring.BufferOverlap \
 test -z "${ATS_MAKE}" && ATS_MAKE="make"
 test ! -z "${WORKSPACE}" && cd "${WORKSPACE}/src"
 
-# Check to see if this is a Github PR build (so not a github branch per-se)
-test "${JOB_NAME#*-github}" != "${JOB_NAME}" && ATS_BRANCH="github"
-
 # Where to store the results, special case for the CI
 output="/tmp"
 
@@ -103,6 +100,3 @@ fi
 if [ -x "/admin/bin/clean-clang.sh" ]; then
     /admin/bin/clean-clang.sh
 fi
-
-# Exit with the scan-build exit code (thanks to --status-bugs)
-exit $status

ci/jenkins/bin/environment.sh

@@ -47,6 +47,11 @@ export TODAY=$(/bin/date +'%m%d%Y')
 # Extract the current branch (default to master). ToDo: Can we do this better ?
 ATS_BRANCH=master
 
+# Make sure to leave these, for the HTTP cache tests
+test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x
+test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x
+
+# These should be maintained and cleaned up as needed.
 test "${JOB_NAME#*-7.1.x}" != "${JOB_NAME}" && ATS_BRANCH=7.1.x
 test "${JOB_NAME#*-8.0.x}" != "${JOB_NAME}" && ATS_BRANCH=8.0.x
 test "${JOB_NAME#*-8.1.x}" != "${JOB_NAME}" && ATS_BRANCH=8.1.x
@@ -59,6 +64,9 @@ test "${JOB_NAME#*-10.1.x}" != "${JOB_NAME}" && ATS_BRANCH=10.1.x
 test "${JOB_NAME#*-10.2.x}" != "${JOB_NAME}" && ATS_BRANCH=10.2.x
 test "${JOB_NAME#*-10.3.x}" != "${JOB_NAME}" && ATS_BRANCH=10.3.x
 
+# Special case for the full build of clang analyzer
+test "${JOB_NAME}" == "clang-analyzer-full" && ATS_BRANCH=FULL
+
 export ATS_BRANCH
 echo "Branch is $ATS_BRANCH"
 

configs/records.config.default.in

@@ -118,7 +118,7 @@ CONFIG proxy.config.http.cache.heuristic_lm_factor FLOAT 0.10
 ##############################################################################
 CONFIG proxy.config.net.connections_throttle INT 30000
 CONFIG proxy.config.net.max_connections_in INT 30000
-CONFIG proxy.config.net.max_connections_active_in INT 10000
+CONFIG proxy.config.net.max_requests_in INT 0
 
 ##############################################################################
 # RAM and disk cache configurations. Docs:

configs/sni.yaml.default

@@ -25,4 +25,4 @@
 #     verify_client: MODERATE
 #   - fqdn: two.com
 #     tunnel_route: two.com
-#     ip_allow = '10.0.0.1-10.0.0.255'
+#     ip_allow: '10.0.0.1-10.0.0.255'

doc/admin-guide/files/records.config.en.rst

@@ -537,10 +537,10 @@ Local Manager
    process as, which also has the effect of setting ownership of configuration
    and log files.
 
-   As of version 2.1.1 if the user_id is prefixed with pound character (``#``)
+   If the user_id is prefixed with pound character (``#``),
    the remainder of the string is considered to be a
    `numeric user identifier <http://en.wikipedia.org/wiki/User_identifier>`_.
-   If the value is set to ``#-1`` |TS| will not change the user during startup.
+   If the value is set to ``#-1``, |TS| will not change the user during startup.
 
    .. important::
 
@@ -999,7 +999,9 @@ mptcp
 .. ts:cv:: CONFIG proxy.config.http.attach_server_session_to_client INT 0
    :overridable:
 
-   Control the re-use of an server session by a user agent (client) session.
+   Control the re-use of an server session by a user agent (client) session. Currently only applies to user
+   agents using HTTP/1.0 or HTTP/1.1. For other HTTP versions, the origin connection is always returned to the
+   session sharing pool or closed.
 
    If a user agent performs more than one HTTP transaction on its connection to |TS| a server session must be
    obtained for the second (and subsequent) transaction as for the first. This settings affects how that server session
@@ -1452,7 +1454,7 @@ Origin Server Connect Attempts
    :reloadable:
    :overridable:
 
-   Maximum number of connection retries |TS| can make while an origin is marked dead.  Typically this value is smaller than
+   Maximum number of connection attempts |TS| can make while an origin is marked dead per request.  Typically this value is smaller than
    `proxy.config.http.connect_attempts_max_retries`_ so an error is returned to the client faster and also to reduce the load on the dead origin.
    The timeout interval `proxy.config.http.connect_attempts_timeout`_ in seconds is used with this setting.
 
@@ -4271,6 +4273,8 @@ Sockets
         SO_KEEPALIVE (2)
         SO_LINGER (4) - with a timeout of 0 seconds
         TCP_FASTOPEN (8)
+        PACKET_MARK (16)
+        PACKET_TOS (32)
 
 .. note::
 
@@ -4303,6 +4307,8 @@ Sockets
         SO_KEEPALIVE (2)
         SO_LINGER (4) - with a timeout of 0 seconds
         TCP_FASTOPEN (8)
+        PACKET_MARK (16)
+        PACKET_TOS (32)
 
 .. note::
 

doc/admin-guide/monitoring/statistics/core/ssl.en.rst

@@ -74,9 +74,6 @@ SSL/TLS
    The number of SSL connections to origin servers which were terminated due to
    unsupported SSL/TLS protocol versions, since statistics collection began.
 
-.. ts:stat:: global proxy.process.ssl.ssl_error_read_eos integer
-   :type: counter
-
 .. ts:stat:: global proxy.process.ssl.ssl_error_ssl integer
    :type: counter
 
@@ -229,4 +226,3 @@ SSL/TLS
 
    Incoming client SSL connections terminated due to an unsupported or disabled
    version of SSL/TLS, since statistics collection began.
-

doc/admin-guide/plugins/access_control.en.rst

@@ -379,16 +379,16 @@ Configuration files
 
 * Format the ``access_control.log``
 
-.. code-block:: bash
-
-  access_control_format = format {
-    Format = '%<cqtq> sub=%<{@TokenSubject}cqh> tid=%<{@TokenId}cqh> status=%<{@TokenStatus}cqh> cache=%<{x-cache}psh> key=%<{x-cache-key}psh>'  }
-
-  log.ascii {
-    Filename = 'access_control',
-    Format = access_control_format
-  }
-
+.. code-block:: yaml
+
+   logging:
+     formats:
+     - format: '%<cqtq> sub=%<{@TokenSubject}cqh> tid=%<{@TokenId}cqh> status=%<{@TokenStatus}cqh> cache=%<{x-cache}psh> key=%<{x-cache-key}psh>'
+       name: access_control_format
+     logs:
+     - filename: access_control
+       format: access_control_format
+       mode: ascii
 
 * X-Debug plugin added to ``plugin.config``
 

doc/admin-guide/plugins/lua.en.rst

@@ -299,6 +299,46 @@ Here is an example:
 
 :ref:`TOP <admin-plugins-ts-lua>`
 
+ts.status
+---------
+**syntax:** *ts.status(MESSAGE)*
+
+**context:** global
+
+**description**: Log the MESSAGE to error.log as status
+
+:ref:`TOP <admin-plugins-ts-lua>`
+
+ts.note
+-------
+**syntax:** *ts.note(MESSAGE)*
+
+**context:** global
+
+**description**: Log the MESSAGE to error.log as note
+
+:ref:`TOP <admin-plugins-ts-lua>`
+
+ts.warning
+----------
+**syntax:** *ts.warning(MESSAGE)*
+
+**context:** global
+
+**description**: Log the MESSAGE to error.log as warning
+
+:ref:`TOP <admin-plugins-ts-lua>`
+
+ts.alert
+--------
+**syntax:** *ts.alert(MESSAGE)*
+
+**context:** global
+
+**description**: Log the MESSAGE to error.log as alert
+
+:ref:`TOP <admin-plugins-ts-lua>`
+
 TS Basic Internal Information
 -----------------------------
 **syntax:** *ts.get_install_dir()*
@@ -2770,6 +2810,28 @@ Here is an example:
 
 :ref:`TOP <admin-plugins-ts-lua>`
 
+ts.http.get_server_protocol_stack
+---------------------------------
+**syntax:** *ts.http.get_server_protocol_stack()*
+
+**context:** do_global_read_response or later
+
+**description:** This function can be used to get server protocol stack information
+
+Here is an example:
+
+::
+
+    function do_global_read_response()
+        local stack = {ts.http.get_server_protocol_stack()}
+        for k,v in pairs(stack) do
+          ts.debug(v)
+        end
+        return 0
+    end
+
+:ref:`TOP <admin-plugins-ts-lua>`
+
 ts.http.server_push
 -------------------
 **syntax:** *ts.http.server_push()*

doc/admin-guide/plugins/multiplexer.en.rst

@@ -24,7 +24,8 @@
 
 |Name| is a remap plug-in that allows a request to be multiplexed one or more times and sent to
 different remap entries. Both headers and body (in case of POST or PUT methods, only) are copied
-into the new requests.
+into the new requests. Optionally POST/PUT requests can be skipped via
+``pparam=proxy.config.multiplexer.skip_post_put=1``.
 
 Description
 ===========
@@ -59,6 +60,7 @@ Example remap.config::
     map http://www.example.com/a http://www.example.com/ @plugin=multiplexer.so @pparam=host1.example.com
     map http://www.example.com/b http://www.example.com/ @plugin=multiplexer.so @pparam=host2.example.com
     map http://www.example.com/c http://www.example.com/ @plugin=multiplexer.so @pparam=host1.example.com @pparam=host2.example.com
+    map http://www.example.com/d http://www.example.com/ @plugin=multiplexer.so @pparam=host1.example.com @pparam=host2.example.com @pparam=proxy.config.multiplexer.skip_post_put=1
 
 
 Implementation

doc/admin-guide/plugins/url_sig.en.rst

@@ -384,3 +384,20 @@ Example
     { [data not shown]
     * Connection #0 to host localhost left intact
 
+
+Replay test support
+===================
+
+To assist in log replay an option is available in the config file which
+will ignore the expiration date.  This allows all url_sig tests to
+pass the expiration date.
+
+The config file option to enable this is::
+
+    ignore_expiry = true
+
+Once updated, touch `remap.config` then issue a
+:option:`traffic_ctl config reload` to make the settings active.
+
+Do NOT deploy this to production as it will disable valid checks
+on signed urls!

doc/appendices/command-line/traffic_manager.en.rst

@@ -52,6 +52,14 @@ SIGHUP
 SIGINT, SIGTERM
   These signals cause :program:`traffic_manager` to exit after also shutting down :program:`traffic_server`.
 
+SIGUSR2
+  This signal causes the :program:`traffic_manager` and :program:`traffic_server` processes to close
+  and reopen their file descriptors for all of their log files. This allows the use of external
+  tools to handle log rotation and retention. For instance, logrotate(8) can be configured to rotate
+  the various |ATS| logs and, via the logrotate postrotate script, send a `-SIGUSR2` to the
+  :program:`traffic_manager` process. After the signal is received, |ATS| will stop logging to the
+  now-rolled files and will reopen log files with the originally configured log names.
+
 Exponential Back-off Delay
 ==========================